Merge pull request #3166 from pi-hole/release/v4.4

Add use-application-dns.net = NXDOMAIN in ProcessDNSSettings rather t…

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+patreon: pihole
+custom: https://pi-hole.net/donate

advanced/01-pihole.conf

@@ -18,6 +18,8 @@
 #                    WITHIN /etc/dnsmasq.d/yourname.conf                      #
 ###############################################################################
 
+addn-hosts=/etc/pihole/gravity.list
+addn-hosts=/etc/pihole/black.list
 addn-hosts=/etc/pihole/local.list
 
 domain-needed

advanced/Scripts/list.sh

@@ -11,45 +11,46 @@
 # Globals
 basename=pihole
 piholeDir=/etc/"${basename}"
-gravityDBfile="${piholeDir}/gravity.db"
+whitelist="${piholeDir}"/whitelist.txt
+blacklist="${piholeDir}"/blacklist.txt
 
+readonly regexlist="/etc/pihole/regex.list"
 reload=false
 addmode=true
 verbose=true
 wildcard=false
-web=false
 
 domList=()
 
-listType=""
-listname=""
+listMain=""
+listAlt=""
 
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
 
 
 helpFunc() {
-    if [[ "${listType}" == "whitelist" ]]; then
+    if [[ "${listMain}" == "${whitelist}" ]]; then
         param="w"
-        type="whitelist"
-    elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then
+        type="white"
+    elif [[ "${listMain}" == "${regexlist}" && "${wildcard}" == true ]]; then
         param="-wild"
-        type="wildcard blacklist"
-    elif [[ "${listType}" == "regex" ]]; then
+        type="wildcard black"
+    elif [[ "${listMain}" == "${regexlist}" ]]; then
         param="-regex"
-        type="regex filter"
+        type="regex black"
     else
         param="b"
-        type="blacklist"
+        type="black"
     fi
 
     echo "Usage: pihole -${param} [options] <domain> <domain2 ...>
 Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com'
-${type^} one or more domains
+${type^}list one or more domains
 
 Options:
-  -d, --delmode       Remove domain(s) from the ${type}
-  -nr, --noreload     Update ${type} without reloading the DNS server
+  -d, --delmode       Remove domain(s) from the ${type}list
+  -nr, --noreload     Update ${type}list without refreshing dnsmasq
   -q, --quiet         Make output less verbose
   -h, --help          Show this help dialog
   -l, --list          Display all your ${type}listed domains
@@ -72,7 +73,7 @@ HandleOther() {
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then
+        if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
             validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
@@ -87,143 +88,175 @@ HandleOther() {
     fi
 }
 
-ProcessDomainList() {
-    if [[ "${listType}" == "regex" ]]; then
-        # Regex filter list
-        listname="regex filters"
-    else
-        # Whitelist / Blacklist
-        listname="${listType}"
+PoplistFile() {
+    # Check whitelist file exists, and if not, create it
+    if [[ ! -f "${whitelist}" ]]; then
+        touch "${whitelist}"
+    fi
+
+    # Check blacklist file exists, and if not, create it
+    if [[ ! -f "${blacklist}" ]]; then
+        touch "${blacklist}"
     fi
 
     for dom in "${domList[@]}"; do
-        # Format domain into regex filter if requested
-        if [[ "${wildcard}" == true ]]; then
-            dom="(^|\\.)${dom//\./\\.}$"
-        fi
-
-        # Logic: If addmode then add to desired list and remove from the other;
-        # if delmode then remove from desired list but do not add to the other
+        # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
-            AddDomain "${dom}" "${listType}"
-            if [[ ! "${listType}" == "regex" ]]; then
-                RemoveDomain "${dom}" "${listAlt}"
-            fi
+            AddDomain "${dom}" "${listMain}"
+            RemoveDomain "${dom}" "${listAlt}"
         else
-            RemoveDomain "${dom}" "${listType}"
+            RemoveDomain "${dom}" "${listMain}"
         fi
   done
 }
 
 AddDomain() {
-    local domain list num
-    # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
-    domain="$1"
     list="$2"
+    domain=$(EscapeRegexp "$1")
 
-    # Is the domain in the list we want to add it to?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
+    [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
+    [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
 
-    if [[ "${num}" -ne 0 ]]; then
-      if [[ "${verbose}" == true ]]; then
-          echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
-      fi
-      return
+    if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
+        [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
+        [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
+        bool=true
+        # Is the domain in the list we want to add it to?
+        grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
+
+        if [[ "${bool}" == false ]]; then
+            # Domain not found in the whitelist file, add it!
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} Adding ${1} to ${listname}..."
+            fi
+            reload=true
+            # Add it to the list we want to add it to
+            echo "$1" >> "${list}"
+        else
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
+            fi
+        fi
+    elif [[ "${list}" == "${regexlist}" ]]; then
+        [[ -z "${type}" ]] && type="--wildcard-only"
+        bool=true
+        domain="${1}"
+
+        [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
+
+        # Is the domain in the list?
+        # Search only for exactly matching lines
+        grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
+
+        if [[ "${bool}" == false ]]; then
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} Adding ${domain} to regex list..."
+            fi
+            reload="restart"
+            echo "$domain" >> "${regexlist}"
+        else
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} ${domain} already exists in regex list, no need to add!"
+            fi
+        fi
     fi
-
-    # Domain not found in the table, add it!
-    if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} Adding ${1} to the ${listname}..."
-    fi
-    reload=true
-    # Insert only the domain here. The enabled and date_added fields will be filled
-    # with their default values (enabled = true, date_added = current timestamp)
-    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES ('${domain}');"
 }
 
 RemoveDomain() {
-    local domain list num
-    # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
-    domain="$1"
     list="$2"
+    domain=$(EscapeRegexp "$1")
 
-    # Is the domain in the list we want to remove it from?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
+    [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
+    [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
 
-    if [[ "${num}" -eq 0 ]]; then
-      if [[ "${verbose}" == true ]]; then
-          echo -e "  ${INFO} ${1} does not exist in ${list}, no need to remove!"
-      fi
-      return
+    if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
+        bool=true
+        [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
+        [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
+        # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa
+        grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
+        if [[ "${bool}" == true ]]; then
+            # Remove it from the other one
+            echo -e "  ${INFO} Removing $1 from ${listname}..."
+            # /I flag: search case-insensitive
+            sed -i "/${domain}/Id" "${list}"
+            reload=true
+        else
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
+            fi
+        fi
+    elif [[ "${list}" == "${regexlist}" ]]; then
+        [[ -z "${type}" ]] && type="--wildcard-only"
+        domain="${1}"
+
+        [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
+
+        bool=true
+        # Is it in the list?
+        grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
+        if [[ "${bool}" == true ]]; then
+            # Remove it from the other one
+            echo -e "  ${INFO} Removing $domain from regex list..."
+            local lineNumber
+            lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:)
+            sed -i "${lineNumber}d" "${list}"
+            reload=true
+        else
+            if [[ "${verbose}" == true ]]; then
+                echo -e "  ${INFO} ${domain} does not exist in regex list, no need to remove!"
+            fi
+        fi
     fi
+}
 
-    # Domain found in the table, remove it!
-    if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} Removing ${1} from the ${listname}..."
-    fi
-    reload=true
-    # Remove it from the current list
-    sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = '${domain}';"
+# Update Gravity
+Reload() {
+    echo ""
+    pihole -g --skip-download "${type:-}"
 }
 
 Displaylist() {
-    local list listname count num_pipes domain enabled status nicedate
-
-    listname="${listType}"
-    data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)"
-
-    if [[ -z $data ]]; then
-        echo -e "Not showing empty ${listname}"
-    else
-        echo -e "Displaying ${listname}:"
+    if [[ -f ${listMain} ]]; then
+        if [[ "${listMain}" == "${whitelist}" ]]; then
+            string="gravity resistant domains"
+        else
+            string="domains caught in the sinkhole"
+        fi
+        verbose=false
+        echo -e "Displaying $string:\n"
         count=1
-        while IFS= read -r line
-        do
-            # Count number of pipes seen in this line
-            # This is necessary because we can only detect the pipe separating the fields
-            # from the end backwards as the domain (which is the first field) may contain
-            # pipe symbols as they are perfectly valid regex filter control characters
-            num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")"
-
-            # Extract domain and enabled status based on the obtained number of pipe characters
-            domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")"
-            enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")"
-            datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")"
-
-            # Translate boolean status into human readable string
-            if [[ "${enabled}" -eq 1 ]]; then
-                status="enabled"
-            else
-                status="disabled"
-            fi
-
-            # Get nice representation of numerical date stored in database
-            nicedate=$(date --rfc-2822 -d "@${datemod}")
-
-            echo "  ${count}: ${domain} (${status}, last modified ${nicedate})"
+        while IFS= read -r RD || [ -n "${RD}" ]; do
+            echo "  ${count}: ${RD}"
             count=$((count+1))
-        done <<< "${data}"
+        done < "${listMain}"
+    else
+        echo -e "  ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}"
     fi
     exit 0;
 }
 
 NukeList() {
-    sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};"
+    if [[ -f "${listMain}" ]]; then
+        # Back up original list
+        cp "${listMain}" "${listMain}.bck~"
+        # Empty out file
+        echo "" > "${listMain}"
+    fi
 }
 
 for var in "$@"; do
     case "${var}" in
-        "-w" | "whitelist"   ) listType="whitelist"; listAlt="blacklist";;
-        "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
-        "--wild" | "wildcard" ) listType="regex"; wildcard=true;;
-        "--regex" | "regex"   ) listType="regex";;
+        "-w" | "whitelist"   ) listMain="${whitelist}"; listAlt="${blacklist}";;
+        "-b" | "blacklist"   ) listMain="${blacklist}"; listAlt="${whitelist}";;
+        "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;;
+        "--regex" | "regex"   ) listMain="${regexlist}";;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;
         "-h" | "--help"      ) helpFunc;;
         "-l" | "--list"      ) Displaylist;;
         "--nuke"             ) NukeList;;
-        "--web"              ) web=true;;
         *                    ) HandleOther "${var}";;
     esac
 done
@@ -234,13 +267,9 @@ if [[ $# = 0 ]]; then
     helpFunc
 fi
 
-ProcessDomainList
-
-# Used on web interface
-if $web; then
-echo "DONE"
-fi
+PoplistFile
 
 if [[ "${reload}" != false ]]; then
-    pihole restartdns reload
+    # Ensure that "restart" is used for Wildcard updates
+    Reload "${reload}"
 fi

advanced/Scripts/piholeARPTable.sh

@@ -1,73 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC1090
-
-# Pi-hole: A black hole for Internet advertisements
-# (c) 2019 Pi-hole, LLC (https://pi-hole.net)
-# Network-wide ad blocking via your own hardware.
-#
-# ARP table interaction
-#
-# This file is copyright under the latest version of the EUPL.
-# Please see LICENSE file for your rights under this license.
-
-coltable="/opt/pihole/COL_TABLE"
-if [[ -f ${coltable} ]]; then
-    source ${coltable}
-fi
-
-# Determine database location
-# Obtain DBFILE=... setting from pihole-FTL.db
-# Constructed to return nothing when
-# a) the setting is not present in the config file, or
-# b) the setting is commented out (e.g. "#DBFILE=...")
-FTLconf="/etc/pihole/pihole-FTL.conf"
-if [ -e "$FTLconf" ]; then
-    DBFILE="$(sed -n -e 's/^\s*DBFILE\s*=\s*//p' ${FTLconf})"
-fi
-# Test for empty string. Use standard path in this case.
-if [ -z "$DBFILE" ]; then
-    DBFILE="/etc/pihole/pihole-FTL.db"
-fi
-
-
-flushARP(){
-    local output
-    if [[ "${args[1]}" != "quiet" ]]; then
-        echo -ne "  ${INFO} Flushing network table ..."
-    fi
-
-    # Flush ARP cache to avoid re-adding of dead entries
-    if ! output=$(ip neigh flush all 2>&1); then
-        echo -e "${OVER}  ${CROSS} Failed to clear ARP cache"
-        echo "  Output: ${output}"
-        return 1
-    fi
-
-    # Truncate network_addresses table in pihole-FTL.db
-    # This needs to be done before we can truncate the network table due to
-    # foreign key contraints
-    if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then
-        echo -e "${OVER}  ${CROSS} Failed to truncate network_addresses table"
-        echo "  Database location: ${DBFILE}"
-        echo "  Output: ${output}"
-        return 1
-    fi
-
-    # Truncate network table in pihole-FTL.db
-    if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then
-        echo -e "${OVER}  ${CROSS} Failed to truncate network table"
-        echo "  Database location: ${DBFILE}"
-        echo "  Output: ${output}"
-        return 1
-    fi
-
-    if [[ "${args[1]}" != "quiet" ]]; then
-        echo -e "${OVER}  ${TICK} Flushed network table"
-    fi
-}
-
-args=("$@")
-
-case "${args[0]}" in
-    "arpflush"            ) flushARP;;
-esac

advanced/Scripts/piholeCheckout.sh

@@ -46,6 +46,12 @@ checkout() {
     local corebranches
     local webbranches
 
+    # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole
+    local funcOutput
+    funcOutput=$(get_binary_name) #Store output of get_binary_name here
+    local binary
+    binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL)
+
     # Avoid globbing
     set -f
 
@@ -86,11 +92,9 @@ checkout() {
         fi
         #echo -e "  ${TICK} Pi-hole Core"
 
-        get_binary_name
         local path
         path="development/${binary}"
         echo "development" > /etc/pihole/ftlbranch
-        chmod 644 /etc/pihole/ftlbranch
     elif [[ "${1}" == "master" ]] ; then
         # Shortcut to check out master branches
         echo -e "  ${INFO} Shortcut \"master\" detected - checking out master branches..."
@@ -101,11 +105,9 @@ checkout() {
             fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo "  ${CROSS} Unable to pull Web master branch"; exit 1; }
         fi
         #echo -e "  ${TICK} Web Interface"
-        get_binary_name
         local path
         path="master/${binary}"
         echo "master" > /etc/pihole/ftlbranch
-        chmod 644 /etc/pihole/ftlbranch
     elif [[ "${1}" == "core" ]] ; then
         str="Fetching branches from ${piholeGitUrl}"
         echo -ne "  ${INFO} $str"
@@ -161,14 +163,12 @@ checkout() {
         fi
         checkout_pull_branch "${webInterfaceDir}" "${2}"
     elif [[ "${1}" == "ftl" ]] ; then
-        get_binary_name
         local path
         path="${2}/${binary}"
 
         if check_download_exists "$path"; then
             echo "  ${TICK} Branch ${2} exists"
             echo "${2}" > /etc/pihole/ftlbranch
-            chmod 644 /etc/pihole/ftlbranch
             FTLinstall "${binary}"
             restart_service pihole-FTL
             enable_service pihole-FTL

advanced/Scripts/piholeDebug.sh

@@ -89,12 +89,16 @@ PIHOLE_WILDCARD_CONFIG_FILE="${DNSMASQ_D_DIRECTORY}/03-wildcard.conf"
 WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf"
 #WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf"
 
+PIHOLE_DEFAULT_AD_LISTS="${PIHOLE_DIRECTORY}/adlists.default"
+PIHOLE_USER_DEFINED_AD_LISTS="${PIHOLE_DIRECTORY}/adlists.list"
+PIHOLE_BLACKLIST_FILE="${PIHOLE_DIRECTORY}/blacklist.txt"
+PIHOLE_BLOCKLIST_FILE="${PIHOLE_DIRECTORY}/gravity.list"
 PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log"
 PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*"
 PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list"
 PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate"
 PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf"
-PIHOLE_GRAVITY_DB_FILE="${PIHOLE_DIRECTORY}/gravity.db"
+PIHOLE_WHITELIST_FILE="${PIHOLE_DIRECTORY}/whitelist.txt"
 
 PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole"
 PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE"
@@ -138,11 +142,16 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}"
 "${PIHOLE_DHCP_CONFIG_FILE}"
 "${PIHOLE_WILDCARD_CONFIG_FILE}"
 "${WEB_SERVER_CONFIG_FILE}"
+"${PIHOLE_DEFAULT_AD_LISTS}"
+"${PIHOLE_USER_DEFINED_AD_LISTS}"
+"${PIHOLE_BLACKLIST_FILE}"
+"${PIHOLE_BLOCKLIST_FILE}"
 "${PIHOLE_INSTALL_LOG_FILE}"
 "${PIHOLE_RAW_BLOCKLIST_FILES}"
 "${PIHOLE_LOCAL_HOSTS_FILE}"
 "${PIHOLE_LOGROTATE_FILE}"
 "${PIHOLE_SETUP_VARS_FILE}"
+"${PIHOLE_WHITELIST_FILE}"
 "${PIHOLE_COMMAND}"
 "${PIHOLE_COLTABLE_FILE}"
 "${FTL_PID}"
@@ -634,19 +643,21 @@ ping_internet() {
 }
 
 compare_port_to_service_assigned() {
-    local service_name="${1}"
-    # The programs we use may change at some point, so they are in a varible here
-    local resolver="pihole-FTL"
-    local web_server="lighttpd"
-    local ftl="pihole-FTL"
+    local service_name
+    local expected_service
+    local port
+
+    service_name="${2}"
+    expected_service="${1}"
+    port="${3}"
 
     # If the service is a Pi-hole service, highlight it in green
-    if [[ "${service_name}" == "${resolver}" ]] || [[ "${service_name}" == "${web_server}" ]] || [[ "${service_name}" == "${ftl}" ]]; then
-        log_write "[${COL_GREEN}${port_number}${COL_NC}] is in use by ${COL_GREEN}${service_name}${COL_NC}"
+    if [[ "${service_name}" == "${expected_service}" ]]; then
+        log_write "[${COL_GREEN}${port}${COL_NC}] is in use by ${COL_GREEN}${service_name}${COL_NC}"
     # Otherwise,
     else
         # Show the service name in red since it's non-standard
-        log_write "[${COL_RED}${port_number}${COL_NC}] is in use by ${COL_RED}${service_name}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_PORTS})"
+        log_write "[${COL_RED}${port}${COL_NC}] is in use by ${COL_RED}${service_name}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_PORTS})"
     fi
 }
 
@@ -680,11 +691,11 @@ check_required_ports() {
         fi
         # Use a case statement to determine if the right services are using the right ports
         case "$(echo "$port_number" | rev | cut -d: -f1 | rev)" in
-            53) compare_port_to_service_assigned  "${resolver}"
+            53) compare_port_to_service_assigned  "${resolver}" "${service_name}" 53
                 ;;
-            80) compare_port_to_service_assigned  "${web_server}"
+            80) compare_port_to_service_assigned  "${web_server}" "${service_name}" 80
                 ;;
-            4711) compare_port_to_service_assigned  "${ftl}"
+            4711) compare_port_to_service_assigned  "${ftl}" "${service_name}" 4711
                 ;;
             # If it's not a default port that Pi-hole needs, just print it out for the user to see
             *) log_write "${port_number} ${service_name} (${protocol_type})";
@@ -784,7 +795,7 @@ dig_at() {
     # This helps emulate queries to different domains that a user might query
     # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains
     local random_url
-    random_url=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1")
+    random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}")
 
     # First, do a dig on localhost to see if Pi-hole can use itself to block a domain
     if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then
@@ -966,7 +977,8 @@ list_files_in_dir() {
         if [[ -d "${dir_to_parse}/${each_file}" ]]; then
             # If it's a directoy, do nothing
             :
-        elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \
+        elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_BLOCKLIST_FILE}" ]] || \
+            [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \
             [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \
             [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \
             [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_SETUP_VARS_FILE}" ]] || \
@@ -1051,70 +1063,31 @@ head_tail_log() {
     IFS="$OLD_IFS"
 }
 
-show_db_entries() {
-    local title="${1}"
-    local query="${2}"
-    local widths="${3}"
-
-    echo_current_diagnostic "${title}"
-
-    OLD_IFS="$IFS"
-    IFS=$'\r\n'
-    local entries=()
-    mapfile -t entries < <(\
-        sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \
-            -cmd ".headers on" \
-            -cmd ".mode column" \
-            -cmd ".width ${widths}" \
-            "${query}"\
-    )
-
-    for line in "${entries[@]}"; do
-        log_write "   ${line}"
-    done
-
-    IFS="$OLD_IFS"
-}
-
-show_adlists() {
-    show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50"
-}
-
-show_whitelist() {
-    show_db_entries "Whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50"
-}
-
-show_blacklist() {
-    show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
-}
-
-show_regexlist() {
-    show_db_entries "Regexlist" "SELECT * FROM regex" "4 100 7 10 13 50"
-}
-
 analyze_gravity_list() {
-    echo_current_diagnostic "Gravity List and Database"
-
-    local gravity_permissions
-    gravity_permissions=$(ls -ld "${PIHOLE_GRAVITY_DB_FILE}")
-    log_write "${COL_GREEN}${gravity_permissions}${COL_NC}"
-
-    local gravity_size
-    gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity")
-    log_write "   Size (excluding blacklist): ${COL_CYAN}${gravity_size}${COL_NC} entries"
-    log_write ""
-
+    echo_current_diagnostic "Gravity list"
+    local head_line
+    local tail_line
+    # Put the current Internal Field Separator into another variable so it can be restored later
     OLD_IFS="$IFS"
+    # Get the lines that are in the file(s) and store them in an array for parsing later
     IFS=$'\r\n'
-    local gravity_sample=()
-    mapfile -t gravity_sample < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10")
-    log_write "   ${COL_CYAN}----- First 10 Domains -----${COL_NC}"
-
-    for line in "${gravity_sample[@]}"; do
-        log_write "   ${line}"
+    local gravity_permissions
+    gravity_permissions=$(ls -ld "${PIHOLE_BLOCKLIST_FILE}")
+    log_write "${COL_GREEN}${gravity_permissions}${COL_NC}"
+    local gravity_head=()
+    mapfile -t gravity_head < <(head -n 4 ${PIHOLE_BLOCKLIST_FILE})
+    log_write "   ${COL_CYAN}-----head of $(basename ${PIHOLE_BLOCKLIST_FILE})------${COL_NC}"
+    for head_line in "${gravity_head[@]}"; do
+        log_write "   ${head_line}"
     done
-
     log_write ""
+    local gravity_tail=()
+    mapfile -t gravity_tail < <(tail -n 4 ${PIHOLE_BLOCKLIST_FILE})
+    log_write "   ${COL_CYAN}-----tail of $(basename ${PIHOLE_BLOCKLIST_FILE})------${COL_NC}"
+    for tail_line in "${gravity_tail[@]}"; do
+        log_write "   ${tail_line}"
+    done
+    # Set the IFS back to what it was
     IFS="$OLD_IFS"
 }
 
@@ -1265,10 +1238,6 @@ process_status
 parse_setup_vars
 check_x_headers
 analyze_gravity_list
-show_adlists
-show_whitelist
-show_blacklist
-show_regexlist
 show_content_of_pihole_files
 parse_locale
 analyze_pihole_log

advanced/Scripts/piholeLogFlush.sh

@@ -39,9 +39,8 @@ if [[ "$@" == *"once"* ]]; then
         # Note that moving the file is not an option, as
         # dnsmasq would happily continue writing into the
         # moved file (it will have the same file handler)
-        cp -p /var/log/pihole.log /var/log/pihole.log.1
+        cp /var/log/pihole.log /var/log/pihole.log.1
         echo " " > /var/log/pihole.log
-        chmod 644 /var/log/pihole.log
     fi
 else
     # Manual flushing
@@ -54,7 +53,6 @@ else
         echo " " > /var/log/pihole.log
         if [ -f /var/log/pihole.log.1 ]; then
             echo " " > /var/log/pihole.log.1
-            chmod 644 /var/log/pihole.log.1
         fi
     fi
     # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)

advanced/Scripts/query.sh

@@ -11,7 +11,8 @@
 
 # Globals
 piholeDir="/etc/pihole"
-gravityDBfile="${piholeDir}/gravity.db"
+adListsList="$piholeDir/adlists.list"
+wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
 options="$*"
 adlist=""
 all=""
@@ -22,10 +23,27 @@ matchType="match"
 colfile="/opt/pihole/COL_TABLE"
 source "${colfile}"
 
+# Print each subdomain
+# e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com"
+processWildcards() {
+    IFS="." read -r -a array <<< "${1}"
+    for (( i=${#array[@]}-1; i>=0; i-- )); do
+        ar=""
+        for (( j=${#array[@]}-1; j>${#array[@]}-i-2; j-- )); do
+            if [[ $j == $((${#array[@]}-1)) ]]; then
+                ar="${array[$j]}"
+            else
+                ar="${array[$j]}.${ar}"
+            fi
+        done
+        echo "${ar}"
+    done
+}
+
 # Scan an array of files for matching strings
 scanList(){
     # Escape full stops
-    local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" type="${3:-}"
+    local domain="${1//./\\.}" lists="${2}" type="${3:-}"
 
     # Prevent grep from printing file path
     cd "$piholeDir" || exit 1
@@ -36,14 +54,9 @@ scanList(){
     # /dev/null forces filename to be printed when only one list has been generated
     # shellcheck disable=SC2086
     case "${type}" in
-		"exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
-		# Create array of regexps
-		# Iterate through each regexp and check whether it matches the domainQuery
-		# If it does, print the matching regexp and continue looping
-		# Input 1 - regexps | Input 2 - domainQuery
-		"regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \
-					<(echo "${lists}") <(echo "${domain}") 2>/dev/null;;
-		*       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
+        "exact" ) grep -i -E "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
+        "wc"    ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;;
+        *       ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;;
     esac
 }
 
@@ -60,6 +73,11 @@ Options:
   exit 0
 fi
 
+if [[ ! -e "$adListsList" ]]; then
+    echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}"
+    exit 1
+fi
+
 # Handle valid options
 if [[ "${options}" == *"-bp"* ]]; then
     exact="exact"; blockpage=true
@@ -89,80 +107,48 @@ if [[ -n "${str:-}" ]]; then
     exit 1
 fi
 
-scanDatabaseTable() {
-    local domain table type querystr result
-    domain="$(printf "%q" "${1}")"
-    table="${2}"
-    type="${3:-}"
-
-    # As underscores are legitimate parts of domains, we escape them when using the LIKE operator.
-    # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this
-    # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
-    # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
-    case "${type}" in
-		"exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
-		*       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
-    esac
-
-    # Send prepared query to gravity database
-    result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
-    if [[ -z "${result}" ]]; then
-        # Return early when there are no matches in this table
-        return
-    fi
-
-    # Mark domain as having been white-/blacklist matched (global variable)
+# Scan Whitelist and Blacklist
+lists="whitelist.txt blacklist.txt"
+mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")"
+if [[ -n "${results[*]}" ]]; then
     wbMatch=true
-
-    # Print table name
-    echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
-
-    # Loop over results and print them
-    mapfile -t results <<< "${result}"
+    # Loop through each result in order to print unique file title once
     for result in "${results[@]}"; do
+        fileName="${result%%.*}"
         if [[ -n "${blockpage}" ]]; then
             echo "π ${result}"
             exit 0
+        elif [[ -n "${exact}" ]]; then
+            echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
+        else
+            # Only print filename title once per file
+            if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
+                echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
+                fileName_prev="${fileName}"
+            fi
+        echo "   ${result#*:}"
         fi
-        echo "   ${result}"
     done
-}
+fi
 
-# Scan Whitelist and Blacklist
-scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
-scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
-
-# Scan Regex table
-mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)
-
-# If we have regexps to process
-if [[ "${#regexList[@]}" -ne 0 ]]; then
-	# Split regexps over a new line
-	str_regexList=$(printf '%s\n' "${regexList[@]}")
-	# Check domainQuery against regexps
-	mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexList}" "regex")
-	# If there were regex matches
-	if [[  "${#regexMatches[@]}" -ne 0 ]]; then
-		# Split matching regexps over a new line
-		str_regexMatches=$(printf '%s\n' "${regexMatches[@]}")
-		# Form a "matched" message
-		str_message="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}"
-		# Form a "results" message
-		str_result="${COL_BOLD}${str_regexMatches}${COL_NC}"
-		# If we are displaying more than just the source of the block
-		if [[ -z "${blockpage}" ]]; then
-			# Set the wildcard match flag
-			wcMatch=true
-			# Echo the "matched" message, indented by one space
-			echo " ${str_message}"
-			# Echo the "results" message, each line indented by three spaces
-			# shellcheck disable=SC2001
-			echo "${str_result}" | sed 's/^/   /'
-		else
-			echo "π Regex list"
-			exit 0
-		fi
-	fi
+# Scan Wildcards
+if [[ -e "${wildcardlist}" ]]; then
+    # Determine all subdomains, domain and TLDs
+    mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")"
+    for match in "${wildcards[@]}"; do
+        # Search wildcard list for matches
+        mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")"
+        if [[ -n "${results[*]}" ]]; then
+            if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then
+                wcMatch=true
+                echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:"
+            fi
+            case "${blockpage}" in
+                true ) echo "π ${wildcardlist##*/}"; exit 0;;
+                *    ) echo "   *.${match}";;
+            esac
+        fi
+    done
 fi
 
 # Get version sorted *.domains filenames (without dir path)
@@ -171,6 +157,20 @@ lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)")
 # Query blocklists for occurences of domain
 mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")"
 
+# Remove unwanted content from $results
+# Each line in $results is formatted as such: [fileName]:[line]
+# 1. Delete lines starting with #
+# 2. Remove comments after domain
+# 3. Remove hosts format IP address
+# 4. Remove any lines that no longer contain the queried domain name (in case the matched domain name was in a comment)
+esc_domain="${domainQuery//./\\.}"
+mapfile -t results <<< "$(IFS=$'\n'; sed \
+	-e "/:#/d" \
+	-e "s/[ \\t]#.*//g" \
+	-e "s/:.*[ \\t]/:/g" \
+	-e "/${esc_domain}/!d" \
+	<<< "${results[*]}")"
+
 # Handle notices
 if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then
     echo -e "  ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists"
@@ -184,24 +184,13 @@ elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then
     exit 0
 fi
 
-# Remove unwanted content from non-exact $results
-if [[ -z "${exact}" ]]; then
-    # Delete lines starting with #
-    # Remove comments after domain
-    # Remove hosts format IP address
-    mapfile -t results <<< "$(IFS=$'\n'; sed \
-        -e "/:#/d" \
-        -e "s/[ \\t]#.*//g" \
-        -e "s/:.*[ \\t]/:/g" \
-        <<< "${results[*]}")"
-    # Exit if result was in a comment
-    [[ -z "${results[*]}" ]] && exit 0
-fi
-
 # Get adlist file content as array
 if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
-    # Retrieve source URLs from gravity database
-    mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
+    for adlistUrl in $(< "${adListsList}"); do
+        if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then
+            adlists+=("${adlistUrl}")
+        fi
+    done
 fi
 
 # Print "Exact matches for" title

advanced/Scripts/update.sh

@@ -31,7 +31,6 @@ source "/opt/pihole/COL_TABLE"
 # make_repo() sourced from basic-install.sh
 # update_repo() source from basic-install.sh
 # getGitFiles() sourced from basic-install.sh
-# get_binary_name() sourced from basic-install.sh
 # FTLcheckUpdate() sourced from basic-install.sh
 
 GitCheckUpdateAvail() {
@@ -129,7 +128,12 @@ main() {
         fi
     fi
 
-    if FTLcheckUpdate > /dev/null; then
+    local funcOutput
+    funcOutput=$(get_binary_name) #Store output of get_binary_name here
+    local binary
+    binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL)
+
+    if FTLcheckUpdate "${binary}" > /dev/null; then
         FTL_update=true
         echo -e "  ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}"
     else
@@ -194,6 +198,14 @@ main() {
         ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --reconfigure --unattended || \
         echo -e "${basicError}" && exit 1
     fi
+
+    if [[ "${FTL_update}" == true || "${core_update}" == true || "${web_update}" == true ]]; then
+        # Force an update of the updatechecker
+        /opt/pihole/updatecheck.sh
+        /opt/pihole/updatecheck.sh x remote
+        echo -e "  ${INFO} Local version file information updated."
+    fi
+
     echo ""
     exit 0
 }

advanced/Scripts/updatecheck.sh

@@ -51,7 +51,6 @@ if [[ "$2" == "remote" ]]; then
 
     GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")"
     echo -n "${GITHUB_CORE_VERSION}" > "${GITHUB_VERSION_FILE}"
-    chmod 644 "${GITHUB_VERSION_FILE}"
 
     if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
         GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")"
@@ -67,7 +66,6 @@ else
 
     CORE_BRANCH="$(get_local_branch /etc/.pihole)"
     echo -n "${CORE_BRANCH}" > "${LOCAL_BRANCH_FILE}"
-    chmod 644 "${LOCAL_BRANCH_FILE}"
 
     if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
         WEB_BRANCH="$(get_local_branch /var/www/html/admin)"
@@ -81,7 +79,6 @@ else
 
     CORE_VERSION="$(get_local_version /etc/.pihole)"
     echo -n "${CORE_VERSION}" > "${LOCAL_VERSION_FILE}"
-    chmod 644 "${LOCAL_VERSION_FILE}"
 
     if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
         WEB_VERSION="$(get_local_version /var/www/html/admin)"

advanced/Scripts/webpage.sh

@@ -16,8 +16,7 @@ readonly dhcpconfig="/etc/dnsmasq.d/02-pihole-dhcp.conf"
 readonly FTLconf="/etc/pihole/pihole-FTL.conf"
 # 03 -> wildcards
 readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf"
-
-readonly gravityDBfile="/etc/pihole/gravity.db"
+readonly PI_HOLE_BIN_DIR="/usr/local/bin"
 
 coltable="/opt/pihole/COL_TABLE"
 if [[ -f ${coltable} ]]; then
@@ -212,6 +211,11 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423
         add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}"
         add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}"
     fi
+
+    # Prevent Firefox from automatically switching over to DNS-over-HTTPS
+    # This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https
+    # (sourced 7th September 2019)
+    add_dnsmasq_setting "server=/use-application-dns.net/"
 }
 
 SetDNSServers() {
@@ -276,7 +280,7 @@ Reboot() {
 }
 
 RestartDNS() {
-    /usr/local/bin/pihole restartdns
+    "${PI_HOLE_BIN_DIR}"/pihole restartdns
 }
 
 SetQueryLogOptions() {
@@ -324,7 +328,6 @@ dhcp-option=option:router,${DHCP_ROUTER}
 dhcp-leasefile=/etc/pihole/dhcp.leases
 #quiet-dhcp
 " > "${dhcpconfig}"
-    chmod 644 "${dhcpconfig}"
 
     if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then
         echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}"
@@ -396,17 +399,19 @@ SetWebUILayout() {
 }
 
 CustomizeAdLists() {
-    local address
-    address="${args[3]}"
+    list="/etc/pihole/adlists.list"
 
     if [[ "${args[2]}" == "enable" ]]; then
-        sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'"
+        sed -i "\\@${args[3]}@s/^#http/http/g" "${list}"
     elif [[ "${args[2]}" == "disable" ]]; then
-        sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'"
+        sed -i "\\@${args[3]}@s/^http/#http/g" "${list}"
     elif [[ "${args[2]}" == "add" ]]; then
-        sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address) VALUES ('${address}')"
+        if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then
+            echo "${args[3]}" >> ${list}
+        fi
     elif [[ "${args[2]}" == "del" ]]; then
-        sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'"
+        var=$(echo "${args[3]}" | sed 's/\//\\\//g')
+        sed -i "/${var}/Id" "${list}"
     else
         echo "Not permitted"
         return 1
@@ -550,13 +555,11 @@ addAudit()
     do
         echo "${var}" >> /etc/pihole/auditlog.list
     done
-    chmod 644 /etc/pihole/auditlog.list
 }
 
 clearAudit()
 {
     echo -n "" > /etc/pihole/auditlog.list
-    chmod 644 /etc/pihole/auditlog.list
 }
 
 SetPrivacyLevel() {

advanced/Templates/gravity.db.sql

@@ -1,142 +0,0 @@
-PRAGMA FOREIGN_KEYS=ON;
-
-CREATE TABLE "group"
-(
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	name TEXT NOT NULL,
-	description TEXT
-);
-
-CREATE TABLE whitelist
-(
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	domain TEXT UNIQUE NOT NULL,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
-);
-
-CREATE TABLE whitelist_by_group
-(
-	whitelist_id INTEGER NOT NULL REFERENCES whitelist (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (whitelist_id, group_id)
-);
-
-CREATE TABLE blacklist
-(
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	domain TEXT UNIQUE NOT NULL,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
-);
-
-CREATE TABLE blacklist_by_group
-(
-	blacklist_id INTEGER NOT NULL REFERENCES blacklist (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (blacklist_id, group_id)
-);
-
-CREATE TABLE regex
-(
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	domain TEXT UNIQUE NOT NULL,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
-);
-
-CREATE TABLE regex_by_group
-(
-	regex_id INTEGER NOT NULL REFERENCES regex (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (regex_id, group_id)
-);
-
-CREATE TABLE adlist
-(
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	address TEXT UNIQUE NOT NULL,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
-);
-
-CREATE TABLE adlist_by_group
-(
-	adlist_id INTEGER NOT NULL REFERENCES adlist (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (adlist_id, group_id)
-);
-
-CREATE TABLE gravity
-(
-	domain TEXT PRIMARY KEY
-);
-CREATE TABLE info
-(
-	property TEXT PRIMARY KEY,
-	value TEXT NOT NULL
-);
-
-INSERT INTO info VALUES("version","1");
-
-CREATE VIEW vw_gravity AS SELECT domain
-    FROM gravity
-    WHERE domain NOT IN (SELECT domain from vw_whitelist);
-
-CREATE VIEW vw_whitelist AS SELECT DISTINCT domain
-    FROM whitelist
-    LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id
-    LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id
-    WHERE whitelist.enabled = 1 AND (whitelist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY whitelist.id;
-
-CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
-    BEGIN
-      UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
-    END;
-
-CREATE VIEW vw_blacklist AS SELECT DISTINCT domain
-    FROM blacklist
-    LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id
-    LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id
-    WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY blacklist.id;
-
-CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
-    BEGIN
-      UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
-    END;
-
-CREATE VIEW vw_regex AS SELECT DISTINCT domain
-    FROM regex
-    LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id
-    LEFT JOIN "group" ON "group".id = regex_by_group.group_id
-    WHERE regex.enabled = 1 AND (regex_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY regex.id;
-
-CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
-    BEGIN
-      UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
-    END;
-
-CREATE VIEW vw_adlist AS SELECT DISTINCT address
-    FROM adlist
-    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id
-    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
-    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY adlist.id;
-
-CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist
-    BEGIN
-      UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address;
-    END;
-

advanced/bash-completion/pihole

@@ -7,7 +7,7 @@ _pihole() {
 
 	case "${prev}" in
 		"pihole")
-			opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush"
+			opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist"
 			COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
 		;;
 		"whitelist"|"blacklist"|"wildcard"|"regex")

automated install/basic-install.sh

@@ -65,6 +65,7 @@ PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update versi
 # This directory is where the Pi-hole scripts will be installed
 PI_HOLE_INSTALL_DIR="/opt/pihole"
 PI_HOLE_CONFIG_DIR="/etc/pihole"
+PI_HOLE_BIN_DIR="/usr/local/bin"
 PI_HOLE_BLOCKPAGE_DIR="${webroot}/pihole"
 useUpdateVars=false
 
@@ -74,8 +75,8 @@ regexFile="/etc/pihole/regex.list"
 # this script can run
 IPV4_ADDRESS=""
 IPV6_ADDRESS=""
-# By default, query logging is disabled and the dashboard is set to be installed
-QUERY_LOGGING=false
+# By default, query logging is enabled and the dashboard is set to be installed
+QUERY_LOGGING=true
 INSTALL_WEB_INTERFACE=true
 PRIVACY_LEVEL=0
 
@@ -84,8 +85,13 @@ if [ -z "${USER}" ]; then
 fi
 
 
-# Find the rows and columns will default to 80x24 if it can not be detected
-screen_size=$(stty size || printf '%d %d' 24 80)
+# Check if we are running on a real terminal and find the rows and columns
+# If there is no real terminal, we will default to 80x24
+if [ -t 0 ] ; then
+  screen_size=$(stty size)
+else
+  screen_size="24 80"
+fi
 # Set rows variable to contain first number
 printf -v rows '%d' "${screen_size%% *}"
 # Set columns variable to contain second number
@@ -118,7 +124,7 @@ done
 # If the color table file exists,
 if [[ -f "${coltable}" ]]; then
     # source it
-    source "${coltable}"
+    source ${coltable}
 # Otherwise,
 else
     # Set these values so the installer can still run in color
@@ -133,9 +139,6 @@ else
     OVER="\\r\\033[K"
 fi
 
-# Define global binary variable
-binary="tbd"
-
 # A simple function that just echoes out our logo in ASCII format
 # This lets users know that it is a Pi-hole, LLC product
 show_ascii_berry() {
@@ -182,26 +185,26 @@ if is_command apt-get ; then
     # A variable to store the command used to update the package cache
     UPDATE_PKG_CACHE="${PKG_MANAGER} update"
     # An array for something...
-    PKG_INSTALL=("${PKG_MANAGER}" --yes --no-install-recommends install)
+    PKG_INSTALL=(${PKG_MANAGER} --yes --no-install-recommends install)
     # grep -c will return 1 retVal on 0 matches, block this throwing the set -e with an OR TRUE
     PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true"
     # Some distros vary slightly so these fixes for dependencies may apply
     # on Ubuntu 18.04.1 LTS we need to add the universe repository to gain access to dialog and dhcpcd5
     APT_SOURCES="/etc/apt/sources.list"
     if awk 'BEGIN{a=1;b=0}/bionic main/{a=0}/bionic.*universe/{b=1}END{exit a + b}' ${APT_SOURCES}; then
-        if ! whiptail --defaultno --title "Dependencies Require Update to Allowed Repositories" --yesno "Would you like to enable 'universe' repository?\\n\\nThis repository is required by the following packages:\\n\\n- dhcpcd5\\n- dialog" "${r}" "${c}"; then
+        if ! whiptail --defaultno --title "Dependencies Require Update to Allowed Repositories" --yesno "Would you like to enable 'universe' repository?\\n\\nThis repository is required by the following packages:\\n\\n- dhcpcd5\\n- dialog" ${r} ${c}; then
             printf "  %b Aborting installation: dependencies could not be installed.\\n" "${CROSS}"
             exit # exit the installer
         else
             printf "  %b Enabling universe package repository for Ubuntu Bionic\\n" "${INFO}"
-            cp -p ${APT_SOURCES} ${APT_SOURCES}.backup # Backup current repo list
+            cp ${APT_SOURCES} ${APT_SOURCES}.backup # Backup current repo list
             printf "  %b Backed up current configuration to %s\\n" "${TICK}" "${APT_SOURCES}.backup"
             add-apt-repository universe
             printf "  %b Enabled %s\\n" "${TICK}" "'universe' repository"
         fi
     fi
     # Debian 7 doesn't have iproute2 so if the dry run install is successful,
-    if "${PKG_MANAGER}" install --dry-run iproute2 > /dev/null 2>&1; then
+    if ${PKG_MANAGER} install --dry-run iproute2 > /dev/null 2>&1; then
         # we can install it
         iproute_pkg="iproute2"
     # Otherwise,
@@ -222,7 +225,7 @@ if is_command apt-get ; then
     # Check if installed php is v 7.0, or newer to determine packages to install
     if [[ "$phpInsNewer" != true ]]; then
         # Prefer the php metapackage if it's there
-        if "${PKG_MANAGER}" install --dry-run php > /dev/null 2>&1; then
+        if ${PKG_MANAGER} install --dry-run php > /dev/null 2>&1; then
             phpVer="php"
         # fall back on the php5 packages
         else
@@ -233,19 +236,19 @@ if is_command apt-get ; then
         phpVer="php$phpInsMajor.$phpInsMinor"
     fi
     # We also need the correct version for `php-sqlite` (which differs across distros)
-    if "${PKG_MANAGER}" install --dry-run "${phpVer}-sqlite3" > /dev/null 2>&1; then
+    if ${PKG_MANAGER} install --dry-run ${phpVer}-sqlite3 > /dev/null 2>&1; then
         phpSqlite="sqlite3"
     else
         phpSqlite="sqlite"
     fi
     # Since our install script is so large, we need several other programs to successfully get a machine provisioned
     # These programs are stored in an array so they can be looped through later
-    INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git "${iproute_pkg}" whiptail)
+    INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git ${iproute_pkg} whiptail)
     # Pi-hole itself has several dependencies that also need to be installed
     PIHOLE_DEPS=(cron curl dnsutils iputils-ping lsof netcat psmisc sudo unzip wget idn2 sqlite3 libcap2-bin dns-root-data resolvconf libcap2)
     # The Web dashboard has some that also need to be installed
     # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code
-    PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-${phpSqlite}")
+    PIHOLE_WEB_DEPS=(lighttpd ${phpVer}-common ${phpVer}-cgi ${phpVer}-${phpSqlite})
     # The Web server user,
     LIGHTTPD_USER="www-data"
     # group,
@@ -281,7 +284,7 @@ elif is_command rpm ; then
 
     # Fedora and family update cache on every PKG_INSTALL call, no need for a separate update.
     UPDATE_PKG_CACHE=":"
-    PKG_INSTALL=("${PKG_MANAGER}" install -y)
+    PKG_INSTALL=(${PKG_MANAGER} install -y)
     PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l"
     INSTALLER_DEPS=(dialog git iproute newt procps-ng which chkconfig)
     PIHOLE_DEPS=(bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc sqlite libcap)
@@ -319,7 +322,7 @@ elif is_command rpm ; then
 
         # The default php on CentOS 7.x is 5.4 which is EOL
         # Check if the version of PHP available via installed repositories is >= to PHP 7
-        AVAILABLE_PHP_VERSION=$("${PKG_MANAGER}" info php | grep -i version | grep -o '[0-9]\+' | head -1)
+        AVAILABLE_PHP_VERSION=$(${PKG_MANAGER} info php | grep -i version | grep -o '[0-9]\+' | head -1)
         if [[ $AVAILABLE_PHP_VERSION -ge $SUPPORTED_CENTOS_PHP_VERSION ]]; then
             # Since PHP 7 is available by default, install via default PHP package names
             : # do nothing as PHP is current
@@ -329,7 +332,7 @@ elif is_command rpm ; then
             rpm -q ${REMI_PKG} &> /dev/null || rc=$?
         if [[ $rc -ne 0 ]]; then
             # The PHP version available via default repositories is older than version 7
-            if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" "${r}" "${c}"; then
+            if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" ${r} ${c}; then
                 # User decided to NOT update PHP from REMI, attempt to install the default available PHP version
                 printf "  %b User opt-out of PHP 7 upgrade on CentOS. Deprecated PHP may be in use.\\n" "${INFO}"
                 : # continue with unsupported php version
@@ -352,7 +355,7 @@ elif is_command rpm ; then
     fi
     else
         # Warn user of unsupported version of Fedora or CentOS
-        if ! whiptail --defaultno --title "Unsupported RPM based distribution" --yesno "Would you like to continue installation on an unsupported RPM based distribution?\\n\\nPlease ensure the following packages have been installed manually:\\n\\n- lighttpd\\n- lighttpd-fastcgi\\n- PHP version 7+" "${r}" "${c}"; then
+        if ! whiptail --defaultno --title "Unsupported RPM based distribution" --yesno "Would you like to continue installation on an unsupported RPM based distribution?\\n\\nPlease ensure the following packages have been installed manually:\\n\\n- lighttpd\\n- lighttpd-fastcgi\\n- PHP version 7+" ${r} ${c}; then
             printf "  %b Aborting installation due to unsupported RPM based distribution\\n" "${CROSS}"
             exit # exit the installer
         else
@@ -414,9 +417,6 @@ make_repo() {
     fi
     # Clone the repo and return the return code from this command
     git clone -q --depth 20 "${remoteRepo}" "${directory}" &> /dev/null || return $?
-    # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git)
-    chmod -R a+rX "${directory}"
-
     # Show a colored message showing it's status
     printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
     # Always return 0? Not sure this is correct
@@ -450,8 +450,6 @@ update_repo() {
     git pull --quiet &> /dev/null || return $?
     # Show a completion message
     printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
-    # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git)
-    chmod -R a+rX "${directory}"
     # Move back into the original directory
     cd "${curdir}" &> /dev/null || return 1
     return 0
@@ -499,8 +497,6 @@ resetRepo() {
     printf "  %b %s..." "${INFO}" "${str}"
     # Use git to remove the local changes
     git reset --hard &> /dev/null || return $?
-    # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git)
-    chmod -R a+rX "${directory}"
     # And show the status
     printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
     # Returning success anyway?
@@ -544,15 +540,15 @@ get_available_interfaces() {
 # A function for displaying the dialogs the user sees when first running the installer
 welcomeDialogs() {
     # Display the welcome dialog using an appropriately sized window via the calculation conducted earlier in the script
-    whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "\\n\\nThis installer will transform your device into a network-wide ad blocker!" "${r}" "${c}"
+    whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "\\n\\nThis installer will transform your device into a network-wide ad blocker!" ${r} ${c}
 
     # Request that users donate if they enjoy the software since we all work on it in our free time
-    whiptail --msgbox --backtitle "Plea" --title "Free and open source" "\\n\\nThe Pi-hole is free, but powered by your donations:  http://pi-hole.net/donate" "${r}" "${c}"
+    whiptail --msgbox --backtitle "Plea" --title "Free and open source" "\\n\\nThe Pi-hole is free, but powered by your donations:  http://pi-hole.net/donate" ${r} ${c}
 
     # Explain the need for a static address
     whiptail --msgbox --backtitle "Initiating network interface" --title "Static IP Needed" "\\n\\nThe Pi-hole is a SERVER so it needs a STATIC IP ADDRESS to function properly.
 
-In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." "${r}" "${c}"
+In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." ${r} ${c}
 }
 
 # We need to make sure there is enough space before installing, so there is a function to check this
@@ -639,7 +635,7 @@ chooseInterface() {
         # Feed the available interfaces into this while loop
         done <<< "${availableInterfaces}"
         # The whiptail command that will be run, stored in a variable
-        chooseInterfaceCmd=(whiptail --separate-output --radiolist "Choose An Interface (press space to select)" "${r}" "${c}" "${interfaceCount}")
+        chooseInterfaceCmd=(whiptail --separate-output --radiolist "Choose An Interface (press space to select)" ${r} ${c} ${interfaceCount})
         # Now run the command using the interfaces saved into the array
         chooseInterfaceOptions=$("${chooseInterfaceCmd[@]}" "${interfacesArray[@]}" 2>&1 >/dev/tty) || \
         # If the user chooses Cancel, exit
@@ -720,7 +716,7 @@ useIPv6dialog() {
     # If the IPV6_ADDRESS contains a value
     if [[ ! -z "${IPV6_ADDRESS}" ]]; then
         # Display that IPv6 is supported and will be used
-        whiptail --msgbox --backtitle "IPv6..." --title "IPv6 Supported" "$IPV6_ADDRESS will be used to block ads." "${r}" "${c}"
+        whiptail --msgbox --backtitle "IPv6..." --title "IPv6 Supported" "$IPV6_ADDRESS will be used to block ads." ${r} ${c}
     fi
 }
 
@@ -730,7 +726,7 @@ use4andor6() {
     local useIPv4
     local useIPv6
     # Let use select IPv4 and/or IPv6 via a checklist
-    cmd=(whiptail --separate-output --checklist "Select Protocols (press space to select)" "${r}" "${c}" 2)
+    cmd=(whiptail --separate-output --checklist "Select Protocols (press space to select)" ${r} ${c} 2)
     # In an array, show the options available:
     # IPv4 (on by default)
     options=(IPv4 "Block ads over IPv4" on
@@ -779,11 +775,11 @@ getStaticIPv4Settings() {
     # This is useful for users that are using DHCP reservations; then we can just use the information gathered via our functions
     if whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address?
           IP address:    ${IPV4_ADDRESS}
-          Gateway:       ${IPv4gw}" "${r}" "${c}"; then
+          Gateway:       ${IPv4gw}" ${r} ${c}; then
         # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict.
         whiptail --msgbox --backtitle "IP information" --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict.  But in most cases the router is smart enough to not do that.
 If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want.
-It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." "${r}" "${c}"
+It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." ${r} ${c}
     # Nothing else to do since the variables are already set above
     else
     # Otherwise, we need to ask the user to input their desired settings.
@@ -792,13 +788,13 @@ It is also possible to use a DHCP reservation, but if you are going to do that,
     until [[ "${ipSettingsCorrect}" = True ]]; do
 
         # Ask for the IPv4 address
-        IPV4_ADDRESS=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" "${r}" "${c}" "${IPV4_ADDRESS}" 3>&1 1>&2 2>&3) || \
+        IPV4_ADDRESS=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" ${r} ${c} "${IPV4_ADDRESS}" 3>&1 1>&2 2>&3) || \
         # Cancelling IPv4 settings window
         { ipSettingsCorrect=False; echo -e "  ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; }
         printf "  %b Your static IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}"
 
         # Ask for the gateway
-        IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" "${r}" "${c}" "${IPv4gw}" 3>&1 1>&2 2>&3) || \
+        IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" ${r} ${c} "${IPv4gw}" 3>&1 1>&2 2>&3) || \
         # Cancelling gateway settings window
         { ipSettingsCorrect=False; echo -e "  ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; }
         printf "  %b Your static IPv4 gateway: %s\\n" "${INFO}" "${IPv4gw}"
@@ -806,7 +802,7 @@ It is also possible to use a DHCP reservation, but if you are going to do that,
         # Give the user a chance to review their settings before moving on
         if whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct?
             IP address: ${IPV4_ADDRESS}
-            Gateway:    ${IPv4gw}" "${r}" "${c}"; then
+            Gateway:    ${IPv4gw}" ${r} ${c}; then
                 # After that's done, the loop ends and we move on
                 ipSettingsCorrect=True
         else
@@ -854,7 +850,7 @@ setIFCFG() {
         # Put the IP in variables without the CIDR notation
         printf -v CIDR "%s" "${IPV4_ADDRESS##*/}"
         # Backup existing interface configuration:
-        cp -p "${IFCFG_FILE}" "${IFCFG_FILE}".pihole.orig
+        cp "${IFCFG_FILE}" "${IFCFG_FILE}".pihole.orig
         # Build Interface configuration file using the GLOBAL variables we have
         {
         echo "# Configured via Pi-hole installer"
@@ -868,8 +864,6 @@ setIFCFG() {
         echo "DNS2=$PIHOLE_DNS_2"
         echo "USERCTL=no"
         }> "${IFCFG_FILE}"
-        chmod 644 "${IFCFG_FILE}"
-        chown root:root "${IFCFG_FILE}"
         # Use ip to immediately set the new address
         ip addr replace dev "${PIHOLE_INTERFACE}" "${IPV4_ADDRESS}"
         # If NetworkMangler command line interface exists and ready to mangle,
@@ -934,7 +928,7 @@ valid_ip() {
         # and set the new one to a dot (period)
         IFS='.'
         # Put the IP into an array
-        ip=("${ip}")
+        ip=(${ip})
         # Restore the IFS to what it was
         IFS=${OIFS}
         ## Evaluate each octet by checking if it's less than or equal to 255 (the max for each octet)
@@ -944,7 +938,7 @@ valid_ip() {
         stat=$?
     fi
     # Return the exit code
-    return "${stat}"
+    return ${stat}
 }
 
 # A function to choose the upstream DNS provider(s)
@@ -974,7 +968,7 @@ setDNS() {
     # Restore the IFS to what it was
     IFS=${OIFS}
     # In a whiptail dialog, show the options
-    DNSchoices=$(whiptail --separate-output --menu "Select Upstream DNS Provider. To use your own, select Custom." "${r}" "${c}" 7 \
+    DNSchoices=$(whiptail --separate-output --menu "Select Upstream DNS Provider. To use your own, select Custom." ${r} ${c} 7 \
     "${DNSChooseOptions[@]}" 2>&1 >/dev/tty) || \
     # exit if Cancel is selected
     { printf "  %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; }
@@ -1004,7 +998,7 @@ setDNS() {
             fi
 
             # Dialog for the user to enter custom upstream servers
-            piholeDNS=$(whiptail --backtitle "Specify Upstream DNS Provider(s)"  --inputbox "Enter your desired upstream DNS provider(s), separated by a comma.\\n\\nFor example '8.8.8.8, 8.8.4.4'" "${r}" "${c}" "${prePopulate}" 3>&1 1>&2 2>&3) || \
+            piholeDNS=$(whiptail --backtitle "Specify Upstream DNS Provider(s)"  --inputbox "Enter your desired upstream DNS provider(s), separated by a comma.\\n\\nFor example '8.8.8.8, 8.8.4.4'" ${r} ${c} "${prePopulate}" 3>&1 1>&2 2>&3) || \
             { printf "  %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; }
             # Clean user input and replace whitespace with comma.
             piholeDNS=$(sed 's/[, \t]\+/,/g' <<< "${piholeDNS}")
@@ -1037,7 +1031,7 @@ setDNS() {
             # Otherwise,
             else
                 # Show the settings
-                if (whiptail --backtitle "Specify Upstream DNS Provider(s)" --title "Upstream DNS Provider(s)" --yesno "Are these settings correct?\\n    DNS Server 1:   $PIHOLE_DNS_1\\n    DNS Server 2:   ${PIHOLE_DNS_2}" "${r}" "${c}"); then
+                if (whiptail --backtitle "Specify Upstream DNS Provider(s)" --title "Upstream DNS Provider(s)" --yesno "Are these settings correct?\\n    DNS Server 1:   $PIHOLE_DNS_1\\n    DNS Server 2:   ${PIHOLE_DNS_2}" ${r} ${c}); then
                 # and break from the loop since the servers are valid
                 DNSSettingsCorrect=True
                 # Otherwise,
@@ -1076,21 +1070,21 @@ setLogging() {
     local LogChoices
 
     # Ask if the user wants to log queries
-    LogToggleCommand=(whiptail --separate-output --radiolist "Do you want to log queries to pihole.log?" "${r}" "${c}" 6)
-    # The default selection is off to reduce stressing the SD card
-    LogChooseOptions=("Off (Recommended)" "" on
-        "On" "" off)
+    LogToggleCommand=(whiptail --separate-output --radiolist "Do you want to log queries?" "${r}" "${c}" 6)
+    # The default selection is on
+    LogChooseOptions=("On (Recommended)" "" on
+        Off "" off)
     # Get the user's choice
     LogChoices=$("${LogToggleCommand[@]}" "${LogChooseOptions[@]}" 2>&1 >/dev/tty) || (printf "  %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" && exit 1)
     case ${LogChoices} in
         # If it's on
-        "On")
+        "On (Recommended)")
             printf "  %b Logging On.\\n" "${INFO}"
             # Set the GLOBAL variable to true so we know what they selected
             QUERY_LOGGING=true
             ;;
         # Otherwise, it's off,
-        "Off (Recommended)")
+        Off)
             printf "  %b Logging Off.\\n" "${INFO}"
             # So set it to false
             QUERY_LOGGING=false
@@ -1128,7 +1122,7 @@ setAdminFlag() {
     local WebChoices
 
     # Similar to the logging function, ask what the user wants
-    WebToggleCommand=(whiptail --separate-output --radiolist "Do you wish to install the web admin interface?" "${r}" "${c}" 6)
+    WebToggleCommand=(whiptail --separate-output --radiolist "Do you wish to install the web admin interface?" ${r} ${c} 6)
     # with the default being enabled
     WebChooseOptions=("On (Recommended)" "" on
         Off "" off)
@@ -1177,12 +1171,11 @@ chooseBlocklists() {
         mv "${adlistFile}" "${adlistFile}.old"
     fi
     # Let user select (or not) blocklists via a checklist
-    cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 7)
+    cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 6)
     # In an array, show the options available (all off by default):
     options=(StevenBlack "StevenBlack's Unified Hosts List" on
         MalwareDom "MalwareDomains" on
         Cameleon "Cameleon" on
-        ZeusTracker "ZeusTracker" on
         DisconTrack "Disconnect.me Tracking" on
         DisconAd "Disconnect.me Ads" on
         HostsFile "Hosts-file.net Ads" on)
@@ -1194,7 +1187,6 @@ chooseBlocklists() {
     do
         appendToListsFile "${choice}"
     done
-    chmod 644 "${adlistFile}"
 }
 
 # Accept a string parameter, it must be one of the default lists
@@ -1205,7 +1197,6 @@ appendToListsFile() {
         StevenBlack  )  echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";;
         MalwareDom   )  echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";;
         Cameleon     )  echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";;
-        ZeusTracker  )  echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";;
         DisconTrack  )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";;
         DisconAd     )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";;
         HostsFile    )  echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";;
@@ -1223,7 +1214,6 @@ installDefaultBlocklists() {
     appendToListsFile StevenBlack
     appendToListsFile MalwareDom
     appendToListsFile Cameleon
-    appendToListsFile ZeusTracker
     appendToListsFile DisconTrack
     appendToListsFile DisconAd
     appendToListsFile HostsFile
@@ -1235,7 +1225,6 @@ version_check_dnsmasq() {
     local dnsmasq_conf="/etc/dnsmasq.conf"
     local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig"
     local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list"
-    local dnsmasq_pihole_id_string2="# Dnsmasq config for Pi-hole's FTLDNS"
     local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original"
     local dnsmasq_pihole_01_snippet="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf"
     local dnsmasq_pihole_01_location="/etc/dnsmasq.d/01-pihole.conf"
@@ -1243,17 +1232,16 @@ version_check_dnsmasq() {
     # If the dnsmasq config file exists
     if [[ -f "${dnsmasq_conf}" ]]; then
         printf "  %b Existing dnsmasq.conf found..." "${INFO}"
-        # If a specific string is found within this file, we presume it's from older versions on Pi-hole,
-        if grep -q "${dnsmasq_pihole_id_string}" "${dnsmasq_conf}" ||
-           grep -q "${dnsmasq_pihole_id_string2}" "${dnsmasq_conf}"; then
+        # If gravity.list is found within this file, we presume it's from older versions on Pi-hole,
+        if grep -q ${dnsmasq_pihole_id_string} ${dnsmasq_conf}; then
             printf " it is from a previous Pi-hole install.\\n"
             printf "  %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}"
             # so backup the original file
-            mv -f "${dnsmasq_conf}" "${dnsmasq_conf_orig}"
+            mv -f ${dnsmasq_conf} ${dnsmasq_conf_orig}
             printf "%b  %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}"  "${TICK}"
             printf "  %b Restoring default dnsmasq.conf..." "${INFO}"
             # and replace it with the default
-            install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}"
+            cp ${dnsmasq_original_config} ${dnsmasq_conf}
             printf "%b  %b Restoring default dnsmasq.conf...\\n" "${OVER}"  "${TICK}"
         # Otherwise,
         else
@@ -1264,47 +1252,47 @@ version_check_dnsmasq() {
         # If a file cannot be found,
         printf "  %b No dnsmasq.conf found... restoring default dnsmasq.conf..." "${INFO}"
         # restore the default one
-        install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}"
+        cp ${dnsmasq_original_config} ${dnsmasq_conf}
         printf "%b  %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}"  "${TICK}"
     fi
 
     printf "  %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf..." "${INFO}"
     # Check to see if dnsmasq directory exists (it may not due to being a fresh install and dnsmasq no longer being a dependency)
     if [[ ! -d "/etc/dnsmasq.d"  ]];then
-        install -d -m 755 "/etc/dnsmasq.d"
+        mkdir "/etc/dnsmasq.d"
     fi
     # Copy the new Pi-hole DNS config file into the dnsmasq.d directory
-    install -D -m 644 -T "${dnsmasq_pihole_01_snippet}" "${dnsmasq_pihole_01_location}"
+    cp ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location}
     printf "%b  %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}"  "${TICK}"
     # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier
     # First, swap in the interface to listen on
-    sed -i "s/@INT@/$PIHOLE_INTERFACE/" "${dnsmasq_pihole_01_location}"
+    sed -i "s/@INT@/$PIHOLE_INTERFACE/" ${dnsmasq_pihole_01_location}
     if [[ "${PIHOLE_DNS_1}" != "" ]]; then
         # Then swap in the primary DNS server
-        sed -i "s/@DNS1@/$PIHOLE_DNS_1/" "${dnsmasq_pihole_01_location}"
+        sed -i "s/@DNS1@/$PIHOLE_DNS_1/" ${dnsmasq_pihole_01_location}
     else
         #
-        sed -i '/^server=@DNS1@/d' "${dnsmasq_pihole_01_location}"
+        sed -i '/^server=@DNS1@/d' ${dnsmasq_pihole_01_location}
     fi
     if [[ "${PIHOLE_DNS_2}" != "" ]]; then
         # Then swap in the primary DNS server
-        sed -i "s/@DNS2@/$PIHOLE_DNS_2/" "${dnsmasq_pihole_01_location}"
+        sed -i "s/@DNS2@/$PIHOLE_DNS_2/" ${dnsmasq_pihole_01_location}
     else
         #
-        sed -i '/^server=@DNS2@/d' "${dnsmasq_pihole_01_location}"
+        sed -i '/^server=@DNS2@/d' ${dnsmasq_pihole_01_location}
     fi
 
     #
-    sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' "${dnsmasq_conf}"
+    sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' ${dnsmasq_conf}
 
     # If the user does not want to enable logging,
     if [[ "${QUERY_LOGGING}" == false ]] ; then
         # Disable it by commenting out the directive in the DNS config file
-        sed -i 's/^log-queries/#log-queries/' "${dnsmasq_pihole_01_location}"
+        sed -i 's/^log-queries/#log-queries/' ${dnsmasq_pihole_01_location}
     # Otherwise,
     else
         # enable it by uncommenting the directive in the DNS config file
-        sed -i 's/^#log-queries/log-queries/' "${dnsmasq_pihole_01_location}"
+        sed -i 's/^#log-queries/log-queries/' ${dnsmasq_pihole_01_location}
     fi
 }
 
@@ -1349,7 +1337,7 @@ installScripts() {
         install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./advanced/Scripts/*.sh
         install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./automated\ install/uninstall.sh
         install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./advanced/Scripts/COL_TABLE
-        install -o "${USER}" -Dm755 -t /usr/local/bin/ pihole
+        install -o "${USER}" -Dm755 -t "${PI_HOLE_BIN_DIR}" pihole
         install -Dm644 ./advanced/bash-completion/pihole /etc/bash_completion.d/pihole
         printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
 
@@ -1372,7 +1360,6 @@ installConfigs() {
     # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6
     # Some values may be empty (for example: DNS servers without IPv6 support)
     echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
-    chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
 
     # Install empty file if it does not exist
     if [[ ! -r "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" ]]; then
@@ -1391,18 +1378,19 @@ installConfigs() {
     if [[ "${INSTALL_WEB_SERVER}" == true ]]; then
         # and if the Web server conf directory does not exist,
         if [[ ! -d "/etc/lighttpd" ]]; then
-            # make it and set the owners
-            install -d -m 755 -o "${USER}" -g root /etc/lighttpd
+            # make it
+            mkdir /etc/lighttpd
+            # and set the owners
+            chown "${USER}":root /etc/lighttpd
         # Otherwise, if the config file already exists
         elif [[ -f "/etc/lighttpd/lighttpd.conf" ]]; then
             # back up the original
             mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig
         fi
         # and copy in the config file Pi-hole needs
-        install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf
+        cp ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf
         # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it
         touch /etc/lighttpd/external.conf
-        chmod 644 /etc/lighttpd/external.conf
         # if there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config
         if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then
             sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' /etc/lighttpd/lighttpd.conf
@@ -1433,16 +1421,16 @@ install_manpage() {
     fi
     if [[ ! -d "/usr/local/share/man/man8" ]]; then
         # if not present, create man8 directory
-        install -d -m 755 /usr/local/share/man/man8
+        mkdir /usr/local/share/man/man8
     fi
     if [[ ! -d "/usr/local/share/man/man5" ]]; then
-        # if not present, create man5 directory
-        install -d -m 755 /usr/local/share/man/man5
+        # if not present, create man8 directory
+        mkdir /usr/local/share/man/man5
     fi
     # Testing complete, copy the files & update the man db
-    install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8
-    install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8
-    install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5
+    cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8
+    cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8
+    cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5
     if mandb -q &>/dev/null; then
         # Updated successfully
         printf "%b  %b man pages installed and database updated\\n" "${OVER}" "${TICK}"
@@ -1647,7 +1635,7 @@ install_dependent_packages() {
     # Install Fedora/CentOS packages
     for i in "$@"; do
         printf "  %b Checking for %s..." "${INFO}" "${i}"
-        if "${PKG_MANAGER}" -q list installed "${i}" &> /dev/null; then
+        if ${PKG_MANAGER} -q list installed "${i}" &> /dev/null; then
             printf "%b  %b Checking for %s" "${OVER}" "${TICK}" "${i}"
         else
             printf "%b  %b Checking for %s (will be installed)" "${OVER}" "${INFO}" "${i}"
@@ -1671,7 +1659,7 @@ installPiholeWeb() {
     # Install the directory
     install -d -m 0755 ${PI_HOLE_BLOCKPAGE_DIR}
     # and the blockpage
-    install -D -m 644 ${PI_HOLE_LOCAL_REPO}/advanced/{index,blockingpage}.* ${PI_HOLE_BLOCKPAGE_DIR}/
+    install -D ${PI_HOLE_LOCAL_REPO}/advanced/{index,blockingpage}.* ${PI_HOLE_BLOCKPAGE_DIR}/
 
     # Remove superseded file
     if [[ -e "${PI_HOLE_BLOCKPAGE_DIR}/index.js" ]]; then
@@ -1698,17 +1686,17 @@ installPiholeWeb() {
     local str="Installing sudoer file"
     printf "\\n  %b %s..." "${INFO}" "${str}"
     # Make the .d directory if it doesn't exist
-    install -d -m 755 /etc/sudoers.d/
+    mkdir -p /etc/sudoers.d/
     # and copy in the pihole sudoers file
     install -m 0640 ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.sudo /etc/sudoers.d/pihole
     # Add lighttpd user (OS dependent) to sudoers file
-    echo "${LIGHTTPD_USER} ALL=NOPASSWD: /usr/local/bin/pihole" >> /etc/sudoers.d/pihole
+    echo "${LIGHTTPD_USER} ALL=NOPASSWD: ${PI_HOLE_BIN_DIR}/pihole" >> /etc/sudoers.d/pihole
 
     # If the Web server user is lighttpd,
     if [[ "$LIGHTTPD_USER" == "lighttpd" ]]; then
         # Allow executing pihole via sudo with Fedora
-        # Usually /usr/local/bin is not permitted as directory for sudoable programs
-        echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin" >> /etc/sudoers.d/pihole
+        # Usually /usr/local/bin ${PI_HOLE_BIN_DIR} is not permitted as directory for sudoable programs
+        echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:${PI_HOLE_BIN_DIR}" >> /etc/sudoers.d/pihole
     fi
     # Set the strict permissions on the file
     chmod 0440 /etc/sudoers.d/pihole
@@ -1721,8 +1709,7 @@ installCron() {
     local str="Installing latest Cron script"
     printf "\\n  %b %s..." "${INFO}" "${str}"
     # Copy the cron file over from the local repo
-    # File must not be world or group writeable and must be owned by root
-    install -D -m 644 -T -o root -g root ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.cron /etc/cron.d/pihole
+    cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.cron /etc/cron.d/pihole
     # Randomize gravity update time
     sed -i "s/59 1 /$((1 + RANDOM % 58)) $((3 + RANDOM % 2))/" /etc/cron.d/pihole
     # Randomize update checker time
@@ -1765,7 +1752,7 @@ configureFirewall() {
     # If a firewall is running,
     if firewall-cmd --state &> /dev/null; then
         # ask if the user wants to install Pi-hole's default firewall rules
-        whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" "${r}" "${c}" || \
+        whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" ${r} ${c} || \
         { printf "  %b Not installing firewall rulesets.\\n" "${INFO}"; return 0; }
         printf "  %b Configuring FirewallD for httpd and pihole-FTL\\n" "${TICK}"
         # Allow HTTP and DNS traffic
@@ -1778,7 +1765,7 @@ configureFirewall() {
     # If chain Policy is not ACCEPT or last Rule is not ACCEPT
     # then check and insert our Rules above the DROP/REJECT Rule.
         if iptables -S INPUT | head -n1 | grep -qv '^-P.*ACCEPT$' || iptables -S INPUT | tail -n1 | grep -qv '^-\(A\|P\).*ACCEPT$'; then
-            whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" "${r}" "${c}" || \
+            whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" ${r} ${c} || \
             { printf "  %b Not installing firewall rulesets.\\n" "${INFO}"; return 0; }
             printf "  %b Installing new IPTables firewall rulesets\\n" "${TICK}"
             # Check chain first, otherwise a new rule will duplicate old ones
@@ -1830,7 +1817,6 @@ finalExports() {
     echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}"
     echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}"
     }>> "${setupVars}"
-    chmod 644 "${setupVars}"
 
     # Set the privacy level
     sed -i '/PRIVACYLEVEL/d' "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf"
@@ -1853,7 +1839,7 @@ installLogrotate() {
     local str="Installing latest logrotate script"
     printf "\\n  %b %s..." "${INFO}" "${str}"
     # Copy the file over from the local repo
-    install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate
+    cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate
     # Different operating systems have different user / group
     # settings for logrotate that makes it impossible to create
     # a static logrotate file that will work with e.g.
@@ -1872,26 +1858,29 @@ installLogrotate() {
 # At some point in the future this list can be pruned, for now we'll need it to ensure updates don't break.
 # Refactoring of install script has changed the name of a couple of variables. Sort them out here.
 accountForRefactor() {
-    sed -i 's/piholeInterface/PIHOLE_INTERFACE/g' "${setupVars}"
-    sed -i 's/IPv4_address/IPV4_ADDRESS/g' "${setupVars}"
-    sed -i 's/IPv4addr/IPV4_ADDRESS/g' "${setupVars}"
-    sed -i 's/IPv6_address/IPV6_ADDRESS/g' "${setupVars}"
-    sed -i 's/piholeIPv6/IPV6_ADDRESS/g' "${setupVars}"
-    sed -i 's/piholeDNS1/PIHOLE_DNS_1/g' "${setupVars}"
-    sed -i 's/piholeDNS2/PIHOLE_DNS_2/g' "${setupVars}"
-    sed -i 's/^INSTALL_WEB=/INSTALL_WEB_INTERFACE=/' "${setupVars}"
+    sed -i 's/piholeInterface/PIHOLE_INTERFACE/g' ${setupVars}
+    sed -i 's/IPv4_address/IPV4_ADDRESS/g' ${setupVars}
+    sed -i 's/IPv4addr/IPV4_ADDRESS/g' ${setupVars}
+    sed -i 's/IPv6_address/IPV6_ADDRESS/g' ${setupVars}
+    sed -i 's/piholeIPv6/IPV6_ADDRESS/g' ${setupVars}
+    sed -i 's/piholeDNS1/PIHOLE_DNS_1/g' ${setupVars}
+    sed -i 's/piholeDNS2/PIHOLE_DNS_2/g' ${setupVars}
+    sed -i 's/^INSTALL_WEB=/INSTALL_WEB_INTERFACE=/' ${setupVars}
     # Add 'INSTALL_WEB_SERVER', if its not been applied already: https://github.com/pi-hole/pi-hole/pull/2115
     if ! grep -q '^INSTALL_WEB_SERVER=' ${setupVars}; then
         local webserver_installed=false
         if grep -q '^INSTALL_WEB_INTERFACE=true' ${setupVars}; then
             webserver_installed=true
         fi
-        echo -e "INSTALL_WEB_SERVER=$webserver_installed" >> "${setupVars}"
+        echo -e "INSTALL_WEB_SERVER=$webserver_installed" >> ${setupVars}
     fi
 }
 
 # Install base files and web interface
 installPihole() {
+    # Create the pihole user
+    create_pihole_user
+
     # If the user wants to install the Web interface,
     if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
         if [[ ! -d "${webroot}" ]]; then
@@ -1903,9 +1892,6 @@ installPihole() {
             # Set the owner and permissions
             chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} ${webroot}
             chmod 0775 ${webroot}
-            # Repair permissions if /var/www/html is not world readable
-            chmod a+rx /var/www
-            chmod a+rx /var/www/html
             # Give pihole access to the Web server group
             usermod -a -G ${LIGHTTPD_GROUP} pihole
             # If the lighttpd command is executable,
@@ -1968,7 +1954,7 @@ checkSelinux() {
         # If it's enforcing,
         if [[ "${enforceMode}" == "Enforcing" ]]; then
             # Explain Pi-hole does not support it yet
-            whiptail --defaultno --title "SELinux Enforcing Detected" --yesno "SELinux is being ENFORCED on your system! \\n\\nPi-hole currently does not support SELinux, but you may still continue with the installation.\\n\\nNote: Web Admin will not be fully functional unless you set your policies correctly\\n\\nContinue installing Pi-hole?" "${r}" "${c}" || \
+            whiptail --defaultno --title "SELinux Enforcing Detected" --yesno "SELinux is being ENFORCED on your system! \\n\\nPi-hole currently does not support SELinux, but you may still continue with the installation.\\n\\nNote: Web Admin will not be fully functional unless you set your policies correctly\\n\\nContinue installing Pi-hole?" ${r} ${c} || \
             { printf "\\n  %bSELinux Enforcing detected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; }
             printf "  %b Continuing installation with SELinux Enforcing\\n" "${INFO}"
             printf "      %b Please refer to official SELinux documentation to create a custom policy\\n" "${INFO}"
@@ -2007,7 +1993,7 @@ If you set a new IP address, you should restart the Pi.
 
 The install log is in /etc/pihole.
 
-${additional}" "${r}" "${c}"
+${additional}" ${r} ${c}
 }
 
 update_dialogs() {
@@ -2028,7 +2014,7 @@ update_dialogs() {
     opt2b="This will reset your Pi-hole and allow you to enter new settings."
 
     # Display the information to the user
-    UpdateCmd=$(whiptail --title "Existing Install Detected!" --menu "\\n\\nWe have detected an existing install.\\n\\nPlease choose from the following options: \\n($strAdd)" "${r}" "${c}" 2 \
+    UpdateCmd=$(whiptail --title "Existing Install Detected!" --menu "\\n\\nWe have detected an existing install.\\n\\nPlease choose from the following options: \\n($strAdd)" ${r} ${c} 2 \
     "${opt1a}"  "${opt1b}" \
     "${opt2a}"  "${opt2b}" 3>&2 2>&1 1>&3) || \
     { printf "  %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; }
@@ -2117,8 +2103,6 @@ checkout_pull_branch() {
     printf "  %b %s" "${INFO}" "$str"
     git checkout "${branch}" --quiet || return 1
     printf "%b  %b %s\\n" "${OVER}" "${TICK}" "$str"
-    # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git)
-    chmod -R a+rX "${directory}"
 
     git_pull=$(git pull || return 1)
 
@@ -2167,21 +2151,15 @@ clone_or_update_repos() {
 }
 
 # Download FTL binary to random temp directory and install FTL binary
+# Disable directive for SC2120 a value _can_ be passed to this function, but it is passed from an external script that sources this one
+# shellcheck disable=SC2120
 FTLinstall() {
+
     # Local, named variables
     local latesttag
     local str="Downloading and Installing FTL"
     printf "  %b %s..." "${INFO}" "${str}"
 
-    # Find the latest version tag for FTL
-    latesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep "Location" | awk -F '/' '{print $NF}')
-    # Tags should always start with v, check for that.
-    if [[ ! "${latesttag}" == v* ]]; then
-        printf "%b  %b %s\\n" "${OVER}" "${CROSS}" "${str}"
-        printf "  %bError: Unable to get latest release location from GitHub%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"
-        return 1
-    fi
-
     # Move into the temp ftl directory
     pushd "$(mktemp -d)" > /dev/null || { printf "Unable to make temporary directory for FTL binary download\\n"; return 1; }
 
@@ -2197,9 +2175,12 @@ FTLinstall() {
         ftlBranch="master"
     fi
 
+    local binary
+    binary="${1}"
+
     # Determine which version of FTL to download
     if [[ "${ftlBranch}" == "master" ]];then
-        url="https://github.com/pi-hole/FTL/releases/download/${latesttag%$'\r'}"
+        url="https://github.com/pi-hole/ftl/releases/latest/download"
     else
         url="https://ftl.pi-hole.net/${ftlBranch}"
     fi
@@ -2215,8 +2196,6 @@ FTLinstall() {
 
             # Before stopping FTL, we download the macvendor database
             curl -sSL "https://ftl.pi-hole.net/macvendor.db" -o "${PI_HOLE_CONFIG_DIR}/macvendor.db" || true
-            chmod 644 "${PI_HOLE_CONFIG_DIR}/macvendor.db"
-            chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/macvendor.db"
 
             # Stop pihole-FTL service if available
             stop_service pihole-FTL &> /dev/null
@@ -2267,7 +2246,6 @@ disable_dnsmasq() {
     fi
     # Create /etc/dnsmasq.conf
     echo "conf-dir=/etc/dnsmasq.d" > "${conffile}"
-    chmod 644 "${conffile}"
 }
 
 get_binary_name() {
@@ -2275,6 +2253,8 @@ get_binary_name() {
     local machine
     machine=$(uname -m)
 
+    local l_binary
+
     local str="Detecting architecture"
     printf "  %b %s..." "${INFO}" "${str}"
     # If the machine is arm or aarch
@@ -2290,24 +2270,24 @@ get_binary_name() {
         if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then
             printf "%b  %b Detected ARM-aarch64 architecture\\n" "${OVER}" "${TICK}"
             # set the binary to be used
-            binary="pihole-FTL-aarch64-linux-gnu"
+            l_binary="pihole-FTL-aarch64-linux-gnu"
         #
         elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then
             #
             if [[ "${rev}" -gt 6 ]]; then
                 printf "%b  %b Detected ARM-hf architecture (armv7+)\\n" "${OVER}" "${TICK}"
                 # set the binary to be used
-                binary="pihole-FTL-arm-linux-gnueabihf"
+                l_binary="pihole-FTL-arm-linux-gnueabihf"
             # Otherwise,
             else
                 printf "%b  %b Detected ARM-hf architecture (armv6 or lower) Using ARM binary\\n" "${OVER}" "${TICK}"
                 # set the binary to be used
-                binary="pihole-FTL-arm-linux-gnueabi"
+                l_binary="pihole-FTL-arm-linux-gnueabi"
             fi
         else
             printf "%b  %b Detected ARM architecture\\n" "${OVER}" "${TICK}"
             # set the binary to be used
-            binary="pihole-FTL-arm-linux-gnueabi"
+            l_binary="pihole-FTL-arm-linux-gnueabi"
         fi
     elif [[ "${machine}" == "x86_64" ]]; then
         # This gives the architecture of packages dpkg installs (for example, "i386")
@@ -2320,12 +2300,12 @@ get_binary_name() {
         # in the past (see https://github.com/pi-hole/pi-hole/pull/2004)
         if [[ "${dpkgarch}" == "i386" ]]; then
             printf "%b  %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}"
-            binary="pihole-FTL-linux-x86_32"
+            l_binary="pihole-FTL-linux-x86_32"
         else
             # 64bit
             printf "%b  %b Detected x86_64 architecture\\n" "${OVER}" "${TICK}"
             # set the binary to be used
-            binary="pihole-FTL-linux-x86_64"
+            l_binary="pihole-FTL-linux-x86_64"
         fi
     else
         # Something else - we try to use 32bit executable and warn the user
@@ -2336,13 +2316,13 @@ get_binary_name() {
         else
             printf "%b  %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}"
         fi
-        binary="pihole-FTL-linux-x86_32"
+        l_binary="pihole-FTL-linux-x86_32"
     fi
+
+    echo ${l_binary}
 }
 
 FTLcheckUpdate() {
-    get_binary_name
-
     #In the next section we check to see if FTL is already installed (in case of pihole -r).
     #If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download
     printf "  %b Checking for existing FTL binary...\\n" "${INFO}"
@@ -2358,6 +2338,9 @@ FTLcheckUpdate() {
         ftlBranch="master"
     fi
 
+    local binary
+    binary="${1}"
+
     local remoteSha1
     local localSha1
 
@@ -2400,7 +2383,12 @@ FTLcheckUpdate() {
             local FTLversion
             FTLversion=$(/usr/bin/pihole-FTL tag)
             local FTLlatesttag
-            FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep 'Location' | awk -F '/' '{print $NF}' | tr -d '\r\n')
+
+            if ! FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep --color=never -i Location | awk -F / '{print $NF}' | tr -d '[:cntrl:]'); then
+                # There was an issue while retrieving the latest version
+                printf "  %b Failed to retrieve latest FTL release metadata" "${CROSS}"
+                return 3
+            fi
 
             if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then
                 return 0
@@ -2428,8 +2416,10 @@ FTLcheckUpdate() {
 FTLdetect() {
     printf "\\n  %b FTL Checks...\\n\\n" "${INFO}"
 
-    if FTLcheckUpdate ; then
-        FTLinstall || return 1
+    printf "  %b" "${2}"
+
+    if FTLcheckUpdate "${1}"; then
+        FTLinstall "${1}" || return 1
     fi
 }
 
@@ -2449,7 +2439,6 @@ copy_to_install_log() {
     # Copy the contents of file descriptor 3 into the install log
     # Since we use color codes such as '\e[1;33m', they should be removed
     sed 's/\[[0-9;]\{1,5\}m//g' < /proc/$$/fd/3 > "${installLogLoc}"
-    chmod 644 "${installLogLoc}"
 }
 
 main() {
@@ -2534,7 +2523,7 @@ main() {
         # Display welcome dialogs
         welcomeDialogs
         # Create directory for Pi-hole storage
-        install -d -m 755 /etc/pihole/
+        mkdir -p /etc/pihole/
         # Determine available interfaces
         get_available_interfaces
         # Find interfaces and let the user choose one
@@ -2556,7 +2545,7 @@ main() {
         installDefaultBlocklists
 
         # Source ${setupVars} to use predefined user variables in the functions
-        source "${setupVars}"
+        source ${setupVars}
 
         # Get the privacy level if it exists (default is 0)
         if [[ -f "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" ]]; then
@@ -2592,8 +2581,15 @@ main() {
     fi
     # Create the pihole user
     create_pihole_user
+
     # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole
-    if ! FTLdetect; then
+    local funcOutput
+    funcOutput=$(get_binary_name) #Store output of get_binary_name here
+    local binary
+    binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL)
+    local theRest
+    theRest="${funcOutput%pihole-FTL*}" # Print the rest of get_binary_name's output to display (cut out from first instance of "pihole-FTL")
+    if ! FTLdetect "${binary}" "${theRest}"; then
         printf "  %b FTL Engine not installed\\n" "${CROSS}"
         exit 1
     fi
@@ -2613,7 +2609,7 @@ main() {
             pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8)
             # shellcheck disable=SC1091
             . /opt/pihole/webpage.sh
-            echo "WEBPASSWORD=$(HashPassword "${pw}")" >> "${setupVars}"
+            echo "WEBPASSWORD=$(HashPassword ${pw})" >> ${setupVars}
         fi
     fi
 
@@ -2686,7 +2682,7 @@ main() {
 
     if [[ "${INSTALL_TYPE}" == "Update" ]]; then
         printf "\\n"
-        /usr/local/bin/pihole version --current
+        "${PI_HOLE_BIN_DIR}"/pihole version --current
     fi
 }
 

automated install/uninstall.sh

@@ -156,7 +156,7 @@ removeNoPurge() {
 
     # Restore Resolved
     if [[ -e /etc/systemd/resolved.conf.orig ]]; then
-        ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf
+        ${SUDO} cp /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf
         systemctl reload-or-restart systemd-resolved
     fi
 

gravity.sh

@@ -23,27 +23,31 @@ PIHOLE_COMMAND="/usr/local/bin/${basename}"
 
 piholeDir="/etc/${basename}"
 
-# Legacy (pre v5.0) list file locations
+adListFile="${piholeDir}/adlists.list"
+adListDefault="${piholeDir}/adlists.default"
+
 whitelistFile="${piholeDir}/whitelist.txt"
 blacklistFile="${piholeDir}/blacklist.txt"
 regexFile="${piholeDir}/regex.list"
-adListFile="${piholeDir}/adlists.list"
 
+adList="${piholeDir}/gravity.list"
+blackList="${piholeDir}/black.list"
 localList="${piholeDir}/local.list"
 VPNList="/etc/openvpn/ipp.txt"
 
-piholeGitDir="/etc/.pihole"
-gravityDBfile="${piholeDir}/gravity.db"
-gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql"
-optimize_database=false
-
 domainsExtension="domains"
 matterAndLight="${basename}.0.matterandlight.txt"
 parsedMatter="${basename}.1.parsedmatter.txt"
+whitelistMatter="${basename}.2.whitelistmatter.txt"
+accretionDisc="${basename}.3.accretionDisc.txt"
 preEventHorizon="list.preEventHorizon"
 
+skipDownload="false"
+
 resolver="pihole-FTL"
 
+haveSourceUrls=true
+
 # Source setupVars from install script
 setupVars="${piholeDir}/setupVars.conf"
 if [[ -f "${setupVars}" ]];then
@@ -79,104 +83,17 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then
   echo -e "  ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}"
 fi
 
-# Generate new sqlite3 file from schema template
-generate_gravity_database() {
-  sqlite3 "${gravityDBfile}" < "${gravityDBschema}"
-}
-
-# Import domains from file and store them in the specified database table
-database_table_from_file() {
-  # Define locals
-  local table source backup_path backup_file
-  table="${1}"
-  source="${2}"
-  backup_path="${piholeDir}/migration_backup"
-  backup_file="${backup_path}/$(basename "${2}")"
-
-  # Truncate table
-  output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 )
-  status="$?"
-
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to truncate ${table} database ${gravityDBfile}\\n  ${output}"
-    gravity_Cleanup "error"
-  fi
-
-  local tmpFile
-  tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")"
-  local timestamp
-  timestamp="$(date --utc +'%s')"
-  local inputfile
-  if [[ "${table}" == "gravity" ]]; then
-    # No need to modify the input data for the gravity table
-    inputfile="${source}"
+# Determine if Pi-hole blocking is disabled
+# If this is the case, we want to update
+#  gravity.list.bck and black.list.bck instead of
+#  gravity.list and black.list
+detect_pihole_blocking_status() {
+  if [[ "${BLOCKING_ENABLED}" == false ]]; then
+    echo -e "  ${INFO} Pi-hole blocking is disabled"
+    adList="${adList}.bck"
+    blackList="${blackList}.bck"
   else
-    # Apply format for white-, blacklist, regex, and adlist tables
-    local rowid
-    declare -i rowid
-    rowid=1
-    # Read file line by line
-    grep -v '^ *#' < "${source}" | while IFS= read -r domain
-    do
-      # Only add non-empty lines
-      if [[ ! -z "${domain}" ]]; then
-        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
-        rowid+=1
-      fi
-    done
-    inputfile="${tmpFile}"
-  fi
-  # Store domains in database table specified by ${table}
-  # Use printf as .mode and .import need to be on separate lines
-  # see https://unix.stackexchange.com/a/445615/83260
-  output=$( { printf ".timeout 10000\\n.mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
-  status="$?"
-
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to fill table ${table} in database ${gravityDBfile}\\n  ${output}"
-    gravity_Cleanup "error"
-  fi
-
-  # Delete tmpfile
-  rm "${tmpFile}" > /dev/null 2>&1 || \
-      echo -e "  ${CROSS} Unable to remove ${tmpFile}"
-
-  # Move source file to backup directory, create directory if not existing
-  mkdir -p "${backup_path}"
-  mv "${source}" "${backup_file}" 2> /dev/null || \
-      echo -e "  ${CROSS} Unable to backup ${source} to ${backup_path}"
-}
-
-# Migrate pre-v5.0 list files to database-based Pi-hole versions
-migrate_to_database() {
-  # Create database file only if not present
-  if [ -e "${gravityDBfile}" ]; then
-    return 0
-  fi
-
-  echo -e "  ${INFO} Creating new gravity database"
-  generate_gravity_database
-
-  # Migrate list files to new database
-  if [[ -e "${adListFile}" ]]; then
-    # Store adlist domains in database
-    echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
-    database_table_from_file "adlist" "${adListFile}"
-  fi
-  if [[ -e "${blacklistFile}" ]]; then
-    # Store blacklisted domains in database
-    echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
-    database_table_from_file "blacklist" "${blacklistFile}"
-  fi
-  if [[ -e "${whitelistFile}" ]]; then
-    # Store whitelisted domains in database
-    echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
-    database_table_from_file "whitelist" "${whitelistFile}"
-  fi
-  if [[ -e "${regexFile}" ]]; then
-    # Store regex domains in database
-    echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
-    database_table_from_file "regex" "${regexFile}"
+    echo -e "  ${INFO} Pi-hole blocking is enabled"
   fi
 }
 
@@ -184,8 +101,8 @@ migrate_to_database() {
 gravity_CheckDNSResolutionAvailable() {
   local lookupDomain="pi.hole"
 
-  # Determine if $localList does not exist, and ensure it is not empty
-  if [[ ! -e "${localList}" ]] || [[ -s "${localList}" ]]; then
+  # Determine if $localList does not exist
+  if [[ ! -e "${localList}" ]]; then
     lookupDomain="raw.githubusercontent.com"
   fi
 
@@ -236,13 +153,19 @@ gravity_CheckDNSResolutionAvailable() {
   gravity_CheckDNSResolutionAvailable
 }
 
-# Retrieve blocklist URLs and parse domains from adlist.list
+# Retrieve blocklist URLs and parse domains from adlists.list
 gravity_GetBlocklistUrls() {
   echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
 
-  # Retrieve source URLs from gravity database
-  # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true)
-  mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
+  if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then
+    # Remove superceded $adListDefault file
+    rm "${adListDefault}" 2> /dev/null || \
+      echo -e "  ${CROSS} Unable to remove ${adListDefault}"
+  fi
+
+  # Retrieve source URLs from $adListFile
+  # Logic: Remove comments and empty lines
+  mapfile -t sources <<< "$(grep -v -E "^(#|$)" "${adListFile}" 2> /dev/null)"
 
   # Parse source domains from $sources
   mapfile -t sourceDomains <<< "$(
@@ -259,12 +182,11 @@ gravity_GetBlocklistUrls() {
 
   if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then
     echo -e "${OVER}  ${TICK} ${str}"
-    return 0
   else
     echo -e "${OVER}  ${CROSS} ${str}"
     echo -e "  ${INFO} No source list found, or it is empty"
     echo ""
-    return 1
+    haveSourceUrls=false
   fi
 }
 
@@ -292,9 +214,11 @@ gravity_SetDownloadOptions() {
       *) cmd_ext="";;
     esac
 
-    echo -e "  ${INFO} Target: ${domain} (${url##*/})"
-    gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
-    echo ""
+    if [[ "${skipDownload}" == false ]]; then
+      echo -e "  ${INFO} Target: ${domain} (${url##*/})"
+      gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
+      echo ""
+    fi
   done
   gravity_Blackbody=true
 }
@@ -339,7 +263,7 @@ gravity_DownloadBlocklistFromUrl() {
     else
         printf -v port "%s" "${PIHOLE_DNS_1#*#}"
     fi
-    ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}")
+    ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1)
     if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then
       port=443;
     else port=80
@@ -411,17 +335,14 @@ gravity_ParseFileIntoDomains() {
     # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious
     # This helps with that and makes it easier to read
     # It also helps with debugging so each stage of the script can be researched more in depth
-    # 1) Remove carriage returns
-    # 2) Convert all characters to lowercase
-    # 3) Remove lines containing "#" or "/"
-    # 4) Remove leading tabs, spaces, etc.
-    # 5) Delete lines not matching domain names
-    < "${source}" tr -d '\r' | \
-    tr '[:upper:]' '[:lower:]' | \
-    sed -r '/(\/|#).*$/d' | \
-    sed -r 's/^.*\s+//g' | \
-    sed -r '/([^\.]+\.)+[^\.]{2,}/!d' >  "${destination}"
-    chmod 644 "${destination}"
+    # Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only.
+    # Last awk command takes non-commented lines and if they have 2 fields, take the right field (the domain) and leave
+    # the left (IP address), otherwise grab the single field.
+
+    < ${source} awk -F '#' '{print $1}' | \
+    awk -F '/' '{print $1}' | \
+    awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | \
+    sed -nr -e 's/\.{2,}/./g' -e '/\./p' >  ${destination}
     return 0
   fi
 
@@ -432,48 +353,7 @@ gravity_ParseFileIntoDomains() {
   # Determine how to parse individual source file formats
   if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then
     # Compare $firstLine against lower case words found in Adblock lists
-    echo -ne "  ${INFO} Format: Adblock"
-
-    # Define symbols used as comments: [!
-    # "||.*^" includes the "Example 2" domains we can extract
-    # https://adblockplus.org/filter-cheatsheet
-    abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
-
-    # Parse Adblock lists by extracting "Example 2" domains
-    # Logic: Ignore lines which do not include comments or domain name anchor
-    awk ''"${abpFilter}"' {
-      # Remove valid adblock type options
-      gsub(/\$?~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
-      # Remove starting domain name anchor "||" and ending seperator "^"
-      gsub(/^(\|\|)|(\^)/, "", $0)
-      # Remove invalid characters (*/,=$)
-      if($0 ~ /[*\/,=\$]/) { $0="" }
-      # Remove lines which are only IPv4 addresses
-      if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
-      if($0) { print $0 }
-    }' "${source}" > "${destination}"
-    chmod 644 "${destination}"
-
-    # Determine if there are Adblock exception rules
-    # https://adblockplus.org/filters
-    if grep -q "^@@||" "${source}" &> /dev/null; then
-      # Parse Adblock lists by extracting exception rules
-      # Logic: Ignore lines which do not include exception format "@@||example.com^"
-      awk -F "[|^]" '/^@@\|\|.*\^/ {
-        # Remove valid adblock type options
-        gsub(/\$?~?(third-party)/, "", $0)
-        # Remove invalid characters (*/,=$)
-        if($0 ~ /[*\/,=\$]/) { $0="" }
-        if($3) { print $3 }
-      }' "${source}" > "${destination}.exceptionsFile.tmp"
-
-      # Remove exceptions
-      comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}"
-      mv "${source}" "${destination}"
-      chmod 644 "${destination}"
-    fi
-
-    echo -e "${OVER}  ${TICK} Format: Adblock"
+    echo -e "  ${CROSS} Format: Adblock (list type not supported)"
   elif grep -q "^address=/" "${source}" &> /dev/null; then
     # Parse Dnsmasq format lists
     echo -e "  ${CROSS} Format: Dnsmasq (list type not supported)"
@@ -494,13 +374,11 @@ gravity_ParseFileIntoDomains() {
       # Print if nonempty
       length { print }
     ' "${source}" 2> /dev/null > "${destination}"
-    chmod 644 "${destination}"
 
     echo -e "${OVER}  ${TICK} Format: URL"
   else
     # Default: Keep hosts/domains file in same format as it was downloaded
     output=$( { mv "${source}" "${destination}"; } 2>&1 )
-    chmod 644 "${destination}"
 
     if [[ ! -e "${destination}" ]]; then
       echo -e "\\n  ${CROSS} Unable to move tmp file to ${piholeDir}
@@ -515,11 +393,12 @@ gravity_ConsolidateDownloadedBlocklists() {
   local str lastLine
 
   str="Consolidating blocklists"
-  echo -ne "  ${INFO} ${str}..."
+  if [[ "${haveSourceUrls}" == true ]]; then
+    echo -ne "  ${INFO} ${str}..."
+  fi
 
   # Empty $matterAndLight if it already exists, otherwise, create it
   : > "${piholeDir}/${matterAndLight}"
-  chmod 644 "${piholeDir}/${matterAndLight}"
 
   # Loop through each *.domains file
   for i in "${activeDomains[@]}"; do
@@ -535,8 +414,9 @@ gravity_ConsolidateDownloadedBlocklists() {
       fi
     fi
   done
-  echo -e "${OVER}  ${TICK} ${str}"
-
+  if [[ "${haveSourceUrls}" == true ]]; then
+    echo -e "${OVER}  ${TICK} ${str}"
+  fi
 }
 
 # Parse consolidated list into (filtered, unique) domains-only format
@@ -544,45 +424,67 @@ gravity_SortAndFilterConsolidatedList() {
   local str num
 
   str="Extracting domains from blocklists"
-  echo -ne "  ${INFO} ${str}..."
+  if [[ "${haveSourceUrls}" == true ]]; then
+    echo -ne "  ${INFO} ${str}..."
+  fi
 
-  # Parse into file
+  # Parse into hosts file
   gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}"
 
   # Format $parsedMatter line total as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
-
-  echo -e "${OVER}  ${TICK} ${str}"
-  echo -e "  ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains"
+  if [[ "${haveSourceUrls}" == true ]]; then
+    echo -e "${OVER}  ${TICK} ${str}"
+  fi
+  echo -e "  ${INFO} Number of domains being pulled in by gravity: ${COL_BLUE}${num}${COL_NC}"
 
   str="Removing duplicate domains"
-  echo -ne "  ${INFO} ${str}..."
-  sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
-  chmod 644 "${piholeDir}/${preEventHorizon}"
-  echo -e "${OVER}  ${TICK} ${str}"
+  if [[ "${haveSourceUrls}" == true ]]; then
+    echo -ne "  ${INFO} ${str}..."
+  fi
 
-  # Format $preEventHorizon line total as currency
-  num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-  str="Storing ${COL_BLUE}${num}${COL_NC} unique blocking domains in database"
-  echo -ne "  ${INFO} ${str}..."
-  database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}"
-  echo -e "${OVER}  ${TICK} ${str}"
+  sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
+
+  if [[ "${haveSourceUrls}" == true ]]; then
+    echo -e "${OVER}  ${TICK} ${str}"
+    # Format $preEventHorizon line total as currency
+    num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
+    echo -e "  ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}"
+  fi
 }
 
-# Report number of entries in a table
-gravity_Table_Count() {
-  local table="${1}"
-  local str="${2}"
-  local num
-  num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} WHERE enabled = 1;")"
-  echo -e "  ${INFO} Number of ${str}: ${num}"
+# Whitelist user-defined domains
+gravity_Whitelist() {
+  local num str
+
+  if [[ ! -f "${whitelistFile}" ]]; then
+    echo -e "  ${INFO} Nothing to whitelist!"
+    return 0
+  fi
+
+  num=$(wc -l < "${whitelistFile}")
+  str="Number of whitelisted domains: ${num}"
+  echo -ne "  ${INFO} ${str}..."
+
+  # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile
+  comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}"
+
+  echo -e "${OVER}  ${INFO} ${str}"
 }
 
 # Output count of blacklisted domains and regex filters
-gravity_ShowCount() {
-  gravity_Table_Count "blacklist" "blacklisted domains"
-  gravity_Table_Count "whitelist" "whitelisted domains"
-  gravity_Table_Count "regex" "regex filters"
+gravity_ShowBlockCount() {
+  local num
+
+  if [[ -f "${blacklistFile}" ]]; then
+    num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
+    echo -e "  ${INFO} Number of blacklisted domains: ${num}"
+  fi
+
+  if [[ -f "${regexFile}" ]]; then
+    num=$(grep -cv "^#" "${regexFile}")
+    echo -e "  ${INFO} Number of regex filters: ${num}"
+  fi
 }
 
 # Parse list of domains into hosts format
@@ -602,7 +504,7 @@ gravity_ParseDomainsIntoHosts() {
 }
 
 # Create "localhost" entries into hosts format
-gravity_generateLocalList() {
+gravity_ParseLocalDomains() {
   local hostname
 
   if [[ -s "/etc/hostname" ]]; then
@@ -618,7 +520,6 @@ gravity_generateLocalList() {
 
   # Empty $localList if it already exists, otherwise, create it
   : > "${localList}"
-  chmod 644 "${localList}"
 
   gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
 
@@ -628,6 +529,40 @@ gravity_generateLocalList() {
   fi
 }
 
+# Create primary blacklist entries
+gravity_ParseBlacklistDomains() {
+  local output status
+
+  # Empty $accretionDisc if it already exists, otherwise, create it
+  : > "${piholeDir}/${accretionDisc}"
+
+  if [[ -f "${piholeDir}/${whitelistMatter}" ]]; then
+    mv "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}"
+  else
+    # There was no whitelist file, so use preEventHorizon instead of whitelistMatter.
+    cp "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}"
+  fi
+
+  # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it
+  output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+}
+
+# Create user-added blacklist entries
+gravity_ParseUserDomains() {
+  if [[ ! -f "${blacklistFile}" ]]; then
+    return 0
+  fi
+  # Copy the file over as /etc/pihole/black.list so dnsmasq can use it
+  cp "${blacklistFile}" "${blackList}" 2> /dev/null || \
+    echo -e "\\n  ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}"
+}
+
 # Trap Ctrl-C
 gravity_Trap() {
   trap '{ echo -e "\\n\\n  ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
@@ -659,21 +594,6 @@ gravity_Cleanup() {
 
   echo -e "${OVER}  ${TICK} ${str}"
 
-  if ${optimize_database} ; then
-    str="Optimizing domains database"
-    echo -ne "  ${INFO} ${str}..."
-    # Run VACUUM command on database to optimize it
-    output=$( { sqlite3 "${gravityDBfile}" "VACUUM;"; } 2>&1 )
-    status="$?"
-
-    if [[ "${status}" -ne 0 ]]; then
-      echo -e "\\n  ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n  ${output}"
-      error="error"
-    else
-      echo -e "${OVER}  ${TICK} ${str}"
-    fi
-  fi
-
   # Only restart DNS service if offline
   if ! pidof ${resolver} &> /dev/null; then
     "${PIHOLE_COMMAND}" restartdns
@@ -700,17 +620,17 @@ Options:
 for var in "$@"; do
   case "${var}" in
     "-f" | "--force" ) forceDelete=true;;
-    "-o" | "--optimize" ) optimize_database=true;;
     "-h" | "--help" ) helpFunc;;
+    "-sd" | "--skip-download" ) skipDownload=true;;
+    "-b" | "--blacklist-only" ) listType="blacklist";;
+    "-w" | "--whitelist-only" ) listType="whitelist";;
+    "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";;
   esac
 done
 
 # Trap Ctrl-C
 gravity_Trap
 
-# Move possibly existing legacy files to the gravity database
-migrate_to_database
-
 if [[ "${forceDelete:-}" == true ]]; then
   str="Deleting existing list cache"
   echo -ne "${INFO} ${str}..."
@@ -719,24 +639,56 @@ if [[ "${forceDelete:-}" == true ]]; then
   echo -e "${OVER}  ${TICK} ${str}"
 fi
 
-# Gravity downloads blocklists next
-gravity_CheckDNSResolutionAvailable
-if gravity_GetBlocklistUrls; then
-  gravity_SetDownloadOptions
-  # Build preEventHorizon
+detect_pihole_blocking_status
+
+# Determine which functions to run
+if [[ "${skipDownload}" == false ]]; then
+  # Gravity needs to download blocklists
+  gravity_CheckDNSResolutionAvailable
+  gravity_GetBlocklistUrls
+  if [[ "${haveSourceUrls}" == true ]]; then
+    gravity_SetDownloadOptions
+  fi
   gravity_ConsolidateDownloadedBlocklists
   gravity_SortAndFilterConsolidatedList
+else
+  # Gravity needs to modify Blacklist/Whitelist/Wildcards
+  echo -e "  ${INFO} Using cached Event Horizon list..."
+  numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
+  echo -e "  ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
 fi
 
-# Create local.list
-gravity_generateLocalList
-gravity_ShowCount
+# Perform when downloading blocklists, or modifying the whitelist
+if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then
+  gravity_Whitelist
+fi
+
+convert_wildcard_to_regex
+gravity_ShowBlockCount
+
+# Perform when downloading blocklists, or modifying the white/blacklist (not wildcards)
+if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then
+  str="Parsing domains into hosts format"
+  echo -ne "  ${INFO} ${str}..."
+
+  gravity_ParseUserDomains
+
+  # Perform when downloading blocklists
+  if [[ ! "${listType:-}" == "blacklist" ]]; then
+    gravity_ParseLocalDomains
+    gravity_ParseBlacklistDomains
+  fi
+
+  echo -e "${OVER}  ${TICK} ${str}"
+
+  gravity_Cleanup
+fi
 
-gravity_Cleanup
 echo ""
 
 # Determine if DNS has been restarted by this instance of gravity
 if [[ -z "${dnsWasOffline:-}" ]]; then
-  "${PIHOLE_COMMAND}" restartdns reload
+  # Use "force-reload" when restarting dnsmasq for everything but Wildcards
+  "${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}"
 fi
 "${PIHOLE_COMMAND}" status

manpages/pihole.8

@@ -351,12 +351,6 @@ Switching Pi-hole subsystem branches
 .br
     Switch to core development branch
 .br
-
-\fBpihole arpflush\fR
-.br
-    Flush information stored in Pi-hole's network tables
-.br
-
 .SH "SEE ALSO"
 
 \fBlighttpd\fR(8), \fBpihole-FTL\fR(8)

pihole

@@ -10,11 +10,14 @@
 # Please see LICENSE file for your rights under this license.
 
 readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
+readonly gravitylist="/etc/pihole/gravity.list"
+readonly blacklist="/etc/pihole/black.list"
 
-# setupVars is not readonly here because in some functions (checkout),
+# setupVars and PI_HOLE_BIN_DIR are not readonly here because in some funcitons (checkout),
 # it might get set again when the installer is sourced. This causes an
 # error due to modifying a readonly variable.
 setupVars="/etc/pihole/setupVars.conf"
+PI_HOLE_BIN_DIR="/usr/local/bin"
 
 readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE"
 source "${colfile}"
@@ -54,11 +57,6 @@ flushFunc() {
   exit 0
 }
 
-arpFunc() {
-  "${PI_HOLE_SCRIPT_DIR}"/piholeARPTable.sh "$@"
-  exit 0
-}
-
 updatePiholeFunc() {
   shift
   "${PI_HOLE_SCRIPT_DIR}"/update.sh "$@"
@@ -101,20 +99,16 @@ versionFunc() {
 
 restartDNS() {
   local svcOption svc str output status
-  svcOption="${1:-}"
+  svcOption="${1:-restart}"
 
-  # Determine if we should reload or restart restart
+  # Determine if we should reload or restart
   if [[ "${svcOption}" =~ "reload" ]]; then
-    # Using SIGHUP will NOT re-read any *.conf files
+    # Reload has been requested
+    # Note: This will NOT re-read any *.conf files
     svc="killall -s SIGHUP ${resolver}"
   else
-    # Get PID of resolver to determine if it needs to start or restart
-    if pidof pihole-FTL &> /dev/null; then
-      svcOption="restart"
-    else
-      svcOption="start"
-    fi
-    svc="service ${resolver} ${svcOption}"
+    # A full restart has been requested
+    svc="service ${resolver} restart"
   fi
 
   # Print output to Terminal, but not to Web Admin
@@ -151,6 +145,14 @@ Time:
       echo -e "  ${INFO} Blocking already disabled, nothing to do"
       exit 0
     fi
+    if [[ -e "${gravitylist}" ]]; then
+      mv "${gravitylist}" "${gravitylist}.bck"
+      echo "" > "${gravitylist}"
+    fi
+    if [[ -e "${blacklist}" ]]; then
+      mv "${blacklist}" "${blacklist}.bck"
+      echo "" > "${blacklist}"
+    fi
     if [[ $# > 1 ]]; then
       local error=false
       if [[ "${2}" == *"s" ]]; then
@@ -159,7 +161,7 @@ Time:
           local str="Disabling blocking for ${tt} seconds"
           echo -e "  ${INFO} ${str}..."
           local str="Blocking will be re-enabled in ${tt} seconds"
-          nohup bash -c "sleep ${tt}; pihole enable" </dev/null &>/dev/null &
+          nohup bash -c "sleep ${tt}; ${PI_HOLE_BIN_DIR}/pihole enable" </dev/null &>/dev/null &
         else
           local error=true
         fi
@@ -170,7 +172,7 @@ Time:
           echo -e "  ${INFO} ${str}..."
           local str="Blocking will be re-enabled in ${tt} minutes"
           tt=$((${tt}*60))
-          nohup bash -c "sleep ${tt}; pihole enable" </dev/null &>/dev/null &
+          nohup bash -c "sleep ${tt}; ${PI_HOLE_BIN_DIR}/pihole enable" </dev/null &>/dev/null &
         else
           local error=true
         fi
@@ -199,6 +201,12 @@ Time:
     echo -e "  ${INFO} Enabling blocking"
     local str="Pi-hole Enabled"
 
+    if [[ -e "${gravitylist}.bck" ]]; then
+      mv "${gravitylist}.bck" "${gravitylist}"
+    fi
+    if [[ -e "${blacklist}.bck" ]]; then
+      mv "${blacklist}.bck" "${blacklist}"
+    fi
     sed -i "/BLOCKING_ENABLED=/d" "${setupVars}"
     echo "BLOCKING_ENABLED=true" >> "${setupVars}"
   fi
@@ -226,7 +234,7 @@ Options:
     sed -i 's/^QUERY_LOGGING=true/QUERY_LOGGING=false/' /etc/pihole/setupVars.conf
     if [[ "${2}" != "noflush" ]]; then
       # Flush logs
-      pihole -f
+      "${PI_HOLE_BIN_DIR}"/pihole -f
     fi
     echo -e "  ${INFO} Disabling logging..."
     local str="Logging has been disabled!"
@@ -279,7 +287,7 @@ statusFunc() {
       *) echo -e "  ${INFO} Pi-hole blocking will be enabled";;
     esac
     # Enable blocking
-    pihole enable
+    "${PI_HOLE_BIN_DIR}"/pihole enable
   fi
 }
 
@@ -302,7 +310,7 @@ tailFunc() {
   # Colour everything else as gray
   tail -f /var/log/pihole.log | sed -E \
     -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \
-    -e "s,(.*(gravity |black |regex | config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
+    -e "s,(.*(gravity.list|black.list|regex.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
     -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \
     -e "s,.*,${COL_GRAY}&${COL_NC},"
   exit 0
@@ -408,8 +416,7 @@ Options:
                         Add '-h' for more info on disable usage
   restartdns          Restart Pi-hole subsystems
   checkout            Switch Pi-hole subsystems to a different Github branch
-                        Add '-h' for more info on checkout usage
-  arpflush            Flush information stored in Pi-hole's network tables";
+                        Add '-h' for more info on checkout usage";
   exit 0
 }
 
@@ -436,8 +443,8 @@ fi
 case "${1}" in
   "-w" | "whitelist"            ) listFunc "$@";;
   "-b" | "blacklist"            ) listFunc "$@";;
-  "--wild" | "wildcard"         ) listFunc "$@";;
-  "--regex" | "regex"           ) listFunc "$@";;
+  "--wild" | "wildcard"          ) listFunc "$@";;
+  "--regex" | "regex"            ) listFunc "$@";;
   "-d" | "debug"                ) debugFunc "$@";;
   "-f" | "flush"                ) flushFunc "$@";;
   "-up" | "updatePihole"        ) updatePiholeFunc "$@";;
@@ -458,6 +465,5 @@ case "${1}" in
   "checkout"                    ) piholeCheckoutFunc "$@";;
   "tricorder"                   ) tricorderFunc;;
   "updatechecker"               ) updateCheckFunc "$@";;
-  "arpflush"                    ) arpFunc "$@";;
   *                             ) helpFunc;;
 esac

test/test_automated_install.py

@@ -399,7 +399,10 @@ def test_FTL_detect_aarch64_no_errors(Pihole):
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
     create_pihole_user
-    FTLdetect
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     ''')
     expected_stdout = info_box + ' FTL Checks...'
     assert expected_stdout in detectPlatform.stdout
@@ -420,7 +423,10 @@ def test_FTL_detect_armv6l_no_errors(Pihole):
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
     create_pihole_user
-    FTLdetect
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     ''')
     expected_stdout = info_box + ' FTL Checks...'
     assert expected_stdout in detectPlatform.stdout
@@ -442,7 +448,10 @@ def test_FTL_detect_armv7l_no_errors(Pihole):
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
     create_pihole_user
-    FTLdetect
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     ''')
     expected_stdout = info_box + ' FTL Checks...'
     assert expected_stdout in detectPlatform.stdout
@@ -459,7 +468,10 @@ def test_FTL_detect_x86_64_no_errors(Pihole):
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
     create_pihole_user
-    FTLdetect
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     ''')
     expected_stdout = info_box + ' FTL Checks...'
     assert expected_stdout in detectPlatform.stdout
@@ -476,7 +488,10 @@ def test_FTL_detect_unknown_no_errors(Pihole):
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
     create_pihole_user
-    FTLdetect
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     ''')
     expected_stdout = 'Not able to detect architecture (unknown: mips)'
     assert expected_stdout in detectPlatform.stdout
@@ -495,64 +510,14 @@ def test_FTL_download_aarch64_no_errors(Pihole):
     ''')
     download_binary = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    binary="pihole-FTL-aarch64-linux-gnu"
     create_pihole_user
-    FTLinstall
+    FTLinstall "pihole-FTL-aarch64-linux-gnu"
     ''')
     expected_stdout = tick_box + ' Downloading and Installing FTL'
     assert expected_stdout in download_binary.stdout
     assert 'error' not in download_binary.stdout.lower()
 
 
-def test_FTL_download_unknown_fails_no_errors(Pihole):
-    '''
-    confirms unknown binary is not downloaded for FTL engine
-    '''
-    # mock whiptail answers and ensure installer dependencies
-    mock_command('whiptail', {'*': ('', '0')}, Pihole)
-    Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    distro_check
-    install_dependent_packages ${INSTALLER_DEPS[@]}
-    ''')
-    download_binary = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    binary="pihole-FTL-mips"
-    create_pihole_user
-    FTLinstall
-    ''')
-    expected_stdout = cross_box + ' Downloading and Installing FTL'
-    assert expected_stdout in download_binary.stdout
-    error1 = 'Error: URL https://github.com/pi-hole/FTL/releases/download/'
-    assert error1 in download_binary.stdout
-    error2 = 'not found'
-    assert error2 in download_binary.stdout
-
-
-def test_FTL_download_binary_unset_no_errors(Pihole):
-    '''
-    confirms unset binary variable does not download FTL engine
-    '''
-    # mock whiptail answers and ensure installer dependencies
-    mock_command('whiptail', {'*': ('', '0')}, Pihole)
-    Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    distro_check
-    install_dependent_packages ${INSTALLER_DEPS[@]}
-    ''')
-    download_binary = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    create_pihole_user
-    FTLinstall
-    ''')
-    expected_stdout = cross_box + ' Downloading and Installing FTL'
-    assert expected_stdout in download_binary.stdout
-    error1 = 'Error: URL https://github.com/pi-hole/FTL/releases/download/'
-    assert error1 in download_binary.stdout
-    error2 = 'not found'
-    assert error2 in download_binary.stdout
-
-
 def test_FTL_binary_installed_and_responsive_no_errors(Pihole):
     '''
     confirms FTL binary is copied and functional in installed location
@@ -560,7 +525,10 @@ def test_FTL_binary_installed_and_responsive_no_errors(Pihole):
     installed_binary = Pihole.run('''
     source /opt/pihole/basic-install.sh
     create_pihole_user
-    FTLdetect
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     pihole-FTL version
     ''')
     expected_stdout = 'v'