Merge pull request #3166 from pi-hole/release/v4.4

Add use-application-dns.net = NXDOMAIN in ProcessDNSSettings rather t…

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+patreon: pihole
+custom: https://pi-hole.net/donate

advanced/01-pihole.conf

@@ -43,8 +43,3 @@ log-facility=/var/log/pihole.log
 local-ttl=2
 
 log-async
-
-# If a DHCP client claims that its name is "wpad", ignore that.
-# This fixes a security hole. see CERT Vulnerability VU#598349
-dhcp-name-match=set:wpad-ignore,wpad
-dhcp-ignore-names=tag:wpad-ignore

advanced/Scripts/piholeCheckout.sh

@@ -46,6 +46,12 @@ checkout() {
     local corebranches
     local webbranches
 
+    # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole
+    local funcOutput
+    funcOutput=$(get_binary_name) #Store output of get_binary_name here
+    local binary
+    binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL)
+
     # Avoid globbing
     set -f
 
@@ -86,7 +92,6 @@ checkout() {
         fi
         #echo -e "  ${TICK} Pi-hole Core"
 
-        get_binary_name
         local path
         path="development/${binary}"
         echo "development" > /etc/pihole/ftlbranch
@@ -100,7 +105,6 @@ checkout() {
             fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo "  ${CROSS} Unable to pull Web master branch"; exit 1; }
         fi
         #echo -e "  ${TICK} Web Interface"
-        get_binary_name
         local path
         path="master/${binary}"
         echo "master" > /etc/pihole/ftlbranch
@@ -159,7 +163,6 @@ checkout() {
         fi
         checkout_pull_branch "${webInterfaceDir}" "${2}"
     elif [[ "${1}" == "ftl" ]] ; then
-        get_binary_name
         local path
         path="${2}/${binary}"
 

advanced/Scripts/piholeDebug.sh

@@ -643,19 +643,21 @@ ping_internet() {
 }
 
 compare_port_to_service_assigned() {
-    local service_name="${1}"
-    # The programs we use may change at some point, so they are in a varible here
-    local resolver="pihole-FTL"
-    local web_server="lighttpd"
-    local ftl="pihole-FTL"
+    local service_name
+    local expected_service
+    local port
+
+    service_name="${2}"
+    expected_service="${1}"
+    port="${3}"
 
     # If the service is a Pi-hole service, highlight it in green
-    if [[ "${service_name}" == "${resolver}" ]] || [[ "${service_name}" == "${web_server}" ]] || [[ "${service_name}" == "${ftl}" ]]; then
-        log_write "[${COL_GREEN}${port_number}${COL_NC}] is in use by ${COL_GREEN}${service_name}${COL_NC}"
+    if [[ "${service_name}" == "${expected_service}" ]]; then
+        log_write "[${COL_GREEN}${port}${COL_NC}] is in use by ${COL_GREEN}${service_name}${COL_NC}"
     # Otherwise,
     else
         # Show the service name in red since it's non-standard
-        log_write "[${COL_RED}${port_number}${COL_NC}] is in use by ${COL_RED}${service_name}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_PORTS})"
+        log_write "[${COL_RED}${port}${COL_NC}] is in use by ${COL_RED}${service_name}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_PORTS})"
     fi
 }
 
@@ -689,11 +691,11 @@ check_required_ports() {
         fi
         # Use a case statement to determine if the right services are using the right ports
         case "$(echo "$port_number" | rev | cut -d: -f1 | rev)" in
-            53) compare_port_to_service_assigned  "${resolver}"
+            53) compare_port_to_service_assigned  "${resolver}" "${service_name}" 53
                 ;;
-            80) compare_port_to_service_assigned  "${web_server}"
+            80) compare_port_to_service_assigned  "${web_server}" "${service_name}" 80
                 ;;
-            4711) compare_port_to_service_assigned  "${ftl}"
+            4711) compare_port_to_service_assigned  "${ftl}" "${service_name}" 4711
                 ;;
             # If it's not a default port that Pi-hole needs, just print it out for the user to see
             *) log_write "${port_number} ${service_name} (${protocol_type})";

advanced/Scripts/query.sh

@@ -54,7 +54,7 @@ scanList(){
     # /dev/null forces filename to be printed when only one list has been generated
     # shellcheck disable=SC2086
     case "${type}" in
-        "exact" ) grep -i -E -l "(^|(?<!#)\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
+        "exact" ) grep -i -E "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
         "wc"    ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;;
         *       ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;;
     esac
@@ -157,6 +157,20 @@ lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)")
 # Query blocklists for occurences of domain
 mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")"
 
+# Remove unwanted content from $results
+# Each line in $results is formatted as such: [fileName]:[line]
+# 1. Delete lines starting with #
+# 2. Remove comments after domain
+# 3. Remove hosts format IP address
+# 4. Remove any lines that no longer contain the queried domain name (in case the matched domain name was in a comment)
+esc_domain="${domainQuery//./\\.}"
+mapfile -t results <<< "$(IFS=$'\n'; sed \
+	-e "/:#/d" \
+	-e "s/[ \\t]#.*//g" \
+	-e "s/:.*[ \\t]/:/g" \
+	-e "/${esc_domain}/!d" \
+	<<< "${results[*]}")"
+
 # Handle notices
 if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then
     echo -e "  ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists"
@@ -170,20 +184,6 @@ elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then
     exit 0
 fi
 
-# Remove unwanted content from non-exact $results
-if [[ -z "${exact}" ]]; then
-    # Delete lines starting with #
-    # Remove comments after domain
-    # Remove hosts format IP address
-    mapfile -t results <<< "$(IFS=$'\n'; sed \
-        -e "/:#/d" \
-        -e "s/[ \\t]#.*//g" \
-        -e "s/:.*[ \\t]/:/g" \
-        <<< "${results[*]}")"
-    # Exit if result was in a comment
-    [[ -z "${results[*]}" ]] && exit 0
-fi
-
 # Get adlist file content as array
 if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
     for adlistUrl in $(< "${adListsList}"); do

advanced/Scripts/update.sh

@@ -31,7 +31,6 @@ source "/opt/pihole/COL_TABLE"
 # make_repo() sourced from basic-install.sh
 # update_repo() source from basic-install.sh
 # getGitFiles() sourced from basic-install.sh
-# get_binary_name() sourced from basic-install.sh
 # FTLcheckUpdate() sourced from basic-install.sh
 
 GitCheckUpdateAvail() {
@@ -129,7 +128,12 @@ main() {
         fi
     fi
 
-    if FTLcheckUpdate > /dev/null; then
+    local funcOutput
+    funcOutput=$(get_binary_name) #Store output of get_binary_name here
+    local binary
+    binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL)
+
+    if FTLcheckUpdate "${binary}" > /dev/null; then
         FTL_update=true
         echo -e "  ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}"
     else
@@ -194,6 +198,14 @@ main() {
         ${PI_HOLE_FILES_DIR}/automated\ install/basic-install.sh --reconfigure --unattended || \
         echo -e "${basicError}" && exit 1
     fi
+
+    if [[ "${FTL_update}" == true || "${core_update}" == true || "${web_update}" == true ]]; then
+        # Force an update of the updatechecker
+        /opt/pihole/updatecheck.sh
+        /opt/pihole/updatecheck.sh x remote
+        echo -e "  ${INFO} Local version file information updated."
+    fi
+
     echo ""
     exit 0
 }

advanced/Scripts/webpage.sh

@@ -16,6 +16,7 @@ readonly dhcpconfig="/etc/dnsmasq.d/02-pihole-dhcp.conf"
 readonly FTLconf="/etc/pihole/pihole-FTL.conf"
 # 03 -> wildcards
 readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf"
+readonly PI_HOLE_BIN_DIR="/usr/local/bin"
 
 coltable="/opt/pihole/COL_TABLE"
 if [[ -f ${coltable} ]]; then
@@ -210,6 +211,11 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423
         add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}"
         add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}"
     fi
+
+    # Prevent Firefox from automatically switching over to DNS-over-HTTPS
+    # This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https
+    # (sourced 7th September 2019)
+    add_dnsmasq_setting "server=/use-application-dns.net/"
 }
 
 SetDNSServers() {
@@ -274,7 +280,7 @@ Reboot() {
 }
 
 RestartDNS() {
-    /usr/local/bin/pihole restartdns
+    "${PI_HOLE_BIN_DIR}"/pihole restartdns
 }
 
 SetQueryLogOptions() {
@@ -363,6 +369,14 @@ EnableDHCP() {
     delete_dnsmasq_setting "dhcp-"
     delete_dnsmasq_setting "quiet-dhcp"
 
+    # If a DHCP client claims that its name is "wpad", ignore that.
+    # This fixes a security hole. see CERT Vulnerability VU#598349
+    # We also ignore "localhost" as Windows behaves strangely if a
+    # device claims this host name
+    add_dnsmasq_setting "dhcp-name-match=set:hostname-ignore,wpad
+dhcp-name-match=set:hostname-ignore,localhost
+dhcp-ignore-names=tag:hostname-ignore"
+
     ProcessDHCPSettings
 
     RestartDNS

advanced/Templates/pihole-FTL.service

@@ -13,7 +13,14 @@ FTLUSER=pihole
 PIDFILE=/var/run/pihole-FTL.pid
 
 get_pid() {
-    pidof "pihole-FTL"
+    # First, try to obtain PID from PIDFILE
+    if [ -s "${PIDFILE}" ]; then
+        cat "${PIDFILE}"
+        return
+    fi
+
+    # If the PIDFILE is empty or not available, obtain the PID using pidof
+    pidof "pihole-FTL" | awk '{print $(NF)}'
 }
 
 is_running() {

advanced/lighttpd.conf.debian

@@ -27,7 +27,7 @@ server.modules = (
 )
 
 server.document-root        = "/var/www/html"
-server.error-handler-404    = "pihole/index.php"
+server.error-handler-404    = "/pihole/index.php"
 server.upload-dirs          = ( "/var/cache/lighttpd/uploads" )
 server.errorlog             = "/var/log/lighttpd/error.log"
 server.pid-file             = "/var/run/lighttpd.pid"

advanced/lighttpd.conf.fedora

@@ -28,7 +28,7 @@ server.modules = (
 )
 
 server.document-root        = "/var/www/html"
-server.error-handler-404    = "pihole/index.php"
+server.error-handler-404    = "/pihole/index.php"
 server.upload-dirs          = ( "/var/cache/lighttpd/uploads" )
 server.errorlog             = "/var/log/lighttpd/error.log"
 server.pid-file             = "/var/run/lighttpd.pid"

automated install/basic-install.sh

@@ -31,7 +31,7 @@ set -e
 # List of supported DNS servers
 DNS_SERVERS=$(cat << EOM
 Google (ECS);8.8.8.8;8.8.4.4;2001:4860:4860:0:0:0:0:8888;2001:4860:4860:0:0:0:0:8844
-OpenDNS (ECS);208.67.222.222;208.67.220.220;2620:0:ccc::2;2620:0:ccd::2
+OpenDNS (ECS);208.67.222.222;208.67.220.220;2620:119:35::35;2620:119:53::53
 Level3;4.2.2.1;4.2.2.2;;
 Comodo;8.26.56.26;8.20.247.20;;
 DNS.WATCH;84.200.69.80;84.200.70.40;2001:1608:10:25:0:0:1c04:b12f;2001:1608:10:25:0:0:9249:d69b
@@ -65,6 +65,7 @@ PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update versi
 # This directory is where the Pi-hole scripts will be installed
 PI_HOLE_INSTALL_DIR="/opt/pihole"
 PI_HOLE_CONFIG_DIR="/etc/pihole"
+PI_HOLE_BIN_DIR="/usr/local/bin"
 PI_HOLE_BLOCKPAGE_DIR="${webroot}/pihole"
 useUpdateVars=false
 
@@ -84,8 +85,13 @@ if [ -z "${USER}" ]; then
 fi
 
 
-# Find the rows and columns will default to 80x24 if it can not be detected
-screen_size=$(stty size || printf '%d %d' 24 80)
+# Check if we are running on a real terminal and find the rows and columns
+# If there is no real terminal, we will default to 80x24
+if [ -t 0 ] ; then
+  screen_size=$(stty size)
+else
+  screen_size="24 80"
+fi
 # Set rows variable to contain first number
 printf -v rows '%d' "${screen_size%% *}"
 # Set columns variable to contain second number
@@ -133,9 +139,6 @@ else
     OVER="\\r\\033[K"
 fi
 
-# Define global binary variable
-binary="tbd"
-
 # A simple function that just echoes out our logo in ASCII format
 # This lets users know that it is a Pi-hole, LLC product
 show_ascii_berry() {
@@ -283,7 +286,7 @@ elif is_command rpm ; then
     UPDATE_PKG_CACHE=":"
     PKG_INSTALL=(${PKG_MANAGER} install -y)
     PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l"
-    INSTALLER_DEPS=(dialog git iproute newt procps-ng which)
+    INSTALLER_DEPS=(dialog git iproute newt procps-ng which chkconfig)
     PIHOLE_DEPS=(bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc sqlite libcap)
     PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo)
     LIGHTTPD_USER="lighttpd"
@@ -1168,12 +1171,11 @@ chooseBlocklists() {
         mv "${adlistFile}" "${adlistFile}.old"
     fi
     # Let user select (or not) blocklists via a checklist
-    cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 7)
+    cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 6)
     # In an array, show the options available (all off by default):
     options=(StevenBlack "StevenBlack's Unified Hosts List" on
         MalwareDom "MalwareDomains" on
         Cameleon "Cameleon" on
-        ZeusTracker "ZeusTracker" on
         DisconTrack "Disconnect.me Tracking" on
         DisconAd "Disconnect.me Ads" on
         HostsFile "Hosts-file.net Ads" on)
@@ -1195,7 +1197,6 @@ appendToListsFile() {
         StevenBlack  )  echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";;
         MalwareDom   )  echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";;
         Cameleon     )  echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";;
-        ZeusTracker  )  echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";;
         DisconTrack  )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";;
         DisconAd     )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";;
         HostsFile    )  echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";;
@@ -1213,7 +1214,6 @@ installDefaultBlocklists() {
     appendToListsFile StevenBlack
     appendToListsFile MalwareDom
     appendToListsFile Cameleon
-    appendToListsFile ZeusTracker
     appendToListsFile DisconTrack
     appendToListsFile DisconAd
     appendToListsFile HostsFile
@@ -1337,7 +1337,7 @@ installScripts() {
         install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./advanced/Scripts/*.sh
         install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./automated\ install/uninstall.sh
         install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./advanced/Scripts/COL_TABLE
-        install -o "${USER}" -Dm755 -t /usr/local/bin/ pihole
+        install -o "${USER}" -Dm755 -t "${PI_HOLE_BIN_DIR}" pihole
         install -Dm644 ./advanced/bash-completion/pihole /etc/bash_completion.d/pihole
         printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
 
@@ -1605,7 +1605,6 @@ install_dependent_packages() {
 
     # Install packages passed in via argument array
     # No spinner - conflicts with set -e
-    declare -a argArray1=("${!1}")
     declare -a installArray
 
     # Debian based package install - debconf will download the entire package list
@@ -1615,7 +1614,7 @@ install_dependent_packages() {
     # installed by us, and remove only the installed packages, and not the entire list.
     if is_command debconf-apt-progress ; then
         # For each package,
-        for i in "${argArray1[@]}"; do
+        for i in "$@"; do
             printf "  %b Checking for %s..." "${INFO}" "${i}"
             if dpkg-query -W -f='${Status}' "${i}" 2>/dev/null | grep "ok installed" &> /dev/null; then
                 printf "%b  %b Checking for %s\\n" "${OVER}" "${TICK}" "${i}"
@@ -1634,7 +1633,7 @@ install_dependent_packages() {
     fi
 
     # Install Fedora/CentOS packages
-    for i in "${argArray1[@]}"; do
+    for i in "$@"; do
         printf "  %b Checking for %s..." "${INFO}" "${i}"
         if ${PKG_MANAGER} -q list installed "${i}" &> /dev/null; then
             printf "%b  %b Checking for %s" "${OVER}" "${TICK}" "${i}"
@@ -1691,13 +1690,13 @@ installPiholeWeb() {
     # and copy in the pihole sudoers file
     install -m 0640 ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.sudo /etc/sudoers.d/pihole
     # Add lighttpd user (OS dependent) to sudoers file
-    echo "${LIGHTTPD_USER} ALL=NOPASSWD: /usr/local/bin/pihole" >> /etc/sudoers.d/pihole
+    echo "${LIGHTTPD_USER} ALL=NOPASSWD: ${PI_HOLE_BIN_DIR}/pihole" >> /etc/sudoers.d/pihole
 
     # If the Web server user is lighttpd,
     if [[ "$LIGHTTPD_USER" == "lighttpd" ]]; then
         # Allow executing pihole via sudo with Fedora
-        # Usually /usr/local/bin is not permitted as directory for sudoable programs
-        echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin" >> /etc/sudoers.d/pihole
+        # Usually /usr/local/bin ${PI_HOLE_BIN_DIR} is not permitted as directory for sudoable programs
+        echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:${PI_HOLE_BIN_DIR}" >> /etc/sudoers.d/pihole
     fi
     # Set the strict permissions on the file
     chmod 0440 /etc/sudoers.d/pihole
@@ -2152,21 +2151,15 @@ clone_or_update_repos() {
 }
 
 # Download FTL binary to random temp directory and install FTL binary
+# Disable directive for SC2120 a value _can_ be passed to this function, but it is passed from an external script that sources this one
+# shellcheck disable=SC2120
 FTLinstall() {
+
     # Local, named variables
     local latesttag
     local str="Downloading and Installing FTL"
     printf "  %b %s..." "${INFO}" "${str}"
 
-    # Find the latest version tag for FTL
-    latesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep "Location" | awk -F '/' '{print $NF}')
-    # Tags should always start with v, check for that.
-    if [[ ! "${latesttag}" == v* ]]; then
-        printf "%b  %b %s\\n" "${OVER}" "${CROSS}" "${str}"
-        printf "  %bError: Unable to get latest release location from GitHub%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"
-        return 1
-    fi
-
     # Move into the temp ftl directory
     pushd "$(mktemp -d)" > /dev/null || { printf "Unable to make temporary directory for FTL binary download\\n"; return 1; }
 
@@ -2182,9 +2175,12 @@ FTLinstall() {
         ftlBranch="master"
     fi
 
+    local binary
+    binary="${1}"
+
     # Determine which version of FTL to download
     if [[ "${ftlBranch}" == "master" ]];then
-        url="https://github.com/pi-hole/FTL/releases/download/${latesttag%$'\r'}"
+        url="https://github.com/pi-hole/ftl/releases/latest/download"
     else
         url="https://ftl.pi-hole.net/${ftlBranch}"
     fi
@@ -2257,6 +2253,8 @@ get_binary_name() {
     local machine
     machine=$(uname -m)
 
+    local l_binary
+
     local str="Detecting architecture"
     printf "  %b %s..." "${INFO}" "${str}"
     # If the machine is arm or aarch
@@ -2272,24 +2270,24 @@ get_binary_name() {
         if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then
             printf "%b  %b Detected ARM-aarch64 architecture\\n" "${OVER}" "${TICK}"
             # set the binary to be used
-            binary="pihole-FTL-aarch64-linux-gnu"
+            l_binary="pihole-FTL-aarch64-linux-gnu"
         #
         elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then
             #
             if [[ "${rev}" -gt 6 ]]; then
                 printf "%b  %b Detected ARM-hf architecture (armv7+)\\n" "${OVER}" "${TICK}"
                 # set the binary to be used
-                binary="pihole-FTL-arm-linux-gnueabihf"
+                l_binary="pihole-FTL-arm-linux-gnueabihf"
             # Otherwise,
             else
                 printf "%b  %b Detected ARM-hf architecture (armv6 or lower) Using ARM binary\\n" "${OVER}" "${TICK}"
                 # set the binary to be used
-                binary="pihole-FTL-arm-linux-gnueabi"
+                l_binary="pihole-FTL-arm-linux-gnueabi"
             fi
         else
             printf "%b  %b Detected ARM architecture\\n" "${OVER}" "${TICK}"
             # set the binary to be used
-            binary="pihole-FTL-arm-linux-gnueabi"
+            l_binary="pihole-FTL-arm-linux-gnueabi"
         fi
     elif [[ "${machine}" == "x86_64" ]]; then
         # This gives the architecture of packages dpkg installs (for example, "i386")
@@ -2302,12 +2300,12 @@ get_binary_name() {
         # in the past (see https://github.com/pi-hole/pi-hole/pull/2004)
         if [[ "${dpkgarch}" == "i386" ]]; then
             printf "%b  %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}"
-            binary="pihole-FTL-linux-x86_32"
+            l_binary="pihole-FTL-linux-x86_32"
         else
             # 64bit
             printf "%b  %b Detected x86_64 architecture\\n" "${OVER}" "${TICK}"
             # set the binary to be used
-            binary="pihole-FTL-linux-x86_64"
+            l_binary="pihole-FTL-linux-x86_64"
         fi
     else
         # Something else - we try to use 32bit executable and warn the user
@@ -2318,13 +2316,13 @@ get_binary_name() {
         else
             printf "%b  %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}"
         fi
-        binary="pihole-FTL-linux-x86_32"
+        l_binary="pihole-FTL-linux-x86_32"
     fi
+
+    echo ${l_binary}
 }
 
 FTLcheckUpdate() {
-    get_binary_name
-
     #In the next section we check to see if FTL is already installed (in case of pihole -r).
     #If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download
     printf "  %b Checking for existing FTL binary...\\n" "${INFO}"
@@ -2340,6 +2338,9 @@ FTLcheckUpdate() {
         ftlBranch="master"
     fi
 
+    local binary
+    binary="${1}"
+
     local remoteSha1
     local localSha1
 
@@ -2382,7 +2383,12 @@ FTLcheckUpdate() {
             local FTLversion
             FTLversion=$(/usr/bin/pihole-FTL tag)
             local FTLlatesttag
-            FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep 'Location' | awk -F '/' '{print $NF}' | tr -d '\r\n')
+
+            if ! FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep --color=never -i Location | awk -F / '{print $NF}' | tr -d '[:cntrl:]'); then
+                # There was an issue while retrieving the latest version
+                printf "  %b Failed to retrieve latest FTL release metadata" "${CROSS}"
+                return 3
+            fi
 
             if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then
                 return 0
@@ -2410,8 +2416,10 @@ FTLcheckUpdate() {
 FTLdetect() {
     printf "\\n  %b FTL Checks...\\n\\n" "${INFO}"
 
-    if FTLcheckUpdate ; then
-        FTLinstall || return 1
+    printf "  %b" "${2}"
+
+    if FTLcheckUpdate "${1}"; then
+        FTLinstall "${1}" || return 1
     fi
 }
 
@@ -2506,7 +2514,7 @@ main() {
     notify_package_updates_available
 
     # Install packages used by this installation script
-    install_dependent_packages INSTALLER_DEPS[@]
+    install_dependent_packages "${INSTALLER_DEPS[@]}"
 
     # Check if SELinux is Enforcing
     checkSelinux
@@ -2557,7 +2565,7 @@ main() {
         dep_install_list+=("${PIHOLE_WEB_DEPS[@]}")
     fi
 
-    install_dependent_packages dep_install_list[@]
+    install_dependent_packages "${dep_install_list[@]}"
     unset dep_install_list
 
     # On some systems, lighttpd is not enabled on first install. We need to enable it here if the user
@@ -2571,8 +2579,17 @@ main() {
     else
         LIGHTTPD_ENABLED=false
     fi
+    # Create the pihole user
+    create_pihole_user
+
     # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole
-    if ! FTLdetect; then
+    local funcOutput
+    funcOutput=$(get_binary_name) #Store output of get_binary_name here
+    local binary
+    binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL)
+    local theRest
+    theRest="${funcOutput%pihole-FTL*}" # Print the rest of get_binary_name's output to display (cut out from first instance of "pihole-FTL")
+    if ! FTLdetect "${binary}" "${theRest}"; then
         printf "  %b FTL Engine not installed\\n" "${CROSS}"
         exit 1
     fi
@@ -2665,7 +2682,7 @@ main() {
 
     if [[ "${INSTALL_TYPE}" == "Update" ]]; then
         printf "\\n"
-        /usr/local/bin/pihole version --current
+        "${PI_HOLE_BIN_DIR}"/pihole version --current
     fi
 }
 

automated install/uninstall.sh

@@ -55,13 +55,13 @@ fi
 # Compatability
 if [ -x "$(command -v apt-get)" ]; then
     # Debian Family
-    PKG_REMOVE="${PKG_MANAGER} -y remove --purge"
+    PKG_REMOVE=("${PKG_MANAGER}" -y remove --purge)
     package_check() {
         dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed"
     }
 elif [ -x "$(command -v rpm)" ]; then
     # Fedora Family
-    PKG_REMOVE="${PKG_MANAGER} remove -y"
+    PKG_REMOVE=("${PKG_MANAGER}" remove -y)
     package_check() {
         rpm -qa | grep "^$1-" > /dev/null
     }
@@ -80,7 +80,7 @@ removeAndPurge() {
                 case ${yn} in
                     [Yy]* )
                         echo -ne "  ${INFO} Removing ${i}...";
-                        ${SUDO} "${PKG_REMOVE} ${i}" &> /dev/null;
+                        ${SUDO} "${PKG_REMOVE[@]}" "${i}" &> /dev/null;
                         echo -e "${OVER}  ${INFO} Removed ${i}";
                         break;;
                     [Nn]* ) echo -e "  ${INFO} Skipped ${i}"; break;;
@@ -132,12 +132,15 @@ removeNoPurge() {
     fi
 
     if package_check lighttpd > /dev/null; then
-        ${SUDO} rm -rf /etc/lighttpd/ &> /dev/null
-        echo -e "  ${TICK} Removed lighttpd"
-    else
-        if [ -f /etc/lighttpd/lighttpd.conf.orig ]; then
+        if [[ -f /etc/lighttpd/lighttpd.conf.orig ]]; then
             ${SUDO} mv /etc/lighttpd/lighttpd.conf.orig /etc/lighttpd/lighttpd.conf
         fi
+
+        if [[ -f /etc/lighttpd/external.conf ]]; then
+            ${SUDO} rm /etc/lighttpd/external.conf
+        fi
+
+        echo -e "  ${TICK} Removed lighttpd configs"
     fi
 
     ${SUDO} rm -f /etc/dnsmasq.d/adList.conf &> /dev/null

gravity.sh

@@ -263,7 +263,7 @@ gravity_DownloadBlocklistFromUrl() {
     else
         printf -v port "%s" "${PIHOLE_DNS_1#*#}"
     fi
-    ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}")
+    ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1)
     if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then
       port=443;
     else port=80
@@ -353,46 +353,7 @@ gravity_ParseFileIntoDomains() {
   # Determine how to parse individual source file formats
   if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then
     # Compare $firstLine against lower case words found in Adblock lists
-    echo -ne "  ${INFO} Format: Adblock"
-
-    # Define symbols used as comments: [!
-    # "||.*^" includes the "Example 2" domains we can extract
-    # https://adblockplus.org/filter-cheatsheet
-    abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
-
-    # Parse Adblock lists by extracting "Example 2" domains
-    # Logic: Ignore lines which do not include comments or domain name anchor
-    awk ''"${abpFilter}"' {
-      # Remove valid adblock type options
-      gsub(/\$?~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
-      # Remove starting domain name anchor "||" and ending seperator "^"
-      gsub(/^(\|\|)|(\^)/, "", $0)
-      # Remove invalid characters (*/,=$)
-      if($0 ~ /[*\/,=\$]/) { $0="" }
-      # Remove lines which are only IPv4 addresses
-      if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
-      if($0) { print $0 }
-    }' "${source}" > "${destination}"
-
-    # Determine if there are Adblock exception rules
-    # https://adblockplus.org/filters
-    if grep -q "^@@||" "${source}" &> /dev/null; then
-      # Parse Adblock lists by extracting exception rules
-      # Logic: Ignore lines which do not include exception format "@@||example.com^"
-      awk -F "[|^]" '/^@@\|\|.*\^/ {
-        # Remove valid adblock type options
-        gsub(/\$?~?(third-party)/, "", $0)
-        # Remove invalid characters (*/,=$)
-        if($0 ~ /[*\/,=\$]/) { $0="" }
-        if($3) { print $3 }
-      }' "${source}" > "${destination}.exceptionsFile.tmp"
-
-      # Remove exceptions
-      comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}"
-      mv "${source}" "${destination}"
-    fi
-
-    echo -e "${OVER}  ${TICK} Format: Adblock"
+    echo -e "  ${CROSS} Format: Adblock (list type not supported)"
   elif grep -q "^address=/" "${source}" &> /dev/null; then
     # Parse Dnsmasq format lists
     echo -e "  ${CROSS} Format: Dnsmasq (list type not supported)"

manpages/pihole.8

@@ -35,7 +35,7 @@ pihole -g\fR
 .br
 \fBpihole\fR \fB-l\fR (\fBon|off|off noflush\fR)
 .br
-\fBpihole -up \fR[--checkonly]
+\fBpihole -up \fR[--check-only]
 .br
 \fBpihole -v\fR [-p|-a|-f] [-c|-l|-hash]
 .br

pihole

@@ -13,10 +13,11 @@ readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
 readonly gravitylist="/etc/pihole/gravity.list"
 readonly blacklist="/etc/pihole/black.list"
 
-# setupVars is not readonly here because in some funcitons (checkout),
+# setupVars and PI_HOLE_BIN_DIR are not readonly here because in some funcitons (checkout),
 # it might get set again when the installer is sourced. This causes an
 # error due to modifying a readonly variable.
 setupVars="/etc/pihole/setupVars.conf"
+PI_HOLE_BIN_DIR="/usr/local/bin"
 
 readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE"
 source "${colfile}"
@@ -98,20 +99,16 @@ versionFunc() {
 
 restartDNS() {
   local svcOption svc str output status
-  svcOption="${1:-}"
+  svcOption="${1:-restart}"
 
-  # Determine if we should reload or restart restart
+  # Determine if we should reload or restart
   if [[ "${svcOption}" =~ "reload" ]]; then
-    # Using SIGHUP will NOT re-read any *.conf files
+    # Reload has been requested
+    # Note: This will NOT re-read any *.conf files
     svc="killall -s SIGHUP ${resolver}"
   else
-    # Get PID of resolver to determine if it needs to start or restart
-    if pidof pihole-FTL &> /dev/null; then
-      svcOption="restart"
-    else
-      svcOption="start"
-    fi
-    svc="service ${resolver} ${svcOption}"
+    # A full restart has been requested
+    svc="service ${resolver} restart"
   fi
 
   # Print output to Terminal, but not to Web Admin
@@ -164,7 +161,7 @@ Time:
           local str="Disabling blocking for ${tt} seconds"
           echo -e "  ${INFO} ${str}..."
           local str="Blocking will be re-enabled in ${tt} seconds"
-          nohup bash -c "sleep ${tt}; pihole enable" </dev/null &>/dev/null &
+          nohup bash -c "sleep ${tt}; ${PI_HOLE_BIN_DIR}/pihole enable" </dev/null &>/dev/null &
         else
           local error=true
         fi
@@ -175,7 +172,7 @@ Time:
           echo -e "  ${INFO} ${str}..."
           local str="Blocking will be re-enabled in ${tt} minutes"
           tt=$((${tt}*60))
-          nohup bash -c "sleep ${tt}; pihole enable" </dev/null &>/dev/null &
+          nohup bash -c "sleep ${tt}; ${PI_HOLE_BIN_DIR}/pihole enable" </dev/null &>/dev/null &
         else
           local error=true
         fi
@@ -237,7 +234,7 @@ Options:
     sed -i 's/^QUERY_LOGGING=true/QUERY_LOGGING=false/' /etc/pihole/setupVars.conf
     if [[ "${2}" != "noflush" ]]; then
       # Flush logs
-      pihole -f
+      "${PI_HOLE_BIN_DIR}"/pihole -f
     fi
     echo -e "  ${INFO} Disabling logging..."
     local str="Logging has been disabled!"
@@ -290,7 +287,7 @@ statusFunc() {
       *) echo -e "  ${INFO} Pi-hole blocking will be enabled";;
     esac
     # Enable blocking
-    pihole enable
+    "${PI_HOLE_BIN_DIR}"/pihole enable
   fi
 }
 

test/fedora.Dockerfile

@@ -1,4 +1,4 @@
-FROM fedora:latest
+FROM fedora:30
 
 ENV GITDIR /etc/.pihole
 ENV SCRIPTDIR /opt/pihole

test/test_automated_install.py

@@ -398,7 +398,11 @@ def test_FTL_detect_aarch64_no_errors(Pihole):
     )
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    FTLdetect
+    create_pihole_user
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     ''')
     expected_stdout = info_box + ' FTL Checks...'
     assert expected_stdout in detectPlatform.stdout
@@ -418,7 +422,11 @@ def test_FTL_detect_armv6l_no_errors(Pihole):
     mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole)
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    FTLdetect
+    create_pihole_user
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     ''')
     expected_stdout = info_box + ' FTL Checks...'
     assert expected_stdout in detectPlatform.stdout
@@ -439,7 +447,11 @@ def test_FTL_detect_armv7l_no_errors(Pihole):
     mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole)
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    FTLdetect
+    create_pihole_user
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     ''')
     expected_stdout = info_box + ' FTL Checks...'
     assert expected_stdout in detectPlatform.stdout
@@ -455,7 +467,11 @@ def test_FTL_detect_x86_64_no_errors(Pihole):
     '''
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    FTLdetect
+    create_pihole_user
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     ''')
     expected_stdout = info_box + ' FTL Checks...'
     assert expected_stdout in detectPlatform.stdout
@@ -471,7 +487,11 @@ def test_FTL_detect_unknown_no_errors(Pihole):
     mock_command('uname', {'-m': ('mips', '0')}, Pihole)
     detectPlatform = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    FTLdetect
+    create_pihole_user
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     ''')
     expected_stdout = 'Not able to detect architecture (unknown: mips)'
     assert expected_stdout in detectPlatform.stdout
@@ -481,56 +501,34 @@ def test_FTL_download_aarch64_no_errors(Pihole):
     '''
     confirms only aarch64 package is downloaded for FTL engine
     '''
+    # mock whiptail answers and ensure installer dependencies
+    mock_command('whiptail', {'*': ('', '0')}, Pihole)
+    Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    distro_check
+    install_dependent_packages ${INSTALLER_DEPS[@]}
+    ''')
     download_binary = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    binary="pihole-FTL-aarch64-linux-gnu"
-    FTLinstall
+    create_pihole_user
+    FTLinstall "pihole-FTL-aarch64-linux-gnu"
     ''')
     expected_stdout = tick_box + ' Downloading and Installing FTL'
     assert expected_stdout in download_binary.stdout
     assert 'error' not in download_binary.stdout.lower()
 
 
-def test_FTL_download_unknown_fails_no_errors(Pihole):
-    '''
-    confirms unknown binary is not downloaded for FTL engine
-    '''
-    download_binary = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    binary="pihole-FTL-mips"
-    FTLinstall
-    ''')
-    expected_stdout = cross_box + ' Downloading and Installing FTL'
-    assert expected_stdout in download_binary.stdout
-    error1 = 'Error: URL https://github.com/pi-hole/FTL/releases/download/'
-    assert error1 in download_binary.stdout
-    error2 = 'not found'
-    assert error2 in download_binary.stdout
-
-
-def test_FTL_download_binary_unset_no_errors(Pihole):
-    '''
-    confirms unset binary variable does not download FTL engine
-    '''
-    download_binary = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    FTLinstall
-    ''')
-    expected_stdout = cross_box + ' Downloading and Installing FTL'
-    assert expected_stdout in download_binary.stdout
-    error1 = 'Error: URL https://github.com/pi-hole/FTL/releases/download/'
-    assert error1 in download_binary.stdout
-    error2 = 'not found'
-    assert error2 in download_binary.stdout
-
-
 def test_FTL_binary_installed_and_responsive_no_errors(Pihole):
     '''
     confirms FTL binary is copied and functional in installed location
     '''
     installed_binary = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    FTLdetect
+    create_pihole_user
+    funcOutput=$(get_binary_name)
+    binary="pihole-FTL${funcOutput##*pihole-FTL}"
+    theRest="${funcOutput%pihole-FTL*}"
+    FTLdetect "${binary}" "${theRest}"
     pihole-FTL version
     ''')
     expected_stdout = 'v'