Initial work on stubbing for lua.

Signed-off-by: Dan Schaper <dan.schaper@pi-hole.net>

.github/workflows/test.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        distro: [debian_9, debian_10, debian_11, ubuntu_16, ubuntu_18, ubuntu_20, ubuntu_21, centos_7, centos_8, fedora_32, fedora_33]
+        distro: [debian_9, debian_10, ubuntu_16, ubuntu_18, ubuntu_20, centos_7, centos_8, fedora_32, fedora_33]
     env:
       DISTRO: ${{matrix.distro}}
     steps:

advanced/Scripts/chronometer.sh

@@ -498,6 +498,10 @@ chronoFunc() {
         printFunc " RAM usage: " "$ram_perc%" "$ram_info"
         printFunc " HDD usage: " "$disk_perc" "$disk_info"
 
+        if [[ "$scr_lines" -gt 17 ]] && [[ "$chrono_width" != "small" ]]; then
+            printFunc "  LAN addr: " "${IPV4_ADDRESS/\/*/}" "$lan_info"
+        fi
+
         if [[ "$DHCP_ACTIVE" == "true" ]]; then
             printFunc "DHCP usage: " "$ph_dhcp_percent%" "$dhcp_info"
         fi

advanced/Scripts/piholeDebug.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # Pi-hole: A black hole for Internet advertisements
-# (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+# (c) 2021 Pi-hole (https://pi-hole.net)
 # Network-wide ad blocking via your own hardware.
 #
 # Generates pihole_debug.log to be used for troubleshooting.
@@ -198,6 +198,33 @@ show_disclaimer(){
     log_write "${DISCLAIMER}"
 }
 
+check_for_ftl(){
+    echo_current_diagnostic "Checking for pihole-FTL binary"
+
+    declare -g FTL_PATH
+    read -r FTL_PATH < <(which pihole-FTL)
+    if [ -z "${FTL_PATH}" ]; then
+        log_write "${CROSS} ${COL_RED} Unable to find pihole-FTL binary.${COL_NC}"
+        # Non-zero return value
+        return 2
+    else
+        log_write "${TICK} pihole-FTL: ${COL_GREEN}${FTL_PATH}${COL_NC}"
+    fi
+
+}
+
+check_for_lua(){
+    echo_current_diagnostic "Checking for lua capabilities"
+
+    if ! (${FTL_PATH} lua -v &>/dev/null); then
+        log_write "${CROSS} ${COL_RED} pihole-FTL binary does not have lua capabilites.${COL_NC}"
+        # Non-zero return value
+        return
+    else
+        log_write "${TICK} pihole-FTL: ${COL_GREEN}lua found!${COL_NC}"
+    fi
+}
+
 source_setup_variables() {
     # Display the current test that is running
     log_write "\\n${COL_PURPLE}*** [ INITIALIZING ]${COL_NC} Sourcing setup variables"
@@ -410,12 +437,12 @@ os_check() {
     # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net
     # and determines whether or not the script is running on one of those systems
     local remote_os_domain valid_os valid_version detected_os detected_version cmdResult digReturnCode response
-    remote_os_domain=${OS_CHECK_DOMAIN_NAME:-"versions.pi-hole.net"}
+    remote_os_domain="versions.pi-hole.net"
 
     detected_os=$(grep "\bID\b" /etc/os-release | cut -d '=' -f2 | tr -d '"')
     detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"')
 
-    cmdResult="$(dig +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)"
+    cmdResult="$(dig +short -t txt ${remote_os_domain} @ns1.pi-hole.net 2>&1; echo $?)"
     #Get the return code of the previous command (last line)
     digReturnCode="${cmdResult##*$'\n'}"
 
@@ -859,13 +886,13 @@ dig_at() {
 
     # Store the arguments as variables with names
     local protocol="${1}"
+    local IP="${2}"
     echo_current_diagnostic "Name resolution (IPv${protocol}) using a random blocked domain and a known ad-serving domain"
     # Set more local variables
     # We need to test name resolution locally, via Pi-hole, and via a public resolver
     local local_dig
+    local pihole_dig
     local remote_dig
-    local interfaces
-    local addresses
     # Use a static domain that we know has IPv4 and IPv6 to avoid false positives
     # Sometimes the randomly chosen domains don't use IPv6, or something else is wrong with them
     local remote_url="doubleclick.com"
@@ -874,15 +901,15 @@ dig_at() {
     if [[ ${protocol} == "6" ]]; then
         # Set the IPv6 variables and record type
         local local_address="::1"
+        local pihole_address="${IP}"
         local remote_address="2001:4860:4860::8888"
-        local sed_selector="inet6"
         local record_type="AAAA"
     # Otherwise, it should be 4
     else
         # so use the IPv4 values
         local local_address="127.0.0.1"
+        local pihole_address="${IP}"
         local remote_address="8.8.8.8"
-        local sed_selector="inet"
         local record_type="A"
     fi
 
@@ -892,51 +919,32 @@ dig_at() {
     local random_url
     random_url=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1")
 
+    # First, do a dig on localhost to see if Pi-hole can use itself to block a domain
+    if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then
+        # If it can, show success
+        log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} via ${COL_CYAN}localhost$COL_NC (${local_address})"
+    else
+        # Otherwise, show a failure
+        log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} via ${COL_RED}localhost${COL_NC} (${local_address})"
+    fi
+
     # Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address
     # This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is
     # just asing itself locally
-    # The default timeouts and tries are reduced in case the DNS server isn't working, so the user isn't
-    # waiting for too long
-    #
-    # Turn off history expansion such that the "!" in the sed command cannot do silly things
-    set +H
-    # Get interfaces
-    # sed logic breakdown:
-    #     / master /d;
-    #          Removes all interfaces that are slaves of others (e.g. virtual docker interfaces)
-    #     /UP/!d;
-    #          Removes all interfaces which are not UP
-    #     s/^[0-9]*: //g;
-    #          Removes interface index
-    #     s/: <.*//g;
-    #          Removes everything after the interface name
-    interfaces="$(ip link show | sed "/ master /d;/UP/!d;s/^[0-9]*: //g;s/: <.*//g;")"
+    # The default timeouts and tries are reduced in case the DNS server isn't working, so the user isn't waiting for too long
 
-    while IFS= read -r iface ; do
-        # Get addresses of current interface
-        # sed logic breakdown:
-        #     /inet(|6) /!d;
-        #          Removes all lines from ip a that do not contain either "inet " or "inet6 "
-        #     s/^.*inet(|6) //g;
-        #          Removes all leading whitespace as well as the "inet " or "inet6 " string
-        #     s/\/.*$//g;
-        #          Removes CIDR and everything thereafter (e.g., scope properties)
-        addresses="$(ip address show dev "${iface}" | sed "/${sed_selector} /!d;s/^.*${sed_selector} //g;s/\/.*$//g;")"
-        while IFS= read -r local_address ; do
-            # Check if Pi-hole can use itself to block a domain
-            if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" +short "${record_type}"); then
-                # If it can, show success
-                log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})"
-            else
-                # Otherwise, show a failure
-                log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})"
-            fi
-        done <<< "${addresses}"
-    done <<< "${interfaces}"
+    # If Pi-hole can dig itself from it's IP (not the loopback address)
+    if pihole_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${pihole_address}" +short "${record_type}"); then
+        # show a success
+        log_write "${TICK} ${random_url} ${COL_GREEN}is ${pihole_dig}${COL_NC} via ${COL_CYAN}Pi-hole${COL_NC} (${pihole_address})"
+    else
+        # Otherwise, show a failure
+        log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} via ${COL_RED}Pi-hole${COL_NC} (${pihole_address})"
+    fi
 
     # Finally, we need to make sure legitimate queries can out to the Internet using an external, public DNS server
     # We are using the static remote_url here instead of a random one because we know it works with IPv4 and IPv6
-    if remote_dig=$(dig +tries=1 +time=2 -"${protocol}" "${remote_url}" @"${remote_address}" +short "${record_type}" | head -n1); then
+    if remote_dig=$(dig +tries=1 +time=2 -"${protocol}" "${remote_url}" @${remote_address} +short "${record_type}" | head -n1); then
         # If successful, the real IP of the domain will be returned instead of Pi-hole's IP
         log_write "${TICK} ${remote_url} ${COL_GREEN}is ${remote_dig}${COL_NC} via ${COL_CYAN}a remote, public DNS server${COL_NC} (${remote_address})"
     else
@@ -1051,7 +1059,7 @@ parse_file() {
     local file_lines
     # For each line in the file,
     for file_lines in "${file_info[@]}"; do
-        if [[ -n "${file_lines}" ]]; then
+        if [[ ! -z "${file_lines}" ]]; then
             # don't include the Web password hash
             [[ "${file_lines}" =~ ^\#.*$  || ! "${file_lines}" || "${file_lines}" == "WEBPASSWORD="* ]] && continue
             # otherwise, display the lines of the file
@@ -1065,8 +1073,12 @@ parse_file() {
 check_name_resolution() {
     # Check name resolution from localhost, Pi-hole's IP, and Google's name severs
     # using the function we created earlier
-    dig_at 4
-    dig_at 6
+    dig_at 4 "${IPV4_ADDRESS%/*}"
+    # If IPv6 enabled,
+    if [[ "${IPV6_ADDRESS}" ]]; then
+        # check resolution
+        dig_at 6 "${IPV6_ADDRESS%/*}"
+    fi
 }
 
 # This function can check a directory exists
@@ -1296,74 +1308,56 @@ analyze_gravity_list() {
     IFS="$OLD_IFS"
 }
 
-obfuscated_pihole_log() {
-  local pihole_log=("$@")
-  local line
-  local error_to_check_for
-  local line_to_obfuscate
-  local obfuscated_line
-  for line in "${pihole_log[@]}"; do
-      # A common error in the pihole.log is when there is a non-hosts formatted file
-      # that the DNS server is attempting to read.  Since it's not formatted
-      # correctly, there will be an entry for "bad address at line n"
-      # So we can check for that here and highlight it in red so the user can see it easily
-      error_to_check_for=$(echo "${line}" | grep 'bad address at')
-      # Some users may not want to have the domains they visit sent to us
-      # To that end, we check for lines in the log that would contain a domain name
-      line_to_obfuscate=$(echo "${line}" | grep ': query\|: forwarded\|: reply')
-      # If the variable contains a value, it found an error in the log
-      if [[ -n ${error_to_check_for} ]]; then
-          # So we can print it in red to make it visible to the user
-          log_write "   ${CROSS} ${COL_RED}${line}${COL_NC} (${FAQ_BAD_ADDRESS})"
-      else
-          # If the variable does not a value (the current default behavior), so do not obfuscate anything
-          if [[ -z ${OBFUSCATE} ]]; then
-              log_write "   ${line}"
-          # Othwerise, a flag was passed to this command to obfuscate domains in the log
-          else
-              # So first check if there are domains in the log that should be obfuscated
-              if [[ -n ${line_to_obfuscate} ]]; then
-                  # If there are, we need to use awk to replace only the domain name (the 6th field in the log)
-                  # so we substitute the domain for the placeholder value
-                  obfuscated_line=$(echo "${line_to_obfuscate}" | awk -v placeholder="${OBFUSCATED_PLACEHOLDER}" '{sub($6,placeholder); print $0}')
-                  log_write "   ${obfuscated_line}"
-              else
-                  log_write "   ${line}"
-              fi
-          fi
-      fi
-  done
-}
-
 analyze_pihole_log() {
-  echo_current_diagnostic "Pi-hole log"
-  local pihole_log_head=()
-  local pihole_log_tail=()
-  local pihole_log_permissions
-  local logging_enabled
-
-  logging_enabled=$(grep -c "^log-queries" /etc/dnsmasq.d/01-pihole.conf)
-  if [[ "${logging_enabled}" == "0" ]]; then
-      # Inform user that logging has been disabled and pihole.log does not contain queries
-      log_write "${INFO} Query logging is disabled"
-      log_write ""
-  fi
-  # Put the current Internal Field Separator into another variable so it can be restored later
-  OLD_IFS="$IFS"
-  # Get the lines that are in the file(s) and store them in an array for parsing later
-  IFS=$'\r\n'
-  pihole_log_permissions=$(ls -ld "${PIHOLE_LOG}")
-  log_write "${COL_GREEN}${pihole_log_permissions}${COL_NC}"
-  mapfile -t pihole_log_head < <(head -n 20 ${PIHOLE_LOG})
-  log_write "   ${COL_CYAN}-----head of $(basename ${PIHOLE_LOG})------${COL_NC}"
-  obfuscated_pihole_log "${pihole_log_head[@]}"
-  log_write ""
-  mapfile -t pihole_log_tail < <(tail -n 20 ${PIHOLE_LOG})
-  log_write "   ${COL_CYAN}-----tail of $(basename ${PIHOLE_LOG})------${COL_NC}"
-  obfuscated_pihole_log "${pihole_log_tail[@]}"
-  log_write ""
-  # Set the IFS back to what it was
-  IFS="$OLD_IFS"
+    echo_current_diagnostic "Pi-hole log"
+    local head_line
+    # Put the current Internal Field Separator into another variable so it can be restored later
+    OLD_IFS="$IFS"
+    # Get the lines that are in the file(s) and store them in an array for parsing later
+    IFS=$'\r\n'
+    local pihole_log_permissions
+    pihole_log_permissions=$(ls -ld "${PIHOLE_LOG}")
+    log_write "${COL_GREEN}${pihole_log_permissions}${COL_NC}"
+    local pihole_log_head=()
+    mapfile -t pihole_log_head < <(head -n 20 ${PIHOLE_LOG})
+    log_write "   ${COL_CYAN}-----head of $(basename ${PIHOLE_LOG})------${COL_NC}"
+    local error_to_check_for
+    local line_to_obfuscate
+    local obfuscated_line
+    for head_line in "${pihole_log_head[@]}"; do
+        # A common error in the pihole.log is when there is a non-hosts formatted file
+        # that the DNS server is attempting to read.  Since it's not formatted
+        # correctly, there will be an entry for "bad address at line n"
+        # So we can check for that here and highlight it in red so the user can see it easily
+        error_to_check_for=$(echo "${head_line}" | grep 'bad address at')
+        # Some users may not want to have the domains they visit sent to us
+        # To that end, we check for lines in the log that would contain a domain name
+        line_to_obfuscate=$(echo "${head_line}" | grep ': query\|: forwarded\|: reply')
+        # If the variable contains a value, it found an error in the log
+        if [[ -n ${error_to_check_for} ]]; then
+            # So we can print it in red to make it visible to the user
+            log_write "   ${CROSS} ${COL_RED}${head_line}${COL_NC} (${FAQ_BAD_ADDRESS})"
+        else
+            # If the variable does not a value (the current default behavior), so do not obfuscate anything
+            if [[ -z ${OBFUSCATE} ]]; then
+                log_write "   ${head_line}"
+            # Othwerise, a flag was passed to this command to obfuscate domains in the log
+            else
+                # So first check if there are domains in the log that should be obfuscated
+                if [[ -n ${line_to_obfuscate} ]]; then
+                    # If there are, we need to use awk to replace only the domain name (the 6th field in the log)
+                    # so we substitute the domain for the placeholder value
+                    obfuscated_line=$(echo "${line_to_obfuscate}" | awk -v placeholder="${OBFUSCATED_PLACEHOLDER}" '{sub($6,placeholder); print $0}')
+                    log_write "   ${obfuscated_line}"
+                else
+                    log_write "   ${head_line}"
+                fi
+            fi
+        fi
+    done
+    log_write ""
+    # Set the IFS back to what it was
+    IFS="$OLD_IFS"
 }
 
 tricorder_use_nc_or_curl() {
@@ -1454,28 +1448,40 @@ make_temporary_log
 initialize_debug
 # setupVars.conf needs to be sourced before the networking so the values are
 # available to the other functions
-source_setup_variables
-check_component_versions
-check_critical_program_versions
-diagnose_operating_system
-check_selinux
-check_firewalld
-processor_check
-check_networking
-check_name_resolution
-check_dhcp_servers
-process_status
-ftl_full_status
-parse_setup_vars
-check_x_headers
-analyze_gravity_list
-show_groups
-show_domainlist
-show_clients
-show_adlists
-show_content_of_pihole_files
-show_messages
-parse_locale
-analyze_pihole_log
-copy_to_debug_log
-upload_to_tricorder
+if ! check_for_ftl; then
+    log_write "${COL_RED}Unable to complete debug run. Please contact support for assistance."
+    log_write "Please note the error that is displayed above.${COL_NC}"
+    #Non-zero return value
+    exit 2
+fi
+if ! check_for_lua; then
+    log_write "${COL_RED}Unable to complete debug run. Please contact support for assistance."
+    log_write "Please note the error that is displayed above.${COL_NC}"
+    #Non-zero return value
+    exit 2
+fi
+# source_setup_variables
+# check_component_versions
+# check_critical_program_versions
+# diagnose_operating_system
+# check_selinux
+# check_firewalld
+# processor_check
+# check_networking
+# check_name_resolution
+# check_dhcp_servers
+# process_status
+# ftl_full_status
+# parse_setup_vars
+# check_x_headers
+# analyze_gravity_list
+# show_groups
+# show_domainlist
+# show_clients
+# show_adlists
+# show_content_of_pihole_files
+# show_messages
+# parse_locale
+# analyze_pihole_log
+# copy_to_debug_log
+# upload_to_tricorder

advanced/Scripts/webpage.sh

@@ -273,6 +273,11 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423
         fi
     fi
 
+    # Prevent Firefox from automatically switching over to DNS-over-HTTPS
+    # This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https
+    # (sourced 7th September 2019)
+    add_dnsmasq_setting "server=/use-application-dns.net/"
+
     # We need to process DHCP settings here as well to account for possible
     # changes in the non-FQDN forwarding. This cannot be done in 01-pihole.conf
     # as we don't want to delete all local=/.../ lines so it's much safer to
@@ -421,7 +426,7 @@ dhcp-leasefile=/etc/pihole/dhcp.leases
         echo "#quiet-dhcp6
 #enable-ra
 dhcp-option=option6:dns-server,[::]
-dhcp-range=::100,::1ff,constructor:${interface},ra-names,slaac,64,3600
+dhcp-range=::100,::1ff,constructor:${interface},ra-names,slaac,${leasetime}
 ra-param=*,0,0
 " >> "${dhcpconfig}"
     fi
@@ -711,7 +716,7 @@ RemoveCustomDNSAddress() {
     host="${args[3]}"
 
     if valid_ip "${ip}" || valid_ip6 "${ip}" ; then
-        sed -i "/^${ip} ${host}$/d" "${dnscustomfile}"
+        sed -i "/${ip} ${host}/d" "${dnscustomfile}"
     else
         echo -e "  ${CROSS} Invalid IP has been passed"
         exit 1
@@ -743,7 +748,7 @@ RemoveCustomCNAMERecord() {
     if [[ -n "${validDomain}" ]]; then
         validTarget="$(checkDomain "${target}")"
         if [[ -n "${validDomain}" ]]; then
-            sed -i "/cname=${validDomain},${validTarget}$/d" "${dnscustomcnamefile}"
+            sed -i "/cname=${validDomain},${validTarget}/d" "${dnscustomcnamefile}"
         else
             echo "  ${CROSS} Invalid Target Passed!"
             exit 1

advanced/lighttpd.conf.debian

@@ -20,6 +20,7 @@ server.modules = (
     "mod_accesslog",
     "mod_auth",
     "mod_expire",
+    "mod_compress",
     "mod_redirect",
     "mod_setenv",
     "mod_rewrite"
@@ -40,6 +41,26 @@ index-file.names            = ( "index.php", "index.html", "index.lighttpd.html"
 url.access-deny             = ( "~", ".inc", ".md", ".yml", ".ini" )
 static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" )
 
+compress.cache-dir = "/var/cache/lighttpd/compress/"
+compress.filetype  = (
+    "application/json",
+    "application/vnd.ms-fontobject",
+    "application/xml",
+    "font/eot",
+    "font/opentype",
+    "font/otf",
+    "font/ttf",
+    "image/bmp",
+    "image/svg+xml",
+    "image/vnd.microsoft.icon",
+    "image/x-icon",
+    "text/css",
+    "text/html",
+    "text/javascript",
+    "text/plain",
+    "text/xml"
+)
+
 mimetype.assign = (
     ".ico"   => "image/x-icon",
     ".jpeg"  => "image/jpeg",

advanced/lighttpd.conf.fedora

@@ -21,6 +21,7 @@ server.modules = (
     "mod_expire",
     "mod_fastcgi",
     "mod_accesslog",
+    "mod_compress",
     "mod_redirect",
     "mod_setenv",
     "mod_rewrite"
@@ -41,6 +42,26 @@ index-file.names            = ( "index.php", "index.html", "index.lighttpd.html"
 url.access-deny             = ( "~", ".inc", ".md", ".yml", ".ini" )
 static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" )
 
+compress.cache-dir = "/var/cache/lighttpd/compress/"
+compress.filetype  = (
+    "application/json",
+    "application/vnd.ms-fontobject",
+    "application/xml",
+    "font/eot",
+    "font/opentype",
+    "font/otf",
+    "font/ttf",
+    "image/bmp",
+    "image/svg+xml",
+    "image/vnd.microsoft.icon",
+    "image/x-icon",
+    "text/css",
+    "text/html",
+    "text/javascript",
+    "text/plain",
+    "text/xml"
+)
+
 mimetype.assign = (
     ".ico"   => "image/x-icon",
     ".jpeg"  => "image/jpeg",

automated install/basic-install.sh

@@ -186,12 +186,12 @@ os_check() {
         # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net
         # and determines whether or not the script is running on one of those systems
         local remote_os_domain valid_os valid_version valid_response detected_os detected_version display_warning cmdResult digReturnCode response
-        remote_os_domain=${OS_CHECK_DOMAIN_NAME:-"versions.pi-hole.net"}
+        remote_os_domain="versions.pi-hole.net"
 
         detected_os=$(grep "\bID\b" /etc/os-release | cut -d '=' -f2 | tr -d '"')
         detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"')
 
-        cmdResult="$(dig +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)"
+        cmdResult="$(dig +short -t txt ${remote_os_domain} @ns1.pi-hole.net 2>&1; echo $?)"
         # Gets the return code of the previous command (last line)
         digReturnCode="${cmdResult##*$'\n'}"
 
@@ -1944,17 +1944,9 @@ finalExports() {
 # Install the logrotate script
 installLogrotate() {
     local str="Installing latest logrotate script"
-    local target=/etc/pihole/logrotate
-
     printf "\\n  %b %s..." "${INFO}" "${str}"
-    if [[ -f ${target} ]]; then
-        printf "\\n\\t%b Existing logrotate file found. No changes made.\\n" "${INFO}"
-        # Return value isn't that important, using 2 to indicate that it's not a fatal error but
-        # the function did not complete. 
-        return 2
-    fi
     # Copy the file over from the local repo
-    install -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target}
+    install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate
     # Different operating systems have different user / group
     # settings for logrotate that makes it impossible to create
     # a static logrotate file that will work with e.g.
@@ -1963,9 +1955,9 @@ installLogrotate() {
     # the local properties of the /var/log directory
     logusergroup="$(stat -c '%U %G' /var/log)"
     # If there is a usergroup for log rotation,
-    if [[ -n "${logusergroup}" ]]; then
+    if [[ ! -z "${logusergroup}" ]]; then
         # replace the line in the logrotate script with that usergroup.
-        sed -i "s/# su #/su ${logusergroup}/g;" ${target}
+        sed -i "s/# su #/su ${logusergroup}/g;" /etc/pihole/logrotate
     fi
     printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
 }
@@ -2042,10 +2034,8 @@ installPihole() {
     fi
     # Install the cron file
     installCron
-    if ! installLogrotate; then
-        printf "  %b Failure in logrotate installation function.\\n" "${CROSS}"
-        # This isn't fatal, no need to exit.    
-    fi
+    # Install the logrotate file
+    installLogrotate
     # Check if dnsmasq is present. If so, disable it and back up any possible
     # config file
     disable_dnsmasq

automated install/uninstall.sh

@@ -206,7 +206,11 @@ removeNoPurge() {
 }
 
 ######### SCRIPT ###########
-echo -e "  ${INFO} Be sure to confirm if any dependencies should not be removed"
+if command -v vcgencmd &> /dev/null; then
+    echo -e "  ${INFO} All dependencies are safe to remove on Raspbian"
+else
+    echo -e "  ${INFO} Be sure to confirm if any dependencies should not be removed"
+fi
 while true; do
     echo -e "  ${INFO} ${COL_YELLOW}The following dependencies may have been added by the Pi-hole install:"
     echo -n "    "

gravity.sh

@@ -47,6 +47,16 @@ domainsExtension="domains"
 setupVars="${piholeDir}/setupVars.conf"
 if [[ -f "${setupVars}" ]];then
   source "${setupVars}"
+
+  # Remove CIDR mask from IPv4/6 addresses
+  IPV4_ADDRESS="${IPV4_ADDRESS%/*}"
+  IPV6_ADDRESS="${IPV6_ADDRESS%/*}"
+
+  # Determine if IPv4/6 addresses exist
+  if [[ -z "${IPV4_ADDRESS}" ]] && [[ -z "${IPV6_ADDRESS}" ]]; then
+    echo -e "  ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}"
+    exit 1
+  fi
 else
   echo -e "  ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC}
   Please run 'pihole -r', and choose the 'reconfigure' option to fix."
@@ -63,8 +73,6 @@ fi
 # have changed
 gravityDBfile="${GRAVITYDB}"
 gravityTEMPfile="${GRAVITYDB}_temp"
-gravityDIR="$(dirname -- "${gravityDBfile}")"
-gravityOLDfile="${gravityDIR}/gravity_old.db"
 
 if [[ -z "${BLOCKINGMODE}" ]] ; then
   BLOCKINGMODE="NULL"
@@ -115,19 +123,8 @@ gravity_swap_databases() {
   fi
   echo -e "${OVER}  ${TICK} ${str}"
 
-  # Swap databases and remove or conditionally rename old database
-  # Number of available blocks on disk
-  availableBlocks=$(stat -f --format "%a" "${gravityDIR}")
-  # Number of blocks, used by gravity.db
-  gravityBlocks=$(stat --format "%b" ${gravityDBfile})
-  # Only keep the old database if available disk space is at least twice the size of the existing gravity.db.
-  # Better be safe than sorry...
-  if [ "${availableBlocks}" -gt "$(("${gravityBlocks}" * 2))" ] && [ -f "${gravityDBfile}" ]; then
-    echo -e "  ${TICK} The old database remains available."
-    mv "${gravityDBfile}" "${gravityOLDfile}"
-  else
-    rm "${gravityDBfile}"
-  fi
+  # Swap databases and remove old database
+  rm "${gravityDBfile}"
   mv "${gravityTEMPfile}" "${gravityDBfile}"
 }
 
@@ -567,7 +564,7 @@ compareLists() {
 # Download specified URL and perform checks on HTTP status and file content
 gravity_DownloadBlocklistFromUrl() {
   local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" saveLocation="${5}" target="${6}" compression="${7}"
-  local heisenbergCompensator="" patternBuffer str httpCode success="" ip
+  local heisenbergCompensator="" patternBuffer str httpCode success=""
 
   # Create temp file to store content on disk instead of RAM
   patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb")
@@ -585,10 +582,7 @@ gravity_DownloadBlocklistFromUrl() {
   blocked=false
   case $BLOCKINGMODE in
     "IP-NODATA-AAAA"|"IP")
-        # Get IP address of this domain
-        ip="$(dig "${domain}" +short)"
-        # Check if this IP matches any IP of the system
-        if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then
+        if [[ $(dig "${domain}" +short | grep "${IPV4_ADDRESS}" -c) -ge 1 ]]; then
           blocked=true
         fi;;
     "NXDOMAIN")
@@ -791,12 +785,43 @@ gravity_ShowCount() {
   gravity_Table_Count "vw_regex_whitelist" "regex whitelist filters"
 }
 
+# Parse list of domains into hosts format
+gravity_ParseDomainsIntoHosts() {
+  awk -v ipv4="$IPV4_ADDRESS" -v ipv6="$IPV6_ADDRESS" '{
+    # Remove windows CR line endings
+    sub(/\r$/, "")
+    # Parse each line as "ipaddr domain"
+    if(ipv6 && ipv4) {
+      print ipv4" "$0"\n"ipv6" "$0
+    } else if(!ipv6) {
+      print ipv4" "$0
+    } else {
+      print ipv6" "$0
+    }
+  }' >> "${2}" < "${1}"
+}
+
 # Create "localhost" entries into hosts format
 gravity_generateLocalList() {
+  local hostname
+
+  if [[ -s "/etc/hostname" ]]; then
+    hostname=$(< "/etc/hostname")
+  elif command -v hostname &> /dev/null; then
+    hostname=$(hostname -f)
+  else
+    echo -e "  ${CROSS} Unable to determine fully qualified domain name of host"
+    return 0
+  fi
+
+  echo -e "${hostname}\\npi.hole" > "${localList}.tmp"
+
   # Empty $localList if it already exists, otherwise, create it
-  echo "### Do not modify this file, it will be overwritten by pihole -g" > "${localList}"
+  : > "${localList}"
   chmod 644 "${localList}"
 
+  gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
+
   # Add additional LAN hosts provided by OpenVPN (if available)
   if [[ -f "${VPNList}" ]]; then
     awk -F, '{printf $2"\t"$1".vpn\n"}' "${VPNList}" >> "${localList}"
@@ -865,11 +890,6 @@ for var in "$@"; do
   esac
 done
 
-# Remove OLD (backup) gravity file, if it exists
-if [[ -f "${gravityOLDfile}" ]]; then
-  rm "${gravityOLDfile}"
-fi
-
 # Trap Ctrl-C
 gravity_Trap
 

manpages/pihole.8

@@ -56,7 +56,7 @@ Available commands and options:
 
 \fB-w, whitelist\fR [options] [<domain1> <domain2 ...>]
 .br
-    Adds or removes specified domain or domains to the Whitelist
+    Adds or removes specified domain or domains tho the Whitelist
 .br
 
 \fB-b, blacklist\fR [options] [<domain1> <domain2 ...>]

pihole

@@ -363,13 +363,16 @@ tailFunc() {
   fi
   echo -e "  ${INFO} Press Ctrl-C to exit"
 
+  # Retrieve IPv4/6 addresses
+  source /etc/pihole/setupVars.conf
+
   # Strip date from each line
   # Color blocklist/blacklist/wildcard entries as red
   # Color A/AAAA/DHCP strings as white
   # Color everything else as gray
-  tail -f /var/log/pihole.log | grep --line-buffered "${1}" | sed -E \
+  tail -f /var/log/pihole.log | sed -E \
     -e "s,($(date +'%b %d ')| dnsmasq\[[0-9]*\]),,g" \
-    -e "s,(.*(blacklisted |gravity blocked ).*),${COL_RED}&${COL_NC}," \
+    -e "s,(.*(blacklisted |gravity blocked ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
     -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \
     -e "s,.*,${COL_GRAY}&${COL_NC},"
   exit 0
@@ -453,10 +456,7 @@ Debugging Options:
                         Add '-a' to automatically upload the log to tricorder.pi-hole.net
   -f, flush           Flush the Pi-hole log
   -r, reconfigure     Reconfigure or Repair Pi-hole subsystems
-  -t, tail [arg]      View the live output of the Pi-hole log.
-                      Add an optional argument to filter the log
-                      (regular expressions are supported)
-
+  -t, tail            View the live output of the Pi-hole log
 
 Options:
   -a, admin           Web interface options
@@ -530,7 +530,7 @@ case "${1}" in
   "status"                      ) statusFunc "$2";;
   "restartdns"                  ) restartDNS "$2";;
   "-a" | "admin"                ) webpageFunc "$@";;
-  "-t" | "tail"                 ) tailFunc "$2";;
+  "-t" | "tail"                 ) tailFunc;;
   "checkout"                    ) piholeCheckoutFunc "$@";;
   "tricorder"                   ) tricorderFunc;;
   "updatechecker"               ) updateCheckFunc "$@";;

supportedos.txt

@@ -0,0 +1,5 @@
+Raspbian=9,10
+Ubuntu=16,18,20
+Debian=9,10
+Fedora=32,33
+CentOS=7,8

test/_centos_7.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_centos_8.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_debian_10.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_debian_11.Dockerfile

@@ -1,17 +0,0 @@
-FROM buildpack-deps:bullseye-scm
-
-ENV GITDIR /etc/.pihole
-ENV SCRIPTDIR /opt/pihole
-
-RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
-ADD . $GITDIR
-RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/
-ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
-
-RUN true && \
-    chmod +x $SCRIPTDIR/*
-
-ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
-
-#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_debian_9.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_fedora_32.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_fedora_33.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_ubuntu_16.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_ubuntu_18.Dockerfile

@@ -12,6 +12,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_ubuntu_20.Dockerfile

@@ -13,6 +13,5 @@ RUN true && \
     chmod +x $SCRIPTDIR/*
 
 ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
 
 #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/_ubuntu_21.Dockerfile

@@ -1,18 +0,0 @@
-FROM buildpack-deps:hirsute-scm
-
-ENV GITDIR /etc/.pihole
-ENV SCRIPTDIR /opt/pihole
-
-RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
-ADD . $GITDIR
-RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/
-ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN true && \
-    chmod +x $SCRIPTDIR/*
-
-ENV PH_TEST true
-ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net
-
-#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/conftest.py

@@ -8,6 +8,8 @@ check_output = testinfra.get_backend(
 
 SETUPVARS = {
     'PIHOLE_INTERFACE': 'eth99',
+    'IPV4_ADDRESS': '1.1.1.1',
+    'IPV6_ADDRESS': 'FE80::240:D0FF:FE48:4672',
     'PIHOLE_DNS_1': '4.2.2.1',
     'PIHOLE_DNS_2': '4.2.2.2'
 }

test/test_automated_install.py

@@ -45,6 +45,8 @@ def test_setupVars_are_sourced_to_global_scope(Pihole):
         # Currently debug test function only
         echo "Outputting sourced variables"
         echo "PIHOLE_INTERFACE=${PIHOLE_INTERFACE}"
+        echo "IPV4_ADDRESS=${IPV4_ADDRESS}"
+        echo "IPV6_ADDRESS=${IPV6_ADDRESS}"
         echo "PIHOLE_DNS_1=${PIHOLE_DNS_1}"
         echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}"
     }
@@ -595,42 +597,3 @@ def test_os_check_passes(Pihole):
     ''')
     expected_stdout = 'Supported OS detected'
     assert expected_stdout in detectOS.stdout
-
-
-def test_package_manager_has_installer_deps(Pihole):
-    ''' Confirms OS is able to install the required packages for the installer'''
-    mock_command('whiptail', {'*': ('', '0')}, Pihole)
-    output = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    distro_check
-    install_dependent_packages ${INSTALLER_DEPS[@]}
-    ''')
-
-    assert 'No package' not in output.stdout  # centos7 still exits 0...
-    assert output.rc == 0
-
-
-def test_package_manager_has_pihole_deps(Pihole):
-    ''' Confirms OS is able to install the required packages for Pi-hole '''
-    mock_command('whiptail', {'*': ('', '0')}, Pihole)
-    output = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    distro_check
-    install_dependent_packages ${PIHOLE_DEPS[@]}
-    ''')
-
-    assert 'No package' not in output.stdout  # centos7 still exits 0...
-    assert output.rc == 0
-
-
-def test_package_manager_has_web_deps(Pihole):
-    ''' Confirms OS is able to install the required packages for web '''
-    mock_command('whiptail', {'*': ('', '0')}, Pihole)
-    output = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    distro_check
-    install_dependent_packages ${PIHOLE_WEB_DEPS[@]}
-    ''')
-
-    assert 'No package' not in output.stdout  # centos7 still exits 0...
-    assert output.rc == 0

test/tox.debian_11.ini

@@ -1,8 +0,0 @@
-[tox]
-envlist = py37
-
-[testenv]
-whitelist_externals = docker
-deps = -rrequirements.txt
-commands = docker build -f _debian_11.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_automated_install.py

test/tox.ubuntu_21.ini

@@ -1,8 +0,0 @@
-[tox]
-envlist = py37
-
-[testenv]
-whitelist_externals = docker
-deps = -rrequirements.txt
-commands = docker build -f _ubuntu_21.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_automated_install.py