Merge pull request #1856 from pi-hole/release/v3.2.1

Pi-hole core v3.2.1

.stickler.yml

@@ -0,0 +1,3 @@
+linters:
+  shellcheck:
+    shell: bash

advanced/Scripts/piholeDebug.sh

@@ -809,8 +809,14 @@ process_status(){
   local i
   # For each process,
   for i in "${PIHOLE_PROCESSES[@]}"; do
-    # get its status via systemctl
-    local status_of_process=$(systemctl is-active "${i}")
+    # If systemd
+    if command -v systemctl &> /dev/null; then
+      # get its status via systemctl
+      local status_of_process=$(systemctl is-active "${i}")
+    else
+      # Otherwise, use the service command
+      local status_of_process=$(service "${i}" status | awk '/Active:/ {print $2}') &> /dev/null
+    fi
     # and print it out to the user
     if [[ "${status_of_process}" == "active" ]]; then
       # If it's active, show it in green

advanced/Scripts/updatecheck.sh

@@ -3,7 +3,7 @@
 # (c) 2017 Pi-hole, LLC (https://pi-hole.net)
 # Network-wide ad blocking via your own hardware.
 #
-# Checks for updates via GitHub
+# Checks for local or remote versions and branches
 #
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
@@ -25,35 +25,42 @@ function json_extract() {
   fi
 }
 
-GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")"
-GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")"
-GITHUB_FTL_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/FTL/releases/latest' 2> /dev/null)")"
-
-echo "${GITHUB_CORE_VERSION} ${GITHUB_WEB_VERSION} ${GITHUB_FTL_VERSION}" > "/etc/pihole/GitHubVersions"
-
 function get_local_branch() {
   # Return active branch
   cd "${1}" 2> /dev/null || return 1
   git rev-parse --abbrev-ref HEAD || return 1
 }
 
-CORE_BRANCH="$(get_local_branch /etc/.pihole)"
-WEB_BRANCH="$(get_local_branch /var/www/html/admin)"
-#FTL_BRANCH="$(pihole-FTL branch)"
-# Don't store FTL branch until the next release of FTL which
-# supports returning the branch in an easy way
-FTL_BRANCH="XXX"
-
-echo "${CORE_BRANCH} ${WEB_BRANCH} ${FTL_BRANCH}" > "/etc/pihole/localbranches"
-
 function get_local_version() {
-  # Return active branch
-  cd "${1}" 2> /dev/null || return 1
-  git describe --long --dirty --tags || return 1
+# Return active branch
+cd "${1}" 2> /dev/null || return 1
+git describe --long --dirty --tags || return 1
 }
 
-CORE_VERSION="$(get_local_version /etc/.pihole)"
-WEB_VERSION="$(get_local_version /var/www/html/admin)"
-FTL_VERSION="$(pihole-FTL version)"
+if [[ "$2" == "remote" ]]; then
 
-echo "${CORE_VERSION} ${WEB_VERSION} ${FTL_VERSION}" > "/etc/pihole/localversions"
+  if [[ "$3" == "reboot" ]]; then
+    sleep 30
+  fi
+
+  GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")"
+  GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")"
+  GITHUB_FTL_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/FTL/releases/latest' 2> /dev/null)")"
+
+  echo "${GITHUB_CORE_VERSION} ${GITHUB_WEB_VERSION} ${GITHUB_FTL_VERSION}" > "/etc/pihole/GitHubVersions"
+
+else
+
+  CORE_BRANCH="$(get_local_branch /etc/.pihole)"
+  WEB_BRANCH="$(get_local_branch /var/www/html/admin)"
+  FTL_BRANCH="$(pihole-FTL branch)"
+
+  echo "${CORE_BRANCH} ${WEB_BRANCH} ${FTL_BRANCH}" > "/etc/pihole/localbranches"
+
+  CORE_VERSION="$(get_local_version /etc/.pihole)"
+  WEB_VERSION="$(get_local_version /var/www/html/admin)"
+  FTL_VERSION="$(pihole-FTL version)"
+
+  echo "${CORE_VERSION} ${WEB_VERSION} ${FTL_VERSION}" > "/etc/pihole/localversions"
+
+fi

advanced/Scripts/webpage.sh

@@ -153,6 +153,7 @@ ProcessDNSSettings() {
 	if [[ "${DNSSEC}" == true ]]; then
 		echo "dnssec
 trust-anchor=.,19036,8,2,49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5
+trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D
 " >> "${dnsmasqconfig}"
 	fi
 

advanced/index.php

@@ -213,6 +213,8 @@ if (explode("-", $phVersion)[1] != "0")
 
 // Please Note: Text is added via CSS to allow an admin to provide a localised
 // language without the need to edit this file
+
+setHeader();
 ?>
 <!DOCTYPE html>
 <!-- Pi-hole: A black hole for Internet advertisements
@@ -224,7 +226,6 @@ if (explode("-", $phVersion)[1] != "0")
 <head>
   <meta charset="UTF-8">
   <?=$viewPort ?>
-  <?=setHeader() ?>
   <meta name="robots" content="noindex,nofollow"/>
   <meta http-equiv="x-dns-prefetch-control" content="off">
   <link rel="shortcut icon" href="<?=$proto ?>://pi.hole/admin/img/favicon.png" type="image/x-icon"/>

advanced/pihole-FTL.service

@@ -25,9 +25,9 @@ start() {
   if is_running; then
     echo "pihole-FTL is already running"
   else
-    touch /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port
+    touch /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log
     chown pihole:pihole /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /etc/pihole
-    chmod 0644 /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port
+    chmod 0644 /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log
     su -s /bin/sh -c "/usr/bin/pihole-FTL" "$FTLUSER"
     echo
   fi

advanced/pihole.cron

@@ -29,5 +29,9 @@
 
 @reboot root /usr/sbin/logrotate /etc/pihole/logrotate
 
-# Pi-hole: Grab remote version and branch every 10 minutes
-*/10 *  * * *   root    PATH="$PATH:/usr/local/bin/" pihole updatechecker
+# Pi-hole: Grab local version and branch every 10 minutes
+*/10 *  * * *   root    PATH="$PATH:/usr/local/bin/" pihole updatechecker local
+
+# Pi-hole: Grab remote version every 24 hours
+59 17  * * *   root    PATH="$PATH:/usr/local/bin/" pihole updatechecker remote
+@reboot root    PATH="$PATH:/usr/local/bin/" pihole updatechecker remote reboot

automated install/basic-install.sh

@@ -715,8 +715,8 @@ setStaticIPv4() {
       }> "${IFCFG_FILE}"
       # Use ip to immediately set the new address
       ip addr replace dev "${PIHOLE_INTERFACE}" "${IPV4_ADDRESS}"
-      # If NetworkMangler command line interface exists,
-      if command -v nmcli &> /dev/null;then
+      # If NetworkMangler command line interface exists and ready to mangle,
+      if command -v nmcli &> /dev/null && nmcli general status &> /dev/null; then
         # Tell NetworkManagler to read our new sysconfig file
         nmcli con load "${IFCFG_FILE}" > /dev/null
       fi
@@ -1243,7 +1243,7 @@ install_dependent_packages() {
         echo -e "${OVER}  ${TICK} Checking for $i"
       else
         #
-        echo -e "${OVER}  ${CROSS} Checking for $i (will be installed)"
+        echo -e "${OVER}  ${INFO} Checking for $i (will be installed)"
         #
         installArray+=("${i}")
       fi
@@ -1268,7 +1268,7 @@ install_dependent_packages() {
     if ${PKG_MANAGER} -q list installed "${i}" &> /dev/null; then
       echo -e "${OVER}  ${TICK} Checking for $i"
     else
-      echo -e "${OVER}  ${CROSS} Checking for $i (will be installed)"
+      echo -e "${OVER}  ${INFO} Checking for $i (will be installed)"
       #
       installArray+=("${i}")
     fi
@@ -1368,7 +1368,9 @@ installCron() {
   # Copy the cron file over from the local repo
   cp ${PI_HOLE_LOCAL_REPO}/advanced/pihole.cron /etc/cron.d/pihole
   # Randomize gravity update time
-  sed -i "s/59 1/$((1 + RANDOM % 58)) $((3 + RANDOM % 2))/" /etc/cron.d/pihole
+  sed -i "s/59 1 /$((1 + RANDOM % 58)) $((3 + RANDOM % 2))/" /etc/cron.d/pihole
+  # Randomize update checker time
+  sed -i "s/59 17/$((1 + RANDOM % 58)) $((12 + RANDOM % 8))/" /etc/cron.d/pihole
   echo -e "${OVER}  ${TICK} ${str}"
 }
 

gravity.sh

@@ -69,7 +69,7 @@ fi
 
 # Determine if DNS resolution is available before proceeding
 gravity_DNSLookup() {
-  local lookupDomain="pi.hole" plural=""
+  local lookupDomain="pi.hole"
 
   # Determine if $localList does not exist
   if [[ ! -e "${localList}" ]]; then
@@ -88,6 +88,19 @@ gravity_DNSLookup() {
     exit 1
   fi
 
+  # If the /etc/resolv.conf contains resolvers other than 127.0.0.1 then the local dnsmasq will not be queried and pi.hole is NXDOMAIN.
+  # This means that even though name resolution is working, the getent hosts check fails and the holddown timer keeps ticking and eventualy fails
+  # So we check the output of the last command and if it failed, attempt to use dig +short as a fallback
+  if timeout 1 dig +short "${lookupDomain}" &> /dev/null; then
+    if [[ -n "${secs:-}" ]]; then
+      echo -e "${OVER}  ${TICK} DNS resolution is now available\\n"
+    fi
+    return 0
+  elif [[ -n "${secs:-}" ]]; then
+    echo -e "${OVER}  ${CROSS} DNS resolution is not available"
+    exit 1
+  fi
+
   # Determine error output message
   if pidof dnsmasq &> /dev/null; then
     echo -e "  ${CROSS} DNS resolution is currently unavailable"
@@ -98,11 +111,10 @@ gravity_DNSLookup() {
 
   # Ensure DNS server is given time to be resolvable
   secs="120"
-  echo -ne "  ${INFO} Waiting up to ${secs} seconds before continuing..."
+  echo -ne "  ${INFO} Time until retry: ${secs}"
   until timeout 1 getent hosts "${lookupDomain}" &> /dev/null; do
     [[ "${secs:-}" -eq 0 ]] && break
-    [[ "${secs:-}" -ne 1 ]] && plural="s"
-    echo -ne "${OVER}  ${INFO} Waiting up to ${secs} second${plural} before continuing..."
+    echo -ne "${OVER}  ${INFO} Time until retry: ${secs}"
     : $((secs--))
     sleep 1
   done
@@ -139,7 +151,8 @@ gravity_Collapse() {
     awk -F '[/:]' '{
       # Remove URL protocol & optional username:password@
       gsub(/(.*:\/\/|.*:.*@)/, "", $0)
-      print $1
+      if(length($1)>0){print $1}
+      else {print "local"}
     }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null
   )"
 
@@ -192,9 +205,10 @@ gravity_Pull() {
   patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb")
 
   # Determine if $saveLocation has read permission
-  if [[ -r "${saveLocation}" ]]; then
+  if [[ -r "${saveLocation}" && $url != "file"* ]]; then
     # Have curl determine if a remote file has been modified since last retrieval
     # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls)
+    # Note: Don't do this for local files, always download them
     heisenbergCompensator="-z ${saveLocation}"
   fi
 
@@ -203,20 +217,32 @@ gravity_Pull() {
   # shellcheck disable=SC2086
   httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null)
 
-  # Determine "Status:" output based on HTTP response
-  case "${httpCode}" in
-    "200") echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success=true;;
-    "304") echo -e "${OVER}  ${TICK} ${str} No changes detected"; success=true;;
-    "000") echo -e "${OVER}  ${CROSS} ${str} Connection Refused";;
-    "403") echo -e "${OVER}  ${CROSS} ${str} Forbidden";;
-    "404") echo -e "${OVER}  ${CROSS} ${str} Not found";;
-    "408") echo -e "${OVER}  ${CROSS} ${str} Time-out";;
-    "451") echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons";;
-    "500") echo -e "${OVER}  ${CROSS} ${str} Internal Server Error";;
-    "504") echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Gateway)";;
-    "521") echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)";;
-    "522") echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)";;
-    *    ) echo -e "${OVER}  ${CROSS} ${str} ${httpCode}";;
+  case $url in
+    # Did we "download" a remote file?
+    "http"*)
+      # Determine "Status:" output based on HTTP response
+      case "${httpCode}" in
+        "200") echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success=true;;
+        "304") echo -e "${OVER}  ${TICK} ${str} No changes detected"; success=true;;
+        "000") echo -e "${OVER}  ${CROSS} ${str} Connection Refused";;
+        "403") echo -e "${OVER}  ${CROSS} ${str} Forbidden";;
+        "404") echo -e "${OVER}  ${CROSS} ${str} Not found";;
+        "408") echo -e "${OVER}  ${CROSS} ${str} Time-out";;
+        "451") echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons";;
+        "500") echo -e "${OVER}  ${CROSS} ${str} Internal Server Error";;
+        "504") echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Gateway)";;
+        "521") echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)";;
+        "522") echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)";;
+        *    ) echo -e "${OVER}  ${CROSS} ${str} ${httpCode}";;
+      esac;;
+    # Did we "download" a local file?
+    "file"*)
+        if [[ -s "${patternBuffer}" ]]; then
+          echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success=true
+        else
+          echo -e "${OVER}  ${CROSS} ${str} Not found / empty list"
+        fi;;
+    *) echo -e "${OVER}  ${CROSS} ${str} ${url} ${httpCode}";;
   esac
 
   # Determine if the blocklist was downloaded and saved correctly
@@ -243,36 +269,22 @@ gravity_Pull() {
 
 # Parse source files into domains format
 gravity_ParseFileIntoDomains() {
-  local source="${1}" destination="${2}" commentPattern firstLine abpFilter
+  local source="${1}" destination="${2}" firstLine abpFilter
 
   # Determine if we are parsing a consolidated list
   if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
-    # Define symbols used as comments: #;@![/
-    commentPattern="[#;@![\\/]"
+    # Remove comments and print only the domain name
+    # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious
+    # This helps with that and makes it easier to read
+    # It also helps with debugging so each stage of the script can be researched more in depth
+    #Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only.
+    #Last awk command takes non-commented lines and if they have 2 fields, take the left field (the domain) and leave
+    #+ the right (IP address), otherwise grab the single field.
 
-    # Parse Domains/Hosts files by removing comments & host IPs
-    # Logic: Ignore lines which begin with comments
-    awk '!/^'"${commentPattern}"'/ {
-      # Determine if there are multiple words seperated by a space
-      if(NF>1) {
-        # Remove comments (including prefixed spaces/tabs)
-        if($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) }
-        # Determine if there are aliased domains
-        if($3) {
-          # Remove IP address
-          $1=""
-          # Remove space which is left in $0 when removing $1
-          gsub("^ ", "", $0)
-          print $0
-        } else if($2) {
-          # Print single domain without IP
-          print $2
-        }
-      # If there are no words seperated by space
-      } else if($1) {
-        print $1
-      }
-    }' "${source}" 2> /dev/null > "${destination}"
+    < ${source} awk -F '#' '{print $1}' | \
+    awk -F '/' '{print $1}' | \
+    awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | \
+    sed -nr -e 's/\.{2,}/./g' -e '/\./p' >  ${destination}
     return 0
   fi
 
@@ -393,7 +405,7 @@ gravity_Filter() {
   # Format $parsedMatter line total as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
   echo -e "${OVER}  ${TICK} ${str}
-  ${INFO} ${COL_BLUE}${num}${COL_NC} domains being pulled in by gravity"
+  ${INFO} Number of domains being pulled in by gravity: ${COL_BLUE}${num}${COL_NC}"
 
   str="Removing duplicate domains"
   echo -ne "  ${INFO} ${str}..."
@@ -402,31 +414,30 @@ gravity_Filter() {
 
   # Format $preEventHorizon line total as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-  echo -e "  ${INFO} ${COL_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon"
+  echo -e "  ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}"
 }
 
 # Whitelist unique blocklist domain sources
 gravity_WhitelistBLD() {
-  local uniqDomains plural="" str 
+  local uniqDomains str
 
   echo ""
 
   # Create array of unique $sourceDomains
   mapfile -t uniqDomains <<< "$(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")"
-  [[ "${#uniqDomains[@]}" -ne 1 ]] && plural="s"
 
-  str="Adding ${#uniqDomains[@]} blocklist source domain${plural} to the whitelist"
+  str="Number of blocklist source domains being added to the whitelist: ${#uniqDomains[@]}"
   echo -ne "  ${INFO} ${str}..."
 
   # Whitelist $uniqDomains
-  "${PIHOLE_COMMAND}" -w -nr -q "${uniqDomains[*]}" &> /dev/null
+  "${PIHOLE_COMMAND}" -w -nr -q ${uniqDomains[*]} &> /dev/null
 
-  echo -e "${OVER}  ${TICK} ${str}"
+  echo -e "${OVER}  ${INFO} ${str}"
 }
 
 # Whitelist user-defined domains
 gravity_Whitelist() {
-  local num plural="" str
+  local num str
 
   if [[ ! -f "${whitelistFile}" ]]; then
     echo -e "  ${INFO} Nothing to whitelist!"
@@ -434,24 +445,22 @@ gravity_Whitelist() {
   fi
 
   num=$(wc -l < "${whitelistFile}")
-  [[ "${num}" -ne 1 ]] && plural="s"
-  str="Whitelisting ${num} domain${plural}"
+  str="Number of whitelisted domains: ${num}"
   echo -ne "  ${INFO} ${str}..."
 
   # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile
   grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${whitelistMatter}"
 
-  echo -e "${OVER}  ${TICK} ${str}"
+  echo -e "${OVER}  ${INFO} ${str}"
 }
 
 # Output count of blacklisted domains and wildcards
 gravity_ShowBlockCount() {
-  local num plural
+  local num
 
   if [[ -f "${blacklistFile}" ]]; then
     num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
-    plural=; [[ "${num}" -ne 1 ]] && plural="s"
-    echo -e "  ${INFO} Blacklisted ${num} domain${plural}"
+    echo -e "  ${INFO} Number of blacklisted domains: ${num}"
   fi
 
   if [[ -f "${wildcardFile}" ]]; then
@@ -460,8 +469,7 @@ gravity_ShowBlockCount() {
     if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then
       num=$(( num/2 ))
     fi
-    plural=; [[ "${num}" -ne 1 ]] && plural="s"
-    echo -e "  ${INFO} Wildcard blocked ${num} domain${plural}"
+    echo -e "  ${INFO} Number of wildcard blocked domains: ${num}"
   fi
 }
 

pihole

@@ -658,6 +658,6 @@ case "${1}" in
   "-t" | "tail"                 ) tailFunc;;
   "checkout"                    ) piholeCheckoutFunc "$@";;
   "tricorder"                   ) tricorderFunc;;
-  "updatechecker"               ) updateCheckFunc;;
+  "updatechecker"               ) updateCheckFunc "$@";;
   *                             ) helpFunc;;
 esac