From 6830b08723c17b1c1b9a43d341b12b0951682bf9 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Mon, 24 Jul 2017 21:24:34 +1000
Subject: [PATCH 01/18] Clean up and optimise Gravity

* Shellcheck validation
* Made variable names, function names, comments and output clearer to understand
* Quoted and braced variables and conditionals
* Fix adlists.list handling logic, and remove any CR line endings
* Make CP/MV/RM provide user-friendly output upon failure
* Change adlists.list retrieval logic
* Moved and fixed adlists.list domain parsing logic
* Create gravity_ParseFileAsDomains() function to handle parsing of source files
* If no changes to a list is detected, print no output
* Ensure each source blocklist has a final newline
* Format number output as currency
* Make array of adlists domain sources unique to prevent redundant whitelisting
* Merged bash IPv4/IPv6 hosts formatting IF statement into an awk one-liner
* Trap Ctrl-C cancellations and run gravity_Cleanup()
* Use new gravity_Cleanup() function on errors and script completion
* Ensure that dnsmasq uses force-reload when gravity is invoked
* Add --wildcard option to ensure dnsmasq is restarted upon b/wlisting of a wildcard

Signed-off-by: WaLLy3K <wally3k@pi-hole.net>
---
 gravity.sh | 674 ++++++++++++++++++++++++++++++-----------------------
 1 file changed, 388 insertions(+), 286 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 168a4350..852356d7 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -1,18 +1,66 @@
 #!/usr/bin/env bash
+# shellcheck disable=SC1090
+
 # Pi-hole: A black hole for Internet advertisements
 # (c) 2017 Pi-hole, LLC (https://pi-hole.net)
 # Network-wide ad blocking via your own hardware.
 #
+# Usage: "pihole -g"
 # Compiles a list of ad-serving domains by downloading them from multiple sources
 #
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-# Run this script as root or under sudo
-
 coltable="/opt/pihole/COL_TABLE"
 source ${coltable}
 
+basename="pihole"
+PIHOLE_COMMAND="/usr/local/bin/${basename}"
+WHITELIST_COMMAND="${PIHOLE_COMMAND} -w"
+
+piholeDir="/etc/${basename}"
+piholeRepo="/etc/.${basename}"
+
+adListFile="${piholeDir}/adlists.list"
+adListDefault="${piholeDir}/adlists.default"
+adListRepoDefault="${piholeRepo}/adlists.default"
+
+whitelistFile="${piholeDir}/whitelist.txt"
+blacklistFile="${piholeDir}/blacklist.txt"
+wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf"
+
+adList="${piholeDir}/gravity.list"
+blackList="${piholeDir}/black.list"
+localList="${piholeDir}/local.list"
+
+domainsExtension="domains"
+matterAndLight="${basename}.0.matterandlight.txt"
+supernova="${basename}.1.supernova.txt"
+preEventHorizon="list.preEventHorizon"
+eventHorizon="${basename}.2.supernova.txt"
+accretionDisc="${basename}.3.accretionDisc.txt"
+
+skipDownload="false"
+
+# Source setupVars from install script
+setupVars="${piholeDir}/setupVars.conf"
+if [[ -f "${setupVars}" ]];then
+  source "${setupVars}"
+
+  # Remove CIDR mask from IPv4/6 addresses
+  IPV4_ADDRESS="${IPV4_ADDRESS%/*}"
+  IPV6_ADDRESS="${IPV6_ADDRESS%/*}"
+else
+  echo -e "  ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC}
+  Please run 'pihole -r', and choose the 'reconfigure' option to fix."
+  exit 1
+fi
+
+# Warn users still using pihole.conf that it no longer has any effect
+if [[ -r "${piholeDir}/pihole.conf" ]]; then
+  echo -e "  ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}"
+fi
+
 helpFunc() {
   echo "Usage: pihole -g
 Update domains from blocklists specified in adlists.list
@@ -23,267 +71,307 @@ Options:
   exit 0
 }
 
-PIHOLE_COMMAND="/usr/local/bin/pihole"
-
-adListFile=/etc/pihole/adlists.list
-adListDefault=/etc/pihole/adlists.default # Deprecated
-adListRepoDefault=/etc/.pihole/adlists.default
-whitelistScript="${PIHOLE_COMMAND} -w"
-whitelistFile=/etc/pihole/whitelist.txt
-blacklistFile=/etc/pihole/blacklist.txt
-readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
-
-# Source the setupVars from install script for the IP
-setupVars=/etc/pihole/setupVars.conf
-if [[ -f "${setupVars}" ]];then
-  . /etc/pihole/setupVars.conf
-else
-  echo -e "  ${COL_LIGHT_RED}Error: /etc/pihole/setupVars.conf missing. Possible installation failure.${COL_NC}
-  Please run 'pihole -r', and choose the 'reconfigure' option to reconfigure."
-  exit 1
-fi
-
-# Remove the /* from the end of the IP addresses
-IPV4_ADDRESS=${IPV4_ADDRESS%/*}
-IPV6_ADDRESS=${IPV6_ADDRESS%/*}
-
-# Variables for various stages of downloading and formatting the list
-basename=pihole
-piholeDir=/etc/${basename}
-adList=${piholeDir}/gravity.list
-blackList=${piholeDir}/black.list
-localList=${piholeDir}/local.list
-justDomainsExtension=domains
-matterAndLight=${basename}.0.matterandlight.txt
-supernova=${basename}.1.supernova.txt
-preEventHorizon=list.preEventHorizon
-eventHorizon=${basename}.2.supernova.txt
-accretionDisc=${basename}.3.accretionDisc.txt
-
-skipDownload=false
-
-# Warn users still using pihole.conf that it no longer has any effect
-if [[ -r ${piholeDir}/pihole.conf ]]; then
-echo -e "  ${COL_LIGHT_RED}pihole.conf file no longer supported. Overrides in this file are ignored.${COL_NC}"
-fi
-
-###########################
-# Collapse - begin formation of pihole
-gravity_collapse() {
-
-  #New Logic:
-  # Does /etc/pihole/adlists.list exist? If so leave it alone
-  #                                      If not, cp /etc/.pihole/adlists.default /etc/pihole/adlists.list
-  # Read from adlists.list
-
-  # The following two blocks will sort out any missing adlists in the /etc/pihole directory, and remove legacy adlists.default
-  if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then
-    rm "${adListDefault}"
-  fi
-
-  if [ ! -f "${adListFile}" ]; then
-    cp "${adListRepoDefault}" "${adListFile}"
-  fi
-
+# Retrieve blocklist URLs from adlists.list
+gravity_Collapse() {
   echo -e "  ${INFO} Neutrino emissions detected..."
-  echo ""
-  local str="Pulling source lists into range"
+
+  # Handle "adlists.list" and "adlists.default" files
+  if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then
+    # Remove superceded $adListDefault file
+    rm "${adListDefault}" 2> /dev/null || \
+      echo -e "  ${CROSS} Unable to remove ${adListDefault}"
+  elif [[ ! -f "${adListFile}" ]]; then
+    # Create "adlists.list"
+    cp "${adListRepoDefault}" "${adListFile}" 2> /dev/null || \
+      echo -e "  ${CROSS} Unable to copy ${adListFile##*/} from ${piholeRepo}"
+  fi
+
+  local str="Pulling blocklist source list into range"
   echo -ne "  ${INFO} ${str}..."
 
-  sources=()
-  while IFS= read -r line || [[ -n "$line" ]]; do
-    # Do not read commented out or blank lines
-    if [[ ${line} = \#* ]] || [[ ! ${line} ]]; then
-      echo "" > /dev/null
-    else
-      sources+=(${line})
-    fi
-  done < ${adListFile}
+  # Retrieve source URLs from $adListFile
+  # Logic: Remove comments, CR line endings and empty lines
+  mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null)
 
-  echo -e "${OVER}  ${TICK} ${str}"
+  # Parse source domains from $sources
+  # Logic: Split by folder/port and remove URL protocol/password
+  mapfile -t sourceDomains < <(
+    awk -F '[/:]' '{
+      gsub(/(.*:\/\/|.*:.*@)/, "", $0)
+      print $1
+    }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null
+  )
+
+  if [[ -n "${sources[*]}" ]] || [[ -n "${sourceDomains[*]}" ]]; then
+    echo -e "${OVER}  ${TICK} ${str}"
+  else
+    echo -e "${OVER}  ${CROSS} ${str}"
+    gravity_Cleanup "error"
+  fi
 }
 
-# patternCheck - check to see if curl downloaded any new files.
-gravity_patternCheck() {
-  patternBuffer=$1
-  success=$2
-  error=$3
-  if [[ "${success}" = true ]]; then
-    # Check if download was successful but list has not been modified
-    if [[ "${error}" == "304" ]]; then
-      echo -e "  ${TICK} No changes detected, transport skipped!"
-    # Check if the patternbuffer is a non-zero length file
-    elif [[ -s "${patternBuffer}" ]]; then
-      # Some blocklists are copyright, they need to be downloaded and stored
-      # as is. They can be processed for content after they have been saved.
-      mv "${patternBuffer}" "${saveLocation}"
-      echo -e "  ${TICK} List updated, transport successful!"
-    else
-      # Empty file -> use previously downloaded list
-      echo -e "  ${INFO} Received empty file, ${COL_LIGHT_GREEN}using cached one${COL_NC} (list not updated!)"
-    fi
+# Parse source file into domains-only format
+gravity_ParseFileAsDomains() {
+  local source destination hostsFilter firstLine abpFilter
+  source="${1}"
+  destination="${2}"
+
+  # Determine how to parse source file
+  if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
+    # Symbols used as comments: "#;@![/"
+    commentPattern="[#;@![\\/]"
+
+    # Parse consolidated file by removing comments and hosts IP's
+    # Logic: Process lines which do not begin with comments
+    awk '!/^'"${commentPattern}"'/ {
+      # If there are multiple words seperated by space
+      if (NF>1) {
+        # Remove comments (Inc. prefixed spaces/tabs)
+        if ($0 ~ /'"${commentPattern}"'/) { gsub("( |	)'"${commentPattern}"'.*", "", $0) }
+        # Print consecutive domains
+        if ($3) {
+          $1=""
+          gsub("^ ", "", $0)
+          print $0
+        # Print single domain
+        } else if ($2) {
+          print $2
+        }
+      # Print single domain
+      } else if($1) {
+        print $1
+      }
+    }' "${source}" 2> /dev/null > "${destination}"
   else
-    # Check if cached list exists
-    if [[ -r "${saveLocation}" ]]; then
-      echo -e "  ${CROSS} List download failed, using cached list (list not updated!)"
+    # Individual file parsing
+    # Logic: comments are kept, and domains are extracted from each line
+    read -r firstLine < "${source}"
+
+    # Determine how to parse individual source file
+    if [[ "${firstLine,,}" =~ "adblock" ]] || [[ "${firstLine,,}" =~ "ublock" ]] || [[ "${firstLine,,}" =~ "! checksum" ]]; then
+      # Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet 
+      abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
+      awk ''"${abpFilter}"' {
+        # Remove valid adblock type options
+        gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
+        # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional)
+        gsub(/(\|\||\^\$?$)/, "", $0)
+        # Remove lines which are only IPv4 addresses or contain "^/*"
+        if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" }
+        # Print if not empty
+        if ($0) { print $0 }
+      }' "${source}" 2> /dev/null > "${destination}"
+      echo -e "  ${TICK} Format: Adblock"
+    elif grep -q -E "^(https?://|([0-9]{1,3}\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then
+      # Parse URLs
+      awk '{
+        # Remove URL protocol, optional "username:password@", and ":?/;"
+        if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) }
+        # Remove lines which are only IPv4 addresses
+        if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
+        if ($0) { print $0 }
+      }' "${source}" 2> /dev/null > "${destination}"
+      echo -e "  ${TICK} Format: URL"
     else
-      echo -e "  ${CROSS} Download failed and no cached list available (list will not be considered)"
+      # Keep hosts/domains file in same format as it was downloaded
+      output=$( { mv "${source}" "${destination}"; } 2>&1 )
+      status="$?"
+
+      if [[ "${status}" -ne 0 ]]; then
+        echo -e "  ${CROSS} Unable to move tmp file to ${piholeDir}
+      ${output}"
+        gravity_Cleanup "error"
+      fi
     fi
   fi
 }
 
-# transport - curl the specified url with any needed command extentions
-gravity_transport() {
-  url=$1
-  cmd_ext=$2
-  agent=$3
+# Determine output based on gravity_Transport() status
+gravity_AdvancedTransport() {
+  local patternBuffer success error output status
+  patternBuffer="${1}"
+  success="${2}"
+  error="${3}"
 
-  # tmp file, so we don't have to store the (long!) lists in RAM
+  if [[ "${success}" = true ]]; then
+    if [[ "${error}" == "304" ]]; then
+      : # Print no output
+    # Check if the patternbuffer is a non-zero length file
+    elif [[ -s "${patternBuffer}" ]]; then
+      # Parse Adblock/URL format blocklists and include comments
+      # HOSTS format is moved as-is
+      gravity_ParseFileAsDomains "${patternBuffer}" "${saveLocation}" "1"
+    else
+      # Fall back to previously cached list if current $patternBuffer is empty
+      echo -e "  ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
+    fi
+  else
+    # Determine if cached list exists
+    if [[ -r "${saveLocation}" ]]; then
+      echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
+    else
+      echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
+    fi
+  fi
+}
+
+# Curl the specified URL with any necessary command extentions
+gravity_Transport() {
+  local url cmd_ext agent
+  url="${1}"
+  cmd_ext="${2}"
+  agent="${3}"
+
+  # Store downloaded content to temp file instead of RAM
   patternBuffer=$(mktemp)
   heisenbergCompensator=""
-  if [[ -r ${saveLocation} ]]; then
+  if [[ -r "${saveLocation}" ]]; then
     # If domain has been saved, add file for date check to only download newer
     heisenbergCompensator="-z ${saveLocation}"
   fi
 
-  # Silently curl url
-  echo -e "${OVER}  ${TICK} ${str}"
   local str="Status:"
-  echo -ne "  ${INFO} ${str} Pending"
-  err=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w %{http_code} -A "${agent}" ${url} -o ${patternBuffer})
+  echo -ne "  ${INFO} ${str} Pending..."
+  # shellcheck disable=SC2086
+  httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null)
 
-  # Analyze http response
-  case "$err" in
-    "200"  ) echo -e "${OVER}  ${TICK} ${str} Success (OK)"; success=true;;
-    "304"  ) echo -e "${OVER}  ${TICK} ${str} Not modified"; success=true;;
-    "403"  ) echo -e "${OVER}  ${CROSS} ${str} Forbidden"; success=false;;
-    "404"  ) echo -e "${OVER}  ${CROSS} ${str} Not found"; success=false;;
-    "408"  ) echo -e "${OVER}  ${CROSS} ${str} Time-out"; success=false;;
-    "451"  ) echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons"; success=false;;
-    "521"  ) echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success=false;;
-    "522"  ) echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success=false;;
-    "500"  ) echo -e "${OVER}  ${CROSS} ${str} Internal Server Error"; success=false;;
-    *      ) echo -e "${OVER}  ${CROSS} ${str} Status $err"; success=false;;
+  # Determine "Status:" output based on HTTP response
+  case "$httpCode" in
+    "200" ) echo -e "${OVER}  ${TICK} ${str} Transport successful"; success=true;;
+    "304" ) echo -e "${OVER}  ${TICK} ${str} No changes detected"; success=true;;
+    "403" ) echo -e "${OVER}  ${CROSS} ${str} Forbidden"; success=false;;
+    "404" ) echo -e "${OVER}  ${CROSS} ${str} Not found"; success=false;;
+    "408" ) echo -e "${OVER}  ${CROSS} ${str} Time-out"; success=false;;
+    "451" ) echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons"; success=false;;
+    "521" ) echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success=false;;
+    "522" ) echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success=false;;
+    "500" ) echo -e "${OVER}  ${CROSS} ${str} Internal Server Error"; success=false;;
+    *     ) echo -e "${OVER}  ${CROSS} ${str} Status $httpCode"; success=false;;
   esac
 
-  # Process result
-  gravity_patternCheck "${patternBuffer}" "${success}" "${err}"
+  # Output additional info if success=false
+  gravity_AdvancedTransport "${patternBuffer}" "${success}" "${httpCode}"
 
-  # Delete temp file if it hasn't been moved
+  # Delete temp file if it has not been moved
   if [[ -f "${patternBuffer}" ]]; then
     rm "${patternBuffer}"
   fi
 }
 
-# spinup - main gravity function
-gravity_spinup() {
+# Define User Agent and options for each blocklist
+gravity_Pull() {
+  local agent url domain cmd_ext str
+
   echo ""
-  # Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and # blank lines
+
+  # Loop through $sources to download each one
   for ((i = 0; i < "${#sources[@]}"; i++)); do
-    url=${sources[$i]}
-    # Get just the domain from the URL
-    domain=$(cut -d'/' -f3 <<< "${url}")
+    url="${sources[$i]}"
+    domain="${sourceDomains[$i]}"
 
     # Save the file as list.#.domain
-    saveLocation=${piholeDir}/list.${i}.${domain}.${justDomainsExtension}
-    activeDomains[$i]=${saveLocation}
+    saveLocation="${piholeDir}/list.${i}.${domain}.${domainsExtension}"
+    activeDomains[$i]="${saveLocation}"
 
     agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36"
 
-    # Use a case statement to download lists that need special cURL commands
-    # to complete properly and reset the user agent when required
+    # Use a case statement to download lists that need special commands
     case "${domain}" in
-      "pgl.yoyo.org")
-        cmd_ext="-d mimetype=plaintext -d hostformat=hosts"
-      ;;
-
-      # Default is a simple request
-      *) cmd_ext=""
+      "pgl.yoyo.org") cmd_ext="-d mimetype=plaintext -d hostformat=hosts";;
+      *) cmd_ext="";;
     esac
 
     if [[ "${skipDownload}" == false ]]; then
-      local str="Aiming tractor beam at $domain"
-      echo -ne "  ${INFO} ${str}..."
+      str="Target: $domain (${url##*/})"
+      echo -e "  ${INFO} ${str}"
+
+      gravity_Transport "$url" "$cmd_ext" "$agent" "$str"
 
-      gravity_transport "$url" "$cmd_ext" "$agent" "$str"
       echo ""
     fi
   done
 }
 
-# Schwarzchild - aggregate domains to one list and add blacklisted domains
-gravity_Schwarzchild() {
-  echo ""
-  # Find all active domains and compile them into one file and remove CRs
-  local str="Aggregating list of domains"
+# Consolidate domains to one list and add blacklisted domains
+gravity_Schwarzschild() {
+  local str lastLine
+
+  str="Consolidating blocklists"
   echo -ne "  ${INFO} ${str}..."
-  
-  truncate -s 0 ${piholeDir}/${matterAndLight}
+
+  # Compile all blacklisted domains into one file and remove CRs
+  truncate -s 0 "${piholeDir}/${matterAndLight}"
   for i in "${activeDomains[@]}"; do
     # Only assimilate list if it is available (download might have failed permanently)
     if [[ -r "${i}" ]]; then
-      cat "${i}" | tr -d '\r' >> ${piholeDir}/${matterAndLight}
+      tr -d '\r' < "${i}" >> "${piholeDir}/${matterAndLight}"
+
+      # Ensure each source blocklist has a final newline
+      lastLine=$(tail -1 "${piholeDir}/${matterAndLight}")
+      [[ "${#lastLine}" -gt 0 ]] && echo "" >> "${piholeDir}/${matterAndLight}"
     fi
   done
 
   echo -e "${OVER}  ${TICK} ${str}"
 }
 
+# Append blacklist entries to eventHorizon if they exist
 gravity_Blacklist() {
-  # Append blacklist entries to eventHorizon if they exist
+  local numBlacklisted plural str
+
   if [[ -f "${blacklistFile}" ]]; then
-    numBlacklisted=$(wc -l < "${blacklistFile}")
-    plural=; [[ "$numBlacklisted" != "1" ]] && plural=s
-    local str="Exact blocked domain${plural}: $numBlacklisted"
+    numBlacklisted=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
+    plural=; [[ "${numBlacklisted}" != "1" ]] && plural=s
+    str="Exact blocked domain${plural}: $numBlacklisted"
     echo -e "  ${INFO} ${str}"
   else
     echo -e "  ${INFO} Nothing to blacklist!"
   fi
 }
 
+# Return number of wildcards in output
 gravity_Wildcard() {
-  # Return number of wildcards in output - don't actually handle wildcards
-  if [[ -f "${wildcardlist}" ]]; then
-    numWildcards=$(grep -c ^ "${wildcardlist}")
+  local numWildcards plural
+
+  if [[ -f "${wildcardFile}" ]]; then
+    numWildcards=$(grep -c ^ "${wildcardFile}")
     if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then
       let numWildcards/=2
     fi
-    plural=; [[ "$numWildcards" != "1" ]] && plural=s
+    plural=; [[ "${numWildcards}" != "1" ]] && plural=s
     echo -e "  ${INFO} Wildcard blocked domain${plural}: $numWildcards"
   else
     echo -e "  ${INFO} No wildcards used!"
   fi
-
 }
 
+# Prevent the domains of adlist sources from being blacklisted by other blocklists
 gravity_Whitelist() {
+  local plural str
+
   echo ""
-  # Prevent our sources from being pulled into the hole
-  plural=; [[ "${sources[@]}" != "1" ]] && plural=s
-  local str="Adding adlist source${plural} to the whitelist"
+  plural=; [[ "${#sources[*]}" != "1" ]] && plural=s
+  str="Adding blocklist source${plural} to the whitelist"
   echo -ne "  ${INFO} ${str}..."
 
-  urls=()
-  for url in "${sources[@]}"; do
-    tmp=$(awk -F '/' '{print $3}' <<< "${url}")
-    urls=("${urls[@]}" ${tmp})
-  done
+  # Create array of unique $sourceDomains
+  # shellcheck disable=SC2046
+  read -r -a uniqDomains <<< $(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")
+
+  ${WHITELIST_COMMAND} -nr -q "${uniqDomains[*]}" > /dev/null
 
   echo -e "${OVER}  ${TICK} ${str}"
 
-  # Ensure adlist domains are in whitelist.txt
-  ${whitelistScript} -nr -q "${urls[@]}" > /dev/null
-
-  # Check whitelist.txt exists.
+  # Test existence of whitelist.txt
   if [[ -f "${whitelistFile}" ]]; then
     # Remove anything in whitelist.txt from the Event Horizon
     numWhitelisted=$(wc -l < "${whitelistFile}")
-    plural=; [[ "$numWhitelisted" != "1" ]] && plural=s
+    plural=; [[ "${numWhitelisted}" != "1" ]] && plural=s
     local str="Whitelisting $numWhitelisted domain${plural}"
     echo -ne "  ${INFO} ${str}..."
 
     # Print everything from preEventHorizon into eventHorizon EXCEPT domains in whitelist.txt
-    grep -F -x -v -f ${whitelistFile} ${piholeDir}/${preEventHorizon} > ${piholeDir}/${eventHorizon}
+    grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${eventHorizon}"
 
     echo -e "${OVER}  ${TICK} ${str}"
   else
@@ -291,140 +379,155 @@ gravity_Whitelist() {
   fi
 }
 
-gravity_unique() {
-  # Sort and remove duplicates
-  local str="Removing duplicate domains"
+# Sort and remove duplicate blacklisted domains
+gravity_Unique() {
+  local str numberOf
+
+  str="Removing duplicate domains"
   echo -ne "  ${INFO} ${str}..."
-
-  sort -u  ${piholeDir}/${supernova} > ${piholeDir}/${preEventHorizon}
-
+  sort -u "${piholeDir}/${supernova}" > "${piholeDir}/${preEventHorizon}"
   echo -e "${OVER}  ${TICK} ${str}"
-  numberOf=$(wc -l < ${piholeDir}/${preEventHorizon})
-  echo -e "  ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the event horizon."
+
+  numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
+  echo -e "  ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
 }
 
-gravity_doHostFormat() {
-  # Check vars from setupVars.conf to see if we're using IPv4, IPv6, Or both.
-  if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then
-    # Both IPv4 and IPv6
-    awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" '{sub(/\r$/,""); print ipv4addr" "$0"\n"ipv6addr" "$0}' >> "${2}" < "${1}"
-  elif [[ -n "${IPV4_ADDRESS}" ]] && [[ -z "${IPV6_ADDRESS}" ]];then
-    # Only IPv4
-    awk -v ipv4addr="$IPV4_ADDRESS" '{sub(/\r$/,""); print ipv4addr" "$0}' >> "${2}" < "${1}"
-  elif [[ -z "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then
-    # Only IPv6
-    awk -v ipv6addr="$IPV6_ADDRESS" '{sub(/\r$/,""); print ipv6addr" "$0}' >> "${2}" < "${1}"
-  elif [[ -z "${IPV4_ADDRESS}" ]] &&[[  -z "${IPV6_ADDRESS}" ]];then
+# Parse list of domains into hosts format
+gravity_ParseDomainsIntoHosts() {
+  if [[ -n "${IPV4_ADDRESS}" ]] || [[ -n "${IPV6_ADDRESS}" ]]; then
+    awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" \
+      '{sub(/\r$/,""); if(ipv4addr) { print ipv4addr" "$0; }; if(ipv6addr) { print ipv6addr" "$0; }}' >> "${2}" < "${1}"
+  else
     echo -e "${OVER}  ${CROSS} ${str}"
-    echo -e "      ${COL_LIGHT_RED}No IP Values found! Please run 'pihole -r' and choose reconfigure to restore values${COL_NC}"
-    exit 1
+    echo -e "  ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}\\n"
+    gravity_Cleanup "error"
   fi
 }
 
-gravity_hostFormatLocal() {
-  # Format domain list as "192.168.x.x domain.com"
-
+# Create "localhost" entries
+gravity_ParseLocalDomains() {
   if [[ -f "/etc/hostname" ]]; then
-    hostname=$(< /etc/hostname)
-  elif [ -x "$(command -v hostname)" ]; then
+    hostname=$(< "/etc/hostname")
+  elif command -v hostname &> /dev/null; then
     hostname=$(hostname -f)
   else
     echo -e "  ${CROSS} Unable to determine fully qualified domain name of host"
   fi
 
-  echo -e "${hostname}\npi.hole" > "${localList}.tmp"
+  echo -e "${hostname}\\npi.hole" > "${localList}.tmp"
+
   # Copy the file over as /etc/pihole/local.list so dnsmasq can use it
-  rm "${localList}"
-  gravity_doHostFormat "${localList}.tmp" "${localList}"
-  rm "${localList}.tmp"
+  rm "${localList}" 2> /dev/null || \
+    echo -e "  ${CROSS} Unable to remove ${localList}"
+  gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
+  rm "${localList}.tmp" 2> /dev/null || \
+    echo -e "  ${CROSS} Unable to remove ${localList}.tmp"
 }
 
-gravity_hostFormatGravity() {
-  # Format domain list as "192.168.x.x domain.com"
-  echo "" > "${piholeDir}/${accretionDisc}"
-  gravity_doHostFormat "${piholeDir}/${eventHorizon}" "${piholeDir}/${accretionDisc}"
+# Create primary blacklist entries
+gravity_ParseBlacklistDomains() {
+  # Create $accretionDisc
+  [[ ! -f "${piholeDir}/${accretionDisc}" ]] && echo "" > "${piholeDir}/${accretionDisc}"
+
+  gravity_ParseDomainsIntoHosts "${piholeDir}/${eventHorizon}" "${piholeDir}/${accretionDisc}"
+
   # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it
-  mv "${piholeDir}/${accretionDisc}" "${adList}"
+  output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 )
+  status="$?"
 
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "  ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}
+  ${output}"
+    gravity_Cleanup "error"
+  fi
 }
 
-gravity_hostFormatBlack() {
+# Create user-added blacklist entries
+gravity_ParseUserDomains() {
   if [[ -f "${blacklistFile}" ]]; then
-    numBlacklisted=$(wc -l < "${blacklistFile}")
-    # Format domain list as "192.168.x.x domain.com"
-    gravity_doHostFormat "${blacklistFile}" "${blackList}.tmp"
+    numBlacklisted=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
+    gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp"
     # Copy the file over as /etc/pihole/black.list so dnsmasq can use it
-    mv "${blackList}.tmp" "${blackList}"
+    mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \
+      echo -e "  ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}"
   else
     echo -e "  ${INFO} Nothing to blacklist!"
   fi
 }
 
-# blackbody - remove any remnant files from script processes
-gravity_blackbody() {
-  # Loop through list files
-  for file in ${piholeDir}/*.${justDomainsExtension}; do
-    # If list is in active array then leave it (noop) else rm the list
-    if [[ " ${activeDomains[@]} " =~ ${file} ]]; then
-      :
-    else
-      rm -f "${file}"
-    fi
-  done
-}
-
-gravity_advanced() {
-  # Remove comments and print only the domain name
-  # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious
-  # This helps with that and makes it easier to read
-  # It also helps with debugging so each stage of the script can be researched more in depth
-  local str="Formatting list of domains to remove comments"
+# Parse consolidated blocklist into domains-only format
+gravity_Advanced() {
+  local str="Extracting domains from blocklists"
   echo -ne "  ${INFO} ${str}..."
 
-  #awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' ${piholeDir}/${matterAndLight} | sed -nr -e 's/\.{2,}/./g' -e '/\./p' >  ${piholeDir}/${supernova}
-  #Above line does not correctly grab domains where comment is on the same line (e.g 'addomain.com #comment')
-  #Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only.
-  #Last awk command takes non-commented lines and if they have 2 fields, take the left field (the domain) and leave
-  #+ the right (IP address), otherwise grab the single field.
-  cat ${piholeDir}/${matterAndLight} | \
-    awk -F '#' '{print $1}' | \
-    awk -F '/' '{print $1}' | \
-    awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | \
-    sed -nr -e 's/\.{2,}/./g' -e '/\./p' >  ${piholeDir}/${supernova}
+  # Parse files as Hosts
+  gravity_ParseFileAsDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${supernova}"
 
-  echo -e "${OVER}  ${TICK} ${str}"
+  numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${supernova}")")
+  echo -e "${OVER}  ${TICK} ${str}
+  ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} domains being pulled in by gravity"
 
-  numberOf=$(wc -l < ${piholeDir}/${supernova})
-  echo -e "  ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} domains being pulled in by gravity"
-
-  gravity_unique
+  gravity_Unique
 }
 
-gravity_reload() {
-  # Reload hosts file
-  echo ""
-  local str="Refreshing lists in dnsmasq"
-  echo -e "  ${INFO} ${str}..."
+# Trap Ctrl-C
+gravity_Trap() {
+  trap '{ echo -e "\\n\\n  ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
+}
 
-  # Ensure /etc/dnsmasq.d/01-pihole.conf is pointing at the correct list!
-  # First escape forward slashes in the path:
-  adList=${adList//\//\\\/}
-  # Now replace the line in dnsmasq file
-  #  sed -i "s/^addn-hosts.*/addn-hosts=$adList/" /etc/dnsmasq.d/01-pihole.conf
+# Clean up after Gravity
+gravity_Cleanup() {
+  local error="${1:-}"
 
-  "${PIHOLE_COMMAND}" restartdns
+  str="Cleaning up debris"
+  echo -ne "  ${INFO} ${str}..."
+
+  rm ${piholeDir}/pihole.*.txt 2> /dev/null
+  rm ${piholeDir}/*.tmp 2> /dev/null
+
+  # Remove any unused .domains files
+  for file in ${piholeDir}/*.${domainsExtension}; do
+    # If list is in active array then leave it (noop) else rm the list
+    if [[ "${activeDomains[*]}" =~ ${file} ]]; then
+      :
+    else
+      rm -f "${file}" 2> /dev/null || \
+        echo -e "  ${CROSS} Failed to remove ${file##*/}"
+    fi
+  done
+
+  echo -e "${OVER}  ${TICK} ${str}"
+  
+  [[ -n "$error" ]] && echo ""
+
+  # Only restart DNS service if offline
+  if ! pidof dnsmasq &> /dev/null; then
+    "${PIHOLE_COMMAND}" restartdns
+  fi
+
+  if [[ -n "$error" ]]; then
+    "${PIHOLE_COMMAND}" status
+    exit 1
+  fi
 }
 
 for var in "$@"; do
   case "${var}" in
-    "-f" | "--force"     ) forceGrav=true;;
-    "-h" | "--help"      ) helpFunc;;
-    "-sd" | "--skip-download"    ) skipDownload=true;;
-    "-b" | "--blacklist-only"    ) blackListOnly=true;;
+    "-f" | "--force" ) forceDelete=true;;
+    "-h" | "--help" ) helpFunc;;
+    "-sd" | "--skip-download" ) skipDownload=true;;
+    "-b" | "--blacklist-only" ) blackListOnly=true;;
+    "-w" | "--wildcard" ) dnsRestart="restart";;
   esac
 done
 
-if [[ "${forceGrav}" == true ]]; then
+# Main Gravity Execution
+gravity_Trap
+
+# Use "force-reload" when restarting dnsmasq for Blacklists and Whitelists
+[[ -z "${dnsRestart}" ]] && dnsRestart="force-reload"
+
+if [[ "${forceDelete}" == true ]]; then
   str="Deleting exising list cache"
   echo -ne "${INFO} ${str}..."
 
@@ -435,42 +538,41 @@ if [[ "${forceGrav}" == true ]]; then
   fi
 fi
 
+# If $blackListOnly is true, only run essential functions
 if [[ ! "${blackListOnly}" == true ]]; then
-  gravity_collapse
-  gravity_spinup
+  gravity_Collapse
+  gravity_Pull
+
   if [[ "${skipDownload}" == false ]]; then
-    gravity_Schwarzchild
-    gravity_advanced
+    gravity_Schwarzschild
+    gravity_Advanced
   else
     echo -e "  ${INFO} Using cached Event Horizon list..."
-    numberOf=$(wc -l < ${piholeDir}/${preEventHorizon})
-    echo -e "  ${INFO} ${COL_LIGHT_BLUE}$numberOf${COL_NC} unique domains trapped in the event horizon."
+    numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
+    echo -e "  ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
   fi
+
   gravity_Whitelist
 fi
+
 gravity_Blacklist
 gravity_Wildcard
 
-str="Formatting domains into a HOSTS file"
+str="Parsing domains into hosts format"
 echo -ne "  ${INFO} ${str}..."
+
 if [[ ! "${blackListOnly}" == true ]]; then
-  gravity_hostFormatLocal
-  gravity_hostFormatGravity
+  gravity_ParseLocalDomains
+  gravity_ParseBlacklistDomains
 fi
-gravity_hostFormatBlack
+
+gravity_ParseUserDomains
 echo -e "${OVER}  ${TICK} ${str}"
 
-gravity_blackbody
-
 if [[ ! "${blackListOnly}" == true ]]; then
-  # Clear no longer needed files...
-  str="Cleaning up un-needed files"
-  echo -ne "  ${INFO} ${str}..."
-
-  rm ${piholeDir}/pihole.*.txt 2> /dev/null
-  
-  echo -e "${OVER}  ${TICK} ${str}"
+  gravity_Cleanup
 fi
 
-gravity_reload
+echo ""
+"${PIHOLE_COMMAND}" restartdns "${dnsRestart}"
 "${PIHOLE_COMMAND}" status

From a2825be81961cee4ddbb37f9e984fe6a27faa99a Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Mon, 24 Jul 2017 21:25:04 +1000
Subject: [PATCH 02/18] Add dnsmasq "force-reload" option

* Made use of $PI_HOLE_SCRIPT_DIR
* Fix #1610 & #1624
* Add "$2" to restartDNS options

Signed-off-by: WaLLy3K <wally3k@pi-hole.net>
---
 pihole | 58 +++++++++++++++++++++++++++++-----------------------------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/pihole b/pihole
index 8da911d8..7ea2cc38 100755
--- a/pihole
+++ b/pihole
@@ -8,12 +8,12 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-colfile="/opt/pihole/COL_TABLE"
-source ${colfile}
-
 readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
 readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
 
+colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE"
+source "${colfile}"
+
 # Must be root to use this tool
 if [[ ! $EUID -eq 0 ]];then
   if [[ -x "$(command -v sudo)" ]]; then
@@ -26,7 +26,7 @@ if [[ ! $EUID -eq 0 ]];then
 fi
 
 webpageFunc() {
-  source /opt/pihole/webpage.sh
+  source "${PI_HOLE_SCRIPT_DIR}/webpage.sh"
   main "$@"
   exit 0
 }
@@ -340,33 +340,33 @@ versionFunc() {
 }
 
 restartDNS() {
-  dnsmasqPid=$(pidof dnsmasq)
-  local str="Restarting DNS service"
-  echo -ne "  ${INFO} ${str}"
-  if [[ "${dnsmasqPid}" ]]; then
-    # Service already running - reload config
-    if [[ -x "$(command -v systemctl)" ]]; then
-      output=$( { systemctl restart dnsmasq; } 2>&1 )
+  local svcOption str output status over
+  
+  # Force-reload is used for "/etc/pihole/*.list" files
+  # Restart is needed for "/etc/dnsmasq.d/*.conf" files
+  svcOption="${1:-}"
+
+  if [[ -z "${svcOption}" ]]; then
+    # Get PID of dnsmasq to determine if it needs to start or restart
+    if pidof dnsmasq &> /dev/null; then
+      svcOption="restart"
     else
-      output=$( { service dnsmasq restart; } 2>&1 )
-    fi
-    if [[ -z "${output}" ]]; then
-      echo -e "${OVER}  ${TICK} ${str}"
-    else
-      echo -e "${OVER}  ${CROSS} ${output}"
+      svcOption="start"
     fi
+  fi
+
+  # Only print output to Terminal, not Web Admin
+  str="${svcOption^}ing DNS service"
+  [[ -t 1 ]] && echo -ne "  ${INFO} ${str}..."
+
+  output=$( { service dnsmasq "${svcOption}"; } 2>&1 )
+  status="$?"
+
+  if [[ "$?" -eq 0 ]]; then
+    [[ -t 1 ]] && echo -e "${OVER}  ${TICK} ${str}"
   else
-    # Service not running, start it up
-    if [[ -x "$(command -v systemctl)" ]]; then
-      output=$( { systemctl start dnsmasq; } 2>&1 )
-    else
-      output=$( { service dnsmasq start; } 2>&1 )
-    fi
-    if [[ -z "${output}" ]]; then
-      echo -e "${OVER}  ${TICK} ${str}"
-    else
-      echo -e "${OVER}  ${CROSS} ${output}"
-    fi
+    [[ ! -t 1 ]] && OVER=""
+    echo -e "${OVER}  ${CROSS} ${output}"
   fi
 }
 
@@ -645,7 +645,7 @@ case "${1}" in
   "enable"                      ) piholeEnable 1;;
   "disable"                     ) piholeEnable 0 "$2";;
   "status"                      ) piholeStatus "$2";;
-  "restartdns"                  ) restartDNS;;
+  "restartdns"                  ) restartDNS "$2";;
   "-a" | "admin"                ) webpageFunc "$@";;
   "-t" | "tail"                 ) tailFunc;;
   "checkout"                    ) piholeCheckoutFunc "$@";;

From 406098e55a61c8cf72ba06e3de682cb938a401e1 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Mon, 24 Jul 2017 21:26:39 +1000
Subject: [PATCH 03/18] Speed up refresh time

* Add "--blacklist-only" to only run essential gravity functions
* Pass "--wildcard" option to `gravity.sh` to ensure dnsmasq is restarted

Signed-off-by: WaLLy3K <wally3k@pi-hole.net>
---
 advanced/Scripts/list.sh | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 86589083..8b6a6e5b 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -138,7 +138,7 @@ AddDomain() {
       if [[ "${verbose}" == true ]]; then
       echo -e "  ${INFO} Adding $1 to wildcard blacklist..."
       fi
-      reload=true
+      reload="restart"
       echo "address=/$1/${IPV4_ADDRESS}" >> "${wildcardlist}"
       if [[ "${#IPV6_ADDRESS}" > 0 ]]; then
         echo "address=/$1/${IPV6_ADDRESS}" >> "${wildcardlist}"
@@ -183,7 +183,7 @@ RemoveDomain() {
       echo -e "  ${INFO} Removing $1 from $listname..."
       # /I flag: search case-insensitive
       sed -i "/address=\/${domain}/Id" "${list}"
-      reload=true
+      reload="restart"
     else
       if [[ "${verbose}" == true ]]; then
         echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
@@ -192,12 +192,16 @@ RemoveDomain() {
   fi
 }
 
+# Update Gravity
 Reload() {
-  # Reload hosts file
   echo ""
-  echo -e "  ${INFO} Updating gravity..."
-  echo ""
-  pihole -g -sd
+
+  # Ensure that "restart" is used for Wildcard updates
+  if [[ "${1}" == "restart" ]]; then
+    local type="--wildcard"
+  fi
+
+  pihole -g --skip-download --blacklist-only "${type:-}"
 }
 
 Displaylist() {
@@ -243,6 +247,7 @@ fi
 
 PoplistFile
 
-if ${reload}; then
-  Reload
+if [[ "${reload}" != false ]]; then
+  # Ensure that "restart" is used for Wildcard updates
+  Reload "${reload}"
 fi

From 61ff0452e135445b6782e24a8c5783c3aad97eff Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Mon, 24 Jul 2017 21:27:12 +1000
Subject: [PATCH 04/18] Remove duplicate code

* Make RestartDNS() use `pihole restartdns`

Signed-off-by: WaLLy3K <wally3k@pi-hole.net>
---
 advanced/Scripts/webpage.sh | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index 42272122..1f90f9c3 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -221,20 +221,7 @@ Reboot() {
 }
 
 RestartDNS() {
-  local str="Restarting DNS service"
-  [[ -t 1 ]] && echo -ne "  ${INFO} ${str}"
-  if command -v systemctl &> /dev/null; then
-    output=$( { systemctl restart dnsmasq; } 2>&1 )
-  else
-    output=$( { service dnsmasq restart; } 2>&1 )
-  fi
-
- if [[ -z "${output}" ]]; then
-    [[ -t 1 ]] && echo -e "${OVER}  ${TICK} ${str}"
-  else
-    [[ ! -t 1 ]] && OVER=""
-    echo -e "${OVER}  ${CROSS} ${output}"
-  fi
+  /usr/local/bin/pihole restartdns
 }
 
 SetQueryLogOptions() {

From 2d8fff099fbaf2e99faab031ef9e928bcea14906 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Thu, 27 Jul 2017 12:34:26 +1000
Subject: [PATCH 05/18] Pass correct options to gravity.sh

* Optimise $validDomain function by using bashism and `grep`
* Add black/white/wildcard variables to pass to Reload()
* Revert reload variable behaviour
* Ensure Reload() function passes correct options to gravity.sh

Signed-off-by: WaLLy3K <wally3k@pi-hole.net>
---
 advanced/Scripts/list.sh | 31 +++++++++++++++----------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 8b6a6e5b..a3f3261a 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -66,15 +66,15 @@ HandleOther() {
   domain="${1,,}"
 
   # Check validity of domain
-  validDomain=$(perl -lne 'print if /^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$/' <<< "${domain}") # Valid chars check
-  validDomain=$(perl -lne 'print if /^.{1,253}$/' <<< "${validDomain}") # Overall length check
-  validDomain=$(perl -lne 'print if /^[^\.]{1,63}(\.[^\.]{1,63})*$/' <<< "${validDomain}") # Length of each label
+  if [[ "${#domain}" -le 253 ]]; then
+    validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check
+    validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label
+  fi
   
-  if [[ -z "${validDomain}" ]]; then
-    echo -e "  ${CROSS} $1 is not a valid argument or domain name!"
-  else
-    echo -e "  ${TICK} $1 is a valid domain name!"
+  if [[ -n "${validDomain}" ]]; then
     domList=("${domList[@]}" ${validDomain})
+  else
+    echo -e "  ${CROSS} ${domain} is not a valid argument or domain name!"
   fi
 }
 
@@ -107,6 +107,8 @@ AddDomain() {
   [[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist"
 
   if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
+    [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
+    [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
     bool=true
     # Is the domain in the list we want to add it to?
     grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
@@ -129,7 +131,7 @@ AddDomain() {
     # Remove the /* from the end of the IP addresses
     IPV4_ADDRESS=${IPV4_ADDRESS%/*}
     IPV6_ADDRESS=${IPV6_ADDRESS%/*}
-
+    [[ -z "${type}" ]] && type="--wildcard-only"
     bool=true
     # Is the domain in the list?
     grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false
@@ -161,6 +163,8 @@ RemoveDomain() {
 
   if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
     bool=true
+    [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
+    [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
     # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa
     grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
     if [[ "${bool}" == true ]]; then
@@ -175,6 +179,7 @@ RemoveDomain() {
       fi
     fi
   elif [[ "${list}" == "${wildcardlist}" ]]; then
+    [[ -z "${type}" ]] && type="--wildcard-only"
     bool=true
     # Is it in the list?
     grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false
@@ -183,7 +188,7 @@ RemoveDomain() {
       echo -e "  ${INFO} Removing $1 from $listname..."
       # /I flag: search case-insensitive
       sed -i "/address=\/${domain}/Id" "${list}"
-      reload="restart"
+      reload=true
     else
       if [[ "${verbose}" == true ]]; then
         echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
@@ -195,13 +200,7 @@ RemoveDomain() {
 # Update Gravity
 Reload() {
   echo ""
-
-  # Ensure that "restart" is used for Wildcard updates
-  if [[ "${1}" == "restart" ]]; then
-    local type="--wildcard"
-  fi
-
-  pihole -g --skip-download --blacklist-only "${type:-}"
+  pihole -g --skip-download "${type:-}"
 }
 
 Displaylist() {

From f24ab8508eb1fa2b87e25f0f1db29ed869562bce Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Thu, 27 Jul 2017 12:34:35 +1000
Subject: [PATCH 06/18] WIP cleanup

* Changed supernova/eventHorizon variables to match their purpose
* Add gravity_DNSLookup() function
* Ensure all comments are clear and relevant
* Use && instead of || in gravity_Collapse()
* Renamed existing functions, and placed them in order of script execution
* Use \t instead of literal tab in gravity_ParseFileIntoDomains()
* Replace instances of "truncate" with : > (e.g: gravity_Schwarzschild())
* Ensure correct variables are local'd
* Use phrase "Cleaning up stray matter" when gravity_Cleanup() is called
* Add black/white/wildcard switches for list.sh
* Ensure necessary functions are called when modifying black/white/wildcards

Signed-off-by: WaLLy3K <wally3k@pi-hole.net>
---
 gravity.sh | 582 ++++++++++++++++++++++++++++-------------------------
 1 file changed, 310 insertions(+), 272 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 852356d7..b222886b 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -12,7 +12,7 @@
 # Please see LICENSE file for your rights under this license.
 
 coltable="/opt/pihole/COL_TABLE"
-source ${coltable}
+source "${coltable}"
 
 basename="pihole"
 PIHOLE_COMMAND="/usr/local/bin/${basename}"
@@ -35,13 +35,16 @@ localList="${piholeDir}/local.list"
 
 domainsExtension="domains"
 matterAndLight="${basename}.0.matterandlight.txt"
-supernova="${basename}.1.supernova.txt"
-preEventHorizon="list.preEventHorizon"
-eventHorizon="${basename}.2.supernova.txt"
+parsedMatter="${basename}.1.parsedmatter.txt"
+whitelistMatter="${basename}.2.whitelistmatter.txt"
 accretionDisc="${basename}.3.accretionDisc.txt"
+preEventHorizon="list.preEventHorizon"
 
 skipDownload="false"
 
+# Use "force-reload" when restarting dnsmasq for everything but Wildcards
+dnsRestart="force-reload"
+
 # Source setupVars from install script
 setupVars="${piholeDir}/setupVars.conf"
 if [[ -f "${setupVars}" ]];then
@@ -61,17 +64,39 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then
   echo -e "  ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}"
 fi
 
-helpFunc() {
-  echo "Usage: pihole -g
-Update domains from blocklists specified in adlists.list
+# Attempt to resolve DNS before proceeding with blocklist generation
+gravity_DNSLookup() {
+  local plu
 
-Options:
-  -f, --force          Force the download of all specified blocklists
-  -h, --help           Show this help dialog"
-  exit 0
+  # Determine if github.com can be resolved
+  if ! timeout 2 nslookup github.com &> /dev/null; then
+    if [[ -n "${secs}" ]]; then
+      echo -e "${OVER}  ${CROSS} DNS resolution is still unavailable, cancelling"
+      exit 1
+    fi
+
+    # Determine error output message
+    if pidof dnsmasq &> /dev/null; then
+      echo -e "  ${CROSS} DNS resolution is temporarily unavailable"
+    else
+      echo -e "  ${CROSS} DNS service is not running"
+      "${PIHOLE_COMMAND}" restartdns
+    fi
+
+    secs="15"
+    while [[ "${secs}" -ne 0 ]]; do
+      [[ "${secs}" -ne 1 ]] && plu="s" || plu=""
+      echo -ne "${OVER}  ${INFO} Waiting $secs second${plu} before continuing..."
+      sleep 1
+      : $((secs--))
+    done
+
+    # Try again
+    gravity_DNSLookup
+  fi
 }
 
-# Retrieve blocklist URLs from adlists.list
+# Retrieve blocklist URLs and parse domains from adlists.list
 gravity_Collapse() {
   echo -e "  ${INFO} Neutrino emissions detected..."
 
@@ -94,7 +119,7 @@ gravity_Collapse() {
   mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null)
 
   # Parse source domains from $sources
-  # Logic: Split by folder/port and remove URL protocol/password
+  # Logic: Split by folder/port, remove URL protocol & optional username:password@
   mapfile -t sourceDomains < <(
     awk -F '[/:]' '{
       gsub(/(.*:\/\/|.*:.*@)/, "", $0)
@@ -102,7 +127,7 @@ gravity_Collapse() {
     }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null
   )
 
-  if [[ -n "${sources[*]}" ]] || [[ -n "${sourceDomains[*]}" ]]; then
+  if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then
     echo -e "${OVER}  ${TICK} ${str}"
   else
     echo -e "${OVER}  ${CROSS} ${str}"
@@ -110,157 +135,9 @@ gravity_Collapse() {
   fi
 }
 
-# Parse source file into domains-only format
-gravity_ParseFileAsDomains() {
-  local source destination hostsFilter firstLine abpFilter
-  source="${1}"
-  destination="${2}"
-
-  # Determine how to parse source file
-  if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
-    # Symbols used as comments: "#;@![/"
-    commentPattern="[#;@![\\/]"
-
-    # Parse consolidated file by removing comments and hosts IP's
-    # Logic: Process lines which do not begin with comments
-    awk '!/^'"${commentPattern}"'/ {
-      # If there are multiple words seperated by space
-      if (NF>1) {
-        # Remove comments (Inc. prefixed spaces/tabs)
-        if ($0 ~ /'"${commentPattern}"'/) { gsub("( |	)'"${commentPattern}"'.*", "", $0) }
-        # Print consecutive domains
-        if ($3) {
-          $1=""
-          gsub("^ ", "", $0)
-          print $0
-        # Print single domain
-        } else if ($2) {
-          print $2
-        }
-      # Print single domain
-      } else if($1) {
-        print $1
-      }
-    }' "${source}" 2> /dev/null > "${destination}"
-  else
-    # Individual file parsing
-    # Logic: comments are kept, and domains are extracted from each line
-    read -r firstLine < "${source}"
-
-    # Determine how to parse individual source file
-    if [[ "${firstLine,,}" =~ "adblock" ]] || [[ "${firstLine,,}" =~ "ublock" ]] || [[ "${firstLine,,}" =~ "! checksum" ]]; then
-      # Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet 
-      abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
-      awk ''"${abpFilter}"' {
-        # Remove valid adblock type options
-        gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
-        # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional)
-        gsub(/(\|\||\^\$?$)/, "", $0)
-        # Remove lines which are only IPv4 addresses or contain "^/*"
-        if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" }
-        # Print if not empty
-        if ($0) { print $0 }
-      }' "${source}" 2> /dev/null > "${destination}"
-      echo -e "  ${TICK} Format: Adblock"
-    elif grep -q -E "^(https?://|([0-9]{1,3}\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then
-      # Parse URLs
-      awk '{
-        # Remove URL protocol, optional "username:password@", and ":?/;"
-        if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) }
-        # Remove lines which are only IPv4 addresses
-        if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
-        if ($0) { print $0 }
-      }' "${source}" 2> /dev/null > "${destination}"
-      echo -e "  ${TICK} Format: URL"
-    else
-      # Keep hosts/domains file in same format as it was downloaded
-      output=$( { mv "${source}" "${destination}"; } 2>&1 )
-      status="$?"
-
-      if [[ "${status}" -ne 0 ]]; then
-        echo -e "  ${CROSS} Unable to move tmp file to ${piholeDir}
-      ${output}"
-        gravity_Cleanup "error"
-      fi
-    fi
-  fi
-}
-
-# Determine output based on gravity_Transport() status
-gravity_AdvancedTransport() {
-  local patternBuffer success error output status
-  patternBuffer="${1}"
-  success="${2}"
-  error="${3}"
-
-  if [[ "${success}" = true ]]; then
-    if [[ "${error}" == "304" ]]; then
-      : # Print no output
-    # Check if the patternbuffer is a non-zero length file
-    elif [[ -s "${patternBuffer}" ]]; then
-      # Parse Adblock/URL format blocklists and include comments
-      # HOSTS format is moved as-is
-      gravity_ParseFileAsDomains "${patternBuffer}" "${saveLocation}" "1"
-    else
-      # Fall back to previously cached list if current $patternBuffer is empty
-      echo -e "  ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
-    fi
-  else
-    # Determine if cached list exists
-    if [[ -r "${saveLocation}" ]]; then
-      echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
-    else
-      echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
-    fi
-  fi
-}
-
-# Curl the specified URL with any necessary command extentions
-gravity_Transport() {
-  local url cmd_ext agent
-  url="${1}"
-  cmd_ext="${2}"
-  agent="${3}"
-
-  # Store downloaded content to temp file instead of RAM
-  patternBuffer=$(mktemp)
-  heisenbergCompensator=""
-  if [[ -r "${saveLocation}" ]]; then
-    # If domain has been saved, add file for date check to only download newer
-    heisenbergCompensator="-z ${saveLocation}"
-  fi
-
-  local str="Status:"
-  echo -ne "  ${INFO} ${str} Pending..."
-  # shellcheck disable=SC2086
-  httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null)
-
-  # Determine "Status:" output based on HTTP response
-  case "$httpCode" in
-    "200" ) echo -e "${OVER}  ${TICK} ${str} Transport successful"; success=true;;
-    "304" ) echo -e "${OVER}  ${TICK} ${str} No changes detected"; success=true;;
-    "403" ) echo -e "${OVER}  ${CROSS} ${str} Forbidden"; success=false;;
-    "404" ) echo -e "${OVER}  ${CROSS} ${str} Not found"; success=false;;
-    "408" ) echo -e "${OVER}  ${CROSS} ${str} Time-out"; success=false;;
-    "451" ) echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons"; success=false;;
-    "521" ) echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success=false;;
-    "522" ) echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success=false;;
-    "500" ) echo -e "${OVER}  ${CROSS} ${str} Internal Server Error"; success=false;;
-    *     ) echo -e "${OVER}  ${CROSS} ${str} Status $httpCode"; success=false;;
-  esac
-
-  # Output additional info if success=false
-  gravity_AdvancedTransport "${patternBuffer}" "${success}" "${httpCode}"
-
-  # Delete temp file if it has not been moved
-  if [[ -f "${patternBuffer}" ]]; then
-    rm "${patternBuffer}"
-  fi
-}
-
-# Define User Agent and options for each blocklist
-gravity_Pull() {
-  local agent url domain cmd_ext str
+# Define options for when retrieving blocklists
+gravity_Supernova() {
+  local url domain agent cmd_ext str
 
   echo ""
 
@@ -282,28 +159,167 @@ gravity_Pull() {
     esac
 
     if [[ "${skipDownload}" == false ]]; then
-      str="Target: $domain (${url##*/})"
+      str="Target: ${domain} (${url##*/})"
       echo -e "  ${INFO} ${str}"
 
-      gravity_Transport "$url" "$cmd_ext" "$agent" "$str"
+      gravity_Pull "${url}" "${cmd_ext}" "${agent}" "${str}"
 
       echo ""
     fi
   done
 }
 
-# Consolidate domains to one list and add blacklisted domains
+# Download specified URL and perform QA
+gravity_Pull() {
+  local url cmd_ext agent heisenbergCompensator patternBuffer str httpCode success
+
+  url="${1}"
+  cmd_ext="${2}"
+  agent="${3}"
+
+  # Store downloaded content to temp file instead of RAM
+  patternBuffer=$(mktemp)
+  heisenbergCompensator=""
+  if [[ -r "${saveLocation}" ]]; then
+    # If domain has been saved, add file for date check to only download newer
+    heisenbergCompensator="-z ${saveLocation}"
+  fi
+
+  str="Status:"
+  echo -ne "  ${INFO} ${str} Pending..."
+  # shellcheck disable=SC2086
+  httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null)
+
+  # Determine "Status:" output based on HTTP response
+  case "$httpCode" in
+    "200" ) echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success="true";;
+    "304" ) echo -e "${OVER}  ${TICK} ${str} No changes detected"; success="true";;
+    "403" ) echo -e "${OVER}  ${CROSS} ${str} Forbidden"; success="false";;
+    "404" ) echo -e "${OVER}  ${CROSS} ${str} Not found"; success="false";;
+    "408" ) echo -e "${OVER}  ${CROSS} ${str} Time-out"; success="false";;
+    "451" ) echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons"; success="false";;
+    "521" ) echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";;
+    "522" ) echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";;
+    "500" ) echo -e "${OVER}  ${CROSS} ${str} Internal Server Error"; success="false";;
+    *     ) echo -e "${OVER}  ${CROSS} ${str} Status $httpCode"; success="false";;
+  esac
+
+  # Determine if the blocklist was downloaded and saved correctly
+  if [[ "${success}" == "true" ]]; then
+    if [[ "${httpCode}" == "304" ]]; then
+      : # Do nothing
+    # Check if patternbuffer is a non-zero length file
+    elif [[ -s "${patternBuffer}" ]]; then
+      # Determine if blocklist is non-standard and parse as appropriate
+      gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}"
+    else
+      # Fall back to previously cached list if patternBuffer is empty
+      echo -e "  ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
+    fi
+  else
+    # Determine if cached list exists
+    if [[ -r "${saveLocation}" ]]; then
+      echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
+    else
+      echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
+    fi
+  fi
+
+  # Delete temp file if it has not been moved
+  if [[ -f "${patternBuffer}" ]]; then
+    rm "${patternBuffer}" 2> /dev/null || \
+      echo -e "  ${CROSS} Unable to remove ${patternBuffer}"
+  fi
+}
+
+# Parse non-standard source files into domains-only format
+gravity_ParseFileIntoDomains() {
+  local source destination commentPattern firstLine abpFilter
+  source="${1}"
+  destination="${2}"
+
+  # Determine how to parse source file
+  if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
+    # Consolidated list parsing: Remove comments and hosts IP's
+
+    # Symbols used as comments: #;@![/
+    commentPattern="[#;@![\\/]"
+
+    # Logic: Process lines which do not begin with comments
+    awk '!/^'"${commentPattern}"'/ {
+      # If there are multiple words seperated by space
+      if (NF>1) {
+        # Remove comments (Inc. prefixed spaces/tabs)
+        if ($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) }
+        # Print consecutive domains
+        if ($3) {
+          $1=""
+          # Remove space which is left in $0 when removing $1
+          gsub("^ ", "", $0)
+          print $0
+        # Print single domain
+        } else if ($2) {
+          print $2
+        }
+      # If there are no words seperated by space
+      } else if($1) {
+        print $1
+      }
+    }' "${source}" 2> /dev/null > "${destination}"
+  else
+    # Individual file parsing: Keep comments, while parsing domains from each line
+    read -r firstLine < "${source}"
+
+    # Determine how to parse individual source file formats
+    if [[ "${firstLine,,}" =~ "adblock" ]] || [[ "${firstLine,,}" =~ "ublock" ]] || [[ "${firstLine,,}" =~ "! checksum" ]]; then
+      # Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet 
+      abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
+      awk ''"${abpFilter}"' {
+        # Remove valid adblock type options
+        gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
+        # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional)
+        gsub(/(\|\||\^\$?$)/, "", $0)
+        # Remove lines which are only IPv4 addresses or contain "^/*"
+        if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" }
+        if ($0) { print $0 }
+      }' "${source}" 2> /dev/null > "${destination}"
+    elif grep -q -E "^(https?://|([0-9]{1,3}\\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then
+      # Parse URLs if source file contains http:// or IPv4
+      awk '{
+        # Remove URL protocol, optional "username:password@", and ":?/;"
+        if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) }
+        # Remove lines which are only IPv4 addresses
+        if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
+        if ($0) { print $0 }
+      }' "${source}" 2> /dev/null > "${destination}"
+    else
+      # Keep hosts/domains file in same format as it was downloaded
+      output=$( { mv "${source}" "${destination}"; } 2>&1 )
+      status="$?"
+
+      if [[ "${status}" -ne 0 ]]; then
+        echo -e "  ${CROSS} Unable to move tmp file to ${piholeDir}
+      ${output}"
+        gravity_Cleanup "error"
+      fi
+    fi
+  fi
+}
+
+# Create unfiltered "Matter and Light" consolidated list
 gravity_Schwarzschild() {
   local str lastLine
 
   str="Consolidating blocklists"
   echo -ne "  ${INFO} ${str}..."
 
-  # Compile all blacklisted domains into one file and remove CRs
-  truncate -s 0 "${piholeDir}/${matterAndLight}"
+  # Empty $matterAndLight if it already exists, otherwise, create it
+  : > "${piholeDir}/${matterAndLight}"
+
   for i in "${activeDomains[@]}"; do
     # Only assimilate list if it is available (download might have failed permanently)
     if [[ -r "${i}" ]]; then
+      # Compile all blacklisted domains into one file and remove CRs
       tr -d '\r' < "${i}" >> "${piholeDir}/${matterAndLight}"
 
       # Ensure each source blocklist has a final newline
@@ -315,39 +331,39 @@ gravity_Schwarzschild() {
   echo -e "${OVER}  ${TICK} ${str}"
 }
 
-# Append blacklist entries to eventHorizon if they exist
-gravity_Blacklist() {
-  local numBlacklisted plural str
+# Parse unfiltered consolidated blocklist into filtered domains-only format
+gravity_Filter() {
+  local str num
 
-  if [[ -f "${blacklistFile}" ]]; then
-    numBlacklisted=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
-    plural=; [[ "${numBlacklisted}" != "1" ]] && plural=s
-    str="Exact blocked domain${plural}: $numBlacklisted"
-    echo -e "  ${INFO} ${str}"
-  else
-    echo -e "  ${INFO} Nothing to blacklist!"
-  fi
+  str="Extracting domains from blocklists"
+  echo -ne "  ${INFO} ${str}..."
+
+  # Parse files as hosts
+  gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}"
+
+  num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
+  echo -e "${OVER}  ${TICK} ${str}
+  ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} domains being pulled in by gravity"
+
+  gravity_Unique
 }
 
-# Return number of wildcards in output
-gravity_Wildcard() {
-  local numWildcards plural
+# Sort and remove duplicate blacklisted domains
+gravity_Unique() {
+  local str num
 
-  if [[ -f "${wildcardFile}" ]]; then
-    numWildcards=$(grep -c ^ "${wildcardFile}")
-    if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then
-      let numWildcards/=2
-    fi
-    plural=; [[ "${numWildcards}" != "1" ]] && plural=s
-    echo -e "  ${INFO} Wildcard blocked domain${plural}: $numWildcards"
-  else
-    echo -e "  ${INFO} No wildcards used!"
-  fi
+  str="Removing duplicate domains"
+  echo -ne "  ${INFO} ${str}..."
+  sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
+  echo -e "${OVER}  ${TICK} ${str}"
+
+  num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
+  echo -e "  ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon"
 }
 
-# Prevent the domains of adlist sources from being blacklisted by other blocklists
-gravity_Whitelist() {
-  local plural str
+# Whitelist blocklist domain sources
+gravity_WhitelistBLD() {
+  local plural str uniqDomains
 
   echo ""
   plural=; [[ "${#sources[*]}" != "1" ]] && plural=s
@@ -361,17 +377,22 @@ gravity_Whitelist() {
   ${WHITELIST_COMMAND} -nr -q "${uniqDomains[*]}" > /dev/null
 
   echo -e "${OVER}  ${TICK} ${str}"
+}
+
+# Whitelist user-defined domains
+gravity_Whitelist() {
+  local plural str num
 
   # Test existence of whitelist.txt
   if [[ -f "${whitelistFile}" ]]; then
     # Remove anything in whitelist.txt from the Event Horizon
-    numWhitelisted=$(wc -l < "${whitelistFile}")
-    plural=; [[ "${numWhitelisted}" != "1" ]] && plural=s
-    local str="Whitelisting $numWhitelisted domain${plural}"
+    num=$(wc -l < "${whitelistFile}")
+    plural=; [[ "${num}" != "1" ]] && plural=s
+    local str="Whitelisting ${num} domain${plural}"
     echo -ne "  ${INFO} ${str}..."
 
-    # Print everything from preEventHorizon into eventHorizon EXCEPT domains in whitelist.txt
-    grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${eventHorizon}"
+    # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in whitelist.txt
+    grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${whitelistMatter}"
 
     echo -e "${OVER}  ${TICK} ${str}"
   else
@@ -379,24 +400,40 @@ gravity_Whitelist() {
   fi
 }
 
-# Sort and remove duplicate blacklisted domains
-gravity_Unique() {
-  local str numberOf
+# Output count of blacklisted domains and wildcards
+gravity_ShowBlockCount() {
+  local num plural str
 
-  str="Removing duplicate domains"
-  echo -ne "  ${INFO} ${str}..."
-  sort -u "${piholeDir}/${supernova}" > "${piholeDir}/${preEventHorizon}"
-  echo -e "${OVER}  ${TICK} ${str}"
+  if [[ -f "${blacklistFile}" ]]; then
+    num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
+    plural=; [[ "${num}" != "1" ]] && plural=s
+    str="Exact blocked domain${plural}: ${num}"
+    echo -e "  ${INFO} ${str}"
+  else
+    echo -e "  ${INFO} Nothing to blacklist!"
+  fi
 
-  numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-  echo -e "  ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
+  if [[ -f "${wildcardFile}" ]]; then
+    num=$(grep -c "^" "${wildcardFile}")
+    # If IPv4 and IPv6 is used, divide total wildcard count by 2
+    if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then
+      num=$(( num/2 ))
+    fi
+    plural=; [[ "${num}" != "1" ]] && plural=s
+    echo -e "  ${INFO} Wildcard blocked domain${plural}: ${num}"
+  else
+    echo -e "  ${INFO} No wildcards used!"
+  fi
 }
 
 # Parse list of domains into hosts format
 gravity_ParseDomainsIntoHosts() {
   if [[ -n "${IPV4_ADDRESS}" ]] || [[ -n "${IPV6_ADDRESS}" ]]; then
-    awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" \
-      '{sub(/\r$/,""); if(ipv4addr) { print ipv4addr" "$0; }; if(ipv6addr) { print ipv6addr" "$0; }}' >> "${2}" < "${1}"
+    awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" '{
+      sub(/\r$/, "")
+      if(ipv4addr) { print ipv4addr" "$0; }
+      if(ipv6addr) { print ipv6addr" "$0; }
+    }' >> "${2}" < "${1}"
   else
     echo -e "${OVER}  ${CROSS} ${str}"
     echo -e "  ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}\\n"
@@ -406,6 +443,8 @@ gravity_ParseDomainsIntoHosts() {
 
 # Create "localhost" entries
 gravity_ParseLocalDomains() {
+  local hostname
+
   if [[ -f "/etc/hostname" ]]; then
     hostname=$(< "/etc/hostname")
   elif command -v hostname &> /dev/null; then
@@ -415,23 +454,19 @@ gravity_ParseLocalDomains() {
   fi
 
   echo -e "${hostname}\\npi.hole" > "${localList}.tmp"
-
-  # Copy the file over as /etc/pihole/local.list so dnsmasq can use it
-  rm "${localList}" 2> /dev/null || \
-    echo -e "  ${CROSS} Unable to remove ${localList}"
   gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
-  rm "${localList}.tmp" 2> /dev/null || \
-    echo -e "  ${CROSS} Unable to remove ${localList}.tmp"
 }
 
 # Create primary blacklist entries
 gravity_ParseBlacklistDomains() {
-  # Create $accretionDisc
-  [[ ! -f "${piholeDir}/${accretionDisc}" ]] && echo "" > "${piholeDir}/${accretionDisc}"
+  local output status
 
-  gravity_ParseDomainsIntoHosts "${piholeDir}/${eventHorizon}" "${piholeDir}/${accretionDisc}"
+  # Empty $accretionDisc if it already exists, otherwise, create it
+  : > "${piholeDir}/${accretionDisc}"
 
-  # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it
+  gravity_ParseDomainsIntoHosts "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}"
+
+  # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it
   output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 )
   status="$?"
 
@@ -445,7 +480,6 @@ gravity_ParseBlacklistDomains() {
 # Create user-added blacklist entries
 gravity_ParseUserDomains() {
   if [[ -f "${blacklistFile}" ]]; then
-    numBlacklisted=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
     gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp"
     # Copy the file over as /etc/pihole/black.list so dnsmasq can use it
     mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \
@@ -455,21 +489,6 @@ gravity_ParseUserDomains() {
   fi
 }
 
-# Parse consolidated blocklist into domains-only format
-gravity_Advanced() {
-  local str="Extracting domains from blocklists"
-  echo -ne "  ${INFO} ${str}..."
-
-  # Parse files as Hosts
-  gravity_ParseFileAsDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${supernova}"
-
-  numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${supernova}")")
-  echo -e "${OVER}  ${TICK} ${str}
-  ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} domains being pulled in by gravity"
-
-  gravity_Unique
-}
-
 # Trap Ctrl-C
 gravity_Trap() {
   trap '{ echo -e "\\n\\n  ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
@@ -479,7 +498,7 @@ gravity_Trap() {
 gravity_Cleanup() {
   local error="${1:-}"
 
-  str="Cleaning up debris"
+  str="Cleaning up stray matter"
   echo -ne "  ${INFO} ${str}..."
 
   rm ${piholeDir}/pihole.*.txt 2> /dev/null
@@ -487,10 +506,8 @@ gravity_Cleanup() {
 
   # Remove any unused .domains files
   for file in ${piholeDir}/*.${domainsExtension}; do
-    # If list is in active array then leave it (noop) else rm the list
-    if [[ "${activeDomains[*]}" =~ ${file} ]]; then
-      :
-    else
+    # If list is not in active array, then remove it
+    if [[ ! "${activeDomains[*]}" =~ ${file} ]]; then
       rm -f "${file}" 2> /dev/null || \
         echo -e "  ${CROSS} Failed to remove ${file##*/}"
     fi
@@ -505,27 +522,38 @@ gravity_Cleanup() {
     "${PIHOLE_COMMAND}" restartdns
   fi
 
+  # Print Pi-hole status if an error occured
   if [[ -n "$error" ]]; then
     "${PIHOLE_COMMAND}" status
     exit 1
   fi
 }
 
+helpFunc() {
+  echo "Usage: pihole -g
+Update domains from blocklists specified in adlists.list
+
+Options:
+  -f, --force          Force the download of all specified blocklists
+  -h, --help           Show this help dialog"
+  exit 0
+}
+
 for var in "$@"; do
   case "${var}" in
     "-f" | "--force" ) forceDelete=true;;
     "-h" | "--help" ) helpFunc;;
     "-sd" | "--skip-download" ) skipDownload=true;;
-    "-b" | "--blacklist-only" ) blackListOnly=true;;
-    "-w" | "--wildcard" ) dnsRestart="restart";;
+    "-b" | "--blacklist-only" ) listType="blacklist";;
+    "-w" | "--whitelist-only" ) listType="whitelist";;
+    "-wild" | "--wildcard-only" ) listType="wildcard";;
   esac
 done
 
-# Main Gravity Execution
 gravity_Trap
 
-# Use "force-reload" when restarting dnsmasq for Blacklists and Whitelists
-[[ -z "${dnsRestart}" ]] && dnsRestart="force-reload"
+# Ensure dnsmasq is restarted when modifying wildcards
+[[ "${listType}" == "wildcard" ]] && dnsRestart="restart"
 
 if [[ "${forceDelete}" == true ]]; then
   str="Deleting exising list cache"
@@ -535,41 +563,51 @@ if [[ "${forceDelete}" == true ]]; then
     echo -e "${OVER}  ${TICK} ${str}"
   else
     echo -e "${OVER}  ${CROSS} ${str}"
+    exit 1
   fi
 fi
 
-# If $blackListOnly is true, only run essential functions
-if [[ ! "${blackListOnly}" == true ]]; then
+# Determine which functions to run
+if [[ "${skipDownload}" == false ]]; then
+  # Gravity needs to download blocklists
+  gravity_DNSLookup
   gravity_Collapse
-  gravity_Pull
-
-  if [[ "${skipDownload}" == false ]]; then
-    gravity_Schwarzschild
-    gravity_Advanced
-  else
-    echo -e "  ${INFO} Using cached Event Horizon list..."
-    numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-    echo -e "  ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
-  fi
+  gravity_Supernova
+  gravity_Schwarzschild
+  gravity_Filter
+  gravity_WhitelistBLD
+else
+  # Gravity needs to modify Blacklist/Whitelist/Wildcards
+  echo -e "  ${INFO} Using cached Event Horizon list..."
+  numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
+  echo -e "  ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
+fi
 
+# Perform when downloading blocklists, or modifying the whitelist
+if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then
   gravity_Whitelist
 fi
 
-gravity_Blacklist
-gravity_Wildcard
+gravity_ShowBlockCount
 
-str="Parsing domains into hosts format"
-echo -ne "  ${INFO} ${str}..."
+# Perform when downloading blocklists, or modifying the blacklist
+if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "blacklist" ]]; then
+  str="Parsing domains into hosts format"
+  echo -ne "  ${INFO} ${str}..."
 
-if [[ ! "${blackListOnly}" == true ]]; then
-  gravity_ParseLocalDomains
-  gravity_ParseBlacklistDomains
+  gravity_ParseUserDomains
+
+  # Perform when downloading blocklists
+  if [[ ! "${listType}" == "blacklist" ]]; then
+    gravity_ParseLocalDomains
+    gravity_ParseBlacklistDomains
+  fi
+
+  echo -e "${OVER}  ${TICK} ${str}"
 fi
 
-gravity_ParseUserDomains
-echo -e "${OVER}  ${TICK} ${str}"
-
-if [[ ! "${blackListOnly}" == true ]]; then
+# Perform when downloading blocklists
+if [[ "${skipDownload}" == false ]]; then
   gravity_Cleanup
 fi
 

From cc4ada99d83fb002634fc320763b2ab565e47843 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Tue, 1 Aug 2017 20:48:43 +1000
Subject: [PATCH 07/18] Use SIGHUP instead of force-reload

---
 pihole | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/pihole b/pihole
index 1579bdbf..6d875d85 100755
--- a/pihole
+++ b/pihole
@@ -347,32 +347,33 @@ versionFunc() {
 }
 
 restartDNS() {
-  local svcOption str output status over
-  
-  # Force-reload is used for "/etc/pihole/*.list" files
-  # Restart is needed for "/etc/dnsmasq.d/*.conf" files
+  local svcOption svc str output status
   svcOption="${1:-}"
 
-  if [[ -z "${svcOption}" ]]; then
+  if [[ "${svcOption}" =~ "reload" ]]; then
+    # SIGHUP does NOT re-read any *.conf files
+    svc="killall -s SIGHUP dnsmasq"
+  elif [[ -z "${svcOption}" ]]; then
     # Get PID of dnsmasq to determine if it needs to start or restart
     if pidof dnsmasq &> /dev/null; then
       svcOption="restart"
     else
       svcOption="start"
     fi
+    svc="service dnsmasq ${svcOption}"
   fi
 
-  # Only print output to Terminal, not Web Admin
+  # Print output to Terminal, not Web Admin
   str="${svcOption^}ing DNS service"
   [[ -t 1 ]] && echo -ne "  ${INFO} ${str}..."
 
-  output=$( { service dnsmasq "${svcOption}"; } 2>&1 )
+  output=$( { ${svc}; } 2>&1 )
   status="$?"
 
-  if [[ "$?" -eq 0 ]]; then
+  if [[ "${status}" -eq 0 ]]; then
     [[ -t 1 ]] && echo -e "${OVER}  ${TICK} ${str}"
   else
-    [[ ! -t 1 ]] && OVER=""
+    [[ ! -t 1 ]] && local OVER=""
     echo -e "${OVER}  ${CROSS} ${output}"
   fi
 }

From 0a00936e99481204154190f3d554926ed8992e48 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Mon, 28 Aug 2017 11:36:02 +1000
Subject: [PATCH 08/18] Update resolver test & added more comments

* Add/update code comments
* Change resolver check to test for pi.hole
* Make resolver check timeout after 10 seconds
* Use > instead of &> where appropriate
* Make resolver check sleep for 30 seconds (effectively waiting up to 50s for dnsmasq to be resolvable)
* Provide confirmation upon success of resolver check availability
* Quotes and Braced remaining variables as appropriate
* Removed duplicate local
---
 gravity.sh | 58 ++++++++++++++++++++++++++++++++----------------------
 1 file changed, 34 insertions(+), 24 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index b222886b..c1254bb2 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -64,26 +64,27 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then
   echo -e "  ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}"
 fi
 
-# Attempt to resolve DNS before proceeding with blocklist generation
+# Determine if DNS resolution is available before proceeding with retrieving blocklists
 gravity_DNSLookup() {
   local plu
 
-  # Determine if github.com can be resolved
-  if ! timeout 2 nslookup github.com &> /dev/null; then
+  # Determine if pi.hole can be resolved
+  if ! timeout 10 nslookup pi.hole > /dev/null; then
     if [[ -n "${secs}" ]]; then
       echo -e "${OVER}  ${CROSS} DNS resolution is still unavailable, cancelling"
       exit 1
     fi
 
     # Determine error output message
-    if pidof dnsmasq &> /dev/null; then
+    if pidof dnsmasq > /dev/null; then
       echo -e "  ${CROSS} DNS resolution is temporarily unavailable"
     else
       echo -e "  ${CROSS} DNS service is not running"
       "${PIHOLE_COMMAND}" restartdns
     fi
 
-    secs="15"
+    # Give time for dnsmasq to be resolvable
+    secs="30"
     while [[ "${secs}" -ne 0 ]]; do
       [[ "${secs}" -ne 1 ]] && plu="s" || plu=""
       echo -ne "${OVER}  ${INFO} Waiting $secs second${plu} before continuing..."
@@ -93,6 +94,9 @@ gravity_DNSLookup() {
 
     # Try again
     gravity_DNSLookup
+  elif [[ -n "${secs}" ]]; then
+    # Print confirmation of resolvability if it had previously failed
+    echo -e "${OVER}  ${TICK} DNS resolution is now available\\n"
   fi
 }
 
@@ -106,7 +110,7 @@ gravity_Collapse() {
     rm "${adListDefault}" 2> /dev/null || \
       echo -e "  ${CROSS} Unable to remove ${adListDefault}"
   elif [[ ! -f "${adListFile}" ]]; then
-    # Create "adlists.list"
+    # Create "adlists.list" by copying "adlists.default" from internal Pi-hole repo
     cp "${adListRepoDefault}" "${adListFile}" 2> /dev/null || \
       echo -e "  ${CROSS} Unable to copy ${adListFile##*/} from ${piholeRepo}"
   fi
@@ -115,11 +119,11 @@ gravity_Collapse() {
   echo -ne "  ${INFO} ${str}..."
 
   # Retrieve source URLs from $adListFile
-  # Logic: Remove comments, CR line endings and empty lines
+  # Awk Logic: Remove comments, CR line endings and empty lines
   mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null)
 
   # Parse source domains from $sources
-  # Logic: Split by folder/port, remove URL protocol & optional username:password@
+  # Awk Logic: Split by folder/port, remove URL protocol & optional username:password@
   mapfile -t sourceDomains < <(
     awk -F '[/:]' '{
       gsub(/(.*:\/\/|.*:.*@)/, "", $0)
@@ -150,9 +154,10 @@ gravity_Supernova() {
     saveLocation="${piholeDir}/list.${i}.${domain}.${domainsExtension}"
     activeDomains[$i]="${saveLocation}"
 
+    # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-)
     agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36"
 
-    # Use a case statement to download lists that need special commands
+    # Provide special commands for blocklists which may need them
     case "${domain}" in
       "pgl.yoyo.org") cmd_ext="-d mimetype=plaintext -d hostformat=hosts";;
       *) cmd_ext="";;
@@ -179,9 +184,10 @@ gravity_Pull() {
 
   # Store downloaded content to temp file instead of RAM
   patternBuffer=$(mktemp)
+
   heisenbergCompensator=""
   if [[ -r "${saveLocation}" ]]; then
-    # If domain has been saved, add file for date check to only download newer
+    # Allow curl to determine if a remote file has been modified since last retrieval
     heisenbergCompensator="-z ${saveLocation}"
   fi
 
@@ -191,7 +197,7 @@ gravity_Pull() {
   httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null)
 
   # Determine "Status:" output based on HTTP response
-  case "$httpCode" in
+  case "${httpCode}" in
     "200" ) echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success="true";;
     "304" ) echo -e "${OVER}  ${TICK} ${str} No changes detected"; success="true";;
     "403" ) echo -e "${OVER}  ${CROSS} ${str} Forbidden"; success="false";;
@@ -201,7 +207,7 @@ gravity_Pull() {
     "521" ) echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";;
     "522" ) echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";;
     "500" ) echo -e "${OVER}  ${CROSS} ${str} Internal Server Error"; success="false";;
-    *     ) echo -e "${OVER}  ${CROSS} ${str} Status $httpCode"; success="false";;
+    *     ) echo -e "${OVER}  ${CROSS} ${str} Status ${httpCode}"; success="false";;
   esac
 
   # Determine if the blocklist was downloaded and saved correctly
@@ -242,14 +248,14 @@ gravity_ParseFileIntoDomains() {
   if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
     # Consolidated list parsing: Remove comments and hosts IP's
 
-    # Symbols used as comments: #;@![/
+    # Define symbols used as comments: #;@![/
     commentPattern="[#;@![\\/]"
 
-    # Logic: Process lines which do not begin with comments
+    # Awk Logic: Process lines which do not begin with comments
     awk '!/^'"${commentPattern}"'/ {
       # If there are multiple words seperated by space
       if (NF>1) {
-        # Remove comments (Inc. prefixed spaces/tabs)
+        # Remove comments (including prefixed spaces/tabs)
         if ($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) }
         # Print consecutive domains
         if ($3) {
@@ -271,8 +277,9 @@ gravity_ParseFileIntoDomains() {
     read -r firstLine < "${source}"
 
     # Determine how to parse individual source file formats
+    # Lists may not capitalise the first line correctly, so compare strings against lower case
     if [[ "${firstLine,,}" =~ "adblock" ]] || [[ "${firstLine,,}" =~ "ublock" ]] || [[ "${firstLine,,}" =~ "! checksum" ]]; then
-      # Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet 
+      # Awk Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet 
       abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
       awk ''"${abpFilter}"' {
         # Remove valid adblock type options
@@ -338,9 +345,10 @@ gravity_Filter() {
   str="Extracting domains from blocklists"
   echo -ne "  ${INFO} ${str}..."
 
-  # Parse files as hosts
+  # Parse into hosts file
   gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}"
 
+  # Format file line count as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
   echo -e "${OVER}  ${TICK} ${str}
   ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} domains being pulled in by gravity"
@@ -357,6 +365,7 @@ gravity_Unique() {
   sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
   echo -e "${OVER}  ${TICK} ${str}"
 
+  # Format file line count as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
   echo -e "  ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon"
 }
@@ -371,6 +380,7 @@ gravity_WhitelistBLD() {
   echo -ne "  ${INFO} ${str}..."
 
   # Create array of unique $sourceDomains
+  # Disable SC2046 as quoting will only return first domain
   # shellcheck disable=SC2046
   read -r -a uniqDomains <<< $(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")
 
@@ -388,7 +398,7 @@ gravity_Whitelist() {
     # Remove anything in whitelist.txt from the Event Horizon
     num=$(wc -l < "${whitelistFile}")
     plural=; [[ "${num}" != "1" ]] && plural=s
-    local str="Whitelisting ${num} domain${plural}"
+    str="Whitelisting ${num} domain${plural}"
     echo -ne "  ${INFO} ${str}..."
 
     # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in whitelist.txt
@@ -429,6 +439,7 @@ gravity_ShowBlockCount() {
 # Parse list of domains into hosts format
 gravity_ParseDomainsIntoHosts() {
   if [[ -n "${IPV4_ADDRESS}" ]] || [[ -n "${IPV6_ADDRESS}" ]]; then
+    # Awk Logic: Remove CR line endings and print IP before domain if IPv4/6 is used
     awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" '{
       sub(/\r$/, "")
       if(ipv4addr) { print ipv4addr" "$0; }
@@ -447,7 +458,7 @@ gravity_ParseLocalDomains() {
 
   if [[ -f "/etc/hostname" ]]; then
     hostname=$(< "/etc/hostname")
-  elif command -v hostname &> /dev/null; then
+  elif command -v hostname > /dev/null; then
     hostname=$(hostname -f)
   else
     echo -e "  ${CROSS} Unable to determine fully qualified domain name of host"
@@ -471,8 +482,7 @@ gravity_ParseBlacklistDomains() {
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
-    echo -e "  ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}
-  ${output}"
+    echo -e "  ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n  ${output}"
     gravity_Cleanup "error"
   fi
 }
@@ -507,7 +517,7 @@ gravity_Cleanup() {
   # Remove any unused .domains files
   for file in ${piholeDir}/*.${domainsExtension}; do
     # If list is not in active array, then remove it
-    if [[ ! "${activeDomains[*]}" =~ ${file} ]]; then
+    if [[ ! "${activeDomains[*]}" =~ "${file}" ]]; then
       rm -f "${file}" 2> /dev/null || \
         echo -e "  ${CROSS} Failed to remove ${file##*/}"
     fi
@@ -515,7 +525,7 @@ gravity_Cleanup() {
 
   echo -e "${OVER}  ${TICK} ${str}"
   
-  [[ -n "$error" ]] && echo ""
+  [[ -n "${error}" ]] && echo ""
 
   # Only restart DNS service if offline
   if ! pidof dnsmasq &> /dev/null; then
@@ -523,7 +533,7 @@ gravity_Cleanup() {
   fi
 
   # Print Pi-hole status if an error occured
-  if [[ -n "$error" ]]; then
+  if [[ -n "${error}" ]]; then
     "${PIHOLE_COMMAND}" status
     exit 1
   fi

From 4d39ab9753f1df520c50b86b9d126353382f6e13 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Thu, 14 Sep 2017 16:39:25 +1000
Subject: [PATCH 09/18] Allow force-reload to be used when restarting DNS

* Remove duplicate coltable variable definition and source
* Minor comment modifications
* Add "$2" to restartdns
---
 pihole | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/pihole b/pihole
index 61ef0e20..625ae8bb 100755
--- a/pihole
+++ b/pihole
@@ -12,10 +12,6 @@
 readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
 readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
 readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE"
-
-source ${colfile}
-
-colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE"
 source "${colfile}"
 
 # Must be root to use this tool
@@ -354,8 +350,9 @@ restartDNS() {
   local svcOption svc str output status
   svcOption="${1:-}"
 
+  # Determine if we should reload or restart dnsmasq
   if [[ "${svcOption}" =~ "reload" ]]; then
-    # SIGHUP does NOT re-read any *.conf files
+    # Using SIGHUP will NOT re-read any *.conf files
     svc="killall -s SIGHUP dnsmasq"
   elif [[ -z "${svcOption}" ]]; then
     # Get PID of dnsmasq to determine if it needs to start or restart
@@ -367,7 +364,7 @@ restartDNS() {
     svc="service dnsmasq ${svcOption}"
   fi
 
-  # Print output to Terminal, not Web Admin
+  # Print output to Terminal, but not to Web Admin
   str="${svcOption^}ing DNS service"
   [[ -t 1 ]] && echo -ne "  ${INFO} ${str}..."
 
@@ -659,7 +656,7 @@ case "${1}" in
   "enable"                      ) piholeEnable 1;;
   "disable"                     ) piholeEnable 0 "$2";;
   "status"                      ) statusFunc "$2";;
-  "restartdns"                  ) restartDNS;;
+  "restartdns"                  ) restartDNS "$2";;
   "-a" | "admin"                ) webpageFunc "$@";;
   "-t" | "tail"                 ) tailFunc;;
   "checkout"                    ) piholeCheckoutFunc "$@";;

From 8191ec01e5ca0699a29f07076f51396049440577 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Thu, 14 Sep 2017 16:39:30 +1000
Subject: [PATCH 10/18] Gravity Bugfixes

* Merge development OpenVPN code
* Determine which domain to resolve depending on existence of $localList
* Re-add code to remove $localList, preventing duplicate local entries
* Minor shellcheck validation fix
---
 gravity.sh | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index c1254bb2..354bb51e 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -32,6 +32,7 @@ wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf"
 adList="${piholeDir}/gravity.list"
 blackList="${piholeDir}/black.list"
 localList="${piholeDir}/local.list"
+VPNList="/etc/openvpn/ipp.txt"
 
 domainsExtension="domains"
 matterAndLight="${basename}.0.matterandlight.txt"
@@ -66,10 +67,17 @@ fi
 
 # Determine if DNS resolution is available before proceeding with retrieving blocklists
 gravity_DNSLookup() {
-  local plu
+  local lookupDomain plu
 
-  # Determine if pi.hole can be resolved
-  if ! timeout 10 nslookup pi.hole > /dev/null; then
+  # Determine which domain to resolve depending on existence of $localList
+  if [[ -e "${localList}" ]]; then
+    lookupDomain="pi.hole"
+  else
+    lookupDomain="raw.githubusercontent.com"
+  fi
+
+  # Determine if domain can be resolved
+  if ! timeout 10 nslookup "${lookupDomain}" > /dev/null; then
     if [[ -n "${secs}" ]]; then
       echo -e "${OVER}  ${CROSS} DNS resolution is still unavailable, cancelling"
       exit 1
@@ -452,20 +460,28 @@ gravity_ParseDomainsIntoHosts() {
   fi
 }
 
-# Create "localhost" entries
+# Create "localhost" entries into hosts format
 gravity_ParseLocalDomains() {
   local hostname
 
   if [[ -f "/etc/hostname" ]]; then
     hostname=$(< "/etc/hostname")
-  elif command -v hostname > /dev/null; then
+  elif command -v hostname &> /dev/null; then
     hostname=$(hostname -f)
   else
     echo -e "  ${CROSS} Unable to determine fully qualified domain name of host"
   fi
 
   echo -e "${hostname}\\npi.hole" > "${localList}.tmp"
+  # Copy the file over as /etc/pihole/local.list so dnsmasq can use it
+  rm "${localList}" 2> /dev/null || true
   gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
+  rm "${localList}.tmp" 2> /dev/null || true
+
+  # Add additional local hosts provided by OpenVPN (if available)
+  if [[ -f "${VPNList}" ]]; then
+    awk -F, '{printf $2"\t"$1"\n"}' "${VPNList}" >> "${localList}"
+  fi
 }
 
 # Create primary blacklist entries
@@ -494,8 +510,6 @@ gravity_ParseUserDomains() {
     # Copy the file over as /etc/pihole/black.list so dnsmasq can use it
     mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \
       echo -e "  ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}"
-  else
-    echo -e "  ${INFO} Nothing to blacklist!"
   fi
 }
 
@@ -517,7 +531,7 @@ gravity_Cleanup() {
   # Remove any unused .domains files
   for file in ${piholeDir}/*.${domainsExtension}; do
     # If list is not in active array, then remove it
-    if [[ ! "${activeDomains[*]}" =~ "${file}" ]]; then
+    if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then
       rm -f "${file}" 2> /dev/null || \
         echo -e "  ${CROSS} Failed to remove ${file##*/}"
     fi

From d3073e5e2391e4338251fc11bf2b52ddbcc22234 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Thu, 14 Sep 2017 17:09:52 +1000
Subject: [PATCH 11/18] Fix array of unique $sourceDomains

---
 gravity.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 354bb51e..39a0e75b 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -388,9 +388,7 @@ gravity_WhitelistBLD() {
   echo -ne "  ${INFO} ${str}..."
 
   # Create array of unique $sourceDomains
-  # Disable SC2046 as quoting will only return first domain
-  # shellcheck disable=SC2046
-  read -r -a uniqDomains <<< $(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")
+  mapfile -t uniqDomains <<< "$(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")"
 
   ${WHITELIST_COMMAND} -nr -q "${uniqDomains[*]}" > /dev/null
 

From ff5411a93a08c78d95e7811239214fbd0708e74c Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Thu, 14 Sep 2017 20:23:49 +1000
Subject: [PATCH 12/18] Add 'Connection Refused' for 000 status

* Shift default dnsRestart value into unset default parameter expansion value
* Change nslookup timeout to 5 seconds
* Use &> instead of >
* Standardise plural code
* Update some comments
* Add "000" when connection is refused
* Condense adblock detection logic
* Add Dnsmasq format detection and parsing logic
* Removed unnecessary echo
* Add dnsWasOffline variable to ensure that if DNS service has been stopped, that it doesn't start and also get reloaded
---
 gravity.sh | 89 +++++++++++++++++++++++++++++++-----------------------
 1 file changed, 51 insertions(+), 38 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 39a0e75b..a2f5d978 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -43,9 +43,6 @@ preEventHorizon="list.preEventHorizon"
 
 skipDownload="false"
 
-# Use "force-reload" when restarting dnsmasq for everything but Wildcards
-dnsRestart="force-reload"
-
 # Source setupVars from install script
 setupVars="${piholeDir}/setupVars.conf"
 if [[ -f "${setupVars}" ]];then
@@ -67,9 +64,10 @@ fi
 
 # Determine if DNS resolution is available before proceeding with retrieving blocklists
 gravity_DNSLookup() {
-  local lookupDomain plu
+  local lookupDomain plural
 
-  # Determine which domain to resolve depending on existence of $localList
+  # Determine which domain should be resolved
+  # "pi.hole" is not always available (e.g: new install), but FTL will not log it
   if [[ -e "${localList}" ]]; then
     lookupDomain="pi.hole"
   else
@@ -77,25 +75,25 @@ gravity_DNSLookup() {
   fi
 
   # Determine if domain can be resolved
-  if ! timeout 10 nslookup "${lookupDomain}" > /dev/null; then
+  if ! timeout 5 nslookup "${lookupDomain}" &> /dev/null; then
     if [[ -n "${secs}" ]]; then
       echo -e "${OVER}  ${CROSS} DNS resolution is still unavailable, cancelling"
       exit 1
     fi
 
     # Determine error output message
-    if pidof dnsmasq > /dev/null; then
+    if pidof dnsmasq &> /dev/null; then
       echo -e "  ${CROSS} DNS resolution is temporarily unavailable"
     else
       echo -e "  ${CROSS} DNS service is not running"
       "${PIHOLE_COMMAND}" restartdns
     fi
 
-    # Give time for dnsmasq to be resolvable
+    # Give time for DNS server to be resolvable
     secs="30"
     while [[ "${secs}" -ne 0 ]]; do
-      [[ "${secs}" -ne 1 ]] && plu="s" || plu=""
-      echo -ne "${OVER}  ${INFO} Waiting $secs second${plu} before continuing..."
+      plural=; [[ "${secs}" -ne 1 ]] && plural="s"
+      echo -ne "${OVER}  ${INFO} Waiting $secs second${plural} before continuing..."
       sleep 1
       : $((secs--))
     done
@@ -127,7 +125,7 @@ gravity_Collapse() {
   echo -ne "  ${INFO} ${str}..."
 
   # Retrieve source URLs from $adListFile
-  # Awk Logic: Remove comments, CR line endings and empty lines
+  # Awk Logic: Remove comments (#@;![), CR (windows) line endings and empty lines
   mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null)
 
   # Parse source domains from $sources
@@ -182,7 +180,7 @@ gravity_Supernova() {
   done
 }
 
-# Download specified URL and perform QA
+# Download specified URL and perform checks on HTTP status and file content
 gravity_Pull() {
   local url cmd_ext agent heisenbergCompensator patternBuffer str httpCode success
 
@@ -195,7 +193,8 @@ gravity_Pull() {
 
   heisenbergCompensator=""
   if [[ -r "${saveLocation}" ]]; then
-    # Allow curl to determine if a remote file has been modified since last retrieval
+    # Make curl determine if a remote file has been modified since last retrieval
+    # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github links)
     heisenbergCompensator="-z ${saveLocation}"
   fi
 
@@ -206,28 +205,29 @@ gravity_Pull() {
 
   # Determine "Status:" output based on HTTP response
   case "${httpCode}" in
-    "200" ) echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success="true";;
-    "304" ) echo -e "${OVER}  ${TICK} ${str} No changes detected"; success="true";;
-    "403" ) echo -e "${OVER}  ${CROSS} ${str} Forbidden"; success="false";;
-    "404" ) echo -e "${OVER}  ${CROSS} ${str} Not found"; success="false";;
-    "408" ) echo -e "${OVER}  ${CROSS} ${str} Time-out"; success="false";;
-    "451" ) echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons"; success="false";;
-    "521" ) echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";;
-    "522" ) echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";;
-    "500" ) echo -e "${OVER}  ${CROSS} ${str} Internal Server Error"; success="false";;
-    *     ) echo -e "${OVER}  ${CROSS} ${str} Status ${httpCode}"; success="false";;
+    "200") echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success="true";;
+    "304") echo -e "${OVER}  ${TICK} ${str} No changes detected"; success="true";;
+    "000") echo -e "${OVER}  ${CROSS} ${str} Connection Refused"; success="false";;
+    "403") echo -e "${OVER}  ${CROSS} ${str} Forbidden"; success="false";;
+    "404") echo -e "${OVER}  ${CROSS} ${str} Not found"; success="false";;
+    "408") echo -e "${OVER}  ${CROSS} ${str} Time-out"; success="false";;
+    "451") echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons"; success="false";;
+    "521") echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";;
+    "522") echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";;
+    "500") echo -e "${OVER}  ${CROSS} ${str} Internal Server Error"; success="false";;
+    *    ) echo -e "${OVER}  ${CROSS} ${str} ${httpCode}"; success="false";;
   esac
 
   # Determine if the blocklist was downloaded and saved correctly
   if [[ "${success}" == "true" ]]; then
     if [[ "${httpCode}" == "304" ]]; then
-      : # Do nothing
-    # Check if patternbuffer is a non-zero length file
+      : # Do not attempt to re-parse file
+    # Check if $patternbuffer is a non-zero length file
     elif [[ -s "${patternBuffer}" ]]; then
       # Determine if blocklist is non-standard and parse as appropriate
       gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}"
     else
-      # Fall back to previously cached list if patternBuffer is empty
+      # Fall back to previously cached list if $patternBuffer is empty
       echo -e "  ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
     fi
   else
@@ -248,9 +248,7 @@ gravity_Pull() {
 
 # Parse non-standard source files into domains-only format
 gravity_ParseFileIntoDomains() {
-  local source destination commentPattern firstLine abpFilter
-  source="${1}"
-  destination="${2}"
+  local source="${1}" destination="${2}" commentPattern firstLine abpFilter
 
   # Determine how to parse source file
   if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
@@ -286,7 +284,7 @@ gravity_ParseFileIntoDomains() {
 
     # Determine how to parse individual source file formats
     # Lists may not capitalise the first line correctly, so compare strings against lower case
-    if [[ "${firstLine,,}" =~ "adblock" ]] || [[ "${firstLine,,}" =~ "ublock" ]] || [[ "${firstLine,,}" =~ "! checksum" ]]; then
+    if [[ "${firstLine,,}" =~ (adblock|ublock|!checksum) ]]; then
       # Awk Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet 
       abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
       awk ''"${abpFilter}"' {
@@ -298,8 +296,8 @@ gravity_ParseFileIntoDomains() {
         if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" }
         if ($0) { print $0 }
       }' "${source}" 2> /dev/null > "${destination}"
+    # Parse URL list if source file contains http:// or IPv4
     elif grep -q -E "^(https?://|([0-9]{1,3}\\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then
-      # Parse URLs if source file contains http:// or IPv4
       awk '{
         # Remove URL protocol, optional "username:password@", and ":?/;"
         if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) }
@@ -307,6 +305,17 @@ gravity_ParseFileIntoDomains() {
         if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
         if ($0) { print $0 }
       }' "${source}" 2> /dev/null > "${destination}"
+    # Parse Dnsmasq format lists
+    elif grep -q "^address=/" "${source}" &> /dev/null; then
+      awk -F/ '{
+        # Print comments
+        if ($0 ~ "#") {
+          print $0 
+        # Print domains
+        } else if ($2) {
+          print $2
+        }
+      }' "${source}" 2> /dev/null > "${destination}"
     else
       # Keep hosts/domains file in same format as it was downloaded
       output=$( { mv "${source}" "${destination}"; } 2>&1 )
@@ -383,7 +392,7 @@ gravity_WhitelistBLD() {
   local plural str uniqDomains
 
   echo ""
-  plural=; [[ "${#sources[*]}" != "1" ]] && plural=s
+  plural=; [[ "${#sources[*]}" != "1" ]] && plural="s"
   str="Adding blocklist source${plural} to the whitelist"
   echo -ne "  ${INFO} ${str}..."
 
@@ -403,7 +412,7 @@ gravity_Whitelist() {
   if [[ -f "${whitelistFile}" ]]; then
     # Remove anything in whitelist.txt from the Event Horizon
     num=$(wc -l < "${whitelistFile}")
-    plural=; [[ "${num}" != "1" ]] && plural=s
+    plural=; [[ "${num}" -ne 1 ]] && plural="s"
     str="Whitelisting ${num} domain${plural}"
     echo -ne "  ${INFO} ${str}..."
 
@@ -422,7 +431,7 @@ gravity_ShowBlockCount() {
 
   if [[ -f "${blacklistFile}" ]]; then
     num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
-    plural=; [[ "${num}" != "1" ]] && plural=s
+    plural=; [[ "${num}" -ne 1 ]] && plural="s"
     str="Exact blocked domain${plural}: ${num}"
     echo -e "  ${INFO} ${str}"
   else
@@ -435,7 +444,7 @@ gravity_ShowBlockCount() {
     if [[ -n "${IPV4_ADDRESS}" ]] && [[ -n "${IPV6_ADDRESS}" ]];then
       num=$(( num/2 ))
     fi
-    plural=; [[ "${num}" != "1" ]] && plural=s
+    plural=; [[ "${num}" -ne 1 ]] && plural="s"
     echo -e "  ${INFO} Wildcard blocked domain${plural}: ${num}"
   else
     echo -e "  ${INFO} No wildcards used!"
@@ -536,12 +545,11 @@ gravity_Cleanup() {
   done
 
   echo -e "${OVER}  ${TICK} ${str}"
-  
-  [[ -n "${error}" ]] && echo ""
 
   # Only restart DNS service if offline
   if ! pidof dnsmasq &> /dev/null; then
     "${PIHOLE_COMMAND}" restartdns
+    dnsWasOffline=true
   fi
 
   # Print Pi-hole status if an error occured
@@ -634,5 +642,10 @@ if [[ "${skipDownload}" == false ]]; then
 fi
 
 echo ""
-"${PIHOLE_COMMAND}" restartdns "${dnsRestart}"
+
+# Determine if DNS has been restarted by this instance of gravity
+if [[ -z "${dnsWasOffline:-}" ]]; then
+  # Use "force-reload" when restarting dnsmasq for everything but Wildcards
+  "${PIHOLE_COMMAND}" restartdns "${dnsRestart:-force-reload}"
+fi
 "${PIHOLE_COMMAND}" status

From c957124fad96efd0a0462afa0e17289fb3fd6059 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Fri, 15 Sep 2017 22:39:17 +1000
Subject: [PATCH 13/18] Optimised parsing of domains on IPv6 servers

* Remove WHITELIST_COMMAND
* Place IPv4/IPv6 availability test underneath setupVars.conf source
* Improved clarity on comments
* Define default lookupDomain on local line
* Use `getent hosts` instead of nslookup (faster)
* Make gravity_DNSLookup() function more readable
* Use bold on "Neutrino emissions detected"
* Swap conditionals around on adlists file handling
* Add comments to both gravity_Collapse() `awk`s
* Removed unnecessary "${str}" from gravity_Pull()
* Merge function variables into local line
* Place .phgbp suffice on mktemp, so patternbuffers can be cleaned up all at once in gravity_Cleanup()
* Removed success="false" from $httpCode case, placed empty success var in local
* Reordered $httpCode case numerically because I can
* Provide error if Dnsmasq format list is being parsed
* Remove IPv4 check when determining URL list (too slow on large lists)
* Check ${#sources[@]} to ensure we're checking the number of entries and not the character count
* Define empty plural in local line, removing unnecessary plural=;
* Optimised readability of gravity_Whitelist()
* Removed uninformative "Nothing to blacklist"/"No wildcards used" text
* Optimised parsing of domains into hosts format on IPv6 enabled servers
* Ensure /etc/hostname is non-zero
* Use `: >` instead of `rm` as consistent with the rest of the script
* Ensured that gravity_Cleanup() removes ${localList}.tmp
* Optimised readability of gravity_ParseUserDomains()
* Moved dnsRestart to ${var} case statement, renaming it to dnsRestartType for readability
* Set default $listType to ensure script passes "bash strict mode"
---
 gravity.sh | 420 ++++++++++++++++++++++++++---------------------------
 1 file changed, 207 insertions(+), 213 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index a2f5d978..c1295852 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -16,7 +16,6 @@ source "${coltable}"
 
 basename="pihole"
 PIHOLE_COMMAND="/usr/local/bin/${basename}"
-WHITELIST_COMMAND="${PIHOLE_COMMAND} -w"
 
 piholeDir="/etc/${basename}"
 piholeRepo="/etc/.${basename}"
@@ -51,87 +50,99 @@ if [[ -f "${setupVars}" ]];then
   # Remove CIDR mask from IPv4/6 addresses
   IPV4_ADDRESS="${IPV4_ADDRESS%/*}"
   IPV6_ADDRESS="${IPV6_ADDRESS%/*}"
+
+  # Determine if IPv4/6 addresses exist
+  if [[ -z "${IPV4_ADDRESS}" ]] && [[ -z "${IPV6_ADDRESS}" ]]; then
+    echo -e "  ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}"
+    exit 1
+  fi
 else
   echo -e "  ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC}
   Please run 'pihole -r', and choose the 'reconfigure' option to fix."
   exit 1
 fi
 
-# Warn users still using pihole.conf that it no longer has any effect
+# Determine if superseded pihole.conf exists
 if [[ -r "${piholeDir}/pihole.conf" ]]; then
   echo -e "  ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}"
 fi
 
-# Determine if DNS resolution is available before proceeding with retrieving blocklists
+# Determine if DNS resolution is available before proceeding
 gravity_DNSLookup() {
-  local lookupDomain plural
+  local lookupDomain="pi.hole" plural=""
 
-  # Determine which domain should be resolved
-  # "pi.hole" is not always available (e.g: new install), but FTL will not log it
-  if [[ -e "${localList}" ]]; then
-    lookupDomain="pi.hole"
-  else
+  # Determine if $localList does not exist
+  if [[ ! -e "${localList}" ]]; then
     lookupDomain="raw.githubusercontent.com"
   fi
 
-  # Determine if domain can be resolved
-  if ! timeout 5 nslookup "${lookupDomain}" &> /dev/null; then
-    if [[ -n "${secs}" ]]; then
-      echo -e "${OVER}  ${CROSS} DNS resolution is still unavailable, cancelling"
-      exit 1
-    fi
-
-    # Determine error output message
-    if pidof dnsmasq &> /dev/null; then
-      echo -e "  ${CROSS} DNS resolution is temporarily unavailable"
-    else
-      echo -e "  ${CROSS} DNS service is not running"
-      "${PIHOLE_COMMAND}" restartdns
-    fi
-
-    # Give time for DNS server to be resolvable
-    secs="30"
-    while [[ "${secs}" -ne 0 ]]; do
-      plural=; [[ "${secs}" -ne 1 ]] && plural="s"
-      echo -ne "${OVER}  ${INFO} Waiting $secs second${plural} before continuing..."
-      sleep 1
-      : $((secs--))
-    done
-
-    # Try again
-    gravity_DNSLookup
-  elif [[ -n "${secs}" ]]; then
+  # Determine if $lookupDomain is resolvable
+  if timeout 5 getent hosts "${lookupDomain}" &> /dev/null; then
     # Print confirmation of resolvability if it had previously failed
-    echo -e "${OVER}  ${TICK} DNS resolution is now available\\n"
+    if [[ -n "${secs:-}" ]]; then
+      echo -e "${OVER}  ${TICK} DNS resolution is now available\\n"
+    fi
+    return 0
+  elif [[ -n "${secs:-}" ]]; then
+    echo -e "${OVER}  ${CROSS} DNS resolution is still unavailable, cancelling"
+    exit 1
   fi
+
+  # Determine error output message
+  if pidof dnsmasq &> /dev/null; then
+    echo -e "  ${CROSS} DNS resolution is currently unavailable"
+  else
+    echo -e "  ${CROSS} DNS service is not running"
+    "${PIHOLE_COMMAND}" restartdns
+  fi
+
+  # Give time for DNS server to be resolvable
+  secs="30"
+  while [[ "${secs}" -ne 0 ]]; do
+    [[ "${secs}" -ne 1 ]] && plural="s"
+    echo -ne "${OVER}  ${INFO} Waiting $secs second${plural} before continuing..."
+    sleep 1
+    : $((secs--))
+  done
+
+  # Try again
+  gravity_DNSLookup
 }
 
 # Retrieve blocklist URLs and parse domains from adlists.list
 gravity_Collapse() {
-  echo -e "  ${INFO} Neutrino emissions detected..."
+  echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
 
-  # Handle "adlists.list" and "adlists.default" files
-  if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then
+  # Determine if adlists file needs handling
+  if [[ ! -f "${adListFile}" ]]; then
+    # Create "adlists.list" by copying "adlists.default" from internal core repo
+    cp "${adListRepoDefault}" "${adListFile}" 2> /dev/null || \
+      echo -e "  ${CROSS} Unable to copy ${adListFile##*/} from ${piholeRepo}"
+  elif [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then
     # Remove superceded $adListDefault file
     rm "${adListDefault}" 2> /dev/null || \
       echo -e "  ${CROSS} Unable to remove ${adListDefault}"
-  elif [[ ! -f "${adListFile}" ]]; then
-    # Create "adlists.list" by copying "adlists.default" from internal Pi-hole repo
-    cp "${adListRepoDefault}" "${adListFile}" 2> /dev/null || \
-      echo -e "  ${CROSS} Unable to copy ${adListFile##*/} from ${piholeRepo}"
   fi
 
   local str="Pulling blocklist source list into range"
   echo -ne "  ${INFO} ${str}..."
 
   # Retrieve source URLs from $adListFile
-  # Awk Logic: Remove comments (#@;![), CR (windows) line endings and empty lines
-  mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null)
+  mapfile -t sources <<< $(
+    # Logic: Remove comments (#@;![)
+    awk '!/^[#@;!\[]/ {
+      # Remove windows CR line endings
+      gsub(/\r$/, "", $0)
+      # Print non-empty line
+      if ($1) { print $1 }
+    }' "${adListFile}" 2> /dev/null
+  )
 
   # Parse source domains from $sources
-  # Awk Logic: Split by folder/port, remove URL protocol & optional username:password@
-  mapfile -t sourceDomains < <(
+  mapfile -t sourceDomains <<< $(
+    # Logic: Split by folder/port
     awk -F '[/:]' '{
+      # Remove URL protocol & optional username:password@
       gsub(/(.*:\/\/|.*:.*@)/, "", $0)
       print $1
     }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null
@@ -151,7 +162,7 @@ gravity_Supernova() {
 
   echo ""
 
-  # Loop through $sources to download each one
+  # Loop through $sources and download each one
   for ((i = 0; i < "${#sources[@]}"; i++)); do
     url="${sources[$i]}"
     domain="${sourceDomains[$i]}"
@@ -170,11 +181,8 @@ gravity_Supernova() {
     esac
 
     if [[ "${skipDownload}" == false ]]; then
-      str="Target: ${domain} (${url##*/})"
-      echo -e "  ${INFO} ${str}"
-
-      gravity_Pull "${url}" "${cmd_ext}" "${agent}" "${str}"
-
+      echo -e "  ${INFO} Target: ${domain} (${url##*/})"
+      gravity_Pull "${url}" "${cmd_ext}" "${agent}"
       echo ""
     fi
   done
@@ -182,19 +190,15 @@ gravity_Supernova() {
 
 # Download specified URL and perform checks on HTTP status and file content
 gravity_Pull() {
-  local url cmd_ext agent heisenbergCompensator patternBuffer str httpCode success
+  local url="${1}" cmd_ext="${2}" agent="${3}" heisenbergCompensator="" patternBuffer str httpCode success=""
 
-  url="${1}"
-  cmd_ext="${2}"
-  agent="${3}"
+  # Create temp file to store content on disk instead of RAM
+  patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb")
 
-  # Store downloaded content to temp file instead of RAM
-  patternBuffer=$(mktemp)
-
-  heisenbergCompensator=""
+  # Determine if $saveLocation has read permission
   if [[ -r "${saveLocation}" ]]; then
-    # Make curl determine if a remote file has been modified since last retrieval
-    # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github links)
+    # Have curl determine if a remote file has been modified since last retrieval
+    # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls)
     heisenbergCompensator="-z ${saveLocation}"
   fi
 
@@ -205,21 +209,21 @@ gravity_Pull() {
 
   # Determine "Status:" output based on HTTP response
   case "${httpCode}" in
-    "200") echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success="true";;
-    "304") echo -e "${OVER}  ${TICK} ${str} No changes detected"; success="true";;
-    "000") echo -e "${OVER}  ${CROSS} ${str} Connection Refused"; success="false";;
-    "403") echo -e "${OVER}  ${CROSS} ${str} Forbidden"; success="false";;
-    "404") echo -e "${OVER}  ${CROSS} ${str} Not found"; success="false";;
-    "408") echo -e "${OVER}  ${CROSS} ${str} Time-out"; success="false";;
-    "451") echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons"; success="false";;
-    "521") echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";;
-    "522") echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";;
-    "500") echo -e "${OVER}  ${CROSS} ${str} Internal Server Error"; success="false";;
-    *    ) echo -e "${OVER}  ${CROSS} ${str} ${httpCode}"; success="false";;
+    "200") echo -e "${OVER}  ${TICK} ${str} Retrieval successful"; success=true;;
+    "304") echo -e "${OVER}  ${TICK} ${str} No changes detected"; success=true;;
+    "000") echo -e "${OVER}  ${CROSS} ${str} Connection Refused";;
+    "403") echo -e "${OVER}  ${CROSS} ${str} Forbidden";;
+    "404") echo -e "${OVER}  ${CROSS} ${str} Not found";;
+    "408") echo -e "${OVER}  ${CROSS} ${str} Time-out";;
+    "451") echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons";;
+    "500") echo -e "${OVER}  ${CROSS} ${str} Internal Server Error";;
+    "521") echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)";;
+    "522") echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)";;
+    *    ) echo -e "${OVER}  ${CROSS} ${str} ${httpCode}";;
   esac
 
   # Determine if the blocklist was downloaded and saved correctly
-  if [[ "${success}" == "true" ]]; then
+  if [[ "${success}" == true ]]; then
     if [[ "${httpCode}" == "304" ]]; then
       : # Do not attempt to re-parse file
     # Check if $patternbuffer is a non-zero length file
@@ -231,46 +235,40 @@ gravity_Pull() {
       echo -e "  ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
     fi
   else
-    # Determine if cached list exists
+    # Determine if cached list has read permission
     if [[ -r "${saveLocation}" ]]; then
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
     else
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
     fi
   fi
-
-  # Delete temp file if it has not been moved
-  if [[ -f "${patternBuffer}" ]]; then
-    rm "${patternBuffer}" 2> /dev/null || \
-      echo -e "  ${CROSS} Unable to remove ${patternBuffer}"
-  fi
 }
 
-# Parse non-standard source files into domains-only format
+# Parse source files into domains format
 gravity_ParseFileIntoDomains() {
   local source="${1}" destination="${2}" commentPattern firstLine abpFilter
 
-  # Determine how to parse source file
+  # Determine if we are parsing a consolidated list
   if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
-    # Consolidated list parsing: Remove comments and hosts IP's
-
     # Define symbols used as comments: #;@![/
     commentPattern="[#;@![\\/]"
 
-    # Awk Logic: Process lines which do not begin with comments
+    # Parse Domains/Hosts files by removing comments & host IPs
+    # Logic: Ignore lines which begin with comments
     awk '!/^'"${commentPattern}"'/ {
-      # If there are multiple words seperated by space
-      if (NF>1) {
+      # Determine if there are multiple words seperated by a space
+      if(NF>1) {
         # Remove comments (including prefixed spaces/tabs)
-        if ($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) }
-        # Print consecutive domains
-        if ($3) {
+        if($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) }
+        # Determine if there are aliased domains
+        if($3) {
+          # Remove IP address
           $1=""
           # Remove space which is left in $0 when removing $1
           gsub("^ ", "", $0)
           print $0
-        # Print single domain
-        } else if ($2) {
+        } else if($2) {
+          # Print single domain without IP
           print $2
         }
       # If there are no words seperated by space
@@ -278,59 +276,60 @@ gravity_ParseFileIntoDomains() {
         print $1
       }
     }' "${source}" 2> /dev/null > "${destination}"
-  else
-    # Individual file parsing: Keep comments, while parsing domains from each line
-    read -r firstLine < "${source}"
+    return 0
+  fi
 
-    # Determine how to parse individual source file formats
-    # Lists may not capitalise the first line correctly, so compare strings against lower case
-    if [[ "${firstLine,,}" =~ (adblock|ublock|!checksum) ]]; then
-      # Awk Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet 
-      abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
-      awk ''"${abpFilter}"' {
-        # Remove valid adblock type options
-        gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
-        # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional)
-        gsub(/(\|\||\^\$?$)/, "", $0)
-        # Remove lines which are only IPv4 addresses or contain "^/*"
-        if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" }
-        if ($0) { print $0 }
-      }' "${source}" 2> /dev/null > "${destination}"
-    # Parse URL list if source file contains http:// or IPv4
-    elif grep -q -E "^(https?://|([0-9]{1,3}\\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then
-      awk '{
-        # Remove URL protocol, optional "username:password@", and ":?/;"
-        if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) }
-        # Remove lines which are only IPv4 addresses
-        if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
-        if ($0) { print $0 }
-      }' "${source}" 2> /dev/null > "${destination}"
+  # Individual file parsing: Keep comments, while parsing domains from each line
+  # We keep comments to respect the list maintainer's licensing
+  read -r firstLine < "${source}"
+
+  # Determine how to parse individual source file formats
+  if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then
+    # Compare $firstLine against lower case words found in Adblock lists
+
+    # Define symbols used as comments: [!
+    # "||.*^" includes the "Example 2" domains we can extract
+    # https://adblockplus.org/filter-cheatsheet 
+    abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
+
+    # Parse Adblock lists by extracting "Example 2" domains
+    # Logic: Ignore lines which do not include comments or domain name anchor
+    awk ''"${abpFilter}"' {
+      # Remove valid adblock type options
+      gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
+      # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional)
+      gsub(/(\|\||\^\$?$)/, "", $0)
+      # Remove lines which are only IPv4 addresses or contain "^/*"
+      if($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" }
+      if($0) { print $0 }
+    }' "${source}" 2> /dev/null > "${destination}"
+  elif grep -q "^address=/" "${source}" &> /dev/null; then
     # Parse Dnsmasq format lists
-    elif grep -q "^address=/" "${source}" &> /dev/null; then
-      awk -F/ '{
-        # Print comments
-        if ($0 ~ "#") {
-          print $0 
-        # Print domains
-        } else if ($2) {
-          print $2
-        }
-      }' "${source}" 2> /dev/null > "${destination}"
-    else
-      # Keep hosts/domains file in same format as it was downloaded
-      output=$( { mv "${source}" "${destination}"; } 2>&1 )
-      status="$?"
+    echo -e "  ${CROSS} ${COL_BOLD}dnsmasq${COL_NC} format lists are not supported"
+  elif grep -q -E "^(https?://|www\\.)" "${source}" &> /dev/null; then
+    # Parse URL list if source file contains "http://" or "www."
+    # Scanning for "^IPv4$" is too slow with large (1M) lists on low-end hardware
 
-      if [[ "${status}" -ne 0 ]]; then
-        echo -e "  ${CROSS} Unable to move tmp file to ${piholeDir}
-      ${output}"
-        gravity_Cleanup "error"
-      fi
+    awk '{
+      # Remove URL protocol, optional "username:password@", and ":?/;"
+      if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) }
+      # Remove lines which are only IPv4 addresses
+      if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
+      if ($0) { print $0 }
+    }' "${source}" 2> /dev/null > "${destination}"
+  else
+    # Default: Keep hosts/domains file in same format as it was downloaded
+    output=$( { mv "${source}" "${destination}"; } 2>&1 )
+
+    if [[ ! -e "${destination}" ]]; then
+      echo -e "  ${CROSS} Unable to move tmp file to ${piholeDir}
+    ${output}"
+      gravity_Cleanup "error"
     fi
   fi
 }
 
-# Create unfiltered "Matter and Light" consolidated list
+# Create (unfiltered) "Matter and Light" consolidated list
 gravity_Schwarzschild() {
   local str lastLine
 
@@ -340,22 +339,25 @@ gravity_Schwarzschild() {
   # Empty $matterAndLight if it already exists, otherwise, create it
   : > "${piholeDir}/${matterAndLight}"
 
+  # Loop through each *.domains file
   for i in "${activeDomains[@]}"; do
-    # Only assimilate list if it is available (download might have failed permanently)
+    # Determine if file has read permissions, as download might have failed
     if [[ -r "${i}" ]]; then
-      # Compile all blacklisted domains into one file and remove CRs
+      # Remove windows CRs from file, and append into $matterAndLight
       tr -d '\r' < "${i}" >> "${piholeDir}/${matterAndLight}"
 
-      # Ensure each source blocklist has a final newline
+      # Ensure that the first line of a new list is on a new line
       lastLine=$(tail -1 "${piholeDir}/${matterAndLight}")
-      [[ "${#lastLine}" -gt 0 ]] && echo "" >> "${piholeDir}/${matterAndLight}"
+      if [[ "${#lastLine}" -gt 0 ]]; then
+        echo "" >> "${piholeDir}/${matterAndLight}"
+      fi
     fi
   done
 
   echo -e "${OVER}  ${TICK} ${str}"
 }
 
-# Parse unfiltered consolidated blocklist into filtered domains-only format
+# Parse consolidated list into (filtered, unique) domains-only format
 gravity_Filter() {
   local str num
 
@@ -365,77 +367,67 @@ gravity_Filter() {
   # Parse into hosts file
   gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}"
 
-  # Format file line count as currency
+  # Format $parsedMatter line total as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
   echo -e "${OVER}  ${TICK} ${str}
   ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} domains being pulled in by gravity"
 
-  gravity_Unique
-}
-
-# Sort and remove duplicate blacklisted domains
-gravity_Unique() {
-  local str num
-
   str="Removing duplicate domains"
   echo -ne "  ${INFO} ${str}..."
   sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
   echo -e "${OVER}  ${TICK} ${str}"
 
-  # Format file line count as currency
+  # Format $preEventHorizon line total as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
   echo -e "  ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon"
 }
 
-# Whitelist blocklist domain sources
+# Whitelist unique blocklist domain sources
 gravity_WhitelistBLD() {
-  local plural str uniqDomains
+  local plural="" str uniqDomains
 
   echo ""
-  plural=; [[ "${#sources[*]}" != "1" ]] && plural="s"
-  str="Adding blocklist source${plural} to the whitelist"
+  [[ "${#sources[@]}" -ne 1 ]] && plural="s"
+  str="Adding blocklist domain source${plural} to the whitelist"
   echo -ne "  ${INFO} ${str}..."
 
   # Create array of unique $sourceDomains
   mapfile -t uniqDomains <<< "$(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")"
 
-  ${WHITELIST_COMMAND} -nr -q "${uniqDomains[*]}" > /dev/null
+  # Whitelist $uniqDomains
+  "${PIHOLE_COMMAND}" -w -nr -q "${uniqDomains[*]}" &> /dev/null
 
   echo -e "${OVER}  ${TICK} ${str}"
 }
 
 # Whitelist user-defined domains
 gravity_Whitelist() {
-  local plural str num
+  local num plural="" str
 
-  # Test existence of whitelist.txt
-  if [[ -f "${whitelistFile}" ]]; then
-    # Remove anything in whitelist.txt from the Event Horizon
-    num=$(wc -l < "${whitelistFile}")
-    plural=; [[ "${num}" -ne 1 ]] && plural="s"
-    str="Whitelisting ${num} domain${plural}"
-    echo -ne "  ${INFO} ${str}..."
-
-    # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in whitelist.txt
-    grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${whitelistMatter}"
-
-    echo -e "${OVER}  ${TICK} ${str}"
-  else
+  if [[ ! -f "${whitelistFile}" ]]; then
     echo -e "  ${INFO} Nothing to whitelist!"
+    return 0
   fi
+
+  num=$(wc -l < "${whitelistFile}")
+  [[ "${num}" -ne 1 ]] && plural="s"
+  str="Whitelisting ${num} domain${plural}"
+  echo -ne "  ${INFO} ${str}..."
+
+  # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile
+  grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${whitelistMatter}"
+
+  echo -e "${OVER}  ${TICK} ${str}"
 }
 
 # Output count of blacklisted domains and wildcards
 gravity_ShowBlockCount() {
-  local num plural str
+  local num plural
 
   if [[ -f "${blacklistFile}" ]]; then
     num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
     plural=; [[ "${num}" -ne 1 ]] && plural="s"
-    str="Exact blocked domain${plural}: ${num}"
-    echo -e "  ${INFO} ${str}"
-  else
-    echo -e "  ${INFO} Nothing to blacklist!"
+    echo -e "  ${INFO} Blacklisted ${num} domain${plural}"
   fi
 
   if [[ -f "${wildcardFile}" ]]; then
@@ -445,47 +437,47 @@ gravity_ShowBlockCount() {
       num=$(( num/2 ))
     fi
     plural=; [[ "${num}" -ne 1 ]] && plural="s"
-    echo -e "  ${INFO} Wildcard blocked domain${plural}: ${num}"
-  else
-    echo -e "  ${INFO} No wildcards used!"
+    echo -e "  ${INFO} Wildcard blocked ${num} domain${plural}"
   fi
 }
 
 # Parse list of domains into hosts format
 gravity_ParseDomainsIntoHosts() {
-  if [[ -n "${IPV4_ADDRESS}" ]] || [[ -n "${IPV6_ADDRESS}" ]]; then
-    # Awk Logic: Remove CR line endings and print IP before domain if IPv4/6 is used
-    awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" '{
-      sub(/\r$/, "")
-      if(ipv4addr) { print ipv4addr" "$0; }
-      if(ipv6addr) { print ipv6addr" "$0; }
-    }' >> "${2}" < "${1}"
-  else
-    echo -e "${OVER}  ${CROSS} ${str}"
-    echo -e "  ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}\\n"
-    gravity_Cleanup "error"
-  fi
+  awk -v ipv4="$IPV4_ADDRESS" -v ipv6="$IPV6_ADDRESS" '{
+    # Remove windows CR line endings
+    sub(/\r$/, "")
+    # Parse each line as "ipaddr domain"
+    if(ipv6 && ipv4) {
+      print ipv4" "$0"\n"ipv6" "$0
+    } else if(!ipv6) {
+      print ipv4" "$0
+    } else {
+      print ipv6" "$0
+    }
+  }' >> "${2}" < "${1}"
 }
 
 # Create "localhost" entries into hosts format
 gravity_ParseLocalDomains() {
   local hostname
 
-  if [[ -f "/etc/hostname" ]]; then
+  if [[ -s "/etc/hostname" ]]; then
     hostname=$(< "/etc/hostname")
   elif command -v hostname &> /dev/null; then
     hostname=$(hostname -f)
   else
     echo -e "  ${CROSS} Unable to determine fully qualified domain name of host"
+    return 0
   fi
 
   echo -e "${hostname}\\npi.hole" > "${localList}.tmp"
-  # Copy the file over as /etc/pihole/local.list so dnsmasq can use it
-  rm "${localList}" 2> /dev/null || true
-  gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
-  rm "${localList}.tmp" 2> /dev/null || true
 
-  # Add additional local hosts provided by OpenVPN (if available)
+  # Empty $localList if it already exists, otherwise, create it
+  : > "${localList}"
+
+  gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
+
+  # Add additional LAN hosts provided by OpenVPN (if available)
   if [[ -f "${VPNList}" ]]; then
     awk -F, '{printf $2"\t"$1"\n"}' "${VPNList}" >> "${localList}"
   fi
@@ -512,12 +504,14 @@ gravity_ParseBlacklistDomains() {
 
 # Create user-added blacklist entries
 gravity_ParseUserDomains() {
-  if [[ -f "${blacklistFile}" ]]; then
-    gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp"
-    # Copy the file over as /etc/pihole/black.list so dnsmasq can use it
-    mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \
-      echo -e "  ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}"
+  if [[ ! -f "${blacklistFile}" ]]; then
+    return 0
   fi
+
+  gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp"
+  # Copy the file over as /etc/pihole/black.list so dnsmasq can use it
+  mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \
+    echo -e "  ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}"
 }
 
 # Trap Ctrl-C
@@ -525,15 +519,17 @@ gravity_Trap() {
   trap '{ echo -e "\\n\\n  ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
 }
 
-# Clean up after Gravity
+# Clean up after Gravity upon exit or cancellation
 gravity_Cleanup() {
   local error="${1:-}"
 
   str="Cleaning up stray matter"
   echo -ne "  ${INFO} ${str}..."
 
+  # Delete tmp content generated by Gravity
   rm ${piholeDir}/pihole.*.txt 2> /dev/null
   rm ${piholeDir}/*.tmp 2> /dev/null
+  rm /tmp/*.phgpb 2> /dev/null
 
   # Remove any unused .domains files
   for file in ${piholeDir}/*.${domainsExtension}; do
@@ -576,16 +572,14 @@ for var in "$@"; do
     "-sd" | "--skip-download" ) skipDownload=true;;
     "-b" | "--blacklist-only" ) listType="blacklist";;
     "-w" | "--whitelist-only" ) listType="whitelist";;
-    "-wild" | "--wildcard-only" ) listType="wildcard";;
+    "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";;
   esac
 done
 
+# Trap Ctrl-C
 gravity_Trap
 
-# Ensure dnsmasq is restarted when modifying wildcards
-[[ "${listType}" == "wildcard" ]] && dnsRestart="restart"
-
-if [[ "${forceDelete}" == true ]]; then
+if [[ "${forceDelete:-}" == true ]]; then
   str="Deleting exising list cache"
   echo -ne "${INFO} ${str}..."
 
@@ -628,7 +622,7 @@ if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "blacklist" ]]; then
   gravity_ParseUserDomains
 
   # Perform when downloading blocklists
-  if [[ ! "${listType}" == "blacklist" ]]; then
+  if [[ ! "${listType:-}" == "blacklist" ]]; then
     gravity_ParseLocalDomains
     gravity_ParseBlacklistDomains
   fi
@@ -646,6 +640,6 @@ echo ""
 # Determine if DNS has been restarted by this instance of gravity
 if [[ -z "${dnsWasOffline:-}" ]]; then
   # Use "force-reload" when restarting dnsmasq for everything but Wildcards
-  "${PIHOLE_COMMAND}" restartdns "${dnsRestart:-force-reload}"
+  "${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}"
 fi
 "${PIHOLE_COMMAND}" status

From d02bf258af2dcabd3de648dcf31d98c62d903083 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Mon, 18 Sep 2017 17:36:03 +1000
Subject: [PATCH 14/18] Improve non-standard list parsing

* Add 504 status (Gateway connection timed out)
* Add text for non-standard list parsing
* Improve adblock parsing
* Ensure adblock exception rules are removed from file
* Ensure "www." is not treated as a URL-format list
* Corrected typo
* Ensure script does not fail if "-f" is used when there are no blocklists generated

Signed off by WaLLy3K <wally3k@pi-hole.net>
---
 gravity.sh | 55 ++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 39 insertions(+), 16 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index c1295852..3bed8cb3 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -217,6 +217,7 @@ gravity_Pull() {
     "408") echo -e "${OVER}  ${CROSS} ${str} Time-out";;
     "451") echo -e "${OVER}  ${CROSS} ${str} Unavailable For Legal Reasons";;
     "500") echo -e "${OVER}  ${CROSS} ${str} Internal Server Error";;
+    "504") echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Gateway)";;
     "521") echo -e "${OVER}  ${CROSS} ${str} Web Server Is Down (Cloudflare)";;
     "522") echo -e "${OVER}  ${CROSS} ${str} Connection Timed Out (Cloudflare)";;
     *    ) echo -e "${OVER}  ${CROSS} ${str} ${httpCode}";;
@@ -286,6 +287,7 @@ gravity_ParseFileIntoDomains() {
   # Determine how to parse individual source file formats
   if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then
     # Compare $firstLine against lower case words found in Adblock lists
+    echo -ne "  ${INFO} Format: Adblock"
 
     # Define symbols used as comments: [!
     # "||.*^" includes the "Example 2" domains we can extract
@@ -296,19 +298,42 @@ gravity_ParseFileIntoDomains() {
     # Logic: Ignore lines which do not include comments or domain name anchor
     awk ''"${abpFilter}"' {
       # Remove valid adblock type options
-      gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
-      # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional)
-      gsub(/(\|\||\^\$?$)/, "", $0)
-      # Remove lines which are only IPv4 addresses or contain "^/*"
-      if($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" }
+      gsub(/\$?~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
+      # Remove starting domain name anchor "||" and ending seperator "^"
+      gsub(/^(\|\|)|(\^)/, "", $0)
+      # Remove invalid characters (*/,=$)
+      if($0 ~ /[*\/,=\$]/) { $0="" }
+      # Remove lines which are only IPv4 addresses
+      if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
       if($0) { print $0 }
-    }' "${source}" 2> /dev/null > "${destination}"
+    }' "${source}" > "${destination}"
+
+    # Determine if there are Adblock exception rules
+    # https://adblockplus.org/filters
+    if grep -q "^@@||" "${source}" &> /dev/null; then
+      # Parse Adblock lists by extracting exception rules
+      # Logic: Ignore lines which do not include exception format "@@||example.com^"
+      awk -F "[|^]" '/^@@\|\|.*\^/ {
+        # Remove valid adblock type options
+        gsub(/\$?~?(third-party)/, "", $0)
+        # Remove invalid characters (*/,=$)
+        if($0 ~ /[*\/,=\$]/) { $0="" }
+        if($3) { print $3 }
+      }' "${source}" > "${destination}.exceptionsFile.tmp"
+
+      # Remove exceptions
+      grep -F -x -v -f "${destination}.exceptionsFile.tmp" "${destination}" > "${source}"
+      mv "${source}" "${destination}"
+    fi
+
+    echo -e "${OVER}  ${TICK} Format: Adblock"
   elif grep -q "^address=/" "${source}" &> /dev/null; then
     # Parse Dnsmasq format lists
-    echo -e "  ${CROSS} ${COL_BOLD}dnsmasq${COL_NC} format lists are not supported"
-  elif grep -q -E "^(https?://|www\\.)" "${source}" &> /dev/null; then
-    # Parse URL list if source file contains "http://" or "www."
+    echo -e "  ${CROSS} Format: Dnsmasq (list type not supported)"
+  elif grep -q -E "^https?://" "${source}" &> /dev/null; then
+    # Parse URL list if source file contains "http://" or "https://"
     # Scanning for "^IPv4$" is too slow with large (1M) lists on low-end hardware
+    echo -ne "  ${INFO} Format: URL"
 
     awk '{
       # Remove URL protocol, optional "username:password@", and ":?/;"
@@ -317,6 +342,8 @@ gravity_ParseFileIntoDomains() {
       if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
       if ($0) { print $0 }
     }' "${source}" 2> /dev/null > "${destination}"
+
+    echo -e "${OVER}  ${TICK} Format: URL"
   else
     # Default: Keep hosts/domains file in same format as it was downloaded
     output=$( { mv "${source}" "${destination}"; } 2>&1 )
@@ -580,15 +607,11 @@ done
 gravity_Trap
 
 if [[ "${forceDelete:-}" == true ]]; then
-  str="Deleting exising list cache"
+  str="Deleting existing list cache"
   echo -ne "${INFO} ${str}..."
 
-  if rm /etc/pihole/list.* 2> /dev/null; then
-    echo -e "${OVER}  ${TICK} ${str}"
-  else
-    echo -e "${OVER}  ${CROSS} ${str}"
-    exit 1
-  fi
+  rm /etc/pihole/list.* 2> /dev/null || true
+  echo -e "${OVER}  ${TICK} ${str}"
 fi
 
 # Determine which functions to run

From 8d8482d60b73c97a513a3fab57cba1b2125d3590 Mon Sep 17 00:00:00 2001
From: Adam Warner <adamw@rner.email>
Date: Tue, 19 Sep 2017 19:02:50 +0100
Subject: [PATCH 15/18] `<<<$()` back to `< <()`

Signed-off-by: Adam Warner <adamw@rner.email>
---
 gravity.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 3bed8cb3..bb69c918 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -128,7 +128,7 @@ gravity_Collapse() {
   echo -ne "  ${INFO} ${str}..."
 
   # Retrieve source URLs from $adListFile
-  mapfile -t sources <<< $(
+  mapfile -t sources < <(
     # Logic: Remove comments (#@;![)
     awk '!/^[#@;!\[]/ {
       # Remove windows CR line endings
@@ -139,7 +139,7 @@ gravity_Collapse() {
   )
 
   # Parse source domains from $sources
-  mapfile -t sourceDomains <<< $(
+  mapfile -t sourceDomains < <(
     # Logic: Split by folder/port
     awk -F '[/:]' '{
       # Remove URL protocol & optional username:password@
@@ -291,7 +291,7 @@ gravity_ParseFileIntoDomains() {
 
     # Define symbols used as comments: [!
     # "||.*^" includes the "Example 2" domains we can extract
-    # https://adblockplus.org/filter-cheatsheet 
+    # https://adblockplus.org/filter-cheatsheet
     abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
 
     # Parse Adblock lists by extracting "Example 2" domains

From a6f9272d4b021dbce152ebfc2ac8150a7a96f9b1 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Wed, 20 Sep 2017 22:25:33 +1000
Subject: [PATCH 16/18] Fix gravity from only parsing one adlist URL

* Redirect `grep` correctly to $sources (instead of using `awk`)
* Redirect $sourceDomains correctly
* Replace use of ${COL_LIGHT_BLUE}
* Add numeric count informing user of unique source domains being whitelisted
---
 gravity.sh | 30 ++++++++++++------------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index bb69c918..cb717aba 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -128,25 +128,18 @@ gravity_Collapse() {
   echo -ne "  ${INFO} ${str}..."
 
   # Retrieve source URLs from $adListFile
-  mapfile -t sources < <(
-    # Logic: Remove comments (#@;![)
-    awk '!/^[#@;!\[]/ {
-      # Remove windows CR line endings
-      gsub(/\r$/, "", $0)
-      # Print non-empty line
-      if ($1) { print $1 }
-    }' "${adListFile}" 2> /dev/null
-  )
+  # Logic: Remove comments and empty lines
+  mapfile -t sources <<< "$(grep -v -E "^(#|$)" "${adListFile}" 2> /dev/null)"
 
   # Parse source domains from $sources
-  mapfile -t sourceDomains < <(
+  mapfile -t sourceDomains <<< "$(
     # Logic: Split by folder/port
     awk -F '[/:]' '{
       # Remove URL protocol & optional username:password@
       gsub(/(.*:\/\/|.*:.*@)/, "", $0)
       print $1
     }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null
-  )
+  )"
 
   if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then
     echo -e "${OVER}  ${TICK} ${str}"
@@ -397,7 +390,7 @@ gravity_Filter() {
   # Format $parsedMatter line total as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
   echo -e "${OVER}  ${TICK} ${str}
-  ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} domains being pulled in by gravity"
+  ${INFO} ${COL_BLUE}${num}${COL_NC} domains being pulled in by gravity"
 
   str="Removing duplicate domains"
   echo -ne "  ${INFO} ${str}..."
@@ -406,20 +399,21 @@ gravity_Filter() {
 
   # Format $preEventHorizon line total as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-  echo -e "  ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon"
+  echo -e "  ${INFO} ${COL_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon"
 }
 
 # Whitelist unique blocklist domain sources
 gravity_WhitelistBLD() {
-  local plural="" str uniqDomains
+  local uniqDomains plural="" str 
 
   echo ""
-  [[ "${#sources[@]}" -ne 1 ]] && plural="s"
-  str="Adding blocklist domain source${plural} to the whitelist"
-  echo -ne "  ${INFO} ${str}..."
 
   # Create array of unique $sourceDomains
   mapfile -t uniqDomains <<< "$(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")"
+  [[ "${#uniqDomains[@]}" -ne 1 ]] && plural="s"
+
+  str="Adding ${#uniqDomains[@]} blocklist source domain${plural} to the whitelist"
+  echo -ne "  ${INFO} ${str}..."
 
   # Whitelist $uniqDomains
   "${PIHOLE_COMMAND}" -w -nr -q "${uniqDomains[*]}" &> /dev/null
@@ -627,7 +621,7 @@ else
   # Gravity needs to modify Blacklist/Whitelist/Wildcards
   echo -e "  ${INFO} Using cached Event Horizon list..."
   numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-  echo -e "  ${INFO} ${COL_LIGHT_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
+  echo -e "  ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
 fi
 
 # Perform when downloading blocklists, or modifying the whitelist

From 2deb2bf03f452f824645abb55deb3252014cf0f1 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Fri, 22 Sep 2017 02:23:43 +1000
Subject: [PATCH 17/18] Fix broken whitelist functionality

---
 gravity.sh | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index cb717aba..881fc6f2 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -631,8 +631,8 @@ fi
 
 gravity_ShowBlockCount
 
-# Perform when downloading blocklists, or modifying the blacklist
-if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "blacklist" ]]; then
+# Perform when downloading blocklists, or modifying the white/blacklist (not wildcards)
+if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then
   str="Parsing domains into hosts format"
   echo -ne "  ${INFO} ${str}..."
 
@@ -645,10 +645,7 @@ if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "blacklist" ]]; then
   fi
 
   echo -e "${OVER}  ${TICK} ${str}"
-fi
 
-# Perform when downloading blocklists
-if [[ "${skipDownload}" == false ]]; then
   gravity_Cleanup
 fi
 

From 47099e2855884feb782479172803cf3e4baf4297 Mon Sep 17 00:00:00 2001
From: WaLLy3K <WaLLy3K@users.noreply.github.com>
Date: Fri, 22 Sep 2017 03:56:53 +1000
Subject: [PATCH 18/18] Ensure resolver test occurs each second

* Ensure that gravity will run the second the resolver is available
* Increase timeout to 120s
---
 gravity.sh | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 881fc6f2..6c3e9ec6 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -77,14 +77,14 @@ gravity_DNSLookup() {
   fi
 
   # Determine if $lookupDomain is resolvable
-  if timeout 5 getent hosts "${lookupDomain}" &> /dev/null; then
+  if timeout 1 getent hosts "${lookupDomain}" &> /dev/null; then
     # Print confirmation of resolvability if it had previously failed
     if [[ -n "${secs:-}" ]]; then
       echo -e "${OVER}  ${TICK} DNS resolution is now available\\n"
     fi
     return 0
   elif [[ -n "${secs:-}" ]]; then
-    echo -e "${OVER}  ${CROSS} DNS resolution is still unavailable, cancelling"
+    echo -e "${OVER}  ${CROSS} DNS resolution is not available"
     exit 1
   fi
 
@@ -96,13 +96,15 @@ gravity_DNSLookup() {
     "${PIHOLE_COMMAND}" restartdns
   fi
 
-  # Give time for DNS server to be resolvable
-  secs="30"
-  while [[ "${secs}" -ne 0 ]]; do
-    [[ "${secs}" -ne 1 ]] && plural="s"
-    echo -ne "${OVER}  ${INFO} Waiting $secs second${plural} before continuing..."
-    sleep 1
+  # Ensure DNS server is given time to be resolvable
+  secs="120"
+  echo -ne "  ${INFO} Waiting up to ${secs} seconds before continuing..."
+  until timeout 1 getent hosts "${lookupDomain}" &> /dev/null; do
+    [[ "${secs:-}" -eq 0 ]] && break
+    [[ "${secs:-}" -ne 1 ]] && plural="s"
+    echo -ne "${OVER}  ${INFO} Waiting up to ${secs} second${plural} before continuing..."
     : $((secs--))
+    sleep 1
   done
 
   # Try again