Use compression (if available) when downloading the ad lists.

Signed-off-by: DL6ER <dl6er@dl6er.de>

advanced/Scripts/database_migration/gravity-db.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1090
+
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2019 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# Updates gravity.db database
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+upgrade_gravityDB(){
+	local database piholeDir auditFile version
+	database="${1}"
+	piholeDir="${2}"
+	auditFile="${piholeDir}/auditlog.list"
+
+	# Get database version
+	version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
+
+	if [[ "$version" == "1" ]]; then
+		# This migration script upgrades the gravity.db file by
+		# adding the domain_audit table
+		sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
+		version=2
+
+		# Store audit domains in database table
+		if [ -e "${auditFile}" ]; then
+			echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
+			# database_table_from_file is defined in gravity.sh
+			database_table_from_file "domain_audit" "${auditFile}"
+		fi
+	fi
+}

advanced/Scripts/database_migration/gravity/1_to_2.sql

@@ -0,0 +1,10 @@
+.timeout 30000
+
+CREATE TABLE domain_audit
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
+);
+
+UPDATE info SET value = 2 WHERE property = 'version';

advanced/Scripts/piholeDebug.sh

@@ -94,7 +94,35 @@ PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*"
 PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list"
 PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate"
 PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf"
-PIHOLE_GRAVITY_DB_FILE="${PIHOLE_DIRECTORY}/gravity.db"
+PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole-FTL.conf"
+
+# Read the value of an FTL config key. The value is printed to stdout.
+#
+# Args:
+# 1. The key to read
+# 2. The default if the setting or config does not exist
+get_ftl_conf_value() {
+    local key=$1
+    local default=$2
+    local value
+
+    # Obtain key=... setting from pihole-FTL.conf
+    if [[ -e "$PIHOLE_FTL_CONF_FILE" ]]; then
+        # Constructed to return nothing when
+        # a) the setting is not present in the config file, or
+        # b) the setting is commented out (e.g. "#DBFILE=...")
+        value="$(sed -n -e "s/^\\s*$key=\\s*//p" ${PIHOLE_FTL_CONF_FILE})"
+    fi
+
+    # Test for missing value. Use default value in this case.
+    if [[ -z "$value" ]]; then
+        value="$default"
+    fi
+
+    echo "$value"
+}
+
+PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "GRAVITYDB" "${PIHOLE_DIRECTORY}/gravity.db")"
 
 PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole"
 PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE"
@@ -105,7 +133,7 @@ FTL_PORT="${RUN_DIRECTORY}/pihole-FTL.port"
 PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log"
 PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*"
 PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log"
-PIHOLE_FTL_LOG="${LOG_DIRECTORY}/pihole-FTL.log"
+PIHOLE_FTL_LOG="$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/pihole-FTL.log")"
 
 PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access.log"
 PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error.log"

advanced/Scripts/query.sh

@@ -115,7 +115,9 @@ scanDatabaseTable() {
     wbMatch=true
 
     # Print table name
-    echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
+    if [[ -z "${blockpage}" ]]; then
+        echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
+    fi
 
     # Loop over results and print them
     mapfile -t results <<< "${result}"
@@ -159,7 +161,7 @@ if [[ "${#regexList[@]}" -ne 0 ]]; then
 			# shellcheck disable=SC2001
 			echo "${str_result}" | sed 's/^/   /'
 		else
-			echo "π Regex list"
+			echo "π .wildcard"
 			exit 0
 		fi
 	fi

advanced/Scripts/webpage.sh

@@ -542,21 +542,45 @@ Teleporter() {
     php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.tar.gz"
 }
 
+checkDomain()
+{
+    local domain validDomain
+    # Convert to lowercase
+    domain="${1,,}"
+    validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
+    validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label
+    echo "${validDomain}"
+}
+
 addAudit()
 {
     shift # skip "-a"
     shift # skip "audit"
-    for var in "$@"
+    local domains validDomain
+    domains=""
+    for domain in "$@"
     do
-        echo "${var}" >> /etc/pihole/auditlog.list
+      # Check domain to be added. Only continue if it is valid
+      validDomain="$(checkDomain "${domain}")"
+      if [[ -n "${validDomain}" ]]; then
+        # Put comma in between domains when there is
+        # more than one domains to be added
+        # SQL INSERT allows adding multiple rows at once using the format
+        ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st');
+        if [[ -n "${domains}" ]]; then
+          domains="${domains},"
+        fi
+        domains="${domains}('${domain}')"
+      fi
     done
-    chmod 644 /etc/pihole/auditlog.list
+    # Insert only the domain here. The date_added field will be
+    # filled with its default value (date_added = current timestamp)
+    sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
 }
 
 clearAudit()
 {
-    echo -n "" > /etc/pihole/auditlog.list
-    chmod 644 /etc/pihole/auditlog.list
+    sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;"
 }
 
 SetPrivacyLevel() {

advanced/Templates/gravity.db.sql

@@ -80,6 +80,7 @@ CREATE TABLE gravity
 (
 	domain TEXT PRIMARY KEY
 );
+
 CREATE TABLE info
 (
 	property TEXT PRIMARY KEY,

advanced/index.php

@@ -102,20 +102,30 @@ if ($blocklistglob === array()) {
     die("[ERROR] There are no domain lists generated lists within <code>/etc/pihole/</code>! Please update gravity by running <code>pihole -g</code>, or repair Pi-hole using <code>pihole -r</code>.");
 }
 
-// Set location of adlists file
-if (is_file("/etc/pihole/adlists.list")) {
-    $adLists = "/etc/pihole/adlists.list";
-} elseif (is_file("/etc/pihole/adlists.default")) {
-    $adLists = "/etc/pihole/adlists.default";
+// Get possible non-standard location of FTL's database
+$FTLsettings = parse_ini_file("/etc/pihole/pihole-FTL.conf");
+if (isset($FTLsettings["GRAVITYDB"])) {
+    $gravityDBFile = $FTLsettings["GRAVITYDB"];
 } else {
-    die("[ERROR] File not found: <code>/etc/pihole/adlists.list</code>");
+    $gravityDBFile = "/etc/pihole/gravity.db";
 }
 
-// Get all URLs starting with "http" or "www" from adlists and re-index array numerically
-$adlistsUrls = array_values(preg_grep("/(^http)|(^www)/i", file($adLists, FILE_IGNORE_NEW_LINES)));
+// Connect to gravity.db
+try {
+    $db = new SQLite3($gravityDBFile, SQLITE3_OPEN_READONLY);
+} catch (Exception $exception) {
+    die("[ERROR]: Failed to connect to gravity.db");
+}
+
+// Get all adlist addresses
+$adlistResults = $db->query("SELECT address FROM vw_adlist");
+$adlistsUrls = array();
+while ($row = $adlistResults->fetchArray()) {
+    array_push($adlistsUrls, $row[0]);
+}
 
 if (empty($adlistsUrls))
-    die("[ERROR]: There are no adlist URL's found within <code>$adLists</code>");
+    die("[ERROR]: There are no adlists enabled");
 
 // Get total number of blocklists (Including Whitelist, Blacklist & Wildcard lists)
 $adlistsCount = count($adlistsUrls) + 3;

automated install/basic-install.sh

@@ -934,7 +934,7 @@ valid_ip() {
         # and set the new one to a dot (period)
         IFS='.'
         # Put the IP into an array
-        ip=("${ip}")
+        read -r -a ip <<< "${ip}"
         # Restore the IFS to what it was
         IFS=${OIFS}
         ## Evaluate each octet by checking if it's less than or equal to 255 (the max for each octet)
@@ -1177,12 +1177,11 @@ chooseBlocklists() {
         mv "${adlistFile}" "${adlistFile}.old"
     fi
     # Let user select (or not) blocklists via a checklist
-    cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 7)
+    cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 6)
     # In an array, show the options available (all off by default):
     options=(StevenBlack "StevenBlack's Unified Hosts List" on
         MalwareDom "MalwareDomains" on
         Cameleon "Cameleon" on
-        ZeusTracker "ZeusTracker" on
         DisconTrack "Disconnect.me Tracking" on
         DisconAd "Disconnect.me Ads" on
         HostsFile "Hosts-file.net Ads" on)
@@ -1205,7 +1204,6 @@ appendToListsFile() {
         StevenBlack  )  echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";;
         MalwareDom   )  echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";;
         Cameleon     )  echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";;
-        ZeusTracker  )  echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";;
         DisconTrack  )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";;
         DisconAd     )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";;
         HostsFile    )  echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";;
@@ -1223,7 +1221,6 @@ installDefaultBlocklists() {
     appendToListsFile StevenBlack
     appendToListsFile MalwareDom
     appendToListsFile Cameleon
-    appendToListsFile ZeusTracker
     appendToListsFile DisconTrack
     appendToListsFile DisconAd
     appendToListsFile HostsFile
@@ -2399,8 +2396,16 @@ FTLcheckUpdate() {
         if [[ ${ftlLoc} ]]; then
             local FTLversion
             FTLversion=$(/usr/bin/pihole-FTL tag)
+            local FTLreleaseData
             local FTLlatesttag
-            FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep 'Location' | awk -F '/' '{print $NF}' | tr -d '\r\n')
+
+            if ! FTLreleaseData=$(curl -sI https://github.com/pi-hole/FTL/releases/latest); then
+                # There was an issue while retrieving the latest version
+                printf "  %b Failed to retrieve latest FTL release metadata" "${CROSS}"
+                return 3
+            fi
+
+            FTLlatesttag=$(grep 'Location' < "${FTLreleaseData}" | awk -F '/' '{print $NF}' | tr -d '\r\n')
 
             if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then
                 return 0

gravity.sh

@@ -17,6 +17,8 @@ coltable="/opt/pihole/COL_TABLE"
 source "${coltable}"
 regexconverter="/opt/pihole/wildcard_regex_converter.sh"
 source "${regexconverter}"
+# shellcheck disable=SC1091
+source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh"
 
 basename="pihole"
 PIHOLE_COMMAND="/usr/local/bin/${basename}"
@@ -112,15 +114,21 @@ database_table_from_file() {
     inputfile="${source}"
   else
     # Apply format for white-, blacklist, regex, and adlist tables
+    # Read file line by line
     local rowid
     declare -i rowid
     rowid=1
-    # Read file line by line
     grep -v '^ *#' < "${source}" | while IFS= read -r domain
     do
       # Only add non-empty lines
-      if [[ ! -z "${domain}" ]]; then
-        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
+      if [[ -n "${domain}" ]]; then
+        if [[ "${table}" == "domain_audit" ]]; then
+          # domain_audit table format (no enable or modified fields)
+          echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
+        else
+          # White-, black-, and regexlist format
+          echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
+        fi
         rowid+=1
       fi
     done
@@ -150,34 +158,36 @@ database_table_from_file() {
 # Migrate pre-v5.0 list files to database-based Pi-hole versions
 migrate_to_database() {
   # Create database file only if not present
-  if [ -e "${gravityDBfile}" ]; then
-    return 0
+  if [ ! -e "${gravityDBfile}" ]; then
+    # Create new database file - note that this will be created in version 1
+    echo -e "  ${INFO} Creating new gravity database"
+    generate_gravity_database
+
+    # Migrate list files to new database
+    if [ -e "${adListFile}" ]; then
+      # Store adlist domains in database
+      echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
+      database_table_from_file "adlist" "${adListFile}"
+    fi
+    if [ -e "${blacklistFile}" ]; then
+      # Store blacklisted domains in database
+      echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
+      database_table_from_file "blacklist" "${blacklistFile}"
+    fi
+    if [ -e "${whitelistFile}" ]; then
+      # Store whitelisted domains in database
+      echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
+      database_table_from_file "whitelist" "${whitelistFile}"
+    fi
+    if [ -e "${regexFile}" ]; then
+      # Store regex domains in database
+      echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
+      database_table_from_file "regex" "${regexFile}"
+    fi
   fi
 
-  echo -e "  ${INFO} Creating new gravity database"
-  generate_gravity_database
-
-  # Migrate list files to new database
-  if [[ -e "${adListFile}" ]]; then
-    # Store adlist domains in database
-    echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
-    database_table_from_file "adlist" "${adListFile}"
-  fi
-  if [[ -e "${blacklistFile}" ]]; then
-    # Store blacklisted domains in database
-    echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
-    database_table_from_file "blacklist" "${blacklistFile}"
-  fi
-  if [[ -e "${whitelistFile}" ]]; then
-    # Store whitelisted domains in database
-    echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
-    database_table_from_file "whitelist" "${whitelistFile}"
-  fi
-  if [[ -e "${regexFile}" ]]; then
-    # Store regex domains in database
-    echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
-    database_table_from_file "regex" "${regexFile}"
-  fi
+  # Check if gravity database needs to be updated
+  upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"
 }
 
 # Determine if DNS resolution is available before proceeding
@@ -301,7 +311,7 @@ gravity_SetDownloadOptions() {
 
 # Download specified URL and perform checks on HTTP status and file content
 gravity_DownloadBlocklistFromUrl() {
-  local url="${1}" cmd_ext="${2}" agent="${3}" heisenbergCompensator="" patternBuffer str httpCode success=""
+  local url="${1}" cmd_ext="${2}" agent="${3}" heisenbergCompensator="" patternBuffer str httpCode success="" compression
 
   # Create temp file to store content on disk instead of RAM
   patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb")
@@ -349,8 +359,18 @@ gravity_DownloadBlocklistFromUrl() {
     echo -ne "  ${INFO} ${str} Pending..."
     cmd_ext="--resolve $domain:$port:$ip $cmd_ext"
   fi
+
+  # Use compression to reduce the amount of data that is transfered
+  # between the Pi-hole and the ad list provider. Use this feature
+  # only if it is supported by the locally available version of curl
+  if curl -V | grep -q "Features:.* libz"; then
+    compression="--compressed"
+  else
+    compression=""
+  fi
+
   # shellcheck disable=SC2086
-  httpCode=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null)
+  httpCode=$(curl -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null)
 
   case $url in
     # Did we "download" a local file?

test/test_automated_install.py

@@ -700,3 +700,42 @@ def test_IPv6_ULA_GUA_test(Pihole):
     ''')
     expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads'
     assert expected_stdout in detectPlatform.stdout
+
+
+def test_validate_ip_valid(Pihole):
+    '''
+    Given a valid IP address, valid_ip returns success
+    '''
+
+    output = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    valid_ip "192.168.1.1"
+    ''')
+
+    assert output.rc == 0
+
+
+def test_validate_ip_invalid_octet(Pihole):
+    '''
+    Given an invalid IP address (large octet), valid_ip returns an error
+    '''
+
+    output = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    valid_ip "1092.168.1.1"
+    ''')
+
+    assert output.rc == 1
+
+
+def test_validate_ip_invalid_letters(Pihole):
+    '''
+    Given an invalid IP address (contains letters), valid_ip returns an error
+    '''
+
+    output = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    valid_ip "not an IP"
+    ''')
+
+    assert output.rc == 1