Use adlist instead of vw_adlist

Signed-off-by: Christian König <ckoenig@posteo.de>

.github/release.yml

@@ -1,7 +0,0 @@
-changelog:
-  exclude:
-    labels:
-      - internal
-    authors:
-      - dependabot
-      - github-actions

.github/workflows/stale.yml

@@ -1,25 +0,0 @@
-name: Mark stale issues
-
-on:
-  schedule:
-  - cron: '0 * * * *'
-  workflow_dispatch:
-
-jobs:
-  stale:
-
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-
-    steps:
-    - uses: actions/stale@v4
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        days-before-stale: 30
-        days-before-close: 5
-        stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.'
-        stale-issue-label: 'Submitter Attention Required'
-        exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed'
-        exempt-all-issue-assignees: true
-        operations-per-run: 300

.github/workflows/sync-back-to-dev.yml

@@ -1,28 +0,0 @@
-name: Sync Back to Development
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  sync-branches:
-    runs-on: ubuntu-latest
-    name: Syncing branches
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Opening pull request
-        id: pull
-        uses: tretuna/sync-branches@1.4.0
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          FROM_BRANCH: 'master'
-          TO_BRANCH: 'development'
-          CONTENT_COMPARISON: true
-      - name: Label the pull request to ignore for release note generation
-        uses: actions-ecosystem/action-add-labels@v1
-        with:
-          labels: internal
-          repo: ${{ github.repository }}
-          number: ${{ steps.pull.outputs.PULL_REQUEST_NUMBER }}

advanced/Scripts/chronometer.sh

@@ -357,7 +357,7 @@ get_sys_stats() {
     ram_used="${ram_raw[1]}"
     ram_total="${ram_raw[2]}"
 
-    if [[ "$(pihole status web 2> /dev/null)" -ge "1" ]]; then
+    if [[ "$(pihole status web 2> /dev/null)" == "1" ]]; then
         ph_status="${COL_LIGHT_GREEN}Active"
     else
         ph_status="${COL_LIGHT_RED}Offline"

advanced/Scripts/piholeDebug.sh

@@ -467,9 +467,6 @@ diagnose_operating_system() {
     # Display the current test that is running
     echo_current_diagnostic "Operating system"
 
-    # If the PIHOLE_DOCKER_TAG variable is set, include this information in the debug output
-    [ -n "${PIHOLE_DOCKER_TAG}" ] && log_write "${INFO} Pi-hole Docker Container: ${PIHOLE_DOCKER_TAG}"
-
     # If there is a /etc/*release file, it's probably a supported operating system, so we can
     if ls /etc/*release 1> /dev/null 2>&1; then
         # display the attributes to the user from the function made earlier
@@ -733,11 +730,11 @@ compare_port_to_service_assigned() {
 
     # If the service is a Pi-hole service, highlight it in green
     if [[ "${service_name}" == "${expected_service}" ]]; then
-        log_write "${TICK} ${COL_GREEN}${port}${COL_NC} is in use by ${COL_GREEN}${service_name}${COL_NC}"
+        log_write "[${COL_GREEN}${port}${COL_NC}] is in use by ${COL_GREEN}${service_name}${COL_NC}"
     # Otherwise,
     else
         # Show the service name in red since it's non-standard
-        log_write "${CROSS} ${COL_RED}${port}${COL_NC} is in use by ${COL_RED}${service_name}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_PORTS})"
+        log_write "[${COL_RED}${port}${COL_NC}] is in use by ${COL_RED}${service_name}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS_PORTS})"
     fi
 }
 
@@ -753,47 +750,36 @@ check_required_ports() {
     # Sort the addresses and remove duplicates
     while IFS= read -r line; do
         ports_in_use+=( "$line" )
-    done < <( ss --listening --numeric --tcp --udp --processes --no-header )
+    done < <( lsof -iTCP -sTCP:LISTEN -P -n +c 10 )
 
     # Now that we have the values stored,
     for i in "${!ports_in_use[@]}"; do
         # loop through them and assign some local variables
         local service_name
-        service_name=$(echo "${ports_in_use[$i]}" | awk '{gsub(/users:\(\("/,"",$7);gsub(/".*/,"",$7);print $7}')
+        service_name=$(echo "${ports_in_use[$i]}" | awk '{print $1}')
         local protocol_type
-        protocol_type=$(echo "${ports_in_use[$i]}" | awk '{print $1}')
+        protocol_type=$(echo "${ports_in_use[$i]}" | awk '{print $5}')
         local port_number
-        port_number="$(echo "${ports_in_use[$i]}" | awk '{print $5}')" #  | awk '{gsub(/^.*:/,"",$5);print $5}')
+        port_number="$(echo "${ports_in_use[$i]}" | awk '{print $9}')"
 
+        # Skip the line if it's the titles of the columns the lsof command produces
+        if [[ "${service_name}" == COMMAND ]]; then
+            continue
+        fi
         # Use a case statement to determine if the right services are using the right ports
-        case "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" in
-            53) compare_port_to_service_assigned  "${resolver}" "${service_name}" "${protocol_type}:${port_number}"
+        case "$(echo "$port_number" | rev | cut -d: -f1 | rev)" in
+            53) compare_port_to_service_assigned  "${resolver}" "${service_name}" 53
                 ;;
-            80) compare_port_to_service_assigned  "${web_server}" "${service_name}" "${protocol_type}:${port_number}"
+            80) compare_port_to_service_assigned  "${web_server}" "${service_name}" 80
                 ;;
-            4711) compare_port_to_service_assigned  "${ftl}" "${service_name}" "${protocol_type}:${port_number}"
+            4711) compare_port_to_service_assigned  "${ftl}" "${service_name}" 4711
                 ;;
             # If it's not a default port that Pi-hole needs, just print it out for the user to see
-            *) log_write "    ${protocol_type}:${port_number} is in use by ${service_name:=<unknown>}";
+            *) log_write "${port_number} ${service_name} (${protocol_type})";
         esac
     done
 }
 
-ip_command() {
-    # Obtain and log information from "ip XYZ show" commands
-    echo_current_diagnostic "${2}"
-    local entries=()
-    mapfile -t entries < <(ip "${1}" show)
-    for line in "${entries[@]}"; do
-        log_write "   ${line}"
-    done
-}
-
-check_ip_command() {
-    ip_command "addr" "Network interfaces and addresses"
-    ip_command "route" "Network routing table"
-}
-
 check_networking() {
     # Runs through several of the functions made earlier; we just clump them
     # together since they are all related to the networking aspect of things
@@ -802,9 +788,7 @@ check_networking() {
     detect_ip_addresses "6"
     ping_gateway "4"
     ping_gateway "6"
-    # Skip the following check if installed in docker container. Unpriv'ed containers do not have access to the information required
-    # to resolve the service name listening - and the container should not start if there was a port conflict anyway
-    [ -z "${PIHOLE_DOCKER_TAG}" ] && check_required_ports
+    check_required_ports
 }
 
 check_x_headers() {
@@ -1402,9 +1386,9 @@ upload_to_tricorder() {
     log_write "${TICK} ${COL_GREEN}** FINISHED DEBUGGING! **${COL_NC}\\n"
 
     # Provide information on what they should do with their token
-    log_write "   * The debug log can be uploaded to tricorder.pi-hole.net for sharing with developers only."
+    log_write "    * The debug log can be uploaded to tricorder.pi-hole.net for sharing with developers only."
 
-    # If pihole -d is running automatically
+    # If pihole -d is running automatically (usually through the dashboard)
     if [[ "${AUTOMATED}" ]]; then
         # let the user know
         log_write "${INFO} Debug script running in automated mode"
@@ -1412,19 +1396,16 @@ upload_to_tricorder() {
         curl_to_tricorder
         # If we're not running in automated mode,
     else
-        # if not being called from the web interface
-        if [[ ! "${WEBCALL}" ]]; then
-            echo ""
-            # give the user a choice of uploading it or not
-            # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem
-            read -r -p "[?] Would you like to upload the log? [y/N] " response
-            case ${response} in
-                # If they say yes, run our function for uploading the log
-                [yY][eE][sS]|[yY]) curl_to_tricorder;;
-                # If they choose no, just exit out of the script
-                *) log_write "    * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n    * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit;
-            esac
-        fi
+        echo ""
+        # give the user a choice of uploading it or not
+        # Users can review the log file locally (or the output of the script since they are the same) and try to self-diagnose their problem
+        read -r -p "[?] Would you like to upload the log? [y/N] " response
+        case ${response} in
+            # If they say yes, run our function for uploading the log
+            [yY][eE][sS]|[yY]) curl_to_tricorder;;
+            # If they choose no, just exit out of the script
+            *) log_write "    * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n    * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit;
+        esac
     fi
     # Check if tricorder.pi-hole.net is reachable and provide token
     # along with some additional useful information
@@ -1444,13 +1425,8 @@ upload_to_tricorder() {
     # If no token was generated
     else
         # Show an error and some help instructions
-        # Skip this if being called from web interface and autmatic mode was not chosen (users opt-out to upload)
-        if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then
-            :
-        else
-            log_write "${CROSS}  ${COL_RED}There was an error uploading your debug log.${COL_NC}"
-            log_write "   * Please try again or contact the Pi-hole team for assistance."
-        fi
+        log_write "${CROSS}  ${COL_RED}There was an error uploading your debug log.${COL_NC}"
+        log_write "   * Please try again or contact the Pi-hole team for assistance."
     fi
     # Finally, show where the log file is no matter the outcome of the function so users can look at it
     log_write "   * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n"
@@ -1469,7 +1445,6 @@ check_selinux
 check_firewalld
 processor_check
 disk_usage
-check_ip_command
 check_networking
 check_name_resolution
 check_dhcp_servers

advanced/Scripts/query.sh

@@ -233,7 +233,7 @@ for result in "${results[@]}"; do
     adlistAddress="${extra/|*/}"
     extra="${extra#*|}"
     if [[ "${extra}" == "0" ]]; then
-        extra=" (disabled)"
+        extra="(disabled)"
     else
         extra=""
     fi
@@ -241,7 +241,7 @@ for result in "${results[@]}"; do
     if [[ -n "${blockpage}" ]]; then
         echo "0 ${adlistAddress}"
     elif [[ -n "${exact}" ]]; then
-        echo "  - ${adlistAddress}${extra}"
+        echo "  - ${adlistAddress} ${extra}"
     else
         if [[ ! "${adlistAddress}" == "${adlistAddress_prev:-}" ]]; then
             count=""
@@ -256,7 +256,7 @@ for result in "${results[@]}"; do
             [[ "${count}" -gt "${max_count}" ]] && continue
             echo "   ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}"
         else
-            echo "   ${match}${extra}"
+            echo "   ${match} ${extra}"
         fi
     fi
 done

advanced/Templates/pihole-FTL.conf

@@ -1,2 +0,0 @@
-#; Pi-hole FTL config file
-#; Comments should start with #; to avoid issues with PHP and bash reading this file

automated install/basic-install.sh

@@ -287,7 +287,7 @@ package_manager_detect() {
         # Packages required to run this install script (stored as an array)
         INSTALLER_DEPS=(git iproute2 whiptail ca-certificates)
         # Packages required to run Pi-hole (stored as an array)
-        PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2 netcat-openbsd)
+        PIHOLE_DEPS=(cron curl iputils-ping lsof psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2 netcat)
         # Packages required for the Web admin interface (stored as an array)
         # It's useful to separate this from Pi-hole, since the two repos are also setup separately
         PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl")
@@ -332,7 +332,7 @@ package_manager_detect() {
         PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l"
         OS_CHECK_DEPS=(grep bind-utils)
         INSTALLER_DEPS=(git iproute newt procps-ng which chkconfig ca-certificates)
-        PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap nmap-ncat)
+        PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap lsof nmap-ncat)
         PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl)
         LIGHTTPD_USER="lighttpd"
         LIGHTTPD_GROUP="lighttpd"
@@ -1299,10 +1299,10 @@ installConfigs() {
     echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
     chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf"
 
-    # Install template file if it does not exist
+    # Install empty file if it does not exist
     if [[ ! -r "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" ]]; then
         install -d -m 0755 ${PI_HOLE_CONFIG_DIR}
-        if ! install -T -o pihole -m 664 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL.conf" "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" &>/dev/null; then
+        if ! install -o pihole -m 664 /dev/null "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" &>/dev/null; then
             printf "  %bError: Unable to initialize configuration file %s/pihole-FTL.conf\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}"
             return 1
         fi
@@ -1731,7 +1731,7 @@ finalExports() {
     # If the setup variable file exists,
     if [[ -e "${setupVars}" ]]; then
         # update the variables in the file
-        sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1\b/d;/PIHOLE_DNS_2\b/d;/QUERY_LOGGING/d;/INSTALL_WEB_SERVER/d;/INSTALL_WEB_INTERFACE/d;/LIGHTTPD_ENABLED/d;/CACHE_SIZE/d;/DNS_FQDN_REQUIRED/d;/DNS_BOGUS_PRIV/d;/DNSMASQ_LISTENING/d;' "${setupVars}"
+        sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1\b/d;/PIHOLE_DNS_2\b/d;/QUERY_LOGGING/d;/INSTALL_WEB_SERVER/d;/INSTALL_WEB_INTERFACE/d;/LIGHTTPD_ENABLED/d;/CACHE_SIZE/d;/DNS_FQDN_REQUIRED/d;/DNS_BOGUS_PRIV/d;' "${setupVars}"
     fi
     # echo the information to the user
     {
@@ -1747,7 +1747,6 @@ finalExports() {
         echo "CACHE_SIZE=${CACHE_SIZE}"
         echo "DNS_FQDN_REQUIRED=${DNS_FQDN_REQUIRED:-true}"
         echo "DNS_BOGUS_PRIV=${DNS_BOGUS_PRIV:-true}"
-        echo "DNSMASQ_LISTENING=${DNSMASQ_LISTENING:-local}"
     }>> "${setupVars}"
     chmod 644 "${setupVars}"
 

gravity.sh

@@ -85,19 +85,27 @@ generate_gravity_database() {
 
 # Copy data from old to new database file and swap them
 gravity_swap_databases() {
-  local str copyGravity
-  str="Building tree"
-  echo -ne "  ${INFO} ${str}..."
+  local str copyGravity enabled_adlists
 
-  # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once
-  output=$( { sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 )
-  status="$?"
+  # Only build an index if there are enabled adlists
+  enabled_adlists="$(sqlite3 "${gravityTEMPfile}" "Select Count(id) from adlist where enabled=1;")"
+  if [[ enabled_adlists -ge 1 ]]; then
+    str="Building tree"
+    echo -ne "  ${INFO} ${str}..."
 
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to build gravity tree in ${gravityTEMPfile}\\n  ${output}"
-    return 1
+    # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once
+    output=$( { sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 )
+    status="$?"
+
+    if [[ "${status}" -ne 0 ]]; then
+        echo -e "\\n  ${CROSS} Unable to build gravity tree in ${gravityTEMPfile}\\n  ${output}"
+        return 1
+    fi
+    echo -e "${OVER}  ${TICK} ${str}"
+  else
+      str="No enabled adlists. Skipping building tree"
+      echo -ne "  ${INFO} ${str}...\\n"
   fi
-  echo -e "${OVER}  ${TICK} ${str}"
 
   str="Swapping databases"
   echo -ne "  ${INFO} ${str}..."
@@ -266,12 +274,9 @@ database_adlist_number() {
 database_adlist_status() {
   # Only try to set the status when this field exists in the gravity database
   if ! gravity_column_exists "adlist" "status"; then
-    echo "Gravity status column is not found!"
     return;
   fi
 
-  echo "Gravity status column found and set to ${2} (ID ${1})"
-
   output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
   status="$?"
 
@@ -405,12 +410,14 @@ gravity_DownloadBlocklists() {
   )"
 
   local str="Pulling blocklist source list into range"
-  echo -e "${OVER}  ${TICK} ${str}"
 
-  if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then
+  if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then
+    echo -e "${OVER}  ${TICK} ${str}"
+  else
+    echo -e "${OVER}  ${CROSS} ${str}"
     echo -e "  ${INFO} No source list found, or it is empty"
     echo ""
-    unset sources
+    return 1
   fi
 
   local url domain agent cmd_ext str target compression

pihole

@@ -223,7 +223,8 @@ Time:
       fi
 
       local str="Pi-hole Disabled"
-      sed -i "s/^BLOCKING_ENABLED=true/BLOCKING_ENABLED=false/" "${setupVars}"
+      sed -i "/BLOCKING_ENABLED=/d" "${setupVars}"
+      echo "BLOCKING_ENABLED=false" >> "${setupVars}"
     fi
   else
     # Enable Pi-hole
@@ -235,7 +236,8 @@ Time:
     echo -e "  ${INFO} Enabling blocking"
     local str="Pi-hole Enabled"
 
-    sed -i "s/^BLOCKING_ENABLED=false/BLOCKING_ENABLED=true/" "${setupVars}"
+    sed -i "/BLOCKING_ENABLED=/d" "${setupVars}"
+    echo "BLOCKING_ENABLED=true" >> "${setupVars}"
   fi
 
   restartDNS reload-lists
@@ -281,29 +283,26 @@ Options:
 }
 
 analyze_ports() {
-  local lv4 lv6 port=${1}
   # FTL is listening at least on at least one port when this
   # function is getting called
   # Check individual address family/protocol combinations
   # For a healthy Pi-hole, they should all be up (nothing printed)
-  lv4="$(ss --ipv4 --listening --numeric --tcp --udp src :${port})"
-  if grep -q "udp " <<< "${lv4}"; then
+  if grep -q "IPv4.*UDP" <<< "${1}"; then
       echo -e "     ${TICK} UDP (IPv4)"
   else
       echo -e "     ${CROSS} UDP (IPv4)"
   fi
-  if grep -q "tcp " <<< "${lv4}"; then
+  if grep -q "IPv4.*TCP" <<< "${1}"; then
       echo -e "     ${TICK} TCP (IPv4)"
   else
       echo -e "     ${CROSS} TCP (IPv4)"
   fi
-  lv6="$(ss --ipv6 --listening --numeric --tcp --udp src :${port})"
-  if grep -q "udp " <<< "${lv6}"; then
+  if grep -q "IPv6.*UDP" <<< "${1}"; then
       echo -e "     ${TICK} UDP (IPv6)"
   else
       echo -e "     ${CROSS} UDP (IPv6)"
   fi
-  if grep -q "tcp " <<< "${lv6}"; then
+  if grep -q "IPv6.*TCP" <<< "${1}"; then
       echo -e "     ${TICK} TCP (IPv6)"
   else
       echo -e "     ${CROSS} TCP (IPv6)"
@@ -325,6 +324,7 @@ statusFunc() {
   else
     #get the port pihole-FTL is listening on by using FTL's telnet API
     port="$(echo ">dns-port >quit" | nc 127.0.0.1 4711)"
+    listening="$(lsof -Pni:${port})"
     if [[ "${port}" == "0" ]]; then
       case "${1}" in
         "web") echo "-1";;
@@ -334,7 +334,7 @@ statusFunc() {
     else
       if [[ "${1}" != "web" ]]; then
         echo -e "  ${TICK} FTL is listening on port ${port}"
-        analyze_ports "${port}"
+        analyze_ports "${listening}"
       fi
     fi
   fi