Optimize gravity database before asking FTL to reload to avoid race-collisions on slow(er) devices

Signed-off-by: DL6ER <dl6er@dl6er.de>

.github/ISSUE_TEMPLATE.md

@@ -1,37 +0,0 @@
-**In raising this issue, I confirm the following:** `{please fill the checkboxes, e.g: [X]}`
-
-- [] I have read and understood the [contributors guide](https://github.com/pi-hole/pi-hole/blob/master/CONTRIBUTING.md).
-- [] The issue I am reporting can be *replicated*.
-- [] The issue I am reporting isn't a duplicate (see [FAQs](https://github.com/pi-hole/pi-hole/wiki/FAQs), [closed issues](https://github.com/pi-hole/pi-hole/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), and [open issues](https://github.com/pi-hole/pi-hole/issues)).
-
-**How familiar are you with the the source code relevant to this issue?:**
-
-`{Replace this with a number from 1 to 10. 1 being not familiar, and 10 being very familiar}`
-
----
-**Expected behavior:**
-
-`{A detailed description of what you expect to see}`
-
-**Actual behavior:**
-
-`{A detailed description and/or screenshots of what you do see}`
-
-**Steps to reproduce:**
-
-`{Detailed steps of how we can reproduce this}`
-
-**Debug token provided by [uploading `pihole -d` log](https://discourse.pi-hole.net/t/the-pihole-command-with-examples/738#debug):**
-
-`{Alphanumeric token}`
-
-**Troubleshooting undertaken, and/or other relevant information:**
-
-`{Steps of what you have done to fix this}`
-
-> * `{Please delete this quoted section when opening your issue}`
-> * You must follow the template instructions. Failure to do so will result in your issue being closed.
-> * Please [submit any feature requests here](https://discourse.pi-hole.net/c/feature-requests), so it is votable and trackable by the community.
-> * Please respect that Pi-hole is developed by volunteers, who can only reply in their spare time.
-> * Detail helps us understand and resolve an issue quicker, but please ensure it's relevant.
-> * _This template was created based on the work of [`udemy-dl`](https://github.com/nishad/udemy-dl/blob/master/LICENSE)._

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,31 +0,0 @@
-**By submitting this pull request, I confirm the following:** 
-*please fill any appropriate checkboxes, e.g: [X]*
-
-- [ ] I have read and understood the [contributors guide](https://github.com/pi-hole/pi-hole/blob/master/CONTRIBUTING.md), as well as this entire template.
-- [ ] I have made only one major change in my proposed changes.
-- [ ] I have commented my proposed changes within the code.
-- [ ] I have tested my proposed changes, and have included unit tests where possible.
-- [ ] I am willing to help maintain this change if there are issues with it later.
-- [ ] I give this submission freely and claim no ownership.
-- [ ] It is compatible with the [EUPL 1.2 license](https://opensource.org/licenses/EUPL-1.1)
-- [ ] I have squashed any insignificant commits. ([`git rebase`](http://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html))
-
-Please make sure you [Sign Off](https://docs.pi-hole.net/guides/github/how-to-signoff/) all commits. Pi-hole enforces the [DCO](https://docs.pi-hole.net/guides/github/contributing/).
-
----
-**What does this PR aim to accomplish?:**
-*A detailed description, screenshots (if necessary), as well as links to any relevant GitHub issues*
-
-
-**How does this PR accomplish the above?:**
-*A detailed description (such as a changelog) and screenshots (if necessary) of the implemented fix*
-
-
-**What documentation changes (if any) are needed to support this PR?:**
-*A detailed list of any necessary changes*
-
-
----
-* You must follow the template instructions. Failure to do so will result in your pull request being closed.
-* Please respect that Pi-hole is developed by volunteers, who can only reply in their spare time.
-

.travis.yml

@@ -1,12 +1,5 @@
-sudo: required
-services:
-  - docker
-language: python
-python:
-  - "3.6"
-install:
-  - pip install -r requirements.txt
-
-script:
-  # tox.ini handles setup, ordering of docker build first, and then run tests
-  - tox
+import:
+  - source: pi-hole/.github:/build-configs/core.yml@main
+    if: branch = master
+  - source: pi-hole/.github:/build-configs/core.yml@latest
+    if: branch != master

advanced/Scripts/piholeDebug.sh

@@ -46,8 +46,8 @@ OBFUSCATED_PLACEHOLDER="<DOMAIN OBFUSCATED>"
 # FAQ URLs for use in showing the debug log
 FAQ_UPDATE_PI_HOLE="${COL_CYAN}https://discourse.pi-hole.net/t/how-do-i-update-pi-hole/249${COL_NC}"
 FAQ_CHECKOUT_COMMAND="${COL_CYAN}https://discourse.pi-hole.net/t/the-pihole-command-with-examples/738#checkout${COL_NC}"
-FAQ_HARDWARE_REQUIREMENTS="${COL_CYAN}https://discourse.pi-hole.net/t/hardware-software-requirements/273${COL_NC}"
-FAQ_HARDWARE_REQUIREMENTS_PORTS="${COL_CYAN}https://discourse.pi-hole.net/t/hardware-software-requirements/273#ports${COL_NC}"
+FAQ_HARDWARE_REQUIREMENTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/${COL_NC}"
+FAQ_HARDWARE_REQUIREMENTS_PORTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#ports${COL_NC}"
 FAQ_GATEWAY="${COL_CYAN}https://discourse.pi-hole.net/t/why-is-a-default-gateway-important-for-pi-hole/3546${COL_NC}"
 FAQ_ULA="${COL_CYAN}https://discourse.pi-hole.net/t/use-ipv6-ula-addresses-for-pi-hole/2127${COL_NC}"
 FAQ_FTL_COMPATIBILITY="${COL_CYAN}https://github.com/pi-hole/FTL#compatibility-list${COL_NC}"
@@ -1264,7 +1264,7 @@ upload_to_tricorder() {
             # If they choose no, just exit out of the script
             *) log_write "    * Log will ${COL_GREEN}NOT${COL_NC} be uploaded to tricorder.\\n    * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n";exit;
         esac
-    fi232623
+    fi
     # Check if tricorder.pi-hole.net is reachable and provide token
     # along with some additional useful information
     if [[ -n "${tricorder_token}" ]]; then

advanced/Templates/pihole-FTL.service

@@ -12,19 +12,8 @@
 FTLUSER=pihole
 PIDFILE=/run/pihole-FTL.pid
 
-get_pid() {
-    # First, try to obtain PID from PIDFILE
-    if [ -s "${PIDFILE}" ]; then
-        cat "${PIDFILE}"
-        return
-    fi
-
-    # If the PIDFILE is empty or not available, obtain the PID using pidof
-    pidof "pihole-FTL" | awk '{print $(NF)}'
-}
-
 is_running() {
-    ps "$(get_pid)" > /dev/null 2>&1
+    pgrep -o "pihole-FTL" > /dev/null 2>&1
 }
 
 
@@ -63,7 +52,7 @@ start() {
 # Stop the service
 stop() {
   if is_running; then
-    kill "$(get_pid)"
+    pkill -o pihole-FTL
     for i in {1..5}; do
       if ! is_running; then
         break
@@ -76,7 +65,7 @@ stop() {
 
     if is_running; then
       echo "Not stopped; may still be shutting down or shutdown may have failed, killing now"
-      kill -9 "$(get_pid)"
+      pkill -o -9 pihole-FTL
       exit 1
     else
       echo "Stopped"

advanced/Templates/pihole.cron

@@ -18,19 +18,19 @@
 #          early morning. Download any updates from the adlists
 #          Squash output to log, then splat the log to stdout on error to allow for
 #          standard crontab job error handling.
-59 1    * * 7   root    PATH="$PATH:/usr/local/bin/" pihole updateGravity >/var/log/pihole_updateGravity.log || cat /var/log/pihole_updateGravity.log
+59 1    * * 7   root    PATH="$PATH:/usr/sbin:/usr/local/bin/" pihole updateGravity >/var/log/pihole_updateGravity.log || cat /var/log/pihole_updateGravity.log
 
 # Pi-hole: Flush the log daily at 00:00
 #          The flush script will use logrotate if available
 #          parameter "once": logrotate only once (default is twice)
 #          parameter "quiet": don't print messages
-00 00   * * *   root    PATH="$PATH:/usr/local/bin/" pihole flush once quiet
+00 00   * * *   root    PATH="$PATH:/usr/sbin:/usr/local/bin/" pihole flush once quiet
 
 @reboot root /usr/sbin/logrotate /etc/pihole/logrotate
 
 # Pi-hole: Grab local version and branch every 10 minutes
-*/10 *  * * *   root    PATH="$PATH:/usr/local/bin/" pihole updatechecker local
+*/10 *  * * *   root    PATH="$PATH:/usr/sbin:/usr/local/bin/" pihole updatechecker local
 
 # Pi-hole: Grab remote version every 24 hours
-59 17  * * *   root    PATH="$PATH:/usr/local/bin/" pihole updatechecker remote
-@reboot root    PATH="$PATH:/usr/local/bin/" pihole updatechecker remote reboot
+59 17  * * *   root    PATH="$PATH:/usr/sbin:/usr/local/bin/" pihole updatechecker remote
+@reboot root    PATH="$PATH:/usr/sbin:/usr/local/bin/" pihole updatechecker remote reboot

automated install/basic-install.sh

@@ -72,8 +72,8 @@ useUpdateVars=false
 adlistFile="/etc/pihole/adlists.list"
 # Pi-hole needs an IP address; to begin, these variables are empty since we don't know what the IP is until
 # this script can run
-IPV4_ADDRESS=""
-IPV6_ADDRESS=""
+IPV4_ADDRESS=${IPV4_ADDRESS}
+IPV6_ADDRESS=${IPV6_ADDRESS}
 # By default, query logging is enabled and the dashboard is set to be installed
 QUERY_LOGGING=true
 INSTALL_WEB_INTERFACE=true
@@ -175,57 +175,69 @@ is_command() {
 }
 
 os_check() {
-    # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net
-    # and determines whether or not the script is running on one of those systems
-    local remote_os_domain valid_os valid_version detected_os_pretty detected_os detected_version display_warning
-    remote_os_domain="versions.pi-hole.net"
-    valid_os=false
-    valid_version=false
-    display_warning=true
+    if [ "$PIHOLE_SKIP_OS_CHECK" != true ]; then
+        # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net
+        # and determines whether or not the script is running on one of those systems
+        local remote_os_domain valid_os valid_version detected_os_pretty detected_os detected_version display_warning
+        remote_os_domain="versions.pi-hole.net"
+        valid_os=false
+        valid_version=false
+        display_warning=true
 
-    detected_os_pretty=$(cat /etc/*release | grep PRETTY_NAME | cut -d '=' -f2- | tr -d '"')
-    detected_os="${detected_os_pretty%% *}"
-    detected_version=$(cat /etc/*release | grep VERSION_ID | cut -d '=' -f2- | tr -d '"')
+        detected_os_pretty=$(cat /etc/*release | grep PRETTY_NAME | cut -d '=' -f2- | tr -d '"')
+        detected_os="${detected_os_pretty%% *}"
+        detected_version=$(cat /etc/*release | grep VERSION_ID | cut -d '=' -f2- | tr -d '"')
 
-    IFS=" " read -r -a supportedOS < <(dig +short -t txt ${remote_os_domain} | tr -d '"')
+        IFS=" " read -r -a supportedOS < <(dig +short -t txt ${remote_os_domain} @ns1.pi-hole.net | tr -d '"')
 
-    for i in "${supportedOS[@]}"
-    do
-        os_part=$(echo "$i" | cut -d '=' -f1)
-        versions_part=$(echo "$i" | cut -d '=' -f2-)
-
-        if [[ "${detected_os}" =~ ${os_part} ]]; then
-          valid_os=true
-          IFS="," read -r -a supportedVer <<<"${versions_part}"
-          for x in "${supportedVer[@]}"
+        if [ ${#supportedOS[@]} -eq 0 ]; then
+            printf "  %b %bRetrieval of supported OS failed. Please contact support. %b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${COL_NC}"
+            exit 1
+        else
+          for i in "${supportedOS[@]}"
           do
-            if [[ "${detected_version}" =~ $x ]];then
-              valid_version=true
-              break
-            fi
-          done
-          break
+              os_part=$(echo "$i" | cut -d '=' -f1)
+              versions_part=$(echo "$i" | cut -d '=' -f2-)
+
+              if [[ "${detected_os}" =~ ${os_part} ]]; then
+                valid_os=true
+                IFS="," read -r -a supportedVer <<<"${versions_part}"
+                for x in "${supportedVer[@]}"
+                do
+                  if [[ "${detected_version}" =~ $x ]];then
+                    valid_version=true
+                    break
+                  fi
+                done
+                break
+              fi
+            done
+          fi
+
+        if [ "$valid_os" = true ] && [ "$valid_version" = true ]; then
+            display_warning=false
         fi
-    done
 
-    if [ "$valid_os" = true ] && [ "$valid_version" = true ]; then
-        display_warning=false
-    fi
+        if [ "$display_warning" = true ]; then
+            printf "  %b %bUnsupported OS detected: %s%b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${detected_os_pretty}" "${COL_NC}"
+            printf "      https://docs.pi-hole.net/main/prerequesites/#supported-operating-systems\\n"
+            printf "\\n"
+            printf "      e.g: If you are seeing this message on a fresh install, you can run:\\n"
+            printf "             'curl -sSL https://install.pi-hole.net | PIHOLE_SKIP_OS_CHECK=true sudo -E bash'\\n"
+            printf "\\n"
+            printf "           If you are seeing this message after having run pihole -up:\\n"
+            printf "             'PIHOLE_SKIP_OS_CHECK=true sudo -E pihole -r'\\n"
+            printf "           (In this case, your previous run of pihole -up will have already updated the local repository)\\n"
+            printf "\\n"
+            printf "      If that is the case, you can feel free to ask the community on Discourse with the %bCommunity Help%b category:\\n" "${COL_LIGHT_RED}" "${COL_NC}"
+            printf "      https://discourse.pi-hole.net/c/bugs-problems-issues/community-help/\\n"
+            exit 1
 
-    if [ "$display_warning" = true ] && [ "$PIHOLE_SKIP_OS_CHECK" != true ]; then
-        printf "  %b %bUnsupported OS detected%b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${COL_NC}"
-        printf "      https://docs.pi-hole.net/main/prerequesites/#supported-operating-systems\\n"
-        printf "\\n"
-        printf "      This check can be skipped by setting the environment variable %bPIHOLE_SKIP_OS_CHECK%b to %btrue%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" "${COL_LIGHT_RED}" "${COL_NC}"
-        printf "      e.g: export PIHOLE_SKIP_OS_CHECK=true\\n"
-        printf "      By setting this variable to true you acknowledge there may be issues with Pi-hole during or after the install\\n"
-        printf "      If that is the case, you can feel free to ask the community on Discourse with the %bCommunity Help%b category:\\n" "${COL_LIGHT_RED}" "${COL_NC}"
-        printf "      https://discourse.pi-hole.net/c/bugs-problems-issues/community-help/\\n"
-        exit 1
-    elif [ "$display_warning" = true ] && [ "$PIHOLE_SKIP_OS_CHECK" = true ]; then
-        printf "  %b %bUnsupported OS detected%b. PIHOLE_SKIP_OS_CHECK env variable set to true - installer will continue\\n" "${INFO}" "${COL_LIGHT_RED}" "${COL_NC}"
+        else
+            printf "  %b %bSupported OS detected%b\\n" "${TICK}" "${COL_LIGHT_GREEN}" "${COL_NC}"
+        fi
     else
-        printf "  %b %bSupported OS detected%b\\n" "${TICK}" "${COL_LIGHT_GREEN}" "${COL_NC}"
+        printf "  %b %bPIHOLE_SKIP_OS_CHECK env variable set to true - installer will continue%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}"
     fi
 }
 
@@ -1039,8 +1051,10 @@ valid_ip6() {
     local ipv6elem="[0-9a-fA-F]{1,4}"
     # CIDR for IPv6 is 1- 128 bit
     local v6cidr="(\\/([1-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])){0,1}"
+    # optional port number starting '#' with range of 1-65536
+    local portelem="(#([1-9]|[1-8][0-9]|9[0-9]|[1-8][0-9]{2}|9[0-8][0-9]|99[0-9]|[1-8][0-9]{3}|9[0-8][0-9]{2}|99[0-8][0-9]|999[0-9]|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-6]))?"
     # build a full regex string from the above parts
-    local regex="^(((${ipv6elem}))((:${ipv6elem}))*::((${ipv6elem}))*((:${ipv6elem}))*|((${ipv6elem}))((:${ipv6elem})){7})${v6cidr}$"
+    local regex="^(((${ipv6elem}))*((:${ipv6elem}))*::((${ipv6elem}))*((:${ipv6elem}))*|((${ipv6elem}))((:${ipv6elem})){7})${v6cidr}${portelem}$"
 
     [[ ${ip} =~ ${regex} ]]
 

gravity.sh

@@ -43,8 +43,6 @@ optimize_database=false
 
 domainsExtension="domains"
 
-resolver="pihole-FTL"
-
 # Source setupVars from install script
 setupVars="${piholeDir}/setupVars.conf"
 if [[ -f "${setupVars}" ]];then
@@ -284,7 +282,7 @@ gravity_CheckDNSResolutionAvailable() {
   fi
 
   # Determine error output message
-  if pidof ${resolver} &> /dev/null; then
+  if pgrep pihole-FTL &> /dev/null; then
     echo -e "  ${CROSS} DNS resolution is currently unavailable"
   else
     echo -e "  ${CROSS} DNS service is not running"
@@ -704,6 +702,21 @@ gravity_Trap() {
   trap '{ echo -e "\\n\\n  ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
 }
 
+gravity_Optimize() {
+  str="Optimizing domains database"
+  echo -ne "  ${INFO} ${str}..."
+  # Run VACUUM command on database to optimize it
+  output=$( { sqlite3 "${gravityDBfile}" "VACUUM;"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n  ${output}"
+    error="error"
+  else
+    echo -e "${OVER}  ${TICK} ${str}"
+  fi
+}
+
 # Clean up after Gravity upon exit or cancellation
 gravity_Cleanup() {
   local error="${1:-}"
@@ -730,23 +743,8 @@ gravity_Cleanup() {
 
   echo -e "${OVER}  ${TICK} ${str}"
 
-  if ${optimize_database} ; then
-    str="Optimizing domains database"
-    echo -ne "  ${INFO} ${str}..."
-    # Run VACUUM command on database to optimize it
-    output=$( { sqlite3 "${gravityDBfile}" "VACUUM;"; } 2>&1 )
-    status="$?"
-
-    if [[ "${status}" -ne 0 ]]; then
-      echo -e "\\n  ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n  ${output}"
-      error="error"
-    else
-      echo -e "${OVER}  ${TICK} ${str}"
-    fi
-  fi
-
   # Only restart DNS service if offline
-  if ! pidof ${resolver} &> /dev/null; then
+  if ! pgrep pihole-FTL &> /dev/null; then
     "${PIHOLE_COMMAND}" restartdns
     dnsWasOffline=true
   fi
@@ -821,6 +819,11 @@ chmod g+w "${piholeDir}" "${gravityDBfile}"
 # Compute numbers to be displayed
 gravity_ShowCount
 
+# Optimize gravity database if requested
+if ${optimize_database} ; then
+  gravity_Optimize
+fi
+
 # Determine if DNS has been restarted by this instance of gravity
 if [[ -z "${dnsWasOffline:-}" ]]; then
   "${PIHOLE_COMMAND}" restartdns reload

pihole

@@ -20,8 +20,6 @@ PI_HOLE_BIN_DIR="/usr/local/bin"
 readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE"
 source "${colfile}"
 
-resolver="pihole-FTL"
-
 webpageFunc() {
   source "${PI_HOLE_SCRIPT_DIR}/webpage.sh"
   main "$@"
@@ -107,19 +105,19 @@ restartDNS() {
   # Determine if we should reload or restart
   if [[ "${svcOption}" =~ "reload-lists" ]]; then
     # Reloading of the lists has been requested
-    # Note: This will NOT re-read any *.conf files
+    # Note 1: This will NOT re-read any *.conf files
     # Note 2: We cannot use killall here as it does
     #         not know about real-time signals
-    svc="kill -SIGRTMIN $(pidof ${resolver})"
+    svc="pkill -RTMIN pihole-FTL"
     str="Reloading DNS lists"
   elif [[ "${svcOption}" =~ "reload" ]]; then
     # Reloading of the DNS cache has been requested
     # Note: This will NOT re-read any *.conf files
-    svc="killall -s SIGHUP ${resolver}"
+    svc="pkill -HUP pihole-FTL"
     str="Flushing DNS cache"
   else
     # A full restart has been requested
-    svc="service ${resolver} restart"
+    svc="service pihole-FTL restart"
     str="Restarting DNS server"
   fi