Merge pull request #1041 from pi-hole/development

[RELEASE] Pi-Hole Core 2.10.2

.github/ISSUE_TEMPLATE.md

@@ -6,16 +6,7 @@
 
 **How familiar are you with the codebase?:**
 
-- [] 1 (very unfamiliar)
-- [] 2
-- [] 3
-- [] 4
-- [] 5
-- [] 6
-- [] 7
-- [] 8
-- [] 9
-- [] 10 (very familiar)
+_{replace this text with a number from 1 to 10, with 1 being not familiar, and 10 being very familiar}_
 
 ---
 **[FEATURE REQUEST | QUESTION | OTHER]:**

.github/PULL_REQUEST_TEMPLATE.md

@@ -10,16 +10,7 @@
 
 **How familiar are you with the codebase?:**
 
-- [] 1 (very unfamiliar)
-- [] 2
-- [] 3
-- [] 4
-- [] 5
-- [] 6
-- [] 7
-- [] 8
-- [] 9
-- [] 10 (very familiar)
+_{replace this text with a number from 1 to 10, with 1 being not familiar, and 10 being very familiar}_
 
 ---
 _{replace this line with your pull request content}_

README.md

@@ -10,7 +10,7 @@
 
 ## The multi-platform, network-wide ad blocker
 
-Block ads for **all** your devices _without_ the need to install client-side software.  The Pi-hole blocks ads the DNS-level, so all your devices are protected.
+Block ads for **all** your devices _without_ the need to install client-side software.  The Pi-hole blocks ads at the DNS-level, so all your devices are protected.
 
 - Web Browsers
 - Cell Phones
@@ -53,7 +53,7 @@ wget -O basic-install.sh https://install.pi-hole.net
 bash basic-install.sh
 ```
 
-Once installed, [configure your router to have **DHCP clients use the Pi as their DNS server**](http://pi-hole.net/faq/can-i-set-the-pi-hole-to-be-the-dns-server-at-my-router-so-i-dont-have-to-change-settings-for-my-devices/) and then any device that connects to your network will have ads blocked without any further configuration. Alternatively, you can manually set each device to [use the Raspberry Pi as its DNS server](http://pi-hole.net/faq/how-do-i-use-the-pi-hole-as-my-dns-server/).
+Once installed, [configure your router to have **DHCP clients use the Pi as their DNS server**](https://discourse.pi-hole.net/t/how-do-i-configure-my-devices-to-use-pi-hole-as-their-dns-server/245) and then any device that connects to your network will have ads blocked without any further configuration. Alternatively, you can manually set each device to [use the Raspberry Pi as its DNS server](http://pi-hole.net/faq/how-do-i-use-the-pi-hole-as-my-dns-server/).
 
 ## Installing the Pi-hole (Click to Watch!)
 <p align="center">

advanced/Scripts/chronometer.sh

@@ -15,8 +15,6 @@
 piLog="/var/log/pihole.log"
 gravity="/etc/pihole/gravity.list"
 
-today=$(date "+%b %e")
-
 . /etc/pihole/setupVars.conf
 
 CalcBlockedDomains() {
@@ -35,7 +33,7 @@ CalcBlockedDomains() {
 
 CalcQueriesToday() {
 	if [ -e "${piLog}" ]; then
-		queriesToday=$(cat "${piLog}" | grep "${today}" | awk '/query/ {print $6}' | wc -l)
+		queriesToday=$(awk '/query\[/ {print $6}' < "${piLog}" | wc -l)
 	else
 		queriesToday="Err."
 	fi
@@ -43,7 +41,7 @@ CalcQueriesToday() {
 
 CalcblockedToday() {
 	if [ -e "${piLog}" ] && [ -e "${gravity}" ];then
-		blockedToday=$(cat ${piLog} | awk '/\/etc\/pihole\/gravity.list/ && !/address/ {print $6}' | wc -l)
+		blockedToday=$(awk '/\/etc\/pihole\/gravity.list/ && !/address/ {print $6}' < "${piLog}" | wc -l)
 	else
 		blockedToday="Err."
 	fi
@@ -104,7 +102,7 @@ normalChrono() {
 
 		echo "Blocking:      ${blockedDomainsTotal}"
 		echo "Queries:       ${queriesToday}" #same total calculation as dashboard
-	  echo "Pi-holed:      ${blockedToday} (${percentBlockedToday}%)"
+		echo "Pi-holed:      ${blockedToday} (${percentBlockedToday}%)"
 
 		sleep 5
 	done

advanced/Scripts/update.sh

@@ -22,9 +22,15 @@ readonly PI_HOLE_FILES_DIR="/etc/.pihole"
 is_repo() {
   # Use git to check if directory is currently under VCS, return the value
   local directory="${1}"
+  local curdir
+  local rc
 
-  git -C "${directory}" status --short &> /dev/null
-  return
+  curdir="${PWD}"
+  cd "${directory}" &> /dev/null || return 1
+  git status --short &> /dev/null
+  rc=$?
+  cd "${curdir}" &> /dev/null || return 1
+  return $rc
 }
 
 prep_repo() {
@@ -46,16 +52,20 @@ make_repo() {
 
 update_repo() {
   local directory="${1}"
-  local retVal=0
+  local curdir
   # Pull the latest commits
 
+  curdir="${PWD}"
+  cd "${directory}" &> /dev/null || return 1
   # Stash all files not tracked for later retrieval
-  git -C "${directory}" stash --all --quiet &> /dev/null || ${retVal}=1
+  git stash --all --quiet &> /dev/null
   # Force a clean working directory for cloning
-  git -C "${directory}" clean --force -d &> /dev/null || ${retVal}=1
+  git clean --force -d &> /dev/null
   # Fetch latest changes and apply
-  git -C "${directory}" pull --quiet &> /dev/null || ${retVal}=1
-  return ${retVal}
+  git pull --quiet &> /dev/null
+  cd "${curdir}" &> /dev/null || return 1
+
+  return
 }
 
 getGitFiles() {

advanced/Scripts/webpage.sh

@@ -61,6 +61,7 @@ SetWebPassword(){
 		echo "WEBPASSWORD=${hash}" >> /etc/pihole/setupVars.conf
 		echo "New password set"
 	else
+		echo "WEBPASSWORD=" >> /etc/pihole/setupVars.conf
 		echo "Password removed"
 	fi
 
@@ -69,7 +70,7 @@ SetWebPassword(){
 SetDNSServers(){
 
 	# Remove setting from file (create backup setupVars.conf.bak)
-	sed -i.bak '/PIHOLE_DNS_1/d;/PIHOLE_DNS_2/d;/DNS_FQDN_REQUIRED/d;' /etc/pihole/setupVars.conf
+	sed -i.bak '/PIHOLE_DNS_1/d;/PIHOLE_DNS_2/d;/DNS_FQDN_REQUIRED/d;/DNS_BOGUS_PRIV/d;' /etc/pihole/setupVars.conf
 	# Save setting to file
 	echo "PIHOLE_DNS_1=${args[2]}" >> /etc/pihole/setupVars.conf
 	if [[ "${args[3]}" != "none" ]]; then
@@ -215,6 +216,20 @@ SetDNSDomainName(){
 
 }
 
+ResolutionSettings() {
+
+	typ=${args[2]}
+	state=${args[3]}
+
+	if [[ "${typ}" == "forward" ]]; then
+		sed -i.bak '/API_GET_UPSTREAM_DNS_HOSTNAME/d;' /etc/pihole/setupVars.conf
+		echo "API_GET_UPSTREAM_DNS_HOSTNAME=${state}" >> /etc/pihole/setupVars.conf
+	elif [[ "${typ}" == "clients" ]]; then
+		sed -i.bak '/API_GET_CLIENT_HOSTNAME/d;' /etc/pihole/setupVars.conf
+		echo "API_GET_CLIENT_HOSTNAME=${state}" >> /etc/pihole/setupVars.conf
+	fi
+}
+
 case "${args[1]}" in
 	"-p" | "password"   ) SetWebPassword;;
 	"-c" | "celsius"    ) unit="C"; SetTemperatureUnit;;
@@ -231,6 +246,7 @@ case "${args[1]}" in
 	"layout"            ) SetWebUILayout;;
 	"-h" | "--help"     ) helpFunc;;
 	"domainname"        ) SetDNSDomainName;;
+	"resolve"           ) ResolutionSettings;;
 	*                   ) helpFunc;;
 esac
 

advanced/pihole.sudo

@@ -9,4 +9,3 @@
 # the Free Software Foundation, either version 2 of the License, or
 # (at your option) any later version.
 
-www-data ALL=NOPASSWD: /usr/local/bin/pihole

advanced/selinux/pihole.te

@@ -1,87 +0,0 @@
-module pihole 1.0;
-
-require {
-        type var_log_t;
-        type unconfined_t;
-        type init_t;
-        type auditd_t;
-        type syslogd_t;
-        type NetworkManager_t;
-        type mdadm_t;
-        type tuned_t;
-        type avahi_t;
-        type irqbalance_t;
-        type system_dbusd_t;
-        type kernel_t;
-        type httpd_sys_script_t;
-        type systemd_logind_t;
-        type httpd_t;
-        type policykit_t;
-        type dnsmasq_t;
-        type udev_t;
-        type postfix_pickup_t;
-        type sshd_t;
-        type crond_t;
-        type getty_t;
-        type lvm_t;
-        type postfix_qmgr_t;
-        type postfix_master_t;
-        class dir { getattr search };
-        class file { read open setattr };
-}
-
-#============= dnsmasq_t ==============
-allow dnsmasq_t var_log_t:file { open setattr };
-
-#============= httpd_t ==============
-allow httpd_t var_log_t:file { read open };
-
-#============= httpd_sys_script_t (class: dir) ==============
-allow httpd_sys_script_t NetworkManager_t:dir { getattr search };
-allow httpd_sys_script_t auditd_t:dir { getattr search };
-allow httpd_sys_script_t avahi_t:dir { getattr search };
-allow httpd_sys_script_t crond_t:dir { getattr search };
-allow httpd_sys_script_t dnsmasq_t:dir { getattr search };
-allow httpd_sys_script_t getty_t:dir { getattr search };
-allow httpd_sys_script_t httpd_t:dir { getattr search };
-allow httpd_sys_script_t init_t:dir { getattr search };
-allow httpd_sys_script_t irqbalance_t:dir { getattr search };
-allow httpd_sys_script_t kernel_t:dir { getattr search };
-allow httpd_sys_script_t lvm_t:dir { getattr search };
-allow httpd_sys_script_t mdadm_t:dir { getattr search };
-allow httpd_sys_script_t policykit_t:dir { getattr search };
-allow httpd_sys_script_t postfix_master_t:dir { getattr search };
-allow httpd_sys_script_t postfix_pickup_t:dir { getattr search };
-allow httpd_sys_script_t postfix_qmgr_t:dir { getattr search };
-allow httpd_sys_script_t sshd_t:dir { getattr search };
-allow httpd_sys_script_t syslogd_t:dir { getattr search };
-allow httpd_sys_script_t system_dbusd_t:dir { getattr search };
-allow httpd_sys_script_t systemd_logind_t:dir { getattr search };
-allow httpd_sys_script_t tuned_t:dir { getattr search };
-allow httpd_sys_script_t udev_t:dir { getattr search };
-allow httpd_sys_script_t unconfined_t:dir { getattr search };
-
-#============= httpd_sys_script_t (class: file) ==============
-allow httpd_sys_script_t NetworkManager_t:file { read open };
-allow httpd_sys_script_t auditd_t:file { read open };
-allow httpd_sys_script_t avahi_t:file { read open };
-allow httpd_sys_script_t crond_t:file { read open };
-allow httpd_sys_script_t dnsmasq_t:file { read open };
-allow httpd_sys_script_t getty_t:file { read open };
-allow httpd_sys_script_t httpd_t:file { read open };
-allow httpd_sys_script_t init_t:file { read open };
-allow httpd_sys_script_t irqbalance_t:file { read open };
-allow httpd_sys_script_t kernel_t:file { read open };
-allow httpd_sys_script_t lvm_t:file { read open };
-allow httpd_sys_script_t mdadm_t:file { read open };
-allow httpd_sys_script_t policykit_t:file { read open };
-allow httpd_sys_script_t postfix_master_t:file { read open };
-allow httpd_sys_script_t postfix_pickup_t:file { read open };
-allow httpd_sys_script_t postfix_qmgr_t:file { read open };
-allow httpd_sys_script_t sshd_t:file { read open };
-allow httpd_sys_script_t syslogd_t:file { read open };
-allow httpd_sys_script_t system_dbusd_t:file { read open };
-allow httpd_sys_script_t systemd_logind_t:file { read open };
-allow httpd_sys_script_t tuned_t:file { read open };
-allow httpd_sys_script_t udev_t:file { read open };
-allow httpd_sys_script_t unconfined_t:file { read open };

automated install/basic-install.sh

@@ -135,8 +135,14 @@ fi
 is_repo() {
   # Use git to check if directory is currently under VCS, return the value
   local directory="${1}"
-  git -C "${directory}" status --short &> /dev/null
-  return
+  if [ -d $directory ]; then
+    # git -C is not used here to support git versions older than 1.8.4
+    curdir=$PWD; cd $directory; git status --short &> /dev/null; rc=$?; cd $curdir
+    return $rc
+  else
+    # non-zero return code if directory does not exist OR is not a valid git repository
+    return 1
+  fi
 }
 
 make_repo() {
@@ -152,7 +158,7 @@ make_repo() {
 update_repo() {
   local directory="${1}"
   # Pull the latest commits
-  echo -n ":::     Updating repo in $1..."
+  echo -n ":::    Updating repo in $1..."
   cd "${directory}" || exit 1
   git stash -q &> /dev/null
   git pull -q &> /dev/null
@@ -628,6 +634,7 @@ installScripts() {
   # Install files from local core repository
   if is_repo "${PI_HOLE_LOCAL_REPO}"; then
     cd "${PI_HOLE_LOCAL_REPO}"
+    install -o "${USER}" -Dm755 -d /opt/pihole
     install -o "${USER}" -Dm755 -t /opt/pihole/ gravity.sh
     install -o "${USER}" -Dm755 -t /opt/pihole/ ./advanced/Scripts/*.sh
     install -o "${USER}" -Dm755 -t /opt/pihole/ ./automated\ install/uninstall.sh
@@ -800,6 +807,15 @@ installPiholeWeb() {
   echo -n "::: Installing sudoer file..."
   mkdir -p /etc/sudoers.d/
   cp /etc/.pihole/advanced/pihole.sudo /etc/sudoers.d/pihole
+  # Add lighttpd user (OS dependent) to sudoers file
+  echo "${LIGHTTPD_USER} ALL=NOPASSWD: /usr/local/bin/pihole" >> /etc/sudoers.d/pihole
+
+  if [[ "$LIGHTTPD_USER" == "lighttpd" ]]; then
+    # Allow executing pihole via sudo with Fedora
+    # Usually /usr/local/bin is not permitted as directory for sudoable programms
+    echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin" >> /etc/sudoers.d/pihole
+  fi
+
   chmod 0440 /etc/sudoers.d/pihole
   echo " done!"
 }
@@ -820,8 +836,12 @@ runGravity() {
     echo "::: Cleaning up previous install (preserving whitelist/blacklist)"
     rm /etc/pihole/list.*
   fi
+  # Test if /etc/pihole/adlists.default exists
+  if [[ ! -e /etc/pihole/adlists.default ]]; then
+    cp /etc/.pihole/adlists.default /etc/pihole/adlists.default
+  fi
   echo "::: Running gravity.sh"
-  /opt/pihole/gravity.sh
+  { /opt/pihole/gravity.sh; }
 }
 
 create_pihole_user() {
@@ -877,7 +897,6 @@ installPihole() {
   installScripts
   installConfigs
   CreateLogFile
-  configureSelinux
   installPiholeWeb
   installCron
   configureFirewall
@@ -908,7 +927,6 @@ updatePihole() {
   installScripts
   installConfigs
   CreateLogFile
-  configureSelinux
   installPiholeWeb
   installCron
   configureFirewall
@@ -916,23 +934,25 @@ updatePihole() {
   runGravity
 }
 
-configureSelinux() {
+
+
+checkSelinux() {
   if [ -x "$(command -v getenforce)" ]; then
-    printf "\n::: SELinux Detected\n"
-    printf ":::\tChecking for SELinux policy development packages..."
-    package_check_install "selinux-policy-devel" > /dev/null
-    echo " installed!"
-    printf ":::\tEnabling httpd server side includes (SSI).. "
-    setsebool -P httpd_ssi_exec on &> /dev/null && echo "Success" || echo "SELinux not enabled"
-    printf "\n:::\tCompiling Pi-Hole SELinux policy..\n"
-    if ! [ -x "$(command -v systemctl)" ]; then
-      sed -i.bak '/systemd/d' /etc/.pihole/advanced/selinux/pihole.te
+    echo ":::"
+    echo -n "::: SELinux Support Detected... Mode: "
+    enforceMode=$(getenforce)
+    echo "${enforceMode}"
+    if [[ "${enforceMode}" == "Enforcing" ]]; then
+      if (whiptail --title "SELinux Enforcing Detected" --yesno "SELinux is being Enforced on your system!\n\nPi-hole currently does not support SELinux, but you may still continue with the installation.\n\nNote: Admin UI Will not function fully without setting your policies correctly\n\nContinue installing Pi-hole?" ${r} ${c}); then
+          echo ":::"
+          echo "::: Continuing installation with SELinux Enforcing."
+          echo "::: Please refer to official SELinux documentation to create a custom policy."
+      else
+          echo ":::"
+          echo "::: Not continuing install after SELinux Enforcing detected."
+          exit 1
+      fi
     fi
-    checkmodule -M -m -o /etc/pihole/pihole.mod /etc/.pihole/advanced/selinux/pihole.te
-    semodule_package -o /etc/pihole/pihole.pp -m /etc/pihole/pihole.mod
-    semodule -i /etc/pihole/pihole.pp
-    rm -f /etc/pihole/pihole.mod
-    semodule -l | grep pihole &> /dev/null && echo "::: Installed Pi-Hole SELinux policy" || echo "::: Warning: Pi-Hole SELinux policy did not install."
   fi
 }
 
@@ -998,7 +1018,8 @@ update_dialogs() {
 }
 
 main() {
-# Check arguments for the undocumented flags
+
+  # Check arguments for the undocumented flags
   for var in "$@"; do
     case "$var" in
       "--reconfigure"  ) reconfigure=true;;
@@ -1033,6 +1054,9 @@ main() {
   # Install packages used by this installation script
   install_dependent_packages INSTALLER_DEPS[@]
 
+   # Check if SELinux is Enforcing
+  checkSelinux
+
   if [[ "${reconfigure}" == true ]]; then
     echo "::: --reconfigure passed to install script. Not downloading/updating local repos"
   else
@@ -1053,10 +1077,10 @@ main() {
     get_available_interfaces
     # Find interfaces and let the user choose one
     chooseInterface
-    # Let the user decide if they want to block ads over IPv4 and/or IPv6
-    use4andor6
     # Decide what upstream DNS Servers to use
     setDNS
+    # Let the user decide if they want to block ads over IPv4 and/or IPv6
+    use4andor6
     # Let the user decide if they want query logging enabled...
     setLogging
 

gravity.sh

@@ -104,16 +104,30 @@ gravity_collapse() {
 # patternCheck - check to see if curl downloaded any new files.
 gravity_patternCheck() {
 	patternBuffer=$1
-	# check if the patternbuffer is a non-zero length file
-	if [[ -s "${patternBuffer}" ]]; then
-		# Some of the blocklists are copyright, they need to be downloaded
-		# and stored as is. They can be processed for content after they
-		# have been saved.
-		mv "${patternBuffer}" "${saveLocation}"
-		echo " List updated, transport successful!"
+	success=$2
+	error=$3
+	if [ $success = true ]; then
+		# check if download was successful but list has not been modified
+		if [ "${error}" == "304" ]; then
+			echo ":::   No changes detected, transport skipped!"
+		# check if the patternbuffer is a non-zero length file
+		elif [[ -s "${patternBuffer}" ]]; then
+			# Some of the blocklists are copyright, they need to be downloaded
+			# and stored as is. They can be processed for content after they
+			# have been saved.
+			mv "${patternBuffer}" "${saveLocation}"
+			echo ":::   List updated, transport successful!"
+		else
+			# Empty file -> use previously downloaded list
+			echo ":::   Received empty file, using cached one (list not updated!)"
+		fi
 	else
-		# curl didn't download any host files, probably because of the date check
-		echo " No changes detected, transport skipped!"
+		# check if cached list exists
+		if [[ -r "${saveLocation}" ]]; then
+			echo ":::   List download failed, using cached list (list not updated!)"
+		else
+			echo ":::   Download failed and no cached list available (list will not be considered)"
+		fi
 	fi
 }
 
@@ -132,9 +146,27 @@ gravity_transport() {
 	fi
 
 	# Silently curl url
-	curl -s -L ${cmd_ext} ${heisenbergCompensator} -A "${agent}" ${url} > ${patternBuffer}
-	# Check for list updates
-	gravity_patternCheck "${patternBuffer}"
+	err=$(curl -s -L ${cmd_ext} ${heisenbergCompensator} -w %{http_code} -A "${agent}" ${url} -o ${patternBuffer})
+
+	echo " done"
+	# Analyze http response
+	echo -n ":::   Status: "
+	case "$err" in
+		"200"  ) echo "Success (OK)"; success=true;;
+		"304"  ) echo "Not modified"; success=true;;
+		"403"  ) echo "Forbidden"; success=false;;
+		"404"  ) echo "Not found"; success=false;;
+		"408"  ) echo "Time-out"; success=false;;
+		"451"  ) echo "Unavailable For Legal Reasons"; success=false;;
+		"521"  ) echo "Web Server Is Down (Cloudflare)"; success=false;;
+		"522"  ) echo "Connection Timed Out (Cloudflare)"; success=false;;
+		"500"  ) echo "Internal Server Error"; success=false;;
+		*      ) echo "Status $err"; success=false;;
+	esac
+
+	# Process result
+	gravity_patternCheck "${patternBuffer}" ${success} "${err}"
+
 }
 
 # spinup - main gravity function
@@ -181,7 +213,10 @@ gravity_Schwarzchild() {
 	echo -n "::: Aggregating list of domains..."
 	truncate -s 0 ${piholeDir}/${matterAndLight}
 	for i in "${activeDomains[@]}"; do
-		cat "${i}" | tr -d '\r' >> ${piholeDir}/${matterAndLight}
+		# Only assimilate list if it is available (download might have faild permanently)
+		if [[ -r "${i}" ]]; then
+			cat "${i}" | tr -d '\r' >> ${piholeDir}/${matterAndLight}
+		fi
 	done
 	echo " done!"
 }
@@ -353,7 +388,7 @@ if [[ "${forceGrav}" == true ]]; then
 fi
 
 #Overwrite adlists.default from /etc/.pihole in case any changes have been made. Changes should be saved in /etc/adlists.list
-cp /etc/.pihole/adlists.default /etc/pihole/adlists.default
+#cp /etc/.pihole/adlists.default /etc/pihole/adlists.default
 gravity_collapse
 gravity_spinup
 if [[ "${skipDownload}" == false ]]; then

pihole

@@ -72,9 +72,9 @@ scanList(){
   list="${2}"
   method="${3}"
   if [[ ${method} == "-exact" ]] ; then
-    grep -E "(^|\s)${domain}($|\s)" "${list}"
+    grep -i -E "(^|\s)${domain}($|\s)" "${list}"
   else
-    grep "${domain}" "${list}"
+    grep -i "${domain}" "${list}"
   fi
 }