Merge remote-tracking branch 'origin/development' into development-v6

This commit is contained in:
Adam Warner
2021-07-05 16:36:31 +01:00
8 changed files with 187 additions and 152 deletions

View File

@@ -498,10 +498,6 @@ chronoFunc() {
printFunc " RAM usage: " "$ram_perc%" "$ram_info"
printFunc " HDD usage: " "$disk_perc" "$disk_info"
if [[ "$scr_lines" -gt 17 ]] && [[ "$chrono_width" != "small" ]]; then
printFunc " LAN addr: " "${IPV4_ADDRESS/\/*/}" "$lan_info"
fi
if [[ "$DHCP_ACTIVE" == "true" ]]; then
printFunc "DHCP usage: " "$ph_dhcp_percent%" "$dhcp_info"
fi

View File

@@ -826,13 +826,13 @@ dig_at() {
# Store the arguments as variables with names
local protocol="${1}"
local IP="${2}"
echo_current_diagnostic "Name resolution (IPv${protocol}) using a random blocked domain and a known ad-serving domain"
# Set more local variables
# We need to test name resolution locally, via Pi-hole, and via a public resolver
local local_dig
local pihole_dig
local remote_dig
local interfaces
local addresses
# Use a static domain that we know has IPv4 and IPv6 to avoid false positives
# Sometimes the randomly chosen domains don't use IPv6, or something else is wrong with them
local remote_url="doubleclick.com"
@@ -841,15 +841,15 @@ dig_at() {
if [[ ${protocol} == "6" ]]; then
# Set the IPv6 variables and record type
local local_address="::1"
local pihole_address="${IP}"
local remote_address="2001:4860:4860::8888"
local sed_selector="inet6"
local record_type="AAAA"
# Otherwise, it should be 4
else
# so use the IPv4 values
local local_address="127.0.0.1"
local pihole_address="${IP}"
local remote_address="8.8.8.8"
local sed_selector="inet"
local record_type="A"
fi
@@ -859,32 +859,51 @@ dig_at() {
local random_url
random_url=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1")
# First, do a dig on localhost to see if Pi-hole can use itself to block a domain
if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then
# If it can, show success
log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} via ${COL_CYAN}localhost$COL_NC (${local_address})"
else
# Otherwise, show a failure
log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} via ${COL_RED}localhost${COL_NC} (${local_address})"
fi
# Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address
# This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is
# just asing itself locally
# The default timeouts and tries are reduced in case the DNS server isn't working, so the user isn't waiting for too long
# The default timeouts and tries are reduced in case the DNS server isn't working, so the user isn't
# waiting for too long
#
# Turn off history expansion such that the "!" in the sed command cannot do silly things
set +H
# Get interfaces
# sed logic breakdown:
# / master /d;
# Removes all interfaces that are slaves of others (e.g. virtual docker interfaces)
# /UP/!d;
# Removes all interfaces which are not UP
# s/^[0-9]*: //g;
# Removes interface index
# s/: <.*//g;
# Removes everything after the interface name
interfaces="$(ip link show | sed "/ master /d;/UP/!d;s/^[0-9]*: //g;s/: <.*//g;")"
# If Pi-hole can dig itself from it's IP (not the loopback address)
if pihole_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${pihole_address}" +short "${record_type}"); then
# show a success
log_write "${TICK} ${random_url} ${COL_GREEN}is ${pihole_dig}${COL_NC} via ${COL_CYAN}Pi-hole${COL_NC} (${pihole_address})"
else
# Otherwise, show a failure
log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} via ${COL_RED}Pi-hole${COL_NC} (${pihole_address})"
fi
while IFS= read -r iface ; do
# Get addresses of current interface
# sed logic breakdown:
# /inet(|6) /!d;
# Removes all lines from ip a that do not contain either "inet " or "inet6 "
# s/^.*inet(|6) //g;
# Removes all leading whitespace as well as the "inet " or "inet6 " string
# s/\/.*$//g;
# Removes CIDR and everything thereafter (e.g., scope properties)
addresses="$(ip address show dev "${iface}" | sed "/${sed_selector} /!d;s/^.*${sed_selector} //g;s/\/.*$//g;")"
while IFS= read -r local_address ; do
# Check if Pi-hole can use itself to block a domain
if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" +short "${record_type}"); then
# If it can, show success
log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})"
else
# Otherwise, show a failure
log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})"
fi
done <<< "${addresses}"
done <<< "${interfaces}"
# Finally, we need to make sure legitimate queries can out to the Internet using an external, public DNS server
# We are using the static remote_url here instead of a random one because we know it works with IPv4 and IPv6
if remote_dig=$(dig +tries=1 +time=2 -"${protocol}" "${remote_url}" @${remote_address} +short "${record_type}" | head -n1); then
if remote_dig=$(dig +tries=1 +time=2 -"${protocol}" "${remote_url}" @"${remote_address}" +short "${record_type}" | head -n1); then
# If successful, the real IP of the domain will be returned instead of Pi-hole's IP
log_write "${TICK} ${remote_url} ${COL_GREEN}is ${remote_dig}${COL_NC} via ${COL_CYAN}a remote, public DNS server${COL_NC} (${remote_address})"
else
@@ -999,7 +1018,7 @@ parse_file() {
local file_lines
# For each line in the file,
for file_lines in "${file_info[@]}"; do
if [[ ! -z "${file_lines}" ]]; then
if [[ -n "${file_lines}" ]]; then
# don't include the Web password hash
[[ "${file_lines}" =~ ^\#.*$ || ! "${file_lines}" || "${file_lines}" == "WEBPASSWORD="* ]] && continue
# otherwise, display the lines of the file
@@ -1013,12 +1032,8 @@ parse_file() {
check_name_resolution() {
# Check name resolution from localhost, Pi-hole's IP, and Google's name severs
# using the function we created earlier
dig_at 4 "${IPV4_ADDRESS%/*}"
# If IPv6 enabled,
if [[ "${IPV6_ADDRESS}" ]]; then
# check resolution
dig_at 6 "${IPV6_ADDRESS%/*}"
fi
dig_at 4
dig_at 6
}
# This function can check a directory exists
@@ -1248,56 +1263,74 @@ analyze_gravity_list() {
IFS="$OLD_IFS"
}
obfuscated_pihole_log() {
local pihole_log=("$@")
local line
local error_to_check_for
local line_to_obfuscate
local obfuscated_line
for line in "${pihole_log[@]}"; do
# A common error in the pihole.log is when there is a non-hosts formatted file
# that the DNS server is attempting to read. Since it's not formatted
# correctly, there will be an entry for "bad address at line n"
# So we can check for that here and highlight it in red so the user can see it easily
error_to_check_for=$(echo "${line}" | grep 'bad address at')
# Some users may not want to have the domains they visit sent to us
# To that end, we check for lines in the log that would contain a domain name
line_to_obfuscate=$(echo "${line}" | grep ': query\|: forwarded\|: reply')
# If the variable contains a value, it found an error in the log
if [[ -n ${error_to_check_for} ]]; then
# So we can print it in red to make it visible to the user
log_write " ${CROSS} ${COL_RED}${line}${COL_NC} (${FAQ_BAD_ADDRESS})"
else
# If the variable does not a value (the current default behavior), so do not obfuscate anything
if [[ -z ${OBFUSCATE} ]]; then
log_write " ${line}"
# Othwerise, a flag was passed to this command to obfuscate domains in the log
else
# So first check if there are domains in the log that should be obfuscated
if [[ -n ${line_to_obfuscate} ]]; then
# If there are, we need to use awk to replace only the domain name (the 6th field in the log)
# so we substitute the domain for the placeholder value
obfuscated_line=$(echo "${line_to_obfuscate}" | awk -v placeholder="${OBFUSCATED_PLACEHOLDER}" '{sub($6,placeholder); print $0}')
log_write " ${obfuscated_line}"
else
log_write " ${line}"
fi
fi
fi
done
}
analyze_pihole_log() {
echo_current_diagnostic "Pi-hole log"
local head_line
# Put the current Internal Field Separator into another variable so it can be restored later
OLD_IFS="$IFS"
# Get the lines that are in the file(s) and store them in an array for parsing later
IFS=$'\r\n'
local pihole_log_permissions
pihole_log_permissions=$(ls -ld "${PIHOLE_LOG}")
log_write "${COL_GREEN}${pihole_log_permissions}${COL_NC}"
local pihole_log_head=()
mapfile -t pihole_log_head < <(head -n 20 ${PIHOLE_LOG})
log_write " ${COL_CYAN}-----head of $(basename ${PIHOLE_LOG})------${COL_NC}"
local error_to_check_for
local line_to_obfuscate
local obfuscated_line
for head_line in "${pihole_log_head[@]}"; do
# A common error in the pihole.log is when there is a non-hosts formatted file
# that the DNS server is attempting to read. Since it's not formatted
# correctly, there will be an entry for "bad address at line n"
# So we can check for that here and highlight it in red so the user can see it easily
error_to_check_for=$(echo "${head_line}" | grep 'bad address at')
# Some users may not want to have the domains they visit sent to us
# To that end, we check for lines in the log that would contain a domain name
line_to_obfuscate=$(echo "${head_line}" | grep ': query\|: forwarded\|: reply')
# If the variable contains a value, it found an error in the log
if [[ -n ${error_to_check_for} ]]; then
# So we can print it in red to make it visible to the user
log_write " ${CROSS} ${COL_RED}${head_line}${COL_NC} (${FAQ_BAD_ADDRESS})"
else
# If the variable does not a value (the current default behavior), so do not obfuscate anything
if [[ -z ${OBFUSCATE} ]]; then
log_write " ${head_line}"
# Othwerise, a flag was passed to this command to obfuscate domains in the log
else
# So first check if there are domains in the log that should be obfuscated
if [[ -n ${line_to_obfuscate} ]]; then
# If there are, we need to use awk to replace only the domain name (the 6th field in the log)
# so we substitute the domain for the placeholder value
obfuscated_line=$(echo "${line_to_obfuscate}" | awk -v placeholder="${OBFUSCATED_PLACEHOLDER}" '{sub($6,placeholder); print $0}')
log_write " ${obfuscated_line}"
else
log_write " ${head_line}"
fi
fi
fi
done
log_write ""
# Set the IFS back to what it was
IFS="$OLD_IFS"
echo_current_diagnostic "Pi-hole log"
local pihole_log_head=()
local pihole_log_tail=()
local pihole_log_permissions
local logging_enabled
logging_enabled=$(grep -c "^log-queries" /etc/dnsmasq.d/01-pihole.conf)
if [[ "${logging_enabled}" == "0" ]]; then
# Inform user that logging has been disabled and pihole.log does not contain queries
log_write "${INFO} Query logging is disabled"
log_write ""
fi
# Put the current Internal Field Separator into another variable so it can be restored later
OLD_IFS="$IFS"
# Get the lines that are in the file(s) and store them in an array for parsing later
IFS=$'\r\n'
pihole_log_permissions=$(ls -ld "${PIHOLE_LOG}")
log_write "${COL_GREEN}${pihole_log_permissions}${COL_NC}"
mapfile -t pihole_log_head < <(head -n 20 ${PIHOLE_LOG})
log_write " ${COL_CYAN}-----head of $(basename ${PIHOLE_LOG})------${COL_NC}"
obfuscated_pihole_log "${pihole_log_head[@]}"
log_write ""
mapfile -t pihole_log_tail < <(tail -n 20 ${PIHOLE_LOG})
log_write " ${COL_CYAN}-----tail of $(basename ${PIHOLE_LOG})------${COL_NC}"
obfuscated_pihole_log "${pihole_log_tail[@]}"
log_write ""
# Set the IFS back to what it was
IFS="$OLD_IFS"
}
tricorder_use_nc_or_curl() {

View File

@@ -273,11 +273,6 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423
fi
fi
# Prevent Firefox from automatically switching over to DNS-over-HTTPS
# This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https
# (sourced 7th September 2019)
add_dnsmasq_setting "server=/use-application-dns.net/"
# We need to process DHCP settings here as well to account for possible
# changes in the non-FQDN forwarding. This cannot be done in 01-pihole.conf
# as we don't want to delete all local=/.../ lines so it's much safer to
@@ -716,7 +711,7 @@ RemoveCustomDNSAddress() {
host="${args[3]}"
if valid_ip "${ip}" || valid_ip6 "${ip}" ; then
sed -i "/${ip} ${host}/d" "${dnscustomfile}"
sed -i "/^${ip} ${host}$/d" "${dnscustomfile}"
else
echo -e " ${CROSS} Invalid IP has been passed"
exit 1
@@ -747,7 +742,7 @@ RemoveCustomCNAMERecord() {
if [[ -n "${validDomain}" ]]; then
validTarget="$(checkDomain "${target}")"
if [[ -n "${validDomain}" ]]; then
sed -i "/cname=${validDomain},${validTarget}/d" "${dnscustomcnamefile}"
sed -i "/cname=${validDomain},${validTarget}$/d" "${dnscustomcnamefile}"
else
echo " ${CROSS} Invalid Target Passed!"
exit 1

View File

@@ -206,11 +206,7 @@ removeNoPurge() {
}
######### SCRIPT ###########
if command -v vcgencmd &> /dev/null; then
echo -e " ${INFO} All dependencies are safe to remove on Raspbian"
else
echo -e " ${INFO} Be sure to confirm if any dependencies should not be removed"
fi
echo -e " ${INFO} Be sure to confirm if any dependencies should not be removed"
while true; do
echo -e " ${INFO} ${COL_YELLOW}The following dependencies may have been added by the Pi-hole install:"
echo -n " "

View File

@@ -47,16 +47,6 @@ domainsExtension="domains"
setupVars="${piholeDir}/setupVars.conf"
if [[ -f "${setupVars}" ]];then
source "${setupVars}"
# Remove CIDR mask from IPv4/6 addresses
IPV4_ADDRESS="${IPV4_ADDRESS%/*}"
IPV6_ADDRESS="${IPV6_ADDRESS%/*}"
# Determine if IPv4/6 addresses exist
if [[ -z "${IPV4_ADDRESS}" ]] && [[ -z "${IPV6_ADDRESS}" ]]; then
echo -e " ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}"
exit 1
fi
else
echo -e " ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC}
Please run 'pihole -r', and choose the 'reconfigure' option to fix."
@@ -73,6 +63,8 @@ fi
# have changed
gravityDBfile="${GRAVITYDB}"
gravityTEMPfile="${GRAVITYDB}_temp"
gravityDIR="$(dirname -- "${gravityDBfile}")"
gravityOLDfile="${gravityDIR}/gravity_old.db"
if [[ -z "${BLOCKINGMODE}" ]] ; then
BLOCKINGMODE="NULL"
@@ -123,8 +115,19 @@ gravity_swap_databases() {
fi
echo -e "${OVER} ${TICK} ${str}"
# Swap databases and remove old database
rm "${gravityDBfile}"
# Swap databases and remove or conditionally rename old database
# Number of available blocks on disk
availableBlocks=$(stat -f --format "%a" "${gravityDIR}")
# Number of blocks, used by gravity.db
gravityBlocks=$(stat --format "%b" ${gravityDBfile})
# Only keep the old database if available disk space is at least twice the size of the existing gravity.db.
# Better be safe than sorry...
if [ "${availableBlocks}" -gt "$(("${gravityBlocks}" * 2))" ] && [ -f "${gravityDBfile}" ]; then
echo -e " ${TICK} The old database remains available."
mv "${gravityDBfile}" "${gravityOLDfile}"
else
rm "${gravityDBfile}"
fi
mv "${gravityTEMPfile}" "${gravityDBfile}"
}
@@ -564,7 +567,7 @@ compareLists() {
# Download specified URL and perform checks on HTTP status and file content
gravity_DownloadBlocklistFromUrl() {
local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" saveLocation="${5}" target="${6}" compression="${7}"
local heisenbergCompensator="" patternBuffer str httpCode success=""
local heisenbergCompensator="" patternBuffer str httpCode success="" ip
# Create temp file to store content on disk instead of RAM
patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb")
@@ -582,7 +585,10 @@ gravity_DownloadBlocklistFromUrl() {
blocked=false
case $BLOCKINGMODE in
"IP-NODATA-AAAA"|"IP")
if [[ $(dig "${domain}" +short | grep "${IPV4_ADDRESS}" -c) -ge 1 ]]; then
# Get IP address of this domain
ip="$(dig "${domain}" +short)"
# Check if this IP matches any IP of the system
if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then
blocked=true
fi;;
"NXDOMAIN")
@@ -785,43 +791,12 @@ gravity_ShowCount() {
gravity_Table_Count "vw_regex_whitelist" "regex whitelist filters"
}
# Parse list of domains into hosts format
gravity_ParseDomainsIntoHosts() {
awk -v ipv4="$IPV4_ADDRESS" -v ipv6="$IPV6_ADDRESS" '{
# Remove windows CR line endings
sub(/\r$/, "")
# Parse each line as "ipaddr domain"
if(ipv6 && ipv4) {
print ipv4" "$0"\n"ipv6" "$0
} else if(!ipv6) {
print ipv4" "$0
} else {
print ipv6" "$0
}
}' >> "${2}" < "${1}"
}
# Create "localhost" entries into hosts format
gravity_generateLocalList() {
local hostname
if [[ -s "/etc/hostname" ]]; then
hostname=$(< "/etc/hostname")
elif command -v hostname &> /dev/null; then
hostname=$(hostname -f)
else
echo -e " ${CROSS} Unable to determine fully qualified domain name of host"
return 0
fi
echo -e "${hostname}\\npi.hole" > "${localList}.tmp"
# Empty $localList if it already exists, otherwise, create it
: > "${localList}"
echo "### Do not modify this file, it will be overwritten by pihole -g" > "${localList}"
chmod 644 "${localList}"
gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
# Add additional LAN hosts provided by OpenVPN (if available)
if [[ -f "${VPNList}" ]]; then
awk -F, '{printf $2"\t"$1".vpn\n"}' "${VPNList}" >> "${localList}"
@@ -890,6 +865,11 @@ for var in "$@"; do
esac
done
# Remove OLD (backup) gravity file, if it exists
if [[ -f "${gravityOLDfile}" ]]; then
rm "${gravityOLDfile}"
fi
# Trap Ctrl-C
gravity_Trap

14
pihole
View File

@@ -363,16 +363,13 @@ tailFunc() {
fi
echo -e " ${INFO} Press Ctrl-C to exit"
# Retrieve IPv4/6 addresses
source /etc/pihole/setupVars.conf
# Strip date from each line
# Color blocklist/blacklist/wildcard entries as red
# Color A/AAAA/DHCP strings as white
# Color everything else as gray
tail -f /var/log/pihole.log | sed -E \
tail -f /var/log/pihole.log | grep --line-buffered "${1}" | sed -E \
-e "s,($(date +'%b %d ')| dnsmasq\[[0-9]*\]),,g" \
-e "s,(.*(blacklisted |gravity blocked ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
-e "s,(.*(blacklisted |gravity blocked ).* is (0.0.0.0|::|NXDOMAIN).*),${COL_RED}&${COL_NC}," \
-e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \
-e "s,.*,${COL_GRAY}&${COL_NC},"
exit 0
@@ -456,7 +453,10 @@ Debugging Options:
Add '-a' to automatically upload the log to tricorder.pi-hole.net
-f, flush Flush the Pi-hole log
-r, reconfigure Reconfigure or Repair Pi-hole subsystems
-t, tail View the live output of the Pi-hole log
-t, tail [arg] View the live output of the Pi-hole log.
Add an optional argument to filter the log
(regular expressions are supported)
Options:
-a, admin Web interface options
@@ -530,7 +530,7 @@ case "${1}" in
"status" ) statusFunc "$2";;
"restartdns" ) restartDNS "$2";;
"-a" | "admin" ) webpageFunc "$@";;
"-t" | "tail" ) tailFunc;;
"-t" | "tail" ) tailFunc "$2";;
"checkout" ) piholeCheckoutFunc "$@";;
"tricorder" ) tricorderFunc;;
"updatechecker" ) updateCheckFunc "$@";;

View File

@@ -8,8 +8,6 @@ check_output = testinfra.get_backend(
SETUPVARS = {
'PIHOLE_INTERFACE': 'eth99',
'IPV4_ADDRESS': '1.1.1.1',
'IPV6_ADDRESS': 'FE80::240:D0FF:FE48:4672',
'PIHOLE_DNS_1': '4.2.2.1',
'PIHOLE_DNS_2': '4.2.2.2'
}

View File

@@ -45,8 +45,6 @@ def test_setupVars_are_sourced_to_global_scope(Pihole):
# Currently debug test function only
echo "Outputting sourced variables"
echo "PIHOLE_INTERFACE=${PIHOLE_INTERFACE}"
echo "IPV4_ADDRESS=${IPV4_ADDRESS}"
echo "IPV6_ADDRESS=${IPV6_ADDRESS}"
echo "PIHOLE_DNS_1=${PIHOLE_DNS_1}"
echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}"
}
@@ -572,3 +570,42 @@ def test_os_check_passes(Pihole):
''')
expected_stdout = 'Supported OS detected'
assert expected_stdout in detectOS.stdout
def test_package_manager_has_installer_deps(Pihole):
''' Confirms OS is able to install the required packages for the installer'''
mock_command('whiptail', {'*': ('', '0')}, Pihole)
output = Pihole.run('''
source /opt/pihole/basic-install.sh
distro_check
install_dependent_packages ${INSTALLER_DEPS[@]}
''')
assert 'No package' not in output.stdout # centos7 still exits 0...
assert output.rc == 0
def test_package_manager_has_pihole_deps(Pihole):
''' Confirms OS is able to install the required packages for Pi-hole '''
mock_command('whiptail', {'*': ('', '0')}, Pihole)
output = Pihole.run('''
source /opt/pihole/basic-install.sh
distro_check
install_dependent_packages ${PIHOLE_DEPS[@]}
''')
assert 'No package' not in output.stdout # centos7 still exits 0...
assert output.rc == 0
def test_package_manager_has_web_deps(Pihole):
''' Confirms OS is able to install the required packages for web '''
mock_command('whiptail', {'*': ('', '0')}, Pihole)
output = Pihole.run('''
source /opt/pihole/basic-install.sh
distro_check
install_dependent_packages ${PIHOLE_WEB_DEPS[@]}
''')
assert 'No package' not in output.stdout # centos7 still exits 0...
assert output.rc == 0