diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 65b42b95..a82d0d51 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -32,4 +32,11 @@ upgrade_gravityDB(){ database_table_from_file "domain_audit" "${auditFile}" fi fi + if [[ "$version" == "2" ]]; then + # This migration script upgrades the gravity.db file by + # renaming the regex table to regex_blacklist, and + # creating a new regex_whitelist table + corresponding linking table and views + sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql" + version=3 + fi } diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index 45b5fa02..6d57a6fe 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -1,5 +1,7 @@ .timeout 30000 +BEGIN TRANSACTION; + CREATE TABLE domain_audit ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -8,3 +10,5 @@ CREATE TABLE domain_audit ); UPDATE info SET value = 2 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql new file mode 100644 index 00000000..fd7c24d2 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -0,0 +1,65 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE regex RENAME TO regex_blacklist; + +CREATE TABLE regex_blacklist_by_group +( + regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_blacklist_id, group_id) +); + +INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group; +DROP TABLE regex_by_group; +DROP VIEW vw_regex; +DROP TRIGGER tr_regex_update; + +CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain + FROM regex_blacklist + LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id + LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id + WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_blacklist.id; + +CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist + BEGIN + UPDATE regex_blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + +CREATE TABLE regex_whitelist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); + +CREATE TABLE regex_whitelist_by_group +( + regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_whitelist_id, group_id) +); + +CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain + FROM regex_whitelist + LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_whitelist_id = regex_whitelist.id + LEFT JOIN "group" ON "group".id = regex_whitelist_by_group.group_id + WHERE regex_whitelist.enabled = 1 AND (regex_whitelist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_whitelist.id; + +CREATE TRIGGER tr_regex_whitelist_update AFTER UPDATE ON regex_whitelist + BEGIN + UPDATE regex_whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + + +UPDATE info SET value = 3 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index fa81348b..6a606665 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -32,12 +32,18 @@ helpFunc() { if [[ "${listType}" == "whitelist" ]]; then param="w" type="whitelist" - elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then + elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then param="-wild" type="wildcard blacklist" - elif [[ "${listType}" == "regex" ]]; then + elif [[ "${listType}" == "regex_blacklist" ]]; then param="-regex" - type="regex filter" + type="regex blacklist filter" + elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then + param="-white-wild" + type="wildcard whitelist" + elif [[ "${listType}" == "regex_whitelist" ]]; then + param="-white-regex" + type="regex whitelist filter" else param="b" type="blacklist" @@ -72,7 +78,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then + if [[ ( "${listType}" == "regex_blacklist" || "${listType}" == "regex_whitelist" ) && "${wildcard}" == false ]]; then validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check @@ -88,12 +94,19 @@ HandleOther() { } ProcessDomainList() { - if [[ "${listType}" == "regex" ]]; then - # Regex filter list - listname="regex filters" + local is_regexlist + if [[ "${listType}" == "regex_blacklist" ]]; then + # Regex black filter list + listname="regex blacklist filters" + is_regexlist=true + elif [[ "${listType}" == "regex_whitelist" ]]; then + # Regex white filter list + listname="regex whitelist filters" + is_regexlist=true else # Whitelist / Blacklist listname="${listType}" + is_regexlist=false fi for dom in "${domList[@]}"; do @@ -106,7 +119,7 @@ ProcessDomainList() { # if delmode then remove from desired list but do not add to the other if ${addmode}; then AddDomain "${dom}" "${listType}" - if [[ ! "${listType}" == "regex" ]]; then + if ! ${is_regexlist}; then RemoveDomain "${dom}" "${listAlt}" fi else @@ -173,7 +186,7 @@ Displaylist() { data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)" if [[ -z $data ]]; then - echo -e "Not showing empty ${listname}" + echo -e "Not showing empty list" else echo -e "Displaying ${listname}:" count=1 @@ -215,8 +228,10 @@ for var in "$@"; do case "${var}" in "-w" | "whitelist" ) listType="whitelist"; listAlt="blacklist";; "-b" | "blacklist" ) listType="blacklist"; listAlt="whitelist";; - "--wild" | "wildcard" ) listType="regex"; wildcard=true;; - "--regex" | "regex" ) listType="regex";; + "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;; + "--regex" | "regex" ) listType="regex_blacklist";; + "--white-regex" | "white-regex" ) listType="regex_whitelist";; + "--white-wild" | "white-wild" ) listType="regex_whitelist"; wildcard=true;; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index d1acb950..84e34416 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1104,20 +1104,27 @@ show_db_entries() { IFS="$OLD_IFS" } +show_groups() { + show_db_entries "Groups" "SELECT * FROM \"group\"" "4 4 30 50" +} + show_adlists() { - show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50" + show_db_entries "Adlists" "SELECT id,address,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM adlist" "4 100 7 19 19 50" + show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 4" } show_whitelist() { - show_db_entries "Whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50" + show_db_entries "Exact whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM whitelist" "4 100 7 19 19 50" + show_db_entries "Exact whitelist groups" "SELECT * FROM whitelist_by_group" "4 4" + show_db_entries "Regex whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM regex_whitelist" "4 100 7 19 19 50" + show_db_entries "Regex whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4" } show_blacklist() { - show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" -} - -show_regexlist() { - show_db_entries "Regexlist" "SELECT * FROM regex" "4 100 7 10 13 50" + show_db_entries "Exact blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM blacklist" "4 100 7 19 19 50" + show_db_entries "Exact blacklist groups" "SELECT * FROM blacklist_by_group" "4 4" + show_db_entries "Regex blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM regex_blacklist" "4 100 7 19 19 50" + show_db_entries "Regex blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4" } analyze_gravity_list() { @@ -1293,10 +1300,10 @@ process_status parse_setup_vars check_x_headers analyze_gravity_list +show_groups show_adlists show_whitelist show_blacklist -show_regexlist show_content_of_pihole_files parse_locale analyze_pihole_log diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 6d061ba5..f40adfbf 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -36,14 +36,14 @@ scanList(){ # /dev/null forces filename to be printed when only one list has been generated # shellcheck disable=SC2086 case "${type}" in - "exact" ) grep -i -E -l "(^|(?/dev/null;; - # Create array of regexps - # Iterate through each regexp and check whether it matches the domainQuery - # If it does, print the matching regexp and continue looping - # Input 1 - regexps | Input 2 - domainQuery - "regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \ - <(echo "${lists}") <(echo "${domain}") 2>/dev/null;; - * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; + "exact" ) grep -i -E -l "(^|(?/dev/null;; + # Create array of regexps + # Iterate through each regexp and check whether it matches the domainQuery + # If it does, print the matching regexp and continue looping + # Input 1 - regexps | Input 2 - domainQuery + "regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \ + <(echo "${lists}") <(echo "${domain}") 2>/dev/null;; + * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; esac } @@ -100,8 +100,8 @@ scanDatabaseTable() { # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. case "${type}" in - "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; - * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; + * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac # Send prepared query to gravity database @@ -130,42 +130,52 @@ scanDatabaseTable() { done } +scanRegexDatabaseTable() { + local domain list + domain="${1}" + list="${2}" + + # Query all regex from the corresponding database tables + mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_${list}" 2> /dev/null) + + # If we have regexps to process + if [[ "${#regexList[@]}" -ne 0 ]]; then + # Split regexps over a new line + str_regexList=$(printf '%s\n' "${regexList[@]}") + # Check domain against regexps + mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex") + # If there were regex matches + if [[ "${#regexMatches[@]}" -ne 0 ]]; then + # Split matching regexps over a new line + str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") + # Form a "matched" message + str_message="${matchType^} found in ${COL_BOLD}Regex ${list}${COL_NC}" + # Form a "results" message + str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" + # If we are displaying more than just the source of the block + if [[ -z "${blockpage}" ]]; then + # Set the wildcard match flag + wcMatch=true + # Echo the "matched" message, indented by one space + echo " ${str_message}" + # Echo the "results" message, each line indented by three spaces + # shellcheck disable=SC2001 + echo "${str_result}" | sed 's/^/ /' + else + echo "π .wildcard" + exit 0 + fi + fi + fi +} + # Scan Whitelist and Blacklist scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" # Scan Regex table -mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null) - -# If we have regexps to process -if [[ "${#regexList[@]}" -ne 0 ]]; then - # Split regexps over a new line - str_regexList=$(printf '%s\n' "${regexList[@]}") - # Check domainQuery against regexps - mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexList}" "regex") - # If there were regex matches - if [[ "${#regexMatches[@]}" -ne 0 ]]; then - # Split matching regexps over a new line - str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") - # Form a "matched" message - str_message="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}" - # Form a "results" message - str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" - # If we are displaying more than just the source of the block - if [[ -z "${blockpage}" ]]; then - # Set the wildcard match flag - wcMatch=true - # Echo the "matched" message, indented by one space - echo " ${str_message}" - # Echo the "results" message, each line indented by three spaces - # shellcheck disable=SC2001 - echo "${str_result}" | sed 's/^/ /' - else - echo "π .wildcard" - exit 0 - fi - fi -fi +scanRegexDatabaseTable "${domainQuery}" "whitelist" +scanRegexDatabaseTable "${domainQuery}" "blacklist" # Get version sorted *.domains filenames (without dir path) lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)") diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9891fd9d..25c66ab7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1910,6 +1910,9 @@ installPihole() { chmod a+rx /var/www/html # Give pihole access to the Web server group usermod -a -G ${LIGHTTPD_GROUP} pihole + # Give lighttpd access to the pihole group so the web interface can + # manage the gravity.db database + usermod -a -G pihole ${LIGHTTPD_USER} # If the lighttpd command is executable, if is_command lighty-enable-mod ; then # enable fastcgi and fastcgi-php diff --git a/gravity.sh b/gravity.sh index 14b32827..2fc26d49 100755 --- a/gravity.sh +++ b/gravity.sh @@ -84,6 +84,10 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { sqlite3 "${gravityDBfile}" < "${gravityDBschema}" + + # Ensure proper permissions are set for the newly created database + chown pihole:pihole "${gravityDBfile}" + chmod g+w "${piholeDir}" "${gravityDBfile}" } # Import domains from file and store them in the specified database table @@ -181,6 +185,8 @@ migrate_to_database() { fi if [ -e "${regexFile}" ]; then # Store regex domains in database + # Important note: We need to add the domains to the "regex" table + # as it will only later be renamed to "regex_blacklist"! echo -e " ${INFO} Migrating content of ${regexFile} into new database" database_table_from_file "regex" "${regexFile}" fi @@ -590,9 +596,10 @@ gravity_Table_Count() { # Output count of blacklisted domains and regex filters gravity_ShowCount() { - gravity_Table_Count "blacklist" "blacklisted domains" - gravity_Table_Count "whitelist" "whitelisted domains" - gravity_Table_Count "regex" "regex filters" + gravity_Table_Count "blacklist" "exact blacklisted domains" + gravity_Table_Count "regex_blacklist" "regex blacklist filters" + gravity_Table_Count "whitelist" "exact whitelisted domains" + gravity_Table_Count "regex_whitelist" "regex whitelist filters" } # Parse list of domains into hosts format diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 065280c7..ed012092 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -66,14 +66,24 @@ Available commands and options: Adds or removes specified domain or domains to the blacklist .br +\fB--regex, regex\fR [options] [ ] +.br + Add or removes specified regex filter to the regex blacklist +.br + +\fB--white-regex\fR [options] [ ] +.br + Add or removes specified regex filter to the regex whitelist +.br + \fB--wild, wildcard\fR [options] [ ] .br Add or removes specified domain to the wildcard blacklist .br -\fB--regex, regex\fR [options] [ ] +\fB--white-wild\fR [options] [ ] .br - Add or removes specified regex filter to the regex blacklist + Add or removes specified domain to the wildcard whitelist .br (Whitelist/Blacklist manipulation options): diff --git a/pihole b/pihole index 9fa65a8f..1d9f0809 100755 --- a/pihole +++ b/pihole @@ -375,8 +375,10 @@ Add '-h' after specific commands for more information on usage Whitelist/Blacklist Options: -w, whitelist Whitelist domain(s) -b, blacklist Blacklist domain(s) - --wild, wildcard Wildcard blacklist domain(s) - --regex, regex Regex blacklist domains(s) + --regex, regex Regex blacklist domains(s) + --white-regex Regex whitelist domains(s) + --wild, wildcard Wildcard blacklist domain(s) + --white-wild Wildcard whitelist domain(s) Add '-h' for more info on whitelist/blacklist usage Debugging Options: @@ -438,6 +440,8 @@ case "${1}" in "-b" | "blacklist" ) listFunc "$@";; "--wild" | "wildcard" ) listFunc "$@";; "--regex" | "regex" ) listFunc "$@";; + "--white-regex" | "white-regex" ) listFunc "$@";; + "--white-wild" | "white-wild" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";;