simple-adblock: update to 1.9.4-1
authorStan Grishin <stangri@melmac.ca>
Thu, 16 Feb 2023 22:59:10 +0000 (22:59 +0000)
committerStan Grishin <stangri@melmac.ca>
Thu, 16 Feb 2023 23:10:22 +0000 (23:10 +0000)
* update default config for new oisd.nl lists
* conf.update file to migrate oisd.nl lists to the new format
* introduce AdBlockPlus lists support (new oisd.nl format)
* longer wait for WAN up/gateway detection
* make load_environemnt only execute once to suppress duplicate
  warnings/errors

PS. While I was testing this, oisd.nl has brought back the old domains
    lists as well, so this version supports both as I'm unclear as to
    why the "big" ABPlus list is only 6.2Mb where as the "big" domains
    list is whopping 19.9Mb.

Signed-off-by: Stan Grishin <stangri@melmac.ca>
net/simple-adblock/Makefile
net/simple-adblock/files/simple-adblock.conf.update
net/simple-adblock/files/simple-adblock.init

index 497c6c3bd0bc1cc6d080f1f19ceb631e76200a7d..012b84fe2acd40b03140eee688eafdeec9eedcc7 100644 (file)
@@ -5,8 +5,8 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=simple-adblock
-PKG_VERSION:=1.9.3
-PKG_RELEASE:=7
+PKG_VERSION:=1.9.4
+PKG_RELEASE:=1
 PKG_MAINTAINER:=Stan Grishin <stangri@melmac.ca>
 PKG_LICENSE:=GPL-3.0-or-later
 
index 2d42d1712e1aeb57847a8e57600df2d6fcff914d..3a3d7beacdb355f61a709cd5bd3eedb889285f35 100644 (file)
@@ -8,8 +8,16 @@ s|raw.githubusercontent.com/StevenBlack/hosts/|cdn.jsdelivr.net/gh/StevenBlack/h
 s|raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/|cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@|g
 s|raw.githubusercontent.com/jawz101/MobileAdTrackers/|cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@|g
 s|http://winhelp2002.mvps.org/hosts.txt|https://winhelp2002.mvps.org/hosts.txt|g
+s|list blocked_domains_url 'https://dbl.oisd.nl/basic'|list blocked_adblockplus_url 'https://small.oisd.nl/'|g
+s|list blocked_domains_url 'https://dbl.oisd.nl/nsfw'|list blocked_adblockplus_url 'https://nsfw.oisd.nl/'|g
+s|list blocked_domains_url 'https://dbl.oisd.nl/'|list blocked_adblockplus_url 'https://big.oisd.nl/'|g
+s|list blocked_hosts_url 'https://hosts.oisd.nl/basic'|list blocked_adblockplus_url 'https://small.oisd.nl/'|g
+s|list blocked_hosts_url 'https://hosts.oisd.nl/nsfw'|list blocked_adblockplus_url 'https://nsfw.oisd.nl/'|g
+s|list blocked_hosts_url 'https://hosts.oisd.nl/'|list blocked_adblockplus_url 'https://big.oisd.nl/'|g
 \|dshield.org|d
 \|www.malwaredomainlist.com/hostslist/hosts.txt|d
 \|https://mirror1.malwaredomains.com/files/justdomains|d
 \|lists.disconnect.me|d
 \|https://cdn.jsdelivr.net/gh/paulgb/BarbBlock/blacklists/domain-list.txt|d
+\|dnsmasq.oisd.nl|d
+\|dnsmasq2.oisd.nl|d
index 89ef3fd7bd079ed65f94bf14c416176cf4b6d19f..0876e218c5f47327bba30b1af3c13a63a1e954ff 100644 (file)
@@ -62,7 +62,8 @@ readonly sharedMemoryError="/dev/shm/$packageName-error"
 readonly sharedMemoryOutput="/dev/shm/$packageName-output"
 readonly hostsFilter='/localhost/d;/^#/d;/^[^0-9]/d;s/^0\.0\.0\.0.//;s/^127\.0\.0\.1.//;s/[[:space:]]*#.*$//;s/[[:cntrl:]]$//;s/[[:space:]]//g;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;'
 readonly domainsFilter='/^#/d;s/[[:space:]]*#.*$//;s/[[:space:]]*$//;s/[[:cntrl:]]$//;/[[:space:]]/d;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;'
-readonly dnsmasqOISDFilter='\|^server=/[[:alnum:]_.-].*/|!d'
+readonly adBlockPlusFilter='/^#/d;/^!/d;s/[[:space:]]*#.*$//;s/^||//;s/\^$//;s/[[:space:]]*$//;s/[[:cntrl:]]$//;/[[:space:]]/d;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;'
+readonly dnsmasqFileFilter='\|^server=/[[:alnum:]_.-].*/|!d'
 readonly _OK_='\033[0;32m\xe2\x9c\x93\033[0m'
 readonly _FAIL_='\033[0;31m\xe2\x9c\x97\033[0m'
 readonly __OK__='\033[0;32m[\xe2\x9c\x93]\033[0m'
@@ -76,6 +77,20 @@ readonly nft="$(command -v nft)"
 readonly canaryDomainsMozilla='use-application-dns.net'
 readonly canaryDomainsiCloud='mask.icloud.com mask-h2.icloud.com'
 
+dl_command=
+dl_flag=
+isSSLSupported=
+outputFilter=
+outputFilterIPv6=
+outputFile=
+outputGzip=
+outputCache=
+awk='awk'
+load_environment_flag=
+
+. /lib/functions/network.sh
+. /usr/share/libubox/jshn.sh
+
 debug() { local i j; for i in "$@"; do eval "j=\$$i"; echo "${i}: ${j} "; done; }
 
 uci_add_list_if_new() {
@@ -112,8 +127,7 @@ get_text() {
                errorNoDnsmasqNftset) 
                        r="dnsmasq nft set support is enabled in $packageName, but dnsmasq is either not installed or installed dnsmasq does not support nft set";;
                errorNoNft) r="dnsmasq nft sets support is enabled in $packageName, but nft is not installed";;
-               errorMkdirFail) r="Unable to create directory for";;
-               errorNoWanGateway) r="The ${serviceName} service failed to discover WAN gateway!";;
+               errorNoWanGateway) r="The ${serviceName} failed to discover WAN gateway";;
                errorOutputDirCreate) r="failed to create directory for %s file";;
                errorOutputFileCreate) r="failed to create $outputFile file";;
                errorFailDNSReload) r="failed to restart/reload DNS resolver";;
@@ -193,9 +207,53 @@ output() {
        fi
 }
 
+load_network() {
+       local param="$1"
+       local i j wan_if wan_gw wan_proto
+       local counter wan_if_timeout='20' wan_gw_timeout='5'
+       counter=0
+       while [ -z "$wan_if" ]; do
+               network_flush_cache
+               network_find_wan wan_if
+               if [ -n "$wan_if" ]; then
+                       output "WAN Interface found: '${wan_if}'.\\n"
+                       break
+               fi
+               if [ "$counter" -gt "$wan_if_timeout" ]; then
+                       output "WAN Interface timeout, assuming 'wan'.\\n"
+                       wan_if='wan'
+                       break
+               fi
+               counter=$((counter+1))
+               output "Waiting to discover WAN Interface...\\n"
+               sleep 1
+       done
+
+       counter=0
+       wan_proto="$(uci -q get "network.${wan_if}.proto")"
+       if [ "$wan_proto" = 'pppoe' ]; then
+               wan_gw_timeout=$((wan_gw_timeout+10))
+       fi
+       while [ "$counter" -le "$wan_gw_timeout" ]; do
+               network_flush_cache
+               network_get_gateway wan_gw "$wan_if"
+               if [ -n "$wan_gw" ]; then
+                       output "WAN Gateway found: '${wan_gw}.'\\n"
+                       return 0
+               fi
+               counter=$((counter+1))
+               output "Waiting to discover $wan_if Gateway...\\n"
+               sleep 1
+       done
+       json add error "errorNoWanGateway"
+       output "${_ERROR_}: $(get_text 'errorNoWanGateway')!\\n"; return 1;
+}
+
 load_environment() {
-       local i j wan_if wan_gw
-       local validation_result="$1" quiet="$2"
+       local i j
+       local validation_result="$1" param="$2"
+
+       [ -z "$load_environment_flag" ] || return 0
 
        if [ "$validation_result" != '0' ]; then
                json add error "errorConfigValidationFail"
@@ -222,7 +280,7 @@ load_environment() {
                case "$dns" in
                        dnsmasq.conf) :;;
                        *)
-                               if [ -z "$quiet" ]; then
+                               if [ "$param" != 'quiet' ]; then
                                        json add warning "warningExternalDnsmasqConfig"
                                        output "${_WARNING_}: $(get_text 'warningExternalDnsmasqConfig')!\\n"
                                fi
@@ -243,14 +301,14 @@ load_environment() {
        case "$dns" in
                dnsmasq.ipset)
                        if dnsmasq -v 2>/dev/null | grep -q 'no-ipset' || ! dnsmasq -v 2>/dev/null | grep -q -w 'ipset'; then
-                               if [ -z "$quiet" ]; then
+                               if [ "$param" != 'quiet' ]; then
                                        json add error "errorNoDnsmasqIpset"
                                        output "${_ERROR_}: $(get_text 'errorNoDnsmasqIpset')!\\n"
                                fi
                                dns='dnsmasq.servers'
                        fi
                        if ! ipset help hash:net; then
-                               if [ -z "$quiet" ]; then
+                               if [ "$param" != 'quiet' ]; then
                                        json add error "errorNoIpset"
                                        output "${_ERROR_}: $(get_text 'errorNoIpset')!\\n"
                                fi
@@ -259,14 +317,14 @@ load_environment() {
                ;;
                dnsmasq.nftset)
                        if dnsmasq -v 2>/dev/null | grep -q 'no-nftset' || ! dnsmasq -v 2>/dev/null | grep -q -w 'nftset'; then
-                               if [ -z "$quiet" ]; then
+                               if [ "$param" != 'quiet' ]; then
                                        json add error "errorNoDnsmasqNftset"
                                        output "${_ERROR_}: $(get_text 'errorNoDnsmasqNftset')!\\n"
                                fi
                                dns='dnsmasq.servers'
                        fi
                        if [ -z "$nft" ]; then
-                               if [ -z "$quiet" ]; then
+                               if [ "$param" != 'quiet' ]; then
                                        json add error "errorNoNft"
                                        output "${_ERROR_}: $(get_text 'errorNoNft')!\\n"
                                fi
@@ -353,14 +411,13 @@ load_environment() {
 
        for i in "$outputFile" "$outputCache" "$outputGzip"; do
                if ! mkdir -p "$(dirname "$i")"; then
-                       json add error "errorOutputDirCreate" "$i"
-                       output "${_ERROR_}: $(get_text 'errorMkdirFail' "$i")!\\n"
+                       if [ "$param" != 'quiet' ]; then
+                               json add error "errorOutputDirCreate" "$i"
+                               output "${_ERROR_}: $(get_text 'errorOutputDirCreate' "$i")!\\n"
+                       fi
                fi
        done
 
-       . /lib/functions/network.sh
-       . /usr/share/libubox/jshn.sh
-
        is_present 'gawk' && awk='gawk'
        if ! is_present '/usr/libexec/grep-gnu' || ! is_present '/usr/libexec/sed-gnu' || \
                ! is_present '/usr/libexec/sort-coreutils' || ! is_present 'gawk'; then
@@ -369,8 +426,8 @@ load_environment() {
                        is_present '/usr/libexec/grep-gnu' || s="$s grep"
                        is_present '/usr/libexec/sed-gnu' || s="$s sed"
                        is_present '/usr/libexec/sort-coreutils' || s="$s coreutils-sort"
-                       if [ -z "$quiet" ]; then
-                               json add warning "errorOutputFileCreate" "${i}"
+                       if [ "$param" != 'quiet' ]; then
+                               json add warning "warningMissingRecommendedPackages" "${i}"
                                output "${_WARNING_}: $(get_text 'warningMissingRecommendedPackages'), install them by running:\\n"
                                output "$s;\\n"
                        fi
@@ -405,12 +462,15 @@ load_environment() {
        else
                unset isSSLSupported
        fi
+       load_environment_flag=1
        cache 'test' && return 0 
        cache 'test_gzip' && return 0
-       network_flush_cache; network_find_wan wan_if; network_get_gateway wan_gw "$wan_if";
-       [ -n "$wan_gw" ] && return 0
-       json add error "errorNoWanGateway"
-       output "${_ERROR_}: $(get_text 'errorNoWanGateway')!\\n"; return 1;
+       if [ "$param" = 'on_boot' ]; then
+               load_network "$param"
+               return "$?"
+       else
+               return 0
+       fi
 }
 
 resolver() {
@@ -555,7 +615,7 @@ json() {
                        case "$param" in
                                triggers)
                                        curReload="$parallel_downloads $debug $download_timeout $allowed_domain $blocked_domain $allowed_domains_url \
-                                               $blocked_domains_url $blocked_hosts_url $dns $config_update_enabled $config_update_url \
+                                               $blocked_adblockplus_url $blocked_domains_url $blocked_hosts_url $dns $config_update_enabled $config_update_url \
                                                $dnsmasq_config_file_url $curl_max_file_size $curl_retry"
                                        curRestart="$compressed_cache $force_dns $led $force_dns_port"
                                        if [ ! -s "$jsonFile" ]; then
@@ -589,7 +649,7 @@ json() {
                        case "$param" in
                                triggers) 
                                        reload="$parallel_downloads $debug $download_timeout $allowed_domain $blocked_domain $allowed_domains_url \
-                                               $blocked_domains_url $blocked_hosts_url $dns $config_update_enabled $config_update_url \
+                                               $blocked_adblockplus_url $blocked_domains_url $blocked_hosts_url $dns $config_update_enabled $config_update_url \
                                                $dnsmasq_config_file_url $curl_max_file_size $curl_retry"
                                        restart="$compressed_cache $force_dns $led $force_dns_port"
                                ;;
@@ -658,14 +718,22 @@ process_url() {
        if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then return 1; fi
        label="${1##*//}"; label="${label%%/*}";
        case "$2" in
-               dnsmasq) label="Dnsmasq: $label"; filter="$dnsmasqOISDFilter";;
-               domains) label="Domains: $label"; filter="$domainsFilter";;
-               hosts) label="Hosts:   $label"; filter="$hostsFilter";;
+               adbp)    label="ADBPlus: $label"; filter="$adBlockPlusFilter"
+               ;;
+               dnsmasq) label="Dnsmasq: $label"; filter="$dnsmasqFileFilter"
+               ;;
+               domains) label="Domains: $label"; filter="$domainsFilter"
+               ;;
+               hosts)   label="Hosts:   $label"; filter="$hostsFilter"
+               ;;
        esac
        case "$3" in
-               allowed) type='Allowed'; D_TMP="$A_TMP";;
-               blocked) type='Blocked'; D_TMP="$B_TMP";;
-               file) type='File'; D_TMP="$B_TMP";;
+               allowed) type='Allowed'; D_TMP="$A_TMP"
+               ;;
+               blocked) type='Blocked'; D_TMP="$B_TMP"
+               ;;
+               file) type='File'; D_TMP="$B_TMP"
+               ;;
        esac
        if [ "${1:0:5}" = "https" ] && [ -z "$isSSLSupported" ]; then
                output 1 "$_FAIL_"
@@ -676,7 +744,7 @@ process_url() {
        while [ -z "$R_TMP" ] || [ -e "$R_TMP" ]; do
                R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)"
        done
-       if ! $dl_command "$1" $dl_flag "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
+       if ! $dl_command "$1" "$dl_flag" "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
                output 1 "$_FAIL_"
                output 2 "[DL] $type $label $__FAIL__\\n"
                echo "errorDownloadingList|${1}" >> "$sharedMemoryError"
@@ -757,6 +825,13 @@ download_lists() {
                        process_url "$hf" 'hosts' 'blocked'
                fi
        done
+       for hf in ${blocked_adblockplus_url}; do
+               if [ "$parallel_downloads" -gt 0 ]; then
+                       process_url "$hf" 'adbp' 'blocked' &
+               else
+                       process_url "$hf" 'adbp' 'blocked'
+               fi
+       done
        for hf in ${blocked_domains_url}; do
                if [ "$parallel_downloads" -gt 0 ]; then
                        process_url "$hf" 'domains' 'blocked' &
@@ -1080,8 +1155,12 @@ adb_check() {
 
 adb_config_update() {
        local R_TMP label
-       local param="$1" validation_result="$3"
-       load_environment "$validation_result" 'quiet' || return 1
+       local param validation_result="$3"
+       case "$1" in
+               on_boot) param="$1";;
+               *) param='quiet';;
+       esac
+       load_environment "$validation_result" "$param" || return 1
        label="${config_update_url##*//}"
        label="${label%%/*}";
        [ "$config_update_enabled" -ne 0 ] || return 0
@@ -1094,7 +1173,7 @@ adb_config_update() {
        while [ -z "$R_TMP" ] || [ -e "$R_TMP" ]; do
                R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)"
        done
-       if ! $dl_command "$config_update_url" $dl_flag "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
+       if ! $dl_command "$config_update_url" "$dl_flag" "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
                output 1 "$_FAIL_\\n"
                output 2 "[DL] Config  Update:  $label $__FAIL__\\n"
                json add error "errorDownloadingConfigUpdate"
@@ -1118,9 +1197,27 @@ adb_sizes() {
        load_environment "$validation_result" 'quiet' || return 1
 
        echo "# $(date)"
-       for i in $blocked_domains_url; do
+       for i in ${blocked_adblockplus_url}; do
+               [ "${i//melmac}" != "$i" ] && continue
+               if $dl_command "$i" "$dl_flag" /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then
+                       echo "# File size: $(du -sh /tmp/sast | $awk '{print $1}')"
+                       if is_greater "$(du -sk /tmp/sast)" "500"; then
+                               echo "# block-list too big for most routers"
+                       elif is_greater "$(du -sk /tmp/sast)" "100"; then
+                               echo "# block-list may be too big for some routers"
+                       fi
+                       rm -rf /tmp/sast
+                       echo "  list blocked_adblockplus_url '$i'"
+                       echo ""
+               else
+                       echo "# site was down on last check"
+                       echo "#  list blocked_adblockplus_url '$i'"
+                       echo ""
+               fi
+       done
+       for i in ${blocked_domains_url}; do
                [ "${i//melmac}" != "$i" ] && continue
-               if $dl_command "$i" $dl_flag /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then
+               if $dl_command "$i" "$dl_flag" /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then
                        echo "# File size: $(du -sh /tmp/sast | $awk '{print $1}')"
                        if is_greater "$(du -sk /tmp/sast)" "500"; then
                                echo "# block-list too big for most routers"
@@ -1136,8 +1233,8 @@ adb_sizes() {
                        echo ""
                fi
        done
-       for i in $blocked_hosts_url; do
-               if $dl_command "$i" $dl_flag /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then
+       for i in ${blocked_hosts_url}; do
+               if $dl_command "$i" "$dl_flag" /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then
                        echo "# File size: $(du -sh /tmp/sast | $awk '{print $1}')"
                        if is_greater "$(du -sk /tmp/sast)" "500"; then
                                echo "# block-list too big for most routers"
@@ -1157,9 +1254,9 @@ adb_sizes() {
 
 adb_start() {
        local action status error message stats c
-       local validation_result="$3"
+       local param="$1" validation_result="$3"
 
-       load_environment "$validation_result" || return 1
+       load_environment "$validation_result" "$param" || return 1
 
        status="$(json get status)"
        error="$(json get error)"
@@ -1272,7 +1369,11 @@ adb_start() {
        procd_open_data
        json_add_string 'status' "$(json get status)"
        json_add_string 'errors' "$(json get errors)"
-       json_add_int 'entries' "$(wc -l < "$outputFile")"
+       if [ -s "$outputFile" ]; then
+               json_add_int 'entries' "$(wc -l < "$outputFile")"
+       else
+               json_add_int 'entries' '0'
+       fi
        json_add_array firewall
        if [ "$force_dns" -ne 0 ]; then
                for c in $force_dns_port; do
@@ -1460,15 +1561,6 @@ stop_service() { load_validate_config 'config' adb_stop "'$*'"; }
 version() { echo "$PKG_VERSION"; }
 
 load_validate_config() {
-       local dl_command
-       local dl_flag
-       local isSSLSupported
-       local outputFilter
-       local outputFilterIPv6
-       local outputFile
-       local outputGzip
-       local outputCache
-       local awk='awk'
        local enabled
        local force_dns
        local force_dns_port
@@ -1491,6 +1583,7 @@ load_validate_config() {
        local dns_instance
        local allowed_domain
        local allowed_domains_url
+       local blocked_adblockplus_url
        local blocked_domain
        local blocked_domains_url
        local blocked_hosts_url
@@ -1519,6 +1612,7 @@ load_validate_config() {
                'allowed_domain:list(string)' \
                'allowed_domains_url:list(string)' \
                'blocked_domain:list(string)' \
+               'blocked_adblockplus_url:list(string)' \
                'blocked_domains_url:list(string)' \
                'blocked_hosts_url:list(string)' \
                'dnsmasq_config_file_url:string'