banip: release 0.7.1
authorDirk Brenken <dev@brenken.org>
Sat, 6 Feb 2021 21:28:27 +0000 (22:28 +0100)
committerDirk Brenken <dev@brenken.org>
Sun, 7 Feb 2021 07:30:54 +0000 (08:30 +0100)
* add 'ban_extrasources' to handle banIP-unrelated sets for reporting
  and queries
* add set timeouts for local sources (maclist, whitelist, blacklist)

Signed-off-by: Dirk Brenken <dev@brenken.org>
net/banip/Makefile
net/banip/files/README.md
net/banip/files/banip.sh

index 614487147d55c900d3d9e5021d310b7989139324..392f42e4706bc8f87c08bb2e2de06d3390068576 100644 (file)
@@ -6,7 +6,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=banip
-PKG_VERSION:=0.7.0
+PKG_VERSION:=0.7.1
 PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0-or-later
 PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>
index d8add1726e5a7ee9c45068ffefe22ef8528c170b..a9f2f41183e2d020bab61b52601bf4013d9661cc 100644 (file)
@@ -107,47 +107,49 @@ Available commands:
 ## banIP config options
 * Usually the auto pre-configured banIP setup works quite well and no manual overrides are needed
 
-| Option                  | Type   | Default             | Description                                                                                          |
-| :---------------------- | :----- | :------------------ | :--------------------------------------------------------------------------------------------------- |
-| ban_enabled             | option | 0                   | enable the banIP service                                                                             |
-| ban_autodetect          | option | 1                   | auto-detect wan interfaces, devices and subnets                                                      |
-| ban_debug               | option | 0                   | enable banIP related debug logging                                                                   |
-| ban_mail_enabled        | option | 0                   | enable the mail service                                                                              |
-| ban_monitor_enabled     | option | 0                   | enable the log monitor, e.g. to catch failed ssh/luci logins                                         |
-| ban_logsrc_enabled      | option | 0                   | enable the src-related logchain                                                                      |
-| ban_logdst_enabled      | option | 0                   | enable the dst-related logchain                                                                      |
-| ban_autoblacklist       | option | 1                   | add suspicious IPs automatically to the local blacklist                                              |
-| ban_autowhitelist       | option | 1                   | add wan IPs/subnets automatically to the local whitelist                                             |
-| ban_maxqueue            | option | 4                           | size of the download queue to handle downloads and processing in parallel             |
-| ban_reportdir           | option | /tmp/banIP-Report           | directory where banIP stores the report files                                         |
-| ban_backupdir           | option | /tmp/banIP-Backup           | directory where banIP stores the compressed backup files                              |
-| ban_ifaces              | list   | -                           | list option to add logical wan interfaces manually                                    |
-| ban_sources             | list   | -                           | list option to add banIP sources                                                      |
-| ban_countries           | list   | -                           | list option to add certain countries as an alpha-2 ISO code, e.g. 'de' for germany    |
-| ban_asns                | list   | -                           | list option to add certain ASNs (autonomous system number), e.g. '32934' for facebook |
-| ban_chain               | option | banIP                       | name of the root chain used by banIP                                                  |
-| ban_global_settype      | option | src+dst                     | global settype as default for all sources                                             |
-| ban_settype_src         | list   | -                           | special SRC settype for a certain sources                                             |
-| ban_settype_dst         | list   | -                           | special DST settype for a certain sources                                             |
-| ban_settype_all         | list   | -                           | special SRC+DST settype for a certain sources                                         |
-| ban_target_src          | option | DROP                        | default src action (used by log chains as well)                                       |
-| ban_target_dst          | option | REJECT                      | default dst action (used by log chains as well)                                       |
-| ban_lan_inputchains_4   | list   | input_lan_rule              | list option to add IPv4 lan input chains                                              |
-| ban_lan_inputchains_6   | list   | input_lan_rule              | list option to add IPv6 lan input chains                                              |
-| ban_lan_forwardchains_4 | list   | forwarding_lan_rule         | list option to add IPv4 lan forward chains                                            |
-| ban_lan_forwardchains_6 | list   | forwarding_lan_rule         | list option to add IPv6 lan forward chains                                            |
-| ban_wan_inputchains_4   | list   | input_wan_rule              | list option to add IPv4 wan input chains                                              |
-| ban_wan_inputchains_6   | list   | input_wan_rule              | list option to add IPv6 wan input chains                                              |
-| ban_wan_forwardchains_4 | list   | forwarding_wan_rule         | list option to add IPv4 wan forward chains                                            |
-| ban_wan_forwardchains_6 | list   | forwarding_wan_rule         | list option to add IPv6 wan forward chains                                            |
-| ban_mailreceiver        | option | -                           | receiver address for banIP related notification E-Mails                               |
-| ban_mailsender          | option | no-reply@banIP              | sender address for banIP related notification E-Mails                                 |
-| ban_mailtopic           | option | banIP notification          | topic for banIP related notification E-Mails                                          |
-| ban_mailprofile         | option | ban_notify                  | mail profile used in 'msmtp' for banIP related notification E-Mails                   |
-| ban_srcarc              | option | /etc/banip/banip.sources.gz | full path to the compressed source archive file used by banIP                         |
-| ban_maclist             | option | /etc/banip/banip.maclist    | full path to the maclist file used by banIP                                           |
-| ban_blacklist           | option | /etc/banip/banip.blacklist  | full path to the blacklist file used by banIP                                         |
-| ban_whitelist           | option | /etc/banip/banip.whitelist  | full path to the whitelist file used by banIP                                         |
+| Option                  | Type   | Default                       | Description                                                                           |
+| :---------------------- | :----- | :---------------------------- | :------------------------------------------------------------------------------------ |
+| ban_enabled             | option | 0                             | enable the banIP service                                                              |
+| ban_autodetect          | option | 1                             | auto-detect wan interfaces, devices and subnets                                       |
+| ban_debug               | option | 0                             | enable banIP related debug logging                                                    |
+| ban_mail_enabled        | option | 0                             | enable the mail service                                                               |
+| ban_monitor_enabled     | option | 0                             | enable the log monitor, e.g. to catch failed ssh/luci logins                          |
+| ban_logsrc_enabled      | option | 0                             | enable the src-related logchain                                                       |
+| ban_logdst_enabled      | option | 0                             | enable the dst-related logchain                                                       |
+| ban_autoblacklist       | option | 1                             | add suspicious IPs automatically to the local blacklist                               |
+| ban_autowhitelist       | option | 1                             | add wan IPs/subnets automatically to the local whitelist                              |
+| ban_maxqueue            | option | 4                             | size of the download queue to handle downloads and processing in parallel             |
+| ban_reportdir           | option | /tmp/banIP-Report             | directory where banIP stores the report files                                         |
+| ban_backupdir           | option | /tmp/banIP-Backup             | directory where banIP stores the compressed backup files                              |
+| ban_ifaces              | list   | -                             | list option to add logical wan interfaces manually                                    |
+| ban_sources             | list   | -                             | list option to add banIP sources                                                      |
+| ban_countries           | list   | -                             | list option to add certain countries as an alpha-2 ISO code, e.g. 'de' for germany    |
+| ban_asns                | list   | -                             | list option to add certain ASNs (autonomous system number), e.g. '32934' for facebook |
+| ban_chain               | option | banIP                         | name of the root chain used by banIP                                                  |
+| ban_global_settype      | option | src+dst                       | global settype as default for all sources                                             |
+| ban_settype_src         | list   | -                             | special SRC settype for a certain sources                                             |
+| ban_settype_dst         | list   | -                             | special DST settype for a certain sources                                             |
+| ban_settype_all         | list   | -                             | special SRC+DST settype for a certain sources                                         |
+| ban_target_src          | option | DROP                          | default src action (used by log chains as well)                                       |
+| ban_target_dst          | option | REJECT                        | default dst action (used by log chains as well)                                       |
+| ban_lan_inputchains_4   | list   | input_lan_rule                | list option to add IPv4 lan input chains                                              |
+| ban_lan_inputchains_6   | list   | input_lan_rule                | list option to add IPv6 lan input chains                                              |
+| ban_lan_forwardchains_4 | list   | forwarding_lan_rule           | list option to add IPv4 lan forward chains                                            |
+| ban_lan_forwardchains_6 | list   | forwarding_lan_rule           | list option to add IPv6 lan forward chains                                            |
+| ban_wan_inputchains_4   | list   | input_wan_rule                | list option to add IPv4 wan input chains                                              |
+| ban_wan_inputchains_6   | list   | input_wan_rule                | list option to add IPv6 wan input chains                                              |
+| ban_wan_forwardchains_4 | list   | forwarding_wan_rule           | list option to add IPv4 wan forward chains                                            |
+| ban_wan_forwardchains_6 | list   | forwarding_wan_rule           | list option to add IPv6 wan forward chains                                            |
+| ban_mailreceiver        | option | -                             | receiver address for banIP related notification E-Mails                               |
+| ban_mailsender          | option | no-reply@banIP                | sender address for banIP related notification E-Mails                                 |
+| ban_mailtopic           | option | banIP notification            | topic for banIP related notification E-Mails                                          |
+| ban_mailprofile         | option | ban_notify                    | mail profile used in 'msmtp' for banIP related notification E-Mails                   |
+| ban_srcarc              | option | /etc/banip/banip.sources.gz   | full path to the compressed source archive file used by banIP                         |
+| ban_localsources        | list   | maclist, whitelist, blacklist | limit the selection to certain local sources                                          |
+| ban_extrasources        | list   | -                             | add additional, non-banIP related IPSets e.g. for reporting or queries                |
+| ban_maclist_timeout     | option | -                             | individual maclist IPSet timeout                                                      |
+| ban_whitelist_timeout   | option | -                             | individual whitelist IPSet timeout                                                    |
+| ban_blacklist_timeout   | option | -                             | individual blacklist IPSet timeout                                                    |
   
 ## Examples
 **list/edit banIP sources:**
@@ -334,7 +336,7 @@ password        <password>
 </code></pre>
 Finally enable E-Mail support and add a valid E-Mail receiver address in LuCI.
   
-**Edit, add new adblock sources:**  
+**Edit, add new banIP sources:**  
 The banIP blocklist sources are stored in an external, compressed JSON file '/etc/banip/banip.sources.gz'. 
 This file is directly parsed in LuCI and accessible via CLI, just call _/etc/init.d/banip list_.
 
index 34cd90eae90f76515d65fc7db8ab38b81e631f00..aca1c8538a09fc179bdb0747a2156e5965dc6246 100755 (executable)
@@ -12,7 +12,7 @@
 export LC_ALL=C
 export PATH="/usr/sbin:/usr/bin:/sbin:/bin"
 set -o pipefail
-ban_ver="0.7.0"
+ban_ver="0.7.1"
 ban_enabled="0"
 ban_mail_enabled="0"
 ban_proto4_enabled="0"
@@ -151,6 +151,9 @@ f_conf()
                        elif [ "${option}" = "ban_localsources" ]
                        then
                                eval "${option}=\"$(printf "%s" "${ban_localsources}")${value} \""
+                       elif [ "${option}" = "ban_extrasources" ]
+                       then
+                               eval "${option}=\"$(printf "%s" "${ban_extrasources}")${value} \""
                        elif [ "${option}" = "ban_settype_src" ]
                        then
                                eval "${option}=\"$(printf "%s" "${ban_settype_src}")${value} \""
@@ -233,7 +236,7 @@ f_conf()
        ban_logterms="${ban_logterms:-"dropbear sshd luci"}"
        f_log "debug" "f_conf  ::: ifaces: ${ban_ifaces:-"-"}, chain: ${ban_chain}, set_type: ${ban_global_settype}, log_chains (src/dst): ${ban_logchain_src}/${ban_logchain_dst}, targets (src/dst): ${ban_target_src}/${ban_target_dst}"
        f_log "debug" "f_conf  ::: lan_inputs (4/6): ${ban_lan_inputchains_4}/${ban_lan_inputchains_6}, lan_forwards (4/6): ${ban_lan_forwardchains_4}/${ban_lan_forwardchains_6}, wan_inputs (4/6): ${ban_wan_inputchains_4}/${ban_wan_inputchains_6}, wan_forwards (4/6): ${ban_wan_forwardchains_4}/${ban_wan_forwardchains_6}"
-       f_log "debug" "f_conf  ::: local_sources: ${ban_localsources:-"-"}, log_terms: ${ban_logterms:-"-"}, log_prefixes (src/dst): ${ban_logprefix_src}/${ban_logprefix_dst}, log_options (src/dst): ${ban_logopts_src}/${ban_logopts_dst}"
+       f_log "debug" "f_conf  ::: local_sources: ${ban_localsources:-"-"}, extra_sources: ${ban_extrasources:-"-"}, log_terms: ${ban_logterms:-"-"}, log_prefixes (src/dst): ${ban_logprefix_src}/${ban_logprefix_dst}, log_options (src/dst): ${ban_logopts_src}/${ban_logopts_dst}"
 }
 
 # check environment
@@ -733,12 +736,22 @@ f_ipset()
                "create")
                        if [ "${src_name}" = "maclist" ] && [ -s "${tmp_file}" ] && [ -z "$("${ban_ipset_cmd}" -q -n list "${src_name}")" ]
                        then
-                               "${ban_ipset_cmd}" create "${src_name}" hash:mac maxelem 262144 counters
+                               "${ban_ipset_cmd}" create "${src_name}" hash:mac maxelem 262144 counters timeout "${ban_maclist_timeout:-"0"}"
                                out_rc="${?}"
                        elif [ -s "${tmp_file}" ] && [ -z "$("${ban_ipset_cmd}" -q -n list "${src_name}")" ]
                        then
-                               "${ban_ipset_cmd}" create "${src_name}" hash:net hashsize 64 maxelem 262144 family "${src_ipver}" counters
-                               out_rc="${?}"
+                               if [ "${src_name%_*}" = "whitelist" ]
+                               then
+                                       "${ban_ipset_cmd}" create "${src_name}" hash:net hashsize 64 maxelem 262144 family "${src_ipver}" counters timeout "${ban_whitelist_timeout:-"0"}"
+                                       out_rc="${?}"
+                               elif [ "${src_name%_*}" = "blacklist" ]
+                               then
+                                       "${ban_ipset_cmd}" create "${src_name}" hash:net hashsize 64 maxelem 262144 family "${src_ipver}" counters timeout "${ban_blacklist_timeout:-"0"}"
+                                       out_rc="${?}"
+                               else
+                                       "${ban_ipset_cmd}" create "${src_name}" hash:net hashsize 64 maxelem 262144 family "${src_ipver}" counters
+                                       out_rc="${?}"
+                               fi
                        else
                                "${ban_ipset_cmd}" -q flush "${src_name}"
                                out_rc="${?}"
@@ -1338,7 +1351,7 @@ f_query()
                query_start="$(date "+%s")"
                printf "%s\n%s\n%s\n" ":::" "::: search '${search}' in banIP related IPSets" ":::"
 
-               for src in ${ban_localsources} ${ban_sources}
+               for src in ${ban_localsources} ${ban_sources} ${ban_extrasources}
                do
                        if [ "${src}" = "maclist" ] && [ -n "$("${ban_ipset_cmd}" -q -n list "${src}")" ]
                        then
@@ -1394,19 +1407,24 @@ f_report()
                > "${report_txt}"
                printf "%s\n" "{" >> "${report_json}"
                printf "\t%s\n" "\"ipsets\": {" >> "${report_json}"
-               for src in ${ban_localsources} ${ban_sources}
+               for src in ${ban_localsources} ${ban_sources} ${ban_extrasources}
                do
-                       if [ -n "$(printf "%s\n" "${ban_settype_src}" | grep -F "${src}")" ]
-                       then
-                               set_type="src"
-                       elif [ -n "$(printf "%s\n" "${ban_settype_dst}" | grep -F "${src}")" ]
+                       if [ -n "$(printf "%s" "${ban_extrasources}" | grep -F "${src}")" ]
                        then
-                               set_type="dst"
-                       elif [ -n "$(printf "%s\n" "${ban_settype_all}" | grep -F "${src}")" ]
-                       then
-                               set_type="src+dst"
+                               set_type="n/a"
                        else
-                               set_type="${ban_global_settype}"
+                               if [ -n "$(printf "%s\n" "${ban_settype_src}" | grep -F "${src}")" ]
+                               then
+                                       set_type="src"
+                               elif [ -n "$(printf "%s\n" "${ban_settype_dst}" | grep -F "${src}")" ]
+                               then
+                                       set_type="dst"
+                               elif [ -n "$(printf "%s\n" "${ban_settype_all}" | grep -F "${src}")" ]
+                               then
+                                       set_type="src+dst"
+                               else
+                                       set_type="${ban_global_settype}"
+                               fi
                        fi
                        if [ "${src}" = "maclist" ]
                        then
@@ -1414,7 +1432,7 @@ f_report()
                                if [ -n "${src_list}" ]
                                then
                                        cnt="$(printf "%s" "${src_list}" | awk '/^Number of entries:/{print $4}')"
-                                       cnt_acc="$(printf "%s" "${src_list}" | grep -cE "^(([0-9A-Z][0-9A-Z]:){5}[0-9A-Z]{2} packets [1-9]+)")"
+                                       cnt_acc="$(printf "%s" "${src_list}" | grep -cE " packets [1-9]+")"
                                        cnt_acc_sum=$((cnt_acc_sum+cnt_acc))
                                        cnt_mac_sum="${cnt}"
                                        cnt_sum=$((cnt_sum+cnt))
@@ -1430,7 +1448,7 @@ f_report()
                                        printf "\t\t\t%s\n" "\"count_mac\": \"${cnt}\"," >> "${report_json}"
                                        printf "\t\t\t%s" "\"count_acc\": \"${cnt_acc}\"" >> "${report_json}"
                                        printf ",\n\t\t\t%s" "\"member_acc\": [" >> "${report_json}"
-                                       printf "%s" "${src_list}" | awk '/^(([0-9A-Z][0-9A-Z]:){5}[0-9A-Z]{2} packets [1-9]+)/{print $1,$3}' | \
+                                       printf "%s" "${src_list}" | awk 'match($0,/ packets [1-9]+/){printf "%s %s\n",$1,substr($0,RSTART+9,RLENGTH-9)}' | \
                                                awk 'BEGIN{i=0};{i=i+1;if(i==1){printf "\n\t\t\t\t\t{\n\t\t\t\t\t\t\"member\": \"%s\",\n\t\t\t\t\t\t\"packets\": \"%s\"\n\t\t\t\t\t}",$1,$2}else{printf ",\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"member\": \"%s\",\n\t\t\t\t\t\t\t\"packets\": \"%s\"\n\t\t\t\t\t\t}",$1,$2}}' >> "${report_json}"
                                        printf "\n\t\t\t%s\n" "]" >> "${report_json}"
                                        printf "\t\t%s" "}" >> "${report_json}"
@@ -1443,9 +1461,9 @@ f_report()
                                        if [ -n "${src_list}" ]
                                        then
                                                cnt="$(printf "%s\n" "${src_list}" | awk '/^Number of entries:/{print $4}')"
-                                               cnt_cidr="$(printf "%s\n" "${src_list}" | grep -cE "(/[0-9]{1,3} packets)")"
+                                               cnt_cidr="$(printf "%s\n" "${src_list}" | grep -cE "/[0-9]{1,3} packets [0-9]+")"
                                                cnt_ip=$((cnt-cnt_cidr-cnt_mac))
-                                               cnt_acc="$(printf "%s\n" "${src_list}" | grep -cE "( packets [1-9]+)")"
+                                               cnt_acc="$(printf "%s\n" "${src_list}" | grep -cE " packets [1-9]+")"
                                                cnt_cidr_sum=$((cnt_cidr_sum+cnt_cidr))
                                                cnt_ip_sum=$((cnt_ip_sum+cnt_ip))
                                                cnt_acc_sum=$((cnt_acc_sum+cnt_acc))
@@ -1462,7 +1480,7 @@ f_report()
                                                printf "\t\t\t%s\n" "\"count_mac\": \"0\"," >> "${report_json}"
                                                printf "\t\t\t%s" "\"count_acc\": \"${cnt_acc}\"" >> "${report_json}"
                                                printf ",\n\t\t\t%s" "\"member_acc\": [" >> "${report_json}"
-                                               printf "%s" "${src_list}" | awk '/( packets [1-9]+)/{print $1,$3}' | \
+                                               printf "%s" "${src_list}" | awk 'match($0,/ packets [1-9]+/){printf "%s %s\n",$1,substr($0,RSTART+9,RLENGTH-9)}' | \
                                                        awk 'BEGIN{i=0};{i=i+1;if(i==1){printf "\n\t\t\t\t\t{\n\t\t\t\t\t\t\"member\": \"%s\",\n\t\t\t\t\t\t\"packets\": \"%s\"\n\t\t\t\t\t}",$1,$2}else{printf ",\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\"member\": \"%s\",\n\t\t\t\t\t\t\t\"packets\": \"%s\"\n\t\t\t\t\t\t}",$1,$2}}' >> "${report_json}"
                                                printf "\n\t\t\t%s\n" "]" >> "${report_json}"
                                                printf "\t\t%s" "}" >> "${report_json}"