mirror/eworm-de-routeros-scripts
1
0
Fork 0
mirror of https://github.com/eworm-de/routeros-scripts.git synced 2024-05-11 05:55:19 +00:00
Code Releases Activity

Merge branch 'fw-addr-lists' into next

Browse Source
This commit is contained in:
Christian Hesse
2024-04-06 01:14:34 +02:00
parent fadc59c562 c4d2ea19dd
commit d430d32a30
2 changed files with 65 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
doc/fw-addr-lists.md
View File

@ -4,7 +4,7 @@ Download, import and update firewall address-lists
[![GitHub stars](https://img.shields.io/github/stars/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=red)](https://github.com/eworm-de/routeros-scripts/stargazers) [![GitHub stars](https://img.shields.io/github/stars/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=red)](https://github.com/eworm-de/routeros-scripts/stargazers)
[![GitHub forks](https://img.shields.io/github/forks/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=green)](https://github.com/eworm-de/routeros-scripts/network) [![GitHub forks](https://img.shields.io/github/forks/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=green)](https://github.com/eworm-de/routeros-scripts/network)
[![GitHub watchers](https://img.shields.io/github/watchers/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=blue)](https://github.com/eworm-de/routeros-scripts/watchers) [![GitHub watchers](https://img.shields.io/github/watchers/eworm-de/routeros-scripts?logo=GitHub&style=flat&color=blue)](https://github.com/eworm-de/routeros-scripts/watchers)
[![required RouterOS version](https://img.shields.io/badge/RouterOS-7.12-yellow?style=flat)](https://mikrotik.com/download/changelogs/) [![required RouterOS version](https://img.shields.io/badge/RouterOS-7.13-yellow?style=flat)](https://mikrotik.com/download/changelogs/)
[![Telegram group @routeros_scripts](https://img.shields.io/badge/Telegram-%40routeros__scripts-%2326A5E4?logo=telegram&style=flat)](https://t.me/routeros_scripts) [![Telegram group @routeros_scripts](https://img.shields.io/badge/Telegram-%40routeros__scripts-%2326A5E4?logo=telegram&style=flat)](https://t.me/routeros_scripts)
[![donate with PayPal](https://img.shields.io/badge/Like_it%3F-Donate!-orange?logo=githubsponsors&logoColor=orange&style=flat)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A4ZXBD6YS2W8J) [![donate with PayPal](https://img.shields.io/badge/Like_it%3F-Donate!-orange?logo=githubsponsors&logoColor=orange&style=flat)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A4ZXBD6YS2W8J)
@ -29,6 +29,10 @@ see situation when the lists are not populated.
To mitigate man-in-the-middle attacks with altered lists the server's To mitigate man-in-the-middle attacks with altered lists the server's
certificate is checked. certificate is checked.
> ⚠️ **Warning**: The script does not limit the size of a list, but keep in
> mind that huge lists can exhaust your device's resources (RAM and CPU),
> and may take a long time to process.
Requirements and installation Requirements and installation
----------------------------- -----------------------------

96
fw-addr-lists.rsc
View File

@ -3,7 +3,7 @@
# Copyright (c) 2023-2024 Christian Hesse <mail@eworm.de> # Copyright (c) 2023-2024 Christian Hesse <mail@eworm.de>
# https://git.eworm.de/cgit/routeros-scripts/about/COPYING.md # https://git.eworm.de/cgit/routeros-scripts/about/COPYING.md
# #
# requires RouterOS, version=7.12 # requires RouterOS, version=7.13
# #
# download, import and update firewall address-lists # download, import and update firewall address-lists
# https://git.eworm.de/cgit/routeros-scripts/about/doc/fw-addr-lists.md # https://git.eworm.de/cgit/routeros-scripts/about/doc/fw-addr-lists.md
@ -19,7 +19,8 @@
:global CertificateAvailable; :global CertificateAvailable;
:global EitherOr; :global EitherOr;
:global FetchUserAgentStr; :global FetchHuge;
:global HumanReadableNum;
:global LogPrint; :global LogPrint;
:global LogPrintOnce; :global LogPrintOnce;
:global ScriptLock; :global ScriptLock;
@ -50,25 +51,25 @@
:local Failure false; :local Failure false;
:foreach List in=$FwList do={ :foreach List in=$FwList do={
:local CheckCertificate "no"; :local CheckCertificate false;
:local Data false; :local Data false;
:local TimeOut [ $EitherOr [ :totime ($List->"timeout") ] $FwAddrListTimeOut ]; :local TimeOut [ $EitherOr [ :totime ($List->"timeout") ] $FwAddrListTimeOut ];
:if ([ :len ($List->"cert") ] > 0) do={ :if ([ :len ($List->"cert") ] > 0) do={
:set CheckCertificate "yes-without-crl"; :set CheckCertificate true;
:if ([ $CertificateAvailable ($List->"cert") ] = false) do={ :if ([ $CertificateAvailable ($List->"cert") ] = false) do={
$LogPrint warning $ScriptName ("Downloading required certificate failed, trying anyway."); $LogPrint warning $ScriptName ("Downloading required certificate (" . $FwListName . \
" / " . $List->"url" . ") failed, trying anyway.");
} }
} }
:for I from=1 to=5 do={ :for I from=1 to=5 do={
:if ($Data = false) do={ :if ($Data = false) do={
:do { :set Data [ $FetchHuge $ScriptName ($List->"url") $CheckCertificate ];
:set Data ([ /tool/fetch check-certificate=$CheckCertificate output=user \ :if ($Data = false) do={
http-header-field=({ [ $FetchUserAgentStr $ScriptName ] }) ($List->"url") as-value ]->"data");
} on-error={
:if ($I < 5) do={ :if ($I < 5) do={
$LogPrint debug $ScriptName ("Failed downloading, " . $I . ". try: " . $List->"url"); $LogPrint debug $ScriptName ("Failed downloading for list '" . $FwListName . \
"', " . $I . ". try from: " . $List->"url");
:delay (($I * $I) . "s"); :delay (($I * $I) . "s");
} }
} }
@ -78,54 +79,67 @@
:if ($Data = false) do={ :if ($Data = false) do={
:set Data ""; :set Data "";
:set Failure true; :set Failure true;
$LogPrint warning $ScriptName ("Failed downloading list from: " . $List->"url"); $LogPrint warning $ScriptName ("Failed downloading for list '" . $FwListName . \
} "' from: " . $List->"url");
} else={
:if ([ :len $Data ] > 63000) do={ $LogPrint debug $ScriptName ("Downloaded " . [ $HumanReadableNum [ :len $Data ] 1024 ] . \
$LogPrintOnce warning $ScriptName ("The list is huge and may be truncated: " . $List->"url"); "B for list '" . $FwListName . "' from: " . $List->"url");
} }
:while ([ :len $Data ] != 0) do={ :while ([ :len $Data ] != 0) do={
:local Line [ :pick $Data 0 [ :find $Data "\n" ] ]; :local Line [ :pick $Data 0 [ :find $Data "\n" ] ];
:local Address ([ :pick $Line 0 [ $FindDelim $Line ] ] . ($List->"cidr")); :local Address ([ :pick $Line 0 [ $FindDelim $Line ] ] . ($List->"cidr"));
:if ($Address ~ "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(/[0-9]{1,2})?\$" || \ :do {
$Address ~ "^[\\.a-zA-Z0-9-]+\\.[a-zA-Z]{2,}\$") do={ :if ($Address ~ "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(/[0-9]{1,2})?\$") do={
:set ($IPv4Addresses->$Address) $TimeOut; :set ($IPv4Addresses->$Address) $TimeOut;
} :error true;
:if ($Address ~ "^[0-9a-zA-Z]*:[0-9a-zA-Z:\\.]+(/[0-9]{1,3})?\$" || \ }
$Address ~ "^[\\.a-zA-Z0-9-]+\\.[a-zA-Z]{2,}\$") do={ :if ($Address ~ "^[0-9a-zA-Z]*:[0-9a-zA-Z:\\.]+(/[0-9]{1,3})?\$") do={
:set ($IPv6Addresses->$Address) $TimeOut; :set ($IPv6Addresses->$Address) $TimeOut;
} :error true;
}
:if ($Address ~ "^[\\.a-zA-Z0-9-]+\\.[a-zA-Z]{2,}\$") do={
:set ($IPv4Addresses->$Address) $TimeOut;
:set ($IPv6Addresses->$Address) $TimeOut;
:error true;
}
} on-error={ }
:set Data [ :pick $Data ([ :len $Line ] + 1) [ :len $Data ] ]; :set Data [ :pick $Data ([ :len $Line ] + 1) [ :len $Data ] ];
} }
} }
:foreach Entry in=[ /ip/firewall/address-list/find where list=$FwListName comment=$ListComment ] do={ :foreach Entry in=[ /ip/firewall/address-list/find where \
list=$FwListName comment=$ListComment ] do={
:local Address [ /ip/firewall/address-list/get $Entry address ]; :local Address [ /ip/firewall/address-list/get $Entry address ];
:if ([ :typeof ($IPv4Addresses->$Address) ] = "time") do={ :if ([ :typeof ($IPv4Addresses->$Address) ] = "time") do={
$LogPrint debug $ScriptName ("Renewing IPv4 address for " . ($IPv4Addresses->$Address) . ": " . $Address); $LogPrint debug $ScriptName ("Renewing IPv4 address in list '" . $FwListName . \
"' with " . ($IPv4Addresses->$Address) . ": " . $Address);
/ip/firewall/address-list/set $Entry timeout=($IPv4Addresses->$Address); /ip/firewall/address-list/set $Entry timeout=($IPv4Addresses->$Address);
:set ($IPv4Addresses->$Address); :set ($IPv4Addresses->$Address);
:set CntRenew ($CntRenew + 1); :set CntRenew ($CntRenew + 1);
} else={ } else={
:if ($Failure = false) do={ :if ($Failure = false) do={
$LogPrint debug $ScriptName ("Removing IPv4 address: " . $Address); $LogPrint debug $ScriptName ("Removing IPv4 address from list '" . $FwListName . \
"': " . $Address);
/ip/firewall/address-list/remove $Entry; /ip/firewall/address-list/remove $Entry;
:set CntRemove ($CntRemove + 1); :set CntRemove ($CntRemove + 1);
} }
} }
} }
:foreach Entry in=[ /ipv6/firewall/address-list/find where list=$FwListName comment=$ListComment ] do={ :foreach Entry in=[ /ipv6/firewall/address-list/find where \
list=$FwListName comment=$ListComment ] do={
:local Address [ /ipv6/firewall/address-list/get $Entry address ]; :local Address [ /ipv6/firewall/address-list/get $Entry address ];
:if ([ :typeof ($IPv6Addresses->$Address) ] = "time") do={ :if ([ :typeof ($IPv6Addresses->$Address) ] = "time") do={
$LogPrint debug $ScriptName ("Renewing IPv6 address for " . ($IPv6Addresses->$Address) . ": " . $Address); $LogPrint debug $ScriptName ("Renewing IPv6 address in list '" . $FwListName . \
"' with " . ($IPv6Addresses->$Address) . ": " . $Address);
/ipv6/firewall/address-list/set $Entry timeout=($IPv6Addresses->$Address); /ipv6/firewall/address-list/set $Entry timeout=($IPv6Addresses->$Address);
:set ($IPv6Addresses->$Address); :set ($IPv6Addresses->$Address);
:set CntRenew ($CntRenew + 1); :set CntRenew ($CntRenew + 1);
} else={ } else={
:if ($Failure = false) do={ :if ($Failure = false) do={
$LogPrint debug $ScriptName ("Removing: " . $Address); $LogPrint debug $ScriptName ("Removing IPv6 address from list '" . $FwListName . \
"': " . $Address);
/ipv6/firewall/address-list/remove $Entry; /ipv6/firewall/address-list/remove $Entry;
:set CntRemove ($CntRemove + 1); :set CntRemove ($CntRemove + 1);
} }
@ -133,27 +147,37 @@
} }
:foreach Address,Timeout in=$IPv4Addresses do={ :foreach Address,Timeout in=$IPv4Addresses do={
$LogPrint debug $ScriptName ("Adding IPv4 address for " . $Timeout . ": " . $Address); $LogPrint debug $ScriptName ("Adding IPv4 address to list '" . $FwListName . \
"' with " . $Timeout . ": " . $Address);
:do { :do {
/ip/firewall/address-list/add list=$FwListName comment=$ListComment address=$Address timeout=$Timeout; /ip/firewall/address-list/add list=$FwListName comment=$ListComment \
address=$Address timeout=$Timeout;
:set ($IPv4Addresses->$Address); :set ($IPv4Addresses->$Address);
:set CntAdd ($CntAdd + 1); :set CntAdd ($CntAdd + 1);
} on-error={ } on-error={
$LogPrint warning $ScriptName ("Failed to add IPv4 address " . $Address . " to list '" . $FwListName . "'."); $LogPrint warning $ScriptName ("Failed to add IPv4 address to list '" . $FwListName . \
"': " . $Address);
} }
} }
:foreach Address,Timeout in=$IPv6Addresses do={ :foreach Address,Timeout in=$IPv6Addresses do={
$LogPrint debug $ScriptName ("Adding IPv6 address for " . $Timeout . ": " . $Address); $LogPrint debug $ScriptName ("Adding IPv6 address to list '" . $FwListName . \
"' with " . $Timeout . ": " . $Address);
:do { :do {
/ipv6/firewall/address-list/add list=$FwListName comment=$ListComment address=$Address timeout=$Timeout; /ipv6/firewall/address-list/add list=$FwListName comment=$ListComment \
address=$Address timeout=$Timeout;
:set ($IPv6Addresses->$Address); :set ($IPv6Addresses->$Address);
:set CntAdd ($CntAdd + 1); :set CntAdd ($CntAdd + 1);
} on-error={ } on-error={
$LogPrint warning $ScriptName ("Failed to add IPv6 address " . $Address . " to list '" . $FwListName . "'."); $LogPrint warning $ScriptName ("Failed to add IPv6 address to list '" . $FwListName . \
"': " . $Address);
} }
} }
$LogPrint info $ScriptName ("list: " . $FwListName . " -- added: " . $CntAdd . " - renewed: " . $CntRenew . " - removed: " . $CntRemove); $LogPrint info $ScriptName ("list: " . $FwListName . \
" (" . [ $HumanReadableNum ($CntAdd + $CntRenew) 1000 ] . ")" . \
" -- added: " . [ $HumanReadableNum $CntAdd 1000 ] . \
" - renewed: " . [ $HumanReadableNum $CntRenew 1000 ] . \
" - removed: " . [ $HumanReadableNum $CntRemove 1000 ]);
} }
} on-error={ } } on-error={ }