diff --git a/pihole/blocklists/build/build.log b/pihole/blocklists/build/build.log index 4c4022e5..dda49e44 100755 --- a/pihole/blocklists/build/build.log +++ b/pihole/blocklists/build/build.log @@ -3,10 +3,10 @@ Initialize ... =========================================== Bereits aktuell. -[master b8f9f93] auto-update - 2 files changed, 585 insertions(+), 1682 deletions(-) - create mode 100644 pihole/blocklists/data/tmp - delete mode 100644 pihole/blocklists/data/white.list.notracking +Auf Branch master +Ihr Branch ist auf demselben Stand wie 'origin/master'. + +nichts zu committen, Arbeitsverzeichnis unverändert Bereits aktuell. Auf Branch main @@ -21,17 +21,17 @@ Ihr Branch ist auf demselben Stand wie 'origin/main'. nichts zu committen, Arbeitsverzeichnis unverändert Bereits aktuell. -[master 6a1a362] auto-update - 2 files changed, 106 insertions(+), 73 deletions(-) +[master 4a3e7ad] auto-update + 1 file changed, 98 insertions(+), 100 deletions(-) =========================================== Whitelist ... =========================================== -893 /media/nas/git/rpi/pihole/blocklists/data/white.list.referral +887 /media/nas/git/rpi/pihole/blocklists/data/white.list.referral 43 /media/nas/git/rpi/pihole/blocklists/data/white.list.shc -3100 /media/nas/git/rpi/pihole/blocklists/data/white.list.oisd +3108 /media/nas/git/rpi/pihole/blocklists/data/white.list.oisd 577 /media/nas/git/rpi/pihole/blocklists/data/white.list.adguard ### Compile exclude Blocklist ### @@ -89,25 +89,26 @@ Initialize ... 37 | 739 | white | http | online | unchanged | https://raw.githubusercontent.com/rspamd/maps/master/rspamd/dmarc_whitelist.inc 38 | 79 | white | http | online | unchanged | https://raw.githubusercontent.com/zebpalmer/dns_blocklists/master/whitelist.txt 39 | 678 | white | local | online | unchanged | white.list - 40 | 893 | white | local | online | changed | white.list.referral + 40 | 887 | white | local | online | changed | white.list.referral 41 | 15 | white | local | online | unchanged | white.list.important 42 | 577 | white | local | online | unchanged | white.list.adguard - 43 | 3100 | white | local | online | unchanged | white.list.oisd + 43 | 3108 | white | local | online | changed | white.list.oisd 44 | 43 | white | local | online | unchanged | white.list.shc # Build exclude Domainlist ... Stats exclude: -** Source (raw): 110596 -== Source (unique): 58493 (-52103) -++ FLD from SLD: 64837 (+6344) -++ FLD: 64929 (+92) -++ WWW: 92684 (+27755) +** Source (raw): 110598 +== Source (unique): 58922 (-51676) +++ FLD from SLD: 65273 (+6351) +++ FLD: 65364 (+91) +++ WWW: 93125 (+27761) -***************************************************** -* No changes to the previous repo version detected! * -***************************************************** +93125 unique Domains - Version 2021.1023.215016 +MD5 Domains RAW: ab5f4bbea6d07ec8be9af9cd6478a4e3 + +# Push exclude to local Repositories ... exclude - Done! @@ -154,21 +155,16 @@ Stats dead: dead - Done! Bereits aktuell. -[master 14c9121] white-dead auto-update - 3 files changed, 2 insertions(+), 586 deletions(-) - delete mode 100644 pihole/blocklists/data/tmp +[master 22e71fe] white-dead auto-update + 11 files changed, 468 insertions(+), 443 deletions(-) Bereits aktuell. -Auf Branch main -Ihr Branch ist auf demselben Stand wie 'origin/main'. - -nichts zu committen, Arbeitsverzeichnis unverändert +[main 3e78d1f] white-dead auto-update + 1 file changed, 1 insertion(+) Bereits aktuell. -Auf Branch main -Ihr Branch ist auf demselben Stand wie 'origin/main'. - -nichts zu committen, Arbeitsverzeichnis unverändert +[main 432288e] white-dead auto-update + 1 file changed, 1 insertion(+) =========================================== Black ... @@ -218,16 +214,20 @@ Stats black: -- White(*): 14649 (-630) ++ Block: 14668 (+19) -- Unblock: 14666 (-2) +-- Unblock(*): 14646 (-20) -***************************************************** -* No changes to the previous repo version detected! * -***************************************************** +14646 unique Domains - Version 2021.1023.215058 +MD5 Domains RAW: ce8a1edde3bfe78329d9c5ddaedf1c6f + +# Push black to local Repositories ... black - Done! Bereits aktuell. -[master 2867940] black auto-update - 1 file changed, 3 insertions(+), 3 deletions(-) +[master 39d6eca] black auto-update + 7 files changed, 47 insertions(+), 1305 deletions(-) + rewrite pihole/blocklists/build/black/out/black.domains.wildcard.removed (99%) + rewrite pihole/blocklists/data/black.list.wc.removed (99%) Bereits aktuell. Auf Branch main @@ -327,7 +327,7 @@ Initialize ... 2 | 407 | hosts | http | online | unchanged | https://curben.gitlab.io/malware-filter/pup-filter-hosts.txt 3 | 8884 | hosts | http | online | unchanged | https://curben.gitlab.io/malware-filter/urlhaus-filter-hosts.txt 4 | 3496 | hosts | http | online | unchanged | https://gitlab.com/ZeroDot1/CoinBlockerLists/raw/master/hosts_browser - 5 | 33669 | hosts | http | online | changed | https://hole.cert.pl/domains/domains_hosts.txt + 5 | 33675 | hosts | http | online | changed | https://hole.cert.pl/domains/domains_hosts.txt 6 | 550 | hosts | http | online | unchanged | https://paulgb.github.io/BarbBlock/blacklists/hosts-file.txt 7 | 5997 | hosts | http | online | unchanged | https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt 8 | 2204 | hosts | http | online | unchanged | https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts @@ -347,19 +347,19 @@ Initialize ... 22 | 549 | domains | http | online | unchanged | https://feeds.alphasoc.net/ryuk.txt 23 | 9233 | domains | http | online | unchanged | https://gitlab.com/KevinThomas0/cryptoscamdb-lists/-/raw/master/cryptoscamdb-blocklist.txt 24 | 365 | domains | http | online | unchanged | https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt - 25 | 33669 | domains | http | online | changed | https://hole.cert.pl/domains/domains.txt + 25 | 33675 | domains | http | online | changed | https://hole.cert.pl/domains/domains.txt 26 | 73323 | domains | http | online | unchanged | https://joewein.net/dl/bl/dom-bl-base.txt 27 | 767 | domains | http | online | unchanged | https://joewein.net/dl/bl/dom-bl.txt 28 | 2000 | domains | http | online | unchanged | https://kriskintel.com/feeds/ktip_covid_domains.txt - 29 | 1994 | domains | http | online | unchanged | https://kriskintel.com/feeds/ktip_malicious_domains.txt + 29 | 1998 | domains | http | online | unchanged | https://kriskintel.com/feeds/ktip_malicious_domains.txt 30 | 397 | domains | http | online | unchanged | https://kriskintel.com/feeds/ktip_ransomware_feeds.txt 31 | 2245 | domains | http | online | unchanged | https://orca.pet/notonmyshift/domains.txt 32 | 52 | domains | http | online | unchanged | https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt 33 | 44826 | domains | http | online | unchanged | https://phishing.army/download/phishing_army_blocklist.txt - 34 | 54300 | domains | http | online | unchanged | https://phishing.army/download/phishing_army_blocklist_extended.txt + 34 | 54484 | domains | http | online | unchanged | https://phishing.army/download/phishing_army_blocklist_extended.txt 35 | 1406 | domains | http | online | unchanged | https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/domains.txt 36 | 27 | domains | http | online | unchanged | https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/fake-domains.txt - 37 | 35404 | domains | http | online | unchanged | https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADomains.txt + 37 | 35464 | domains | http | online | unchanged | https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADomains.txt 38 | 675 | domains | http | online | unchanged | https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Cryptocurrency 39 | 22769 | domains | http | online | unchanged | https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Malware 40 | 179 | domains | http | online | unchanged | https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Risk @@ -373,7 +373,7 @@ Initialize ... 48 | 3239 | domains | http | online | unchanged | https://raw.githubusercontent.com/iam-py-test/my_filters_001/main/Alternative%20list%20formats/antimalware_domains.txt 49 | 2079 | domains | http | online | unchanged | https://raw.githubusercontent.com/matomo-org/referrer-spam-blacklist/master/spammers.txt 50 | 71289 | domains | http | online | unchanged | https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-ACTIVE.txt - 51 | 495 | domains | http | online | unchanged | https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-NEW-today.txt + 51 | 550 | domains | http | online | changed | https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-NEW-today.txt 52 | 10000 | domains | http | online | unchanged | https://raw.githubusercontent.com/prodaft/malware-ioc/master/FluBot/v3.7_5000_domain.txt 53 | 10000 | domains | http | online | unchanged | https://raw.githubusercontent.com/prodaft/malware-ioc/master/FluBot/v3.7_germany.txt 54 | 10000 | domains | http | online | unchanged | https://raw.githubusercontent.com/prodaft/malware-ioc/master/FluBot/v3.8_domains.txt @@ -400,21 +400,21 @@ Initialize ... 75 | 77 | domains | http | online | unchanged | https://www.botvrij.eu/data/ioclist.domain.raw 76 | 29 | domains | http | online | unchanged | https://www.botvrij.eu/data/ioclist.hostname.raw 77 | 35077 | domains | http | online | unchanged | https://www.stopforumspam.com/downloads/toxic_domains_whole.txt - 78 | 101569 | domains | http | online | changed | https://www.usom.gov.tr/url-list.txt + 78 | 101569 | domains | http | online | unchanged | https://www.usom.gov.tr/url-list.txt 79 | 29 | domains | local | online | unchanged | black.list.threat-intelligence # Build threat-intelligence Domainlist ... Stats threat-intelligence: -** Source (raw): 1187890 -== Source (unique): 860267 (-327623) --- White: 859499 (-768) --- White(*): 857979 (-1520) --- Dead: 367703 (-490276) +** Source (raw): 1188205 +== Source (unique): 860348 (-327857) +-- White: 859576 (-772) +-- White(*): 857939 (-1637) +-- Dead: 367666 (-490273) -367703 unique Domains - Version 2021.1023.194013 -MD5 Domains RAW: dfd2b5701be1ee3069e201d9a817fe78 +367666 unique Domains - Version 2021.1023.215119 +MD5 Domains RAW: ca75400fb4442f7ca12fc672e463eeed # Convert threat-intelligence to Hostlist ... @@ -440,10 +440,10 @@ Prepare domain list for compiling ... done. ] } ℹ Start compiling threat-intelligence.adblock.raw -ℹ Original length is 347542 -ℹ Length after applying transformations is 347542 -ℹ The list was compressed from 347545 to 319608 -ℹ Final length of the list is 319614 +ℹ Original length is 347498 +ℹ Length after applying transformations is 347498 +ℹ The list was compressed from 347501 to 319497 +ℹ Final length of the list is 319503 ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/threat-intelligence/out/threat-intelligence.adblock ℹ Finished compiling @@ -454,16 +454,16 @@ Prepare domain list for compiling ... done. threat-intelligence - Done! Bereits aktuell. -[master 2f5b722] threat-intelligence auto-update - 13 files changed, 11070 insertions(+), 10972 deletions(-) +[master a5787e0] threat-intelligence auto-update + 13 files changed, 369 insertions(+), 96 deletions(-) Bereits aktuell. -[main 484a6d8] threat-intelligence auto-update - 2 files changed, 36 insertions(+), 20 deletions(-) +[main 309345f] threat-intelligence auto-update + 2 files changed, 73 insertions(+), 28 deletions(-) Bereits aktuell. -[main 036791a] threat-intelligence auto-update - 2 files changed, 35 insertions(+), 20 deletions(-) +[main b0aa9ce] threat-intelligence auto-update + 2 files changed, 64 insertions(+), 28 deletions(-) =========================================== Personal ... @@ -503,7 +503,7 @@ Initialize ... # Download and convert Sourcelists ... Nr | Count | Format | Source | Status | File | URL/File - 1 | 14666 | domains | local | online | unchanged | black.list + 1 | 14646 | domains | local | online | changed | black.list 2 | 29 | domains | local | online | unchanged | black.list.threat-intelligence 3 | 2 | domains | local | online | unchanged | black.list.affiliatetracking 4 | 3672 | hosts | http | online | unchanged | https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext @@ -515,37 +515,68 @@ Initialize ... Stats personal: -** Source (raw): 27043 -== Source (unique): 23181 (-3862) --- White: 21884 (-1297) --- White(*): 21849 (-35) --- Dead: 20912 (-937) -++ Block: 21082 (+170) --- Unblock: 21080 (-2) -++ FLD: 21220 (+140) -++ WWW: 27904 (+6684) +** Source (raw): 27023 +== Source (unique): 23161 (-3862) +-- White: 21864 (-1297) +-- White(*): 21825 (-39) +-- Dead: 20888 (-937) +++ Block: 21058 (+170) +-- Unblock: 21056 (-2) +-- Unblock(*): 21056 (-0) +++ FLD: 21196 (+140) +++ WWW: 27879 (+6683) -***************************************************** -* No changes to the previous repo version detected! * -***************************************************** +27879 unique Domains - Version 2021.1023.215624 +MD5 Domains RAW: e4dc9f52fabd52af4a5dc60755ea8ae2 + +# Convert personal to Hostlist ... + +# Convert personal to AdBlocklist ... + +Prepare domain list for compiling ... done. + +ℹ Starting @adguard/hostlist-compiler v1.0.12 +ℹ Starting the compiler +ℹ Configuration: { + "name": "personal", + "sources": [ + { + "source": "personal.adblock.raw", + "type": "adblock", + "transformations": [ + "Validate" + ] + } + ], + "transformations": [ + "Compress" + ] +} +ℹ Start compiling personal.adblock.raw +ℹ Original length is 20804 +ℹ Length after applying transformations is 20803 +ℹ The list was compressed from 20806 to 13581 +ℹ Final length of the list is 13587 +ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/personal/out/personal.adblock +ℹ Finished compiling + +# Attach header to personal Domainlist ... + +# Push personal to local Repositories ... personal - Done! Bereits aktuell. -[master 0694621] personal auto-update - 2 files changed, 4 insertions(+), 86 deletions(-) +[master d5fbf4d] personal auto-update + 12 files changed, 45 insertions(+), 109 deletions(-) Bereits aktuell. -Auf Branch main -Ihr Branch ist auf demselben Stand wie 'origin/main'. - -nichts zu committen, Arbeitsverzeichnis unverändert +[main d83d5c9] personal auto-update + 2 files changed, 19 insertions(+), 20 deletions(-) Bereits aktuell. -Auf Branch main -Ihr Branch ist auf demselben Stand wie 'origin/main'. - -nichts zu committen, Arbeitsverzeichnis unverändert +[main f065cf1] personal auto-update + 2 files changed, 19 insertions(+), 19 deletions(-) =========================================== Fake ... @@ -611,8 +642,10 @@ Stats fake: fake - Done! Bereits aktuell. -[master c8a6cd5] fake auto-update - 1 file changed, 3 insertions(+), 3 deletions(-) +Auf Branch master +Ihr Branch ist auf demselben Stand wie 'origin/master'. + +nichts zu committen, Arbeitsverzeichnis unverändert Bereits aktuell. Auf Branch main @@ -669,7 +702,7 @@ Initialize ... Nr | Count | Format | Source | Status | File | URL/File 1 | 1013 | domains | http | online | unchanged | https://raw.githubusercontent.com/jkrejcha/AdmiraList/master/AdmiraList.txt - 2 | 15702 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt + 2 | 15712 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt 3 | 701 | domains | http | online | unchanged | https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/adservers-and-trackers.txt 4 | 9922 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/adv/domains 5 | 1486 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/tracker/domains @@ -680,54 +713,85 @@ Initialize ... 10 | 244 | hosts | http | online | unchanged | https://raw.githubusercontent.com/d3ward/toolz/master/src/d3host.txt 11 | 1734 | hosts | http | online | unchanged | https://raw.githubusercontent.com/infinitytec/blocklists/master/ads-and-trackers.txt 12 | 335 | hosts | http | online | unchanged | https://raw.githubusercontent.com/shreyasminocha/shady-hosts/main/hosts - 13 | 16962 | hosts | http | online | changed | https://someonewhocares.org/hosts/ + 13 | 16962 | hosts | http | online | unchanged | https://someonewhocares.org/hosts/ 14 | 1555 | hosts | http | OFFLINE | unchanged | USE LOCAL COPY: urlhaus.abuse.ch_downloads_hostfile_.txt - 15 | 40653 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt + 15 | 40609 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt 16 | 13720 | domains | http | online | unchanged | https://raw.githubusercontent.com/adguardteam/cname-trackers/master/combined_disguised_trackers_justdomains.txt 17 | 18564 | adblock | http | online | unchanged | https://easylist.to/easylist/easylist.txt - 18 | 16183 | adblock | http | online | unchanged | https://easylist.to/easylist/easyprivacy.txt + 18 | 16184 | adblock | http | online | unchanged | https://easylist.to/easylist/easyprivacy.txt 19 | 9314 | adblock | http | online | unchanged | https://raw.githubusercontent.com/easylist/easylist/master/easyprivacy/easyprivacy_specific_cname.txt 20 | 2016 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/TOP_EU_US_Ads_Trackers_ABP 21 | 4924 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/EU_US%2Bmost_used_ad_and_tracking_networks - 22 | 14666 | domains | local | online | unchanged | black.list + 22 | 14646 | domains | local | online | unchanged | black.list 23 | 29 | domains | local | online | unchanged | black.list.threat-intelligence # Build light Domainlist ... Stats light: -** Source (raw): 186960 -== Source (unique): 108116 (-78844) --- Exclude: 108054 (-62) --- White: 105806 (-2248) --- White(*): 104938 (-868) --- Dead: 87788 (-17150) -++ Block: 87958 (+170) --- Unblock: 87956 (-2) -++ FLD: 88722 (+766) -++ WWW: 127384 (+38662) +** Source (raw): 186907 +== Source (unique): 108069 (-78838) +-- Exclude: 108007 (-62) +-- White: 105758 (-2249) +-- White(*): 104842 (-916) +-- Dead: 87745 (-17097) +++ Block: 87915 (+170) +-- Unblock: 87913 (-2) +-- Unblock(*): 87913 (-0) +++ FLD: 88679 (+766) +++ WWW: 127352 (+38673) -***************************************************** -* No changes to the previous repo version detected! * -***************************************************** +127352 unique Domains - Version 2021.1023.215800 +MD5 Domains RAW: d233178ecedc434689cb1db83907252a + +# Convert light to Hostlist ... + +# Convert light to AdBlocklist ... + +Prepare domain list for compiling ... done. + +ℹ Starting @adguard/hostlist-compiler v1.0.12 +ℹ Starting the compiler +ℹ Configuration: { + "name": "light", + "sources": [ + { + "source": "light.adblock.raw", + "type": "adblock", + "transformations": [ + "Validate" + ] + } + ], + "transformations": [ + "Compress" + ] +} +ℹ Start compiling light.adblock.raw +ℹ Original length is 86381 +ℹ Length after applying transformations is 86380 +ℹ The list was compressed from 86383 to 72971 +ℹ Final length of the list is 72977 +ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/light/out/light.adblock +ℹ Finished compiling + +# Attach header to light Domainlist ... + +# Push light to local Repositories ... light - Done! Bereits aktuell. -[master 1a9d9df] light auto-update - 5 files changed, 9 insertions(+), 111 deletions(-) +[master 4ffe92b] light auto-update + 11 files changed, 67 insertions(+), 87 deletions(-) Bereits aktuell. -Auf Branch main -Ihr Branch ist auf demselben Stand wie 'origin/main'. - -nichts zu committen, Arbeitsverzeichnis unverändert +[main 1383e1f] light auto-update + 2 files changed, 27 insertions(+), 28 deletions(-) Bereits aktuell. -Auf Branch main -Ihr Branch ist auf demselben Stand wie 'origin/main'. - -nichts zu committen, Arbeitsverzeichnis unverändert +[main b1fa8e8] light auto-update + 2 files changed, 27 insertions(+), 27 deletions(-) =========================================== Multi ... @@ -775,7 +839,7 @@ Initialize ... 2 | 636 | domains | http | online | unchanged | https://v.firebog.net/hosts/Admiral.txt 3 | 15120 | domains | http | online | unchanged | https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt 4 | 13720 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/combined_disguised_trackers_justdomains.txt - 5 | 15702 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt + 5 | 15712 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt 6 | 701 | domains | http | online | unchanged | https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/adservers-and-trackers.txt 7 | 9922 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/adv/domains 8 | 1486 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/tracker/domains @@ -805,14 +869,14 @@ Initialize ... 32 | 8624 | hosts | http | online | unchanged | https://raw.githubusercontent.com/guardicore/labs_campaigns/master/Autodiscover/autodiscover-tlds.txt 33 | 1555 | hosts | http | OFFLINE | unchanged | USE LOCAL COPY: urlhaus.abuse.ch_downloads_hostfile_.txt 34 | 373 | hosts | http | online | unchanged | https://raw.githubusercontent.com/Laicure/hosts/master/shithosts - 35 | 40653 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt + 35 | 40609 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt 36 | 1019 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_11_Mobile/filter.txt 37 | 729 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_14_Annoyances/filter.txt 38 | 5523 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_16_French/filter.txt 39 | 4649 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_1_Russian/filter.txt 40 | 6551 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_224_Chinese/filter.txt - 41 | 22148 | adblock | http | online | changed | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_2_English/filter.txt - 42 | 16808 | adblock | http | online | changed | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_3_Spyware/filter.txt + 41 | 22148 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_2_English/filter.txt + 42 | 16808 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_3_Spyware/filter.txt 43 | 50 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_4_Social/filter.txt 44 | 403 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_6_German/filter.txt 45 | 336 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_7_Japanese/filter.txt @@ -834,7 +898,7 @@ Initialize ... 61 | 25 | adblock | http | online | unchanged | https://easylist-downloads.adblockplus.org/koreanlist.txt 62 | 5516 | adblock | http | online | unchanged | https://easylist-downloads.adblockplus.org/liste_fr.txt 63 | 18564 | adblock | http | online | unchanged | https://easylist.to/easylist/easylist.txt - 64 | 16183 | adblock | http | online | unchanged | https://easylist.to/easylist/easyprivacy.txt + 64 | 16184 | adblock | http | online | unchanged | https://easylist.to/easylist/easyprivacy.txt 65 | 55 | adblock | http | online | unchanged | https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt 66 | 86 | adblock | http | online | unchanged | https://raw.github.com/tomasko126/easylistczechandslovak/master/filters.txt 67 | 685 | adblock | http | online | unchanged | https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt @@ -855,45 +919,76 @@ Initialize ... 82 | 4 | adblock | http | online | unchanged | https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resource-abuse.txt 83 | 2016 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/TOP_EU_US_Ads_Trackers_ABP 84 | 4924 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/EU_US%2Bmost_used_ad_and_tracking_networks - 85 | 14666 | domains | local | online | unchanged | black.list + 85 | 14646 | domains | local | online | unchanged | black.list 86 | 29 | domains | local | online | unchanged | black.list.threat-intelligence # Build multi Domainlist ... Stats multi: -** Source (raw): 946278 -== Source (unique): 459073 (-487205) --- Exclude: 455033 (-4040) --- White: 451579 (-3454) --- White(*): 450047 (-1532) --- Dead: 351731 (-98316) -++ Block: 351901 (+170) --- Unblock: 351899 (-2) -++ FLD: 352574 (+675) -++ WWW: 471817 (+119243) +** Source (raw): 946225 +== Source (unique): 459055 (-487170) +-- Exclude: 455015 (-4040) +-- White: 451560 (-3455) +-- White(*): 449095 (-2465) +-- Dead: 350812 (-98283) +++ Block: 350982 (+170) +-- Unblock: 350980 (-2) +-- Unblock(*): 350980 (-0) +++ FLD: 351655 (+675) +++ WWW: 470908 (+119253) -***************************************************** -* No changes to the previous repo version detected! * -***************************************************** +470908 unique Domains - Version 2021.1023.215929 +MD5 Domains RAW: fd81abfe944299cb865b9f5380337f3a + +# Convert multi to Hostlist ... + +# Convert multi to AdBlocklist ... + +Prepare domain list for compiling ... done. + +ℹ Starting @adguard/hostlist-compiler v1.0.12 +ℹ Starting the compiler +ℹ Configuration: { + "name": "multi", + "sources": [ + { + "source": "multi.adblock.raw", + "type": "adblock", + "transformations": [ + "Validate" + ] + } + ], + "transformations": [ + "Compress" + ] +} +ℹ Start compiling multi.adblock.raw +ℹ Original length is 307113 +ℹ Length after applying transformations is 307111 +ℹ The list was compressed from 307114 to 256413 +ℹ Final length of the list is 256419 +ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/multi/out/multi.adblock +ℹ Finished compiling + +# Attach header to multi Domainlist ... + +# Push multi to local Repositories ... multi - Done! Bereits aktuell. -[master 0d0dc30] multi auto-update - 4 files changed, 11 insertions(+), 174 deletions(-) +[master ce216b8] multi auto-update + 9 files changed, 47 insertions(+), 51 deletions(-) Bereits aktuell. -Auf Branch main -Ihr Branch ist auf demselben Stand wie 'origin/main'. - -nichts zu committen, Arbeitsverzeichnis unverändert +[main edc509a] multi auto-update + 2 files changed, 20 insertions(+), 21 deletions(-) Bereits aktuell. -Auf Branch main -Ihr Branch ist auf demselben Stand wie 'origin/main'. - -nichts zu committen, Arbeitsverzeichnis unverändert +[main 5a8a8a9] multi auto-update + 2 files changed, 20 insertions(+), 20 deletions(-) =========================================== Affiliate & Tracking ... @@ -974,23 +1069,23 @@ Initialize ... 6 | 22 | domains | http | online | unchanged | https://raw.githubusercontent.com/nextdns/cname-cloaking-blocklist/master/domains 7 | 429 | domains | http | online | unchanged | https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains 8 | 5 | domains | http | online | unchanged | https://raw.githubusercontent.com/nextdns/metadata/master/security/parked-domains-cname - 9 | 892 | domains | local | online | changed | white.list.referral + 9 | 887 | domains | local | online | changed | white.list.referral 10 | 2 | domains | local | online | unchanged | black.list.affiliatetracking # Build affiliatetracking Domainlist ... Stats affiliatetracking: -** Source (raw): 149617 -== Source (unique): 128364 (-21253) --- Exclude: 113536 (-14828) --- White: 113461 (-75) --- White(*): 113182 (-279) --- Dead: 111235 (-1947) -++ Block: 111420 (+185) --- Unblock: 111420 (-0) -++ FLD: 111706 (+286) -++ WWW: 112186 (+480) +** Source (raw): 149612 +== Source (unique): 127930 (-21682) +-- Exclude: 113104 (-14826) +-- White: 113045 (-59) +-- White(*): 112767 (-278) +-- Dead: 110821 (-1946) +++ Block: 111006 (+185) +-- Unblock: 111006 (-0) +++ FLD: 111292 (+286) +++ WWW: 111383 (+91) ***************************************************** * No changes to the previous repo version detected! * @@ -999,8 +1094,8 @@ Stats affiliatetracking: affiliatetracking - Done! Bereits aktuell. -[master 745d5a3] affiliatetracking auto-update - 3 files changed, 3 insertions(+), 88 deletions(-) +[master 83fd807] affiliatetracking auto-update + 5 files changed, 438 insertions(+), 520 deletions(-) Bereits aktuell. Auf Branch main @@ -1054,46 +1149,176 @@ Initialize ... # Download and convert Sourcelists ... Nr | Count | Format | Source | Status | File | URL/File - 1 | 471817 | domains | local | online | unchanged | multi.txt + 1 | 470908 | domains | local | online | changed | multi.txt 2 | 46559 | domains | local | online | unchanged | fake.txt - 3 | 971566 | domains | http | online | unchanged | https://dbl.oisd.nl - 4 | 40653 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt + 3 | 971925 | domains | http | online | changed | https://dbl.oisd.nl + 4 | 40609 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt # Build big Domainlist ... Stats big: -** Source (raw): 1530595 -== Source (unique): 1242557 (-288038) --- White: 1242293 (-264) --- White(*): 1240371 (-1922) -++ Block: 1240390 (+19) --- Unblock: 1240338 (-52) -++ FLD: 1240554 (+216) -++ WWW: 1252608 (+12054) +** Source (raw): 1530001 +== Source (unique): 1241902 (-288099) +-- White: 1241638 (-264) +-- White(*): 1239716 (-1922) +++ Block: 1239735 (+19) +-- Unblock: 1239683 (-52) +-- Unblock(*): 1239602 (-81) +++ FLD: 1239818 (+216) +++ WWW: 1251825 (+12007) -***************************************************** -* No changes to the previous repo version detected! * -***************************************************** +1251825 unique Domains - Version 2021.1023.220430 +MD5 Domains RAW: 9a2fc41d9b00ca38e4675a0024f4ca53 + +# Convert big to Hostlist ... + +# Convert big to AdBlocklist ... + +ℹ Starting @adguard/hostlist-compiler v1.0.12 +ℹ Starting the compiler +ℹ Configuration: { + "name": "Big", + "sources": [ + { + "source": "/media/nas/git/adguard/multi.adblock", + "type": "adblock", + "transformations": [ + "Validate" + ] + }, + { + "source": "/media/nas/git/adguard/fake.adblock", + "type": "adblock", + "transformations": [ + "Validate" + ] + }, + { + "source": "https://abp.oisd.nl/", + "type": "adblock", + "exclusions_sources": [ + "../exclusions.txt", + "../../../data/adblock.exclusions.referral", + "../../../data/adblock.exclusions.list", + "../../../data/adblock.exclusions.important", + "https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt" + ], + "transformations": [ + "Validate" + ] + }, + { + "source": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt", + "type": "adblock", + "exclusions_sources": [ + "../exclusions.txt", + "../../../data/adblock.exclusions.referral", + "../../../data/adblock.exclusions.list", + "../../../data/adblock.exclusions.important" + ], + "transformations": [ + "Validate" + ] + }, + { + "source": "https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/rules.txt", + "exclusions_sources": [ + "../exclusions.txt", + "../../../data/adblock.exclusions.referral", + "../../../data/adblock.exclusions.important" + ], + "transformations": [ + "Validate" + ], + "type": "adblock" + }, + { + "source": "../rules.txt", + "exclusions_sources": [ + "../exclusions.txt", + "../../../data/adblock.exclusions.referral", + "../../../data/adblock.exclusions.important" + ], + "transformations": [ + "Validate" + ], + "type": "adblock" + }, + { + "source": "https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt", + "transformations": [ + "Validate" + ], + "type": "adblock" + }, + { + "source": "../exceptions.txt", + "transformations": [ + "Validate" + ], + "type": "adblock" + } + ], + "transformations": [ + "Compress" + ] +} +ℹ Start compiling /media/nas/git/adguard/multi.adblock +ℹ Original length is 256539 +ℹ Length after applying transformations is 256539 +ℹ Start compiling /media/nas/git/adguard/fake.adblock +ℹ Original length is 23682 +ℹ Length after applying transformations is 23682 +ℹ Start compiling https://abp.oisd.nl/ +ℹ Original length is 368421 +ℹ Filtering the list of rules using 1439 exclusion rules +ℹ Excluded 319 rules. 368102 rules left. +ℹ Length after applying transformations is 368101 +ℹ Start compiling https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt +ℹ Original length is 41680 +ℹ Filtering the list of rules using 1114 exclusion rules +ℹ Excluded 57 rules. 41623 rules left. +ℹ Length after applying transformations is 41623 +ℹ Start compiling https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/rules.txt +ℹ Original length is 576 +ℹ Filtering the list of rules using 472 exclusion rules +ℹ Excluded 0 rules. 576 rules left. +ℹ Length after applying transformations is 570 +ℹ Start compiling ../rules.txt +ℹ Original length is 2 +ℹ Filtering the list of rules using 472 exclusion rules +ℹ Excluded 0 rules. 2 rules left. +ℹ Length after applying transformations is 2 +ℹ Start compiling https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt +ℹ Original length is 179 +ℹ Length after applying transformations is 179 +ℹ Start compiling ../exceptions.txt +ℹ Original length is 30 +ℹ Length after applying transformations is 30 +ℹ The list was compressed from 690750 to 490797 +ℹ Final length of the list is 490803 +ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/big/out/big.adblock +ℹ Finished compiling + +# Attach header to big Domainlist ... + +# Push big to local Repositories ... big - Done! Bereits aktuell. -[master 31a478f] big auto-update - 2 files changed, 58 insertions(+), 238 deletions(-) - rewrite pihole/blocklists/build/big/out/big.stats (83%) +[master bb093ed] big auto-update + 12 files changed, 1672 insertions(+), 2224 deletions(-) + rewrite pihole/blocklists/build/big/out/big.domains.wildcard.removed (99%) Bereits aktuell. -Auf Branch main -Ihr Branch ist auf demselben Stand wie 'origin/main'. - -nichts zu committen, Arbeitsverzeichnis unverändert +[main 9c26ed2] big auto-update + 2 files changed, 308 insertions(+), 105 deletions(-) Bereits aktuell. -Auf Branch main -Ihr Branch ist auf demselben Stand wie 'origin/main'. - -nichts zu committen, Arbeitsverzeichnis unverändert +[main bd81652] big auto-update + 2 files changed, 67 insertions(+), 24 deletions(-) =========================================== Steven Black @@ -1204,19 +1429,19 @@ Initialize ... # Download and convert Sourcelists ... Nr | Count | Format | Source | Status | File | URL/File - 1 | 1252608 | domains | local | online | unchanged | big.txt - 2 | 112186 | domains | local | online | unchanged | affiliatetracking.txt - 3 | 367703 | domains | local | online | changed | threat-intelligence.txt + 1 | 1251825 | domains | local | online | changed | big.txt + 2 | 111383 | domains | local | online | changed | affiliatetracking.txt + 3 | 367666 | domains | local | online | changed | threat-intelligence.txt # Build experimental Domainlist ... Stats experimental: -** Source (raw): 1732497 -== Source (unique): 1458021 (-274476) +** Source (raw): 1730874 +== Source (unique): 1456282 (-274592) -1458021 unique Domains - Version 2021.1023.195241 -MD5 Domains RAW: cf7e71a365c0fc609ca374e750b9ef9e +1456282 unique Domains - Version 2021.1023.221245 +MD5 Domains RAW: da1e83e75139a8d2a0694be3cdc548a3 # Convert experimental to Hostlist ... @@ -1276,22 +1501,22 @@ MD5 Domains RAW: cf7e71a365c0fc609ca374e750b9ef9e ] } ℹ Start compiling /media/nas/git/adguard/big.adblock -ℹ Original length is 490776 -ℹ Length after applying transformations is 490776 +ℹ Original length is 489786 +ℹ Length after applying transformations is 489786 ℹ Start compiling /media/nas/git/adguard/affiliatetracking.adblock -ℹ Original length is 80836 -ℹ Length after applying transformations is 80836 +ℹ Original length is 80442 +ℹ Length after applying transformations is 80442 ℹ Start compiling /media/nas/git/adguard/threat-intelligence.adblock -ℹ Original length is 319726 -ℹ Length after applying transformations is 319726 +ℹ Original length is 319615 +ℹ Length after applying transformations is 319615 ℹ Start compiling ../rules.txt ℹ Original length is 1 ℹ Length after applying transformations is 1 ℹ Start compiling ../exceptions.txt ℹ Original length is 1 ℹ Length after applying transformations is 1 -ℹ The list was compressed from 891355 to 638954 -ℹ Final length of the list is 638960 +ℹ The list was compressed from 889860 to 637567 +ℹ Final length of the list is 637573 ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/experimental/out/experimental.adblock ℹ Finished compiling @@ -1300,8 +1525,8 @@ MD5 Domains RAW: cf7e71a365c0fc609ca374e750b9ef9e experimental - Done! Bereits aktuell. -[master e7657bb] experimental auto-update - 8 files changed, 104 insertions(+), 25 deletions(-) +[master a8c9972] experimental auto-update + 10 files changed, 16645 insertions(+), 24872 deletions(-) Bereits aktuell. Auf Branch main diff --git a/pihole/blocklists/build/buildList.sh b/pihole/blocklists/build/buildList.sh index 65df7880..f7ba35b8 100755 --- a/pihole/blocklists/build/buildList.sh +++ b/pihole/blocklists/build/buildList.sh @@ -88,12 +88,12 @@ getDomainsFromList() { extractValidDomains 0 "$list" >>"$2" done <"$1" sortList "$2" - + # Extend FLD from SLD if [ "$4" == "1" ]; then extendFLDfromSLD "$2" fi - + # Extend missing WWW/FLD if [ "$3" == "1" ]; then extendWWW "$2" @@ -121,9 +121,9 @@ convertWildcardToRegex() { extractValidDomains() { # Sourcetype $1: 0=Domains / 1=Hosts / 2=Adblock / 3=Whitelist / 4=Deadlist - + dos2unix -q "$2" - + # Domains if [ "$1" == "0" ]; then cat <"$2" | sed -e 's/^[[:space:]]*//' | awk '{print $1}' | grep -Ev '^\s*$|^#|^!' | sed 's/^\.//' | sed 's/^\*\.//' | grep -P "$regex" @@ -135,17 +135,17 @@ extractValidDomains() { # Adblock if [ "$1" == "2" ]; then cat <"$2" | sed -e 's/^[[:space:]]*//' | grep ^\|\|.* | sed 's/[\|^]//g' | - sed 's/$popup,third-party//' | - sed 's/$important,all//' | - sed 's/$third-party//' | - sed 's/$popup//' | - sed 's/$important//' | - sed 's/$all//' | - sed 's/$document//' | - sed 's/$doc//' | - sed 's/$3p//' | - sed 's/$1p//' | - grep -P "$regex" + sed 's/$popup,third-party//' | + sed 's/$important,all//' | + sed 's/$third-party//' | + sed 's/$popup//' | + sed 's/$important//' | + sed 's/$all//' | + sed 's/$document//' | + sed 's/$doc//' | + sed 's/$3p//' | + sed 's/$1p//' | + grep -P "$regex" fi # Whitelist if [ "$1" == "3" ]; then @@ -174,14 +174,14 @@ extendFLD() { extendFLDfromSLD() { cat <"$1" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!|^www\.|ip6\.arpa$' | awk '{print $1}' | grep -P '(^[a-zA-Z0-9].*\.(?:[a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]*$))' | grep -P "$regex" >"$1.sld" cat <"$1.sld" | grep -Po '([a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]{2,}$)|([a-zA-Z0-9\-]*\.{1}(ac|co|gov|ltd|me|net|nhs|nic|org|plc|sch|aaa|aca|acct|ae|ar|biz|br|cn|club|ebiz|gr|gb|game|fin|eu|hu|id|idv|in|info|jp|jur|law|mex|no|nom|or|pp|pro|qc|se|sa|ru|recht|radio|uk|us|uy|za|web)\.{1}[a-zA-Z]{2,}$)' | grep -P "$regex" >"$1.fld" - + cat <"$1.sld" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.1" cat <"$1.fld.1" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.2" cat <"$1.fld.2" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.3" cat <"$1.fld.3" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.4" cat <"$1.fld.4" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.5" cat <"$1.fld.5" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.6" - + cat <"$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6" >>"$1" rm -f "$1.sld" "$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6" sortList "$1" @@ -277,14 +277,14 @@ printHeader() { LinkToFilename() { echo "$1" | sed 's/https:\/\///' | sed 's/www\.//' | sed 's/bitbucket.org\///' | - sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' | - sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' | - sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' | - sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' | - sed 's/\/domain?format=plain//' | - sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" | - sed 's/[\/]/_/g' | - awk '{print tolower($0)}' + sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' | + sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' | + sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' | + sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' | + sed 's/\/domain?format=plain//' | + sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" | + sed 's/[\/]/_/g' | + awk '{print tolower($0)}' } convertWhiteToAdblock() { @@ -356,10 +356,10 @@ echo "" echo "$sourcedir" exit 1 fi - + echo "### Compile $name Blocklist ###" echo "" - + # Print config echo "Config:" echo "" @@ -374,18 +374,18 @@ echo "" printConfig "-- Whitelists" "$whitelists" printConfigLocal "- Unblock" "$unblock" printConfig "-- Unblocklists" "$unblocklists" - + # Init echo "Initialize ..." - + dos2unix -q "$sourcedir"/* - + echo "" - + # Download and convert Sourcelists echo "# Download and convert Sourcelists ..." echo "" - + i=0 ( printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "Nr" "Count" "Format" "Source" "Status" "File" "URL/File" @@ -393,20 +393,20 @@ echo "" if [ "${url:0:1}" == "#" ] || [ "${url:0:1}" == "" ]; then continue fi - + ((i++)) - + # Determine list type if inConfig "white"; then listtype=3 # Whitelist listtypename="white" - elif inConfig "dead"; then + elif inConfig "dead"; then listtype=4 # Deadlist listtypename="dead" else listtype=0 # Domains listtypename="domains" - + if [ "${url:0:1}" == "!" ]; then url=$(echo "$url" | awk '{print $2}') listtype=1 # Hosts @@ -418,9 +418,9 @@ echo "" listtypename="adblock" fi fi - + echo "$i. $listtypename - $url" >>"$domains".sourceurls - + # Processing according to source and list type if [ "${url:0:4}" == "http" ]; then # online source="http" @@ -436,14 +436,14 @@ echo "" extractValidDomains $listtype "$url" >"$file.domains" url="$filename" fi - + # Check if the source has been changed if compareFiles "$file" "$file.domains"; then filestatus="unchanged" else filestatus="changed" fi - + # If the downloaded source contains domains, use them, # otherwise use a cached local copy of the source. c=$(wc -l "$file.domains" | awk '{print $1}') @@ -461,130 +461,128 @@ echo "" mv "$file.domains" "$file" status="online" fi - + # Add domains to the domains list if [ -s "$file" ]; then cat "$file" >>"$domains" fi - + printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "$i" "$c" "$listtypename" "$source" "$status" "$filestatus" "$url" done <"$sourcelists" ) | tee "$header.ext" sed -i 's/^/# /' "$header.ext" echo "" - + # Build Domainlist echo "# Build $name Domainlist ..." echo "" - + echo "Stats $name:" echo "" - + echo -n "** Source (raw): " cdo=$(wc -l "$domains" | awk '{print $1}') echo "$cdo" - + echo -n "== Source (unique): " sortList "$domains" printDecCount - + # Add Domains from personal Blacklists if [ -s "$black" ]; then cat <"$black" >>"$domains".blacklist fi getDomainsFromList "$blacklists" "$domains".blacklist 0 0 - + if [ -s "$domains".blacklist ]; then echo -n "++ Black: " addDomains "$domains".blacklist "$domains" printIncCount fi - + # Remove excluded Domains getDomainsFromList "$excludelists" "$domains".excludelist 0 0 - + if [ -s "$domains".excludelist ]; then echo -n "-- Exclude: " removeDomains "$domains".excludelist "$domains" printDecCount fi - + # Save domain list for later dead domains check cp "$domains" "$domains".checkfordead - + # Remove whitelisted Domains if [ -s "$white" ]; then cat <"$white" >>"$domains".whitelist fi getDomainsFromList "$whitelists" "$domains".whitelist 1 1 - + if [ -s "$domains".whitelist ]; then echo -n "-- White: " removeDomains "$domains".whitelist "$domains" - + # Build Adblock whitelist if inConfig "adblockwhite"; then if [ -s "$white" ]; then convertWhiteToAdblock "$white" "$sourcedir"/exceptions.txt fi fi - + printDecCount fi - + # Remove wildcard whitelisted Domains getRegexDomainsFromWilcardList "$whitelists" "$domains".whitelist.wildcards - + if [ -s "$domains".whitelist.wildcards ]; then echo -n "-- White(*): " removeDomainsWildcard "$domains".whitelist.wildcards "$domains" printDecCount fi - + # Remove dead Domains getDomainsFromList "$deadlists" "$domains".deadlist 0 0 - + if [ -s "$domains".deadlist ]; then echo -n "-- Dead: " removeDomains "$domains".deadlist "$domains" printDecCount fi - + # Add prioritized blocked Domains if [ -s "$block" ]; then cat <"$block" >>"$domains".prioblocklist fi getDomainsFromList "$prioblocklists" "$domains".prioblocklist 1 0 - + if [ -s "$domains".prioblocklist ]; then echo -n "++ Block: " addDomains "$domains".prioblocklist "$domains" printIncCount fi - + # Unblock prioritized whitelisted Domains if [ -s "$unblock" ]; then cat <"$unblock" >>"$domains".unblocklist fi getDomainsFromList "$unblocklists" "$domains".unblocklist 1 1 - + if [ -s "$domains".unblocklist ]; then echo -n "-- Unblock: " removeDomains "$domains".unblocklist "$domains" - - # Build Adblock unblocklist - if inConfig "adblockwhite"; then - if [ -s "$unblock" ]; then - cat <"$unblock" >>"$domains".unblocklist.adblock - fi - getDomainsFromList "$unblocklists" "$domains".unblocklist.adblock 0 0 - convertWhiteToAdblock "$domains".unblocklist.adblock "$outdir"/"$name".unblock.adblock - rm -f "$domains".unblocklist.adblock - fi - printDecCount fi - + + # Unblock prioritized wildcard whitelisted Domains + getRegexDomainsFromWilcardList "$unblocklists" "$domains".unblocklist.wildcards + + if [ -s "$domains".unblocklist.wildcards ]; then + echo -n "-- Unblock(*): " + removeDomainsWildcard "$domains".unblocklist.wildcards "$domains" + printDecCount + fi + # Extend by missing WWW/FLD Domains if inConfig "extendWWWFLD"; then if inConfig "white"; then @@ -592,16 +590,16 @@ echo "" extendFLDfromSLD "$domains" printIncCount fi - + echo -n "++ FLD: " extendFLD "$domains" printIncCount - + echo -n "++ WWW: " extendWWW "$domains" printIncCount fi - + # Check if there are changes to the previous repo version getMd5 "$domains" >"$outdir"/md5.new if compareFiles "$outdir/md5.new" "$outdir/md5.old"; then @@ -613,30 +611,30 @@ echo "" exit 0 fi getMd5 "$domains" >"$outdir"/md5.old - + # Cleanup rm -f "$hosts" "$adblock" rm -f "$hosts".* "$adblock".* - + # Version echo "" echo "$cdo unique Domains - Version $version" echo "MD5 Domains RAW: $(getMd5 "$domains")" echo "" - + # Convert to Hostlist if inConfig "hosts"; then echo "# Convert $name to Hostlist ... " rm -f "$hosts" - + if ! inConfig "noheader"; then printHeader "$cdo" >"$hosts" fi - + sed -e 's/^/0.0.0.0 /' "$domains" >>"$hosts" echo "" fi - + # Convert to Adblocklist if inConfig "adblock"; then echo "# Convert $name to AdBlocklist ..." @@ -645,7 +643,7 @@ echo "" cp $jsonbase "$json" sed -i "s/tmp/$name/g" "$json" fi - + if ! inConfig "no_adblock_convert"; then echo -n "Prepare domain list for compiling ... " if inConfig "adblock_important"; then @@ -656,7 +654,7 @@ echo "" echo "done." echo "" fi - + cd "$outdir" || exit if inConfig "verbose"; then hostlist-compiler -v -c "$json" -o "$adblock" @@ -665,58 +663,58 @@ echo "" fi rm -f "$adblock".raw cd "$sourcedir" || exit - + sortList "$adblock" - + if ! inConfig "noheader"; then c=$(grep -Evc '^\s*$|^#|^!' "$adblock") printHeader "$c" | sed 's/^\#/\!/' >"$adblock".tmp fi - + cat <"$adblock" | grep -Ev '^\s*$|^#|^!' >>"$adblock".tmp mv "$adblock".tmp "$adblock" - + if inConfig "adblocktowildcard"; then cat <"$adblock" | grep -Ev '^\s*$|^#|^!' | sed 's/||/*./' | sed 's/[\^]//g' >"$adblock".wildcards fi - + echo "" fi - + if ! inConfig "noheader"; then # Attach header echo "# Attach header to $name Domainlist ..." echo "" - + printHeader "$cdo" >"$domains".tmp cat <"$domains" >>"$domains".tmp mv "$domains".tmp "$domains" fi - + # Push to local repository if inConfig "localpush"; then echo "# Push $name to local Repositories ..." echo "" - + dos2unix -q "$outdir"/* - + if inConfig "white" || inConfig "dead"; then cp "$domains" "$repodata"/"$name".list - elif inConfig "block"; then + elif inConfig "block"; then cp "$domains" "$repodata"/"$name".list.block - elif inConfig "black"; then + elif inConfig "black"; then cp "$domains" "$repodata"/"$name".list cp "$domains".whitelist.removed "$repodata"/"$name".list.wl.removed cp "$domains".wildcard.removed "$repodata"/"$name".list.wc.removed else cp "$domains" "$repodomains"/"$name".txt cp "$stats" "$repodomains" - + if inConfig "hosts"; then cp "$hosts" "$repohosts"/"$name".txt cp "$stats" "$repohosts" fi - + if inConfig "adblock"; then cp "$adblock" "$repoadguard"/"$name".adblock cp "$stats" "$repoadguard" diff --git a/pihole/blocklists/build/updateWhitelist.sh b/pihole/blocklists/build/updateWhitelist.sh index 6af23165..16d46e7f 100755 --- a/pihole/blocklists/build/updateWhitelist.sh +++ b/pihole/blocklists/build/updateWhitelist.sh @@ -49,6 +49,26 @@ convertWhiteToAdBlock() { done <"$1" } +convertWhiteToAdBlockSubdomains() { + rm -f "$2" + while IFS= read -r domain || [ -n "$domain" ]; do + if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then + echo "$domain" | sed 's/^\#/\!/' >>"$2" + continue + fi + if [ "${domain:0:2}" == "*." ]; then + echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2" + else + if [[ $domain =~ \* ]]; then + echo "@@/$domain/" >>"$2" + else + echo "@@||$domain^|" >>"$2" + fi + fi + done <"$1" + sortList $2 +} + # Start echo '===========================================' echo 'Whitelist ...' @@ -62,7 +82,8 @@ tmp=$data/tmp # Referral curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains >$tmp -curl -s -L https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt >>$tmp +curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains | +awk '{print $1}' | grep -Ev '^\s*$|^#|^!|^www' | sed -e 's/^/\*\./' >>$tmp generateWhitelist $tmp $tmp.1 cat <$tmp.1 >>$data/white.list.referral rm -f $tmp.1 @@ -81,35 +102,35 @@ wc -l $data/white.list.oisd # AdGuard curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt | - grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | - grep -P "$regex" >$tmp +grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | +grep -P "$regex" >$tmp curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt | - grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | - grep -P "$regex" | sed -e 's/^/*./' >>$tmp +grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | +grep -P "$regex" | sed -e 's/^/*./' >>$tmp curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt | - grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' | - grep -P "$regex" >>$tmp +grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' | +grep -P "$regex" >>$tmp curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt | - grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp +grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt | - grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | - grep -P "$regex" >>$tmp +grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | +grep -P "$regex" >>$tmp curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt | - grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | - grep -P "$regex" | sed -e 's/^/*./' >>$tmp +grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | +grep -P "$regex" | sed -e 's/^/*./' >>$tmp curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt | - grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | - grep -P "$regex" >>$tmp +grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | +grep -P "$regex" >>$tmp curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt | - grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | - grep -P "$regex" | sed -e 's/^/*./' >>$tmp +grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | +grep -P "$regex" | sed -e 's/^/*./' >>$tmp generateWhitelist $tmp $data/white.list.adguard wc -l $data/white.list.adguard @@ -123,12 +144,12 @@ cp $data/white.list.referral /media/nas/git/hosts # Convert personal whitelist to AdBlock format convertWhiteToAdBlock "$data/white.list" /media/nas/git/adguard/whitelist.adguard -convertWhiteToAdBlock "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard +convertWhiteToAdBlockSubdomains "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard # Build host-compiler exclusion lists cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.list" cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.important" -cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.referral" +cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.referral" echo '===========================================' echo 'Deadlist ...'