zelo72
/
rpi
mirror of https://github.com/Zelo72/rpi
1
0
Fork 0
Code Issues Releases Wiki Activity

build auto-update

This commit is contained in:
Zelo72 2021-10-23 22:15:32 +02:00
parent a8c99727fb
commit 456d0fbcd6
3 changed files with 557 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

615
pihole/blocklists/build/build.log
View File

@ -3,10 +3,10 @@ Initialize ...
=========================================== ===========================================
Bereits aktuell. Bereits aktuell.
[master b8f9f93] auto-update Auf Branch master
2 files changed, 585 insertions(+), 1682 deletions(-) Ihr Branch ist auf demselben Stand wie 'origin/master'.
create mode 100644 pihole/blocklists/data/tmp
delete mode 100644 pihole/blocklists/data/white.list.notracking nichts zu committen, Arbeitsverzeichnis unverändert
Bereits aktuell. Bereits aktuell.
Auf Branch main Auf Branch main
@ -21,17 +21,17 @@ Ihr Branch ist auf demselben Stand wie 'origin/main'.
nichts zu committen, Arbeitsverzeichnis unverändert nichts zu committen, Arbeitsverzeichnis unverändert
Bereits aktuell. Bereits aktuell.
[master 6a1a362] auto-update [master 4a3e7ad] auto-update
2 files changed, 106 insertions(+), 73 deletions(-) 1 file changed, 98 insertions(+), 100 deletions(-)
=========================================== ===========================================
Whitelist ... Whitelist ...
=========================================== ===========================================
893 /media/nas/git/rpi/pihole/blocklists/data/white.list.referral 887 /media/nas/git/rpi/pihole/blocklists/data/white.list.referral
43 /media/nas/git/rpi/pihole/blocklists/data/white.list.shc 43 /media/nas/git/rpi/pihole/blocklists/data/white.list.shc
3100 /media/nas/git/rpi/pihole/blocklists/data/white.list.oisd 3108 /media/nas/git/rpi/pihole/blocklists/data/white.list.oisd
577 /media/nas/git/rpi/pihole/blocklists/data/white.list.adguard 577 /media/nas/git/rpi/pihole/blocklists/data/white.list.adguard
### Compile exclude Blocklist ### ### Compile exclude Blocklist ###
@ -89,25 +89,26 @@ Initialize ...
37 | 739 | white | http | online | unchanged | https://raw.githubusercontent.com/rspamd/maps/master/rspamd/dmarc_whitelist.inc 37 | 739 | white | http | online | unchanged | https://raw.githubusercontent.com/rspamd/maps/master/rspamd/dmarc_whitelist.inc
38 | 79 | white | http | online | unchanged | https://raw.githubusercontent.com/zebpalmer/dns_blocklists/master/whitelist.txt 38 | 79 | white | http | online | unchanged | https://raw.githubusercontent.com/zebpalmer/dns_blocklists/master/whitelist.txt
39 | 678 | white | local | online | unchanged | white.list 39 | 678 | white | local | online | unchanged | white.list
40 | 893 | white | local | online | changed | white.list.referral 40 | 887 | white | local | online | changed | white.list.referral
41 | 15 | white | local | online | unchanged | white.list.important 41 | 15 | white | local | online | unchanged | white.list.important
42 | 577 | white | local | online | unchanged | white.list.adguard 42 | 577 | white | local | online | unchanged | white.list.adguard
43 | 3100 | white | local | online | unchanged | white.list.oisd 43 | 3108 | white | local | online | changed | white.list.oisd
44 | 43 | white | local | online | unchanged | white.list.shc 44 | 43 | white | local | online | unchanged | white.list.shc
# Build exclude Domainlist ... # Build exclude Domainlist ...
Stats exclude: Stats exclude:
** Source (raw): 110596 ** Source (raw): 110598
== Source (unique): 58493 (-52103) == Source (unique): 58922 (-51676)
++ FLD from SLD: 64837 (+6344) ++ FLD from SLD: 65273 (+6351)
++ FLD: 64929 (+92) ++ FLD: 65364 (+91)
++ WWW: 92684 (+27755) ++ WWW: 93125 (+27761)
***************************************************** 93125 unique Domains - Version 2021.1023.215016
* No changes to the previous repo version detected! * MD5 Domains RAW: ab5f4bbea6d07ec8be9af9cd6478a4e3
*****************************************************
# Push exclude to local Repositories ...
exclude - Done! exclude - Done!
@ -154,21 +155,16 @@ Stats dead:
dead - Done! dead - Done!
Bereits aktuell. Bereits aktuell.
[master 14c9121] white-dead auto-update [master 22e71fe] white-dead auto-update
3 files changed, 2 insertions(+), 586 deletions(-) 11 files changed, 468 insertions(+), 443 deletions(-)
delete mode 100644 pihole/blocklists/data/tmp
Bereits aktuell. Bereits aktuell.
Auf Branch main [main 3e78d1f] white-dead auto-update
Ihr Branch ist auf demselben Stand wie 'origin/main'. 1 file changed, 1 insertion(+)
nichts zu committen, Arbeitsverzeichnis unverändert
Bereits aktuell. Bereits aktuell.
Auf Branch main [main 432288e] white-dead auto-update
Ihr Branch ist auf demselben Stand wie 'origin/main'. 1 file changed, 1 insertion(+)
nichts zu committen, Arbeitsverzeichnis unverändert
=========================================== ===========================================
Black ... Black ...
@ -218,16 +214,20 @@ Stats black:
-- White(*): 14649 (-630) -- White(*): 14649 (-630)
++ Block: 14668 (+19) ++ Block: 14668 (+19)
-- Unblock: 14666 (-2) -- Unblock: 14666 (-2)
-- Unblock(*): 14646 (-20)
***************************************************** 14646 unique Domains - Version 2021.1023.215058
* No changes to the previous repo version detected! * MD5 Domains RAW: ce8a1edde3bfe78329d9c5ddaedf1c6f
*****************************************************
# Push black to local Repositories ...
black - Done! black - Done!
Bereits aktuell. Bereits aktuell.
[master 2867940] black auto-update [master 39d6eca] black auto-update
1 file changed, 3 insertions(+), 3 deletions(-) 7 files changed, 47 insertions(+), 1305 deletions(-)
rewrite pihole/blocklists/build/black/out/black.domains.wildcard.removed (99%)
rewrite pihole/blocklists/data/black.list.wc.removed (99%)
Bereits aktuell. Bereits aktuell.
Auf Branch main Auf Branch main
@ -327,7 +327,7 @@ Initialize ...
2 | 407 | hosts | http | online | unchanged | https://curben.gitlab.io/malware-filter/pup-filter-hosts.txt 2 | 407 | hosts | http | online | unchanged | https://curben.gitlab.io/malware-filter/pup-filter-hosts.txt
3 | 8884 | hosts | http | online | unchanged | https://curben.gitlab.io/malware-filter/urlhaus-filter-hosts.txt 3 | 8884 | hosts | http | online | unchanged | https://curben.gitlab.io/malware-filter/urlhaus-filter-hosts.txt
4 | 3496 | hosts | http | online | unchanged | https://gitlab.com/ZeroDot1/CoinBlockerLists/raw/master/hosts_browser 4 | 3496 | hosts | http | online | unchanged | https://gitlab.com/ZeroDot1/CoinBlockerLists/raw/master/hosts_browser
5 | 33669 | hosts | http | online | changed | https://hole.cert.pl/domains/domains_hosts.txt 5 | 33675 | hosts | http | online | changed | https://hole.cert.pl/domains/domains_hosts.txt
6 | 550 | hosts | http | online | unchanged | https://paulgb.github.io/BarbBlock/blacklists/hosts-file.txt 6 | 550 | hosts | http | online | unchanged | https://paulgb.github.io/BarbBlock/blacklists/hosts-file.txt
7 | 5997 | hosts | http | online | unchanged | https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt 7 | 5997 | hosts | http | online | unchanged | https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareHosts.txt
8 | 2204 | hosts | http | online | unchanged | https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts 8 | 2204 | hosts | http | online | unchanged | https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Risk/hosts
@ -347,19 +347,19 @@ Initialize ...
22 | 549 | domains | http | online | unchanged | https://feeds.alphasoc.net/ryuk.txt 22 | 549 | domains | http | online | unchanged | https://feeds.alphasoc.net/ryuk.txt
23 | 9233 | domains | http | online | unchanged | https://gitlab.com/KevinThomas0/cryptoscamdb-lists/-/raw/master/cryptoscamdb-blocklist.txt 23 | 9233 | domains | http | online | unchanged | https://gitlab.com/KevinThomas0/cryptoscamdb-lists/-/raw/master/cryptoscamdb-blocklist.txt
24 | 365 | domains | http | online | unchanged | https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt 24 | 365 | domains | http | online | unchanged | https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt
25 | 33669 | domains | http | online | changed | https://hole.cert.pl/domains/domains.txt 25 | 33675 | domains | http | online | changed | https://hole.cert.pl/domains/domains.txt
26 | 73323 | domains | http | online | unchanged | https://joewein.net/dl/bl/dom-bl-base.txt 26 | 73323 | domains | http | online | unchanged | https://joewein.net/dl/bl/dom-bl-base.txt
27 | 767 | domains | http | online | unchanged | https://joewein.net/dl/bl/dom-bl.txt 27 | 767 | domains | http | online | unchanged | https://joewein.net/dl/bl/dom-bl.txt
28 | 2000 | domains | http | online | unchanged | https://kriskintel.com/feeds/ktip_covid_domains.txt 28 | 2000 | domains | http | online | unchanged | https://kriskintel.com/feeds/ktip_covid_domains.txt
29 | 1994 | domains | http | online | unchanged | https://kriskintel.com/feeds/ktip_malicious_domains.txt 29 | 1998 | domains | http | online | unchanged | https://kriskintel.com/feeds/ktip_malicious_domains.txt
30 | 397 | domains | http | online | unchanged | https://kriskintel.com/feeds/ktip_ransomware_feeds.txt 30 | 397 | domains | http | online | unchanged | https://kriskintel.com/feeds/ktip_ransomware_feeds.txt
31 | 2245 | domains | http | online | unchanged | https://orca.pet/notonmyshift/domains.txt 31 | 2245 | domains | http | online | unchanged | https://orca.pet/notonmyshift/domains.txt
32 | 52 | domains | http | online | unchanged | https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt 32 | 52 | domains | http | online | unchanged | https://osint.digitalside.it/Threat-Intel/lists/latestdomains.txt
33 | 44826 | domains | http | online | unchanged | https://phishing.army/download/phishing_army_blocklist.txt 33 | 44826 | domains | http | online | unchanged | https://phishing.army/download/phishing_army_blocklist.txt
34 | 54300 | domains | http | online | unchanged | https://phishing.army/download/phishing_army_blocklist_extended.txt 34 | 54484 | domains | http | online | unchanged | https://phishing.army/download/phishing_army_blocklist_extended.txt
35 | 1406 | domains | http | online | unchanged | https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/domains.txt 35 | 1406 | domains | http | online | unchanged | https://raw.githubusercontent.com/AmnestyTech/investigations/master/2021-07-18_nso/domains.txt
36 | 27 | domains | http | online | unchanged | https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/fake-domains.txt 36 | 27 | domains | http | online | unchanged | https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/fake-domains.txt
37 | 35404 | domains | http | online | unchanged | https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADomains.txt 37 | 35464 | domains | http | online | unchanged | https://raw.githubusercontent.com/PolishFiltersTeam/KADhosts/master/KADomains.txt
38 | 675 | domains | http | online | unchanged | https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Cryptocurrency 38 | 675 | domains | http | online | unchanged | https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Cryptocurrency
39 | 22769 | domains | http | online | unchanged | https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Malware 39 | 22769 | domains | http | online | unchanged | https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Malware
40 | 179 | domains | http | online | unchanged | https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Risk 40 | 179 | domains | http | online | unchanged | https://raw.githubusercontent.com/ShadowWhisperer/BlockLists/master/Lists/Risk
@ -373,7 +373,7 @@ Initialize ...
48 | 3239 | domains | http | online | unchanged | https://raw.githubusercontent.com/iam-py-test/my_filters_001/main/Alternative%20list%20formats/antimalware_domains.txt 48 | 3239 | domains | http | online | unchanged | https://raw.githubusercontent.com/iam-py-test/my_filters_001/main/Alternative%20list%20formats/antimalware_domains.txt
49 | 2079 | domains | http | online | unchanged | https://raw.githubusercontent.com/matomo-org/referrer-spam-blacklist/master/spammers.txt 49 | 2079 | domains | http | online | unchanged | https://raw.githubusercontent.com/matomo-org/referrer-spam-blacklist/master/spammers.txt
50 | 71289 | domains | http | online | unchanged | https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-ACTIVE.txt 50 | 71289 | domains | http | online | unchanged | https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-ACTIVE.txt
51 | 495 | domains | http | online | unchanged | https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-NEW-today.txt 51 | 550 | domains | http | online | changed | https://raw.githubusercontent.com/mitchellkrogza/Phishing.Database/master/phishing-domains-NEW-today.txt
52 | 10000 | domains | http | online | unchanged | https://raw.githubusercontent.com/prodaft/malware-ioc/master/FluBot/v3.7_5000_domain.txt 52 | 10000 | domains | http | online | unchanged | https://raw.githubusercontent.com/prodaft/malware-ioc/master/FluBot/v3.7_5000_domain.txt
53 | 10000 | domains | http | online | unchanged | https://raw.githubusercontent.com/prodaft/malware-ioc/master/FluBot/v3.7_germany.txt 53 | 10000 | domains | http | online | unchanged | https://raw.githubusercontent.com/prodaft/malware-ioc/master/FluBot/v3.7_germany.txt
54 | 10000 | domains | http | online | unchanged | https://raw.githubusercontent.com/prodaft/malware-ioc/master/FluBot/v3.8_domains.txt 54 | 10000 | domains | http | online | unchanged | https://raw.githubusercontent.com/prodaft/malware-ioc/master/FluBot/v3.8_domains.txt
@ -400,21 +400,21 @@ Initialize ...
75 | 77 | domains | http | online | unchanged | https://www.botvrij.eu/data/ioclist.domain.raw 75 | 77 | domains | http | online | unchanged | https://www.botvrij.eu/data/ioclist.domain.raw
76 | 29 | domains | http | online | unchanged | https://www.botvrij.eu/data/ioclist.hostname.raw 76 | 29 | domains | http | online | unchanged | https://www.botvrij.eu/data/ioclist.hostname.raw
77 | 35077 | domains | http | online | unchanged | https://www.stopforumspam.com/downloads/toxic_domains_whole.txt 77 | 35077 | domains | http | online | unchanged | https://www.stopforumspam.com/downloads/toxic_domains_whole.txt
78 | 101569 | domains | http | online | changed | https://www.usom.gov.tr/url-list.txt 78 | 101569 | domains | http | online | unchanged | https://www.usom.gov.tr/url-list.txt
79 | 29 | domains | local | online | unchanged | black.list.threat-intelligence 79 | 29 | domains | local | online | unchanged | black.list.threat-intelligence
# Build threat-intelligence Domainlist ... # Build threat-intelligence Domainlist ...
Stats threat-intelligence: Stats threat-intelligence:
** Source (raw): 1187890 ** Source (raw): 1188205
== Source (unique): 860267 (-327623) == Source (unique): 860348 (-327857)
-- White: 859499 (-768) -- White: 859576 (-772)
-- White(*): 857979 (-1520) -- White(*): 857939 (-1637)
-- Dead: 367703 (-490276) -- Dead: 367666 (-490273)
367703 unique Domains - Version 2021.1023.194013 367666 unique Domains - Version 2021.1023.215119
MD5 Domains RAW: dfd2b5701be1ee3069e201d9a817fe78 MD5 Domains RAW: ca75400fb4442f7ca12fc672e463eeed
# Convert threat-intelligence to Hostlist ... # Convert threat-intelligence to Hostlist ...
@ -440,10 +440,10 @@ Prepare domain list for compiling ... done.
] ]
} }
ℹ Start compiling threat-intelligence.adblock.raw ℹ Start compiling threat-intelligence.adblock.raw
ℹ Original length is 347542 ℹ Original length is 347498
ℹ Length after applying transformations is 347542 ℹ Length after applying transformations is 347498
ℹ The list was compressed from 347545 to 319608 ℹ The list was compressed from 347501 to 319497
ℹ Final length of the list is 319614 ℹ Final length of the list is 319503
ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/threat-intelligence/out/threat-intelligence.adblock ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/threat-intelligence/out/threat-intelligence.adblock
ℹ Finished compiling ℹ Finished compiling
@ -454,16 +454,16 @@ Prepare domain list for compiling ... done.
threat-intelligence - Done! threat-intelligence - Done!
Bereits aktuell. Bereits aktuell.
[master 2f5b722] threat-intelligence auto-update [master a5787e0] threat-intelligence auto-update
13 files changed, 11070 insertions(+), 10972 deletions(-) 13 files changed, 369 insertions(+), 96 deletions(-)
Bereits aktuell. Bereits aktuell.
[main 484a6d8] threat-intelligence auto-update [main 309345f] threat-intelligence auto-update
2 files changed, 36 insertions(+), 20 deletions(-) 2 files changed, 73 insertions(+), 28 deletions(-)
Bereits aktuell. Bereits aktuell.
[main 036791a] threat-intelligence auto-update [main b0aa9ce] threat-intelligence auto-update
2 files changed, 35 insertions(+), 20 deletions(-) 2 files changed, 64 insertions(+), 28 deletions(-)
=========================================== ===========================================
Personal ... Personal ...
@ -503,7 +503,7 @@ Initialize ...
# Download and convert Sourcelists ... # Download and convert Sourcelists ...
Nr | Count | Format | Source | Status | File | URL/File Nr | Count | Format | Source | Status | File | URL/File
1 | 14666 | domains | local | online | unchanged | black.list 1 | 14646 | domains | local | online | changed | black.list
2 | 29 | domains | local | online | unchanged | black.list.threat-intelligence 2 | 29 | domains | local | online | unchanged | black.list.threat-intelligence
3 | 2 | domains | local | online | unchanged | black.list.affiliatetracking 3 | 2 | domains | local | online | unchanged | black.list.affiliatetracking
4 | 3672 | hosts | http | online | unchanged | https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext 4 | 3672 | hosts | http | online | unchanged | https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext
@ -515,37 +515,68 @@ Initialize ...
Stats personal: Stats personal:
** Source (raw): 27043 ** Source (raw): 27023
== Source (unique): 23181 (-3862) == Source (unique): 23161 (-3862)
-- White: 21884 (-1297) -- White: 21864 (-1297)
-- White(*): 21849 (-35) -- White(*): 21825 (-39)
-- Dead: 20912 (-937) -- Dead: 20888 (-937)
++ Block: 21082 (+170) ++ Block: 21058 (+170)
-- Unblock: 21080 (-2) -- Unblock: 21056 (-2)
++ FLD: 21220 (+140) -- Unblock(*): 21056 (-0)
++ WWW: 27904 (+6684) ++ FLD: 21196 (+140)
++ WWW: 27879 (+6683)
***************************************************** 27879 unique Domains - Version 2021.1023.215624
* No changes to the previous repo version detected! * MD5 Domains RAW: e4dc9f52fabd52af4a5dc60755ea8ae2
*****************************************************
# Convert personal to Hostlist ...
# Convert personal to AdBlocklist ...
Prepare domain list for compiling ... done.
ℹ Starting @adguard/hostlist-compiler v1.0.12
ℹ Starting the compiler
ℹ Configuration: {
"name": "personal",
"sources": [
{
"source": "personal.adblock.raw",
"type": "adblock",
"transformations": [
"Validate"
]
}
],
"transformations": [
"Compress"
]
}
ℹ Start compiling personal.adblock.raw
ℹ Original length is 20804
ℹ Length after applying transformations is 20803
ℹ The list was compressed from 20806 to 13581
ℹ Final length of the list is 13587
ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/personal/out/personal.adblock
ℹ Finished compiling
# Attach header to personal Domainlist ...
# Push personal to local Repositories ...
personal - Done! personal - Done!
Bereits aktuell. Bereits aktuell.
[master 0694621] personal auto-update [master d5fbf4d] personal auto-update
2 files changed, 4 insertions(+), 86 deletions(-) 12 files changed, 45 insertions(+), 109 deletions(-)
Bereits aktuell. Bereits aktuell.
Auf Branch main [main d83d5c9] personal auto-update
Ihr Branch ist auf demselben Stand wie 'origin/main'. 2 files changed, 19 insertions(+), 20 deletions(-)
nichts zu committen, Arbeitsverzeichnis unverändert
Bereits aktuell. Bereits aktuell.
Auf Branch main [main f065cf1] personal auto-update
Ihr Branch ist auf demselben Stand wie 'origin/main'. 2 files changed, 19 insertions(+), 19 deletions(-)
nichts zu committen, Arbeitsverzeichnis unverändert
=========================================== ===========================================
Fake ... Fake ...
@ -611,8 +642,10 @@ Stats fake:
fake - Done! fake - Done!
Bereits aktuell. Bereits aktuell.
[master c8a6cd5] fake auto-update Auf Branch master
1 file changed, 3 insertions(+), 3 deletions(-) Ihr Branch ist auf demselben Stand wie 'origin/master'.
nichts zu committen, Arbeitsverzeichnis unverändert
Bereits aktuell. Bereits aktuell.
Auf Branch main Auf Branch main
@ -669,7 +702,7 @@ Initialize ...
Nr | Count | Format | Source | Status | File | URL/File Nr | Count | Format | Source | Status | File | URL/File
1 | 1013 | domains | http | online | unchanged | https://raw.githubusercontent.com/jkrejcha/AdmiraList/master/AdmiraList.txt 1 | 1013 | domains | http | online | unchanged | https://raw.githubusercontent.com/jkrejcha/AdmiraList/master/AdmiraList.txt
2 | 15702 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt 2 | 15712 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt
3 | 701 | domains | http | online | unchanged | https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/adservers-and-trackers.txt 3 | 701 | domains | http | online | unchanged | https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/adservers-and-trackers.txt
4 | 9922 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/adv/domains 4 | 9922 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/adv/domains
5 | 1486 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/tracker/domains 5 | 1486 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/tracker/domains
@ -680,54 +713,85 @@ Initialize ...
10 | 244 | hosts | http | online | unchanged | https://raw.githubusercontent.com/d3ward/toolz/master/src/d3host.txt 10 | 244 | hosts | http | online | unchanged | https://raw.githubusercontent.com/d3ward/toolz/master/src/d3host.txt
11 | 1734 | hosts | http | online | unchanged | https://raw.githubusercontent.com/infinitytec/blocklists/master/ads-and-trackers.txt 11 | 1734 | hosts | http | online | unchanged | https://raw.githubusercontent.com/infinitytec/blocklists/master/ads-and-trackers.txt
12 | 335 | hosts | http | online | unchanged | https://raw.githubusercontent.com/shreyasminocha/shady-hosts/main/hosts 12 | 335 | hosts | http | online | unchanged | https://raw.githubusercontent.com/shreyasminocha/shady-hosts/main/hosts
13 | 16962 | hosts | http | online | changed | https://someonewhocares.org/hosts/ 13 | 16962 | hosts | http | online | unchanged | https://someonewhocares.org/hosts/
14 | 1555 | hosts | http | OFFLINE | unchanged | USE LOCAL COPY: urlhaus.abuse.ch_downloads_hostfile_.txt 14 | 1555 | hosts | http | OFFLINE | unchanged | USE LOCAL COPY: urlhaus.abuse.ch_downloads_hostfile_.txt
15 | 40653 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt 15 | 40609 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
16 | 13720 | domains | http | online | unchanged | https://raw.githubusercontent.com/adguardteam/cname-trackers/master/combined_disguised_trackers_justdomains.txt 16 | 13720 | domains | http | online | unchanged | https://raw.githubusercontent.com/adguardteam/cname-trackers/master/combined_disguised_trackers_justdomains.txt
17 | 18564 | adblock | http | online | unchanged | https://easylist.to/easylist/easylist.txt 17 | 18564 | adblock | http | online | unchanged | https://easylist.to/easylist/easylist.txt
18 | 16183 | adblock | http | online | unchanged | https://easylist.to/easylist/easyprivacy.txt 18 | 16184 | adblock | http | online | unchanged | https://easylist.to/easylist/easyprivacy.txt
19 | 9314 | adblock | http | online | unchanged | https://raw.githubusercontent.com/easylist/easylist/master/easyprivacy/easyprivacy_specific_cname.txt 19 | 9314 | adblock | http | online | unchanged | https://raw.githubusercontent.com/easylist/easylist/master/easyprivacy/easyprivacy_specific_cname.txt
20 | 2016 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/TOP_EU_US_Ads_Trackers_ABP 20 | 2016 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/TOP_EU_US_Ads_Trackers_ABP
21 | 4924 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/EU_US%2Bmost_used_ad_and_tracking_networks 21 | 4924 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/EU_US%2Bmost_used_ad_and_tracking_networks
22 | 14666 | domains | local | online | unchanged | black.list 22 | 14646 | domains | local | online | unchanged | black.list
23 | 29 | domains | local | online | unchanged | black.list.threat-intelligence 23 | 29 | domains | local | online | unchanged | black.list.threat-intelligence
# Build light Domainlist ... # Build light Domainlist ...
Stats light: Stats light:
** Source (raw): 186960 ** Source (raw): 186907
== Source (unique): 108116 (-78844) == Source (unique): 108069 (-78838)
-- Exclude: 108054 (-62) -- Exclude: 108007 (-62)
-- White: 105806 (-2248) -- White: 105758 (-2249)
-- White(*): 104938 (-868) -- White(*): 104842 (-916)
-- Dead: 87788 (-17150) -- Dead: 87745 (-17097)
++ Block: 87958 (+170) ++ Block: 87915 (+170)
-- Unblock: 87956 (-2) -- Unblock: 87913 (-2)
++ FLD: 88722 (+766) -- Unblock(*): 87913 (-0)
++ WWW: 127384 (+38662) ++ FLD: 88679 (+766)
++ WWW: 127352 (+38673)
***************************************************** 127352 unique Domains - Version 2021.1023.215800
* No changes to the previous repo version detected! * MD5 Domains RAW: d233178ecedc434689cb1db83907252a
*****************************************************
# Convert light to Hostlist ...
# Convert light to AdBlocklist ...
Prepare domain list for compiling ... done.
ℹ Starting @adguard/hostlist-compiler v1.0.12
ℹ Starting the compiler
ℹ Configuration: {
"name": "light",
"sources": [
{
"source": "light.adblock.raw",
"type": "adblock",
"transformations": [
"Validate"
]
}
],
"transformations": [
"Compress"
]
}
ℹ Start compiling light.adblock.raw
ℹ Original length is 86381
ℹ Length after applying transformations is 86380
ℹ The list was compressed from 86383 to 72971
ℹ Final length of the list is 72977
ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/light/out/light.adblock
ℹ Finished compiling
# Attach header to light Domainlist ...
# Push light to local Repositories ...
light - Done! light - Done!
Bereits aktuell. Bereits aktuell.
[master 1a9d9df] light auto-update [master 4ffe92b] light auto-update
5 files changed, 9 insertions(+), 111 deletions(-) 11 files changed, 67 insertions(+), 87 deletions(-)
Bereits aktuell. Bereits aktuell.
Auf Branch main [main 1383e1f] light auto-update
Ihr Branch ist auf demselben Stand wie 'origin/main'. 2 files changed, 27 insertions(+), 28 deletions(-)
nichts zu committen, Arbeitsverzeichnis unverändert
Bereits aktuell. Bereits aktuell.
Auf Branch main [main b1fa8e8] light auto-update
Ihr Branch ist auf demselben Stand wie 'origin/main'. 2 files changed, 27 insertions(+), 27 deletions(-)
nichts zu committen, Arbeitsverzeichnis unverändert
=========================================== ===========================================
Multi ... Multi ...
@ -775,7 +839,7 @@ Initialize ...
2 | 636 | domains | http | online | unchanged | https://v.firebog.net/hosts/Admiral.txt 2 | 636 | domains | http | online | unchanged | https://v.firebog.net/hosts/Admiral.txt
3 | 15120 | domains | http | online | unchanged | https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt 3 | 15120 | domains | http | online | unchanged | https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt
4 | 13720 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/combined_disguised_trackers_justdomains.txt 4 | 13720 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/cname-trackers/master/combined_disguised_trackers_justdomains.txt
5 | 15702 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt 5 | 15712 | domains | http | online | unchanged | https://raw.githubusercontent.com/AdroitAdorKhan/antipopads-re/master/formats/domains.txt
6 | 701 | domains | http | online | unchanged | https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/adservers-and-trackers.txt 6 | 701 | domains | http | online | unchanged | https://raw.githubusercontent.com/DRSDavidSoft/additional-hosts/master/domains/blacklist/adservers-and-trackers.txt
7 | 9922 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/adv/domains 7 | 9922 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/adv/domains
8 | 1486 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/tracker/domains 8 | 1486 | domains | http | online | unchanged | https://raw.githubusercontent.com/cbuijs/shallalist/master/tracker/domains
@ -805,14 +869,14 @@ Initialize ...
32 | 8624 | hosts | http | online | unchanged | https://raw.githubusercontent.com/guardicore/labs_campaigns/master/Autodiscover/autodiscover-tlds.txt 32 | 8624 | hosts | http | online | unchanged | https://raw.githubusercontent.com/guardicore/labs_campaigns/master/Autodiscover/autodiscover-tlds.txt
33 | 1555 | hosts | http | OFFLINE | unchanged | USE LOCAL COPY: urlhaus.abuse.ch_downloads_hostfile_.txt 33 | 1555 | hosts | http | OFFLINE | unchanged | USE LOCAL COPY: urlhaus.abuse.ch_downloads_hostfile_.txt
34 | 373 | hosts | http | online | unchanged | https://raw.githubusercontent.com/Laicure/hosts/master/shithosts 34 | 373 | hosts | http | online | unchanged | https://raw.githubusercontent.com/Laicure/hosts/master/shithosts
35 | 40653 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt 35 | 40609 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
36 | 1019 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_11_Mobile/filter.txt 36 | 1019 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_11_Mobile/filter.txt
37 | 729 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_14_Annoyances/filter.txt 37 | 729 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_14_Annoyances/filter.txt
38 | 5523 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_16_French/filter.txt 38 | 5523 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_16_French/filter.txt
39 | 4649 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_1_Russian/filter.txt 39 | 4649 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_1_Russian/filter.txt
40 | 6551 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_224_Chinese/filter.txt 40 | 6551 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_224_Chinese/filter.txt
41 | 22148 | adblock | http | online | changed | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_2_English/filter.txt 41 | 22148 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_2_English/filter.txt
42 | 16808 | adblock | http | online | changed | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_3_Spyware/filter.txt 42 | 16808 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_3_Spyware/filter.txt
43 | 50 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_4_Social/filter.txt 43 | 50 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_4_Social/filter.txt
44 | 403 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_6_German/filter.txt 44 | 403 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_6_German/filter.txt
45 | 336 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_7_Japanese/filter.txt 45 | 336 | adblock | http | online | unchanged | https://raw.githubusercontent.com/AdguardTeam/FiltersRegistry/master/filters/filter_7_Japanese/filter.txt
@ -834,7 +898,7 @@ Initialize ...
61 | 25 | adblock | http | online | unchanged | https://easylist-downloads.adblockplus.org/koreanlist.txt 61 | 25 | adblock | http | online | unchanged | https://easylist-downloads.adblockplus.org/koreanlist.txt
62 | 5516 | adblock | http | online | unchanged | https://easylist-downloads.adblockplus.org/liste_fr.txt 62 | 5516 | adblock | http | online | unchanged | https://easylist-downloads.adblockplus.org/liste_fr.txt
63 | 18564 | adblock | http | online | unchanged | https://easylist.to/easylist/easylist.txt 63 | 18564 | adblock | http | online | unchanged | https://easylist.to/easylist/easylist.txt
64 | 16183 | adblock | http | online | unchanged | https://easylist.to/easylist/easyprivacy.txt 64 | 16184 | adblock | http | online | unchanged | https://easylist.to/easylist/easyprivacy.txt
65 | 55 | adblock | http | online | unchanged | https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt 65 | 55 | adblock | http | online | unchanged | https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt
66 | 86 | adblock | http | online | unchanged | https://raw.github.com/tomasko126/easylistczechandslovak/master/filters.txt 66 | 86 | adblock | http | online | unchanged | https://raw.github.com/tomasko126/easylistczechandslovak/master/filters.txt
67 | 685 | adblock | http | online | unchanged | https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt 67 | 685 | adblock | http | online | unchanged | https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt
@ -855,45 +919,76 @@ Initialize ...
82 | 4 | adblock | http | online | unchanged | https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resource-abuse.txt 82 | 4 | adblock | http | online | unchanged | https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resource-abuse.txt
83 | 2016 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/TOP_EU_US_Ads_Trackers_ABP 83 | 2016 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/TOP_EU_US_Ads_Trackers_ABP
84 | 4924 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/EU_US%2Bmost_used_ad_and_tracking_networks 84 | 4924 | adblock | http | online | unchanged | https://raw.githubusercontent.com/Kees1958/W3C_annual_most_used_survey_blocklist/master/EU_US%2Bmost_used_ad_and_tracking_networks
85 | 14666 | domains | local | online | unchanged | black.list 85 | 14646 | domains | local | online | unchanged | black.list
86 | 29 | domains | local | online | unchanged | black.list.threat-intelligence 86 | 29 | domains | local | online | unchanged | black.list.threat-intelligence
# Build multi Domainlist ... # Build multi Domainlist ...
Stats multi: Stats multi:
** Source (raw): 946278 ** Source (raw): 946225
== Source (unique): 459073 (-487205) == Source (unique): 459055 (-487170)
-- Exclude: 455033 (-4040) -- Exclude: 455015 (-4040)
-- White: 451579 (-3454) -- White: 451560 (-3455)
-- White(*): 450047 (-1532) -- White(*): 449095 (-2465)
-- Dead: 351731 (-98316) -- Dead: 350812 (-98283)
++ Block: 351901 (+170) ++ Block: 350982 (+170)
-- Unblock: 351899 (-2) -- Unblock: 350980 (-2)
++ FLD: 352574 (+675) -- Unblock(*): 350980 (-0)
++ WWW: 471817 (+119243) ++ FLD: 351655 (+675)
++ WWW: 470908 (+119253)
***************************************************** 470908 unique Domains - Version 2021.1023.215929
* No changes to the previous repo version detected! * MD5 Domains RAW: fd81abfe944299cb865b9f5380337f3a
*****************************************************
# Convert multi to Hostlist ...
# Convert multi to AdBlocklist ...
Prepare domain list for compiling ... done.
ℹ Starting @adguard/hostlist-compiler v1.0.12
ℹ Starting the compiler
ℹ Configuration: {
"name": "multi",
"sources": [
{
"source": "multi.adblock.raw",
"type": "adblock",
"transformations": [
"Validate"
]
}
],
"transformations": [
"Compress"
]
}
ℹ Start compiling multi.adblock.raw
ℹ Original length is 307113
ℹ Length after applying transformations is 307111
ℹ The list was compressed from 307114 to 256413
ℹ Final length of the list is 256419
ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/multi/out/multi.adblock
ℹ Finished compiling
# Attach header to multi Domainlist ...
# Push multi to local Repositories ...
multi - Done! multi - Done!
Bereits aktuell. Bereits aktuell.
[master 0d0dc30] multi auto-update [master ce216b8] multi auto-update
4 files changed, 11 insertions(+), 174 deletions(-) 9 files changed, 47 insertions(+), 51 deletions(-)
Bereits aktuell. Bereits aktuell.
Auf Branch main [main edc509a] multi auto-update
Ihr Branch ist auf demselben Stand wie 'origin/main'. 2 files changed, 20 insertions(+), 21 deletions(-)
nichts zu committen, Arbeitsverzeichnis unverändert
Bereits aktuell. Bereits aktuell.
Auf Branch main [main 5a8a8a9] multi auto-update
Ihr Branch ist auf demselben Stand wie 'origin/main'. 2 files changed, 20 insertions(+), 20 deletions(-)
nichts zu committen, Arbeitsverzeichnis unverändert
=========================================== ===========================================
Affiliate & Tracking ... Affiliate & Tracking ...
@ -974,23 +1069,23 @@ Initialize ...
6 | 22 | domains | http | online | unchanged | https://raw.githubusercontent.com/nextdns/cname-cloaking-blocklist/master/domains 6 | 22 | domains | http | online | unchanged | https://raw.githubusercontent.com/nextdns/cname-cloaking-blocklist/master/domains
7 | 429 | domains | http | online | unchanged | https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains 7 | 429 | domains | http | online | unchanged | https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains
8 | 5 | domains | http | online | unchanged | https://raw.githubusercontent.com/nextdns/metadata/master/security/parked-domains-cname 8 | 5 | domains | http | online | unchanged | https://raw.githubusercontent.com/nextdns/metadata/master/security/parked-domains-cname
9 | 892 | domains | local | online | changed | white.list.referral 9 | 887 | domains | local | online | changed | white.list.referral
10 | 2 | domains | local | online | unchanged | black.list.affiliatetracking 10 | 2 | domains | local | online | unchanged | black.list.affiliatetracking
# Build affiliatetracking Domainlist ... # Build affiliatetracking Domainlist ...
Stats affiliatetracking: Stats affiliatetracking:
** Source (raw): 149617 ** Source (raw): 149612
== Source (unique): 128364 (-21253) == Source (unique): 127930 (-21682)
-- Exclude: 113536 (-14828) -- Exclude: 113104 (-14826)
-- White: 113461 (-75) -- White: 113045 (-59)
-- White(*): 113182 (-279) -- White(*): 112767 (-278)
-- Dead: 111235 (-1947) -- Dead: 110821 (-1946)
++ Block: 111420 (+185) ++ Block: 111006 (+185)
-- Unblock: 111420 (-0) -- Unblock: 111006 (-0)
++ FLD: 111706 (+286) ++ FLD: 111292 (+286)
++ WWW: 112186 (+480) ++ WWW: 111383 (+91)
***************************************************** *****************************************************
* No changes to the previous repo version detected! * * No changes to the previous repo version detected! *
@ -999,8 +1094,8 @@ Stats affiliatetracking:
affiliatetracking - Done! affiliatetracking - Done!
Bereits aktuell. Bereits aktuell.
[master 745d5a3] affiliatetracking auto-update [master 83fd807] affiliatetracking auto-update
3 files changed, 3 insertions(+), 88 deletions(-) 5 files changed, 438 insertions(+), 520 deletions(-)
Bereits aktuell. Bereits aktuell.
Auf Branch main Auf Branch main
@ -1054,46 +1149,176 @@ Initialize ...
# Download and convert Sourcelists ... # Download and convert Sourcelists ...
Nr | Count | Format | Source | Status | File | URL/File Nr | Count | Format | Source | Status | File | URL/File
1 | 471817 | domains | local | online | unchanged | multi.txt 1 | 470908 | domains | local | online | changed | multi.txt
2 | 46559 | domains | local | online | unchanged | fake.txt 2 | 46559 | domains | local | online | unchanged | fake.txt
3 | 971566 | domains | http | online | unchanged | https://dbl.oisd.nl 3 | 971925 | domains | http | online | changed | https://dbl.oisd.nl
4 | 40653 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt 4 | 40609 | adblock | http | online | unchanged | https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
# Build big Domainlist ... # Build big Domainlist ...
Stats big: Stats big:
** Source (raw): 1530595 ** Source (raw): 1530001
== Source (unique): 1242557 (-288038) == Source (unique): 1241902 (-288099)
-- White: 1242293 (-264) -- White: 1241638 (-264)
-- White(*): 1240371 (-1922) -- White(*): 1239716 (-1922)
++ Block: 1240390 (+19) ++ Block: 1239735 (+19)
-- Unblock: 1240338 (-52) -- Unblock: 1239683 (-52)
++ FLD: 1240554 (+216) -- Unblock(*): 1239602 (-81)
++ WWW: 1252608 (+12054) ++ FLD: 1239818 (+216)
++ WWW: 1251825 (+12007)
***************************************************** 1251825 unique Domains - Version 2021.1023.220430
* No changes to the previous repo version detected! * MD5 Domains RAW: 9a2fc41d9b00ca38e4675a0024f4ca53
*****************************************************
# Convert big to Hostlist ...
# Convert big to AdBlocklist ...
ℹ Starting @adguard/hostlist-compiler v1.0.12
ℹ Starting the compiler
ℹ Configuration: {
"name": "Big",
"sources": [
{
"source": "/media/nas/git/adguard/multi.adblock",
"type": "adblock",
"transformations": [
"Validate"
]
},
{
"source": "/media/nas/git/adguard/fake.adblock",
"type": "adblock",
"transformations": [
"Validate"
]
},
{
"source": "https://abp.oisd.nl/",
"type": "adblock",
"exclusions_sources": [
"../exclusions.txt",
"../../../data/adblock.exclusions.referral",
"../../../data/adblock.exclusions.list",
"../../../data/adblock.exclusions.important",
"https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt"
],
"transformations": [
"Validate"
]
},
{
"source": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt",
"type": "adblock",
"exclusions_sources": [
"../exclusions.txt",
"../../../data/adblock.exclusions.referral",
"../../../data/adblock.exclusions.list",
"../../../data/adblock.exclusions.important"
],
"transformations": [
"Validate"
]
},
{
"source": "https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/rules.txt",
"exclusions_sources": [
"../exclusions.txt",
"../../../data/adblock.exclusions.referral",
"../../../data/adblock.exclusions.important"
],
"transformations": [
"Validate"
],
"type": "adblock"
},
{
"source": "../rules.txt",
"exclusions_sources": [
"../exclusions.txt",
"../../../data/adblock.exclusions.referral",
"../../../data/adblock.exclusions.important"
],
"transformations": [
"Validate"
],
"type": "adblock"
},
{
"source": "https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt",
"transformations": [
"Validate"
],
"type": "adblock"
},
{
"source": "../exceptions.txt",
"transformations": [
"Validate"
],
"type": "adblock"
}
],
"transformations": [
"Compress"
]
}
ℹ Start compiling /media/nas/git/adguard/multi.adblock
ℹ Original length is 256539
ℹ Length after applying transformations is 256539
ℹ Start compiling /media/nas/git/adguard/fake.adblock
ℹ Original length is 23682
ℹ Length after applying transformations is 23682
ℹ Start compiling https://abp.oisd.nl/
ℹ Original length is 368421
ℹ Filtering the list of rules using 1439 exclusion rules
ℹ Excluded 319 rules. 368102 rules left.
ℹ Length after applying transformations is 368101
ℹ Start compiling https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt
ℹ Original length is 41680
ℹ Filtering the list of rules using 1114 exclusion rules
ℹ Excluded 57 rules. 41623 rules left.
ℹ Length after applying transformations is 41623
ℹ Start compiling https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/rules.txt
ℹ Original length is 576
ℹ Filtering the list of rules using 472 exclusion rules
ℹ Excluded 0 rules. 576 rules left.
ℹ Length after applying transformations is 570
ℹ Start compiling ../rules.txt
ℹ Original length is 2
ℹ Filtering the list of rules using 472 exclusion rules
ℹ Excluded 0 rules. 2 rules left.
ℹ Length after applying transformations is 2
ℹ Start compiling https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt
ℹ Original length is 179
ℹ Length after applying transformations is 179
ℹ Start compiling ../exceptions.txt
ℹ Original length is 30
ℹ Length after applying transformations is 30
ℹ The list was compressed from 690750 to 490797
ℹ Final length of the list is 490803
ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/big/out/big.adblock
ℹ Finished compiling
# Attach header to big Domainlist ...
# Push big to local Repositories ...
big - Done! big - Done!
Bereits aktuell. Bereits aktuell.
[master 31a478f] big auto-update [master bb093ed] big auto-update
2 files changed, 58 insertions(+), 238 deletions(-) 12 files changed, 1672 insertions(+), 2224 deletions(-)
rewrite pihole/blocklists/build/big/out/big.stats (83%) rewrite pihole/blocklists/build/big/out/big.domains.wildcard.removed (99%)
Bereits aktuell. Bereits aktuell.
Auf Branch main [main 9c26ed2] big auto-update
Ihr Branch ist auf demselben Stand wie 'origin/main'. 2 files changed, 308 insertions(+), 105 deletions(-)
nichts zu committen, Arbeitsverzeichnis unverändert
Bereits aktuell. Bereits aktuell.
Auf Branch main [main bd81652] big auto-update
Ihr Branch ist auf demselben Stand wie 'origin/main'. 2 files changed, 67 insertions(+), 24 deletions(-)
nichts zu committen, Arbeitsverzeichnis unverändert
=========================================== ===========================================
Steven Black Steven Black
@ -1204,19 +1429,19 @@ Initialize ...
# Download and convert Sourcelists ... # Download and convert Sourcelists ...
Nr | Count | Format | Source | Status | File | URL/File Nr | Count | Format | Source | Status | File | URL/File
1 | 1252608 | domains | local | online | unchanged | big.txt 1 | 1251825 | domains | local | online | changed | big.txt
2 | 112186 | domains | local | online | unchanged | affiliatetracking.txt 2 | 111383 | domains | local | online | changed | affiliatetracking.txt
3 | 367703 | domains | local | online | changed | threat-intelligence.txt 3 | 367666 | domains | local | online | changed | threat-intelligence.txt
# Build experimental Domainlist ... # Build experimental Domainlist ...
Stats experimental: Stats experimental:
** Source (raw): 1732497 ** Source (raw): 1730874
== Source (unique): 1458021 (-274476) == Source (unique): 1456282 (-274592)
1458021 unique Domains - Version 2021.1023.195241 1456282 unique Domains - Version 2021.1023.221245
MD5 Domains RAW: cf7e71a365c0fc609ca374e750b9ef9e MD5 Domains RAW: da1e83e75139a8d2a0694be3cdc548a3
# Convert experimental to Hostlist ... # Convert experimental to Hostlist ...
@ -1276,22 +1501,22 @@ MD5 Domains RAW: cf7e71a365c0fc609ca374e750b9ef9e
] ]
} }
ℹ Start compiling /media/nas/git/adguard/big.adblock ℹ Start compiling /media/nas/git/adguard/big.adblock
ℹ Original length is 490776 ℹ Original length is 489786
ℹ Length after applying transformations is 490776 ℹ Length after applying transformations is 489786
ℹ Start compiling /media/nas/git/adguard/affiliatetracking.adblock ℹ Start compiling /media/nas/git/adguard/affiliatetracking.adblock
ℹ Original length is 80836 ℹ Original length is 80442
ℹ Length after applying transformations is 80836 ℹ Length after applying transformations is 80442
ℹ Start compiling /media/nas/git/adguard/threat-intelligence.adblock ℹ Start compiling /media/nas/git/adguard/threat-intelligence.adblock
ℹ Original length is 319726 ℹ Original length is 319615
ℹ Length after applying transformations is 319726 ℹ Length after applying transformations is 319615
ℹ Start compiling ../rules.txt ℹ Start compiling ../rules.txt
ℹ Original length is 1 ℹ Original length is 1
ℹ Length after applying transformations is 1 ℹ Length after applying transformations is 1
ℹ Start compiling ../exceptions.txt ℹ Start compiling ../exceptions.txt
ℹ Original length is 1 ℹ Original length is 1
ℹ Length after applying transformations is 1 ℹ Length after applying transformations is 1
ℹ The list was compressed from 891355 to 638954 ℹ The list was compressed from 889860 to 637567
ℹ Final length of the list is 638960 ℹ Final length of the list is 637573
ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/experimental/out/experimental.adblock ℹ Writing output to /media/nas/git/rpi/pihole/blocklists/build/experimental/out/experimental.adblock
ℹ Finished compiling ℹ Finished compiling
@ -1300,8 +1525,8 @@ MD5 Domains RAW: cf7e71a365c0fc609ca374e750b9ef9e
experimental - Done! experimental - Done!
Bereits aktuell. Bereits aktuell.
[master e7657bb] experimental auto-update [master a8c9972] experimental auto-update
8 files changed, 104 insertions(+), 25 deletions(-) 10 files changed, 16645 insertions(+), 24872 deletions(-)
Bereits aktuell. Bereits aktuell.
Auf Branch main Auf Branch main

198
pihole/blocklists/build/buildList.sh
View File

@ -88,12 +88,12 @@ getDomainsFromList() {
extractValidDomains 0 "$list" >>"$2" extractValidDomains 0 "$list" >>"$2"
done <"$1" done <"$1"
sortList "$2" sortList "$2"
# Extend FLD from SLD # Extend FLD from SLD
if [ "$4" == "1" ]; then if [ "$4" == "1" ]; then
extendFLDfromSLD "$2" extendFLDfromSLD "$2"
fi fi
# Extend missing WWW/FLD # Extend missing WWW/FLD
if [ "$3" == "1" ]; then if [ "$3" == "1" ]; then
extendWWW "$2" extendWWW "$2"
@ -121,9 +121,9 @@ convertWildcardToRegex() {
extractValidDomains() { extractValidDomains() {
# Sourcetype $1: 0=Domains / 1=Hosts / 2=Adblock / 3=Whitelist / 4=Deadlist # Sourcetype $1: 0=Domains / 1=Hosts / 2=Adblock / 3=Whitelist / 4=Deadlist
dos2unix -q "$2" dos2unix -q "$2"
# Domains # Domains
if [ "$1" == "0" ]; then if [ "$1" == "0" ]; then
cat <"$2" | sed -e 's/^[[:space:]]*//' | awk '{print $1}' | grep -Ev '^\s*$|^#|^!' | sed 's/^\.//' | sed 's/^\*\.//' | grep -P "$regex" cat <"$2" | sed -e 's/^[[:space:]]*//' | awk '{print $1}' | grep -Ev '^\s*$|^#|^!' | sed 's/^\.//' | sed 's/^\*\.//' | grep -P "$regex"
@ -135,17 +135,17 @@ extractValidDomains() {
# Adblock # Adblock
if [ "$1" == "2" ]; then if [ "$1" == "2" ]; then
cat <"$2" | sed -e 's/^[[:space:]]*//' | grep ^\|\|.* | sed 's/[\|^]//g' | cat <"$2" | sed -e 's/^[[:space:]]*//' | grep ^\|\|.* | sed 's/[\|^]//g' |
sed 's/$popup,third-party//' | sed 's/$popup,third-party//' |
sed 's/$important,all//' | sed 's/$important,all//' |
sed 's/$third-party//' | sed 's/$third-party//' |
sed 's/$popup//' | sed 's/$popup//' |
sed 's/$important//' | sed 's/$important//' |
sed 's/$all//' | sed 's/$all//' |
sed 's/$document//' | sed 's/$document//' |
sed 's/$doc//' | sed 's/$doc//' |
sed 's/$3p//' | sed 's/$3p//' |
sed 's/$1p//' | sed 's/$1p//' |
grep -P "$regex" grep -P "$regex"
fi fi
# Whitelist # Whitelist
if [ "$1" == "3" ]; then if [ "$1" == "3" ]; then
@ -174,14 +174,14 @@ extendFLD() {
extendFLDfromSLD() { extendFLDfromSLD() {
cat <"$1" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!|^www\.|ip6\.arpa$' | awk '{print $1}' | grep -P '(^[a-zA-Z0-9].*\.(?:[a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]*$))' | grep -P "$regex" >"$1.sld" cat <"$1" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!|^www\.|ip6\.arpa$' | awk '{print $1}' | grep -P '(^[a-zA-Z0-9].*\.(?:[a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]*$))' | grep -P "$regex" >"$1.sld"
cat <"$1.sld" | grep -Po '([a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]{2,}$)|([a-zA-Z0-9\-]*\.{1}(ac|co|gov|ltd|me|net|nhs|nic|org|plc|sch|aaa|aca|acct|ae|ar|biz|br|cn|club|ebiz|gr|gb|game|fin|eu|hu|id|idv|in|info|jp|jur|law|mex|no|nom|or|pp|pro|qc|se|sa|ru|recht|radio|uk|us|uy|za|web)\.{1}[a-zA-Z]{2,}$)' | grep -P "$regex" >"$1.fld" cat <"$1.sld" | grep -Po '([a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]{2,}$)|([a-zA-Z0-9\-]*\.{1}(ac|co|gov|ltd|me|net|nhs|nic|org|plc|sch|aaa|aca|acct|ae|ar|biz|br|cn|club|ebiz|gr|gb|game|fin|eu|hu|id|idv|in|info|jp|jur|law|mex|no|nom|or|pp|pro|qc|se|sa|ru|recht|radio|uk|us|uy|za|web)\.{1}[a-zA-Z]{2,}$)' | grep -P "$regex" >"$1.fld"
cat <"$1.sld" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.1" cat <"$1.sld" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.1"
cat <"$1.fld.1" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.2" cat <"$1.fld.1" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.2"
cat <"$1.fld.2" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.3" cat <"$1.fld.2" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.3"
cat <"$1.fld.3" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.4" cat <"$1.fld.3" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.4"
cat <"$1.fld.4" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.5" cat <"$1.fld.4" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.5"
cat <"$1.fld.5" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.6" cat <"$1.fld.5" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.6"
cat <"$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6" >>"$1" cat <"$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6" >>"$1"
rm -f "$1.sld" "$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6" rm -f "$1.sld" "$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6"
sortList "$1" sortList "$1"
@ -277,14 +277,14 @@ printHeader() {
LinkToFilename() { LinkToFilename() {
echo "$1" | sed 's/https:\/\///' | sed 's/www\.//' | sed 's/bitbucket.org\///' | echo "$1" | sed 's/https:\/\///' | sed 's/www\.//' | sed 's/bitbucket.org\///' |
sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' | sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' |
sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' | sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' |
sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' | sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' |
sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' | sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' |
sed 's/\/domain?format=plain//' | sed 's/\/domain?format=plain//' |
sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" | sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" |
sed 's/[\/]/_/g' | sed 's/[\/]/_/g' |
awk '{print tolower($0)}' awk '{print tolower($0)}'
} }
convertWhiteToAdblock() { convertWhiteToAdblock() {
@ -356,10 +356,10 @@ echo ""
echo "$sourcedir" echo "$sourcedir"
exit 1 exit 1
fi fi
echo "### Compile $name Blocklist ###" echo "### Compile $name Blocklist ###"
echo "" echo ""
# Print config # Print config
echo "Config:" echo "Config:"
echo "" echo ""
@ -374,18 +374,18 @@ echo ""
printConfig "-- Whitelists" "$whitelists" printConfig "-- Whitelists" "$whitelists"
printConfigLocal "- Unblock" "$unblock" printConfigLocal "- Unblock" "$unblock"
printConfig "-- Unblocklists" "$unblocklists" printConfig "-- Unblocklists" "$unblocklists"
# Init # Init
echo "Initialize ..." echo "Initialize ..."
dos2unix -q "$sourcedir"/* dos2unix -q "$sourcedir"/*
echo "" echo ""
# Download and convert Sourcelists # Download and convert Sourcelists
echo "# Download and convert Sourcelists ..." echo "# Download and convert Sourcelists ..."
echo "" echo ""
i=0 i=0
( (
printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "Nr" "Count" "Format" "Source" "Status" "File" "URL/File" printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "Nr" "Count" "Format" "Source" "Status" "File" "URL/File"
@ -393,20 +393,20 @@ echo ""
if [ "${url:0:1}" == "#" ] || [ "${url:0:1}" == "" ]; then if [ "${url:0:1}" == "#" ] || [ "${url:0:1}" == "" ]; then
continue continue
fi fi
((i++)) ((i++))
# Determine list type # Determine list type
if inConfig "white"; then if inConfig "white"; then
listtype=3 # Whitelist listtype=3 # Whitelist
listtypename="white" listtypename="white"
elif inConfig "dead"; then elif inConfig "dead"; then
listtype=4 # Deadlist listtype=4 # Deadlist
listtypename="dead" listtypename="dead"
else else
listtype=0 # Domains listtype=0 # Domains
listtypename="domains" listtypename="domains"
if [ "${url:0:1}" == "!" ]; then if [ "${url:0:1}" == "!" ]; then
url=$(echo "$url" | awk '{print $2}') url=$(echo "$url" | awk '{print $2}')
listtype=1 # Hosts listtype=1 # Hosts
@ -418,9 +418,9 @@ echo ""
listtypename="adblock" listtypename="adblock"
fi fi
fi fi
echo "$i. $listtypename - $url" >>"$domains".sourceurls echo "$i. $listtypename - $url" >>"$domains".sourceurls
# Processing according to source and list type # Processing according to source and list type
if [ "${url:0:4}" == "http" ]; then # online if [ "${url:0:4}" == "http" ]; then # online
source="http" source="http"
@ -436,14 +436,14 @@ echo ""
extractValidDomains $listtype "$url" >"$file.domains" extractValidDomains $listtype "$url" >"$file.domains"
url="$filename" url="$filename"
fi fi
# Check if the source has been changed # Check if the source has been changed
if compareFiles "$file" "$file.domains"; then if compareFiles "$file" "$file.domains"; then
filestatus="unchanged" filestatus="unchanged"
else else
filestatus="changed" filestatus="changed"
fi fi
# If the downloaded source contains domains, use them, # If the downloaded source contains domains, use them,
# otherwise use a cached local copy of the source. # otherwise use a cached local copy of the source.
c=$(wc -l "$file.domains" | awk '{print $1}') c=$(wc -l "$file.domains" | awk '{print $1}')
@ -461,130 +461,128 @@ echo ""
mv "$file.domains" "$file" mv "$file.domains" "$file"
status="online" status="online"
fi fi
# Add domains to the domains list # Add domains to the domains list
if [ -s "$file" ]; then if [ -s "$file" ]; then
cat "$file" >>"$domains" cat "$file" >>"$domains"
fi fi
printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "$i" "$c" "$listtypename" "$source" "$status" "$filestatus" "$url" printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "$i" "$c" "$listtypename" "$source" "$status" "$filestatus" "$url"
done <"$sourcelists" done <"$sourcelists"
) | tee "$header.ext" ) | tee "$header.ext"
sed -i 's/^/# /' "$header.ext" sed -i 's/^/# /' "$header.ext"
echo "" echo ""
# Build Domainlist # Build Domainlist
echo "# Build $name Domainlist ..." echo "# Build $name Domainlist ..."
echo "" echo ""
echo "Stats $name:" echo "Stats $name:"
echo "" echo ""
echo -n "** Source (raw): " echo -n "** Source (raw): "
cdo=$(wc -l "$domains" | awk '{print $1}') cdo=$(wc -l "$domains" | awk '{print $1}')
echo "$cdo" echo "$cdo"
echo -n "== Source (unique): " echo -n "== Source (unique): "
sortList "$domains" sortList "$domains"
printDecCount printDecCount
# Add Domains from personal Blacklists # Add Domains from personal Blacklists
if [ -s "$black" ]; then if [ -s "$black" ]; then
cat <"$black" >>"$domains".blacklist cat <"$black" >>"$domains".blacklist
fi fi
getDomainsFromList "$blacklists" "$domains".blacklist 0 0 getDomainsFromList "$blacklists" "$domains".blacklist 0 0
if [ -s "$domains".blacklist ]; then if [ -s "$domains".blacklist ]; then
echo -n "++ Black: " echo -n "++ Black: "
addDomains "$domains".blacklist "$domains" addDomains "$domains".blacklist "$domains"
printIncCount printIncCount
fi fi
# Remove excluded Domains # Remove excluded Domains
getDomainsFromList "$excludelists" "$domains".excludelist 0 0 getDomainsFromList "$excludelists" "$domains".excludelist 0 0
if [ -s "$domains".excludelist ]; then if [ -s "$domains".excludelist ]; then
echo -n "-- Exclude: " echo -n "-- Exclude: "
removeDomains "$domains".excludelist "$domains" removeDomains "$domains".excludelist "$domains"
printDecCount printDecCount
fi fi
# Save domain list for later dead domains check # Save domain list for later dead domains check
cp "$domains" "$domains".checkfordead cp "$domains" "$domains".checkfordead
# Remove whitelisted Domains # Remove whitelisted Domains
if [ -s "$white" ]; then if [ -s "$white" ]; then
cat <"$white" >>"$domains".whitelist cat <"$white" >>"$domains".whitelist
fi fi
getDomainsFromList "$whitelists" "$domains".whitelist 1 1 getDomainsFromList "$whitelists" "$domains".whitelist 1 1
if [ -s "$domains".whitelist ]; then if [ -s "$domains".whitelist ]; then
echo -n "-- White: " echo -n "-- White: "
removeDomains "$domains".whitelist "$domains" removeDomains "$domains".whitelist "$domains"
# Build Adblock whitelist # Build Adblock whitelist
if inConfig "adblockwhite"; then if inConfig "adblockwhite"; then
if [ -s "$white" ]; then if [ -s "$white" ]; then
convertWhiteToAdblock "$white" "$sourcedir"/exceptions.txt convertWhiteToAdblock "$white" "$sourcedir"/exceptions.txt
fi fi
fi fi
printDecCount printDecCount
fi fi
# Remove wildcard whitelisted Domains # Remove wildcard whitelisted Domains
getRegexDomainsFromWilcardList "$whitelists" "$domains".whitelist.wildcards getRegexDomainsFromWilcardList "$whitelists" "$domains".whitelist.wildcards
if [ -s "$domains".whitelist.wildcards ]; then if [ -s "$domains".whitelist.wildcards ]; then
echo -n "-- White(*): " echo -n "-- White(*): "
removeDomainsWildcard "$domains".whitelist.wildcards "$domains" removeDomainsWildcard "$domains".whitelist.wildcards "$domains"
printDecCount printDecCount
fi fi
# Remove dead Domains # Remove dead Domains
getDomainsFromList "$deadlists" "$domains".deadlist 0 0 getDomainsFromList "$deadlists" "$domains".deadlist 0 0
if [ -s "$domains".deadlist ]; then if [ -s "$domains".deadlist ]; then
echo -n "-- Dead: " echo -n "-- Dead: "
removeDomains "$domains".deadlist "$domains" removeDomains "$domains".deadlist "$domains"
printDecCount printDecCount
fi fi
# Add prioritized blocked Domains # Add prioritized blocked Domains
if [ -s "$block" ]; then if [ -s "$block" ]; then
cat <"$block" >>"$domains".prioblocklist cat <"$block" >>"$domains".prioblocklist
fi fi
getDomainsFromList "$prioblocklists" "$domains".prioblocklist 1 0 getDomainsFromList "$prioblocklists" "$domains".prioblocklist 1 0
if [ -s "$domains".prioblocklist ]; then if [ -s "$domains".prioblocklist ]; then
echo -n "++ Block: " echo -n "++ Block: "
addDomains "$domains".prioblocklist "$domains" addDomains "$domains".prioblocklist "$domains"
printIncCount printIncCount
fi fi
# Unblock prioritized whitelisted Domains # Unblock prioritized whitelisted Domains
if [ -s "$unblock" ]; then if [ -s "$unblock" ]; then
cat <"$unblock" >>"$domains".unblocklist cat <"$unblock" >>"$domains".unblocklist
fi fi
getDomainsFromList "$unblocklists" "$domains".unblocklist 1 1 getDomainsFromList "$unblocklists" "$domains".unblocklist 1 1
if [ -s "$domains".unblocklist ]; then if [ -s "$domains".unblocklist ]; then
echo -n "-- Unblock: " echo -n "-- Unblock: "
removeDomains "$domains".unblocklist "$domains" removeDomains "$domains".unblocklist "$domains"
# Build Adblock unblocklist
if inConfig "adblockwhite"; then
if [ -s "$unblock" ]; then
cat <"$unblock" >>"$domains".unblocklist.adblock
fi
getDomainsFromList "$unblocklists" "$domains".unblocklist.adblock 0 0
convertWhiteToAdblock "$domains".unblocklist.adblock "$outdir"/"$name".unblock.adblock
rm -f "$domains".unblocklist.adblock
fi
printDecCount printDecCount
fi fi
# Unblock prioritized wildcard whitelisted Domains
getRegexDomainsFromWilcardList "$unblocklists" "$domains".unblocklist.wildcards
if [ -s "$domains".unblocklist.wildcards ]; then
echo -n "-- Unblock(*): "
removeDomainsWildcard "$domains".unblocklist.wildcards "$domains"
printDecCount
fi
# Extend by missing WWW/FLD Domains # Extend by missing WWW/FLD Domains
if inConfig "extendWWWFLD"; then if inConfig "extendWWWFLD"; then
if inConfig "white"; then if inConfig "white"; then
@ -592,16 +590,16 @@ echo ""
extendFLDfromSLD "$domains" extendFLDfromSLD "$domains"
printIncCount printIncCount
fi fi
echo -n "++ FLD: " echo -n "++ FLD: "
extendFLD "$domains" extendFLD "$domains"
printIncCount printIncCount
echo -n "++ WWW: " echo -n "++ WWW: "
extendWWW "$domains" extendWWW "$domains"
printIncCount printIncCount
fi fi
# Check if there are changes to the previous repo version # Check if there are changes to the previous repo version
getMd5 "$domains" >"$outdir"/md5.new getMd5 "$domains" >"$outdir"/md5.new
if compareFiles "$outdir/md5.new" "$outdir/md5.old"; then if compareFiles "$outdir/md5.new" "$outdir/md5.old"; then
@ -613,30 +611,30 @@ echo ""
exit 0 exit 0
fi fi
getMd5 "$domains" >"$outdir"/md5.old getMd5 "$domains" >"$outdir"/md5.old
# Cleanup # Cleanup
rm -f "$hosts" "$adblock" rm -f "$hosts" "$adblock"
rm -f "$hosts".* "$adblock".* rm -f "$hosts".* "$adblock".*
# Version # Version
echo "" echo ""
echo "$cdo unique Domains - Version $version" echo "$cdo unique Domains - Version $version"
echo "MD5 Domains RAW: $(getMd5 "$domains")" echo "MD5 Domains RAW: $(getMd5 "$domains")"
echo "" echo ""
# Convert to Hostlist # Convert to Hostlist
if inConfig "hosts"; then if inConfig "hosts"; then
echo "# Convert $name to Hostlist ... " echo "# Convert $name to Hostlist ... "
rm -f "$hosts" rm -f "$hosts"
if ! inConfig "noheader"; then if ! inConfig "noheader"; then
printHeader "$cdo" >"$hosts" printHeader "$cdo" >"$hosts"
fi fi
sed -e 's/^/0.0.0.0 /' "$domains" >>"$hosts" sed -e 's/^/0.0.0.0 /' "$domains" >>"$hosts"
echo "" echo ""
fi fi
# Convert to Adblocklist # Convert to Adblocklist
if inConfig "adblock"; then if inConfig "adblock"; then
echo "# Convert $name to AdBlocklist ..." echo "# Convert $name to AdBlocklist ..."
@ -645,7 +643,7 @@ echo ""
cp $jsonbase "$json" cp $jsonbase "$json"
sed -i "s/tmp/$name/g" "$json" sed -i "s/tmp/$name/g" "$json"
fi fi
if ! inConfig "no_adblock_convert"; then if ! inConfig "no_adblock_convert"; then
echo -n "Prepare domain list for compiling ... " echo -n "Prepare domain list for compiling ... "
if inConfig "adblock_important"; then if inConfig "adblock_important"; then
@ -656,7 +654,7 @@ echo ""
echo "done." echo "done."
echo "" echo ""
fi fi
cd "$outdir" || exit cd "$outdir" || exit
if inConfig "verbose"; then if inConfig "verbose"; then
hostlist-compiler -v -c "$json" -o "$adblock" hostlist-compiler -v -c "$json" -o "$adblock"
@ -665,58 +663,58 @@ echo ""
fi fi
rm -f "$adblock".raw rm -f "$adblock".raw
cd "$sourcedir" || exit cd "$sourcedir" || exit
sortList "$adblock" sortList "$adblock"
if ! inConfig "noheader"; then if ! inConfig "noheader"; then
c=$(grep -Evc '^\s*$|^#|^!' "$adblock") c=$(grep -Evc '^\s*$|^#|^!' "$adblock")
printHeader "$c" | sed 's/^\#/\!/' >"$adblock".tmp printHeader "$c" | sed 's/^\#/\!/' >"$adblock".tmp
fi fi
cat <"$adblock" | grep -Ev '^\s*$|^#|^!' >>"$adblock".tmp cat <"$adblock" | grep -Ev '^\s*$|^#|^!' >>"$adblock".tmp
mv "$adblock".tmp "$adblock" mv "$adblock".tmp "$adblock"
if inConfig "adblocktowildcard"; then if inConfig "adblocktowildcard"; then
cat <"$adblock" | grep -Ev '^\s*$|^#|^!' | sed 's/||/*./' | sed 's/[\^]//g' >"$adblock".wildcards cat <"$adblock" | grep -Ev '^\s*$|^#|^!' | sed 's/||/*./' | sed 's/[\^]//g' >"$adblock".wildcards
fi fi
echo "" echo ""
fi fi
if ! inConfig "noheader"; then if ! inConfig "noheader"; then
# Attach header # Attach header
echo "# Attach header to $name Domainlist ..." echo "# Attach header to $name Domainlist ..."
echo "" echo ""
printHeader "$cdo" >"$domains".tmp printHeader "$cdo" >"$domains".tmp
cat <"$domains" >>"$domains".tmp cat <"$domains" >>"$domains".tmp
mv "$domains".tmp "$domains" mv "$domains".tmp "$domains"
fi fi
# Push to local repository # Push to local repository
if inConfig "localpush"; then if inConfig "localpush"; then
echo "# Push $name to local Repositories ..." echo "# Push $name to local Repositories ..."
echo "" echo ""
dos2unix -q "$outdir"/* dos2unix -q "$outdir"/*
if inConfig "white" || inConfig "dead"; then if inConfig "white" || inConfig "dead"; then
cp "$domains" "$repodata"/"$name".list cp "$domains" "$repodata"/"$name".list
elif inConfig "block"; then elif inConfig "block"; then
cp "$domains" "$repodata"/"$name".list.block cp "$domains" "$repodata"/"$name".list.block
elif inConfig "black"; then elif inConfig "black"; then
cp "$domains" "$repodata"/"$name".list cp "$domains" "$repodata"/"$name".list
cp "$domains".whitelist.removed "$repodata"/"$name".list.wl.removed cp "$domains".whitelist.removed "$repodata"/"$name".list.wl.removed
cp "$domains".wildcard.removed "$repodata"/"$name".list.wc.removed cp "$domains".wildcard.removed "$repodata"/"$name".list.wc.removed
else else
cp "$domains" "$repodomains"/"$name".txt cp "$domains" "$repodomains"/"$name".txt
cp "$stats" "$repodomains" cp "$stats" "$repodomains"
if inConfig "hosts"; then if inConfig "hosts"; then
cp "$hosts" "$repohosts"/"$name".txt cp "$hosts" "$repohosts"/"$name".txt
cp "$stats" "$repohosts" cp "$stats" "$repohosts"
fi fi
if inConfig "adblock"; then if inConfig "adblock"; then
cp "$adblock" "$repoadguard"/"$name".adblock cp "$adblock" "$repoadguard"/"$name".adblock
cp "$stats" "$repoadguard" cp "$stats" "$repoadguard"

57
pihole/blocklists/build/updateWhitelist.sh
View File

@ -49,6 +49,26 @@ convertWhiteToAdBlock() {
done <"$1" done <"$1"
} }
convertWhiteToAdBlockSubdomains() {
rm -f "$2"
while IFS= read -r domain || [ -n "$domain" ]; do
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
echo "$domain" | sed 's/^\#/\!/' >>"$2"
continue
fi
if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
else
if [[ $domain =~ \* ]]; then
echo "@@/$domain/" >>"$2"
else
echo "@@||$domain^|" >>"$2"
fi
fi
done <"$1"
sortList $2
}
# Start # Start
echo '===========================================' echo '==========================================='
echo 'Whitelist ...' echo 'Whitelist ...'
@ -62,7 +82,8 @@ tmp=$data/tmp
# Referral # Referral
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains >$tmp curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains >$tmp
curl -s -L https://raw.githubusercontent.com/anudeepND/whitelist/master/domains/referral-sites.txt >>$tmp curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains |
awk '{print $1}' | grep -Ev '^\s*$|^#|^!|^www' | sed -e 's/^/\*\./' >>$tmp
generateWhitelist $tmp $tmp.1 generateWhitelist $tmp $tmp.1
cat <$tmp.1 >>$data/white.list.referral cat <$tmp.1 >>$data/white.list.referral
rm -f $tmp.1 rm -f $tmp.1
@ -81,35 +102,35 @@ wc -l $data/white.list.oisd
# AdGuard # AdGuard
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt | curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >$tmp grep -P "$regex" >$tmp
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt | curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt | curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' | grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt | curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt | curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt | curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt | curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt | curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp grep -P "$regex" | sed -e 's/^/*./' >>$tmp
generateWhitelist $tmp $data/white.list.adguard generateWhitelist $tmp $data/white.list.adguard
wc -l $data/white.list.adguard wc -l $data/white.list.adguard
@ -123,12 +144,12 @@ cp $data/white.list.referral /media/nas/git/hosts
# Convert personal whitelist to AdBlock format # Convert personal whitelist to AdBlock format
convertWhiteToAdBlock "$data/white.list" /media/nas/git/adguard/whitelist.adguard convertWhiteToAdBlock "$data/white.list" /media/nas/git/adguard/whitelist.adguard
convertWhiteToAdBlock "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard convertWhiteToAdBlockSubdomains "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard
# Build host-compiler exclusion lists # Build host-compiler exclusion lists
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.list" cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.list"
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.important" cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.important"
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.referral" cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.referral"
echo '===========================================' echo '==========================================='
echo 'Deadlist ...' echo 'Deadlist ...'