zelo72
/
rpi
mirror of https://github.com/Zelo72/rpi
1
0
Fork 0
Code Issues Releases Wiki Activity

build auto-update

This commit is contained in:
Zelo72 2021-10-25 12:36:45 +02:00
parent 86c08d033d
commit e4760e1b63
3 changed files with 662 additions and 710 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1080
pihole/blocklists/build/build.log
View File

File diff suppressed because it is too large Load Diff

198
pihole/blocklists/build/buildList.sh
View File

@ -80,20 +80,24 @@ version=$(date +'%Y.%m%d.%H%M%S')
# Helpfunctions # Helpfunctions
getDomainsFromList() { getDomainsFromList() {
type=$5
if [ -z "$5" ]; then
type="0"
fi
if [ -s "$1" ]; then if [ -s "$1" ]; then
while IFS= read -r list || [ -n "$list" ]; do while IFS= read -r list || [ -n "$list" ]; do
if [ "${list:0:1}" == "#" ] || [ "${list:0:1}" == "" ]; then if [ "${list:0:1}" == "#" ] || [ "${list:0:1}" == "" ]; then
continue continue
fi fi
extractValidDomains 0 "$list" >>"$2" extractValidDomains $type "$list" >>"$2"
done <"$1" done <"$1"
sortList "$2" sortList "$2"
# Extend FLD from SLD # Extend FLD from SLD
if [ "$4" == "1" ]; then if [ "$4" == "1" ]; then
extendFLDfromSLD "$2" extendFLDfromSLD "$2"
fi fi
# Extend missing WWW/FLD # Extend missing WWW/FLD
if [ "$3" == "1" ]; then if [ "$3" == "1" ]; then
extendWWW "$2" extendWWW "$2"
@ -121,9 +125,9 @@ convertWildcardToRegex() {
extractValidDomains() { extractValidDomains() {
# Sourcetype $1: 0=Domains / 1=Hosts / 2=Adblock / 3=Whitelist / 4=Deadlist # Sourcetype $1: 0=Domains / 1=Hosts / 2=Adblock / 3=Whitelist / 4=Deadlist
dos2unix -q "$2" dos2unix -q "$2"
# Domains # Domains
if [ "$1" == "0" ]; then if [ "$1" == "0" ]; then
cat <"$2" | sed -e 's/^[[:space:]]*//' | awk '{print $1}' | grep -Ev '^\s*$|^#|^!' | sed 's/^\.//' | sed 's/^\*\.//' | grep -P "$regex" cat <"$2" | sed -e 's/^[[:space:]]*//' | awk '{print $1}' | grep -Ev '^\s*$|^#|^!' | sed 's/^\.//' | sed 's/^\*\.//' | grep -P "$regex"
@ -135,17 +139,17 @@ extractValidDomains() {
# Adblock # Adblock
if [ "$1" == "2" ]; then if [ "$1" == "2" ]; then
cat <"$2" | sed -e 's/^[[:space:]]*//' | grep ^\|\|.* | sed 's/[\|^]//g' | cat <"$2" | sed -e 's/^[[:space:]]*//' | grep ^\|\|.* | sed 's/[\|^]//g' |
sed 's/$popup,third-party//' | sed 's/$popup,third-party//' |
sed 's/$important,all//' | sed 's/$important,all//' |
sed 's/$third-party//' | sed 's/$third-party//' |
sed 's/$popup//' | sed 's/$popup//' |
sed 's/$important//' | sed 's/$important//' |
sed 's/$all//' | sed 's/$all//' |
sed 's/$document//' | sed 's/$document//' |
sed 's/$doc//' | sed 's/$doc//' |
sed 's/$3p//' | sed 's/$3p//' |
sed 's/$1p//' | sed 's/$1p//' |
grep -P "$regex" grep -P "$regex"
fi fi
# Whitelist # Whitelist
if [ "$1" == "3" ]; then if [ "$1" == "3" ]; then
@ -174,14 +178,14 @@ extendFLD() {
extendFLDfromSLD() { extendFLDfromSLD() {
cat <"$1" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!|^www\.|ip6\.arpa$' | awk '{print $1}' | grep -P '(^[a-zA-Z0-9].*\.(?:[a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]*$))' | grep -P "$regex" >"$1.sld" cat <"$1" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!|^www\.|ip6\.arpa$' | awk '{print $1}' | grep -P '(^[a-zA-Z0-9].*\.(?:[a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]*$))' | grep -P "$regex" >"$1.sld"
cat <"$1.sld" | grep -Po '([a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]{2,}$)|([a-zA-Z0-9\-]*\.{1}(ac|co|gov|ltd|me|net|nhs|nic|org|plc|sch|aaa|aca|acct|ae|ar|biz|br|cn|club|ebiz|gr|gb|game|fin|eu|hu|id|idv|in|info|jp|jur|law|mex|no|nom|or|pp|pro|qc|se|sa|ru|recht|radio|uk|us|uy|za|web)\.{1}[a-zA-Z]{2,}$)' | grep -P "$regex" >"$1.fld" cat <"$1.sld" | grep -Po '([a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]{2,}$)|([a-zA-Z0-9\-]*\.{1}(ac|co|gov|ltd|me|net|nhs|nic|org|plc|sch|aaa|aca|acct|ae|ar|biz|br|cn|club|ebiz|gr|gb|game|fin|eu|hu|id|idv|in|info|jp|jur|law|mex|no|nom|or|pp|pro|qc|se|sa|ru|recht|radio|uk|us|uy|za|web)\.{1}[a-zA-Z]{2,}$)' | grep -P "$regex" >"$1.fld"
cat <"$1.sld" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.1" cat <"$1.sld" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.1"
cat <"$1.fld.1" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.2" cat <"$1.fld.1" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.2"
cat <"$1.fld.2" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.3" cat <"$1.fld.2" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.3"
cat <"$1.fld.3" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.4" cat <"$1.fld.3" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.4"
cat <"$1.fld.4" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.5" cat <"$1.fld.4" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.5"
cat <"$1.fld.5" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.6" cat <"$1.fld.5" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.6"
cat <"$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6" >>"$1" cat <"$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6" >>"$1"
rm -f "$1.sld" "$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6" rm -f "$1.sld" "$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6"
sortList "$1" sortList "$1"
@ -277,14 +281,14 @@ printHeader() {
LinkToFilename() { LinkToFilename() {
echo "$1" | sed 's/https:\/\///' | sed 's/www\.//' | sed 's/bitbucket.org\///' | echo "$1" | sed 's/https:\/\///' | sed 's/www\.//' | sed 's/bitbucket.org\///' |
sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' | sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' |
sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' | sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' |
sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' | sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' |
sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' | sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' |
sed 's/\/domain?format=plain//' | sed 's/\/domain?format=plain//' |
sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" | sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" |
sed 's/[\/]/_/g' | sed 's/[\/]/_/g' |
awk '{print tolower($0)}' awk '{print tolower($0)}'
} }
convertWhiteToAdblock() { convertWhiteToAdblock() {
@ -295,13 +299,9 @@ convertWhiteToAdblock() {
continue continue
fi fi
if [ "${domain:0:2}" == "*." ]; then if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2" echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
else else
if [[ $domain =~ \* ]]; then echo "@@|$domain^" >>"$2"
echo "@@/$domain/" >>"$2"
else
echo "@@|$domain^|" >>"$2"
fi
fi fi
done <"$1" done <"$1"
fi fi
@ -356,10 +356,10 @@ echo ""
echo "$sourcedir" echo "$sourcedir"
exit 1 exit 1
fi fi
echo "### Compile $name Blocklist ###" echo "### Compile $name Blocklist ###"
echo "" echo ""
# Print config # Print config
echo "Config:" echo "Config:"
echo "" echo ""
@ -374,18 +374,18 @@ echo ""
printConfig "-- Whitelists" "$whitelists" printConfig "-- Whitelists" "$whitelists"
printConfigLocal "- Unblock" "$unblock" printConfigLocal "- Unblock" "$unblock"
printConfig "-- Unblocklists" "$unblocklists" printConfig "-- Unblocklists" "$unblocklists"
# Init # Init
echo "Initialize ..." echo "Initialize ..."
dos2unix -q "$sourcedir"/* dos2unix -q "$sourcedir"/*
echo "" echo ""
# Download and convert Sourcelists # Download and convert Sourcelists
echo "# Download and convert Sourcelists ..." echo "# Download and convert Sourcelists ..."
echo "" echo ""
i=0 i=0
( (
printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "Nr" "Count" "Format" "Source" "Status" "File" "URL/File" printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "Nr" "Count" "Format" "Source" "Status" "File" "URL/File"
@ -393,20 +393,20 @@ echo ""
if [ "${url:0:1}" == "#" ] || [ "${url:0:1}" == "" ]; then if [ "${url:0:1}" == "#" ] || [ "${url:0:1}" == "" ]; then
continue continue
fi fi
((i++)) ((i++))
# Determine list type # Determine list type
if inConfig "white"; then if inConfig "white"; then
listtype=3 # Whitelist listtype=3 # Whitelist
listtypename="white" listtypename="white"
elif inConfig "dead"; then elif inConfig "dead"; then
listtype=4 # Deadlist listtype=4 # Deadlist
listtypename="dead" listtypename="dead"
else else
listtype=0 # Domains listtype=0 # Domains
listtypename="domains" listtypename="domains"
if [ "${url:0:1}" == "!" ]; then if [ "${url:0:1}" == "!" ]; then
url=$(echo "$url" | awk '{print $2}') url=$(echo "$url" | awk '{print $2}')
listtype=1 # Hosts listtype=1 # Hosts
@ -418,9 +418,9 @@ echo ""
listtypename="adblock" listtypename="adblock"
fi fi
fi fi
echo "$i. $listtypename - $url" >>"$domains".sourceurls echo "$i. $listtypename - $url" >>"$domains".sourceurls
# Processing according to source and list type # Processing according to source and list type
if [ "${url:0:4}" == "http" ]; then # online if [ "${url:0:4}" == "http" ]; then # online
source="http" source="http"
@ -436,14 +436,14 @@ echo ""
extractValidDomains $listtype "$url" >"$file.domains" extractValidDomains $listtype "$url" >"$file.domains"
url="$filename" url="$filename"
fi fi
# Check if the source has been changed # Check if the source has been changed
if compareFiles "$file" "$file.domains"; then if compareFiles "$file" "$file.domains"; then
filestatus="unchanged" filestatus="unchanged"
else else
filestatus="changed" filestatus="changed"
fi fi
# If the downloaded source contains domains, use them, # If the downloaded source contains domains, use them,
# otherwise use a cached local copy of the source. # otherwise use a cached local copy of the source.
c=$(wc -l "$file.domains" | awk '{print $1}') c=$(wc -l "$file.domains" | awk '{print $1}')
@ -461,128 +461,128 @@ echo ""
mv "$file.domains" "$file" mv "$file.domains" "$file"
status="online" status="online"
fi fi
# Add domains to the domains list # Add domains to the domains list
if [ -s "$file" ]; then if [ -s "$file" ]; then
cat "$file" >>"$domains" cat "$file" >>"$domains"
fi fi
printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "$i" "$c" "$listtypename" "$source" "$status" "$filestatus" "$url" printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "$i" "$c" "$listtypename" "$source" "$status" "$filestatus" "$url"
done <"$sourcelists" done <"$sourcelists"
) | tee "$header.ext" ) | tee "$header.ext"
sed -i 's/^/# /' "$header.ext" sed -i 's/^/# /' "$header.ext"
echo "" echo ""
# Build Domainlist # Build Domainlist
echo "# Build $name Domainlist ..." echo "# Build $name Domainlist ..."
echo "" echo ""
echo "Stats $name:" echo "Stats $name:"
echo "" echo ""
echo -n "** Source (raw): " echo -n "** Source (raw): "
cdo=$(wc -l "$domains" | awk '{print $1}') cdo=$(wc -l "$domains" | awk '{print $1}')
echo "$cdo" echo "$cdo"
echo -n "== Source (unique): " echo -n "== Source (unique): "
sortList "$domains" sortList "$domains"
printDecCount printDecCount
# Add Domains from personal Blacklists # Add Domains from personal Blacklists
if [ -s "$black" ]; then if [ -s "$black" ]; then
cat <"$black" >>"$domains".blacklist cat <"$black" >>"$domains".blacklist
fi fi
getDomainsFromList "$blacklists" "$domains".blacklist 0 0 getDomainsFromList "$blacklists" "$domains".blacklist 0 0
if [ -s "$domains".blacklist ]; then if [ -s "$domains".blacklist ]; then
echo -n "++ Black: " echo -n "++ Black: "
addDomains "$domains".blacklist "$domains" addDomains "$domains".blacklist "$domains"
printIncCount printIncCount
fi fi
# Remove excluded Domains # Remove excluded Domains
getDomainsFromList "$excludelists" "$domains".excludelist 0 0 getDomainsFromList "$excludelists" "$domains".excludelist 0 0
if [ -s "$domains".excludelist ]; then if [ -s "$domains".excludelist ]; then
echo -n "-- Exclude: " echo -n "-- Exclude: "
removeDomains "$domains".excludelist "$domains" removeDomains "$domains".excludelist "$domains"
printDecCount printDecCount
fi fi
# Save domain list for later dead domains check # Save domain list for later dead domains check
cp "$domains" "$domains".checkfordead cp "$domains" "$domains".checkfordead
# Remove whitelisted Domains # Remove whitelisted Domains
if [ -s "$white" ]; then if [ -s "$white" ]; then
cat <"$white" >>"$domains".whitelist cat <"$white" >>"$domains".whitelist
fi fi
getDomainsFromList "$whitelists" "$domains".whitelist 1 1 getDomainsFromList "$whitelists" "$domains".whitelist 1 1 3
if [ -s "$domains".whitelist ]; then if [ -s "$domains".whitelist ]; then
echo -n "-- White: " echo -n "-- White: "
removeDomains "$domains".whitelist "$domains" removeDomains "$domains".whitelist "$domains"
# Build Adblock whitelist # Build Adblock whitelist
if inConfig "adblockwhite"; then if inConfig "adblockwhite"; then
if [ -s "$white" ]; then if [ -s "$white" ]; then
convertWhiteToAdblock "$white" "$sourcedir"/exceptions.txt convertWhiteToAdblock "$white" "$sourcedir"/exceptions.txt
fi fi
fi fi
printDecCount printDecCount
fi fi
# Remove wildcard whitelisted Domains # Remove wildcard whitelisted Domains
getRegexDomainsFromWilcardList "$whitelists" "$domains".whitelist.wildcards getRegexDomainsFromWilcardList "$whitelists" "$domains".whitelist.wildcards
if [ -s "$domains".whitelist.wildcards ]; then if [ -s "$domains".whitelist.wildcards ]; then
echo -n "-- White(*): " echo -n "-- White(*): "
removeDomainsWildcard "$domains".whitelist.wildcards "$domains" removeDomainsWildcard "$domains".whitelist.wildcards "$domains"
printDecCount printDecCount
fi fi
# Remove dead Domains # Remove dead Domains
getDomainsFromList "$deadlists" "$domains".deadlist 0 0 getDomainsFromList "$deadlists" "$domains".deadlist 0 0
if [ -s "$domains".deadlist ]; then if [ -s "$domains".deadlist ]; then
echo -n "-- Dead: " echo -n "-- Dead: "
removeDomains "$domains".deadlist "$domains" removeDomains "$domains".deadlist "$domains"
printDecCount printDecCount
fi fi
# Add prioritized blocked Domains # Add prioritized blocked Domains
if [ -s "$block" ]; then if [ -s "$block" ]; then
cat <"$block" >>"$domains".prioblocklist cat <"$block" >>"$domains".prioblocklist
fi fi
getDomainsFromList "$prioblocklists" "$domains".prioblocklist 1 0 getDomainsFromList "$prioblocklists" "$domains".prioblocklist 1 0
if [ -s "$domains".prioblocklist ]; then if [ -s "$domains".prioblocklist ]; then
echo -n "++ Block: " echo -n "++ Block: "
addDomains "$domains".prioblocklist "$domains" addDomains "$domains".prioblocklist "$domains"
printIncCount printIncCount
fi fi
# Unblock prioritized whitelisted Domains # Unblock prioritized whitelisted Domains
if [ -s "$unblock" ]; then if [ -s "$unblock" ]; then
cat <"$unblock" >>"$domains".unblocklist cat <"$unblock" >>"$domains".unblocklist
fi fi
getDomainsFromList "$unblocklists" "$domains".unblocklist 1 1 getDomainsFromList "$unblocklists" "$domains".unblocklist 1 1
if [ -s "$domains".unblocklist ]; then if [ -s "$domains".unblocklist ]; then
echo -n "-- Unblock: " echo -n "-- Unblock: "
removeDomains "$domains".unblocklist "$domains" removeDomains "$domains".unblocklist "$domains"
printDecCount printDecCount
fi fi
# Unblock prioritized wildcard whitelisted Domains # Unblock prioritized wildcard whitelisted Domains
getRegexDomainsFromWilcardList "$unblocklists" "$domains".unblocklist.wildcards getRegexDomainsFromWilcardList "$unblocklists" "$domains".unblocklist.wildcards
if [ -s "$domains".unblocklist.wildcards ]; then if [ -s "$domains".unblocklist.wildcards ]; then
echo -n "-- Unblock(*): " echo -n "-- Unblock(*): "
removeDomainsWildcard "$domains".unblocklist.wildcards "$domains" removeDomainsWildcard "$domains".unblocklist.wildcards "$domains"
printDecCount printDecCount
fi fi
# Extend by missing WWW/FLD Domains # Extend by missing WWW/FLD Domains
if inConfig "extendWWWFLD"; then if inConfig "extendWWWFLD"; then
if inConfig "white"; then if inConfig "white"; then
@ -590,16 +590,16 @@ echo ""
extendFLDfromSLD "$domains" extendFLDfromSLD "$domains"
printIncCount printIncCount
fi fi
echo -n "++ FLD: " echo -n "++ FLD: "
extendFLD "$domains" extendFLD "$domains"
printIncCount printIncCount
echo -n "++ WWW: " echo -n "++ WWW: "
extendWWW "$domains" extendWWW "$domains"
printIncCount printIncCount
fi fi
# Check if there are changes to the previous repo version # Check if there are changes to the previous repo version
getMd5 "$domains" >"$outdir"/md5.new getMd5 "$domains" >"$outdir"/md5.new
if compareFiles "$outdir/md5.new" "$outdir/md5.old"; then if compareFiles "$outdir/md5.new" "$outdir/md5.old"; then
@ -611,30 +611,30 @@ echo ""
exit 0 exit 0
fi fi
getMd5 "$domains" >"$outdir"/md5.old getMd5 "$domains" >"$outdir"/md5.old
# Cleanup # Cleanup
rm -f "$hosts" "$adblock" rm -f "$hosts" "$adblock"
rm -f "$hosts".* "$adblock".* rm -f "$hosts".* "$adblock".*
# Version # Version
echo "" echo ""
echo "$cdo unique Domains - Version $version" echo "$cdo unique Domains - Version $version"
echo "MD5 Domains RAW: $(getMd5 "$domains")" echo "MD5 Domains RAW: $(getMd5 "$domains")"
echo "" echo ""
# Convert to Hostlist # Convert to Hostlist
if inConfig "hosts"; then if inConfig "hosts"; then
echo "# Convert $name to Hostlist ... " echo "# Convert $name to Hostlist ... "
rm -f "$hosts" rm -f "$hosts"
if ! inConfig "noheader"; then if ! inConfig "noheader"; then
printHeader "$cdo" >"$hosts" printHeader "$cdo" >"$hosts"
fi fi
sed -e 's/^/0.0.0.0 /' "$domains" >>"$hosts" sed -e 's/^/0.0.0.0 /' "$domains" >>"$hosts"
echo "" echo ""
fi fi
# Convert to Adblocklist # Convert to Adblocklist
if inConfig "adblock"; then if inConfig "adblock"; then
echo "# Convert $name to AdBlocklist ..." echo "# Convert $name to AdBlocklist ..."
@ -643,7 +643,7 @@ echo ""
cp $jsonbase "$json" cp $jsonbase "$json"
sed -i "s/tmp/$name/g" "$json" sed -i "s/tmp/$name/g" "$json"
fi fi
if ! inConfig "no_adblock_convert"; then if ! inConfig "no_adblock_convert"; then
echo -n "Prepare domain list for compiling ... " echo -n "Prepare domain list for compiling ... "
if inConfig "adblock_important"; then if inConfig "adblock_important"; then
@ -654,7 +654,7 @@ echo ""
echo "done." echo "done."
echo "" echo ""
fi fi
cd "$outdir" || exit cd "$outdir" || exit
if inConfig "verbose"; then if inConfig "verbose"; then
hostlist-compiler -v -c "$json" -o "$adblock" hostlist-compiler -v -c "$json" -o "$adblock"
@ -663,58 +663,58 @@ echo ""
fi fi
rm -f "$adblock".raw rm -f "$adblock".raw
cd "$sourcedir" || exit cd "$sourcedir" || exit
sortList "$adblock" sortList "$adblock"
if ! inConfig "noheader"; then if ! inConfig "noheader"; then
c=$(grep -Evc '^\s*$|^#|^!' "$adblock") c=$(grep -Evc '^\s*$|^#|^!' "$adblock")
printHeader "$c" | sed 's/^\#/\!/' >"$adblock".tmp printHeader "$c" | sed 's/^\#/\!/' >"$adblock".tmp
fi fi
cat <"$adblock" | grep -Ev '^\s*$|^#|^!' >>"$adblock".tmp cat <"$adblock" | grep -Ev '^\s*$|^#|^!' >>"$adblock".tmp
mv "$adblock".tmp "$adblock" mv "$adblock".tmp "$adblock"
if inConfig "adblocktowildcard"; then if inConfig "adblocktowildcard"; then
cat <"$adblock" | grep -Ev '^\s*$|^#|^!' | sed 's/||/*./' | sed 's/[\^]//g' >"$adblock".wildcards cat <"$adblock" | grep -Ev '^\s*$|^#|^!' | sed 's/||/*./' | sed 's/[\^]//g' >"$adblock".wildcards
fi fi
echo "" echo ""
fi fi
if ! inConfig "noheader"; then if ! inConfig "noheader"; then
# Attach header # Attach header
echo "# Attach header to $name Domainlist ..." echo "# Attach header to $name Domainlist ..."
echo "" echo ""
printHeader "$cdo" >"$domains".tmp printHeader "$cdo" >"$domains".tmp
cat <"$domains" >>"$domains".tmp cat <"$domains" >>"$domains".tmp
mv "$domains".tmp "$domains" mv "$domains".tmp "$domains"
fi fi
# Push to local repository # Push to local repository
if inConfig "localpush"; then if inConfig "localpush"; then
echo "# Push $name to local Repositories ..." echo "# Push $name to local Repositories ..."
echo "" echo ""
dos2unix -q "$outdir"/* dos2unix -q "$outdir"/*
if inConfig "white" || inConfig "dead"; then if inConfig "white" || inConfig "dead"; then
cp "$domains" "$repodata"/"$name".list cp "$domains" "$repodata"/"$name".list
elif inConfig "block"; then elif inConfig "block"; then
cp "$domains" "$repodata"/"$name".list.block cp "$domains" "$repodata"/"$name".list.block
elif inConfig "black"; then elif inConfig "black"; then
cp "$domains" "$repodata"/"$name".list cp "$domains" "$repodata"/"$name".list
cp "$domains".whitelist.removed "$repodata"/"$name".list.wl.removed cp "$domains".whitelist.removed "$repodata"/"$name".list.wl.removed
cp "$domains".wildcard.removed "$repodata"/"$name".list.wc.removed cp "$domains".wildcard.removed "$repodata"/"$name".list.wc.removed
else else
cp "$domains" "$repodomains"/"$name".txt cp "$domains" "$repodomains"/"$name".txt
cp "$stats" "$repodomains" cp "$stats" "$repodomains"
if inConfig "hosts"; then if inConfig "hosts"; then
cp "$hosts" "$repohosts"/"$name".txt cp "$hosts" "$repohosts"/"$name".txt
cp "$stats" "$repohosts" cp "$stats" "$repohosts"
fi fi
if inConfig "adblock"; then if inConfig "adblock"; then
cp "$adblock" "$repoadguard"/"$name".adblock cp "$adblock" "$repoadguard"/"$name".adblock
cp "$stats" "$repoadguard" cp "$stats" "$repoadguard"

94
pihole/blocklists/build/updateWhitelist.sh
View File

@ -32,43 +32,27 @@ sortList() {
convertWhiteToAdBlock() { convertWhiteToAdBlock() {
rm -f "$2" rm -f "$2"
includesubdomains=$3
if [ -z "$3" ]; then
includesubdomains="0"
fi
while IFS= read -r domain || [ -n "$domain" ]; do while IFS= read -r domain || [ -n "$domain" ]; do
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
echo "$domain" | sed 's/^\#/\!/' >>"$2" echo "$domain" | sed 's/^\#/\!/' >>"$2"
continue continue
fi fi
if [ "${domain:0:2}" == "*." ]; then if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2" echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
else else
if [[ $domain =~ \* ]]; then if [ $includesubdomains == "0" ]; then
echo "@@/$domain/" >>"$2" echo "@@|$domain^" >>"$2"
else else
echo "@@|$domain^|" >>"$2" echo "@@||$domain^" >>"$2"
fi fi
fi fi
done <"$1" done <"$1"
} }
convertWhiteToAdBlockSubdomains() {
rm -f "$2"
while IFS= read -r domain || [ -n "$domain" ]; do
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
echo "$domain" | sed 's/^\#/\!/' >>"$2"
continue
fi
if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
else
if [[ $domain =~ \* ]]; then
echo "@@/$domain/" >>"$2"
else
echo "@@||$domain^|" >>"$2"
fi
fi
done <"$1"
sortList $2
}
# Start # Start
echo '===========================================' echo '==========================================='
echo 'Whitelist ...' echo 'Whitelist ...'
@ -83,7 +67,7 @@ tmp=$data/tmp
# Referral # Referral
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains >$tmp curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains >$tmp
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains | curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains |
awk '{print $1}' | grep -Ev '^\s*$|^#|^!|^www' | sed -e 's/^/\*\./' >>$tmp awk '{print $1}' | grep -Ev '^\s*$|^#|^!|^www' | sed -e 's/^/\*\./' >>$tmp
generateWhitelist $tmp $tmp.1 generateWhitelist $tmp $tmp.1
cat <$tmp.1 >>$data/white.list.referral cat <$tmp.1 >>$data/white.list.referral
rm -f $tmp.1 rm -f $tmp.1
@ -102,35 +86,35 @@ wc -l $data/white.list.oisd
# AdGuard # AdGuard
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt | curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >$tmp grep -P "$regex" >$tmp
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt | curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt | curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' | grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt | curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt | curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt | curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt | curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt | curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' | grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp grep -P "$regex" | sed -e 's/^/*./' >>$tmp
generateWhitelist $tmp $data/white.list.adguard generateWhitelist $tmp $data/white.list.adguard
wc -l $data/white.list.adguard wc -l $data/white.list.adguard
@ -144,26 +128,26 @@ cp $data/white.list.referral /media/nas/git/hosts
# Convert personal whitelist to AdBlock format # Convert personal whitelist to AdBlock format
convertWhiteToAdBlock "$data/white.list" /media/nas/git/adguard/whitelist.adguard convertWhiteToAdBlock "$data/white.list" /media/nas/git/adguard/whitelist.adguard
convertWhiteToAdBlockSubdomains "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard convertWhiteToAdBlock "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard 1
# Build host-compiler exclusion lists # Build host-compiler exclusion lists
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.list" cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.list"
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' | cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' | sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' | sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >> "$data/adblock.exclusions.list" sort -u >>"$data/adblock.exclusions.list"
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.important" cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.important"
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' | cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' | sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' | sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >> "$data/adblock.exclusions.important" sort -u >>"$data/adblock.exclusions.important"
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.referral" cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.referral"
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' | cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' | sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' | sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >> "$data/adblock.exclusions.referral" sort -u >>"$data/adblock.exclusions.referral"
echo '===========================================' echo '==========================================='
echo 'Deadlist ...' echo 'Deadlist ...'