zelo72
/
rpi
mirror of https://github.com/Zelo72/rpi
1
0
Fork 0
Code Issues Releases Wiki Activity

build auto-update

This commit is contained in:
Zelo72 2021-10-25 12:36:45 +02:00
parent 86c08d033d
commit e4760e1b63
3 changed files with 662 additions and 710 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1080
pihole/blocklists/build/build.log
View File

File diff suppressed because it is too large Load Diff

60
pihole/blocklists/build/buildList.sh
View File

@ -80,12 +80,16 @@ version=$(date +'%Y.%m%d.%H%M%S')
# Helpfunctions
getDomainsFromList() {
type=$5
if [ -z "$5" ]; then
type="0"
fi
if [ -s "$1" ]; then
while IFS= read -r list || [ -n "$list" ]; do
if [ "${list:0:1}" == "#" ] || [ "${list:0:1}" == "" ]; then
continue
fi
extractValidDomains 0 "$list" >>"$2"
extractValidDomains $type "$list" >>"$2"
done <"$1"
sortList "$2"
@ -135,17 +139,17 @@ extractValidDomains() {
# Adblock
if [ "$1" == "2" ]; then
cat <"$2" | sed -e 's/^[[:space:]]*//' | grep ^\|\|.* | sed 's/[\|^]//g' |
sed 's/$popup,third-party//' |
sed 's/$important,all//' |
sed 's/$third-party//' |
sed 's/$popup//' |
sed 's/$important//' |
sed 's/$all//' |
sed 's/$document//' |
sed 's/$doc//' |
sed 's/$3p//' |
sed 's/$1p//' |
grep -P "$regex"
sed 's/$popup,third-party//' |
sed 's/$important,all//' |
sed 's/$third-party//' |
sed 's/$popup//' |
sed 's/$important//' |
sed 's/$all//' |
sed 's/$document//' |
sed 's/$doc//' |
sed 's/$3p//' |
sed 's/$1p//' |
grep -P "$regex"
fi
# Whitelist
if [ "$1" == "3" ]; then
@ -277,14 +281,14 @@ printHeader() {
LinkToFilename() {
echo "$1" | sed 's/https:\/\///' | sed 's/www\.//' | sed 's/bitbucket.org\///' |
sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' |
sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' |
sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' |
sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' |
sed 's/\/domain?format=plain//' |
sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" |
sed 's/[\/]/_/g' |
awk '{print tolower($0)}'
sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' |
sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' |
sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' |
sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' |
sed 's/\/domain?format=plain//' |
sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" |
sed 's/[\/]/_/g' |
awk '{print tolower($0)}'
}
convertWhiteToAdblock() {
@ -295,13 +299,9 @@ convertWhiteToAdblock() {
continue
fi
if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
else
if [[ $domain =~ \* ]]; then
echo "@@/$domain/" >>"$2"
else
echo "@@|$domain^|" >>"$2"
fi
echo "@@|$domain^" >>"$2"
fi
done <"$1"
fi
@ -400,7 +400,7 @@ echo ""
if inConfig "white"; then
listtype=3 # Whitelist
listtypename="white"
elif inConfig "dead"; then
elif inConfig "dead"; then
listtype=4 # Deadlist
listtypename="dead"
else
@ -516,7 +516,7 @@ echo ""
if [ -s "$white" ]; then
cat <"$white" >>"$domains".whitelist
fi
getDomainsFromList "$whitelists" "$domains".whitelist 1 1
getDomainsFromList "$whitelists" "$domains".whitelist 1 1 3
if [ -s "$domains".whitelist ]; then
echo -n "-- White: "
@ -700,9 +700,9 @@ echo ""
if inConfig "white" || inConfig "dead"; then
cp "$domains" "$repodata"/"$name".list
elif inConfig "block"; then
elif inConfig "block"; then
cp "$domains" "$repodata"/"$name".list.block
elif inConfig "black"; then
elif inConfig "black"; then
cp "$domains" "$repodata"/"$name".list
cp "$domains".whitelist.removed "$repodata"/"$name".list.wl.removed
cp "$domains".wildcard.removed "$repodata"/"$name".list.wc.removed

90
pihole/blocklists/build/updateWhitelist.sh
View File

@ -32,43 +32,27 @@ sortList() {
convertWhiteToAdBlock() {
rm -f "$2"
includesubdomains=$3
if [ -z "$3" ]; then
includesubdomains="0"
fi
while IFS= read -r domain || [ -n "$domain" ]; do
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
echo "$domain" | sed 's/^\#/\!/' >>"$2"
continue
fi
if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
else
if [[ $domain =~ \* ]]; then
echo "@@/$domain/" >>"$2"
if [ $includesubdomains == "0" ]; then
echo "@@|$domain^" >>"$2"
else
echo "@@|$domain^|" >>"$2"
echo "@@||$domain^" >>"$2"
fi
fi
done <"$1"
}
convertWhiteToAdBlockSubdomains() {
rm -f "$2"
while IFS= read -r domain || [ -n "$domain" ]; do
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
echo "$domain" | sed 's/^\#/\!/' >>"$2"
continue
fi
if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
else
if [[ $domain =~ \* ]]; then
echo "@@/$domain/" >>"$2"
else
echo "@@||$domain^|" >>"$2"
fi
fi
done <"$1"
sortList $2
}
# Start
echo '==========================================='
echo 'Whitelist ...'
@ -83,7 +67,7 @@ tmp=$data/tmp
# Referral
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains >$tmp
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains |
awk '{print $1}' | grep -Ev '^\s*$|^#|^!|^www' | sed -e 's/^/\*\./' >>$tmp
awk '{print $1}' | grep -Ev '^\s*$|^#|^!|^www' | sed -e 's/^/\*\./' >>$tmp
generateWhitelist $tmp $tmp.1
cat <$tmp.1 >>$data/white.list.referral
rm -f $tmp.1
@ -102,35 +86,35 @@ wc -l $data/white.list.oisd
# AdGuard
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >$tmp
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >$tmp
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp
grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
generateWhitelist $tmp $data/white.list.adguard
wc -l $data/white.list.adguard
@ -144,26 +128,26 @@ cp $data/white.list.referral /media/nas/git/hosts
# Convert personal whitelist to AdBlock format
convertWhiteToAdBlock "$data/white.list" /media/nas/git/adguard/whitelist.adguard
convertWhiteToAdBlockSubdomains "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard
convertWhiteToAdBlock "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard 1
# Build host-compiler exclusion lists
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.list"
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.list"
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >> "$data/adblock.exclusions.list"
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >>"$data/adblock.exclusions.list"
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.important"
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.important"
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >> "$data/adblock.exclusions.important"
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >>"$data/adblock.exclusions.important"
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.referral"
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.referral"
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >> "$data/adblock.exclusions.referral"
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >>"$data/adblock.exclusions.referral"
echo '==========================================='
echo 'Deadlist ...'