zelo72
/
rpi
mirror of https://github.com/Zelo72/rpi
1
0
Fork 0
Code Issues Releases Wiki Activity

build auto-update

This commit is contained in:
Zelo72 2021-10-25 12:36:45 +02:00
parent 86c08d033d
commit e4760e1b63
3 changed files with 662 additions and 710 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1080
pihole/blocklists/build/build.log
View File

File diff suppressed because it is too large Load Diff

16
pihole/blocklists/build/buildList.sh
View File

@ -80,12 +80,16 @@ version=$(date +'%Y.%m%d.%H%M%S')
# Helpfunctions
getDomainsFromList() {
type=$5
if [ -z "$5" ]; then
type="0"
fi
if [ -s "$1" ]; then
while IFS= read -r list || [ -n "$list" ]; do
if [ "${list:0:1}" == "#" ] || [ "${list:0:1}" == "" ]; then
continue
fi
extractValidDomains 0 "$list" >>"$2"
extractValidDomains $type "$list" >>"$2"
done <"$1"
sortList "$2"
@ -295,13 +299,9 @@ convertWhiteToAdblock() {
continue
fi
if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
else
if [[ $domain =~ \* ]]; then
echo "@@/$domain/" >>"$2"
else
echo "@@|$domain^|" >>"$2"
fi
echo "@@|$domain^" >>"$2"
fi
done <"$1"
fi
@ -516,7 +516,7 @@ echo ""
if [ -s "$white" ]; then
cat <"$white" >>"$domains".whitelist
fi
getDomainsFromList "$whitelists" "$domains".whitelist 1 1
getDomainsFromList "$whitelists" "$domains".whitelist 1 1 3
if [ -s "$domains".whitelist ]; then
echo -n "-- White: "

76
pihole/blocklists/build/updateWhitelist.sh
View File

@ -32,43 +32,27 @@ sortList() {
convertWhiteToAdBlock() {
rm -f "$2"
includesubdomains=$3
if [ -z "$3" ]; then
includesubdomains="0"
fi
while IFS= read -r domain || [ -n "$domain" ]; do
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
echo "$domain" | sed 's/^\#/\!/' >>"$2"
continue
fi
if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
else
if [[ $domain =~ \* ]]; then
echo "@@/$domain/" >>"$2"
if [ $includesubdomains == "0" ]; then
echo "@@|$domain^" >>"$2"
else
echo "@@|$domain^|" >>"$2"
echo "@@||$domain^" >>"$2"
fi
fi
done <"$1"
}
convertWhiteToAdBlockSubdomains() {
rm -f "$2"
while IFS= read -r domain || [ -n "$domain" ]; do
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
echo "$domain" | sed 's/^\#/\!/' >>"$2"
continue
fi
if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
else
if [[ $domain =~ \* ]]; then
echo "@@/$domain/" >>"$2"
else
echo "@@||$domain^|" >>"$2"
fi
fi
done <"$1"
sortList $2
}
# Start
echo '==========================================='
echo 'Whitelist ...'
@ -102,35 +86,35 @@ wc -l $data/white.list.oisd
# AdGuard
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >$tmp
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >$tmp
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp
grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" >>$tmp
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
generateWhitelist $tmp $data/white.list.adguard
wc -l $data/white.list.adguard
@ -144,26 +128,26 @@ cp $data/white.list.referral /media/nas/git/hosts
# Convert personal whitelist to AdBlock format
convertWhiteToAdBlock "$data/white.list" /media/nas/git/adguard/whitelist.adguard
convertWhiteToAdBlockSubdomains "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard
convertWhiteToAdBlock "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard 1
# Build host-compiler exclusion lists
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.list"
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.list"
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >> "$data/adblock.exclusions.list"
sort -u >>"$data/adblock.exclusions.list"
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.important"
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.important"
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >> "$data/adblock.exclusions.important"
sort -u >>"$data/adblock.exclusions.important"
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.referral"
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.referral"
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
sort -u >> "$data/adblock.exclusions.referral"
sort -u >>"$data/adblock.exclusions.referral"
echo '==========================================='
echo 'Deadlist ...'