mirror of https://github.com/Zelo72/rpi
build auto-update
This commit is contained in:
parent
86c08d033d
commit
e4760e1b63
File diff suppressed because it is too large
Load Diff
|
@ -80,12 +80,16 @@ version=$(date +'%Y.%m%d.%H%M%S')
|
||||||
|
|
||||||
# Helpfunctions
|
# Helpfunctions
|
||||||
getDomainsFromList() {
|
getDomainsFromList() {
|
||||||
|
type=$5
|
||||||
|
if [ -z "$5" ]; then
|
||||||
|
type="0"
|
||||||
|
fi
|
||||||
if [ -s "$1" ]; then
|
if [ -s "$1" ]; then
|
||||||
while IFS= read -r list || [ -n "$list" ]; do
|
while IFS= read -r list || [ -n "$list" ]; do
|
||||||
if [ "${list:0:1}" == "#" ] || [ "${list:0:1}" == "" ]; then
|
if [ "${list:0:1}" == "#" ] || [ "${list:0:1}" == "" ]; then
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
extractValidDomains 0 "$list" >>"$2"
|
extractValidDomains $type "$list" >>"$2"
|
||||||
done <"$1"
|
done <"$1"
|
||||||
sortList "$2"
|
sortList "$2"
|
||||||
|
|
||||||
|
@ -135,17 +139,17 @@ extractValidDomains() {
|
||||||
# Adblock
|
# Adblock
|
||||||
if [ "$1" == "2" ]; then
|
if [ "$1" == "2" ]; then
|
||||||
cat <"$2" | sed -e 's/^[[:space:]]*//' | grep ^\|\|.* | sed 's/[\|^]//g' |
|
cat <"$2" | sed -e 's/^[[:space:]]*//' | grep ^\|\|.* | sed 's/[\|^]//g' |
|
||||||
sed 's/$popup,third-party//' |
|
sed 's/$popup,third-party//' |
|
||||||
sed 's/$important,all//' |
|
sed 's/$important,all//' |
|
||||||
sed 's/$third-party//' |
|
sed 's/$third-party//' |
|
||||||
sed 's/$popup//' |
|
sed 's/$popup//' |
|
||||||
sed 's/$important//' |
|
sed 's/$important//' |
|
||||||
sed 's/$all//' |
|
sed 's/$all//' |
|
||||||
sed 's/$document//' |
|
sed 's/$document//' |
|
||||||
sed 's/$doc//' |
|
sed 's/$doc//' |
|
||||||
sed 's/$3p//' |
|
sed 's/$3p//' |
|
||||||
sed 's/$1p//' |
|
sed 's/$1p//' |
|
||||||
grep -P "$regex"
|
grep -P "$regex"
|
||||||
fi
|
fi
|
||||||
# Whitelist
|
# Whitelist
|
||||||
if [ "$1" == "3" ]; then
|
if [ "$1" == "3" ]; then
|
||||||
|
@ -277,14 +281,14 @@ printHeader() {
|
||||||
|
|
||||||
LinkToFilename() {
|
LinkToFilename() {
|
||||||
echo "$1" | sed 's/https:\/\///' | sed 's/www\.//' | sed 's/bitbucket.org\///' |
|
echo "$1" | sed 's/https:\/\///' | sed 's/www\.//' | sed 's/bitbucket.org\///' |
|
||||||
sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' |
|
sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' |
|
||||||
sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' |
|
sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' |
|
||||||
sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' |
|
sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' |
|
||||||
sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' |
|
sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' |
|
||||||
sed 's/\/domain?format=plain//' |
|
sed 's/\/domain?format=plain//' |
|
||||||
sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" |
|
sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" |
|
||||||
sed 's/[\/]/_/g' |
|
sed 's/[\/]/_/g' |
|
||||||
awk '{print tolower($0)}'
|
awk '{print tolower($0)}'
|
||||||
}
|
}
|
||||||
|
|
||||||
convertWhiteToAdblock() {
|
convertWhiteToAdblock() {
|
||||||
|
@ -295,13 +299,9 @@ convertWhiteToAdblock() {
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
if [ "${domain:0:2}" == "*." ]; then
|
if [ "${domain:0:2}" == "*." ]; then
|
||||||
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
|
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
|
||||||
else
|
else
|
||||||
if [[ $domain =~ \* ]]; then
|
echo "@@|$domain^" >>"$2"
|
||||||
echo "@@/$domain/" >>"$2"
|
|
||||||
else
|
|
||||||
echo "@@|$domain^|" >>"$2"
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
done <"$1"
|
done <"$1"
|
||||||
fi
|
fi
|
||||||
|
@ -400,7 +400,7 @@ echo ""
|
||||||
if inConfig "white"; then
|
if inConfig "white"; then
|
||||||
listtype=3 # Whitelist
|
listtype=3 # Whitelist
|
||||||
listtypename="white"
|
listtypename="white"
|
||||||
elif inConfig "dead"; then
|
elif inConfig "dead"; then
|
||||||
listtype=4 # Deadlist
|
listtype=4 # Deadlist
|
||||||
listtypename="dead"
|
listtypename="dead"
|
||||||
else
|
else
|
||||||
|
@ -516,7 +516,7 @@ echo ""
|
||||||
if [ -s "$white" ]; then
|
if [ -s "$white" ]; then
|
||||||
cat <"$white" >>"$domains".whitelist
|
cat <"$white" >>"$domains".whitelist
|
||||||
fi
|
fi
|
||||||
getDomainsFromList "$whitelists" "$domains".whitelist 1 1
|
getDomainsFromList "$whitelists" "$domains".whitelist 1 1 3
|
||||||
|
|
||||||
if [ -s "$domains".whitelist ]; then
|
if [ -s "$domains".whitelist ]; then
|
||||||
echo -n "-- White: "
|
echo -n "-- White: "
|
||||||
|
@ -700,9 +700,9 @@ echo ""
|
||||||
|
|
||||||
if inConfig "white" || inConfig "dead"; then
|
if inConfig "white" || inConfig "dead"; then
|
||||||
cp "$domains" "$repodata"/"$name".list
|
cp "$domains" "$repodata"/"$name".list
|
||||||
elif inConfig "block"; then
|
elif inConfig "block"; then
|
||||||
cp "$domains" "$repodata"/"$name".list.block
|
cp "$domains" "$repodata"/"$name".list.block
|
||||||
elif inConfig "black"; then
|
elif inConfig "black"; then
|
||||||
cp "$domains" "$repodata"/"$name".list
|
cp "$domains" "$repodata"/"$name".list
|
||||||
cp "$domains".whitelist.removed "$repodata"/"$name".list.wl.removed
|
cp "$domains".whitelist.removed "$repodata"/"$name".list.wl.removed
|
||||||
cp "$domains".wildcard.removed "$repodata"/"$name".list.wc.removed
|
cp "$domains".wildcard.removed "$repodata"/"$name".list.wc.removed
|
||||||
|
|
|
@ -32,43 +32,27 @@ sortList() {
|
||||||
|
|
||||||
convertWhiteToAdBlock() {
|
convertWhiteToAdBlock() {
|
||||||
rm -f "$2"
|
rm -f "$2"
|
||||||
|
includesubdomains=$3
|
||||||
|
if [ -z "$3" ]; then
|
||||||
|
includesubdomains="0"
|
||||||
|
fi
|
||||||
while IFS= read -r domain || [ -n "$domain" ]; do
|
while IFS= read -r domain || [ -n "$domain" ]; do
|
||||||
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
|
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
|
||||||
echo "$domain" | sed 's/^\#/\!/' >>"$2"
|
echo "$domain" | sed 's/^\#/\!/' >>"$2"
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
if [ "${domain:0:2}" == "*." ]; then
|
if [ "${domain:0:2}" == "*." ]; then
|
||||||
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
|
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
|
||||||
else
|
else
|
||||||
if [[ $domain =~ \* ]]; then
|
if [ $includesubdomains == "0" ]; then
|
||||||
echo "@@/$domain/" >>"$2"
|
echo "@@|$domain^" >>"$2"
|
||||||
else
|
else
|
||||||
echo "@@|$domain^|" >>"$2"
|
echo "@@||$domain^" >>"$2"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
done <"$1"
|
done <"$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
convertWhiteToAdBlockSubdomains() {
|
|
||||||
rm -f "$2"
|
|
||||||
while IFS= read -r domain || [ -n "$domain" ]; do
|
|
||||||
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
|
|
||||||
echo "$domain" | sed 's/^\#/\!/' >>"$2"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
if [ "${domain:0:2}" == "*." ]; then
|
|
||||||
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
|
|
||||||
else
|
|
||||||
if [[ $domain =~ \* ]]; then
|
|
||||||
echo "@@/$domain/" >>"$2"
|
|
||||||
else
|
|
||||||
echo "@@||$domain^|" >>"$2"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done <"$1"
|
|
||||||
sortList $2
|
|
||||||
}
|
|
||||||
|
|
||||||
# Start
|
# Start
|
||||||
echo '==========================================='
|
echo '==========================================='
|
||||||
echo 'Whitelist ...'
|
echo 'Whitelist ...'
|
||||||
|
@ -83,7 +67,7 @@ tmp=$data/tmp
|
||||||
# Referral
|
# Referral
|
||||||
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains >$tmp
|
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains >$tmp
|
||||||
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains |
|
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains |
|
||||||
awk '{print $1}' | grep -Ev '^\s*$|^#|^!|^www' | sed -e 's/^/\*\./' >>$tmp
|
awk '{print $1}' | grep -Ev '^\s*$|^#|^!|^www' | sed -e 's/^/\*\./' >>$tmp
|
||||||
generateWhitelist $tmp $tmp.1
|
generateWhitelist $tmp $tmp.1
|
||||||
cat <$tmp.1 >>$data/white.list.referral
|
cat <$tmp.1 >>$data/white.list.referral
|
||||||
rm -f $tmp.1
|
rm -f $tmp.1
|
||||||
|
@ -102,35 +86,35 @@ wc -l $data/white.list.oisd
|
||||||
|
|
||||||
# AdGuard
|
# AdGuard
|
||||||
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
|
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
|
||||||
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||||
grep -P "$regex" >$tmp
|
grep -P "$regex" >$tmp
|
||||||
|
|
||||||
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
|
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
|
||||||
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||||
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||||
|
|
||||||
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
|
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
|
||||||
grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' |
|
grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' |
|
||||||
grep -P "$regex" >>$tmp
|
grep -P "$regex" >>$tmp
|
||||||
|
|
||||||
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
|
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
|
||||||
grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||||
|
|
||||||
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
|
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
|
||||||
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||||
grep -P "$regex" >>$tmp
|
grep -P "$regex" >>$tmp
|
||||||
|
|
||||||
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
|
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
|
||||||
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||||
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||||
|
|
||||||
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
|
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
|
||||||
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||||
grep -P "$regex" >>$tmp
|
grep -P "$regex" >>$tmp
|
||||||
|
|
||||||
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
|
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
|
||||||
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||||
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||||
|
|
||||||
generateWhitelist $tmp $data/white.list.adguard
|
generateWhitelist $tmp $data/white.list.adguard
|
||||||
wc -l $data/white.list.adguard
|
wc -l $data/white.list.adguard
|
||||||
|
@ -144,26 +128,26 @@ cp $data/white.list.referral /media/nas/git/hosts
|
||||||
|
|
||||||
# Convert personal whitelist to AdBlock format
|
# Convert personal whitelist to AdBlock format
|
||||||
convertWhiteToAdBlock "$data/white.list" /media/nas/git/adguard/whitelist.adguard
|
convertWhiteToAdBlock "$data/white.list" /media/nas/git/adguard/whitelist.adguard
|
||||||
convertWhiteToAdBlockSubdomains "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard
|
convertWhiteToAdBlock "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard 1
|
||||||
|
|
||||||
# Build host-compiler exclusion lists
|
# Build host-compiler exclusion lists
|
||||||
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.list"
|
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.list"
|
||||||
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
||||||
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
||||||
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
||||||
sort -u >> "$data/adblock.exclusions.list"
|
sort -u >>"$data/adblock.exclusions.list"
|
||||||
|
|
||||||
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.important"
|
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.important"
|
||||||
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
||||||
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
||||||
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
||||||
sort -u >> "$data/adblock.exclusions.important"
|
sort -u >>"$data/adblock.exclusions.important"
|
||||||
|
|
||||||
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.referral"
|
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.referral"
|
||||||
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
||||||
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
||||||
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
||||||
sort -u >> "$data/adblock.exclusions.referral"
|
sort -u >>"$data/adblock.exclusions.referral"
|
||||||
|
|
||||||
echo '==========================================='
|
echo '==========================================='
|
||||||
echo 'Deadlist ...'
|
echo 'Deadlist ...'
|
||||||
|
|
Loading…
Reference in New Issue