mirror of https://github.com/Zelo72/rpi
build auto-update
This commit is contained in:
Zelo72
parent
86c08d033d
commit
e4760e1b63
File diff suppressed because it is too large
Load Diff
|
|
@ -80,20 +80,24 @@ version=$(date +'%Y.%m%d.%H%M%S')
|
|||
|
||||
# Helpfunctions
|
||||
getDomainsFromList() {
|
||||
type=$5
|
||||
if [ -z "$5" ]; then
|
||||
type="0"
|
||||
fi
|
||||
if [ -s "$1" ]; then
|
||||
while IFS= read -r list || [ -n "$list" ]; do
|
||||
if [ "${list:0:1}" == "#" ] || [ "${list:0:1}" == "" ]; then
|
||||
continue
|
||||
fi
|
||||
extractValidDomains 0 "$list" >>"$2"
|
||||
extractValidDomains $type "$list" >>"$2"
|
||||
done <"$1"
|
||||
sortList "$2"
|
||||
|
||||
|
||||
# Extend FLD from SLD
|
||||
if [ "$4" == "1" ]; then
|
||||
extendFLDfromSLD "$2"
|
||||
fi
|
||||
|
||||
|
||||
# Extend missing WWW/FLD
|
||||
if [ "$3" == "1" ]; then
|
||||
extendWWW "$2"
|
||||
|
|
@ -121,9 +125,9 @@ convertWildcardToRegex() {
|
|||
|
||||
extractValidDomains() {
|
||||
# Sourcetype $1: 0=Domains / 1=Hosts / 2=Adblock / 3=Whitelist / 4=Deadlist
|
||||
|
||||
|
||||
dos2unix -q "$2"
|
||||
|
||||
|
||||
# Domains
|
||||
if [ "$1" == "0" ]; then
|
||||
cat <"$2" | sed -e 's/^[[:space:]]*//' | awk '{print $1}' | grep -Ev '^\s*$|^#|^!' | sed 's/^\.//' | sed 's/^\*\.//' | grep -P "$regex"
|
||||
|
|
@ -135,17 +139,17 @@ extractValidDomains() {
|
|||
# Adblock
|
||||
if [ "$1" == "2" ]; then
|
||||
cat <"$2" | sed -e 's/^[[:space:]]*//' | grep ^\|\|.* | sed 's/[\|^]//g' |
|
||||
sed 's/$popup,third-party//' |
|
||||
sed 's/$important,all//' |
|
||||
sed 's/$third-party//' |
|
||||
sed 's/$popup//' |
|
||||
sed 's/$important//' |
|
||||
sed 's/$all//' |
|
||||
sed 's/$document//' |
|
||||
sed 's/$doc//' |
|
||||
sed 's/$3p//' |
|
||||
sed 's/$1p//' |
|
||||
grep -P "$regex"
|
||||
sed 's/$popup,third-party//' |
|
||||
sed 's/$important,all//' |
|
||||
sed 's/$third-party//' |
|
||||
sed 's/$popup//' |
|
||||
sed 's/$important//' |
|
||||
sed 's/$all//' |
|
||||
sed 's/$document//' |
|
||||
sed 's/$doc//' |
|
||||
sed 's/$3p//' |
|
||||
sed 's/$1p//' |
|
||||
grep -P "$regex"
|
||||
fi
|
||||
# Whitelist
|
||||
if [ "$1" == "3" ]; then
|
||||
|
|
@ -174,14 +178,14 @@ extendFLD() {
|
|||
extendFLDfromSLD() {
|
||||
cat <"$1" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!|^www\.|ip6\.arpa$' | awk '{print $1}' | grep -P '(^[a-zA-Z0-9].*\.(?:[a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]*$))' | grep -P "$regex" >"$1.sld"
|
||||
cat <"$1.sld" | grep -Po '([a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]{2,}$)|([a-zA-Z0-9\-]*\.{1}(ac|co|gov|ltd|me|net|nhs|nic|org|plc|sch|aaa|aca|acct|ae|ar|biz|br|cn|club|ebiz|gr|gb|game|fin|eu|hu|id|idv|in|info|jp|jur|law|mex|no|nom|or|pp|pro|qc|se|sa|ru|recht|radio|uk|us|uy|za|web)\.{1}[a-zA-Z]{2,}$)' | grep -P "$regex" >"$1.fld"
|
||||
|
||||
|
||||
cat <"$1.sld" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.1"
|
||||
cat <"$1.fld.1" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.2"
|
||||
cat <"$1.fld.2" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.3"
|
||||
cat <"$1.fld.3" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.4"
|
||||
cat <"$1.fld.4" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.5"
|
||||
cat <"$1.fld.5" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.6"
|
||||
|
||||
|
||||
cat <"$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6" >>"$1"
|
||||
rm -f "$1.sld" "$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6"
|
||||
sortList "$1"
|
||||
|
|
@ -277,14 +281,14 @@ printHeader() {
|
|||
|
||||
LinkToFilename() {
|
||||
echo "$1" | sed 's/https:\/\///' | sed 's/www\.//' | sed 's/bitbucket.org\///' |
|
||||
sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' |
|
||||
sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' |
|
||||
sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' |
|
||||
sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' |
|
||||
sed 's/\/domain?format=plain//' |
|
||||
sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" |
|
||||
sed 's/[\/]/_/g' |
|
||||
awk '{print tolower($0)}'
|
||||
sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' |
|
||||
sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' |
|
||||
sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' |
|
||||
sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' |
|
||||
sed 's/\/domain?format=plain//' |
|
||||
sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" |
|
||||
sed 's/[\/]/_/g' |
|
||||
awk '{print tolower($0)}'
|
||||
}
|
||||
|
||||
convertWhiteToAdblock() {
|
||||
|
|
@ -295,13 +299,9 @@ convertWhiteToAdblock() {
|
|||
continue
|
||||
fi
|
||||
if [ "${domain:0:2}" == "*." ]; then
|
||||
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
|
||||
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
|
||||
else
|
||||
if [[ $domain =~ \* ]]; then
|
||||
echo "@@/$domain/" >>"$2"
|
||||
else
|
||||
echo "@@|$domain^|" >>"$2"
|
||||
fi
|
||||
echo "@@|$domain^" >>"$2"
|
||||
fi
|
||||
done <"$1"
|
||||
fi
|
||||
|
|
@ -356,10 +356,10 @@ echo ""
|
|||
echo "$sourcedir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
echo "### Compile $name Blocklist ###"
|
||||
echo ""
|
||||
|
||||
|
||||
# Print config
|
||||
echo "Config:"
|
||||
echo ""
|
||||
|
|
@ -374,18 +374,18 @@ echo ""
|
|||
printConfig "-- Whitelists" "$whitelists"
|
||||
printConfigLocal "- Unblock" "$unblock"
|
||||
printConfig "-- Unblocklists" "$unblocklists"
|
||||
|
||||
|
||||
# Init
|
||||
echo "Initialize ..."
|
||||
|
||||
|
||||
dos2unix -q "$sourcedir"/*
|
||||
|
||||
|
||||
echo ""
|
||||
|
||||
|
||||
# Download and convert Sourcelists
|
||||
echo "# Download and convert Sourcelists ..."
|
||||
echo ""
|
||||
|
||||
|
||||
i=0
|
||||
(
|
||||
printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "Nr" "Count" "Format" "Source" "Status" "File" "URL/File"
|
||||
|
|
@ -393,20 +393,20 @@ echo ""
|
|||
if [ "${url:0:1}" == "#" ] || [ "${url:0:1}" == "" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
|
||||
((i++))
|
||||
|
||||
|
||||
# Determine list type
|
||||
if inConfig "white"; then
|
||||
listtype=3 # Whitelist
|
||||
listtypename="white"
|
||||
elif inConfig "dead"; then
|
||||
elif inConfig "dead"; then
|
||||
listtype=4 # Deadlist
|
||||
listtypename="dead"
|
||||
else
|
||||
listtype=0 # Domains
|
||||
listtypename="domains"
|
||||
|
||||
|
||||
if [ "${url:0:1}" == "!" ]; then
|
||||
url=$(echo "$url" | awk '{print $2}')
|
||||
listtype=1 # Hosts
|
||||
|
|
@ -418,9 +418,9 @@ echo ""
|
|||
listtypename="adblock"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
echo "$i. $listtypename - $url" >>"$domains".sourceurls
|
||||
|
||||
|
||||
# Processing according to source and list type
|
||||
if [ "${url:0:4}" == "http" ]; then # online
|
||||
source="http"
|
||||
|
|
@ -436,14 +436,14 @@ echo ""
|
|||
extractValidDomains $listtype "$url" >"$file.domains"
|
||||
url="$filename"
|
||||
fi
|
||||
|
||||
|
||||
# Check if the source has been changed
|
||||
if compareFiles "$file" "$file.domains"; then
|
||||
filestatus="unchanged"
|
||||
else
|
||||
filestatus="changed"
|
||||
fi
|
||||
|
||||
|
||||
# If the downloaded source contains domains, use them,
|
||||
# otherwise use a cached local copy of the source.
|
||||
c=$(wc -l "$file.domains" | awk '{print $1}')
|
||||
|
|
@ -461,128 +461,128 @@ echo ""
|
|||
mv "$file.domains" "$file"
|
||||
status="online"
|
||||
fi
|
||||
|
||||
|
||||
# Add domains to the domains list
|
||||
if [ -s "$file" ]; then
|
||||
cat "$file" >>"$domains"
|
||||
fi
|
||||
|
||||
|
||||
printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "$i" "$c" "$listtypename" "$source" "$status" "$filestatus" "$url"
|
||||
done <"$sourcelists"
|
||||
) | tee "$header.ext"
|
||||
sed -i 's/^/# /' "$header.ext"
|
||||
echo ""
|
||||
|
||||
|
||||
# Build Domainlist
|
||||
echo "# Build $name Domainlist ..."
|
||||
echo ""
|
||||
|
||||
|
||||
echo "Stats $name:"
|
||||
echo ""
|
||||
|
||||
|
||||
echo -n "** Source (raw): "
|
||||
cdo=$(wc -l "$domains" | awk '{print $1}')
|
||||
echo "$cdo"
|
||||
|
||||
|
||||
echo -n "== Source (unique): "
|
||||
sortList "$domains"
|
||||
printDecCount
|
||||
|
||||
|
||||
# Add Domains from personal Blacklists
|
||||
if [ -s "$black" ]; then
|
||||
cat <"$black" >>"$domains".blacklist
|
||||
fi
|
||||
getDomainsFromList "$blacklists" "$domains".blacklist 0 0
|
||||
|
||||
|
||||
if [ -s "$domains".blacklist ]; then
|
||||
echo -n "++ Black: "
|
||||
addDomains "$domains".blacklist "$domains"
|
||||
printIncCount
|
||||
fi
|
||||
|
||||
|
||||
# Remove excluded Domains
|
||||
getDomainsFromList "$excludelists" "$domains".excludelist 0 0
|
||||
|
||||
|
||||
if [ -s "$domains".excludelist ]; then
|
||||
echo -n "-- Exclude: "
|
||||
removeDomains "$domains".excludelist "$domains"
|
||||
printDecCount
|
||||
fi
|
||||
|
||||
|
||||
# Save domain list for later dead domains check
|
||||
cp "$domains" "$domains".checkfordead
|
||||
|
||||
|
||||
# Remove whitelisted Domains
|
||||
if [ -s "$white" ]; then
|
||||
cat <"$white" >>"$domains".whitelist
|
||||
fi
|
||||
getDomainsFromList "$whitelists" "$domains".whitelist 1 1
|
||||
|
||||
getDomainsFromList "$whitelists" "$domains".whitelist 1 1 3
|
||||
|
||||
if [ -s "$domains".whitelist ]; then
|
||||
echo -n "-- White: "
|
||||
removeDomains "$domains".whitelist "$domains"
|
||||
|
||||
|
||||
# Build Adblock whitelist
|
||||
if inConfig "adblockwhite"; then
|
||||
if [ -s "$white" ]; then
|
||||
convertWhiteToAdblock "$white" "$sourcedir"/exceptions.txt
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
printDecCount
|
||||
fi
|
||||
|
||||
|
||||
# Remove wildcard whitelisted Domains
|
||||
getRegexDomainsFromWilcardList "$whitelists" "$domains".whitelist.wildcards
|
||||
|
||||
|
||||
if [ -s "$domains".whitelist.wildcards ]; then
|
||||
echo -n "-- White(*): "
|
||||
removeDomainsWildcard "$domains".whitelist.wildcards "$domains"
|
||||
printDecCount
|
||||
fi
|
||||
|
||||
|
||||
# Remove dead Domains
|
||||
getDomainsFromList "$deadlists" "$domains".deadlist 0 0
|
||||
|
||||
|
||||
if [ -s "$domains".deadlist ]; then
|
||||
echo -n "-- Dead: "
|
||||
removeDomains "$domains".deadlist "$domains"
|
||||
printDecCount
|
||||
fi
|
||||
|
||||
|
||||
# Add prioritized blocked Domains
|
||||
if [ -s "$block" ]; then
|
||||
cat <"$block" >>"$domains".prioblocklist
|
||||
fi
|
||||
getDomainsFromList "$prioblocklists" "$domains".prioblocklist 1 0
|
||||
|
||||
|
||||
if [ -s "$domains".prioblocklist ]; then
|
||||
echo -n "++ Block: "
|
||||
addDomains "$domains".prioblocklist "$domains"
|
||||
printIncCount
|
||||
fi
|
||||
|
||||
|
||||
# Unblock prioritized whitelisted Domains
|
||||
if [ -s "$unblock" ]; then
|
||||
cat <"$unblock" >>"$domains".unblocklist
|
||||
fi
|
||||
getDomainsFromList "$unblocklists" "$domains".unblocklist 1 1
|
||||
|
||||
|
||||
if [ -s "$domains".unblocklist ]; then
|
||||
echo -n "-- Unblock: "
|
||||
removeDomains "$domains".unblocklist "$domains"
|
||||
printDecCount
|
||||
fi
|
||||
|
||||
|
||||
# Unblock prioritized wildcard whitelisted Domains
|
||||
getRegexDomainsFromWilcardList "$unblocklists" "$domains".unblocklist.wildcards
|
||||
|
||||
|
||||
if [ -s "$domains".unblocklist.wildcards ]; then
|
||||
echo -n "-- Unblock(*): "
|
||||
removeDomainsWildcard "$domains".unblocklist.wildcards "$domains"
|
||||
printDecCount
|
||||
fi
|
||||
|
||||
|
||||
# Extend by missing WWW/FLD Domains
|
||||
if inConfig "extendWWWFLD"; then
|
||||
if inConfig "white"; then
|
||||
|
|
@ -590,16 +590,16 @@ echo ""
|
|||
extendFLDfromSLD "$domains"
|
||||
printIncCount
|
||||
fi
|
||||
|
||||
|
||||
echo -n "++ FLD: "
|
||||
extendFLD "$domains"
|
||||
printIncCount
|
||||
|
||||
|
||||
echo -n "++ WWW: "
|
||||
extendWWW "$domains"
|
||||
printIncCount
|
||||
fi
|
||||
|
||||
|
||||
# Check if there are changes to the previous repo version
|
||||
getMd5 "$domains" >"$outdir"/md5.new
|
||||
if compareFiles "$outdir/md5.new" "$outdir/md5.old"; then
|
||||
|
|
@ -611,30 +611,30 @@ echo ""
|
|||
exit 0
|
||||
fi
|
||||
getMd5 "$domains" >"$outdir"/md5.old
|
||||
|
||||
|
||||
# Cleanup
|
||||
rm -f "$hosts" "$adblock"
|
||||
rm -f "$hosts".* "$adblock".*
|
||||
|
||||
|
||||
# Version
|
||||
echo ""
|
||||
echo "$cdo unique Domains - Version $version"
|
||||
echo "MD5 Domains RAW: $(getMd5 "$domains")"
|
||||
echo ""
|
||||
|
||||
|
||||
# Convert to Hostlist
|
||||
if inConfig "hosts"; then
|
||||
echo "# Convert $name to Hostlist ... "
|
||||
rm -f "$hosts"
|
||||
|
||||
|
||||
if ! inConfig "noheader"; then
|
||||
printHeader "$cdo" >"$hosts"
|
||||
fi
|
||||
|
||||
|
||||
sed -e 's/^/0.0.0.0 /' "$domains" >>"$hosts"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
|
||||
# Convert to Adblocklist
|
||||
if inConfig "adblock"; then
|
||||
echo "# Convert $name to AdBlocklist ..."
|
||||
|
|
@ -643,7 +643,7 @@ echo ""
|
|||
cp $jsonbase "$json"
|
||||
sed -i "s/tmp/$name/g" "$json"
|
||||
fi
|
||||
|
||||
|
||||
if ! inConfig "no_adblock_convert"; then
|
||||
echo -n "Prepare domain list for compiling ... "
|
||||
if inConfig "adblock_important"; then
|
||||
|
|
@ -654,7 +654,7 @@ echo ""
|
|||
echo "done."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
|
||||
cd "$outdir" || exit
|
||||
if inConfig "verbose"; then
|
||||
hostlist-compiler -v -c "$json" -o "$adblock"
|
||||
|
|
@ -663,58 +663,58 @@ echo ""
|
|||
fi
|
||||
rm -f "$adblock".raw
|
||||
cd "$sourcedir" || exit
|
||||
|
||||
|
||||
sortList "$adblock"
|
||||
|
||||
|
||||
if ! inConfig "noheader"; then
|
||||
c=$(grep -Evc '^\s*$|^#|^!' "$adblock")
|
||||
printHeader "$c" | sed 's/^\#/\!/' >"$adblock".tmp
|
||||
fi
|
||||
|
||||
|
||||
cat <"$adblock" | grep -Ev '^\s*$|^#|^!' >>"$adblock".tmp
|
||||
mv "$adblock".tmp "$adblock"
|
||||
|
||||
|
||||
if inConfig "adblocktowildcard"; then
|
||||
cat <"$adblock" | grep -Ev '^\s*$|^#|^!' | sed 's/||/*./' | sed 's/[\^]//g' >"$adblock".wildcards
|
||||
fi
|
||||
|
||||
|
||||
echo ""
|
||||
fi
|
||||
|
||||
|
||||
if ! inConfig "noheader"; then
|
||||
# Attach header
|
||||
echo "# Attach header to $name Domainlist ..."
|
||||
echo ""
|
||||
|
||||
|
||||
printHeader "$cdo" >"$domains".tmp
|
||||
cat <"$domains" >>"$domains".tmp
|
||||
mv "$domains".tmp "$domains"
|
||||
fi
|
||||
|
||||
|
||||
# Push to local repository
|
||||
if inConfig "localpush"; then
|
||||
echo "# Push $name to local Repositories ..."
|
||||
echo ""
|
||||
|
||||
|
||||
dos2unix -q "$outdir"/*
|
||||
|
||||
|
||||
if inConfig "white" || inConfig "dead"; then
|
||||
cp "$domains" "$repodata"/"$name".list
|
||||
elif inConfig "block"; then
|
||||
elif inConfig "block"; then
|
||||
cp "$domains" "$repodata"/"$name".list.block
|
||||
elif inConfig "black"; then
|
||||
elif inConfig "black"; then
|
||||
cp "$domains" "$repodata"/"$name".list
|
||||
cp "$domains".whitelist.removed "$repodata"/"$name".list.wl.removed
|
||||
cp "$domains".wildcard.removed "$repodata"/"$name".list.wc.removed
|
||||
else
|
||||
cp "$domains" "$repodomains"/"$name".txt
|
||||
cp "$stats" "$repodomains"
|
||||
|
||||
|
||||
if inConfig "hosts"; then
|
||||
cp "$hosts" "$repohosts"/"$name".txt
|
||||
cp "$stats" "$repohosts"
|
||||
fi
|
||||
|
||||
|
||||
if inConfig "adblock"; then
|
||||
cp "$adblock" "$repoadguard"/"$name".adblock
|
||||
cp "$stats" "$repoadguard"
|
||||
|
|
|
|||
|
|
@ -32,43 +32,27 @@ sortList() {
|
|||
|
||||
convertWhiteToAdBlock() {
|
||||
rm -f "$2"
|
||||
includesubdomains=$3
|
||||
if [ -z "$3" ]; then
|
||||
includesubdomains="0"
|
||||
fi
|
||||
while IFS= read -r domain || [ -n "$domain" ]; do
|
||||
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
|
||||
echo "$domain" | sed 's/^\#/\!/' >>"$2"
|
||||
continue
|
||||
fi
|
||||
if [ "${domain:0:2}" == "*." ]; then
|
||||
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
|
||||
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
|
||||
else
|
||||
if [[ $domain =~ \* ]]; then
|
||||
echo "@@/$domain/" >>"$2"
|
||||
if [ $includesubdomains == "0" ]; then
|
||||
echo "@@|$domain^" >>"$2"
|
||||
else
|
||||
echo "@@|$domain^|" >>"$2"
|
||||
echo "@@||$domain^" >>"$2"
|
||||
fi
|
||||
fi
|
||||
done <"$1"
|
||||
}
|
||||
|
||||
convertWhiteToAdBlockSubdomains() {
|
||||
rm -f "$2"
|
||||
while IFS= read -r domain || [ -n "$domain" ]; do
|
||||
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
|
||||
echo "$domain" | sed 's/^\#/\!/' >>"$2"
|
||||
continue
|
||||
fi
|
||||
if [ "${domain:0:2}" == "*." ]; then
|
||||
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^|" >>"$2"
|
||||
else
|
||||
if [[ $domain =~ \* ]]; then
|
||||
echo "@@/$domain/" >>"$2"
|
||||
else
|
||||
echo "@@||$domain^|" >>"$2"
|
||||
fi
|
||||
fi
|
||||
done <"$1"
|
||||
sortList $2
|
||||
}
|
||||
|
||||
# Start
|
||||
echo '==========================================='
|
||||
echo 'Whitelist ...'
|
||||
|
|
@ -83,7 +67,7 @@ tmp=$data/tmp
|
|||
# Referral
|
||||
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains >$tmp
|
||||
curl -s -L https://raw.githubusercontent.com/nextdns/metadata/master/privacy/affiliate-tracking-domains |
|
||||
awk '{print $1}' | grep -Ev '^\s*$|^#|^!|^www' | sed -e 's/^/\*\./' >>$tmp
|
||||
awk '{print $1}' | grep -Ev '^\s*$|^#|^!|^www' | sed -e 's/^/\*\./' >>$tmp
|
||||
generateWhitelist $tmp $tmp.1
|
||||
cat <$tmp.1 >>$data/white.list.referral
|
||||
rm -f $tmp.1
|
||||
|
|
@ -102,35 +86,35 @@ wc -l $data/white.list.oisd
|
|||
|
||||
# AdGuard
|
||||
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
|
||||
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" >$tmp
|
||||
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" >$tmp
|
||||
|
||||
curl -s -L https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt |
|
||||
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||
|
||||
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
|
||||
grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" >>$tmp
|
||||
grep -Ev '^\s*$|^#|^!' | sed 's/[\|^]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" >>$tmp
|
||||
|
||||
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt |
|
||||
grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||
grep -Ev '^\s*$|^#|^!|^\||^\/' | grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||
|
||||
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
|
||||
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" >>$tmp
|
||||
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" >>$tmp
|
||||
|
||||
curl -s -L https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt |
|
||||
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||
|
||||
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
|
||||
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" >>$tmp
|
||||
grep -P '^\@\@' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" >>$tmp
|
||||
|
||||
curl -s -L https://raw.githubusercontent.com/DandelionSprout/adfilt/master/AdGuard%20Home%20Compilation%20List/AdGuardHomeCompilationList.txt |
|
||||
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||
grep -P '^\@\@\|\|' | sed 's/[\|^\@]//g' | sed 's/$important//' |
|
||||
grep -P "$regex" | sed -e 's/^/*./' >>$tmp
|
||||
|
||||
generateWhitelist $tmp $data/white.list.adguard
|
||||
wc -l $data/white.list.adguard
|
||||
|
|
@ -144,26 +128,26 @@ cp $data/white.list.referral /media/nas/git/hosts
|
|||
|
||||
# Convert personal whitelist to AdBlock format
|
||||
convertWhiteToAdBlock "$data/white.list" /media/nas/git/adguard/whitelist.adguard
|
||||
convertWhiteToAdBlockSubdomains "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard
|
||||
convertWhiteToAdBlock "$data/white.list.referral" /media/nas/git/adguard/whitelist.referral.adguard 1
|
||||
|
||||
# Build host-compiler exclusion lists
|
||||
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.list"
|
||||
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
||||
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
||||
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
||||
sort -u >> "$data/adblock.exclusions.list"
|
||||
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.list"
|
||||
cat <"$data/white.list" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
||||
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
||||
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
||||
sort -u >>"$data/adblock.exclusions.list"
|
||||
|
||||
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.important"
|
||||
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.important"
|
||||
cat <"$data/white.list.important" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
||||
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
||||
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
||||
sort -u >> "$data/adblock.exclusions.important"
|
||||
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
||||
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
||||
sort -u >>"$data/adblock.exclusions.important"
|
||||
|
||||
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' > "$data/adblock.exclusions.referral"
|
||||
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
||||
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
||||
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
||||
sort -u >> "$data/adblock.exclusions.referral"
|
||||
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | sed 's/\*\.//' | sort -u | sed -e 's/^/|/' | sed -e 's/$/^/' >"$data/adblock.exclusions.referral"
|
||||
cat <"$data/white.list.referral" | grep -Ev '^\s*$|^#|^!' | grep -E '^\*\.' |
|
||||
sed 's/^\*//' | sed 's/^.//' | sed 's/\./\\./g' | sed -e 's/^/\(\\|\|\\.\|\^\)/' |
|
||||
sed -e 's/^/\//' | sed -e 's/$/\(\$\|\\\^\)\//' |
|
||||
sort -u >>"$data/adblock.exclusions.referral"
|
||||
|
||||
echo '==========================================='
|
||||
echo 'Deadlist ...'
|
||||
|
|
|
|||
Loading…
Reference in New Issue