rpi/pihole/blocklists/build/buildList.sh

775 lines
23 KiB
Bash
Executable File

#!/bin/bash
SCRIPT=$(readlink -f "$0")
SCRIPTPATH=$(dirname "$SCRIPT")
cd "$SCRIPTPATH" || exit
# Test internet connection
if ! (ping -c1 -w2 google.de >/dev/null) && ! (ping -c1 -w2 cloudflare.com >/dev/null); then
echo "No Internet connection! The script will be terminated!"
exit 1
fi
# Base
name=$1
basedir=/media/nas/git/rpi/pihole/blocklists/build
builddir=/media/nas/git/rpi/pihole/blocklists/build
sourcedir=$basedir/$name
mkdir -p "$sourcedir"
# Config
getConfigValue() {
grep -w "$1" "$2" | awk -F'=' '{print $2}'
}
inConfig() {
if cat <"$config" | grep -Ev '^\s*$|^#|^!' | grep -qw "$1"; then
return 0
else
return 1
fi
}
config=$sourcedir/$name.config
touch "$config"
# Build config
buildconfig=$builddir/build.config
touch $buildconfig
dos2unix -q $buildconfig
repodomains=$(getConfigValue "repodomains" $buildconfig)
repohosts=$(getConfigValue "repohosts" $buildconfig)
repoadguard=$(getConfigValue "repoadguard" $buildconfig)
repodata=$(getConfigValue "repodata" $buildconfig)
# Domain based configuration files
white=$sourcedir/$name.white
black=$sourcedir/$name.black
block=$sourcedir/$name.block
unblock=$sourcedir/$name.unblock
touch "$white" "$block" "$black" "$unblock"
# List based configuration files
whitelists=$sourcedir/$name.config.whitelists
deadlists=$sourcedir/$name.config.deadlists
excludelists=$sourcedir/$name.config.excludelists
sourcelists=$sourcedir/$name.config.sourcelists
blacklists=$sourcedir/$name.config.blacklists
prioblocklists=$sourcedir/$name.config.prioblocklists
unblocklists=$sourcedir/$name.config.unblocklists
touch "$whitelists" "$deadlists" "$excludelists" "$sourcelists" "$blacklists" "$prioblocklists" "$unblocklists"
# Header configuration
header=$sourcedir/$name.header
touch "$header"
# AdGuard host compiler configuration
json=$sourcedir/$name.json
jsonbase=$basedir/adblock.json
# Input
if inConfig "white" || inConfig "dead"; then
indir=$(getConfigValue "reposourceswhite" $buildconfig)
else
indir=$(getConfigValue "reposources" $buildconfig)
fi
mkdir -p "$indir"
# Output
outdir=$sourcedir/out
mkdir -p "$outdir"
domains=$outdir/$name.domains
hosts=$outdir/$name.hosts
adblock=$outdir/$name.adblock
stats=$outdir/$name.stats
# Valid domain regex
regex="(?=^.{4,253}$)(^(?:[a-zA-Z0-9](?:(?:[a-zA-Z0-9\-]){0,61}[a-zA-Z0-9])?\.)+([a-zA-Z]{2,}|xn--[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])$)"
# Version
version=$(date +'%Y.%m%d.%H%M%S')
# Helpfunctions
getDomainsFromList() {
type=$5
if [ -z "$5" ]; then
type="0"
fi
if [ -s "$1" ]; then
while IFS= read -r list || [ -n "$list" ]; do
if [ "${list:0:1}" == "#" ] || [ "${list:0:1}" == "" ]; then
continue
fi
extractValidDomains $type "$list" >>"$2"
done <"$1"
sortList "$2"
# Extend FLD from SLD
if [ "$4" == "1" ]; then
extendFLDfromSLD "$2"
fi
# Extend missing WWW/FLD
if [ "$3" == "1" ]; then
extendWWW "$2"
extendFLD "$2"
fi
fi
}
getRegexDomainsFromWilcardList() {
if [ -s "$1" ]; then
while IFS= read -r list || [ -n "$list" ]; do
if [ "${list:0:1}" == "#" ] || [ "${list:0:1}" == "" ]; then
continue
fi
convertWildcardToRegex "$list" >>"$2"
done <"$1"
sortList "$2"
if inConfig "debug"; then
cp "$2" "$2".regex
fi
fi
}
convertWildcardToRegex() {
dos2unix -q "$1"
cat <"$1" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!' | awk '{print $1}' | grep -E '^\*\.|^\.' |
sed 's/^\./\(\.\|\^\)/' | sed 's/^\*\./\(\.\|\^\)/' | sed 's/\./\\./g' | sed 's/\-/\\-/g' | sed 's/\*/\.\*/g' | sed -e 's/$/\$/' >tmp
cat <"$1" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!' | awk '{print $1}' | grep -Ev '^\*\.|^\.' | grep -E '\*' |
sed -e 's/^/\^/' | sed 's/\./\\./g' | sed 's/\-/\\-/g' | sed 's/\*/\.\*/g' | sed -e 's/$/\$/' >>tmp
sort -u tmp
rm -f tmp
}
extractValidDomains() {
# Sourcetype $1: 0=Domains / 1=Hosts / 2=Adblock / 3=Whitelist / 4=Deadlist
dos2unix -q "$2"
# Domains
if [ "$1" == "0" ]; then
cat <"$2" | sed -e 's/^[[:space:]]*//' | awk '{print $1}' | grep -Ev '^\s*$|^#|^!' | sed 's/^\.//' | sed 's/^\*\.//' | grep -P "$regex"
fi
# Hosts
if [ "$1" == "1" ]; then
cat <"$2" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!|localhost.localdomain' | awk '{print $2}' | sed 's/^\.//' | sed 's/^\*\.//' | grep -P "$regex"
fi
# Adblock
if [ "$1" == "2" ]; then
cat <"$2" | sed -e 's/^[[:space:]]*//' | grep ^\|\|.* | sed 's/[\|^]//g' |
sed 's/$popup,third-party//' |
sed 's/$important,all//' |
sed 's/$third-party//' |
sed 's/$popup//' |
sed 's/$important//' |
sed 's/$all//' |
sed 's/$document//' |
sed 's/$doc//' |
sed 's/$3p//' |
sed 's/$1p//' |
grep -P "$regex"
fi
# Whitelist
if [ "$1" == "3" ]; then
cat <"$2" | sed -e 's/^[[:space:]]*//' | awk '{print $1}' | grep -Ev '^\s*$|^#|^!' # | sed 's/^\./\*\./' | sed 's/^\*\.\*\./\*\./'
fi
# Deadlist
if [ "$1" == "4" ]; then
cat <"$2" | sed -e 's/^[[:space:]]*//' | awk '{print $1}' | grep -Ev '^\s*$|^#|^!' | sed 's/^\.//' | sed 's/^\*\.//'
fi
}
extendWWW() {
cat <"$1" | grep -Ev '^www\.' | grep -P '(^[a-zA-Z0-9](?:[a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]*$))' | sed -e 's/^/www./' | grep -P "$regex" >"$1.www"
cat "$1.www" >>"$1"
rm "$1.www"
sortList "$1"
}
extendFLD() {
cat <"$1" | grep -P '(^[wW]{3}\.(?:[a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]*$))' | sed -r 's/^(www\.)//' | grep -P "$regex" >"$1.fld"
cat "$1.fld" >>"$1"
rm "$1.fld"
sortList "$1"
}
extendFLDfromSLD() {
cat <"$1" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!|^www\.|ip6\.arpa$' | awk '{print $1}' | grep -P '(^[a-zA-Z0-9].*\.(?:[a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]*$))' | grep -P "$regex" >"$1.sld"
cat <"$1.sld" | grep -Po '([a-zA-Z0-9\-]*\.{1}[a-zA-Z0-9\-]{2,}$)|([a-zA-Z0-9\-]*\.{1}(ac|co|gov|ltd|me|net|nhs|nic|org|plc|sch|aaa|aca|acct|ae|ar|biz|br|cn|club|ebiz|gr|gb|game|fin|eu|hu|id|idv|in|info|jp|jur|law|mex|no|nom|or|pp|pro|qc|se|sa|ru|recht|radio|uk|us|uy|za|web)\.{1}[a-zA-Z]{2,}$)' | grep -P "$regex" >"$1.fld"
cat <"$1.sld" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.1"
cat <"$1.fld.1" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.2"
cat <"$1.fld.2" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.3"
cat <"$1.fld.3" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.4"
cat <"$1.fld.4" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.5"
cat <"$1.fld.5" | sed 's/\./ /' | awk '{print $2}' | grep -P "$regex" >"$1.fld.6"
cat <"$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6" >>"$1"
rm -f "$1.sld" "$1.fld" "$1.fld.1" "$1.fld.2" "$1.fld.3" "$1.fld.4" "$1.fld.5" "$1.fld.6"
sortList "$1"
}
sortList() {
touch "$1"
if [ -s "$1" ]; then
sort -uf "$1" >"$1.sorted"
mv "$1.sorted" "$1"
fi
}
printIncCount() {
c=$(wc -l "$domains" | awk '{print $1}')
echo "$c (+$((c - cdo)))"
cdo=$c
}
printDecCount() {
c=$(wc -l "$domains" | awk '{print $1}')
echo "$c (-$((cdo - c)))"
cdo=$c
}
removeDomains() {
if [ -s "$1" ] && [ -s "$2" ]; then
comm -13 <(tr '[:upper:]' '[:lower:]' <"$1" | sort) <(tr '[:upper:]' '[:lower:]' <"$2" | sort) | awk '{print $1}' >"$2.tmp"
if inConfig "debug"; then
echo "# ---- START ----" >>"$1.removed"
comm -12 <(tr '[:upper:]' '[:lower:]' <"$1" | sort) <(tr '[:upper:]' '[:lower:]' <"$2" | sort) | awk '{print $1}' >>"$1.removed"
echo "# ---- END ----" >>"$1.removed"
fi
mv "$2.tmp" "$2"
rm "$1"
fi
}
removeDomainsWildcard() {
if [ -s "$1" ]; then
while IFS= read -r domain || [ -n "$domain" ]; do
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
continue
fi
cat <"$2" | rg -N "$domain" >>"$2.wildcard"
done <"$1"
rm "$1"
if [ -s "$2.wildcard" ]; then
sortList "$2.wildcard"
removeDomains "$2.wildcard" "$2"
fi
fi
}
addDomains() {
if [ -s "$1" ]; then
if inConfig "debug"; then
comm -23 <(tr '[:upper:]' '[:lower:]' <"$1" | sort) <(tr '[:upper:]' '[:lower:]' <"$2" | sort) | awk '{print $1}' >"$1.added"
fi
cat <"$1" | sed -e 's/^[[:space:]]*//' | grep -Ev '^\s*$|^#|^!' | awk '{print $1}' | grep -P "$regex" >>"$2"
sortList "$2"
rm "$1"
fi
}
printConfig() {
if [ -s "$2" ]; then
c=$(wc -l <(cat <"$2" | grep -Ev '^\s*$|^#') | awk '{print $1}')
if [ "$c" != "0" ]; then
echo "$1 - $c:"
echo ""
cat <"$2" | grep -Ev '^\s*$|^#' | sed -e 's/^/ /'
echo ""
fi
fi
}
printConfigLocal() {
if [ -s "$2" ]; then
c=$(wc -l <(cat <"$2" | grep -Ev '^\s*$|^#') | awk '{print $1}')
if [ "$c" != "0" ]; then
echo " $1 - $c:"
echo ""
echo "$2" | grep -Po '[a-zA-Z0-9-_\.]*$' | sed -e 's/^/ /'
echo ""
fi
fi
}
printHeader() {
cat "$header"
echo "#"
cat "$header".ext
echo "#"
echo "# $1 unique Domains - Version $version"
echo "#"
}
LinkToFilename() {
echo "$1" | sed 's/http:\/\///' | sed 's/https:\/\///' | sed 's/www\.//' | sed 's/bitbucket.org\///' |
sed 's/raw.githubusercontent.com\///' | sed 's/gist.githubusercontent.com\///' |
sed 's/gitlab.com\///' | sed 's/github.com\///' | sed 's/gitlab\.//' | sed 's/\/raw\master\//_/' |
sed 's/\/raw\//_/' | sed 's/\/master\//_/' | sed 's/\/main\//_/' | sed 's/.txt//' |
sed 's/\.php?hostformat=hosts&showintro=0&mimetype=plaintext//' |
sed 's/\/domain?format=plain//' |
sed 's/%20//g' | sed 's/%2B//g' | sed "s/[\']//g" |
sed 's/[\/?=&]/_/g' |
awk '{print tolower($0)}'
}
convertWhiteToAdblock() {
rm -f "$2"
if [ -s "$1" ]; then
while IFS= read -r domain || [ -n "$domain" ]; do
if [ "${domain:0:1}" == "#" ] || [ "${domain:0:1}" == "" ]; then
continue
fi
if [ "${domain:0:2}" == "*." ]; then
echo "@@||$(echo "$domain" | sed 's/^\*\.//')^" >>"$2"
else
echo "@@|$domain^" >>"$2"
fi
done <"$1"
fi
touch "$2"
}
getMd5() {
if [ -s "$1" ]; then
md5sum "$1" | awk '{print $1}'
else
echo "0"
fi
}
compareFiles() {
md51=1
md52=2
if [ -s "$1" ]; then
md51=$(getMd5 "$1")
fi
if [ -s "$2" ]; then
md52=$(getMd5 "$2")
fi
if [ "$md51" == "$md52" ]; then
return 0
else
return 1
fi
}
# Start
echo ""
echo "Start: $(date +'%Y.%m-%d-%H:%M:%S')"
# Cleanup
touch "$outdir"/md5.old
rm -f "$domains" "$stats"
rm -f "$domains".* "$stats".*
rm -f "$outdir"/md5.new
cd "$sourcedir" || exit
echo ""
(
if [ ! -s "$sourcelists" ]; then
echo "No configuration files found. Empty configuration files were created."
echo "$sourcedir"
exit 1
fi
echo "### Compile $name Blocklist ###"
echo ""
# Print config
echo "Config:"
echo ""
printConfig "** Build" "$config"
printConfigLocal "+ Blacklist" "$black"
printConfig "++ Blacklists" "$blacklists"
printConfigLocal "+ Prioritized Block" "$block"
printConfig "++ Prioritized Blocklists" "$prioblocklists"
printConfig "-- Excludelists" "$excludelists"
printConfig "-- Deadlists" "$deadlists"
printConfigLocal "- White" "$white"
printConfig "-- Whitelists" "$whitelists"
printConfigLocal "- Unblock" "$unblock"
printConfig "-- Unblocklists" "$unblocklists"
# Init
echo "Initialize ..."
dos2unix -q "$sourcedir"/*
echo ""
# Download and convert Sourcelists
echo "# Download and convert Sourcelists ..."
echo ""
i=0
(
printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "Nr" "Count" "Format" "Source" "Status" "File" "URL/File"
while IFS= read -r url || [ -n "$url" ]; do
if [ "${url:0:1}" == "#" ] || [ "${url:0:1}" == "" ]; then
continue
fi
((i++))
# Determine list type
if inConfig "white"; then
listtype=3 # Whitelist
listtypename="white"
elif inConfig "dead"; then
listtype=4 # Deadlist
listtypename="dead"
else
listtype=0 # Domains
listtypename="domains"
if [ "${url:0:1}" == "!" ]; then
url=$(echo "$url" | awk '{print $2}')
listtype=1 # Hosts
listtypename="hosts"
fi
if [ "${url:0:1}" == "&" ]; then
url=$(echo "$url" | awk '{print $2}')
listtype=2 # Adblock
listtypename="adblock"
fi
fi
echo "$i. $listtypename - $url" >>"$domains".sourceurls
# Processing according to source and list type
if [ "${url:0:4}" == "http" ]; then # online
source="http"
filename=$(LinkToFilename "$url").txt
file=$indir/$filename
curl -s -L "$url" >"$file.tmp"
extractValidDomains $listtype "$file.tmp" >"$file.domains"
rm "$file.tmp"
else # local
source="local"
filename=$(echo "$url" | grep -Po '[a-zA-Z0-9-_\.]*$')
file=$indir/$filename
extractValidDomains $listtype "$url" >"$file.domains"
url="$filename"
fi
# Check if the source has been changed
if compareFiles "$file" "$file.domains"; then
filestatus="unchanged"
else
filestatus="changed"
fi
# If the downloaded source contains domains, use them,
# otherwise use a cached local copy of the source.
c=$(wc -l "$file.domains" | awk '{print $1}')
if [ ! "$c" ] || [ "$c" == "0" ]; then
c=0
filestatus="OFFLINE"
rm "$file.domains"
if [ -s "$file" ]; then
c=$(wc -l "$file" | awk '{print $1}')
url="USE LOCAL COPY: $filename"
filestatus="unchanged"
fi
status="OFFLINE"
else
mv "$file.domains" "$file"
status="online"
fi
# Add domains to the domains list
if [ -s "$file" ]; then
cat "$file" >>"$domains"
fi
#echo "$file" >>$builddir/sourcefiles.txt
printf "%+4s | %+7s | %-7s | %-6s | %-7s | %-9s | %s\n" "$i" "$c" "$listtypename" "$source" "$status" "$filestatus" "$url"
done <"$sourcelists"
) | tee "$header.ext"
sed -i 's/^/# /' "$header.ext"
echo ""
# Build Domainlist
echo "# Build $name Domainlist ..."
echo ""
echo "Stats $name:"
echo ""
echo -n "** Source (raw): "
cdo=$(wc -l "$domains" | awk '{print $1}')
echo "$cdo"
echo -n "== Source (unique): "
sortList "$domains"
printDecCount
# Save unique domain list
if inConfig "unique"; then
cp "$domains" "$domains".unique
fi
# Add Domains from personal Blacklists
if [ -s "$black" ]; then
cat <"$black" >>"$domains".blacklist
fi
getDomainsFromList "$blacklists" "$domains".blacklist 0 0
if [ -s "$domains".blacklist ]; then
echo -n "++ Black: "
addDomains "$domains".blacklist "$domains"
printIncCount
fi
# Remove excluded Domains
getDomainsFromList "$excludelists" "$domains".excludelist 0 0
if [ -s "$domains".excludelist ]; then
echo -n "-- Exclude: "
removeDomains "$domains".excludelist "$domains"
printDecCount
fi
# Save domain list for later dead domains check
if inConfig "checkfordead"; then
cp "$domains" "$domains".checkfordead
fi
# Remove whitelisted Domains
if [ -s "$white" ]; then
cat <"$white" >>"$domains".whitelist
fi
getDomainsFromList "$whitelists" "$domains".whitelist 1 1 3
if [ -s "$domains".whitelist ]; then
echo -n "-- White: "
removeDomains "$domains".whitelist "$domains"
# Build Adblock whitelist
if inConfig "adblockwhite"; then
if [ -s "$white" ]; then
convertWhiteToAdblock "$white" "$sourcedir"/exceptions.txt
fi
fi
printDecCount
fi
# Remove wildcard whitelisted Domains
if [ -s "$white" ]; then
cat <"$white" | grep -E '\*' >"$domains".whitelist.wildcards.tmp
convertWildcardToRegex "$domains".whitelist.wildcards.tmp >>"$domains".whitelist.wildcards
rm -f "$domains".whitelist.wildcards.tmp
fi
getRegexDomainsFromWilcardList "$whitelists" "$domains".whitelist.wildcards
if [ -s "$domains".whitelist.wildcards ]; then
echo -n "-- White(*): "
removeDomainsWildcard "$domains".whitelist.wildcards "$domains"
printDecCount
fi
# Remove dead Domains
getDomainsFromList "$deadlists" "$domains".deadlist 0 0
if [ -s "$domains".deadlist ]; then
echo -n "-- Dead: "
removeDomains "$domains".deadlist "$domains"
printDecCount
fi
# Add prioritized blocked Domains
if [ -s "$block" ]; then
cat <"$block" >>"$domains".prioblocklist
fi
getDomainsFromList "$prioblocklists" "$domains".prioblocklist 1 0
if [ -s "$domains".prioblocklist ]; then
echo -n "++ Block: "
addDomains "$domains".prioblocklist "$domains"
printIncCount
fi
# Unblock prioritized whitelisted Domains
if [ -s "$unblock" ]; then
cat <"$unblock" >>"$domains".unblocklist
fi
getDomainsFromList "$unblocklists" "$domains".unblocklist 1 1
if [ -s "$domains".unblocklist ]; then
echo -n "-- Unblock: "
removeDomains "$domains".unblocklist "$domains"
printDecCount
fi
# Unblock prioritized wildcard whitelisted Domains
if [ -s "$unblock" ]; then
cat <"$unblock" | grep -E '\*' >"$domains".unblocklist.wildcards.tmp
convertWildcardToRegex "$domains".unblocklist.wildcards.tmp >>"$domains".unblocklist.wildcards
rm -f "$domains".unblocklist.wildcards.tmp
fi
getRegexDomainsFromWilcardList "$unblocklists" "$domains".unblocklist.wildcards
if [ -s "$domains".unblocklist.wildcards ]; then
echo -n "-- Unblock(*): "
removeDomainsWildcard "$domains".unblocklist.wildcards "$domains"
printDecCount
fi
# Extend by missing WWW/FLD Domains
if inConfig "extendWWWFLD"; then
if inConfig "white"; then
echo -n "++ FLD from SLD: "
extendFLDfromSLD "$domains"
printIncCount
fi
echo -n "++ FLD: "
extendFLD "$domains"
printIncCount
echo -n "++ WWW: "
extendWWW "$domains"
printIncCount
fi
# Check if there are changes to the previous repo version
getMd5 "$domains" >"$outdir"/md5.new
if compareFiles "$outdir/md5.new" "$outdir/md5.old"; then
echo ""
echo "*****************************************************"
echo "* No changes to the previous repo version detected! *"
echo "*****************************************************"
echo ""
exit 0
fi
getMd5 "$domains" >"$outdir"/md5.old
# Cleanup
rm -f "$hosts" "$adblock"
rm -f "$hosts".* "$adblock".*
# Version
echo ""
echo "$cdo unique Domains - Version $version"
echo "MD5 Domains RAW: $(getMd5 "$domains")"
echo ""
# Convert to Hostlist
if inConfig "hosts"; then
echo "# Convert $name to Hostlist ... "
rm -f "$hosts"
if ! inConfig "noheader"; then
printHeader "$cdo" >"$hosts"
fi
sed -e 's/^/0.0.0.0 /' "$domains" >>"$hosts"
echo ""
fi
# Convert to Adblocklist
if inConfig "adblock"; then
echo "# Convert $name to AdBlocklist ..."
echo ""
if [ ! -f "$json" ]; then
cp $jsonbase "$json"
sed -i "s/tmp/$name/g" "$json"
fi
if ! inConfig "no_adblock_convert"; then
echo -n "Prepare domain list for compiling ... "
if inConfig "adblock_important"; then
cat <"$domains" | grep -Ev '^\s*$|^#|^!|^www\.' | sed -e 's/^/||/' | sed -e 's/$/^$important/' >"$adblock".raw
else
cat <"$domains" | grep -Ev '^\s*$|^#|^!|^www\.' | sed -e 's/^/||/' | sed -e 's/$/^/' >"$adblock".raw
fi
echo "done."
echo ""
fi
cd "$outdir" || exit
if inConfig "verbose"; then
hostlist-compiler -v -c "$json" -o "$adblock"
else
hostlist-compiler -c "$json" -o "$adblock"
fi
rm -f "$adblock".raw
cd "$sourcedir" || exit
sortList "$adblock"
if ! inConfig "noheader"; then
c=$(grep -Evc '^\s*$|^#|^!' "$adblock")
printHeader "$c" | sed 's/^\#/\!/' >"$adblock".tmp
fi
cat <"$adblock" | grep -Ev '^\s*$|^#|^!' >>"$adblock".tmp
mv "$adblock".tmp "$adblock"
if inConfig "adblocktowildcard"; then
cat <"$adblock" | grep -Ev '^\s*$|^#|^!' | sed 's/||/*./' | sed 's/[\^]//g' >"$adblock".wildcards
fi
echo ""
fi
if ! inConfig "noheader"; then
# Attach header
echo "# Attach header to $name Domainlist ..."
echo ""
printHeader "$cdo" >"$domains".tmp
cat <"$domains" >>"$domains".tmp
mv "$domains".tmp "$domains"
fi
# Push to local repository
if inConfig "localpush"; then
echo "# Push $name to local Repositories ..."
echo ""
dos2unix -q "$outdir"/*
if inConfig "white" || inConfig "dead"; then
cp "$domains" "$repodata"/"$name".list
elif inConfig "block"; then
cp "$domains" "$repodata"/"$name".list.block
elif inConfig "black"; then
touch "$domains".whitelist.removed "$domains".wildcard.removed
cp "$domains" "$repodata"/"$name".list
cp "$domains".whitelist.removed "$repodata"/"$name".list.wl.removed
cp "$domains".wildcard.removed "$repodata"/"$name".list.wc.removed
else
cp "$domains" "$repodomains"/"$name".txt
cp "$stats" "$repodomains"
if inConfig "hosts"; then
cp "$hosts" "$repohosts"/"$name".txt
cp "$stats" "$repohosts"
fi
if inConfig "adblock"; then
cp "$adblock" "$repoadguard"/"$name".adblock
cp "$stats" "$repoadguard"
fi
fi
fi
) | tee "$stats"
echo "End: $(date +'%Y.%m-%d-%H:%M:%S')"
echo ""
# Cleanup
rm -f "$header.ext"
sortList $builddir/sourcefiles.txt
echo "$name - Done!"
echo ""