/jffs/scripts/hosts-update-script.sh

#!/bin/sh
# This is Hosts Update Script (HUS)
# Created for use in RT-N66U with [Fork] Asuswrt-Merlin 374.43 LTS releases
# Default path: /jffs/scripts/hosts-update-script.sh
# More info: //4pda.ru/forum/index.php?showtopic=491608&view=findpost&p=91762386


PATH=/opt/sbin:/opt/bin:/opt/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin


# 0.0.0.0 is defined as a non-routable meta-address used to designate an invalid, unknown, or non applicable target. Using 0.0.0.0 is empirically faster, possibly because there's no wait for a timeout resolution
ENDPOINT_IP4="0.0.0.0"
BLIST="/tmp/mnt/microsd/dnsmasq/blocking_black.list"
WLIST="/tmp/mnt/microsd/dnsmasq/blocking_white.list"
TMPDIR="/tmp/mnt/microsd/dnsmasq/blocking_initial.list"
SRTDIR="/tmp/mnt/microsd/dnsmasq/blocking_sorted.list"
TARGET="/tmp/mnt/microsd/dnsmasq/hosts.blocked"


logger "HUS: start"
while [ $(date "+%Y") -lt 2019 ]; do
    sleep 5
done
logger "HUS: time seems synchronized now, starting hosts update"
rm -f "$TARGET" "$TMPDIR" "$SRTDIR"
touch "$TARGET"


# Download and process the files needed to make the lists (enable/add more, if you want)
######## ENABLED START ########

# focus on ad related domains
LISTNAME="Peter Lowe"
logger "HUS: downloading hosts file $LISTNAME"
wget -qO- "https://pgl.yoyo.org/as/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext" | awk -v r="$ENDPOINT_IP4" '{sub(/^127.0.0.1/, r)} $0 ~ "^"r' > "$TMPDIR"
logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"

# focus on malware
LISTNAME="MalwareDomainList.com"
logger "HUS: downloading hosts file $LISTNAME"
wget -qO- "https://www.malwaredomainlist.com/hostslist/hosts.txt" | awk -v r="$ENDPOINT_IP4" '{sub(/^127.0.0.1/, r)} $0 ~ "^"r' >> "$TMPDIR"
logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"

# focus on mobile ads
LISTNAME="AdAway default blocklist"
logger "HUS: downloading hosts file $LISTNAME"
wget -qO- "https://adaway.org/hosts.txt" | awk -v r="$ENDPOINT_IP4" '{sub(/^127.0.0.1/, r)} $0 ~ "^"r' >> "$TMPDIR"
logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"

# broad blocklist
LISTNAME="by Dan Pollock"
logger "HUS: downloading hosts file $LISTNAME"
wget -qO- "https://someonewhocares.org/hosts/hosts" | awk -v r="$ENDPOINT_IP4" '{sub(/^127.0.0.1/, r)} $0 ~ "^"r' >> "$TMPDIR"
logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"

# focus on malicious bitcoin mining sites
LISTNAME="NoCoin Filter List"
logger "HUS: downloading hosts file $LISTNAME"
wget -qO- "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"

# focus on malvertising by disconnect.me
LISTNAME="Malvertising list by Disconnect"
logger "HUS: downloading hosts file $LISTNAME"
wget -qO- "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt" | grep -v -e ^# -e ^$ | awk -v r="$ENDPOINT_IP4 " '{sub(//, r)} $0 ~ "^"r' >> "$TMPDIR"
logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"

# focus on Windows installers ads sources
LISTNAME="Windows installers ads sources"
logger "HUS: downloading hosts file $LISTNAME"
wget -qO- "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/UncheckyAds/hosts" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"

# focus on sites which have used DMCA takedowns to force removal from other blacklists
LISTNAME="BarbBlock"
logger "HUS: downloading hosts file $LISTNAME"
wget -qO- "https://paulgb.github.io/BarbBlock/blacklists/domain-list.txt" | grep -v -e ^# -e ^$ | awk -v r="$ENDPOINT_IP4 " '{sub(//, r)} $0 ~ "^"r' >> "$TMPDIR"
logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"

# broad blocklist
LISTNAME="MVPS"
logger "HUS: downloading hosts file $LISTNAME"
wget -qO- "https://winhelp2002.mvps.org/hosts.txt" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"

# focus on ads and tracking domains
LISTNAME="Lightswitch05's ads-and-tracking-extended"
logger "HUS: downloading hosts file $LISTNAME"
wget -qO- "https://www.github.developerdan.com/hosts/lists/ads-and-tracking-extended.txt" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"

######## ENABLED FINISH ########


######## DISABLED START ########
# focus on intrusive online consultants and callback widgets
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://raw.githubusercontent.com/pafnuty/onlineConsultantBlocker/master/hosts" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on zeus botnet by abuse.ch
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://zeustracker.abuse.ch/blocklist.php?download=hostfile" | awk -v r="$ENDPOINT_IP4" '{sub(/^127.0.0.1/, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on ransomware by abuse.ch
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt" | grep -v -e ^# -e ^$ | awk -v r="$ENDPOINT_IP4 " '{sub(//, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on malware
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://mirror1.malwaredomains.com/files/justdomains" | grep -v -e ^# -e ^$ | awk -v r="$ENDPOINT_IP4 " '{sub(//, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# generic blocklist, fewer false positives down to the low sensitivity list with more false positives
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://dshield.org/feeds/suspiciousdomains_High.txt" | grep -v -e ^# -e ^$ | awk -v r="$ENDPOINT_IP4 " '{sub(//, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# broad blocklist
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://hosts-file.net/ad_servers.txt" | awk -v r="$ENDPOINT_IP4" '{sub(/^127.0.0.1/, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# generic blocklist
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://dshield.org/feeds/suspiciousdomains_Medium.txt" | grep -v -e ^# -e ^$ | awk -v r="$ENDPOINT_IP4 " '{sub(//, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# generic blocklist
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://dshield.org/feeds/suspiciousdomains_Low.txt" | grep -v -e ^# -e ^$ | awk -v r="$ENDPOINT_IP4 " '{sub(//, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on windows telemetry domains
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on malware by disconnect.me
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt" | grep -v -e ^# -e ^$ | awk -v r="$ENDPOINT_IP4 " '{sub(//, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on malicious bitcoin mining sites
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://raw.githubusercontent.com/greatis/Anti-WebMiner/master/hosts" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on dead websites (against typos in URLs)
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://raw.githubusercontent.com/FadeMind/hosts.extras/master/add.Dead/hosts" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on malicious bitcoin mining sites
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://gitlab.com/ZeroDot1/CoinBlockerLists/raw/master/hosts" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# broad blocklist
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://raw.githubusercontent.com/mitchellkrogza/Badd-Boyz-Hosts/master/hosts" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on 2o7 network tracking
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://raw.githubusercontent.com/azet12/KADhosts/master/KADhosts.txt" | awk -v r="$ENDPOINT_IP4" '{sub(/^0.0.0.0/, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on mobile trackers
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://vqs.gryzor.com/overall_trackers.txt" | awk -v r="$ENDPOINT_IP4" '{sub(/^127.0.0.1/, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
#
# focus on trackers
#LISTNAME=""
#logger "HUS: downloading hosts file $LISTNAME"
#wget -qO- "https://quidsup.net/notrack/blocklist.php?download | grep -v -e ^# -e ^$ | awk -v r="$ENDPOINT_IP4 " '{sub(//, r)} $0 ~ "^"r' >> "$TMPDIR"
#logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
######## DISABLED FINISH ########


# Add blacklist, if non-empty
if [ -s "$BLIST" ]
then
    logger "HUS: adding blacklist entries"
    awk -v r="$ENDPOINT_IP4" '/^[^#]/ { print r,$1 }' "$BLIST" >> "$TMPDIR"
    logger "HUS: $(wc -l $TMPDIR | awk '{print $1;}')"
else
    logger "HUS: no blacklist found"
fi

# Sort list
logger "HUS: sorting list"
awk '{sub(/\r$/,"");print $1,$2}' "$TMPDIR" | sort -u > "$SRTDIR"
logger "HUS: $(wc -l $SRTDIR | awk '{print $1;}')"


# Filter (if applicable)
if [ -s "$WLIST" ]
then
    # Filter the blacklist, suppressing whitelist matches
    # This is relatively slow
    logger "HUS: filtering blacklist, suppressing whitelist matches"
    egrep -v "^[[:space:]]*$" "$WLIST" | awk '/^[^#]/ {sub(/\r$/,"");print $1}' | grep -vf - "$SRTDIR" > "$TARGET"
else
    logger "HUS: no whitelist found"
    cat "$SRTDIR" > "$TARGET"
fi


logger "HUS: hosts file has been successfully updated."
logger "HUS: $(wc -l $TARGET)"


# Delete files used to build list to free up the limited space
rm -f "$TMPDIR" "$SRTDIR"

# Restart dnsmasq
#killall -SIGHUP dnsmasq
service restart_dnsmasq

logger "HUS: finish"