#!/bin/bash

while IFS=',' read -r POS HOSTNAME; do

	if cat checked.csv | grep -q ^$HOSTNAME; then
		echo "Skipping $HOSTNAME, already checked."
		continue
	fi

	ISWPORG=$(curl -s -L -m 5 $HOSTNAME/readme.html 2>&1 | grep "wordpress.org/support")
	ISWPCOM=$(curl -s -L -m 5 --head $HOSTNAME 2>&1 | grep "visit automattic.com/jobs")
	ISWPORGCONTENT=$(curl -s -L -m 5 $HOSTNAME 2>&1 > html/$HOSTNAME.txt | grep "/wp-content/")

	if [[ $ISWPORG || $ISWPCOM || $ISWPORGCONTENT ]]; then
		echo "$POS - $HOSTNAME is WP"
		echo "$POS,$HOSTNAME" >> topwp.csv
	else
		echo "$POS - $HOSTNAME is not WP"
	fi

	echo $HOSTNAME >> checked.csv

done < top-1m.csv