4chan-countries-scrape.fish

#!/usr/bin/fish

mkdir pol-threads
pushd pol-threads
curl https://a.4cdn.org/pol/catalog.json > catalog.json
for value in (cat catalog.json | jq ".[].threads|.[]|.no")
	curl "https://a.4cdn.org/pol/thread/$value.json" > "$value.json"
end

for jfile in 3*.json; cat "$jfile" | jq ".posts|.[]|.country_name"; end | sort | uniq > countries.txt

cat countries.txt
cat countries.txt | wc -l
# 100

popd

mkdir int-threads
pushd int-threads

curl https://a.4cdn.org/int/catalog.json > catalog.json
for value in (cat catalog.json | jq ".[].threads|.[]|.no")
	curl "https://a.4cdn.org/int/thread/$value.json" > "$value.json"
end

for jfile in 3*.json; cat "$jfile" | jq ".posts|.[]|.country_name"; end | sort | uniq > countries.txt

cat countries.txt
cat countries.txt | wc -l

popd