normie_scraper

#THREAD https://boards.4channel.org/g/thread/69235901
#SHA256 (pgmg6p.7z) = 04635f8f3034709f8b5d6dec64683cd853ad4c89654232dd64d3866b949038a3
#DB https://files.catbox.moe/pgmg6p.7z

#WORD STATS
#site popularity - https://pastebin.com/fDAAzUiS
#fag https://pastebin.com/kNaP3phv
#normie https://pastebin.com/8Fx1Wsa5
#incel https://pastebin.com/7feY2nWC


#!/bin/bash
for text in normie normalfag
    do
    for year in {2010..2018}
        do
        for month in {1..12}
            do
            month=$(printf '%02i' $month)
            # This can be optimized.
            sqlite3 \
                "$1" \
                "with month_posts as (
                    select
                        timestamp,
                        text
                    from
                        posts
                    where
                        strftime(
                            '%Y%m',
                            timestamp,
                            'unixepoch'
                        ) == '$year$month'
                )
                select
                    '$text',
                    '$year',
                    '$month',
                    (
                        select
                            count(text)
                        from
                            month_posts
                    )
                    as total,
                    (
                        select
                            count(text)
                        from
                            month_posts
                        where
                            text like '%$text%'
                    )
                    as count
                where
                    total > 0"
        done
    done
done

#!/bin/bash
# https://github.com/dinedal/textql
textql \
    -header \
    -sql "select
        normalfag.year,
        normalfag.month,
        normalfag.total,
        normalfag.count as normalfag_count,
        normie.count as normie_count
    from
        (
            select
                year,
                month,
                total,
                count
            from
                data
            where
                text = 'normalfag'
        )
        as normalfag
    join
        (
            select
                year,
                month,
                total,
                count
            from
                data
            where
                text = 'normie'
        )
        as normie
    on
        normalfag.year = normie.year
            and
        normalfag.month = normie.month
            and
        normie.count > normalfag.count" \
    "$1"


#!/usr/bin/gnuplot --persist
set title '"Normalfag" vs "normie"'
set ylabel 'Count'
set xlabel 'Date'
set grid

set datafile separator ","
set xdata time
set timefmt '%Y,%m'
set format x "%Y-%m"

plot \
    "< grep normalfag data.txt" using 2:5 title "normalfag" with lines, \
    "< grep normie data.txt" using 2:5 title "normie" with lines