# -*- coding: utf-8 -*-
"""
@author: Anon

Changelog:
2020/08/02: Added ?_=RANDOM to get not recoded file version / Added timeout and retries request function / Thread level
2020/08/03  Added Post level data scrapping (com & photos)
"""
import json, requests, urllib.request, time, socket, os

def file_as_bytes(file):
    with file:
        return file.read()

def GetFile(Thread, myPath):
    socket.setdefaulttimeout(15)
    t0 = time.time()
    Filepath = ("https://media.8kun.top/file_store/" + str(Thread["tim"]) + str(Thread['ext']+ "?_=RANDOM"))
    LocalFileName = str(Thread['filename']) + str(Thread['ext'])
    fullfilename = os.path.join(myPath , LocalFileName)
    for attempt in range(3):
        try:
             urllib.request.urlretrieve(Filepath, fullfilename)
        except Exception as x:
            print('It failed :(', x.__class__.__name__)
            print('Failed url: ' + Filepath)
        else:
            print('Downloaded: ' + LocalFileName)
            break
        finally:
            t1 = time.time()
            print('Took', t1 - t0, 'seconds')

def main(myPath):
    t0 = time.time()
    ThreadsR = requests.get('https://8kun.top/vichan/catalog.json');
    Pages = ThreadsR.json()
    FilesCount = 0
    ThreadCount = 0
    PostCount = 0

    with open(myPath + "Data.txt", 'a', encoding='utf-8') as outfile:
        print("-----------------------Vichan-----------------------", file=outfile)
        for Page in Pages:
            Threads = Page['threads']
            for Thread in Threads:
                print ("|     Tread     | Replies: "  + str(Thread['replies']), file=outfile)
                ThreadCount += 1
                ThreadJsonUrl = "https://8kun.top/vichan/res/" + str(Thread['no']) +".json"
                print(ThreadJsonUrl)
                PostsR = requests.get(ThreadJsonUrl);
                PostList = PostsR.json()
                for Post in PostList["posts"]:
                    print ("|     POST     |:" + str(Post['no']) + " | Time: " + str(Post['time']) + ' | com: '  + str(Post['com']) + '\n', file=outfile)
                    PostCount += 1
                    if 'tim' in Post:
                         GetFile(Post, myPath)
                         FilesCount += 1
    t1 = time.time()
    print("TotalFiles collected: " + str(FilesCount ), file=outfile)
    print("Total threads collected: " + str(FilesCount ), file=outfile)
    print("Total posts collected: " + str(FilesCount ), file=outfile)
    print("Total time: ", t1 - t0, 'seconds', file=outfile)


myPath = r"C:/Tmp/ChanArchiveOutput/"
main(myPath)