#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import requests
import time
import os
import re

my_url = "https://file.wikileaks.org/file/?fbclid=IwAR1dMnhHMZmI-etF53fp3LF3xqDWzepBp4gxYno173UqBuwLaOuh-JRc1K4"


r = requests.get(my_url)
assert r.status_code == 200


wikileaks_index = re.findall('"([^"]*)"', r.text)[2:]


try:
    os.makedirs("wikileaks_dump")
except FileExistsError:
    pass


for fname in wikileaks_index:
    tmp_url = f"{my_url}/{fname}"
    r = requests.get(tmp_url)
    fpath = os.path.join("wikileaks_dump",fname).rstrip("/")
    with open(fpath, 'wb') as fd:
        for chunk in r.iter_content(chunk_size=128):
            fd.write(chunk)
    print(time.ctime(), "saved",fname)