# gatgher_link.py
import requests as r
from bs4 import BeautifulSoup

def preprocess_article(article : str):
    lines = article.splitlines()
    return [lines[1], lines[5]]

def get_html(link):
    """
    return links to websites without img, potentially mp4
    """
    html=r.get(link).text
    soup = BeautifulSoup(html, 'html.parser')
    photos = soup.find_all('div', class_ = 'card-list__items')
    soup = BeautifulSoup(str(photos), 'html.parser')
    articles = soup.find_all('article')
    articles = [preprocess_article(str(article)) for article in articles]
    articles = [
            f"https://coomer.su{article[0][9:47]}"
            for article in articles
            if article[1].find("img") == -1
            ]

    return articles

def main():
    link = "https://{hehe}.com?o="

    links = [
            link + str(50*i)
            for i in range(0, 32)
            ]
    sublinks = []
    for link in links:
        sublinks = sublinks + get_html(link)
    with open("links.txt", "w+", encoding="utf-8") as f:
        f.writelines(sublinks)

if __name__ == "__main__":
    main()

#preprocess links
import requests as r
from bs4 import BeautifulSoup
import threading


def preprocess_article(article: str):
    lines = article.splitlines()
    return [lines[1], lines[5]]


sublinks = []
faulty = []


def get_download_link(link):
    """
    return links to websites without img, potentially mp4
    """
    html = r.get(link).text
    soup = BeautifulSoup(html, "html.parser")
    photos = soup.find("a", class_="post__attachment-link")
    if photos == -1:
        faulty.append(link)
        return
    photos = str(photos)
    begin = photos.find("href")
    photos = photos[begin + 6 :]
    end = photos.find('"')
    photos = photos[:end]
    link = photos
    sublinks.append(link)
    print(len(sublinks))


def download_links(batch):
    for link in batch:
        try:
            get_download_link(link)
        except Exception:
            print(link)


def write_lines(filename, lines):
    with open(filename, "w+") as f:
        for line in lines:
            print(line)
            f.write(line)
            f.write("\n")

def main():
    with open("links.txt", "r", encoding="utf-8") as f:
        links = [link.strip() for link in f.readlines()]

    batch_size = 50
    threads = []
    for i in range(0, int(len(links) / batch_size) + 1):
        begin = i * batch_size
        end = (i + 1) * batch_size - 1
        if end >= len(links):
            end = len(links) - 1
            batch = links[begin:end]
            threads.append(threading.Thread(target=download_links, args=(batch,)))
            break
        batch = links[begin:end]
        threads.append(threading.Thread(target=download_links, args=(batch,)))
    for t in threads:
        t.start()
    for t in threads:
        t.join()

    write_lines("video_links.txt", sublinks)
    write_lines("fault_video_links.txt", faulty)


if __name__ == "__main__":
    main()

#download links
import threading
import os
import uuid
import requests

with open("video_links.txt", "r") as f:
    links = [
            line.strip()
            for line in f.readlines()
            ]

t = []
def download_video(url, directory="videos"):
    # Ensure the directory exists
    os.makedirs(directory, exist_ok=True)

    # Generate a unique file name
    unique_name = f"{uuid.uuid4()}.mp4"
    file_path = os.path.join(directory, unique_name)

    try:
        # Download the video
        response = requests.get(url, stream=True)
        response.raise_for_status()  # Raise an error for HTTP issues

        # Write to the file
        with open(file_path, "wb") as f:
            for chunk in response.iter_content(chunk_size=1024):
                if chunk:  # Filter out keep-alive new chunks
                    f.write(chunk)

        print(f"Video downloaded successfully: {file_path}")
        return file_path
    except requests.RequestException as e:
        print(f"Failed to download the video: {e}")
        return None

def download_links(links):
    for link in links:
        download_video(link)

threads = []
batch_size = 10
for i in range(0, int(len(links) / batch_size) + 1):
    begin = i * batch_size
    end = (i + 1) * batch_size - 1
    if end >= len(links):
        end = len(links) - 1
        batch = links[begin:end]
        threads.append(threading.Thread(target=download_links, args=(batch,)))
        break
    batch = links[begin:end]
    threads.append(threading.Thread(target=download_links, args=(batch,)))
for t in threads:
    t.start()
for t in threads:
    t.join()