youiv_search.py

# -*- coding: utf-8 -*-

import requests
import re
import sys
import codecs
import os
import time

class G:
    cache_dir_exists = False
    post_index_exists = False
    post_index = ""
    index_index_exists = False
    index_index = ""

G.cache_dir_exists = os.path.isdir("youiv cache")
if G.cache_dir_exists:
    G.post_index_exists = os.path.isfile("youiv cache/post/index.txt")
    G.index_index_exists = os.path.isfile("youiv cache/index/index.txt")

headers = {
    'User-agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:29.0)'
    ' Gecko/20100101 Firefox/29.0'
}

def request(url, headers):
    while True:
        r = requests.get(url, headers=headers)
        if r.status_code == 200:
            return r.text
        else:
            #print "HTTP ERROR -> ", str(r.status_code) + " (" + url + ")"
            #time.sleep(10)
            return "HTTP ERROR -> " + str(r.status_code)

def get_post_text(suburl, headers):
    if not G.cache_dir_exists:
        os.makedirs("youiv cache")
        os.makedirs("youiv cache/index")
        os.makedirs("youiv cache/post")
        G.cache_dir_exists = True

    subsuburl_list = suburl.split("-", 2)
    subsuburl = subsuburl_list[0] + "-" + subsuburl_list[1]

    url = "https://youiv.tv/" + suburl

    index_file_exists = False
    fname = suburl

    if G.post_index_exists:
        f = open("youiv cache/post/index.txt", "r")
        lines = f.read().split("\n")
        for l in lines:
            if l.find(subsuburl) >= 0:
                index_file_exists = True
                fname = l
                break
        f.close()

    if not os.path.isfile("youiv cache/post/" + fname):
        text = request(url, headers)

        f = open("youiv cache/post/" + fname, "wb")
        f.write(text.encode("utf-8"))
        f.close()

        if not G.post_index_exists:
            f = open("youiv cache/post/index.txt", "w")
            f.write(fname + "\n")
            f.close()
            G.post_index_exists = True
        else:
            f = open("youiv cache/post/index.txt", "a")
            f.write(fname + "\n")
            f.close()

        return text
    else:
        f = open("youiv cache/post/" + fname, encoding='utf-8')
        text = f.read()
        f.close()
        return text

def get_index_text(page_num, headers):
    if not G.cache_dir_exists:
        os.makedirs("youiv cache")
        os.makedirs("youiv cache/index")
        os.makedirs("youiv cache/post")
        G.cache_dir_exists = True

    url = "https://youiv.tv/u15-279-" + str(page_num) + ".html"
    last_time = time.time()

    lines = []
    index_file_exists = False
    if G.index_index_exists:
        f = open("youiv cache/index/index.txt", "r")
        lines = f.read().split("\n")

        line_num = 0
        while line_num < len(lines):
            line = lines[line_num].split()
            if len(line) >= 2 and line[1] == str(page_num):
                index_file_exists = True
                last_time = float(line[0])
                break
            line_num += 1

        f.close()

    if not os.path.isfile("youiv cache/index/" + str(page_num)) or time.time() - last_time >= 86400: # If not cached or page in cache is over a day old
        text = request(url, headers)

        f = open("youiv cache/index/" + str(page_num), "wb")
        f.write(text.encode("utf-8"))
        f.close()

        if not G.index_index_exists:
            f = open("youiv cache/index/index.txt", "w")
            f.write(str(time.time()) + " " + str(page_num) + "\n")
            f.close()
            G.index_index_exists = True
        else:
            write_lines = []

            f = open("youiv cache/index/index.txt", "r")

            line_num = 0
            while line_num < len(lines):
                line = lines[line_num].split()
                if not (len(line) >= 2 and line[1] == str(page_num)):
                    write_lines.append(" ".join(line))
                line_num += 1

            f.close()

            f = open("youiv cache/index/index.txt", "w")
            f.write("\n".join(write_lines))
            f.write(str(time.time()) + " " + str(page_num) + "\n")
            f.close()

        return text
    else:
        f = open("youiv cache/index/" + str(page_num), encoding='utf-8')
        text = f.read()
        f.close()
        return text

def search_post(suburl, prod_company, code_number):
    page_text = get_post_text(suburl, headers)
    #m = re.search("(" + prod_company + "|" + prod_company.upper() + "|" + prod_company[0].upper() + prod_company[1:] + ")\\s*(\\-)?\\s*" + code_number, page_text)
    m = re.search(prod_company + "\\s*(\\-)?\\s*" + code_number, page_text.lower())

    return True if m else False

print("1. Search by title")
print("2. Search by video code (much slower)")
print("3. Search by both (about as fast as 1 but not as accurate as 2)")
option = input("Option: ")

prod_company = ""
code_number = ""

if option == "1":
    query = input("Type in title or sub-phrase: ")
elif option == "2":
    query = input("Type in the video code (Ex. TSBS-81085): ")
    video_code = "".join(query.split()).lower().replace("-", "")
    prod_company = re.search("[a-z]+", video_code).group(0)
    code_number = re.search("[0-9]+", video_code).group(0)
else:
    query = input("Type in title or sub-phrase: ")
    query2 = input("Type in the video code (Ex. TSBS-81085): ")
    video_code = "".join(query2.split()).lower().replace("-", "")
    prod_company = re.search("[a-z]+", video_code).group(0)
    code_number = re.search("[0-9]+", video_code).group(0)

start_page = int(input("Starting page #: "))
max_page = int(input("Search up to page # (Type 0 to search all pages): "))

page_text = get_index_text(1, headers)

m = re.search("\\<a href\\=\\\"([a-z]|[A-Z]|[0-9]|(\\-))+\\.html\\\" class=\\\"last\\\"\\>\\.\\.\\. [0-9]+\\<\\/a\\>", page_text)
subtext = m.group(0)
last_page = int(re.search("\\.\\.\\.( |\\t|\\n|\\r)*[0-9]+", subtext).group(0)[3:].strip())
#print(last_page)

if max_page > 0:
    last_page = min(last_page, max_page)

results = []

for i in range(max(2, start_page + 1), last_page + 2):
#for i in range(2, 3):
    print("Searching page " + str(i - 1) + "...")

    find_start = 0
    found = page_text.find("<li style", find_start)

    while found >= 0:
        find_start = found + len("<li style")
        url_start = page_text.find("<a href=", find_start) + len("<a href=") + 1
        url_end = page_text.find("\"", url_start)
        suburl = page_text[url_start:url_end]
        url = "https://youiv.tv/" + suburl

        title_start = page_text.find("title=\"", url_end) + len("title=\"")
        title_end = page_text.find("\"", title_start)
        title = page_text[title_start:title_end]

        if option == "1":
            if "".join(title.split()).lower().find("".join(query.split()).lower()) >= 0:
                results.append((url, title))
        elif option == "2":
            if search_post(suburl, prod_company, code_number):
                print("\nFOUND: " + url + " - " + title + "\n")
                answer = input("Continue searching? y/n: ")
                if answer != "y":
                    sys.exit(0)
                results.append((url, title))
        else:
            if "".join(title.split()).lower().find("".join(query.split()).lower()) >= 0:
                if search_post(suburl, prod_company, code_number):
                    print("\nFOUND: " + url + " - " + title + "\n")
                    answer = input("Continue searching? y/n: ")
                    if answer != "y":
                        sys.exit(0)
                    results.append((url, title))

        found = page_text.find("<li style", find_start)

    if i <= last_page:
        page_text = get_index_text(i, headers)

print("\nRESULTS:\n")

if len(results) == 0:
    print("- No results :( - ")
else:
    for r in results:
        print(r[0] + " - " + r[1])