get anidb needed columns from html file

from bs4 import BeautifulSoup
from pathlib import Path
import sqlite3
import concurrent.futures

def strip_check(text):
    if isinstance(text, str):
        return text.strip()
    else:
        return ""

needed_data = []

html_files = Path().rglob('anime/*.html')


def fetch_records(file_path):
    # Read the HTML file
    with open(file_path, 'r', encoding='utf-8') as file:
        content = file.read()

    print("processing: " + file_path.name)

    # Parse the HTML content
    soup = BeautifulSoup(content, 'html.parser')
    anime = {
            "ID": "",
            "anidbID": "",
            "title": "",
            "type": "",
            "startdate": "",
            "enddate": "",
            "related": "",
            "creators": "",
            "description": "",
            "rating": "",
            "picture": "",
            "categories": "",
            "characters": "",
            "epnos": "",
            "airdates": "",
            "episodetitles": "",
            "createddate": "2024-08-30 14:39:08",
        }

    # Anime ID and anidbID:
    anime['ID'] = file_path.name.replace('.html', '')
    anime['anidbID'] = file_path.name.replace('.html', '')
    #print('Anime ID and anidbID are: ' + anime['anidbID'])

    # Anime title
    anime_title = soup.select_one('h1.anime')
    if(anime_title and hasattr(anime_title,'text')):
        anime['title'] = strip_check(anime_title.text[7:])
    else:
        anime['title'] = "Title for Anime No." + anime['anidbID'] = file_path.name.replace('.html', '')
    #print('Anime title is: ' + anime['title'])

    # Anime type
    anime_type = soup.select_one('tr.type td')
    if(anime_type and hasattr(anime_type, 'text')):
        anime['type'] = strip_check(anime_type.text.split(',')[0])
    else:
        anime['type'] = "Anime"
    #print('Anime type is: ' + anime['type'])

    # Anime startdate
    anime_startdata = soup.select_one('tr.year span[itemprop="startDate"]')
    if(anime_startdata and hasattr(anime_startdata, 'content')):
        anime['startdate'] = strip_check(anime_startdata['content'])
    else:
        anime_startdata = soup.select_one('tr.year span[itemprop="datePublished"]')
        if(anime_startdata and hasattr(anime_startdata, 'content')):
            anime['startdate'] = strip_check(anime_startdata['content'])
        else:
            anime['startdate'] = '1999-12-30'
    #print('Anime startdate is: ' + anime['startdate'])

    # Anime enddate
    anime_enddate = soup.select_one('tr.year span[itemprop="endDate"]')
    if(anime_enddate and hasattr(anime_enddate,'content')):
        anime['enddate'] = strip_check(anime_enddate['content'])
    else:
        anime_enddate = soup.select_one('tr.year span[itemprop="datePublished"]')
        if(anime_enddate and hasattr(anime_enddate,'content')):
            anime['enddate'] = strip_check(anime_enddate['content'])
        else:
            anime['enddate'] = anime['startdate']
    #print('Anime endDate is: ' + anime['enddate'])

    # Anime related
    related_anime_list = soup.select('div.relations.direct a.name-colored')
    for related_anime in related_anime_list:
        anime['related'] += strip_check(related_anime.text) + '|'
    if anime['related'].endswith('|'):
        anime['related'] = anime['related'][:-1]
    #print('Anime related is|are: ' + anime['related'])


    # Anime creators
    creators_list = soup.select('div.extra table.stafflist td.creator a')
    for creator in creators_list:
        anime['creators'] += strip_check(creator.text) + '|'
    if anime['creators'].endswith('|'):
        anime['creators'] = anime['creators'][:-1]
    #print('Anime creators is|are: ' + anime['creators'])

    # Anime description
    description = soup.select_one('div[itemprop="description"]')
    if(description and hasattr(description, 'text')):
        anime['description'] = strip_check(description.text)
    else:
        anime['description'] = "No description"
    #print('Anime description is: ' + anime['description'])

    # Anime rating
    rating = soup.select_one('tr.rating span[itemprop="ratingValue"]')
    if(rating and hasattr(rating, 'text')):
        anime['rating'] = strip_check(rating.text)
    else:
        anime['rating'] = "0.00"
    #print('Anime rating is: ' + anime['rating'])

    # Anime picture
    picture = soup.select_one('meta[property="og:image"]')
    if(hasattr(picture, 'content')):
        anime['picture'] = strip_check(picture['content'].split('/')[-1])
    else:
        anime['picture'] = "No picture"
    #print('Anime picture is: ' + anime['picture'])

    # Anime categories
    categories_list = soup.select('div.animetags span.tagname')
    for category in categories_list:
        anime['categories'] += strip_check(category.text) + '|'
    if anime['categories'].endswith('|'):
        anime['categories'] = anime['categories'][:-1]
    #print('Anime categories is|are: ' + anime['categories'])

    # Anime characters
    characters_list = soup.select('div[itemprop="character"] span[itemprop="name"]')
    for character in characters_list:
        anime['characters'] += strip_check(character.text) + '|'
    if anime['characters'].endswith('|'):
        anime['characters'] = anime['characters'][:-1]
    #print('Anime characters is|are: ' + anime['characters'])

    # Anime epnos
    epnos_list = soup.select('tr[itemprop="episode"] abbr[itemprop="episodeNumber"]')
    for epnos in epnos_list:
        anime['epnos'] += strip_check(epnos.text) + '|'
    if anime['epnos'].endswith('|'):
        anime['epnos'] = anime['epnos'][:-1]
    #print('Anime epnos is|are: ' + anime['epnos'])

    # Anime airdates
    airdates_list = soup.select('tr[itemprop="episode"] td[itemprop="datePublished"]')
    for airdate in airdates_list:
        anime['airdates'] += strip_check(airdate['content']) + '|'
    if anime['airdates'].endswith('|'):
        anime['airdates'] = anime['airdates'][:-1]
    #print('Anime airdates is|are: ' + anime['airdates'])

    # Anime episodetitles
    episodetitles_list = soup.select('tr[itemprop="episode"] label[itemprop="name"]')
    for episodetitle in episodetitles_list:
        anime['episodetitles'] += strip_check(episodetitle.text) + '|'
    if anime['episodetitles'].endswith('|'):
        anime['episodetitles'] = anime['episodetitles'][:-1]
    #print('Anime episodetitles is|are: ' + anime['episodetitles'])

    return anime

def escape_string(value):
    return sqlite3.Connection('').execute('SELECT quote(?)', (value,)).fetchone()[0]


#for file_path in html_files:
#    needed_data.append(fetch_records(file_path))

def process_files_concurrently(html_files, max_workers):
    needed_data = []
    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
        future_to_file = {executor.submit(fetch_records, file_path): file_path for file_path in html_files}
        for future in concurrent.futures.as_completed(future_to_file):
            file_path = future_to_file[future]
            try:
                data = future.result()
                needed_data.append(data)
            except Exception as exc:
                print(f'{file_path} generated an exception: {exc}')
    return needed_data

max_workers = 50  # Number of files to process concurrently
needed_data = process_files_concurrently(html_files, max_workers)

with open('anidb_dump.sql', 'w', encoding='utf-8') as file:
    for anime in needed_data:
        columns = ', '.join(anime.keys())
        values = ', '.join(escape_string(value) if isinstance(value, str) else str(value) for value in anime.values())
        sql = f'INSERT INTO anidb ({columns}) VALUES ({values});\n'
        file.write(sql)


print('SQL file generated')