from bs4 import BeautifulSoup as BS
from openpyxl import Workbook
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from itertools import *
from time import sleep
import requests
import asyncio
import aiohttp

def add_university_data(ws_universities, id_num, name, link, city):
    ws_universities.append([id_num, name, link, city])


def add_item_data(ws_items ,id_num, name, link):
    ws_items.append([id_num, name, link])


def add_directions_data(ws_directions, id_num, directions, num):
    ws_directions.append([id_num, directions, num])

universities_data = []
items_data = []
direction_data = []


async def get_page_data(session, page, headers):
    print(f'Страница №{page + 1} / 250')
    url = f'https://www.ucheba.ru/for-abiturients/vuz/rossiya?s={page}0'

    async with session.get(url=url, headers=headers) as response:

        response_text = await response.text()

        # Основная страница
        src = response_text
        soup = BS(src, "lxml")
        all_universiti_hrefs = soup.find_all('a', 'js_webstat')

        count_university = 1

        for link in all_universiti_hrefs:

            count_items = 0
            count_derection = 0

            link_text = link.text
            link_href = 'https://www.ucheba.ru' + link.get('href')

            response = session.get(url=link_href, headers=headers)
            src = response

            soup = BS(src, 'lxml')
            universities_citi = soup.find(
                'ul', 'params-list').find_all('li')[0].text.strip()

            universities_data.append(
                {
                count_university: [link_text, link_href, universities_citi]
                }
            )

            response = session.get(url=link_href, headers=headers)
            src = response.text

            soup = BS(src, 'lxml')

            receipt_group = soup.find('div', class_='ege-groups-list')

            try:
                for ul in receipt_group.find_all('div', class_='ege-groups-list__item'):
                    items = []

                    for li in ul.find('ul', class_='ege-groups-list__subjects subjects-list').find_all('li'):
                        items.append(li.text.replace(
                            '\n', '').replace(' ', '-'))

                    items_name = '_'.join(items)

                    items_href = 'https://www.ucheba.ru' + ul.find('div', class_='ege-groups-list__info').find(
                        'a', class_='ege-groups-list__programs-link').get('href')

                    items_data.append(
                        {
                        f'{count_university}_{count_items}': [items_name, items_href]
                        }
                    )

                    src = session.get(url=items_href, headers=headers)
                    soup = BS(src.text, 'lxml')

                    if not soup.find('div', class_='paginator mt-25'):

                        all_derection = [i.text for i in soup.find_all(
                            'a', class_='js_webstat')]
                        all_passing_score = []
                        for i in soup.find_all('div', class_='search-results-options'):
                            if isinstance(i.text.split()[2]) or i.text.split()[2] == '—':
                                all_passing_score.append(i.text.split()[2])
                            else:
                                all_passing_score.append(i.text.split()[3])

                        for d, p in zip_longest(all_derection, all_passing_score, fillvalue=' '):
                            direction_data.append(
                                {
                                f'{count_university}_{count_items}_{count_derection}': [d, p]
                                }
                            )
                            count_derection += 1
                    else:

                        page = soup.find(
                            'div', class_='paginator mt-25').find_all('a')

                        for i in page:

                            items_href = items_href[:items_href[::-
                                                                1].page('//'[0])-1] + i.get('href')
                            src = session.get(url=items_href, headers=headers)
                            soup = BS(src.text, 'lxml')

                            all_derection = [j.text for j in soup.find_all(
                                'a', class_='js_webstat')]
                            all_passing_score = []

                            for j in soup.find_all('div', class_='search-results-options'):

                                if isinstance(j.text.split()[2]) or j.text.split()[2] == '—':
                                    all_passing_score.append(j.text.split()[2])
                                else:
                                    all_passing_score.append(j.text.split()[3])

                            for d, p in zip_longest(all_derection, all_passing_score, fillvalue=' '):
                                direction_data.append(
                                    {
                                    f'{count_university}_{count_items}_{count_derection}': [d, p]
                                    }
                                )
                        count_items += 1

            except:
                print('Ошибка')
                continue

            count_university += 1
    print(f'Обработана страница {page}')

async def gather_data():
    auth_data = {
        "password": "Vi31128282",
        "rememberMe": "true",
        "username": "brykovvita173@gmail.com"
    }
    auth_url = 'https://api.ucheba.ru/v1/auth'

    headers = {
        "accept": 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
        'Content-Type': 'application/json',
        'cookie': 'sessionSource=%7B%22ref%22%3A%22https%3A%2F%2Fwww.google.com%2F%22%2C%22req%22%3A%22%2Ffor-abiturients%2Fvuz%2Frossiya%22%2C%22utm%22%3Anull%7D; locationIds=%5B3%5D; advUserId=805f961b-06c9-43bd-9a8d-2eec536fc4cf; _ym_uid=1678554080752447596; _ym_d=1678554080; _ga=GA1.2.2134169335.1678554081; _gid=GA1.2.1188182247.1678554081; tmr_lvid=e168b7d9e791badcff6caa177b5666a2; tmr_lvidTS=1678556330118; advUserId=805f961b-06c9-43bd-9a8d-2eec536fc4cf; _ym_isad=2; noticePermissionVkBlock=block; noticePermissionVkCounter=1; uchru_token=eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJpYXQiOjE2NzkwNzg1ODksImV4cCI6MTcxMDcwMDk4OSwicm9sZXMiOlsiUk9MRV9VU0VSIl0sImlkIjo4MDQyMjN9.oCAsDcDZ57g4G2ZMyuwD_tpFgHwCWXwAyKQK5NxH4oCL8xbLqX9AVZSKCfAJlLwd0UB8gp1XrFwxL6Dw45zHukRK6nYAL-sgJGGUrkTrdLuszPYg4cy9Yil3KyHN-HzfNvocxxmijOqwqxqvpjNrSw_Kig1hMnrQcWp2Zh37dP-IGeLcGvpSau9vzYrV0W9piKAYk7P4mKishorB4DDEPzg4S1P0EfAGxd2iFDJExqbJbp8j7M13EdGpJ7GkhbWKEJvqWbEYYqBjdhlTueLqH0_b0kN9BIbG-jHw7iVYmn2-89AJQP6vm1TYH0MCqPe8YvOELqRfx0L3mi3y7XPaGw; userTags=%7B%22ege_request_messenger%22%3A1678554078%2C%22platform_web_desktop%22%3A1678556149%2C%22ege_interested%22%3A1648917327%2C%22service_main_page%22%3A1678977992%2C%22service_catalog_vuz%22%3A1678963910%7D; _ym_visorc=b; tmr_detect=0%7C1679078599245; bannerSession=start; bannerSessionCount=%5B1038%5D; notices=%5B%22VkPermissionNoticeId%22%2C%22bannerNotice%22%5D; snuch=6fqg228gic0tg1k7g58rn63vpb; _gat=1; bannerSessionCount=%5B1038%2C1379%5D'
    }

    async with aiohttp.ClientSession() as session:
        # authorize first
        await session.post(auth_url, json=auth_data, headers=headers)

        # get the initial page
        url = 'https://www.ucheba.ru/for-abiturients/vuz/rossiya'
        response = await session.get(url=url, headers=headers)

        soup = BS(await response.text(), 'lxml')
        pages_count = soup.find('div', 'paginator mt-25').find('ul').find_all('li')[-1].text

        tasks = []

        for page in range(0, int(pages_count) + 1):
            task = asyncio.create_task(get_page_data(session, page, headers))
            tasks.append(task)

        await asyncio.gather(*tasks)

def main():
    asyncio.run(gather_data())

    wb = Workbook()
    ws_universities = wb.active

    ws_universities.title = 'Universities'
    ws_universities.append(
        ['ID', 'Название вуза', 'Ссылка на страницу вуза', 'Город вуза'])

    ws_items = wb.create_sheet('Items')
    ws_items.append(['ID', 'Предметы для поступления',
                    'Ссылка на все направления по предметам'])

    ws_directions = wb.create_sheet('Directions')
    ws_directions.append(['ID', 'Направления', 'Минимальный балл'])

    wb.save(
            f'/home/kukuruzka-vitya/CODE/za_python/parsing/pars_university_first/university.xlsx')

    for data_un in universities_data:
        for university_num, items in data_un.items():
            add_university_data(ws_universities, university_num, items[0], items[1], items[2])

    for data_un in items_data:
        for university_num, items in data_un.items():
            add_university_data(ws_items ,university_num.split('_')[0], items[0], items[1])

    for data_un in direction_data:
        for university_num, items in data_un.items():
            add_university_data(ws_directions ,university_num.split('_')[0], items[0], items[1])

    wb.save(
        f'/home/kukuruzka-vitya/CODE/za_python/parsing/pars_university_first/university.xlsx')

if __name__ == '__main__':
    main()