routes_search

# coding: utf-8
"""
This module implements searching for airline-routes on several web-sites.

To get routes use find_routes_on_multi_search().
To print results in console use print_routes().
"""
from datetime import datetime
from abc import ABCMeta, abstractmethod
import io
from itertools import product
from lxml import html
from lxml.etree import XMLSyntaxError
from requests.exceptions import RequestException
import requests


class Route(object):
    """Forms route.

    :param departure_date: date departure :class:'datetime' object.
    :param route_prices: tuple, first element - dictionary(faretype=price), faretype, price are strings, second element - currency
     string
    :param route_duration: route duration string
    :param origin_city: city origin string
    :param destination_city: city destination string
    :param route_flights: list of :class:'Flight' objects
    """
    def __init__(self, departure_date, route_prices, route_duration, origin_city,
                 destination_city, route_flights):
        self.route_flights = route_flights
        self.route_prices = route_prices
        self.route_duration = route_duration
        self.origin_city = origin_city
        self.destination_city = destination_city
        self.departure_date = departure_date
        self.lowest_price = self.find_lowest_price()

    def find_lowest_price(self):
        """Finds the lowest price of this route.

        :return: float
        """
        all_route_prices = self.route_prices[0].values()
        lowest_price = min(map(float, all_route_prices))
        return lowest_price

    def __repr__(self):
        repr_string = u'_' * 40
        repr_string += u'\n{0}\t{1}\t{2}\t{3}\t{4} {5}\n'\
            .format(self.origin_city, self.departure_date.date(), self.route_duration,
                    self.destination_city, self.lowest_price, self.route_prices[1])
        repr_string += u'Prices[{}]:\n'.format(self.route_prices[1])
        for fare_type in self.route_prices[0]:
            repr_string += u'{0} - {1}\n'.format(fare_type, self.route_prices[0][fare_type])
        for flight in self.route_flights:
            repr_string += str(flight)
        return repr_string.encode('utf-8')


class Flight(object):
    """Forms flight.

    :param flight_code: flight code string
    :param departure_time: time departure :class:'datetime' object or string
    :param arrival_time: time arrival :class:'datetime' object or string
    :param origin_code_iata: origin IATA code string
    :param destination_code_iata: destination IATA code string
    """
    def __init__(self, flight_code, departure_time, arrival_time, origin_code_iata,
                 destination_code_iata):
        self.departure_time = departure_time
        self.arrival_time = arrival_time
        self.flight_code = flight_code
        self.origin_code_iata = origin_code_iata
        self.destination_code_iata = destination_code_iata

    def __repr__(self):
        repr_string = u'\n\t{0}\t{1}\t{2} ---> {3}\t{4}'\
            .format(self.flight_code, self.departure_time, self.origin_code_iata,
                    self.destination_code_iata, self.arrival_time)
        return repr_string.encode('utf-8')


class SearchForRoutes(object):
    """Base class for searching on sites.

    :param origin_code_iata: origin IATA code string
    :param destination_code_iata: destination code IATA string
    :param route_dates: list of departure and return dates :class:'datetime' objects
    """
    __metaclass__ = ABCMeta

    def __init__(self, origin_code_iata, destination_code_iata, route_dates):
        self.search_request_url = ''
        self.validation_url = ''
        self.route_dates = route_dates
        self.origin_code_iata = origin_code_iata
        self.destination_code_iata = destination_code_iata
        self.routes_found = []
        self.output_file_encoding = 'utf-8'

    def _decode_json(self, response_json):    # Decodes JSON
        try:
            decoded_json = response_json.json()
        except ValueError:
            raise ValueError('({})JSON decoding failed, routes not found'
                             .format(self.__class__.__name__))
        return decoded_json

    def _decode_html_string(self, html_string):    # Parses string and creates DOM
        try:
            page_html = html.fromstring(html_string)
        except XMLSyntaxError:
            raise ValueError('({})HTML string failed, routes not found'
                             .format(self.__class__.__name__))
        return page_html

    @abstractmethod
    def make_request_params(self, session):
        """Makes parameters for searching request.

        :param session: for current search-request on particular web-site :class:'Session'
        :return tuple: HTTP-request method string and dictionary of special parameters
        (headers, body, JSON etc.)
        """
        return None, None

    @abstractmethod
    def validate_codes_iata(self, session):
        """Validates codes IATA for search-request.

        :param session: for current search-request on particular web-site :class:'Session'
        :return list of two valid codes IATA (or other parameters instead codes,
        which needs for search)
        if no valid dates then raises ValueError
        """
        return None, None

    def send_request_and_check(self, session, request_method, url, request_params):
        """Sends request and checks server response.

        :param session: for current search-request on particular web-site :class:'Session'
        :param request_method: HTTP-request method string
        :param url: URL for request
        :param request_params: of special parameters (headers, body, JSON etc.) for request
        :return :class:'Response' object
        if request failed then raises RequestException
        """
        try:
            response = session.request(request_method, url, **request_params)
            response.raise_for_status()
        except RequestException, error:
            print '({}) request failed'.format(self.__class__.__name__)
            raise error
        return response

    def search_request(self):
        """Make search-request and finds routes."""
        session = requests.Session()
        self.origin_code_iata, self.destination_code_iata = self.validate_codes_iata(session)
        request_method, request_params = self.make_request_params(session)
        search_response = self.send_request_and_check(session, request_method,
                                                      self.search_request_url, request_params)
        self.routes_found = self.extract_data_from_response(search_response)

    @abstractmethod
    def extract_data_from_response(self, response):
        """Extracts useful information from server response.

        :param response: :class:'Response' object
        :return list of :class:'Route" objects
        """
        pass

    def output_extracted_routes(self):
        """Prints routes' information on console and writes in Result.txt file."""
        with io.open('Result.txt', 'a', encoding=self.output_file_encoding) as output_file:
            for route in self.routes_found:
                print route
                output_file.write(route)
        print "Export of results is completed"

    def search(self):
        """Search on particular web-site and output routes' information on console and file."""
        self.search_request()
        if self.routes_found:
            self.output_extracted_routes()


class AeroflotSearch(SearchForRoutes):
    """Route-search on Aeroflot web-site."""
    def __init__(self, origin_code_iata, destination_code_iata, route_dates):
        SearchForRoutes.__init__(self, origin_code_iata, destination_code_iata, route_dates)
        self.search_request_url = 'https://www.aeroflot.ru/sb/booking/api/app/search/v2'
        self.validation_url = 'https://www.aeroflot.ru/sb/booking/api/app/cities/v1'

    @staticmethod
    def _find_in_json(code, cities):    # Finds code IATA in JSON for validation
        for city in cities['data']['cities']:
            if code == city['code']:
                return code
            else:
                for airport in city['airports']:
                    if code == airport['code']:
                        return code
        return None

    @staticmethod
    def _make_validation_request_params():    # Makes validation request parameters
        validation_params = {'json': {'lang': 'ru'}}
        return validation_params

    def validate_codes_iata(self, session):
        """Validates codes IATA for search-request for Aeroflot web-site."""
        valid_codes_iata = []
        validation_params = self._make_validation_request_params()
        cities_json = self.send_request_and_check(session, 'POST', self.validation_url,
                                                  validation_params)
        cities = self._decode_json(cities_json)
        for code_iata in self.origin_code_iata, self.destination_code_iata:
            code_found = self._find_in_json(code_iata, cities)
            if not code_found:
                raise ValueError('(AeroflotSearch) invalid code IATA: {}'.format(code_iata))
            valid_codes_iata.append(code_found)
        print '(AeroflotSearch) valid codes IATA'
        return valid_codes_iata

    def make_request_params(self, session):
        """Makes HTTP-request method and parameters for Aeroflot."""
        origin_code_iata = self.origin_code_iata
        destination_code_iata = self.destination_code_iata
        json_param_routes = []
        for date in self.route_dates:
            route = {
                "origin": origin_code_iata,
                "destination": destination_code_iata,
                "departure": date.strftime('%Y-%m-%d')
            }
            json_param_routes.append(route)
            origin_code_iata, destination_code_iata = destination_code_iata, origin_code_iata
        json_params = {
            "routes": json_param_routes,
            "cabin": "econom",
            "country": "ru",
            "adults": 1,
            "combined": False,
            "lang": "ru"
        }
        search_request_method = 'POST'
        search_request_params = {'json': json_params}
        return search_request_method, search_request_params

    @staticmethod
    def _extract_route_prices(route):    # Extracts dictionary of prices from route info
        route_prices = dict()
        currency = route['prices'][0]['currency']
        for price in route['prices']:
            route_price = price['total_amount']
            route_faretype = price['fare_group_name']
            route_prices[route_faretype] = route_price
        return route_prices, currency

    @staticmethod
    def _extract_flights_from_route(route):    # Extracts flights from route info
        route_flights = []
        for leg in route['legs']:
            for flight in leg['segments']:
                number = flight['flight_number']
                air_code = flight['airline_code']
                raw_departure_time = flight['departure']
                departure_time = datetime.strptime(raw_departure_time, '%Y-%m-%d %H:%M')
                arrival_time = flight['arrival'][-5:]
                origin_code_iata = flight['origin']['airport_code']
                destination_code_iata = flight['destination']['airport_code']
                one_flight = Flight(air_code + number, departure_time, arrival_time,
                                    origin_code_iata, destination_code_iata)
                route_flights.append(one_flight)
        return route_flights

    def _extract_routes(self, direction):    # Extracts routes from direction
        routes_extracted = []
        city_origin = direction[0]['legs'][0]['segments'][0]['origin']['city_name']
        city_destination = direction[0]['legs'][0]['segments'][-1]['destination']['city_name']
        raw_departure_date = direction[0]['legs'][0]['segments'][0]['departure'][:10]
        departure_date = datetime.strptime(raw_departure_date, '%Y-%m-%d')
        for route in direction:
            route_prices = self._extract_route_prices(route)
            route_flights = self._extract_flights_from_route(route)
            route_duration = route['time_name']
            one_route = Route(departure_date, route_prices, route_duration, city_origin,
                              city_destination, route_flights)
            routes_extracted.append(one_route)
        return routes_extracted

    def extract_data_from_response(self, response):
        """Extracts routes from Aeroflot JSON-response."""
        routes_found = []
        json_routes = self._decode_json(response)
        if not json_routes['data']['itineraries']:
            raise ValueError('(AeroflotSearch) routes not found')
        for direction in json_routes['data']['itineraries']:
            routes_found += self._extract_routes(direction)
        return routes_found


class NordwindSearch(SearchForRoutes):
    """Route-search on Nordwind web-site."""
    def __init__(self, origin_code_iata, destination_code_iata, route_dates):
        SearchForRoutes.__init__(self, origin_code_iata, destination_code_iata, route_dates)
        self.search_request_url = 'https://airbook.nordwindairlines.ru/online/json/' \
                                  'search-variants-mono-brand-cartesian'
        self.validation_url = 'https://airbook.nordwindairlines.ru/online/json/dependence-cities'

    @staticmethod
    def _find_in_json(code, cities):    # Finds code IATA in JSON for validation
        for city in cities['origin']:
            if code == city['codeEn']:
                return code
        return None

    def validate_codes_iata(self, session):
        """Validates codes IATA for search-request for Nordwind web-site."""
        valid_codes_iata = []
        validation_params = {}
        cities_json = self.send_request_and_check(session, 'GET', self.validation_url,
                                                  validation_params)
        cities = self._decode_json(cities_json)
        for code_iata in self.origin_code_iata, self.destination_code_iata:
            code_found = self._find_in_json(code_iata, cities)
            if not code_found:
                raise ValueError('(NordWindSearch) invalid code IATA: {}'.format(code_iata))
            valid_codes_iata.append(code_found)
        print '(NordWindSearch) valid codes IATA'
        return valid_codes_iata

    def make_request_params(self, session):
        """Makes HTTP-request method and parameters for Nordwind."""
        origin_code_iata = self.origin_code_iata
        destination_code_iata = self.destination_code_iata
        param_routes = {}
        segment_count = 0
        for date in self.route_dates:
            route = {
                'date[%s]' % segment_count: date.strftime('%d.%m.%Y'),
                'origin-city-code[%s]' % segment_count: origin_code_iata,
                'destination-city-code[%s]' % segment_count: destination_code_iata
            }
            origin_code_iata, destination_code_iata = destination_code_iata, origin_code_iata
            param_routes.update(route)
            segment_count += 1
        requst_params = {
            'segmentsCount': segment_count,
            'lang': 'ru',
            'count-aaa': 1
        }
        requst_params.update(param_routes)
        params = {'params': requst_params}
        method = 'GET'
        return method, params

    @staticmethod
    def _extract_flights_from_route(route):    # Extracts flights from route info
        route_flights = []
        for flight in route['flights']:
            number = flight['racenumber']
            air_code = flight['carrier']
            depart_time = flight['departuretime']
            depart_date = flight['departuredate']
            departure_time = datetime.strptime(depart_date + depart_time, '%d.%m.%Y%H:%M')
            arrival_time = flight['arrivaltime']
            origin_code_iata = flight['originport']
            destination_code_iata = flight['destinationport']
            one_flight = Flight(air_code + number, departure_time,
                                arrival_time, origin_code_iata, destination_code_iata)
            route_flights.append(one_flight)
        return route_flights

    @staticmethod
    def _extract_route_prices(routes_price_list, chain_id):    # Extracts prices from route info
        route_prices = dict()
        currency = routes_price_list[chain_id][0]['currency']
        for price in routes_price_list[chain_id]:
            route_price = price['price']
            route_faretype = price['brand'][:12]
            route_prices[route_faretype] = route_price
        return route_prices, currency

    def _extract_routes(self, json_routes, routes_price_list):    # Extracts routes from JSON
        extracted_routes = []
        for route in json_routes['flights']:
            chain_id = route['chainId']
            route_flights = self._extract_flights_from_route(route)
            route_prices = self._extract_route_prices(routes_price_list, chain_id)
            route_duration = route['flights'][0]['flighttime']
            city_origin = route['flights'][0]['origincityName']
            city_destination = route['flights'][0]['destinationcityName']
            departure_date = datetime.strptime(route['flights'][0]['departuredate'], '%d.%m.%Y')
            one_route = Route(departure_date, route_prices, route_duration, city_origin,
                              city_destination, route_flights)
            extracted_routes.append(one_route)
        return extracted_routes

    def extract_data_from_response(self, response):
        """Extracts routes from Nordwind JSON-response."""
        json_routes = self._decode_json(response)
        if 'error' in json_routes:
            raise ValueError('(NordwindSearch) routes not found')
        routes_price_list = {}
        for prices in json_routes['prices']:
            routes_price_list.update(prices)
        routes_found = self._extract_routes(json_routes, routes_price_list)
        return routes_found


class FlynikiSearch(SearchForRoutes):
    """Route-search on Flyniki web-site."""
    def __init__(self, origin_code_iata, destination_code_iata, route_dates):
        SearchForRoutes.__init__(self, origin_code_iata, destination_code_iata, route_dates)
        self.search_request_url = 'https://www.flyniki.com/en/booking/flight/vacancy.php'
        self.validation_url = 'https://www.flyniki.com/en/site/json/suggestAirport.php'

    @staticmethod
    def _find_in_json(code, cities):    # Finds code IATA in JSON for validation, returns city name
        for city in cities['suggestList']:
            if code == city['code']:
                return city['name']
        return None

    def validate_codes_iata(self, session):
        """Validates codes IATA for search-request for Flyniki web-site.

        :param session: for current search-request on particular web-site :class:'Session'
        :return list of two city names, which are used for creating search-request
        """
        valid_cities = []
        payload = {'searchflightid': 0,
                   'suggestsource[]': 'activeairports',
                   'departures[]': self.origin_code_iata,
                   'destinations[]': self.destination_code_iata,
                   'routesource[0]': 'airberlin',
                   'routesource[1]': 'partner'}
        validation_params = {'params': payload}
        for search_for in 'departures', 'destinations':
            validation_params['params'].update(dict(searchfor=search_for))
            key = '{}[]'.format(search_for)
            code_iata = validation_params['params'][key]
            cities_json = self.send_request_and_check(session, 'GET', self.validation_url,
                                                      validation_params)
            cities = self._decode_json(cities_json)
            if not cities['suggestList']:
                raise ValueError('(FlyNikiSearch) invalid code IATA: {}'.format(code_iata))
            city_found = self._find_in_json(code_iata, cities)
            if not city_found:
                raise ValueError('(FlyNikiSearch) invalid code IATA: {}'.format(code_iata))
            valid_cities.append(city_found)
        print '(FlynikiSearch) valid codes IATA'
        return valid_cities

    def get_sid(self, session):
        """Makes simple request and returns SID number for future search.

        :param session: for current search-request on particular web-site :class:'Session'
        :return sid string
        """
        sid_request_headers = {'Content-Type': 'application/x-www-form-urlencoded'}
        sid_request_body = r'market=RU&language=en&bookingmask_' \
                           r'widget_dateformat=dd/mm/yy&returnDate={}'\
            .format(datetime.now().strftime('%d/%m/%Y'))
        sid_request_params = {'headers': sid_request_headers, 'data': sid_request_body}
        sid_request_url = 'https://www.flyniki.com/en/start.php'
        response = self.send_request_and_check(session, 'POST', sid_request_url, sid_request_params)
        sid = response.url.split('=')[1]
        return sid

    def make_request_params(self, session):
        """Makes HTTP-request method and parameters for Flyniki."""
        origin_code_iata = self.origin_code_iata
        destination_code_iata = self.destination_code_iata
        sid = self.get_sid(session)
        request_data = r'_ajax[requestParams][adultCount]=1&_ajax[templates][]=main&' \
                       r'_ajax[templates][]=priceoverview&_ajax[templates][]=infos&' \
                       r'_ajax[templates][]=flightinfo&_ajax[requestParams][departure]={0}&' \
                       r'_ajax[requestParams][destination]={1}'\
            .format(origin_code_iata, destination_code_iata)
        request_data += r'&_ajax[requestParams][outboundDate]={0}&' \
                        r'_ajax[requestParams][returnDate]={1}'\
            .format(self.route_dates[0].strftime('%Y-%m-%d'),
                    self.route_dates[-1].strftime('%Y-%m-%d'))
        if len(self.route_dates) == 1:
            request_data += r'&_ajax[requestParams][oneway]=on'
        request_headers = {'Content-Type': 'application/x-www-form-urlencoded'}
        request_payload = {'sid': sid}
        search_request_params = {'headers': request_headers, 'data': request_data,
                                 'params': request_payload}
        search_request_method = 'POST'
        return search_request_method, search_request_params

    @staticmethod
    def _extract_flights_from_route(route):    # Extracts flights from route info
        route_flights = []
        for flight in route.xpath('./following-sibling::tr[1]//tbody/tr'):
            flight_number = flight.xpath('./td[4]/text()')[0]
            departure_time = flight.xpath('./td[2]/span/time/text()')[0]
            arrival_time = flight.xpath('./td[3]/span/time/text()')[0]
            origin = flight.xpath('./td[2]/span/text()')[1]
            destination = flight.xpath('./td[3]/span/text()')[1]
            origin_code_iata = origin.split(',')[1].strip()
            destination_code_iata = destination.split(',')[1].strip()
            one_flight = Flight(flight_number, departure_time, arrival_time, origin_code_iata,
                                destination_code_iata)
            route_flights.append(one_flight)
        return route_flights

    @staticmethod
    def _extract_route_prices(route):    # Extracts prices from route info
        route_prices = dict()
        for price in route.xpath('./td[position() > 4][label]'):
            route_price = price.xpath('./label/div/span/text()')[0]
            faretype = price.xpath('.//input[@name="faretype"]/@value')[0]
            faregroup = price.xpath('.//input[@name="faregroup"]/@value')[0]
            route_prices[faregroup + faretype] = route_price
        return route_prices

    def _extract_routes(self, direction, currency):    # Extracts routes from direction
        routes_extracted = []
        route_title = direction.xpath('.//div[@class="vacancy_route"]/text()')[0]
        cities = route_title.split(',')[0]
        city_origin = cities.split(u'–')[0]
        city_destination = cities.split(u'–')[1]
        raw_departure_date = route_title.split(',')[-1]
        departure_date = datetime.strptime(raw_departure_date.strip(), '%d/%m/%y')
        for route in direction.xpath('.//tr[@role="group"]'):
            route_flights = self._extract_flights_from_route(route)
            route_prices = self._extract_route_prices(route), currency
            route_duration = route.xpath('./td[4]/span/text()')[0]
            one_route = Route(departure_date, route_prices, route_duration, city_origin,
                              city_destination, route_flights)
            routes_extracted.append(one_route)
        return routes_extracted

    def extract_data_from_response(self, response):
        """Extracts routes from Flyniki mixed JSON/HTML-response."""
        routes_found = []
        json_html = self._decode_json(response)
        html_page = self._decode_html_string(json_html['templates']['main'])
        currency_list = html_page.xpath('.//th/@aria-label')
        if not currency_list:
            raise ValueError('(FlynikiSearch) routes not found')
        currency = currency_list[0].split()[0]
        directions = html_page.xpath('//div[@id="flighttables"]/div[div[@class="row"]]')
        for direction in directions:
            routes_found += self._extract_routes(direction, currency)
        return routes_found


class S7Search(SearchForRoutes):
    """Route-search on S7 web-site."""
    def __init__(self, origin_code_iata, destination_code_iata, route_dates):
        SearchForRoutes.__init__(self, origin_code_iata, destination_code_iata, route_dates)
        self.search_request_url = 'https://travelwith.s7.ru/ajax/actions/updateFlightsSearch.action'
        self.validation_url = 'https://www.s7.ru/app/LocationService'

    @staticmethod
    def _find_in_json(code, cities):    # Finds code IATA in JSON for validation, returns
        for city in cities['c']:        # special code
            if code == city['iata']:
                return city['code']
        return None

    def validate_codes_iata(self, session):
        """Validates codes IATA for search-request for S7 web-site

        :return list of two special search-parameters
        """
        valid_codes_iata = []
        payload = {'action': 'get_locations',
                   'searchType': 'avia'}
        validation_params = {'params': payload}
        for code_iata in self.origin_code_iata, self.destination_code_iata:
            validation_params['params'].update({'str': code_iata})
            cities_json = self.send_request_and_check(session, 'GET', self.validation_url,
                                                      validation_params)
            cities = self._decode_json(cities_json)
            if not cities['c']:
                raise ValueError('(S7Search) invalid code IATA: {}'.format(code_iata))
            code_found = self._find_in_json(code_iata, cities)
            if not code_found:
                raise ValueError('(S7Search) invalid code IATA: {}'.format(code_iata))
            valid_codes_iata.append(code_found)
        print '(S7Search) valid codes IATA'
        return valid_codes_iata

    def make_request_params(self, session):
        """Makes HTTP-request method and parameters for S7"""
        origin_code_iata = self.origin_code_iata
        destination_code_iata = self.destination_code_iata
        request_data = r'model.page=FLIGHTS_SELECT_PAGE&model.milesEnabled=true&' \
                       r'model.directFlightsOnly=false&model.flexible=false&' \
                       r'model.redemption=false&model.currencyType=RUB&' \
                       r'model.multiCitySearchType=PRICE&model.departurePoint={0}&' \
                       r'model.arrivalPoint={1}&model.departureDate={2}&' \
                       r'model.adultsCount=1'.format(origin_code_iata, destination_code_iata,
                                                     self.route_dates[0].strftime('%d.%m.%Y'))
        if len(self.route_dates) == 2:
            request_data += r'&model.routeType=ROUND_TRIP&model.arrivalDate={}'\
                .format(self.route_dates[1].strftime('%d.%m.%Y'))
        else:
            request_data += r'&model.routeType=ONE_WAY'
        request_headers = {'Content-Type': 'application/x-www-form-urlencoded'}
        search_request_params = {'headers': request_headers, 'data': request_data}
        search_request_method = 'POST'
        return search_request_method, search_request_params

    @staticmethod
    def _extract_route_prices(route):    # Extracts prices from route info
        route_prices = dict()
        for price in route.xpath('.//div[div[@class="radiobutton"]]'):
            route_price = price.xpath('.//span[@data-qa="amount"]/text()')[0].replace(u'\xa0', '')
            route_faretype = price.xpath('./@data-tariff-type')[0]
            route_prices[route_faretype] = route_price
        return route_prices

    @staticmethod
    def _extract_flights_from_route(route):    # Extracts flights from route info
        flights = route.xpath('.//div[@data-qa="extended_info"]')
        route_flights = []
        if len(flights) == 1:
            flights_info = [route]
        else:
            flights_info = flights
        for flight in flights_info:
            flight_number = flight.xpath('.//span[@data-qa="number_flightItem"]/text()')[0]
            departure_time = flight.xpath('.//time[@data-qa="timeDeparture_flightItem"]/text()')[0]
            arrival_time = flight.xpath('.//time[@data-qa="timeArrived_flightItem"]/text()')[0]
            origin_code_iata = \
                flight.xpath('.//span[@data-qa="airportDeparture_flightItem"]/text()')[0]
            destination_code_iata = \
                flight.xpath('.//span[@data-qa="airportArrived_flightItem"]/text()')[0]
            one_flight = Flight(flight_number, departure_time, arrival_time, origin_code_iata,
                                destination_code_iata)
            route_flights.append(one_flight)
        return route_flights

    def _extract_routes(self, index, direction, currency):    # Extracts routes from direction
        routes_extracted = []
        cities = direction.xpath('.//div[@class="route"]/text()')
        city_origin = cities[0].strip()
        city_destination = cities[2].strip()
        departure_date = self.route_dates[index]
        for route in direction.xpath('.//div[@data-qa="listFlight"]/div[@data-qa]'):
            route_prices = self._extract_route_prices(route), currency
            route_flights = self._extract_flights_from_route(route)
            route_duration = \
                route.xpath('.//div[@data-qa="durationTotal_flightItemShort"]/text()')[0]
            one_route = Route(departure_date, route_prices, route_duration, city_origin,
                              city_destination, route_flights)
            routes_extracted.append(one_route)
        return routes_extracted

    def extract_data_from_response(self, response):
        """Extracts routes from S7 HTML-response."""
        routes_found = []
        html_page = self._decode_html_string(response.text)
        currency_list = html_page.xpath('//span[@data-qa="currency"][1]/text()')
        if not currency_list:
            raise ValueError('(S7Search) routes not found')
        currency = currency_list[0]
        directions = html_page.xpath('//div[@data-qa="selectFlight_block"]/div[@data-qa]')
        for index, direction in enumerate(directions):
            routes_found += self._extract_routes(index, direction, currency)
        return routes_found


def validate_date(date):
    """Validates date and converts it to datetime object.

    :param date: date string
    :return :class:'datetime' object if date string in format 'dd-mm-yyyy' and date not in the past
    """
    date_object = datetime.strptime(date.strip(), '%d-%m-%Y')
    if date_object.date() >= datetime.now().date():
        return date_object
    else:
        raise ValueError('invalid date: {}'.format(date_object.date()))


def input_date(date_message, required_date=True):
    """Requests date input.

    :param date_message:  message string for user
    :param required_date: (optional)flag that shows mandatory date (default=True)
    :return :class:'datetime' object
    """
    valid_date = None
    while not valid_date:
        try:
            raw_date = raw_input(date_message).strip()
            if required_date:
                valid_date = validate_date(raw_date)
            else:
                if raw_date:
                    valid_date = validate_date(raw_date)
            return valid_date
        except ValueError, error:
            print error


def input_codes_of_sites():
    """Request web-sites codes input and checks it.

    :return list of web-sites for search
    """
    sites = []
    while not sites:
        input_string = raw_input('{0}Code of company: '.format(air_companies_info()))
        if input_string:
            try:
                codes = map(int, input_string.split())
            except ValueError, error:
                print error
                continue
            sites = [SITES_DICT[code] for code in codes if code in SITES_DICT]
        else:
            sites = SITES_DICT.values()
    return sites


def input_code_iata(message):
    """Request code IATA input and performs simple check.

    :param message: message string for user
    :return string
    """
    code_iata = ''
    while not code_iata:
        code_iata = raw_input(message).upper().strip()
        if code_iata.isalpha() and len(code_iata) == 3:
            return code_iata
        else:
            print 'Invalid code IATA'
            code_iata = ''


def air_companies_info():
    """Constructs string of information about web-sites and their codes

    :return string
    """
    info = '\n'
    for code in sorted(SITES_DICT.keys()):
        info += '{1:<15} - {0}\n'.format(code, SITES_DICT[code].__name__)
    return info


def find_routes_in_multi_search():
    """ Request input of all necessary data, executes search on several web-sites
    and collects all routes.
    Repeats action until it gets routes.

    :return list of :class:'Route' objects
    """
    routes_found = []
    sites = input_codes_of_sites()
    while not routes_found:
        origin_code_iata = input_code_iata('Origin code IATA: ')
        destination_code_iata = input_code_iata('Destination code IATA: ')
        departure_date = input_date('Departure date (dd-mm-yyyy): ')
        return_date = input_date('Return date (dd-mm-yyyy): ', required_date=False)
        dates = sorted([date for date in (departure_date, return_date) if date])
        for site in sites:
            one_searcher = site(origin_code_iata, destination_code_iata, dates)
            try:
                one_searcher.search_request()
            except (ValueError, RequestException), error:
                print error
                continue
            routes_found += one_searcher.routes_found
    result_routes = _process_routes(routes_found)
    return result_routes


def _process_routes(routes):
    """Constructs sorted by price list of routes/return routes

    :param routes: list of :class:'Route' objects
    :return list of :class:'Route' objects or list of return-routes (pair of Routes in tuple)
    """
    key_date = routes[0].departure_date
    one_direction = [route for route in routes if route.departure_date == key_date]
    return_direction = [route for route in routes if route.departure_date != key_date]
    if return_direction:
        return_routes = product(one_direction, return_direction)
        prepared_routes = sorted(return_routes, key=lambda x: x[0].lowest_price + x[1].lowest_price)
    else:
        prepared_routes = sorted(routes, key=lambda x: x.lowest_price)
    return prepared_routes


def print_routes(routes, maximum_routes_number):
    """Prints routes/return routes information and prices on console

    :param routes: list of :class:'Route' objects or list of return-routes
    :param maximum_routes_number: number of routes to be printed
    """
    quantity_of_routes = min((len(routes), maximum_routes_number))
    routes = routes[:quantity_of_routes]
    if isinstance(routes[0], Route):
        for route in routes:
            print route
    elif isinstance(routes[0], tuple):
        for route in routes:
            full_price = '{0}\nFull price: {1} {2}\n'\
                .format('*' * 40, (route[0].lowest_price + route[1].lowest_price),
                        route[0].route_prices[1].encode('utf-8'))
            print '{0}\n{1}\n{2}'.format(route[0], route[1], full_price)
    else:
        print 'No routes'


SITES_DICT = {1: AeroflotSearch, 2: S7Search, 3: NordwindSearch, 4: FlynikiSearch}

if __name__ == '__main__':
    ROUTES = find_routes_in_multi_search()
    print_routes(ROUTES, 25)