Спринт 9. Параллелим методы поисковой системы. Задание 2.

-----------------------------------------------------------------------
main.cpp
-----------------------------------------------------------------------
#include "process_queries.h"
#include "search_server.h"

#include <execution>
#include <iostream>
#include <random>
#include <string>
#include <vector>

#include "log_duration.h"

using namespace std;

string GenerateWord(mt19937& generator, int max_length) {
    const int length = uniform_int_distribution(1, max_length)(generator);
    string word;
    word.reserve(length);
    for (int i = 0; i < length; ++i) {
        word.push_back(uniform_int_distribution('a', 'z')(generator));
    }
    return word;
}

vector<string> GenerateDictionary(mt19937& generator, int word_count, int max_length) {
    vector<string> words;
    words.reserve(word_count);
    for (int i = 0; i < word_count; ++i) {
        words.push_back(GenerateWord(generator, max_length));
    }
    sort(words.begin(), words.end());
    words.erase(unique(words.begin(), words.end()), words.end());
    return words;
}

string GenerateQuery(mt19937& generator, const vector<string>& dictionary, int word_count, double minus_prob = 0) {
    string query;
    for (int i = 0; i < word_count; ++i) {
        if (!query.empty()) {
            query.push_back(' ');
        }
        if (uniform_real_distribution<>(0, 1)(generator) < minus_prob) {
            query.push_back('-');
        }
        query += dictionary[uniform_int_distribution<int>(0, dictionary.size() - 1)(generator)];
    }
    return query;
}

vector<string> GenerateQueries(mt19937& generator, const vector<string>& dictionary, int query_count, int max_word_count) {
    vector<string> queries;
    queries.reserve(query_count);
    for (int i = 0; i < query_count; ++i) {
        queries.push_back(GenerateQuery(generator, dictionary, max_word_count));
    }
    return queries;
}

template <typename ExecutionPolicy>
void Test(string_view mark, SearchServer search_server, const string& query, ExecutionPolicy&& policy) {
    LOG_DURATION(mark);
    const int document_count = search_server.GetDocumentCount();
    int word_count = 0;
    for (int id = 0; id < document_count; ++id) {
        const auto [words, status] = search_server.MatchDocument(policy, query, id);
        word_count += words.size();
    }
    cout << word_count << endl;
}

#define TEST(policy) Test(#policy, search_server, query, execution::policy)

int main() {
    mt19937 generator;

    const auto dictionary = GenerateDictionary(generator, 1000, 10);
    const auto documents = GenerateQueries(generator, dictionary, 10'000, 70);

    const string query = GenerateQuery(generator, dictionary, 500, 0.1);

    SearchServer search_server(dictionary[0]);
    for (size_t i = 0; i < documents.size(); ++i) {
        search_server.AddDocument(i, documents[i], DocumentStatus::ACTUAL, {1, 2, 3});
    }

    TEST(seq);
    TEST(par);
}
-----------------------------------------------------------------------
document.cpp
-----------------------------------------------------------------------

empty

-----------------------------------------------------------------------
document.h
-----------------------------------------------------------------------
#pragma once
#include <iostream>

struct Document {
    Document() = default;
    Document(int id, double relevance, int rating)
        : id(id)
        , relevance(relevance)
        , rating(rating) {
    }
    int id = 0;
    double relevance = 0.0;
    int rating = 0;
};

enum class DocumentStatus {
    ACTUAL,
    IRRELEVANT,
    BANNED,
    REMOVED,
};
-----------------------------------------------------------------------
paginator.h
-----------------------------------------------------------------------
#pragma once
#include <algorithm>
#include <vector>
#include <cassert>
#include <iostream>
#include "document.h"

using namespace std::string_literals;

template <typename IteratorRanges>
class IteratorRange {
public:
    explicit IteratorRange(IteratorRanges begin, IteratorRanges end) : begin_(begin), end_(end), size_(distance(begin, end)){}
    IteratorRanges begin() const  {
        return begin_;
    }
    IteratorRanges end() const {
        return end_;
    }
    size_t size() const {
        return size_;
    }
private:
    IteratorRanges begin_;
    IteratorRanges end_;
    size_t size_;
};

std::ostream& operator<<(std::ostream& out, const Document& document) {
    out << "{ document_id = "s << document.id
        << ", relevance = "s << document.relevance
        << ", rating = "s << document.rating << " }"s;
    return out;
}

template <typename To_Out>
std::ostream& operator<<(std::ostream& out, const IteratorRange<To_Out>& sheet) {
    auto helper = sheet.begin();
    while(helper != sheet.end()){
        out<< *helper;
        //out<< "Test";
        ++helper;
    }
    return out;
}

template <typename Paginatorr>
class Paginator {
public :
    Paginator(const Paginatorr& result_begin, const Paginatorr& result_end, size_t size_of_sheet){
        auto full_size = distance(result_begin, result_end);
        Paginatorr helper = result_begin;
        for (auto i = 0; i < full_size/size_of_sheet; ++i) {
            sheets.push_back(IteratorRange<Paginatorr>(helper, helper+ size_of_sheet));
            helper=helper+size_of_sheet;
        }
        if (helper!= result_end) {
            sheets.push_back(IteratorRange<Paginatorr>(helper, result_end));
        }
    }
    auto begin() const {
        return sheets.begin();
    }
    auto end() const {
        return sheets.end();
    }
    size_t size() {
        return sheets.size();
    }
private:
    std::vector<IteratorRange<Paginatorr>> sheets;
};

template <typename Container>
auto Paginate(const Container& c, size_t page_size) {
    return Paginator(begin(c), end(c), page_size);
}
-----------------------------------------------------------------------
process_queries.cpp
-----------------------------------------------------------------------
#include <execution>
#include <algorithm>
#include <list>

#include "process_queries.h"

std::vector<std::vector<Document>> ProcessQueries(const SearchServer& search_server,
    const std::vector<std::string>& queries){

    std::vector<std::vector<Document>> helper(queries.size());
    std::transform(std::execution::par,
                   queries.begin(),
                   queries.end(),
                   helper.begin(),
                   [&search_server](std::string query){
                        return search_server.FindTopDocuments(query);
                    });
    return helper;
}

std::vector<Document> ProcessQueriesJoined(const SearchServer& search_server,
    const std::vector<std::string>& queries){

    return std::transform_reduce(std::execution::par,
                                 queries.begin(),
                                 queries.end(),
                                 std::vector<Document>{},
                                 [](std::vector<Document> lhs, std::vector<Document> const& rhs){
                                     lhs.insert(lhs.end(), rhs.begin(), rhs.end());
                                     return lhs;
                                 },
                                 [&search_server](const std::string& query){
                                     return search_server.FindTopDocuments(query);
                                 });
}
-----------------------------------------------------------------------
process_queries.h
-----------------------------------------------------------------------
#pragma once
#include "search_server.h"
#include <vector>
#include <string>

std::vector<std::vector<Document>> ProcessQueries(const SearchServer& search_server,
    const std::vector<std::string>& queries);

std::vector<Document> ProcessQueriesJoined(const SearchServer& search_server,
    const std::vector<std::string>& queries);
-----------------------------------------------------------------------
read_input_functions.cpp
-----------------------------------------------------------------------
#include "read_input_functions.h"
#include <iostream>

using namespace std::string_literals;

std::string ReadLine() {
    std::string s;
    std::getline(std::cin, s);
    return s;
}

int ReadLineWithNumber() {
    int result;
    std::cin >> result;
    ReadLine();
    return result;
}
-----------------------------------------------------------------------
read_input_functions.h
-----------------------------------------------------------------------
#pragma once
#include "document.h"
#include <string>

std::string ReadLine();
int ReadLineWithNumber();
-----------------------------------------------------------------------
request_queue.cpp
-----------------------------------------------------------------------
#include "request_queue.h"

std::vector<Document> RequestQueue::AddFindRequest(const std::string& raw_query, DocumentStatus status) {
    return AddFindRequest(raw_query, [&status](int document_id, DocumentStatus document_status, int rating) {return document_status == status;});
}

std::vector<Document> RequestQueue::AddFindRequest(const std::string& raw_query) {
    return AddFindRequest(raw_query, DocumentStatus::ACTUAL);
}

int RequestQueue::GetNoResultRequests() const {
    return count_if(requests_.begin(), requests_.end(), [](QueryResult query){return query.query_result == 0;});
}
-----------------------------------------------------------------------
request_queue.h
-----------------------------------------------------------------------
#pragma once
#include "search_server.h"
#include <deque>

class RequestQueue {
public:
    RequestQueue(const SearchServer& search_server) : search_request(search_server){}
    template <typename DocumentPredicate>
    std::vector<Document> AddFindRequest(const std::string& raw_query, DocumentPredicate document_predicate);
    std::vector<Document> AddFindRequest(const std::string& raw_query, DocumentStatus status);
    std::vector<Document> AddFindRequest(const std::string& raw_query);
    int GetNoResultRequests() const;
private:
    struct QueryResult {
        bool query_result;
    };
    const SearchServer& search_request;
    std::deque<QueryResult> requests_;
    const static int min_in_day_ = 1440;
};

template <typename DocumentPredicate>
std::vector<Document> RequestQueue::AddFindRequest(const std::string& raw_query, DocumentPredicate document_predicate) {
    std::vector<Document> helper = search_request.FindTopDocuments(raw_query, document_predicate);

    QueryResult query;
    query.query_result = (helper.empty()==false);

    if (!(requests_.size() < min_in_day_)){
        requests_.pop_front();
    }
    requests_.push_back(query);
    return helper;
}
-----------------------------------------------------------------------
search_server.cpp
-----------------------------------------------------------------------
#include <numeric>
#include "search_server.h"

void SearchServer::AddDocument(int document_id,
                               const std::string& document,
                               DocumentStatus status,
                               const std::vector<int>& ratings) {

    if ((document_id < 0) || (documents_.count(document_id) > 0)) {
        throw std::invalid_argument("Invalid document ID"s);
    }

    const auto words = SplitIntoWordsNoStop(document);
    const double inv_word_count = 1.0 / words.size();

    for (const std::string& word : words) {
        ids_of_docs_to_word_freqs_[document_id][word] += inv_word_count;
        word_to_document_freqs_[word][document_id] += inv_word_count;
    }

    documents_.emplace(document_id, DocumentData{ComputeAverageRating(ratings), status});
    document_ids_.push_back(document_id);
}

std::vector<Document> SearchServer::FindTopDocuments(const std::string& raw_query,
                                                     DocumentStatus status) const {
    return FindTopDocuments(raw_query,
                            [status](int document_id,
                                     DocumentStatus document_status,
                                     int rating) {
                                         return document_status == status;
                                     }
                           );
}

std::vector<Document> SearchServer::FindTopDocuments(const std::string& raw_query) const {
    return FindTopDocuments(raw_query,
                            DocumentStatus::ACTUAL);
}

int SearchServer::GetDocumentCount() const {
    return documents_.size();
}

std::vector<int>::const_iterator SearchServer::begin() const {
    return document_ids_.begin();
}

std::vector<int>::const_iterator SearchServer::end() const {
    return document_ids_.end();
}

const std::map<std::string, double>& SearchServer::GetWordFrequencies(int document_id) const {
    static const std::map<std::string, double> emptyes;
    return (!ids_of_docs_to_word_freqs_.count(document_id))
            ? emptyes
            : ids_of_docs_to_word_freqs_.at(document_id);
}

void SearchServer::RemoveDocument(int document_id) {
    RemoveDocument(std::execution::seq,
                   document_id);
}

void SearchServer::RemoveDocument(const std::execution::sequenced_policy&,
                                  int document_id) {
    auto helper = std::find(document_ids_.begin(),
                            document_ids_.end(),
                            document_id);

    if (helper == document_ids_.end()) {
        return;
    } else {
        document_ids_.erase(helper);
    }

    documents_.erase(document_id);
    std::for_each(word_to_document_freqs_.begin(),
                  word_to_document_freqs_.end(),
                  [&](auto& helper) {
                      helper.second.erase(document_id);
                  });
}

void SearchServer::RemoveDocument(const std::execution::parallel_policy&,
                                  int document_id) {
    auto helper = std::find(std::execution::par,
                            document_ids_.begin(),
                            document_ids_.end(),
                            document_id);

    if (helper == document_ids_.end()) {
        return;
    } else {
        document_ids_.erase(helper);
    }

    documents_.erase(document_id);
    std::for_each(std::execution::par,
                   word_to_document_freqs_.begin(),
                   word_to_document_freqs_.end(),
                   [&](auto& helper) {
                       helper.second.erase(document_id);
                   });
}

std::tuple<std::vector<std::string>, DocumentStatus> SearchServer::MatchDocument(const std::string& raw_query,
                                                                                 int document_id) const {
    return MatchDocument(std::execution::seq,
                         raw_query,
                         document_id);
}

std::tuple<std::vector<std::string>, DocumentStatus> SearchServer::MatchDocument(const std::execution::sequenced_policy&,
                                                                                 const std::string& raw_query,
                                                                                 int document_id) const {

    if ((document_id < 0) || (documents_.count(document_id) <= 0)) {
        throw std::invalid_argument("Non-existent document ID"s);
    }

    const auto result = ParseQuery(raw_query, true);
    std::vector<std::string> matched_words;

    for (const std::string& word : result.minus_words) {
            if (word_to_document_freqs_.count(word) == 0) {
                continue;
            }
            if (word_to_document_freqs_.at(word).count(document_id)) {
                 return {{}, documents_.at(document_id).status};
            }
        }

        for (const std::string& word : result.plus_words) {
            if (word_to_document_freqs_.count(word) == 0) {
                continue;
            }
            if (word_to_document_freqs_.at(word).count(document_id)) {
                matched_words.push_back(word);
            }
        }

        return {matched_words, documents_.at(document_id).status};
}

std::tuple<std::vector<std::string>, DocumentStatus> SearchServer::MatchDocument(const std::execution::parallel_policy&,
                                                                                 const std::string& raw_query,
                                                                                 int document_id) const {
    if ((document_id < 0) || (documents_.count(document_id) <= 0)) {
        throw std::invalid_argument("Non-existent document ID"s);
    }

    const auto result = ParseQuery(raw_query, false);
    std::vector<std::string> matched_words(result.plus_words.size());

    const auto check = [this, document_id](const std::string& word) {
        const auto helper = word_to_document_freqs_.find(word);
        return helper!= word_to_document_freqs_.end() &&
                        helper->second.count(document_id);
    };

    if (std::any_of(std::execution::par,
                    result.minus_words.begin(),
                    result.minus_words.end(),
                    check)) {
                        return {{}, documents_.at(document_id).status};
    }

    auto end = std::copy_if(std::execution::par,
                            result.plus_words.begin(),
                            result.plus_words.end(),
                            matched_words.begin(),
                            check);

    std::sort(matched_words.begin(), end);
    end = std::unique(matched_words.begin(), end);
    matched_words.erase(end, matched_words.end());

    return {matched_words,
            documents_.at(document_id).status};
}

bool SearchServer::IsStopWord(const std::string& word) const {
    return stop_words_.count(word) > 0;
}

bool SearchServer::IsValidWord(const std::string& word) {
    return none_of(word.begin(),
                   word.end(),
                   [](char c) {
                       return c >= '\0' && c < ' ';
                   }
    );
}

std::vector<std::string> SearchServer::SplitIntoWordsNoStop(const std::string& text) const {
    std::vector<std::string> words;
    for (const std::string& word : SplitIntoWords(text)) {
        if (!IsValidWord(word)) {
            throw std::invalid_argument("Word "s + word + " is invalid"s);
        }
        if (!IsStopWord(word)) {
            words.push_back(word);
        }
    }
    return words;
}

int SearchServer::ComputeAverageRating(const std::vector<int>& ratings) {
    if (ratings.empty()) {
        return 0;
    }
    int rating_sum = std::accumulate(ratings.begin(),
                                     ratings.end(),
                                     0);
    return rating_sum /static_cast<int>(ratings.size());
}

SearchServer::QueryWord SearchServer::ParseQueryWord(const std::string& text) const {
    if (text.empty()) {
        throw std::invalid_argument("Query word is empty"s);
    }

    std::string word = text;
    bool is_minus = false;

    if (word[0] == '-') {
        is_minus = true;
        word = word.substr(1);
    }

    if (word.empty() || word[0] == '-' || !IsValidWord(word)) {
        throw std::invalid_argument("Query word "s + text + " is invalid");
    }
    return {word, is_minus, IsStopWord(word)};
}

SearchServer::Query SearchServer::ParseQuery(const std::string& text,
                                             bool sort) const {
    Query result;
    for (const std::string& word : SplitIntoWords(text)) {
        const auto query_word = ParseQueryWord(word);
        if (!query_word.is_stop) {
            if (query_word.is_minus) {
                result.minus_words.push_back(query_word.data);
            } else {
                result.plus_words.push_back(query_word.data);
            }
        }
    }
    if (sort) {
        std::sort(result.minus_words.begin(),
                  result.minus_words.end());
        std::sort(result.plus_words.begin(),
                  result.plus_words.end());

        result.minus_words.erase(std::unique(result.minus_words.begin(),
                                             result.minus_words.end()),
                                             result.minus_words.end());
        result.plus_words.erase(std::unique(result.plus_words.begin(),
                                            result.plus_words.end()),
                                            result.plus_words.end());
    }
    return result;
}

double SearchServer::ComputeWordInverseDocumentFreq(const std::string& word) const {
    return log(GetDocumentCount() * 1.0 / word_to_document_freqs_.at(word).size());
}
-----------------------------------------------------------------------
search_server.h
-----------------------------------------------------------------------
#pragma once
#include <tuple>
#include <algorithm>
#include <cmath>
#include <iostream>
#include <map>
#include <set>
#include <stdexcept>
#include <execution>
#include <string>
#include <vector>
#include "read_input_functions.h"
#include "string_processing.h"
#include "log_duration.h"

using namespace std::string_literals;

class SearchServer {
public:
    template <typename StringContainer>
    SearchServer(const StringContainer& stop_words);
    SearchServer(const std::string& stop_words_text) : SearchServer(SplitIntoWords(stop_words_text)){}

    void AddDocument(int document_id,
                     const std::string& document,
                     DocumentStatus status,
                     const std::vector<int>& ratings);

    template <typename DocumentPredicate>
    std::vector<Document> FindTopDocuments(const std::string& raw_query,
                                           DocumentPredicate document_predicate) const;
    std::vector<Document> FindTopDocuments(const std::string& raw_query,
                                           DocumentStatus status) const;
    std::vector<Document> FindTopDocuments(const std::string& raw_query) const;

    int GetDocumentCount() const;

    std::vector<int>::const_iterator begin() const;
    std::vector<int>::const_iterator end() const;

    const std::map<std::string, double>& GetWordFrequencies(int document_id) const;

    void RemoveDocument(int document_id);
    void RemoveDocument(const std::execution::sequenced_policy&,
                        int document_id);
    void RemoveDocument(const std::execution::parallel_policy&,
                        int document_id);

    std::tuple<std::vector<std::string>, DocumentStatus> MatchDocument(const std::string& raw_query,
                                                                       int document_id) const;
    std::tuple<std::vector<std::string>, DocumentStatus> MatchDocument(const std::execution::sequenced_policy&,
                                                                       const std::string& raw_query,
                                                                       int document_id) const;
    std::tuple<std::vector<std::string>, DocumentStatus> MatchDocument(const std::execution::parallel_policy&,
                                                                       const std::string& raw_query,
                                                                       int document_id) const;

private:
    struct DocumentData {
        int rating;
        DocumentStatus status;
    };

    const double EPSILON = 1e-6;
    const int MAX_RESULT_DOCUMENT_COUNT = 5;

    const std::set<std::string> stop_words_;
    std::map<std::string, std::map<int, double>> word_to_document_freqs_;
    std::map<int, DocumentData> documents_;
    std::vector<int> document_ids_;
    std::map<int, std::map<std::string, double>> ids_of_docs_to_word_freqs_;

    bool IsStopWord(const std::string& word) const;
    static bool IsValidWord(const std::string& word);
    std::vector<std::string> SplitIntoWordsNoStop(const std::string& text) const;
    static int ComputeAverageRating(const std::vector<int>& ratings);

    struct QueryWord {
        std::string data;
        bool is_minus;
        bool is_stop;
    };

    QueryWord ParseQueryWord(const std::string& text) const;

    struct Query {
        std::vector<std::string> plus_words;
        std::vector<std::string> minus_words;
    };

    Query ParseQuery(const std::string& text, bool sort) const;
    double ComputeWordInverseDocumentFreq(const std::string& word) const;

    template <typename DocumentPredicate>
    std::vector<Document> FindAllDocuments(const Query& query,
                                           DocumentPredicate document_predicate) const;
};

template <typename StringContainer>
SearchServer::SearchServer(const StringContainer& stop_words) : stop_words_(MakeUniqueNonEmptyStrings(stop_words)){
    if (!all_of(stop_words_.begin(), stop_words_.end(), IsValidWord)) {
        throw std::invalid_argument("Some of stop words are invalid"s);
    }
}

template <typename DocumentPredicate>
std::vector<Document> SearchServer::FindTopDocuments(const std::string& raw_query,
                                                     DocumentPredicate document_predicate) const {
    const auto query = ParseQuery(raw_query, true);
    auto matched_documents = FindAllDocuments(query,
                                              document_predicate);
    sort(matched_documents.begin(), matched_documents.end(),
    [this](const Document& lhs, const Document& rhs) {
        if (std::abs(lhs.relevance - rhs.relevance) < EPSILON) {
            return lhs.rating > rhs.rating;
        } else {
            return lhs.relevance > rhs.relevance;
        }
    });
    if (matched_documents.size() > MAX_RESULT_DOCUMENT_COUNT) {
        matched_documents.resize(MAX_RESULT_DOCUMENT_COUNT);
    }
        return matched_documents;
}

template <typename DocumentPredicate>
std::vector<Document> SearchServer::FindAllDocuments(const Query& query,
                                                     DocumentPredicate document_predicate) const {
    std::map<int, double> document_to_relevance;
    for (const std::string& word : query.plus_words) {
        if (word_to_document_freqs_.count(word) == 0) {
            continue;
        }
    const double inverse_document_freq = ComputeWordInverseDocumentFreq(word);
        for (const auto [document_id, term_freq] : word_to_document_freqs_.at(word)) {
            const auto& document_data = documents_.at(document_id);
            if (document_predicate(document_id, document_data.status, document_data.rating)) {
                document_to_relevance[document_id] += term_freq * inverse_document_freq;
            }
        }
    }

    for (const std::string& word : query.minus_words) {
       if (word_to_document_freqs_.count(word) == 0) {
            continue;
        }
        for (const auto [document_id, _] : word_to_document_freqs_.at(word)) {
            document_to_relevance.erase(document_id);
        }
    }

    std::vector<Document> matched_documents;
    for (const auto [document_id, relevance] : document_to_relevance) {
        matched_documents.push_back(
            {document_id, relevance, documents_.at(document_id).rating});
    }
    return matched_documents;
}
-----------------------------------------------------------------------
string_processing.cpp
-----------------------------------------------------------------------
#include "string_processing.h"

std::vector<std::string> SplitIntoWords(const std::string& text) {
    std::vector<std::string> words;
    std::string word;
    for (const char c : text) {
        if (c == ' ') {
            if (!word.empty()) {
                words.push_back(word);
                word.clear();
            }
        } else {
            word += c;
        }
    }
    if (!word.empty()) {
        words.push_back(word);
    }
    return words;
}
-----------------------------------------------------------------------
string_processing.h
-----------------------------------------------------------------------
#pragma once
#include <set>
#include <vector>
#include <string>

std::vector<std::string> SplitIntoWords(const std::string& text);

template <typename StringContainer>
std::set<std::string> MakeUniqueNonEmptyStrings(const StringContainer& strings) {
    std::set<std::string> non_empty_strings;
    for (const std::string& str : strings) {
        if (!str.empty()) {
            non_empty_strings.insert(str);
        }
    }
    return non_empty_strings;
}
-----------------------------------------------------------------------
test_example_functions.cpp
-----------------------------------------------------------------------

empty

-----------------------------------------------------------------------
test_example_functions.h
-----------------------------------------------------------------------

empty