API tools faq
paste
shushus

Спринт 6. Дедупликатор документов.

shushus
Dec 26th, 2022
543
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
C++ 22.78 KB | None | 0 0
raw download clone embed print report
  1. -----------------------------------------------------------------------
  2. main.cpp
  3. -----------------------------------------------------------------------
  4. #include "document.h"
  5. #include "paginator.h"
  6. #include "read_input_functions.h"
  7. #include "string_processing.h"
  8. #include "search_server.h"
  9. #include "request_queue.h"
  10. #include "remove_duplicates.h"
  11. #include "test_example_functions.h"
  12.  
  13. using namespace std;
  14.  
  15. int main() {
  16.     SearchServer search_server("and with"s);
  17.     AddDocument(search_server, 1, "funny pet and nasty rat"s, DocumentStatus::ACTUAL, {7, 2, 7});
  18.     AddDocument(search_server, 2, "funny pet with curly hair"s, DocumentStatus::ACTUAL, {1, 2});
  19.     AddDocument(search_server, 3, "funny pet with curly hair"s, DocumentStatus::ACTUAL, {1, 2});
  20.     AddDocument(search_server, 4, "funny pet and curly hair"s, DocumentStatus::ACTUAL, {1, 2});
  21.     AddDocument(search_server, 5, "funny funny pet and nasty nasty rat"s, DocumentStatus::ACTUAL, {1, 2});
  22.     AddDocument(search_server, 6, "funny pet and not very nasty rat"s, DocumentStatus::ACTUAL, {1, 2});
  23.     AddDocument(search_server, 7, "very nasty rat and not very funny pet"s, DocumentStatus::ACTUAL, {1, 2});
  24.     AddDocument(search_server, 8, "pet with rat and rat and rat"s, DocumentStatus::ACTUAL, {1, 2});
  25.     AddDocument(search_server, 9, "nasty rat with curly hair"s, DocumentStatus::ACTUAL, {1, 2});
  26.     cout << "Before duplicates removed: "s << search_server.GetDocumentCount() << endl;
  27.     RemoveDuplicates(search_server);
  28.     cout << "After duplicates removed: "s << search_server.GetDocumentCount() << endl;
  29. }
  30. -----------------------------------------------------------------------
  31. document.cpp
  32. -----------------------------------------------------------------------
  33.  
  34. empty
  35.  
  36. -----------------------------------------------------------------------
  37. document.h
  38. -----------------------------------------------------------------------
  39. #pragma once
  40. #include <iostream>
  41.  
  42. struct Document {
  43.     Document() = default;
  44.     Document(int id, double relevance, int rating)
  45.         : id(id)
  46.         , relevance(relevance)
  47.         , rating(rating) {
  48.     }
  49.     int id = 0;
  50.     double relevance = 0.0;
  51.     int rating = 0;
  52. };
  53.  
  54. enum class DocumentStatus {
  55.     ACTUAL,
  56.     IRRELEVANT,
  57.     BANNED,
  58.     REMOVED,
  59. };
  60. -----------------------------------------------------------------------
  61. log_duration.h
  62. -----------------------------------------------------------------------
  63. #pragma once
  64. #include <chrono>
  65. #include <iostream>
  66.  
  67. #define PROFILE_CONCAT_INTERNAL(X, Y) X##Y
  68. #define PROFILE_CONCAT(X, Y) PROFILE_CONCAT_INTERNAL(X, Y)
  69. #define UNIQUE_VAR_NAME_PROFILE PROFILE_CONCAT(profileGuard, __LINE__)
  70. #define LOG_DURATION(x) LogDuration UNIQUE_VAR_NAME_PROFILE(x)
  71. #define LOG_DURATION_STREAM(x, y) LogDuration UNIQUE_VAR_NAME_PROFILE(x, y)
  72.  
  73. class LogDuration {
  74. public:
  75.     using Clock = std::chrono::steady_clock;
  76.     LogDuration(const std::string& id, std::ostream& out = std::cerr) : id_(id), out_(out) {}
  77.     ~LogDuration() {
  78.         using namespace std::chrono;
  79.         using namespace std::literals;
  80.  
  81.         const auto end_time = Clock::now();
  82.         const auto dur = end_time - start_time_;
  83.         out_ << id_ << ": "s << duration_cast<milliseconds>(dur).count() << " ms"s << std::endl;
  84.     }
  85. private:
  86.     const std::string id_;
  87.     const Clock::time_point start_time_ = Clock::now();
  88.     std::ostream& out_;
  89. };
  90. -----------------------------------------------------------------------
  91. paginator.h
  92. -----------------------------------------------------------------------
  93. #pragma once
  94. #include <algorithm>
  95. #include <vector>
  96. #include <cassert>
  97. #include <iostream>
  98. #include "document.h"
  99.  
  100. using namespace std::string_literals;
  101.  
  102. template <typename IteratorRanges>
  103. class IteratorRange {
  104. public:
  105.     explicit IteratorRange(IteratorRanges begin, IteratorRanges end) : begin_(begin), end_(end), size_(distance(begin, end)){}
  106.     IteratorRanges begin() const  {
  107.         return begin_;
  108.     }
  109.     IteratorRanges end() const {
  110.         return end_;
  111.     }
  112.     size_t size() const {
  113.         return size_;
  114.     }
  115. private:
  116.     IteratorRanges begin_;
  117.     IteratorRanges end_;
  118.     size_t size_;
  119. };
  120.  
  121. std::ostream& operator<<(std::ostream& out, const Document& document) {
  122.     out << "{ document_id = "s << document.id
  123.         << ", relevance = "s << document.relevance
  124.         << ", rating = "s << document.rating << " }"s;
  125.     return out;
  126. }
  127.  
  128. template <typename To_Out>
  129. std::ostream& operator<<(std::ostream& out, const IteratorRange<To_Out>& sheet) {
  130.     auto helper = sheet.begin();
  131.     while(helper != sheet.end()){
  132.         out<< *helper;
  133.         //out<< "Test";
  134.         ++helper;
  135.     }
  136.     return out;
  137. }
  138.  
  139. template <typename Paginatorr>
  140. class Paginator {
  141. public :
  142.     Paginator(const Paginatorr& result_begin, const Paginatorr& result_end, size_t size_of_sheet){
  143.         auto full_size = distance(result_begin, result_end);
  144.         Paginatorr helper = result_begin;
  145.         for (auto i = 0; i < full_size/size_of_sheet; ++i) {
  146.             sheets.push_back(IteratorRange<Paginatorr>(helper, helper+ size_of_sheet));
  147.             helper=helper+size_of_sheet;
  148.         }
  149.         if (helper!= result_end) {
  150.             sheets.push_back(IteratorRange<Paginatorr>(helper, result_end));
  151.         }
  152.     }
  153.     auto begin() const {
  154.         return sheets.begin();
  155.     }
  156.     auto end() const {
  157.         return sheets.end();
  158.     }
  159.     size_t size() {
  160.         return sheets.size();
  161.     }
  162. private:
  163.     std::vector<IteratorRange<Paginatorr>> sheets;
  164. };
  165.  
  166. template <typename Container>
  167. auto Paginate(const Container& c, size_t page_size) {
  168.     return Paginator(begin(c), end(c), page_size);
  169. }
  170. -----------------------------------------------------------------------
  171. read_input_functions.cpp
  172. -----------------------------------------------------------------------
  173. #include "read_input_functions.h"
  174. #include <iostream>
  175.  
  176. using namespace std::string_literals;
  177.  
  178. std::string ReadLine() {
  179.     std::string s;
  180.     std::getline(std::cin, s);
  181.     return s;
  182. }
  183.  
  184. int ReadLineWithNumber() {
  185.     int result;
  186.     std::cin >> result;
  187.     ReadLine();
  188.     return result;
  189. }
  190. -----------------------------------------------------------------------
  191. read_input_functions.h
  192. -----------------------------------------------------------------------
  193. #pragma once
  194. #include "document.h"
  195. #include <string>
  196.  
  197. std::string ReadLine();
  198. int ReadLineWithNumber();
  199. -----------------------------------------------------------------------
  200. request_queue.cpp
  201. -----------------------------------------------------------------------
  202. #include "request_queue.h"
  203.  
  204. std::vector<Document> RequestQueue::AddFindRequest(const std::string& raw_query, DocumentStatus status) {
  205.     return AddFindRequest(raw_query, [&status](int document_id, DocumentStatus document_status, int rating) {return document_status == status;});
  206. }
  207.    
  208. std::vector<Document> RequestQueue::AddFindRequest(const std::string& raw_query) {
  209.     return AddFindRequest(raw_query, DocumentStatus::ACTUAL);
  210. }
  211.    
  212. int RequestQueue::GetNoResultRequests() const {
  213.     return count_if(requests_.begin(), requests_.end(), [](QueryResult query){return query.query_result == 0;});
  214. }
  215. -----------------------------------------------------------------------
  216. request_queue.h
  217. -----------------------------------------------------------------------
  218. #pragma once
  219. #include "search_server.h"
  220. #include <deque>
  221.  
  222. class RequestQueue {
  223. public:
  224.     RequestQueue(const SearchServer& search_server) : search_request(search_server){}  
  225.     template <typename DocumentPredicate>
  226.     std::vector<Document> AddFindRequest(const std::string& raw_query, DocumentPredicate document_predicate);
  227.     std::vector<Document> AddFindRequest(const std::string& raw_query, DocumentStatus status);
  228.     std::vector<Document> AddFindRequest(const std::string& raw_query);
  229.     int GetNoResultRequests() const;
  230. private:
  231.     struct QueryResult {
  232.         bool query_result;
  233.     };
  234.     const SearchServer& search_request;
  235.     std::deque<QueryResult> requests_;
  236.     const static int min_in_day_ = 1440;
  237. };
  238.  
  239. template <typename DocumentPredicate>
  240. std::vector<Document> RequestQueue::AddFindRequest(const std::string& raw_query, DocumentPredicate document_predicate) {
  241.     std::vector<Document> helper = search_request.FindTopDocuments(raw_query, document_predicate);
  242.  
  243.     QueryResult query;
  244.     query.query_result = (helper.empty()==false);
  245.  
  246.     if (!(requests_.size() < min_in_day_)){
  247.         requests_.pop_front();
  248.     }
  249.     requests_.push_back(query);
  250.     return helper;
  251. }
  252. -----------------------------------------------------------------------
  253. search_server.cpp
  254. -----------------------------------------------------------------------
  255. #include <numeric>
  256. #include "search_server.h"
  257.  
  258. void SearchServer::AddDocument(int document_id, const std::string& document, DocumentStatus status,const std::vector<int>& ratings){
  259.     if ((document_id < 0) || (documents_.count(document_id) > 0)) {
  260.         throw std::invalid_argument("Invalid document_id"s);
  261.     }
  262.     const auto words = SplitIntoWordsNoStop(document);
  263.  
  264.     const double inv_word_count = 1.0 / words.size();
  265.     for (const std::string& word : words) {
  266.         ids_of_docs_to_word_freqs_[document_id][word] += inv_word_count;
  267.         word_to_document_freqs_[word][document_id] += inv_word_count;
  268.     }
  269.     documents_.emplace(document_id, DocumentData{ComputeAverageRating(ratings), status});
  270.     document_ids_.insert(document_id);
  271. }
  272.  
  273. std::vector<Document> SearchServer::FindTopDocuments(const std::string& raw_query, DocumentStatus status) const {
  274.     return FindTopDocuments(raw_query, [status](int document_id, DocumentStatus document_status, int rating) {
  275.             return document_status == status;
  276.         });
  277. }
  278.  
  279. std::vector<Document> SearchServer::FindTopDocuments(const std::string& raw_query) const {
  280.     return FindTopDocuments(raw_query, DocumentStatus::ACTUAL);
  281. }
  282.  
  283. int SearchServer::GetDocumentCount() const {
  284.     return documents_.size();
  285. }
  286.  
  287. std::set<int> ::const_iterator SearchServer::begin() const {
  288.     return document_ids_.begin();
  289. }
  290.  
  291. std::set<int> ::const_iterator SearchServer::end() const {
  292.     return document_ids_.end();
  293. }
  294.  
  295. const std::map<std::string, double>& SearchServer::GetWordFrequencies(int document_id) const {
  296.     static const std::map<std::string, double> emptyes;
  297.     return (!ids_of_docs_to_word_freqs_.count(document_id)) ? emptyes : ids_of_docs_to_word_freqs_.at(document_id);
  298. }
  299.  
  300. void SearchServer::RemoveDocument(int document_id) {
  301.     if (document_ids_.find(document_id) != document_ids_.end()) {
  302.         for (auto& [word, _] : ids_of_docs_to_word_freqs_[document_id]) {
  303.             auto erase_word = word_to_document_freqs_[word].find(document_id);
  304.             word_to_document_freqs_[word].erase(erase_word);
  305.         }
  306.     }
  307.    
  308.     documents_.erase(document_id);
  309.     document_ids_.erase(document_id);
  310.     ids_of_docs_to_word_freqs_.erase(document_id);
  311. }
  312.  
  313. std::tuple<std::vector<std::string>, DocumentStatus> SearchServer::MatchDocument(const std::string& raw_query, int document_id) const {
  314.     const auto query = ParseQuery(raw_query);
  315.     std::vector<std::string> matched_words;
  316.     for (const std::string& word : query.plus_words) {
  317.         if (word_to_document_freqs_.count(word) == 0) {
  318.             continue;
  319.         }
  320.         if (word_to_document_freqs_.at(word).count(document_id)) {
  321.             matched_words.push_back(word);
  322.         }
  323.     }
  324.     for (const std::string& word : query.minus_words) {
  325.         if (word_to_document_freqs_.count(word) == 0) {
  326.             continue;
  327.         }
  328.         if (word_to_document_freqs_.at(word).count(document_id)) {
  329.             matched_words.clear();
  330.             break;
  331.         }
  332.     }
  333.     return {matched_words, documents_.at(document_id).status};
  334. }
  335.  
  336. bool SearchServer::IsStopWord(const std::string& word) const {
  337.     return stop_words_.count(word) > 0;
  338. }
  339.  
  340. bool SearchServer::IsValidWord(const std::string& word) {
  341.     return none_of(word.begin(), word.end(), [](char c) {
  342.         return c >= '\0' && c < ' ';
  343.     });
  344. }
  345.  
  346. std::vector<std::string> SearchServer::SplitIntoWordsNoStop(const std::string& text) const {
  347.     std::vector<std::string> words;
  348.     for (const std::string& word : SplitIntoWords(text)) {
  349.         if (!IsValidWord(word)) {
  350.             throw std::invalid_argument("Word "s + word + " is invalid"s);
  351.         }
  352.         if (!IsStopWord(word)) {
  353.             words.push_back(word);
  354.         }
  355.     }
  356.     return words;
  357. }
  358.  
  359. int SearchServer::ComputeAverageRating(const std::vector<int>& ratings) {
  360.     if (ratings.empty()) {
  361.         return 0;
  362.     }
  363.     int rating_sum = std::accumulate(ratings.begin(), ratings.end(), 0);
  364.     return rating_sum / static_cast<int>(ratings.size());
  365. }
  366.  
  367. SearchServer::QueryWord SearchServer::ParseQueryWord(const std::string& text) const {
  368.     if (text.empty()) {
  369.         throw std::invalid_argument("Query word is empty"s);
  370.     }
  371.     std::string word = text;
  372.     bool is_minus = false;
  373.     if (word[0] == '-') {
  374.         is_minus = true;
  375.         word = word.substr(1);
  376.     }
  377.     if (word.empty() || word[0] == '-' || !IsValidWord(word)) {
  378.         throw std::invalid_argument("Query word "s + text + " is invalid");
  379.     }
  380.  
  381.     return {word, is_minus, IsStopWord(word)};
  382. }
  383.  
  384. SearchServer::Query SearchServer::ParseQuery(const std::string& text) const {
  385.     Query result;
  386.     for (const std::string& word : SplitIntoWords(text)) {
  387.         const auto query_word = ParseQueryWord(word);
  388.         if (!query_word.is_stop) {
  389.             if (query_word.is_minus) {
  390.                 result.minus_words.insert(query_word.data);
  391.             } else {
  392.                 result.plus_words.insert(query_word.data);
  393.             }
  394.         }
  395.     }
  396.     return result;
  397. }
  398.  
  399. double SearchServer::ComputeWordInverseDocumentFreq(const std::string& word) const {
  400.     return log(GetDocumentCount() * 1.0 / word_to_document_freqs_.at(word).size());
  401. }
  402. -----------------------------------------------------------------------
  403. search_server.h
  404. -----------------------------------------------------------------------
  405. #pragma once
  406. #include <tuple>
  407. #include <algorithm>
  408. #include <cmath>
  409. #include <iostream>
  410. #include <map>
  411. #include <set>
  412. #include <stdexcept>
  413. #include <string>
  414. #include <vector>
  415. #include "read_input_functions.h"
  416. #include "string_processing.h"
  417. #include "log_duration.h"
  418.  
  419. using namespace std::string_literals;
  420.  
  421. class SearchServer {
  422. public:
  423.     template <typename StringContainer>
  424.     SearchServer(const StringContainer& stop_words);
  425.     SearchServer(const std::string& stop_words_text) : SearchServer(SplitIntoWords(stop_words_text)){}
  426.    
  427.     void AddDocument(int document_id, const std::string& document, DocumentStatus status, const std::vector<int>& ratings);
  428.    
  429.     template <typename DocumentPredicate>
  430.     std::vector<Document> FindTopDocuments(const std::string& raw_query, DocumentPredicate document_predicate) const;
  431.     std::vector<Document> FindTopDocuments(const std::string& raw_query, DocumentStatus status) const;
  432.     std::vector<Document> FindTopDocuments(const std::string& raw_query) const;
  433.    
  434.     int GetDocumentCount() const;
  435.    
  436.     std::set<int> ::const_iterator begin() const;
  437.     std::set<int> ::const_iterator end() const;
  438.    
  439.     const std::map<std::string, double>& GetWordFrequencies(int document_id) const;
  440.    
  441.     void RemoveDocument(int document_id);
  442.    
  443.     std::tuple<std::vector<std::string>, DocumentStatus> MatchDocument(const std::string& raw_query, int document_id) const;
  444.  
  445. private:
  446.     struct DocumentData {
  447.         int rating;
  448.         DocumentStatus status;
  449.     };
  450.    
  451.     const double EPSILON = 1e-6;
  452.     const int MAX_RESULT_DOCUMENT_COUNT = 5;
  453.    
  454.     const std::set<std::string> stop_words_;
  455.     std::map<std::string, std::map<int, double>> word_to_document_freqs_;
  456.     std::map<int, DocumentData> documents_;
  457.     std::set<int> document_ids_;
  458.     std::map<int, std::map<std::string, double>> ids_of_docs_to_word_freqs_;
  459.  
  460.     bool IsStopWord(const std::string& word) const;
  461.     static bool IsValidWord(const std::string& word);
  462.     std::vector<std::string> SplitIntoWordsNoStop(const std::string& text) const;
  463.     static int ComputeAverageRating(const std::vector<int>& ratings);
  464.  
  465.     struct QueryWord {
  466.         std::string data;
  467.         bool is_minus;
  468.         bool is_stop;
  469.     };
  470.  
  471.     QueryWord ParseQueryWord(const std::string& text) const;
  472.  
  473.     struct Query {
  474.         std::set<std::string> plus_words;
  475.         std::set<std::string> minus_words;
  476.     };
  477.  
  478.     Query ParseQuery(const std::string& text) const;
  479.     double ComputeWordInverseDocumentFreq(const std::string& word) const;
  480.  
  481.     template <typename DocumentPredicate>
  482.     std::vector<Document> FindAllDocuments(const Query& query, DocumentPredicate document_predicate) const;
  483. };
  484.  
  485. template <typename StringContainer>
  486. SearchServer::SearchServer(const StringContainer& stop_words) : stop_words_(MakeUniqueNonEmptyStrings(stop_words)){
  487.     if (!all_of(stop_words_.begin(), stop_words_.end(), IsValidWord)) {
  488.         throw std::invalid_argument("Some of stop words are invalid"s);
  489.     }
  490. }
  491.  
  492. template <typename DocumentPredicate>
  493. std::vector<Document> SearchServer::FindTopDocuments(const std::string& raw_query, DocumentPredicate document_predicate) const {
  494.     const auto query = ParseQuery(raw_query);
  495.     auto matched_documents = FindAllDocuments(query, document_predicate);
  496.     sort(matched_documents.begin(), matched_documents.end(),
  497.     [this](const Document& lhs, const Document& rhs) {
  498.         if (std::abs(lhs.relevance - rhs.relevance) < EPSILON) {
  499.             return lhs.rating > rhs.rating;
  500.         } else {
  501.             return lhs.relevance > rhs.relevance;
  502.         }
  503.     });
  504.     if (matched_documents.size() > MAX_RESULT_DOCUMENT_COUNT) {
  505.         matched_documents.resize(MAX_RESULT_DOCUMENT_COUNT);
  506.     }
  507.         return matched_documents;
  508. }
  509.  
  510. template <typename DocumentPredicate>
  511. std::vector<Document> SearchServer::FindAllDocuments(const Query& query, DocumentPredicate document_predicate) const {
  512.     std::map<int, double> document_to_relevance;
  513.     for (const std::string& word : query.plus_words) {
  514.         if (word_to_document_freqs_.count(word) == 0) {
  515.             continue;
  516.         }
  517.     const double inverse_document_freq = ComputeWordInverseDocumentFreq(word);
  518.         for (const auto [document_id, term_freq] : word_to_document_freqs_.at(word)) {
  519.             const auto& document_data = documents_.at(document_id);
  520.             if (document_predicate(document_id, document_data.status, document_data.rating)) {
  521.                 document_to_relevance[document_id] += term_freq * inverse_document_freq;
  522.             }
  523.         }
  524.     }
  525.  
  526.     for (const std::string& word : query.minus_words) {
  527.        if (word_to_document_freqs_.count(word) == 0) {
  528.             continue;
  529.         }
  530.         for (const auto [document_id, _] : word_to_document_freqs_.at(word)) {
  531.             document_to_relevance.erase(document_id);
  532.         }
  533.     }
  534.  
  535.     std::vector<Document> matched_documents;
  536.     for (const auto [document_id, relevance] : document_to_relevance) {
  537.         matched_documents.push_back(
  538.             {document_id, relevance, documents_.at(document_id).rating});
  539.     }
  540.     return matched_documents;
  541. }
  542. -----------------------------------------------------------------------
  543. string_processing.cpp
  544. -----------------------------------------------------------------------
  545. #include "string_processing.h"
  546.  
  547. std::vector<std::string> SplitIntoWords(const std::string& text) {
  548.     std::vector<std::string> words;
  549.     std::string word;
  550.     for (const char c : text) {
  551.         if (c == ' ') {
  552.             if (!word.empty()) {
  553.                 words.push_back(word);
  554.                 word.clear();
  555.             }
  556.         } else {
  557.             word += c;
  558.         }
  559.     }
  560.     if (!word.empty()) {
  561.         words.push_back(word);
  562.     }
  563.     return words;
  564. }
  565. -----------------------------------------------------------------------
  566. string_processing.h
  567. -----------------------------------------------------------------------
  568. #pragma once
  569. #include <set>
  570. #include <vector>
  571. #include <string>
  572.  
  573. std::vector<std::string> SplitIntoWords(const std::string& text);
  574.  
  575. template <typename StringContainer>
  576. std::set<std::string> MakeUniqueNonEmptyStrings(const StringContainer& strings) {
  577.     std::set<std::string> non_empty_strings;
  578.     for (const std::string& str : strings) {
  579.         if (!str.empty()) {
  580.             non_empty_strings.insert(str);
  581.         }
  582.     }
  583.     return non_empty_strings;
  584. }
  585. -----------------------------------------------------------------------
  586. test_example_functions.cpp
  587. -----------------------------------------------------------------------
  588. #include "test_example_functions.h"
  589.  
  590. void AddDocument(SearchServer& search_server, int document_id, const std::string& document, DocumentStatus status, const std::vector<int>& ratings){
  591.     try{
  592.         search_server.AddDocument(document_id, document, status, ratings);
  593.     }catch(const std::invalid_argument& e){
  594.         std::cout << "Can not add a document "s << document_id << ": "s << e.what() << std::endl;
  595.     }
  596. }
  597. -----------------------------------------------------------------------
  598. test_example_functions.h
  599. -----------------------------------------------------------------------
  600. #pragma once
  601. #include "search_server.h"
  602. using namespace std;
  603.  
  604. void AddDocument(SearchServer& search_server, int document_id, const std::string& document, DocumentStatus status, const std::vector<int>& ratings);
  605. -----------------------------------------------------------------------
  606. remove_duplicates.cpp
  607. -----------------------------------------------------------------------
  608. #include "remove_duplicates.h"
  609. using namespace std::string_literals;
  610.  
  611. void RemoveDuplicates(SearchServer& search_server) {
  612.     //LOG_DURATION_STREAM("Remove duplicates", std::cout);
  613.     std::set<int> id_remove;
  614.     std::map<std::set<std::string>, int> unique_word_plus_id;
  615.     for (const int document_id : search_server) {
  616.         std::map<std::string, double> all_words;
  617.        
  618.         all_words = search_server.GetWordFrequencies(document_id);
  619.         std::set<std::string> unique_words;
  620.        
  621.         for (auto [word, _] : all_words) {
  622.             unique_words.insert(word);
  623.         }
  624.        
  625.         if (unique_word_plus_id.count(unique_words)) {
  626.             id_remove.insert(document_id);
  627.         } else {
  628.             unique_word_plus_id.insert(std::pair{unique_words, document_id});
  629.         }
  630.     }
  631.     for (auto id_: id_remove) {
  632.         std::cout<<"Found duplicate document id "<< id_ <<std::endl;
  633.         search_server.RemoveDocument(id_);
  634.     }
  635. }
  636. -----------------------------------------------------------------------
  637. remove_duplicates.h
  638. -----------------------------------------------------------------------
  639. #pragma once
  640. #include "search_server.h"
  641.  
  642. void RemoveDuplicates(SearchServer& search_server);
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!