Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #include <algorithm>
- #include <cassert>
- #include <string>
- #include <vector>
- using namespace std;
- template <typename Token>
- using Sentence = vector<Token>;
- template <typename TokenForwardIt>
- TokenForwardIt FindSentenceEnd(TokenForwardIt tokens_begin, TokenForwardIt tokens_end) {
- const TokenForwardIt before_sentence_end
- = adjacent_find(tokens_begin, tokens_end, [](const auto& left_token, const auto& right_token) {
- return left_token.IsEndSentencePunctuation() && !right_token.IsEndSentencePunctuation();
- });
- return before_sentence_end == tokens_end ? tokens_end : next(before_sentence_end);
- }
- template <typename Token>
- vector<Sentence<Token>> SplitIntoSentences(vector<Token> tokens) {
- vector<Sentence<Token>> helper;
- auto iter_begin = tokens.begin(), iter_end = tokens.end(), iter = tokens.begin();
- while (iter_begin != iter_end) {
- auto token = FindSentenceEnd(iter_begin, iter_end);
- if (iter == token) {
- helper.push_back({make_move_iterator(iter_begin), make_move_iterator(token)});
- iter_begin = iter;
- }
- ++iter;
- }
- return helper;
- }
- struct TestToken {
- string data;
- bool is_end_sentence_punctuation = false;
- bool IsEndSentencePunctuation() const {
- return is_end_sentence_punctuation;
- }
- bool operator==(const TestToken& other) const {
- return data == other.data && is_end_sentence_punctuation == other.is_end_sentence_punctuation;
- }
- };
- ostream& operator<<(ostream& stream, const TestToken& token) {
- return stream << token.data;
- }
- void TestSplitting() {
- assert(SplitIntoSentences(vector<TestToken>({{"Split"s}, {"into"s}, {"sentences"s}, {"!"s}}))
- == vector<Sentence<TestToken>>({{{"Split"s}, {"into"s}, {"sentences"s}, {"!"s}}}));
- assert(SplitIntoSentences(vector<TestToken>({{"Split"s}, {"into"s}, {"sentences"s}, {"!"s, true}}))
- == vector<Sentence<TestToken>>({{{"Split"s}, {"into"s}, {"sentences"s}, {"!"s, true}}}));
- assert(SplitIntoSentences(vector<TestToken>(
- {{"Split"s}, {"into"s}, {"sentences"s}, {"!"s, true}, {"!"s, true}, {"Without"s}, {"copies"s}, {"."s, true}}))
- == vector<Sentence<TestToken>>({
- {{"Split"s}, {"into"s}, {"sentences"s}, {"!"s, true}, {"!"s, true}},
- {{"Without"s}, {"copies"s}, {"."s, true}},
- }));
- }
- int main() {
- TestSplitting();
- }
Advertisement
Add Comment
Please, Sign In to add comment