Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #include <algorithm>
- #include <cassert>
- #include <string>
- #include <vector>
- #include <utility>
- using namespace std;
- template <typename Token>
- using Sentence = vector<Token>;
- template <typename TokenForwardIt>
- TokenForwardIt FindSentenceEnd(TokenForwardIt tokens_begin, TokenForwardIt tokens_end) {
- const TokenForwardIt before_sentence_end
- = adjacent_find(tokens_begin, tokens_end, [](const auto& left_token, const auto& right_token) {
- return left_token.IsEndSentencePunctuation() && !right_token.IsEndSentencePunctuation();
- });
- return before_sentence_end == tokens_end ? tokens_end : next(before_sentence_end);
- }
- template <typename Token>
- vector<Sentence<Token>> SplitIntoSentences(vector<Token> tokens) {
- vector<Sentence<Token>> vector_helper; Sentence<Token> helper_part;
- for (auto begin = tokens.begin();;){
- auto end = FindSentenceEnd(begin, tokens.end());
- move(begin, end, std::back_inserter(helper_part));
- vector_helper.push_back(move(helper_part));
- if (end == tokens.end()){
- break;
- }
- begin = end;
- }
- return vector_helper;
- }
- struct TestToken {
- string data;
- bool is_end_sentence_punctuation = false;
- bool IsEndSentencePunctuation() const {
- return is_end_sentence_punctuation;
- }
- bool operator==(const TestToken& other) const {
- return data == other.data && is_end_sentence_punctuation == other.is_end_sentence_punctuation;
- }
- };
- ostream& operator<<(ostream& stream, const TestToken& token) {
- return stream << token.data;
- }
- void TestSplitting() {
- assert(SplitIntoSentences(vector<TestToken>({{"Split"s}, {"into"s}, {"sentences"s}, {"!"s}}))
- == vector<Sentence<TestToken>>({{{"Split"s}, {"into"s}, {"sentences"s}, {"!"s}}}));
- assert(SplitIntoSentences(vector<TestToken>({{"Split"s}, {"into"s}, {"sentences"s}, {"!"s, true}}))
- == vector<Sentence<TestToken>>({{{"Split"s}, {"into"s}, {"sentences"s}, {"!"s, true}}}));
- assert(SplitIntoSentences(vector<TestToken>(
- {{"Split"s}, {"into"s}, {"sentences"s}, {"!"s, true}, {"!"s, true}, {"Without"s}, {"copies"s}, {"."s, true}}))
- == vector<Sentence<TestToken>>({
- {{"Split"s}, {"into"s}, {"sentences"s}, {"!"s, true}, {"!"s, true}},
- {{"Without"s}, {"copies"s}, {"."s, true}},
- }));
- }
- int main() {
- TestSplitting();
- }
Advertisement
Add Comment
Please, Sign In to add comment