Advertisement
Guest User

String tokenizer using a string of multiple delimiters

a guest
Oct 12th, 2015
257
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
C++ 2.30 KB | None | 0 0
  1. #include <iostream>
  2. #include <fstream>
  3. #include <algorithm>
  4. #include <vector>
  5. #include <string>
  6.  
  7. constexpr auto str_end = std::string::npos;
  8.  
  9. using std::vector;
  10. using std::string;
  11. using std::ostream;
  12. using std::ifstream;
  13. using std::getline;
  14. using std::cout;
  15. using std::endl;
  16.  
  17. // stream vector elements out
  18. template <typename T>
  19. ostream& operator<<(ostream& os, const vector<T>& vec) {
  20.   for (const auto& elem : vec) os << elem << ' ';
  21.  
  22.   return os;
  23. }
  24.  
  25. // given a string with delimiters inside, parse it into
  26. //  individual tokens stored in a vector<string>
  27. void tokenize(const string& str, vector<string>& tokens,
  28.               const string& delimiters = " ") {
  29.   auto last_pos = str.find_first_not_of(delimiters, 0);      // first token
  30.   auto curr_pos = str.find_first_of(delimiters, last_pos);   // next delim
  31.  
  32.   while (curr_pos != str_end || last_pos != str_end) {
  33.     tokens.emplace_back(str.substr(last_pos, curr_pos - last_pos));    
  34.     last_pos = str.find_first_not_of(delimiters, curr_pos);  // next token
  35.     curr_pos = str.find_first_of(delimiters, last_pos);      // next delim
  36.   }
  37. }
  38.  
  39. /*
  40. // general wrapper ?
  41. // -exclude string T's somehow?
  42. template <typename T>
  43. void tokenize(const string& str, vector<T>& tokens,
  44.               const string& delimiters = " ") {
  45.   vector<string> str_tokens;
  46.   tokenize(str, str_tokens, delims);
  47.  
  48.   for (const auto& e : str_tokens)
  49.     tokens.emplace_back(  ??? (e));  // need T's type, then the right conversion    
  50. }
  51. */
  52.  
  53. // int wrapper
  54. void tokenize(const string& str, vector<int>& tokens,
  55.               const string& delimiters = " ") {
  56.   vector<string> str_tokens;
  57.   tokenize(str, str_tokens, delims);
  58.  
  59.   for (const auto& e : str_tokens)
  60.     tokens.emplace_back(stoi(e));  // ints    
  61. }
  62.  
  63. // double wrapper
  64. void tokenize(const string& str, vector<double>& tokens,
  65.               const string& delimiters = " ") {
  66.   vector<string> str_tokens;
  67.   tokenize(str, str_tokens, delims);
  68.  
  69.   for (const auto& e : str_tokens)
  70.     tokens.emplace_back(stod(e));  // doubles      
  71. }
  72.  
  73. int main() {
  74.   ifstream fs{"data"};
  75.   string tmp{""};
  76.   const string delims{"[,]"};
  77.   //vector<string> tokens;
  78.   //vector<int> tokens;
  79.   vector<double> tokens;
  80.  
  81.   while (getline(fs, tmp)) tokenize(tmp, tokens, delims);
  82.  
  83.   cout << tokens << endl;
  84. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement