I have read http://www.codeproject.com/KB/recipes/Tokenizer.aspx and I want to have the last example ( at the end, just before all the graphs) "Extending Delimiter Predicates" in my main, but I don't get the same output tokens as the author of the article does when I assign the token_list to a vector, why?
How to put the real result in a list or a vector? I want to have this :
- list0 abc
- list1 123, mno xyz
- list2 i\,jk
But I have something like :
- list0 abc;"123, mno xyz",i\,jk
- list1 123, mno xyz",i\,jk
- list2 i\,jk
Source sample :
class extended_predicate
{
public:
extended_predicate(const std::string& delimiters)
: escape_(false),
in_bracket_range_(false),
mdp_(delimiters)
{}
inline bool operator()(const unsigned char c) const
{
if (escape_)
{
escape_ = false;
return false;
}
else if ('\\' == c)
{
escape_ = true;
return false;
}
else if ('"' == c)
{
in_bracket_range_ = !in_bracket_range_;
return true;
}
else if (in_bracket_range_)
return false;
else
return mdp_(c);
}
inline void reset()
{
escape_ = false;
in_bracket_range_ = false;
}
private:
mutable bool escape_;
mutable bool in_bracket_range_;
mutable strtk::multiple_char_delimiter_predicate mdp_;
};
int main()
{
std::string str = "abc;\"123, mno xyz\",i\\,jk";
strtk::std_string::token_list_type token_list;
strtk::split(extended_predicate(".,; "),
str,
std::back_inserter(token_list),
strtk::split_options::compress_delimiters);
return 0;
}
