00001
00002
00003
00004
00005
00006
00007
00008
00009 #ifndef TOKENIZER_H
00010 #define TOKENIZER_H
00011
00012 #include <functional>
00013 #include <string>
00014 #include <vector>
00015 #include <algorithm>
00016
00017 using namespace std;
00018
00019 class CSpace:public unary_function < char, bool >
00020 {
00021 public:
00022 bool operator () (char c) const;
00023 };
00024
00025 bool CSpace::operator()(char c) const
00026 {
00027 return (' ' == c);
00028 }
00029
00030 class CSeparator:public unary_function < char, bool >
00031 {
00032 public:
00033 CSeparator::CSeparator (string const &rostr):m_ostr (rostr)
00034 {
00035 }
00036 bool operator () (char c) const;
00037
00038 private:
00039 string m_ostr;
00040 };
00041
00042 bool CSeparator::operator()(char c) const
00043 {
00044 unsigned int iFind = m_ostr.find(c);
00045 if(iFind != string::npos)
00046 return true;
00047 else
00048 return false;
00049 }
00050
00051 template < typename Pred = CSpace > class CTokenizer
00052 {
00053 public:
00054 static void Tokenize (vector < string > &roResult,
00055 string const &rostr, Pred const &roPred =
00056 Pred ());
00057 };
00058
00059 template < typename Pred >
00060 void CTokenizer < Pred >::Tokenize (vector < string > &roResult,
00061 string const &rostr,
00062 Pred const &roPred)
00063 {
00064
00065 roResult.clear ();
00066 string::const_iterator it = rostr.begin ();
00067 string::const_iterator itTokenEnd = rostr.begin ();
00068 while (it != rostr.end ())
00069 {
00070
00071 while (roPred (*it))
00072 it++;
00073
00074 itTokenEnd = find_if (it, rostr.end (), roPred);
00075
00076 if (it < itTokenEnd)
00077 roResult.push_back (string (it, itTokenEnd));
00078 it = itTokenEnd;
00079 }
00080 }
00081
00082 #endif