michael@0: // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
michael@0: // Use of this source code is governed by a BSD-style license that can be
michael@0: // found in the LICENSE file.
michael@0: 
michael@0: #ifndef BASE_STRING_TOKENIZER_H_
michael@0: #define BASE_STRING_TOKENIZER_H_
michael@0: 
michael@0: #include <string>
michael@0: 
michael@0: // StringTokenizerT is a simple string tokenizer class.  It works like an
michael@0: // iterator that with each step (see the Advance method) updates members that
michael@0: // refer to the next token in the input string.  The user may optionally
michael@0: // configure the tokenizer to return delimiters.
michael@0: //
michael@0: //
michael@0: // EXAMPLE 1:
michael@0: //
michael@0: //   StringTokenizer t("this is a test", " ");
michael@0: //   while (t.GetNext()) {
michael@0: //     printf("%s\n", t.token().c_str());
michael@0: //   }
michael@0: //
michael@0: // Output:
michael@0: //
michael@0: //   this
michael@0: //   is
michael@0: //   a
michael@0: //   test
michael@0: //
michael@0: //
michael@0: // EXAMPLE 2:
michael@0: //
michael@0: //   StringTokenizer t("no-cache=\"foo, bar\", private", ", ");
michael@0: //   t.set_quote_chars("\"");
michael@0: //   while (t.GetNext()) {
michael@0: //     printf("%s\n", t.token().c_str());
michael@0: //   }
michael@0: //
michael@0: // Output:
michael@0: //
michael@0: //   no-cache="foo, bar"
michael@0: //   private
michael@0: //
michael@0: //
michael@0: // EXAMPLE 3:
michael@0: //
michael@0: //   bool next_is_option = false, next_is_value = false;
michael@0: //   std::string input = "text/html; charset=UTF-8; foo=bar";
michael@0: //   StringTokenizer t(input, "; =");
michael@0: //   t.set_options(StringTokenizer::RETURN_DELIMS);
michael@0: //   while (t.GetNext()) {
michael@0: //     if (t.token_is_delim()) {
michael@0: //       switch (*t.token_begin()) {
michael@0: //         case ';':
michael@0: //           next_is_option = true;
michael@0: //           break;
michael@0: //         case '=':
michael@0: //           next_is_value = true;
michael@0: //           break;
michael@0: //       }
michael@0: //     } else {
michael@0: //       const char* label;
michael@0: //       if (next_is_option) {
michael@0: //         label = "option-name";
michael@0: //         next_is_option = false;
michael@0: //       } else if (next_is_value) {
michael@0: //         label = "option-value";
michael@0: //         next_is_value = false;
michael@0: //       } else {
michael@0: //         label = "mime-type";
michael@0: //       }
michael@0: //       printf("%s: %s\n", label, t.token().c_str());
michael@0: //     }
michael@0: //   }
michael@0: //
michael@0: //
michael@0: template <class str, class const_iterator>
michael@0: class StringTokenizerT {
michael@0:  public:
michael@0:   typedef typename str::value_type char_type;
michael@0: 
michael@0:   // Options that may be pass to set_options()
michael@0:   enum {
michael@0:     // Specifies the delimiters should be returned as tokens
michael@0:     RETURN_DELIMS = 1 << 0,
michael@0:   };
michael@0: 
michael@0:   StringTokenizerT(const str& string,
michael@0:                    const str& delims) {
michael@0:     Init(string.begin(), string.end(), delims);
michael@0:   }
michael@0: 
michael@0:   StringTokenizerT(const_iterator string_begin,
michael@0:                    const_iterator string_end,
michael@0:                    const str& delims) {
michael@0:     Init(string_begin, string_end, delims);
michael@0:   }
michael@0: 
michael@0:   // Set the options for this tokenizer.  By default, this is 0.
michael@0:   void set_options(int options) { options_ = options; }
michael@0: 
michael@0:   // Set the characters to regard as quotes.  By default, this is empty.  When
michael@0:   // a quote char is encountered, the tokenizer will switch into a mode where
michael@0:   // it ignores delimiters that it finds.  It switches out of this mode once it
michael@0:   // finds another instance of the quote char.  If a backslash is encountered
michael@0:   // within a quoted string, then the next character is skipped.
michael@0:   void set_quote_chars(const str& quotes) { quotes_ = quotes; }
michael@0: 
michael@0:   // Call this method to advance the tokenizer to the next delimiter.  This
michael@0:   // returns false if the tokenizer is complete.  This method must be called
michael@0:   // before calling any of the token* methods.
michael@0:   bool GetNext() {
michael@0:     AdvanceState state;
michael@0:     token_is_delim_ = false;
michael@0:     for (;;) {
michael@0:       token_begin_ = token_end_;
michael@0:       if (token_end_ == end_)
michael@0:         return false;
michael@0:       ++token_end_;
michael@0:       if (AdvanceOne(&state, *token_begin_))
michael@0:         break;
michael@0:       if (options_ & RETURN_DELIMS) {
michael@0:         token_is_delim_ = true;
michael@0:         return true;
michael@0:       }
michael@0:       // else skip over delim
michael@0:     }
michael@0:     while (token_end_ != end_ && AdvanceOne(&state, *token_end_))
michael@0:       ++token_end_;
michael@0:     return true;
michael@0:   }
michael@0: 
michael@0:   // Returns true if token is a delimiter.  When the tokenizer is constructed
michael@0:   // with the RETURN_DELIMS option, this method can be used to check if the
michael@0:   // returned token is actually a delimiter.
michael@0:   bool token_is_delim() const { return token_is_delim_; }
michael@0: 
michael@0:   // If GetNext() returned true, then these methods may be used to read the
michael@0:   // value of the token.
michael@0:   const_iterator token_begin() const { return token_begin_; }
michael@0:   const_iterator token_end() const { return token_end_; }
michael@0:   str token() const { return str(token_begin_, token_end_); }
michael@0: 
michael@0:  private:
michael@0:   void Init(const_iterator string_begin,
michael@0:             const_iterator string_end,
michael@0:             const str& delims) {
michael@0:     token_end_ = string_begin;
michael@0:     end_ = string_end;
michael@0:     delims_ = delims;
michael@0:     options_ = 0;
michael@0:   }
michael@0: 
michael@0:   bool IsDelim(char_type c) const {
michael@0:     return delims_.find(c) != str::npos;
michael@0:   }
michael@0: 
michael@0:   bool IsQuote(char_type c) const {
michael@0:     return quotes_.find(c) != str::npos;
michael@0:   }
michael@0: 
michael@0:   struct AdvanceState {
michael@0:     bool in_quote;
michael@0:     bool in_escape;
michael@0:     char_type quote_char;
michael@0:     AdvanceState() : in_quote(false), in_escape(false) {}
michael@0:   };
michael@0: 
michael@0:   // Returns true if a delimiter was not hit.
michael@0:   bool AdvanceOne(AdvanceState* state, char_type c) {
michael@0:     if (state->in_quote) {
michael@0:       if (state->in_escape) {
michael@0:         state->in_escape = false;
michael@0:       } else if (c == '\\') {
michael@0:         state->in_escape = true;
michael@0:       } else if (c == state->quote_char) {
michael@0:         state->in_quote = false;
michael@0:       }
michael@0:     } else {
michael@0:       if (IsDelim(c))
michael@0:         return false;
michael@0:       state->in_quote = IsQuote(state->quote_char = c);
michael@0:     }
michael@0:     return true;
michael@0:   }
michael@0: 
michael@0:   const_iterator token_begin_;
michael@0:   const_iterator token_end_;
michael@0:   const_iterator end_;
michael@0:   str delims_;
michael@0:   str quotes_;
michael@0:   int options_;
michael@0:   bool token_is_delim_;
michael@0: };
michael@0: 
michael@0: typedef StringTokenizerT<std::string, std::string::const_iterator>
michael@0:     StringTokenizer;
michael@0: typedef StringTokenizerT<std::wstring, std::wstring::const_iterator>
michael@0:     WStringTokenizer;
michael@0: typedef StringTokenizerT<std::string, const char*> CStringTokenizer;
michael@0: 
michael@0: #endif  // BASE_STRING_TOKENIZER_H_