The Tor Browser / file revision

 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.

 // Use of this source code is governed by a BSD-style license that can be

 // found in the LICENSE file.

 #ifndef BASE_STRING_TOKENIZER_H_

 #define BASE_STRING_TOKENIZER_H_

 #include <string>

 // StringTokenizerT is a simple string tokenizer class.  It works like an

 // iterator that with each step (see the Advance method) updates members that

 // refer to the next token in the input string.  The user may optionally

 // configure the tokenizer to return delimiters.

 //

 //

 // EXAMPLE 1:

 //

 //   StringTokenizer t("this is a test", " ");

 //   while (t.GetNext()) {

 //     printf("%s\n", t.token().c_str());

 //   }

 //

 // Output:

 //

 //   this

 //   is

 //   a

 //   test

 //

 //

 // EXAMPLE 2:

 //

 //   StringTokenizer t("no-cache=\"foo, bar\", private", ", ");

 //   t.set_quote_chars("\"");

 //   while (t.GetNext()) {

 //     printf("%s\n", t.token().c_str());

 //   }

 //

 // Output:

 //

 //   no-cache="foo, bar"

 //   private

 //

 //

 // EXAMPLE 3:

 //

 //   bool next_is_option = false, next_is_value = false;

 //   std::string input = "text/html; charset=UTF-8; foo=bar";

 //   StringTokenizer t(input, "; =");

 //   t.set_options(StringTokenizer::RETURN_DELIMS);

 //   while (t.GetNext()) {

 //     if (t.token_is_delim()) {

 //       switch (*t.token_begin()) {

 //         case ';':

 //           next_is_option = true;

 //           break;

 //         case '=':

 //           next_is_value = true;

 //           break;

 //       }

 //     } else {

 //       const char* label;

 //       if (next_is_option) {

 //         label = "option-name";

 //         next_is_option = false;

 //       } else if (next_is_value) {

 //         label = "option-value";

 //         next_is_value = false;

 //       } else {

 //         label = "mime-type";

 //       }

 //       printf("%s: %s\n", label, t.token().c_str());

 //     }

 //   }

 //

 //

 template <class str, class const_iterator>

 class StringTokenizerT {

  public:

   typedef typename str::value_type char_type;

   // Options that may be pass to set_options()

   enum {

     // Specifies the delimiters should be returned as tokens

     RETURN_DELIMS = 1 << 0,

   };

   StringTokenizerT(const str& string,

                    const str& delims) {

     Init(string.begin(), string.end(), delims);

   }

   StringTokenizerT(const_iterator string_begin,

                    const_iterator string_end,

                    const str& delims) {

     Init(string_begin, string_end, delims);

   }

   // Set the options for this tokenizer.  By default, this is 0.

   void set_options(int options) { options_ = options; }

   // Set the characters to regard as quotes.  By default, this is empty.  When

   // a quote char is encountered, the tokenizer will switch into a mode where

   // it ignores delimiters that it finds.  It switches out of this mode once it

   // finds another instance of the quote char.  If a backslash is encountered

   // within a quoted string, then the next character is skipped.

   void set_quote_chars(const str& quotes) { quotes_ = quotes; }

   // Call this method to advance the tokenizer to the next delimiter.  This

   // returns false if the tokenizer is complete.  This method must be called

   // before calling any of the token* methods.

   bool GetNext() {

     AdvanceState state;

     token_is_delim_ = false;

     for (;;) {

       token_begin_ = token_end_;

       if (token_end_ == end_)

         return false;

       ++token_end_;

       if (AdvanceOne(&state, *token_begin_))

         break;

       if (options_ & RETURN_DELIMS) {

         token_is_delim_ = true;

         return true;

       }

       // else skip over delim

     }

     while (token_end_ != end_ && AdvanceOne(&state, *token_end_))

       ++token_end_;

     return true;

   }

   // Returns true if token is a delimiter.  When the tokenizer is constructed

   // with the RETURN_DELIMS option, this method can be used to check if the

   // returned token is actually a delimiter.

   bool token_is_delim() const { return token_is_delim_; }

   // If GetNext() returned true, then these methods may be used to read the

   // value of the token.

   const_iterator token_begin() const { return token_begin_; }

   const_iterator token_end() const { return token_end_; }

   str token() const { return str(token_begin_, token_end_); }

  private:

   void Init(const_iterator string_begin,

             const_iterator string_end,

             const str& delims) {

     token_end_ = string_begin;

     end_ = string_end;

     delims_ = delims;

     options_ = 0;

   }

   bool IsDelim(char_type c) const {

     return delims_.find(c) != str::npos;

   }

   bool IsQuote(char_type c) const {

     return quotes_.find(c) != str::npos;

   }

   struct AdvanceState {

     bool in_quote;

     bool in_escape;

     char_type quote_char;

     AdvanceState() : in_quote(false), in_escape(false) {}

   };

   // Returns true if a delimiter was not hit.

   bool AdvanceOne(AdvanceState* state, char_type c) {

     if (state->in_quote) {

       if (state->in_escape) {

         state->in_escape = false;

       } else if (c == '\\') {

         state->in_escape = true;

       } else if (c == state->quote_char) {

         state->in_quote = false;

       }

     } else {

       if (IsDelim(c))

         return false;

       state->in_quote = IsQuote(state->quote_char = c);

     }

     return true;

   }

   const_iterator token_begin_;

   const_iterator token_end_;

   const_iterator end_;

   str delims_;

   str quotes_;

   int options_;

   bool token_is_delim_;

 };

 typedef StringTokenizerT<std::string, std::string::const_iterator>

     StringTokenizer;

 typedef StringTokenizerT<std::wstring, std::wstring::const_iterator>

     WStringTokenizer;

 typedef StringTokenizerT<std::string, const char*> CStringTokenizer;

 #endif  // BASE_STRING_TOKENIZER_H_