public PatternTokenizer(String str,
    Pattern pattern,
    boolean toLowerCase) 
{
    
      this.str = str;
      this.matcher = pattern.matcher(str);
      this.toLowerCase = toLowerCase;
}

 public Token next() 
{
      if (matcher == null) return null;
      while (true) { // loop takes care of leading and trailing boundary cases
        int start = pos;
        int end;
        boolean isMatch = matcher.find();
        if (isMatch) {
          end = matcher.start();
          pos = matcher.end();
        } else { 
          end = str.length();
          matcher = null; // we're finished
        }
        if (start != end) { // non-empty match (header/trailer)
          String text = str.substring(start, end);
          if (toLowerCase) text = text.toLowerCase(locale);
          return new Token(text, start, end);
        }
        if (!isMatch) return null;
      }
}