Source code: com/port80/eclipse/util/JavaCodeScanner.java

   /**
    * Modified from org.eclipse.jdt.internal.compiler.parser.Scanner.java
    * . Added token for '#'.
    */
   package com.port80.eclipse.util;
   
   import java.util.ArrayList;
   import java.util.Iterator;
   import java.util.List;
  
  import org.eclipse.jdt.core.compiler.IScanner;
  import org.eclipse.jdt.core.compiler.InvalidInputException;
  import org.eclipse.jdt.internal.compiler.ast.StringLiteral;
  import org.eclipse.jdt.internal.compiler.parser.NLSLine;
  
  public class JavaCodeScanner implements IScanner, IJavaCodeSymbol {
  
    /* APIs ares
     - getNextToken() which return the current type of the token
       (this value is not memorized by the scanner)
     - getCurrentTokenSource() which provides with the token "REAL" source
       (aka all unicode have been transformed into a correct char)
     - sourceStart gives the position into the stream
     - currentPosition-1 gives the sourceEnd position into the stream 
    */
  
    // 1.4 feature 
    private boolean assertMode;
    public boolean useAssertAsAnIndentifier = false;
    //flag indicating if processed source contains occurrences of keyword assert 
    public boolean containsAssertKeyword = false;
  
    public boolean recordLineSeparator;
    public char currentCharacter;
    public int startPosition;
    public int currentPosition;
    public int initialPosition, eofPosition;
    // after this position eof are generated instead of real token from the source
  
    public boolean tokenizeComments;
    public boolean tokenizeWhiteSpace;
  
    //source should be viewed as a window (aka a part)
    //of a entire very large stream
    public char source[];
  
    //unicode support
    public char[] withoutUnicodeBuffer;
    public int withoutUnicodePtr; //when == 0 ==> no unicode in the current token
    public boolean unicodeAsBackSlash = false;
  
    public boolean scanningFloatLiteral = false;
  
    //support for /** comments
    //public char[][] comments = new char[10][];
    public int[] commentStops = new int[10];
    public int[] commentStarts = new int[10];
    public int commentPtr = -1; // no comment test with commentPtr value -1
  
    //diet parsing support - jump over some method body when requested
    public boolean diet = false;
  
    //support for the  poor-line-debuggers ....
    //remember the position of the cr/lf
    public int[] lineEnds = new int[250];
    public int linePtr = -1;
    public boolean wasAcr = false;
  
    public static final String END_OF_SOURCE = "End_Of_Source"; //$NON-NLS-1$
  
    public static final String INVALID_HEXA = "Invalid_Hexa_Literal"; //$NON-NLS-1$
    public static final String INVALID_OCTAL = "Invalid_Octal_Literal"; //$NON-NLS-1$
    public static final String INVALID_CHARACTER_CONSTANT = "Invalid_Character_Constant"; //$NON-NLS-1$
    public static final String INVALID_ESCAPE = "Invalid_Escape"; //$NON-NLS-1$
    public static final String INVALID_INPUT = "Invalid_Input"; //$NON-NLS-1$
    public static final String INVALID_UNICODE_ESCAPE = "Invalid_Unicode_Escape"; //$NON-NLS-1$
    public static final String INVALID_FLOAT = "Invalid_Float_Literal"; //$NON-NLS-1$
  
    public static final String NULL_SOURCE_STRING = "Null_Source_String"; //$NON-NLS-1$
    public static final String UNTERMINATED_STRING = "Unterminated_String"; //$NON-NLS-1$
    public static final String UNTERMINATED_COMMENT = "Unterminated_Comment"; //$NON-NLS-1$
    public static final String INVALID_CHAR_IN_STRING = "Invalid_Char_In_String"; //$NON-NLS-1$
  
    //----------------optimized identifier managment------------------
    static final char[] charArray_a = new char[] { 'a' },
      charArray_b = new char[] { 'b' },
      charArray_c = new char[] { 'c' },
      charArray_d = new char[] { 'd' },
      charArray_e = new char[] { 'e' },
      charArray_f = new char[] { 'f' },
      charArray_g = new char[] { 'g' },
      charArray_h = new char[] { 'h' },
      charArray_i = new char[] { 'i' },
      charArray_j = new char[] { 'j' },
      charArray_k = new char[] { 'k' },
      charArray_l = new char[] { 'l' },
      charArray_m = new char[] { 'm' },
      charArray_n = new char[] { 'n' },
      charArray_o = new char[] { 'o' },
     charArray_p = new char[] { 'p' },
     charArray_q = new char[] { 'q' },
     charArray_r = new char[] { 'r' },
     charArray_s = new char[] { 's' },
     charArray_t = new char[] { 't' },
     charArray_u = new char[] { 'u' },
     charArray_v = new char[] { 'v' },
     charArray_w = new char[] { 'w' },
     charArray_x = new char[] { 'x' },
     charArray_y = new char[] { 'y' },
     charArray_z = new char[] { 'z' };
 
   static final char[] initCharArray = new char[] { '\u0000', '\u0000', '\u0000', '\u0000', '\u0000', '\u0000' };
   static final int TableSize = 30, InternalTableSize = 6; //30*6 = 180 entries
   public static final int OptimizedLength = 6;
   public /*static*/
   final char[][][][] charArray_length = new char[OptimizedLength][TableSize][InternalTableSize][];
   // support for detecting non-externalized string literals
   int currentLineNr = -1;
   int previousLineNr = -1;
   NLSLine currentLine = null;
   List lines = new ArrayList();
   public static final String TAG_PREFIX = "//$NON-NLS-"; //$NON-NLS-1$
   public static final int TAG_PREFIX_LENGTH = TAG_PREFIX.length();
   public static final String TAG_POSTFIX = "$"; //$NON-NLS-1$
   public static final int TAG_POSTFIX_LENGTH = TAG_POSTFIX.length();
   public StringLiteral[] nonNLSStrings = null;
   public boolean checkNonExternalizedStringLiterals = true;
   public boolean wasNonExternalizedStringLiteral = false;
 
   /*static*/ {
     for (int i = 0; i < 6; i++) {
       for (int j = 0; j < TableSize; j++) {
         for (int k = 0; k < InternalTableSize; k++) {
           charArray_length[i][j][k] = initCharArray;
         }
       }
     }
   }
   static int newEntry2 = 0, newEntry3 = 0, newEntry4 = 0, newEntry5 = 0, newEntry6 = 0;
 
   public static final int RoundBracket = 0;
   public static final int SquareBracket = 1;
   public static final int CurlyBracket = 2;
   public static final int BracketKinds = 3;
   public JavaCodeScanner() {
     this(false, false);
   }
   public JavaCodeScanner(boolean tokenizeComments, boolean tokenizeWhiteSpace) {
     this(tokenizeComments, tokenizeWhiteSpace, false);
   }
   public final boolean atEnd() {
     // This code is not relevant if source is 
     // Only a part of the real stream input
 
     return source.length == currentPosition;
   }
   public char[] getCurrentIdentifierSource() {
     //return the token REAL source (aka unicodes are precomputed)
 
     char[] result;
     if (withoutUnicodePtr != 0)
       //0 is used as a fast test flag so the real first char is in position 1
       System.arraycopy(
         withoutUnicodeBuffer,
         1,
         result = new char[withoutUnicodePtr],
         0,
         withoutUnicodePtr);
     else {
       int length = currentPosition - startPosition;
       switch (length) { // see OptimizedLength
         case 1 :
           return optimizedCurrentTokenSource1();
         case 2 :
           return optimizedCurrentTokenSource2();
         case 3 :
           return optimizedCurrentTokenSource3();
         case 4 :
           return optimizedCurrentTokenSource4();
         case 5 :
           return optimizedCurrentTokenSource5();
         case 6 :
           return optimizedCurrentTokenSource6();
       }
       //no optimization
       System.arraycopy(source, startPosition, result = new char[length], 0, length);
     }
     return result;
   }
   public int getCurrentTokenEndPosition() {
     return this.currentPosition - 1;
   }
   public final char[] getCurrentTokenSource() {
     // Return the token REAL source (aka unicodes are precomputed)
 
     char[] result;
     if (withoutUnicodePtr != 0)
       // 0 is used as a fast test flag so the real first char is in position 1
       System.arraycopy(
         withoutUnicodeBuffer,
         1,
         result = new char[withoutUnicodePtr],
         0,
         withoutUnicodePtr);
     else {
       int length;
       System.arraycopy(
         source,
         startPosition,
         result = new char[length = currentPosition - startPosition],
         0,
         length);
     }
     return result;
   }
   public final char[] getCurrentTokenSourceString() {
     //return the token REAL source (aka unicodes are precomputed).
     //REMOVE the two " that are at the beginning and the end.
 
     char[] result;
     if (withoutUnicodePtr != 0)
       //0 is used as a fast test flag so the real first char is in position 1
       System.arraycopy(withoutUnicodeBuffer, 2,
       //2 is 1 (real start) + 1 (to jump over the ")
       result = new char[withoutUnicodePtr - 2], 0, withoutUnicodePtr - 2);
     else {
       int length;
       System.arraycopy(
         source,
         startPosition + 1,
         result = new char[length = currentPosition - startPosition - 2],
         0,
         length);
     }
     return result;
   }
   public int getCurrentTokenStartPosition() {
     return this.startPosition;
   }
   /*
    * Search the source position corresponding to the end of a given line number
    *
    * Line numbers are 1-based, and relative to the scanner initialPosition. 
    * Character positions are 0-based.
    *
    * In case the given line number is inconsistent, answers -1.
    */
   public final int getLineEnd(int lineNumber) {
 
     if (lineEnds == null)
       return -1;
     if (lineNumber >= lineEnds.length)
       return -1;
     if (lineNumber <= 0)
       return -1;
 
     if (lineNumber == lineEnds.length - 1)
       return eofPosition;
     return lineEnds[lineNumber - 1];
     // next line start one character behind the lineEnd of the previous line
   }
   /**
    * Search the source position corresponding to the beginning of a given line number
    *
    * Line numbers are 1-based, and relative to the scanner initialPosition. 
    * Character positions are 0-based.
    *
    * e.g.  getLineStart(1) --> 0  i.e. first line starts at character 0.
    *
    * In case the given line number is inconsistent, answers -1.
    */
   public final int getLineStart(int lineNumber) {
 
     if (lineEnds == null)
       return -1;
     if (lineNumber >= lineEnds.length)
       return -1;
     if (lineNumber <= 0)
       return -1;
 
     if (lineNumber == 1)
       return initialPosition;
     return lineEnds[lineNumber - 2] + 1;
     // next line start one character behind the lineEnd of the previous line
   }
   public final boolean getNextChar(char testedChar) {
     //BOOLEAN
     //handle the case of unicode.
     //when a unicode appears then we must use a buffer that holds char internal values
     //At the end of this method currentCharacter holds the new visited char
     //and currentPosition points right next after it
     //Both previous lines are true if the currentCharacter is == to the testedChar
     //On false, no side effect has occured.
 
     //ALL getNextChar.... ARE OPTIMIZED COPIES 
 
     int temp = currentPosition;
     try {
       if (((currentCharacter = source[currentPosition++]) == '\\')
         && (source[currentPosition] == 'u')) {
         //-------------unicode traitement ------------
         int c1, c2, c3, c4;
         int unicodeSize = 6;
         currentPosition++;
         while (source[currentPosition] == 'u') {
           currentPosition++;
           unicodeSize++;
         }
 
         if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15 || c1 < 0)
           || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15 || c2 < 0)
           || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15 || c3 < 0)
           || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15 || c4 < 0)) {
           currentPosition = temp;
           return false;
         }
 
         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
         if (currentCharacter != testedChar) {
           currentPosition = temp;
           return false;
         }
         unicodeAsBackSlash = currentCharacter == '\\';
 
         //need the unicode buffer
         if (withoutUnicodePtr == 0) {
           //buffer all the entries that have been left aside....
           withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
           System.arraycopy(
             source,
             startPosition,
             withoutUnicodeBuffer,
             1,
             withoutUnicodePtr);
         }
         //fill the buffer with the char
         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
         return true;
 
       } //-------------end unicode traitement--------------
       else {
         if (currentCharacter != testedChar) {
           currentPosition = temp;
           return false;
         }
         unicodeAsBackSlash = false;
         if (withoutUnicodePtr != 0)
           withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
         return true;
       }
     } catch (IndexOutOfBoundsException e) {
       unicodeAsBackSlash = false;
       currentPosition = temp;
       return false;
     }
   }
   public final int getNextChar(char testedChar1, char testedChar2) {
     //INT 0 : testChar1 \\\\///\\\\ 1 : testedChar2 \\\\///\\\\ -1 : others
     //test can be done with (x==0) for the first and (x>0) for the second
     //handle the case of unicode.
     //when a unicode appears then we must use a buffer that holds char internal values
     //At the end of this method currentCharacter holds the new visited char
     //and currentPosition points right next after it
     //Both previous lines are true if the currentCharacter is == to the testedChar1/2
     //On false, no side effect has occured.
 
     //ALL getNextChar.... ARE OPTIMIZED COPIES 
 
     int temp = currentPosition;
     try {
       int result;
       if (((currentCharacter = source[currentPosition++]) == '\\')
         && (source[currentPosition] == 'u')) {
         //-------------unicode traitement ------------
         int c1, c2, c3, c4;
         int unicodeSize = 6;
         currentPosition++;
         while (source[currentPosition] == 'u') {
           currentPosition++;
           unicodeSize++;
         }
 
         if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15 || c1 < 0)
           || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15 || c2 < 0)
           || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15 || c3 < 0)
           || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15 || c4 < 0)) {
           currentPosition = temp;
           return 2;
         }
 
         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
         if (currentCharacter == testedChar1)
           result = 0;
         else if (currentCharacter == testedChar2)
           result = 1;
         else {
           currentPosition = temp;
           return -1;
         }
 
         //need the unicode buffer
         if (withoutUnicodePtr == 0) {
           //buffer all the entries that have been left aside....
           withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
           System.arraycopy(
             source,
             startPosition,
             withoutUnicodeBuffer,
             1,
             withoutUnicodePtr);
         }
         //fill the buffer with the char
         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
         return result;
       } //-------------end unicode traitement--------------
       else {
         if (currentCharacter == testedChar1)
           result = 0;
         else if (currentCharacter == testedChar2)
           result = 1;
         else {
           currentPosition = temp;
           return -1;
         }
 
         if (withoutUnicodePtr != 0)
           withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
         return result;
       }
     } catch (IndexOutOfBoundsException e) {
       currentPosition = temp;
       return -1;
     }
   }
   public final boolean getNextCharAsDigit() {
     //BOOLEAN
     //handle the case of unicode.
     //when a unicode appears then we must use a buffer that holds char internal values
     //At the end of this method currentCharacter holds the new visited char
     //and currentPosition points right next after it
     //Both previous lines are true if the currentCharacter is a digit
     //On false, no side effect has occured.
 
     //ALL getNextChar.... ARE OPTIMIZED COPIES 
 
     int temp = currentPosition;
     try {
       if (((currentCharacter = source[currentPosition++]) == '\\')
         && (source[currentPosition] == 'u')) {
         //-------------unicode traitement ------------
         int c1, c2, c3, c4;
         int unicodeSize = 6;
         currentPosition++;
         while (source[currentPosition] == 'u') {
           currentPosition++;
           unicodeSize++;
         }
 
         if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15 || c1 < 0)
           || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15 || c2 < 0)
           || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15 || c3 < 0)
           || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15 || c4 < 0)) {
           currentPosition = temp;
           return false;
         }
 
         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
         if (!Character.isDigit(currentCharacter)) {
           currentPosition = temp;
           return false;
         }
 
         //need the unicode buffer
         if (withoutUnicodePtr == 0) {
           //buffer all the entries that have been left aside....
           withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
           System.arraycopy(
             source,
             startPosition,
             withoutUnicodeBuffer,
             1,
             withoutUnicodePtr);
         }
         //fill the buffer with the char
         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
         return true;
       } //-------------end unicode traitement--------------
       else {
         if (!Character.isDigit(currentCharacter)) {
           currentPosition = temp;
           return false;
         }
         if (withoutUnicodePtr != 0)
           withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
         return true;
       }
     } catch (IndexOutOfBoundsException e) {
       currentPosition = temp;
       return false;
     }
   }
   public final boolean getNextCharAsDigit(int radix) {
     //BOOLEAN
     //handle the case of unicode.
     //when a unicode appears then we must use a buffer that holds char internal values
     //At the end of this method currentCharacter holds the new visited char
     //and currentPosition points right next after it
     //Both previous lines are true if the currentCharacter is a digit base on radix
     //On false, no side effect has occured.
 
     //ALL getNextChar.... ARE OPTIMIZED COPIES 
 
     int temp = currentPosition;
     try {
       if (((currentCharacter = source[currentPosition++]) == '\\')
         && (source[currentPosition] == 'u')) {
         //-------------unicode traitement ------------
         int c1, c2, c3, c4;
         int unicodeSize = 6;
         currentPosition++;
         while (source[currentPosition] == 'u') {
           currentPosition++;
           unicodeSize++;
         }
 
         if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15 || c1 < 0)
           || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15 || c2 < 0)
           || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15 || c3 < 0)
           || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15 || c4 < 0)) {
           currentPosition = temp;
           return false;
         }
 
         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
         if (Character.digit(currentCharacter, radix) == -1) {
           currentPosition = temp;
           return false;
         }
 
         //need the unicode buffer
         if (withoutUnicodePtr == 0) {
           //buffer all the entries that have been left aside....
           withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
           System.arraycopy(
             source,
             startPosition,
             withoutUnicodeBuffer,
             1,
             withoutUnicodePtr);
         }
         //fill the buffer with the char
         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
         return true;
       } //-------------end unicode traitement--------------
       else {
         if (Character.digit(currentCharacter, radix) == -1) {
           currentPosition = temp;
           return false;
         }
         if (withoutUnicodePtr != 0)
           withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
         return true;
       }
     } catch (IndexOutOfBoundsException e) {
       currentPosition = temp;
       return false;
     }
   }
   public boolean getNextCharAsJavaIdentifierPart() {
     //BOOLEAN
     //handle the case of unicode.
     //when a unicode appears then we must use a buffer that holds char internal values
     //At the end of this method currentCharacter holds the new visited char
     //and currentPosition points right next after it
     //Both previous lines are true if the currentCharacter is a JavaIdentifierPart
     //On false, no side effect has occured.
 
     //ALL getNextChar.... ARE OPTIMIZED COPIES 
 
     int temp = currentPosition;
     try {
       if (((currentCharacter = source[currentPosition++]) == '\\')
         && (source[currentPosition] == 'u')) {
         //-------------unicode traitement ------------
         int c1, c2, c3, c4;
         int unicodeSize = 6;
         currentPosition++;
         while (source[currentPosition] == 'u') {
           currentPosition++;
           unicodeSize++;
         }
 
         if (((c1 = Character.getNumericValue(source[currentPosition++])) > 15 || c1 < 0)
           || ((c2 = Character.getNumericValue(source[currentPosition++])) > 15 || c2 < 0)
           || ((c3 = Character.getNumericValue(source[currentPosition++])) > 15 || c3 < 0)
           || ((c4 = Character.getNumericValue(source[currentPosition++])) > 15 || c4 < 0)) {
           currentPosition = temp;
           return false;
         }
 
         currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
         if (!Character.isJavaIdentifierPart(currentCharacter)) {
           currentPosition = temp;
           return false;
         }
 
         //need the unicode buffer
         if (withoutUnicodePtr == 0) {
           //buffer all the entries that have been left aside....
           withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
           System.arraycopy(
             source,
             startPosition,
             withoutUnicodeBuffer,
             1,
             withoutUnicodePtr);
         }
         //fill the buffer with the char
         withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
         return true;
       } //-------------end unicode traitement--------------
       else {
         if (!Character.isJavaIdentifierPart(currentCharacter)) {
           currentPosition = temp;
           return false;
         }
 
         if (withoutUnicodePtr != 0)
           withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
         return true;
       }
     } catch (IndexOutOfBoundsException e) {
       currentPosition = temp;
       return false;
     }
   }
   public int getNextToken() throws InvalidInputException {
 
     this.wasAcr = false;
     if (diet) {
       jumpOverMethodBody();
       diet = false;
       return currentPosition > source.length ? TokenNameEOF : TokenNameRBRACE;
     }
     try {
       while (true) { //loop for jumping over comments
         withoutUnicodePtr = 0;
         //start with a new token (even comment written with unicode )
 
         // ---------Consume white space and handles startPosition---------
         int whiteStart = currentPosition;
         boolean isWhiteSpace;
         do {
           startPosition = currentPosition;
           if (((currentCharacter = source[currentPosition++]) == '\\')
             && (source[currentPosition] == 'u')) {
             isWhiteSpace = jumpOverUnicodeWhiteSpace();
           } else {
             if ((currentCharacter == '\r') || (currentCharacter == '\n')) {
               checkNonExternalizeString();
               if (recordLineSeparator) {
                 pushLineSeparator();
               } else {
                 currentLine = null;
               }
             }
             isWhiteSpace =
               (currentCharacter == ' ')
                 || Character.isWhitespace(currentCharacter);
           }
         } while (isWhiteSpace);
         if (tokenizeWhiteSpace && (whiteStart != currentPosition - 1)) {
           // reposition scanner in case we are interested by spaces as tokens
           currentPosition--;
           startPosition = whiteStart;
           return TokenNameWHITESPACE;
         }
         //little trick to get out in the middle of a source compuation
         if (currentPosition > eofPosition)
           return TokenNameEOF;
 
         // ---------Identify the next token-------------
 
         switch (currentCharacter) {
           case '(' :
             return TokenNameLPAREN;
           case ')' :
             return TokenNameRPAREN;
           case '{' :
             return TokenNameLBRACE;
           case '}' :
             return TokenNameRBRACE;
           case '[' :
             return TokenNameLBRACKET;
           case ']' :
             return TokenNameRBRACKET;
           case ';' :
             return TokenNameSEMICOLON;
           case ',' :
             return TokenNameCOMMA;
           case '.' :
             if (getNextCharAsDigit())
               return scanNumber(true);
             return TokenNameDOT;
           case '+' :
             {
               int test;
               if ((test = getNextChar('+', '=')) == 0)
                 return TokenNamePLUS_PLUS;
               if (test > 0)
                 return TokenNamePLUS_EQUAL;
               return TokenNamePLUS;
             }
           case '-' :
             {
               int test;
               if ((test = getNextChar('-', '=')) == 0)
                 return TokenNameMINUS_MINUS;
               if (test > 0)
                 return TokenNameMINUS_EQUAL;
               return TokenNameMINUS;
             }
           case '~' :
             return TokenNameTWIDDLE;
           case '!' :
             if (getNextChar('='))
               return TokenNameNOT_EQUAL;
             return TokenNameNOT;
           case '*' :
             if (getNextChar('='))
               return TokenNameMULTIPLY_EQUAL;
             return TokenNameMULTIPLY;
           case '%' :
             if (getNextChar('='))
               return TokenNameREMAINDER_EQUAL;
             return TokenNameREMAINDER;
           case '<' :
             {
               int test;
               if ((test = getNextChar('=', '<')) == 0)
                 return TokenNameLESS_EQUAL;
               if (test > 0) {
                 if (getNextChar('='))
                   return TokenNameLEFT_SHIFT_EQUAL;
                 return TokenNameLEFT_SHIFT;
               }
               return TokenNameLESS;
             }
           case '>' :
             {
               int test;
               if ((test = getNextChar('=', '>')) == 0)
                 return TokenNameGREATER_EQUAL;
               if (test > 0) {
                 if ((test = getNextChar('=', '>')) == 0)
                   return TokenNameRIGHT_SHIFT_EQUAL;
                 if (test > 0) {
                   if (getNextChar('='))
                     return TokenNameUNSIGNED_RIGHT_SHIFT_EQUAL;
                   return TokenNameUNSIGNED_RIGHT_SHIFT;
                 }
                 return TokenNameRIGHT_SHIFT;
               }
               return TokenNameGREATER;
             }
           case '=' :
             if (getNextChar('='))
               return TokenNameEQUAL_EQUAL;
             return TokenNameEQUAL;
           case '&' :
             {
               int test;
               if ((test = getNextChar('&', '=')) == 0)
                 return TokenNameAND_AND;
               if (test > 0)
                 return TokenNameAND_EQUAL;
               return TokenNameAND;
             }
           case '|' :
             {
               int test;
               if ((test = getNextChar('|', '=')) == 0)
                 return TokenNameOR_OR;
               if (test > 0)
                 return TokenNameOR_EQUAL;
               return TokenNameOR;
             }
           case '^' :
             if (getNextChar('='))
               return TokenNameXOR_EQUAL;
             return TokenNameXOR;
           case '?' :
             return TokenNameQUESTION;
           case ':' :
             return TokenNameCOLON;
           case '\'' :
             {
               int test;
               if ((test = getNextChar('\n', '\r')) == 0) {
                 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
               }
               if (test > 0) {
                 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
                 for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
                   if (currentPosition + lookAhead
                     == source.length)
                     break;
                   if (source[currentPosition + lookAhead]
                     == '\n')
                     break;
                   if (source[currentPosition + lookAhead]
                     == '\'') {
                     currentPosition += lookAhead + 1;
                     break;
                   }
                 }
                 throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
               }
             }
             if (getNextChar('\'')) {
               // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
               for (int lookAhead = 0; lookAhead < 3; lookAhead++) {
                 if (currentPosition + lookAhead == source.length)
                   break;
                 if (source[currentPosition + lookAhead] == '\n')
                   break;
                 if (source[currentPosition + lookAhead] == '\'') {
                   currentPosition += lookAhead + 1;
                   break;
                 }
               }
               throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
             }
             if (getNextChar('\\'))
               scanEscapeCharacter();
             else { // consume next character
               unicodeAsBackSlash = false;
               if (((currentCharacter = source[currentPosition++]) == '\\')
                 && (source[currentPosition] == 'u')) {
                 getNextUnicodeChar();
               } else {
                 if (withoutUnicodePtr != 0) {
                   withoutUnicodeBuffer[++withoutUnicodePtr] =
                     currentCharacter;
                 }
               }
             }
             if (getNextChar('\''))
               return TokenNameCharacterLiteral;
             // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
             for (int lookAhead = 0; lookAhead < 20; lookAhead++) {
               if (currentPosition + lookAhead == source.length)
                 break;
               if (source[currentPosition + lookAhead] == '\n')
                 break;
               if (source[currentPosition + lookAhead] == '\'') {
                 currentPosition += lookAhead + 1;
                 break;
               }
             }
             throw new InvalidInputException(INVALID_CHARACTER_CONSTANT);
           case '"' :
             try {
               // consume next character
               unicodeAsBackSlash = false;
               if (((currentCharacter = source[currentPosition++]) == '\\')
                 && (source[currentPosition] == 'u')) {
                 getNextUnicodeChar();
               } else {
                 if (withoutUnicodePtr != 0) {
                   withoutUnicodeBuffer[++withoutUnicodePtr] =
                     currentCharacter;
                 }
               }
 
               while (currentCharacter != '"') {
                 /**** \r and \n are not valid in string literals ****/
                 if ((currentCharacter == '\n')
                   || (currentCharacter == '\r')) {
                   // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
                   for (int lookAhead = 0;
                     lookAhead < 50;
                     lookAhead++) {
                     if (currentPosition + lookAhead
                       == source.length)
                       break;
                     if (source[currentPosition + lookAhead]
                       == '\n')
                       break;
                     if (source[currentPosition + lookAhead]
                       == '\"') {
                       currentPosition += lookAhead
                         + 1;
                       break;
                     }
                   }
                   throw new InvalidInputException(INVALID_CHAR_IN_STRING);
                 }
                 if (currentCharacter == '\\') {
                   int escapeSize = currentPosition;
                   boolean backSlashAsUnicodeInString =
                     unicodeAsBackSlash;
                   //scanEscapeCharacter make a side effect on this value and we need the previous value few lines down this one
                   scanEscapeCharacter();
                   escapeSize = currentPosition - escapeSize;
                   if (withoutUnicodePtr == 0) {
                     //buffer all the entries that have been left aside....
                     withoutUnicodePtr =
                       currentPosition
                         - escapeSize
                         - 1
                         - startPosition;
                     System.arraycopy(
                       source,
                       startPosition,
                       withoutUnicodeBuffer,
                       1,
                       withoutUnicodePtr);
                     withoutUnicodeBuffer[++withoutUnicodePtr] =
                       currentCharacter;
                   } else { //overwrite the / in the buffer
                     withoutUnicodeBuffer[withoutUnicodePtr] =
                       currentCharacter;
                     if (backSlashAsUnicodeInString) { //there are TWO \ in the stream where only one is correct
                       withoutUnicodePtr--;
                     }
                   }
                 }
                 // consume next character
                 unicodeAsBackSlash = false;
                 if (((currentCharacter = source[currentPosition++])
                   == '\\')
                   && (source[currentPosition] == 'u')) {
                   getNextUnicodeChar();
                 } else {
                   if (withoutUnicodePtr != 0) {
                     withoutUnicodeBuffer[++withoutUnicodePtr] =
                       currentCharacter;
                   }
                 }
 
               }
             } catch (IndexOutOfBoundsException e) {
               throw new InvalidInputException(UNTERMINATED_STRING);
             } catch (InvalidInputException e) {
               if (e.getMessage().equals(INVALID_ESCAPE)) {
                 // relocate if finding another quote fairly close: thus unicode '/u000D' will be fully consumed
                 for (int lookAhead = 0; lookAhead < 50; lookAhead++) {
                   if (currentPosition + lookAhead
                     == source.length)
                     break;
                   if (source[currentPosition + lookAhead]
                     == '\n')
                     break;
                   if (source[currentPosition + lookAhead]
                     == '\"') {
                     currentPosition += lookAhead + 1;
                     break;
                   }
                 }
 
               }
               throw e; // rethrow
             }
             if (checkNonExternalizedStringLiterals) { // check for presence of  NLS tags //$NON-NLS-?$ where ? is an int.
               if (currentLine == null) {
                 currentLine = new NLSLine();
                 lines.add(currentLine);
               }
               currentLine.add(
                 new StringLiteral(
                   getCurrentTokenSourceString(),
                   startPosition,
                   currentPosition - 1));
             }
             return TokenNameStringLiteral;
           case '/' :
             {
               int test;
               if ((test = getNextChar('/', '*')) == 0) { //line comment 
                 int endPositionForLineComment = 0;
                 try { //get the next char 
                   if (((currentCharacter =
                     source[currentPosition++])
                     == '\\')
                     && (source[currentPosition] == 'u')) {
                     //-------------unicode traitement ------------
                     int c1 = 0, c2 = 0, c3 = 0, c4 = 0;
                     currentPosition++;
                     while (source[currentPosition]
                       == 'u') {
                       currentPosition++;
                     }
                     if ((c1 =
                       Character.getNumericValue(
                         source[currentPosition++]))
                       > 15
                       || c1 < 0
                       || (c2 =
                        Character
                          .getNumericValue(
                          source[currentPosition++]))
                        > 15
                      || c2 < 0
                      || (c3 =
                        Character
                          .getNumericValue(
                          source[currentPosition++]))
                        > 15
                      || c3 < 0
                      || (c4 =
                        Character
                          .getNumericValue(
                          source[currentPosition++]))
                        > 15
                      || c4 < 0) {
                      throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
                    } else {
                      currentCharacter =
                        (char) (((c1 * 16 + c2)
                          * 16
                          + c3)
                          * 16
                          + c4);
                    }
                  }

                  //handle the \\u case manually into comment
                  if (currentCharacter == '\\') {
                    if (source[currentPosition] == '\\')
                      currentPosition++;
                  } //jump over the \\
                  boolean isUnicode = false;
                  while (currentCharacter != '\r'
                    && currentCharacter != '\n') {
                    //get the next char
                    isUnicode = false;
                    if (((currentCharacter =
                      source[currentPosition++])
                      == '\\')
                      && (source[currentPosition]
                        == 'u')) {
                      isUnicode = true;
                      //-------------unicode traitement ------------
                      int c1 = 0,
                        c2 = 0,
                        c3 = 0,
                        c4 = 0;
                      currentPosition++;
                      while (source[currentPosition]
                        == 'u') {
                        currentPosition++;
                      }
                      if ((c1 =
                        Character
                          .getNumericValue(
                          source[currentPosition++]))
                        > 15
                        || c1 < 0
                        || (c2 =
                          Character
                            .getNumericValue(
                            source[currentPosition++]))
                          > 15
                        || c2 < 0
                        || (c3 =
                          Character
                            .getNumericValue(
                            source[currentPosition++]))
                          > 15
                        || c3 < 0
                        || (c4 =
                          Character
                            .getNumericValue(
                            source[currentPosition++]))
                          > 15
                        || c4 < 0) {
                        throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
                      } else {
                        currentCharacter =
                          (char) (((c1
                            * 16
                            + c2)
                            * 16
                            + c3)
                            * 16
                            + c4);
                      }
                    }
                    //handle the \\u case manually into comment
                    if (currentCharacter == '\\') {
                      if (source[currentPosition]
                        == '\\')
                        currentPosition++;
                    } //jump over the \\
                  }
                  if (isUnicode) {
                    endPositionForLineComment =
                      currentPosition - 6;
                  } else {
                    endPositionForLineComment =
                      currentPosition - 1;
                  }
                  recordComment(false);
                  if ((currentCharacter == '\r')
                    || (currentCharacter == '\n')) {
                    checkNonExternalizeString();
                    if (recordLineSeparator) {
                      if (isUnicode) {
                        pushUnicodeLineSeparator();
                      } else {
                        pushLineSeparator();
                      }
                    } else {
                      currentLine = null;
                    }
                  }
                  if (tokenizeComments) {
                    if (!isUnicode) {
                      currentPosition =
                        endPositionForLineComment;
                      // reset one character behind
                    }
                    return TokenNameCOMMENT_LINE;
                  }
                } catch (IndexOutOfBoundsException e) { //an eof will them be generated
                  if (tokenizeComments) {
                    currentPosition--;
                    // reset one character behind
                    return TokenNameCOMMENT_LINE;
                  }
                }
                break;
              }
              if (test > 0) { //traditional and annotation comment
                boolean isJavadoc = false, star = false;
                // consume next character
                unicodeAsBackSlash = false;
                if (((currentCharacter = source[currentPosition++])
                  == '\\')
                  && (source[currentPosition] == 'u')) {
                  getNextUnicodeChar();
                } else {
                  if (withoutUnicodePtr != 0) {
                    withoutUnicodeBuffer[++withoutUnicodePtr] =
                      currentCharacter;
                  }
                }

                if (currentCharacter == '*') {
                  isJavadoc = true;
                  star = true;
                }
                if ((currentCharacter == '\r')
                  || (currentCharacter == '\n')) {
                  checkNonExternalizeString();
                  if (recordLineSeparator) {
                    pushLineSeparator();
                  } else {
                    currentLine = null;
                  }
                }
                try { //get the next char 
                  if (((currentCharacter =
                    source[currentPosition++])
                    == '\\')
                    && (source[currentPosition] == 'u')) {
                    //-------------unicode traitement ------------
                    getNextUnicodeChar();
                  }
                  //handle the \\u case manually into comment
                  if (currentCharacter == '\\') {
                    if (source[currentPosition] == '\\')
                      currentPosition++;
                    //jump over the \\
                  }
                  // empty comment is not a javadoc /**/
                  if (currentCharacter == '/') {
                    isJavadoc = false;
                  }
                  //loop until end of comment */
                  while ((currentCharacter != '/') || (!star)) {
                    if ((currentCharacter == '\r')
                      || (currentCharacter == '\n')) {
                      checkNonExternalizeString();
                      if (recordLineSeparator) {
                        pushLineSeparator();
                      } else {
                        currentLine = null;
                      }
                    }
                    star = currentCharacter == '*';
                    //get next char
                    if (((currentCharacter =
                      source[currentPosition++])
                      == '\\')
                      && (source[currentPosition]
                        == 'u')) {
                      //-------------unicode traitement ------------
                      getNextUnicodeChar();
                    }
                    //handle the \\u case manually into comment
                    if (currentCharacter == '\\') {
                      if (source[currentPosition]
                        == '\\')
                        currentPosition++;
                    } //jump over the \\
                  }
                  recordComment(isJavadoc);
                  if (tokenizeComments) {
                    if (isJavadoc)
                      return TokenNameCOMMENT_JAVADOC;
                    return TokenNameCOMMENT_BLOCK;
                  }
                } catch (IndexOutOfBoundsException e) {
                  throw new InvalidInputException(UNTERMINATED_COMMENT);
                }
                break;
              }
              if (getNextChar('='))
                return TokenNameDIVIDE_EQUAL;
              return TokenNameDIVIDE;
            }
          case '\u001a' :
            if (atEnd())
              return TokenNameEOF;
            //the atEnd may not be <currentPosition == source.length> if source is only some part of a real (external) stream
            throw new InvalidInputException("Ctrl-Z"); //$NON-NLS-1$

          default :
            if (Character.isJavaIdentifierStart(currentCharacter))
              return scanIdentifierOrKeyword();
          if (currentCharacter == '#')
            return TokenNameHASH;
          if (currentCharacter == '@')
            return TokenNameEACH;
            if (Character.isDigit(currentCharacter))
              return scanNumber(false);
            return TokenNameERROR;
        }
      }
    } //-----------------end switch while try--------------------
    catch (IndexOutOfBoundsException e) {
    }
    return TokenNameEOF;
  }
  public final void getNextUnicodeChar() throws IndexOutOfBoundsException, InvalidInputException {
    //VOID
    //handle the case of unicode.
    //when a unicode appears then we must use a buffer that holds char internal values
    //At the end of this method currentCharacter holds the new visited char
    //and currentPosition points right next after it

    //ALL getNextChar.... ARE OPTIMIZED COPIES 

    int c1 = 0, c2 = 0, c3 = 0, c4 = 0, unicodeSize = 6;
    currentPosition++;
    while (source[currentPosition] == 'u') {
      currentPosition++;
      unicodeSize++;
    }

    if ((c1 = Character.getNumericValue(source[currentPosition++])) > 15
      || c1 < 0
      || (c2 = Character.getNumericValue(source[currentPosition++])) > 15
      || c2 < 0
      || (c3 = Character.getNumericValue(source[currentPosition++])) > 15
      || c3 < 0
      || (c4 = Character.getNumericValue(source[currentPosition++])) > 15
      || c4 < 0) {
      throw new InvalidInputException(INVALID_UNICODE_ESCAPE);
    } else {
      currentCharacter = (char) (((c1 * 16 + c2) * 16 + c3) * 16 + c4);
      //need the unicode buffer
      if (withoutUnicodePtr == 0) {
        //buffer all the entries that have been left aside....
        withoutUnicodePtr = currentPosition - unicodeSize - startPosition;
        System.arraycopy(source, startPosition, withoutUnicodeBuffer, 1, withoutUnicodePtr);
      }
      //fill the buffer with the char
      withoutUnicodeBuffer[++withoutUnicodePtr] = currentCharacter;
    }
    unicodeAsBackSlash = currentCharacter == '\\';
  }
  /* Tokenize a method body, assuming that curly brackets are properly balanced.
   */
  public final void jumpOverMethodBody() {

    this.wasAcr = false;
    int found = 1;
    try {
      while (true) { //loop for jumping over comments
        // ---------Consume white space and handles startPosition---------
        boolean isWhiteSpace;
        do {
          startPosition = currentPosition;
          if (((currentCharacter = source[currentPosition++]) == '\\')
            && (source[currentPosition] == 'u')) {
            isWhiteSpace = jumpOverUnicodeWhiteSpace();
          } else {
            if (recordLineSeparator
              && ((currentCharacter == '\r') || (currentCharacter == '\n')))
              pushLineSeparator();
            isWhiteSpace = Character.isWhitespace(currentCharacter);
          }
        } while (isWhiteSpace);

        // -------consume token until } is found---------
        switch (currentCharacter) {
          case '{' :
            found++;
            break;
          case '}' :
            found--;
            if (found == 0)
              return;
            break;
          case '\'' :
            {
              boolean test;
              test = getNextChar('\\');
              if (test) {
                try {
                  scanEscapeCharacter();
                } catch (InvalidInputException ex) {
                };
              } else {
                try { // consume next character
                  unicodeAsBackSlash = false;
                  if (((currentCharacter =
                    source[currentPosition++])
                    == '\\')
                    && (source[currentPosition] == 'u')) {
                    getNextUnicodeChar();
                  } else {
                    if (withoutUnicodePtr != 0) {
                      withoutUnicodeBuffer[++withoutUnicodePtr] =
                        currentCharacter;
                    }
                  }
                } catch (InvalidInputException ex) {
                };
              }
              getNextChar('\'');
              break;
            }
          case '"' :
            try {
              try { // consume next character
                unicodeAsBackSlash = false;
                if (((currentCharacter = source[currentPosition++])
                  == '\\')
                  && (source[currentPosition] == 'u')) {
                  getNextUnicodeChar();
                } else {
                  if (withoutUnicodePtr != 0) {
                    withoutUnicodeBuffer[++withoutUnicodePtr] =
                      currentCharacter;
                  }
                }
              } catch (InvalidInputException ex) {
              };
              while (currentCharacter != '"') {
                if (currentCharacter == '\r') {
                  if (source[currentPosition] == '\n')
                    currentPosition++;
                  break;
                  // the string cannot go further that the line
                }
                if (currentCharacter == '\n') {
                  break;
                  // the string cannot go further that the line
                }
                if (currentCharacter == '\\') {
                  try {
                    scanEscapeCharacter();
                  } catch (InvalidInputException ex) {
                  };
                }
                try { // consume next character
                  unicodeAsBackSlash = false;
                  if (((currentCharacter =
                    source[currentPosition++]