|
|||||||||
| Home >> All >> com >> trapezium >> [ parse overview ] | PREV CLASS NEXT CLASS | ||||||||
SUMMARY:  JAVADOC |  SOURCE |  DOWNLOAD | NESTED | FIELD | CONSTR | METHOD |
DETAIL: FIELD | CONSTR | METHOD | ||||||||
com.trapezium.parse
Class TokenEnumerator
java.lang.Objectcom.trapezium.parse.TokenEnumerator
- All Implemented Interfaces:
- java.io.Serializable, TokenTypes
- public class TokenEnumerator
- extends java.lang.Object
- implements TokenTypes, java.io.Serializable
- extends java.lang.Object
Converts an InputStream into a sequence of tokens used by parsing.
1.0 architecture had Token objects, this replaced in 1.1 with less elegant, but more memory efficient byte array with run-length-encoding of initial spaces to store VRML file in memory.
Each token still contains the same information as previously, except is now represented as an "int", and is accessed only through the TokenEnumerator. The information associated with each token is:
- line number
- offset in line of start of token
- size of token (number of characters it contains)
- type of token
Quoted strings are implemented with two token types: QuotedString and QuotedStringContinuation. This is done to avoid the complication of tokens existing on more than one line. This class works with TokenFactory. Arrays and primitive types rather than objects are used to increase the size of a file that can be handled before OutOfMemory exception occurs.
- Since:
- 1.0
- Version:
- 1.11, 20 Jan 1998
| Nested Class Summary | |
(package private) class |
TokenEnumerator.LineReporter
this class is not used by Vorlon, part of callback progress reporting |
| Field Summary | |
protected int |
byteArrayBoundary
|
(package private) com.trapezium.util.ByteString |
byteString
|
(package private) static int |
counter
|
protected boolean |
dirtyFileData
Set by any method which changes fileData |
(package private) com.trapezium.util.ProgressIndicator |
factoryResponseListener
|
protected byte[] |
fileData
|
protected int |
fileDataIdx
|
(package private) java.lang.String |
fileUrl
|
(package private) int |
firstTokenOnLastLineChecked
|
(package private) static int |
InitialByteArraySize
|
(package private) static int |
InitialLineArraySize
|
(package private) static int |
InitialTokenArraySize
|
static int |
lastId
|
(package private) int |
lastLineChecked
Get the number of tokens on a line |
(package private) int |
lastLineReported
|
protected int |
lineArrayBoundary
The number of entries possible in the lineIdx array |
protected boolean[] |
lineBreakArray
|
protected int[] |
lineIdx
One array entry per line |
protected int[] |
lineNumberArray
One array entry per token |
protected short[] |
lineOffsetArray
|
(package private) TokenEnumerator.LineReporter |
lineReporter
|
(package private) int |
myId
|
protected int |
numberLines
The number of valid lines |
protected int |
numberTokens
The number of valid token entries |
static long |
presetLength
|
(package private) boolean |
skipComments
default is to skip comment tokens |
(package private) static int |
SmallInitialByteArraySize
|
(package private) static int |
SmallInitialTokenArraySize
|
(package private) int |
startOffsetOfNextLine
|
(package private) static int |
tabIndentSize
|
protected int |
tokenArrayBoundary
The number of entries possible in the above arrays |
(package private) com.trapezium.util.ReturnInteger |
tokenOffsetReturn
create tokens for a line of text |
protected int |
tokenScannerOffset
Internal cursor, optimization, offset into above arrays during scanning |
protected short[] |
tokenSizeArray
|
(package private) com.trapezium.util.ReturnInteger |
tokenSizeReturn
|
protected byte[] |
tokenTypeArray
|
(package private) com.trapezium.util.ReturnInteger |
tokenTypeReturn
|
| Fields inherited from interface com.trapezium.parse.TokenTypes |
BadNumber, CommentToken, EmptyLine, Keyword1Token, Keyword2Token, LeftBrace, LeftBracket, NameToken, NumberToken, QuotedString, QuotedStringContinuation, RightBrace, RightBracket, WhiteToken |
| Constructor Summary | |
TokenEnumerator()
used when recreating token enumerator based on old one |
|
TokenEnumerator(java.io.InputStream is,
java.lang.String inFile)
Class constructor |
|
TokenEnumerator(java.io.InputStream is,
java.lang.String inFile,
boolean allowUnterminatedString)
Class constructor |
|
TokenEnumerator(java.io.InputStream is,
java.lang.String inFile,
com.trapezium.util.ProgressIndicator frl,
java.io.File fileSource)
Class constructor |
|
TokenEnumerator(java.io.InputStream is,
java.lang.String inFile,
com.trapezium.util.ProgressIndicator frl,
java.io.File fileSource,
boolean allowUnterminatedString)
|
|
TokenEnumerator(int byteArraySize,
int tokenArraySize,
int lineArraySize)
Create a token enumerator with built in arrays of a specific size |
|
TokenEnumerator(java.lang.String s)
enumerate sequence of tokens from a string |
|
TokenEnumerator(TokenEnumerator source)
Create one tokenEnumerator using the same arrays as another. |
|
| Method Summary | |
void |
addLine(int offset,
int len,
byte[] sourceArray,
int numberSourceTokens,
int sourceTokenOffset,
short[] sourceLineOffsetArray,
short[] sourceTokenSizeArray,
boolean[] sourceLineBreakArray,
byte[] sourceTokenTypeArray)
|
void |
addLine(int offset,
int len,
byte[] sourceArray,
TokenFactory t)
Optimized add line, takes bytes from source array. |
void |
addLine(java.lang.StringBuffer line,
TokenFactory t)
Add a line to the list of lines |
void |
addLine(java.lang.String line,
TokenFactory t)
Add a line to the list of lines |
void |
addLineCapacity()
|
(package private) void |
addToken(int lineNumber,
byte type,
int offset,
int size)
|
void |
append(int tokenOffset,
java.lang.StringBuffer accumulator)
|
void |
breakLineAt(int offset)
mark the token as one that is associated with a line break |
char |
charAt(int charOffset)
get the character at a particular offset in the current token |
char |
charAt(int charOffset,
int tokenOffset)
Get the character at a particular offset for a token |
boolean |
copy2f(float[] result,
int last)
|
boolean |
copy3f(float[] result,
int last)
|
void |
detailDump()
|
void |
detailDump(boolean details)
|
void |
detailDump(int tokenOffset)
|
(package private) void |
detailDump(java.io.PrintStream ps,
boolean allDetails)
|
void |
detailDump(java.lang.String fileName)
Dump detailed TokenEnumerator data to a file. |
void |
disableCommentSkipping()
getNextToken() will return comment tokens |
void |
dump()
debugging dump of TokenEnumerator |
(package private) float |
eGetFloat(int tokenOffset)
|
void |
enableCommentSkipping()
getNextToken() will not return comment tokens |
void |
ensureByteCapacity(int len)
Make sure the fileData[] byte array has enough room for len more bytes. |
void |
ensureLineCapacity(int nLines)
Make sure the lineIdx[] array has enough room for nLines more lines. |
(package private) void |
ensureTokenCapacity(int nTokens)
|
protected void |
expandSize(int lineNumber,
int size)
Expand the size of a line by shifting bytes in fileData, and shifting offset values in lineIdx |
int |
getByteArrayBoundary()
get the number of bytes available in the byte array |
int |
getByteArraySize()
|
int |
getByteOffset(int tokenOffset)
Get the byte offset of a token in fileData byte array |
protected com.trapezium.util.ByteString |
getByteString(int fileDataOffset)
|
byte[] |
getCharArray(int t1,
int t2)
create a byte array for a sequence of tokens |
byte[] |
getCharArray(int t1,
int t2,
int t3)
|
int |
getCurrentTokenOffset()
what is the current token offset |
byte[] |
getFileData()
Get the byte array used to store file text. |
int |
getFileDataIdx()
Get the number of bytes used in the byte array. |
java.lang.String |
getFileUrl()
|
int |
getFirstTokenOnLine(int lineNumber)
|
float |
getFloat(int offset)
Get the float value of a token |
int |
getIntValue(int offset)
Get the integer value of the token at a particular offset. |
int |
getLineArrayBoundary()
Get the size of the line based array |
int |
getLineArraySize()
Get the amount of the line based array that is in use + 500 |
java.lang.String |
getLineAt(int offset)
get the String object for a line at a particular offset |
java.lang.String |
getLineAt(int offset,
int sizeLimit)
get the String object for a line at a particular offset, but limit size |
boolean[] |
getLineBreakArray()
Get the lineBreakArray |
char |
getLineChar(int lineNumber,
int charOffset)
get the character at a particular offset in a line |
int[] |
getLineIdx()
Get the lineIdx array |
int |
getLineIdx(int lineNo)
Get the offset in the byte array of the first byte of a particular line |
int |
getLineNumber(int offset)
Get the line number of a token |
int[] |
getLineNumberArray()
Get the lineNumberArray |
int |
getLineOffset(int offset)
Get the offset within a line of a particular token, does not include leading white space |
short[] |
getLineOffsetArray()
Get the lineOffsetArray |
int |
getLineSize(int lineNo)
Get the number of bytes occupied by a line, including 0 bytes, and assuming that next line immediately follows. |
int |
getNextNumber(int tokenOffset)
|
int |
getNextToken()
get the next token |
int |
getNextToken(int prevToken)
get the token following a particular token |
java.lang.String |
getNospaceLineAt(int offset)
|
(package private) java.lang.String |
getNospaceString(int lineByteOffset)
|
int |
getNumberLines()
Get the number of lines |
int |
getNumberTokens()
Get the number of tokens created |
int |
getNumberTokensOnLine(int lineNumber)
|
int |
getSize(int tokenOffset)
Get size of a token, does not include leading white space |
int |
getState()
state is just the current offset |
(package private) java.lang.String |
getString(int lineByteOffset,
boolean spaceToTab)
Get the String at a particular offset in the byte array |
(package private) java.lang.String |
getString(int lineByteOffset,
int sizeLimit)
|
java.lang.String |
getTabLineAt(int offset)
|
int |
getTokenArrayBoundary()
Get the size of the token based arrays |
int |
getTokenArraySize()
get the number of tokens allocated in token array +1000 (not used by vorlon) |
short[] |
getTokenSizeArray()
Get the tokenSizeArray |
java.lang.String |
getTokenType(int offset)
get a string form of the type of the token at a particular offset |
byte[] |
getTokenTypeArray()
Get the tokenTypeArray |
int |
getType(int offset)
get an int form of the type of the token at a particular offset |
boolean |
hasChar(int offset,
char cval)
|
boolean |
hasLines()
Check if the TokenEnumerator has any lines |
boolean |
hasMoreElements()
are there more tokens available |
void |
incLineNumbers(int line)
|
(package private) void |
increaseLineCapacity()
Increase the capacity of the lineIdx[] array |
(package private) void |
init(long byteArraySize)
Initialize all array sizes based on file length, initialize byte array also. |
(package private) void |
init(long byteArraySize,
boolean initializeByteArray)
Initialize all array sizes based on file length |
(package private) void |
init(long byteArraySize,
int tokenArraySize,
int lineArraySize)
Initialize all arrays |
(package private) void |
init(long byteArraySize,
int tokenArraySize,
int lineArraySize,
boolean initializeByteArray)
Initialize all arrays, with possible exception of byte array. |
void |
insert(int tokenOffset,
java.lang.String tok1,
byte tokType)
Insert one token of specific type |
void |
insert(int tokenOffset,
java.lang.String tok1,
java.lang.String tok2)
Insert two tokens |
protected void |
insertLine(int newLineNumber,
int byteOffset)
Create a new line at a specific offset, assumes fileData already set up. |
boolean |
isComment(int offset)
is this a comment token? |
boolean |
isContinuationString(int offset)
is the token a quoted string continuation? |
boolean |
isDirty()
Check if TokenEnumerator needs retokenizing |
boolean |
isFloat(int offset)
|
boolean |
isLeftBrace(int offset)
is the token a left brace? |
boolean |
isLeftBracket(int offset)
is the token a left bracket? |
boolean |
isLineBreak(int offset)
is the token associated with a line break? |
boolean |
isName(int offset)
is the token a name? |
boolean |
isNegativeOne(int offset)
|
boolean |
isNumber(int offset)
is the token a number? |
boolean |
isQuotedString(int offset)
is the token a quoted string? |
boolean |
isRightBrace(int offset)
is the token a right brace? |
boolean |
isRightBracket(int offset)
is the token a right bracket? |
boolean |
isRightBracketOrBrace(int offset)
is the token at a particular offset a right bracket or a right brace |
boolean |
isSpecialCharacter(int offset)
is the token a bracket or brace? |
int |
length()
get the length of the current token |
int |
length(int offset)
Get the length of a particular token |
void |
lineDump()
|
(package private) void |
loadLines(java.io.InputStream inStream,
com.trapezium.util.ProgressIndicator frl,
boolean allowUnterminatedString)
|
boolean |
matches(int tokenOffset,
java.lang.String s)
compare beginning of token to a string |
boolean |
nearlySameAs(int tokenOffset,
java.lang.String s)
compare token to a string, case insensitive |
void |
notifyByLine(int lineNumber)
report back based on line |
void |
notifyByToken(int tokenOffset)
report back based on token |
void |
notifyLineNumbers(com.trapezium.util.ProgressIndicator frl)
set up call back to report token scanning progress. |
(package private) void |
processLine(TokenFactory t,
com.trapezium.util.ByteString line,
int lineNumber)
|
protected boolean |
processLine(TokenFactory t,
int fileDataOffset,
int lineNumber)
Create tokens for a line |
private void |
readObject(java.io.ObjectInputStream stream)
Read object form ObjectInputStream, and reconstruct transient fields. |
void |
removeLines(int startLine,
int n)
Remove a set of lines |
void |
removeTokens(int startToken,
int nTokens)
Remove a set of tokens |
void |
replace(int tokenOffset,
java.lang.String newToken)
Replace a token with a specific String value |
void |
retokenize()
Retokenize the data in the "fileData" byte array. |
(package private) void |
rightShiftBytes(int boundary,
int amount)
Shift all data bytes right by the specified amount starting at a specific boundary |
(package private) void |
rightShiftTokens(int boundary,
int amount)
Shift all token arrays right by the specified amount starting at a specific boundary |
boolean |
sameAs(int token1offset,
int token2offset)
compare two tokens |
boolean |
sameAs(int tokenOffset,
java.lang.String s)
compare token to a string |
boolean |
sameAs(java.lang.String s)
compare current token to a string |
boolean |
sameAs(TokenEnumerator other)
compare the current token to one in another TokenEnumerator |
void |
saveLines(java.io.PrintStream ps)
save lines to a PrintStream |
void |
setFileDataIdx(int fileDataIdx)
Set the next free location in the byte array |
void |
setFileUrl(java.lang.String fileUrl)
|
void |
setLineIdx(int lineNo,
int idxVal)
Set the offset in the byte array of a line |
void |
setNumberLines(int n)
Set the number of lines |
void |
setState(int state)
restore previous state, used for lookahead |
static void |
setTabIndentSize(int n)
|
(package private) void |
skipCommentTokens()
skip comment tokens |
int |
skipNonNumbers()
Skip to the first number token, or end of stream, whichever comes first. |
void |
skipTo(int tokenType)
skip to the next token of a particular type |
int |
skipToNumber(int offset)
Skip to the a specific number token |
void |
snarfArrays(TokenEnumerator source)
copy array objects from one TokenEnumerator to another |
protected void |
split_line(int tokenLine,
int tokenLineOffset,
boolean splitOnSpace)
split a line at a particualr offset |
void |
startLineWith(int tokenOffset)
Make token be the first token in a line |
java.lang.String |
toString(int tokenOffset)
Convert a token to a string, does not include leading white space |
int |
valueAfterDot(int offset)
|
int |
valueBeforeDot(int offset)
|
void |
wipeout()
set all array objects to null, so they can be garbage collected |
int |
zerosAfterDot(int offset)
|
| Methods inherited from class java.lang.Object |
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
| Field Detail |
tabIndentSize
static int tabIndentSize
InitialByteArraySize
static int InitialByteArraySize
InitialTokenArraySize
static int InitialTokenArraySize
InitialLineArraySize
static int InitialLineArraySize
SmallInitialByteArraySize
static int SmallInitialByteArraySize
SmallInitialTokenArraySize
static int SmallInitialTokenArraySize
lineNumberArray
protected transient int[] lineNumberArray
- One array entry per token
lineOffsetArray
protected transient short[] lineOffsetArray
lineBreakArray
protected transient boolean[] lineBreakArray
tokenTypeArray
protected transient byte[] tokenTypeArray
tokenSizeArray
protected transient short[] tokenSizeArray
tokenArrayBoundary
protected int tokenArrayBoundary
- The number of entries possible in the above arrays
numberTokens
protected int numberTokens
- The number of valid token entries
tokenScannerOffset
protected int tokenScannerOffset
- Internal cursor, optimization, offset into above arrays during scanning
dirtyFileData
protected transient boolean dirtyFileData
- Set by any method which changes fileData
lineIdx
protected int[] lineIdx
- One array entry per line
numberLines
protected int numberLines
- The number of valid lines
lineArrayBoundary
protected int lineArrayBoundary
- The number of entries possible in the lineIdx array
lastLineChecked
int lastLineChecked
- Get the number of tokens on a line
firstTokenOnLastLineChecked
int firstTokenOnLastLineChecked
startOffsetOfNextLine
int startOffsetOfNextLine
fileData
protected byte[] fileData
fileDataIdx
protected int fileDataIdx
byteArrayBoundary
protected int byteArrayBoundary
fileUrl
java.lang.String fileUrl
lastId
public static int lastId
myId
int myId
presetLength
public static long presetLength
lineReporter
transient TokenEnumerator.LineReporter lineReporter
counter
static int counter
tokenOffsetReturn
com.trapezium.util.ReturnInteger tokenOffsetReturn
- create tokens for a line of text
tokenTypeReturn
com.trapezium.util.ReturnInteger tokenTypeReturn
tokenSizeReturn
com.trapezium.util.ReturnInteger tokenSizeReturn
byteString
transient com.trapezium.util.ByteString byteString
factoryResponseListener
transient com.trapezium.util.ProgressIndicator factoryResponseListener
lastLineReported
int lastLineReported
skipComments
boolean skipComments
- default is to skip comment tokens
| Constructor Detail |
TokenEnumerator
public TokenEnumerator(java.io.InputStream is, java.lang.String inFile) throws java.io.FileNotFoundException, java.io.IOException
- Class constructor
TokenEnumerator
public TokenEnumerator(java.io.InputStream is, java.lang.String inFile, boolean allowUnterminatedString) throws java.io.FileNotFoundException, java.io.IOException
- Class constructor
TokenEnumerator
public TokenEnumerator(java.io.InputStream is, java.lang.String inFile, com.trapezium.util.ProgressIndicator frl, java.io.File fileSource) throws java.io.FileNotFoundException, java.io.IOException
- Class constructor
TokenEnumerator
public TokenEnumerator(java.io.InputStream is, java.lang.String inFile, com.trapezium.util.ProgressIndicator frl, java.io.File fileSource, boolean allowUnterminatedString) throws java.io.FileNotFoundException, java.io.IOException
TokenEnumerator
public TokenEnumerator(java.lang.String s)
- enumerate sequence of tokens from a string
TokenEnumerator
public TokenEnumerator()
- used when recreating token enumerator based on old one
TokenEnumerator
public TokenEnumerator(int byteArraySize,
int tokenArraySize,
int lineArraySize)
- Create a token enumerator with built in arrays of a specific size
TokenEnumerator
public TokenEnumerator(TokenEnumerator source)
- Create one tokenEnumerator using the same arrays as another.
Original intent was to allow arrays to be reused during reformatting,
however, this only works if same comment skipping strategy in place,
if not, there is risk that arrays wipe out each other.
| Method Detail |
setTabIndentSize
public static void setTabIndentSize(int n)
getNumberTokens
public int getNumberTokens()
- Get the number of tokens created
hasLines
public boolean hasLines()
- Check if the TokenEnumerator has any lines
getNumberLines
public int getNumberLines()
- Get the number of lines
setNumberLines
public void setNumberLines(int n)
- Set the number of lines
getTokenArraySize
public int getTokenArraySize()
- get the number of tokens allocated in token array +1000
(not used by vorlon)
getLineNumberArray
public int[] getLineNumberArray()
- Get the lineNumberArray
getLineOffsetArray
public short[] getLineOffsetArray()
- Get the lineOffsetArray
getLineBreakArray
public boolean[] getLineBreakArray()
- Get the lineBreakArray
getTokenTypeArray
public byte[] getTokenTypeArray()
- Get the tokenTypeArray
getTokenSizeArray
public short[] getTokenSizeArray()
- Get the tokenSizeArray
getTokenArrayBoundary
public int getTokenArrayBoundary()
- Get the size of the token based arrays
getLineIdx
public int[] getLineIdx()
- Get the lineIdx array
setLineIdx
public void setLineIdx(int lineNo,
int idxVal)
- Set the offset in the byte array of a line
getLineArrayBoundary
public int getLineArrayBoundary()
- Get the size of the line based array
getLineArraySize
public int getLineArraySize()
- Get the amount of the line based array that is in use + 500
isDirty
public boolean isDirty()
- Check if TokenEnumerator needs retokenizing
getNumberTokensOnLine
public int getNumberTokensOnLine(int lineNumber)
getFirstTokenOnLine
public int getFirstTokenOnLine(int lineNumber)
removeTokens
public void removeTokens(int startToken,
int nTokens)
- Remove a set of tokens
insertLine
protected void insertLine(int newLineNumber,
int byteOffset)
- Create a new line at a specific offset, assumes
fileData already set up.
expandSize
protected void expandSize(int lineNumber,
int size)
- Expand the size of a line by shifting bytes in fileData, and shifting
offset values in lineIdx
removeLines
public void removeLines(int startLine,
int n)
- Remove a set of lines
getLineIdx
public int getLineIdx(int lineNo)
- Get the offset in the byte array of the first byte of a particular line
getLineSize
public int getLineSize(int lineNo)
- Get the number of bytes occupied by a line, including 0 bytes,
and assuming that next line immediately follows.
getFileData
public byte[] getFileData()
- Get the byte array used to store file text.
getFileDataIdx
public int getFileDataIdx()
- Get the number of bytes used in the byte array.
setFileDataIdx
public void setFileDataIdx(int fileDataIdx)
- Set the next free location in the byte array
wipeout
public void wipeout()
- set all array objects to null, so they can be garbage collected
startLineWith
public void startLineWith(int tokenOffset)
- Make token be the first token in a line
split_line
protected void split_line(int tokenLine,
int tokenLineOffset,
boolean splitOnSpace)
- split a line at a particualr offset
insert
public void insert(int tokenOffset,
java.lang.String tok1,
byte tokType)
- Insert one token of specific type
insert
public void insert(int tokenOffset,
java.lang.String tok1,
java.lang.String tok2)
- Insert two tokens
replace
public void replace(int tokenOffset,
java.lang.String newToken)
- Replace a token with a specific String value
rightShiftTokens
void rightShiftTokens(int boundary,
int amount)
- Shift all token arrays right by the specified amount starting at a specific boundary
rightShiftBytes
void rightShiftBytes(int boundary,
int amount)
- Shift all data bytes right by the specified amount starting at a specific boundary
snarfArrays
public void snarfArrays(TokenEnumerator source)
- copy array objects from one TokenEnumerator to another
getByteArrayBoundary
public int getByteArrayBoundary()
- get the number of bytes available in the byte array
getByteArraySize
public int getByteArraySize()
getFileUrl
public java.lang.String getFileUrl()
setFileUrl
public void setFileUrl(java.lang.String fileUrl)
init
void init(long byteArraySize)
- Initialize all array sizes based on file length, initialize byte array also.
init
void init(long byteArraySize,
boolean initializeByteArray)
- Initialize all array sizes based on file length
init
void init(long byteArraySize,
int tokenArraySize,
int lineArraySize)
- Initialize all arrays
init
void init(long byteArraySize,
int tokenArraySize,
int lineArraySize,
boolean initializeByteArray)
- Initialize all arrays, with possible exception of byte array.
readObject
private void readObject(java.io.ObjectInputStream stream) throws java.io.IOException
- Read object form ObjectInputStream, and reconstruct transient fields.
loadLines
void loadLines(java.io.InputStream inStream, com.trapezium.util.ProgressIndicator frl, boolean allowUnterminatedString)
retokenize
public void retokenize()
- Retokenize the data in the "fileData" byte array.
addLineCapacity
public void addLineCapacity()
increaseLineCapacity
void increaseLineCapacity()
- Increase the capacity of the lineIdx[] array
ensureLineCapacity
public void ensureLineCapacity(int nLines)
- Make sure the lineIdx[] array has enough room for nLines more lines.
ensureTokenCapacity
void ensureTokenCapacity(int nTokens)
addLine
public void addLine(java.lang.String line, TokenFactory t)
- Add a line to the list of lines
addLine
public void addLine(java.lang.StringBuffer line, TokenFactory t)
- Add a line to the list of lines
addLine
public void addLine(int offset,
int len,
byte[] sourceArray,
TokenFactory t)
- Optimized add line, takes bytes from source array.
addLine
public void addLine(int offset,
int len,
byte[] sourceArray,
int numberSourceTokens,
int sourceTokenOffset,
short[] sourceLineOffsetArray,
short[] sourceTokenSizeArray,
boolean[] sourceLineBreakArray,
byte[] sourceTokenTypeArray)
ensureByteCapacity
public void ensureByteCapacity(int len)
- Make sure the fileData[] byte array has enough room for len more bytes.
lineDump
public void lineDump()
saveLines
public void saveLines(java.io.PrintStream ps)
- save lines to a PrintStream
dump
public void dump()
- debugging dump of TokenEnumerator
detailDump
public void detailDump(int tokenOffset)
detailDump
public void detailDump()
detailDump
public void detailDump(boolean details)
detailDump
public void detailDump(java.lang.String fileName)
- Dump detailed TokenEnumerator data to a file.
detailDump
void detailDump(java.io.PrintStream ps, boolean allDetails)
addToken
void addToken(int lineNumber,
byte type,
int offset,
int size)
breakLineAt
public void breakLineAt(int offset)
- mark the token as one that is associated with a line break
isLineBreak
public boolean isLineBreak(int offset)
- is the token associated with a line break?
getLineAt
public java.lang.String getLineAt(int offset)
- get the String object for a line at a particular offset
getLineAt
public java.lang.String getLineAt(int offset, int sizeLimit)
- get the String object for a line at a particular offset, but limit size
getNospaceLineAt
public java.lang.String getNospaceLineAt(int offset)
getTabLineAt
public java.lang.String getTabLineAt(int offset)
getNospaceString
java.lang.String getNospaceString(int lineByteOffset)
getString
java.lang.String getString(int lineByteOffset, boolean spaceToTab)
- Get the String at a particular offset in the byte array
getString
java.lang.String getString(int lineByteOffset, int sizeLimit)
sameAs
public boolean sameAs(TokenEnumerator other)
- compare the current token to one in another TokenEnumerator
getLineNumber
public int getLineNumber(int offset)
- Get the line number of a token
incLineNumbers
public void incLineNumbers(int line)
getLineOffset
public int getLineOffset(int offset)
- Get the offset within a line of a particular token, does not include leading white space
charAt
public char charAt(int charOffset)
- get the character at a particular offset in the current token
getLineChar
public char getLineChar(int lineNumber,
int charOffset)
- get the character at a particular offset in a line
charAt
public char charAt(int charOffset,
int tokenOffset)
- Get the character at a particular offset for a token
getByteOffset
public int getByteOffset(int tokenOffset)
- Get the byte offset of a token in fileData byte array
length
public int length()
- get the length of the current token
length
public int length(int offset)
- Get the length of a particular token
getIntValue
public int getIntValue(int offset)
- Get the integer value of the token at a particular offset.
isNegativeOne
public boolean isNegativeOne(int offset)
valueBeforeDot
public int valueBeforeDot(int offset)
zerosAfterDot
public int zerosAfterDot(int offset)
valueAfterDot
public int valueAfterDot(int offset)
isRightBracketOrBrace
public boolean isRightBracketOrBrace(int offset)
- is the token at a particular offset a right bracket or a right brace
isQuotedString
public boolean isQuotedString(int offset)
- is the token a quoted string?
isRightBracket
public boolean isRightBracket(int offset)
- is the token a right bracket?
isLeftBracket
public boolean isLeftBracket(int offset)
- is the token a left bracket?
isRightBrace
public boolean isRightBrace(int offset)
- is the token a right brace?
isLeftBrace
public boolean isLeftBrace(int offset)
- is the token a left brace?
isSpecialCharacter
public boolean isSpecialCharacter(int offset)
- is the token a bracket or brace?
isContinuationString
public boolean isContinuationString(int offset)
- is the token a quoted string continuation?
isNumber
public boolean isNumber(int offset)
- is the token a number?
isName
public boolean isName(int offset)
- is the token a name?
skipTo
public void skipTo(int tokenType)
- skip to the next token of a particular type
skipNonNumbers
public int skipNonNumbers()
- Skip to the first number token, or end of stream, whichever comes first.
skipToNumber
public int skipToNumber(int offset)
- Skip to the a specific number token
isComment
public boolean isComment(int offset)
- is this a comment token?
sameAs
public boolean sameAs(java.lang.String s)
- compare current token to a string
sameAs
public boolean sameAs(int tokenOffset,
java.lang.String s)
- compare token to a string
nearlySameAs
public boolean nearlySameAs(int tokenOffset,
java.lang.String s)
- compare token to a string, case insensitive
matches
public boolean matches(int tokenOffset,
java.lang.String s)
- compare beginning of token to a string
sameAs
public boolean sameAs(int token1offset,
int token2offset)
- compare two tokens
getSize
public int getSize(int tokenOffset)
- Get size of a token, does not include leading white space
toString
public java.lang.String toString(int tokenOffset)
- Convert a token to a string, does not include leading white space
getFloat
public float getFloat(int offset)
- Get the float value of a token
eGetFloat
float eGetFloat(int tokenOffset)
isFloat
public boolean isFloat(int offset)
hasChar
public boolean hasChar(int offset,
char cval)
append
public void append(int tokenOffset,
java.lang.StringBuffer accumulator)
processLine
void processLine(TokenFactory t, com.trapezium.util.ByteString line, int lineNumber)
processLine
protected boolean processLine(TokenFactory t, int fileDataOffset, int lineNumber)
- Create tokens for a line
getByteString
protected com.trapezium.util.ByteString getByteString(int fileDataOffset)