Source code: com/trapezium/parse/TokenEnumerator.java
1 /*
2 * @(#)TokenEnumerator.java
3 *
4 * Copyright (c) 1998 by Trapezium Development LLC. All Rights Reserved.
5 *
6 * The information in this file is the property of Trapezium Development LLC
7 * and may be used only in accordance with the terms of the license granted
8 * by Trapezium.
9 *
10 */
11 package com.trapezium.parse;
12 import com.trapezium.util.ReturnInteger;
13 import com.trapezium.util.ByteString;
14 import com.trapezium.util.ProgressIndicator;
15
16 import java.util.Vector;
17 import java.io.File;
18 import java.io.FileNotFoundException;
19 import java.io.IOException;
20 import java.io.InputStreamReader;
21 import java.io.BufferedReader;
22 import java.io.InputStream;
23 import java.io.FileInputStream;
24 import java.io.PrintStream;
25 import java.io.FileOutputStream;
26 import com.trapezium.util.GlobalProgressIndicator;
27
28 /**
29 * Converts an InputStream into a sequence of tokens used by parsing.
30 * <P>
31 * 1.0 architecture had Token objects, this replaced in 1.1 with less elegant,
32 * but more memory efficient byte array with run-length-encoding of initial spaces
33 * to store VRML file in memory.
34 * <P>
35 * Each token still contains the same information as previously, except is
36 * now represented as an "int", and is accessed only through the TokenEnumerator.
37 * The information associated with each token is:
38 * <OL>
39 * <LI>line number</LI>
40 * <LI>offset in line of start of token</LI>
41 * <LI>size of token (number of characters it contains)</LI>
42 * <LI>type of token</LI>
43 * </OL>
44 * <P>
45 * Quoted strings are implemented with two token types: QuotedString
46 * and QuotedStringContinuation. This is done to avoid the complication
47 * of tokens existing on more than one line.
48 *
49 * This class works with TokenFactory. Arrays and primitive types rather than
50 * objects are used to increase the size of a file that can be handled before
51 * OutOfMemory exception occurs.
52 *
53 * @author Johannes N. Johannsen
54 * @version 1.11, 20 Jan 1998
55 *
56 * @since 1.0
57 */
58 public class TokenEnumerator implements TokenTypes, java.io.Serializable {
59 static int tabIndentSize = 0;
60 static public void setTabIndentSize( int n ) {
61 tabIndentSize = n;
62 }
63 static int InitialByteArraySize = 10000;
64 static int InitialTokenArraySize = 5000;
65 static int InitialLineArraySize = 5000;
66
67 static int SmallInitialByteArraySize = 100;
68 static int SmallInitialTokenArraySize = 20;
69
70 /** One array entry per token */
71 transient protected int[] lineNumberArray;
72 transient protected short[] lineOffsetArray;
73 transient protected boolean[] lineBreakArray;
74 transient protected byte[] tokenTypeArray;
75 transient protected short[] tokenSizeArray;
76
77 /** The number of entries possible in the above arrays */
78 protected int tokenArrayBoundary;
79
80 /** The number of valid token entries */
81 protected int numberTokens;
82
83 /** Internal cursor, optimization, offset into above arrays during scanning */
84 protected int tokenScannerOffset;
85
86 /** Set by any method which changes fileData */
87 transient protected boolean dirtyFileData;
88
89 /** One array entry per line */
90 protected int[] lineIdx;
91
92 /** The number of valid lines */
93 protected int numberLines;
94
95 /** The number of entries possible in the lineIdx array */
96 protected int lineArrayBoundary;
97
98 /** Get the number of tokens created */
99 public int getNumberTokens() {
100 return( numberTokens );
101 }
102
103 /********************* LINE METHODS **************************/
104 /** Check if the TokenEnumerator has any lines */
105 public boolean hasLines() {
106 return( numberLines > 0 );
107 }
108
109 /** Get the number of lines */
110 public int getNumberLines() {
111 return( numberLines );
112 }
113
114 /** Set the number of lines */
115 public void setNumberLines( int n ) {
116 numberLines = n;
117 }
118
119 /**
120 * get the number of tokens allocated in token array +1000
121 * (not used by vorlon)
122 */
123 public int getTokenArraySize() {
124 return( numberTokens + 1000 );
125 }
126
127 /** Get the lineNumberArray */
128 public int[] getLineNumberArray() {
129 return( lineNumberArray );
130 }
131
132 /** Get the lineOffsetArray */
133 public short[] getLineOffsetArray() {
134 return( lineOffsetArray );
135 }
136
137 /** Get the lineBreakArray */
138 public boolean[] getLineBreakArray() {
139 return( lineBreakArray );
140 }
141
142 /** Get the tokenTypeArray
143 *
144 * @see TokenTypes
145 */
146 public byte[] getTokenTypeArray() {
147 return( tokenTypeArray );
148 }
149
150 /** Get the tokenSizeArray */
151 public short[] getTokenSizeArray() {
152 return( tokenSizeArray );
153 }
154
155 /** Get the size of the token based arrays */
156 public int getTokenArrayBoundary() {
157 return( tokenArrayBoundary );
158 }
159
160 /** Get the lineIdx array */
161 public int[] getLineIdx() {
162 return( lineIdx );
163 }
164
165 /** Set the offset in the byte array of a line */
166 public void setLineIdx( int lineNo, int idxVal ) {
167 lineIdx[ lineNo ] = idxVal;
168 }
169
170 /** Get the size of the line based array */
171 public int getLineArrayBoundary() {
172 return( lineArrayBoundary );
173 }
174
175 /**
176 * Get the amount of the line based array that is in use + 500
177 */
178 public int getLineArraySize() {
179 return( numberLines + 500 );
180 }
181
182 /** Check if TokenEnumerator needs retokenizing
183 *
184 * @return true if the TokenEnumerator needs retokenizing, otherwise false
185 */
186 public boolean isDirty() {
187 return( dirtyFileData );
188 }
189
190 /** Get the number of tokens on a line */
191 int lastLineChecked = -1;
192 int firstTokenOnLastLineChecked = -1;
193 int startOffsetOfNextLine = -1;
194 public int getNumberTokensOnLine( int lineNumber ) {
195 int startOffset = 0;
196 if ( lineNumber > lastLineChecked ) {
197 if ( startOffsetOfNextLine != -1 ) {
198 startOffset = startOffsetOfNextLine;
199 }
200 }
201 boolean foundIt = false;
202 int result = 0;
203 for ( int i = startOffset; i < numberTokens; i++ ) {
204 if ( !foundIt ) {
205 if ( lineNumberArray[ i ] == lineNumber ) {
206 firstTokenOnLastLineChecked = i;
207 lastLineChecked = lineNumber;
208 result = 1;
209 foundIt = true;
210 } else if ( lineNumberArray[i] > lineNumber ) {
211 break;
212 }
213 } else {
214 if ( lineNumberArray[ i ] != lineNumber ) {
215 startOffsetOfNextLine = i + 1;
216 return( result );
217 }
218 result++;
219 }
220 }
221 return( result );
222 }
223
224 public int getFirstTokenOnLine( int lineNumber ) {
225 if ( lineNumber == lastLineChecked ) {
226 // System.out.println( "ret1" + firstTokenOnLastLineChecked );
227 return( firstTokenOnLastLineChecked );
228 } else if ( lineNumber > lastLineChecked ) {
229 for ( int i = startOffsetOfNextLine; i < numberTokens; i++ ) {
230 if ( lineNumberArray[ i ] == lineNumber ) {
231 // System.out.println( "ret 2" );
232 return( i );
233 } else if ( lineNumberArray[i] > lineNumber ) {
234 return( -1 );
235 }
236 }
237 return( -1 );
238 }
239 for ( int i = 0; i < numberTokens; i++ ) {
240 if ( lineNumberArray[i] == lineNumber ) {
241 // System.out.println( "ret3" );
242 return( i );
243 } else if ( lineNumberArray[i] > lineNumber ) {
244 return( -1 );
245 }
246 }
247
248 return( -1 );
249 }
250
251 /** Remove a set of tokens */
252 public void removeTokens( int startToken, int nTokens ) {
253 dirtyFileData = true;
254 int end = numberTokens - nTokens;
255 for ( int i = startToken; i < end; i++ ) {
256 lineNumberArray[i] = lineNumberArray[ i + nTokens ];
257 lineOffsetArray[i] = lineOffsetArray[ i + nTokens ];
258 lineBreakArray[i] = lineBreakArray[ i + nTokens ];
259 tokenTypeArray[i] = tokenTypeArray[ i + nTokens ];
260 tokenSizeArray[i] = tokenSizeArray[ i + nTokens ];
261 }
262 numberTokens -= nTokens;
263 }
264
265 /** Create a new line at a specific offset, assumes
266 * fileData already set up.
267 *
268 * @param newLineNumber the new line number
269 * @param byteOffset the offset into the fileData array of the
270 * text of the line
271 */
272 protected void insertLine( int newLineNumber, int byteOffset ) {
273 dirtyFileData = true;
274 addLineCapacity();
275 for ( int i = numberLines; i > newLineNumber; i-- ) {
276 lineIdx[ i ] = lineIdx[ i - 1 ];
277 }
278 numberLines++;
279 lineIdx[ newLineNumber ] = byteOffset;
280 }
281
282 /** Expand the size of a line by shifting bytes in fileData, and shifting
283 * offset values in lineIdx
284 */
285 protected void expandSize( int lineNumber, int size ) {
286 // size must be at least 20
287 if ( size < 20 ) {
288 size = 20;
289 }
290
291 // if we are on the last line number, the allocated size is the remainder of
292 // the fileData size.
293 int allocatedLineSize = 0;
294 if ( lineNumber == ( numberLines - 1 )) {
295 allocatedLineSize = byteArrayBoundary - lineIdx[ lineNumber ];
296 // Otherwise, the allocated line size is the difference between this line and
297 // the next non-error indicating line
298 } else {
299 int nextLine = lineNumber + 1;
300 if ( nextLine >= numberLines ) {
301 allocatedLineSize = byteArrayBoundary - lineIdx[ lineNumber ];
302 } else {
303 allocatedLineSize = lineIdx[ nextLine ] - lineIdx[ lineNumber ];
304 }
305 }
306
307 int actualLineSize = 0;
308 int scanner = lineIdx[ lineNumber ];
309 while (( scanner < byteArrayBoundary ) && ( fileData[ scanner ] != 0 )) {
310 actualLineSize++;
311 scanner++;
312 }
313 if ( fileData[ scanner ] == 0 ) {
314 actualLineSize++;
315 }
316
317 // If we don't have enough room for the expansion, make sure fileData
318 // size can accommodate it, then shift data of all subsequent lines
319 if (( allocatedLineSize - actualLineSize ) < size ) {
320 ensureByteCapacity( size );
321 if ( lineNumber != ( numberLines - 1 )) {
322 int zboundary = lineIdx[ lineNumber ];
323 while ( fileData[ zboundary ] != 0 ) {
324 zboundary++;
325 }
326 zboundary++;
327 rightShiftBytes( zboundary, size );
328 int boundary = lineIdx[ lineNumber ];
329 for ( int i = 0; i < numberLines; i++ ) {
330 if ( lineIdx[i] > boundary ) {
331 lineIdx[ i ] += size;
332 }
333 }
334 }
335 }
336 }
337
338 /** Remove a set of lines */
339 public void removeLines( int startLine, int n ) {
340 dirtyFileData = true;
341 int end = numberLines - n;
342 for ( int i = startLine; i < end; i++ ) {
343 lineIdx[ i ] = lineIdx[ i + n ];
344 }
345 numberLines -= n;
346 for ( int i = 0; i < numberTokens; i++ ) {
347 int lineNo = lineNumberArray[i];
348 if ( lineNo >= startLine ) {
349 lineNo -= n;
350 lineNumberArray[i] = lineNo;
351 }
352 }
353 }
354
355 /**
356 * Get the offset in the byte array of the first byte of a particular line
357 */
358 public int getLineIdx( int lineNo ) {
359 return( lineIdx[ lineNo ] );
360 }
361
362 /** Get the number of bytes occupied by a line, including 0 bytes,
363 * and assuming that next line immediately follows.
364 */
365 public int getLineSize( int lineNo ) {
366 if ( lineNo == ( numberLines - 1 )) {
367 return( fileDataIdx - lineIdx[ lineNo ] );
368 } else {
369 return( lineIdx[ lineNo + 1 ] - lineIdx[ lineNo ] );
370 }
371 }
372
373 // This version is going with the very simple form of one byte per character
374 // with new line characters represented by 0 byte
375 protected byte[] fileData;
376 protected int fileDataIdx = 0;
377 protected int byteArrayBoundary;
378
379 /**
380 * Get the byte array used to store file text.
381 */
382 public byte[] getFileData() {
383 return( fileData );
384 }
385
386
387 /** Get the number of bytes used in the byte array. */
388 public int getFileDataIdx() {
389 return( fileDataIdx );
390 }
391
392 /** Set the next free location in the byte array */
393 public void setFileDataIdx( int fileDataIdx ) {
394 this.fileDataIdx = fileDataIdx;
395 }
396
397 /**
398 * set all array objects to null, so they can be garbage collected
399 */
400 public void wipeout() {
401 fileData = null;
402 lineIdx = null;
403 lineNumberArray = null;
404 lineOffsetArray = null;
405 lineBreakArray = null;
406 tokenSizeArray = null;
407 }
408
409 /** Make token be the first token in a line */
410 public void startLineWith( int tokenOffset ) {
411 if ( tokenOffset == 0 ) {
412 return;
413 }
414 int tokenLine = lineNumberArray[ tokenOffset ];
415 int prevTokenLine = lineNumberArray[ tokenOffset - 1 ];
416 if ( prevTokenLine != tokenLine ) {
417 return;
418 }
419
420 // split the lines
421 int tokenByteOffset = getByteOffset( tokenOffset );
422 int tokenLineOffset = lineOffsetArray[ tokenOffset ];
423 boolean splitOnSpace = true;
424 if ( tokenByteOffset == ( getByteOffset( tokenOffset - 1 ) + getSize( tokenOffset - 1 ))) {
425 splitOnSpace = false;
426 }
427 split_line( tokenLine, tokenLineOffset, splitOnSpace );
428
429 // now all the line numbers of all tokens have to be adjusted
430 for ( int i = tokenOffset; i < numberTokens; i++ ) {
431 lineNumberArray[ i ] += 1;
432 }
433
434 // all the tokens on the new line have to be adjusted so that first token is at offset 0
435 int scanner = tokenOffset;
436 while ( lineNumberArray[ scanner ] == ( tokenLine + 1 )) {
437 lineOffsetArray[ scanner ] -= tokenLineOffset;
438 scanner++;
439 }
440 }
441
442 /** split a line at a particualr offset
443 *
444 * @param tokenLine the line to split
445 * @param tokenLineOffset offset into the bytes where to split, takes into
446 * account rle of spaces
447 * @param splitOnSpace
448 */
449 protected void split_line( int tokenLine, int tokenLineOffset, boolean splitOnSpace ) {
450 dirtyFileData = true;
451 int tokenByteOffset = lineIdx[ tokenLine ] + tokenLineOffset;
452 insertLine( tokenLine + 1, tokenByteOffset );
453
454 // at this point, lineIdx is set correctly. However, if line is split
455 // where there is no space between tokens, we have to shift fileData
456 // by one byte, and adjust all lineIdx entries
457 if ( !splitOnSpace ) {
458 rightShiftBytes( tokenByteOffset, 1 );
459 for ( int i = tokenLine + 1; i < numberLines; i++ ) {
460 lineIdx[i] += 1;
461 }
462 fileData[ tokenByteOffset ] = 0;
463 } else {
464 fileData[ tokenByteOffset - 1 ] = 0;
465 }
466 }
467
468 /** Insert one token of specific type */
469 public void insert( int tokenOffset, String tok1, byte tokType ) {
470 int len1 = tok1.length();
471 int length = len1 + 1;
472 ensureByteCapacity( length );
473 ensureTokenCapacity( 1 );
474 rightShiftTokens( tokenOffset, 1 );
475 int tokenByteOffset = getByteOffset( tokenOffset );
476 rightShiftBytes( tokenByteOffset, length );
477 for ( int i = 0; i < len1; i++ ) {
478 fileData[ tokenByteOffset++ ] = (byte)tok1.charAt( i );
479 }
480 fileData[ tokenByteOffset++ ] = (byte)' ';
481 tokenTypeArray[ tokenOffset ] = tokType;
482 tokenSizeArray[ tokenOffset ] = (short)len1;
483 int scanner = tokenOffset + 1;
484 int insertionLine = lineNumberArray[ tokenOffset ];
485 while ( lineNumberArray[ scanner ] == insertionLine ) {
486 lineOffsetArray[ scanner ] += length;
487 scanner++;
488 if ( scanner >= numberTokens ) {
489 break;
490 }
491 }
492 for ( int i = insertionLine + 1; i < numberLines; i++ ) {
493 lineIdx[i] += length;
494 }
495 }
496
497 /** Insert two tokens */
498 public void insert( int tokenOffset, String tok1, String tok2 ) {
499 int len1 = tok1.length();
500 int len2 = tok2.length();
501 int length = len1 + len2 + 2;
502 ensureByteCapacity( length );
503 ensureTokenCapacity( 2 );
504 rightShiftTokens( tokenOffset, 2 );
505 int tokenByteOffset = getByteOffset( tokenOffset );
506 rightShiftBytes( tokenByteOffset, length );
507 for ( int i = 0; i < len1; i++ ) {
508 fileData[ tokenByteOffset++ ] = (byte)tok1.charAt( i );
509 }
510 fileData[ tokenByteOffset++ ] = (byte)' ';
511 for ( int i = 0; i < len2; i++ ) {
512 fileData[ tokenByteOffset++ ] = (byte)tok2.charAt( i );
513 }
514 fileData[ tokenByteOffset++ ] = (byte)' ';
515 lineBreakArray[ tokenOffset ] = true;
516 tokenTypeArray[ tokenOffset ] = NameToken;
517 tokenSizeArray[ tokenOffset ] = (short)len1;
518 int insertionLine = lineNumberArray[ tokenOffset ];
519 lineNumberArray[ tokenOffset + 1 ] = insertionLine;
520 lineOffsetArray[ tokenOffset + 1 ] = (short)(lineOffsetArray[ tokenOffset ] + len1 + 1);
521 lineBreakArray[ tokenOffset + 1 ] = false;
522 tokenTypeArray[ tokenOffset + 1 ] = NameToken;
523 tokenSizeArray[ tokenOffset + 1 ] = (short)len2;
524 int scanner = tokenOffset + 2;
525 while ( lineNumberArray[ scanner ] == insertionLine ) {
526 lineOffsetArray[ scanner ] += length;
527 scanner++;
528 if ( scanner >= numberTokens ) {
529 break;
530 }
531 }
532 for ( int i = insertionLine + 1; i < numberLines; i++ ) {
533 lineIdx[i] += length;
534 }
535 }
536
537 /** Replace a token with a specific String value */
538 public void replace( int tokenOffset, String newToken ) {
539 // new length - original length is the change
540 int newTokenLength = newToken.length();
541 int lengthChange = newTokenLength - tokenSizeArray[ tokenOffset ];
542 ensureByteCapacity( lengthChange );
543 int tokenByteOffset = getByteOffset( tokenOffset );
544 if ( lengthChange > 0 ) {
545 rightShiftBytes( tokenByteOffset, lengthChange );
546 } else {
547 for ( int i = tokenByteOffset; i < fileDataIdx; i++ ) {
548 fileData[ i ] = fileData[ i - lengthChange ];
549 if ( fileData[i] == 0 ) {
550 break;
551 }
552 }
553 }
554 for ( int i = 0; i < newTokenLength; i++ ) {
555 fileData[ tokenByteOffset++ ] = (byte)newToken.charAt( i );
556 }
557 tokenSizeArray[ tokenOffset ] = (short)newTokenLength;
558 int insertionLine = lineNumberArray[ tokenOffset ];
559 int scanner = tokenOffset + 1;
560 while ( lineNumberArray[ scanner ] == insertionLine ) {
561 lineOffsetArray[ scanner ] += lengthChange;
562 scanner++;
563 if ( scanner >= numberTokens ) {
564 break;
565 }
566 }
567 if ( lengthChange > 0 ) {
568 for ( int i = insertionLine + 1; i < numberLines; i++ ) {
569 lineIdx[i] += lengthChange;
570 }
571 }
572 }
573
574
575 /** Shift all token arrays right by the specified amount starting at a specific boundary */
576 void rightShiftTokens( int boundary, int amount ) {
577 for ( int i = numberTokens - 1; i >= boundary; i-- ) {
578 lineNumberArray[ i + amount ] = lineNumberArray[i];
579 lineOffsetArray[ i + amount ] = lineOffsetArray[i];
580 lineBreakArray[ i + amount ] = lineBreakArray[i];
581 tokenTypeArray[ i + amount ] = tokenTypeArray[i];
582 tokenSizeArray[ i + amount ] = tokenSizeArray[i];
583 }
584 numberTokens += amount;
585 }
586
587
588 /** Shift all data bytes right by the specified amount starting at a specific boundary */
589 void rightShiftBytes( int boundary, int amount ) {
590 // first right shift all the fileData
591 for ( int i = fileDataIdx - 1; i >= boundary; i-- ) {
592 fileData[ i + amount ] = fileData[ i ];
593 }
594 fileDataIdx += amount;
595 }
596
597 /**
598 * copy array objects from one TokenEnumerator to another
599 */
600 public void snarfArrays( TokenEnumerator source ) {
601 // fileData = source.getFileData();
602 // lineIdx = source.getLineIdx();
603 lineNumberArray = source.getLineNumberArray();
604 lineOffsetArray = source.getLineOffsetArray();
605 lineBreakArray = source.getLineBreakArray();
606 tokenTypeArray = source.getTokenTypeArray();
607 tokenSizeArray = source.getTokenSizeArray();
608 tokenArrayBoundary = source.getTokenArrayBoundary();
609 lineArrayBoundary = source.getLineArrayBoundary();
610 // byteArrayBoundary = source.getByteArrayBoundary();
611 }
612
613
614 /** get the number of bytes available in the byte array */
615 public int getByteArrayBoundary() {
616 return( byteArrayBoundary );
617 }
618 public int getByteArraySize() {
619 return( fileDataIdx + 10000 );
620 }
621
622 String fileUrl;
623 public String getFileUrl() {
624 return( fileUrl );
625 }
626 public void setFileUrl( String fileUrl ) {
627 this.fileUrl = fileUrl;
628 }
629
630 /** Class constructor
631 *
632 * @param is InputStream containing the data
633 * @param inFile url used to create the InputStream
634 */
635 public TokenEnumerator( InputStream is, String inFile ) throws FileNotFoundException, IOException {
636 this( is, inFile, null, null );
637 }
638
639 /** Class constructor
640 *
641 * @param is InputStream containing the data
642 * @param inFile url used to create the InputStream
643 * @param allowUnterminatedString if true, QuotedStringContinuation tokens are allowed,
644 * otherwise, QuotedStrings are automatically ended on the line they start on even if there is
645 * no matching quote
646 */
647 public TokenEnumerator( InputStream is, String inFile, boolean allowUnterminatedString ) throws FileNotFoundException, IOException {
648 this( is, inFile, null, null, allowUnterminatedString );
649 }
650
651 static public int lastId = 0;
652 int myId;
653
654 /** Class constructor
655 *
656 * @param is InputStream containing the data
657 * @param inFile url used to create the InputStream
658 * @param frl callback to give progress
659 * @param fileSource File object if a local file
660 */
661 public TokenEnumerator( InputStream is, String inFile, ProgressIndicator frl, File fileSource ) throws FileNotFoundException, IOException {
662 this( is, inFile, frl, fileSource, true );
663 }
664
665 public TokenEnumerator( InputStream is, String inFile, ProgressIndicator frl, File fileSource, boolean allowUnterminatedString ) throws FileNotFoundException, IOException {
666 lastId++;
667 myId = lastId;
668 fileUrl = inFile;
669 long fileLength = 0;
670 if ( fileSource != null ) {
671 fileLength = fileSource.length();
672 if ( fileLength < InitialByteArraySize ) {
673 fileLength += fileLength/2;
674 } else {
675 fileLength = ( fileSource.length()/100000 ) * 100000 + 100000;
676 }
677 }
678 if ( fileLength == 0 ) {
679 fileLength = InitialByteArraySize;
680 }
681 init( fileLength );
682 loadLines( is, frl, allowUnterminatedString );
683 }
684
685 /** enumerate sequence of tokens from a string */
686 public TokenEnumerator( String s ) {
687 init( SmallInitialByteArraySize, SmallInitialTokenArraySize, 3 );
688 TokenFactory t = new TokenFactory();
689 addLine( s, t );
690 tokenScannerOffset = -1;
691 }
692
693 /** used when recreating token enumerator based on old one */
694 public TokenEnumerator() {
695 lastId++;
696 myId = lastId;
697 init( InitialByteArraySize, InitialTokenArraySize, InitialLineArraySize );
698 }
699
700 /** Create a token enumerator with built in arrays of a specific size */
701 public TokenEnumerator( int byteArraySize, int tokenArraySize, int lineArraySize ) {
702 init( byteArraySize, tokenArraySize, lineArraySize );
703 }
704
705 /** Create one tokenEnumerator using the same arrays as another.
706 * Original intent was to allow arrays to be reused during reformatting,
707 * however, this only works if same comment skipping strategy in place,
708 * if not, there is risk that arrays wipe out each other.
709 */
710 public TokenEnumerator( TokenEnumerator source ) {
711 lastId++;
712 myId = lastId;
713 snarfArrays( source );
714 if (( source.getByteArrayBoundary() - source.getFileDataIdx() ) < InitialByteArraySize ) {
715 fileData = new byte[ source.getByteArrayBoundary() + InitialByteArraySize ];
716 byteArrayBoundary = source.getByteArrayBoundary();
717 } else {
718 fileData = new byte[ source.getByteArrayBoundary() ];
719 byteArrayBoundary = source.getByteArrayBoundary();
720 }
721 lineIdx = new int[ source.getLineArrayBoundary() ];
722 fileUrl = source.getFileUrl();
723 }
724
725 /** Initialize all array sizes based on file length, initialize byte array also.
726 *
727 * @param byteArraySize estimated size of the byte array
728 */
729 void init( long byteArraySize ) {
730 init( byteArraySize, true );
731 }
732
733 /** Initialize all array sizes based on file length
734 *
735 * @param byteArraySize estimated size of byte array
736 * @param initializeByteArray true if the byte array is initialized also,
737 * it is not initialized in the case of reading from serialized version.
738 */
739 void init( long byteArraySize, boolean initializeByteArray ) {
740 int tokLength = (int)byteArraySize/4;
741 if ( tokLength <= 0 ) {
742 tokLength = 100;
743 }
744 int lineLength = (int)byteArraySize/6;
745 if ( lineLength <= 0 ) {
746 lineLength = 50;
747 }
748 if ( !initializeByteArray ) {
749 lineLength = lineArrayBoundary;
750 }
751 init( byteArraySize, tokLength, lineLength, initializeByteArray );
752 }
753
754 /** Initialize all arrays
755 *
756 * @param byteArraySize estimated size of byte array
757 * @param tokenArraySize estimated size of toke array
758 * @param lineArraySize estimated number of lines
759 */
760 void init( long byteArraySize, int tokenArraySize, int lineArraySize ) {
761 init( byteArraySize, tokenArraySize, lineArraySize, true );
762 }
763
764 /** Initialize all arrays, with possible exception of byte array.
765 *
766 * @param byteArraySize estimated size of byte array
767 * @param tokenArraySize estimated size of token array
768 * @param lineArraySize estimated number of lines
769 * @param initializeByteArray true if the byte array is to be initialized also.
770 */
771 void init( long byteArraySize, int tokenArraySize, int lineArraySize, boolean initializeByteArray ) {
772 try {
773 tokenScannerOffset = 0;
774 numberTokens = 0;
775 lineNumberArray = new int[ tokenArraySize ];
776 lineOffsetArray = new short[ tokenArraySize ];
777 lineBreakArray = new boolean[ tokenArraySize ];
778 tokenTypeArray = new byte[ tokenArraySize ];
779 tokenSizeArray = new short[ tokenArraySize ];
780 tokenArrayBoundary = tokenArraySize;
781 lineArrayBoundary = lineArraySize;
782 if ( initializeByteArray ) {
783 fileData = new byte[(int)byteArraySize];
784 lineIdx = new int[ lineArraySize ];
785 numberLines = 0;
786 }
787 byteArrayBoundary = (int)byteArraySize;
788 } catch ( Exception e ) {
789 System.out.println( "Exception " + e );
790 }
791 }
792
793 /** Read object form ObjectInputStream, and reconstruct transient fields.
794 *
795 */
796 private void readObject( java.io.ObjectInputStream stream ) throws java.io.IOException {
797 try {
798 stream.defaultReadObject();
799 init( fileData.length, false );
800 byteString = new ByteString();
801 lineReporter = new LineReporter();
802 retokenize();
803 } catch ( Exception e ) {
804 e.printStackTrace();
805 }
806 }
807
808 static public long presetLength = 0;
809 void loadLines( InputStream inStream, ProgressIndicator frl, boolean allowUnterminatedString ) {
810 TokenFactory t = new TokenFactory( allowUnterminatedString );
811 long sizeInBytes = 0;
812 try {
813 LineReader lr = new LineReader( inStream );
814 int counter = 0;
815 while ( true ) {
816 String s = lr.readLine();
817 if ( s == null ) {
818 break;
819 } else if ( counter == 0 ) {
820 // funny case where zero length file has bogus line result with single char 0xff
821 if ( s.length() == 1 ) {
822 break;
823 }
824 }
825 addLine( s, t );
826 sizeInBytes += s.length() + 2;
827 if ( counter == 100 ) {
828 if (( frl != null ) && ( presetLength != 0 )) {
829 frl.setPercent( (int)(sizeInBytes*100/presetLength));
830 }
831 counter = 0;
832 }
833 counter++;
834 if ( GlobalProgressIndicator.abortCurrentProcess ) {
835 return;
836 }
837 }
838 } catch( Exception e ) {
839 try {
840 e.printStackTrace();
841 inStream.close();
842 if ( frl != null ) {
843 lineReporter.setLineCount( numberLines );
844 }
845 } catch( Exception e2 ) {
846 }
847 }
848 }
849
850
851
852 /** Retokenize the data in the "fileData" byte array. */
853 public void retokenize() {
854 TokenFactory t = new TokenFactory();
855 int replacementLineNumber = 0;
856 numberTokens = 0;
857 for ( int i = 0; i < numberLines; i++ ) {
858 if ( processLine( t, lineIdx[ i ], replacementLineNumber )) {
859 lineIdx[ replacementLineNumber ] = lineIdx[ i ];
860 replacementLineNumber++;
861 }
862 }
863 numberLines = replacementLineNumber;
864 tokenScannerOffset = 0;
865 dirtyFileData = false;
866 }
867
868 public void addLineCapacity() {
869 if ( numberLines == lineArrayBoundary ) {
870 increaseLineCapacity();
871 }
872 }
873
874 /** Increase the capacity of the lineIdx[] array */
875 void increaseLineCapacity() {
876 int newSize = lineArrayBoundary + InitialLineArraySize;
877 int[] temp = new int[ newSize ];
878 System.arraycopy( lineIdx, 0, temp, 0, lineArrayBoundary );
879 lineIdx = temp;
880 lineArrayBoundary = newSize;
881 }
882
883 /** Make sure the lineIdx[] array has enough room for <B>nLines</B> more lines.
884 *
885 * @param nLines the number of new lines to add
886 */
887 public void ensureLineCapacity( int nLines ) {
888 if (( numberLines + nLines ) >= lineArrayBoundary ) {
889 increaseLineCapacity();
890 }
891 }
892
893 void ensureTokenCapacity( int nTokens ) {
894 if (( numberTokens + nTokens ) >= tokenArrayBoundary ) {
895 int newSize = tokenArrayBoundary + InitialTokenArraySize;
896 int[] temp = new int[ newSize ];
897 System.arraycopy( lineNumberArray, 0, temp, 0, tokenArrayBoundary );
898 lineNumberArray = temp;
899 temp = null;
900 System.gc();
901 short[] stemp = new short[ newSize ];
902 System.arraycopy( lineOffsetArray, 0, stemp, 0, tokenArrayBoundary );
903 lineOffsetArray = stemp;
904 stemp = null;
905 System.gc();
906 byte[] bytetemp = new byte[ newSize ];
907 System.arraycopy( tokenTypeArray, 0, bytetemp, 0, tokenArrayBoundary );
908 tokenTypeArray = bytetemp;
909 bytetemp = null;
910 System.gc();
911 stemp = new short[ newSize ];
912 System.arraycopy( tokenSizeArray, 0, stemp, 0, tokenArrayBoundary );
913 tokenSizeArray = stemp;
914 stemp = null;
915 System.gc();
916 boolean[] btemp = new boolean[ newSize ];
917 System.arraycopy( lineBreakArray, 0, btemp, 0, tokenArrayBoundary );
918 lineBreakArray = btemp;
919 tokenArrayBoundary = newSize;
920 temp = null;
921 btemp = null;
922 System.gc();
923 }
924 }
925
926 /** Add a line to the list of lines */
927 public void addLine( String line, TokenFactory t ) {
928 addLineCapacity();
929 lineIdx[ numberLines ] = fileDataIdx;
930 int len = line.length();
931 // System.out.println( "ensure byte capacity " + len );
932 ensureByteCapacity( len );
933 // rle encode leading spaces
934 int spaceCount = 0;
935 for ( int i = 0; i < len; i++ ) {
936 char x = line.charAt( i );
937 if (( x == ' ' ) || ( x == '\t' )) {
938 spaceCount++;
939 } else {
940 break;
941 }
942 }
943 if ( spaceCount > 0 ) {
944 if ( spaceCount > 100 ) {
945 spaceCount = 100;
946 }
947 fileData[ fileDataIdx++ ] = (byte)(spaceCount | 128);
948 }
949 // System.out.println( "spaceCount is " + spaceCount + ", fileDataIdx " + fileDataIdx + ", byteArrayBoundary " + byteArrayBoundary );
950 int internalCount = 0;
951 int internalLine = 0;
952 for ( int i = spaceCount; i < len; i++ ) {
953 fileData[ fileDataIdx++ ] = (byte)line.charAt( i );
954 internalCount++;
955 if (( internalCount > 1000 ) && ( line.charAt( i ) == ' ' )) {
956 internalLine++;
957 //System.out.println( "internal line " + internalLine );
958 fileData[ fileDataIdx++ ] = 0;
959 byteString.setup( fileData, lineIdx[ numberLines ] );
960 processLine( t, byteString, numberLines );
961 numberLines++;
962 addLineCapacity();
963 lineIdx[ numberLines ] = fileDataIdx;
964 internalCount = 0;
965 }
966 }
967 fileData[ fileDataIdx++ ] = 0;
968 byteString.setup( fileData, lineIdx[ numberLines ] );
969 processLine( t, byteString, numberLines );
970 numberLines++;
971 }
972
973 /** Add a line to the list of lines */
974 public void addLine( StringBuffer line, TokenFactory t ) {
975 addLineCapacity();
976 lineIdx[ numberLines ] = fileDataIdx;
977 int len = line.length();
978 ensureByteCapacity( len );
979 // rle encode leading spaces
980 int spaceCount = 0;
981 for ( int i = 0; i < len; i++ ) {
982 char x = line.charAt( i );
983 if (( x == ' ' ) || ( x == '\t' )) {
984 spaceCount++;
985 } else {
986 break;
987 }
988 }
989 if ( spaceCount > 0 ) {
990 if ( spaceCount > 100 ) {
991 spaceCount = 100;
992 }
993 fileData[ fileDataIdx++ ] = (byte)(spaceCount | 128);
994 }
995 for ( int i = spaceCount; i < len; i++ ) {
996 fileData[ fileDataIdx++ ] = (byte)line.charAt( i );
997 }
998 fileData[ fileDataIdx++ ] = 0;
999 byteString.setup( fileData, lineIdx[ numberLines ] );
1000 processLine( t, byteString, numberLines );
1001 numberLines++;
1002 }
1003
1004 /** Optimized add line, takes bytes from source array.
1005 *
1006 * @param offset offset into source array
1007 * @param len number of bytes occupied by original line, including terminating byte
1008 * @param sourceArray byte array containing text of original line
1009 * @param t TokenFactory required by TokenEnumerator
1010 */
1011 public void addLine( int offset, int len, byte[] sourceArray, TokenFactory t ) {
1012 addLineCapacity();
1013 lineIdx[ numberLines ] = fileDataIdx;
1014 ensureByteCapacity( len );
1015 System.arraycopy( sourceArray, offset, fileData, fileDataIdx, len );
1016 fileDataIdx += len;
1017// fileData[ fileDataIdx++ ] = 0;
1018 processLine( t, lineIdx[ numberLines ], numberLines );
1019 numberLines++;
1020 }
1021
1022 public void addLine( int offset, int len, byte[] sourceArray,
1023 int numberSourceTokens, int sourceTokenOffset,
1024 short[] sourceLineOffsetArray, short[] sourceTokenSizeArray,
1025 boolean[] sourceLineBreakArray, byte[] sourceTokenTypeArray ) {
1026 addLineCapacity();
1027 lineIdx[ numberLines ] = fileDataIdx;
1028 ensureByteCapacity( len );
1029 System.arraycopy( sourceArray, offset, fileData, fileDataIdx, len );
1030 fileDataIdx += len;
1031 if ( numberSourceTokens > 0 ) {
1032 ensureTokenCapacity( numberSourceTokens );
1033 for ( int i = 0; i < numberSourceTokens; i++ ) {
1034 lineNumberArray[numberTokens + i] = numberLines;
1035 }
1036// System.out.println( "addLine numberSourceTokens " + numberSourceTokens + ", sourceTokenOffset " + sourceTokenOffset );
1037// System.out.println( "numberTokens " + numberTokens );
1038// System.out.println( "sourceLineOffsetArray size is " + sourceLineOffsetArray.length );
1039// System.out.println( "lineOffsetArray size is " + lineOffsetArray.length );
1040 System.arraycopy( sourceLineOffsetArray, sourceTokenOffset,
1041 lineOffsetArray, numberTokens, numberSourceTokens );
1042 System.arraycopy( sourceTokenSizeArray, sourceTokenOffset,
1043 tokenSizeArray, numberTokens, numberSourceTokens );
1044 System.arraycopy( sourceLineBreakArray, sourceTokenOffset,
1045 lineBreakArray, numberTokens, numberSourceTokens );
1046 System.arraycopy( sourceTokenTypeArray, sourceTokenOffset,
1047 tokenTypeArray, numberTokens, numberSourceTokens );
1048 numberTokens += numberSourceTokens;
1049 }
1050 numberLines++;
1051 }
1052
1053
1054 /** Make sure the fileData[] byte array has enough room for <B>len</B> more bytes.
1055 *
1056 * @param len number of bytes more that are to be added
1057 */
1058 public void ensureByteCapacity( int len ) {
1059 len++;
1060 while (( fileDataIdx + len ) >= byteArrayBoundary ) {
1061 int newSize = byteArrayBoundary;
1062 if ( byteArrayBoundary > 300000 ) {
1063 newSize += 300000;
1064 } else {
1065 newSize = newSize * 2;
1066 }
1067 byte[] btemp = new byte[ newSize ];
1068 System.arraycopy( fileData, 0, btemp, 0, byteArrayBoundary );
1069 fileData = btemp;
1070 btemp = null;
1071 Runtime.getRuntime().gc();
1072 byteArrayBoundary = newSize;
1073 }
1074 }
1075
1076 public void lineDump() {
1077 System.out.println( numberLines + " lines." );
1078 for ( int i = 0; i < numberLines; i++ ) {
1079 System.out.println( (i+1) + " " + lineIdx[i] + ": " + getLineAt( i ));
1080 }
1081 }
1082
1083 /** save lines to a PrintStream */
1084 public void saveLines( PrintStream ps ) {
1085 for ( int i = 0; i < numberLines; i++ ) {
1086 String s = getLineAt( i );
1087 ps.println( s );
1088 }
1089 }
1090
1091 /** debugging dump of TokenEnumerator */
1092 public void dump() {
1093 System.out.println( "TokenEnumerator dump: FileDataIdx is " + fileDataIdx );
1094 int scanner = 0;
1095 StringBuffer buf = new StringBuffer();
1096 while ( scanner < fileDataIdx ) {
1097 if ( fileData[ scanner ] == 0 ) {
1098 String s = new String( buf );
1099 System.out.println( s );
1100 buf = new StringBuffer();
1101 } else if (( fileData[ scanner ] & 128 ) != 0 ) {
1102 int rleSpace = fileData[ scanner ] & 127;
1103 for ( int i = 0; i < rleSpace; i++ ) {
1104 buf.append( ' ' );
1105 }
1106 } else {
1107 buf.append( (char)fileData[ scanner ] );
1108 }
1109 scanner++;
1110 }
1111 System.out.println( "Done with TokenEnumerator dump" );
1112 }
1113
1114 public void detailDump( int tokenOffset ) {
1115 System.out.println( "token: " + tokenOffset + ", '" + toString( tokenOffset ) + "'" );
1116 System.out.println( "..isLineBreak: " + isLineBreak( tokenOffset ));
1117 System.out.println( "..line " + getLineNumber( tokenOffset ) + ", offset " + getLineOffset( tokenOffset ));
1118 System.out.println( "..line is '" + getLineAt( getLineNumber( tokenOffset )) + "'" );
1119 System.out.println( "..type is " + getTokenType( tokenOffset ));
1120 }
1121
1122 public void detailDump() {
1123 detailDump( System.out, true );
1124 }
1125
1126 public void detailDump( boolean details ) {
1127 detailDump( System.out, details );
1128 }
1129
1130 /** Dump detailed TokenEnumerator data to a file.
1131 *
1132 * @param fileName file to dump to
1133 */
1134 public void detailDump( String fileName ) {
1135 try {
1136 detailDump( new PrintStream( new FileOutputStream( fileName )), true );
1137 } catch ( Exception e ) {
1138 }
1139 }
1140
1141 void detailDump( PrintStream ps, boolean allDetails ) {
1142 for ( int i = 0; i < numberLines; i++ ) {
1143 ps.println( (i+1) + ": " + getLineAt( i ));
1144 }
1145 if ( allDetails ) {
1146/* for ( int i = 0; i < fileDataIdx; i++ ) {
1147 if ( fileData[i] != 0 ) {
1148 if (( fileData[i] & 128 ) != 0 ) {
1149 ps.println( i + ": rle " + (fileData[i] & 127) + " spaces" );
1150 } else {
1151 char c = (char)fileData[i];
1152 ps.println( i + ": '" + c + "'" );
1153 }
1154 } else {
1155 ps.println( "zero" );
1156 }
1157 }*/
1158 for ( int i = 0; i < numberLines; i++ ) {
1159 ps.println( "line " + i + " at " + lineIdx[i] );
1160 }
1161 for ( int i = 0; i < numberTokens; i++ ) {
1162 ps.println( "token " + i + ", line " + getLineNumber( i ) + ", lineoffset " + getLineOffset( i ) + ", type " + getTokenType( i ) + ", value '" + toString( i ) + "'" );
1163// ps.println( "token " + i + ", line " + getLineNumber( i ) + ", lineoffset " + getLineOffset( i ) + ", type " + getTokenType( i ));
1164 }
1165 }
1166 }
1167
1168 //
1169 // add a token to the token enumerator, which updates the lineNumberArray,
1170 // lineOffsetArray, tokenTypeArray, and tokenSizeArray. The lineBreakArray
1171 // is for formatting only, and is updated in the specific VrmlElement subclasses
1172 // through a call to "breakLineAt".
1173 //
1174 void addToken( int lineNumber, byte type, int offset, int size ) {
1175 if ( type == QuotedStringContinuation ) {
1176 offset = 0;
1177 }
1178 ensureTokenCapacity( 1 );
1179 lineNumberArray[ numberTokens ] = lineNumber;
1180 lineOffsetArray[ numberTokens ] = (short)offset;
1181 tokenSizeArray[ numberTokens ] = (short)size;
1182 lineBreakArray[ numberTokens ] = false;
1183 tokenTypeArray[ numberTokens ] = type;
1184 numberTokens++;
1185 }
1186
1187 /** mark the token as one that is associated with a line break */
1188 public void breakLineAt( int offset ) {
1189 lineBreakArray[ offset ] = true;
1190 }
1191
1192 /** is the token associated with a line break? */
1193 public boolean isLineBreak( int offset ) {
1194 return( lineBreakArray[ offset ] );
1195 }
1196
1197 /** this class is not used by Vorlon, part of callback progress reporting */
1198 class LineReporter {
1199 ProgressIndicator frl = null;
1200 int lastReportedLine;
1201 int lineCount;
1202 int lastPercent;
1203
1204 public void setProgressIndicator( ProgressIndicator l ) {
1205 frl = l;
1206 lastReportedLine = 0;
1207 lastPercent = 0;
1208 }
1209
1210 public void setLineCount( int lineCount ) {
1211 this.lineCount = lineCount;
1212 }
1213
1214 public void report( int lineNumber ) {
1215 if (( frl != null ) && ( lineCount != 0 )) {
1216 int currentPercent = lineNumber*100/lineCount;
1217 if ( currentPercent != lastPercent ) {
1218 frl.setPercent( lastPercent );
1219 lastPercent = currentPercent;
1220 }
1221 }
1222 }
1223 }
1224
1225 transient LineReporter lineReporter = new LineReporter();
1226
1227 /** get the String object for a line at a particular offset */
1228 public String getLineAt( int offset ) {
1229 if ( offset >= numberLines ) {
1230 return( null );
1231 }
1232 return( getString( lineIdx[ offset ], false ));
1233 }
1234
1235 /** get the String object for a line at a particular offset, but limit size */
1236 public String getLineAt( int offset, int sizeLimit ) {
1237 if ( offset >= numberLines ) {
1238 return( null );
1239 }
1240// System.out.println( "getLineAt " + offset + " with sizeLimit " + sizeLimit );
1241 return( getString( lineIdx[ offset ], sizeLimit ));
1242 }
1243
1244 public String getNospaceLineAt( int offset ) {
1245 if ( offset >= numberLines ) {
1246 return( null );
1247 }
1248 return( getNospaceString( lineIdx[ offset ] ));
1249 }
1250
1251 public String getTabLineAt( int offset ) {
1252 if ( offset >= numberLines ) {
1253 return( null );
1254 }
1255 return( getString( lineIdx[ offset ], true ));
1256 }
1257
1258 String getNospaceString( int lineByteOffset ) {
1259 StringBuffer sb = new StringBuffer();
1260 while ( fileData[ lineByteOffset ] != 0 ) {
1261 if (( fileData[ lineByteOffset ] & 128 ) != 0 ) {
1262 lineByteOffset++;
1263 continue;
1264 } else if ( fileData[ lineByteOffset ] == (byte)'\t' ) {
1265 sb.append( ' ' );
1266 } else {
1267 sb.append( (char)fileData[ lineByteOffset ] );
1268 }
1269 lineByteOffset++;
1270 }
1271 return( new String( sb ));
1272 }
1273
1274 /** Get the String at a particular offset in the byte array */
1275 String getString( int lineByteOffset, boolean spaceToTab ) {
1276 StringBuffer sb = new StringBuffer();
1277 int firstByteOffset = lineByteOffset;
1278 while ( fileData[ lineByteOffset ] != 0 ) {
1279 if (( firstByteOffset == lineByteOffset ) && (( fileData[ lineByteOffset ] & 128 ) != 0 )) {
1280 int rleSpace = fileData[ lineByteOffset ] & 127;
1281 if (( tabIndentSize == 0 ) || !spaceToTab ) {
1282 for ( int i = 0; i < rleSpace; i++ ) {
1283 sb.append( ' ' );
1284 }
1285 } else {
1286 int ntabs = rleSpace/tabIndentSize;
1287 int remainderSpaces = rleSpace%tabIndentSize;
1288 for ( int i = 0; i < ntabs; i++ ) {
1289 sb.append( '\t' );
1290 }
1291 for ( int i = 0; i < remainderSpaces; i++ ) {
1292 sb.append( ' ' );
1293 }
1294 }
1295 } else if ( fileData[ lineByteOffset ] == (byte)'\t' ) {
1296 sb.append( ' ' );
1297 } else {
1298 if (( fileData[ lineByteOffset ] & 128 ) != 0 ) {
1299 int ival = fileData[ lineByteOffset ] & 127;
1300 ival += 128;
1301 System.out.println( "byte is " + ival );
1302 }
1303 sb.append( (char)fileData[ lineByteOffset ] );
1304 }
1305 lineByteOffset++;
1306 }
1307 return( new String( sb ));
1308 }
1309
1310 String getString( int lineByteOffset, int sizeLimit ) {
1311 StringBuffer sb = new StringBuffer();
1312 int currentSize = 0;
1313 while ( fileData[ lineByteOffset ] != 0 ) {
1314 if (( fileData[ lineByteOffset ] & 128 ) != 0 ) {
1315 int rleSpace = fileData[ lineByteOffset ] & 127;
1316 for ( int i = 0; i < rleSpace; i++ ) {
1317 sb.append( ' ' );
1318 }
1319 currentSize += rleSpace;
1320 } else if ( fileData[ lineByteOffset ] == (byte)'\t' ) {
1321 sb.append( ' ' );
1322 currentSize++;
1323 } else {
1324 sb.append( (char)fileData[ lineByteOffset ] );
1325 currentSize++;
1326 }
1327 lineByteOffset++;
1328 if ( currentSize >= sizeLimit ) {
1329 break;
1330 }
1331 }
1332 return( new String( sb ));
1333 }
1334
1335 /** compare the current token to one in another TokenEnumerator
1336 *
1337 * @return true if the two tokens are identical, otherwise false.
1338 */
1339 public boolean sameAs( TokenEnumerator other ) {
1340 int l1 = length();
1341 int l2 = other.length();
1342 if ( l1 != l2 ) {
1343 return( false );
1344 }
1345 for ( int i = 0; i < l1; i++ ) {
1346 if ( charAt( i ) != other.charAt( i )) {
1347 return( false );
1348 }
1349 }
1350 return( true );
1351 }
1352
1353 /** Get the line number of a token */
1354 public int getLineNumber( int offset ) {
1355 return( lineNumberArray[ offset ] );
1356 }
1357
1358 public void incLineNumbers( int line ) {
1359// for ( int i = 0; i < numberTokens; i++ ) {
1360// System.out.println( "before " + i + " line is " + getLineNumber( i ));
1361// }
1362 for ( int i = numberTokens - 1; i >= 0; i-- ) {
1363 int lno = lineNumberArray[ i ];
1364 if ( lno >= line ) {
1365 lno++;
1366 lineNumberArray[i] = lno;
1367 } else {
1368 break;
13