Source code: jsource/syntax/tokenmarker/TokenMarker.java
1 /*
2 * TokenMarker.java - Generic token marker
3 * Copyright (C) 1998, 1999 Slava Pestov
4 * Revised for JSource 2002 Panagiotis Plevrakis
5 *
6 * You may use and modify this package for any purpose. Redistribution is
7 * permitted, in both source and binary form, provided that this notice
8 * remains intact in all source distributions of this package.
9 */
10
11 package jsource.syntax.tokenmarker;
12
13
14 import javax.swing.text.Segment;
15 import java.util.*;
16
17
18 /**
19 * A token marker that splits lines of text into tokens. Each token carries
20 * a length field and an indentification tag that can be mapped to a color
21 * for painting that token.<p>
22 *
23 * For performance reasons, the linked list of tokens is reused after each
24 * line is tokenized. Therefore, the return value of <code>markTokens</code>
25 * should only be used for immediate painting. Notably, it cannot be
26 * cached.
27 *
28 * @author Slava Pestov
29 */
30 public abstract class TokenMarker {
31
32 /**
33 * A wrapper for the lower-level <code>markTokensImpl</code> method
34 * that is called to split a line up into tokens.
35 * @param line The line
36 * @param lineIndex The line number
37 */
38 public Token markTokens(Segment line, int lineIndex) {
39 if (lineIndex >= length) {
40 throw new IllegalArgumentException("Tokenizing invalid line: " + lineIndex);
41 }
42
43 lastToken = null;
44
45 LineInfo info = lineInfo[lineIndex];
46 LineInfo prev;
47
48 if (lineIndex == 0)
49 prev = null;
50 else
51 prev = lineInfo[lineIndex - 1];
52
53 byte oldToken = info.token;
54 byte token = markTokensImpl(prev == null ? Token.NULL : prev.token, line, lineIndex);
55
56 info.token = token;
57
58 /*
59 * This is a foul hack. It stops nextLineRequested
60 * from being cleared if the same line is marked twice.
61 *
62 * Why is this necessary? It's all JEditTextArea's fault.
63 * When something is inserted into the text, firing a
64 * document event, the insertUpdate() method shifts the
65 * caret (if necessary) by the amount inserted.
66 *
67 * All caret movement is handled by the select() method,
68 * which eventually pipes the new position to scrollTo()
69 * and calls repaint().
70 *
71 * Note that at this point in time, the new line hasn't
72 * yet been painted; the caret is moved first.
73 *
74 * scrollTo() calls offsetToX(), which tokenizes the line
75 * unless it is being called on the last line painted
76 * (in which case it uses the text area's painter cached
77 * token list). What scrollTo() does next is irrelevant.
78 *
79 * After scrollTo() has done it's job, repaint() is
80 * called, and eventually we end up in paintLine(), whose
81 * job is to paint the changed line. It, too, calls
82 * markTokens().
83 *
84 * The problem was that if the line started a multiline
85 * token, the first markTokens() (done in offsetToX())
86 * would set nextLineRequested (because the line end
87 * token had changed) but the second would clear it
88 * (because the line was the same that time) and therefore
89 * paintLine() would never know that it needed to repaint
90 * subsequent lines.
91 *
92 * This bug took me ages to track down, that's why I wrote
93 * all the relevant info down so that others wouldn't
94 * duplicate it.
95 */
96 if (!(lastLine == lineIndex && nextLineRequested))
97 nextLineRequested = (oldToken != token);
98
99 lastLine = lineIndex;
100
101 addToken(0, Token.END);
102
103 return firstToken;
104 }
105
106 /**
107 * An abstract method that splits a line up into tokens. It
108 * should parse the line, and call <code>addToken()</code> to
109 * add syntax tokens to the token list. Then, it should return
110 * the initial token type for the next line.<p>
111 *
112 * For example if the current line contains the start of a
113 * multiline comment that doesn't end on that line, this method
114 * should return the comment token type so that it continues on
115 * the next line.
116 *
117 * @param token The initial token type for this line
118 * @param line The line to be tokenized
119 * @param lineIndex The index of the line in the document,
120 * starting at 0
121 * @return The initial token type for the next line
122 */
123 protected abstract byte markTokensImpl(byte token, Segment line,
124 int lineIndex);
125
126 /**
127 * Returns if the token marker supports tokens that span multiple
128 * lines. If this is true, the object using this token marker is
129 * required to pass all lines in the document to the
130 * <code>markTokens()</code> method (in turn).<p>
131 *
132 * The default implementation returns true; it should be overridden
133 * to return false on simpler token markers for increased speed.
134 */
135 public boolean supportsMultilineTokens() {
136 return true;
137 }
138
139 /**
140 * Informs the token marker that lines have been inserted into
141 * the document. This inserts a gap in the <code>lineInfo</code>
142 * array.
143 * @param index The first line number
144 * @param lines The number of lines
145 */
146 public void insertLines(int index, int lines) {
147 if (lines <= 0)
148 return;
149 length += lines;
150 ensureCapacity(length);
151 int len = index + lines;
152
153 System.arraycopy(lineInfo, index, lineInfo, len,
154 lineInfo.length - len);
155
156 for (int i = index + lines - 1; i >= index; i--) {
157 lineInfo[i] = new LineInfo();
158 }
159 }
160
161 /**
162 * Informs the token marker that line have been deleted from
163 * the document. This removes the lines in question from the
164 * <code>lineInfo</code> array.
165 * @param index The first line number
166 * @param lines The number of lines
167 */
168 public void deleteLines(int index, int lines) {
169 if (lines <= 0)
170 return;
171 int len = index + lines;
172
173 length -= lines;
174 System.arraycopy(lineInfo, len, lineInfo,
175 index, lineInfo.length - len);
176 }
177
178 /**
179 * Returns the number of lines in this token marker.
180 */
181 public int getLineCount() {
182 return length;
183 }
184
185 /**
186 * Returns true if the next line should be repainted. This
187 * will return true after a line has been tokenized that starts
188 * a multiline token that continues onto the next line.
189 */
190 public boolean isNextLineRequested() {
191 return nextLineRequested;
192 }
193
194 /**
195 * The first token in the list. This should be used as the return
196 * value from <code>markTokens()</code>.
197 */
198 protected Token firstToken;
199
200 /**
201 * The last token in the list. New tokens are added here.
202 * This should be set to null before a new line is to be tokenized.
203 */
204 protected Token lastToken;
205
206 /**
207 * An array for storing information about lines. It is enlarged and
208 * shrunk automatically by the <code>insertLines()</code> and
209 * <code>deleteLines()</code> methods.
210 */
211 protected LineInfo[] lineInfo;
212
213 /**
214 * The number of lines in the model being tokenized. This can be
215 * less than the length of the <code>lineInfo</code> array.
216 */
217 protected int length;
218
219 /**
220 * The last tokenized line.
221 */
222 protected int lastLine;
223
224 /**
225 * True if the next line should be painted.
226 */
227 protected boolean nextLineRequested;
228
229 /**
230 * Creates a new <code>TokenMarker</code>. This DOES NOT create
231 * a lineInfo array; an initial call to <code>insertLines()</code>
232 * does that.
233 */
234 protected TokenMarker() {
235 lastLine = -1;
236 }
237
238 /**
239 * Ensures that the <code>lineInfo</code> array can contain the
240 * specified index. This enlarges it if necessary. No action is
241 * taken if the array is large enough already.<p>
242 *
243 * It should be unnecessary to call this under normal
244 * circumstances; <code>insertLine()</code> should take care of
245 * enlarging the line info array automatically.
246 *
247 * @param index The array index
248 */
249 protected void ensureCapacity(int index) {
250 if (lineInfo == null)
251 lineInfo = new LineInfo[index + 1];
252 else if (lineInfo.length <= index) {
253 LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2];
254
255 System.arraycopy(lineInfo, 0, lineInfoN, 0,
256 lineInfo.length);
257 lineInfo = lineInfoN;
258 }
259 }
260
261 /**
262 * Adds a token to the token list.
263 * @param length The length of the token
264 * @param id The id of the token
265 */
266 protected void addToken(int length, byte id) {
267 if (id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST)
268 throw new InternalError("Invalid id: " + id);
269
270 if (length == 0 && id != Token.END)
271 return;
272
273 if (firstToken == null) {
274 firstToken = new Token(length, id);
275 lastToken = firstToken;
276 } else if (lastToken == null) {
277 lastToken = firstToken;
278 firstToken.length = length;
279 firstToken.id = id;
280 } else if (lastToken.next == null) {
281 lastToken.next = new Token(length, id);
282 lastToken = lastToken.next;
283 } else {
284 lastToken = lastToken.next;
285 lastToken.length = length;
286 lastToken.id = id;
287 }
288 }
289
290 /**
291 * Inner class for storing information about tokenized lines.
292 */
293 public class LineInfo {
294
295 /**
296 * Creates a new LineInfo object with token = Token.NULL
297 * and obj = null.
298 */
299 public LineInfo() {}
300
301 /**
302 * Creates a new LineInfo object with the specified
303 * parameters.
304 */
305 public LineInfo(byte token, Object obj) {
306 this.token = token;
307 this.obj = obj;
308 }
309
310 /**
311 * The id of the last token of the line.
312 */
313 public byte token;
314
315 /**
316 * This is for use by the token marker implementations
317 * themselves. It can be used to store anything that
318 * is an object and that needs to exist on a per-line
319 * basis.
320 */
321 public Object obj;
322 }
323 }