Source code: org/outerj/pollo/texteditor/TokenMarker.java
1 package org.outerj.pollo.texteditor;
2 /*
3 * TokenMarker.java - Generic token marker
4 * Copyright (C) 1998, 1999 Slava Pestov
5 *
6 * You may use and modify this package for any purpose. Redistribution is
7 * permitted, in both source and binary form, provided that this notice
8 * remains intact in all source distributions of this package.
9 */
10
11 import javax.swing.text.Segment;
12
13 /**
14 * A token marker that splits lines of text into tokens. Each token carries
15 * a length field and an indentification tag that can be mapped to a color
16 * for painting that token.<p>
17 *
18 * For performance reasons, the linked list of tokens is reused after each
19 * line is tokenized. Therefore, the return value of <code>markTokens</code>
20 * should only be used for immediate painting. Notably, it cannot be
21 * cached.
22 *
23 * @author Slava Pestov
24 * @version $Id: TokenMarker.java,v 1.3 2002/08/27 20:44:03 bdumon Exp $
25 *
26 * @see org.gjt.sp.jedit.syntax.Token
27 */
28 public abstract class TokenMarker
29 {
30 /**
31 * A wrapper for the lower-level <code>markTokensImpl</code> method
32 * that is called to split a line up into tokens.
33 * @param line The line
34 * @param lineIndex The line number
35 */
36 public Token markTokens(Segment line, int lineIndex)
37 {
38 if(lineIndex >= length)
39 {
40 throw new IllegalArgumentException("Tokenizing invalid line: "
41 + lineIndex);
42 }
43
44 lastToken = null;
45
46 LineInfo info = lineInfo[lineIndex];
47 LineInfo prev;
48 if(lineIndex == 0)
49 prev = null;
50 else
51 prev = lineInfo[lineIndex - 1];
52
53 byte oldToken = info.token;
54 byte token = markTokensImpl(prev == null ?
55 Token.NULL : prev.token,line,lineIndex);
56
57 info.token = token;
58
59 /*
60 * This is a foul hack. It stops nextLineRequested
61 * from being cleared if the same line is marked twice.
62 *
63 * Why is this necessary? It's all JEditTextArea's fault.
64 * When something is inserted into the text, firing a
65 * document event, the insertUpdate() method shifts the
66 * caret (if necessary) by the amount inserted.
67 *
68 * All caret movement is handled by the select() method,
69 * which eventually pipes the new position to scrollTo()
70 * and calls repaint().
71 *
72 * Note that at this point in time, the new line hasn't
73 * yet been painted; the caret is moved first.
74 *
75 * scrollTo() calls offsetToX(), which tokenizes the line
76 * unless it is being called on the last line painted
77 * (in which case it uses the text area's painter cached
78 * token list). What scrollTo() does next is irrelevant.
79 *
80 * After scrollTo() has done it's job, repaint() is
81 * called, and eventually we end up in paintLine(), whose
82 * job is to paint the changed line. It, too, calls
83 * markTokens().
84 *
85 * The problem was that if the line started a multiline
86 * token, the first markTokens() (done in offsetToX())
87 * would set nextLineRequested (because the line end
88 * token had changed) but the second would clear it
89 * (because the line was the same that time) and therefore
90 * paintLine() would never know that it needed to repaint
91 * subsequent lines.
92 *
93 * This bug took me ages to track down, that's why I wrote
94 * all the relevant info down so that others wouldn't
95 * duplicate it.
96 */
97 if(!(lastLine == lineIndex && nextLineRequested))
98 nextLineRequested = (oldToken != token);
99
100 lastLine = lineIndex;
101
102 addToken(0,Token.END);
103
104 return firstToken;
105 }
106
107 /**
108 * An abstract method that splits a line up into tokens. It
109 * should parse the line, and call <code>addToken()</code> to
110 * add syntax tokens to the token list. Then, it should return
111 * the initial token type for the next line.<p>
112 *
113 * For example if the current line contains the start of a
114 * multiline comment that doesn't end on that line, this method
115 * should return the comment token type so that it continues on
116 * the next line.
117 *
118 * @param token The initial token type for this line
119 * @param line The line to be tokenized
120 * @param lineIndex The index of the line in the document,
121 * starting at 0
122 * @return The initial token type for the next line
123 */
124 protected abstract byte markTokensImpl(byte token, Segment line,
125 int lineIndex);
126
127 /**
128 * Returns if the token marker supports tokens that span multiple
129 * lines. If this is true, the object using this token marker is
130 * required to pass all lines in the document to the
131 * <code>markTokens()</code> method (in turn).<p>
132 *
133 * The default implementation returns true; it should be overridden
134 * to return false on simpler token markers for increased speed.
135 */
136 public boolean supportsMultilineTokens()
137 {
138 return true;
139 }
140
141 /**
142 * Informs the token marker that lines have been inserted into
143 * the document. This inserts a gap in the <code>lineInfo</code>
144 * array.
145 * @param index The first line number
146 * @param lines The number of lines
147 */
148 public void insertLines(int index, int lines)
149 {
150 if(lines <= 0)
151 return;
152 length += lines;
153 ensureCapacity(length);
154 int len = index + lines;
155 System.arraycopy(lineInfo,index,lineInfo,len,
156 lineInfo.length - len);
157
158 for(int i = index + lines - 1; i >= index; i--)
159 {
160 lineInfo[i] = new LineInfo();
161 }
162 }
163
164 /**
165 * Informs the token marker that line have been deleted from
166 * the document. This removes the lines in question from the
167 * <code>lineInfo</code> array.
168 * @param index The first line number
169 * @param lines The number of lines
170 */
171 public void deleteLines(int index, int lines)
172 {
173 if (lines <= 0)
174 return;
175 int len = index + lines;
176 length -= lines;
177 System.arraycopy(lineInfo,len,lineInfo,
178 index,lineInfo.length - len);
179 }
180
181 /**
182 * Returns the number of lines in this token marker.
183 */
184 public int getLineCount()
185 {
186 return length;
187 }
188
189 /**
190 * Returns true if the next line should be repainted. This
191 * will return true after a line has been tokenized that starts
192 * a multiline token that continues onto the next line.
193 */
194 public boolean isNextLineRequested()
195 {
196 return nextLineRequested;
197 }
198
199 // protected members
200
201 /**
202 * The first token in the list. This should be used as the return
203 * value from <code>markTokens()</code>.
204 */
205 protected Token firstToken;
206
207 /**
208 * The last token in the list. New tokens are added here.
209 * This should be set to null before a new line is to be tokenized.
210 */
211 protected Token lastToken;
212
213 /**
214 * An array for storing information about lines. It is enlarged and
215 * shrunk automatically by the <code>insertLines()</code> and
216 * <code>deleteLines()</code> methods.
217 */
218 protected LineInfo[] lineInfo;
219
220 /**
221 * The number of lines in the model being tokenized. This can be
222 * less than the length of the <code>lineInfo</code> array.
223 */
224 protected int length;
225
226 /**
227 * The last tokenized line.
228 */
229 protected int lastLine;
230
231 /**
232 * True if the next line should be painted.
233 */
234 protected boolean nextLineRequested;
235
236 /**
237 * Creates a new <code>TokenMarker</code>. This DOES NOT create
238 * a lineInfo array; an initial call to <code>insertLines()</code>
239 * does that.
240 */
241 protected TokenMarker()
242 {
243 lastLine = -1;
244 }
245
246 /**
247 * Ensures that the <code>lineInfo</code> array can contain the
248 * specified index. This enlarges it if necessary. No action is
249 * taken if the array is large enough already.<p>
250 *
251 * It should be unnecessary to call this under normal
252 * circumstances; <code>insertLine()</code> should take care of
253 * enlarging the line info array automatically.
254 *
255 * @param index The array index
256 */
257 protected void ensureCapacity(int index)
258 {
259 if(lineInfo == null)
260 lineInfo = new LineInfo[index + 1];
261 else if(lineInfo.length <= index)
262 {
263 LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2];
264 System.arraycopy(lineInfo,0,lineInfoN,0,
265 lineInfo.length);
266 lineInfo = lineInfoN;
267 }
268 }
269
270 /**
271 * Adds a token to the token list.
272 * @param length The length of the token
273 * @param id The id of the token
274 */
275 protected void addToken(int length, byte id)
276 {
277 if(id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST)
278 throw new InternalError("Invalid id: " + id);
279
280 if(length == 0 && id != Token.END)
281 return;
282
283 if(firstToken == null)
284 {
285 firstToken = new Token(length,id);
286 lastToken = firstToken;
287 }
288 else if(lastToken == null)
289 {
290 lastToken = firstToken;
291 firstToken.length = length;
292 firstToken.id = id;
293 }
294 else if(lastToken.next == null)
295 {
296 lastToken.next = new Token(length,id);
297 lastToken = lastToken.next;
298 }
299 else
300 {
301 lastToken = lastToken.next;
302 lastToken.length = length;
303 lastToken.id = id;
304 }
305 }
306
307 /**
308 * Inner class for storing information about tokenized lines.
309 */
310 public class LineInfo
311 {
312 /**
313 * Creates a new LineInfo object with token = Token.NULL
314 * and obj = null.
315 */
316 public LineInfo()
317 {
318 }
319
320 /**
321 * Creates a new LineInfo object with the specified
322 * parameters.
323 */
324 public LineInfo(byte token, Object obj)
325 {
326 this.token = token;
327 this.obj = obj;
328 }
329
330 /**
331 * The id of the last token of the line.
332 */
333 public byte token;
334
335 /**
336 * This is for use by the token marker implementations
337 * themselves. It can be used to store anything that
338 * is an object and that needs to exist on a per-line
339 * basis.
340 */
341 public Object obj;
342 }
343 }