Source code: jsource/syntax/tokenmarker/Token.java
1 /*
2 * Token.java - Generic token
3 * Copyright (C) 1998, 1999 Slava Pestov
4 * Revised for JSource 2002 Panagiotis Plevrakis
5 *
6 * You may use and modify this package for any purpose. Redistribution is
7 * permitted, in both source and binary form, provided that this notice
8 * remains intact in all source distributions of this package.
9 */
10
11 package jsource.syntax.tokenmarker;
12
13
14 /**
15 * A linked list of tokens. Each token has three fields - a token
16 * identifier, which is a byte value that can be looked up in the
17 * array returned by <code>SyntaxDocument.getColors()</code>
18 * to get a color value, a length value which is the length of the
19 * token in the text, and a pointer to the next token in the list.
20 *
21 * @author Slava Pestov
22 */
23
24 public class Token {
25
26 /**
27 * Normal text token id. This should be used to mark
28 * normal text.
29 */
30 public static final byte NULL = 0;
31
32 /**
33 * Comment 1 token id. This can be used to mark a comment.
34 */
35 public static final byte COMMENT1 = 1;
36
37 /**
38 * Comment 2 token id. This can be used to mark a comment.
39 */
40 public static final byte COMMENT2 = 2;
41
42 /**
43 * Literal 1 token id. This can be used to mark a string
44 * literal (eg, C mode uses this to mark "..." literals)
45 */
46 public static final byte LITERAL1 = 3;
47
48 /**
49 * Literal 2 token id. This can be used to mark an object
50 * literal (eg, Java mode uses this to mark true, false, etc)
51 */
52 public static final byte LITERAL2 = 4;
53
54 /**
55 * Label token id. This can be used to mark labels
56 * (eg, C mode uses this to mark ...: sequences)
57 */
58 public static final byte LABEL = 5;
59
60 /**
61 * Keyword 1 token id. This can be used to mark a
62 * keyword. This should be used for general language
63 * constructs.
64 */
65 public static final byte KEYWORD1 = 6;
66
67 /**
68 * Keyword 2 token id. This can be used to mark a
69 * keyword. This should be used for preprocessor
70 * commands, or variables.
71 */
72 public static final byte KEYWORD2 = 7;
73
74 /**
75 * Keyword 3 token id. This can be used to mark a
76 * keyword. This should be used for data types.
77 */
78 public static final byte KEYWORD3 = 8;
79
80 /**
81 * Operator token id. This can be used to mark an
82 * operator. (eg, SQL mode marks +, -, etc with this
83 * token type)
84 */
85 public static final byte OPERATOR = 9;
86
87 /**
88 * Invalid token id. This can be used to mark invalid
89 * or incomplete tokens, so the user can easily spot
90 * syntax errors.
91 */
92 public static final byte INVALID = 10;
93
94 /**
95 * The total number of defined token ids.
96 */
97 public static final byte ID_COUNT = 11;
98
99 /**
100 * The first id that can be used for internal state
101 * in a token marker.
102 */
103 public static final byte INTERNAL_FIRST = 100;
104
105 /**
106 * The last id that can be used for internal state
107 * in a token marker.
108 */
109 public static final byte INTERNAL_LAST = 126;
110
111 /**
112 * The token type, that along with a length of 0
113 * marks the end of the token list.
114 */
115 public static final byte END = 127;
116
117 /**
118 * The length of this token.
119 */
120 public int length;
121
122 /**
123 * The id of this token.
124 */
125 public byte id;
126
127 /**
128 * The next token in the linked list.
129 */
130 public Token next;
131
132 /**
133 * Creates a new token.
134 * @param length The length of the token
135 * @param id The id of the token
136 */
137 public Token(int length, byte id) {
138 this.length = length;
139 this.id = id;
140 }
141
142 /**
143 * Returns a string representation of this token.
144 */
145 public String toString() {
146 return "[id=" + id + ",length=" + length + "]";
147 }
148 }