Source code: org/pqt/autorib/tokenizer/Tokenizer.java
1 //AutoRIB
2 // Copyright © 1998 - 2002, P W Quint
3 //
4 // Contact: autorib00@aol.com
5 //
6 // This library is free software; you can redistribute it and/or
7 // modify it under the terms of the GNU General Public
8 // License as published by the Free Software Foundation; either
9 // version 2 of the License, or (at your option) any later version.
10 //
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 // General Public License for more details.
15 //
16 // You should have received a copy of the GNU General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
20 package org.pqt.autorib.tokenizer;
21
22 import java.io.*;
23 import java.util.Iterator;
24 import java.util.Vector;
25 import org.pqt.autorib.util.Point;
26
27 /** Reads Characters from a given stream, ignoring comments
28 * . It recognises the following kinds of input (which can be stored
29 * in a Token object):
30 * <p>
31 * <p>- a word;
32 * <p>- a number;
33 * <p>- a string (delimeted by "") comment characters are not recognized
34 * inside a string - but an error is raised if a newline is read
35 * <p>- an array of numbers (enclosed in [])
36 * <p>
37 * This class is not used directly - rather its subclasses RIBTokenizer and
38 * InstrTokenizer are used. The advantage of having this common superclass
39 * is that it makes it comparatively easy to read RIB from an instructions
40 * file.
41 * <p> A count of the current line number is maintained, to
42 * allow this to be used in error reporting
43 * @version $Header: /home/CVSRepository/autoribnew/org/pqt/autorib/tokenizer/Tokenizer.java,v 1.5 2003/12/02 17:10:52 remote Exp $
44 */
45 public class Tokenizer extends Object {
46
47 /** true if there is a token that has been pushed back into
48 * the buffer and is ready for reading (i.e is in the
49 * variable token)
50 */
51 protected boolean pushedBack;
52 protected int tokensRead;
53 /** the file name this tokenizer is attached to */
54 public String fileName;
55 /** the reader for the file */
56 public Reader f;
57 /** a stream tokenizer to read the file */
58 public StreamTokenizer t;
59 /** the most recently read Token */
60 public Token token;
61
62 /** set up a tokenizer to read the named file
63 * @param name the file to read from
64 * @throws FileNotFoundException
65 */
66 public Tokenizer(String name) throws FileNotFoundException {
67 f = new FileReader(name);
68 t = new StreamTokenizer(f);
69 init();
70 fileName = name;
71 }
72
73 /** Create a tokenizer based on a StringTokenizer, to read
74 * from the given file name
75 * @param st an existing string tokenizer
76 * @param name a file name
77 */
78 public Tokenizer(StreamTokenizer st, String name) {
79 t = st;
80 init();
81 fileName = name;
82 }
83
84 /** attach a tokenizer to an existing (opened) stream, which is attached to
85 * a file of the given name
86 * @param is the stream
87 * @param name name of the file the stream is attached
88 * (this is used in reporting error
89 * messages e.g. line x of file <I>name</I>
90 */
91 public Tokenizer(InputStream is, String name) {
92 f = new BufferedReader(new InputStreamReader(is));
93 t = new StreamTokenizer(f);
94 init();
95 fileName = name;
96 }
97
98 private void init() {
99 t.resetSyntax();
100 t.eolIsSignificant(false);
101 t.quoteChar('"');
102 t.ordinaryChar('[');
103 t.ordinaryChar(']');
104 t.ordinaryChar('{');
105 t.ordinaryChar('}');
106 t.wordChars('a','z');
107 t.wordChars('A','Z');
108 t.wordChars('.','.');
109 t.wordChars('-','-');
110 t.wordChars('+','+');
111 t.wordChars('0','9');
112 t.whitespaceChars(0,' ');
113 pushedBack = false;
114 tokensRead = 0;
115 }
116
117
118 /** Get a token of any permissable kind
119 * @throws FormatException if there is a syntax error
120 * @throws IOException
121 * @return the type of token read (one of the constants
122 * defined in the Token class: STRING, FLOAT etc)
123 */
124 public int getToken() throws FormatException, IOException {
125 int tt; char c;
126 if (pushedBack)
127 pushedBack = false;
128 else {
129 token = new Token();
130 tt = t.nextToken();
131 if (tt == StreamTokenizer.TT_EOF)
132 token.tokenType = Token.EOF;
133 else {
134 if (tt == '[')
135 getArray_();
136 else if (tt == '"') {
137 token.stringVal = t.sval;
138 token.tokenType = Token.STRING;
139 }
140 else if (tt == StreamTokenizer.TT_WORD) {
141 c = t.sval.charAt(0);
142 if (((c == '.') || (c == '-')) ||
143 ((c >= '0') && (c <= '9'))) {
144 try {token.floatVal = new Float(t.sval);}
145 catch (NumberFormatException e)
146 { throw new FormatException("Incorrect number format",
147 this); }
148 token.tokenType = Token.FLOAT;
149 }
150 else {
151 token.tokenType = Token.REQUEST;
152 token.requestVal = t.sval;
153 }
154 }
155 else if (tt == '{')
156 token.tokenType = Token.OPENBRACE;
157 else if (tt == '}')
158 token.tokenType = Token.CLOSEBRACE;
159 else
160 throw new FormatException("Unrecognised Token",
161 this);
162 }
163 }
164 tokensRead++;
165 return token.tokenType;
166 }
167
168 /** Read a string token
169 * @throws FormatException thrown if no string is found
170 * @throws IOException
171 * @return the String read
172 *
173 */
174 public String getString() throws FormatException, IOException {
175 if (getToken() != Token.STRING)
176 throw new FormatException("String expected", this);
177 return token.stringVal;
178 }
179
180 /** read in a float, and return it. If no float is found raise an error
181 * @throws FormatException if no float found
182 * @throws IOException
183 * @return the float read
184 */
185 public float getNumber() throws FormatException, IOException {
186 if (getToken() != Token.FLOAT)
187 throw new FormatException("Number expected", this);
188 return token.floatVal.floatValue();
189 }
190
191 /** Used internally to read the remainder of an array after
192 * the initial [ has been read
193 * @throws FormatException if there is a syntax error in the array
194 * @throws IOException
195 */
196 public void getArray_() throws FormatException, IOException {
197 Vector v = new Vector(30);
198 Float f; char c;
199 int tt;
200 boolean Continue = true;
201 while (Continue) {
202 tt = t.nextToken();
203 if (tt == ']') {
204 Continue = false;
205 v.trimToSize();
206 }
207 else if (tt == t.TT_WORD) {
208 c = t.sval.charAt(0);
209 if (((c == '.') || (c == '-')) ||
210 ((c >= '0') && (c <= '9'))) {
211 try {f = new Float(t.sval);}
212 catch (NumberFormatException e)
213 { throw new FormatException("Incorrect number format",
214 this); }
215 v.addElement(f);
216 }
217 else
218 throw new FormatException("] expected",this);
219 }
220 else if (tt == '"')
221 v.addElement(t.sval);
222 else
223 throw new FormatException("] expected",this);
224 }
225 token.tokenType = Token.ARRAY;
226 token.arrayVal = v;
227 }
228
229 /** read in an array, and return it. If no array is found raise an error
230 * @throws FormatException if array contains a syntax error
231 * @throws IOException
232 * @return
233 */
234 public Vector getArray() throws FormatException,
235 IOException {
236 int tt;
237 tt = t.nextToken();
238 token = new Token();
239 if (tt != '[')
240 throw new FormatException("[ expected", this);
241 else
242 getArray_();
243 tokensRead++;
244 return token.arrayVal;
245 }
246
247 /** read in an array, and return it as a token. If no array is found raise an error
248 * @throws FormatException if array contains a syntax error
249 * @throws IOException
250 * @return a Token containing the array
251 */
252 public Token getArrayToken() throws FormatException,
253 IOException {
254 getArray();
255 return token;
256 }
257
258 /** Read in an RIB (or Instr) request name e.g Display
259 * @throws FormatException if there is a syntax error - note that the
260 * validity of the request itself is not cheked at
261 * this stage
262 * @throws IOException
263 * @return the request name as a string
264 */
265
266 public String getRequest() throws FormatException, IOException {
267 if (getToken() != Token.REQUEST)
268 throw new FormatException("Request expected",this);
269 return token.requestVal;
270 }
271
272 /**
273 * Push back the last token read, so that it will be
274 * read by the next read. Multiple Pushbacks will be ignored,
275 * as will a call to pushBack at the beginning of the file.
276 */
277 public void pushBack() {
278 if ((tokensRead > 0) && !pushedBack) {
279 pushedBack = true;
280 tokensRead--;
281 }
282 }
283
284 /** return the current line number in the instruction or rib file
285 * @return the line number
286 */
287 public int lineno() {
288 return t.lineno();
289 }
290
291 /** read in an array of floating point numbers
292 * @return the array of floating point numbers or throws a format exception
293 **/
294 public float[] getFloatArray() throws FormatException, IOException {
295 Object o;
296 Vector v = getArray();
297 int j = 0;
298 float f[] = new float[v.size()];
299 Iterator i = v.iterator();
300 while (i.hasNext()) {
301 o = i.next();
302 if (!(o instanceof Float))
303 throw new FormatException("Floating point array expected", this);
304 else
305 f[j] = ((Float) o).floatValue();
306 j++;
307 }
308 return f;
309 }
310
311 /** read and return a single float value - discarding any enclosing [ ]
312 * @return a float value, or throws a FormatException*/
313 public float getSingleFloat() throws FormatException, IOException {
314 int i = getToken();
315 if (i == Token.FLOAT)
316 return token.floatVal.floatValue();
317 if ((i == Token.ARRAY)) {
318 if ((token.arrayVal.size() == 1) &&
319 (token.arrayVal.get(0) instanceof Float))
320 return ((Float)token.arrayVal.get(0)).floatValue();
321 }
322 throw new FormatException("Floating point value expected",this);
323 }
324
325 /** Get a single string, discarding any enclosing brackets
326 * @return a float value, or throws a FormatException*/
327 public String getSingleString() throws FormatException, IOException {
328 int i = getToken();
329 if (i == Token.STRING)
330 return token.stringVal;
331 if ((i == Token.ARRAY)) {
332 if ((token.arrayVal.size() == 1) &&
333 (token.arrayVal.get(0) instanceof String))
334 return ((String)token.arrayVal.get(0));
335 }
336 throw new FormatException("String value expected",this);
337 }
338
339 /** Get a single point from the input stream
340 *@return a Point value, throws an exception if a point is not found*/
341 public Point getPoint() throws FormatException, IOException {
342 int i = getToken(), count = 0;
343 Point p = new Point();
344 Object o;
345 if (i == Token.ARRAY) {
346 Iterator j = token.arrayVal.iterator();
347 while (j.hasNext()) {
348 o = j.next();
349 if (o instanceof Float)
350 p.value[count] = ((Float)o).floatValue();
351 else
352 throw new FormatException("Point expected",this);
353 count++;
354 }
355 }
356 else
357 throw new FormatException("Point expected",this);
358 return p;
359 }
360
361 }