public Token next() throws IOException {
Token t = input.next();
if (t == null)
return null;
String text = t.termText();
System.out.println(text);
int minPoz = -1;
int poz;
for (int i = 0; i < apostrophes.length(); i++) {
poz = text.indexOf(apostrophes.charAt(i));
if (poz != -1)
minPoz = (minPoz == -1) ? poz : Math.min(poz, minPoz);
}
if (minPoz != -1
&& articles.contains(text.substring(0, minPoz).toLowerCase()))
text = text.substring(minPoz + 1);
return new Token(text, t.startOffset(), t.endOffset(), t.type());
}
Returns the next input Token whith termText() without elisioned start |