Source code: raining/client/HttpClient.java
1 /*
2 * $Author: rahul_kumar $
3 * $Id: HttpClient.java,v 1.9 2003/10/12 17:56:05 rahul_kumar Exp $
4 *
5 * Copyright (C) 2003 Rahul Kumar. All rights reserved.
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 *
21 */
22
23 package raining.client;
24 import raining.core.*;
25 import java.net.URL;
26 import org.apache.commons.cli.*;
27 import java.io.*;
28 import java.util.*;
29 import sun.net.www.*;
30
31 /**
32 * A sample HTTPClient that extends NioSocket.
33 *
34 * Command line for:
35 * - file to take URLs from
36 * - use proxy, server, port, authstring
37 *
38 * FOr Testing or loading (this should go into another extends)
39 * - how many concurrent connections (A)
40 * - how long to keep hitting URL
41 * - any pause between A's
42 * - max hits
43 *
44 * RK modified on 20031004 13:08:05
45 * Made proxy setting once and for all. User should just set and forget.
46 * I am removing the stupid idea of allowing setting encoded string in
47 * the command line. just makes things more complic for developers.
48 *
49 * TODO As much as i dislike wasting my time on the proxy thing, i wd
50 * like to make it static, so we dont have to keep messing with it in
51 * each URL. May make life simpler for those instantiating this.
52 *
53 * Rather than wait for -1, we have to do as follows for HTTP
54 * Content-Length: XXXX gives you the length after the header
55 * Header end is 2 "\r\n\r\n". We can close after this length s
56 * crossed. also we shd not give the header.
57 */
58
59 public class HttpClient extends NioSocket {
60
61 public static final String P = "HttpClient";
62 public static void main (String args[]){
63
64 String urlarray[] = null;
65 //String proxyServer = null;
66 //int proxyPort = 8080;
67 //String proxyAuth = null;
68 //String proxyBase64 = null;
69 //boolean useProxy = false;
70 String filename = null;
71
72 ClientParser clp = new ClientParser(args, "HttpClient");
73 clp.parse();
74
75 urlarray = clp.getUrlArray();
76 // the following gets still need to be moved off, this is adding
77 // to the work one has to do, even after using clp.
78 useProxy = clp.getUseProxy();
79 if (useProxy){
80 HttpClient.setProxy(clp.getProxyServer(), clp.getProxyPort());
81 // this is a silly option that complicates the interface. OUT
82 // WITH IT.
83 //proxyBase64 = clp.getProxyBase64();
84
85 if (clp.getProxyAuth() != null)
86 HttpClient.setProxyAuthString(clp.getProxyAuth());
87 else
88 System.err.println( "warning: Auth String not passed for connecting to proxy !");
89 }
90
91 // START WORK
92 //
93 try {
94 if (urlarray.length >0){
95 for( int i = 0; i < urlarray.length; i++ ){
96 HttpClient h = new HttpClient (urlarray[i]);
97 h.push();
98 }
99 }
100 NioSocket.stopWhenIdle(true);
101 NioSocket.start();
102 } catch (Exception exc) { System.err.println( " Http 21 EXC:"+ exc.toString()); exc.printStackTrace(); }
103
104 }
105
106 protected URL u = null;
107 String host = null;
108 String path = null;
109 String absoluteUrl = null;
110 int port = 80;
111
112 // having proxy setting with each connection, crapped up the code of
113 // callers and extenders so much that i am making it static. Users
114 // should set it once and forget about it.
115 static String proxyServer = null;
116 static int proxyPort = 8080;
117 static String proxyAuth = null;
118 static String proxyBase64 = null;
119 static boolean useProxy = false;
120
121 String filename = null;
122
123 String GETString = null;
124 boolean getrequired = false;
125
126 // java.net.HttpURLConnection
127 /**
128 * An <code>int</code> representing the three digit HTTP Status-Code.
129 * <ul>
130 * <li> 1xx: Informational
131 * <li> 2xx: Success
132 * <li> 3xx: Redirection
133 * <li> 4xx: Client Error
134 * <li> 5xx: Server Error
135 * </ul>
136 */
137 protected int responseCode = -1;
138
139 /**
140 * The HTTP response message.
141 */
142 protected String responseMessage = null;
143
144
145 /** Constructor that takes absolute URL and splits it.
146 * e.g. http://www.benegal.net/
147 */
148 public HttpClient (String Url) throws java.net.MalformedURLException, java.io.IOException, Exception {
149 super();
150 this.absoluteUrl = Url;
151 u = new URL (Url);
152 host = u.getHost();
153 path = u.getPath();
154 if (path.length()==0)
155 path="/";
156 port = u.getPort();
157 if (port == -1) port = u.getDefaultPort();
158
159
160 }
161 /** Constructor to be used if you have created a GETString already.
162 * this is required if you wish to blast one URL with many
163 * connections. The sequence would be:
164 * 1. use the blank cons
165 * 2. create a GET String using makeGETString with Url and save
166 * In a loop:
167 * 3. set the URL with setGETString.
168 * 4. set the port and host (setHostPort or setProxy)
169 * 5. push
170 */
171 public HttpClient () throws java.io.IOException, Exception {
172 super();
173 getrequired = true;
174 }
175 /** set the string to be used for pushing. Must be used if yuo used
176 * a blank constructor.
177 */
178 public HttpClient setGETString (String s){
179 this.GETString = s;
180 getrequired = false;
181 return this;
182 }
183 /** tell HTTPClients to use a proxy server */
184 public static void setProxy(String hostname, int port){
185 useProxy = true;
186 proxyServer = hostname;
187 proxyPort = port;
188 }
189 /** required by caching clients (non-proxy) on a connection level.
190 */
191 public HttpClient setHostPort(String hostname, int port){
192 this.host = hostname;
193 this.port = port;
194 return this;
195 }
196 public static void setProxyAuthString(String authString){
197 proxyAuth = authString;
198 proxyBase64 = encode (authString);
199 }
200 /** encode a clear text auth string, into Base 64, needed for
201 * proxy authentication cases.
202 * put as a method so that extenders can call.
203 */
204 public static String encode (String authString) {
205 return "Basic " + new sun.misc.BASE64Encoder().encode(authString.getBytes());
206
207 }
208 /**
209 * gave the user the option of specifying the encoded string on the
210 * command line. Moved out since it complicates the interface for
211 * others.
212 public static void setProxyBase64String (String base64String ){
213 proxyBase64 = base64String;
214 }
215 */
216 /** push the GET Url.
217 * I think even the string creations should go into a method, for
218 * those overriding. Also, it needs to be StringBuffered.
219 */
220 public void push () throws java.io.IOException, Exception {
221
222 // user did not send in a get string
223 if (getrequired){
224 throw new Exception ("Get String required, if you use blank constructor for HttpClient.");
225 }
226
227 if (GETString == null){
228 makeGETString();
229 }
230 //System.out.println( "HttpClient 196 push() : GETSTRING:"+GETString+"p:"+port+"h:"+host);
231 if (useProxy)
232 super.create_client_socket( proxyServer,proxyPort, GETString);
233 else
234 super.create_client_socket( host , port, GETString);
235
236 }
237 /** Return the GET String given an absolute url as a string, for the
238 * purpose of caching a reusing. This one caters to non proxy cases.
239 * This helps a caller cache a
240 * string and use it multiple times.
241 * Is the URL parse light, or
242 * should we do a parse here itself.
243 */
244 public static URLDetails makeGETString (String Url)
245 throws java.net.MalformedURLException {
246 URL u = new URL (Url);
247 String host = u.getHost();
248 String path = u.getPath();
249 if (path.length()==0)
250 path="/";
251 int port = u.getPort();
252 if (port == -1) port = u.getDefaultPort();
253
254 String GETString = "GET "+path+" HTTP/1.1\r\n"+
255 "Host: "+host+ "\r\n\r\n";
256
257 URLDetails urld = new URLDetails(host, port, path, GETString);
258 //return (GETString);
259 return (urld);
260 }
261 /** return the GET String, this one caters to proxy cases.
262 * Pass the encoded string as the
263 * second parameter.
264 */
265 public static URLDetails makeGETString (String Url, String proxyBase64)
266 throws java.net.MalformedURLException {
267 URL u = new URL (Url);
268 String host = u.getHost();
269 String path = u.getPath();
270 if (path.length()==0)
271 path="/";
272 int port = u.getPort();
273 if (port == -1) port = u.getDefaultPort();
274
275 String GETString = "GET "+Url+" HTTP/1.1\r\n"+
276 "Proxy-Authorization: "+proxyBase64 + "\r\n"+
277 "Host: "+host+ "\r\n\r\n";
278
279 URLDetails urld = new URLDetails(host, port, path, GETString);
280 return (urld);
281 //return (GETString);
282 }
283
284 public void makeGETString() {
285
286
287 if (useProxy){
288 GETString = "GET "+absoluteUrl+" HTTP/1.1\r\n"+
289 "Proxy-Authorization: "+proxyBase64 + "\r\n"+
290 "Host: "+host+ "\r\n\r\n";
291 }
292 else
293 {
294 GETString = "GET "+path+" HTTP/1.1\r\n"+
295 "Host: "+host+ "\r\n\r\n";
296 }
297 }
298 /** we implement our own processing here.
299 * e.g. we could write to disk or db.
300 * we would do any blocking writes in a thread if we want
301 * scalability.
302 */
303 // public static final char DOTCHAR = '.';
304 public void handle_read_complete(String mdata){
305 System.out.println( mdata );
306 System.err.println( "S = " + id +" Total:" + totalBytesRead);
307 if (debug){
308 if (totalBytesRead == 0) System.out.println( " XXXXXX " + id);
309 }
310
311 }
312 /** as read from header. If not found in header, it remains 0, so
313 * that the read may terminate fast, since it may be erroneous.
314 */
315 int contentLength = 0;
316 /** location where the header ends, updated only after "\r\n\r\n" detected. */
317 int header_end = 0;
318 boolean seen_header = false;
319 MessageHeader properties = null;
320
321 /**
322 * Implementation of how a HTTPClient handles a read.
323 * Takes into account parsing the header and looking for content
324 * length, and closing channel if content has been obtained,
325 * withuot waiting for channel to get a -1.
326 *
327 * should we minus header length from bytes read or handle it
328 * internally.
329 * XXX how is the person here to know abt rsb. if he uses
330 */
331 public int handle_read (){
332
333 int count = super.handle_read();
334 int pos;
335 if (count >0) {
336 if (!seen_header){
337 /*
338 if ( (pos=rsb.indexOf("Content-Length:")) != -1 ){
339 int pos1 = rsb.indexOf("\r\n", pos +14);
340 String snum = rsb.substring(pos+15, pos1);
341 content_length = Integer.parseInt(snum.trim());
342 seen_header = true;
343 }
344 */
345 if ( (header_end=rsb.indexOf("\r\n\r\n")) != -1 ){
346 header_end += 4;
347 try {
348 properties = new MessageHeader(new ByteArrayInputStream(rsb.toString().getBytes()));
349 int resp = getResponseCode();
350 getContentLength();
351 System.out.print( properties.findValue(null) + "-" +
352 properties.findValue("content-length"));
353 //System.out.println( properties.findValue("content-type"));
354 //System.out.println( properties.findValue("location"));
355 //System.out.println( properties.toString());
356 //System.out.println( "RSB:["+rsb.toString() +"]");
357
358
359 } catch (Exception exc) {
360 System.err.println( P+" L EXC:"+ exc.toString()); exc.printStackTrace(); }
361 // XXX Need to test out with a real HTTP server
362 // so i know all situations.
363 // i could be deleting the data itself!
364 //rsb.delete(0, header_end);
365 seen_header = true;
366 }
367 }
368 if (seen_header){
369 if ( (totalBytesRead-header_end) >= contentLength){
370 handle_read_complete( getReadData() );
371 handle_close();
372 }
373 }
374 } // count >0
375
376 return count;
377
378 } // handle
379
380 /** return content of file as array of Strings.
381 * Used by client parser for reading up url list.
382 * Should close file pointer in finally.
383 */
384 public static String[] getFileContentsAsArray (String filename){
385 BufferedReader br = null;
386 try {
387 br = new BufferedReader( new FileReader(filename));
388 java.util.ArrayList al = new java.util.ArrayList();
389 String line;
390 while ( (line = br.readLine())!=null){
391 al.add (line.trim());
392 }
393 if (al.size()>0){
394 String ret[] = new String[al.size()];
395 ret = (String[]) al.toArray(ret);
396 return ret;
397 }
398 br.close(); //RK added on 20031205 22:51:58
399 } catch (Exception exc) { System.err.println( "EXC:"+ exc.toString()); exc.printStackTrace(); }
400 return null;
401 }
402 /** Call this routine to get the content-length associated with this
403 * object.
404 */
405 public int getContentLength() {
406 if (properties == null) return -1;
407 int l = contentLength;
408 if (l < 0) {
409 try {
410 l = Integer.parseInt(properties.findValue("content-length"));
411 contentLength = l;
412 } catch(Exception e) {
413 }
414 }
415 return l;
416 }
417 public int getResponseCode() throws IOException {
418 if (responseCode != -1) {
419 return responseCode;
420 }
421
422 /*
423 * If the response code is >= 400 then an IOException is
424 * thrown by getInputStream the first time only.
425 */
426
427 if (properties == null) return -1;
428
429 String resp = getHeaderField(0);
430 /* should have no leading/trailing LWS
431 * expedite the typical case by assuming it has
432 * form "HTTP/1.x <WS> 2XX <mumble>"
433 */
434 int ind;
435 try {
436 ind = resp.indexOf(' ');
437 while(resp.charAt(ind) == ' ')
438 ind++;
439 responseCode = Integer.parseInt(resp.substring(ind, ind + 3));
440 responseMessage = resp.substring(ind + 4).trim();
441 return responseCode;
442 } catch (Exception e) {
443 return responseCode;
444 }
445 }
446 public String getResponseMessage() throws IOException {
447 getResponseCode();
448 return responseMessage;
449 }
450 // sun.net.www.URLConnection
451 /**
452 * Return the value for the nth header field. Returns null if
453 * there are fewer than n fields. This can be used in conjunction
454 * with getHeaderFieldKey to iterate through all the headers in the message.
455 */
456 public String getHeaderField(int n) {
457 MessageHeader props = properties;
458 return props == null ? null : props.getValue(n);
459 }
460
461
462
463
464
465
466 } // end of class
467 /*
468 class HeaderParser {
469 String s = null;
470 public HeaderParser (StringBuffer sb){
471 s = sb.toString();
472 }
473 public ParsedHeader parse (){
474 String rows[] = split(s, '\n');
475 for( int i = 0; i < rows.length; i++ ){
476 if (rows[i].startsWith("HTTP/")){
477
478 }
479 }
480
481
482
483 }
484 */
485
486 /** splits a given string on given character separator returning
487 * an array of strings.
488 * @author Rahul Kumar June 17, 2001
489 * How to deal with consecutive delimiters - i am returning blank
490 * strings.
491 public static String[] split (String st, char sep) {
492
493 ArrayList alist = new ArrayList();
494
495 int len = st.length();
496 int pos = 0;
497 int fin = 0;
498
499 // while not end of string, and you can find a match
500 while (pos < len && (fin = st.indexOf(sep, pos)) != -1){
501 alist.add(st.substring(pos, fin ));
502 pos = fin + 1;
503 }
504 // Push remainder if it's not empty
505 String remainder = st.substring(pos);
506 if (remainder.length() != 0)
507 {
508 alist.add(remainder);
509 }
510
511 // Return list as an array of strings
512 String[] ret = new String[alist.size()];
513 alist.toArray(ret);
514 return ret;
515 } // end of split
516
517 */
518
519