View Javadoc

1   /*
2    * @(#)HTTPResponse.java				0.3-3 06/05/2001
3    *
4    *  This file is part of the HTTPClient package
5    *  Copyright (C) 1996-2001 Ronald Tschalär
6    *
7    *  This library is free software; you can redistribute it and/or
8    *  modify it under the terms of the GNU Lesser General Public
9    *  License as published by the Free Software Foundation; either
10   *  version 2 of the License, or (at your option) any later version.
11   *
12   *  This library is distributed in the hope that it will be useful,
13   *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14   *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15   *  Lesser General Public License for more details.
16   *
17   *  You should have received a copy of the GNU Lesser General Public
18   *  License along with this library; if not, write to the Free
19   *  Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
20   *  MA 02111-1307, USA
21   *
22   *  For questions, suggestions, bug-reports, enhancement-requests etc.
23   *  I may be contacted at:
24   *
25   *  ronald@innovation.ch
26   *
27   *  The HTTPClient's home page is located at:
28   *
29   *  http://www.innovation.ch/java/HTTPClient/ 
30   *
31   * This file contains modifications for use with "The Grinder"
32   * (http://grinder.sourceforge.net) under the terms of the LGPL. They
33   * are marked below with the comment "GRINDER MODIFICATION".
34   *
35   */
36  
37  package HTTPClient;
38  
39  import java.io.IOException;
40  import java.io.InterruptedIOException;
41  import java.io.InputStream;
42  import java.io.ByteArrayInputStream;
43  import java.net.URL;
44  import java.util.Date;
45  import java.util.Enumeration;
46  
47  
48  
49  /**
50   * This defines the http-response class returned by the requests. It's
51   * basically a wrapper around the Response class which first lets all
52   * the modules handle the response before finally giving the info to
53   * the user.
54   *
55   * @version	0.3-3  06/05/2001
56   * @author	Ronald Tschalär
57   * @since	0.3
58   */
59  public class HTTPResponse implements HTTPClientModuleConstants
60  {
61      /** the list of modules */
62      private HTTPClientModule[]  modules;
63  
64      /** the timeout for reads */
65      private int          timeout;
66  
67      /** the request */
68      private Request      request = null;
69  
70      /** the current response */
71              Response     response = null;
72  
73      /** the HttpOutputStream to synchronize on */
74      private HttpOutputStream out_stream = null;
75  
76      /** our input stream from the stream demux */
77      private InputStream  inp_stream;
78  
79      /** the status code returned. */
80      private int          StatusCode;
81  
82      /** the reason line associated with the status code. */
83      private String       ReasonLine;
84  
85      /** the HTTP version of the response. */
86      private String       Version;
87  
88      /** the original URI used. */
89      private URI          OriginalURI = null;
90  
91      /** the final URI of the document. */
92      private URI          EffectiveURI = null;
93  
94      /** any headers which were received and do not fit in the above list. */
95      private CIHashtable  Headers = null;
96  
97      /** any trailers which were received and do not fit in the above list. */
98      private CIHashtable  Trailers = null;
99  
100     /** the ContentLength of the data. */
101     private int          ContentLength = -1;
102 
103     /** the data (body) returned. */
104     private byte[]       Data = null;
105 
106     /** signals if we have got and parsed the headers yet? */
107     private boolean      initialized = false;
108 
109     /** signals if we have got the trailers yet? */
110     private boolean      got_trailers = false;
111 
112     /** marks this response as aborted (stop() in HTTPConnection) */
113     private boolean      aborted = false;
114 
115     /** should the request be retried by the application? */
116     private boolean      retry = false;
117 
118     /** the method used in the request */
119     private String       method = null;
120 
121     /** ++GRINDER MODIFICATION **/
122     /** The time to first byte */
123     private long         ttfb;
124     /** --GRINDER MODIFICATION **/
125 
126     // Constructors
127 
128     /**
129      * Creates a new HTTPResponse.
130      *
131      * @param modules the list of modules handling this response
132      * @param timeout the timeout to be used on stream read()'s
133      */
134     HTTPResponse(HTTPClientModule[] modules, int timeout, Request orig)
135     {
136 	this.modules = modules;
137 	this.timeout = timeout;
138 	try
139 	{
140 	    int qp = orig.getRequestURI().indexOf('?');
141 	    this.OriginalURI = new URI(orig.getConnection().getProtocol(),
142 				       null,
143 				       orig.getConnection().getHost(),
144 				       orig.getConnection().getPort(),
145 				       qp < 0 ? orig.getRequestURI() :
146 					 orig.getRequestURI().substring(0, qp),
147 				       qp < 0 ? null :
148 					 orig.getRequestURI().substring(qp+1),
149 				       null);
150 	}
151 	catch (ParseException pe)
152 	    { }
153 	this.method = orig.getMethod();
154     }
155 
156 
157     /**
158      * @param req the request
159      * @param resp the response
160      */
161     void set(Request req, Response resp)
162     {
163 	this.request   = req;
164 	this.response  = resp;
165 	resp.http_resp = this;
166 	resp.timeout   = timeout;
167 	this.aborted   = resp.final_resp;
168     }
169 
170 
171     /**
172      * @param req the request
173      * @param resp the response
174      */
175     void set(Request req, HttpOutputStream out_stream)
176     {
177 	this.request    = req;
178 	this.out_stream = out_stream;
179     }
180 
181 
182     // Methods
183 
184     /**
185      * Give the status code for this request. These are grouped as follows:
186      * <UL>
187      *   <LI> 1xx - Informational (new in HTTP/1.1)
188      *   <LI> 2xx - Success
189      *   <LI> 3xx - Redirection
190      *   <LI> 4xx - Client Error
191      *   <LI> 5xx - Server Error
192      * </UL>
193      *
194      * @exception IOException if any exception occurs on the socket.
195      * @exception ModuleException if any module encounters an exception.
196      */
197     public final int getStatusCode()  throws IOException, ModuleException
198     {
199 	if (!initialized)  handleResponse();
200 	return StatusCode;
201     }
202 
203     /**
204      * Give the reason line associated with the status code.
205      *
206      * @exception IOException If any exception occurs on the socket.
207      * @exception ModuleException if any module encounters an exception.
208      */
209     public final String getReasonLine()  throws IOException, ModuleException
210     {
211 	if (!initialized)  handleResponse();
212 	return ReasonLine;
213     }
214 
215     /**
216      * Get the HTTP version used for the response.
217      *
218      * @exception IOException If any exception occurs on the socket.
219      * @exception ModuleException if any module encounters an exception.
220      */
221     public final String getVersion()  throws IOException, ModuleException
222     {
223 	if (!initialized)  handleResponse();
224 	return Version;
225     }
226 
227     /**
228      * Get the name and type of server.
229      *
230      * @deprecated This method is a remnant of V0.1; use
231      *             <code>getHeader("Server")</code> instead.
232      * @see #getHeader(java.lang.String)
233      * @exception IOException If any exception occurs on the socket.
234      * @exception ModuleException if any module encounters an exception.
235      */
236     public final String getServer()  throws IOException, ModuleException
237     {
238 	if (!initialized)  handleResponse();
239 	return getHeader("Server");
240     }
241 
242 
243     /**
244      * Get the original URI used in the request.
245      *
246      * @return the URI used in primary request
247      */
248     public final URI getOriginalURI()
249     {
250 	return OriginalURI;
251     }
252 
253 
254     /**
255      * Get the final URL of the document. This is set if the original
256      * request was deferred via the "moved" (301, 302, or 303) return
257      * status.
258      *
259      * @return the effective URL, or null if no redirection occured
260      * @exception IOException If any exception occurs on the socket.
261      * @exception ModuleException if any module encounters an exception.
262      * @deprecated use getEffectiveURI() instead
263      * @see #getEffectiveURI
264      */
265     public final URL getEffectiveURL()  throws IOException, ModuleException
266     {
267 	if (!initialized)  handleResponse();
268 	if (EffectiveURI != null)
269 	    return EffectiveURI.toURL();
270 	return null;
271     }
272 
273 
274     /**
275      * Get the final URI of the document. If the request was redirected
276      * via the "moved" (301, 302, 303, or 307) return status this returns
277      * the URI used in the last redirection; otherwise it returns the
278      * original URI.
279      *
280      * @return the effective URI
281      * @exception IOException If any exception occurs on the socket.
282      * @exception ModuleException if any module encounters an exception.
283      */
284     public final URI getEffectiveURI()  throws IOException, ModuleException
285     {
286 	if (!initialized)  handleResponse();
287 	if (EffectiveURI != null)
288 	    return EffectiveURI;
289 	return OriginalURI;
290     }
291 
292 
293     /**
294      * Retrieves the value for a given header.
295      *
296      * @param  hdr the header name.
297      * @return the value for the header, or null if non-existent.
298      * @exception IOException If any exception occurs on the socket.
299      * @exception ModuleException if any module encounters an exception.
300      */
301     public String getHeader(String hdr)  throws IOException, ModuleException
302     {
303 	if (!initialized)  handleResponse();
304 	return (String) Headers.get(hdr.trim());
305     }
306 
307     /**
308      * Retrieves the value for a given header. The value is parsed as an
309      * int.
310      *
311      * @param  hdr the header name.
312      * @return the value for the header if the header exists
313      * @exception NumberFormatException if the header's value is not a number
314      *                                  or if the header does not exist.
315      * @exception IOException if any exception occurs on the socket.
316      * @exception ModuleException if any module encounters an exception.
317      */
318     public int getHeaderAsInt(String hdr)
319 		throws IOException, ModuleException, NumberFormatException
320     {
321 	String val = getHeader(hdr);
322 	if (val == null)
323 	    throw new NumberFormatException("null");
324 	return Integer.parseInt(val);
325     }
326 
327     /**
328      * Retrieves the value for a given header. The value is parsed as a
329      * date; if this fails it is parsed as a long representing the number
330      * of seconds since 12:00 AM, Jan 1st, 1970. If this also fails an
331      * exception is thrown.
332      * <br>Note: When sending dates use Util.httpDate().
333      *
334      * @param  hdr the header name.
335      * @return the value for the header, or null if non-existent.
336      * @exception IllegalArgumentException if the header's value is neither a
337      *            legal date nor a number.
338      * @exception IOException if any exception occurs on the socket.
339      * @exception ModuleException if any module encounters an exception.
340      */
341     public Date getHeaderAsDate(String hdr)
342 		throws IOException, IllegalArgumentException, ModuleException
343     {
344 	String raw_date = getHeader(hdr);
345 	if (raw_date == null) return null;
346 
347 	// asctime() format is missing an explicit GMT specifier
348 	if (raw_date.toUpperCase().indexOf("GMT") == -1  &&
349 	    raw_date.indexOf(' ') > 0)
350 	    raw_date += " GMT";
351 
352 	Date   date;
353 
354 	try
355 	    { date = Util.parseHttpDate(raw_date); }
356 	catch (IllegalArgumentException iae)
357 	{
358 	    // some servers erroneously send a number, so let's try that
359 	    long time;
360 	    try
361 		{ time = Long.parseLong(raw_date); }
362 	    catch (NumberFormatException nfe)
363 		{ throw iae; }	// give up
364 	    if (time < 0)  time = 0;
365 	    date = new Date(time * 1000L);
366 	}
367 
368 	return date;
369     }
370 
371     /**
372      * Returns an enumeration of all the headers available via getHeader().
373      *
374      * @exception IOException If any exception occurs on the socket.
375      * @exception ModuleException if any module encounters an exception.
376      */
377     public Enumeration listHeaders()  throws IOException, ModuleException
378     {
379 	if (!initialized)  handleResponse();
380 	return Headers.keys();
381     }
382 
383     /** ++GRINDER MODIFICATION **/
384     public long getTimeToFirstByte(){
385             return ttfb;
386     }
387     /** --GRINDER MODIFICATION **/
388 
389     /**
390      * Retrieves the value for a given trailer. This should not be invoked
391      * until all response data has been read. If invoked before it will
392      * call <code>getData()</code> to force the data to be read.
393      *
394      * @param  trailer the trailer name.
395      * @return the value for the trailer, or null if non-existent.
396      * @exception IOException If any exception occurs on the socket.
397      * @exception ModuleException if any module encounters an exception.
398      * @see #getData()
399      */
400     public String getTrailer(String trailer) throws IOException, ModuleException
401     {
402 	if (!got_trailers)  getTrailers();
403 	return (String) Trailers.get(trailer.trim());
404     }
405 
406     /**
407      * Retrieves the value for a given tailer. The value is parsed as an
408      * int.
409      *
410      * @param  trailer the tailer name.
411      * @return the value for the trailer if the trailer exists
412      * @exception NumberFormatException if the trailer's value is not a number
413      *                                  or if the trailer does not exist.
414      * @exception IOException if any exception occurs on the socket.
415      * @exception ModuleException if any module encounters an exception.
416      */
417     public int getTrailerAsInt(String trailer)
418 		throws IOException, ModuleException, NumberFormatException
419     {
420 	String val = getTrailer(trailer);
421 	if (val == null)
422 	    throw new NumberFormatException("null");
423 	return Integer.parseInt(val);
424     }
425 
426     /**
427      * Retrieves the value for a given trailer. The value is parsed as a
428      * date; if this fails it is parsed as a long representing the number
429      * of seconds since 12:00 AM, Jan 1st, 1970. If this also fails an
430      * IllegalArgumentException is thrown.
431      * <br>Note: When sending dates use Util.httpDate().
432      *
433      * @param  trailer the trailer name.
434      * @return the value for the trailer, or null if non-existent.
435      * @exception IllegalArgumentException if the trailer's value is neither a
436      *            legal date nor a number.
437      * @exception IOException if any exception occurs on the socket.
438      * @exception ModuleException if any module encounters an exception.
439      */
440     public Date getTrailerAsDate(String trailer)
441 		throws IOException, IllegalArgumentException, ModuleException
442     {
443 	String raw_date = getTrailer(trailer);
444 	if (raw_date == null) return null;
445 
446 	// asctime() format is missing an explicit GMT specifier
447 	if (raw_date.toUpperCase().indexOf("GMT") == -1  &&
448 	    raw_date.indexOf(' ') > 0)
449 	    raw_date += " GMT";
450 
451 	Date   date;
452 
453 	try
454 	    { date = Util.parseHttpDate(raw_date); }
455 	catch (IllegalArgumentException iae)
456 	{
457 	    // some servers erroneously send a number, so let's try that
458 	    long time;
459 	    try
460 		{ time = Long.parseLong(raw_date); }
461 	    catch (NumberFormatException nfe)
462 		{ throw iae; }	// give up
463 	    if (time < 0)  time = 0;
464 	    date = new Date(time * 1000L);
465 	}
466 
467 	return date;
468     }
469 
470     /**
471      * Returns an enumeration of all the trailers available via getTrailer().
472      *
473      * @exception IOException If any exception occurs on the socket.
474      * @exception ModuleException if any module encounters an exception.
475      */
476     public Enumeration listTrailers()  throws IOException, ModuleException
477     {
478 	if (!got_trailers)  getTrailers();
479 	return Trailers.keys();
480     }
481 
482 
483     /**
484      * Reads all the response data into a byte array. Note that this method
485      * won't return until <em>all</em> the data has been received (so for
486      * instance don't invoke this method if the server is doing a server
487      * push). If <code>getInputStream()</code> had been previously invoked
488      * then this method only returns any unread data remaining on the stream
489      * and then closes it.
490      *
491      * <P>Note to the unwary: code like
492      *<PRE>
493      *     System.out.println("The data: " + resp.getData())
494      *</PRE>
495      * will probably not do what you want - use
496      *<PRE>
497      *     System.out.println("The data: " + resp.getText())
498      *</PRE>
499      * instead.
500      *
501      * @see #getInputStream()
502      * @return an array containing the data (body) returned. If no data
503      *         was returned then it's set to a zero-length array.
504      * @exception IOException If any io exception occured while reading
505      *			      the data
506      * @exception ModuleException if any module encounters an exception.
507      */
508     public synchronized byte[] getData()  throws IOException, ModuleException
509     {
510 	if (!initialized)  handleResponse();
511 
512 	if (Data == null)
513 	{
514 	    try
515 		{ readResponseData(inp_stream); }
516 	    catch (InterruptedIOException ie)		// don't intercept
517 		{ throw ie; }
518 	    catch (IOException ioe)
519 	    {
520 		Log.write(Log.RESP, "HResp: (\"" + method + " " +
521 				    OriginalURI.getPathAndQuery() + "\")");
522 		Log.write(Log.RESP, "       ", ioe);
523 
524 		try { inp_stream.close(); } catch (Exception e) { }
525 		throw ioe;
526 	    }
527 
528 	    inp_stream.close();
529 	}
530 
531 	return Data;
532     }
533 
534     /**
535      * Reads all the response data into a buffer and turns it into a string
536      * using the appropriate character converter. Since this uses {@link
537      * #getData() getData()}, the caveats of that method apply here as well.
538      *
539      * @see #getData()
540      * @return the body as a String. If no data was returned then an empty
541      *         string is returned.
542      * @exception IOException If any io exception occured while reading
543      *			      the data, or if the content is not text
544      * @exception ModuleException if any module encounters an exception.
545      * @exception ParseException if an error occured trying to parse the
546      *                           content-type header field
547      */
548     public synchronized String getText()
549 	throws IOException, ModuleException, ParseException
550     {
551 	String ct = getHeader("Content-Type");
552 
553 	/** ++GRINDER MODIFICATION **/
554 	if (ct == null) {
555 	    return new String(getData(), "ISO-8859-1");
556 	}
557 
558 	// if (ct == null  ||  !ct.toLowerCase().startsWith("text/"))
559         // throw new IOException("Content-Type `" + ct + "' is not a text type");
560 	/** --GRINDER MODIFICATION **/
561 
562 	String charset = Util.getParameter("charset", ct);
563 	if (charset == null)
564 	    charset = "ISO-8859-1";
565 
566 	return new String(getData(), charset);
567     }
568 
569     /**
570      * Gets an input stream from which the returned data can be read. Note
571      * that if <code>getData()</code> had been previously invoked it will
572      * actually return a ByteArrayInputStream created from that data.
573      *
574      * @see #getData()
575      * @return the InputStream.
576      * @exception IOException If any exception occurs on the socket.
577      * @exception ModuleException if any module encounters an exception.
578      */
579     public synchronized InputStream getInputStream()
580 	    throws IOException, ModuleException
581     {
582 	if (!initialized)  handleResponse();
583 
584 	if (Data == null)
585 	    return inp_stream;
586 	else
587 	{
588 	    getData();		// ensure complete data is read
589 	    return new ByteArrayInputStream(Data);
590 	}
591     }
592 
593 
594     /**
595      * Should the request be retried by the application? If the application
596      * used an <var>HttpOutputStream</var> in the request then various
597      * modules (such as the redirection and authorization modules) are not
598      * able to resend the request themselves. Instead, it becomes the
599      * application's responsibility. The application can check this flag, and
600      * if it's set, resend the exact same request. The modules such as the
601      * RedirectionModule or AuthorizationModule will then recognize the resend
602      * and fix up or redirect the request as required (i.e. they defer their
603      * normal action until the resend).
604      *
605      * <P>If the application resends the request then it <strong>must</strong>
606      * use the same <var>HttpOutputStream</var> instance. This is because the
607      * modules use this to recognize the retried request and to perform the
608      * necessary work on the request before it's sent.
609      *
610      * <P>Here is a skeleton example of usage:
611      * <PRE>
612      *     OutputStream out = new HttpOutputStream(1234);
613      *     do
614      *     {
615      *         rsp = con.Post("/cgi-bin/my_cgi", out);
616      *         out.write(...);
617      *         out.close();
618      *     } while (rsp.retryRequest());
619      *
620      *     if (rsp.getStatusCode() >= 300)
621      *         ...
622      * </PRE>
623      *
624      * <P>Note that for this to ever return true, the java system property
625      * <var>HTTPClient.deferStreamed</var> must be set to true at the beginning
626      * of the application (before the HTTPConnection class is loaded). This
627      * prevents unwary applications from causing inadvertent memory leaks. If
628      * an application does set this, then it <em>must</em> resend any request
629      * whose response returns true here in order to prevent memory leaks (a
630      * switch to JDK 1.2 will allow us to use weak references and eliminate
631      * this problem).
632      *
633      * @return true if the request should be retried.
634      * @exception IOException If any exception occurs on the socket.
635      * @exception ModuleException if any module encounters an exception.
636      */
637     public boolean retryRequest()  throws IOException, ModuleException
638     {
639 	if (!initialized)
640 	{
641 	    try
642 		{ handleResponse(); }
643 	    catch (RetryException re)
644 		{ this.retry = response.retry; }
645 	}
646 	return retry;
647     }
648 
649 
650     /**
651      * produces a full list of headers and their values, one per line.
652      *
653      * @return a string containing the headers
654      */
655     public String toString()
656     {
657 	if (!initialized)
658 	{
659 	    try
660 		{ handleResponse(); }
661 	    catch (Exception e)
662 	    {
663 		if (!(e instanceof InterruptedIOException))
664 		{
665 		    Log.write(Log.RESP, "HResp: (\"" + method + " " +
666 				        OriginalURI.getPathAndQuery() + "\")");
667 		    Log.write(Log.RESP, "       ", e);
668 		}
669 		return "Failed to read headers: " + e;
670 	    }
671 	}
672 
673 	String nl = System.getProperty("line.separator", "\n");
674 
675 	StringBuffer str = new StringBuffer(Version);
676 	str.append(' ');
677 	str.append(StatusCode);
678 	str.append(' ');
679 	str.append(ReasonLine);
680 	str.append(nl);
681 
682 	if (EffectiveURI != null)
683 	{
684 	    str.append("Effective-URI: ");
685 	    str.append(EffectiveURI);
686 	    str.append(nl);
687 	}
688 
689 	Enumeration hdr_list = Headers.keys();
690 	while (hdr_list.hasMoreElements())
691 	{
692 	    String hdr = (String) hdr_list.nextElement();
693 	    str.append(hdr);
694 	    str.append(": ");
695 	    str.append(Headers.get(hdr));
696 	    str.append(nl);
697 	}
698 
699 	return str.toString();
700     }
701 
702 
703     // Helper Methods
704 
705 
706     HTTPClientModule[] getModules()
707     {
708 	return modules;
709     }
710 
711 
712     /**
713      * Processes a Response. This is done by calling the response handler
714      * in each module. When all is done, the various fields of this instance
715      * are intialized from the last Response.
716      *
717      * @exception IOException if any handler throws an IOException.
718      * @exception ModuleException if any module encounters an exception.
719      * @return true if a new request was generated. This is used for internal
720      *         subrequests only
721      */
722     synchronized boolean handleResponse()  throws IOException, ModuleException
723     {
724 	if (initialized)  return false;
725 
726 
727 	/* first get the response if necessary */
728 
729 	if (out_stream != null)
730 	{
731 	    response           = out_stream.getResponse();
732 	    response.http_resp = this;
733 	    out_stream         = null;
734 	}
735 
736 
737 	/* go through modules and handle them */
738 
739 	doModules: while (true)
740 	{
741 
742 	Phase1: for (int idx=0; idx<modules.length && !aborted; idx++)
743 	{
744 	    try
745 		{ modules[idx].responsePhase1Handler(response, request); }
746 	    catch (RetryException re)
747 	    {
748 		if (re.restart)
749 		    continue doModules;
750 		else
751 		    throw re;
752 	    }
753 	}
754 
755 	Phase2: for (int idx=0; idx<modules.length && !aborted; idx++)
756 	{
757             int sts = modules[idx].responsePhase2Handler(response, request);
758             switch (sts)
759             {
760                 case RSP_CONTINUE:	// continue processing
761                     break;
762 
763                 case RSP_RESTART:	// restart response processing
764                     idx = -1;
765 		    continue doModules;
766 
767                 case RSP_SHORTCIRC:	// stop processing and return
768                     break doModules;
769 
770                 case RSP_REQUEST:	// go to phase 1
771                 case RSP_NEWCON_REQ:	// process the request using a new con
772 		    response.getInputStream().close();
773 		    if (handle_trailers) invokeTrailerHandlers(true);
774 		    if (request.internal_subrequest)  return true;
775 		    request.getConnection().
776 				handleRequest(request, this, response, true);
777 		    if (initialized)  break doModules;
778 
779                     idx = -1;
780 		    continue doModules;
781 
782                 case RSP_SEND:		// send the request immediately
783                 case RSP_NEWCON_SND:	// send the request using a new con
784 		    response.getInputStream().close();
785 		    if (handle_trailers) invokeTrailerHandlers(true);
786 		    if (request.internal_subrequest)  return true;
787 		    request.getConnection().
788 				handleRequest(request, this, response, false);
789                     idx = -1;
790 		    continue doModules;
791 
792                 default:                // not valid
793                     throw new Error("HTTPClient Internal Error: invalid status"+
794                                     " " + sts + " returned by module " +
795                                     modules[idx].getClass().getName());
796 	    }
797 	}
798 
799 	Phase3: for (int idx=0; idx<modules.length && !aborted; idx++)
800 	{
801             modules[idx].responsePhase3Handler(response, request);
802 	}
803 
804 	break doModules;
805 	}
806 
807 	/* force a read on the response in case none of the modules did */
808 	response.getStatusCode();
809 
810 	/* all done, so copy data */
811 	if (!request.internal_subrequest)
812 	    init(response);
813 
814 	if (handle_trailers)
815 	    invokeTrailerHandlers(false);
816 
817 	return false;
818     }
819 
820 
821     /**
822      * Copies the relevant fields from Response and marks this as initialized.
823      *
824      * @param resp the Response class to copy from
825      */
826     void init(Response resp)
827     {
828 	if (initialized)  return;
829 
830 	this.StatusCode    = resp.StatusCode;
831 	this.ReasonLine    = resp.ReasonLine;
832 	this.Version       = resp.Version;
833 	this.EffectiveURI  = resp.EffectiveURI;
834 	this.ContentLength = resp.ContentLength;
835 	this.Headers       = resp.Headers;
836 	this.inp_stream    = resp.inp_stream;
837 	this.Data          = resp.Data;
838 	this.retry         = resp.retry;
839 	this.ttfb          = resp.getTtfb();
840 	initialized        = true;
841     }
842 
843 
844     private boolean handle_trailers  = false;
845     private boolean trailers_handled = false;
846 
847     /**
848      * This is invoked by the RespInputStream when it is close()'d. It
849      * just invokes the trailer handler in each module.
850      *
851      * @param force invoke the handlers even if not initialized yet?
852      * @exception IOException     if thrown by any module
853      * @exception ModuleException if thrown by any module
854      */
855     void invokeTrailerHandlers(boolean force)
856 	    throws IOException, ModuleException
857     {
858 	if (trailers_handled)  return;
859 
860 	if (!force  &&  !initialized)
861 	{
862 	    handle_trailers = true;
863 	    return;
864 	}
865 
866 	for (int idx=0; idx<modules.length && !aborted; idx++)
867 	{
868             modules[idx].trailerHandler(response, request);
869 	}
870 
871 	trailers_handled = true;
872     }
873 
874 
875     /**
876      * Mark this request as having been aborted. It's invoked by
877      * HTTPConnection.stop().
878      */
879     void markAborted()
880     {
881 	aborted = true;
882     }
883 
884 
885     /**
886      * Gets any trailers from the response if we haven't already done so.
887      */
888     private synchronized void getTrailers()  throws IOException, ModuleException
889     {
890 	if (got_trailers)  return;
891 	if (!initialized)  handleResponse();
892 
893 	response.getTrailer("Any");
894 	Trailers = response.Trailers;
895 	got_trailers = true;
896 
897 	invokeTrailerHandlers(false);
898     }
899 
900 
901     /**
902      * Reads the response data received. Does not return until either
903      * Content-Length bytes have been read or EOF is reached.
904      * 
905      * @inp the input stream from which to read the data
906      * @exception IOException if any read on the input stream fails
907      */
908     private void readResponseData(InputStream inp)
909 	    throws IOException, ModuleException
910     {
911     /** ++GRINDER MODIFICATION * */
912     // if (ContentLength == 0)
913     // return;
914     /** --GRINDER MODIFICATION * */
915 
916     if (Data == null)
917       Data = new byte[0];
918 
919     /** ++GRINDER MODIFICATION * */
920     if (ContentLength == 0)
921       return;
922     /** --GRINDER MODIFICATION * */
923 
924     // read response data
925     int off = Data.length;
926 
927     try {
928       /** ++GRINDER MODIFICATION * */
929       // // check Content-length header in case CE-Module removed it
930       // if (getHeader("Content-Length") != null)
931       // {
932       // int rcvd = 0;
933       // Data = new byte[ContentLength];
934       //
935       // do
936       // {
937       // off += rcvd;
938       // rcvd = inp.read(Data, off, ContentLength-off);
939       // } while (rcvd != -1 && off+rcvd < ContentLength);
940       //
941       // /* Don't do this!
942       // * If we do, then getData() won't work after a getInputStream()
943       // * because we'll never get all the expected data. Instead, let
944       // * the underlying RespInputStream throw the EOF.
945       // if (rcvd == -1) // premature EOF
946       // {
947       // throw new EOFException("Encountered premature EOF while " +
948       // "reading headers: received " + off +
949       // " bytes instead of the expected " +
950       // ContentLength + " bytes");
951       // }
952       // */
953       // }
954       // else
955       // {
956       // int inc = 1000,
957       // rcvd = 0;
958       //
959       // do
960       // {
961       // off += rcvd;
962       // Data = Util.resizeArray(Data, off+inc);
963       // } while ((rcvd = inp.read(Data, off, inc)) != -1);
964       //
965       // Data = Util.resizeArray(Data, off);
966       // }
967       
968       final HTTPConnection.BandwidthLimiterFactory
969         bandwidthLimiterFactory =
970           request.getConnection().getBandwidthLimiterFactory();
971 
972       final HTTPConnection.BandwidthLimiter bandwidthLimiter =
973         bandwidthLimiterFactory.create();
974       
975       final boolean fixedSize;
976 
977       // Check Content-length header in case CE-Module removed it.
978       if (getHeader("Content-Length") != null) {
979         // As per the original code, we don't raise problems about unexpected
980         // EOFs if the available data doesn't match the content length.
981         Data = new byte[ContentLength];
982         fixedSize = true;
983       }
984       else {
985         Data = new byte[1000];
986         fixedSize = false;
987       }
988 
989       int rcvd = 0;
990 
991       do {
992         off += rcvd;
993 
994         if (fixedSize) {
995           if (off >= Data.length) {
996             break;
997           }
998         }
999         else {
1000           // Grow exponentially so that the number of copies for an N byte
1001           // response is O(ln N). We resize every time we've used at least half
1002           // the remaining bytes.
1003           if ((Data.length - off) < Data.length / 2) {
1004             Data = Util.resizeArray(Data,  Data.length * 2);
1005           }
1006         }
1007         
1008         final int maximumBytes =
1009           Math.min(Data.length - off, bandwidthLimiter.maximumBytes(off));
1010 
1011         rcvd = inp.read(Data, off, maximumBytes);
1012       }
1013       while (rcvd != -1);
1014 
1015       if (off < Data.length) {
1016         Data = Util.resizeArray(Data, off);
1017       }
1018 
1019       /** --GRINDER MODIFICATION * */
1020     }
1021     catch (IOException ioe)
1022 	{
1023       Data = Util.resizeArray(Data, off);
1024       throw ioe;
1025     }
1026 	finally
1027 	{
1028 	    try
1029 		{ inp.close(); }
1030 	    catch (IOException ioe)
1031 		{ }
1032       }
1033       }
1034 
1035 
1036     int getTimeout()
1037     {
1038 	return timeout;
1039     }
1040 }