hunt.http.codec.http.decode.HttpParser source code

1 module hunt.http.codec.http.decode.HttpParser;
2 
3 import hunt.http.codec.http.model;
4 import hunt.http.codec.http.hpack.HpackEncoder;
5 
6 import hunt.container;
7 import hunt.datetime;
8 import hunt.lang.exception;
9 import hunt.logging;
10 import hunt.string;
11 import hunt.util.TypeUtils;
12 
13 import core.time;
14 
15 import std.algorithm;
16 import std.array;
17 import std.container.array;
18 import std.conv;
19 import std.string;
20 
21 
22 private bool contains(T)(T[] items, T item) {
23     return items.canFind(item);
24 }
25 
26 
27 /* ------------------------------------------------------------ */
28 
29 /**
30  * A Parser for 1.0 and 1.1 as defined by RFC7230
31  * <p>
32  * This parser parses HTTP client and server messages from buffers
33  * passed in the {@link #parseNext(ByteBuffer)} method.  The parsed
34  * elements of the HTTP message are passed as event calls to the
35  * {@link HttpHandler} instance the parser is constructed with.
36  * If the passed handler is a {@link RequestHandler} then server side
37  * parsing is performed and if it is a {@link ResponseHandler}, then
38  * client side parsing is done.
39  * </p>
40  * <p>
41  * The contract of the {@link HttpHandler} API is that if a call returns
42  * true then the call to {@link #parseNext(ByteBuffer)} will return as
43  * soon as possible also with a true response.  Typically this indicates
44  * that the parsing has reached a stage where the caller should process
45  * the events accumulated by the handler.    It is the preferred calling
46  * style that handling such as calling a servlet to process a request,
47  * should be done after a true return from {@link #parseNext(ByteBuffer)}
48  * rather than from within the scope of a call like
49  * {@link RequestHandler#messageComplete()}
50  * </p>
51  * <p>
52  * For performance, the parse is heavily dependent on the
53  * {@link Trie#getBest(ByteBuffer, int, int)} method to look ahead in a
54  * single pass for both the structure ( : and CRLF ) and semantic (which
55  * header and value) of a header.  Specifically the static {@link HttpHeader#CACHE}
56  * is used to lookup common combinations of headers and values
57  * (eg. "Connection: close"), or just header names (eg. "Connection:" ).
58  * For headers who's value is not known statically (eg. Host, COOKIE) then a
59  * per parser dynamic Trie of {@link HttpFields} from previous parsed messages
60  * is used to help the parsing of subsequent messages.
61  * </p>
62  * <p>
63  * The parser can work in varying compliance modes:
64  * <dl>
65  * <dt>RFC7230</dt><dd>(default) Compliance with RFC7230</dd>
66  * <dt>RFC2616</dt><dd>Wrapped headers and HTTP/0.9 supported</dd>
67  * <dt>LEGACY</dt><dd>(aka STRICT) Adherence to Servlet Specification requirement for
68  * exact case of header names, bypassing the header caches, which are case insensitive,
69  * otherwise equivalent to RFC2616</dd>
70  * </dl>
71  *
72  * @see <a href="http://tools.ietf.org/html/rfc7230">RFC 7230</a>
73  */
74 class HttpParser {
75 
76     enum INITIAL_URI_LENGTH = 256;
77     private MonoTime startTime;
78 
79     /**
80      * Cache of common {@link HttpField}s including: <UL>
81      * <LI>Common static combinations such as:<UL>
82      * <li>Connection: close
83      * <li>Accept-Encoding: gzip
84      * <li>Content-Length: 0
85      * </ul>
86      * <li>Combinations of Content-Type header for common mime types by common charsets
87      * <li>Most common headers with null values so that a lookup will at least
88      * determine the header name even if the name:value combination is not cached
89      * </ul>
90      */
91     __gshared Trie!HttpField CACHE;
92 
93     // States
94     enum FieldState {
95         FIELD,
96         IN_NAME,
97         VALUE,
98         IN_VALUE,
99         WS_AFTER_NAME,
100     }
101 
102     // States
103     enum State {
104         START,
105         METHOD,
106         RESPONSE_VERSION,
107         SPACE1,
108         STATUS,
109         URI,
110         SPACE2,
111         REQUEST_VERSION,
112         REASON,
113         PROXY,
114         HEADER,
115         CONTENT,
116         EOF_CONTENT,
117         CHUNKED_CONTENT,
118         CHUNK_SIZE,
119         CHUNK_PARAMS,
120         CHUNK,
121         TRAILER,
122         END,
123         CLOSE,  // The associated stream/endpoint should be closed
124         CLOSED  // The associated stream/endpoint is at EOF
125     }
126 
127     private static State[] __idleStates = [State.START, State.END, State.CLOSE, State.CLOSED];
128     private static State[] __completeStates = [State.END, State.CLOSE, State.CLOSED];
129 
130     private HttpHandler _handler;
131     private RequestHandler _requestHandler;
132     private ResponseHandler _responseHandler;
133     private ComplianceHandler _complianceHandler;
134     private int _maxHeaderBytes;
135     private HttpCompliance _compliance;
136     private HttpComplianceSection[] _compliances;
137     private HttpField _field;
138     private HttpHeader _header;
139     private string _headerString;
140     private string _valueString;
141     private int _responseStatus;
142     private int _headerBytes;
143     private bool _host;
144     private bool _headerComplete;
145 
146     /* ------------------------------------------------------------------------------- */
147     private  State _state = State.START;
148     private  FieldState _fieldState = FieldState.FIELD;
149     private  bool _eof;
150     private HttpMethod _method;
151     private string _methodString;
152     private HttpVersion _version;
153     private StringBuilder _uri;
154     private EndOfContent _endOfContent;
155     private long _contentLength = -1;
156     private long _contentPosition;
157     private int _chunkLength;
158     private int _chunkPosition;
159     private bool _headResponse;
160     private bool _cr;
161     private ByteBuffer _contentChunk;
162     private Trie!HttpField _fieldCache;
163 
164     private int _length;
165     private StringBuilder _string; 
166 
167     shared static this() {
168         CACHE = new ArrayTrie!HttpField(2048);
169         CACHE.put(new HttpField(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE));
170         CACHE.put(new HttpField(HttpHeader.CONNECTION, HttpHeaderValue.KEEP_ALIVE));
171         CACHE.put(new HttpField(HttpHeader.CONNECTION, HttpHeaderValue.UPGRADE));
172         CACHE.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip"));
173         CACHE.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip, deflate"));
174         CACHE.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip, deflate, br"));
175         CACHE.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip,deflate,sdch"));
176         CACHE.put(new HttpField(HttpHeader.ACCEPT_LANGUAGE, "en-US,en;q=0.5"));
177         CACHE.put(new HttpField(HttpHeader.ACCEPT_LANGUAGE, "en-GB,en-US;q=0.8,en;q=0.6"));
178 
179         CACHE.put(new HttpField(HttpHeader.ACCEPT_LANGUAGE, 
180             "en-AU,en;q=0.9,it-IT;q=0.8,it;q=0.7,en-GB;q=0.6,en-US;q=0.5"));
181             
182         CACHE.put(new HttpField(HttpHeader.ACCEPT_CHARSET, "ISO-8859-1,utf-8;q=0.7,*;q=0.3"));
183         CACHE.put(new HttpField(HttpHeader.ACCEPT, "*/*"));
184         CACHE.put(new HttpField(HttpHeader.ACCEPT, "image/png,image/*;q=0.8,*/*;q=0.5"));
185         CACHE.put(new HttpField(HttpHeader.ACCEPT, 
186             "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"));
187 
188         CACHE.put(new HttpField(HttpHeader.ACCEPT, 
189             "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"));
190 
191         CACHE.put(new HttpField(HttpHeader.ACCEPT_RANGES, HttpHeaderValue.BYTES));
192         CACHE.put(new HttpField(HttpHeader.PRAGMA, "no-cache"));
193 
194         CACHE.put(new HttpField(HttpHeader.CACHE_CONTROL, 
195             "private, no-cache, no-cache=Set-Cookie, proxy-revalidate"));
196 
197         CACHE.put(new HttpField(HttpHeader.CACHE_CONTROL, "no-cache"));
198         CACHE.put(new HttpField(HttpHeader.CACHE_CONTROL, "max-age=0"));
199         CACHE.put(new HttpField(HttpHeader.CONTENT_LENGTH, "0"));
200         CACHE.put(new HttpField(HttpHeader.CONTENT_ENCODING, "gzip"));
201         CACHE.put(new HttpField(HttpHeader.CONTENT_ENCODING, "deflate"));
202         CACHE.put(new HttpField(HttpHeader.TRANSFER_ENCODING, "chunked"));
203         CACHE.put(new HttpField(HttpHeader.EXPIRES, "Fri, 01 Jan 1990 00:00:00 GMT"));
204 
205         // Add common Content types as fields
206         foreach (string type ; ["text/plain", "text/html", "text/xml", "text/json", 
207             "application/json", "application/x-www-form-urlencoded"]) {
208             HttpField field = new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type);
209             CACHE.put(field);
210 
211             foreach (string charset ; ["utf-8", "iso-8859-1"]) {
212                 CACHE.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ ";charset=" ~ charset));
213                 CACHE.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ "; charset=" ~ charset));
214                 CACHE.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ ";charset=" ~ charset.toUpper()));
215                 CACHE.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ "; charset=" ~ charset.toUpper()));
216             }
217         }
218 
219         // Add headers with null values so HttpParser can avoid looking up name again for unknown values
220         foreach (HttpHeader h ; HttpHeader.values()) {
221             // trace(h.toString());
222             if (!CACHE.put(new HttpField(h, cast(string) null))) {
223                 // FIXME: Needing refactor or cleanup -@zxp at 9/25/2018, 8:11:29 PM
224                 // 
225                 // warning(h.toString());
226                 // throw new IllegalStateException("CACHE FULL");
227             }
228         }
229 
230         // Add some more common headers
231         CACHE.put(new HttpField(HttpHeader.REFERER, cast(string) null));
232         CACHE.put(new HttpField(HttpHeader.IF_MODIFIED_SINCE, cast(string) null));
233         CACHE.put(new HttpField(HttpHeader.IF_NONE_MATCH, cast(string) null));
234         CACHE.put(new HttpField(HttpHeader.AUTHORIZATION, cast(string) null));
235         CACHE.put(new HttpField(HttpHeader.COOKIE, cast(string) null));
236     }
237 
238     private static HttpCompliance getCompliance() {
239         return HttpCompliance.RFC7230;
240     }
241 
242     /* ------------------------------------------------------------------------------- */
243     this(RequestHandler handler) {
244         this(handler, -1, getCompliance());
245     }
246 
247     /* ------------------------------------------------------------------------------- */
248     this(ResponseHandler handler) {
249         this(handler, -1, getCompliance());
250     }
251 
252     /* ------------------------------------------------------------------------------- */
253     this(RequestHandler handler, int maxHeaderBytes) {
254         this(handler, maxHeaderBytes, getCompliance());
255     }
256 
257     /* ------------------------------------------------------------------------------- */
258     this(ResponseHandler handler, int maxHeaderBytes) {
259         this(handler, maxHeaderBytes, getCompliance());
260     }
261 
262     /* ------------------------------------------------------------------------------- */
263     this(RequestHandler handler, HttpCompliance compliance) {
264         this(handler, -1, compliance);
265     }
266 
267     /* ------------------------------------------------------------------------------- */
268     this(RequestHandler handler, int maxHeaderBytes, HttpCompliance compliance) {
269         this(handler, null, maxHeaderBytes, compliance is null ? getCompliance() : compliance);
270     }
271 
272     /* ------------------------------------------------------------------------------- */
273     this(ResponseHandler handler, int maxHeaderBytes, HttpCompliance compliance) {
274         this(null, handler, maxHeaderBytes, compliance is null ? getCompliance() : compliance);
275     }
276 
277     /* ------------------------------------------------------------------------------- */
278     private this(RequestHandler requestHandler, ResponseHandler responseHandler, 
279         int maxHeaderBytes, HttpCompliance compliance) {
280         version (HUNT_DEBUG) {
281             warning("create http parser");
282         }
283         _string = new StringBuilder();
284         _uri = new StringBuilder(INITIAL_URI_LENGTH);
285         _handler = requestHandler !is null ? cast(HttpHandler)requestHandler : cast(HttpHandler)responseHandler;
286         _requestHandler = requestHandler;
287         _responseHandler = responseHandler;
288         _maxHeaderBytes = maxHeaderBytes;
289         _compliance = compliance;
290         _compliances = compliance.sections();
291         _complianceHandler = cast(ComplianceHandler)_handler;
292     }
293 
294     /* ------------------------------------------------------------------------------- */
295     HttpHandler getHandler() {
296         return _handler;
297     }
298 
299     /* ------------------------------------------------------------------------------- */
300 
301     /**
302      * Check RFC compliance violation
303      *
304      * @param violation The compliance section violation
305      * @param reason    The reason for the violation
306      * @return True if the current compliance level is set so as to Not allow this violation
307      */
308     protected bool complianceViolation(HttpComplianceSection violation, string reason) {
309         if (_compliances.contains(violation))
310             return true;
311 
312         if (_complianceHandler !is null)
313             _complianceHandler.onComplianceViolation(_compliance, violation, reason);
314 
315         return false;
316     }
317 
318     /* ------------------------------------------------------------------------------- */
319     protected void handleViolation(HttpComplianceSection section, string reason) {
320         if (_complianceHandler !is null)
321             _complianceHandler.onComplianceViolation(_compliance, section, reason);
322     }
323 
324     /* ------------------------------------------------------------------------------- */
325     protected string caseInsensitiveHeader(string orig, string normative) {
326         if (_compliances.contains(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE))
327             return normative;
328         if (!orig.equals(normative))
329             handleViolation(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE, orig);
330         return orig;
331     }
332 
333     /* ------------------------------------------------------------------------------- */
334     long getContentLength() {
335         return _contentLength;
336     }
337 
338     /* ------------------------------------------------------------ */
339     long getContentRead() {
340         return _contentPosition;
341     }
342 
343     /* ------------------------------------------------------------ */
344 
345     /**
346      * Set if a HEAD response is expected
347      *
348      * @param head true if head response is expected
349      */
350     void setHeadResponse(bool head) {
351         _headResponse = head;
352     }
353 
354     /* ------------------------------------------------------------------------------- */
355     protected void setResponseStatus(int status) {
356         _responseStatus = status;
357     }
358 
359     /* ------------------------------------------------------------------------------- */
360     State getState() {
361         return _state;
362     }
363 
364     /* ------------------------------------------------------------------------------- */
365     bool inContentState() {
366         return _state >= State.CONTENT && _state < State.END;
367     }
368 
369     /* ------------------------------------------------------------------------------- */
370     bool inHeaderState() {
371         return _state < State.CONTENT;
372     }
373 
374     /* ------------------------------------------------------------------------------- */
375     bool isChunking() {
376         return _endOfContent == EndOfContent.CHUNKED_CONTENT;
377     }
378 
379     /* ------------------------------------------------------------ */
380     bool isStart() {
381         return isState(State.START);
382     }
383 
384     /* ------------------------------------------------------------ */
385     bool isClose() {
386         return isState(State.CLOSE);
387     }
388 
389     /* ------------------------------------------------------------ */
390     bool isClosed() {
391         return isState(State.CLOSED);
392     }
393 
394     /* ------------------------------------------------------------ */
395     bool isIdle() {
396         return __idleStates.contains(_state);
397     }
398 
399     /* ------------------------------------------------------------ */
400     bool isComplete() {
401         return __completeStates.contains(_state);
402     }
403 
404     /* ------------------------------------------------------------------------------- */
405     bool isState(State state) {
406         return _state == state;
407     }
408 
409     /* ------------------------------------------------------------------------------- */
410     enum CharState {
411         ILLEGAL, CR, LF, LEGAL
412     }
413 
414     private static CharState[] __charState;
415 
416     static this() {
417         // token          = 1*tchar
418         // tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
419         //                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
420         //                / DIGIT / ALPHA
421         //                ; any VCHAR, except delimiters
422         // quoted-string  = DQUOTE *( qdtext / quoted-pair ) DQUOTE
423         // qdtext         = HTAB / SP /%x21 / %x23-5B / %x5D-7E / obs-text
424         // obs-text       = %x80-FF
425         // comment        = "(" *( ctext / quoted-pair / comment ) ")"
426         // ctext          = HTAB / SP / %x21-27 / %x2A-5B / %x5D-7E / obs-text
427         // quoted-pair    = "\" ( HTAB / SP / VCHAR / obs-text )
428 
429         __charState = new CharState[256];
430         __charState[0..$] = CharState.ILLEGAL;
431         // Arrays.fill(__charState, CharState.ILLEGAL);
432         __charState[HttpTokens.LINE_FEED] = CharState.LF;
433         __charState[HttpTokens.CARRIAGE_RETURN] = CharState.CR;
434         __charState[HttpTokens.TAB] = CharState.LEGAL;
435         __charState[HttpTokens.SPACE] = CharState.LEGAL;
436 
437         __charState['!'] = CharState.LEGAL;
438         __charState['#'] = CharState.LEGAL;
439         __charState['$'] = CharState.LEGAL;
440         __charState['%'] = CharState.LEGAL;
441         __charState['&'] = CharState.LEGAL;
442         __charState['\''] = CharState.LEGAL;
443         __charState['*'] = CharState.LEGAL;
444         __charState['+'] = CharState.LEGAL;
445         __charState['-'] = CharState.LEGAL;
446         __charState['.'] = CharState.LEGAL;
447         __charState['^'] = CharState.LEGAL;
448         __charState['_'] = CharState.LEGAL;
449         __charState['`'] = CharState.LEGAL;
450         __charState['|'] = CharState.LEGAL;
451         __charState['~'] = CharState.LEGAL;
452 
453         __charState['"'] = CharState.LEGAL;
454 
455         __charState['\\'] = CharState.LEGAL;
456         __charState['('] = CharState.LEGAL;
457         __charState[')'] = CharState.LEGAL;
458         __charState[0x21 .. 0x27 + 1] = CharState.LEGAL;
459         __charState[0x2A .. 0x5B + 1] = CharState.LEGAL;
460         __charState[0x5D .. 0x7E + 1] = CharState.LEGAL;
461         __charState[0x80 .. 0xFF + 1] = CharState.LEGAL;
462     }
463 
464     /* ------------------------------------------------------------------------------- */
465     private byte next(ByteBuffer buffer) {
466         byte ch = buffer.get();
467 
468         CharState s = __charState[0xff & ch];
469         switch (s) {
470             case CharState.ILLEGAL:
471                 throw new IllegalCharacterException(_state, ch, buffer);
472 
473             case CharState.LF:
474                 _cr = false;
475                 break;
476 
477             case CharState.CR:
478                 if (_cr)
479                     throw new BadMessageException("Bad EOL");
480 
481                 _cr = true;
482                 if (buffer.hasRemaining()) {
483                     // Don't count the CRs and LFs of the chunked encoding.
484                     if (_maxHeaderBytes > 0 && (_state == State.HEADER || _state == State.TRAILER))
485                         _headerBytes++;
486                     return next(buffer);
487                 }
488 
489                 // Can return 0 here to indicate the need for more characters,
490                 // because a real 0 in the buffer would cause a BadMessage below
491                 return 0;
492 
493             case CharState.LEGAL:
494                 if (_cr)
495                     throw new BadMessageException("Bad EOL");
496                 break;
497             
498             default:
499                 break;
500         }
501 
502         return ch;
503     }
504 
505     /* ------------------------------------------------------------------------------- */
506     /* Quick lookahead for the start state looking for a request method or a HTTP version,
507      * otherwise skip white space until something else to parse.
508      */
509     private bool quickStart(ByteBuffer buffer) {
510         if (_requestHandler !is null) {
511             _method = HttpMethod.lookAheadGet(buffer);
512             if (_method != HttpMethod.Null) {
513                 _methodString = _method.asString();
514                 buffer.position(cast(int)(buffer.position() + _methodString.length + 1));
515 
516                 setState(State.SPACE1);
517                 return false;
518             }
519         } else if (_responseHandler !is null) {
520             _version = HttpVersion.lookAheadGet(buffer);
521             if (_version != HttpVersion.Null) {
522                 buffer.position(cast(int) (buffer.position() + _version.asString().length + 1));
523                 setState(State.SPACE1);
524                 return false;
525             }
526         }
527 
528         // Quick start look
529         while (_state == State.START && buffer.hasRemaining()) {
530             int ch = next(buffer);
531 
532             if (ch > HttpTokens.SPACE) {
533                 _string.setLength(0);
534                 _string.append(cast(char) ch);
535                 setState(_requestHandler !is null ? State.METHOD : State.RESPONSE_VERSION);
536                 return false;
537             } else if (ch == 0)
538                 break;
539             else if (ch < 0)
540                 throw new BadMessageException();
541 
542             // count this white space as a header byte to avoid DOS
543             if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
544                 warningf("padding is too large >%d", _maxHeaderBytes);
545                 throw new BadMessageException(HttpStatus.BAD_REQUEST_400);
546             }
547         }
548         return false;
549     }
550 
551     /* ------------------------------------------------------------------------------- */
552     private void setString(string s) {
553         _string.setLength(0);
554         _string.append(s);
555         _length = cast(int)s.length;
556     }
557 
558     /* ------------------------------------------------------------------------------- */
559     private string takeString() {
560         _string.setLength(_length);
561         string s = _string.toString();
562         _string.setLength(0);
563         _length = -1;
564         return s;
565     }
566 
567     /* ------------------------------------------------------------------------------- */
568     private bool handleHeaderContentMessage() {
569         version (HUNT_DEBUG) trace("handling headers ...");
570         bool handle_header = _handler.headerComplete();
571         _headerComplete = true;
572         version (HUNT_DEBUG) trace("handling content ...");
573         bool handle_content = _handler.contentComplete();
574         version (HUNT_DEBUG) trace("handling message ...");
575         bool handle_message = _handler.messageComplete();
576         return handle_header || handle_content || handle_message;
577     }
578 
579     /* ------------------------------------------------------------------------------- */
580     private bool handleContentMessage() {
581         bool handle_content = _handler.contentComplete();
582         bool handle_message = _handler.messageComplete();
583         return handle_content || handle_message;
584     }
585 
586     /* ------------------------------------------------------------------------------- */
587     /* Parse a request or response line
588      */
589     private bool parseLine(ByteBuffer buffer) {
590         bool handle = false;
591 
592         // Process headers
593         while (_state < State.HEADER && buffer.hasRemaining() && !handle) {
594             // process each character
595             byte b = next(buffer);
596             if (b == 0)
597                 break;
598 
599             if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
600                 if (_state == State.URI) {
601                     warningf("URI is too large >%d", _maxHeaderBytes);
602                     throw new BadMessageException(HttpStatus.URI_TOO_LONG_414);
603                 } else {
604                     if (_requestHandler !is null)
605                         warningf("request is too large >%d", _maxHeaderBytes);
606                     else
607                         warningf("response is too large >%d", _maxHeaderBytes);
608                     throw new BadMessageException(HttpStatus.REQUEST_HEADER_FIELDS_TOO_LARGE_431);
609                 }
610             }
611 
612             switch (_state) {
613                 case State.METHOD:
614                     if (b == HttpTokens.SPACE) {
615                         _length = _string.length;
616                         _methodString = takeString();
617 
618                         if (_compliances.contains(HttpComplianceSection.METHOD_CASE_SENSITIVE)) {
619                             HttpMethod method = HttpMethod.get(_methodString);
620                             if (method != HttpMethod.Null)
621                                 _methodString = method.asString();
622                         } else {
623                             HttpMethod method = HttpMethod.getInsensitive(_methodString);
624 
625                             if (method != HttpMethod.Null) {
626                                 if (method.asString() != (_methodString))
627                                     handleViolation(HttpComplianceSection.METHOD_CASE_SENSITIVE, _methodString);
628                                 _methodString = method.asString();
629                             }
630                         }
631 
632                         setState(State.SPACE1);
633                     } else if (b < HttpTokens.SPACE) {
634                         if (b == HttpTokens.LINE_FEED)
635                             throw new BadMessageException("No URI");
636                         else
637                             throw new IllegalCharacterException(_state, b, buffer);
638                     } else
639                         _string.append(cast(char) b);
640                     break;
641 
642                 case State.RESPONSE_VERSION:
643                     if (b == HttpTokens.SPACE) {
644                         _length = _string.length;
645                         string ver = takeString();
646                         _version = HttpVersion.fromString(ver);
647                         if (_version == HttpVersion.Null)
648                             throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Unknown Version");
649                         setState(State.SPACE1);
650                     } else if (b < HttpTokens.SPACE)
651                         throw new IllegalCharacterException(_state, b, buffer);
652                     else
653                         _string.append(cast(char) b);
654                     break;
655 
656                 case State.SPACE1:
657                     if (b > HttpTokens.SPACE || b < 0) {
658                         if (_responseHandler !is null) {
659                             setState(State.STATUS);
660                             setResponseStatus(b - '0');
661                         } else {
662                             _uri.reset();
663                             setState(State.URI);
664                             // quick scan for space or EoBuffer
665                             if (buffer.hasArray()) {
666                                 byte[] array = buffer.array();
667                                 int p = buffer.arrayOffset() + buffer.position();
668                                 int l = buffer.arrayOffset() + buffer.limit();
669                                 int i = p;
670                                 while (i < l && array[i] > HttpTokens.SPACE)
671                                     i++;
672 
673                                 int len = i - p;
674                                 _headerBytes += len;
675 
676                                 if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
677                                     warningf("URI is too large >%d", _maxHeaderBytes);
678                                     throw new BadMessageException(HttpStatus.URI_TOO_LONG_414);
679                                 }
680                                 _uri.append(array, p - 1, len + 1);
681                                 buffer.position(i - buffer.arrayOffset());
682                             } else
683                                 _uri.append(b);
684                         }
685                     } else if (b < HttpTokens.SPACE) {
686                         throw new BadMessageException(HttpStatus.BAD_REQUEST_400, 
687                             _requestHandler !is null ? "No URI" : "No Status");
688                     }
689                     break;
690 
691                 case State.STATUS:
692                     if (b == HttpTokens.SPACE) {
693                         setState(State.SPACE2);
694                     } else if (b >= '0' && b <= '9') {
695                         _responseStatus = _responseStatus * 10 + (b - '0');
696                     } else if (b < HttpTokens.SPACE && b >= 0) {
697                         setState(State.HEADER);
698                         handle = _responseHandler.startResponse(_version, _responseStatus, null) || handle;
699                     } else {
700                         throw new BadMessageException();
701                     }
702                     break;
703 
704                 case State.URI:
705                     if (b == HttpTokens.SPACE) {
706                         setState(State.SPACE2);
707                     } else if (b < HttpTokens.SPACE && b >= 0) {
708                         // HTTP/0.9
709                         if (complianceViolation(HttpComplianceSection.NO_HTTP_0_9, "No request version"))
710                             throw new BadMessageException("HTTP/0.9 not supported");
711                         handle = _requestHandler.startRequest(_methodString, _uri.toString(), HttpVersion.HTTP_0_9);
712                         setState(State.END);
713                         BufferUtils.clear(buffer);
714                         handle = handleHeaderContentMessage() || handle;
715                     } else {
716                         _uri.append(b);
717                     }
718                     break;
719 
720                 case State.SPACE2:
721                     if (b > HttpTokens.SPACE) {
722                         _string.setLength(0);
723                         _string.append(cast(char) b);
724                         if (_responseHandler !is null) {
725                             _length = 1;
726                             setState(State.REASON);
727                         } else {
728                             setState(State.REQUEST_VERSION);
729 
730                             // try quick look ahead for HTTP Version
731                             HttpVersion ver;
732                             if (buffer.position() > 0 && buffer.hasArray()) {
733                                 ver = HttpVersion.lookAheadGet(buffer.array(), 
734                                     buffer.arrayOffset() + buffer.position() - 1, 
735                                     buffer.arrayOffset() + buffer.limit());
736                             } else {
737                                 string key = buffer.getString(0, buffer.remaining());
738                                 ver = HttpVersion.fromString(key);
739                             }
740 
741                             if (ver != HttpVersion.Null) {
742                                 int pos = cast(int)(buffer.position() + ver.asString().length - 1);
743                                 if (pos < buffer.limit()) {
744                                     byte n = buffer.get(pos);
745                                     if (n == HttpTokens.CARRIAGE_RETURN) {
746                                         _cr = true;
747                                         _version = ver;
748                                         _string.setLength(0);
749                                         buffer.position(pos + 1);
750                                     } else if (n == HttpTokens.LINE_FEED) {
751                                         _version = ver;
752                                         _string.setLength(0);
753                                         buffer.position(pos);
754                                     }
755                                 }
756                             }
757                         }
758                     } else if (b == HttpTokens.LINE_FEED) {
759                         if (_responseHandler !is null) {
760                             setState(State.HEADER);
761                             handle = _responseHandler.startResponse(_version, _responseStatus, null) || handle;
762                         } else {
763                             // HTTP/0.9
764                             if (complianceViolation(HttpComplianceSection.NO_HTTP_0_9, "No request version"))
765                                 throw new BadMessageException("HTTP/0.9 not supported");
766 
767                             handle = _requestHandler.startRequest(_methodString, _uri.toString(), HttpVersion.HTTP_0_9);
768                             setState(State.END);
769                             BufferUtils.clear(buffer);
770                             handle = handleHeaderContentMessage() || handle;
771                         }
772                     } else if (b < 0)
773                         throw new BadMessageException();
774                     break;
775 
776                 case State.REQUEST_VERSION:
777                     if (b == HttpTokens.LINE_FEED) {
778                         if (_version == HttpVersion.Null) {
779                             _length = _string.length;
780                             _version = HttpVersion.fromString(takeString());
781                         }
782                         if (_version == HttpVersion.Null)
783                             throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Unknown Version");
784 
785                         // Should we try to cache header fields?
786                         if (_fieldCache is null && 
787                             _version.getVersion() >= HttpVersion.HTTP_1_1.getVersion() && 
788                             _handler.getHeaderCacheSize() > 0) {
789                             int header_cache = _handler.getHeaderCacheSize();
790                             _fieldCache = new ArrayTernaryTrie!HttpField(header_cache);
791                         }
792 
793                         setState(State.HEADER);
794 
795                         handle = _requestHandler.startRequest(_methodString, _uri.toString(), _version) || handle;
796                         continue;
797                     } else if (b >= HttpTokens.SPACE)
798                         _string.append(cast(char) b);
799                     else
800                         throw new BadMessageException();
801 
802                     break;
803 
804                 case State.REASON:
805                     if (b == HttpTokens.LINE_FEED) {
806                         string reason = takeString();
807                         setState(State.HEADER);
808                         handle = _responseHandler.startResponse(_version, _responseStatus, reason) || handle;
809                         continue;
810                     } else if (b >= HttpTokens.SPACE || ((b < 0) && (b >= -96))) {
811                         _string.append(cast(char) (0xff & b));
812                         if (b != ' ' && b != '\t')
813                             _length = _string.length;
814                     } else
815                         throw new BadMessageException();
816                     break;
817 
818                 default:
819                     throw new IllegalStateException(_state.to!string());
820             }
821         }
822 
823         return handle;
824     }
825 
826     private void parsedHeader() {
827         // handler last header if any.  Delayed to here just in case there was a continuation line (above)
828         if (!_headerString.empty() || !_valueString.empty()) {
829             // Handle known headers
830             version(HUNT_DEBUG) {
831                 tracef("parsing header: %s, original name: %s, value: %s ", 
832                     _header.toString(), _headerString, _valueString);
833             }
834 
835             if (_header != HttpHeader.Null) {
836                 bool canAddToConnectionTrie = false;
837                 if(_header == HttpHeader.CONTENT_LENGTH) {
838                     if (_endOfContent == EndOfContent.CONTENT_LENGTH) {
839                         throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Duplicate Content-Length");
840                     } else if (_endOfContent != EndOfContent.CHUNKED_CONTENT) {
841                         _contentLength = convertContentLength(_valueString);
842                         if (_contentLength <= 0)
843                             _endOfContent = EndOfContent.NO_CONTENT;
844                         else
845                             _endOfContent = EndOfContent.CONTENT_LENGTH;
846                     }
847                 }
848                 else if(_header == HttpHeader.TRANSFER_ENCODING){
849                     if (HttpHeaderValue.CHUNKED.isSame(_valueString)) {
850                         _endOfContent = EndOfContent.CHUNKED_CONTENT;
851                         _contentLength = -1;
852                     } else {
853                         string[] values = new QuotedCSV(_valueString).getValues();
854                         if (values.length > 0 && HttpHeaderValue.CHUNKED.isSame(values[$ - 1])) {
855                             _endOfContent = EndOfContent.CHUNKED_CONTENT;
856                             _contentLength = -1;
857                         } else {
858                             foreach(string v; values) {
859                                 if(HttpHeaderValue.CHUNKED.isSame(v)) {
860                                     throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Bad chunking");
861                                 }
862                             }
863                         } 
864                     }
865                 }                    
866                 else if(_header == HttpHeader.HOST) {
867                     _host = true;
868                     if ((_field is null) && !_valueString.empty()) {
869                         string headerStr = _compliances.contains(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE) ? 
870                                     _header.asString() : _headerString;
871                         _field = new HostPortHttpField(_header, headerStr, _valueString);
872                         canAddToConnectionTrie = _fieldCache !is null;
873                     }
874                 }
875                 else if(_header == HttpHeader.CONNECTION) {
876                     // Don't cache headers if not persistent
877                     if (HttpHeaderValue.CLOSE.isSame(_valueString)) 
878                         _fieldCache = null;
879                     else {
880                         string[] values = new QuotedCSV(_valueString).getValues();
881                         foreach(string v; values) {
882                             if(HttpHeaderValue.CLOSE.isSame(v)) {
883                                 _fieldCache = null;
884                                 break;
885                             }
886                         }
887                     }
888                 }
889                 else if(_header == HttpHeader.AUTHORIZATION || _header == HttpHeader.ACCEPT ||
890                 _header == HttpHeader.ACCEPT_CHARSET || _header ==  HttpHeader.ACCEPT_ENCODING ||
891                 _header == HttpHeader.ACCEPT_LANGUAGE || _header == HttpHeader.COOKIE || 
892                 _header == HttpHeader.CACHE_CONTROL || _header == HttpHeader.USER_AGENT) {
893                     canAddToConnectionTrie = _fieldCache !is null && _field is null;
894                 }
895 
896                 if (canAddToConnectionTrie && !_fieldCache.isFull() 
897                     && _header != HttpHeader.Null && !_valueString.empty()) {
898                     if (_field is null)
899                         _field = new HttpField(_header, caseInsensitiveHeader(_headerString, _header.asString()), _valueString);
900                     _fieldCache.put(_field);
901                 }
902             }
903             _handler.parsedHeader(_field !is null ? _field : new HttpField(_header, _headerString, _valueString));
904         }
905 
906         _headerString = _valueString = null;
907         _header = HttpHeader.Null;
908         _field = null;
909     }
910 
911     private void parsedTrailer() {
912         // handler last header if any.  Delayed to here just in case there was a continuation line (above)
913         if (!_headerString.empty() || !_valueString.empty()) 
914             _handler.parsedTrailer(_field !is null ? _field : new HttpField(_header, _headerString, _valueString));
915 
916         _headerString = _valueString = null;
917         _header = HttpHeader.Null;
918         _field = null;
919     }
920 
921     private long convertContentLength(string valueString) {
922         try {
923             return to!long(valueString);
924         } catch (Exception e) {
925             warning("parse long exception: ", e);
926             throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Invalid Content-Length Value");
927         }
928     }
929 
930     /* ------------------------------------------------------------------------------- */
931     /*
932      * Parse the message headers and return true if the handler has signaled for a return
933      */
934     protected bool parseFields(ByteBuffer buffer) {
935         // Process headers
936         while ((_state == State.HEADER || _state == State.TRAILER) && buffer.hasRemaining()) {
937             // process each character
938             byte b = next(buffer);
939             if (b == 0)
940                 break;
941 
942             if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
943                 bool header = _state == State.HEADER;
944                 warningf("%s is too large %s>%s", header ? "Header" : "Trailer", _headerBytes, _maxHeaderBytes);
945                 throw new BadMessageException(header ?
946                         HttpStatus.REQUEST_HEADER_FIELDS_TOO_LARGE_431 :
947                         HttpStatus.PAYLOAD_TOO_LARGE_413);
948             }
949 
950             switch (_fieldState) {
951                 case FieldState.FIELD:
952                     switch (b) {
953                         case HttpTokens.COLON:
954                         case HttpTokens.SPACE:
955                         case HttpTokens.TAB: {
956                             if (complianceViolation(HttpComplianceSection.NO_FIELD_FOLDING, _headerString))
957                                 throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Header Folding");
958 
959                             // header value without name - continuation?
960                             if (_valueString == null || _valueString.empty()) {
961                                 _string.setLength(0);
962                                 _length = 0;
963                             } else {
964                                 setString(_valueString);
965                                 _string.append(' ');
966                                 _length++;
967                                 _valueString = null;
968                             }
969                             setState(FieldState.VALUE);
970                             break;
971                         }
972 
973                         case HttpTokens.LINE_FEED: {
974                             // process previous header
975                             if (_state == State.HEADER)
976                                 parsedHeader();
977                             else
978                                 parsedTrailer();
979 
980                             _contentPosition = 0;
981 
982                             // End of headers or trailers?
983                             if (_state == State.TRAILER) {
984                                 setState(State.END);
985                                 return _handler.messageComplete();
986                             }
987 
988                             // Was there a required host header?
989                             if (!_host && _version == HttpVersion.HTTP_1_1 && _requestHandler !is null) {
990                                 throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "No Host");
991                             }
992 
993                             // is it a response that cannot have a body?
994                             if (_responseHandler !is null && // response
995                                     (_responseStatus == 304 || // not-modified response
996                                             _responseStatus == 204 || // no-content response
997                                             _responseStatus < 200)) // 1xx response
998                                 _endOfContent = EndOfContent.NO_CONTENT; // ignore any other headers set
999 
1000                                 // else if we don't know framing
1001                             else if (_endOfContent == EndOfContent.UNKNOWN_CONTENT) {
1002                                 if (_responseStatus == 0  // request
1003                                         || _responseStatus == 304 // not-modified response
1004                                         || _responseStatus == 204 // no-content response
1005                                         || _responseStatus < 200) // 1xx response
1006                                     _endOfContent = EndOfContent.NO_CONTENT;
1007                                 else
1008                                     _endOfContent = EndOfContent.EOF_CONTENT;
1009                             }
1010 
1011                             // How is the message ended?
1012                             switch (_endOfContent) {
1013                                 case EndOfContent.EOF_CONTENT: {
1014                                     setState(State.EOF_CONTENT);
1015                                     bool handle = _handler.headerComplete();
1016                                     _headerComplete = true;
1017                                     return handle;
1018                                 }
1019                                 case EndOfContent.CHUNKED_CONTENT: {
1020                                     setState(State.CHUNKED_CONTENT);
1021                                     bool handle = _handler.headerComplete();
1022                                     _headerComplete = true;
1023                                     return handle;
1024                                 }
1025                                 case EndOfContent.NO_CONTENT: {
1026                                     version (HUNT_DEBUG) trace("parsing done for no content");
1027                                     setState(State.END);
1028                                     return handleHeaderContentMessage();
1029                                 }
1030                                 default: {
1031                                     setState(State.CONTENT);
1032                                     bool handle = _handler.headerComplete();
1033                                     _headerComplete = true;
1034                                     return handle;
1035                                 }
1036                             }
1037                         }
1038 
1039                         default: {
1040                             // process previous header
1041                             if (_state == State.HEADER)
1042                                 parsedHeader();
1043                             else
1044                                 parsedTrailer();
1045 
1046                             // handle new header
1047                             if (buffer.hasRemaining()) {
1048                                 // Try a look ahead for the known header name and value.
1049                                 HttpField cached_field = null;
1050                                 if(_fieldCache !is null)
1051                                     cached_field = _fieldCache.getBest(buffer, -1, buffer.remaining());
1052                                 // TODO: Tasks pending completion -@zxp at 10/23/2018, 8:03:47 PM
1053                                 // Can't handle Sec-WebSocket-Key
1054                                 // if (cached_field is null)
1055                                 //     cached_field = CACHE.getBest(buffer, -1, buffer.remaining());
1056 
1057                                 if (cached_field !is null) {
1058                                     string n = cached_field.getName();
1059                                     string v = cached_field.getValue();
1060 
1061                                     if (!_compliances.contains(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE)) {
1062                                         // Have to get the fields exactly from the buffer to match case
1063                                          // BufferUtils.toString(buffer, buffer.position() - 1, n.length, StandardCharsets.US_ASCII);
1064                                         string en = buffer.getString(buffer.position() - 1, cast(int)n.length);
1065                                         if (!n.equals(en)) {
1066                                             handleViolation(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE, en);
1067                                             n = en;
1068                                             cached_field = new HttpField(cached_field.getHeader(), n, v);
1069                                         }
1070                                     }
1071 
1072                                     if (v != null && !_compliances.contains(HttpComplianceSection.CASE_INSENSITIVE_FIELD_VALUE_CACHE)) {
1073                                         // BufferUtils.toString(buffer, buffer.position() + n.length + 1, v.length, StandardCharsets.ISO_8859_1);
1074                                         string ev = buffer.getString(buffer.position() + n.length + 1, v.length);
1075                                         if (!v.equals(ev)) {
1076                                             handleViolation(HttpComplianceSection.CASE_INSENSITIVE_FIELD_VALUE_CACHE, ev ~ "!=" ~ v);
1077                                             v = ev;
1078                                             cached_field = new HttpField(cached_field.getHeader(), n, v);
1079                                         }
1080                                     }
1081 
1082                                     _header = cached_field.getHeader();
1083                                     _headerString = n;
1084 
1085                                     if (v == null) {
1086                                         // Header only
1087                                         setState(FieldState.VALUE);
1088                                         _string.setLength(0);
1089                                         _length = 0;
1090                                         buffer.position(cast(int)(buffer.position() + n.length + 1));
1091                                         break;
1092                                     } else {
1093                                         // Header and value
1094                                         int pos = cast(int) (buffer.position() + n.length + v.length + 1);
1095                                         byte peek = buffer.get(pos);
1096 
1097                                         if (peek == HttpTokens.CARRIAGE_RETURN || peek == HttpTokens.LINE_FEED) {
1098                                             _field = cached_field;
1099                                             _valueString = v;
1100                                             setState(FieldState.IN_VALUE);
1101 
1102                                             if (peek == HttpTokens.CARRIAGE_RETURN) {
1103                                                 _cr = true;
1104                                                 buffer.position(pos + 1);
1105                                             } else
1106                                                 buffer.position(pos);
1107                                             break;
1108                                         } else {
1109                                             setState(FieldState.IN_VALUE);
1110                                             setString(v);
1111                                             buffer.position(pos);
1112                                             break;
1113                                         }
1114                                     }
1115                                 }
1116                             }
1117 
1118                             // New header
1119                             setState(FieldState.IN_NAME);
1120                             _string.setLength(0);
1121                             _string.append(cast(char) b);
1122                             _length = 1;
1123                         }
1124                     }
1125                     break;
1126 
1127                 case FieldState.IN_NAME:
1128                     switch (b) {
1129                         case HttpTokens.SPACE:
1130                         case HttpTokens.TAB:
1131                             //Ignore trailing whitespaces ?
1132                             if (!complianceViolation(HttpComplianceSection.NO_WS_AFTER_FIELD_NAME, null)) {
1133                                 _headerString = takeString();
1134                                 _header = HttpHeader.get(_headerString);
1135                                 _length = -1;
1136                                 setState(FieldState.WS_AFTER_NAME);
1137                                 break;
1138                             }
1139                             throw new IllegalCharacterException(_state, b, buffer);
1140 
1141                         case HttpTokens.COLON:
1142                             _headerString = takeString();
1143                             _header = HttpHeader.get(_headerString);
1144                             _length = -1;
1145                             setState(FieldState.VALUE);
1146                             break;
1147 
1148                         case HttpTokens.LINE_FEED:
1149                             _headerString = takeString();
1150                             _header = HttpHeader.get(_headerString);
1151                             _string.setLength(0);
1152                             _valueString = "";
1153                             _length = -1;
1154 
1155                             if (!complianceViolation(HttpComplianceSection.FIELD_COLON, _headerString)) {
1156                                 setState(FieldState.FIELD);
1157                                 break;
1158                             }
1159                             throw new IllegalCharacterException(_state, b, buffer);
1160 
1161                         default:
1162                             if (b < 0)
1163                                 throw new IllegalCharacterException(_state, b, buffer);
1164 
1165                             _string.append(cast(char) b);
1166                             _length = _string.length;
1167                             break;
1168                     }
1169                     break;
1170 
1171                 case FieldState.WS_AFTER_NAME:
1172 
1173                     switch (b) {
1174                         case HttpTokens.SPACE:
1175                         case HttpTokens.TAB:
1176                             break;
1177 
1178                         case HttpTokens.COLON:
1179                             setState(FieldState.VALUE);
1180                             break;
1181 
1182                         case HttpTokens.LINE_FEED:
1183                             if (!complianceViolation(HttpComplianceSection.FIELD_COLON, _headerString)) {
1184                                 setState(FieldState.FIELD);
1185                                 break;
1186                             }
1187                             throw new IllegalCharacterException(_state, b, buffer);
1188 
1189                         default:
1190                             throw new IllegalCharacterException(_state, b, buffer);
1191                     }
1192                     break;
1193 
1194                 case FieldState.VALUE:
1195                     switch (b) {
1196                         case HttpTokens.LINE_FEED:
1197                             _string.setLength(0);
1198                             _valueString = "";
1199                             _length = -1;
1200 
1201                             setState(FieldState.FIELD);
1202                             break;
1203 
1204                         case HttpTokens.SPACE:
1205                         case HttpTokens.TAB:
1206                             break;
1207 
1208                         default:
1209                             _string.append(cast(char) (0xff & b));
1210                             _length = _string.length;
1211                             setState(FieldState.IN_VALUE);
1212                             break;
1213                     }
1214                     break;
1215 
1216                 case FieldState.IN_VALUE:
1217                     switch (b) {
1218                         case HttpTokens.LINE_FEED:
1219                             if (_length > 0) {
1220                                 _valueString = takeString();
1221                                 _length = -1;
1222                             }
1223                             setState(FieldState.FIELD);
1224                             break;
1225 
1226                         case HttpTokens.SPACE:
1227                         case HttpTokens.TAB:
1228                             _string.append(cast(char) (0xff & b));
1229                             break;
1230 
1231                         default:
1232                             _string.append(cast(char) (0xff & b));
1233                             _length = _string.length;
1234                             break;
1235                     }
1236                     break;
1237 
1238                 default:
1239                     throw new IllegalStateException(_state.to!string());
1240 
1241             }
1242         }
1243 
1244         return false;
1245     }
1246 
1247     /* ------------------------------------------------------------------------------- */
1248 
1249     /**
1250      * Parse until next Event.
1251      *
1252      * @param buffer the buffer to parse
1253      * @return True if an {@link RequestHandler} method was called and it returned true;
1254      */
1255     bool parseNext(ByteBuffer buffer) {
1256         version(HUNT_DEBUG) {
1257             tracef("parseNext s=%s %s", _state, BufferUtils.toDetailString(buffer));
1258             // tracef("buffer: %s", BufferUtils.toHexString(buffer));
1259 
1260             // startTime = MonoTime.currTime;
1261             // _requestHandler.startRequest("GET", "/plaintext", HttpVersion.HTTP_1_1);
1262             // setState(State.END);
1263             // _handler.messageComplete();
1264             // BufferUtils.clear(buffer);
1265             // return false;
1266         }
1267         try {
1268             // Start a request/response
1269             if (_state == State.START) {
1270                 version(HUNT_METRIC) {
1271                     startTime = MonoTime.currTime;
1272                     info("start a new parsing process...");
1273                 }                
1274                 _version = HttpVersion.Null;
1275                 _method = HttpMethod.Null;
1276                 _methodString = null;
1277                 _endOfContent = EndOfContent.UNKNOWN_CONTENT;
1278                 _header = HttpHeader.Null;
1279                 if (quickStart(buffer))
1280                     return true;
1281             }
1282 
1283             // Request/response line
1284             if (_state >= State.START && _state < State.HEADER) {
1285                 if (parseLine(buffer)) {
1286                     tracef("after parseLine =>%s", buffer.toString());
1287                     // return true;
1288                 }
1289             }
1290 
1291             // parse headers
1292             if (_state == State.HEADER) {
1293                 if (parseFields(buffer)) {
1294                     version(HUNT_DEBUG) tracef("after parseFields =>%s", buffer.toString());
1295                     return true;
1296                 }
1297             }
1298 
1299             // parse content
1300             if (_state >= State.CONTENT && _state < State.TRAILER) {
1301                 // Handle HEAD response
1302                 if (_responseStatus > 0 && _headResponse) {
1303                     setState(State.END);
1304                     return handleContentMessage();
1305                 } else {
1306                     if (parseContent(buffer))
1307                         return true;
1308                 }
1309             }
1310 
1311             // parse headers
1312             if (_state == State.TRAILER) {
1313                 if (parseFields(buffer))
1314                     return true;
1315             }
1316 
1317             // handle end states
1318             if (_state == State.END) {
1319                 // eat white space
1320                 while (buffer.remaining() > 0 && buffer.get(buffer.position()) <= HttpTokens.SPACE)
1321                     buffer.get();
1322             } else if (isClose() || isClosed()) {
1323                 BufferUtils.clear(buffer);
1324             }
1325 
1326             // Handle EOF
1327             if (_eof && !buffer.hasRemaining()) {
1328                 switch (_state) {
1329                     case State.CLOSED:
1330                         break;
1331 
1332                     case State.START:
1333                         setState(State.CLOSED);
1334                         _handler.earlyEOF();
1335                         break;
1336 
1337                     case State.END:
1338                     case State.CLOSE:
1339                         setState(State.CLOSED);
1340                         break;
1341 
1342                     case State.EOF_CONTENT:
1343                     case State.TRAILER:
1344                         if (_fieldState == FieldState.FIELD) {
1345                             // Be forgiving of missing last CRLF
1346                             setState(State.CLOSED);
1347                             return handleContentMessage();
1348                         }
1349                         setState(State.CLOSED);
1350                         _handler.earlyEOF();
1351                         break;
1352 
1353                     case State.CONTENT:
1354                     case State.CHUNKED_CONTENT:
1355                     case State.CHUNK_SIZE:
1356                     case State.CHUNK_PARAMS:
1357                     case State.CHUNK:
1358                         setState(State.CLOSED);
1359                         _handler.earlyEOF();
1360                         break;
1361 
1362                     default:
1363                         version(HUNT_DEBUG)
1364                             tracef("%s EOF in %s", this, _state);
1365                         setState(State.CLOSED);
1366                         _handler.badMessage(new BadMessageException(HttpStatus.BAD_REQUEST_400));
1367                         break;
1368                 }
1369             }
1370         } catch (BadMessageException x) {
1371             BufferUtils.clear(buffer);
1372             badMessage(x);
1373         } catch (Exception x) {
1374             BufferUtils.clear(buffer);
1375             badMessage(new BadMessageException(HttpStatus.BAD_REQUEST_400, 
1376                 _requestHandler !is null ? "Bad Request" : "Bad Response", x));
1377         }
1378         return false;
1379     }
1380 
1381     protected void badMessage(BadMessageException x) {
1382         // version(HUNT_DEBUG)
1383             warning("Parse exception: " ~ this.toString() ~ " for " ~ _handler.toString(), x.toString());
1384         version(HUNT_DEBUG) {
1385             Throwable t = x;
1386             while((t = t.next) !is null) {
1387                 error(t.msg);
1388             }
1389         }
1390         setState(State.CLOSE);
1391         if (_headerComplete)
1392             _handler.earlyEOF();
1393         else
1394             _handler.badMessage(x);
1395     }
1396 
1397     protected bool parseContent(ByteBuffer buffer) {
1398         int remaining = buffer.remaining();
1399         if (remaining == 0 && _state == State.CONTENT) {
1400             long content = _contentLength - _contentPosition;
1401             if (content == 0) {
1402                 setState(State.END);
1403                 return handleContentMessage();
1404             }
1405         }
1406 
1407         // Handle _content
1408         byte ch;
1409         while (_state < State.TRAILER && remaining > 0) {
1410             switch (_state) {
1411                 case State.EOF_CONTENT:
1412                     _contentChunk = buffer.slice(); // buffer.asReadOnlyBuffer();
1413                     _contentPosition += remaining;
1414                     buffer.position(buffer.position() + remaining);
1415                     if (_handler.content(_contentChunk))
1416                         return true;
1417                     break;
1418 
1419                 case State.CONTENT: {
1420                     long content = _contentLength - _contentPosition;
1421                     if (content == 0) {
1422                         setState(State.END);
1423                         return handleContentMessage();
1424                     } else {
1425                         _contentChunk = buffer.slice(); // buffer.asReadOnlyBuffer();
1426 
1427                         // limit content by expected size
1428                         if (remaining > content) {
1429                             // We can cast remaining to an int as we know that it is smaller than
1430                             // or equal to length which is already an int.
1431                             _contentChunk.limit(_contentChunk.position() + cast(int) content);
1432                         }
1433 
1434                         _contentPosition += _contentChunk.remaining();
1435                         buffer.position(buffer.position() + _contentChunk.remaining());
1436 
1437                         if (_handler.content(_contentChunk))
1438                             return true;
1439 
1440                         if (_contentPosition == _contentLength) {
1441                             setState(State.END);
1442                             return handleContentMessage();
1443                         }
1444                     }
1445                     break;
1446                 }
1447 
1448                 case State.CHUNKED_CONTENT: {
1449                     ch = next(buffer);
1450                     if (ch > HttpTokens.SPACE) {
1451                         _chunkLength = TypeUtils.convertHexDigit(ch);
1452                         _chunkPosition = 0;
1453                         setState(State.CHUNK_SIZE);
1454                     }
1455 
1456                     break;
1457                 }
1458 
1459                 case State.CHUNK_SIZE: {
1460                     ch = next(buffer);
1461                     if (ch == 0)
1462                         break;
1463                     if (ch == HttpTokens.LINE_FEED) {
1464                         if (_chunkLength == 0) {
1465                             setState(State.TRAILER);
1466                             if (_handler.contentComplete())
1467                                 return true;
1468                         } else
1469                             setState(State.CHUNK);
1470                     } else if (ch <= HttpTokens.SPACE || ch == HttpTokens.SEMI_COLON)
1471                         setState(State.CHUNK_PARAMS);
1472                     else
1473                         _chunkLength = _chunkLength * 16 + TypeUtils.convertHexDigit(ch);
1474                     break;
1475                 }
1476 
1477                 case State.CHUNK_PARAMS: {
1478                     ch = next(buffer);
1479                     if (ch == HttpTokens.LINE_FEED) {
1480                         if (_chunkLength == 0) {
1481                             setState(State.TRAILER);
1482                             if (_handler.contentComplete())
1483                                 return true;
1484                         } else
1485                             setState(State.CHUNK);
1486                     }
1487                     break;
1488                 }
1489 
1490                 case State.CHUNK: {
1491                     int chunk = _chunkLength - _chunkPosition;
1492                     if (chunk == 0) {
1493                         setState(State.CHUNKED_CONTENT);
1494                     } else {
1495                         _contentChunk = buffer.slice(); // buffer.asReadOnlyBuffer();
1496 
1497                         if (remaining > chunk)
1498                             _contentChunk.limit(_contentChunk.position() + chunk);
1499                         chunk = _contentChunk.remaining();
1500 
1501                         _contentPosition += chunk;
1502                         _chunkPosition += chunk;
1503                         buffer.position(buffer.position() + chunk);
1504                         if (_handler.content(_contentChunk))
1505                             return true;
1506                     }
1507                     break;
1508                 }
1509 
1510                 case State.CLOSED: {
1511                     // BufferUtils.clear(buffer);
1512                     buffer.clear();
1513                     return false;
1514                 }
1515 
1516                 default:
1517                     break;
1518 
1519             }
1520 
1521             remaining = buffer.remaining();
1522         }
1523         return false;
1524     }
1525 
1526     /* ------------------------------------------------------------------------------- */
1527     bool isAtEOF()
1528     {
1529         return _eof;
1530     }
1531 
1532     /* ------------------------------------------------------------------------------- */
1533 
1534     /**
1535      * Signal that the associated data source is at EOF
1536      */
1537     void atEOF() {
1538         version(HUNT_DEBUG)
1539             tracef("atEOF %s", this);
1540         _eof = true;
1541     }
1542 
1543     /* ------------------------------------------------------------------------------- */
1544 
1545     /**
1546      * Request that the associated data source be closed
1547      */
1548     void close() {
1549         version(HUNT_DEBUG)
1550             tracef("close %s", this);
1551         setState(State.CLOSE);
1552     }
1553 
1554     /* ------------------------------------------------------------------------------- */
1555     void reset() {
1556         version(HUNT_DEBUG)
1557             tracef("reset %s", this);
1558 
1559         // reset state
1560         if (_state == State.CLOSE || _state == State.CLOSED)
1561             return;
1562 
1563         setState(State.START);
1564         _endOfContent = EndOfContent.UNKNOWN_CONTENT;
1565         _contentLength = -1;
1566         _contentPosition = 0;
1567         _responseStatus = 0;
1568         _contentChunk = null;
1569         _headerBytes = 0;
1570         _host = false;
1571         _headerComplete = false;
1572     }
1573 
1574     /* ------------------------------------------------------------------------------- */
1575     protected void setState(State state) {
1576         version(HUNT_DEBUG)
1577             tracef("%s --> %s", _state, state);
1578         _state = state;
1579     
1580         version(HUNT_METRIC) {
1581             if(state == State.END) {
1582                 Duration timeElapsed = MonoTime.currTime - startTime;
1583                 warningf("parsing ended in: %d microseconds", timeElapsed.total!(TimeUnit.Microsecond)());
1584             }
1585         }
1586     }
1587 
1588     /* ------------------------------------------------------------------------------- */
1589     protected void setState(FieldState state) {
1590         // version(HUNT_DEBUG)
1591         //     tracef("%s:%s --> %s", _state, _field, state);
1592         _fieldState = state;
1593     }
1594 
1595     /* ------------------------------------------------------------------------------- */
1596     Trie!HttpField getFieldCache() {
1597         return _fieldCache;
1598     }
1599 
1600     HttpField getCachedField(string name) {
1601         return _fieldCache.get(name);
1602     }
1603     
1604 
1605     /* ------------------------------------------------------------------------------- */
1606     override
1607     string toString() {
1608         return format("%s{s=%s,%d of %d}",
1609                 typeof(this).stringof,
1610                 _state,
1611                 _contentPosition,
1612                 _contentLength);
1613     }
1614 
1615     /* ------------------------------------------------------------ */
1616     /* ------------------------------------------------------------ */
1617     /* ------------------------------------------------------------ */
1618     /* Event Handler interface
1619      * These methods return true if the caller should process the events
1620      * so far received (eg return from parseNext and call HttpChannel.handle).
1621      * If multiple callbacks are called in sequence (eg
1622      * headerComplete then messageComplete) from the same point in the parsing
1623      * then it is sufficient for the caller to process the events only once.
1624      */
1625     interface HttpHandler {
1626         bool content(ByteBuffer item);
1627 
1628         bool headerComplete();
1629 
1630         bool contentComplete();
1631 
1632         bool messageComplete();
1633 
1634         /**
1635          * This is the method called by parser when a HTTP Header name and value is found
1636          *
1637          * @param field The field parsed
1638          */
1639         void parsedHeader(HttpField field);
1640 
1641         /**
1642          * This is the method called by parser when a HTTP Trailer name and value is found
1643          *
1644          * @param field The field parsed
1645          */
1646         void parsedTrailer(HttpField field);
1647 
1648         /* ------------------------------------------------------------ */
1649 
1650         /**
1651          * Called to signal that an EOF was received unexpectedly
1652          * during the parsing of a HTTP message
1653          */
1654         void earlyEOF();
1655 
1656         /* ------------------------------------------------------------ */
1657 
1658         /**
1659          * Called to signal that a bad HTTP message has been received.
1660          *
1661          * @param failure the failure with the bad message information
1662          */
1663         void badMessage(BadMessageException failure);
1664         // {
1665         //     // badMessage(failure.getCode(), failure.getReason());
1666         //     throw new BadMessageException(failure.getCode(), failure.getReason());
1667         // }
1668 
1669         /**
1670          * @deprecated use {@link #badMessage(BadMessageException)} instead
1671          */
1672         // deprecated("")
1673         void badMessage(int status, string reason);
1674 
1675         /* ------------------------------------------------------------ */
1676 
1677         /**
1678          * @return the size in bytes of the per parser header cache
1679          */
1680         int getHeaderCacheSize();
1681 
1682         string toString();
1683     }
1684 
1685     /* ------------------------------------------------------------------------------- */
1686     /* ------------------------------------------------------------------------------- */
1687     /* ------------------------------------------------------------------------------- */
1688     interface RequestHandler : HttpHandler {
1689         /**
1690          * This is the method called by parser when the HTTP request line is parsed
1691          *
1692          * @param method  The method
1693          * @param uri     The raw bytes of the URI.  These are copied into a ByteBuffer that will not be changed until this parser is reset and reused.
1694          * @param version the http version in use
1695          * @return true if handling parsing should return.
1696          */
1697         bool startRequest(string method, string uri, HttpVersion ver);
1698 
1699     }
1700 
1701     /* ------------------------------------------------------------------------------- */
1702     /* ------------------------------------------------------------------------------- */
1703     /* ------------------------------------------------------------------------------- */
1704     interface ResponseHandler :HttpHandler {
1705         /**
1706          * This is the method called by parser when the HTTP request line is parsed
1707          *
1708          * @param version the http version in use
1709          * @param status  the response status
1710          * @param reason  the response reason phrase
1711          * @return true if handling parsing should return
1712          */
1713         bool startResponse(HttpVersion ver, int status, string reason);
1714     }
1715 
1716     /* ------------------------------------------------------------------------------- */
1717     /* ------------------------------------------------------------------------------- */
1718     /* ------------------------------------------------------------------------------- */
1719     interface ComplianceHandler : HttpHandler {
1720         // deprecated("")
1721         // default void onComplianceViolation(HttpCompliance compliance, HttpCompliance required, string reason) {
1722         // }
1723 
1724         void onComplianceViolation(HttpCompliance compliance, HttpComplianceSection v, string details);
1725         // {
1726         //     // onComplianceViolation(compliance, HttpCompliance.requiredCompliance(violation), details);
1727         //     warning(details);
1728         // }
1729     }
1730 
1731     /* ------------------------------------------------------------------------------- */
1732     
1733     private static class IllegalCharacterException : BadMessageException {
1734         private this(State state, byte ch, ByteBuffer buffer) {
1735             super(400, format("Illegal character 0x%X", ch));
1736             // Bug #460642 - don't reveal buffers to end user
1737             warningf("Illegal character 0x%X in state=%s for buffer %s", ch, state, BufferUtils.toDetailString(buffer));
1738         }
1739     }
1740 }