hunt.http.codec.http.decode.HttpParser source code

1 module hunt.http.codec.http.decode.HttpParser;
2 
3 import hunt.http.codec.http.model;
4 import hunt.http.codec.http.hpack.HpackEncoder;
5 
6 import hunt.http.HttpField;
7 import hunt.http.HttpHeader;
8 import hunt.http.HttpMethod;
9 import hunt.http.HttpRequest;
10 import hunt.http.HttpResponse;
11 import hunt.http.HttpStatus;
12 import hunt.http.HttpVersion;
13 import hunt.http.QuotedCSV;
14 
15 import hunt.collection;
16 import hunt.Exceptions;
17 import hunt.logging;
18 import hunt.text.Common;
19 import hunt.util.StringBuilder;
20 import hunt.util.ConverterUtils;
21 import hunt.util.DateTime;
22 
23 import core.time;
24 
25 import std.algorithm;
26 import std.array;
27 import std.concurrency : initOnce;
28 import std.container.array;
29 import std.conv;
30 import std.string;
31 
32 
33 private bool contains(T)(T[] items, T item) {
34     return items.canFind(item);
35 }
36 
37 
38 /* ------------------------------------------------------------ */
39 
40 /**
41  * A Parser for 1.0 and 1.1 as defined by RFC7230
42  * <p>
43  * This parser parses HTTP client and server messages from buffers
44  * passed in the {@link #parseNext(ByteBuffer)} method.  The parsed
45  * elements of the HTTP message are passed as event calls to the
46  * {@link HttpHandler} instance the parser is constructed with.
47  * If the passed handler is a {@link RequestHandler} then server side
48  * parsing is performed and if it is a {@link ResponseHandler}, then
49  * client side parsing is done.
50  * </p>
51  * <p>
52  * The contract of the {@link HttpHandler} API is that if a call returns
53  * true then the call to {@link #parseNext(ByteBuffer)} will return as
54  * soon as possible also with a true response.  Typically this indicates
55  * that the parsing has reached a stage where the caller should process
56  * the events accumulated by the handler.    It is the preferred calling
57  * style that handling such as calling a servlet to process a request,
58  * should be done after a true return from {@link #parseNext(ByteBuffer)}
59  * rather than from within the scope of a call like
60  * {@link RequestHandler#messageComplete()}
61  * </p>
62  * <p>
63  * For performance, the parse is heavily dependent on the
64  * {@link Trie#getBest(ByteBuffer, int, int)} method to look ahead in a
65  * single pass for both the structure ( : and CRLF ) and semantic (which
66  * header and value) of a header.  Specifically the static {@link HttpHeader#CACHE}
67  * is used to lookup common combinations of headers and values
68  * (eg. "Connection: close"), or just header names (eg. "Connection:" ).
69  * For headers who's value is not known statically (eg. Host, COOKIE) then a
70  * per parser dynamic Trie of {@link HttpFields} from previous parsed messages
71  * is used to help the parsing of subsequent messages.
72  * </p>
73  * <p>
74  * The parser can work in varying compliance modes:
75  * <dl>
76  * <dt>RFC7230</dt><dd>(default) Compliance with RFC7230</dd>
77  * <dt>RFC2616</dt><dd>Wrapped headers and HTTP/0.9 supported</dd>
78  * <dt>LEGACY</dt><dd>(aka STRICT) Adherence to Servlet Specification requirement for
79  * exact case of header names, bypassing the header caches, which are case insensitive,
80  * otherwise equivalent to RFC2616</dd>
81  * </dl>
82  *
83  * @see <a href="http://tools.ietf.org/html/rfc7230">RFC 7230</a>
84  */
85 class HttpParser {
86 
87     enum INITIAL_URI_LENGTH = 256;
88     private MonoTime startTime;
89 
90     /**
91      * Cache of common {@link HttpField}s including: <UL>
92      * <LI>Common static combinations such as:<UL>
93      * <li>Connection: close
94      * <li>Accept-Encoding: gzip
95      * <li>Content-Length: 0
96      * </ul>
97      * <li>Combinations of Content-Type header for common mime types by common charsets
98      * <li>Most common headers with null values so that a lookup will at least
99      * determine the header name even if the name:value combination is not cached
100      * </ul>
101      */
102     // __gshared Trie!HttpField CACHE;
103     static Trie!HttpField CACHE() {
104         __gshared Trie!HttpField inst;
105         return initOnce!inst(initFieldCache());
106     }
107 
108     // States
109     enum FieldState {
110         FIELD,
111         IN_NAME,
112         VALUE,
113         IN_VALUE,
114         WS_AFTER_NAME,
115     }
116 
117     // States
118     enum State {
119         START,
120         METHOD,
121         RESPONSE_VERSION,
122         SPACE1,
123         STATUS,
124         URI,
125         SPACE2,
126         REQUEST_VERSION,
127         REASON,
128         PROXY,
129         HEADER,
130         CONTENT,
131         EOF_CONTENT,
132         CHUNKED_CONTENT,
133         CHUNK_SIZE,
134         CHUNK_PARAMS,
135         CHUNK,
136         TRAILER,
137         END,
138         CLOSE,  // The associated stream/endpoint should be closed
139         CLOSED  // The associated stream/endpoint is at EOF
140     }
141 
142     private static State[] __idleStates = [State.START, State.END, State.CLOSE, State.CLOSED];
143     private static State[] __completeStates = [State.END, State.CLOSE, State.CLOSED];
144 
145     private HttpParsingHandler _handler;
146     private HttpRequestParsingHandler _requestHandler;
147     private HttpResponseParsingHandler _responseHandler;
148     private ComplianceParsingHandler _complianceHandler;
149     private int _maxHeaderBytes;
150     private HttpCompliance _compliance;
151     private HttpComplianceSection[] _compliances;
152     private HttpField _field;
153     private HttpHeader _header;
154     private string _headerString;
155     private string _valueString;
156     private int _responseStatus;
157     private int _headerBytes;
158     private bool _host;
159     private bool _headerComplete;
160 
161     /* ------------------------------------------------------------------------------- */
162     private  State _state = State.START;
163     private  FieldState _fieldState = FieldState.FIELD;
164     private  bool _eof;
165     private HttpMethod _method;
166     private string _methodString;
167     private HttpVersion _version;
168     private StringBuilder _uri;
169     private EndOfContent _endOfContent;
170     private long _contentLength = -1;
171     private long _contentPosition;
172     private int _chunkLength;
173     private int _chunkPosition;
174     private bool _headResponse;
175     private bool _cr;
176     private ByteBuffer _contentChunk;
177     private Trie!HttpField _fieldCache;
178 
179     private int _length;
180     private StringBuilder _string; 
181 
182     /* ------------------------------------------------------------------------------- */
183     this(HttpRequestParsingHandler handler) {
184         this(handler, -1, getCompliance());
185     }
186 
187     /* ------------------------------------------------------------------------------- */
188     this(HttpResponseParsingHandler handler) {
189         this(handler, -1, getCompliance());
190     }
191 
192     /* ------------------------------------------------------------------------------- */
193     this(HttpRequestParsingHandler handler, int maxHeaderBytes) {
194         this(handler, maxHeaderBytes, getCompliance());
195     }
196 
197     /* ------------------------------------------------------------------------------- */
198     this(HttpResponseParsingHandler handler, int maxHeaderBytes) {
199         this(handler, maxHeaderBytes, getCompliance());
200     }
201 
202     /* ------------------------------------------------------------------------------- */
203     this(HttpRequestParsingHandler handler, HttpCompliance compliance) {
204         this(handler, -1, compliance);
205     }
206 
207     /* ------------------------------------------------------------------------------- */
208     this(HttpRequestParsingHandler handler, int maxHeaderBytes, HttpCompliance compliance) {
209         this(handler, null, maxHeaderBytes, compliance is null ? getCompliance() : compliance);
210     }
211 
212     /* ------------------------------------------------------------------------------- */
213     this(HttpResponseParsingHandler handler, int maxHeaderBytes, HttpCompliance compliance) {
214         this(null, handler, maxHeaderBytes, compliance is null ? getCompliance() : compliance);
215     }
216 
217     /* ------------------------------------------------------------------------------- */
218     private this(HttpRequestParsingHandler requestHandler, HttpResponseParsingHandler responseHandler, 
219         int maxHeaderBytes, HttpCompliance compliance) {
220         version (HUNT_HTTP_DEBUG) {
221             trace("create http parser");
222         }
223         _string = new StringBuilder();
224         _uri = new StringBuilder(INITIAL_URI_LENGTH);
225         if(requestHandler !is null)
226             _handler = requestHandler;
227         else
228             _handler = responseHandler;
229         // _handler = requestHandler !is null ? cast()requestHandler : responseHandler;
230         _requestHandler = requestHandler;
231         _responseHandler = responseHandler;
232         _maxHeaderBytes = maxHeaderBytes;
233         _compliance = compliance;
234         _compliances = compliance.sections();
235         _complianceHandler = cast(ComplianceParsingHandler)_handler;
236     }
237 
238 
239     private static Trie!HttpField initFieldCache() {
240         Trie!HttpField cache = new ArrayTrie!HttpField(2048);
241         cache.put(new HttpField(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE));
242         cache.put(new HttpField(HttpHeader.CONNECTION, HttpHeaderValue.KEEP_ALIVE));
243         cache.put(new HttpField(HttpHeader.CONNECTION, HttpHeaderValue.UPGRADE));
244         cache.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip"));
245         cache.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip, deflate"));
246         cache.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip, deflate, br"));
247         cache.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip,deflate,sdch"));
248         cache.put(new HttpField(HttpHeader.ACCEPT_LANGUAGE, "en-US,en;q=0.5"));
249         cache.put(new HttpField(HttpHeader.ACCEPT_LANGUAGE, "en-GB,en-US;q=0.8,en;q=0.6"));
250 
251         cache.put(new HttpField(HttpHeader.ACCEPT_LANGUAGE, 
252             "en-AU,en;q=0.9,it-IT;q=0.8,it;q=0.7,en-GB;q=0.6,en-US;q=0.5"));
253             
254         cache.put(new HttpField(HttpHeader.ACCEPT_CHARSET, "ISO-8859-1,utf-8;q=0.7,*;q=0.3"));
255         cache.put(new HttpField(HttpHeader.ACCEPT, "*/*"));
256         cache.put(new HttpField(HttpHeader.ACCEPT, "image/png,image/*;q=0.8,*/*;q=0.5"));
257         cache.put(new HttpField(HttpHeader.ACCEPT, 
258             "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"));
259 
260         cache.put(new HttpField(HttpHeader.ACCEPT, 
261             "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"));
262 
263         cache.put(new HttpField(HttpHeader.ACCEPT_RANGES, HttpHeaderValue.BYTES));
264         cache.put(new HttpField(HttpHeader.PRAGMA, "no-cache"));
265 
266         cache.put(new HttpField(HttpHeader.CACHE_CONTROL, 
267             "private, no-cache, no-cache=Set-Cookie, proxy-revalidate"));
268 
269         cache.put(new HttpField(HttpHeader.CACHE_CONTROL, "no-cache"));
270         cache.put(new HttpField(HttpHeader.CACHE_CONTROL, "max-age=0"));
271         cache.put(new HttpField(HttpHeader.CONTENT_LENGTH, "0"));
272         cache.put(new HttpField(HttpHeader.CONTENT_ENCODING, "gzip"));
273         cache.put(new HttpField(HttpHeader.CONTENT_ENCODING, "deflate"));
274         cache.put(new HttpField(HttpHeader.TRANSFER_ENCODING, "chunked"));
275         cache.put(new HttpField(HttpHeader.EXPIRES, "Fri, 01 Jan 1990 00:00:00 GMT"));
276 
277         // Add common Content types as fields
278         foreach (string type ; ["text/plain", "text/html", "text/xml", "text/json", 
279             "application/json", "application/x-www-form-urlencoded"]) {
280             HttpField field = new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type);
281             cache.put(field);
282 
283             foreach (string charset ; ["utf-8", "iso-8859-1"]) {
284                 cache.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ ";charset=" ~ charset));
285                 cache.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ "; charset=" ~ charset));
286                 cache.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ ";charset=" ~ charset.toUpper()));
287                 cache.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ "; charset=" ~ charset.toUpper()));
288             }
289         }
290 
291         // Add headers with null values so HttpParser can avoid looking up name again for unknown values
292         foreach (HttpHeader h ; HttpHeader.values()) {
293             // trace(h.toString());
294             if (!cache.put(new HttpField(h, cast(string) null))) {
295                 // FIXME: Needing refactor or cleanup -@zxp at 9/25/2018, 8:11:29 PM
296                 // 
297                 // warning(h.toString());
298                 // throw new IllegalStateException("CACHE FULL");
299             }
300         }
301 
302         // Add some more common headers
303         cache.put(new HttpField(HttpHeader.REFERER, cast(string) null));
304         cache.put(new HttpField(HttpHeader.IF_MODIFIED_SINCE, cast(string) null));
305         cache.put(new HttpField(HttpHeader.IF_NONE_MATCH, cast(string) null));
306         cache.put(new HttpField(HttpHeader.AUTHORIZATION, cast(string) null));
307         cache.put(new HttpField(HttpHeader.COOKIE, cast(string) null));
308 
309         return cache;
310     }
311 
312     private static HttpCompliance getCompliance() {
313         return HttpCompliance.RFC7230;
314     }
315 
316 
317     /* ------------------------------------------------------------------------------- */
318     HttpParsingHandler getHandler() {
319         return _handler;
320     }
321 
322     /* ------------------------------------------------------------------------------- */
323 
324     /**
325      * Check RFC compliance violation
326      *
327      * @param violation The compliance section violation
328      * @param reason    The reason for the violation
329      * @return True if the current compliance level is set so as to Not allow this violation
330      */
331     protected bool complianceViolation(HttpComplianceSection violation, string reason) {
332         if (_compliances.contains(violation))
333             return true;
334 
335         if (_complianceHandler !is null)
336             _complianceHandler.onComplianceViolation(_compliance, violation, reason);
337 
338         return false;
339     }
340 
341     /* ------------------------------------------------------------------------------- */
342     protected void handleViolation(HttpComplianceSection section, string reason) {
343         if (_complianceHandler !is null)
344             _complianceHandler.onComplianceViolation(_compliance, section, reason);
345     }
346 
347     /* ------------------------------------------------------------------------------- */
348     protected string caseInsensitiveHeader(string orig, string normative) {
349         if (_compliances.contains(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE))
350             return normative;
351         if (!orig.equals(normative))
352             handleViolation(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE, orig);
353         return orig;
354     }
355 
356     /* ------------------------------------------------------------------------------- */
357     long getContentLength() {
358         return _contentLength;
359     }
360 
361     /* ------------------------------------------------------------ */
362     long getContentRead() {
363         return _contentPosition;
364     }
365 
366     /* ------------------------------------------------------------ */
367 
368     /**
369      * Set if a HEAD response is expected
370      *
371      * @param head true if head response is expected
372      */
373     void setHeadResponse(bool head) {
374         _headResponse = head;
375     }
376 
377     /* ------------------------------------------------------------------------------- */
378     protected void setResponseStatus(int status) {
379         _responseStatus = status;
380     }
381 
382     /* ------------------------------------------------------------------------------- */
383     State getState() {
384         return _state;
385     }
386 
387     /* ------------------------------------------------------------------------------- */
388     bool inContentState() {
389         return _state >= State.CONTENT && _state < State.END;
390     }
391 
392     /* ------------------------------------------------------------------------------- */
393     bool inHeaderState() {
394         return _state < State.CONTENT;
395     }
396 
397     /* ------------------------------------------------------------------------------- */
398     bool isChunking() {
399         return _endOfContent == EndOfContent.CHUNKED_CONTENT;
400     }
401 
402     /* ------------------------------------------------------------ */
403     bool isStart() {
404         return isState(State.START);
405     }
406 
407     /* ------------------------------------------------------------ */
408     bool isClose() {
409         return isState(State.CLOSE);
410     }
411 
412     /* ------------------------------------------------------------ */
413     bool isClosed() {
414         return isState(State.CLOSED);
415     }
416 
417     /* ------------------------------------------------------------ */
418     bool isIdle() {
419         return __idleStates.contains(_state);
420     }
421 
422     /* ------------------------------------------------------------ */
423     bool isComplete() {
424         return __completeStates.contains(_state);
425     }
426 
427     /* ------------------------------------------------------------------------------- */
428     bool isState(State state) {
429         return _state == state;
430     }
431 
432     /* ------------------------------------------------------------------------------- */
433     enum CharState {
434         ILLEGAL, CR, LF, LEGAL
435     }
436 
437     private __gshared CharState[] __charState;
438 
439     shared static this() {
440         // token          = 1*tchar
441         // tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
442         //                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
443         //                / DIGIT / ALPHA
444         //                ; any VCHAR, except delimiters
445         // quoted-string  = DQUOTE *( qdtext / quoted-pair ) DQUOTE
446         // qdtext         = HTAB / SP /%x21 / %x23-5B / %x5D-7E / obs-text
447         // obs-text       = %x80-FF
448         // comment        = "(" *( ctext / quoted-pair / comment ) ")"
449         // ctext          = HTAB / SP / %x21-27 / %x2A-5B / %x5D-7E / obs-text
450         // quoted-pair    = "\" ( HTAB / SP / VCHAR / obs-text )
451 
452         __charState = new CharState[256];
453         __charState[0..$] = CharState.ILLEGAL;
454         // Arrays.fill(__charState, CharState.ILLEGAL);
455         __charState[HttpTokens.LINE_FEED] = CharState.LF;
456         __charState[HttpTokens.CARRIAGE_RETURN] = CharState.CR;
457         __charState[HttpTokens.TAB] = CharState.LEGAL;
458         __charState[HttpTokens.SPACE] = CharState.LEGAL;
459 
460         __charState['!'] = CharState.LEGAL;
461         __charState['#'] = CharState.LEGAL;
462         __charState['$'] = CharState.LEGAL;
463         __charState['%'] = CharState.LEGAL;
464         __charState['&'] = CharState.LEGAL;
465         __charState['\''] = CharState.LEGAL;
466         __charState['*'] = CharState.LEGAL;
467         __charState['+'] = CharState.LEGAL;
468         __charState['-'] = CharState.LEGAL;
469         __charState['.'] = CharState.LEGAL;
470         __charState['^'] = CharState.LEGAL;
471         __charState['_'] = CharState.LEGAL;
472         __charState['`'] = CharState.LEGAL;
473         __charState['|'] = CharState.LEGAL;
474         __charState['~'] = CharState.LEGAL;
475 
476         __charState['"'] = CharState.LEGAL;
477 
478         __charState['\\'] = CharState.LEGAL;
479         __charState['('] = CharState.LEGAL;
480         __charState[')'] = CharState.LEGAL;
481         __charState[0x21 .. 0x27 + 1] = CharState.LEGAL;
482         __charState[0x2A .. 0x5B + 1] = CharState.LEGAL;
483         __charState[0x5D .. 0x7E + 1] = CharState.LEGAL;
484         __charState[0x80 .. 0xFF + 1] = CharState.LEGAL;
485     }
486 
487     /* ------------------------------------------------------------------------------- */
488     private byte next(ByteBuffer buffer) {
489         byte ch = buffer.get();
490 
491         CharState s = __charState[0xff & ch];
492         switch (s) {
493             case CharState.ILLEGAL:
494                 throw new IllegalCharacterException(_state, ch, buffer);
495 
496             case CharState.LF:
497                 _cr = false;
498                 break;
499 
500             case CharState.CR:
501                 if (_cr)
502                     throw new BadMessageException("Bad EOL");
503 
504                 _cr = true;
505                 if (buffer.hasRemaining()) {
506                     // Don't count the CRs and LFs of the chunked encoding.
507                     if (_maxHeaderBytes > 0 && (_state == State.HEADER || _state == State.TRAILER))
508                         _headerBytes++;
509                     return next(buffer);
510                 }
511 
512                 // Can return 0 here to indicate the need for more characters,
513                 // because a real 0 in the buffer would cause a BadMessage below
514                 return 0;
515 
516             case CharState.LEGAL:
517                 if (_cr)
518                     throw new BadMessageException("Bad EOL");
519                 break;
520             
521             default:
522                 break;
523         }
524 
525         return ch;
526     }
527 
528     /* ------------------------------------------------------------------------------- */
529     /* Quick lookahead for the start state looking for a request method or a HTTP version,
530      * otherwise skip white space until something else to parse.
531      */
532     private bool quickStart(ByteBuffer buffer) {
533         if (_requestHandler !is null) {
534             _method = HttpMethod.lookAheadGet(buffer);
535             if (_method != HttpMethod.Null) {
536                 _methodString = _method.asString();
537                 buffer.position(cast(int)(buffer.position() + _methodString.length + 1));
538 
539                 setState(State.SPACE1);
540                 return false;
541             }
542         } else if (_responseHandler !is null) {
543             _version = HttpVersion.lookAheadGet(buffer);
544             if (_version != HttpVersion.Null) {
545                 buffer.position(cast(int) (buffer.position() + _version.asString().length + 1));
546                 setState(State.SPACE1);
547                 return false;
548             }
549         }
550 
551         // Quick start look
552         while (_state == State.START && buffer.hasRemaining()) {
553             int ch = next(buffer);
554 
555             if (ch > HttpTokens.SPACE) {
556                 _string.setLength(0);
557                 _string.append(cast(char) ch);
558                 setState(_requestHandler !is null ? State.METHOD : State.RESPONSE_VERSION);
559                 return false;
560             } else if (ch == 0)
561                 break;
562             else if (ch < 0)
563                 throw new BadMessageException();
564 
565             // count this white space as a header byte to avoid DOS
566             if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
567                 warningf("padding is too large >%d", _maxHeaderBytes);
568                 throw new BadMessageException(HttpStatus.BAD_REQUEST_400);
569             }
570         }
571         return false;
572     }
573 
574     /* ------------------------------------------------------------------------------- */
575     private void setString(string s) {
576         _string.setLength(0);
577         _string.append(s);
578         _length = cast(int)s.length;
579     }
580 
581     /* ------------------------------------------------------------------------------- */
582     private string takeString() {
583         _string.setLength(_length);
584         string s = _string.toString();
585         _string.setLength(0);
586         _length = -1;
587         return s;
588     }
589 
590     /* ------------------------------------------------------------------------------- */
591     private bool handleHeaderContentMessage() {
592         version (HUNT_HTTP_DEBUG_MORE) trace("handling headers ...");
593         bool handle_header = _handler.headerComplete();
594         _headerComplete = true;
595         version (HUNT_HTTP_DEBUG_MORE) trace("handling content ...");
596         bool handle_content = _handler.contentComplete();
597         version (HUNT_HTTP_DEBUG_MORE) trace("handling message ...");
598         bool handle_message = _handler.messageComplete();
599         return handle_header || handle_content || handle_message;
600     }
601 
602     /* ------------------------------------------------------------------------------- */
603     private bool handleContentMessage() {
604         bool handle_content = _handler.contentComplete();
605         bool handle_message = _handler.messageComplete();
606         return handle_content || handle_message;
607     }
608 
609     /* ------------------------------------------------------------------------------- */
610     /* Parse a request or response line
611      */
612     private bool parseLine(ByteBuffer buffer) {
613         bool handle = false;
614 
615         // Process headers
616         while (_state < State.HEADER && buffer.hasRemaining() && !handle) {
617             // process each character
618             byte b = next(buffer);
619             if (b == 0)
620                 break;
621 
622             if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
623                 if (_state == State.URI) {
624                     warningf("URI is too large >%d", _maxHeaderBytes);
625                     throw new BadMessageException(HttpStatus.URI_TOO_LONG_414);
626                 } else {
627                     if (_requestHandler !is null)
628                         warningf("request is too large >%d", _maxHeaderBytes);
629                     else
630                         warningf("response is too large >%d", _maxHeaderBytes);
631                     throw new BadMessageException(HttpStatus.REQUEST_HEADER_FIELDS_TOO_LARGE_431);
632                 }
633             }
634 
635             switch (_state) {
636                 case State.METHOD:
637                     if (b == HttpTokens.SPACE) {
638                         _length = _string.length;
639                         _methodString = takeString();
640 
641                         if (_compliances.contains(HttpComplianceSection.METHOD_CASE_SENSITIVE)) {
642                             HttpMethod method = HttpMethod.get(_methodString);
643                             if (method != HttpMethod.Null)
644                                 _methodString = method.asString();
645                         } else {
646                             HttpMethod method = HttpMethod.getInsensitive(_methodString);
647 
648                             if (method != HttpMethod.Null) {
649                                 if (method.asString() != (_methodString))
650                                     handleViolation(HttpComplianceSection.METHOD_CASE_SENSITIVE, _methodString);
651                                 _methodString = method.asString();
652                             }
653                         }
654 
655                         setState(State.SPACE1);
656                     } else if (b < HttpTokens.SPACE) {
657                         if (b == HttpTokens.LINE_FEED)
658                             throw new BadMessageException("No URI");
659                         else
660                             throw new IllegalCharacterException(_state, b, buffer);
661                     } else
662                         _string.append(cast(char) b);
663                     break;
664 
665                 case State.RESPONSE_VERSION:
666                     if (b == HttpTokens.SPACE) {
667                         _length = _string.length;
668                         string ver = takeString();
669                         _version = HttpVersion.fromString(ver);
670                         if (_version == HttpVersion.Null)
671                             throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Unknown Version");
672                         setState(State.SPACE1);
673                     } else if (b < HttpTokens.SPACE)
674                         throw new IllegalCharacterException(_state, b, buffer);
675                     else
676                         _string.append(cast(char) b);
677                     break;
678 
679                 case State.SPACE1:
680                     if (b > HttpTokens.SPACE || b < 0) {
681                         if (_responseHandler !is null) {
682                             setState(State.STATUS);
683                             setResponseStatus(b - '0');
684                         } else {
685                             _uri.reset();
686                             setState(State.URI);
687                             // quick scan for space or EoBuffer
688                             if (buffer.hasArray()) {
689                                 byte[] array = buffer.array();
690                                 int p = buffer.arrayOffset() + buffer.position();
691                                 int l = buffer.arrayOffset() + buffer.limit();
692                                 int i = p;
693                                 while (i < l && array[i] > HttpTokens.SPACE)
694                                     i++;
695 
696                                 int len = i - p;
697                                 _headerBytes += len;
698 
699                                 if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
700                                     warningf("URI is too large >%d", _maxHeaderBytes);
701                                     throw new BadMessageException(HttpStatus.URI_TOO_LONG_414);
702                                 }
703                                 _uri.append(array, p - 1, len + 1);
704                                 buffer.position(i - buffer.arrayOffset());
705                             } else
706                                 _uri.append(b);
707                         }
708                     } else if (b < HttpTokens.SPACE) {
709                         throw new BadMessageException(HttpStatus.BAD_REQUEST_400, 
710                             _requestHandler !is null ? "No URI" : "No Status");
711                     }
712                     break;
713 
714                 case State.STATUS:
715                     if (b == HttpTokens.SPACE) {
716                         setState(State.SPACE2);
717                     } else if (b >= '0' && b <= '9') {
718                         _responseStatus = _responseStatus * 10 + (b - '0');
719                     } else if (b < HttpTokens.SPACE && b >= 0) {
720                         setState(State.HEADER);
721                         handle = _responseHandler.startResponse(_version, _responseStatus, null) || handle;
722                     } else {
723                         throw new BadMessageException();
724                     }
725                     break;
726 
727                 case State.URI:
728                     if (b == HttpTokens.SPACE) {
729                         setState(State.SPACE2);
730                     } else if (b < HttpTokens.SPACE && b >= 0) {
731                         // HTTP/0.9
732                         if (complianceViolation(HttpComplianceSection.NO_HTTP_0_9, "No request version")) {
733                             throw new BadMessageException("HTTP/0.9 not supported");
734                         }
735 
736                         if(_requestHandler !is null) {
737                             handle = _requestHandler.startRequest(_methodString, _uri.toString(), HttpVersion.HTTP_0_9);
738                         } else {
739                             warning("no requestHandler defined");
740                         }
741 
742                         setState(State.END);
743                         BufferUtils.clear(buffer);
744                         handle = handleHeaderContentMessage() || handle;
745                     } else {
746                         _uri.append(b);
747                     }
748                     break;
749 
750                 case State.SPACE2:
751                     if (b > HttpTokens.SPACE) {
752                         _string.setLength(0);
753                         _string.append(cast(char) b);
754                         if (_responseHandler !is null) {
755                             _length = 1;
756                             setState(State.REASON);
757                         } else {
758                             setState(State.REQUEST_VERSION);
759 
760                             // try quick look ahead for HTTP Version
761                             HttpVersion ver;
762                             if (buffer.position() > 0 && buffer.hasArray()) {
763                                 ver = HttpVersion.lookAheadGet(buffer.array(), 
764                                     buffer.arrayOffset() + buffer.position() - 1, 
765                                     buffer.arrayOffset() + buffer.limit());
766                             } else {
767                                 string key = buffer.getString(0, buffer.remaining());
768                                 ver = HttpVersion.fromString(key);
769                             }
770 
771                             if (ver != HttpVersion.Null) {
772                                 int pos = cast(int)(buffer.position() + ver.asString().length - 1);
773                                 if (pos < buffer.limit()) {
774                                     byte n = buffer.get(pos);
775                                     if (n == HttpTokens.CARRIAGE_RETURN) {
776                                         _cr = true;
777                                         _version = ver;
778                                         _string.setLength(0);
779                                         buffer.position(pos + 1);
780                                     } else if (n == HttpTokens.LINE_FEED) {
781                                         _version = ver;
782                                         _string.setLength(0);
783                                         buffer.position(pos);
784                                     }
785                                 }
786                             }
787                         }
788                     } else if (b == HttpTokens.LINE_FEED) {
789                         if (_responseHandler !is null) {
790                             setState(State.HEADER);
791                             handle = _responseHandler.startResponse(_version, _responseStatus, null) || handle;
792                         } else {
793                             // HTTP/0.9
794                             if (complianceViolation(HttpComplianceSection.NO_HTTP_0_9, "No request version"))
795                                 throw new BadMessageException("HTTP/0.9 not supported");
796                             
797                             if(_requestHandler is null) {
798                                 warning("no requestHandler defined");
799                             } else {
800                                 handle = _requestHandler.startRequest(_methodString, _uri.toString(), HttpVersion.HTTP_0_9);
801                             }
802                             
803                             setState(State.END);
804                             BufferUtils.clear(buffer);
805                             handle = handleHeaderContentMessage() || handle;
806                         }
807                     } else if (b < 0)
808                         throw new BadMessageException();
809                     break;
810 
811                 case State.REQUEST_VERSION:
812                     if (b == HttpTokens.LINE_FEED) {
813                         if (_version == HttpVersion.Null) {
814                             _length = _string.length;
815                             _version = HttpVersion.fromString(takeString());
816                         }
817                         if (_version == HttpVersion.Null)
818                             throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Unknown Version");
819 
820                         // Should we try to cache header fields?
821                         if (_fieldCache is null && 
822                             _version.getVersion() >= HttpVersion.HTTP_1_1.getVersion() && 
823                             _handler.getHeaderCacheSize() > 0) {
824                             int header_cache = _handler.getHeaderCacheSize();
825                             _fieldCache = new ArrayTernaryTrie!HttpField(header_cache);
826                         }
827 
828                         setState(State.HEADER);
829                         
830                         if(_requestHandler is null) {
831                             warning("no requestHandler defined");
832                         } else {
833                             handle = _requestHandler.startRequest(_methodString, _uri.toString(), _version) || handle;
834                         }
835                         continue;
836                     } else if (b >= HttpTokens.SPACE)
837                         _string.append(cast(char) b);
838                     else
839                         throw new BadMessageException();
840 
841                     break;
842 
843                 case State.REASON:
844                     if (b == HttpTokens.LINE_FEED) {
845                         string reason = takeString();
846                         setState(State.HEADER);
847                         handle = _responseHandler.startResponse(_version, _responseStatus, reason) || handle;
848                         continue;
849                     } else if (b >= HttpTokens.SPACE || ((b < 0) && (b >= -96))) {
850                         _string.append(cast(char) (0xff & b));
851                         if (b != ' ' && b != '\t')
852                             _length = _string.length;
853                     } else
854                         throw new BadMessageException();
855                     break;
856 
857                 default:
858                     throw new IllegalStateException(_state.to!string());
859             }
860         }
861 
862         return handle;
863     }
864 
865     private void parsedHeader() {
866         // handler last header if any.  Delayed to here just in case there was a continuation line (above)
867         if (!_headerString.empty() || !_valueString.empty()) {
868             // Handle known headers
869             version(HUNT_HTTP_DEBUG_MORE) {
870                 tracef("parsing header: %s, original name: %s, value: %s ", 
871                     _header.toString(), _headerString, _valueString);
872             }
873 
874             if (_header != HttpHeader.Null) {
875                 bool canAddToConnectionTrie = false;
876                 if(_header == HttpHeader.CONTENT_LENGTH) {
877                     if (_endOfContent == EndOfContent.CONTENT_LENGTH) {
878                         throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Duplicate Content-Length");
879                     } else if (_endOfContent != EndOfContent.CHUNKED_CONTENT) {
880                         _contentLength = convertContentLength(_valueString);
881                         if (_contentLength <= 0)
882                             _endOfContent = EndOfContent.NO_CONTENT;
883                         else
884                             _endOfContent = EndOfContent.CONTENT_LENGTH;
885                     }
886                 }
887                 else if(_header == HttpHeader.TRANSFER_ENCODING){
888                     if (HttpHeaderValue.CHUNKED.isSame(_valueString)) {
889                         _endOfContent = EndOfContent.CHUNKED_CONTENT;
890                         _contentLength = -1;
891                     } else {
892                         string[] values = new QuotedCSV(_valueString).getValues();
893                         if (values.length > 0 && HttpHeaderValue.CHUNKED.isSame(values[$ - 1])) {
894                             _endOfContent = EndOfContent.CHUNKED_CONTENT;
895                             _contentLength = -1;
896                         } else {
897                             foreach(string v; values) {
898                                 if(HttpHeaderValue.CHUNKED.isSame(v)) {
899                                     throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Bad chunking");
900                                 }
901                             }
902                         } 
903                     }
904                 }                    
905                 else if(_header == HttpHeader.HOST) {
906                     _host = true;
907                     if ((_field is null) && !_valueString.empty()) {
908                         string headerStr = _compliances.contains(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE) ? 
909                                     _header.asString() : _headerString;
910                         _field = new HostPortHttpField(_header, headerStr, _valueString);
911                         canAddToConnectionTrie = _fieldCache !is null;
912                     }
913                 }
914                 else if(_header == HttpHeader.CONNECTION) {
915                     // Don't cache headers if not persistent
916                     if (HttpHeaderValue.CLOSE.isSame(_valueString)) 
917                         _fieldCache = null;
918                     else {
919                         string[] values = new QuotedCSV(_valueString).getValues();
920                         foreach(string v; values) {
921                             if(HttpHeaderValue.CLOSE.isSame(v)) {
922                                 _fieldCache = null;
923                                 break;
924                             }
925                         }
926                     }
927                 }
928                 else if(_header == HttpHeader.AUTHORIZATION || _header == HttpHeader.ACCEPT ||
929                 _header == HttpHeader.ACCEPT_CHARSET || _header ==  HttpHeader.ACCEPT_ENCODING ||
930                 _header == HttpHeader.ACCEPT_LANGUAGE || _header == HttpHeader.COOKIE || 
931                 _header == HttpHeader.CACHE_CONTROL || _header == HttpHeader.USER_AGENT) {
932                     canAddToConnectionTrie = _fieldCache !is null && _field is null;
933                 }
934 
935                 if (canAddToConnectionTrie && !_fieldCache.isFull() 
936                     && _header != HttpHeader.Null && !_valueString.empty()) {
937                     if (_field is null)
938                         _field = new HttpField(_header, caseInsensitiveHeader(_headerString, _header.asString()), _valueString);
939                     _fieldCache.put(_field);
940                 }
941             }
942             _handler.parsedHeader(_field !is null ? _field : new HttpField(_header, _headerString, _valueString));
943         }
944 
945         _headerString = _valueString = null;
946         _header = HttpHeader.Null;
947         _field = null;
948     }
949 
950     private void parsedTrailer() {
951         // handler last header if any.  Delayed to here just in case there was a continuation line (above)
952         if (!_headerString.empty() || !_valueString.empty()) 
953             _handler.parsedTrailer(_field !is null ? _field : new HttpField(_header, _headerString, _valueString));
954 
955         _headerString = _valueString = null;
956         _header = HttpHeader.Null;
957         _field = null;
958     }
959 
960     private long convertContentLength(string valueString) {
961         try {
962             return to!long(valueString);
963         } catch (Exception e) {
964             warning("parse long exception: ", e);
965             throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Invalid Content-Length Value");
966         }
967     }
968 
969     /* ------------------------------------------------------------------------------- */
970     /*
971      * Parse the message headers and return true if the handler has signaled for a return
972      */
973     protected bool parseFields(ByteBuffer buffer) {
974         // Process headers
975         while ((_state == State.HEADER || _state == State.TRAILER) && buffer.hasRemaining()) {
976             // process each character
977             byte b = next(buffer);
978             if (b == 0)
979                 break;
980 
981             if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
982                 bool header = _state == State.HEADER;
983                 warningf("%s is too large %s>%s", header ? "Header" : "Trailer", _headerBytes, _maxHeaderBytes);
984                 throw new BadMessageException(header ?
985                         HttpStatus.REQUEST_HEADER_FIELDS_TOO_LARGE_431 :
986                         HttpStatus.PAYLOAD_TOO_LARGE_413);
987             }
988 
989             switch (_fieldState) {
990                 case FieldState.FIELD:
991                     switch (b) {
992                         case HttpTokens.COLON:
993                         case HttpTokens.SPACE:
994                         case HttpTokens.TAB: {
995                             if (complianceViolation(HttpComplianceSection.NO_FIELD_FOLDING, _headerString))
996                                 throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Header Folding");
997 
998                             // header value without name - continuation?
999                             if (_valueString == null || _valueString.empty()) {
1000                                 _string.setLength(0);
1001                                 _length = 0;
1002                             } else {
1003                                 setString(_valueString);
1004                                 _string.append(' ');
1005                                 _length++;
1006                                 _valueString = null;
1007                             }
1008                             setState(FieldState.VALUE);
1009                             break;
1010                         }
1011 
1012                         case HttpTokens.LINE_FEED: {
1013                             // process previous header
1014                             if (_state == State.HEADER)
1015                                 parsedHeader();
1016                             else
1017                                 parsedTrailer();
1018 
1019                             _contentPosition = 0;
1020 
1021                             // End of headers or trailers?
1022                             if (_state == State.TRAILER) {
1023                                 setState(State.END);
1024                                 return _handler.messageComplete();
1025                             }
1026 
1027                             // Was there a required host header?
1028                             if (!_host && _version == HttpVersion.HTTP_1_1 && _requestHandler !is null) {
1029                                 throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "No Host");
1030                             }
1031 
1032                             // is it a response that cannot have a body?
1033                             if (_responseHandler !is null && // response
1034                                     (_responseStatus == 304 || // not-modified response
1035                                             _responseStatus == 204 || // no-content response
1036                                             _responseStatus < 200)) // 1xx response
1037                                 _endOfContent = EndOfContent.NO_CONTENT; // ignore any other headers set
1038 
1039                                 // else if we don't know framing
1040                             else if (_endOfContent == EndOfContent.UNKNOWN_CONTENT) {
1041                                 if (_responseStatus == 0  // request
1042                                         || _responseStatus == 304 // not-modified response
1043                                         || _responseStatus == 204 // no-content response
1044                                         || _responseStatus < 200) // 1xx response
1045                                     _endOfContent = EndOfContent.NO_CONTENT;
1046                                 else
1047                                     _endOfContent = EndOfContent.EOF_CONTENT;
1048                             }
1049 
1050                             // How is the message ended?
1051                             switch (_endOfContent) {
1052                                 case EndOfContent.EOF_CONTENT: {
1053                                     setState(State.EOF_CONTENT);
1054                                     bool handle = _handler.headerComplete();
1055                                     _headerComplete = true;
1056                                     return handle;
1057                                 }
1058                                 case EndOfContent.CHUNKED_CONTENT: {
1059                                     setState(State.CHUNKED_CONTENT);
1060                                     bool handle = _handler.headerComplete();
1061                                     _headerComplete = true;
1062                                     return handle;
1063                                 }
1064                                 case EndOfContent.NO_CONTENT: {
1065                                     version (HUNT_HTTP_DEBUG) trace("parsing done for no content");
1066                                     setState(State.END);
1067                                     return handleHeaderContentMessage();
1068                                 }
1069                                 default: {
1070                                     setState(State.CONTENT);
1071                                     bool handle = _handler.headerComplete();
1072                                     _headerComplete = true;
1073                                     return handle;
1074                                 }
1075                             }
1076                         }
1077 
1078                         default: {
1079                             // process previous header
1080                             if (_state == State.HEADER)
1081                                 parsedHeader();
1082                             else
1083                                 parsedTrailer();
1084 
1085                             // handle new header
1086                             if (buffer.hasRemaining()) {
1087                                 // Try a look ahead for the known header name and value.
1088                                 HttpField cached_field = null;
1089                                 if(_fieldCache !is null)
1090                                     cached_field = _fieldCache.getBest(buffer, -1, buffer.remaining());
1091                                 // TODO: Tasks pending completion -@zxp at 10/23/2018, 8:03:47 PM
1092                                 // Can't handle Sec-WebSocket-Key
1093                                 // if (cached_field is null)
1094                                 //     cached_field = CACHE.getBest(buffer, -1, buffer.remaining());
1095 
1096                                 if (cached_field !is null) {
1097                                     string n = cached_field.getName();
1098                                     string v = cached_field.getValue();
1099 
1100                                     if (!_compliances.contains(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE)) {
1101                                         // Have to get the fields exactly from the buffer to match case
1102                                          // BufferUtils.toString(buffer, buffer.position() - 1, n.length, StandardCharsets.US_ASCII);
1103                                         string en = buffer.getString(buffer.position() - 1, cast(int)n.length);
1104                                         if (!n.equals(en)) {
1105                                             handleViolation(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE, en);
1106                                             n = en;
1107                                             cached_field = new HttpField(cached_field.getHeader(), n, v);
1108                                         }
1109                                     }
1110 
1111                                     if (v != null && !_compliances.contains(HttpComplianceSection.CASE_INSENSITIVE_FIELD_VALUE_CACHE)) {
1112                                         // BufferUtils.toString(buffer, buffer.position() + n.length + 1, v.length, StandardCharsets.ISO_8859_1);
1113                                         string ev = buffer.getString(buffer.position() + n.length + 1, v.length);
1114                                         if (!v.equals(ev)) {
1115                                             handleViolation(HttpComplianceSection.CASE_INSENSITIVE_FIELD_VALUE_CACHE, ev ~ "!=" ~ v);
1116                                             v = ev;
1117                                             cached_field = new HttpField(cached_field.getHeader(), n, v);
1118                                         }
1119                                     }
1120 
1121                                     _header = cached_field.getHeader();
1122                                     _headerString = n;
1123 
1124                                     if (v == null) {
1125                                         // Header only
1126                                         setState(FieldState.VALUE);
1127                                         _string.setLength(0);
1128                                         _length = 0;
1129                                         buffer.position(cast(int)(buffer.position() + n.length + 1));
1130                                         break;
1131                                     } else {
1132                                         // Header and value
1133                                         int pos = cast(int) (buffer.position() + n.length + v.length + 1);
1134                                         byte peek = buffer.get(pos);
1135 
1136                                         if (peek == HttpTokens.CARRIAGE_RETURN || peek == HttpTokens.LINE_FEED) {
1137                                             _field = cached_field;
1138                                             _valueString = v;
1139                                             setState(FieldState.IN_VALUE);
1140 
1141                                             if (peek == HttpTokens.CARRIAGE_RETURN) {
1142                                                 _cr = true;
1143                                                 buffer.position(pos + 1);
1144                                             } else
1145                                                 buffer.position(pos);
1146                                             break;
1147                                         } else {
1148                                             setState(FieldState.IN_VALUE);
1149                                             setString(v);
1150                                             buffer.position(pos);
1151                                             break;
1152                                         }
1153                                     }
1154                                 }
1155                             }
1156 
1157                             // New header
1158                             setState(FieldState.IN_NAME);
1159                             _string.setLength(0);
1160                             _string.append(cast(char) b);
1161                             _length = 1;
1162                         }
1163                     }
1164                     break;
1165 
1166                 case FieldState.IN_NAME:
1167                     switch (b) {
1168                         case HttpTokens.SPACE:
1169                         case HttpTokens.TAB:
1170                             //Ignore trailing whitespaces ?
1171                             if (!complianceViolation(HttpComplianceSection.NO_WS_AFTER_FIELD_NAME, null)) {
1172                                 _headerString = takeString();
1173                                 _header = HttpHeader.get(_headerString);
1174                                 _length = -1;
1175                                 setState(FieldState.WS_AFTER_NAME);
1176                                 break;
1177                             }
1178                             throw new IllegalCharacterException(_state, b, buffer);
1179 
1180                         case HttpTokens.COLON:
1181                             _headerString = takeString();
1182                             _header = HttpHeader.get(_headerString);
1183                             _length = -1;
1184                             setState(FieldState.VALUE);
1185                             break;
1186 
1187                         case HttpTokens.LINE_FEED:
1188                             _headerString = takeString();
1189                             _header = HttpHeader.get(_headerString);
1190                             _string.setLength(0);
1191                             _valueString = "";
1192                             _length = -1;
1193 
1194                             if (!complianceViolation(HttpComplianceSection.FIELD_COLON, _headerString)) {
1195                                 setState(FieldState.FIELD);
1196                                 break;
1197                             }
1198                             throw new IllegalCharacterException(_state, b, buffer);
1199 
1200                         default:
1201                             if (b < 0)
1202                                 throw new IllegalCharacterException(_state, b, buffer);
1203 
1204                             _string.append(cast(char) b);
1205                             _length = _string.length;
1206                             break;
1207                     }
1208                     break;
1209 
1210                 case FieldState.WS_AFTER_NAME:
1211 
1212                     switch (b) {
1213                         case HttpTokens.SPACE:
1214                         case HttpTokens.TAB:
1215                             break;
1216 
1217                         case HttpTokens.COLON:
1218                             setState(FieldState.VALUE);
1219                             break;
1220 
1221                         case HttpTokens.LINE_FEED:
1222                             if (!complianceViolation(HttpComplianceSection.FIELD_COLON, _headerString)) {
1223                                 setState(FieldState.FIELD);
1224                                 break;
1225                             }
1226                             throw new IllegalCharacterException(_state, b, buffer);
1227 
1228                         default:
1229                             throw new IllegalCharacterException(_state, b, buffer);
1230                     }
1231                     break;
1232 
1233                 case FieldState.VALUE:
1234                     switch (b) {
1235                         case HttpTokens.LINE_FEED:
1236                             _string.setLength(0);
1237                             _valueString = "";
1238                             _length = -1;
1239 
1240                             setState(FieldState.FIELD);
1241                             break;
1242 
1243                         case HttpTokens.SPACE:
1244                         case HttpTokens.TAB:
1245                             break;
1246 
1247                         default:
1248                             _string.append(cast(char) (0xff & b));
1249                             _length = _string.length;
1250                             setState(FieldState.IN_VALUE);
1251                             break;
1252                     }
1253                     break;
1254 
1255                 case FieldState.IN_VALUE:
1256                     switch (b) {
1257                         case HttpTokens.LINE_FEED:
1258                             if (_length > 0) {
1259                                 _valueString = takeString();
1260                                 _length = -1;
1261                             }
1262                             setState(FieldState.FIELD);
1263                             break;
1264 
1265                         case HttpTokens.SPACE:
1266                         case HttpTokens.TAB:
1267                             _string.append(cast(char) (0xff & b));
1268                             break;
1269 
1270                         default:
1271                             _string.append(cast(char) (0xff & b));
1272                             _length = _string.length;
1273                             break;
1274                     }
1275                     break;
1276 
1277                 default:
1278                     throw new IllegalStateException(_state.to!string());
1279 
1280             }
1281         }
1282 
1283         return false;
1284     }
1285 
1286     /* ------------------------------------------------------------------------------- */
1287 
1288     /**
1289      * Parse until next Event.
1290      *
1291      * @param buffer the buffer to parse
1292      * @return True if an {@link HttpRequestParsingHandler} method was called and it returned true;
1293      */
1294     bool parseNext(ByteBuffer buffer) {
1295         version(HUNT_HTTP_DEBUG_MORE) {
1296             tracef("parseNext s=%s %s", _state, BufferUtils.toDetailString(buffer));
1297             // tracef("buffer: %s", BufferUtils.toHexString(buffer));
1298 
1299             // startTime = MonoTime.currTime;
1300             // _requestHandler.startRequest("GET", "/plaintext", HttpVersion.HTTP_1_1);
1301             // setState(State.END);
1302             // _handler.messageComplete();
1303             // BufferUtils.clear(buffer);
1304             // return false;
1305         }
1306         try {
1307             // Start a request/response
1308             if (_state == State.START) {
1309                 version(HUNT_METRIC) {
1310                     startTime = MonoTime.currTime;
1311                     info("start a new parsing process...");
1312                 }                
1313                 _version = HttpVersion.Null;
1314                 _method = HttpMethod.Null;
1315                 _methodString = null;
1316                 _endOfContent = EndOfContent.UNKNOWN_CONTENT;
1317                 _header = HttpHeader.Null;
1318                 if (quickStart(buffer))
1319                     return true;
1320             }
1321 
1322             // Request/response line
1323             if (_state >= State.START && _state < State.HEADER) {
1324                 if (parseLine(buffer)) {
1325                     tracef("after parseLine =>%s", buffer.toString());
1326                     // return true;
1327                 }
1328             }
1329 
1330             // parse headers
1331             if (_state == State.HEADER) {
1332                 if (parseFields(buffer)) {
1333                     version(HUNT_HTTP_DEBUG_MORE) tracef("after parseFields =>%s", buffer.toString());
1334                     return true;
1335                 }
1336             }
1337 
1338             // parse content
1339             if (_state >= State.CONTENT && _state < State.TRAILER) {
1340                 // Handle HEAD response
1341                 if (_responseStatus > 0 && _headResponse) {
1342                     setState(State.END);
1343                     return handleContentMessage();
1344                 } else {
1345                     if (parseContent(buffer))
1346                         return true;
1347                 }
1348             }
1349 
1350             // parse headers
1351             if (_state == State.TRAILER) {
1352                 if (parseFields(buffer))
1353                     return true;
1354             }
1355 
1356             // handle end states
1357             if (_state == State.END) {
1358                 // eat white space
1359                 while (buffer.remaining() > 0 && buffer.get(buffer.position()) <= HttpTokens.SPACE)
1360                     buffer.get();
1361             } else if (isClose() || isClosed()) {
1362                 BufferUtils.clear(buffer);
1363             }
1364 
1365             // Handle EOF
1366             if (_eof && !buffer.hasRemaining()) {
1367                 switch (_state) {
1368                     case State.CLOSED:
1369                         break;
1370 
1371                     case State.START:
1372                         setState(State.CLOSED);
1373                         _handler.earlyEOF();
1374                         break;
1375 
1376                     case State.END:
1377                     case State.CLOSE:
1378                         setState(State.CLOSED);
1379                         break;
1380 
1381                     case State.EOF_CONTENT:
1382                     case State.TRAILER:
1383                         if (_fieldState == FieldState.FIELD) {
1384                             // Be forgiving of missing last CRLF
1385                             setState(State.CLOSED);
1386                             return handleContentMessage();
1387                         }
1388                         setState(State.CLOSED);
1389                         _handler.earlyEOF();
1390                         break;
1391 
1392                     case State.CONTENT:
1393                     case State.CHUNKED_CONTENT:
1394                     case State.CHUNK_SIZE:
1395                     case State.CHUNK_PARAMS:
1396                     case State.CHUNK:
1397                         setState(State.CLOSED);
1398                         _handler.earlyEOF();
1399                         break;
1400 
1401                     default:
1402                         version(HUNT_HTTP_DEBUG)
1403                             tracef("%s EOF in %s", this, _state);
1404                         setState(State.CLOSED);
1405                         _handler.badMessage(new BadMessageException(HttpStatus.BAD_REQUEST_400));
1406                         break;
1407                 }
1408             }
1409         } catch (BadMessageException x) {
1410             BufferUtils.clear(buffer);
1411             badMessage(x);
1412         } catch (Exception x) {
1413             version(HUNT_DEBUG) warning(x.msg);
1414             version(HUNT_HTTP_DEBUG) warning(x);
1415             BufferUtils.clear(buffer);
1416             badMessage(new BadMessageException(HttpStatus.BAD_REQUEST_400, 
1417                 _requestHandler !is null ? "Bad Request" : "Bad Response", x));
1418         }
1419         return false;
1420     }
1421 
1422     protected void badMessage(BadMessageException x) {
1423         version(HUNT_DEBUG)
1424             warning("Parse exception: " ~ this.toString() ~ " for " ~ _handler.toString() ~ 
1425                 ", Exception: ", x.msg);
1426 
1427         version(HUNT_HTTP_DEBUG) {
1428             Throwable t = x;
1429             while((t = t.next) !is null) {
1430                 error(t.msg);
1431             }
1432         }
1433         
1434         setState(State.CLOSE);
1435         if (_headerComplete)
1436             _handler.earlyEOF();
1437         else
1438             _handler.badMessage(x);
1439     }
1440 
1441     protected bool parseContent(ByteBuffer buffer) {
1442         int remaining = buffer.remaining();
1443         if (remaining == 0 && _state == State.CONTENT) {
1444             long content = _contentLength - _contentPosition;
1445             if (content == 0) {
1446                 setState(State.END);
1447                 return handleContentMessage();
1448             }
1449         }
1450 
1451         // Handle _content
1452         byte ch;
1453         while (_state < State.TRAILER && remaining > 0) {
1454             switch (_state) {
1455                 case State.EOF_CONTENT:
1456                     _contentChunk = buffer.slice(); // buffer.asReadOnlyBuffer();
1457                     _contentPosition += remaining;
1458                     buffer.position(buffer.position() + remaining);
1459                     if (_handler.content(_contentChunk))
1460                         return true;
1461                     break;
1462 
1463                 case State.CONTENT: {
1464                     long content = _contentLength - _contentPosition;
1465                     if (content == 0) {
1466                         setState(State.END);
1467                         return handleContentMessage();
1468                     } else {
1469                         _contentChunk = buffer.slice(); // buffer.asReadOnlyBuffer();
1470 
1471                         // limit content by expected size
1472                         if (remaining > content) {
1473                             // We can cast remaining to an int as we know that it is smaller than
1474                             // or equal to length which is already an int.
1475                             _contentChunk.limit(_contentChunk.position() + cast(int) content);
1476                         }
1477 
1478                         _contentPosition += _contentChunk.remaining();
1479                         buffer.position(buffer.position() + _contentChunk.remaining());
1480                         version(HUNT_HTTP_DEBUG) trace("setting content...");
1481                         if (_handler.content(_contentChunk))
1482                             return true;
1483 
1484                         if (_contentPosition == _contentLength) {
1485                             setState(State.END);
1486                             return handleContentMessage();
1487                         }
1488                     }
1489                     break;
1490                 }
1491 
1492                 case State.CHUNKED_CONTENT: {
1493                     ch = next(buffer);
1494                     if (ch > HttpTokens.SPACE) {
1495                         _chunkLength = ConverterUtils.convertHexDigit(ch);
1496                         _chunkPosition = 0;
1497                         setState(State.CHUNK_SIZE);
1498                     }
1499 
1500                     break;
1501                 }
1502 
1503                 case State.CHUNK_SIZE: {
1504                     ch = next(buffer);
1505                     if (ch == 0)
1506                         break;
1507                     if (ch == HttpTokens.LINE_FEED) {
1508                         if (_chunkLength == 0) {
1509                             setState(State.TRAILER);
1510                             if (_handler.contentComplete())
1511                                 return true;
1512                         } else
1513                             setState(State.CHUNK);
1514                     } else if (ch <= HttpTokens.SPACE || ch == HttpTokens.SEMI_COLON)
1515                         setState(State.CHUNK_PARAMS);
1516                     else
1517                         _chunkLength = _chunkLength * 16 + ConverterUtils.convertHexDigit(ch);
1518                     break;
1519                 }
1520 
1521                 case State.CHUNK_PARAMS: {
1522                     ch = next(buffer);
1523                     if (ch == HttpTokens.LINE_FEED) {
1524                         if (_chunkLength == 0) {
1525                             setState(State.TRAILER);
1526                             if (_handler.contentComplete())
1527                                 return true;
1528                         } else
1529                             setState(State.CHUNK);
1530                     }
1531                     break;
1532                 }
1533 
1534                 case State.CHUNK: {
1535                     int chunk = _chunkLength - _chunkPosition;
1536                     if (chunk == 0) {
1537                         setState(State.CHUNKED_CONTENT);
1538                     } else {
1539                         _contentChunk = buffer.slice(); // buffer.asReadOnlyBuffer();
1540 
1541                         if (remaining > chunk)
1542                             _contentChunk.limit(_contentChunk.position() + chunk);
1543                         chunk = _contentChunk.remaining();
1544 
1545                         _contentPosition += chunk;
1546                         _chunkPosition += chunk;
1547                         buffer.position(buffer.position() + chunk);
1548                         if (_handler.content(_contentChunk))
1549                             return true;
1550                     }
1551                     break;
1552                 }
1553 
1554                 case State.CLOSED: {
1555                     // BufferUtils.clear(buffer);
1556                     buffer.clear();
1557                     return false;
1558                 }
1559 
1560                 default:
1561                     break;
1562 
1563             }
1564 
1565             remaining = buffer.remaining();
1566         }
1567         return false;
1568     }
1569 
1570     /* ------------------------------------------------------------------------------- */
1571     bool isAtEOF()
1572     {
1573         return _eof;
1574     }
1575 
1576     /* ------------------------------------------------------------------------------- */
1577 
1578     /**
1579      * Signal that the associated data source is at EOF
1580      */
1581     void atEOF() {
1582         version(HUNT_HTTP_DEBUG)
1583             tracef("atEOF %s", this);
1584         _eof = true;
1585     }
1586 
1587     /* ------------------------------------------------------------------------------- */
1588 
1589     /**
1590      * Request that the associated data source be closed
1591      */
1592     void close() {
1593         version(HUNT_HTTP_DEBUG)
1594             tracef("close %s", this);
1595         setState(State.CLOSE);
1596     }
1597 
1598     /* ------------------------------------------------------------------------------- */
1599     void reset() {
1600         version(HUNT_HTTP_DEBUG_MORE)
1601             tracef("reset %s", this);
1602 
1603         // reset state
1604         if (_state == State.CLOSE || _state == State.CLOSED)
1605             return;
1606 
1607         setState(State.START);
1608         _endOfContent = EndOfContent.UNKNOWN_CONTENT;
1609         _contentLength = -1;
1610         _contentPosition = 0;
1611         _responseStatus = 0;
1612         _contentChunk = null;
1613         _headerBytes = 0;
1614         _host = false;
1615         _headerComplete = false;
1616     }
1617 
1618     /* ------------------------------------------------------------------------------- */
1619     protected void setState(State state) {
1620         // version(HUNT_HTTP_DEBUG)
1621         //     tracef("%s --> %s", _state, state);
1622         _state = state;
1623     
1624         version(HUNT_METRIC) {
1625             if(state == State.END) {
1626                 Duration timeElapsed = MonoTime.currTime - startTime;
1627                 warningf("parsing ended in: %d microseconds", timeElapsed.total!(TimeUnit.Microsecond)());
1628             }
1629         }
1630     }
1631 
1632     /* ------------------------------------------------------------------------------- */
1633     protected void setState(FieldState state) {
1634         // version(HUNT_HTTP_DEBUG)
1635         //     tracef("%s:%s --> %s", _state, _field, state);
1636         _fieldState = state;
1637     }
1638 
1639     /* ------------------------------------------------------------------------------- */
1640     Trie!HttpField getFieldCache() {
1641         return _fieldCache;
1642     }
1643 
1644     HttpField getCachedField(string name) {
1645         return _fieldCache.get(name);
1646     }
1647     
1648 
1649     /* ------------------------------------------------------------------------------- */
1650     override
1651     string toString() {
1652         return format("%s{s=%s,%d of %d}",
1653                 typeof(this).stringof,
1654                 _state,
1655                 _contentPosition,
1656                 _contentLength);
1657     }
1658 
1659     deprecated("Using HttpParsingHandler instead.")
1660     alias HttpHandler = HttpParsingHandler;
1661 
1662     deprecated("Using HttpRequestParsingHandler instead.")
1663     alias RequestHandler = HttpRequestParsingHandler;
1664 
1665     deprecated("Using HttpResponseParsingHandler instead.")
1666     alias ResponseHandler = HttpResponseParsingHandler;
1667 
1668     deprecated("Using ComplianceParsingHandler instead.")
1669     alias ComplianceHandler = ComplianceParsingHandler;
1670 
1671 
1672 
1673     // /* ------------------------------------------------------------------------------- */
1674     // /* ------------------------------------------------------------------------------- */
1675     // /* ------------------------------------------------------------------------------- */
1676     // interface ComplianceHandler : HttpHandler {
1677     //     // deprecated("")
1678     //     // default void onComplianceViolation(HttpCompliance compliance, HttpCompliance required, string reason) {
1679     //     // }
1680 
1681     //     void onComplianceViolation(HttpCompliance compliance, HttpComplianceSection v, string details);
1682     //     // {
1683     //     //     // onComplianceViolation(compliance, HttpCompliance.requiredCompliance(violation), details);
1684     //     //     warning(details);
1685     //     // }
1686     // }
1687 
1688     /* ------------------------------------------------------------------------------- */
1689     
1690     private static class IllegalCharacterException : BadMessageException {
1691         private this(State state, byte ch, ByteBuffer buffer) {
1692             super(400, format("Illegal character 0x%X", ch));
1693             // Bug #460642 - don't reveal buffers to end user
1694             warningf("Illegal character 0x%X in state=%s for buffer %s", ch, state, BufferUtils.toDetailString(buffer));
1695         }
1696     }
1697 }
1698 
1699 
1700 /* ------------------------------------------------------------ */
1701 /* ------------------------------------------------------------ */
1702 /* ------------------------------------------------------------ */
1703 /* Event producing interface while parsing http 
1704  * These methods return true if the caller should process the events
1705  * so far received (eg return from parseNext and call HttpChannel.handle).
1706  * If multiple callbacks are called in sequence (eg
1707  * headerComplete then messageComplete) from the same point in the parsing
1708  * then it is sufficient for the caller to process the events only once.
1709  */
1710 interface HttpParsingHandler {
1711 
1712     bool content(ByteBuffer item);
1713 
1714     bool headerComplete();
1715 
1716     bool contentComplete();
1717 
1718     bool messageComplete();
1719 
1720     /**
1721      * This is the method called by parser when a HTTP Header name and value is found
1722      *
1723      * @param field The field parsed
1724      */
1725     void parsedHeader(HttpField field);
1726 
1727     /**
1728      * This is the method called by parser when a HTTP Trailer name and value is found
1729      *
1730      * @param field The field parsed
1731      */
1732     void parsedTrailer(HttpField field);
1733 
1734     /* ------------------------------------------------------------ */
1735 
1736     /**
1737      * Called to signal that an EOF was received unexpectedly
1738      * during the parsing of a HTTP message
1739      */
1740     void earlyEOF();
1741 
1742     /* ------------------------------------------------------------ */
1743 
1744     /**
1745      * Called to signal that a bad HTTP message has been received.
1746      *
1747      * @param failure the failure with the bad message information
1748      */
1749     void badMessage(BadMessageException failure);
1750     // {
1751     //     // badMessage(failure.getCode(), failure.getReason());
1752     //     throw new BadMessageException(failure.getCode(), failure.getReason());
1753     // }
1754 
1755     /**
1756      * @deprecated use {@link #badMessage(BadMessageException)} instead
1757      */
1758     // deprecated("")
1759     void badMessage(int status, string reason);
1760 
1761     /* ------------------------------------------------------------ */
1762 
1763     /**
1764      * @return the size in bytes of the per parser header cache
1765      */
1766     int getHeaderCacheSize();
1767 
1768     string toString();
1769 }
1770 
1771 /* ------------------------------------------------------------------------------- */
1772 /* ------------------------------------------------------------------------------- */
1773 /* ------------------------------------------------------------------------------- */
1774 interface HttpRequestParsingHandler : HttpParsingHandler {
1775     /**
1776      * This is the method called by parser when the HTTP request line is parsed
1777      *
1778      * @param method  The method
1779      * @param uri     The raw bytes of the URI.  These are copied into a ByteBuffer that will not be changed until this parser is reset and reused.
1780      * @param version the http version in use
1781      * @return true if handling parsing should return.
1782      */
1783     bool startRequest(string method, string uri, HttpVersion ver);
1784 
1785 }
1786 
1787 /* ------------------------------------------------------------------------------- */
1788 /* ------------------------------------------------------------------------------- */
1789 /* ------------------------------------------------------------------------------- */
1790 interface HttpResponseParsingHandler : HttpParsingHandler {
1791     /**
1792      * This is the method called by parser when the HTTP request line is parsed
1793      *
1794      * @param version the http version in use
1795      * @param status  the response status
1796      * @param reason  the response reason phrase
1797      * @return true if handling parsing should return
1798      */
1799     bool startResponse(HttpVersion ver, int status, string reason);
1800 }
1801 
1802 /* ------------------------------------------------------------------------------- */
1803 /* ------------------------------------------------------------------------------- */
1804 /* ------------------------------------------------------------------------------- */
1805 interface ComplianceParsingHandler : HttpParsingHandler {
1806     // deprecated("")
1807     // default void onComplianceViolation(HttpCompliance compliance, HttpCompliance required, string reason) {
1808     // }
1809 
1810     void onComplianceViolation(HttpCompliance compliance, HttpComplianceSection v, string details);
1811     // {
1812     //     // onComplianceViolation(compliance, HttpCompliance.requiredCompliance(violation), details);
1813     //     warning(details);
1814     // }
1815 }
1816 
1817 deprecated("Using HttpRequestParsingHandler instead.")
1818 alias HttpRequestHandler = HttpRequestParsingHandler;
1819 
1820 deprecated("Using HttpResponseParsingHandler instead.")
1821 alias HttpResponseHandler = HttpResponseParsingHandler;