hunt.http.codec.http.decode.HttpParser source code

1 module hunt.http.codec.http.decode.HttpParser;
2 
3 import hunt.http.codec.http.model;
4 import hunt.http.codec.http.hpack.HpackEncoder;
5 
6 import hunt.http.HttpField;
7 import hunt.http.HttpHeader;
8 import hunt.http.HttpMethod;
9 import hunt.http.HttpRequest;
10 import hunt.http.HttpResponse;
11 import hunt.http.HttpStatus;
12 import hunt.http.HttpVersion;
13 import hunt.http.QuotedCSV;
14 
15 import hunt.collection;
16 import hunt.Exceptions;
17 import hunt.logging;
18 import hunt.text.Common;
19 import hunt.util.StringBuilder;
20 import hunt.util.ConverterUtils;
21 import hunt.util.DateTime;
22 
23 import core.time;
24 
25 import std.algorithm;
26 import std.array;
27 import std.concurrency : initOnce;
28 import std.container.array;
29 import std.conv;
30 import std.string;
31 
32 alias HttpParserState = HttpParser.State;
33 
34 private bool contains(T)(T[] items, T item) {
35     return items.canFind(item);
36 }
37 
38 
39 /* ------------------------------------------------------------ */
40 
41 /**
42  * A Parser for 1.0 and 1.1 as defined by RFC7230
43  * <p>
44  * This parser parses HTTP client and server messages from buffers
45  * passed in the {@link #parseNext(ByteBuffer)} method.  The parsed
46  * elements of the HTTP message are passed as event calls to the
47  * {@link HttpHandler} instance the parser is constructed with.
48  * If the passed handler is a {@link RequestHandler} then server side
49  * parsing is performed and if it is a {@link ResponseHandler}, then
50  * client side parsing is done.
51  * </p>
52  * <p>
53  * The contract of the {@link HttpHandler} API is that if a call returns
54  * true then the call to {@link #parseNext(ByteBuffer)} will return as
55  * soon as possible also with a true response.  Typically this indicates
56  * that the parsing has reached a stage where the caller should process
57  * the events accumulated by the handler.    It is the preferred calling
58  * style that handling such as calling a servlet to process a request,
59  * should be done after a true return from {@link #parseNext(ByteBuffer)}
60  * rather than from within the scope of a call like
61  * {@link RequestHandler#messageComplete()}
62  * </p>
63  * <p>
64  * For performance, the parse is heavily dependent on the
65  * {@link Trie#getBest(ByteBuffer, int, int)} method to look ahead in a
66  * single pass for both the structure ( : and CRLF ) and semantic (which
67  * header and value) of a header.  Specifically the static {@link HttpHeader#CACHE}
68  * is used to lookup common combinations of headers and values
69  * (eg. "Connection: close"), or just header names (eg. "Connection:" ).
70  * For headers who's value is not known statically (eg. Host, COOKIE) then a
71  * per parser dynamic Trie of {@link HttpFields} from previous parsed messages
72  * is used to help the parsing of subsequent messages.
73  * </p>
74  * <p>
75  * The parser can work in varying compliance modes:
76  * <dl>
77  * <dt>RFC7230</dt><dd>(default) Compliance with RFC7230</dd>
78  * <dt>RFC2616</dt><dd>Wrapped headers and HTTP/0.9 supported</dd>
79  * <dt>LEGACY</dt><dd>(aka STRICT) Adherence to Servlet Specification requirement for
80  * exact case of header names, bypassing the header caches, which are case insensitive,
81  * otherwise equivalent to RFC2616</dd>
82  * </dl>
83  *
84  * @see <a href="http://tools.ietf.org/html/rfc7230">RFC 7230</a>
85  */
86 class HttpParser {
87 
88     enum INITIAL_URI_LENGTH = 256;
89     private MonoTime startTime;
90 
91     /**
92      * Cache of common {@link HttpField}s including: <UL>
93      * <LI>Common static combinations such as:<UL>
94      * <li>Connection: close
95      * <li>Accept-Encoding: gzip
96      * <li>Content-Length: 0
97      * </ul>
98      * <li>Combinations of Content-Type header for common mime types by common charsets
99      * <li>Most common headers with null values so that a lookup will at least
100      * determine the header name even if the name:value combination is not cached
101      * </ul>
102      */
103     // __gshared Trie!HttpField CACHE;
104     static Trie!HttpField CACHE() {
105         __gshared Trie!HttpField inst;
106         return initOnce!inst(initFieldCache());
107     }
108 
109     // States
110     enum FieldState {
111         FIELD,
112         IN_NAME,
113         VALUE,
114         IN_VALUE,
115         WS_AFTER_NAME,
116     }
117 
118     // States
119     enum State {
120         START,
121         METHOD,
122         RESPONSE_VERSION,
123         SPACE1,
124         STATUS,
125         URI,
126         SPACE2,
127         REQUEST_VERSION,
128         REASON,
129         PROXY,
130         HEADER,
131         CONTENT,
132         EOF_CONTENT,
133         CHUNKED_CONTENT,
134         CHUNK_SIZE,
135         CHUNK_PARAMS,
136         CHUNK,
137         TRAILER,
138         END,
139         CLOSE,  // The associated stream/endpoint should be closed
140         CLOSED  // The associated stream/endpoint is at EOF
141     }
142 
143     private static State[] __idleStates = [State.START, State.END, State.CLOSE, State.CLOSED];
144     private static State[] __completeStates = [State.END, State.CLOSE, State.CLOSED];
145 
146     private HttpParsingHandler _handler;
147     private HttpRequestParsingHandler _requestHandler;
148     private HttpResponseParsingHandler _responseHandler;
149     private ComplianceParsingHandler _complianceHandler;
150     private int _maxHeaderBytes;
151     private HttpCompliance _compliance;
152     private HttpComplianceSection[] _compliances;
153     private HttpField _field;
154     private HttpHeader _header;
155     private string _headerString;
156     private string _valueString;
157     private int _responseStatus;
158     private int _headerBytes;
159     private bool _host;
160     private bool _headerComplete;
161 
162     /* ------------------------------------------------------------------------------- */
163     private  State _state = State.START;
164     private  FieldState _fieldState = FieldState.FIELD;
165     private  bool _eof;
166     private HttpMethod _method;
167     private string _methodString;
168     private HttpVersion _version;
169     private StringBuilder _uri;
170     private EndOfContent _endOfContent;
171     private long _contentLength = -1;
172     private long _contentPosition;
173     private int _chunkLength;
174     private int _chunkPosition;
175     private bool _headResponse;
176     private bool _cr;
177     private ByteBuffer _contentChunk;
178     private Trie!HttpField _fieldCache;
179 
180     private int _length;
181     private StringBuilder _string; 
182 
183     /* ------------------------------------------------------------------------------- */
184     this(HttpRequestParsingHandler handler) {
185         this(handler, -1, getCompliance());
186     }
187 
188     /* ------------------------------------------------------------------------------- */
189     this(HttpResponseParsingHandler handler) {
190         this(handler, -1, getCompliance());
191     }
192 
193     /* ------------------------------------------------------------------------------- */
194     this(HttpRequestParsingHandler handler, int maxHeaderBytes) {
195         this(handler, maxHeaderBytes, getCompliance());
196     }
197 
198     /* ------------------------------------------------------------------------------- */
199     this(HttpResponseParsingHandler handler, int maxHeaderBytes) {
200         this(handler, maxHeaderBytes, getCompliance());
201     }
202 
203     /* ------------------------------------------------------------------------------- */
204     this(HttpRequestParsingHandler handler, HttpCompliance compliance) {
205         this(handler, -1, compliance);
206     }
207 
208     /* ------------------------------------------------------------------------------- */
209     this(HttpRequestParsingHandler handler, int maxHeaderBytes, HttpCompliance compliance) {
210         this(handler, null, maxHeaderBytes, compliance is null ? getCompliance() : compliance);
211     }
212 
213     /* ------------------------------------------------------------------------------- */
214     this(HttpResponseParsingHandler handler, int maxHeaderBytes, HttpCompliance compliance) {
215         this(null, handler, maxHeaderBytes, compliance is null ? getCompliance() : compliance);
216     }
217 
218     /* ------------------------------------------------------------------------------- */
219     private this(HttpRequestParsingHandler requestHandler, HttpResponseParsingHandler responseHandler, 
220         int maxHeaderBytes, HttpCompliance compliance) {
221         version (HUNT_HTTP_DEBUG) {
222             trace("create http parser");
223         }
224         _string = new StringBuilder();
225         _uri = new StringBuilder(INITIAL_URI_LENGTH);
226         if(requestHandler !is null)
227             _handler = requestHandler;
228         else
229             _handler = responseHandler;
230         // _handler = requestHandler !is null ? cast()requestHandler : responseHandler;
231         _requestHandler = requestHandler;
232         _responseHandler = responseHandler;
233         _maxHeaderBytes = maxHeaderBytes;
234         _compliance = compliance;
235         _compliances = compliance.sections();
236         _complianceHandler = cast(ComplianceParsingHandler)_handler;
237     }
238 
239 
240     private static Trie!HttpField initFieldCache() {
241         Trie!HttpField cache = new ArrayTrie!HttpField(2048);
242         cache.put(new HttpField(HttpHeader.CONNECTION, HttpHeaderValue.CLOSE));
243         cache.put(new HttpField(HttpHeader.CONNECTION, HttpHeaderValue.KEEP_ALIVE));
244         cache.put(new HttpField(HttpHeader.CONNECTION, HttpHeaderValue.UPGRADE));
245         cache.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip"));
246         cache.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip, deflate"));
247         cache.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip, deflate, br"));
248         cache.put(new HttpField(HttpHeader.ACCEPT_ENCODING, "gzip,deflate,sdch"));
249         cache.put(new HttpField(HttpHeader.ACCEPT_LANGUAGE, "en-US,en;q=0.5"));
250         cache.put(new HttpField(HttpHeader.ACCEPT_LANGUAGE, "en-GB,en-US;q=0.8,en;q=0.6"));
251 
252         cache.put(new HttpField(HttpHeader.ACCEPT_LANGUAGE, 
253             "en-AU,en;q=0.9,it-IT;q=0.8,it;q=0.7,en-GB;q=0.6,en-US;q=0.5"));
254             
255         cache.put(new HttpField(HttpHeader.ACCEPT_CHARSET, "ISO-8859-1,utf-8;q=0.7,*;q=0.3"));
256         cache.put(new HttpField(HttpHeader.ACCEPT, "*/*"));
257         cache.put(new HttpField(HttpHeader.ACCEPT, "image/png,image/*;q=0.8,*/*;q=0.5"));
258         cache.put(new HttpField(HttpHeader.ACCEPT, 
259             "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"));
260 
261         cache.put(new HttpField(HttpHeader.ACCEPT, 
262             "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"));
263 
264         cache.put(new HttpField(HttpHeader.ACCEPT_RANGES, HttpHeaderValue.BYTES));
265         cache.put(new HttpField(HttpHeader.PRAGMA, "no-cache"));
266 
267         cache.put(new HttpField(HttpHeader.CACHE_CONTROL, 
268             "private, no-cache, no-cache=Set-Cookie, proxy-revalidate"));
269 
270         cache.put(new HttpField(HttpHeader.CACHE_CONTROL, "no-cache"));
271         cache.put(new HttpField(HttpHeader.CACHE_CONTROL, "max-age=0"));
272         cache.put(new HttpField(HttpHeader.CONTENT_LENGTH, "0"));
273         cache.put(new HttpField(HttpHeader.CONTENT_ENCODING, "gzip"));
274         cache.put(new HttpField(HttpHeader.CONTENT_ENCODING, "deflate"));
275         cache.put(new HttpField(HttpHeader.TRANSFER_ENCODING, "chunked"));
276         cache.put(new HttpField(HttpHeader.EXPIRES, "Fri, 01 Jan 1990 00:00:00 GMT"));
277 
278         // Add common Content types as fields
279         foreach (string type ; ["text/plain", "text/html", "text/xml", "text/json", 
280             "application/json", "application/x-www-form-urlencoded"]) {
281             HttpField field = new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type);
282             cache.put(field);
283 
284             foreach (string charset ; ["utf-8", "iso-8859-1"]) {
285                 cache.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ ";charset=" ~ charset));
286                 cache.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ "; charset=" ~ charset));
287                 cache.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ ";charset=" ~ charset.toUpper()));
288                 cache.put(new PreEncodedHttpField(HttpHeader.CONTENT_TYPE, type ~ "; charset=" ~ charset.toUpper()));
289             }
290         }
291 
292         // Add headers with null values so HttpParser can avoid looking up name again for unknown values
293         foreach (HttpHeader h ; HttpHeader.values()) {
294             // trace(h.toString());
295             if (!cache.put(new HttpField(h, cast(string) null))) {
296                 // FIXME: Needing refactor or cleanup -@zxp at 9/25/2018, 8:11:29 PM
297                 // 
298                 // warning(h.toString());
299                 // throw new IllegalStateException("CACHE FULL");
300             }
301         }
302 
303         // Add some more common headers
304         cache.put(new HttpField(HttpHeader.REFERER, cast(string) null));
305         cache.put(new HttpField(HttpHeader.IF_MODIFIED_SINCE, cast(string) null));
306         cache.put(new HttpField(HttpHeader.IF_NONE_MATCH, cast(string) null));
307         cache.put(new HttpField(HttpHeader.AUTHORIZATION, cast(string) null));
308         cache.put(new HttpField(HttpHeader.COOKIE, cast(string) null));
309 
310         return cache;
311     }
312 
313     private static HttpCompliance getCompliance() {
314         return HttpCompliance.RFC7230;
315     }
316 
317 
318     /* ------------------------------------------------------------------------------- */
319     HttpParsingHandler getHandler() {
320         return _handler;
321     }
322 
323     /* ------------------------------------------------------------------------------- */
324 
325     /**
326      * Check RFC compliance violation
327      *
328      * @param violation The compliance section violation
329      * @param reason    The reason for the violation
330      * @return True if the current compliance level is set so as to Not allow this violation
331      */
332     protected bool complianceViolation(HttpComplianceSection violation, string reason) {
333         if (_compliances.contains(violation))
334             return true;
335 
336         if (_complianceHandler !is null)
337             _complianceHandler.onComplianceViolation(_compliance, violation, reason);
338 
339         return false;
340     }
341 
342     /* ------------------------------------------------------------------------------- */
343     protected void handleViolation(HttpComplianceSection section, string reason) {
344         if (_complianceHandler !is null)
345             _complianceHandler.onComplianceViolation(_compliance, section, reason);
346     }
347 
348     /* ------------------------------------------------------------------------------- */
349     protected string caseInsensitiveHeader(string orig, string normative) {
350         if (_compliances.contains(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE))
351             return normative;
352         if (!orig.equals(normative))
353             handleViolation(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE, orig);
354         return orig;
355     }
356 
357     /* ------------------------------------------------------------------------------- */
358     long getContentLength() {
359         return _contentLength;
360     }
361 
362     /* ------------------------------------------------------------ */
363     long getContentRead() {
364         return _contentPosition;
365     }
366 
367     /* ------------------------------------------------------------ */
368 
369     /**
370      * Set if a HEAD response is expected
371      *
372      * @param head true if head response is expected
373      */
374     void setHeadResponse(bool head) {
375         _headResponse = head;
376     }
377 
378     /* ------------------------------------------------------------------------------- */
379     protected void setResponseStatus(int status) {
380         _responseStatus = status;
381     }
382 
383     /* ------------------------------------------------------------------------------- */
384     State getState() {
385         return _state;
386     }
387 
388     /* ------------------------------------------------------------------------------- */
389     bool inContentState() {
390         return _state >= State.CONTENT && _state < State.END;
391     }
392 
393     /* ------------------------------------------------------------------------------- */
394     bool inHeaderState() {
395         return _state < State.CONTENT;
396     }
397 
398     /* ------------------------------------------------------------------------------- */
399     bool isChunking() {
400         return _endOfContent == EndOfContent.CHUNKED_CONTENT;
401     }
402 
403     /* ------------------------------------------------------------ */
404     bool isStart() {
405         return isState(State.START);
406     }
407 
408     /* ------------------------------------------------------------ */
409     bool isClose() {
410         return isState(State.CLOSE);
411     }
412 
413     /* ------------------------------------------------------------ */
414     bool isClosed() {
415         return isState(State.CLOSED);
416     }
417 
418     /* ------------------------------------------------------------ */
419     bool isIdle() {
420         return __idleStates.contains(_state);
421     }
422 
423     /* ------------------------------------------------------------ */
424     bool isComplete() {
425         return __completeStates.contains(_state);
426     }
427 
428     /* ------------------------------------------------------------------------------- */
429     bool isState(State state) {
430         return _state == state;
431     }
432 
433     /* ------------------------------------------------------------------------------- */
434     enum CharState {
435         ILLEGAL, CR, LF, LEGAL
436     }
437 
438     private __gshared CharState[] __charState;
439 
440     shared static this() {
441         // token          = 1*tchar
442         // tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
443         //                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
444         //                / DIGIT / ALPHA
445         //                ; any VCHAR, except delimiters
446         // quoted-string  = DQUOTE *( qdtext / quoted-pair ) DQUOTE
447         // qdtext         = HTAB / SP /%x21 / %x23-5B / %x5D-7E / obs-text
448         // obs-text       = %x80-FF
449         // comment        = "(" *( ctext / quoted-pair / comment ) ")"
450         // ctext          = HTAB / SP / %x21-27 / %x2A-5B / %x5D-7E / obs-text
451         // quoted-pair    = "\" ( HTAB / SP / VCHAR / obs-text )
452 
453         __charState = new CharState[256];
454         __charState[0..$] = CharState.ILLEGAL;
455         // Arrays.fill(__charState, CharState.ILLEGAL);
456         __charState[HttpTokens.LINE_FEED] = CharState.LF;
457         __charState[HttpTokens.CARRIAGE_RETURN] = CharState.CR;
458         __charState[HttpTokens.TAB] = CharState.LEGAL;
459         __charState[HttpTokens.SPACE] = CharState.LEGAL;
460 
461         __charState['!'] = CharState.LEGAL;
462         __charState['#'] = CharState.LEGAL;
463         __charState['$'] = CharState.LEGAL;
464         __charState['%'] = CharState.LEGAL;
465         __charState['&'] = CharState.LEGAL;
466         __charState['\''] = CharState.LEGAL;
467         __charState['*'] = CharState.LEGAL;
468         __charState['+'] = CharState.LEGAL;
469         __charState['-'] = CharState.LEGAL;
470         __charState['.'] = CharState.LEGAL;
471         __charState['^'] = CharState.LEGAL;
472         __charState['_'] = CharState.LEGAL;
473         __charState['`'] = CharState.LEGAL;
474         __charState['|'] = CharState.LEGAL;
475         __charState['~'] = CharState.LEGAL;
476 
477         __charState['"'] = CharState.LEGAL;
478 
479         __charState['\\'] = CharState.LEGAL;
480         __charState['('] = CharState.LEGAL;
481         __charState[')'] = CharState.LEGAL;
482         __charState[0x21 .. 0x27 + 1] = CharState.LEGAL;
483         __charState[0x2A .. 0x5B + 1] = CharState.LEGAL;
484         __charState[0x5D .. 0x7E + 1] = CharState.LEGAL;
485         __charState[0x80 .. 0xFF + 1] = CharState.LEGAL;
486     }
487 
488     /* ------------------------------------------------------------------------------- */
489     private byte next(ByteBuffer buffer) {
490         byte ch = buffer.get();
491 
492         CharState s = __charState[0xff & ch];
493         switch (s) {
494             case CharState.ILLEGAL:
495                 throw new IllegalCharacterException(_state, ch, buffer);
496 
497             case CharState.LF:
498                 _cr = false;
499                 break;
500 
501             case CharState.CR:
502                 if (_cr)
503                     throw new BadMessageException("Bad EOL");
504 
505                 _cr = true;
506                 if (buffer.hasRemaining()) {
507                     // Don't count the CRs and LFs of the chunked encoding.
508                     if (_maxHeaderBytes > 0 && (_state == State.HEADER || _state == State.TRAILER))
509                         _headerBytes++;
510                     return next(buffer);
511                 }
512 
513                 // Can return 0 here to indicate the need for more characters,
514                 // because a real 0 in the buffer would cause a BadMessage below
515                 return 0;
516 
517             case CharState.LEGAL:
518                 if (_cr)
519                     throw new BadMessageException("Bad EOL");
520                 break;
521             
522             default:
523                 break;
524         }
525 
526         return ch;
527     }
528 
529     /* ------------------------------------------------------------------------------- */
530     /* Quick lookahead for the start state looking for a request method or a HTTP version,
531      * otherwise skip white space until something else to parse.
532      */
533     private bool quickStart(ByteBuffer buffer) {
534         if (_requestHandler !is null) {
535             _method = HttpMethod.lookAheadGet(buffer);
536             if (_method != HttpMethod.Null) {
537                 _methodString = _method.asString();
538                 buffer.position(cast(int)(buffer.position() + _methodString.length + 1));
539 
540                 setState(State.SPACE1);
541                 return false;
542             }
543         } else if (_responseHandler !is null) {
544             _version = HttpVersion.lookAheadGet(buffer);
545             if (_version != HttpVersion.Null) {
546                 buffer.position(cast(int) (buffer.position() + _version.asString().length + 1));
547                 setState(State.SPACE1);
548                 return false;
549             }
550         }
551 
552         // Quick start look
553         while (_state == State.START && buffer.hasRemaining()) {
554             int ch = next(buffer);
555 
556             if (ch > HttpTokens.SPACE) {
557                 _string.setLength(0);
558                 _string.append(cast(char) ch);
559                 setState(_requestHandler !is null ? State.METHOD : State.RESPONSE_VERSION);
560                 return false;
561             } else if (ch == 0)
562                 break;
563             else if (ch < 0)
564                 throw new BadMessageException();
565 
566             // count this white space as a header byte to avoid DOS
567             if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
568                 warningf("padding is too large >%d", _maxHeaderBytes);
569                 throw new BadMessageException(HttpStatus.BAD_REQUEST_400);
570             }
571         }
572         return false;
573     }
574 
575     /* ------------------------------------------------------------------------------- */
576     private void setString(string s) {
577         _string.setLength(0);
578         _string.append(s);
579         _length = cast(int)s.length;
580     }
581 
582     /* ------------------------------------------------------------------------------- */
583     private string takeString() {
584         _string.setLength(_length);
585         string s = _string.toString();
586         _string.setLength(0);
587         _length = -1;
588         return s;
589     }
590 
591     /* ------------------------------------------------------------------------------- */
592     private bool handleHeaderContentMessage() {
593         version (HUNT_HTTP_DEBUG_MORE) trace("handling headers ...");
594         bool handle_header = _handler.headerComplete();
595         _headerComplete = true;
596         version (HUNT_HTTP_DEBUG_MORE) trace("handling content ...");
597         bool handle_content = _handler.contentComplete();
598         version (HUNT_HTTP_DEBUG_MORE) trace("handling message ...");
599         bool handle_message = _handler.messageComplete();
600         return handle_header || handle_content || handle_message;
601     }
602 
603     /* ------------------------------------------------------------------------------- */
604     private bool handleContentMessage() {
605         bool handle_content = _handler.contentComplete();
606         bool handle_message = _handler.messageComplete();
607         return handle_content || handle_message;
608     }
609 
610     /* ------------------------------------------------------------------------------- */
611     /* Parse a request or response line
612      */
613     private bool parseLine(ByteBuffer buffer) {
614         bool handle = false;
615 
616         // Process headers
617         while (_state < State.HEADER && buffer.hasRemaining() && !handle) {
618             // process each character
619             byte b = next(buffer);
620             if (b == 0)
621                 break;
622 
623             if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
624                 if (_state == State.URI) {
625                     warningf("URI is too large >%d", _maxHeaderBytes);
626                     throw new BadMessageException(HttpStatus.URI_TOO_LONG_414);
627                 } else {
628                     if (_requestHandler !is null)
629                         warningf("request is too large >%d", _maxHeaderBytes);
630                     else
631                         warningf("response is too large >%d", _maxHeaderBytes);
632                     throw new BadMessageException(HttpStatus.REQUEST_HEADER_FIELDS_TOO_LARGE_431);
633                 }
634             }
635 
636             switch (_state) {
637                 case State.METHOD:
638                     if (b == HttpTokens.SPACE) {
639                         _length = _string.length;
640                         _methodString = takeString();
641 
642                         if (_compliances.contains(HttpComplianceSection.METHOD_CASE_SENSITIVE)) {
643                             HttpMethod method = HttpMethod.get(_methodString);
644                             if (method != HttpMethod.Null)
645                                 _methodString = method.asString();
646                         } else {
647                             HttpMethod method = HttpMethod.getInsensitive(_methodString);
648 
649                             if (method != HttpMethod.Null) {
650                                 if (method.asString() != (_methodString))
651                                     handleViolation(HttpComplianceSection.METHOD_CASE_SENSITIVE, _methodString);
652                                 _methodString = method.asString();
653                             }
654                         }
655 
656                         setState(State.SPACE1);
657                     } else if (b < HttpTokens.SPACE) {
658                         if (b == HttpTokens.LINE_FEED)
659                             throw new BadMessageException("No URI");
660                         else
661                             throw new IllegalCharacterException(_state, b, buffer);
662                     } else
663                         _string.append(cast(char) b);
664                     break;
665 
666                 case State.RESPONSE_VERSION:
667                     if (b == HttpTokens.SPACE) {
668                         _length = _string.length;
669                         string ver = takeString();
670                         _version = HttpVersion.fromString(ver);
671                         if (_version == HttpVersion.Null)
672                             throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Unknown Version");
673                         setState(State.SPACE1);
674                     } else if (b < HttpTokens.SPACE)
675                         throw new IllegalCharacterException(_state, b, buffer);
676                     else
677                         _string.append(cast(char) b);
678                     break;
679 
680                 case State.SPACE1:
681                     if (b > HttpTokens.SPACE || b < 0) {
682                         if (_responseHandler !is null) {
683                             setState(State.STATUS);
684                             setResponseStatus(b - '0');
685                         } else {
686                             _uri.reset();
687                             setState(State.URI);
688                             // quick scan for space or EoBuffer
689                             if (buffer.hasArray()) {
690                                 byte[] array = buffer.array();
691                                 int p = buffer.arrayOffset() + buffer.position();
692                                 int l = buffer.arrayOffset() + buffer.limit();
693                                 int i = p;
694                                 while (i < l && array[i] > HttpTokens.SPACE)
695                                     i++;
696 
697                                 int len = i - p;
698                                 _headerBytes += len;
699 
700                                 if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
701                                     warningf("URI is too large >%d", _maxHeaderBytes);
702                                     throw new BadMessageException(HttpStatus.URI_TOO_LONG_414);
703                                 }
704                                 _uri.append(array, p - 1, len + 1);
705                                 buffer.position(i - buffer.arrayOffset());
706                             } else
707                                 _uri.append(b);
708                         }
709                     } else if (b < HttpTokens.SPACE) {
710                         throw new BadMessageException(HttpStatus.BAD_REQUEST_400, 
711                             _requestHandler !is null ? "No URI" : "No Status");
712                     }
713                     break;
714 
715                 case State.STATUS:
716                     if (b == HttpTokens.SPACE) {
717                         setState(State.SPACE2);
718                     } else if (b >= '0' && b <= '9') {
719                         _responseStatus = _responseStatus * 10 + (b - '0');
720                     } else if (b < HttpTokens.SPACE && b >= 0) {
721                         setState(State.HEADER);
722                         handle = _responseHandler.startResponse(_version, _responseStatus, null) || handle;
723                     } else {
724                         throw new BadMessageException();
725                     }
726                     break;
727 
728                 case State.URI:
729                     if (b == HttpTokens.SPACE) {
730                         setState(State.SPACE2);
731                     } else if (b < HttpTokens.SPACE && b >= 0) {
732                         // HTTP/0.9
733                         if (complianceViolation(HttpComplianceSection.NO_HTTP_0_9, "No request version")) {
734                             throw new BadMessageException("HTTP/0.9 not supported");
735                         }
736 
737                         if(_requestHandler !is null) {
738                             handle = _requestHandler.startRequest(_methodString, _uri.toString(), HttpVersion.HTTP_0_9);
739                         } else {
740                             warning("no requestHandler defined");
741                         }
742 
743                         setState(State.END);
744                         BufferUtils.clear(buffer);
745                         handle = handleHeaderContentMessage() || handle;
746                     } else {
747                         _uri.append(b);
748                     }
749                     break;
750 
751                 case State.SPACE2:
752                     if (b > HttpTokens.SPACE) {
753                         _string.setLength(0);
754                         _string.append(cast(char) b);
755                         if (_responseHandler !is null) {
756                             _length = 1;
757                             setState(State.REASON);
758                         } else {
759                             setState(State.REQUEST_VERSION);
760 
761                             // try quick look ahead for HTTP Version
762                             HttpVersion ver;
763                             if (buffer.position() > 0 && buffer.hasArray()) {
764                                 ver = HttpVersion.lookAheadGet(buffer.array(), 
765                                     buffer.arrayOffset() + buffer.position() - 1, 
766                                     buffer.arrayOffset() + buffer.limit());
767                             } else {
768                                 string key = cast(string)buffer.peek(0, buffer.remaining()).idup;
769                                 ver = HttpVersion.fromString(key);
770                             }
771 
772                             if (ver != HttpVersion.Null) {
773                                 int pos = cast(int)(buffer.position() + ver.asString().length - 1);
774                                 if (pos < buffer.limit()) {
775                                     byte n = buffer.get(pos);
776                                     if (n == HttpTokens.CARRIAGE_RETURN) {
777                                         _cr = true;
778                                         _version = ver;
779                                         _string.setLength(0);
780                                         buffer.position(pos + 1);
781                                     } else if (n == HttpTokens.LINE_FEED) {
782                                         _version = ver;
783                                         _string.setLength(0);
784                                         buffer.position(pos);
785                                     }
786                                 }
787                             }
788                         }
789                     } else if (b == HttpTokens.LINE_FEED) {
790                         if (_responseHandler !is null) {
791                             setState(State.HEADER);
792                             handle = _responseHandler.startResponse(_version, _responseStatus, null) || handle;
793                         } else {
794                             // HTTP/0.9
795                             if (complianceViolation(HttpComplianceSection.NO_HTTP_0_9, "No request version"))
796                                 throw new BadMessageException("HTTP/0.9 not supported");
797                             
798                             if(_requestHandler is null) {
799                                 warning("no requestHandler defined");
800                             } else {
801                                 handle = _requestHandler.startRequest(_methodString, _uri.toString(), HttpVersion.HTTP_0_9);
802                             }
803                             
804                             setState(State.END);
805                             BufferUtils.clear(buffer);
806                             handle = handleHeaderContentMessage() || handle;
807                         }
808                     } else if (b < 0)
809                         throw new BadMessageException();
810                     break;
811 
812                 case State.REQUEST_VERSION:
813                     if (b == HttpTokens.LINE_FEED) {
814                         if (_version == HttpVersion.Null) {
815                             _length = _string.length;
816                             _version = HttpVersion.fromString(takeString());
817                         }
818                         if (_version == HttpVersion.Null)
819                             throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Unknown Version");
820 
821                         // Should we try to cache header fields?
822                         if (_fieldCache is null && 
823                             _version.getVersion() >= HttpVersion.HTTP_1_1.getVersion() && 
824                             _handler.getHeaderCacheSize() > 0) {
825                             int header_cache = _handler.getHeaderCacheSize();
826                             _fieldCache = new ArrayTernaryTrie!HttpField(header_cache);
827                         }
828 
829                         setState(State.HEADER);
830                         
831                         if(_requestHandler is null) {
832                             warning("no requestHandler defined");
833                         } else {
834                             handle = _requestHandler.startRequest(_methodString, _uri.toString(), _version) || handle;
835                         }
836                         continue;
837                     } else if (b >= HttpTokens.SPACE)
838                         _string.append(cast(char) b);
839                     else
840                         throw new BadMessageException();
841 
842                     break;
843 
844                 case State.REASON:
845                     if (b == HttpTokens.LINE_FEED) {
846                         string reason = takeString();
847                         setState(State.HEADER);
848                         handle = _responseHandler.startResponse(_version, _responseStatus, reason) || handle;
849                         continue;
850                     } else if (b >= HttpTokens.SPACE || ((b < 0) && (b >= -96))) {
851                         _string.append(cast(char) (0xff & b));
852                         if (b != ' ' && b != '\t')
853                             _length = _string.length;
854                     } else
855                         throw new BadMessageException();
856                     break;
857 
858                 default:
859                     throw new IllegalStateException(_state.to!string());
860             }
861         }
862 
863         return handle;
864     }
865 
866     private void parsedHeader() {
867         // handler last header if any.  Delayed to here just in case there was a continuation line (above)
868         if (!_headerString.empty() || !_valueString.empty()) {
869             // Handle known headers
870             version(HUNT_HTTP_DEBUG_MORE) {
871                 tracef("parsing header: %s, original name: %s, value: %s ", 
872                     _header.toString(), _headerString, _valueString);
873             }
874 
875             if (_header != HttpHeader.Null) {
876                 bool canAddToConnectionTrie = false;
877                 if(_header == HttpHeader.CONTENT_LENGTH) {
878                     if (_endOfContent == EndOfContent.CONTENT_LENGTH) {
879                         throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Duplicate Content-Length");
880                     } else if (_endOfContent != EndOfContent.CHUNKED_CONTENT) {
881                         _contentLength = convertContentLength(_valueString);
882                         if (_contentLength <= 0)
883                             _endOfContent = EndOfContent.NO_CONTENT;
884                         else
885                             _endOfContent = EndOfContent.CONTENT_LENGTH;
886                     }
887                 }
888                 else if(_header == HttpHeader.TRANSFER_ENCODING){
889                     if (HttpHeaderValue.CHUNKED.isSame(_valueString)) {
890                         _endOfContent = EndOfContent.CHUNKED_CONTENT;
891                         _contentLength = -1;
892                     } else {
893                         string[] values = new QuotedCSV(_valueString).getValues();
894                         if (values.length > 0 && HttpHeaderValue.CHUNKED.isSame(values[$ - 1])) {
895                             _endOfContent = EndOfContent.CHUNKED_CONTENT;
896                             _contentLength = -1;
897                         } else {
898                             foreach(string v; values) {
899                                 if(HttpHeaderValue.CHUNKED.isSame(v)) {
900                                     throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Bad chunking");
901                                 }
902                             }
903                         } 
904                     }
905                 }                    
906                 else if(_header == HttpHeader.HOST) {
907                     _host = true;
908                     if ((_field is null) && !_valueString.empty()) {
909                         string headerStr = _compliances.contains(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE) ? 
910                                     _header.asString() : _headerString;
911                         _field = new HostPortHttpField(_header, headerStr, _valueString);
912                         canAddToConnectionTrie = _fieldCache !is null;
913                     }
914                 }
915                 else if(_header == HttpHeader.CONNECTION) {
916                     // Don't cache headers if not persistent
917                     if (HttpHeaderValue.CLOSE.isSame(_valueString)) 
918                         _fieldCache = null;
919                     else {
920                         string[] values = new QuotedCSV(_valueString).getValues();
921                         foreach(string v; values) {
922                             if(HttpHeaderValue.CLOSE.isSame(v)) {
923                                 _fieldCache = null;
924                                 break;
925                             }
926                         }
927                     }
928                 }
929                 else if(_header == HttpHeader.AUTHORIZATION || _header == HttpHeader.ACCEPT ||
930                 _header == HttpHeader.ACCEPT_CHARSET || _header ==  HttpHeader.ACCEPT_ENCODING ||
931                 _header == HttpHeader.ACCEPT_LANGUAGE || _header == HttpHeader.COOKIE || 
932                 _header == HttpHeader.CACHE_CONTROL || _header == HttpHeader.USER_AGENT) {
933                     canAddToConnectionTrie = _fieldCache !is null && _field is null;
934                 }
935 
936                 if (canAddToConnectionTrie && !_fieldCache.isFull() 
937                     && _header != HttpHeader.Null && !_valueString.empty()) {
938                     if (_field is null)
939                         _field = new HttpField(_header, caseInsensitiveHeader(_headerString, _header.asString()), _valueString);
940                     _fieldCache.put(_field);
941                 }
942             }
943             _handler.parsedHeader(_field !is null ? _field : new HttpField(_header, _headerString, _valueString));
944         }
945 
946         _headerString = _valueString = null;
947         _header = HttpHeader.Null;
948         _field = null;
949     }
950 
951     private void parsedTrailer() {
952         // handler last header if any.  Delayed to here just in case there was a continuation line (above)
953         if (!_headerString.empty() || !_valueString.empty()) 
954             _handler.parsedTrailer(_field !is null ? _field : new HttpField(_header, _headerString, _valueString));
955 
956         _headerString = _valueString = null;
957         _header = HttpHeader.Null;
958         _field = null;
959     }
960 
961     private long convertContentLength(string valueString) {
962         try {
963             return to!long(valueString);
964         } catch (Exception e) {
965             warning("parse long exception: ", e);
966             throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Invalid Content-Length Value");
967         }
968     }
969 
970     /* ------------------------------------------------------------------------------- */
971     /*
972      * Parse the message headers and return true if the handler has signaled for a return
973      */
974     protected bool parseFields(ByteBuffer buffer) {
975         // Process headers
976         while ((_state == State.HEADER || _state == State.TRAILER) && buffer.hasRemaining()) {
977             // process each character
978             byte b = next(buffer);
979             if (b == 0)
980                 break;
981 
982             if (_maxHeaderBytes > 0 && ++_headerBytes > _maxHeaderBytes) {
983                 bool header = _state == State.HEADER;
984                 warningf("%s is too large %s>%s", header ? "Header" : "Trailer", _headerBytes, _maxHeaderBytes);
985                 throw new BadMessageException(header ?
986                         HttpStatus.REQUEST_HEADER_FIELDS_TOO_LARGE_431 :
987                         HttpStatus.PAYLOAD_TOO_LARGE_413);
988             }
989 
990             switch (_fieldState) {
991                 case FieldState.FIELD:
992                     switch (b) {
993                         case HttpTokens.COLON:
994                         case HttpTokens.SPACE:
995                         case HttpTokens.TAB: {
996                             if (complianceViolation(HttpComplianceSection.NO_FIELD_FOLDING, _headerString))
997                                 throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "Header Folding");
998 
999                             // header value without name - continuation?
1000                             if (_valueString == null || _valueString.empty()) {
1001                                 _string.setLength(0);
1002                                 _length = 0;
1003                             } else {
1004                                 setString(_valueString);
1005                                 _string.append(' ');
1006                                 _length++;
1007                                 _valueString = null;
1008                             }
1009                             setState(FieldState.VALUE);
1010                             break;
1011                         }
1012 
1013                         case HttpTokens.LINE_FEED: {
1014                             // process previous header
1015                             if (_state == State.HEADER)
1016                                 parsedHeader();
1017                             else
1018                                 parsedTrailer();
1019 
1020                             _contentPosition = 0;
1021 
1022                             // End of headers or trailers?
1023                             if (_state == State.TRAILER) {
1024                                 setState(State.END);
1025                                 return _handler.messageComplete();
1026                             }
1027 
1028                             // Was there a required host header?
1029                             if (!_host && _version == HttpVersion.HTTP_1_1 && _requestHandler !is null) {
1030                                 throw new BadMessageException(HttpStatus.BAD_REQUEST_400, "No Host");
1031                             }
1032 
1033                             // is it a response that cannot have a body?
1034                             if (_responseHandler !is null && // response
1035                                     (_responseStatus == 304 || // not-modified response
1036                                             _responseStatus == 204 || // no-content response
1037                                             _responseStatus < 200)) // 1xx response
1038                                 _endOfContent = EndOfContent.NO_CONTENT; // ignore any other headers set
1039 
1040                                 // else if we don't know framing
1041                             else if (_endOfContent == EndOfContent.UNKNOWN_CONTENT) {
1042                                 if (_responseStatus == 0  // request
1043                                         || _responseStatus == 304 // not-modified response
1044                                         || _responseStatus == 204 // no-content response
1045                                         || _responseStatus < 200) // 1xx response
1046                                     _endOfContent = EndOfContent.NO_CONTENT;
1047                                 else
1048                                     _endOfContent = EndOfContent.EOF_CONTENT;
1049                             }
1050 
1051                             // How is the message ended?
1052                             switch (_endOfContent) {
1053                                 case EndOfContent.EOF_CONTENT: {
1054                                     setState(State.EOF_CONTENT);
1055                                     bool handle = _handler.headerComplete();
1056                                     _headerComplete = true;
1057                                     return handle;
1058                                 }
1059                                 case EndOfContent.CHUNKED_CONTENT: {
1060                                     setState(State.CHUNKED_CONTENT);
1061                                     bool handle = _handler.headerComplete();
1062                                     _headerComplete = true;
1063                                     return handle;
1064                                 }
1065                                 case EndOfContent.NO_CONTENT: {
1066                                     version (HUNT_HTTP_DEBUG) trace("parsing done for no content");
1067                                     setState(State.END);
1068                                     return handleHeaderContentMessage();
1069                                 }
1070                                 default: {
1071                                     setState(State.CONTENT);
1072                                     bool handle = _handler.headerComplete();
1073                                     _headerComplete = true;
1074                                     return handle;
1075                                 }
1076                             }
1077                         }
1078 
1079                         default: {
1080                             // process previous header
1081                             if (_state == State.HEADER)
1082                                 parsedHeader();
1083                             else
1084                                 parsedTrailer();
1085 
1086                             // handle new header
1087                             if (buffer.hasRemaining()) {
1088                                 // Try a look ahead for the known header name and value.
1089                                 HttpField cached_field = null;
1090                                 if(_fieldCache !is null)
1091                                     cached_field = _fieldCache.getBest(buffer, -1, buffer.remaining());
1092                                 // TODO: Tasks pending completion -@zxp at 10/23/2018, 8:03:47 PM
1093                                 // Can't handle Sec-WebSocket-Key
1094                                 // if (cached_field is null)
1095                                 //     cached_field = CACHE.getBest(buffer, -1, buffer.remaining());
1096 
1097                                 if (cached_field !is null) {
1098                                     string n = cached_field.getName();
1099                                     string v = cached_field.getValue();
1100 
1101                                     if (!_compliances.contains(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE)) {
1102                                         // Have to get the fields exactly from the buffer to match case
1103                                          // BufferUtils.toString(buffer, buffer.position() - 1, n.length, StandardCharsets.US_ASCII);
1104                                         string en = cast(string)buffer.peek(buffer.position() - 1, cast(int)n.length).idup;
1105                                         if (!n.equals(en)) {
1106                                             handleViolation(HttpComplianceSection.FIELD_NAME_CASE_INSENSITIVE, en);
1107                                             n = en;
1108                                             cached_field = new HttpField(cached_field.getHeader(), n, v);
1109                                         }
1110                                     }
1111 
1112                                     if (v != null && !_compliances.contains(HttpComplianceSection.CASE_INSENSITIVE_FIELD_VALUE_CACHE)) {
1113                                         // BufferUtils.toString(buffer, buffer.position() + n.length + 1, v.length, StandardCharsets.ISO_8859_1);
1114                                         string ev = cast(string)buffer.peek(buffer.position() + n.length + 1, v.length).idup;
1115                                         if (!v.equals(ev)) {
1116                                             handleViolation(HttpComplianceSection.CASE_INSENSITIVE_FIELD_VALUE_CACHE, ev ~ "!=" ~ v);
1117                                             v = ev;
1118                                             cached_field = new HttpField(cached_field.getHeader(), n, v);
1119                                         }
1120                                     }
1121 
1122                                     _header = cached_field.getHeader();
1123                                     _headerString = n;
1124 
1125                                     if (v == null) {
1126                                         // Header only
1127                                         setState(FieldState.VALUE);
1128                                         _string.setLength(0);
1129                                         _length = 0;
1130                                         buffer.position(cast(int)(buffer.position() + n.length + 1));
1131                                         break;
1132                                     } else {
1133                                         // Header and value
1134                                         int pos = cast(int) (buffer.position() + n.length + v.length + 1);
1135                                         byte peek = buffer.get(pos);
1136 
1137                                         if (peek == HttpTokens.CARRIAGE_RETURN || peek == HttpTokens.LINE_FEED) {
1138                                             _field = cached_field;
1139                                             _valueString = v;
1140                                             setState(FieldState.IN_VALUE);
1141 
1142                                             if (peek == HttpTokens.CARRIAGE_RETURN) {
1143                                                 _cr = true;
1144                                                 buffer.position(pos + 1);
1145                                             } else
1146                                                 buffer.position(pos);
1147                                             break;
1148                                         } else {
1149                                             setState(FieldState.IN_VALUE);
1150                                             setString(v);
1151                                             buffer.position(pos);
1152                                             break;
1153                                         }
1154                                     }
1155                                 }
1156                             }
1157 
1158                             // New header
1159                             setState(FieldState.IN_NAME);
1160                             _string.setLength(0);
1161                             _string.append(cast(char) b);
1162                             _length = 1;
1163                         }
1164                     }
1165                     break;
1166 
1167                 case FieldState.IN_NAME:
1168                     switch (b) {
1169                         case HttpTokens.SPACE:
1170                         case HttpTokens.TAB:
1171                             //Ignore trailing whitespaces ?
1172                             if (!complianceViolation(HttpComplianceSection.NO_WS_AFTER_FIELD_NAME, null)) {
1173                                 _headerString = takeString();
1174                                 _header = HttpHeader.get(_headerString);
1175                                 _length = -1;
1176                                 setState(FieldState.WS_AFTER_NAME);
1177                                 break;
1178                             }
1179                             throw new IllegalCharacterException(_state, b, buffer);
1180 
1181                         case HttpTokens.COLON:
1182                             _headerString = takeString();
1183                             _header = HttpHeader.get(_headerString);
1184                             _length = -1;
1185                             setState(FieldState.VALUE);
1186                             break;
1187 
1188                         case HttpTokens.LINE_FEED:
1189                             _headerString = takeString();
1190                             _header = HttpHeader.get(_headerString);
1191                             _string.setLength(0);
1192                             _valueString = "";
1193                             _length = -1;
1194 
1195                             if (!complianceViolation(HttpComplianceSection.FIELD_COLON, _headerString)) {
1196                                 setState(FieldState.FIELD);
1197                                 break;
1198                             }
1199                             throw new IllegalCharacterException(_state, b, buffer);
1200 
1201                         default:
1202                             if (b < 0)
1203                                 throw new IllegalCharacterException(_state, b, buffer);
1204 
1205                             _string.append(cast(char) b);
1206                             _length = _string.length;
1207                             break;
1208                     }
1209                     break;
1210 
1211                 case FieldState.WS_AFTER_NAME:
1212 
1213                     switch (b) {
1214                         case HttpTokens.SPACE:
1215                         case HttpTokens.TAB:
1216                             break;
1217 
1218                         case HttpTokens.COLON:
1219                             setState(FieldState.VALUE);
1220                             break;
1221 
1222                         case HttpTokens.LINE_FEED:
1223                             if (!complianceViolation(HttpComplianceSection.FIELD_COLON, _headerString)) {
1224                                 setState(FieldState.FIELD);
1225                                 break;
1226                             }
1227                             throw new IllegalCharacterException(_state, b, buffer);
1228 
1229                         default:
1230                             throw new IllegalCharacterException(_state, b, buffer);
1231                     }
1232                     break;
1233 
1234                 case FieldState.VALUE:
1235                     switch (b) {
1236                         case HttpTokens.LINE_FEED:
1237                             _string.setLength(0);
1238                             _valueString = "";
1239                             _length = -1;
1240 
1241                             setState(FieldState.FIELD);
1242                             break;
1243 
1244                         case HttpTokens.SPACE:
1245                         case HttpTokens.TAB:
1246                             break;
1247 
1248                         default:
1249                             _string.append(cast(char) (0xff & b));
1250                             _length = _string.length;
1251                             setState(FieldState.IN_VALUE);
1252                             break;
1253                     }
1254                     break;
1255 
1256                 case FieldState.IN_VALUE:
1257                     switch (b) {
1258                         case HttpTokens.LINE_FEED:
1259                             if (_length > 0) {
1260                                 _valueString = takeString();
1261                                 _length = -1;
1262                             }
1263                             setState(FieldState.FIELD);
1264                             break;
1265 
1266                         case HttpTokens.SPACE:
1267                         case HttpTokens.TAB:
1268                             _string.append(cast(char) (0xff & b));
1269                             break;
1270 
1271                         default:
1272                             _string.append(cast(char) (0xff & b));
1273                             _length = _string.length;
1274                             break;
1275                     }
1276                     break;
1277 
1278                 default:
1279                     throw new IllegalStateException(_state.to!string());
1280 
1281             }
1282         }
1283 
1284         return false;
1285     }
1286 
1287     /* ------------------------------------------------------------------------------- */
1288 
1289     /**
1290      * Parse until next Event.
1291      *
1292      * @param buffer the buffer to parse
1293      * @return True if an {@link HttpRequestParsingHandler} method was called and it returned true;
1294      */
1295     bool parseNext(ByteBuffer buffer) {
1296 
1297         version(HUNT_HTTP_DEBUG_MORE) {
1298             tracef("parseNext s=%s %s", _state, BufferUtils.toDetailString(buffer));
1299             // tracef("buffer: %s", BufferUtils.toHexString(buffer));
1300 
1301             // startTime = MonoTime.currTime;
1302             // _requestHandler.startRequest("GET", "/plaintext", HttpVersion.HTTP_1_1);
1303             // setState(State.END);
1304             // _handler.messageComplete();
1305             // BufferUtils.clear(buffer);
1306             // return false;
1307         }
1308         try {
1309             // Start a request/response
1310             if (_state == State.START) {
1311                 version(HUNT_METRIC) {
1312                     startTime = MonoTime.currTime;
1313                     info("start a new parsing process...");
1314                 }                
1315                 _version = HttpVersion.Null;
1316                 _method = HttpMethod.Null;
1317                 _methodString = null;
1318                 _endOfContent = EndOfContent.UNKNOWN_CONTENT;
1319                 _header = HttpHeader.Null;
1320                 if (quickStart(buffer))
1321                     return true;
1322             }
1323 
1324             // Request/response line
1325             if (_state >= State.START && _state < State.HEADER) {
1326                 if (parseLine(buffer)) {
1327                     version(HUNT_HTTP_DEBUG) tracef("after parseLine =>%s", buffer.toString());
1328                     // return true;
1329                 }
1330             }
1331 
1332             // parse headers
1333             if (_state == State.HEADER) {
1334                 if (parseFields(buffer)) {
1335                     version(HUNT_HTTP_DEBUG_MORE) tracef("after parseFields =>%s", buffer.toString());
1336                     return true;
1337                 }
1338             }
1339 
1340             // parse content
1341             if (_state >= State.CONTENT && _state < State.TRAILER) {
1342                 // Handle HEAD response
1343                 if (_responseStatus > 0 && _headResponse) {
1344                     setState(State.END);
1345                     return handleContentMessage();
1346                 } else {
1347                     if (parseContent(buffer))
1348                         return true;
1349                 }
1350             }
1351 
1352             // parse headers
1353             if (_state == State.TRAILER) {
1354                 if (parseFields(buffer))
1355                     return true;
1356             }
1357 
1358             // handle end states
1359             if (_state == State.END) {
1360                 // eat white space
1361                 while (buffer.remaining() > 0 && buffer.get(buffer.position()) <= HttpTokens.SPACE)
1362                     buffer.get();
1363             } else if (isClose() || isClosed()) {
1364                 BufferUtils.clear(buffer);
1365             }
1366 
1367             // Handle EOF
1368             if (_eof && !buffer.hasRemaining()) {
1369                 switch (_state) {
1370                     case State.CLOSED:
1371                         break;
1372 
1373                     case State.START:
1374                         setState(State.CLOSED);
1375                         _handler.earlyEOF();
1376                         break;
1377 
1378                     case State.END:
1379                     case State.CLOSE:
1380                         setState(State.CLOSED);
1381                         break;
1382 
1383                     case State.EOF_CONTENT:
1384                     case State.TRAILER:
1385                         if (_fieldState == FieldState.FIELD) {
1386                             // Be forgiving of missing last CRLF
1387                             setState(State.CLOSED);
1388                             return handleContentMessage();
1389                         }
1390                         setState(State.CLOSED);
1391                         _handler.earlyEOF();
1392                         break;
1393 
1394                     case State.CONTENT:
1395                     case State.CHUNKED_CONTENT:
1396                     case State.CHUNK_SIZE:
1397                     case State.CHUNK_PARAMS:
1398                     case State.CHUNK:
1399                         setState(State.CLOSED);
1400                         _handler.earlyEOF();
1401                         break;
1402 
1403                     default:
1404                         version(HUNT_HTTP_DEBUG)
1405                             tracef("%s EOF in %s", this, _state);
1406                         setState(State.CLOSED);
1407                         _handler.badMessage(new BadMessageException(HttpStatus.BAD_REQUEST_400));
1408                         break;
1409                 }
1410             }
1411         } catch (BadMessageException x) {
1412             BufferUtils.clear(buffer);
1413             badMessage(x);
1414         } catch (Exception x) {
1415             version(HUNT_DEBUG) warning(x.msg);
1416             version(HUNT_HTTP_DEBUG) warning(x);
1417             BufferUtils.clear(buffer);
1418             badMessage(new BadMessageException(HttpStatus.BAD_REQUEST_400, 
1419                 _requestHandler !is null ? "Bad Request" : "Bad Response", x));
1420         }
1421         return false;
1422     }
1423 
1424     protected void badMessage(BadMessageException x) {
1425         version(HUNT_DEBUG)
1426             warning("Parse exception: " ~ this.toString() ~ " for " ~ _handler.toString() ~ 
1427                 ", Exception: ", x.msg);
1428 
1429         version(HUNT_HTTP_DEBUG) {
1430             Throwable t = x;
1431             while((t = t.next) !is null) {
1432                 error(t.msg);
1433             }
1434         }
1435         
1436         setState(State.CLOSE);
1437         if (_headerComplete)
1438             _handler.earlyEOF();
1439         else
1440             _handler.badMessage(x);
1441     }
1442 
1443     protected bool parseContent(ByteBuffer buffer) {
1444         int remaining = buffer.remaining();
1445         if (remaining == 0 && _state == State.CONTENT) {
1446             long content = _contentLength - _contentPosition;
1447             if (content == 0) {
1448                 setState(State.END);
1449                 return handleContentMessage();
1450             }
1451         }
1452 
1453         // Handle _content
1454         byte ch;
1455         while (_state < State.TRAILER && remaining > 0) {
1456             switch (_state) {
1457                 case State.EOF_CONTENT:
1458                     _contentChunk = buffer.slice(); // buffer.asReadOnlyBuffer();
1459                     _contentPosition += remaining;
1460                     buffer.position(buffer.position() + remaining);
1461                     if (_handler.content(_contentChunk))
1462                         return true;
1463                     break;
1464 
1465                 case State.CONTENT: {
1466                     long content = _contentLength - _contentPosition;
1467                     if (content == 0) {
1468                         setState(State.END);
1469                         return handleContentMessage();
1470                     } else {
1471                         _contentChunk = buffer.slice(); // buffer.asReadOnlyBuffer();
1472 
1473                         // limit content by expected size
1474                         if (remaining > content) {
1475                             // We can cast remaining to an int as we know that it is smaller than
1476                             // or equal to length which is already an int.
1477                             _contentChunk.limit(_contentChunk.position() + cast(int) content);
1478                         }
1479 
1480                         _contentPosition += _contentChunk.remaining();
1481                         buffer.position(buffer.position() + _contentChunk.remaining());
1482                         version(HUNT_HTTP_DEBUG) trace("setting content...");
1483                         if (_handler.content(_contentChunk))
1484                             return true;
1485 
1486                         if (_contentPosition == _contentLength) {
1487                             setState(State.END);
1488                             return handleContentMessage();
1489                         }
1490                     }
1491                     break;
1492                 }
1493 
1494                 case State.CHUNKED_CONTENT: {
1495                     ch = next(buffer);
1496                     if (ch > HttpTokens.SPACE) {
1497                         _chunkLength = ConverterUtils.convertHexDigit(ch);
1498                         _chunkPosition = 0;
1499                         setState(State.CHUNK_SIZE);
1500                     }
1501 
1502                     break;
1503                 }
1504 
1505                 case State.CHUNK_SIZE: {
1506                     ch = next(buffer);
1507                     if (ch == 0)
1508                         break;
1509                     if (ch == HttpTokens.LINE_FEED) {
1510                         if (_chunkLength == 0) {
1511                             setState(State.TRAILER);
1512                             if (_handler.contentComplete())
1513                                 return true;
1514                         } else
1515                             setState(State.CHUNK);
1516                     } else if (ch <= HttpTokens.SPACE || ch == HttpTokens.SEMI_COLON)
1517                         setState(State.CHUNK_PARAMS);
1518                     else
1519                         _chunkLength = _chunkLength * 16 + ConverterUtils.convertHexDigit(ch);
1520                     break;
1521                 }
1522 
1523                 case State.CHUNK_PARAMS: {
1524                     ch = next(buffer);
1525                     if (ch == HttpTokens.LINE_FEED) {
1526                         if (_chunkLength == 0) {
1527                             setState(State.TRAILER);
1528                             if (_handler.contentComplete())
1529                                 return true;
1530                         } else
1531                             setState(State.CHUNK);
1532                     }
1533                     break;
1534                 }
1535 
1536                 case State.CHUNK: {
1537                     int chunk = _chunkLength - _chunkPosition;
1538                     if (chunk == 0) {
1539                         setState(State.CHUNKED_CONTENT);
1540                     } else {
1541                         _contentChunk = buffer.slice(); // buffer.asReadOnlyBuffer();
1542 
1543                         if (remaining > chunk)
1544                             _contentChunk.limit(_contentChunk.position() + chunk);
1545                         chunk = _contentChunk.remaining();
1546 
1547                         _contentPosition += chunk;
1548                         _chunkPosition += chunk;
1549                         buffer.position(buffer.position() + chunk);
1550                         if (_handler.content(_contentChunk))
1551                             return true;
1552                     }
1553                     break;
1554                 }
1555 
1556                 case State.CLOSED: {
1557                     // BufferUtils.clear(buffer);
1558                     buffer.clear();
1559                     return false;
1560                 }
1561 
1562                 default:
1563                     break;
1564 
1565             }
1566 
1567             remaining = buffer.remaining();
1568         }
1569         return false;
1570     }
1571 
1572     /* ------------------------------------------------------------------------------- */
1573     bool isAtEOF()
1574     {
1575         return _eof;
1576     }
1577 
1578     /* ------------------------------------------------------------------------------- */
1579 
1580     /**
1581      * Signal that the associated data source is at EOF
1582      */
1583     void atEOF() {
1584         version(HUNT_HTTP_DEBUG)
1585             tracef("atEOF %s", this);
1586         _eof = true;
1587     }
1588 
1589     /* ------------------------------------------------------------------------------- */
1590 
1591     /**
1592      * Request that the associated data source be closed
1593      */
1594     void close() {
1595         version(HUNT_HTTP_DEBUG)
1596             tracef("close %s", this);
1597         setState(State.CLOSE);
1598     }
1599 
1600     /* ------------------------------------------------------------------------------- */
1601     void reset() {
1602         version(HUNT_HTTP_DEBUG_MORE)
1603             tracef("reset %s", this);
1604 
1605         // reset state
1606         if (_state == State.CLOSE || _state == State.CLOSED)
1607             return;
1608 
1609         setState(State.START);
1610         _endOfContent = EndOfContent.UNKNOWN_CONTENT;
1611         _contentLength = -1;
1612         _contentPosition = 0;
1613         _responseStatus = 0;
1614         _contentChunk = null;
1615         _headerBytes = 0;
1616         _host = false;
1617         _headerComplete = false;
1618     }
1619 
1620     /* ------------------------------------------------------------------------------- */
1621     protected void setState(State state) {
1622         version(HUNT_HTTP_DEBUG)
1623             tracef("%s --> %s", _state, state);
1624         _state = state;
1625     
1626         version(HUNT_METRIC) {
1627             if(state == State.END) {
1628                 Duration timeElapsed = MonoTime.currTime - startTime;
1629                 warningf("parsing ended in: %d microseconds", timeElapsed.total!(TimeUnit.Microsecond)());
1630             }
1631         }
1632     }
1633 
1634     /* ------------------------------------------------------------------------------- */
1635     protected void setState(FieldState state) {
1636         // version(HUNT_HTTP_DEBUG)
1637         //     tracef("%s:%s --> %s", _state, _field, state);
1638         _fieldState = state;
1639     }
1640 
1641     /* ------------------------------------------------------------------------------- */
1642     Trie!HttpField getFieldCache() {
1643         return _fieldCache;
1644     }
1645 
1646     HttpField getCachedField(string name) {
1647         return _fieldCache.get(name);
1648     }
1649     
1650 
1651     /* ------------------------------------------------------------------------------- */
1652     override
1653     string toString() {
1654         return format("%s{s=%s,%d of %d}",
1655                 typeof(this).stringof,
1656                 _state,
1657                 _contentPosition,
1658                 _contentLength);
1659     }
1660 
1661     deprecated("Using HttpParsingHandler instead.")
1662     alias HttpHandler = HttpParsingHandler;
1663 
1664     deprecated("Using HttpRequestParsingHandler instead.")
1665     alias RequestHandler = HttpRequestParsingHandler;
1666 
1667     deprecated("Using HttpResponseParsingHandler instead.")
1668     alias ResponseHandler = HttpResponseParsingHandler;
1669 
1670     deprecated("Using ComplianceParsingHandler instead.")
1671     alias ComplianceHandler = ComplianceParsingHandler;
1672 
1673 
1674 
1675     // /* ------------------------------------------------------------------------------- */
1676     // /* ------------------------------------------------------------------------------- */
1677     // /* ------------------------------------------------------------------------------- */
1678     // interface ComplianceHandler : HttpHandler {
1679     //     // deprecated("")
1680     //     // default void onComplianceViolation(HttpCompliance compliance, HttpCompliance required, string reason) {
1681     //     // }
1682 
1683     //     void onComplianceViolation(HttpCompliance compliance, HttpComplianceSection v, string details);
1684     //     // {
1685     //     //     // onComplianceViolation(compliance, HttpCompliance.requiredCompliance(violation), details);
1686     //     //     warning(details);
1687     //     // }
1688     // }
1689 
1690     /* ------------------------------------------------------------------------------- */
1691     
1692     private static class IllegalCharacterException : BadMessageException {
1693         private this(State state, byte ch, ByteBuffer buffer) {
1694             super(400, format("Illegal character 0x%X", ch));
1695             // Bug #460642 - don't reveal buffers to end user
1696             warningf("Illegal character 0x%X in state=%s for buffer %s", ch, state, BufferUtils.toDetailString(buffer));
1697         }
1698     }
1699 }
1700 
1701 
1702 /* ------------------------------------------------------------ */
1703 /* ------------------------------------------------------------ */
1704 /* ------------------------------------------------------------ */
1705 /* Event producing interface while parsing http 
1706  * These methods return true if the caller should process the events
1707  * so far received (eg return from parseNext and call HttpChannel.handle).
1708  * If multiple callbacks are called in sequence (eg
1709  * headerComplete then messageComplete) from the same point in the parsing
1710  * then it is sufficient for the caller to process the events only once.
1711  */
1712 interface HttpParsingHandler {
1713 
1714     bool content(ByteBuffer item);
1715 
1716     bool headerComplete();
1717 
1718     bool contentComplete();
1719 
1720     bool messageComplete();
1721 
1722     /**
1723      * This is the method called by parser when a HTTP Header name and value is found
1724      *
1725      * @param field The field parsed
1726      */
1727     void parsedHeader(HttpField field);
1728 
1729     /**
1730      * This is the method called by parser when a HTTP Trailer name and value is found
1731      *
1732      * @param field The field parsed
1733      */
1734     void parsedTrailer(HttpField field);
1735 
1736     /* ------------------------------------------------------------ */
1737 
1738     /**
1739      * Called to signal that an EOF was received unexpectedly
1740      * during the parsing of a HTTP message
1741      */
1742     void earlyEOF();
1743 
1744     /* ------------------------------------------------------------ */
1745 
1746     /**
1747      * Called to signal that a bad HTTP message has been received.
1748      *
1749      * @param failure the failure with the bad message information
1750      */
1751     void badMessage(BadMessageException failure);
1752     // {
1753     //     // badMessage(failure.getCode(), failure.getReason());
1754     //     throw new BadMessageException(failure.getCode(), failure.getReason());
1755     // }
1756 
1757     /**
1758      * @deprecated use {@link #badMessage(BadMessageException)} instead
1759      */
1760     // deprecated("")
1761     void badMessage(int status, string reason);
1762 
1763     /* ------------------------------------------------------------ */
1764 
1765     /**
1766      * @return the size in bytes of the per parser header cache
1767      */
1768     int getHeaderCacheSize();
1769 
1770     string toString();
1771 }
1772 
1773 /* ------------------------------------------------------------------------------- */
1774 /* ------------------------------------------------------------------------------- */
1775 /* ------------------------------------------------------------------------------- */
1776 interface HttpRequestParsingHandler : HttpParsingHandler {
1777     /**
1778      * This is the method called by parser when the HTTP request line is parsed
1779      *
1780      * @param method  The method
1781      * @param uri     The raw bytes of the URI.  These are copied into a ByteBuffer that will not be changed until this parser is reset and reused.
1782      * @param version the http version in use
1783      * @return true if handling parsing should return.
1784      */
1785     bool startRequest(string method, string uri, HttpVersion ver);
1786 
1787 }
1788 
1789 /* ------------------------------------------------------------------------------- */
1790 /* ------------------------------------------------------------------------------- */
1791 /* ------------------------------------------------------------------------------- */
1792 interface HttpResponseParsingHandler : HttpParsingHandler {
1793     /**
1794      * This is the method called by parser when the HTTP request line is parsed
1795      *
1796      * @param version the http version in use
1797      * @param status  the response status
1798      * @param reason  the response reason phrase
1799      * @return true if handling parsing should return
1800      */
1801     bool startResponse(HttpVersion ver, int status, string reason);
1802 }
1803 
1804 /* ------------------------------------------------------------------------------- */
1805 /* ------------------------------------------------------------------------------- */
1806 /* ------------------------------------------------------------------------------- */
1807 interface ComplianceParsingHandler : HttpParsingHandler {
1808     // deprecated("")
1809     // default void onComplianceViolation(HttpCompliance compliance, HttpCompliance required, string reason) {
1810     // }
1811 
1812     void onComplianceViolation(HttpCompliance compliance, HttpComplianceSection v, string details);
1813     // {
1814     //     // onComplianceViolation(compliance, HttpCompliance.requiredCompliance(violation), details);
1815     //     warning(details);
1816     // }
1817 }
1818 
1819 deprecated("Using HttpRequestParsingHandler instead.")
1820 alias HttpRequestHandler = HttpRequestParsingHandler;
1821 
1822 deprecated("Using HttpResponseParsingHandler instead.")
1823 alias HttpResponseHandler = HttpResponseParsingHandler;