1 module hunt.http.codec.http.hpack.HpackEncoder;
2 
3 import hunt.http.codec.http.hpack.HpackContext;
4 import hunt.http.codec.http.hpack.Huffman;
5 import hunt.http.codec.http.hpack.NBitInteger;
6 
7 // import hunt.http.codec.http.model;
8 
9 import hunt.http.HttpField;
10 import hunt.http.HttpFields;
11 import hunt.http.HttpHeader;
12 import hunt.http.HttpMetaData;
13 import hunt.http.HttpRequest;
14 import hunt.http.HttpResponse;
15 import hunt.http.HttpStatus;
16 import hunt.http.HttpScheme;
17 import hunt.http.HttpVersion;
18 
19 import hunt.io.ByteBuffer;
20 import hunt.io.BufferUtils;
21 
22 import hunt.http.codec.http.encode.Http1FieldPreEncoder;
23 import hunt.http.codec.http.encode.HttpFieldPreEncoder;
24 import hunt.http.codec.http.hpack.NBitInteger;
25 import hunt.http.codec.http.hpack.Huffman;
26 import hunt.http.codec.http.hpack.HpackContext;
27 
28 // import hunt.http.HttpHeader;
29 // import hunt.http.HttpVersion;
30 
31 import hunt.Exceptions;
32 import hunt.util.ConverterUtils;
33 
34 
35 // import hunt.http.HttpField;
36 // import hunt.http.HttpHeader;
37 // import hunt.http.HttpVersion;
38 
39 import hunt.logging;
40 
41 import std.range;
42 import std.algorithm;
43 import std.conv;
44 
45 alias Entry = HpackContext.Entry;
46 alias StaticEntry = HpackContext.StaticEntry;
47 
48 /**
49 */
50 class HpackEncoder {
51 
52     private __gshared static HttpField[599] __status;
53 
54     enum HttpHeader[] __DO_NOT_HUFFMAN = [            
55                     HttpHeader.AUTHORIZATION,
56                     HttpHeader.CONTENT_MD5,
57                     HttpHeader.PROXY_AUTHENTICATE,
58                     HttpHeader.PROXY_AUTHORIZATION];
59 
60     enum HttpHeader[] __DO_NOT_INDEX = [
61                     // HttpHeader.C_PATH,  // TODO more data needed
62                     // HttpHeader.DATE,    // TODO more data needed
63                     HttpHeader.AUTHORIZATION,
64                     HttpHeader.CONTENT_MD5,
65                     HttpHeader.CONTENT_RANGE,
66                     HttpHeader.ETAG,
67                     HttpHeader.IF_MODIFIED_SINCE,
68                     HttpHeader.IF_UNMODIFIED_SINCE,
69                     HttpHeader.IF_NONE_MATCH,
70                     HttpHeader.IF_RANGE,
71                     HttpHeader.IF_MATCH,
72                     HttpHeader.LOCATION,
73                     HttpHeader.RANGE,
74                     HttpHeader.RETRY_AFTER,
75                     // HttpHeader.EXPIRES,
76                     HttpHeader.LAST_MODIFIED,
77                     HttpHeader.SET_COOKIE,
78                     HttpHeader.SET_COOKIE2];
79 
80 
81     enum HttpHeader[] __NEVER_INDEX = [
82                     HttpHeader.AUTHORIZATION,
83                     HttpHeader.SET_COOKIE,
84                     HttpHeader.SET_COOKIE2];
85 
86     shared static this() {
87         foreach (HttpStatus.Code code ; HttpStatus.Code.values())
88             __status[code.getCode()] = new PreEncodedHttpField(HttpHeader.C_STATUS, std.conv.to!(string)(code.getCode()));
89     }
90 
91     private HpackContext _context;
92     private bool _debug;
93     private int _remoteMaxDynamicTableSize;
94     private int _localMaxDynamicTableSize;
95     private int _maxHeaderListSize;
96     private int _headerListSize;
97 
98     this() {
99         this(4096, 4096, -1);
100     }
101 
102     this(int localMaxDynamicTableSize) {
103         this(localMaxDynamicTableSize, 4096, -1);
104     }
105 
106     this(int localMaxDynamicTableSize, int remoteMaxDynamicTableSize) {
107         this(localMaxDynamicTableSize, remoteMaxDynamicTableSize, -1);
108     }
109 
110     this(int localMaxDynamicTableSize, int remoteMaxDynamicTableSize, int maxHeaderListSize) {
111         _context = new HpackContext(remoteMaxDynamicTableSize);
112         _remoteMaxDynamicTableSize = remoteMaxDynamicTableSize;
113         _localMaxDynamicTableSize = localMaxDynamicTableSize;
114         _maxHeaderListSize = maxHeaderListSize;
115         _debug = true; //log.isDebugEnabled();
116     }
117 
118     int getMaxHeaderListSize() {
119         return _maxHeaderListSize;
120     }
121 
122     void setMaxHeaderListSize(int maxHeaderListSize) {
123         _maxHeaderListSize = maxHeaderListSize;
124     }
125 
126     HpackContext getHpackContext() {
127         return _context;
128     }
129 
130     void setRemoteMaxDynamicTableSize(int remoteMaxDynamicTableSize) {
131         _remoteMaxDynamicTableSize = remoteMaxDynamicTableSize;
132     }
133 
134     void setLocalMaxDynamicTableSize(int localMaxDynamicTableSize) {
135         _localMaxDynamicTableSize = localMaxDynamicTableSize;
136     }
137 
138     void encode(ByteBuffer buffer, HttpMetaData metadata) {
139         version(HUNT_DEBUG)
140             tracef("CtxTbl[%x] encoding", _context.toHash());
141 
142         _headerListSize = 0;
143         int pos = buffer.position();
144 
145         // Check the dynamic table sizes!
146         int maxDynamicTableSize = std.algorithm.min(_remoteMaxDynamicTableSize, _localMaxDynamicTableSize);
147         if (maxDynamicTableSize != _context.getMaxDynamicTableSize())
148             encodeMaxDynamicTableSize(buffer, maxDynamicTableSize);
149 
150         // Add Request/response meta fields
151         if (metadata.isRequest()) {
152             HttpRequest request = cast(HttpRequest) metadata;
153 
154             // TODO optimise these to avoid HttpField creation
155             string scheme = request.getURI().getScheme();
156             encode(buffer, new HttpField(HttpHeader.C_SCHEME, scheme.empty ? HttpScheme.HTTP : scheme));
157             encode(buffer, new HttpField(HttpHeader.C_METHOD, request.getMethod()));
158             encode(buffer, new HttpField(HttpHeader.C_AUTHORITY, request.getURI().getAuthority()));
159             encode(buffer, new HttpField(HttpHeader.C_PATH, request.getURI().getPathQuery()));
160         } else if (metadata.isResponse()) {
161             HttpResponse response = cast(HttpResponse) metadata;
162             int code = response.getStatus();
163             HttpField status = code < __status.length ? __status[code] : null;
164             if (status is null)
165                 status = new HttpField.IntValueHttpField(HttpHeader.C_STATUS, code);
166             encode(buffer, status);
167         }
168 
169         // Add all the other fields
170         foreach (HttpField field ; metadata)
171             encode(buffer, field);
172 
173         // Check size
174         if (_maxHeaderListSize > 0 && _headerListSize > _maxHeaderListSize) {
175             warningf("Header list size too large %s > %s for %s", _headerListSize, _maxHeaderListSize);
176             version(HUNT_DEBUG)
177                 tracef("metadata=%s", metadata);
178         }
179 
180         version(HUNT_DEBUG)
181             tracef("CtxTbl[%x] encoded %d octets", _context.toHash(), buffer.position() - pos);
182     }
183 
184     void encodeMaxDynamicTableSize(ByteBuffer buffer, int maxDynamicTableSize) {
185         if (maxDynamicTableSize > _remoteMaxDynamicTableSize)
186             throw new IllegalArgumentException("");
187         buffer.put(cast(byte) 0x20);
188         NBitInteger.encode(buffer, 5, maxDynamicTableSize);
189         _context.resize(maxDynamicTableSize);
190     }
191 
192     void encode(ByteBuffer buffer, HttpField field) {
193         if (field.getValue() == null)
194             field = new HttpField(field.getHeader(), field.getName(), "");
195 
196         int field_size = cast(int)(field.getName().length + field.getValue().length);
197         _headerListSize += field_size + 32;
198 
199         int p = _debug ? buffer.position() : -1;
200         string encoding = null;
201 
202         HttpHeader he = field.getHeader();
203         // tracef("encoding: %s,  hash: %d", field.toString(), field.toHash());
204 
205         // Is there an entry for the field?
206         Entry entry = _context.get(field);
207         if (entry !is null) {
208             // Known field entry, so encode it as indexed
209             if (entry.isStatic()) {
210                 buffer.put((cast(StaticEntry) entry).getEncodedField());
211                 version(HUNT_DEBUG)
212                     encoding = "IdxFieldS1";
213             } else {
214                 int index = _context.index(entry);
215                 buffer.put(cast(byte) 0x80);
216                 NBitInteger.encode(buffer, 7, index);
217                 version(HUNT_DEBUG)
218                     encoding = "IdxField" ~ (entry.isStatic() ? "S" : "") ~ to!string(1 + NBitInteger.octectsNeeded(7, index));
219             }
220         } else {
221             // Unknown field entry, so we will have to send literally.
222             bool indexed;
223 
224             // But do we know it's name?
225             HttpHeader header = field.getHeader();
226 
227             // Select encoding strategy
228             if (header == HttpHeader.Null) {
229                 // Select encoding strategy for unknown header names
230                 Entry name = _context.get(field.getName());
231 
232                 if (typeid(field) == typeid(PreEncodedHttpField)) {
233                     int i = buffer.position();
234                     (cast(PreEncodedHttpField) field).putTo(buffer, HttpVersion.HTTP_2);
235                     byte b = buffer.get(i);
236                     indexed = b < 0 || b >= 0x40;
237                     version(HUNT_DEBUG)
238                         encoding = indexed ? "PreEncodedIdx" : "PreEncoded";
239                 }
240                 // has the custom header name been seen before?
241                 else if (name is null) {
242                     // unknown name and value, so let's index this just in case it is
243                     // the first time we have seen a custom name or a custom field.
244                     // unless the name is changing, this is worthwhile
245                     indexed = true;
246                     encodeName(buffer, cast(byte) 0x40, 6, field.getName(), null);
247                     encodeValue(buffer, true, field.getValue());
248                     version(HUNT_DEBUG)
249                         encoding = "LitHuffNHuffVIdx";
250                 } else {
251                     // known custom name, but unknown value.
252                     // This is probably a custom field with changing value, so don't index.
253                     indexed = false;
254                     encodeName(buffer, cast(byte) 0x00, 4, field.getName(), null);
255                     encodeValue(buffer, true, field.getValue());
256                     version(HUNT_DEBUG)
257                         encoding = "LitHuffNHuffV!Idx";
258                 }
259             } else {
260                 // Select encoding strategy for known header names
261                 Entry name = _context.get(header);
262                 if(name is null)
263                     warningf("no entry found for header: %s", header.toString());
264                 // else
265                 //     tracef("Entry=%s, header: name=%s, ordinal=%d", name.toString(), header.toString(), header.ordinal());
266 
267                 if (typeid(field) == typeid(PreEncodedHttpField)) {
268                     // Preencoded field
269                     int i = buffer.position();
270                     (cast(PreEncodedHttpField) field).putTo(buffer, HttpVersion.HTTP_2);
271                     byte b = buffer.get(i);
272                     indexed = b < 0 || b >= 0x40;
273                     version(HUNT_DEBUG)
274                         encoding = indexed ? "PreEncodedIdx" : "PreEncoded";
275                 } else if (__DO_NOT_INDEX.contains(header)) {
276                     // Non indexed field
277                     indexed = false;
278                     bool never_index = __NEVER_INDEX.contains(header);
279                     bool huffman = !__DO_NOT_HUFFMAN.contains(header);
280                     encodeName(buffer, never_index ? cast(byte) 0x10 : cast(byte) 0x00, 4, header.asString(), name);
281                     encodeValue(buffer, huffman, field.getValue());
282 
283                     version(HUNT_DEBUG)
284                     {
285                         encoding = "Lit" ~ ((name is null) ? "HuffN" : ("IdxN" ~ (name.isStatic() ? "S" : "") ~ 
286                                 to!string(1 + NBitInteger.octectsNeeded(4, _context.index(name))))) ~
287                                 (huffman ? "HuffV" : "LitV") ~
288                                 (indexed ? "Idx" : (never_index ? "!!Idx" : "!Idx"));
289                         
290                     }
291                 } else if (field_size >= _context.getMaxDynamicTableSize() || header == HttpHeader.CONTENT_LENGTH &&
292                  field.getValue().length > 2) {
293                     // Non indexed if field too large or a content length for 3 digits or more
294                     indexed = false;
295                     encodeName(buffer, cast(byte) 0x00, 4, header.asString(), name);
296                     encodeValue(buffer, true, field.getValue());
297                     version(HUNT_DEBUG)
298                         encoding = "LitIdxNS" ~ to!string(1 + NBitInteger.octectsNeeded(4, _context.index(name))) ~ "HuffV!Idx";
299                 } else {
300                     // indexed
301                     indexed = true;
302                     bool huffman = !__DO_NOT_HUFFMAN.contains(header);
303                     encodeName(buffer, cast(byte) 0x40, 6, header.asString(), name);
304                     encodeValue(buffer, huffman, field.getValue());
305                     version(HUNT_DEBUG){
306                         encoding = ((name is null) ? "LitHuffN" : ("LitIdxN" ~ (name.isStatic() ? "S" : "") ~ 
307                         to!string(1 + NBitInteger.octectsNeeded(6, _context.index(name))))) ~
308                                 (huffman ? "HuffVIdx" : "LitVIdx");
309                     }
310                 }
311             }
312 
313             // If we want the field referenced, then we add it to our
314             // table and reference set.
315             if (indexed)
316                 if (_context.add(field) is null)
317                     throw new IllegalStateException("");
318         }
319 
320         version(HUNT_HTTP_DEBUG) 
321         {
322             int e = buffer.position();
323             tracef("encode %s: '%s' to '%s'", encoding, field, 
324                 ConverterUtils.toHexString(buffer.array(), buffer.arrayOffset() + p, e - p));
325         }
326     }
327 
328     private void encodeName(ByteBuffer buffer, byte mask, int bits, string name, Entry entry) {
329         buffer.put(mask);
330         if (entry is null) {
331             // leave name index bits as 0
332             // Encode the name always with lowercase huffman
333             buffer.put(cast(byte) 0x80);
334             NBitInteger.encode(buffer, 7, Huffman.octetsNeededLC(name));
335             Huffman.encodeLC(buffer, name);
336         } else {
337             NBitInteger.encode(buffer, bits, _context.index(entry));
338         }
339     }
340 
341     static void encodeValue(ByteBuffer buffer, bool huffman, string value) {
342         if (huffman) {
343             // huffman literal value
344             buffer.put(cast(byte) 0x80);
345             NBitInteger.encode(buffer, 7, Huffman.octetsNeeded(value));
346             Huffman.encode(buffer, value);
347         } else {
348             // add literal assuming iso_8859_1
349             buffer.put(cast(byte) 0x00);
350             NBitInteger.encode(buffer, 7, cast(int)value.length);
351             for (size_t i = 0; i < value.length; i++) {
352                 char c = value[i];
353                 if (c < ' ' || c > 127)
354                     throw new IllegalArgumentException("");
355                 buffer.put(cast(byte) c);
356             }
357         }
358     }
359 }
360 
361 
362 /**
363 */
364 class HpackFieldPreEncoder : HttpFieldPreEncoder {
365 
366 	override
367 	HttpVersion getHttpVersion() {
368 		return HttpVersion.HTTP_2;
369 	}
370 
371 	override
372 	byte[] getEncodedField(HttpHeader header, string name, string value) {
373 		bool not_indexed = HpackEncoder.__DO_NOT_INDEX.contains(header);
374 
375 		ByteBuffer buffer = BufferUtils.allocate(cast(int) (name.length + value.length + 10));
376 		BufferUtils.clearToFill(buffer);
377 		bool huffman;
378 		int bits;
379 
380 		if (not_indexed) {
381 			// Non indexed field
382 			bool never_index = HpackEncoder.__NEVER_INDEX.contains(header);
383 			huffman = !HpackEncoder.__DO_NOT_HUFFMAN.contains(header);
384 			buffer.put(never_index ? cast(byte) 0x10 : cast(byte) 0x00);
385 			bits = 4;
386 		} else if (header == HttpHeader.CONTENT_LENGTH && value.length > 1) {
387 			// Non indexed content length for 2 digits or more
388 			buffer.put(cast(byte) 0x00);
389 			huffman = true;
390 			bits = 4;
391 		} else {
392 			// indexed
393 			buffer.put(cast(byte) 0x40);
394 			huffman = !HpackEncoder.__DO_NOT_HUFFMAN.contains(header);
395 			bits = 6;
396 		}
397 
398 		int name_idx = HpackContext.staticIndex(header);
399 		if (name_idx > 0)
400 			NBitInteger.encode(buffer, bits, name_idx);
401 		else {
402 			buffer.put(cast(byte) 0x80);
403 			NBitInteger.encode(buffer, 7, Huffman.octetsNeededLC(name));
404 			Huffman.encodeLC(buffer, name);
405 		}
406 
407 		HpackEncoder.encodeValue(buffer, huffman, value);
408 
409 		BufferUtils.flipToFlush(buffer, 0);
410 		return BufferUtils.toArray(buffer);
411 	}
412 }
413 
414 
415 /**
416  * Pre encoded HttpField.
417  * <p>A HttpField that will be cached and used many times can be created as
418  * a {@link PreEncodedHttpField}, which will use the {@link HttpFieldPreEncoder}
419  * instances discovered by the {@link ServiceLoader} to pre-encode the header
420  * for each version of HTTP in use.  This will save garbage
421  * and CPU each time the field is encoded into a response.
422  * </p>
423  */
424 class PreEncodedHttpField : HttpField {
425     private __gshared static HttpFieldPreEncoder[] __encoders;
426 
427     private byte[][] _encodedField = new byte[][2];
428 
429     shared static this()
430     {
431         __encoders ~= new HpackFieldPreEncoder();
432         __encoders ~= new Http1FieldPreEncoder();
433         // __encoders = [ null,
434         //     new Http1FieldPreEncoder()];
435     }
436 
437     this(HttpHeader header, string name, string value) {
438         super(header, name, value);
439 
440         foreach (HttpFieldPreEncoder e ; __encoders) {
441             if(e is null)
442                 continue;
443             _encodedField[e.getHttpVersion() == HttpVersion.HTTP_2 ? 1 : 0] = e.getEncodedField(header, header.asString(), value);
444         }
445     }
446 
447     this(HttpHeader header, string value) {
448         this(header, header.asString(), value);
449     }
450 
451     this(string name, string value) {
452         this(HttpHeader.Null, name, value);
453     }
454 
455     void putTo(ByteBuffer bufferInFillMode, HttpVersion ver) {
456         bufferInFillMode.put(_encodedField[ver == HttpVersion.HTTP_2 ? 1 : 0]);
457     }
458 }