1 module hunt.http.codec.http.hpack.HpackEncoder;
2 
3 import hunt.http.codec.http.hpack.HpackContext;
4 import hunt.http.codec.http.hpack.Huffman;
5 import hunt.http.codec.http.hpack.NBitInteger;
6 
7 // import hunt.http.codec.http.model;
8 
9 import hunt.http.codec.http.model.HttpField;
10 import hunt.http.codec.http.model.HttpFields;
11 import hunt.http.codec.http.model.HttpHeader;
12 import hunt.http.codec.http.model.HttpStatus;
13 import hunt.http.codec.http.model.HttpScheme;
14 import hunt.http.codec.http.model.HttpVersion;
15 import hunt.http.codec.http.model.MetaData;
16 
17 import hunt.container.ByteBuffer;
18 import hunt.container.BufferUtils;
19 
20 import hunt.http.codec.http.encode.Http1FieldPreEncoder;
21 import hunt.http.codec.http.encode.HttpFieldPreEncoder;
22 import hunt.http.codec.http.hpack.NBitInteger;
23 import hunt.http.codec.http.hpack.Huffman;
24 import hunt.http.codec.http.hpack.HpackContext;
25 
26 // import hunt.http.codec.http.model.HttpHeader;
27 // import hunt.http.codec.http.model.HttpVersion;
28 
29 import hunt.lang.exception;
30 import hunt.util.TypeUtils;
31 
32 
33 // import hunt.http.codec.http.model.HttpField;
34 // import hunt.http.codec.http.model.HttpHeader;
35 // import hunt.http.codec.http.model.HttpVersion;
36 
37 import hunt.logging;
38 
39 import std.range;
40 import std.algorithm;
41 import std.conv;
42 
43 alias Entry = HpackContext.Entry;
44 alias StaticEntry = HpackContext.StaticEntry;
45 
46 /**
47 */
48 class HpackEncoder {
49 
50     private __gshared static HttpField[599] __status;
51 
52     enum HttpHeader[] __DO_NOT_HUFFMAN = [            
53                     HttpHeader.AUTHORIZATION,
54                     HttpHeader.CONTENT_MD5,
55                     HttpHeader.PROXY_AUTHENTICATE,
56                     HttpHeader.PROXY_AUTHORIZATION];
57 
58     enum HttpHeader[] __DO_NOT_INDEX = [
59                     // HttpHeader.C_PATH,  // TODO more data needed
60                     // HttpHeader.DATE,    // TODO more data needed
61                     HttpHeader.AUTHORIZATION,
62                     HttpHeader.CONTENT_MD5,
63                     HttpHeader.CONTENT_RANGE,
64                     HttpHeader.ETAG,
65                     HttpHeader.IF_MODIFIED_SINCE,
66                     HttpHeader.IF_UNMODIFIED_SINCE,
67                     HttpHeader.IF_NONE_MATCH,
68                     HttpHeader.IF_RANGE,
69                     HttpHeader.IF_MATCH,
70                     HttpHeader.LOCATION,
71                     HttpHeader.RANGE,
72                     HttpHeader.RETRY_AFTER,
73                     // HttpHeader.EXPIRES,
74                     HttpHeader.LAST_MODIFIED,
75                     HttpHeader.SET_COOKIE,
76                     HttpHeader.SET_COOKIE2];
77 
78 
79     enum HttpHeader[] __NEVER_INDEX = [
80                     HttpHeader.AUTHORIZATION,
81                     HttpHeader.SET_COOKIE,
82                     HttpHeader.SET_COOKIE2];
83 
84     shared static this() {
85         foreach (HttpStatus.Code code ; HttpStatus.Code.values())
86             __status[code.getCode()] = new PreEncodedHttpField(HttpHeader.C_STATUS, std.conv.to!(string)(code.getCode()));
87     }
88 
89     private HpackContext _context;
90     private bool _debug;
91     private int _remoteMaxDynamicTableSize;
92     private int _localMaxDynamicTableSize;
93     private int _maxHeaderListSize;
94     private int _headerListSize;
95 
96     this() {
97         this(4096, 4096, -1);
98     }
99 
100     this(int localMaxDynamicTableSize) {
101         this(localMaxDynamicTableSize, 4096, -1);
102     }
103 
104     this(int localMaxDynamicTableSize, int remoteMaxDynamicTableSize) {
105         this(localMaxDynamicTableSize, remoteMaxDynamicTableSize, -1);
106     }
107 
108     this(int localMaxDynamicTableSize, int remoteMaxDynamicTableSize, int maxHeaderListSize) {
109         _context = new HpackContext(remoteMaxDynamicTableSize);
110         _remoteMaxDynamicTableSize = remoteMaxDynamicTableSize;
111         _localMaxDynamicTableSize = localMaxDynamicTableSize;
112         _maxHeaderListSize = maxHeaderListSize;
113         _debug = true; //log.isDebugEnabled();
114     }
115 
116     int getMaxHeaderListSize() {
117         return _maxHeaderListSize;
118     }
119 
120     void setMaxHeaderListSize(int maxHeaderListSize) {
121         _maxHeaderListSize = maxHeaderListSize;
122     }
123 
124     HpackContext getHpackContext() {
125         return _context;
126     }
127 
128     void setRemoteMaxDynamicTableSize(int remoteMaxDynamicTableSize) {
129         _remoteMaxDynamicTableSize = remoteMaxDynamicTableSize;
130     }
131 
132     void setLocalMaxDynamicTableSize(int localMaxDynamicTableSize) {
133         _localMaxDynamicTableSize = localMaxDynamicTableSize;
134     }
135 
136     void encode(ByteBuffer buffer, MetaData metadata) {
137         version(HUNT_DEBUG)
138             tracef("CtxTbl[%x] encoding", _context.toHash());
139 
140         _headerListSize = 0;
141         int pos = buffer.position();
142 
143         // Check the dynamic table sizes!
144         int maxDynamicTableSize = std.algorithm.min(_remoteMaxDynamicTableSize, _localMaxDynamicTableSize);
145         if (maxDynamicTableSize != _context.getMaxDynamicTableSize())
146             encodeMaxDynamicTableSize(buffer, maxDynamicTableSize);
147 
148         // Add Request/response meta fields
149         if (metadata.isRequest()) {
150             HttpRequest request = cast(HttpRequest) metadata;
151 
152             // TODO optimise these to avoid HttpField creation
153             string scheme = request.getURI().getScheme();
154             encode(buffer, new HttpField(HttpHeader.C_SCHEME, scheme.empty ? HttpScheme.HTTP : scheme));
155             encode(buffer, new HttpField(HttpHeader.C_METHOD, request.getMethod()));
156             encode(buffer, new HttpField(HttpHeader.C_AUTHORITY, request.getURI().getAuthority()));
157             encode(buffer, new HttpField(HttpHeader.C_PATH, request.getURI().getPathQuery()));
158         } else if (metadata.isResponse()) {
159             HttpResponse response = cast(HttpResponse) metadata;
160             int code = response.getStatus();
161             HttpField status = code < __status.length ? __status[code] : null;
162             if (status is null)
163                 status = new HttpField.IntValueHttpField(HttpHeader.C_STATUS, code);
164             encode(buffer, status);
165         }
166 
167         // Add all the other fields
168         foreach (HttpField field ; metadata)
169             encode(buffer, field);
170 
171         // Check size
172         if (_maxHeaderListSize > 0 && _headerListSize > _maxHeaderListSize) {
173             warningf("Header list size too large %s > %s for %s", _headerListSize, _maxHeaderListSize);
174             version(HUNT_DEBUG)
175                 tracef("metadata=%s", metadata);
176         }
177 
178         version(HUNT_DEBUG)
179             tracef("CtxTbl[%x] encoded %d octets", _context.toHash(), buffer.position() - pos);
180     }
181 
182     void encodeMaxDynamicTableSize(ByteBuffer buffer, int maxDynamicTableSize) {
183         if (maxDynamicTableSize > _remoteMaxDynamicTableSize)
184             throw new IllegalArgumentException("");
185         buffer.put(cast(byte) 0x20);
186         NBitInteger.encode(buffer, 5, maxDynamicTableSize);
187         _context.resize(maxDynamicTableSize);
188     }
189 
190     void encode(ByteBuffer buffer, HttpField field) {
191         if (field.getValue() == null)
192             field = new HttpField(field.getHeader(), field.getName(), "");
193 
194         int field_size = cast(int)(field.getName().length + field.getValue().length);
195         _headerListSize += field_size + 32;
196 
197         int p = _debug ? buffer.position() : -1;
198         string encoding = null;
199 
200         HttpHeader he = field.getHeader();
201         // tracef("encoding: %s,  hash: %d", field.toString(), field.toHash());
202 
203         // Is there an entry for the field?
204         Entry entry = _context.get(field);
205         if (entry !is null) {
206             // Known field entry, so encode it as indexed
207             if (entry.isStatic()) {
208                 buffer.put((cast(StaticEntry) entry).getEncodedField());
209                 version(HUNT_DEBUG)
210                     encoding = "IdxFieldS1";
211             } else {
212                 int index = _context.index(entry);
213                 buffer.put(cast(byte) 0x80);
214                 NBitInteger.encode(buffer, 7, index);
215                 version(HUNT_DEBUG)
216                     encoding = "IdxField" ~ (entry.isStatic() ? "S" : "") ~ to!string(1 + NBitInteger.octectsNeeded(7, index));
217             }
218         } else {
219             // Unknown field entry, so we will have to send literally.
220             bool indexed;
221 
222             // But do we know it's name?
223             HttpHeader header = field.getHeader();
224 
225             // Select encoding strategy
226             if (header == HttpHeader.Null) {
227                 // Select encoding strategy for unknown header names
228                 Entry name = _context.get(field.getName());
229 
230                 if (typeid(field) == typeid(PreEncodedHttpField)) {
231                     int i = buffer.position();
232                     (cast(PreEncodedHttpField) field).putTo(buffer, HttpVersion.HTTP_2);
233                     byte b = buffer.get(i);
234                     indexed = b < 0 || b >= 0x40;
235                     version(HUNT_DEBUG)
236                         encoding = indexed ? "PreEncodedIdx" : "PreEncoded";
237                 }
238                 // has the custom header name been seen before?
239                 else if (name is null) {
240                     // unknown name and value, so let's index this just in case it is
241                     // the first time we have seen a custom name or a custom field.
242                     // unless the name is changing, this is worthwhile
243                     indexed = true;
244                     encodeName(buffer, cast(byte) 0x40, 6, field.getName(), null);
245                     encodeValue(buffer, true, field.getValue());
246                     version(HUNT_DEBUG)
247                         encoding = "LitHuffNHuffVIdx";
248                 } else {
249                     // known custom name, but unknown value.
250                     // This is probably a custom field with changing value, so don't index.
251                     indexed = false;
252                     encodeName(buffer, cast(byte) 0x00, 4, field.getName(), null);
253                     encodeValue(buffer, true, field.getValue());
254                     version(HUNT_DEBUG)
255                         encoding = "LitHuffNHuffV!Idx";
256                 }
257             } else {
258                 // Select encoding strategy for known header names
259                 Entry name = _context.get(header);
260                 if(name is null)
261                     warningf("no entry found for header: %s", header.toString());
262                 // else
263                 //     tracef("Entry=%s, header: name=%s, ordinal=%d", name.toString(), header.toString(), header.ordinal());
264 
265                 if (typeid(field) == typeid(PreEncodedHttpField)) {
266                     // Preencoded field
267                     int i = buffer.position();
268                     (cast(PreEncodedHttpField) field).putTo(buffer, HttpVersion.HTTP_2);
269                     byte b = buffer.get(i);
270                     indexed = b < 0 || b >= 0x40;
271                     version(HUNT_DEBUG)
272                         encoding = indexed ? "PreEncodedIdx" : "PreEncoded";
273                 } else if (__DO_NOT_INDEX.contains(header)) {
274                     // Non indexed field
275                     indexed = false;
276                     bool never_index = __NEVER_INDEX.contains(header);
277                     bool huffman = !__DO_NOT_HUFFMAN.contains(header);
278                     encodeName(buffer, never_index ? cast(byte) 0x10 : cast(byte) 0x00, 4, header.asString(), name);
279                     encodeValue(buffer, huffman, field.getValue());
280 
281                     version(HUNT_DEBUG)
282                     {
283                         encoding = "Lit" ~ ((name is null) ? "HuffN" : ("IdxN" ~ (name.isStatic() ? "S" : "") ~ 
284                                 to!string(1 + NBitInteger.octectsNeeded(4, _context.index(name))))) ~
285                                 (huffman ? "HuffV" : "LitV") ~
286                                 (indexed ? "Idx" : (never_index ? "!!Idx" : "!Idx"));
287                         
288                     }
289                 } else if (field_size >= _context.getMaxDynamicTableSize() || header == HttpHeader.CONTENT_LENGTH &&
290                  field.getValue().length > 2) {
291                     // Non indexed if field too large or a content length for 3 digits or more
292                     indexed = false;
293                     encodeName(buffer, cast(byte) 0x00, 4, header.asString(), name);
294                     encodeValue(buffer, true, field.getValue());
295                     version(HUNT_DEBUG)
296                         encoding = "LitIdxNS" ~ to!string(1 + NBitInteger.octectsNeeded(4, _context.index(name))) ~ "HuffV!Idx";
297                 } else {
298                     // indexed
299                     indexed = true;
300                     bool huffman = !__DO_NOT_HUFFMAN.contains(header);
301                     encodeName(buffer, cast(byte) 0x40, 6, header.asString(), name);
302                     encodeValue(buffer, huffman, field.getValue());
303                     version(HUNT_DEBUG){
304                         encoding = ((name is null) ? "LitHuffN" : ("LitIdxN" ~ (name.isStatic() ? "S" : "") ~ 
305                         to!string(1 + NBitInteger.octectsNeeded(6, _context.index(name))))) ~
306                                 (huffman ? "HuffVIdx" : "LitVIdx");
307                     }
308                 }
309             }
310 
311             // If we want the field referenced, then we add it to our
312             // table and reference set.
313             if (indexed)
314                 if (_context.add(field) is null)
315                     throw new IllegalStateException("");
316         }
317 
318         // version(HUNT_DEBUG) 
319         {
320             int e = buffer.position();
321             tracef("encode %s:'%s' to '%s'", encoding, field, TypeUtils.toHexString(buffer.array(), buffer.arrayOffset() + p, e - p));
322         }
323     }
324 
325     private void encodeName(ByteBuffer buffer, byte mask, int bits, string name, Entry entry) {
326         buffer.put(mask);
327         if (entry is null) {
328             // leave name index bits as 0
329             // Encode the name always with lowercase huffman
330             buffer.put(cast(byte) 0x80);
331             NBitInteger.encode(buffer, 7, Huffman.octetsNeededLC(name));
332             Huffman.encodeLC(buffer, name);
333         } else {
334             NBitInteger.encode(buffer, bits, _context.index(entry));
335         }
336     }
337 
338     static void encodeValue(ByteBuffer buffer, bool huffman, string value) {
339         if (huffman) {
340             // huffman literal value
341             buffer.put(cast(byte) 0x80);
342             NBitInteger.encode(buffer, 7, Huffman.octetsNeeded(value));
343             Huffman.encode(buffer, value);
344         } else {
345             // add literal assuming iso_8859_1
346             buffer.put(cast(byte) 0x00);
347             NBitInteger.encode(buffer, 7, cast(int)value.length);
348             for (size_t i = 0; i < value.length; i++) {
349                 char c = value[i];
350                 if (c < ' ' || c > 127)
351                     throw new IllegalArgumentException("");
352                 buffer.put(cast(byte) c);
353             }
354         }
355     }
356 }
357 
358 
359 /**
360 */
361 class HpackFieldPreEncoder : HttpFieldPreEncoder {
362 
363 	override
364 	HttpVersion getHttpVersion() {
365 		return HttpVersion.HTTP_2;
366 	}
367 
368 	override
369 	byte[] getEncodedField(HttpHeader header, string name, string value) {
370 		bool not_indexed = HpackEncoder.__DO_NOT_INDEX.contains(header);
371 
372 		ByteBuffer buffer = BufferUtils.allocate(cast(int) (name.length + value.length + 10));
373 		BufferUtils.clearToFill(buffer);
374 		bool huffman;
375 		int bits;
376 
377 		if (not_indexed) {
378 			// Non indexed field
379 			bool never_index = HpackEncoder.__NEVER_INDEX.contains(header);
380 			huffman = !HpackEncoder.__DO_NOT_HUFFMAN.contains(header);
381 			buffer.put(never_index ? cast(byte) 0x10 : cast(byte) 0x00);
382 			bits = 4;
383 		} else if (header == HttpHeader.CONTENT_LENGTH && value.length > 1) {
384 			// Non indexed content length for 2 digits or more
385 			buffer.put(cast(byte) 0x00);
386 			huffman = true;
387 			bits = 4;
388 		} else {
389 			// indexed
390 			buffer.put(cast(byte) 0x40);
391 			huffman = !HpackEncoder.__DO_NOT_HUFFMAN.contains(header);
392 			bits = 6;
393 		}
394 
395 		int name_idx = HpackContext.staticIndex(header);
396 		if (name_idx > 0)
397 			NBitInteger.encode(buffer, bits, name_idx);
398 		else {
399 			buffer.put(cast(byte) 0x80);
400 			NBitInteger.encode(buffer, 7, Huffman.octetsNeededLC(name));
401 			Huffman.encodeLC(buffer, name);
402 		}
403 
404 		HpackEncoder.encodeValue(buffer, huffman, value);
405 
406 		BufferUtils.flipToFlush(buffer, 0);
407 		return BufferUtils.toArray(buffer);
408 	}
409 }
410 
411 
412 /**
413  * Pre encoded HttpField.
414  * <p>A HttpField that will be cached and used many times can be created as
415  * a {@link PreEncodedHttpField}, which will use the {@link HttpFieldPreEncoder}
416  * instances discovered by the {@link ServiceLoader} to pre-encode the header
417  * for each version of HTTP in use.  This will save garbage
418  * and CPU each time the field is encoded into a response.
419  * </p>
420  */
421 class PreEncodedHttpField : HttpField {
422     private __gshared static HttpFieldPreEncoder[] __encoders;
423 
424     private byte[][] _encodedField = new byte[][2];
425 
426     shared static this()
427     {
428         __encoders ~= new HpackFieldPreEncoder();
429         __encoders ~= new Http1FieldPreEncoder();
430         // __encoders = [ null,
431         //     new Http1FieldPreEncoder()];
432     }
433 
434     this(HttpHeader header, string name, string value) {
435         super(header, name, value);
436 
437         foreach (HttpFieldPreEncoder e ; __encoders) {
438             if(e is null)
439                 continue;
440             _encodedField[e.getHttpVersion() == HttpVersion.HTTP_2 ? 1 : 0] = e.getEncodedField(header, header.asString(), value);
441         }
442     }
443 
444     this(HttpHeader header, string value) {
445         this(header, header.asString(), value);
446     }
447 
448     this(string name, string value) {
449         this(HttpHeader.Null, name, value);
450     }
451 
452     void putTo(ByteBuffer bufferInFillMode, HttpVersion ver) {
453         bufferInFillMode.put(_encodedField[ver == HttpVersion.HTTP_2 ? 1 : 0]);
454     }
455 }