1 module hunt.http.codec.http.hpack.HpackEncoder; 2 3 import hunt.http.codec.http.hpack.HpackContext; 4 import hunt.http.codec.http.hpack.Huffman; 5 import hunt.http.codec.http.hpack.NBitInteger; 6 7 // import hunt.http.codec.http.model; 8 9 import hunt.http.HttpField; 10 import hunt.http.HttpFields; 11 import hunt.http.HttpHeader; 12 import hunt.http.HttpMetaData; 13 import hunt.http.HttpRequest; 14 import hunt.http.HttpResponse; 15 import hunt.http.HttpStatus; 16 import hunt.http.HttpScheme; 17 import hunt.http.HttpVersion; 18 19 import hunt.io.ByteBuffer; 20 import hunt.io.BufferUtils; 21 22 import hunt.http.codec.http.encode.Http1FieldPreEncoder; 23 import hunt.http.codec.http.encode.HttpFieldPreEncoder; 24 import hunt.http.codec.http.hpack.NBitInteger; 25 import hunt.http.codec.http.hpack.Huffman; 26 import hunt.http.codec.http.hpack.HpackContext; 27 28 // import hunt.http.HttpHeader; 29 // import hunt.http.HttpVersion; 30 31 import hunt.Exceptions; 32 import hunt.util.ConverterUtils; 33 34 35 // import hunt.http.HttpField; 36 // import hunt.http.HttpHeader; 37 // import hunt.http.HttpVersion; 38 39 import hunt.logging; 40 41 import std.range; 42 import std.algorithm; 43 import std.conv; 44 45 alias Entry = HpackContext.Entry; 46 alias StaticEntry = HpackContext.StaticEntry; 47 48 /** 49 */ 50 class HpackEncoder { 51 52 private __gshared static HttpField[599] __status; 53 54 enum HttpHeader[] __DO_NOT_HUFFMAN = [ 55 HttpHeader.AUTHORIZATION, 56 HttpHeader.CONTENT_MD5, 57 HttpHeader.PROXY_AUTHENTICATE, 58 HttpHeader.PROXY_AUTHORIZATION]; 59 60 enum HttpHeader[] __DO_NOT_INDEX = [ 61 // HttpHeader.C_PATH, // TODO more data needed 62 // HttpHeader.DATE, // TODO more data needed 63 HttpHeader.AUTHORIZATION, 64 HttpHeader.CONTENT_MD5, 65 HttpHeader.CONTENT_RANGE, 66 HttpHeader.ETAG, 67 HttpHeader.IF_MODIFIED_SINCE, 68 HttpHeader.IF_UNMODIFIED_SINCE, 69 HttpHeader.IF_NONE_MATCH, 70 HttpHeader.IF_RANGE, 71 HttpHeader.IF_MATCH, 72 HttpHeader.LOCATION, 73 HttpHeader.RANGE, 74 HttpHeader.RETRY_AFTER, 75 // HttpHeader.EXPIRES, 76 HttpHeader.LAST_MODIFIED, 77 HttpHeader.SET_COOKIE, 78 HttpHeader.SET_COOKIE2]; 79 80 81 enum HttpHeader[] __NEVER_INDEX = [ 82 HttpHeader.AUTHORIZATION, 83 HttpHeader.SET_COOKIE, 84 HttpHeader.SET_COOKIE2]; 85 86 shared static this() { 87 foreach (HttpStatus.Code code ; HttpStatus.Code.values()) 88 __status[code.getCode()] = new PreEncodedHttpField(HttpHeader.C_STATUS, std.conv.to!(string)(code.getCode())); 89 } 90 91 private HpackContext _context; 92 private bool _debug; 93 private int _remoteMaxDynamicTableSize; 94 private int _localMaxDynamicTableSize; 95 private int _maxHeaderListSize; 96 private int _headerListSize; 97 98 this() { 99 this(4096, 4096, -1); 100 } 101 102 this(int localMaxDynamicTableSize) { 103 this(localMaxDynamicTableSize, 4096, -1); 104 } 105 106 this(int localMaxDynamicTableSize, int remoteMaxDynamicTableSize) { 107 this(localMaxDynamicTableSize, remoteMaxDynamicTableSize, -1); 108 } 109 110 this(int localMaxDynamicTableSize, int remoteMaxDynamicTableSize, int maxHeaderListSize) { 111 _context = new HpackContext(remoteMaxDynamicTableSize); 112 _remoteMaxDynamicTableSize = remoteMaxDynamicTableSize; 113 _localMaxDynamicTableSize = localMaxDynamicTableSize; 114 _maxHeaderListSize = maxHeaderListSize; 115 _debug = true; //log.isDebugEnabled(); 116 } 117 118 int getMaxHeaderListSize() { 119 return _maxHeaderListSize; 120 } 121 122 void setMaxHeaderListSize(int maxHeaderListSize) { 123 _maxHeaderListSize = maxHeaderListSize; 124 } 125 126 HpackContext getHpackContext() { 127 return _context; 128 } 129 130 void setRemoteMaxDynamicTableSize(int remoteMaxDynamicTableSize) { 131 _remoteMaxDynamicTableSize = remoteMaxDynamicTableSize; 132 } 133 134 void setLocalMaxDynamicTableSize(int localMaxDynamicTableSize) { 135 _localMaxDynamicTableSize = localMaxDynamicTableSize; 136 } 137 138 void encode(ByteBuffer buffer, HttpMetaData metadata) { 139 version(HUNT_DEBUG) 140 tracef("CtxTbl[%x] encoding", _context.toHash()); 141 142 _headerListSize = 0; 143 int pos = buffer.position(); 144 145 // Check the dynamic table sizes! 146 int maxDynamicTableSize = std.algorithm.min(_remoteMaxDynamicTableSize, _localMaxDynamicTableSize); 147 if (maxDynamicTableSize != _context.getMaxDynamicTableSize()) 148 encodeMaxDynamicTableSize(buffer, maxDynamicTableSize); 149 150 // Add Request/response meta fields 151 if (metadata.isRequest()) { 152 HttpRequest request = cast(HttpRequest) metadata; 153 154 // TODO optimise these to avoid HttpField creation 155 string scheme = request.getURI().getScheme(); 156 encode(buffer, new HttpField(HttpHeader.C_SCHEME, scheme.empty ? HttpScheme.HTTP : scheme)); 157 encode(buffer, new HttpField(HttpHeader.C_METHOD, request.getMethod())); 158 encode(buffer, new HttpField(HttpHeader.C_AUTHORITY, request.getURI().getAuthority())); 159 encode(buffer, new HttpField(HttpHeader.C_PATH, request.getURI().getPathQuery())); 160 } else if (metadata.isResponse()) { 161 HttpResponse response = cast(HttpResponse) metadata; 162 int code = response.getStatus(); 163 HttpField status = code < __status.length ? __status[code] : null; 164 if (status is null) 165 status = new HttpField.IntValueHttpField(HttpHeader.C_STATUS, code); 166 encode(buffer, status); 167 } 168 169 // Add all the other fields 170 foreach (HttpField field ; metadata) 171 encode(buffer, field); 172 173 // Check size 174 if (_maxHeaderListSize > 0 && _headerListSize > _maxHeaderListSize) { 175 warningf("Header list size too large %s > %s for %s", _headerListSize, _maxHeaderListSize); 176 version(HUNT_DEBUG) 177 tracef("metadata=%s", metadata); 178 } 179 180 version(HUNT_DEBUG) 181 tracef("CtxTbl[%x] encoded %d octets", _context.toHash(), buffer.position() - pos); 182 } 183 184 void encodeMaxDynamicTableSize(ByteBuffer buffer, int maxDynamicTableSize) { 185 if (maxDynamicTableSize > _remoteMaxDynamicTableSize) 186 throw new IllegalArgumentException(""); 187 buffer.put(cast(byte) 0x20); 188 NBitInteger.encode(buffer, 5, maxDynamicTableSize); 189 _context.resize(maxDynamicTableSize); 190 } 191 192 void encode(ByteBuffer buffer, HttpField field) { 193 if (field.getValue() == null) 194 field = new HttpField(field.getHeader(), field.getName(), ""); 195 196 int field_size = cast(int)(field.getName().length + field.getValue().length); 197 _headerListSize += field_size + 32; 198 199 int p = _debug ? buffer.position() : -1; 200 string encoding = null; 201 202 HttpHeader he = field.getHeader(); 203 // tracef("encoding: %s, hash: %d", field.toString(), field.toHash()); 204 205 // Is there an entry for the field? 206 Entry entry = _context.get(field); 207 if (entry !is null) { 208 // Known field entry, so encode it as indexed 209 if (entry.isStatic()) { 210 buffer.put((cast(StaticEntry) entry).getEncodedField()); 211 version(HUNT_DEBUG) 212 encoding = "IdxFieldS1"; 213 } else { 214 int index = _context.index(entry); 215 buffer.put(cast(byte) 0x80); 216 NBitInteger.encode(buffer, 7, index); 217 version(HUNT_DEBUG) 218 encoding = "IdxField" ~ (entry.isStatic() ? "S" : "") ~ to!string(1 + NBitInteger.octectsNeeded(7, index)); 219 } 220 } else { 221 // Unknown field entry, so we will have to send literally. 222 bool indexed; 223 224 // But do we know it's name? 225 HttpHeader header = field.getHeader(); 226 227 // Select encoding strategy 228 if (header == HttpHeader.Null) { 229 // Select encoding strategy for unknown header names 230 Entry name = _context.get(field.getName()); 231 232 if (typeid(field) == typeid(PreEncodedHttpField)) { 233 int i = buffer.position(); 234 (cast(PreEncodedHttpField) field).putTo(buffer, HttpVersion.HTTP_2); 235 byte b = buffer.get(i); 236 indexed = b < 0 || b >= 0x40; 237 version(HUNT_DEBUG) 238 encoding = indexed ? "PreEncodedIdx" : "PreEncoded"; 239 } 240 // has the custom header name been seen before? 241 else if (name is null) { 242 // unknown name and value, so let's index this just in case it is 243 // the first time we have seen a custom name or a custom field. 244 // unless the name is changing, this is worthwhile 245 indexed = true; 246 encodeName(buffer, cast(byte) 0x40, 6, field.getName(), null); 247 encodeValue(buffer, true, field.getValue()); 248 version(HUNT_DEBUG) 249 encoding = "LitHuffNHuffVIdx"; 250 } else { 251 // known custom name, but unknown value. 252 // This is probably a custom field with changing value, so don't index. 253 indexed = false; 254 encodeName(buffer, cast(byte) 0x00, 4, field.getName(), null); 255 encodeValue(buffer, true, field.getValue()); 256 version(HUNT_DEBUG) 257 encoding = "LitHuffNHuffV!Idx"; 258 } 259 } else { 260 // Select encoding strategy for known header names 261 Entry name = _context.get(header); 262 if(name is null) 263 warningf("no entry found for header: %s", header.toString()); 264 // else 265 // tracef("Entry=%s, header: name=%s, ordinal=%d", name.toString(), header.toString(), header.ordinal()); 266 267 if (typeid(field) == typeid(PreEncodedHttpField)) { 268 // Preencoded field 269 int i = buffer.position(); 270 (cast(PreEncodedHttpField) field).putTo(buffer, HttpVersion.HTTP_2); 271 byte b = buffer.get(i); 272 indexed = b < 0 || b >= 0x40; 273 version(HUNT_DEBUG) 274 encoding = indexed ? "PreEncodedIdx" : "PreEncoded"; 275 } else if (__DO_NOT_INDEX.contains(header)) { 276 // Non indexed field 277 indexed = false; 278 bool never_index = __NEVER_INDEX.contains(header); 279 bool huffman = !__DO_NOT_HUFFMAN.contains(header); 280 encodeName(buffer, never_index ? cast(byte) 0x10 : cast(byte) 0x00, 4, header.asString(), name); 281 encodeValue(buffer, huffman, field.getValue()); 282 283 version(HUNT_DEBUG) 284 { 285 encoding = "Lit" ~ ((name is null) ? "HuffN" : ("IdxN" ~ (name.isStatic() ? "S" : "") ~ 286 to!string(1 + NBitInteger.octectsNeeded(4, _context.index(name))))) ~ 287 (huffman ? "HuffV" : "LitV") ~ 288 (indexed ? "Idx" : (never_index ? "!!Idx" : "!Idx")); 289 290 } 291 } else if (field_size >= _context.getMaxDynamicTableSize() || header == HttpHeader.CONTENT_LENGTH && 292 field.getValue().length > 2) { 293 // Non indexed if field too large or a content length for 3 digits or more 294 indexed = false; 295 encodeName(buffer, cast(byte) 0x00, 4, header.asString(), name); 296 encodeValue(buffer, true, field.getValue()); 297 version(HUNT_DEBUG) 298 encoding = "LitIdxNS" ~ to!string(1 + NBitInteger.octectsNeeded(4, _context.index(name))) ~ "HuffV!Idx"; 299 } else { 300 // indexed 301 indexed = true; 302 bool huffman = !__DO_NOT_HUFFMAN.contains(header); 303 encodeName(buffer, cast(byte) 0x40, 6, header.asString(), name); 304 encodeValue(buffer, huffman, field.getValue()); 305 version(HUNT_DEBUG){ 306 encoding = ((name is null) ? "LitHuffN" : ("LitIdxN" ~ (name.isStatic() ? "S" : "") ~ 307 to!string(1 + NBitInteger.octectsNeeded(6, _context.index(name))))) ~ 308 (huffman ? "HuffVIdx" : "LitVIdx"); 309 } 310 } 311 } 312 313 // If we want the field referenced, then we add it to our 314 // table and reference set. 315 if (indexed) 316 if (_context.add(field) is null) 317 throw new IllegalStateException(""); 318 } 319 320 version(HUNT_HTTP_DEBUG) 321 { 322 int e = buffer.position(); 323 tracef("encode %s: '%s' to '%s'", encoding, field, 324 ConverterUtils.toHexString(buffer.array(), buffer.arrayOffset() + p, e - p)); 325 } 326 } 327 328 private void encodeName(ByteBuffer buffer, byte mask, int bits, string name, Entry entry) { 329 buffer.put(mask); 330 if (entry is null) { 331 // leave name index bits as 0 332 // Encode the name always with lowercase huffman 333 buffer.put(cast(byte) 0x80); 334 NBitInteger.encode(buffer, 7, Huffman.octetsNeededLC(name)); 335 Huffman.encodeLC(buffer, name); 336 } else { 337 NBitInteger.encode(buffer, bits, _context.index(entry)); 338 } 339 } 340 341 static void encodeValue(ByteBuffer buffer, bool huffman, string value) { 342 if (huffman) { 343 // huffman literal value 344 buffer.put(cast(byte) 0x80); 345 NBitInteger.encode(buffer, 7, Huffman.octetsNeeded(value)); 346 Huffman.encode(buffer, value); 347 } else { 348 // add literal assuming iso_8859_1 349 buffer.put(cast(byte) 0x00); 350 NBitInteger.encode(buffer, 7, cast(int)value.length); 351 for (size_t i = 0; i < value.length; i++) { 352 char c = value[i]; 353 if (c < ' ' || c > 127) 354 throw new IllegalArgumentException(""); 355 buffer.put(cast(byte) c); 356 } 357 } 358 } 359 } 360 361 362 /** 363 */ 364 class HpackFieldPreEncoder : HttpFieldPreEncoder { 365 366 override 367 HttpVersion getHttpVersion() { 368 return HttpVersion.HTTP_2; 369 } 370 371 override 372 byte[] getEncodedField(HttpHeader header, string name, string value) { 373 bool not_indexed = HpackEncoder.__DO_NOT_INDEX.contains(header); 374 375 ByteBuffer buffer = BufferUtils.allocate(cast(int) (name.length + value.length + 10)); 376 BufferUtils.clearToFill(buffer); 377 bool huffman; 378 int bits; 379 380 if (not_indexed) { 381 // Non indexed field 382 bool never_index = HpackEncoder.__NEVER_INDEX.contains(header); 383 huffman = !HpackEncoder.__DO_NOT_HUFFMAN.contains(header); 384 buffer.put(never_index ? cast(byte) 0x10 : cast(byte) 0x00); 385 bits = 4; 386 } else if (header == HttpHeader.CONTENT_LENGTH && value.length > 1) { 387 // Non indexed content length for 2 digits or more 388 buffer.put(cast(byte) 0x00); 389 huffman = true; 390 bits = 4; 391 } else { 392 // indexed 393 buffer.put(cast(byte) 0x40); 394 huffman = !HpackEncoder.__DO_NOT_HUFFMAN.contains(header); 395 bits = 6; 396 } 397 398 int name_idx = HpackContext.staticIndex(header); 399 if (name_idx > 0) 400 NBitInteger.encode(buffer, bits, name_idx); 401 else { 402 buffer.put(cast(byte) 0x80); 403 NBitInteger.encode(buffer, 7, Huffman.octetsNeededLC(name)); 404 Huffman.encodeLC(buffer, name); 405 } 406 407 HpackEncoder.encodeValue(buffer, huffman, value); 408 409 BufferUtils.flipToFlush(buffer, 0); 410 return BufferUtils.toArray(buffer); 411 } 412 } 413 414 415 /** 416 * Pre encoded HttpField. 417 * <p>A HttpField that will be cached and used many times can be created as 418 * a {@link PreEncodedHttpField}, which will use the {@link HttpFieldPreEncoder} 419 * instances discovered by the {@link ServiceLoader} to pre-encode the header 420 * for each version of HTTP in use. This will save garbage 421 * and CPU each time the field is encoded into a response. 422 * </p> 423 */ 424 class PreEncodedHttpField : HttpField { 425 private __gshared static HttpFieldPreEncoder[] __encoders; 426 427 private byte[][] _encodedField = new byte[][2]; 428 429 shared static this() 430 { 431 __encoders ~= new HpackFieldPreEncoder(); 432 __encoders ~= new Http1FieldPreEncoder(); 433 // __encoders = [ null, 434 // new Http1FieldPreEncoder()]; 435 } 436 437 this(HttpHeader header, string name, string value) { 438 super(header, name, value); 439 440 foreach (HttpFieldPreEncoder e ; __encoders) { 441 if(e is null) 442 continue; 443 _encodedField[e.getHttpVersion() == HttpVersion.HTTP_2 ? 1 : 0] = e.getEncodedField(header, header.asString(), value); 444 } 445 } 446 447 this(HttpHeader header, string value) { 448 this(header, header.asString(), value); 449 } 450 451 this(string name, string value) { 452 this(HttpHeader.Null, name, value); 453 } 454 455 void putTo(ByteBuffer bufferInFillMode, HttpVersion ver) { 456 bufferInFillMode.put(_encodedField[ver == HttpVersion.HTTP_2 ? 1 : 0]); 457 } 458 }