1 module hunt.http.codec.http.hpack.HpackEncoder; 2 3 import hunt.http.codec.http.hpack.HpackContext; 4 import hunt.http.codec.http.hpack.Huffman; 5 import hunt.http.codec.http.hpack.NBitInteger; 6 7 // import hunt.http.codec.http.model; 8 9 import hunt.http.codec.http.model.HttpField; 10 import hunt.http.codec.http.model.HttpFields; 11 import hunt.http.codec.http.model.HttpHeader; 12 import hunt.http.codec.http.model.HttpStatus; 13 import hunt.http.codec.http.model.HttpScheme; 14 import hunt.http.codec.http.model.HttpVersion; 15 import hunt.http.codec.http.model.MetaData; 16 17 import hunt.container.ByteBuffer; 18 import hunt.container.BufferUtils; 19 20 import hunt.http.codec.http.encode.Http1FieldPreEncoder; 21 import hunt.http.codec.http.encode.HttpFieldPreEncoder; 22 import hunt.http.codec.http.hpack.NBitInteger; 23 import hunt.http.codec.http.hpack.Huffman; 24 import hunt.http.codec.http.hpack.HpackContext; 25 26 // import hunt.http.codec.http.model.HttpHeader; 27 // import hunt.http.codec.http.model.HttpVersion; 28 29 import hunt.lang.exception; 30 import hunt.util.TypeUtils; 31 32 33 // import hunt.http.codec.http.model.HttpField; 34 // import hunt.http.codec.http.model.HttpHeader; 35 // import hunt.http.codec.http.model.HttpVersion; 36 37 import hunt.logging; 38 39 import std.range; 40 import std.algorithm; 41 import std.conv; 42 43 alias Entry = HpackContext.Entry; 44 alias StaticEntry = HpackContext.StaticEntry; 45 46 /** 47 */ 48 class HpackEncoder { 49 50 private __gshared static HttpField[599] __status; 51 52 enum HttpHeader[] __DO_NOT_HUFFMAN = [ 53 HttpHeader.AUTHORIZATION, 54 HttpHeader.CONTENT_MD5, 55 HttpHeader.PROXY_AUTHENTICATE, 56 HttpHeader.PROXY_AUTHORIZATION]; 57 58 enum HttpHeader[] __DO_NOT_INDEX = [ 59 // HttpHeader.C_PATH, // TODO more data needed 60 // HttpHeader.DATE, // TODO more data needed 61 HttpHeader.AUTHORIZATION, 62 HttpHeader.CONTENT_MD5, 63 HttpHeader.CONTENT_RANGE, 64 HttpHeader.ETAG, 65 HttpHeader.IF_MODIFIED_SINCE, 66 HttpHeader.IF_UNMODIFIED_SINCE, 67 HttpHeader.IF_NONE_MATCH, 68 HttpHeader.IF_RANGE, 69 HttpHeader.IF_MATCH, 70 HttpHeader.LOCATION, 71 HttpHeader.RANGE, 72 HttpHeader.RETRY_AFTER, 73 // HttpHeader.EXPIRES, 74 HttpHeader.LAST_MODIFIED, 75 HttpHeader.SET_COOKIE, 76 HttpHeader.SET_COOKIE2]; 77 78 79 enum HttpHeader[] __NEVER_INDEX = [ 80 HttpHeader.AUTHORIZATION, 81 HttpHeader.SET_COOKIE, 82 HttpHeader.SET_COOKIE2]; 83 84 shared static this() { 85 foreach (HttpStatus.Code code ; HttpStatus.Code.values()) 86 __status[code.getCode()] = new PreEncodedHttpField(HttpHeader.C_STATUS, std.conv.to!(string)(code.getCode())); 87 } 88 89 private HpackContext _context; 90 private bool _debug; 91 private int _remoteMaxDynamicTableSize; 92 private int _localMaxDynamicTableSize; 93 private int _maxHeaderListSize; 94 private int _headerListSize; 95 96 this() { 97 this(4096, 4096, -1); 98 } 99 100 this(int localMaxDynamicTableSize) { 101 this(localMaxDynamicTableSize, 4096, -1); 102 } 103 104 this(int localMaxDynamicTableSize, int remoteMaxDynamicTableSize) { 105 this(localMaxDynamicTableSize, remoteMaxDynamicTableSize, -1); 106 } 107 108 this(int localMaxDynamicTableSize, int remoteMaxDynamicTableSize, int maxHeaderListSize) { 109 _context = new HpackContext(remoteMaxDynamicTableSize); 110 _remoteMaxDynamicTableSize = remoteMaxDynamicTableSize; 111 _localMaxDynamicTableSize = localMaxDynamicTableSize; 112 _maxHeaderListSize = maxHeaderListSize; 113 _debug = true; //log.isDebugEnabled(); 114 } 115 116 int getMaxHeaderListSize() { 117 return _maxHeaderListSize; 118 } 119 120 void setMaxHeaderListSize(int maxHeaderListSize) { 121 _maxHeaderListSize = maxHeaderListSize; 122 } 123 124 HpackContext getHpackContext() { 125 return _context; 126 } 127 128 void setRemoteMaxDynamicTableSize(int remoteMaxDynamicTableSize) { 129 _remoteMaxDynamicTableSize = remoteMaxDynamicTableSize; 130 } 131 132 void setLocalMaxDynamicTableSize(int localMaxDynamicTableSize) { 133 _localMaxDynamicTableSize = localMaxDynamicTableSize; 134 } 135 136 void encode(ByteBuffer buffer, MetaData metadata) { 137 version(HUNT_DEBUG) 138 tracef("CtxTbl[%x] encoding", _context.toHash()); 139 140 _headerListSize = 0; 141 int pos = buffer.position(); 142 143 // Check the dynamic table sizes! 144 int maxDynamicTableSize = std.algorithm.min(_remoteMaxDynamicTableSize, _localMaxDynamicTableSize); 145 if (maxDynamicTableSize != _context.getMaxDynamicTableSize()) 146 encodeMaxDynamicTableSize(buffer, maxDynamicTableSize); 147 148 // Add Request/response meta fields 149 if (metadata.isRequest()) { 150 HttpRequest request = cast(HttpRequest) metadata; 151 152 // TODO optimise these to avoid HttpField creation 153 string scheme = request.getURI().getScheme(); 154 encode(buffer, new HttpField(HttpHeader.C_SCHEME, scheme.empty ? HttpScheme.HTTP : scheme)); 155 encode(buffer, new HttpField(HttpHeader.C_METHOD, request.getMethod())); 156 encode(buffer, new HttpField(HttpHeader.C_AUTHORITY, request.getURI().getAuthority())); 157 encode(buffer, new HttpField(HttpHeader.C_PATH, request.getURI().getPathQuery())); 158 } else if (metadata.isResponse()) { 159 HttpResponse response = cast(HttpResponse) metadata; 160 int code = response.getStatus(); 161 HttpField status = code < __status.length ? __status[code] : null; 162 if (status is null) 163 status = new HttpField.IntValueHttpField(HttpHeader.C_STATUS, code); 164 encode(buffer, status); 165 } 166 167 // Add all the other fields 168 foreach (HttpField field ; metadata) 169 encode(buffer, field); 170 171 // Check size 172 if (_maxHeaderListSize > 0 && _headerListSize > _maxHeaderListSize) { 173 warningf("Header list size too large %s > %s for %s", _headerListSize, _maxHeaderListSize); 174 version(HUNT_DEBUG) 175 tracef("metadata=%s", metadata); 176 } 177 178 version(HUNT_DEBUG) 179 tracef("CtxTbl[%x] encoded %d octets", _context.toHash(), buffer.position() - pos); 180 } 181 182 void encodeMaxDynamicTableSize(ByteBuffer buffer, int maxDynamicTableSize) { 183 if (maxDynamicTableSize > _remoteMaxDynamicTableSize) 184 throw new IllegalArgumentException(""); 185 buffer.put(cast(byte) 0x20); 186 NBitInteger.encode(buffer, 5, maxDynamicTableSize); 187 _context.resize(maxDynamicTableSize); 188 } 189 190 void encode(ByteBuffer buffer, HttpField field) { 191 if (field.getValue() == null) 192 field = new HttpField(field.getHeader(), field.getName(), ""); 193 194 int field_size = cast(int)(field.getName().length + field.getValue().length); 195 _headerListSize += field_size + 32; 196 197 int p = _debug ? buffer.position() : -1; 198 string encoding = null; 199 200 HttpHeader he = field.getHeader(); 201 // tracef("encoding: %s, hash: %d", field.toString(), field.toHash()); 202 203 // Is there an entry for the field? 204 Entry entry = _context.get(field); 205 if (entry !is null) { 206 // Known field entry, so encode it as indexed 207 if (entry.isStatic()) { 208 buffer.put((cast(StaticEntry) entry).getEncodedField()); 209 version(HUNT_DEBUG) 210 encoding = "IdxFieldS1"; 211 } else { 212 int index = _context.index(entry); 213 buffer.put(cast(byte) 0x80); 214 NBitInteger.encode(buffer, 7, index); 215 version(HUNT_DEBUG) 216 encoding = "IdxField" ~ (entry.isStatic() ? "S" : "") ~ to!string(1 + NBitInteger.octectsNeeded(7, index)); 217 } 218 } else { 219 // Unknown field entry, so we will have to send literally. 220 bool indexed; 221 222 // But do we know it's name? 223 HttpHeader header = field.getHeader(); 224 225 // Select encoding strategy 226 if (header == HttpHeader.Null) { 227 // Select encoding strategy for unknown header names 228 Entry name = _context.get(field.getName()); 229 230 if (typeid(field) == typeid(PreEncodedHttpField)) { 231 int i = buffer.position(); 232 (cast(PreEncodedHttpField) field).putTo(buffer, HttpVersion.HTTP_2); 233 byte b = buffer.get(i); 234 indexed = b < 0 || b >= 0x40; 235 version(HUNT_DEBUG) 236 encoding = indexed ? "PreEncodedIdx" : "PreEncoded"; 237 } 238 // has the custom header name been seen before? 239 else if (name is null) { 240 // unknown name and value, so let's index this just in case it is 241 // the first time we have seen a custom name or a custom field. 242 // unless the name is changing, this is worthwhile 243 indexed = true; 244 encodeName(buffer, cast(byte) 0x40, 6, field.getName(), null); 245 encodeValue(buffer, true, field.getValue()); 246 version(HUNT_DEBUG) 247 encoding = "LitHuffNHuffVIdx"; 248 } else { 249 // known custom name, but unknown value. 250 // This is probably a custom field with changing value, so don't index. 251 indexed = false; 252 encodeName(buffer, cast(byte) 0x00, 4, field.getName(), null); 253 encodeValue(buffer, true, field.getValue()); 254 version(HUNT_DEBUG) 255 encoding = "LitHuffNHuffV!Idx"; 256 } 257 } else { 258 // Select encoding strategy for known header names 259 Entry name = _context.get(header); 260 if(name is null) 261 warningf("no entry found for header: %s", header.toString()); 262 // else 263 // tracef("Entry=%s, header: name=%s, ordinal=%d", name.toString(), header.toString(), header.ordinal()); 264 265 if (typeid(field) == typeid(PreEncodedHttpField)) { 266 // Preencoded field 267 int i = buffer.position(); 268 (cast(PreEncodedHttpField) field).putTo(buffer, HttpVersion.HTTP_2); 269 byte b = buffer.get(i); 270 indexed = b < 0 || b >= 0x40; 271 version(HUNT_DEBUG) 272 encoding = indexed ? "PreEncodedIdx" : "PreEncoded"; 273 } else if (__DO_NOT_INDEX.contains(header)) { 274 // Non indexed field 275 indexed = false; 276 bool never_index = __NEVER_INDEX.contains(header); 277 bool huffman = !__DO_NOT_HUFFMAN.contains(header); 278 encodeName(buffer, never_index ? cast(byte) 0x10 : cast(byte) 0x00, 4, header.asString(), name); 279 encodeValue(buffer, huffman, field.getValue()); 280 281 version(HUNT_DEBUG) 282 { 283 encoding = "Lit" ~ ((name is null) ? "HuffN" : ("IdxN" ~ (name.isStatic() ? "S" : "") ~ 284 to!string(1 + NBitInteger.octectsNeeded(4, _context.index(name))))) ~ 285 (huffman ? "HuffV" : "LitV") ~ 286 (indexed ? "Idx" : (never_index ? "!!Idx" : "!Idx")); 287 288 } 289 } else if (field_size >= _context.getMaxDynamicTableSize() || header == HttpHeader.CONTENT_LENGTH && 290 field.getValue().length > 2) { 291 // Non indexed if field too large or a content length for 3 digits or more 292 indexed = false; 293 encodeName(buffer, cast(byte) 0x00, 4, header.asString(), name); 294 encodeValue(buffer, true, field.getValue()); 295 version(HUNT_DEBUG) 296 encoding = "LitIdxNS" ~ to!string(1 + NBitInteger.octectsNeeded(4, _context.index(name))) ~ "HuffV!Idx"; 297 } else { 298 // indexed 299 indexed = true; 300 bool huffman = !__DO_NOT_HUFFMAN.contains(header); 301 encodeName(buffer, cast(byte) 0x40, 6, header.asString(), name); 302 encodeValue(buffer, huffman, field.getValue()); 303 version(HUNT_DEBUG){ 304 encoding = ((name is null) ? "LitHuffN" : ("LitIdxN" ~ (name.isStatic() ? "S" : "") ~ 305 to!string(1 + NBitInteger.octectsNeeded(6, _context.index(name))))) ~ 306 (huffman ? "HuffVIdx" : "LitVIdx"); 307 } 308 } 309 } 310 311 // If we want the field referenced, then we add it to our 312 // table and reference set. 313 if (indexed) 314 if (_context.add(field) is null) 315 throw new IllegalStateException(""); 316 } 317 318 // version(HUNT_DEBUG) 319 { 320 int e = buffer.position(); 321 tracef("encode %s:'%s' to '%s'", encoding, field, TypeUtils.toHexString(buffer.array(), buffer.arrayOffset() + p, e - p)); 322 } 323 } 324 325 private void encodeName(ByteBuffer buffer, byte mask, int bits, string name, Entry entry) { 326 buffer.put(mask); 327 if (entry is null) { 328 // leave name index bits as 0 329 // Encode the name always with lowercase huffman 330 buffer.put(cast(byte) 0x80); 331 NBitInteger.encode(buffer, 7, Huffman.octetsNeededLC(name)); 332 Huffman.encodeLC(buffer, name); 333 } else { 334 NBitInteger.encode(buffer, bits, _context.index(entry)); 335 } 336 } 337 338 static void encodeValue(ByteBuffer buffer, bool huffman, string value) { 339 if (huffman) { 340 // huffman literal value 341 buffer.put(cast(byte) 0x80); 342 NBitInteger.encode(buffer, 7, Huffman.octetsNeeded(value)); 343 Huffman.encode(buffer, value); 344 } else { 345 // add literal assuming iso_8859_1 346 buffer.put(cast(byte) 0x00); 347 NBitInteger.encode(buffer, 7, cast(int)value.length); 348 for (size_t i = 0; i < value.length; i++) { 349 char c = value[i]; 350 if (c < ' ' || c > 127) 351 throw new IllegalArgumentException(""); 352 buffer.put(cast(byte) c); 353 } 354 } 355 } 356 } 357 358 359 /** 360 */ 361 class HpackFieldPreEncoder : HttpFieldPreEncoder { 362 363 override 364 HttpVersion getHttpVersion() { 365 return HttpVersion.HTTP_2; 366 } 367 368 override 369 byte[] getEncodedField(HttpHeader header, string name, string value) { 370 bool not_indexed = HpackEncoder.__DO_NOT_INDEX.contains(header); 371 372 ByteBuffer buffer = BufferUtils.allocate(cast(int) (name.length + value.length + 10)); 373 BufferUtils.clearToFill(buffer); 374 bool huffman; 375 int bits; 376 377 if (not_indexed) { 378 // Non indexed field 379 bool never_index = HpackEncoder.__NEVER_INDEX.contains(header); 380 huffman = !HpackEncoder.__DO_NOT_HUFFMAN.contains(header); 381 buffer.put(never_index ? cast(byte) 0x10 : cast(byte) 0x00); 382 bits = 4; 383 } else if (header == HttpHeader.CONTENT_LENGTH && value.length > 1) { 384 // Non indexed content length for 2 digits or more 385 buffer.put(cast(byte) 0x00); 386 huffman = true; 387 bits = 4; 388 } else { 389 // indexed 390 buffer.put(cast(byte) 0x40); 391 huffman = !HpackEncoder.__DO_NOT_HUFFMAN.contains(header); 392 bits = 6; 393 } 394 395 int name_idx = HpackContext.staticIndex(header); 396 if (name_idx > 0) 397 NBitInteger.encode(buffer, bits, name_idx); 398 else { 399 buffer.put(cast(byte) 0x80); 400 NBitInteger.encode(buffer, 7, Huffman.octetsNeededLC(name)); 401 Huffman.encodeLC(buffer, name); 402 } 403 404 HpackEncoder.encodeValue(buffer, huffman, value); 405 406 BufferUtils.flipToFlush(buffer, 0); 407 return BufferUtils.toArray(buffer); 408 } 409 } 410 411 412 /** 413 * Pre encoded HttpField. 414 * <p>A HttpField that will be cached and used many times can be created as 415 * a {@link PreEncodedHttpField}, which will use the {@link HttpFieldPreEncoder} 416 * instances discovered by the {@link ServiceLoader} to pre-encode the header 417 * for each version of HTTP in use. This will save garbage 418 * and CPU each time the field is encoded into a response. 419 * </p> 420 */ 421 class PreEncodedHttpField : HttpField { 422 private __gshared static HttpFieldPreEncoder[] __encoders; 423 424 private byte[][] _encodedField = new byte[][2]; 425 426 shared static this() 427 { 428 __encoders ~= new HpackFieldPreEncoder(); 429 __encoders ~= new Http1FieldPreEncoder(); 430 // __encoders = [ null, 431 // new Http1FieldPreEncoder()]; 432 } 433 434 this(HttpHeader header, string name, string value) { 435 super(header, name, value); 436 437 foreach (HttpFieldPreEncoder e ; __encoders) { 438 if(e is null) 439 continue; 440 _encodedField[e.getHttpVersion() == HttpVersion.HTTP_2 ? 1 : 0] = e.getEncodedField(header, header.asString(), value); 441 } 442 } 443 444 this(HttpHeader header, string value) { 445 this(header, header.asString(), value); 446 } 447 448 this(string name, string value) { 449 this(HttpHeader.Null, name, value); 450 } 451 452 void putTo(ByteBuffer bufferInFillMode, HttpVersion ver) { 453 bufferInFillMode.put(_encodedField[ver == HttpVersion.HTTP_2 ? 1 : 0]); 454 } 455 }