1 module hunt.http.codec.http.decode.MultipartFormParser; 2 3 import hunt.http.codec.http.decode.MultipartParser; 4 import hunt.http.codec.http.model.MultiException; 5 6 import hunt.http.MultipartOptions; 7 import hunt.http.MultipartForm; 8 9 import hunt.collection; 10 import hunt.stream; 11 import hunt.Exceptions; 12 import hunt.logging; 13 import hunt.text.Common; 14 import hunt.text.QuotedStringTokenizer; 15 import hunt.text.StringUtils; 16 17 import std.array; 18 import std.conv; 19 import std.concurrency : initOnce; 20 import std.file; 21 import std.path; 22 import std.regex; 23 import std.string; 24 import std.uni; 25 26 deprecated("Using MultipartFormParser instead.") 27 alias MultipartFormInputStream = MultipartFormParser; 28 29 30 void deleteOnExit(string file) { 31 if(file.exists) { 32 version(HUNT_DEBUG) infof("File removed: %s", file); 33 file.remove(); 34 } else { 35 version(HUNT_DEBUG) warningf("File not exists: %s", file); 36 } 37 } 38 39 /** 40 * MultipartFormParser 41 * <p> 42 * Handle a MultipartForm Mime input stream, breaking it up on the boundary into files and strings. 43 * 44 * @see <a href="https://tools.ietf.org/html/rfc7578">https://tools.ietf.org/html/rfc7578</a> 45 */ 46 class MultipartFormParser { 47 48 static MultiMap!(Part) EMPTY_MAP() { 49 __gshared MultiMap!(Part) inst; 50 return initOnce!inst(new MultiMap!(Part)(Collections.emptyMap!(string, List!(Part))())); 51 } 52 // __gshared MultipartOptions DEFAULT_MULTIPART_CONFIG; 53 54 private int _bufferSize = 16 * 1024; 55 protected InputStream _in; 56 protected MultipartOptions _config; 57 protected string _contentType; 58 protected MultiMap!(Part) _parts; 59 protected Exception _err; 60 protected string _tmpDir; 61 protected string _contextTmpDir; 62 protected bool _deleteOnExit; 63 protected bool _parsed; 64 65 deprecated("It's removed. Just using MultipartForm instead.") 66 alias MultiPart = MultipartForm; 67 68 /** 69 * @param input Request input stream 70 * @param contentType Content-Type header 71 * @param config MultipartOptions 72 * @param contextTmpDir tempdir 73 */ 74 this(InputStream input, string contentType, MultipartOptions config, string contextTmpDir) { 75 _contentType = contentType; 76 _config = config; 77 if (contextTmpDir.empty) 78 _contextTmpDir = tempDir(); 79 else 80 _contextTmpDir = contextTmpDir; 81 82 if (_config is null) { 83 string rootPath = dirName(thisExePath); 84 string abslutePath = buildPath(rootPath, _contextTmpDir); 85 if (!abslutePath.exists()) 86 abslutePath.mkdirRecurse(); 87 _config = new MultipartOptions(abslutePath); 88 } 89 90 // if (input instanceof ServletInputStream) { 91 // if (((ServletInputStream) input).isFinished()) { 92 // _parts = EMPTY_MAP; 93 // _parsed = true; 94 // return; 95 // } 96 // } 97 _in = new BufferedInputStream(input); 98 } 99 100 /** 101 * @return whether the list of parsed parts is empty 102 */ 103 bool isEmpty() { 104 if (_parts is null) 105 return true; 106 107 List!(Part)[] values = _parts.values(); 108 foreach (List!(Part) partList ; values) { 109 if (partList.size() != 0) 110 return false; 111 } 112 113 return true; 114 } 115 116 /** 117 * Get the already parsed parts. 118 * 119 * @return the parts that were parsed 120 */ 121 // deprecated("") 122 // Collection!(Part) getParsedParts() { 123 // if (_parts is null) 124 // return Collections.emptyList(); 125 126 // Collection<List!(Part)> values = _parts.values(); 127 // List!(Part) parts = new ArrayList<>(); 128 // for (List!(Part) o : values) { 129 // List!(Part) asList = LazyList.getList(o, false); 130 // parts.addAll(asList); 131 // } 132 // return parts; 133 // } 134 135 /** 136 * Delete any tmp storage for parts, and clear out the parts list. 137 */ 138 void deleteParts() { 139 if (!_parsed) 140 return; 141 142 Part[] parts; 143 try { 144 parts = getParts(); 145 } catch (IOException e) { 146 throw new RuntimeException(e); 147 } 148 MultiException err = new MultiException(); 149 150 foreach (Part p ; parts) { 151 try { 152 (cast(MultipartForm) p).cleanUp(); 153 } catch (Exception e) { 154 err.add(e); 155 } 156 } 157 _parts.clear(); 158 159 err.ifExceptionThrowRuntime(); 160 } 161 162 /** 163 * Parse, if necessary, the multipart data and return the list of Parts. 164 * 165 * @return the parts 166 * @throws IOException if unable to get the parts 167 */ 168 Part[] getParts() { 169 if (!_parsed) 170 parse(); 171 throwIfError(); 172 173 List!(Part)[] values = _parts.values(); 174 Part[] parts; 175 foreach (List!(Part) o ; values) { 176 foreach(Part p; o) { 177 parts ~= p; 178 } 179 } 180 return parts; 181 } 182 183 /** 184 * Get the named Part. 185 * 186 * @param name the part name 187 * @return the parts 188 * @throws IOException if unable to get the part 189 */ 190 Part getPart(string name) { 191 if (!_parsed) 192 parse(); 193 throwIfError(); 194 return _parts.getValue(name, 0); 195 } 196 197 /** 198 * Throws an exception if one has been latched. 199 * 200 * @throws IOException the exception (if present) 201 */ 202 protected void throwIfError() { 203 if (_err !is null) { 204 _err.next = (new Exception("")); 205 auto ioException = cast(IOException) _err; 206 if (ioException !is null) 207 throw ioException; 208 auto illegalStateException = cast(IllegalStateException) _err; 209 if (illegalStateException !is null) 210 throw illegalStateException; 211 throw new IllegalStateException(_err); 212 } 213 } 214 215 /** 216 * Parse, if necessary, the multipart stream. 217 */ 218 protected void parse() { 219 // have we already parsed the input? 220 if (_parsed) 221 return; 222 _parsed = true; 223 224 scope(exit) _in.close(); 225 226 try { 227 doParse(); 228 } catch (Exception e) { 229 warningf("Error occurred while parsing: %s", e.msg); 230 version(HUNT_HTTP_DEBUG) warning(e); 231 _err = e; 232 } 233 } 234 235 private void doParse() { 236 // initialize 237 _parts = new MultiMap!Part(); 238 239 // if its not a multipart request, don't parse it 240 if (_contentType.empty() || !_contentType.startsWith("multipart/form-data")) 241 return; 242 243 // sort out the location to which to write the files 244 string location = _config.getLocation(); 245 if (location.empty()) 246 _tmpDir = _contextTmpDir; 247 else 248 _tmpDir = buildPath(_contextTmpDir, location); 249 250 version(HUNT_HTTP_DEBUG) { 251 if (!_tmpDir.exists()) 252 _tmpDir.mkdirRecurse(); 253 } 254 255 string contentTypeBoundary = ""; 256 int bstart = cast(int)_contentType.indexOf("boundary="); 257 if (bstart >= 0) { 258 ptrdiff_t bend = _contentType.indexOf(";", bstart); 259 bend = (bend < 0 ? _contentType.length : bend); 260 contentTypeBoundary = QuotedStringTokenizer.unquote(value(_contentType[bstart .. bend]).strip()); 261 } 262 263 Handler handler = new Handler(); 264 MultipartParser parser = new MultipartParser(handler, contentTypeBoundary); 265 266 // Create a buffer to store data from stream // 267 byte[] data = new byte[_bufferSize]; 268 int len = 0; 269 270 /** 271 * keep running total of size of bytes read from input and throw an exception if exceeds MultipartOptions._maxRequestSize 272 */ 273 long total = 0; 274 long maxRequestSize = _config.getMaxRequestSize(); 275 276 version(HUNT_HTTP_DEBUG) { 277 int size = _in.available(); 278 tracef("available: %d, maxRequestSize: %d", size, maxRequestSize); 279 if(size == 0) { 280 warning("no data available in inputStream"); 281 } 282 } 283 284 while (true) { 285 len = _in.read(data); 286 287 if (len > 0) { 288 total += len; 289 if (maxRequestSize > 0 && total > maxRequestSize) { 290 string msg = format("Request exceeds maxRequestSize (%d)", maxRequestSize); 291 version(HUNT_DEBUG) warning(msg); 292 _err = new IllegalStateException(msg); 293 return; 294 } 295 296 ByteBuffer buffer = BufferUtils.toBuffer(data); 297 buffer.limit(len); 298 if (parser.parse(buffer, false)) 299 break; 300 301 if (buffer.hasRemaining()) 302 throw new IllegalStateException("Buffer did not fully consume"); 303 304 } else if (len == -1) { 305 version(HUNT_HTTP_DEBUG) trace("no more data avaiable"); 306 parser.parse(BufferUtils.EMPTY_BUFFER, true); 307 break; 308 } 309 } 310 311 // check for exceptions 312 if (_err !is null) { 313 return; 314 } 315 316 // check we read to the end of the message 317 if (parser.getState() != MultipartParser.State.END) { 318 if (parser.getState() == MultipartParser.State.PREAMBLE) 319 _err = new IOException("Missing initial multi part boundary"); 320 else 321 _err = new IOException("Incomplete Multipart"); 322 } 323 324 version(HUNT_HTTP_DEBUG) { 325 if(_err is null) { 326 info("Parsing completed"); 327 } else { 328 warningf("Parsing Completed with error: %s", _err); 329 } 330 } 331 } 332 333 class Handler : MultipartParserHandler { 334 private MultipartForm _part = null; 335 private string contentDisposition = null; 336 private string contentType = null; 337 private MultiMap!string headers; 338 339 this() { 340 super(); 341 headers = new MultiMap!string(); 342 } 343 344 override 345 bool messageComplete() { 346 return true; 347 } 348 349 override 350 void parsedField(string key, string value) { 351 // Add to headers and mark if one of these fields. // 352 headers.put(std.uni.toLower(key), value); 353 if (key.equalsIgnoreCase("content-disposition")) 354 contentDisposition = value; 355 else if (key.equalsIgnoreCase("content-type")) 356 contentType = value; 357 358 // Transfer encoding is not longer considers as it is deprecated as per 359 // https://tools.ietf.org/html/rfc7578#section-4.7 360 361 } 362 363 override 364 bool headerComplete() { 365 version(HUNT_HTTP_DEBUG) { 366 tracef("headerComplete %s", this); 367 } 368 369 try { 370 // Extract content-disposition 371 bool form_data = false; 372 if (contentDisposition is null) { 373 throw new IOException("Missing content-disposition"); 374 } 375 376 QuotedStringTokenizer tok = new QuotedStringTokenizer(contentDisposition, ";", false, true); 377 string name = null; 378 string filename = null; 379 while (tok.hasMoreTokens()) { 380 string t = tok.nextToken().strip(); 381 string tl = std.uni.toLower(t); 382 if (tl.startsWith("form-data")) 383 form_data = true; 384 else if (tl.startsWith("name=")) 385 name = value(t); 386 else if (tl.startsWith("filename=")) 387 filename = filenameValue(t); 388 } 389 390 // Check disposition 391 if (!form_data) 392 throw new IOException("Part not form-data"); 393 394 // It is valid for reset and submit buttons to have an empty name. 395 // If no name is supplied, the browser skips sending the info for that field. 396 // However, if you supply the empty string as the name, the browser sends the 397 // field, with name as the empty string. So, only continue this loop if we 398 // have not yet seen a name field. 399 if (name is null) 400 throw new IOException("No name in part"); 401 402 403 // create the new part 404 _part = new MultipartForm(name, filename, _config); 405 _part.setTmpDir(_tmpDir); 406 _part.setHeaders(headers); 407 _part.setContentType(contentType); 408 _parts.add(name, _part); 409 410 try { 411 _part.open(); 412 } catch (IOException e) { 413 version(HUNT_HTTP_DEBUG) warning(e.msg); 414 version(HUNT_HTTP_DEBUG) warning(e); 415 _err = e; 416 return true; 417 } 418 } catch (Exception e) { 419 _err = e; 420 return true; 421 } 422 423 return false; 424 } 425 426 override 427 bool content(ByteBuffer buffer, bool last) { 428 if (_part is null) 429 return false; 430 431 if (BufferUtils.hasContent(buffer)) { 432 try { 433 _part.write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); 434 } catch (IOException e) { 435 _err = e; 436 return true; 437 } 438 } 439 440 if (last) { 441 try { 442 _part.close(); 443 } catch (IOException e) { 444 _err = e; 445 return true; 446 } 447 } 448 449 return false; 450 } 451 452 override 453 void startPart() { 454 reset(); 455 } 456 457 override 458 void earlyEOF() { 459 version(HUNT_HTTP_DEBUG) 460 tracef("Early EOF %s", this.outer); 461 } 462 463 void reset() { 464 _part = null; 465 contentDisposition = null; 466 contentType = null; 467 headers = new MultiMap!string(); 468 } 469 } 470 471 void setDeleteOnExit(bool deleteOnExit) { 472 _deleteOnExit = deleteOnExit; 473 } 474 475 bool isDeleteOnExit() { 476 return _deleteOnExit; 477 } 478 479 /* ------------------------------------------------------------ */ 480 private string value(string nameEqualsValue) { 481 ptrdiff_t idx = nameEqualsValue.indexOf('='); 482 string value = nameEqualsValue[idx + 1 .. $].strip(); 483 return QuotedStringTokenizer.unquoteOnly(value); 484 } 485 486 /* ------------------------------------------------------------ */ 487 private string filenameValue(string nameEqualsValue) { 488 ptrdiff_t idx = nameEqualsValue.indexOf('='); 489 string value = nameEqualsValue[idx + 1 .. $].strip(); 490 auto pattern = ctRegex!(".??[a-z,A-Z]\\:\\\\[^\\\\].*"); 491 // if (value.matches(".??[a-z,A-Z]\\:\\\\[^\\\\].*")) { 492 auto m = matchFirst(value, pattern); 493 if(!m.empty) { 494 // incorrectly escaped IE filenames that have the whole path 495 // we just strip any leading & trailing quotes and leave it as is 496 char first = value[0]; 497 if (first == '"' || first == '\'') 498 value = value[1 .. $]; 499 char last = value[$ - 1]; 500 if (last == '"' || last == '\'') 501 value = value[0 .. $ - 1]; 502 503 return value; 504 } else 505 // unquote the string, but allow any backslashes that don't 506 // form a valid escape sequence to remain as many browsers 507 // even on *nix systems will not escape a filename containing 508 // backslashes 509 return QuotedStringTokenizer.unquoteOnly(value, true); 510 } 511 512 } 513