1 module hunt.http.codec.http.decode.MultipartFormParser;
2 
3 import hunt.http.codec.http.decode.MultipartParser;
4 import hunt.http.codec.http.model.MultiException;
5 
6 import hunt.http.MultipartOptions;
7 import hunt.http.MultipartForm;
8 
9 import hunt.collection;
10 import hunt.stream;
11 import hunt.Exceptions;
12 import hunt.logging;
13 import hunt.text.Common;
14 import hunt.text.QuotedStringTokenizer;
15 import hunt.text.StringUtils;
16 
17 import std.array;
18 import std.conv;
19 import std.concurrency : initOnce;
20 import std.file;
21 import std.path;
22 import std.regex;
23 import std.string;
24 import std.uni;
25 
26 deprecated("Using MultipartFormParser instead.")
27 alias MultipartFormInputStream = MultipartFormParser;
28 
29 
30 void deleteOnExit(string file) {
31     if(file.exists) {
32         version(HUNT_DEBUG) infof("File removed: %s", file);
33         file.remove();
34     } else {
35         version(HUNT_DEBUG) warningf("File not exists: %s", file);
36     }
37 }
38 
39 /**
40  * MultipartFormParser
41  * <p>
42  * Handle a MultipartForm Mime input stream, breaking it up on the boundary into files and strings.
43  *
44  * @see <a href="https://tools.ietf.org/html/rfc7578">https://tools.ietf.org/html/rfc7578</a>
45  */
46 class MultipartFormParser {
47 
48     static MultiMap!(Part) EMPTY_MAP() {
49         __gshared MultiMap!(Part) inst;
50         return initOnce!inst(new MultiMap!(Part)(Collections.emptyMap!(string, List!(Part))()));
51     }
52     // __gshared MultipartOptions DEFAULT_MULTIPART_CONFIG;
53 
54     private int _bufferSize = 16 * 1024;
55     protected InputStream _in;
56     protected MultipartOptions _config;
57     protected string _contentType;
58     protected MultiMap!(Part) _parts;
59     protected Exception _err;
60     protected string _tmpDir;
61     protected string _contextTmpDir;
62     protected bool _deleteOnExit;
63     protected bool _parsed;
64 
65     deprecated("It's removed. Just using MultipartForm instead.")
66     alias MultiPart = MultipartForm;
67 
68     /**
69      * @param input         Request input stream
70      * @param contentType   Content-Type header
71      * @param config        MultipartOptions
72      * @param contextTmpDir tempdir
73      */
74     this(InputStream input, string contentType, MultipartOptions config, string contextTmpDir) {
75         _contentType = contentType;
76         _config = config;
77         if (contextTmpDir.empty)
78             _contextTmpDir = tempDir();
79         else
80             _contextTmpDir = contextTmpDir;
81 
82         if (_config is null) {
83             string rootPath = dirName(thisExePath);
84             string abslutePath = buildPath(rootPath, _contextTmpDir);
85             if (!abslutePath.exists())
86                 abslutePath.mkdirRecurse();
87             _config = new MultipartOptions(abslutePath);
88         }
89 
90         // if (input instanceof ServletInputStream) {
91         //     if (((ServletInputStream) input).isFinished()) {
92         //         _parts = EMPTY_MAP;
93         //         _parsed = true;
94         //         return;
95         //     }
96         // }
97         _in = new BufferedInputStream(input);
98     }
99 
100     /**
101      * @return whether the list of parsed parts is empty
102      */
103     bool isEmpty() {
104         if (_parts is null)
105             return true;
106 
107         List!(Part)[] values = _parts.values();
108         foreach (List!(Part) partList ; values) {
109             if (partList.size() != 0)
110                 return false;
111         }
112 
113         return true;
114     }
115 
116     /**
117      * Get the already parsed parts.
118      *
119      * @return the parts that were parsed
120      */
121     // deprecated("")
122     // Collection!(Part) getParsedParts() {
123     //     if (_parts is null)
124     //         return Collections.emptyList();
125 
126     //     Collection<List!(Part)> values = _parts.values();
127     //     List!(Part) parts = new ArrayList<>();
128     //     for (List!(Part) o : values) {
129     //         List!(Part) asList = LazyList.getList(o, false);
130     //         parts.addAll(asList);
131     //     }
132     //     return parts;
133     // }
134 
135     /**
136      * Delete any tmp storage for parts, and clear out the parts list.
137      */
138     void deleteParts() {
139         if (!_parsed)
140             return;
141 
142         Part[] parts;
143         try {
144             parts = getParts();
145         } catch (IOException e) {
146             throw new RuntimeException(e);
147         }
148         MultiException err = new MultiException();
149 
150         foreach (Part p ; parts) {
151             try {
152                 (cast(MultipartForm) p).cleanUp();
153             } catch (Exception e) {
154                 err.add(e);
155             }
156         }
157         _parts.clear();
158 
159         err.ifExceptionThrowRuntime();
160     }
161 
162     /**
163      * Parse, if necessary, the multipart data and return the list of Parts.
164      *
165      * @return the parts
166      * @throws IOException if unable to get the parts
167      */
168     Part[] getParts() {
169         if (!_parsed)
170             parse();
171         throwIfError();
172 
173         List!(Part)[] values = _parts.values();
174         Part[] parts;
175         foreach (List!(Part) o ; values) {
176             foreach(Part p; o) {
177                 parts ~= p;
178             }
179         }
180         return parts;
181     }
182 
183     /**
184      * Get the named Part.
185      *
186      * @param name the part name
187      * @return the parts
188      * @throws IOException if unable to get the part
189      */
190     Part getPart(string name) {
191         if (!_parsed)
192             parse();
193         throwIfError();
194         return _parts.getValue(name, 0);
195     }
196 
197     /**
198      * Throws an exception if one has been latched.
199      *
200      * @throws IOException the exception (if present)
201      */
202     protected void throwIfError() {
203         if (_err !is null) {
204             _err.next = (new Exception(""));
205             auto ioException = cast(IOException) _err;
206             if (ioException !is null)
207                 throw ioException;
208             auto illegalStateException = cast(IllegalStateException) _err;
209             if (illegalStateException !is null)
210                 throw illegalStateException;
211             throw new IllegalStateException(_err);
212         }
213     }
214 
215     /**
216      * Parse, if necessary, the multipart stream.
217      */
218     protected void parse() {
219         // have we already parsed the input?
220         if (_parsed)
221             return;
222         _parsed = true;
223 
224         scope(exit) _in.close();
225 
226         try {
227             doParse();
228         } catch (Exception e) {
229             warningf("Error occurred while parsing: %s", e.msg);
230             version(HUNT_HTTP_DEBUG) warning(e);
231             _err = e;
232         }
233     }
234 
235     private void doParse() {
236         // initialize
237         _parts = new MultiMap!Part();
238 
239         // if its not a multipart request, don't parse it
240         if (_contentType.empty() || !_contentType.startsWith("multipart/form-data"))
241             return;
242 
243         // sort out the location to which to write the files
244         string location = _config.getLocation();
245         if (location.empty())
246             _tmpDir = _contextTmpDir;
247         else 
248             _tmpDir = buildPath(_contextTmpDir, location);
249 
250         version(HUNT_HTTP_DEBUG) {
251             if (!_tmpDir.exists())
252                 _tmpDir.mkdirRecurse();
253         }
254 
255         string contentTypeBoundary = "";
256         int bstart = cast(int)_contentType.indexOf("boundary=");
257         if (bstart >= 0) {
258             ptrdiff_t bend = _contentType.indexOf(";", bstart);
259             bend = (bend < 0 ? _contentType.length : bend);
260             contentTypeBoundary = QuotedStringTokenizer.unquote(value(_contentType[bstart .. bend]).strip());
261         }
262 
263         Handler handler = new Handler();
264         MultipartParser parser = new MultipartParser(handler, contentTypeBoundary);
265 
266         // Create a buffer to store data from stream //
267         byte[] data = new byte[_bufferSize];
268         int len = 0;
269 
270         /**
271          * keep running total of size of bytes read from input and throw an exception if exceeds MultipartOptions._maxRequestSize
272          */
273         long total = 0;
274         long maxRequestSize = _config.getMaxRequestSize();
275 
276         version(HUNT_HTTP_DEBUG) {
277             int size = _in.available();
278             tracef("available: %d, maxRequestSize: %d", size, maxRequestSize);
279             if(size == 0) {
280                 warning("no data available in inputStream");
281             }
282         }
283 
284         while (true) {
285             len = _in.read(data);
286 
287             if (len > 0) {
288                 total += len;
289                 if (maxRequestSize > 0 && total > maxRequestSize) {
290                     string msg = format("Request exceeds maxRequestSize (%d)", maxRequestSize);
291                     version(HUNT_DEBUG) warning(msg);
292                     _err = new IllegalStateException(msg);
293                     return;
294                 }
295 
296                 ByteBuffer buffer = BufferUtils.toBuffer(data);
297                 buffer.limit(len);
298                 if (parser.parse(buffer, false))
299                     break;
300 
301                 if (buffer.hasRemaining())
302                     throw new IllegalStateException("Buffer did not fully consume");
303 
304             } else if (len == -1) {
305                 version(HUNT_HTTP_DEBUG) trace("no more data avaiable");
306                 parser.parse(BufferUtils.EMPTY_BUFFER, true);
307                 break;
308             }
309         }
310 
311         // check for exceptions
312         if (_err !is null) {
313             return;
314         }
315 
316         // check we read to the end of the message
317         if (parser.getState() != MultipartParser.State.END) {
318             if (parser.getState() == MultipartParser.State.PREAMBLE)
319                 _err = new IOException("Missing initial multi part boundary");
320             else
321                 _err = new IOException("Incomplete Multipart");
322         }
323 
324         version(HUNT_HTTP_DEBUG) {
325             if(_err is null) {
326                 info("Parsing completed");
327             } else {
328                 warningf("Parsing Completed with error: %s", _err);
329             }
330         }
331     }
332 
333     class Handler : MultipartParserHandler {
334         private MultipartForm _part = null;
335         private string contentDisposition = null;
336         private string contentType = null;
337         private MultiMap!string headers;
338 
339         this() {
340             super();
341             headers = new MultiMap!string();
342         }
343 
344         override
345         bool messageComplete() {
346             return true;
347         }
348 
349         override
350         void parsedField(string key, string value) {
351             // Add to headers and mark if one of these fields. //
352             headers.put(std.uni.toLower(key), value);
353             if (key.equalsIgnoreCase("content-disposition"))
354                 contentDisposition = value;
355             else if (key.equalsIgnoreCase("content-type"))
356                 contentType = value;
357 
358             // Transfer encoding is not longer considers as it is deprecated as per
359             // https://tools.ietf.org/html/rfc7578#section-4.7
360 
361         }
362 
363         override
364         bool headerComplete() {
365             version(HUNT_HTTP_DEBUG) {
366                 tracef("headerComplete %s", this);
367             }
368 
369             try {
370                 // Extract content-disposition
371                 bool form_data = false;
372                 if (contentDisposition is null) {
373                     throw new IOException("Missing content-disposition");
374                 }
375 
376                 QuotedStringTokenizer tok = new QuotedStringTokenizer(contentDisposition, ";", false, true);
377                 string name = null;
378                 string filename = null;
379                 while (tok.hasMoreTokens()) {
380                     string t = tok.nextToken().strip();
381                     string tl = std.uni.toLower(t);
382                     if (tl.startsWith("form-data"))
383                         form_data = true;
384                     else if (tl.startsWith("name="))
385                         name = value(t);
386                     else if (tl.startsWith("filename="))
387                         filename = filenameValue(t);
388                 }
389 
390                 // Check disposition
391                 if (!form_data)
392                     throw new IOException("Part not form-data");
393 
394                 // It is valid for reset and submit buttons to have an empty name.
395                 // If no name is supplied, the browser skips sending the info for that field.
396                 // However, if you supply the empty string as the name, the browser sends the
397                 // field, with name as the empty string. So, only continue this loop if we
398                 // have not yet seen a name field.
399                 if (name is null)
400                     throw new IOException("No name in part");
401 
402 
403                 // create the new part
404                 _part = new MultipartForm(name, filename, _config);
405                 _part.setTmpDir(_tmpDir);
406                 _part.setHeaders(headers);
407                 _part.setContentType(contentType);
408                 _parts.add(name, _part);
409 
410                 try {
411                     _part.open();
412                 } catch (IOException e) {
413                     version(HUNT_HTTP_DEBUG) warning(e.msg);
414                     version(HUNT_HTTP_DEBUG) warning(e);
415                     _err = e;
416                     return true;
417                 }
418             } catch (Exception e) {
419                 _err = e;
420                 return true;
421             }
422 
423             return false;
424         }
425 
426         override
427         bool content(ByteBuffer buffer, bool last) {
428             if (_part is null)
429                 return false;
430 
431             if (BufferUtils.hasContent(buffer)) {
432                 try {
433                     _part.write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
434                 } catch (IOException e) {
435                     _err = e;
436                     return true;
437                 }
438             }
439 
440             if (last) {
441                 try {
442                     _part.close();
443                 } catch (IOException e) {
444                     _err = e;
445                     return true;
446                 }
447             }
448 
449             return false;
450         }
451 
452         override
453         void startPart() {
454             reset();
455         }
456 
457         override
458         void earlyEOF() {
459             version(HUNT_HTTP_DEBUG)
460                 tracef("Early EOF %s", this.outer);
461         }
462 
463         void reset() {
464             _part = null;
465             contentDisposition = null;
466             contentType = null;
467             headers = new MultiMap!string();
468         }
469     }
470 
471     void setDeleteOnExit(bool deleteOnExit) {
472         _deleteOnExit = deleteOnExit;
473     }
474 
475     bool isDeleteOnExit() {
476         return _deleteOnExit;
477     }
478 
479     /* ------------------------------------------------------------ */
480     private string value(string nameEqualsValue) {
481         ptrdiff_t idx = nameEqualsValue.indexOf('=');
482         string value = nameEqualsValue[idx + 1 .. $].strip();
483         return QuotedStringTokenizer.unquoteOnly(value);
484     }
485 
486     /* ------------------------------------------------------------ */
487     private string filenameValue(string nameEqualsValue) {
488         ptrdiff_t idx = nameEqualsValue.indexOf('=');
489         string value = nameEqualsValue[idx + 1 .. $].strip();
490         auto pattern = ctRegex!(".??[a-z,A-Z]\\:\\\\[^\\\\].*");
491         // if (value.matches(".??[a-z,A-Z]\\:\\\\[^\\\\].*")) {
492         auto m = matchFirst(value, pattern);
493         if(!m.empty) {
494             // incorrectly escaped IE filenames that have the whole path
495             // we just strip any leading & trailing quotes and leave it as is
496             char first = value[0];
497             if (first == '"' || first == '\'')
498                 value = value[1 .. $];
499             char last = value[$ - 1];
500             if (last == '"' || last == '\'')
501                 value = value[0 .. $ - 1];
502 
503             return value;
504         } else
505             // unquote the string, but allow any backslashes that don't
506             // form a valid escape sequence to remain as many browsers
507             // even on *nix systems will not escape a filename containing
508             // backslashes
509             return QuotedStringTokenizer.unquoteOnly(value, true);
510     }
511 
512 }
513