diff --git a/http/vibe/http/client.d b/http/vibe/http/client.d index 78490f8131..a874623feb 100644 --- a/http/vibe/http/client.d +++ b/http/vibe/http/client.d @@ -17,6 +17,7 @@ import vibe.core.log; import vibe.data.json; import vibe.inet.message; import vibe.inet.url; +import vibe.inet.webform : MultiPart, MultiPartBody; import vibe.stream.counting; import vibe.stream.tls; import vibe.stream.operations; @@ -786,6 +787,7 @@ final class HTTPClientRequest : HTTPRequest { FixedAppender!(string, 22) m_contentLengthBuffer; TCPConnection m_rawConn; TLSCertificateInformation m_peerCertificate; + string m_multipartBoundary; } @@ -891,9 +893,138 @@ final class HTTPClientRequest : HTTPRequest { } } - void writePart(MultiPart part) + /** + Writes the body as multipart request that can upload files. + + Also sets the `Content-Length` if it can be calculated. + */ + void writeMultiPartBody(MultiPartBody part) + { + string boundary = randomMultipartBoundary(); + auto length = part.length(boundary); + if (length != 0) + headers["Content-Length"] = length.to!string; + headers["Content-Type"] = part.contentType ~ "; boundary=\"" ~ boundary ~ "\""; + + // call part.write directly instead of begin/write/finalize because it + // also calculates the length for us and expects it to write itself. + part.write(boundary, bodyWriter); + finalize(); + } + + /** + Starts manually writing a multipart request with `writePart` calls + following this call and finalizing using `finalizeMultiPart`. + + This API is for manually writing out the parts, use `writeMultiPartBody` + to do it all in one step instead. + + Sets the content type to the given content type with the boundary. + + Params: + preamble = Text to write in the preamble. It is ignored by HTTP + servers but can be used for example to include additional + information when writing a mail to a non multipart conforming + reader for the user to see at the start. + boundary = The multipart boundary to use to separate the different + parts. If this is null or empty, this function will + automatically generate a cryptographically secure random + boundary to separate the parts. May be at most 70 characters, + otherwise it will be trimmed. + */ + void beginMultiPart(string content_type = "multipart/form-data", string preamble = null, string boundary = null) + { + if (!boundary.length) + boundary = randomMultipartBoundary; + + if (boundary.length > 70) { + logTrace("Boundary '%s' is longer than 70 characters, truncating", boundary); + boundary = boundary[0 .. 70]; + } + + if ("Content-Type" !in headers) + headers["Content-Type"] = content_type ~ "; boundary=\"" ~ boundary ~ "\""; + + m_multipartBoundary = boundary; + + if (preamble.length) { + bodyWriter.write(preamble); + bodyWriter.write("\r\n"); + } + } + + /** + Writes a single multipart part, which is essentially one input in a form + request which can contain headers to specify what it is. You need to + start with `beginMultiPart` and end with `finalizeMultiPart` with this + API. + + Alternatively you can use `writeMultiPartBody` to do everything in one + step. + */ + void writePart(InputStream)(string field_name, InputStream data, + string content_type = "text/plain; charset=\"utf-8\"", bool binary = false) + if (isInputStream!InputStream) { - assert(false, "TODO"); + scope InetHeaderMap headers; + headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; + if (content_type.length) + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + writePart(data, headers); + } + + /// ditto + void writePart(InputStream)(InputStream data, scope const ref InetHeaderMap headers) + if (isInputStream!InputStream) + { + assert(m_multipartBoundary.length, "need to call beginMultiPart and finalizeMultiPart with writePart"); + + bodyWriter.write("--"); + bodyWriter.write(m_multipartBoundary); + bodyWriter.write("\r\n"); + foreach (k, v; headers.byKeyValue) { + bodyWriter.write(k); + bodyWriter.write(": "); + bodyWriter.write(v); + bodyWriter.write("\r\n"); + } + bodyWriter.write("\r\n"); + pipe(data, bodyWriter); + bodyWriter.write("\r\n"); + } + + /** + Finishes writing a multipart response by sending the ending boundary and + finalizing the request. + */ + void finalizeMultiPart(string epilogue = null) + { + bodyWriter.write("--"); + bodyWriter.write(m_multipartBoundary); + bodyWriter.write("--\r\n"); + if (epilogue.length) + { + bodyWriter.write(epilogue); + bodyWriter.write("\r\n"); + } + m_multipartBoundary = null; + finalize(); + } + + /// + unittest { + import vibe.core.file : openFile; + import vibe.inet.webform : MultiPart, MultiPartBody; + + void test(HTTPClientRequest req) { + MultiPartBody part = new MultiPartBody; + part.parts ~= MultiPart.formData("name", "bob"); + part.parts ~= MultiPart.singleFile("picture", "picture.png", "image/png", openFile("res/profilepicture.png")); + part.parts ~= MultiPart.singleFile("upload", NativePath("file.zip")); // auto read & mime detection from filename + req.writeMultiPartBody(part); + } } /** @@ -954,6 +1085,9 @@ final class HTTPClientRequest : HTTPRequest { if (m_headerWritten && !m_bodyWriter) return; + assert(!m_multipartBoundary.length, + "Closed HTTPClientRequest without calling finalizeMultiPart but called beginMultiPart"); + // force the request to be sent if (!m_headerWritten) writeHeader(); else { diff --git a/http/vibe/http/common.d b/http/vibe/http/common.d index 988a22ac9f..4788df2f25 100644 --- a/http/vibe/http/common.d +++ b/http/vibe/http/common.d @@ -11,6 +11,7 @@ public import vibe.http.status; import vibe.core.log; import vibe.core.net; +import vibe.core.path; import vibe.inet.message; import vibe.stream.operations; import vibe.textfilter.urlencode : urlEncode, urlDecode; @@ -340,13 +341,35 @@ class HTTPStatusException : Exception { string debugMessage; } - -final class MultiPart { - string contentType; - - InputStream stream; - //JsonValue json; - string[string] form; +/** + * Creates a random multipart boundary string starting with hyphens containing + * random text data for separation of data in data uploads. + * + * Uses a cryptographically secure random to generate the boundary string. Use + * this if you plan to manually write a multipart/form-data document somewhere. + * + * You should use $(LREF MultiPart) for an easy to use and compatible API + * instead. + */ +string randomMultipartBoundary() +@safe { + import vibe.crypto.cryptorand : secureRNG; + import std.ascii : digits, letters; + + // simple characters which should be supported everywhere without conflict. + // this is 64 characters which makes the random modulo have a very + // convenient uniform distribution. + static immutable string boundaryChars = digits ~ letters ~ "_-"; // ~ "'()+_,-./:=?"; + + auto rng = secureRNG(); + char[64] ret; // can be up to 70 according to spec, 64 should be enough + ubyte[64] randomBuffer; + rng.read(randomBuffer[]); + + ret[0 .. 16] = '-'; // some padding before random + for (int i = 16; i < ret.length; i++) + ret[i] = boundaryChars[randomBuffer[i] % $]; + return ret[].idup; } /** diff --git a/inet/vibe/inet/webform.d b/inet/vibe/inet/webform.d index 1486a7d099..997c1ce0ad 100644 --- a/inet/vibe/inet/webform.d +++ b/inet/vibe/inet/webform.d @@ -337,6 +337,411 @@ private bool parseMultipartFormPart(InputStream)(InputStream stream, ref FormFie return true; } +/** + Represents a single part in a multipart message. Each part can have its own + headers to specify handling of the part for the receiver. + + Use $(LREF MultiPartBody) to manage a collection of parts. +*/ +struct MultiPartField(ContentInputStream) + if (isInputStream!ContentInputStream) +{ + import vibe.stream.memory : createMemoryStream; + + /// Headers for this part of the multipart data. + InetHeaderMap headers; + + private + { + ContentInputStream m_content; + size_t m_contentLength; + } + + @safe: + + /** + Sets the content stream & length to the given parameters. + + Params: + stream = the input stream to read from when writing the MultiPart. + exact_content_length = Set to the content length in bytes to allow + calculation of the `Content-Length` header. Leave 0 to omit. + */ + void setContent(InputStream)(InputStream stream, size_t exact_content_length = 0) + if (is(InputStream == ContentInputStream)) + { + m_content = stream; + m_contentLength = exact_content_length; + } + + /// ditto + void setContent(InputStream)(InputStream stream, size_t exact_content_length = 0) + if (!is(InputStream == InputStreamProxy) + && is(ContentInputStream == InputStreamProxy) + && isInputStream!InputStream) + { + import vibe.internal.interfaceproxy : interfaceProxy; + + m_content = interfaceProxy!(.InputStream)(stream); + m_contentLength = exact_content_length; + } + + /** + Returns the content stream, as previously set from setContent or from + the static constructing methods. + */ + const(ContentInputStream) content() @property const + { + return m_content; + } + + /** + Sets a form-data field with the given name to a static value. + + Params: + field_name = The field name for example as defined in a HTML form. + stream = An InputStream that contains the value for this field. + value = A fixed value string that contains the value for this field. + content = A fixed value that contains the value for this field. + content_type = The content type of the value. If set to empty string + no content type will be sent. + binary = Set to true to set Content-Transfer-Encoding to binary. + exact_content_length = Exact length of what the stream will evaluate + to in bytes. Used for Content-Length calculation if given. + */ + static typeof(this) formData(InputStream)(string field_name, InputStream stream, + string content_type = "text/plain; charset=\"utf-8\"", bool binary = false, + size_t exact_content_length = 0) + if (isInputStream!InputStream) + { + MultiPartField!ContentInputStream ret; + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; + if (content_type.length) + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.setContent(stream, exact_content_length); + return ret; + } + + /// ditto + static typeof(this) formData(string field_name, string value, + string content_type = "text/plain; charset=\"utf-8\"") + { + return formData(field_name, createMemoryStream(cast(ubyte[]) value.dup, false), + content_type, false, value.length); + } + + /// ditto + static typeof(this) formData(string field_name, ubyte[] content, + string content_type = "application/octet-stream") + { + return formData(field_name, createMemoryStream(cast(ubyte[]) content, false), + content_type, true, content.length); + } + + /** + Helper function directly reading from a file calling singleFile with the + InputStream parameter and content length set. + */ + static typeof(this) singleFile(string field_name, NativePath file) + { + import vibe.inet.mimetypes : getMimeTypeForFile; + import vibe.core.file : openFile, FileMode; + + const type = getMimeTypeForFile(file.toString); + const binary = !type.startsWith("text/"); + auto f = openFile(file, FileMode.read); + return singleFile(field_name, file.head.name, type, f, binary, cast(size_t) f.size); + } + + /** + Sets a form-data field with the given name and a filename which is set + inside the Content-Disposition header and a value. + + Params: + field_name = The field name for example as defined in a HTML form. + filename = The filename (without path) to set for this field. + stream = An InputStream that represents the content of the file. + content = The fixed content of the file. + content_type = The content type of the file. If set to empty string + no content type will be sent. + binary = Set to true to set Content-Transfer-Encoding to binary. + exact_content_length = Exact length of what the stream will evaluate + to in bytes. Used for Content-Length calculation if given. + */ + static typeof(this) singleFile(InputStream)(string field_name, string filename, + string content_type, InputStream stream, bool binary = true, + size_t exact_content_length = 0) + if (isInputStream!InputStream) + { + MultiPartField!ContentInputStream ret; + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; + if (content_type.length) + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.setContent(stream, exact_content_length); + return ret; + } + + /// ditto + static typeof(this) singleFile(string field_name, string filename, + string content_type, string content) + { + return singleFile(field_name, filename, content_type, + createMemoryStream(cast(ubyte[]) content.dup, false), false, + content.length); + } + + /// ditto + static typeof(this) singleFile(string field_name, string filename, + string content_type, ubyte[] content) + { + return singleFile(field_name, filename, content_type, + createMemoryStream(content, false), true, content.length); + } + + /** + Helper function directly reading from a file calling multipleFilesPart + with the InputStream parameter and content length set. + */ + static typeof(this) multipleFilesPart(NativePath file) + { + import vibe.inet.mimetypes : getMimeTypeForFile; + import vibe.core.file : openFile, FileMode; + + const type = getMimeTypeForFile(file.toString); + const binary = !type.startsWith("text/"); + auto f = openFile(file, FileMode.read); + return multipleFilesPart(file.head.name, type, f, binary, cast(size_t) f.size); + } + + /** + Creates a part for a multi-file form field to store multiple files in a + single part. Store all multipleFilesPart MultiParts inside a + MultiPartBody and use multipleFiles to associate them all with a single + form field. + + Params: + filename = The filename (without path) to set for this file. + stream = An InputStream that represents the content of the file. + content = The fixed content of the file. + content_type = The content type of the file. If set to empty string + no content type will be sent. + binary = Set to true to set Content-Transfer-Encoding to binary. + exact_content_length = Exact length of what the stream will evaluate + to in bytes. Used for Content-Length calculation if given. + */ + static typeof(this) multipleFilesPart(InputStream)(string filename, + string content_type, InputStream stream, bool binary = false, + size_t exact_content_length = 0) + if (isInputStream!InputStream) + { + MultiPartField!ContentInputStream ret; + ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; + if (content_type.length) + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.setContent(stream, exact_content_length); + return ret; + } + + /// ditto + static typeof(this) multipleFilesPart(string filename, string content_type, + string content) + { + return multipleFilesPart(filename, content_type, + createMemoryStream((() @trusted => cast(ubyte[]) content)(), false), + false, content.length); + } + + /// ditto + static typeof(this) multipleFilesPart(string filename, string content_type, + ubyte[] content) + { + return multipleFilesPart(filename, content_type, + createMemoryStream(content, false), true, content.length); + } + + /** + Creates a field containing multiple values of different kinds inside it + using `multipart/mixed`. Useful for example to attach multiple files + inside a mail multipart. + + You may for example use this to represent a multipart/mixed type to + describe a specific file order of multiple files or you could use + multipart/alternative to represent different file type versions of the + same file for a mail program. + + Params: + name = the field name to associate the mixed multipart to. + multipart = the MultiPartBody containing all the different parts to + attach. + boundary = The boundary to use to split the different parts inside + this nested multipart part. Do not use the same value as for the + parent MultiPartBody, instead generate a new one using + $(REF randomMultipartBoundary, vibe,http,common). + content_type = The subtype for this mixed multipart to have. Common + types include `multipart/mixed` or `multipart/alternative`. + */ + static typeof(this) multipleFiles(string name, MultiPartBody multipart, + string boundary, string content_type = "multipart/mixed") + { + import vibe.stream.memory : createMemoryOutputStream; + + MultiPartField!ContentInputStream ret; + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ name ~ "\""; + ret.headers["Content-Type"] = content_type ~ "; boundary=\"" ~ boundary ~ "\""; + auto stream = createMemoryOutputStream(); + multipart.write(boundary, stream); + ret.setContent(createMemoryStream(stream.data, false), stream.data.length); + return ret; + } + + /** + Calculates the content length of this part in bytes including headers + and boundary length. + + Returns: the length of this part in bytes or 0 if it couldn't be + determined. + */ + size_t getLength(string boundary) const + { + if (m_contentLength == 0) + return 0; + + size_t length; + length += 4 + boundary.length; // --boundary\r\n + foreach (k, v; headers.byKeyValue) + length += 4 + k.length + v.length; // "key: value\r\n" + length += 2; // \r\n + length += m_contentLength; + length += 2; // \r\n + return length; + } + + /** + Writes this MultiPart to the given output stream. + + To use on a HTTPClient use the `writePart` method of `HTTPClientRequest`. + */ + void write(OutputStream)(OutputStream output, string boundary) + if (isOutputStream!OutputStream) + { + output.write("--"); + output.write(boundary); + output.write("\r\n"); + foreach (k, v; headers.byKeyValue) { + output.write(k); + output.write(": "); + output.write(v); + output.write("\r\n"); + } + output.write("\r\n"); + pipe(m_content, output); + output.write("\r\n"); + } +} + +alias MultiPart = MultiPartField!InputStreamProxy; + +/** + Collection container for multiple MultiPart parts, a content type and an + optional preamble/epilogue. + + May represent attachments for emails, form data for HTTP requests or other + multipart internet messages. + + Standards: $(LINK https://tools.ietf.org/html/rfc1521#section-7.2) +*/ +struct MultiPartBody +{ + /** + The mime type of this multipart. For HTTP this is most usually + multipart/form-data to indicate this describing form fields. + */ + string contentType = "multipart/form-data"; + + /** + Extra information to send before/after the multipart data which is + usually ignored for server processing but allows to include text for + example for mail users with non-multipart-supporting mail clients for + instructions how to read this file. + */ + string preamble, epilogue; + + /** + The full collection of parts that this multipart describes. + */ + MultiPart[] parts; + + @safe: + + /** + Computes the length of the parts, preamble, epilogue together in bytes + for sending `Content-Length` headers or calculating progress. + + Returns: the number of bytes the content is gonna take up or `0` if it + cannot be determined in case a part is not a MemoryStream. + */ + size_t length(string boundary) const + { + import vibe.stream.memory : MemoryStream; + + if (!parts.length) + return 0; + + size_t length; + if (preamble.length) + length += preamble.length + 2; // \r\n + foreach (part; parts) { + const subLength = part.getLength(boundary); + if (subLength == 0) + return 0; + length += subLength; + } + length += boundary.length + 6; // "--boundary--\r\n" + if (epilogue.length) + length += epilogue.length + 2; + return length; + } + + /** + Writes the full multipart body to the given output stream with the given + multipart boundary. A secure random boundary can be obtained through + $(REF randomMultipartBoundary, vibe,http,common). + + If you want to send the MultiPartBody in a HTTP request, it is + recommended to use $(REF writePart, vibe,http,client,HTTPClientRequest). + */ + void write(T)(string boundary, T output) + if (isOutputStream!T) + { + import vibe.core.stream : pipe; + + if (!parts.length) + return; + + if (preamble.length) { + output.write(preamble); + output.write("\r\n"); + } + foreach (part; parts) { + part.write(output, boundary); + } + output.write("--"); + output.write(boundary); + output.write("--\r\n"); + if (epilogue.length) + { + output.write(epilogue); + output.write("\r\n"); + } + } +} + /** Encodes a Key-Value map into a form URL encoded string. diff --git a/tests/vibe.http.client.1005/dub.sdl b/tests/vibe.http.client.1005/dub.sdl new file mode 100644 index 0000000000..b33f623f79 --- /dev/null +++ b/tests/vibe.http.client.1005/dub.sdl @@ -0,0 +1,3 @@ +name "tests" +description "Tests multipart encoding" +dependency "vibe-d:http" path="../.." diff --git a/tests/vibe.http.client.1005/source/app.d b/tests/vibe.http.client.1005/source/app.d new file mode 100644 index 0000000000..2fb0fa3ea1 --- /dev/null +++ b/tests/vibe.http.client.1005/source/app.d @@ -0,0 +1,43 @@ +import vibe.core.core; +import vibe.core.file; +import vibe.core.log; +import vibe.core.net; +import vibe.http.client; +import vibe.http.server; +import vibe.inet.webform; +import vibe.stream.memory; +import vibe.stream.operations; + +void main(string[] args) +{ + bool handled = false; + + listenHTTP("127.0.0.1:11005", (scope req, scope res) { + assert(req.form.length == 1); + assert(req.files.length == 1); + assert(req.form["name"] == "bob"); + assert(req.files["picture"].filename == "picture.png"); + auto f = openFile(req.files["picture"].tempPath, FileMode.read); + auto data = f.readAllUTF8; + f.close(); + assert(data == "Totally\0a\0PNG\0file."); + removeFile(req.files["picture"].tempPath); + res.writeBody("ok"); + handled = true; + }); + + requestHTTP("http://127.0.0.1:11005", + (scope req) { + MultiPartBody part; + part.parts ~= MultiPart.formData("name", "bob"); + auto memStream = createMemoryStream(cast(ubyte[]) "Totally\0a\0PNG\0file.", false); + part.parts ~= MultiPart.singleFile("picture", "picture.png", "image/png", memStream, true); + req.writeMultiPartBody(part); + }, + (scope res) { + assert(res.bodyReader.readAllUTF8() == "ok"); + } + ); + + assert(handled); +}