From 228b6de1769de20a115053abc9dbcbb47b55e245 Mon Sep 17 00:00:00 2001 From: Jan Jurzitza Date: Tue, 25 Jul 2017 15:48:06 +0200 Subject: [PATCH 01/10] Start working on #1005 --- http/vibe/http/client.d | 21 ++- http/vibe/http/common.d | 219 ++++++++++++++++++++++- tests/vibe.http.client.1005/dub.sdl | 4 + tests/vibe.http.client.1005/source/app.d | 32 ++++ 4 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 tests/vibe.http.client.1005/dub.sdl create mode 100644 tests/vibe.http.client.1005/source/app.d diff --git a/http/vibe/http/client.d b/http/vibe/http/client.d index 8dda1826d8..94b7839de2 100644 --- a/http/vibe/http/client.d +++ b/http/vibe/http/client.d @@ -803,7 +803,26 @@ final class HTTPClientRequest : HTTPRequest { void writePart(MultiPart part) { - assert(false, "TODO"); + auto boundary = randomMultipartBoundary; + auto length = part.length(boundary); + headers["Content-Type"] = part.contentType ~ "; boundary=\"" ~ boundary ~ "\""; + if (length != 0) + headers["Content-Length"] = length.to!string; + else + headers["Transfer-Encoding"] = "identity"; + part.write(boundary, bodyWriter); + finalize(); + } + + /// + unittest { + void test(HTTPClientRequest req) { + MultiPart part = new MultiPart; + part.parts ~= MultiPartBodyPart.formData("name", "bob"); + part.parts ~= MultiPartBodyPart.singleFile("picture", "picture.png", "image/png", openFile("res/profilepicture.png")); + part.parts ~= MultiPartBodyPart.singleFile("upload", "file.zip"); // auto read & mime detection from filename + req.writePart(part); + } } /** diff --git a/http/vibe/http/common.d b/http/vibe/http/common.d index 160f41962e..46aca8bf90 100644 --- a/http/vibe/http/common.d +++ b/http/vibe/http/common.d @@ -11,6 +11,7 @@ public import vibe.http.status; import vibe.core.log; import vibe.core.net; +import vibe.core.path; import vibe.inet.message; import vibe.stream.operations; import vibe.textfilter.urlencode : urlEncode, urlDecode; @@ -337,13 +338,221 @@ class HTTPStatusException : Exception { string debugMessage; } +final class MultiPartBodyPart +{ + import vibe.stream.memory : createMemoryStream; + + InetHeaderMap headers; + InputStream content; + + @safe: + + static MultiPartBodyPart formData(string field_name, InputStream stream, string content_type = "text/plain; charset=\"utf-8\"", bool binary = false) + { + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.content = stream; + return ret; + } + + static MultiPartBodyPart formData(string field_name, string value, string content_type = "text/plain; charset=\"utf-8\"") + { + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; + ret.headers["Content-Type"] = content_type; + ret.content = createMemoryStream((() @trusted => cast(ubyte[]) value)(), false); + return ret; + } + + static MultiPartBodyPart formData(string field_name, ubyte[] content, string content_type = "application/octet-stream") + { + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.content = createMemoryStream(content, false); + return ret; + } + + static MultiPartBodyPart singleFile(string field_name, Path file) + { + import vibe.inet.mimetypes : getMimeTypeForFile; + import vibe.core.file : openFile, FileMode; + + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ file.head.name ~ "\""; + string type = getMimeTypeForFile(file.toString); + ret.headers["Content-Type"] = type; + if (!type.startsWith("text/")) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.content = openFile(file, FileMode.read); + return ret; + } + + static MultiPartBodyPart singleFile(string field_name, string filename, string content_type, InputStream stream, bool binary = true) + { + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.content = stream; + return ret; + } + + static MultiPartBodyPart singleFile(string field_name, string filename, string content_type, string content) + { + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; + ret.headers["Content-Type"] = content_type; + ret.content = createMemoryStream((() @trusted => cast(ubyte[]) content)(), false); + return ret; + } + + static MultiPartBodyPart singleFile(string field_name, string filename, string content_type, ubyte[] content) + { + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.headers["Content-Type"] = content_type; + ret.content = createMemoryStream(content, false); + return ret; + } + + static MultiPartBodyPart multipleFilesPart(Path file) + { + import vibe.inet.mimetypes : getMimeTypeForFile; + import vibe.core.file : openFile, FileMode; + + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "file; filename=\"" ~ file.head.name ~ "\""; + string type = getMimeTypeForFile(file.toString); + ret.headers["Content-Type"] = type; + if (!type.startsWith("text/")) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.content = openFile(file, FileMode.read); + return ret; + } + + static MultiPartBodyPart multipleFilesPart(string filename, string content_type, InputStream stream) + { + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; + ret.headers["Content-Type"] = content_type; + ret.content = stream; + return ret; + } + + static MultiPartBodyPart multipleFilesPart(string filename, string content_type, string content) + { + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; + ret.headers["Content-Type"] = content_type; + ret.content = createMemoryStream((() @trusted => cast(ubyte[]) content)(), false); + return ret; + } + + static MultiPartBodyPart multipleFilesPart(string filename, string content_type, ubyte[] content) + { + auto ret = new MultiPartBodyPart(); + ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.headers["Content-Type"] = content_type; + ret.content = createMemoryStream(content, false); + return ret; + } + + static MultiPartBodyPart multipleFiles(string name, MultiPart multipart) + { + import vibe.stream.memory : createMemoryOutputStream; + + auto ret = new MultiPartBodyPart(); + string boundary = randomMultipartBoundary; + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ name ~ "\""; + ret.headers["Content-Type"] = "multipart/mixed; boundary=\"" ~ boundary ~ "\""; + auto stream = createMemoryOutputStream(); + multipart.write(boundary, stream); + ret.content = createMemoryStream(stream.data, false); + return ret; + } +} + +final class MultiPart +{ + // https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html + + string contentType = "multipart/form-data"; + string preamble, epilogue; + + MultiPartBodyPart[] parts; + + @safe: + + size_t length(string boundary) const + { + if (!parts.length) + return 0; + + size_t length; + if (preamble.length) + length += preamble.length + 2; + length += boundary.length + 2; // --boundary + foreach (part; parts) { + length += 8 + boundary.length; // \r\n * 3 + boundary (+2) + foreach (k, v; part.headers) + length += 4 + k.length + v.length; // ": \r\n" + auto randomStream = cast(const(RandomAccessStream)) part.content; + if (randomStream) + length += randomStream.size; + else + return 0; // undetectable + } + length += 4; + if (epilogue.length) + length += epilogue.length + 2; + return length; + } + + void write(string boundary, OutputStream output) + { + import vibe.core.stream : pipe; + + if (!parts.length) + return; + + boundary = "--" ~ boundary; + if (preamble.length) + output.write(preamble ~ "\r\n"); + output.write(boundary); + foreach (part; parts) { + output.write("\r\n"); + foreach (k, v; part.headers) + output.write(k ~ ": " ~ v ~ "\r\n"); + output.write("\r\n"); + pipe(part.content, output); + output.write("\r\n"); + output.write(boundary); + } + output.write("--\r\n"); + if (epilogue.length) + output.write(epilogue ~ "\r\n"); + } +} + +string randomMultipartBoundary() +@trusted { + import std.random : uniform; + import std.ascii : digits, letters; -final class MultiPart { - string contentType; + static immutable string boundaryChars = digits ~ letters ~ "_-"; // ~ "'()+_,-./:=?"; - InputStream stream; - //JsonValue json; - string[string] form; + char[64] ret; // can be up to 70 according to spec, but taking poor implementations into account + ret[0 .. 20] = '-'; // some padding before random + for (int i = 20; i < ret.length; i++) + ret[i] = boundaryChars[uniform(0, $)]; + return ret[].idup; } string getHTTPVersionString(HTTPVersion ver) diff --git a/tests/vibe.http.client.1005/dub.sdl b/tests/vibe.http.client.1005/dub.sdl new file mode 100644 index 0000000000..9af498fdea --- /dev/null +++ b/tests/vibe.http.client.1005/dub.sdl @@ -0,0 +1,4 @@ +name "tests" +description "Tests multipart encoding" +dependency "vibe-d:http" path="../.." +versions "VibeDefaultMain" diff --git a/tests/vibe.http.client.1005/source/app.d b/tests/vibe.http.client.1005/source/app.d new file mode 100644 index 0000000000..50717ece4e --- /dev/null +++ b/tests/vibe.http.client.1005/source/app.d @@ -0,0 +1,32 @@ +import vibe.core.core; +import vibe.core.log; +import vibe.core.net; +import vibe.http.client; +import vibe.http.server; +import vibe.stream.operations; +import vibe.stream.memory; + +shared static this() +{ + listenHTTP("127.0.0.1:11005", (scope req, scope res) { + foreach (k, v; req.form) + logInfo("%s: %s", k, v); + foreach (k, v; req.files) + logInfo("%s: %s", k, v.filename); + res.writeBody("Hello world."); + }); + + runTask({ + requestHTTP("http://127.0.0.1:11005", + (scope req) { + MultiPart part = new MultiPart; + part.parts ~= MultiPartBodyPart.formData("name", "bob"); + auto memStream = createMemoryStream(cast(ubyte[]) "Totally\0a\0PNG\0file.", false); + part.parts ~= MultiPartBodyPart.singleFile("picture", "picture.png", "image/png", memStream, true); + req.writePart(part); + }, + (scope res) {} + ); + exitEventLoop(); + }); +} From 38b0eb9c45ac83b093a1c2c3f125c2ec01e70ec0 Mon Sep 17 00:00:00 2001 From: Jan Jurzitza Date: Tue, 25 Jul 2017 15:54:52 +0200 Subject: [PATCH 02/10] Proper testing of MultiPart client --- tests/vibe.http.client.1005/source/app.d | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/vibe.http.client.1005/source/app.d b/tests/vibe.http.client.1005/source/app.d index 50717ece4e..e3a81187fe 100644 --- a/tests/vibe.http.client.1005/source/app.d +++ b/tests/vibe.http.client.1005/source/app.d @@ -1,4 +1,5 @@ import vibe.core.core; +import vibe.core.file; import vibe.core.log; import vibe.core.net; import vibe.http.client; @@ -9,11 +10,16 @@ import vibe.stream.memory; shared static this() { listenHTTP("127.0.0.1:11005", (scope req, scope res) { - foreach (k, v; req.form) - logInfo("%s: %s", k, v); - foreach (k, v; req.files) - logInfo("%s: %s", k, v.filename); - res.writeBody("Hello world."); + assert(req.form.length == 1); + assert(req.files.length == 1); + assert(req.form["name"] == "bob"); + assert(req.files["picture"].filename == "picture.png"); + auto f = openFile(req.files["picture"].tempPath, FileMode.read); + auto data = f.readAllUTF8; + f.close(); + assert(data == "Totally\0a\0PNG\0file."); + removeFile(req.files["picture"].tempPath); + res.writeBody("ok"); }); runTask({ From fa984485f21cef808130cf876b91a8c4dbec54c0 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Tue, 25 Jul 2017 18:11:39 +0200 Subject: [PATCH 03/10] Fix tests --- http/vibe/http/client.d | 2 +- http/vibe/http/common.d | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/http/vibe/http/client.d b/http/vibe/http/client.d index 94b7839de2..2132aa5913 100644 --- a/http/vibe/http/client.d +++ b/http/vibe/http/client.d @@ -820,7 +820,7 @@ final class HTTPClientRequest : HTTPRequest { MultiPart part = new MultiPart; part.parts ~= MultiPartBodyPart.formData("name", "bob"); part.parts ~= MultiPartBodyPart.singleFile("picture", "picture.png", "image/png", openFile("res/profilepicture.png")); - part.parts ~= MultiPartBodyPart.singleFile("upload", "file.zip"); // auto read & mime detection from filename + part.parts ~= MultiPartBodyPart.singleFile("upload", Path("file.zip")); // auto read & mime detection from filename req.writePart(part); } } diff --git a/http/vibe/http/common.d b/http/vibe/http/common.d index 46aca8bf90..8102d68f26 100644 --- a/http/vibe/http/common.d +++ b/http/vibe/http/common.d @@ -387,7 +387,7 @@ final class MultiPartBodyPart ret.headers["Content-Type"] = type; if (!type.startsWith("text/")) ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.content = openFile(file, FileMode.read); + ret.content = cast(InputStream) openFile(file, FileMode.read); return ret; } @@ -432,7 +432,7 @@ final class MultiPartBodyPart ret.headers["Content-Type"] = type; if (!type.startsWith("text/")) ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.content = openFile(file, FileMode.read); + ret.content = cast(InputStream) openFile(file, FileMode.read); return ret; } @@ -515,7 +515,8 @@ final class MultiPart return length; } - void write(string boundary, OutputStream output) + void write(T)(string boundary, T output) + if (isOutputStream!T) { import vibe.core.stream : pipe; From 1ec5b1fb85db79522695eb5e15af62ee0597852e Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sat, 12 Aug 2017 07:03:21 +0200 Subject: [PATCH 04/10] Added possibility to send separate multiparts & reduce GC usage --- http/vibe/http/client.d | 59 +++++++++++++-- http/vibe/http/common.d | 93 +++++++++++++----------- tests/vibe.http.client.1005/source/app.d | 6 +- 3 files changed, 106 insertions(+), 52 deletions(-) diff --git a/http/vibe/http/client.d b/http/vibe/http/client.d index 2132aa5913..24a4923a94 100644 --- a/http/vibe/http/client.d +++ b/http/vibe/http/client.d @@ -695,6 +695,7 @@ final class HTTPClientRequest : HTTPRequest { FixedAppender!(string, 22) m_contentLengthBuffer; TCPConnection m_rawConn; TLSCertificateInformation m_peerCertificate; + bool m_writtenPart = false; } @@ -801,7 +802,10 @@ final class HTTPClientRequest : HTTPRequest { } } - void writePart(MultiPart part) + /** + Writes the body as multipart request that can upload files. + */ + void writePart(MultiPartBody part) { auto boundary = randomMultipartBoundary; auto length = part.length(boundary); @@ -814,13 +818,58 @@ final class HTTPClientRequest : HTTPRequest { finalize(); } + /** + Partially writes the body with a multipart. + You need to supply a boundary which is a random string that should not occur in the content. + The boundary can be generated using `vibe.http.common.randomMultipartBoundary` and must always be the same in one request. + */ + void writePart(MultiPart part, string boundary) + { + if (boundary.length > 70) { + logTrace("Boundary '%s' is longer than 70 characters, truncating", boundary); + boundary.length = 70; + } + + if (m_writtenPart) + bodyWriter.write(boundary); + + if ("Content-Type" !in headers) + headers["Content-Type"] = "multipart/form-data; boundary=\"" ~ boundary ~ "\""; + + boundary = "--" ~ boundary; + bodyWriter.write(boundary); + bodyWriter.write("\r\n"); + foreach (k, v; part.headers) + bodyWriter.write(k ~ ": " ~ v ~ "\r\n"); + bodyWriter.write("\r\n"); + pipe(part.content, bodyWriter); + bodyWriter.write("\r\n"); + m_writtenPart = true; + } + + /** + Finishes writing a multipart response by sending the ending boundary and finalizing the request. + */ + void finalizePart(string boundary, string epilogue = null) + { + bodyWriter.write("--"); + bodyWriter.write(boundary); + bodyWriter.write("--\r\n"); + if (epilogue.length) + { + bodyWriter.write(epilogue); + bodyWriter.write("\r\n"); + } + finalize(); + } + /// unittest { void test(HTTPClientRequest req) { - MultiPart part = new MultiPart; - part.parts ~= MultiPartBodyPart.formData("name", "bob"); - part.parts ~= MultiPartBodyPart.singleFile("picture", "picture.png", "image/png", openFile("res/profilepicture.png")); - part.parts ~= MultiPartBodyPart.singleFile("upload", Path("file.zip")); // auto read & mime detection from filename + MultiPartBody part = new MultiPartBody; + part.parts ~= MultiPart.formData("name", "bob"); + part.parts ~= MultiPart.singleFile("picture", "picture.png", "image/png", openFile("res/profilepicture.png")); + part.parts ~= MultiPart.singleFile("upload", Path("file.zip")); // auto read & mime detection from filename req.writePart(part); } } diff --git a/http/vibe/http/common.d b/http/vibe/http/common.d index 8102d68f26..53e8c98c0c 100644 --- a/http/vibe/http/common.d +++ b/http/vibe/http/common.d @@ -338,18 +338,19 @@ class HTTPStatusException : Exception { string debugMessage; } -final class MultiPartBodyPart +final class MultiPart { import vibe.stream.memory : createMemoryStream; InetHeaderMap headers; - InputStream content; + InterfaceProxy!InputStream content; @safe: - static MultiPartBodyPart formData(string field_name, InputStream stream, string content_type = "text/plain; charset=\"utf-8\"", bool binary = false) + static MultiPart formData(InputStream)(string field_name, InputStream stream, string content_type = "text/plain; charset=\"utf-8\"", bool binary = false) + if (isInputStream!InputStream) { - auto ret = new MultiPartBodyPart(); + auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; ret.headers["Content-Type"] = content_type; if (binary) @@ -358,42 +359,43 @@ final class MultiPartBodyPart return ret; } - static MultiPartBodyPart formData(string field_name, string value, string content_type = "text/plain; charset=\"utf-8\"") + static MultiPart formData(string field_name, string value, string content_type = "text/plain; charset=\"utf-8\"") { - auto ret = new MultiPartBodyPart(); + auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; ret.headers["Content-Type"] = content_type; ret.content = createMemoryStream((() @trusted => cast(ubyte[]) value)(), false); return ret; } - static MultiPartBodyPart formData(string field_name, ubyte[] content, string content_type = "application/octet-stream") + static MultiPart formData(string field_name, ubyte[] content, string content_type = "application/octet-stream") { - auto ret = new MultiPartBodyPart(); + auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; ret.headers["Content-Transfer-Encoding"] = "binary"; ret.content = createMemoryStream(content, false); return ret; } - static MultiPartBodyPart singleFile(string field_name, Path file) + static MultiPart singleFile(string field_name, Path file) { import vibe.inet.mimetypes : getMimeTypeForFile; import vibe.core.file : openFile, FileMode; - auto ret = new MultiPartBodyPart(); + auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ file.head.name ~ "\""; string type = getMimeTypeForFile(file.toString); ret.headers["Content-Type"] = type; if (!type.startsWith("text/")) ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.content = cast(InputStream) openFile(file, FileMode.read); + ret.content = cast(InterfaceProxy!InputStream) openFile(file, FileMode.read); return ret; } - static MultiPartBodyPart singleFile(string field_name, string filename, string content_type, InputStream stream, bool binary = true) + static MultiPart singleFile(InputStream)(string field_name, string filename, string content_type, InputStream stream, bool binary = true) + if (isInputStream!InputStream) { - auto ret = new MultiPartBodyPart(); + auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; ret.headers["Content-Type"] = content_type; if (binary) @@ -402,18 +404,18 @@ final class MultiPartBodyPart return ret; } - static MultiPartBodyPart singleFile(string field_name, string filename, string content_type, string content) + static MultiPart singleFile(string field_name, string filename, string content_type, string content) { - auto ret = new MultiPartBodyPart(); + auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; ret.headers["Content-Type"] = content_type; ret.content = createMemoryStream((() @trusted => cast(ubyte[]) content)(), false); return ret; } - static MultiPartBodyPart singleFile(string field_name, string filename, string content_type, ubyte[] content) + static MultiPart singleFile(string field_name, string filename, string content_type, ubyte[] content) { - auto ret = new MultiPartBodyPart(); + auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; ret.headers["Content-Transfer-Encoding"] = "binary"; ret.headers["Content-Type"] = content_type; @@ -421,54 +423,48 @@ final class MultiPartBodyPart return ret; } - static MultiPartBodyPart multipleFilesPart(Path file) + static MultiPart multipleFilesPart(Path file) { import vibe.inet.mimetypes : getMimeTypeForFile; import vibe.core.file : openFile, FileMode; - auto ret = new MultiPartBodyPart(); + auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "file; filename=\"" ~ file.head.name ~ "\""; string type = getMimeTypeForFile(file.toString); ret.headers["Content-Type"] = type; if (!type.startsWith("text/")) ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.content = cast(InputStream) openFile(file, FileMode.read); + ret.content = cast(InterfaceProxy!InputStream) openFile(file, FileMode.read); return ret; } - static MultiPartBodyPart multipleFilesPart(string filename, string content_type, InputStream stream) + static MultiPart multipleFilesPart(InputStream)(string filename, string content_type, InputStream stream, bool binary = false) + if (isInputStream!InputStream) { - auto ret = new MultiPartBodyPart(); + auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; ret.content = stream; return ret; } - static MultiPartBodyPart multipleFilesPart(string filename, string content_type, string content) + static MultiPart multipleFilesPart(string filename, string content_type, string content) { - auto ret = new MultiPartBodyPart(); - ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; - ret.headers["Content-Type"] = content_type; - ret.content = createMemoryStream((() @trusted => cast(ubyte[]) content)(), false); - return ret; + return multipleFilesPart(filename, content_type, createMemoryStream((() @trusted => cast(ubyte[]) content)(), false), true); } - static MultiPartBodyPart multipleFilesPart(string filename, string content_type, ubyte[] content) + static MultiPart multipleFilesPart(string filename, string content_type, ubyte[] content) { - auto ret = new MultiPartBodyPart(); - ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; - ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.headers["Content-Type"] = content_type; - ret.content = createMemoryStream(content, false); - return ret; + return multipleFilesPart(filename, content_type, createMemoryStream(content, false), true); } - static MultiPartBodyPart multipleFiles(string name, MultiPart multipart) + static MultiPart multipleFiles(string name, MultiPartBody multipart) { import vibe.stream.memory : createMemoryOutputStream; - auto ret = new MultiPartBodyPart(); + auto ret = new MultiPart(); string boundary = randomMultipartBoundary; ret.headers["Content-Disposition"] = "form-data; name=\"" ~ name ~ "\""; ret.headers["Content-Type"] = "multipart/mixed; boundary=\"" ~ boundary ~ "\""; @@ -479,14 +475,14 @@ final class MultiPartBodyPart } } -final class MultiPart +final class MultiPartBody { // https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html string contentType = "multipart/form-data"; string preamble, epilogue; - MultiPartBodyPart[] parts; + MultiPart[] parts; @safe: @@ -524,13 +520,19 @@ final class MultiPart return; boundary = "--" ~ boundary; - if (preamble.length) - output.write(preamble ~ "\r\n"); + if (preamble.length) { + output.write(preamble); + output.write("\r\n"); + } output.write(boundary); foreach (part; parts) { output.write("\r\n"); - foreach (k, v; part.headers) - output.write(k ~ ": " ~ v ~ "\r\n"); + foreach (k, v; part.headers) { + output.write(k); + output.write(": "); + output.write(v); + output.write("\r\n"); + } output.write("\r\n"); pipe(part.content, output); output.write("\r\n"); @@ -538,7 +540,10 @@ final class MultiPart } output.write("--\r\n"); if (epilogue.length) - output.write(epilogue ~ "\r\n"); + { + output.write(epilogue); + output.write("\r\n"); + } } } diff --git a/tests/vibe.http.client.1005/source/app.d b/tests/vibe.http.client.1005/source/app.d index e3a81187fe..3d970d4383 100644 --- a/tests/vibe.http.client.1005/source/app.d +++ b/tests/vibe.http.client.1005/source/app.d @@ -25,10 +25,10 @@ shared static this() runTask({ requestHTTP("http://127.0.0.1:11005", (scope req) { - MultiPart part = new MultiPart; - part.parts ~= MultiPartBodyPart.formData("name", "bob"); + MultiPartBody part = new MultiPartBody; + part.parts ~= MultiPart.formData("name", "bob"); auto memStream = createMemoryStream(cast(ubyte[]) "Totally\0a\0PNG\0file.", false); - part.parts ~= MultiPartBodyPart.singleFile("picture", "picture.png", "image/png", memStream, true); + part.parts ~= MultiPart.singleFile("picture", "picture.png", "image/png", memStream, true); req.writePart(part); }, (scope res) {} From 0d99bf69e6ab300445843ea5e71c49ccf2242c1b Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Tue, 25 Jun 2019 18:34:52 +0200 Subject: [PATCH 05/10] fix InterfaceProxy code --- http/vibe/http/common.d | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/http/vibe/http/common.d b/http/vibe/http/common.d index 53e8c98c0c..5903458413 100644 --- a/http/vibe/http/common.d +++ b/http/vibe/http/common.d @@ -343,7 +343,7 @@ final class MultiPart import vibe.stream.memory : createMemoryStream; InetHeaderMap headers; - InterfaceProxy!InputStream content; + InputStreamProxy content; @safe: @@ -364,7 +364,7 @@ final class MultiPart auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; ret.headers["Content-Type"] = content_type; - ret.content = createMemoryStream((() @trusted => cast(ubyte[]) value)(), false); + emplace(&ret.content, createMemoryStream(cast(ubyte[]) value.dup, false)); return ret; } @@ -373,7 +373,7 @@ final class MultiPart auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.content = createMemoryStream(content, false); + emplace(&ret.content, createMemoryStream(content, false)); return ret; } @@ -388,7 +388,7 @@ final class MultiPart ret.headers["Content-Type"] = type; if (!type.startsWith("text/")) ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.content = cast(InterfaceProxy!InputStream) openFile(file, FileMode.read); + emplace(&ret.content, openFile(file, FileMode.read)); return ret; } @@ -400,7 +400,7 @@ final class MultiPart ret.headers["Content-Type"] = content_type; if (binary) ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.content = stream; + emplace(&ret.content, stream); return ret; } @@ -409,7 +409,7 @@ final class MultiPart auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; ret.headers["Content-Type"] = content_type; - ret.content = createMemoryStream((() @trusted => cast(ubyte[]) content)(), false); + emplace(&ret.content, createMemoryStream((() @trusted => cast(ubyte[]) content)(), false)); return ret; } @@ -419,7 +419,7 @@ final class MultiPart ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; ret.headers["Content-Transfer-Encoding"] = "binary"; ret.headers["Content-Type"] = content_type; - ret.content = createMemoryStream(content, false); + emplace(&ret.content, createMemoryStream(content, false)); return ret; } @@ -434,7 +434,7 @@ final class MultiPart ret.headers["Content-Type"] = type; if (!type.startsWith("text/")) ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.content = cast(InterfaceProxy!InputStream) openFile(file, FileMode.read); + emplace(&ret.content, openFile(file, FileMode.read)); return ret; } @@ -446,7 +446,7 @@ final class MultiPart ret.headers["Content-Type"] = content_type; if (binary) ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.content = stream; + emplace(&ret.content, stream); return ret; } @@ -470,7 +470,7 @@ final class MultiPart ret.headers["Content-Type"] = "multipart/mixed; boundary=\"" ~ boundary ~ "\""; auto stream = createMemoryOutputStream(); multipart.write(boundary, stream); - ret.content = createMemoryStream(stream.data, false); + emplace(&ret.content, createMemoryStream(stream.data, false)); return ret; } } @@ -488,6 +488,8 @@ final class MultiPartBody size_t length(string boundary) const { + import vibe.stream.memory : MemoryStream; + if (!parts.length) return 0; @@ -499,11 +501,19 @@ final class MultiPartBody length += 8 + boundary.length; // \r\n * 3 + boundary (+2) foreach (k, v; part.headers) length += 4 + k.length + v.length; // ": \r\n" - auto randomStream = cast(const(RandomAccessStream)) part.content; - if (randomStream) + + static assert(typeof(part.content).tupleof[1].stringof == "m_intf", + "Can't check the interface type of InterfaceProxy because definition changed"); + + // check if content is a MemoryStream because then we can compute exact lengths. + // with current vibe-core it's not possible to find out the type or even check if it matches one without triggering a fatal assert error, so we access private fields here + // TODO: we would preferably want to support all classes implementing RandomAccessStream here, so we might need to traverse the typeinfo + if ((() @trusted => cast() part.content.tupleof[1])()._typeInfo() is typeid(MemoryStream)) { + auto randomStream = (() @trusted => cast() part.content)().extract!MemoryStream; length += randomStream.size; - else + } else { return 0; // undetectable + } } length += 4; if (epilogue.length) @@ -548,7 +558,7 @@ final class MultiPartBody } string randomMultipartBoundary() -@trusted { +@safe { import std.random : uniform; import std.ascii : digits, letters; From c5da21bc32acec448c4c4ac7ff5275346cc390e5 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sat, 18 Apr 2020 21:00:23 +0200 Subject: [PATCH 06/10] rewrite multipart docs, reduce code redundancy --- http/vibe/http/client.d | 94 ++++++---- http/vibe/http/common.d | 397 +++++++++++++++++++++++++++++----------- 2 files changed, 353 insertions(+), 138 deletions(-) diff --git a/http/vibe/http/client.d b/http/vibe/http/client.d index 8da8da5a03..e3473b9ff8 100644 --- a/http/vibe/http/client.d +++ b/http/vibe/http/client.d @@ -786,7 +786,7 @@ final class HTTPClientRequest : HTTPRequest { FixedAppender!(string, 22) m_contentLengthBuffer; TCPConnection m_rawConn; TLSCertificateInformation m_peerCertificate; - bool m_writtenPart = false; + string m_multipartBoundary; } @@ -894,56 +894,85 @@ final class HTTPClientRequest : HTTPRequest { /** Writes the body as multipart request that can upload files. + + Also sets the `Content-Length` if it can be calculated. */ - void writePart(MultiPartBody part) + void writeMultiPartBody(MultiPartBody part) { - auto boundary = randomMultipartBoundary; + string boundary = randomMultipartBoundary(); auto length = part.length(boundary); - headers["Content-Type"] = part.contentType ~ "; boundary=\"" ~ boundary ~ "\""; if (length != 0) headers["Content-Length"] = length.to!string; - else - headers["Transfer-Encoding"] = "identity"; + headers["Content-Type"] = part.contentType ~ "; boundary=\"" ~ boundary ~ "\""; + + // call part.write directly instead of begin/write/finalize because it + // also calculates the length for us and expects it to write itself. part.write(boundary, bodyWriter); finalize(); } /** - Partially writes the body with a multipart. - You need to supply a boundary which is a random string that should not occur in the content. - The boundary can be generated using `vibe.http.common.randomMultipartBoundary` and must always be the same in one request. - */ - void writePart(MultiPart part, string boundary) + Starts manually writing a multipart request with `writePart` calls + following this call and finalizing using `finalizeMultiPart`. + + This API is for manually writing out the parts, use `writeMultiPartBody` + to do it all in one step instead. + + Sets the content type to the given content type with the boundary. + + Params: + preamble = Text to write in the preamble. It is ignored by HTTP + servers but can be used for example to include additional + information when writing a mail to a non multipart conforming + reader for the user to see at the start. + boundary = The multipart boundary to use to separate the different + parts. If this is null or empty, this function will + automatically generate a cryptographically secure random + boundary to separate the parts. May be at most 70 characters, + otherwise it will be trimmed. + */ + void beginMultiPart(string content_type = "multipart/form-data", string preamble = null, string boundary = null) { + if (!boundary.length) + boundary = randomMultipartBoundary; + if (boundary.length > 70) { logTrace("Boundary '%s' is longer than 70 characters, truncating", boundary); - boundary.length = 70; + boundary = boundary[0 .. 70]; } - if (m_writtenPart) - bodyWriter.write(boundary); - if ("Content-Type" !in headers) - headers["Content-Type"] = "multipart/form-data; boundary=\"" ~ boundary ~ "\""; - - boundary = "--" ~ boundary; - bodyWriter.write(boundary); - bodyWriter.write("\r\n"); - foreach (k, v; part.headers) - bodyWriter.write(k ~ ": " ~ v ~ "\r\n"); - bodyWriter.write("\r\n"); - pipe(part.content, bodyWriter); - bodyWriter.write("\r\n"); - m_writtenPart = true; + headers["Content-Type"] = content_type ~ "; boundary=\"" ~ boundary ~ "\""; + + m_multipartBoundary = boundary; + + if (preamble.length) { + bodyWriter.write(preamble); + bodyWriter.write("\r\n"); + } + } + + /** + Partially writes the body with a multipart. If you plan to manually + write all parts using writePart you need to start with `beginMultiPart` + and end with `finalizeMultiPart`. + + Alternatively you can use `writeMultiPartBody` to do everything in one + step. + */ + void writePart(MultiPart part) + { + part.write(bodyWriter, m_multipartBoundary); } /** - Finishes writing a multipart response by sending the ending boundary and finalizing the request. + Finishes writing a multipart response by sending the ending boundary and + finalizing the request. */ - void finalizePart(string boundary, string epilogue = null) + void finalizeMultiPart(string epilogue = null) { bodyWriter.write("--"); - bodyWriter.write(boundary); + bodyWriter.write(m_multipartBoundary); bodyWriter.write("--\r\n"); if (epilogue.length) { @@ -955,12 +984,15 @@ final class HTTPClientRequest : HTTPRequest { /// unittest { + import vibe.core.file : openFile; + import vibe.http.common : MultiPart, MultiPartBody; + void test(HTTPClientRequest req) { MultiPartBody part = new MultiPartBody; part.parts ~= MultiPart.formData("name", "bob"); part.parts ~= MultiPart.singleFile("picture", "picture.png", "image/png", openFile("res/profilepicture.png")); - part.parts ~= MultiPart.singleFile("upload", Path("file.zip")); // auto read & mime detection from filename - req.writePart(part); + part.parts ~= MultiPart.singleFile("upload", NativePath("file.zip")); // auto read & mime detection from filename + req.writeMultiPartBody(part); } } diff --git a/http/vibe/http/common.d b/http/vibe/http/common.d index 004bf7c6ee..8c6841be1f 100644 --- a/http/vibe/http/common.d +++ b/http/vibe/http/common.d @@ -341,154 +341,335 @@ class HTTPStatusException : Exception { string debugMessage; } +/** + Represents a single part in a multipart message. Each part can have its own + headers to specify handling of the part for the server. + + Use $(LREF MultiPartBody) to manage a collection of parts. +*/ final class MultiPart { import vibe.stream.memory : createMemoryStream; + /// Headers for this part of the multipart data. InetHeaderMap headers; - InputStreamProxy content; + + private + { + InputStreamProxy m_content; + size_t m_contentLength; + } @safe: - static MultiPart formData(InputStream)(string field_name, InputStream stream, string content_type = "text/plain; charset=\"utf-8\"", bool binary = false) + /** + Sets the content stream & length to the given parameters. + + Params: + stream = the input stream to read from when writing the MultiPart. + exact_content_length = Set to the content length in bytes to allow + calculation of the `Content-Length` header. Leave 0 to omit. + */ + void setContent(InputStream)(InputStream stream, + size_t exact_content_length = 0) + if (isInputStream!InputStream) + { + m_content = interfaceProxy!(.InputStream)(stream); + m_contentLength = exact_content_length; + } + + /** + Returns the content stream, wrapped as InputStreamProxy as previously + set from setContent or from the static constructing methods. + */ + inout(InputStreamProxy) content() @property inout + { + return m_content; + } + + /** + Sets a form-data field with the given name to a static value. + + Params: + field_name = The field name for example as defined in a HTML form. + stream = An InputStream that contains the value for this field. + value = A fixed value string that contains the value for this field. + content = A fixed value that contains the value for this field. + content_type = The content type of the value. If set to empty string + no content type will be sent. + binary = Set to true to set Content-Transfer-Encoding to binary. + exact_content_length = Exact length of what the stream will evaluate + to in bytes. Used for Content-Length calculation if given. + */ + static MultiPart formData(InputStream)(string field_name, InputStream stream, + string content_type = "text/plain; charset=\"utf-8\"", bool binary = false, + size_t exact_content_length = 0) if (isInputStream!InputStream) { auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; - ret.headers["Content-Type"] = content_type; + if (content_type.length) + ret.headers["Content-Type"] = content_type; if (binary) ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.content = stream; + ret.setContent(stream, exact_content_length); return ret; } - static MultiPart formData(string field_name, string value, string content_type = "text/plain; charset=\"utf-8\"") + /// ditto + static MultiPart formData(string field_name, string value, + string content_type = "text/plain; charset=\"utf-8\"") { - auto ret = new MultiPart(); - ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; - ret.headers["Content-Type"] = content_type; - emplace(&ret.content, createMemoryStream(cast(ubyte[]) value.dup, false)); - return ret; + return formData(field_name, createMemoryStream(cast(ubyte[]) value.dup, false), + content_type, false, value.length); } - static MultiPart formData(string field_name, ubyte[] content, string content_type = "application/octet-stream") + /// ditto + static MultiPart formData(string field_name, ubyte[] content, + string content_type = "application/octet-stream") { - auto ret = new MultiPart(); - ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; - ret.headers["Content-Transfer-Encoding"] = "binary"; - emplace(&ret.content, createMemoryStream(content, false)); - return ret; + return formData(field_name, createMemoryStream(cast(ubyte[]) content, false), + content_type, true, content.length); } - static MultiPart singleFile(string field_name, Path file) + /** + Helper function directly reading from a file calling singleFile with the + InputStream parameter and content length set. + */ + static MultiPart singleFile(string field_name, NativePath file) { import vibe.inet.mimetypes : getMimeTypeForFile; import vibe.core.file : openFile, FileMode; - auto ret = new MultiPart(); - ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ file.head.name ~ "\""; - string type = getMimeTypeForFile(file.toString); - ret.headers["Content-Type"] = type; - if (!type.startsWith("text/")) - ret.headers["Content-Transfer-Encoding"] = "binary"; - emplace(&ret.content, openFile(file, FileMode.read)); - return ret; - } - - static MultiPart singleFile(InputStream)(string field_name, string filename, string content_type, InputStream stream, bool binary = true) + const type = getMimeTypeForFile(file.toString); + const binary = !type.startsWith("text/"); + auto f = openFile(file, FileMode.read); + return singleFile(field_name, file.head.name, type, f, binary, f.size); + } + + /** + Sets a form-data field with the given name and a filename which is set + inside the Content-Disposition header and a value. + + Params: + field_name = The field name for example as defined in a HTML form. + filename = The filename (without path) to set for this field. + stream = An InputStream that represents the content of the file. + content = The fixed content of the file. + content_type = The content type of the file. If set to empty string + no content type will be sent. + binary = Set to true to set Content-Transfer-Encoding to binary. + exact_content_length = Exact length of what the stream will evaluate + to in bytes. Used for Content-Length calculation if given. + */ + static MultiPart singleFile(InputStream)(string field_name, string filename, + string content_type, InputStream stream, bool binary = true, + size_t exact_content_length = 0) if (isInputStream!InputStream) { auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; - ret.headers["Content-Type"] = content_type; + if (content_type.length) + ret.headers["Content-Type"] = content_type; if (binary) ret.headers["Content-Transfer-Encoding"] = "binary"; - emplace(&ret.content, stream); + ret.setContent(stream, exact_content_length); return ret; } - static MultiPart singleFile(string field_name, string filename, string content_type, string content) + /// ditto + static MultiPart singleFile(string field_name, string filename, + string content_type, string content) { - auto ret = new MultiPart(); - ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; - ret.headers["Content-Type"] = content_type; - emplace(&ret.content, createMemoryStream((() @trusted => cast(ubyte[]) content)(), false)); - return ret; + return singleFile(field_name, filename, content_type, + createMemoryStream(cast(ubyte[]) content.dup, false), false, + content.length); } - static MultiPart singleFile(string field_name, string filename, string content_type, ubyte[] content) + /// ditto + static MultiPart singleFile(string field_name, string filename, + string content_type, ubyte[] content) { - auto ret = new MultiPart(); - ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; - ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.headers["Content-Type"] = content_type; - emplace(&ret.content, createMemoryStream(content, false)); - return ret; + return singleFile(field_name, filename, content_type, + createMemoryStream(content, false), true, content.length); } - static MultiPart multipleFilesPart(Path file) + /** + Helper function directly reading from a file calling multipleFilesPart + with the InputStream parameter and content length set. + */ + static MultiPart multipleFilesPart(NativePath file) { import vibe.inet.mimetypes : getMimeTypeForFile; import vibe.core.file : openFile, FileMode; - auto ret = new MultiPart(); - ret.headers["Content-Disposition"] = "file; filename=\"" ~ file.head.name ~ "\""; - string type = getMimeTypeForFile(file.toString); - ret.headers["Content-Type"] = type; - if (!type.startsWith("text/")) - ret.headers["Content-Transfer-Encoding"] = "binary"; - emplace(&ret.content, openFile(file, FileMode.read)); - return ret; - } - - static MultiPart multipleFilesPart(InputStream)(string filename, string content_type, InputStream stream, bool binary = false) + const type = getMimeTypeForFile(file.toString); + const binary = !type.startsWith("text/"); + auto f = openFile(file, FileMode.read); + return multipleFilesPart(file.head.name, type, f, binary, f.size); + } + + /** + Creates a part for a multi-file form field to store multiple files in a + single part. Store all multipleFilesPart MultiParts inside a + MultiPartBody and use multipleFiles to associate them all with a single + form field. + + Params: + filename = The filename (without path) to set for this file. + stream = An InputStream that represents the content of the file. + content = The fixed content of the file. + content_type = The content type of the file. If set to empty string + no content type will be sent. + binary = Set to true to set Content-Transfer-Encoding to binary. + exact_content_length = Exact length of what the stream will evaluate + to in bytes. Used for Content-Length calculation if given. + */ + static MultiPart multipleFilesPart(InputStream)(string filename, + string content_type, InputStream stream, bool binary = false, + size_t exact_content_length = 0) if (isInputStream!InputStream) { auto ret = new MultiPart(); ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; - ret.headers["Content-Type"] = content_type; - if (binary) - ret.headers["Content-Transfer-Encoding"] = "binary"; - emplace(&ret.content, stream); + if (content_type.length) + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.setContent(stream, exact_content_length); return ret; } - static MultiPart multipleFilesPart(string filename, string content_type, string content) + /// ditto + static MultiPart multipleFilesPart(string filename, string content_type, + string content) { - return multipleFilesPart(filename, content_type, createMemoryStream((() @trusted => cast(ubyte[]) content)(), false), true); + return multipleFilesPart(filename, content_type, + createMemoryStream((() @trusted => cast(ubyte[]) content)(), false), + false, content.length); } - static MultiPart multipleFilesPart(string filename, string content_type, ubyte[] content) + /// ditto + static MultiPart multipleFilesPart(string filename, string content_type, + ubyte[] content) { - return multipleFilesPart(filename, content_type, createMemoryStream(content, false), true); - } - - static MultiPart multipleFiles(string name, MultiPartBody multipart) + return multipleFilesPart(filename, content_type, + createMemoryStream(content, false), true, content.length); + } + + /** + Creates a field containing multiple values of different kinds inside it + using `multipart/mixed`. Useful for example to attach multiple files + inside a mail multipart. + + You may for example use this to represent a multipart/mixed type to + describe a specific file order of multiple files or you could use + multipart/alternative to represent different file type versions of the + same file for a mail program. + + Params: + name = the field name to associate the mixed multipart to. + multipart = the MultiPartBody containing all the different parts to + attach. + content_type = The subtype for this mixed multipart to have. Common + types include `multipart/mixed` or `multipart/alternative`. + */ + static MultiPart multipleFiles(string name, MultiPartBody multipart, + string content_type = "multipart/mixed") { import vibe.stream.memory : createMemoryOutputStream; auto ret = new MultiPart(); string boundary = randomMultipartBoundary; ret.headers["Content-Disposition"] = "form-data; name=\"" ~ name ~ "\""; - ret.headers["Content-Type"] = "multipart/mixed; boundary=\"" ~ boundary ~ "\""; + ret.headers["Content-Type"] = content_type ~ "; boundary=\"" ~ boundary ~ "\""; auto stream = createMemoryOutputStream(); multipart.write(boundary, stream); - emplace(&ret.content, createMemoryStream(stream.data, false)); + ret.setContent(createMemoryStream(stream.data, false), stream.data.length); return ret; } + + /** + Calculates the content length of this part in bytes including headers + and boundary length. + + Returns: the length of this part in bytes or 0 if it couldn't be + determined. + */ + size_t getLength(string boundary) const + { + if (m_contentLength == 0) + return 0; + + size_t length; + length += 4 + boundary.length; // --boundary\r\n + foreach (k, v; headers.byKeyValue) + length += 4 + k.length + v.length; // "key: value\r\n" + length += 2; // \r\n + length += m_contentLength; + length += 2; // \r\n + return length; + } + + /** + Writes this MultiPart to the given output stream. + + To use on a HTTPClient use the `writePart` method of `HTTPClientRequest`. + */ + void write(OutputStream)(OutputStream output, string boundary) + if (isOutputStream!OutputStream) + { + output.write("--"); + output.write(boundary); + output.write("\r\n"); + foreach (k, v; headers.byKeyValue) { + output.write(k); + output.write(": "); + output.write(v); + output.write("\r\n"); + } + output.write("\r\n"); + pipe(content, output); + output.write("\r\n"); + } } +/** + Collection container for multiple MultiPart parts, a content type and an + optional preamble/epilogue. + + Standards: $(LINK https://tools.ietf.org/html/rfc1521#section-7.2) +*/ final class MultiPartBody { - // https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html - + /** + The mime type of this multipart. For HTTP this is most usually + multipart/form-data to indicate this describing form fields. + */ string contentType = "multipart/form-data"; + + /** + Extra information to send before/after the multipart data which is + usually ignored for server processing but allows to include text for + example for mail users with non-multipart-supporting mail clients for + instructions how to read this file. + */ string preamble, epilogue; + /** + The full collection of parts that this multipart describes. + */ MultiPart[] parts; @safe: + /** + Computes the length of the parts, preamble, epilogue together in bytes + for sending `Content-Length` headers or calculating progress. + + Returns: the number of bytes the content is gonna take up or `0` if it + cannot be determined in case a part is not a MemoryStream. + */ size_t length(string boundary) const { import vibe.stream.memory : MemoryStream; @@ -498,32 +679,27 @@ final class MultiPartBody size_t length; if (preamble.length) - length += preamble.length + 2; - length += boundary.length + 2; // --boundary + length += preamble.length + 2; // \r\n foreach (part; parts) { - length += 8 + boundary.length; // \r\n * 3 + boundary (+2) - foreach (k, v; part.headers) - length += 4 + k.length + v.length; // ": \r\n" - - static assert(typeof(part.content).tupleof[1].stringof == "m_intf", - "Can't check the interface type of InterfaceProxy because definition changed"); - - // check if content is a MemoryStream because then we can compute exact lengths. - // with current vibe-core it's not possible to find out the type or even check if it matches one without triggering a fatal assert error, so we access private fields here - // TODO: we would preferably want to support all classes implementing RandomAccessStream here, so we might need to traverse the typeinfo - if ((() @trusted => cast() part.content.tupleof[1])()._typeInfo() is typeid(MemoryStream)) { - auto randomStream = (() @trusted => cast() part.content)().extract!MemoryStream; - length += randomStream.size; - } else { - return 0; // undetectable - } + const subLength = part.getLength(boundary); + if (subLength == 0) + return 0; + length += subLength; } - length += 4; + length += boundary.length + 6; // "--boundary--\r\n" if (epilogue.length) length += epilogue.length + 2; return length; } + /** + Writes the full multipart body to the given output stream with the given + multipart boundary. The boundary can be obtained through + $(LREF randomMultipartBoundary). + + If you want to send the MultiPartBody in a HTTP request, it is + recommended to use $(REF writePart, vibe,http,client,HTTPClientRequest). + */ void write(T)(string boundary, T output) if (isOutputStream!T) { @@ -532,25 +708,15 @@ final class MultiPartBody if (!parts.length) return; - boundary = "--" ~ boundary; if (preamble.length) { output.write(preamble); output.write("\r\n"); } - output.write(boundary); foreach (part; parts) { - output.write("\r\n"); - foreach (k, v; part.headers) { - output.write(k); - output.write(": "); - output.write(v); - output.write("\r\n"); - } - output.write("\r\n"); - pipe(part.content, output); - output.write("\r\n"); - output.write(boundary); + part.write(output, boundary); } + output.write("--"); + output.write(boundary); output.write("--\r\n"); if (epilogue.length) { @@ -560,17 +726,34 @@ final class MultiPartBody } } +/** + * Creates a random multipart boundary string starting with hyphens containing + * random text data for separation of data in data uploads. + * + * Uses a cryptographically secure random to generate the boundary string. Use + * this if you plan to manually write a multipart/form-data document somewhere. + * + * You should use $(LREF MultiPart) for an easy to use and compatible API + * instead. + */ string randomMultipartBoundary() @safe { - import std.random : uniform; + import vibe.crypto.cryptorand : secureRNG; import std.ascii : digits, letters; + // simple characters which should be supported everywhere without conflict. + // this is 64 characters which makes the random modulo have a very + // convenient uniform distribution. static immutable string boundaryChars = digits ~ letters ~ "_-"; // ~ "'()+_,-./:=?"; - char[64] ret; // can be up to 70 according to spec, but taking poor implementations into account - ret[0 .. 20] = '-'; // some padding before random - for (int i = 20; i < ret.length; i++) - ret[i] = boundaryChars[uniform(0, $)]; + auto rng = secureRNG(); + char[64] ret; // can be up to 70 according to spec, 64 should be enough + ubyte[64] randomBuffer; + rng.read(randomBuffer[]); + + ret[0 .. 16] = '-'; // some padding before random + for (int i = 16; i < ret.length; i++) + ret[i] = boundaryChars[randomBuffer[i] % $]; return ret[].idup; } From 3ad28f425bc0ef21da68cf644ad3924308edb566 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sat, 18 Apr 2020 21:02:03 +0200 Subject: [PATCH 07/10] fix multipart test to actually run --- tests/vibe.http.client.1005/dub.sdl | 1 - tests/vibe.http.client.1005/source/app.d | 32 +++++++++++++----------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/vibe.http.client.1005/dub.sdl b/tests/vibe.http.client.1005/dub.sdl index 9af498fdea..b33f623f79 100644 --- a/tests/vibe.http.client.1005/dub.sdl +++ b/tests/vibe.http.client.1005/dub.sdl @@ -1,4 +1,3 @@ name "tests" description "Tests multipart encoding" dependency "vibe-d:http" path="../.." -versions "VibeDefaultMain" diff --git a/tests/vibe.http.client.1005/source/app.d b/tests/vibe.http.client.1005/source/app.d index 3d970d4383..bf9b9e27f2 100644 --- a/tests/vibe.http.client.1005/source/app.d +++ b/tests/vibe.http.client.1005/source/app.d @@ -7,8 +7,10 @@ import vibe.http.server; import vibe.stream.operations; import vibe.stream.memory; -shared static this() +void main(string[] args) { + bool handled = false; + listenHTTP("127.0.0.1:11005", (scope req, scope res) { assert(req.form.length == 1); assert(req.files.length == 1); @@ -20,19 +22,21 @@ shared static this() assert(data == "Totally\0a\0PNG\0file."); removeFile(req.files["picture"].tempPath); res.writeBody("ok"); + handled = true; }); - runTask({ - requestHTTP("http://127.0.0.1:11005", - (scope req) { - MultiPartBody part = new MultiPartBody; - part.parts ~= MultiPart.formData("name", "bob"); - auto memStream = createMemoryStream(cast(ubyte[]) "Totally\0a\0PNG\0file.", false); - part.parts ~= MultiPart.singleFile("picture", "picture.png", "image/png", memStream, true); - req.writePart(part); - }, - (scope res) {} - ); - exitEventLoop(); - }); + requestHTTP("http://127.0.0.1:11005", + (scope req) { + MultiPartBody part = new MultiPartBody; + part.parts ~= MultiPart.formData("name", "bob"); + auto memStream = createMemoryStream(cast(ubyte[]) "Totally\0a\0PNG\0file.", false); + part.parts ~= MultiPart.singleFile("picture", "picture.png", "image/png", memStream, true); + req.writeMultiPartBody(part); + }, + (scope res) { + assert(res.bodyReader.readAllUTF8() == "ok"); + } + ); + + assert(handled); } From 4f8bab28f40b74f9a17f83e965a56724fa671027 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sat, 18 Apr 2020 21:17:33 +0200 Subject: [PATCH 08/10] move MultiPart{,Body} into vibe.inet --- http/vibe/http/client.d | 3 +- http/vibe/http/common.d | 385 ---------------------- inet/vibe/inet/webform.d | 393 +++++++++++++++++++++++ tests/vibe.http.client.1005/source/app.d | 3 +- 4 files changed, 397 insertions(+), 387 deletions(-) diff --git a/http/vibe/http/client.d b/http/vibe/http/client.d index e3473b9ff8..c810533b4b 100644 --- a/http/vibe/http/client.d +++ b/http/vibe/http/client.d @@ -17,6 +17,7 @@ import vibe.core.log; import vibe.data.json; import vibe.inet.message; import vibe.inet.url; +import vibe.inet.webform : MultiPart, MultiPartBody; import vibe.stream.counting; import vibe.stream.tls; import vibe.stream.operations; @@ -985,7 +986,7 @@ final class HTTPClientRequest : HTTPRequest { /// unittest { import vibe.core.file : openFile; - import vibe.http.common : MultiPart, MultiPartBody; + import vibe.inet.webform : MultiPart, MultiPartBody; void test(HTTPClientRequest req) { MultiPartBody part = new MultiPartBody; diff --git a/http/vibe/http/common.d b/http/vibe/http/common.d index 8c6841be1f..4788df2f25 100644 --- a/http/vibe/http/common.d +++ b/http/vibe/http/common.d @@ -341,391 +341,6 @@ class HTTPStatusException : Exception { string debugMessage; } -/** - Represents a single part in a multipart message. Each part can have its own - headers to specify handling of the part for the server. - - Use $(LREF MultiPartBody) to manage a collection of parts. -*/ -final class MultiPart -{ - import vibe.stream.memory : createMemoryStream; - - /// Headers for this part of the multipart data. - InetHeaderMap headers; - - private - { - InputStreamProxy m_content; - size_t m_contentLength; - } - - @safe: - - /** - Sets the content stream & length to the given parameters. - - Params: - stream = the input stream to read from when writing the MultiPart. - exact_content_length = Set to the content length in bytes to allow - calculation of the `Content-Length` header. Leave 0 to omit. - */ - void setContent(InputStream)(InputStream stream, - size_t exact_content_length = 0) - if (isInputStream!InputStream) - { - m_content = interfaceProxy!(.InputStream)(stream); - m_contentLength = exact_content_length; - } - - /** - Returns the content stream, wrapped as InputStreamProxy as previously - set from setContent or from the static constructing methods. - */ - inout(InputStreamProxy) content() @property inout - { - return m_content; - } - - /** - Sets a form-data field with the given name to a static value. - - Params: - field_name = The field name for example as defined in a HTML form. - stream = An InputStream that contains the value for this field. - value = A fixed value string that contains the value for this field. - content = A fixed value that contains the value for this field. - content_type = The content type of the value. If set to empty string - no content type will be sent. - binary = Set to true to set Content-Transfer-Encoding to binary. - exact_content_length = Exact length of what the stream will evaluate - to in bytes. Used for Content-Length calculation if given. - */ - static MultiPart formData(InputStream)(string field_name, InputStream stream, - string content_type = "text/plain; charset=\"utf-8\"", bool binary = false, - size_t exact_content_length = 0) - if (isInputStream!InputStream) - { - auto ret = new MultiPart(); - ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; - if (content_type.length) - ret.headers["Content-Type"] = content_type; - if (binary) - ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.setContent(stream, exact_content_length); - return ret; - } - - /// ditto - static MultiPart formData(string field_name, string value, - string content_type = "text/plain; charset=\"utf-8\"") - { - return formData(field_name, createMemoryStream(cast(ubyte[]) value.dup, false), - content_type, false, value.length); - } - - /// ditto - static MultiPart formData(string field_name, ubyte[] content, - string content_type = "application/octet-stream") - { - return formData(field_name, createMemoryStream(cast(ubyte[]) content, false), - content_type, true, content.length); - } - - /** - Helper function directly reading from a file calling singleFile with the - InputStream parameter and content length set. - */ - static MultiPart singleFile(string field_name, NativePath file) - { - import vibe.inet.mimetypes : getMimeTypeForFile; - import vibe.core.file : openFile, FileMode; - - const type = getMimeTypeForFile(file.toString); - const binary = !type.startsWith("text/"); - auto f = openFile(file, FileMode.read); - return singleFile(field_name, file.head.name, type, f, binary, f.size); - } - - /** - Sets a form-data field with the given name and a filename which is set - inside the Content-Disposition header and a value. - - Params: - field_name = The field name for example as defined in a HTML form. - filename = The filename (without path) to set for this field. - stream = An InputStream that represents the content of the file. - content = The fixed content of the file. - content_type = The content type of the file. If set to empty string - no content type will be sent. - binary = Set to true to set Content-Transfer-Encoding to binary. - exact_content_length = Exact length of what the stream will evaluate - to in bytes. Used for Content-Length calculation if given. - */ - static MultiPart singleFile(InputStream)(string field_name, string filename, - string content_type, InputStream stream, bool binary = true, - size_t exact_content_length = 0) - if (isInputStream!InputStream) - { - auto ret = new MultiPart(); - ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; - if (content_type.length) - ret.headers["Content-Type"] = content_type; - if (binary) - ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.setContent(stream, exact_content_length); - return ret; - } - - /// ditto - static MultiPart singleFile(string field_name, string filename, - string content_type, string content) - { - return singleFile(field_name, filename, content_type, - createMemoryStream(cast(ubyte[]) content.dup, false), false, - content.length); - } - - /// ditto - static MultiPart singleFile(string field_name, string filename, - string content_type, ubyte[] content) - { - return singleFile(field_name, filename, content_type, - createMemoryStream(content, false), true, content.length); - } - - /** - Helper function directly reading from a file calling multipleFilesPart - with the InputStream parameter and content length set. - */ - static MultiPart multipleFilesPart(NativePath file) - { - import vibe.inet.mimetypes : getMimeTypeForFile; - import vibe.core.file : openFile, FileMode; - - const type = getMimeTypeForFile(file.toString); - const binary = !type.startsWith("text/"); - auto f = openFile(file, FileMode.read); - return multipleFilesPart(file.head.name, type, f, binary, f.size); - } - - /** - Creates a part for a multi-file form field to store multiple files in a - single part. Store all multipleFilesPart MultiParts inside a - MultiPartBody and use multipleFiles to associate them all with a single - form field. - - Params: - filename = The filename (without path) to set for this file. - stream = An InputStream that represents the content of the file. - content = The fixed content of the file. - content_type = The content type of the file. If set to empty string - no content type will be sent. - binary = Set to true to set Content-Transfer-Encoding to binary. - exact_content_length = Exact length of what the stream will evaluate - to in bytes. Used for Content-Length calculation if given. - */ - static MultiPart multipleFilesPart(InputStream)(string filename, - string content_type, InputStream stream, bool binary = false, - size_t exact_content_length = 0) - if (isInputStream!InputStream) - { - auto ret = new MultiPart(); - ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; - if (content_type.length) - ret.headers["Content-Type"] = content_type; - if (binary) - ret.headers["Content-Transfer-Encoding"] = "binary"; - ret.setContent(stream, exact_content_length); - return ret; - } - - /// ditto - static MultiPart multipleFilesPart(string filename, string content_type, - string content) - { - return multipleFilesPart(filename, content_type, - createMemoryStream((() @trusted => cast(ubyte[]) content)(), false), - false, content.length); - } - - /// ditto - static MultiPart multipleFilesPart(string filename, string content_type, - ubyte[] content) - { - return multipleFilesPart(filename, content_type, - createMemoryStream(content, false), true, content.length); - } - - /** - Creates a field containing multiple values of different kinds inside it - using `multipart/mixed`. Useful for example to attach multiple files - inside a mail multipart. - - You may for example use this to represent a multipart/mixed type to - describe a specific file order of multiple files or you could use - multipart/alternative to represent different file type versions of the - same file for a mail program. - - Params: - name = the field name to associate the mixed multipart to. - multipart = the MultiPartBody containing all the different parts to - attach. - content_type = The subtype for this mixed multipart to have. Common - types include `multipart/mixed` or `multipart/alternative`. - */ - static MultiPart multipleFiles(string name, MultiPartBody multipart, - string content_type = "multipart/mixed") - { - import vibe.stream.memory : createMemoryOutputStream; - - auto ret = new MultiPart(); - string boundary = randomMultipartBoundary; - ret.headers["Content-Disposition"] = "form-data; name=\"" ~ name ~ "\""; - ret.headers["Content-Type"] = content_type ~ "; boundary=\"" ~ boundary ~ "\""; - auto stream = createMemoryOutputStream(); - multipart.write(boundary, stream); - ret.setContent(createMemoryStream(stream.data, false), stream.data.length); - return ret; - } - - /** - Calculates the content length of this part in bytes including headers - and boundary length. - - Returns: the length of this part in bytes or 0 if it couldn't be - determined. - */ - size_t getLength(string boundary) const - { - if (m_contentLength == 0) - return 0; - - size_t length; - length += 4 + boundary.length; // --boundary\r\n - foreach (k, v; headers.byKeyValue) - length += 4 + k.length + v.length; // "key: value\r\n" - length += 2; // \r\n - length += m_contentLength; - length += 2; // \r\n - return length; - } - - /** - Writes this MultiPart to the given output stream. - - To use on a HTTPClient use the `writePart` method of `HTTPClientRequest`. - */ - void write(OutputStream)(OutputStream output, string boundary) - if (isOutputStream!OutputStream) - { - output.write("--"); - output.write(boundary); - output.write("\r\n"); - foreach (k, v; headers.byKeyValue) { - output.write(k); - output.write(": "); - output.write(v); - output.write("\r\n"); - } - output.write("\r\n"); - pipe(content, output); - output.write("\r\n"); - } -} - -/** - Collection container for multiple MultiPart parts, a content type and an - optional preamble/epilogue. - - Standards: $(LINK https://tools.ietf.org/html/rfc1521#section-7.2) -*/ -final class MultiPartBody -{ - /** - The mime type of this multipart. For HTTP this is most usually - multipart/form-data to indicate this describing form fields. - */ - string contentType = "multipart/form-data"; - - /** - Extra information to send before/after the multipart data which is - usually ignored for server processing but allows to include text for - example for mail users with non-multipart-supporting mail clients for - instructions how to read this file. - */ - string preamble, epilogue; - - /** - The full collection of parts that this multipart describes. - */ - MultiPart[] parts; - - @safe: - - /** - Computes the length of the parts, preamble, epilogue together in bytes - for sending `Content-Length` headers or calculating progress. - - Returns: the number of bytes the content is gonna take up or `0` if it - cannot be determined in case a part is not a MemoryStream. - */ - size_t length(string boundary) const - { - import vibe.stream.memory : MemoryStream; - - if (!parts.length) - return 0; - - size_t length; - if (preamble.length) - length += preamble.length + 2; // \r\n - foreach (part; parts) { - const subLength = part.getLength(boundary); - if (subLength == 0) - return 0; - length += subLength; - } - length += boundary.length + 6; // "--boundary--\r\n" - if (epilogue.length) - length += epilogue.length + 2; - return length; - } - - /** - Writes the full multipart body to the given output stream with the given - multipart boundary. The boundary can be obtained through - $(LREF randomMultipartBoundary). - - If you want to send the MultiPartBody in a HTTP request, it is - recommended to use $(REF writePart, vibe,http,client,HTTPClientRequest). - */ - void write(T)(string boundary, T output) - if (isOutputStream!T) - { - import vibe.core.stream : pipe; - - if (!parts.length) - return; - - if (preamble.length) { - output.write(preamble); - output.write("\r\n"); - } - foreach (part; parts) { - part.write(output, boundary); - } - output.write("--"); - output.write(boundary); - output.write("--\r\n"); - if (epilogue.length) - { - output.write(epilogue); - output.write("\r\n"); - } - } -} - /** * Creates a random multipart boundary string starting with hyphens containing * random text data for separation of data in data uploads. diff --git a/inet/vibe/inet/webform.d b/inet/vibe/inet/webform.d index 1486a7d099..e411f374c2 100644 --- a/inet/vibe/inet/webform.d +++ b/inet/vibe/inet/webform.d @@ -337,6 +337,399 @@ private bool parseMultipartFormPart(InputStream)(InputStream stream, ref FormFie return true; } +/** + Represents a single part in a multipart message. Each part can have its own + headers to specify handling of the part for the receiver. + + Use $(LREF MultiPartBody) to manage a collection of parts. +*/ +final class MultiPart +{ + import vibe.stream.memory : createMemoryStream; + + /// Headers for this part of the multipart data. + InetHeaderMap headers; + + private + { + InputStreamProxy m_content; + size_t m_contentLength; + } + + @safe: + + /** + Sets the content stream & length to the given parameters. + + Params: + stream = the input stream to read from when writing the MultiPart. + exact_content_length = Set to the content length in bytes to allow + calculation of the `Content-Length` header. Leave 0 to omit. + */ + void setContent(InputStream)(InputStream stream, + size_t exact_content_length = 0) + if (isInputStream!InputStream) + { + import vibe.internal.interfaceproxy : interfaceProxy; + + m_content = interfaceProxy!(.InputStream)(stream); + m_contentLength = exact_content_length; + } + + /** + Returns the content stream, wrapped as InputStreamProxy as previously + set from setContent or from the static constructing methods. + */ + inout(InputStreamProxy) content() @property inout + { + return m_content; + } + + /** + Sets a form-data field with the given name to a static value. + + Params: + field_name = The field name for example as defined in a HTML form. + stream = An InputStream that contains the value for this field. + value = A fixed value string that contains the value for this field. + content = A fixed value that contains the value for this field. + content_type = The content type of the value. If set to empty string + no content type will be sent. + binary = Set to true to set Content-Transfer-Encoding to binary. + exact_content_length = Exact length of what the stream will evaluate + to in bytes. Used for Content-Length calculation if given. + */ + static MultiPart formData(InputStream)(string field_name, InputStream stream, + string content_type = "text/plain; charset=\"utf-8\"", bool binary = false, + size_t exact_content_length = 0) + if (isInputStream!InputStream) + { + auto ret = new MultiPart(); + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; + if (content_type.length) + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.setContent(stream, exact_content_length); + return ret; + } + + /// ditto + static MultiPart formData(string field_name, string value, + string content_type = "text/plain; charset=\"utf-8\"") + { + return formData(field_name, createMemoryStream(cast(ubyte[]) value.dup, false), + content_type, false, value.length); + } + + /// ditto + static MultiPart formData(string field_name, ubyte[] content, + string content_type = "application/octet-stream") + { + return formData(field_name, createMemoryStream(cast(ubyte[]) content, false), + content_type, true, content.length); + } + + /** + Helper function directly reading from a file calling singleFile with the + InputStream parameter and content length set. + */ + static MultiPart singleFile(string field_name, NativePath file) + { + import vibe.inet.mimetypes : getMimeTypeForFile; + import vibe.core.file : openFile, FileMode; + + const type = getMimeTypeForFile(file.toString); + const binary = !type.startsWith("text/"); + auto f = openFile(file, FileMode.read); + return singleFile(field_name, file.head.name, type, f, binary, f.size); + } + + /** + Sets a form-data field with the given name and a filename which is set + inside the Content-Disposition header and a value. + + Params: + field_name = The field name for example as defined in a HTML form. + filename = The filename (without path) to set for this field. + stream = An InputStream that represents the content of the file. + content = The fixed content of the file. + content_type = The content type of the file. If set to empty string + no content type will be sent. + binary = Set to true to set Content-Transfer-Encoding to binary. + exact_content_length = Exact length of what the stream will evaluate + to in bytes. Used for Content-Length calculation if given. + */ + static MultiPart singleFile(InputStream)(string field_name, string filename, + string content_type, InputStream stream, bool binary = true, + size_t exact_content_length = 0) + if (isInputStream!InputStream) + { + auto ret = new MultiPart(); + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; + if (content_type.length) + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.setContent(stream, exact_content_length); + return ret; + } + + /// ditto + static MultiPart singleFile(string field_name, string filename, + string content_type, string content) + { + return singleFile(field_name, filename, content_type, + createMemoryStream(cast(ubyte[]) content.dup, false), false, + content.length); + } + + /// ditto + static MultiPart singleFile(string field_name, string filename, + string content_type, ubyte[] content) + { + return singleFile(field_name, filename, content_type, + createMemoryStream(content, false), true, content.length); + } + + /** + Helper function directly reading from a file calling multipleFilesPart + with the InputStream parameter and content length set. + */ + static MultiPart multipleFilesPart(NativePath file) + { + import vibe.inet.mimetypes : getMimeTypeForFile; + import vibe.core.file : openFile, FileMode; + + const type = getMimeTypeForFile(file.toString); + const binary = !type.startsWith("text/"); + auto f = openFile(file, FileMode.read); + return multipleFilesPart(file.head.name, type, f, binary, f.size); + } + + /** + Creates a part for a multi-file form field to store multiple files in a + single part. Store all multipleFilesPart MultiParts inside a + MultiPartBody and use multipleFiles to associate them all with a single + form field. + + Params: + filename = The filename (without path) to set for this file. + stream = An InputStream that represents the content of the file. + content = The fixed content of the file. + content_type = The content type of the file. If set to empty string + no content type will be sent. + binary = Set to true to set Content-Transfer-Encoding to binary. + exact_content_length = Exact length of what the stream will evaluate + to in bytes. Used for Content-Length calculation if given. + */ + static MultiPart multipleFilesPart(InputStream)(string filename, + string content_type, InputStream stream, bool binary = false, + size_t exact_content_length = 0) + if (isInputStream!InputStream) + { + auto ret = new MultiPart(); + ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; + if (content_type.length) + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + ret.setContent(stream, exact_content_length); + return ret; + } + + /// ditto + static MultiPart multipleFilesPart(string filename, string content_type, + string content) + { + return multipleFilesPart(filename, content_type, + createMemoryStream((() @trusted => cast(ubyte[]) content)(), false), + false, content.length); + } + + /// ditto + static MultiPart multipleFilesPart(string filename, string content_type, + ubyte[] content) + { + return multipleFilesPart(filename, content_type, + createMemoryStream(content, false), true, content.length); + } + + /** + Creates a field containing multiple values of different kinds inside it + using `multipart/mixed`. Useful for example to attach multiple files + inside a mail multipart. + + You may for example use this to represent a multipart/mixed type to + describe a specific file order of multiple files or you could use + multipart/alternative to represent different file type versions of the + same file for a mail program. + + Params: + name = the field name to associate the mixed multipart to. + multipart = the MultiPartBody containing all the different parts to + attach. + boundary = The boundary to use to split the different parts inside + this nested multipart part. Do not use the same value as for the + parent MultiPartBody, instead generate a new one using + $(REF randomMultipartBoundary, vibe,http,common). + content_type = The subtype for this mixed multipart to have. Common + types include `multipart/mixed` or `multipart/alternative`. + */ + static MultiPart multipleFiles(string name, MultiPartBody multipart, + string boundary, string content_type = "multipart/mixed") + { + import vibe.stream.memory : createMemoryOutputStream; + + auto ret = new MultiPart(); + ret.headers["Content-Disposition"] = "form-data; name=\"" ~ name ~ "\""; + ret.headers["Content-Type"] = content_type ~ "; boundary=\"" ~ boundary ~ "\""; + auto stream = createMemoryOutputStream(); + multipart.write(boundary, stream); + ret.setContent(createMemoryStream(stream.data, false), stream.data.length); + return ret; + } + + /** + Calculates the content length of this part in bytes including headers + and boundary length. + + Returns: the length of this part in bytes or 0 if it couldn't be + determined. + */ + size_t getLength(string boundary) const + { + if (m_contentLength == 0) + return 0; + + size_t length; + length += 4 + boundary.length; // --boundary\r\n + foreach (k, v; headers.byKeyValue) + length += 4 + k.length + v.length; // "key: value\r\n" + length += 2; // \r\n + length += m_contentLength; + length += 2; // \r\n + return length; + } + + /** + Writes this MultiPart to the given output stream. + + To use on a HTTPClient use the `writePart` method of `HTTPClientRequest`. + */ + void write(OutputStream)(OutputStream output, string boundary) + if (isOutputStream!OutputStream) + { + output.write("--"); + output.write(boundary); + output.write("\r\n"); + foreach (k, v; headers.byKeyValue) { + output.write(k); + output.write(": "); + output.write(v); + output.write("\r\n"); + } + output.write("\r\n"); + pipe(content, output); + output.write("\r\n"); + } +} + +/** + Collection container for multiple MultiPart parts, a content type and an + optional preamble/epilogue. + + May represent attachments for emails, form data for HTTP requests or other + multipart internet messages. + + Standards: $(LINK https://tools.ietf.org/html/rfc1521#section-7.2) +*/ +final class MultiPartBody +{ + /** + The mime type of this multipart. For HTTP this is most usually + multipart/form-data to indicate this describing form fields. + */ + string contentType = "multipart/form-data"; + + /** + Extra information to send before/after the multipart data which is + usually ignored for server processing but allows to include text for + example for mail users with non-multipart-supporting mail clients for + instructions how to read this file. + */ + string preamble, epilogue; + + /** + The full collection of parts that this multipart describes. + */ + MultiPart[] parts; + + @safe: + + /** + Computes the length of the parts, preamble, epilogue together in bytes + for sending `Content-Length` headers or calculating progress. + + Returns: the number of bytes the content is gonna take up or `0` if it + cannot be determined in case a part is not a MemoryStream. + */ + size_t length(string boundary) const + { + import vibe.stream.memory : MemoryStream; + + if (!parts.length) + return 0; + + size_t length; + if (preamble.length) + length += preamble.length + 2; // \r\n + foreach (part; parts) { + const subLength = part.getLength(boundary); + if (subLength == 0) + return 0; + length += subLength; + } + length += boundary.length + 6; // "--boundary--\r\n" + if (epilogue.length) + length += epilogue.length + 2; + return length; + } + + /** + Writes the full multipart body to the given output stream with the given + multipart boundary. A secure random boundary can be obtained through + $(REF randomMultipartBoundary, vibe,http,common). + + If you want to send the MultiPartBody in a HTTP request, it is + recommended to use $(REF writePart, vibe,http,client,HTTPClientRequest). + */ + void write(T)(string boundary, T output) + if (isOutputStream!T) + { + import vibe.core.stream : pipe; + + if (!parts.length) + return; + + if (preamble.length) { + output.write(preamble); + output.write("\r\n"); + } + foreach (part; parts) { + part.write(output, boundary); + } + output.write("--"); + output.write(boundary); + output.write("--\r\n"); + if (epilogue.length) + { + output.write(epilogue); + output.write("\r\n"); + } + } +} + /** Encodes a Key-Value map into a form URL encoded string. diff --git a/tests/vibe.http.client.1005/source/app.d b/tests/vibe.http.client.1005/source/app.d index bf9b9e27f2..7cd8845c12 100644 --- a/tests/vibe.http.client.1005/source/app.d +++ b/tests/vibe.http.client.1005/source/app.d @@ -4,8 +4,9 @@ import vibe.core.log; import vibe.core.net; import vibe.http.client; import vibe.http.server; -import vibe.stream.operations; +import vibe.inet.webform; import vibe.stream.memory; +import vibe.stream.operations; void main(string[] args) { From 0c98b1d20c22f72481b3f3332bea0f2438bb634a Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Sun, 19 Apr 2020 00:20:37 +0200 Subject: [PATCH 09/10] cast file size to size_t for x86 --- inet/vibe/inet/webform.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inet/vibe/inet/webform.d b/inet/vibe/inet/webform.d index e411f374c2..10719758c3 100644 --- a/inet/vibe/inet/webform.d +++ b/inet/vibe/inet/webform.d @@ -442,7 +442,7 @@ final class MultiPart const type = getMimeTypeForFile(file.toString); const binary = !type.startsWith("text/"); auto f = openFile(file, FileMode.read); - return singleFile(field_name, file.head.name, type, f, binary, f.size); + return singleFile(field_name, file.head.name, type, f, binary, cast(size_t) f.size); } /** @@ -504,7 +504,7 @@ final class MultiPart const type = getMimeTypeForFile(file.toString); const binary = !type.startsWith("text/"); auto f = openFile(file, FileMode.read); - return multipleFilesPart(file.head.name, type, f, binary, f.size); + return multipleFilesPart(file.head.name, type, f, binary, cast(size_t) f.size); } /** From d23d9a4e9ca0d80a5cdafbe4e3e6593c4c89f0b3 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Wed, 24 Jun 2020 21:15:03 +0200 Subject: [PATCH 10/10] new more memory efficient multipart api --- http/vibe/http/client.d | 43 ++++++++++++++-- inet/vibe/inet/webform.d | 64 ++++++++++++++---------- tests/vibe.http.client.1005/source/app.d | 2 +- 3 files changed, 77 insertions(+), 32 deletions(-) diff --git a/http/vibe/http/client.d b/http/vibe/http/client.d index c810533b4b..a874623feb 100644 --- a/http/vibe/http/client.d +++ b/http/vibe/http/client.d @@ -954,16 +954,45 @@ final class HTTPClientRequest : HTTPRequest { } /** - Partially writes the body with a multipart. If you plan to manually - write all parts using writePart you need to start with `beginMultiPart` - and end with `finalizeMultiPart`. + Writes a single multipart part, which is essentially one input in a form + request which can contain headers to specify what it is. You need to + start with `beginMultiPart` and end with `finalizeMultiPart` with this + API. Alternatively you can use `writeMultiPartBody` to do everything in one step. */ - void writePart(MultiPart part) + void writePart(InputStream)(string field_name, InputStream data, + string content_type = "text/plain; charset=\"utf-8\"", bool binary = false) + if (isInputStream!InputStream) { - part.write(bodyWriter, m_multipartBoundary); + scope InetHeaderMap headers; + headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; + if (content_type.length) + ret.headers["Content-Type"] = content_type; + if (binary) + ret.headers["Content-Transfer-Encoding"] = "binary"; + writePart(data, headers); + } + + /// ditto + void writePart(InputStream)(InputStream data, scope const ref InetHeaderMap headers) + if (isInputStream!InputStream) + { + assert(m_multipartBoundary.length, "need to call beginMultiPart and finalizeMultiPart with writePart"); + + bodyWriter.write("--"); + bodyWriter.write(m_multipartBoundary); + bodyWriter.write("\r\n"); + foreach (k, v; headers.byKeyValue) { + bodyWriter.write(k); + bodyWriter.write(": "); + bodyWriter.write(v); + bodyWriter.write("\r\n"); + } + bodyWriter.write("\r\n"); + pipe(data, bodyWriter); + bodyWriter.write("\r\n"); } /** @@ -980,6 +1009,7 @@ final class HTTPClientRequest : HTTPRequest { bodyWriter.write(epilogue); bodyWriter.write("\r\n"); } + m_multipartBoundary = null; finalize(); } @@ -1055,6 +1085,9 @@ final class HTTPClientRequest : HTTPRequest { if (m_headerWritten && !m_bodyWriter) return; + assert(!m_multipartBoundary.length, + "Closed HTTPClientRequest without calling finalizeMultiPart but called beginMultiPart"); + // force the request to be sent if (!m_headerWritten) writeHeader(); else { diff --git a/inet/vibe/inet/webform.d b/inet/vibe/inet/webform.d index 10719758c3..997c1ce0ad 100644 --- a/inet/vibe/inet/webform.d +++ b/inet/vibe/inet/webform.d @@ -343,7 +343,8 @@ private bool parseMultipartFormPart(InputStream)(InputStream stream, ref FormFie Use $(LREF MultiPartBody) to manage a collection of parts. */ -final class MultiPart +struct MultiPartField(ContentInputStream) + if (isInputStream!ContentInputStream) { import vibe.stream.memory : createMemoryStream; @@ -352,7 +353,7 @@ final class MultiPart private { - InputStreamProxy m_content; + ContentInputStream m_content; size_t m_contentLength; } @@ -366,9 +367,18 @@ final class MultiPart exact_content_length = Set to the content length in bytes to allow calculation of the `Content-Length` header. Leave 0 to omit. */ - void setContent(InputStream)(InputStream stream, - size_t exact_content_length = 0) - if (isInputStream!InputStream) + void setContent(InputStream)(InputStream stream, size_t exact_content_length = 0) + if (is(InputStream == ContentInputStream)) + { + m_content = stream; + m_contentLength = exact_content_length; + } + + /// ditto + void setContent(InputStream)(InputStream stream, size_t exact_content_length = 0) + if (!is(InputStream == InputStreamProxy) + && is(ContentInputStream == InputStreamProxy) + && isInputStream!InputStream) { import vibe.internal.interfaceproxy : interfaceProxy; @@ -377,10 +387,10 @@ final class MultiPart } /** - Returns the content stream, wrapped as InputStreamProxy as previously - set from setContent or from the static constructing methods. + Returns the content stream, as previously set from setContent or from + the static constructing methods. */ - inout(InputStreamProxy) content() @property inout + const(ContentInputStream) content() @property const { return m_content; } @@ -399,12 +409,12 @@ final class MultiPart exact_content_length = Exact length of what the stream will evaluate to in bytes. Used for Content-Length calculation if given. */ - static MultiPart formData(InputStream)(string field_name, InputStream stream, + static typeof(this) formData(InputStream)(string field_name, InputStream stream, string content_type = "text/plain; charset=\"utf-8\"", bool binary = false, size_t exact_content_length = 0) if (isInputStream!InputStream) { - auto ret = new MultiPart(); + MultiPartField!ContentInputStream ret; ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\""; if (content_type.length) ret.headers["Content-Type"] = content_type; @@ -415,7 +425,7 @@ final class MultiPart } /// ditto - static MultiPart formData(string field_name, string value, + static typeof(this) formData(string field_name, string value, string content_type = "text/plain; charset=\"utf-8\"") { return formData(field_name, createMemoryStream(cast(ubyte[]) value.dup, false), @@ -423,7 +433,7 @@ final class MultiPart } /// ditto - static MultiPart formData(string field_name, ubyte[] content, + static typeof(this) formData(string field_name, ubyte[] content, string content_type = "application/octet-stream") { return formData(field_name, createMemoryStream(cast(ubyte[]) content, false), @@ -434,7 +444,7 @@ final class MultiPart Helper function directly reading from a file calling singleFile with the InputStream parameter and content length set. */ - static MultiPart singleFile(string field_name, NativePath file) + static typeof(this) singleFile(string field_name, NativePath file) { import vibe.inet.mimetypes : getMimeTypeForFile; import vibe.core.file : openFile, FileMode; @@ -460,12 +470,12 @@ final class MultiPart exact_content_length = Exact length of what the stream will evaluate to in bytes. Used for Content-Length calculation if given. */ - static MultiPart singleFile(InputStream)(string field_name, string filename, + static typeof(this) singleFile(InputStream)(string field_name, string filename, string content_type, InputStream stream, bool binary = true, size_t exact_content_length = 0) if (isInputStream!InputStream) { - auto ret = new MultiPart(); + MultiPartField!ContentInputStream ret; ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\""; if (content_type.length) ret.headers["Content-Type"] = content_type; @@ -476,7 +486,7 @@ final class MultiPart } /// ditto - static MultiPart singleFile(string field_name, string filename, + static typeof(this) singleFile(string field_name, string filename, string content_type, string content) { return singleFile(field_name, filename, content_type, @@ -485,7 +495,7 @@ final class MultiPart } /// ditto - static MultiPart singleFile(string field_name, string filename, + static typeof(this) singleFile(string field_name, string filename, string content_type, ubyte[] content) { return singleFile(field_name, filename, content_type, @@ -496,7 +506,7 @@ final class MultiPart Helper function directly reading from a file calling multipleFilesPart with the InputStream parameter and content length set. */ - static MultiPart multipleFilesPart(NativePath file) + static typeof(this) multipleFilesPart(NativePath file) { import vibe.inet.mimetypes : getMimeTypeForFile; import vibe.core.file : openFile, FileMode; @@ -523,12 +533,12 @@ final class MultiPart exact_content_length = Exact length of what the stream will evaluate to in bytes. Used for Content-Length calculation if given. */ - static MultiPart multipleFilesPart(InputStream)(string filename, + static typeof(this) multipleFilesPart(InputStream)(string filename, string content_type, InputStream stream, bool binary = false, size_t exact_content_length = 0) if (isInputStream!InputStream) { - auto ret = new MultiPart(); + MultiPartField!ContentInputStream ret; ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\""; if (content_type.length) ret.headers["Content-Type"] = content_type; @@ -539,7 +549,7 @@ final class MultiPart } /// ditto - static MultiPart multipleFilesPart(string filename, string content_type, + static typeof(this) multipleFilesPart(string filename, string content_type, string content) { return multipleFilesPart(filename, content_type, @@ -548,7 +558,7 @@ final class MultiPart } /// ditto - static MultiPart multipleFilesPart(string filename, string content_type, + static typeof(this) multipleFilesPart(string filename, string content_type, ubyte[] content) { return multipleFilesPart(filename, content_type, @@ -576,12 +586,12 @@ final class MultiPart content_type = The subtype for this mixed multipart to have. Common types include `multipart/mixed` or `multipart/alternative`. */ - static MultiPart multipleFiles(string name, MultiPartBody multipart, + static typeof(this) multipleFiles(string name, MultiPartBody multipart, string boundary, string content_type = "multipart/mixed") { import vibe.stream.memory : createMemoryOutputStream; - auto ret = new MultiPart(); + MultiPartField!ContentInputStream ret; ret.headers["Content-Disposition"] = "form-data; name=\"" ~ name ~ "\""; ret.headers["Content-Type"] = content_type ~ "; boundary=\"" ~ boundary ~ "\""; auto stream = createMemoryOutputStream(); @@ -630,11 +640,13 @@ final class MultiPart output.write("\r\n"); } output.write("\r\n"); - pipe(content, output); + pipe(m_content, output); output.write("\r\n"); } } +alias MultiPart = MultiPartField!InputStreamProxy; + /** Collection container for multiple MultiPart parts, a content type and an optional preamble/epilogue. @@ -644,7 +656,7 @@ final class MultiPart Standards: $(LINK https://tools.ietf.org/html/rfc1521#section-7.2) */ -final class MultiPartBody +struct MultiPartBody { /** The mime type of this multipart. For HTTP this is most usually diff --git a/tests/vibe.http.client.1005/source/app.d b/tests/vibe.http.client.1005/source/app.d index 7cd8845c12..2fb0fa3ea1 100644 --- a/tests/vibe.http.client.1005/source/app.d +++ b/tests/vibe.http.client.1005/source/app.d @@ -28,7 +28,7 @@ void main(string[] args) requestHTTP("http://127.0.0.1:11005", (scope req) { - MultiPartBody part = new MultiPartBody; + MultiPartBody part; part.parts ~= MultiPart.formData("name", "bob"); auto memStream = createMemoryStream(cast(ubyte[]) "Totally\0a\0PNG\0file.", false); part.parts ~= MultiPart.singleFile("picture", "picture.png", "image/png", memStream, true);