Skip to content

Commit

Permalink
Added possibility to send separate multiparts & reduce GC usage
Browse files Browse the repository at this point in the history
  • Loading branch information
WebFreak001 committed Aug 12, 2017
1 parent 513cb5a commit 69736b1
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 52 deletions.
59 changes: 54 additions & 5 deletions http/vibe/http/client.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -664,6 +664,7 @@ final class HTTPClientRequest : HTTPRequest {
bool m_headerWritten = false;
FixedAppender!(string, 22) m_contentLengthBuffer;
NetworkAddress m_localAddress;
bool m_writtenPart = false;
}


Expand Down Expand Up @@ -767,7 +768,10 @@ final class HTTPClientRequest : HTTPRequest {
}
}

void writePart(MultiPart part)
/**
Writes the body as multipart request that can upload files.
*/
void writePart(MultiPartBody part)
{
auto boundary = randomMultipartBoundary;
auto length = part.length(boundary);
Expand All @@ -780,13 +784,58 @@ final class HTTPClientRequest : HTTPRequest {
finalize();
}

/**
Partially writes the body with a multipart.
You need to supply a boundary which is a random string that should not occur in the content.
The boundary can be generated using `vibe.http.common.randomMultipartBoundary` and must always be the same in one request.
*/
void writePart(MultiPart part, string boundary)
{
if (boundary.length > 70) {
logTrace("Boundary '%s' is longer than 70 characters, truncating", boundary);
boundary.length = 70;
}

if (m_writtenPart)
bodyWriter.write(boundary);

if ("Content-Type" !in headers)
headers["Content-Type"] = "multipart/form-data; boundary=\"" ~ boundary ~ "\"";

boundary = "--" ~ boundary;
bodyWriter.write(boundary);
bodyWriter.write("\r\n");
foreach (k, v; part.headers)
bodyWriter.write(k ~ ": " ~ v ~ "\r\n");
bodyWriter.write("\r\n");
pipe(part.content, bodyWriter);
bodyWriter.write("\r\n");
m_writtenPart = true;
}

/**
Finishes writing a multipart response by sending the ending boundary and finalizing the request.
*/
void finalizePart(string boundary, string epilogue = null)
{
bodyWriter.write("--");
bodyWriter.write(boundary);
bodyWriter.write("--\r\n");
if (epilogue.length)
{
bodyWriter.write(epilogue);
bodyWriter.write("\r\n");
}
finalize();
}

///
unittest {
void test(HTTPClientRequest req) {
MultiPart part = new MultiPart;
part.parts ~= MultiPartBodyPart.formData("name", "bob");
part.parts ~= MultiPartBodyPart.singleFile("picture", "picture.png", "image/png", openFile("res/profilepicture.png"));
part.parts ~= MultiPartBodyPart.singleFile("upload", Path("file.zip")); // auto read & mime detection from filename
MultiPartBody part = new MultiPartBody;
part.parts ~= MultiPart.formData("name", "bob");
part.parts ~= MultiPart.singleFile("picture", "picture.png", "image/png", openFile("res/profilepicture.png"));
part.parts ~= MultiPart.singleFile("upload", Path("file.zip")); // auto read & mime detection from filename
req.writePart(part);
}
}
Expand Down
93 changes: 49 additions & 44 deletions http/vibe/http/common.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,18 +337,19 @@ class HTTPStatusException : Exception {
string debugMessage;
}

final class MultiPartBodyPart
final class MultiPart
{
import vibe.stream.memory : createMemoryStream;

InetHeaderMap headers;
InputStream content;
InterfaceProxy!InputStream content;

@safe:

static MultiPartBodyPart formData(string field_name, InputStream stream, string content_type = "text/plain; charset=\"utf-8\"", bool binary = false)
static MultiPart formData(InputStream)(string field_name, InputStream stream, string content_type = "text/plain; charset=\"utf-8\"", bool binary = false)
if (isInputStream!InputStream)
{
auto ret = new MultiPartBodyPart();
auto ret = new MultiPart();
ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"";
ret.headers["Content-Type"] = content_type;
if (binary)
Expand All @@ -357,42 +358,43 @@ final class MultiPartBodyPart
return ret;
}

static MultiPartBodyPart formData(string field_name, string value, string content_type = "text/plain; charset=\"utf-8\"")
static MultiPart formData(string field_name, string value, string content_type = "text/plain; charset=\"utf-8\"")
{
auto ret = new MultiPartBodyPart();
auto ret = new MultiPart();
ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"";
ret.headers["Content-Type"] = content_type;
ret.content = createMemoryStream((() @trusted => cast(ubyte[]) value)(), false);
return ret;
}

static MultiPartBodyPart formData(string field_name, ubyte[] content, string content_type = "application/octet-stream")
static MultiPart formData(string field_name, ubyte[] content, string content_type = "application/octet-stream")
{
auto ret = new MultiPartBodyPart();
auto ret = new MultiPart();
ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"";
ret.headers["Content-Transfer-Encoding"] = "binary";
ret.content = createMemoryStream(content, false);
return ret;
}

static MultiPartBodyPart singleFile(string field_name, Path file)
static MultiPart singleFile(string field_name, Path file)
{
import vibe.inet.mimetypes : getMimeTypeForFile;
import vibe.core.file : openFile, FileMode;

auto ret = new MultiPartBodyPart();
auto ret = new MultiPart();
ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ file.head.name ~ "\"";
string type = getMimeTypeForFile(file.toString);
ret.headers["Content-Type"] = type;
if (!type.startsWith("text/"))
ret.headers["Content-Transfer-Encoding"] = "binary";
ret.content = cast(InputStream) openFile(file, FileMode.read);
ret.content = cast(InterfaceProxy!InputStream) openFile(file, FileMode.read);
return ret;
}

static MultiPartBodyPart singleFile(string field_name, string filename, string content_type, InputStream stream, bool binary = true)
static MultiPart singleFile(InputStream)(string field_name, string filename, string content_type, InputStream stream, bool binary = true)
if (isInputStream!InputStream)
{
auto ret = new MultiPartBodyPart();
auto ret = new MultiPart();
ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\"";
ret.headers["Content-Type"] = content_type;
if (binary)
Expand All @@ -401,73 +403,67 @@ final class MultiPartBodyPart
return ret;
}

static MultiPartBodyPart singleFile(string field_name, string filename, string content_type, string content)
static MultiPart singleFile(string field_name, string filename, string content_type, string content)
{
auto ret = new MultiPartBodyPart();
auto ret = new MultiPart();
ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\"";
ret.headers["Content-Type"] = content_type;
ret.content = createMemoryStream((() @trusted => cast(ubyte[]) content)(), false);
return ret;
}

static MultiPartBodyPart singleFile(string field_name, string filename, string content_type, ubyte[] content)
static MultiPart singleFile(string field_name, string filename, string content_type, ubyte[] content)
{
auto ret = new MultiPartBodyPart();
auto ret = new MultiPart();
ret.headers["Content-Disposition"] = "form-data; name=\"" ~ field_name ~ "\"; filename=\"" ~ filename ~ "\"";
ret.headers["Content-Transfer-Encoding"] = "binary";
ret.headers["Content-Type"] = content_type;
ret.content = createMemoryStream(content, false);
return ret;
}

static MultiPartBodyPart multipleFilesPart(Path file)
static MultiPart multipleFilesPart(Path file)
{
import vibe.inet.mimetypes : getMimeTypeForFile;
import vibe.core.file : openFile, FileMode;

auto ret = new MultiPartBodyPart();
auto ret = new MultiPart();
ret.headers["Content-Disposition"] = "file; filename=\"" ~ file.head.name ~ "\"";
string type = getMimeTypeForFile(file.toString);
ret.headers["Content-Type"] = type;
if (!type.startsWith("text/"))
ret.headers["Content-Transfer-Encoding"] = "binary";
ret.content = cast(InputStream) openFile(file, FileMode.read);
ret.content = cast(InterfaceProxy!InputStream) openFile(file, FileMode.read);
return ret;
}

static MultiPartBodyPart multipleFilesPart(string filename, string content_type, InputStream stream)
static MultiPart multipleFilesPart(InputStream)(string filename, string content_type, InputStream stream, bool binary = false)
if (isInputStream!InputStream)
{
auto ret = new MultiPartBodyPart();
auto ret = new MultiPart();
ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\"";
ret.headers["Content-Type"] = content_type;
if (binary)
ret.headers["Content-Transfer-Encoding"] = "binary";
ret.content = stream;
return ret;
}

static MultiPartBodyPart multipleFilesPart(string filename, string content_type, string content)
static MultiPart multipleFilesPart(string filename, string content_type, string content)
{
auto ret = new MultiPartBodyPart();
ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\"";
ret.headers["Content-Type"] = content_type;
ret.content = createMemoryStream((() @trusted => cast(ubyte[]) content)(), false);
return ret;
return multipleFilesPart(filename, content_type, createMemoryStream((() @trusted => cast(ubyte[]) content)(), false), true);
}

static MultiPartBodyPart multipleFilesPart(string filename, string content_type, ubyte[] content)
static MultiPart multipleFilesPart(string filename, string content_type, ubyte[] content)
{
auto ret = new MultiPartBodyPart();
ret.headers["Content-Disposition"] = "file; filename=\"" ~ filename ~ "\"";
ret.headers["Content-Transfer-Encoding"] = "binary";
ret.headers["Content-Type"] = content_type;
ret.content = createMemoryStream(content, false);
return ret;
return multipleFilesPart(filename, content_type, createMemoryStream(content, false), true);
}

static MultiPartBodyPart multipleFiles(string name, MultiPart multipart)
static MultiPart multipleFiles(string name, MultiPartBody multipart)
{
import vibe.stream.memory : createMemoryOutputStream;

auto ret = new MultiPartBodyPart();
auto ret = new MultiPart();
string boundary = randomMultipartBoundary;
ret.headers["Content-Disposition"] = "form-data; name=\"" ~ name ~ "\"";
ret.headers["Content-Type"] = "multipart/mixed; boundary=\"" ~ boundary ~ "\"";
Expand All @@ -478,14 +474,14 @@ final class MultiPartBodyPart
}
}

final class MultiPart
final class MultiPartBody
{
// https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html

string contentType = "multipart/form-data";
string preamble, epilogue;

MultiPartBodyPart[] parts;
MultiPart[] parts;

@safe:

Expand Down Expand Up @@ -523,21 +519,30 @@ final class MultiPart
return;

boundary = "--" ~ boundary;
if (preamble.length)
output.write(preamble ~ "\r\n");
if (preamble.length) {
output.write(preamble);
output.write("\r\n");
}
output.write(boundary);
foreach (part; parts) {
output.write("\r\n");
foreach (k, v; part.headers)
output.write(k ~ ": " ~ v ~ "\r\n");
foreach (k, v; part.headers) {
output.write(k);
output.write(": ");
output.write(v);
output.write("\r\n");
}
output.write("\r\n");
pipe(part.content, output);
output.write("\r\n");
output.write(boundary);
}
output.write("--\r\n");
if (epilogue.length)
output.write(epilogue ~ "\r\n");
{
output.write(epilogue);
output.write("\r\n");
}
}
}

Expand Down
6 changes: 3 additions & 3 deletions tests/vibe.http.client.1005/source/app.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ shared static this()
runTask({
requestHTTP("http://127.0.0.1:11005",
(scope req) {
MultiPart part = new MultiPart;
part.parts ~= MultiPartBodyPart.formData("name", "bob");
MultiPartBody part = new MultiPartBody;
part.parts ~= MultiPart.formData("name", "bob");
auto memStream = createMemoryStream(cast(ubyte[]) "Totally\0a\0PNG\0file.", false);
part.parts ~= MultiPartBodyPart.singleFile("picture", "picture.png", "image/png", memStream, true);
part.parts ~= MultiPart.singleFile("picture", "picture.png", "image/png", memStream, true);
req.writePart(part);
},
(scope res) {}
Expand Down

0 comments on commit 69736b1

Please sign in to comment.