Ben Copsey

Added the ability to stream the postBody from local disk, drastically cutting me…

…mory use for large posts
... ... @@ -44,10 +44,7 @@
if (!fileData) {
fileData = [[NSMutableDictionary alloc] init];
}
NSMutableDictionary *file = [[[NSMutableDictionary alloc] init] autorelease];
[file setObject:[NSData dataWithContentsOfFile:filePath options:NSUncachedRead error:NULL] forKey:@"data"];
[file setObject:[filePath lastPathComponent] forKey:@"filename"];
[fileData setValue:file forKey:key];
[fileData setValue:filePath forKey:key];
[self setRequestMethod:@"POST"];
}
... ... @@ -69,15 +66,17 @@
[super buildPostBody];
return;
}
NSMutableData *body = [[[NSMutableData alloc] init] autorelease];
if ([fileData count] > 0) {
[self setShouldStreamPostDataFromDisk:YES];
}
// Set your own boundary string only if really obsessive. We don't bother to check if post data contains the boundary, since it's pretty unlikely that it does.
NSString *stringBoundary = @"0xKhTmLbOuNdArY";
[self addRequestHeader:@"Content-Type" value:[NSString stringWithFormat:@"multipart/form-data; boundary=%@",stringBoundary]];
[body appendData:[[NSString stringWithFormat:@"--%@\r\n",stringBoundary] dataUsingEncoding:NSUTF8StringEncoding]];
[self appendPostData:[[NSString stringWithFormat:@"--%@\r\n",stringBoundary] dataUsingEncoding:NSUTF8StringEncoding]];
// Adds post data
NSData *endItemBoundary = [[NSString stringWithFormat:@"\r\n--%@\r\n",stringBoundary] dataUsingEncoding:NSUTF8StringEncoding];
... ... @@ -85,11 +84,11 @@
NSString *key;
int i=0;
while (key = [e nextObject]) {
[body appendData:[[NSString stringWithFormat:@"Content-Disposition: form-data; name=\"%@\"\r\n\r\n",key] dataUsingEncoding:NSUTF8StringEncoding]];
[body appendData:[[postData objectForKey:key] dataUsingEncoding:NSUTF8StringEncoding]];
[self appendPostData:[[NSString stringWithFormat:@"Content-Disposition: form-data; name=\"%@\"\r\n\r\n",key] dataUsingEncoding:NSUTF8StringEncoding]];
[self appendPostData:[[postData objectForKey:key] dataUsingEncoding:NSUTF8StringEncoding]];
i++;
if (i != [postData count] || [fileData count] > 0) { //Only add the boundary if this is not the last item in the post body
[body appendData:endItemBoundary];
[self appendPostData:endItemBoundary];
}
}
... ... @@ -98,22 +97,19 @@
e = [fileData keyEnumerator];
i=0;
while (key = [e nextObject]) {
NSDictionary *fileInfo = [fileData objectForKey:key];
[body appendData:[[NSString stringWithFormat:@"Content-Disposition: form-data; name=\"%@\"; filename=\"%@\"\r\n",key,[fileInfo objectForKey:@"filename"]] dataUsingEncoding:NSUTF8StringEncoding]];
[body appendData:contentTypeHeader];
[body appendData: [fileInfo objectForKey:@"data"]];
NSString *filePath = [fileData objectForKey:key];
[self appendPostData:[[NSString stringWithFormat:@"Content-Disposition: form-data; name=\"%@\"; filename=\"%@\"\r\n",key,[filePath lastPathComponent]] dataUsingEncoding:NSUTF8StringEncoding]];
[self appendPostData:contentTypeHeader];
[self appendPostDataFromFile:filePath];
i++;
// Only add the boundary if this is not the last item in the post body
if (i != [fileData count]) {
[body appendData:endItemBoundary];
[self appendPostData:endItemBoundary];
}
}
[body appendData:[[NSString stringWithFormat:@"\r\n--%@--\r\n",stringBoundary] dataUsingEncoding:NSUTF8StringEncoding]];
[self setPostBody:body];
[self appendPostData:[[NSString stringWithFormat:@"\r\n--%@--\r\n",stringBoundary] dataUsingEncoding:NSUTF8StringEncoding]];
[super buildPostBody];
}
... ...
... ... @@ -41,7 +41,7 @@ typedef enum _ASINetworkErrorType {
NSString *requestMethod;
// Request body
NSData *postBody;
NSMutableData *postBody;
// Dictionary for custom HTTP request headers
NSMutableDictionary *requestHeaders;
... ... @@ -191,6 +191,11 @@ typedef enum _ASINetworkErrorType {
// Custom user information assosiated with the request
NSDictionary *userInfo;
NSString *postBodyFilePath;
NSOutputStream *postBodyWriteStream;
NSInputStream *postBodyReadStream;
BOOL shouldStreamPostDataFromDisk;
}
#pragma mark init / dealloc
... ... @@ -205,6 +210,9 @@ typedef enum _ASINetworkErrorType {
- (void)buildPostBody;
- (void)appendPostData:(NSData *)data;
- (void)appendPostDataFromFile:(NSString *)file;
#pragma mark get information about this request
// Returns the contents of the result as an NSString (not appropriate for binary data - used responseData instead)
... ... @@ -344,7 +352,7 @@ typedef enum _ASINetworkErrorType {
@property (retain) NSDate *lastActivityTime;
@property (assign) NSTimeInterval timeOutSeconds;
@property (retain) NSString *requestMethod;
@property (retain,setter=setPostBody:) NSData *postBody;
@property (retain) NSMutableData *postBody;
@property (assign) unsigned long long contentLength;
@property (assign) unsigned long long partialDownloadSize;
@property (assign) unsigned long long postLength;
... ... @@ -359,4 +367,8 @@ typedef enum _ASINetworkErrorType {
@property (assign) BOOL allowCompressedResponse;
@property (assign) BOOL allowResumeForFileDownloads;
@property (retain) NSDictionary *userInfo;
@property (retain) NSString *postBodyFilePath;
@property (retain) NSOutputStream *postBodyWriteStream;
@property (retain) NSInputStream *postBodyReadStream;
@property (assign) BOOL shouldStreamPostDataFromDisk;
@end
... ...
... ... @@ -144,25 +144,78 @@ static NSError *ASIUnableToCreateRequestError;
[requestHeaders setObject:value forKey:header];
}
-(void)setPostBody:(NSData *)body
{
[postBody release];
postBody = [body retain];
postLength = [postBody length];
[self addRequestHeader:@"Content-Length" value:[NSString stringWithFormat:@"%llu",postLength]];
if (postBody && postLength > 0 && ![requestMethod isEqualToString:@"POST"] && ![requestMethod isEqualToString:@"PUT"]) {
[self setRequestMethod:@"POST"];
}
}
// Subclasses should override this method if they need to create POST content for this request
// This function will be called either just before a request starts, or when postLength is needed, whichever comes first
// postLength must be set by the time this function is complete - calling setPostBody: will do this for you
- (void)buildPostBody
{
if ([self postBodyFilePath] && [self postBodyWriteStream]) {
[[self postBodyWriteStream] close];
[self setPostBodyWriteStream:nil];
[self setPostLength:[[[NSFileManager defaultManager] fileAttributesAtPath:[self postBodyFilePath] traverseLink:NO] fileSize]];
[self addRequestHeader:@"Content-Length" value:[NSString stringWithFormat:@"%llu",postLength]];
if (postBody && postLength > 0 && ![requestMethod isEqualToString:@"POST"] && ![requestMethod isEqualToString:@"PUT"]) {
[self setRequestMethod:@"POST"];
}
} else {
[self setPostLength:[postBody length]];
[self addRequestHeader:@"Content-Length" value:[NSString stringWithFormat:@"%llu",postLength]];
if (postBody && postLength > 0 && ![requestMethod isEqualToString:@"POST"] && ![requestMethod isEqualToString:@"PUT"]) {
[self setRequestMethod:@"POST"];
}
}
haveBuiltPostBody = YES;
}
- (void)setupPostBody
{
if ([self shouldStreamPostDataFromDisk]) {
if (![self postBodyFilePath]) {
[self setPostBodyFilePath:[NSTemporaryDirectory() stringByAppendingPathComponent:[[NSProcessInfo processInfo] globallyUniqueString]]];
}
if (![self postBodyWriteStream]) {
[self setPostBodyWriteStream:[[[NSOutputStream alloc] initToFileAtPath:[self postBodyFilePath] append:NO] autorelease]];
[[self postBodyWriteStream] open];
}
} else {
if (![self postBody]) {
[self setPostBody:[[[NSMutableData alloc] init] autorelease]];
}
}
}
- (void)appendPostData:(NSData *)data
{
[self setupPostBody];
if ([self shouldStreamPostDataFromDisk]) {
[[self postBodyWriteStream] write:[data bytes] maxLength:[data length]];
} else {
[[self postBody] appendData:data];
}
}
- (void)appendPostDataFromFile:(NSString *)file
{
[self setupPostBody];
NSInputStream *stream = [[[NSInputStream alloc] initWithFileAtPath:file] autorelease];
[stream open];
NSMutableData *d;
while ([stream hasBytesAvailable]) {
d = [[NSMutableData alloc] initWithLength:256*1024];
int bytesRead = [stream read:[d mutableBytes] maxLength:256*1024];
if ([self shouldStreamPostDataFromDisk]) {
[[self postBodyWriteStream] write:[d mutableBytes] maxLength:bytesRead];
} else {
NSLog(@"foo");
[[self postBody] appendData:[NSData dataWithBytes:[d mutableBytes] length:bytesRead]];
}
[d release];
}
[stream close];
}
#pragma mark get information about this request
- (BOOL)isFinished
... ... @@ -300,7 +353,7 @@ static NSError *ASIUnableToCreateRequestError;
}
// If this is a post request and we have data to send, add it to the request
// If this is a post request and we have data in memory send, add it to the request
if ([self postBody]) {
CFHTTPMessageSetBody(request, (CFDataRef)postBody);
}
... ... @@ -339,8 +392,14 @@ static NSError *ASIUnableToCreateRequestError;
[self setRawResponseData:[[[NSMutableData alloc] init] autorelease]];
}
// Create the stream for the request.
readStream = CFReadStreamCreateForStreamedHTTPRequest(kCFAllocatorDefault, request,readStream);
if (!readStream) {
if ([self shouldStreamPostDataFromDisk] && [self postBodyFilePath] && [[NSFileManager defaultManager] fileExistsAtPath:[self postBodyFilePath]]) {
[self setPostBodyReadStream:[[[NSInputStream alloc] initWithFileAtPath:[self postBodyFilePath]] autorelease]];
[[self postBodyReadStream] open];
readStream = CFReadStreamCreateForStreamedHTTPRequest(kCFAllocatorDefault, request,(CFReadStreamRef)[self postBodyReadStream]);
} else {
readStream = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, request);
}
if (!readStream) {
[cancelledLock unlock];
[self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:ASIInternalErrorWhileBuildingRequestType userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Unable to create read stream",NSLocalizedDescriptionKey,nil]]];
return;
... ... @@ -1484,4 +1543,8 @@ static NSError *ASIUnableToCreateRequestError;
@synthesize allowCompressedResponse;
@synthesize allowResumeForFileDownloads;
@synthesize userInfo;
@synthesize postBodyFilePath;
@synthesize postBodyWriteStream;
@synthesize postBodyReadStream;
@synthesize shouldStreamPostDataFromDisk;
@end
... ...
... ... @@ -196,10 +196,19 @@
- (IBAction)postWithProgress:(id)sender
{
//Create a 1mb file
NSMutableData *data = [NSMutableData dataWithLength:1024*1024];
//Create a 10MB file
NSMutableData *data = [NSMutableData dataWithLength:1024];
NSString *path = [[[[NSBundle mainBundle] bundlePath] stringByDeletingLastPathComponent] stringByAppendingPathComponent:@"bigfile"];
[data writeToFile:path atomically:NO];
NSOutputStream *stream = [[[NSOutputStream alloc] initToFileAtPath:path append:NO] autorelease];
[stream open];
int i;
for (i=0; i<1024*10; i++) {
[stream write:[data mutableBytes] maxLength:[data length]];
}
[stream close];
[networkQueue cancelAllOperations];
[networkQueue setShowAccurateProgress:YES];
... ...