Start work on storing external resources in files on disk

Ben Copsey
Commit 0950643c6974ad717aef4b1f7a9ab1898f0c4825 0950643c 1 parent dcfd15c9
Showing 3 changed files with 103 additions and 39 deletions
Classes/ASIWebPageRequest.h
Classes/ASIWebPageRequest.m
Mac Sample/AppDelegate.m
--- a/Classes/ASIWebPageRequest.h
View file @0950643
+++ b/Classes/ASIWebPageRequest.h
View file @0950643
@@ -21,6 +21,7 @@ typedef enum _ASIWebContentType {
     ASICSSWebContentType = 2
 } ASIWebContentType;
+
 @interface ASIWebPageRequest : ASIHTTPRequest {
 	ASINetworkQueue *externalResourceQueue;
 	NSMutableDictionary *resourceList;
@@ -31,6 +32,8 @@ typedef enum _ASIWebContentType {
 	ASIWebPageRequest *parentRequest;
 }
+- (NSString *)contentForExternalURL:(NSString *)theURL;
+- (NSString *)cachePathForRequest:(ASIWebPageRequest *)theRequest;
 @property (assign, nonatomic) ASIWebPageRequest *parentRequest;
 @end
--- a/Classes/ASIWebPageRequest.m
View file @0950643
+++ b/Classes/ASIWebPageRequest.m
View file @0950643
@@ -66,8 +66,18 @@ static NSMutableArray *requestsUsingXMLParser = nil;
 - (void)parseAsCSS
 {
 	webContentType = ASICSSWebContentType;
-	NSString *responseCSS = [self responseString];
+
-	if (!responseCSS) {
+	NSString *responseCSS = nil;
+	NSError *err = nil;
+	if ([self downloadDestinationPath]) {
+		responseCSS = [NSString stringWithContentsOfFile:[self downloadDestinationPath] encoding:[self responseEncoding] error:&err];
+	} else {
+		responseCSS = [self responseString];
+	}
+	if (err) {
+		[self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:100 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Unable to read HTML string from response",NSLocalizedDescriptionKey,err,NSUnderlyingErrorKey,nil]]];
+		return;
+	} else if (!responseCSS) {
 		[self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:100 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Unable to read HTML string from response",NSLocalizedDescriptionKey,nil]]];
 		return;
 	}
@@ -99,6 +109,9 @@ static NSMutableArray *requestsUsingXMLParser = nil;
 		[externalResourceRequest setUserInfo:[NSDictionary dictionaryWithObject:theURL forKey:@"Path"]];
 		[externalResourceRequest setParentRequest:self];
 		[externalResourceRequest setShouldResetDownloadProgress:NO];
+		if ([self downloadDestinationPath]) {
+			[externalResourceRequest setDownloadDestinationPath:[self cachePathForRequest:externalResourceRequest]];
+		}
 		[[self externalResourceQueue] addOperation:externalResourceRequest];
 		[externalResourceRequest setShowAccurateProgress:YES];
 	}
@@ -111,20 +124,6 @@ static NSMutableArray *requestsUsingXMLParser = nil;
 - (void)parseAsHTML
 {
 	webContentType = ASIHTMLWebContentType;
-	NSString *responseHTML = [self responseString];
-	if (!responseHTML) {
-		[self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:100 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Unable to read HTML string from response",NSLocalizedDescriptionKey,nil]]];
-		return;
-	}
-
-	NSError *err = nil;
-	if (err) {
-		[self failWithError:err];
-		return;
-	} else if (!responseHTML) {
-		[self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:101 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Unable to convert response string to XHTML",NSLocalizedDescriptionKey,nil]]];
-		return;	
-	}
 	// Only allow parsing of a single document at a time
 	[xmlParsingLock lock];
@@ -134,10 +133,14 @@ static NSMutableArray *requestsUsingXMLParser = nil;
 	}
 	[requestsUsingXMLParser addObject:self];
-	NSData *data = [responseHTML dataUsingEncoding:[self responseEncoding]];
     /* Load XML document */
+	if ([self downloadDestinationPath]) {
+		doc = htmlReadFile([[self downloadDestinationPath] cStringUsingEncoding:NSUTF8StringEncoding], NULL, HTML_PARSE_NONET | HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR);
+	} else {
+		NSData *data = [self responseData];
 		doc = htmlReadMemory([data bytes], (int)[data length], "", NULL, HTML_PARSE_NONET | HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR);
+	}
     if (doc == NULL) {
 		xmlFreeDoc(doc);
 		[self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:101 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Error: unable to parse reponse XML",NSLocalizedDescriptionKey,nil]]];
@@ -172,6 +175,9 @@ static NSMutableArray *requestsUsingXMLParser = nil;
 		[externalResourceRequest setUserInfo:[NSDictionary dictionaryWithObject:theURL forKey:@"Path"]];
 		[externalResourceRequest setParentRequest:self];
 		[externalResourceRequest setShouldResetDownloadProgress:NO];
+		if ([self downloadDestinationPath]) {
+			[externalResourceRequest setDownloadDestinationPath:[self cachePathForRequest:externalResourceRequest]];
+		}
 		[[self externalResourceQueue] addOperation:externalResourceRequest];
 		[externalResourceRequest setShowAccurateProgress:YES];
 		[self incrementDownloadSizeBy:1];
@@ -214,7 +220,11 @@ static NSMutableArray *requestsUsingXMLParser = nil;
 		contentType = @"application/octet-stream";
 	}
 	[requestResponse setObject:contentType forKey:@"ContentType"];
+	if ([self downloadDestinationPath]) {
+		[requestResponse setObject:[externalResourceRequest downloadDestinationPath] forKey:@"DataPath"];
+	} else {
 		[requestResponse setObject:[externalResourceRequest responseData] forKey:@"Data"];
+	}
 }
 - (void)externalResourceFetchFailed:(ASIHTTPRequest *)externalResourceRequest
@@ -225,35 +235,58 @@ static NSMutableArray *requestsUsingXMLParser = nil;
 - (void)finishedFetchingExternalResources:(ASINetworkQueue *)queue
 {
 	if (webContentType == ASICSSWebContentType) {
-		NSMutableString *parsedResponse = [[[self responseString] mutableCopy] autorelease];
+		NSMutableString *parsedResponse;
+		NSError *err = nil;
+		if ([self downloadDestinationPath]) {
+			parsedResponse = [NSMutableString stringWithContentsOfFile:[self downloadDestinationPath] encoding:[self responseEncoding] error:&err];
+		} else {
+			parsedResponse = [[[self responseString] mutableCopy] autorelease];
+		}
+		if (err) {
+			[self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:101 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Error: unable to read response CSS from disk",NSLocalizedDescriptionKey,nil]]];
+			return;
+		}
 		if (![self error]) {
 			for (NSString *resource in [[self resourceList] keyEnumerator]) {
-				NSDictionary *resourceInfo = [[self resourceList] objectForKey:resource];
+				if ([parsedResponse rangeOfString:resource].location != NSNotFound) {
-				NSData *data = [resourceInfo objectForKey:@"Data"];
+					NSString *newURL = [self contentForExternalURL:resource];
-				NSString *contentType = [resourceInfo objectForKey:@"ContentType"];
+					if (newURL) {
-				if (data && contentType) {
+						[parsedResponse replaceOccurrencesOfString:resource withString:newURL options:0 range:NSMakeRange(0, [parsedResponse length])];
-					if (data && contentType) {
-						NSString *newData = [NSString stringWithFormat:@"data:%@;base64,",contentType];
-						newData = [newData stringByAppendingString:[ASIHTTPRequest base64forData:data]];
-						[parsedResponse replaceOccurrencesOfString:resource withString:newData options:0 range:NSMakeRange(0, [parsedResponse length])];
 					}
 				}
 			}
 		}
+		if ([self downloadDestinationPath]) {
+			[parsedResponse writeToFile:[self downloadDestinationPath] atomically:NO encoding:[self responseEncoding] error:&err];
+			if (err) {
+				[self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:101 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Error: unable to write response CSS to disk",NSLocalizedDescriptionKey,nil]]];
+				return;
+			}
+		} else {
 			[self setRawResponseData:(id)[parsedResponse dataUsingEncoding:[self responseEncoding]]];
-
+		}
 	} else {
 		[xmlParsingLock lock];
 		[self updateResourceURLs];
 		xmlChar *bytes = nil;
 		int size = 0;
+		FILE *file = NULL;
+		if ([self downloadDestinationPath]) {
+			file = fdopen([[NSFileHandle fileHandleForWritingAtPath:[self downloadDestinationPath]] fileDescriptor], "w");
+			xmlDocDump(file, doc);
+		} else {
 			xmlDocDumpMemory(doc,&bytes,&size);
 			[self setRawResponseData:[[[NSMutableData alloc] initWithBytes:bytes length:size] autorelease]];
+		}
 		xmlFreeDoc(doc);
 		doc = nil;
+		if (file) {
+			fclose(file);
+		}
+
 		[requestsUsingXMLParser removeObject:self];
 		if (![requestsUsingXMLParser count]) {
 			xmlCleanupParser();
@@ -374,22 +407,18 @@ static NSMutableArray *requestsUsingXMLParser = nil;
 		if ([[nodeName lowercaseString] isEqualToString:@"style"]) {
 			NSArray *externalResources = [[self class] CSSURLsFromString:value];
 			for (NSString *theURL in externalResources) {
-				NSData *data = [[resourceList objectForKey:theURL] objectForKey:@"Data"];
+				if ([value rangeOfString:theURL].location != NSNotFound) {
-				NSString *contentType = [[resourceList objectForKey:theURL] objectForKey:@"ContentType"];
+					NSString *newURL = [self contentForExternalURL:theURL];
-				if (data && contentType) {
+					if (newURL) {
-					NSString *newData = [NSString stringWithFormat:@"data:%@;base64,",contentType];
+						value = [value stringByReplacingOccurrencesOfString:theURL withString:newURL];
-					newData = [newData stringByAppendingString:[ASIHTTPRequest base64forData:data]];
+					}
-					value = [value stringByReplacingOccurrencesOfString:theURL withString:newData];
 				}
 			}
 			xmlNodeSetContent(nodes->nodeTab[i], (xmlChar *)[value cStringUsingEncoding:[self responseEncoding]]);
 		} else {
-			NSData *data = [[resourceList objectForKey:value] objectForKey:@"Data"];
+			NSString *newURL = [self contentForExternalURL:value];
-			NSString *contentType = [[resourceList objectForKey:value] objectForKey:@"ContentType"];
+			if (newURL) {
-			if (data && contentType) {
+				xmlNodeSetContent(nodes->nodeTab[i], (xmlChar *)[newURL cStringUsingEncoding:[self responseEncoding]]);
-				NSString *newData = [NSString stringWithFormat:@"data:%@;base64,",contentType];
-				newData = [newData stringByAppendingString:[ASIHTTPRequest base64forData:data]];
-				xmlNodeSetContent(nodes->nodeTab[i], (xmlChar *)[newData cStringUsingEncoding:[self responseEncoding]]);
 			}
 		}
@@ -423,6 +452,30 @@ static NSMutableArray *requestsUsingXMLParser = nil;
 	return urls;
 }
+- (NSString *)contentForExternalURL:(NSString *)theURL
+{
+	NSData *data;
+	if ([[resourceList objectForKey:theURL] objectForKey:@"DataPath"]) {
+		data = [NSData dataWithContentsOfFile:[[resourceList objectForKey:theURL] objectForKey:@"DataPath"]];
+	} else {
+		data = [[resourceList objectForKey:theURL] objectForKey:@"Data"];
+	}
+	NSString *contentType = [[resourceList objectForKey:theURL] objectForKey:@"ContentType"];
+	if (data && contentType) {
+		NSString *dataURI = [NSString stringWithFormat:@"data:%@;base64,",contentType];
+		dataURI = [dataURI stringByAppendingString:[ASIHTTPRequest base64forData:data]];
+		return dataURI;
+	}
+	return nil;
+}
+
+static int resourceNum = 0;
+- (NSString *)cachePathForRequest:(ASIWebPageRequest *)theRequest
+{
+	resourceNum++;
+	return [NSString stringWithFormat:@"/Users/ben/Desktop/%i",resourceNum];
+}
+
 @synthesize externalResourceQueue;
 @synthesize resourceList;
 @synthesize parentRequest;
--- a/Mac Sample/AppDelegate.m
View file @0950643
+++ b/Mac Sample/AppDelegate.m
View file @0950643
@@ -406,6 +406,7 @@
 	[request setShowAccurateProgress:NO];
 	[request setDownloadProgressDelegate:progressIndicator];
 	[request setDownloadCache:[ASIDownloadCache sharedCache]];
+	[request setDownloadDestinationPath:@"/Users/ben/Desktop/fink"];
 	[[ASIDownloadCache sharedCache] setDefaultCachePolicy:ASIOnlyLoadIfNotCachedCachePolicy];
 	[[ASIDownloadCache sharedCache] setShouldRespectCacheControlHeaders:NO];
 	[request startAsynchronous];
@@ -418,8 +419,15 @@
 - (void)webPageFetchSucceeded:(ASIHTTPRequest *)request
 {
+	if ([request downloadDestinationPath]) {
+		NSString *response = [NSString stringWithContentsOfFile:[request downloadDestinationPath] encoding:[request responseEncoding] error:nil];
+		[webPageSource setString:response];
+		[[webView mainFrame] loadHTMLString:response baseURL:[request url]];	
+	} else {
 		[webPageSource setString:[request responseString]];
 		[[webView mainFrame] loadHTMLString:[request responseString] baseURL:[request url]];
+	}
+
 	[urlField setStringValue:[[request url] absoluteString]];
 }