iOS / asi-http-request

Go to a project
Toggle navigation Toggle navigation pinning
Ben Copsey
0950643c 1 parent dcfd15c9

Start work on storing external resources in files on disk

Inline Side-by-side
Showing 3 changed files with 110 additions and 46 deletions
@@ -21,6 +21,7 @@ typedef enum _ASIWebContentType { @@ -21,6 +21,7 @@ typedef enum _ASIWebContentType {
21 ASICSSWebContentType = 2 21 ASICSSWebContentType = 2
22 } ASIWebContentType; 22 } ASIWebContentType;
23 23
  24 +
24 @interface ASIWebPageRequest : ASIHTTPRequest { 25 @interface ASIWebPageRequest : ASIHTTPRequest {
25 ASINetworkQueue *externalResourceQueue; 26 ASINetworkQueue *externalResourceQueue;
26 NSMutableDictionary *resourceList; 27 NSMutableDictionary *resourceList;
@@ -31,6 +32,8 @@ typedef enum _ASIWebContentType { @@ -31,6 +32,8 @@ typedef enum _ASIWebContentType {
31 ASIWebPageRequest *parentRequest; 32 ASIWebPageRequest *parentRequest;
32 } 33 }
33 34
  35 +- (NSString *)contentForExternalURL:(NSString *)theURL;
  36 +- (NSString *)cachePathForRequest:(ASIWebPageRequest *)theRequest;
34 37
35 @property (assign, nonatomic) ASIWebPageRequest *parentRequest; 38 @property (assign, nonatomic) ASIWebPageRequest *parentRequest;
36 @end 39 @end
@@ -66,8 +66,18 @@ static NSMutableArray *requestsUsingXMLParser = nil; @@ -66,8 +66,18 @@ static NSMutableArray *requestsUsingXMLParser = nil;
66 - (void)parseAsCSS 66 - (void)parseAsCSS
67 { 67 {
68 webContentType = ASICSSWebContentType; 68 webContentType = ASICSSWebContentType;
69 - NSString *responseCSS = [self responseString]; 69 +
70 - if (!responseCSS) { 70 + NSString *responseCSS = nil;
  71 + NSError *err = nil;
  72 + if ([self downloadDestinationPath]) {
  73 + responseCSS = [NSString stringWithContentsOfFile:[self downloadDestinationPath] encoding:[self responseEncoding] error:&err];
  74 + } else {
  75 + responseCSS = [self responseString];
  76 + }
  77 + if (err) {
  78 + [self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:100 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Unable to read HTML string from response",NSLocalizedDescriptionKey,err,NSUnderlyingErrorKey,nil]]];
  79 + return;
  80 + } else if (!responseCSS) {
71 [self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:100 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Unable to read HTML string from response",NSLocalizedDescriptionKey,nil]]]; 81 [self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:100 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Unable to read HTML string from response",NSLocalizedDescriptionKey,nil]]];
72 return; 82 return;
73 } 83 }
@@ -99,6 +109,9 @@ static NSMutableArray *requestsUsingXMLParser = nil; @@ -99,6 +109,9 @@ static NSMutableArray *requestsUsingXMLParser = nil;
99 [externalResourceRequest setUserInfo:[NSDictionary dictionaryWithObject:theURL forKey:@"Path"]]; 109 [externalResourceRequest setUserInfo:[NSDictionary dictionaryWithObject:theURL forKey:@"Path"]];
100 [externalResourceRequest setParentRequest:self]; 110 [externalResourceRequest setParentRequest:self];
101 [externalResourceRequest setShouldResetDownloadProgress:NO]; 111 [externalResourceRequest setShouldResetDownloadProgress:NO];
  112 + if ([self downloadDestinationPath]) {
  113 + [externalResourceRequest setDownloadDestinationPath:[self cachePathForRequest:externalResourceRequest]];
  114 + }
102 [[self externalResourceQueue] addOperation:externalResourceRequest]; 115 [[self externalResourceQueue] addOperation:externalResourceRequest];
103 [externalResourceRequest setShowAccurateProgress:YES]; 116 [externalResourceRequest setShowAccurateProgress:YES];
104 } 117 }
@@ -111,20 +124,6 @@ static NSMutableArray *requestsUsingXMLParser = nil; @@ -111,20 +124,6 @@ static NSMutableArray *requestsUsingXMLParser = nil;
111 - (void)parseAsHTML 124 - (void)parseAsHTML
112 { 125 {
113 webContentType = ASIHTMLWebContentType; 126 webContentType = ASIHTMLWebContentType;
114 - NSString *responseHTML = [self responseString];  
115 - if (!responseHTML) {  
116 - [self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:100 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Unable to read HTML string from response",NSLocalizedDescriptionKey,nil]]];  
117 - return;  
118 - }  
119 -  
120 - NSError *err = nil;  
121 - if (err) {  
122 - [self failWithError:err];  
123 - return;  
124 - } else if (!responseHTML) {  
125 - [self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:101 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Unable to convert response string to XHTML",NSLocalizedDescriptionKey,nil]]];  
126 - return;  
127 - }  
128 127
129 // Only allow parsing of a single document at a time 128 // Only allow parsing of a single document at a time
130 [xmlParsingLock lock]; 129 [xmlParsingLock lock];
@@ -134,10 +133,14 @@ static NSMutableArray *requestsUsingXMLParser = nil; @@ -134,10 +133,14 @@ static NSMutableArray *requestsUsingXMLParser = nil;
134 } 133 }
135 [requestsUsingXMLParser addObject:self]; 134 [requestsUsingXMLParser addObject:self];
136 135
137 - NSData *data = [responseHTML dataUsingEncoding:[self responseEncoding]];  
138 136
139 /* Load XML document */ 137 /* Load XML document */
140 - doc = htmlReadMemory([data bytes], (int)[data length], "", NULL, HTML_PARSE_NONET | HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR); 138 + if ([self downloadDestinationPath]) {
  139 + doc = htmlReadFile([[self downloadDestinationPath] cStringUsingEncoding:NSUTF8StringEncoding], NULL, HTML_PARSE_NONET | HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR);
  140 + } else {
  141 + NSData *data = [self responseData];
  142 + doc = htmlReadMemory([data bytes], (int)[data length], "", NULL, HTML_PARSE_NONET | HTML_PARSE_NOWARNING | HTML_PARSE_NOERROR);
  143 + }
141 if (doc == NULL) { 144 if (doc == NULL) {
142 xmlFreeDoc(doc); 145 xmlFreeDoc(doc);
143 [self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:101 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Error: unable to parse reponse XML",NSLocalizedDescriptionKey,nil]]]; 146 [self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:101 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Error: unable to parse reponse XML",NSLocalizedDescriptionKey,nil]]];
@@ -172,6 +175,9 @@ static NSMutableArray *requestsUsingXMLParser = nil; @@ -172,6 +175,9 @@ static NSMutableArray *requestsUsingXMLParser = nil;
172 [externalResourceRequest setUserInfo:[NSDictionary dictionaryWithObject:theURL forKey:@"Path"]]; 175 [externalResourceRequest setUserInfo:[NSDictionary dictionaryWithObject:theURL forKey:@"Path"]];
173 [externalResourceRequest setParentRequest:self]; 176 [externalResourceRequest setParentRequest:self];
174 [externalResourceRequest setShouldResetDownloadProgress:NO]; 177 [externalResourceRequest setShouldResetDownloadProgress:NO];
  178 + if ([self downloadDestinationPath]) {
  179 + [externalResourceRequest setDownloadDestinationPath:[self cachePathForRequest:externalResourceRequest]];
  180 + }
175 [[self externalResourceQueue] addOperation:externalResourceRequest]; 181 [[self externalResourceQueue] addOperation:externalResourceRequest];
176 [externalResourceRequest setShowAccurateProgress:YES]; 182 [externalResourceRequest setShowAccurateProgress:YES];
177 [self incrementDownloadSizeBy:1]; 183 [self incrementDownloadSizeBy:1];
@@ -214,7 +220,11 @@ static NSMutableArray *requestsUsingXMLParser = nil; @@ -214,7 +220,11 @@ static NSMutableArray *requestsUsingXMLParser = nil;
214 contentType = @"application/octet-stream"; 220 contentType = @"application/octet-stream";
215 } 221 }
216 [requestResponse setObject:contentType forKey:@"ContentType"]; 222 [requestResponse setObject:contentType forKey:@"ContentType"];
217 - [requestResponse setObject:[externalResourceRequest responseData] forKey:@"Data"]; 223 + if ([self downloadDestinationPath]) {
  224 + [requestResponse setObject:[externalResourceRequest downloadDestinationPath] forKey:@"DataPath"];
  225 + } else {
  226 + [requestResponse setObject:[externalResourceRequest responseData] forKey:@"Data"];
  227 + }
218 } 228 }
219 229
220 - (void)externalResourceFetchFailed:(ASIHTTPRequest *)externalResourceRequest 230 - (void)externalResourceFetchFailed:(ASIHTTPRequest *)externalResourceRequest
@@ -225,35 +235,58 @@ static NSMutableArray *requestsUsingXMLParser = nil; @@ -225,35 +235,58 @@ static NSMutableArray *requestsUsingXMLParser = nil;
225 - (void)finishedFetchingExternalResources:(ASINetworkQueue *)queue 235 - (void)finishedFetchingExternalResources:(ASINetworkQueue *)queue
226 { 236 {
227 if (webContentType == ASICSSWebContentType) { 237 if (webContentType == ASICSSWebContentType) {
228 - NSMutableString *parsedResponse = [[[self responseString] mutableCopy] autorelease]; 238 + NSMutableString *parsedResponse;
  239 + NSError *err = nil;
  240 + if ([self downloadDestinationPath]) {
  241 + parsedResponse = [NSMutableString stringWithContentsOfFile:[self downloadDestinationPath] encoding:[self responseEncoding] error:&err];
  242 + } else {
  243 + parsedResponse = [[[self responseString] mutableCopy] autorelease];
  244 + }
  245 + if (err) {
  246 + [self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:101 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Error: unable to read response CSS from disk",NSLocalizedDescriptionKey,nil]]];
  247 + return;
  248 + }
229 if (![self error]) { 249 if (![self error]) {
230 for (NSString *resource in [[self resourceList] keyEnumerator]) { 250 for (NSString *resource in [[self resourceList] keyEnumerator]) {
231 - NSDictionary *resourceInfo = [[self resourceList] objectForKey:resource]; 251 + if ([parsedResponse rangeOfString:resource].location != NSNotFound) {
232 - NSData *data = [resourceInfo objectForKey:@"Data"]; 252 + NSString *newURL = [self contentForExternalURL:resource];
233 - NSString *contentType = [resourceInfo objectForKey:@"ContentType"]; 253 + if (newURL) {
234 - if (data && contentType) { 254 + [parsedResponse replaceOccurrencesOfString:resource withString:newURL options:0 range:NSMakeRange(0, [parsedResponse length])];
235 - if (data && contentType) {  
236 - NSString *newData = [NSString stringWithFormat:@"data:%@;base64,",contentType];  
237 - newData = [newData stringByAppendingString:[ASIHTTPRequest base64forData:data]];  
238 - [parsedResponse replaceOccurrencesOfString:resource withString:newData options:0 range:NSMakeRange(0, [parsedResponse length])];  
239 } 255 }
240 } 256 }
241 } 257 }
242 } 258 }
243 - [self setRawResponseData:(id)[parsedResponse dataUsingEncoding:[self responseEncoding]]]; 259 + if ([self downloadDestinationPath]) {
244 - 260 + [parsedResponse writeToFile:[self downloadDestinationPath] atomically:NO encoding:[self responseEncoding] error:&err];
  261 + if (err) {
  262 + [self failWithError:[NSError errorWithDomain:NetworkRequestErrorDomain code:101 userInfo:[NSDictionary dictionaryWithObjectsAndKeys:@"Error: unable to write response CSS to disk",NSLocalizedDescriptionKey,nil]]];
  263 + return;
  264 + }
  265 + } else {
  266 + [self setRawResponseData:(id)[parsedResponse dataUsingEncoding:[self responseEncoding]]];
  267 + }
245 } else { 268 } else {
246 [xmlParsingLock lock]; 269 [xmlParsingLock lock];
247 270
248 [self updateResourceURLs]; 271 [self updateResourceURLs];
249 xmlChar *bytes = nil; 272 xmlChar *bytes = nil;
250 int size = 0; 273 int size = 0;
251 - xmlDocDumpMemory(doc,&bytes,&size); 274 + FILE *file = NULL;
252 - [self setRawResponseData:[[[NSMutableData alloc] initWithBytes:bytes length:size] autorelease]]; 275 + if ([self downloadDestinationPath]) {
  276 + file = fdopen([[NSFileHandle fileHandleForWritingAtPath:[self downloadDestinationPath]] fileDescriptor], "w");
  277 + xmlDocDump(file, doc);
  278 + } else {
  279 + xmlDocDumpMemory(doc,&bytes,&size);
  280 + [self setRawResponseData:[[[NSMutableData alloc] initWithBytes:bytes length:size] autorelease]];
  281 + }
253 282
254 xmlFreeDoc(doc); 283 xmlFreeDoc(doc);
255 doc = nil; 284 doc = nil;
256 285
  286 + if (file) {
  287 + fclose(file);
  288 + }
  289 +
257 [requestsUsingXMLParser removeObject:self]; 290 [requestsUsingXMLParser removeObject:self];
258 if (![requestsUsingXMLParser count]) { 291 if (![requestsUsingXMLParser count]) {
259 xmlCleanupParser(); 292 xmlCleanupParser();
@@ -374,22 +407,18 @@ static NSMutableArray *requestsUsingXMLParser = nil; @@ -374,22 +407,18 @@ static NSMutableArray *requestsUsingXMLParser = nil;
374 if ([[nodeName lowercaseString] isEqualToString:@"style"]) { 407 if ([[nodeName lowercaseString] isEqualToString:@"style"]) {
375 NSArray *externalResources = [[self class] CSSURLsFromString:value]; 408 NSArray *externalResources = [[self class] CSSURLsFromString:value];
376 for (NSString *theURL in externalResources) { 409 for (NSString *theURL in externalResources) {
377 - NSData *data = [[resourceList objectForKey:theURL] objectForKey:@"Data"]; 410 + if ([value rangeOfString:theURL].location != NSNotFound) {
378 - NSString *contentType = [[resourceList objectForKey:theURL] objectForKey:@"ContentType"]; 411 + NSString *newURL = [self contentForExternalURL:theURL];
379 - if (data && contentType) { 412 + if (newURL) {
380 - NSString *newData = [NSString stringWithFormat:@"data:%@;base64,",contentType]; 413 + value = [value stringByReplacingOccurrencesOfString:theURL withString:newURL];
381 - newData = [newData stringByAppendingString:[ASIHTTPRequest base64forData:data]]; 414 + }
382 - value = [value stringByReplacingOccurrencesOfString:theURL withString:newData];  
383 } 415 }
384 } 416 }
385 xmlNodeSetContent(nodes->nodeTab[i], (xmlChar *)[value cStringUsingEncoding:[self responseEncoding]]); 417 xmlNodeSetContent(nodes->nodeTab[i], (xmlChar *)[value cStringUsingEncoding:[self responseEncoding]]);
386 } else { 418 } else {
387 - NSData *data = [[resourceList objectForKey:value] objectForKey:@"Data"]; 419 + NSString *newURL = [self contentForExternalURL:value];
388 - NSString *contentType = [[resourceList objectForKey:value] objectForKey:@"ContentType"]; 420 + if (newURL) {
389 - if (data && contentType) { 421 + xmlNodeSetContent(nodes->nodeTab[i], (xmlChar *)[newURL cStringUsingEncoding:[self responseEncoding]]);
390 - NSString *newData = [NSString stringWithFormat:@"data:%@;base64,",contentType];  
391 - newData = [newData stringByAppendingString:[ASIHTTPRequest base64forData:data]];  
392 - xmlNodeSetContent(nodes->nodeTab[i], (xmlChar *)[newData cStringUsingEncoding:[self responseEncoding]]);  
393 } 422 }
394 } 423 }
395 424
@@ -423,6 +452,30 @@ static NSMutableArray *requestsUsingXMLParser = nil; @@ -423,6 +452,30 @@ static NSMutableArray *requestsUsingXMLParser = nil;
423 return urls; 452 return urls;
424 } 453 }
425 454
  455 +- (NSString *)contentForExternalURL:(NSString *)theURL
  456 +{
  457 + NSData *data;
  458 + if ([[resourceList objectForKey:theURL] objectForKey:@"DataPath"]) {
  459 + data = [NSData dataWithContentsOfFile:[[resourceList objectForKey:theURL] objectForKey:@"DataPath"]];
  460 + } else {
  461 + data = [[resourceList objectForKey:theURL] objectForKey:@"Data"];
  462 + }
  463 + NSString *contentType = [[resourceList objectForKey:theURL] objectForKey:@"ContentType"];
  464 + if (data && contentType) {
  465 + NSString *dataURI = [NSString stringWithFormat:@"data:%@;base64,",contentType];
  466 + dataURI = [dataURI stringByAppendingString:[ASIHTTPRequest base64forData:data]];
  467 + return dataURI;
  468 + }
  469 + return nil;
  470 +}
  471 +
  472 +static int resourceNum = 0;
  473 +- (NSString *)cachePathForRequest:(ASIWebPageRequest *)theRequest
  474 +{
  475 + resourceNum++;
  476 + return [NSString stringWithFormat:@"/Users/ben/Desktop/%i",resourceNum];
  477 +}
  478 +
426 @synthesize externalResourceQueue; 479 @synthesize externalResourceQueue;
427 @synthesize resourceList; 480 @synthesize resourceList;
428 @synthesize parentRequest; 481 @synthesize parentRequest;
@@ -406,6 +406,7 @@ @@ -406,6 +406,7 @@
406 [request setShowAccurateProgress:NO]; 406 [request setShowAccurateProgress:NO];
407 [request setDownloadProgressDelegate:progressIndicator]; 407 [request setDownloadProgressDelegate:progressIndicator];
408 [request setDownloadCache:[ASIDownloadCache sharedCache]]; 408 [request setDownloadCache:[ASIDownloadCache sharedCache]];
  409 + [request setDownloadDestinationPath:@"/Users/ben/Desktop/fink"];
409 [[ASIDownloadCache sharedCache] setDefaultCachePolicy:ASIOnlyLoadIfNotCachedCachePolicy]; 410 [[ASIDownloadCache sharedCache] setDefaultCachePolicy:ASIOnlyLoadIfNotCachedCachePolicy];
410 [[ASIDownloadCache sharedCache] setShouldRespectCacheControlHeaders:NO]; 411 [[ASIDownloadCache sharedCache] setShouldRespectCacheControlHeaders:NO];
411 [request startAsynchronous]; 412 [request startAsynchronous];
@@ -418,8 +419,15 @@ @@ -418,8 +419,15 @@
418 419
419 - (void)webPageFetchSucceeded:(ASIHTTPRequest *)request 420 - (void)webPageFetchSucceeded:(ASIHTTPRequest *)request
420 { 421 {
421 - [webPageSource setString:[request responseString]]; 422 + if ([request downloadDestinationPath]) {
422 - [[webView mainFrame] loadHTMLString:[request responseString] baseURL:[request url]]; 423 + NSString *response = [NSString stringWithContentsOfFile:[request downloadDestinationPath] encoding:[request responseEncoding] error:nil];
  424 + [webPageSource setString:response];
  425 + [[webView mainFrame] loadHTMLString:response baseURL:[request url]];
  426 + } else {
  427 + [webPageSource setString:[request responseString]];
  428 + [[webView mainFrame] loadHTMLString:[request responseString] baseURL:[request url]];
  429 + }
  430 +
423 [urlField setStringValue:[[request url] absoluteString]]; 431 [urlField setStringValue:[[request url] absoluteString]];
424 } 432 }
425 433