LinkedImageFetcher.m
/* |
File: LinkedImageFetcher.m |
Contains: Downloads an HTML page and then downloads all of the referenced images. |
Written by: DTS |
Copyright: Copyright (c) 2011-2013 Apple Inc. All Rights Reserved. |
Disclaimer: IMPORTANT: This Apple software is supplied to you by Apple Inc. |
("Apple") in consideration of your agreement to the following |
terms, and your use, installation, modification or |
redistribution of this Apple software constitutes acceptance of |
these terms. If you do not agree with these terms, please do |
not use, install, modify or redistribute this Apple software. |
In consideration of your agreement to abide by the following |
terms, and subject to these terms, Apple grants you a personal, |
non-exclusive license, under Apple's copyrights in this |
original Apple software (the "Apple Software"), to use, |
reproduce, modify and redistribute the Apple Software, with or |
without modifications, in source and/or binary forms; provided |
that if you redistribute the Apple Software in its entirety and |
without modifications, you must retain this notice and the |
following text and disclaimers in all such redistributions of |
the Apple Software. Neither the name, trademarks, service marks |
or logos of Apple Inc. may be used to endorse or promote |
products derived from the Apple Software without specific prior |
written permission from Apple. Except as expressly stated in |
this notice, no other rights or licenses, express or implied, |
are granted by Apple herein, including but not limited to any |
patent rights that may be infringed by your derivative works or |
by other works in which the Apple Software may be incorporated. |
The Apple Software is provided by Apple on an "AS IS" basis. |
APPLE MAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING |
WITHOUT LIMITATION THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, |
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, REGARDING |
THE APPLE SOFTWARE OR ITS USE AND OPERATION ALONE OR IN |
COMBINATION WITH YOUR PRODUCTS. |
IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, |
INCIDENTAL OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED |
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) ARISING IN ANY WAY |
OUT OF THE USE, REPRODUCTION, MODIFICATION AND/OR DISTRIBUTION |
OF THE APPLE SOFTWARE, HOWEVER CAUSED AND WHETHER UNDER THEORY |
OF CONTRACT, TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR |
OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE POSSIBILITY OF |
SUCH DAMAGE. |
*/ |
#import "LinkedImageFetcher.h" |
#import "ImageDownloadOperation.h" |
#import "PageGetOperation.h" |
#import "LinkFinder.h" |
@interface LinkedImageFetcher () |
// Read/write versions of public properties |
@property (nonatomic, copy, readwrite) NSError * error; |
// Internal properties |
@property (nonatomic, strong, readonly ) QWatchedOperationQueue * queue; |
@property (nonatomic, strong, readonly ) NSMutableSet * foundPageURLs; |
@property (nonatomic, strong, readonly ) NSMutableDictionary * foundImageURLToPathMap; |
@property (nonatomic, assign, readwrite) NSUInteger runningOperationCount; |
// Forward declarations |
- (void)startPageGet:(NSURL *)pageURL depth:(NSUInteger)depth; |
@end |
@implementation LinkedImageFetcher |
@synthesize URL = _URL; |
@synthesize maximumDepth = _maximumDepth; |
@synthesize imagesDirPath = _imagesDirPath; |
@synthesize delegate = _delegate; |
@synthesize done = _done; |
@synthesize error = _error; |
@synthesize foundPageURLs = _foundPageURLs; |
@synthesize foundImageURLToPathMap = _foundImageURLToPathMap; |
@synthesize runningOperationCount = _runningOperationCount; |
+ (BOOL)isSupportedURL:(NSURL *)url |
{ |
NSString * scheme; |
assert(url != nil); |
scheme = [[url scheme] lowercaseString]; |
return [scheme isEqual:@"http"] || [scheme isEqual:@"https"]; |
} |
- (id)initWithURL:(NSURL *)url |
// See comment in header. |
{ |
assert(url != nil); |
assert([[self class] isSupportedURL:url]); |
self = [super init]; |
if (self != nil) { |
self->_URL = [url copy]; |
self->_imagesDirPath = [[NSTemporaryDirectory() stringByAppendingPathComponent:@"images"] copy]; |
assert(self->_imagesDirPath != nil); |
self->_foundPageURLs = [[NSMutableSet alloc] init]; |
assert(self->_foundPageURLs != nil); |
self->_foundImageURLToPathMap = [[NSMutableDictionary alloc] init]; |
assert(self->_foundImageURLToPathMap != nil); |
} |
return self; |
} |
- (void)dealloc |
{ |
[self->_foundPageURLs release]; |
[self->_foundImageURLToPathMap release]; |
[self->_imagesDirPath release]; |
[self->_queue invalidate]; |
[self->_queue cancelAllOperations]; |
[self->_queue release]; |
[self->_error release]; |
[self->_URL release]; |
[super dealloc]; |
} |
- (QWatchedOperationQueue *)queue |
{ |
if (self->_queue == nil) { |
self->_queue = [[QWatchedOperationQueue alloc] initWithTarget:self]; |
assert(self->_queue != nil); |
} |
return self->_queue; |
} |
- (NSDictionary *)imageURLToPathMap |
// This getter returns a snapshot of the current fetcher state so that, |
// if you call it before the fetcher is done, you don't get a mutable array |
// that's still being mutated. |
{ |
return [[self.foundImageURLToPathMap copy] autorelease]; |
} |
- (BOOL)start |
// See comment in header. |
{ |
BOOL success; |
NSFileManager * fm; |
fm = [NSFileManager defaultManager]; |
assert(fm != nil); |
success = [fm createDirectoryAtPath:self.imagesDirPath withIntermediateDirectories:NO attributes:nil error:NULL]; |
if ( ! success ) { |
// If the create failed, it could be because the directory already exists. |
// So let's get a listing and see if that succeeds. |
success = [fm contentsOfDirectoryAtPath:self.imagesDirPath error:NULL] != nil; |
} |
// Start the main GET operation, that gets the HTML whose links we want |
// to download. |
if (success) { |
[self startPageGet:self.URL depth:0]; |
} |
return success; |
} |
- (void)stopWithError:(NSError *)error |
// An internal method called to stop the fetch and clean things up. |
{ |
assert(error != nil); |
[self.queue invalidate]; |
[self.queue cancelAllOperations]; |
self.error = error; |
// When we set done our client's KVO might release us, meaning that we end |
// up running with an invalid self. This can cause all sorts of problems, |
// so we do my standard retain/autorelease technique to avoid it. |
[[self retain] autorelease]; |
self.done = YES; |
} |
- (void)stop |
// See comment in header. |
{ |
[self stopWithError:[NSError errorWithDomain:NSCocoaErrorDomain code:NSUserCancelledError userInfo:nil]]; |
} |
- (void)logText:(NSString *)text URL:(NSURL *)url depth:(NSUInteger)depth error:(NSError *)error |
// An internal method called to log information about the fetch. |
// This either logs to stdout or via a delegate callback. |
{ |
assert(text != nil); |
assert(url != nil); |
// depth has no constraints |
// error may be nil |
if (self.delegate == nil) { |
// If there's no delegate, we just log to stdout 'cause that's what best suits the |
// command line tool. |
if (error == nil) { |
fprintf(stdout, "%*s%s\n", (int) (depth * 2), "", [[url absoluteString] UTF8String]); |
fprintf(stdout, "%*s %s\n", (int) (depth * 2), "", [text UTF8String]); |
} else { |
fprintf(stdout, "%*s%s\n", (int) (depth * 2), "", [[url absoluteString] UTF8String]); |
fprintf(stdout, "%*s %s: %s %d\n", (int) (depth * 2), "", [text UTF8String], [[error domain] UTF8String], (int) [error code]); |
} |
} else if ([self.delegate respondsToSelector:@selector(linkedImageFetcher:logText:URL:depth:error:)]) { |
[self.delegate linkedImageFetcher:self logText:text URL:url depth:depth error:error]; |
} |
} |
// IMPORTANT: runningOperationCount is only ever modified by the main thread, |
// so we don't have to do any locking. Also, because the 'done' methods are called |
// on the main thread, we don't have to worry about early completion, that is, |
// -parseDone: kicking off download 1, then getting delayed, then download 1 |
// completing, decrementing runningOperationCount, and deciding that we're |
// all done. The decrement of runningOperationCount is done by -downloadDone: |
// and -downloadDone: can't run until we return back to the run loop. |
- (void)operationDidStart |
// Called when an operation has started to increment runningOperationCount. |
{ |
self.runningOperationCount += 1; |
} |
- (void)operationDidFinish |
// Called when an operation has finished to decrement runningOperationCount |
// and complete the whole fetch if it hits zero. |
{ |
assert(self.runningOperationCount != 0); |
self.runningOperationCount -= 1; |
if (self.runningOperationCount == 0) { |
// See comment in -stopWithError:. |
[[self retain] autorelease]; |
self.done = YES; |
} |
} |
- (void)startPageGet:(NSURL *)pageURL depth:(NSUInteger)depth |
// Starts the operation to GET an HTML page. Called for both the |
// initial main page, and for any subsequently linked-to pages. |
{ |
PageGetOperation * op; |
assert([pageURL baseURL] == nil); // must be an absolute URL |
assert( ! [self.foundPageURLs containsObject:pageURL] ); |
[self.foundPageURLs addObject:pageURL]; |
op = [[[PageGetOperation alloc] initWithURL:pageURL depth:depth] autorelease]; |
assert(op != nil); |
[self.queue addOperation:op finishedAction:@selector(pageGetDone:)]; |
[self operationDidStart]; |
// ... continues in -pageGetDone: |
} |
- (void)pageGetDone:(PageGetOperation *)op |
// Called when the GET for an HTML page is done. We start a LinkFinder |
// operation to parse the HTML. |
{ |
assert([op isKindOfClass:[PageGetOperation class]]); |
assert([NSThread isMainThread]); |
if (op.error != nil) { |
// An error getting the main page is fatal to the entire process; an error |
// getting any subsequent pages is just logged. |
if (op.depth == 0) { |
[self stopWithError:op.error]; |
} else { |
[self logText:@"page get error" URL:op.URL depth:op.depth error:op.error]; |
} |
} else { |
LinkFinder * nextOp; |
[self logText:@"page get done" URL:op.URL depth:op.depth error:nil]; |
// Don't use op.URL here, but rather [op.lastResponse URL] so that relatives |
// URLs work in the face of redirection. |
nextOp = [[[LinkFinder alloc] initWithData:op.responseBody fromURL:[op.lastResponse URL] depth:op.depth] autorelease]; |
assert(nextOp != nil); |
nextOp.useRelaxedParsing = YES; |
[self.queue addOperation:nextOp finishedAction:@selector(parseDone:)]; |
[self operationDidStart]; |
// ... continues in -parseDone: |
} |
[self operationDidFinish]; |
} |
- (void)parseDone:(LinkFinder *)op |
// Called when the link finder operation is done. We look at the links |
// and start an appropriate number of page get and image download operations. |
{ |
#pragma unused(op) |
assert([op isKindOfClass:[LinkFinder class]]); |
assert([NSThread isMainThread]); |
if (op.error != nil) { |
// An error parsing the main page is fatal to the entire process; an error |
// parsing any subsequent pages is just logged. |
if (op.depth == 0) { |
[self stopWithError:op.error]; |
} else { |
[self logText:@"page parse error" URL:op.URL depth:op.depth error:op.error]; |
} |
} else { |
NSURL * thisURL; |
NSURL * thisURLAbsolute; |
// We need to use absolute URLs in order to test for membership in |
// foundPageURLs and foundImageURLToPathMap. |
// Process all of the links in the page. But only if we haven't exceeded |
// our maximum depth. And if we haven't already processed that page URL. |
if (op.depth != self.maximumDepth) { |
for (thisURL in op.linkURLs) { |
thisURLAbsolute = [thisURL absoluteURL]; |
assert(thisURLAbsolute != nil); |
if ( [[self class] isSupportedURL:thisURLAbsolute] ) { |
if ([self.foundPageURLs containsObject:thisURLAbsolute]) { |
[self logText:@"page is duplicate" URL:thisURLAbsolute depth:op.depth error:nil]; |
} else if ( ([thisURLAbsolute fragment] != nil) || ([thisURLAbsolute parameterString] != nil) || ([thisURLAbsolute query] != nil) ) { |
[self logText:@"page URL is complex" URL:thisURLAbsolute depth:op.depth error:nil]; |
} else { |
[self startPageGet:thisURLAbsolute depth:op.depth + 1]; |
} |
} else { |
[self logText:@"page URL is unsupported" URL:thisURLAbsolute depth:op.depth error:nil]; |
} |
} |
} |
// Download all of the images in the page, but only if we haven't already |
// downloaded that image. |
for (thisURL in op.imageURLs) { |
thisURLAbsolute = [thisURL absoluteURL]; |
assert(thisURLAbsolute != nil); |
if ( [[self class] isSupportedURL:thisURLAbsolute] ) { |
if ([self.foundImageURLToPathMap objectForKey:thisURLAbsolute] != nil) { |
[self logText:@"image is duplicate" URL:thisURLAbsolute depth:op.depth error:nil]; |
} else { |
ImageDownloadOperation * downloadOperation; |
// Put in a placeholder for the download. |
[self.foundImageURLToPathMap setObject:[NSNull null] forKey:thisURLAbsolute]; |
downloadOperation = [[[ImageDownloadOperation alloc] initWithURL:thisURLAbsolute imagesDirPath:self.imagesDirPath depth:op.depth + 1] autorelease]; |
assert(downloadOperation != nil); |
[self.queue addOperation:downloadOperation finishedAction:@selector(downloadDone:)]; |
[self operationDidStart]; |
// ... continues in -downloadDone: |
} |
} else { |
[self logText:@"image URL is unsupported" URL:thisURLAbsolute depth:op.depth error:nil]; |
} |
} |
} |
[self operationDidFinish]; |
} |
- (void)downloadDone:(ImageDownloadOperation *)op |
// Called when an image download operation is done. |
{ |
#pragma unused(op) |
assert([op isKindOfClass:[ImageDownloadOperation class]]); |
assert([NSThread isMainThread]); |
// Replace the NSNull in the foundImageURLToPathMap with the path to the downloaded |
// file (on success) or the error. Note that we use op.URL here, not [op.lastResponse URL], |
// because this stuff is keyed on the original URL, not the final URL after redirects. |
assert([[self.foundImageURLToPathMap objectForKey:op.URL] isEqual:[NSNull null]]); |
if (op.error != nil) { |
[self.foundImageURLToPathMap setObject:op.error forKey:op.URL]; |
[self logText:@"image download error" URL:op.URL depth:op.depth error:op.error]; |
} else { |
[self.foundImageURLToPathMap setObject:op.imageFilePath forKey:op.URL]; |
[self logText:[NSString stringWithFormat:@"image download to: %@", op.imageFilePath] URL:op.URL depth:op.depth error:nil]; |
} |
[self operationDidFinish]; |
} |
+ (id)fetcherWithURLString:(const char *)urlCStr maximumDepth:(int)maximumDepth |
// See comment in header. |
{ |
LinkedImageFetcher * result; |
NSString * urlStr; |
NSURL * url; |
assert(urlCStr != NULL); |
result = nil; |
// First construct and check the URL. |
url = nil; |
urlStr = @(urlCStr); |
if (urlStr != nil) { |
url = [NSURL URLWithString:urlStr]; |
} |
if (url == nil) { |
fprintf(stderr, "%s: malformed URL: %s\n", getprogname(), urlCStr); |
} else { |
if ( ! [self isSupportedURL:url] ) { |
fprintf(stderr, "%s: unsupported URL scheme: %s\n", getprogname(), [[[url scheme] lowercaseString] UTF8String]); |
url = nil; |
} |
} |
// Then check maximumDepth. If that passes, create the object to return. |
if (url != nil) { |
if (maximumDepth < 0) { |
fprintf(stderr, "%s: maximum depth must be non-negative\n", getprogname()); |
} else { |
result = [[[self alloc] initWithURL:url] autorelease]; |
if (result != nil) { |
result.maximumDepth = maximumDepth; |
} |
} |
} |
return result; |
} |
@end |
Copyright © 2013 Apple Inc. All Rights Reserved. Terms of Use | Privacy Policy | Updated: 2013-03-05