From 379c4fbf517ba6472e677d2dfbe2293ae9b99d7d Mon Sep 17 00:00:00 2001 From: Maksym Shcheglov Date: Mon, 18 Apr 2016 00:21:00 +0100 Subject: [PATCH] Changed the current solution to use extended file attributes as a record's header storage. Defined a function that loads a record header from the file's extended attributes or content (legacy version). Updated unit tests and demo project accordingly. --- .../project.pbxproj | 2 + Sources/SPTPersistentCache.m | 360 ++++++------------ Sources/SPTPersistentCacheHeader.m | 29 ++ Tests/SPTPersistentCacheHeaderTests.m | 136 +++++++ Tests/SPTPersistentCacheTests.m | 101 ++--- Viewer/MainWindowController.m | 20 +- .../SPTPersistentCache/SPTPersistentCache.h | 4 + .../SPTPersistentCacheHeader.h | 11 + 8 files changed, 343 insertions(+), 320 deletions(-) diff --git a/SPTPersistentCacheFramework/SPTPersistentCacheFramework.xcodeproj/project.pbxproj b/SPTPersistentCacheFramework/SPTPersistentCacheFramework.xcodeproj/project.pbxproj index 760e320..9d72b40 100644 --- a/SPTPersistentCacheFramework/SPTPersistentCacheFramework.xcodeproj/project.pbxproj +++ b/SPTPersistentCacheFramework/SPTPersistentCacheFramework.xcodeproj/project.pbxproj @@ -10,6 +10,7 @@ 050076B31C7A4354000819B5 /* SPTPersistentCachePosixWrapper.h in Headers */ = {isa = PBXBuildFile; fileRef = 050076B11C7A4354000819B5 /* SPTPersistentCachePosixWrapper.h */; }; 050076B41C7A4354000819B5 /* SPTPersistentCachePosixWrapper.m in Sources */ = {isa = PBXBuildFile; fileRef = 050076B21C7A4354000819B5 /* SPTPersistentCachePosixWrapper.m */; }; 050076B51C7A4DC7000819B5 /* SPTPersistentCachePosixWrapper.m in Sources */ = {isa = PBXBuildFile; fileRef = 050076B21C7A4354000819B5 /* SPTPersistentCachePosixWrapper.m */; }; + 5EB3C5EB1CEA1B900091739E /* NSError+SPTPersistentCacheDomainErrors.h in Headers */ = {isa = PBXBuildFile; fileRef = DD1D238C1C7785A900D0477A /* NSError+SPTPersistentCacheDomainErrors.h */; }; 9C9E707B1C790F0B00E1CBE6 /* SPTPersistentCacheObjectDescription.m in Sources */ = {isa = PBXBuildFile; fileRef = 9C9E70791C790F0B00E1CBE6 /* SPTPersistentCacheObjectDescription.m */; }; 9C9E707C1C790F0B00E1CBE6 /* SPTPersistentCacheObjectDescription.h in Headers */ = {isa = PBXBuildFile; fileRef = 9C9E707A1C790F0B00E1CBE6 /* SPTPersistentCacheObjectDescription.h */; }; 9C9E707D1C790F3700E1CBE6 /* SPTPersistentCacheObjectDescription.m in Sources */ = {isa = PBXBuildFile; fileRef = 9C9E70791C790F0B00E1CBE6 /* SPTPersistentCacheObjectDescription.m */; }; @@ -236,6 +237,7 @@ DD1D23851C77857E00D0477A /* SPTPersistentCache.h in Headers */, DD1D23B51C7785AE00D0477A /* SPTPersistentCache+Private.h in Headers */, DD1D23BB1C7785AE00D0477A /* SPTPersistentCacheRecord+Private.h in Headers */, + 5EB3C5EB1CEA1B900091739E /* NSError+SPTPersistentCacheDomainErrors.h in Headers */, DD1D23B31C7785AE00D0477A /* crc32iso3309.h in Headers */, DD1D23BD1C7785AE00D0477A /* SPTPersistentCacheResponse+Private.h in Headers */, C4EA65071C7A547000A6091A /* SPTPersistentCacheDebugUtilities.h in Headers */, diff --git a/Sources/SPTPersistentCache.m b/Sources/SPTPersistentCache.m index 7a56bb0..4b96690 100644 --- a/Sources/SPTPersistentCache.m +++ b/Sources/SPTPersistentCache.m @@ -444,104 +444,96 @@ - (void)loadDataForKeySync:(NSString *)key if (![self.fileManager fileExistsAtPath:filePath]) { [self dispatchEmptyResponseWithResult:SPTPersistentCacheResponseCodeNotFound callback:callback onQueue:queue]; return; - } else { - // File exist - NSError *error = nil; - NSMutableData *rawData = [NSMutableData dataWithContentsOfFile:filePath - options:NSDataReadingMappedIfSafe - error:&error]; - if (rawData == nil) { - // File read with error -> inform user - [self dispatchError:error - result:SPTPersistentCacheResponseCodeOperationError - callback:callback - onQueue:queue]; - } else { - SPTPersistentCacheRecordHeader *header = SPTPersistentCacheGetHeaderFromData(rawData.mutableBytes, rawData.length); - - // If not enough data to cast to header, its not the file we can process - if (header == NULL) { - NSError *headerError = [NSError spt_persistentDataCacheErrorWithCode:SPTPersistentCacheLoadingErrorNotEnoughDataToGetHeader]; - [self dispatchError:headerError - result:SPTPersistentCacheResponseCodeOperationError - callback:callback - onQueue:queue]; - return; - } - - SPTPersistentCacheRecordHeader localHeader; - memcpy(&localHeader, header, sizeof(localHeader)); - - // Check header is valid - NSError *headerError = SPTPersistentCacheCheckValidHeader(&localHeader); - if (headerError != nil) { - [self dispatchError:headerError - result:SPTPersistentCacheResponseCodeOperationError - callback:callback - onQueue:queue]; - return; - } + } + // File exist + SPTPersistentCacheRecordHeader header; + NSError *headerError = SPTPersistentCacheGetHeaderFromFileWithPath(filePath, &header); + if (headerError != nil) { + [self dispatchError:headerError + result:SPTPersistentCacheResponseCodeOperationError + callback:callback + onQueue:queue]; + return; + } - const NSUInteger refCount = localHeader.refCount; + const NSUInteger refCount = header.refCount; - // We return locked files even if they expired, GC doesnt collect them too so they valuable to user - // Satisfy Req.#1.2 - if (![self isDataCanBeReturnedWithHeader:&localHeader]) { + // We return locked files even if they expired, GC doesnt collect them too so they valuable to user + // Satisfy Req.#1.2 + if (![self isDataCanBeReturnedWithHeader:&header]) { #ifdef DEBUG_OUTPUT_ENABLED - [self debugOutput:@"PersistentDataCache: Record with key: %@ expired, t:%llu, TTL:%llu", key, localHeader.updateTimeSec, localHeader.ttl]; + [self debugOutput:@"PersistentDataCache: Record with key: %@ expired, t:%llu, TTL:%llu", key, localHeader.updateTimeSec, localHeader.ttl]; #endif - [self dispatchEmptyResponseWithResult:SPTPersistentCacheResponseCodeNotFound - callback:callback - onQueue:queue]; - return; - } - - // Check that payload is correct size - if (localHeader.payloadSizeBytes != [rawData length] - SPTPersistentCacheRecordHeaderSize) { - [self debugOutput:@"PersistentDataCache: Error: Wrong payload size for key:%@ , will return error", key]; - [self dispatchError:[NSError spt_persistentDataCacheErrorWithCode:SPTPersistentCacheLoadingErrorWrongPayloadSize] - result:SPTPersistentCacheResponseCodeOperationError - callback:callback onQueue:queue]; - return; - } - - NSRange payloadRange = NSMakeRange(SPTPersistentCacheRecordHeaderSize, (NSUInteger)localHeader.payloadSizeBytes); - NSData *payload = [rawData subdataWithRange:payloadRange]; - const NSUInteger ttl = (NSUInteger)localHeader.ttl; - + [self dispatchEmptyResponseWithResult:SPTPersistentCacheResponseCodeNotFound + callback:callback + onQueue:queue]; + return; + } - SPTPersistentCacheRecord *record = [[SPTPersistentCacheRecord alloc] initWithData:payload - key:key - refCount:refCount - ttl:ttl]; + NSError *error = nil; + NSData *rawData = [NSData dataWithContentsOfFile:filePath + options:NSDataReadingMappedIfSafe + error:&error]; + if (rawData == nil) { + // File read with error -> inform user + [self dispatchError:error + result:SPTPersistentCacheResponseCodeOperationError + callback:callback + onQueue:queue]; + return; + } + NSData *payload = [self payloadFromRawData:rawData]; + // Check that payload is correct size + if (header.payloadSizeBytes != payload.length) { + [self debugOutput:@"PersistentDataCache: Error: Wrong payload size for key:%@ , will return error", key]; + [self dispatchError:[NSError spt_persistentDataCacheErrorWithCode:SPTPersistentCacheLoadingErrorWrongPayloadSize] + result:SPTPersistentCacheResponseCodeOperationError + callback:callback onQueue:queue]; + return; + } - SPTPersistentCacheResponse *response = [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationSucceeded - error:nil - record:record]; - // If data ttl == 0 we update access time - if (ttl == 0) { - localHeader.updateTimeSec = spt_uint64rint(self.currentDateTimeInterval); - localHeader.crc = SPTPersistentCacheCalculateHeaderCRC(&localHeader); - memcpy(header, &localHeader, sizeof(localHeader)); - - // Write back with updated access attributes - NSError *werror = nil; - if (![rawData writeToFile:filePath options:NSDataWritingAtomic error:&werror]) { - [self debugOutput:@"PersistentDataCache: Error writing back record:%@, error:%@", filePath.lastPathComponent, werror]; - } else { + const NSUInteger ttl = (NSUInteger)header.ttl; + + SPTPersistentCacheRecord *record = [[SPTPersistentCacheRecord alloc] initWithData:payload + key:key + refCount:refCount + ttl:ttl]; + + SPTPersistentCacheResponse *response = [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationSucceeded + error:nil + record:record]; + // If data ttl == 0 we update access time + if (ttl == 0) { + header.updateTimeSec = spt_uint64rint(self.currentDateTimeInterval); + header.crc = SPTPersistentCacheCalculateHeaderCRC(&header); + + // Write back with updated access attributes + NSError* headerSetError = SPTPersistentCacheSetHeaderForFileWithPath(filePath, &header); + if (headerSetError != nil) { + [self debugOutput:@"PersistentDataCache: Error writing back record:%@ error:%@", filePath.lastPathComponent, [headerSetError localizedDescription]]; + } else { #ifdef DEBUG_OUTPUT_ENABLED - [self debugOutput:@"PersistentDataCache: Writing back record:%@ OK", filePath.lastPathComponent]; + [self debugOutput:@"PersistentDataCache: Writing back file header:%@ OK", filePath.lastPathComponent]; #endif - } - } + } + } - // Callback only after we finished everyhing to avoid situation when user gets notified and we are still writting - SPTPersistentCacheSafeDispatch(queue, ^{ - callback(response); - }); - } // if rawData - } // file exist + // Callback only after we finished everyhing to avoid situation when user gets notified and we are still writting + SPTPersistentCacheSafeDispatch(queue, ^{ + callback(response); + }); +} + +/** + * Gets payload from the raw data. Removes the legacy header if needed. + */ +- (NSData*)payloadFromRawData:(NSData*)data +{ + NSMutableData* rawData = [data mutableCopy]; + SPTPersistentCacheRecordHeader *legacyHeader = SPTPersistentCacheGetHeaderFromData(rawData.mutableBytes, rawData.length); + BOOL hasLegacyHeader = legacyHeader != NULL && SPTPersistentCacheCheckValidHeader(legacyHeader) == nil; + return hasLegacyHeader ? [data subdataWithRange:NSMakeRange(SPTPersistentCacheRecordHeaderSize, data.length - SPTPersistentCacheRecordHeaderSize)] : data; } /** @@ -559,26 +551,25 @@ - (NSError *)storeDataSync:(NSData *)data NSString *subDir = [self.dataCacheFileManager subDirectoryPathForKey:key]; [self.fileManager createDirectoryAtPath:subDir withIntermediateDirectories:YES attributes:nil error:nil]; - const NSUInteger payloadLength = [data length]; - const NSUInteger rawDataLength = SPTPersistentCacheRecordHeaderSize + payloadLength; - - NSMutableData *rawData = [NSMutableData dataWithCapacity:rawDataLength]; - - SPTPersistentCacheRecordHeader header = SPTPersistentCacheRecordHeaderMake(ttl, - payloadLength, - spt_uint64rint(self.currentDateTimeInterval), - isLocked); - - [rawData appendBytes:&header length:SPTPersistentCacheRecordHeaderSize]; - [rawData appendData:data]; - NSError *error = nil; - if (![rawData writeToFile:filePath options:NSDataWritingAtomic error:&error]) { + if (![data writeToFile:filePath options:NSDataWritingAtomic error:&error]) { [self debugOutput:@"PersistentDataCache: Error writting to file:%@ , for key:%@. Removing it...", filePath, key]; [self removeDataForKeysSync:@[key]]; [self dispatchError:error result:SPTPersistentCacheResponseCodeOperationError callback:callback onQueue:queue]; } else { + const NSUInteger payloadLength = [data length]; + SPTPersistentCacheRecordHeader header = SPTPersistentCacheRecordHeaderMake(ttl, + payloadLength, + spt_uint64rint(self.currentDateTimeInterval), + isLocked); + NSError* headerSetError = SPTPersistentCacheSetHeaderForFileWithPath(filePath, &header); + if (headerSetError != nil) { + [self debugOutput:@"PersistentDataCache: Error writting header at file path:%@ , error:%@", filePath, [headerSetError localizedDescription]]; + [self removeDataForKeysSync:@[key]]; + [self dispatchError:headerSetError result:SPTPersistentCacheResponseCodeOperationError callback:callback onQueue:queue]; + return headerSetError; + } if (callback != nil) { SPTPersistentCacheResponse *response = [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationSucceeded @@ -595,14 +586,12 @@ - (NSError *)storeDataSync:(NSData *)data } /** - * Method to work safely with opened file referenced by file descriptor. - * Method handles file closing properly in case of errors. - * Descriptor is passed to a jobBlock for further usage. + * Method used to read/write file header. */ -- (SPTPersistentCacheResponse *)guardOpenFileWithPath:(NSString *)filePath - jobBlock:(SPTPersistentCacheFileProcessingBlockType)jobBlock - complain:(BOOL)needComplains - writeBack:(BOOL)writeBack +- (SPTPersistentCacheResponse *)alterHeaderForFileAtPath:(NSString *)filePath + withBlock:(SPTPersistentCacheRecordHeaderGetCallbackType)modifyBlock + writeBack:(BOOL)needWriteBack + complain:(BOOL)needComplains { if (![self.fileManager fileExistsAtPath:filePath]) { if (needComplains) { @@ -610,145 +599,42 @@ - (SPTPersistentCacheResponse *)guardOpenFileWithPath:(NSString *)filePath } return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeNotFound error:nil record:nil]; - } else { - const int SPTPersistentCacheInvalidResult = -1; - const int flags = (writeBack ? O_RDWR : O_RDONLY); - - int fd = open([filePath UTF8String], flags); - if (fd == SPTPersistentCacheInvalidResult) { - const int errorNumber = errno; - NSString *errorDescription = @(strerror(errorNumber)); - [self debugOutput:@"PersistentDataCache: Error opening file:%@ , error:%@", filePath, errorDescription]; - NSError *error = [NSError errorWithDomain:NSPOSIXErrorDomain - code:errorNumber - userInfo:@{ NSLocalizedDescriptionKey: errorDescription }]; - return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationError - error:error - record:nil]; - } - - SPTPersistentCacheResponse *response = jobBlock(fd); - - fd = [self.posixWrapper close:fd]; - if (fd == SPTPersistentCacheInvalidResult) { - const int errorNumber = errno; - NSString *errorDescription = @(strerror(errorNumber)); - [self debugOutput:@"PersistentDataCache: Error closing file:%@ , error:%@", filePath, errorDescription]; - NSError *error = [NSError errorWithDomain:NSPOSIXErrorDomain - code:errorNumber - userInfo:@{ NSLocalizedDescriptionKey: errorDescription }]; - return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationError - error:error - record:nil]; - } - - return response; } -} + SPTPersistentCacheRecordHeader header; + NSError *headerGetError = SPTPersistentCacheGetHeaderFromFileWithPath(filePath, &header); + if (headerGetError != nil) { + [self debugOutput:@"PersistentDataCache: Failed to read the header of file path:%@ , error:%@", filePath, [headerGetError localizedDescription]]; + return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationError + error:headerGetError + record:nil]; + } -/** - * Method used to read/write file header. - */ -- (SPTPersistentCacheResponse *)alterHeaderForFileAtPath:(NSString *)filePath - withBlock:(SPTPersistentCacheRecordHeaderGetCallbackType)modifyBlock - writeBack:(BOOL)needWriteBack - complain:(BOOL)needComplains -{ - return [self guardOpenFileWithPath:filePath jobBlock:^SPTPersistentCacheResponse*(int filedes) { - - SPTPersistentCacheRecordHeader header; - ssize_t readBytes = [self.posixWrapper read:filedes - buffer:&header - bufferSize:SPTPersistentCacheRecordHeaderSize]; - if (readBytes != (ssize_t)SPTPersistentCacheRecordHeaderSize) { - NSError *error = [NSError spt_persistentDataCacheErrorWithCode:SPTPersistentCacheLoadingErrorNotEnoughDataToGetHeader]; - if (readBytes == -1) { - const int errorNumber = errno; - const char *errorString = strerror(errorNumber); - error = [NSError errorWithDomain:NSPOSIXErrorDomain - code:errorNumber - userInfo:@{ NSLocalizedDescriptionKey: @(errorString) }]; - } + modifyBlock(&header); - [self debugOutput:@"PersistentDataCache: Error not enough data to read the header of file path:%@ , error:%@", - filePath, [error localizedDescription]]; + if (needWriteBack) { - return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationError - error:error + uint32_t oldCRC = header.crc; + header.crc = SPTPersistentCacheCalculateHeaderCRC(&header); + + // If nothing has changed we do nothing then + if (oldCRC == header.crc) { + return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationSucceeded + error:nil record:nil]; } - NSError *nsError = SPTPersistentCacheCheckValidHeader(&header); - if (nsError != nil) { - [self debugOutput:@"PersistentDataCache: Error checking header at file path:%@ , error:%@", filePath, nsError]; + NSError* headerSetError = SPTPersistentCacheSetHeaderForFileWithPath(filePath, &header); + if (headerSetError != nil) { + [self debugOutput:@"PersistentDataCache: Error writting header at file path:%@ , error:%@", filePath, [headerSetError localizedDescription]]; return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationError - error:nsError + error:headerSetError record:nil]; } + } - modifyBlock(&header); - - if (needWriteBack) { - - uint32_t oldCRC = header.crc; - header.crc = SPTPersistentCacheCalculateHeaderCRC(&header); - - // If nothing has changed we do nothing then - if (oldCRC == header.crc) { - return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationSucceeded - error:nil - record:nil]; - } - - // Set file pointer to the beginning of the file - off_t seekOffset = [self.posixWrapper lseek:filedes seekType:SEEK_SET seekAmount:0]; - if (seekOffset != 0) { - const int errorNumber = errno; - NSString *errorDescription = @(strerror(errorNumber)); - [self debugOutput:@"PersistentDataCache: Error seeking to begin of file path:%@ , error:%@", filePath, errorDescription]; - NSError *error = [NSError errorWithDomain:NSPOSIXErrorDomain - code:errorNumber - userInfo:@{ NSLocalizedDescriptionKey: errorDescription }]; - return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationError - error:error - record:nil]; - - } else { - ssize_t writtenBytes = [self.posixWrapper write:filedes - buffer:&header - bufferSize:SPTPersistentCacheRecordHeaderSize]; - if (writtenBytes != (ssize_t)SPTPersistentCacheRecordHeaderSize) { - const int errorNumber = errno; - NSString *errorDescription = @(strerror(errorNumber)); - [self debugOutput:@"PersistentDataCache: Error writting header at file path:%@ , error:%@", filePath, errorDescription]; - NSError *error = [NSError errorWithDomain:NSPOSIXErrorDomain - code:errorNumber - userInfo:@{ NSLocalizedDescriptionKey: errorDescription }]; - return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationError - error:error - record:nil]; - - } else { - int result = [self.posixWrapper fsync:filedes]; - if (result == -1) { - const int errorNumber = errno; - NSString *errorDescription = @(strerror(errorNumber)); - [self debugOutput:@"PersistentDataCache: Error flushing file:%@ , error:%@", filePath, errorDescription]; - NSError *error = [NSError errorWithDomain:NSPOSIXErrorDomain - code:errorNumber - userInfo:@{ NSLocalizedDescriptionKey: errorDescription }]; - return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationError - error:error - record:nil]; - } - } - } - } - - return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationSucceeded - error:nil - record:nil]; - } complain:needComplains writeBack:needWriteBack]; + return [[SPTPersistentCacheResponse alloc] initWithResult:SPTPersistentCacheResponseCodeOperationSucceeded + error:nil + record:nil]; } /** diff --git a/Sources/SPTPersistentCacheHeader.m b/Sources/SPTPersistentCacheHeader.m index c43a4b4..8dc6c94 100644 --- a/Sources/SPTPersistentCacheHeader.m +++ b/Sources/SPTPersistentCacheHeader.m @@ -23,9 +23,12 @@ #import "NSError+SPTPersistentCacheDomainErrors.h" #include "crc32iso3309.h" +#include const SPTPersistentCacheMagicType SPTPersistentCacheMagicValue = 0x46545053; // SPTF const size_t SPTPersistentCacheRecordHeaderSize = sizeof(SPTPersistentCacheRecordHeader); +static NSString* const SPTPersistentCacheRecordHeaderAttributeName = @"com.spotify.cache"; +static int const SPTPersistentCacheHeaderInvalidResult = -1; _Static_assert(sizeof(SPTPersistentCacheRecordHeader) == 64, "Struct SPTPersistentCacheRecordHeader has to be packed without padding"); @@ -69,6 +72,32 @@ SPTPersistentCacheRecordHeader SPTPersistentCacheRecordHeaderMake(uint64_t ttl, return (SPTPersistentCacheRecordHeader *)data; } +NSError* SPTPersistentCacheGetHeaderFromFileWithPath(NSString *filePath, SPTPersistentCacheRecordHeader *header) +{ + memset(header, 0, SPTPersistentCacheRecordHeaderSize); + ssize_t size = getxattr([filePath UTF8String], [SPTPersistentCacheRecordHeaderAttributeName UTF8String], header, SPTPersistentCacheRecordHeaderSize, 0, 0); + if (size == SPTPersistentCacheHeaderInvalidResult) { + // try to load the legacy header. the fileHandle object uses file descriptor under the hood and responsible for closing it on deallocation. + NSFileHandle* fileHandle = [NSFileHandle fileHandleForReadingAtPath:filePath]; + NSMutableData* rawData = [[fileHandle readDataOfLength:SPTPersistentCacheRecordHeaderSize] mutableCopy]; + SPTPersistentCacheRecordHeader *legacyHeader = SPTPersistentCacheGetHeaderFromData(rawData.mutableBytes, rawData.length); + // If not enough data to cast to header, its not the file we can process + if (legacyHeader == NULL) { + return [NSError spt_persistentDataCacheErrorWithCode:SPTPersistentCacheLoadingErrorNotEnoughDataToGetHeader]; + } + memcpy(header, legacyHeader, SPTPersistentCacheRecordHeaderSize); + } + return SPTPersistentCacheCheckValidHeader(header); +} + +NSError* SPTPersistentCacheSetHeaderForFileWithPath(NSString *filepath, const SPTPersistentCacheRecordHeader *header) { + int res = setxattr([filepath UTF8String], [SPTPersistentCacheRecordHeaderAttributeName UTF8String], header, SPTPersistentCacheRecordHeaderSize, 0, 0); + if (res == SPTPersistentCacheHeaderInvalidResult) { + return [NSError spt_persistentDataCacheErrorWithCode:SPTPersistentCacheLoadingErrorFileAttributeOperationFail]; + } + return nil; +} + int /*SPTPersistentCacheLoadingError*/ SPTPersistentCacheValidateHeader(const SPTPersistentCacheRecordHeader *header) { if (header == NULL) { diff --git a/Tests/SPTPersistentCacheHeaderTests.m b/Tests/SPTPersistentCacheHeaderTests.m index b85c82c..22e90df 100644 --- a/Tests/SPTPersistentCacheHeaderTests.m +++ b/Tests/SPTPersistentCacheHeaderTests.m @@ -24,6 +24,8 @@ #import #import "SPTPersistentCacheTypeUtilities.h" +static NSString* const SPTCacheRecordFileName = @"cache.record"; + @interface SPTPersistentCacheHeaderTests : XCTestCase @@ -101,4 +103,138 @@ - (void)testSPTPersistentCacheRecordHeaderMake XCTAssertEqual(header.crc, SPTPersistentCacheCalculateHeaderCRC(&header)); } +- (void)testSPTPersistentCacheGetHeaderFromFileWithPath +{ + // GIVEN + NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:SPTCacheRecordFileName]; + [self removeFileAtPath:filePath]; + SPTPersistentCacheRecordHeader header = [self dummyHeader]; + XCTAssertNil([self createRecordAtPath:filePath withHeader:&header]); + + // WHEN + SPTPersistentCacheRecordHeader loadedHeader; + NSError* error = SPTPersistentCacheGetHeaderFromFileWithPath(filePath, &loadedHeader); + + // THEN + XCTAssertNil(error); + XCTAssertEqual(header.reserved1, loadedHeader.reserved1); + XCTAssertEqual(header.reserved2, loadedHeader.reserved2); + XCTAssertEqual(header.reserved3, loadedHeader.reserved3); + XCTAssertEqual(header.reserved4, loadedHeader.reserved4); + XCTAssertEqual(header.flags, loadedHeader.flags); + XCTAssertEqual(header.magic, loadedHeader.magic); + XCTAssertEqual(header.headerSize, loadedHeader.headerSize); + XCTAssertEqual(header.refCount, loadedHeader.refCount); + XCTAssertEqual(header.ttl, loadedHeader.ttl); + XCTAssertEqual(header.payloadSizeBytes, loadedHeader.payloadSizeBytes); + XCTAssertEqual(header.updateTimeSec, loadedHeader.updateTimeSec); + XCTAssertEqual(header.crc, loadedHeader.crc); + + [self removeFileAtPath:filePath]; +} + +- (void)testSPTPersistentCacheGetHeaderFromFileWithPathFailsWithError +{ + // GIVEN + NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:SPTCacheRecordFileName]; + + // WHEN + SPTPersistentCacheRecordHeader loadedHeader; + NSError* error = SPTPersistentCacheGetHeaderFromFileWithPath(filePath, &loadedHeader); + + // THEN + XCTAssertNotNil(error); +} + +- (void)testSPTPersistentCacheGetHeaderFromFileWithPathLegacy +{ + // GIVEN + SPTPersistentCacheRecordHeader legacyHeader = [self dummyHeader]; + NSData* payload = [[NSMutableData dataWithLength:legacyHeader.payloadSizeBytes] copy]; + NSMutableData* rawData = [NSMutableData dataWithBytes:&legacyHeader length:SPTPersistentCacheRecordHeaderSize]; + [rawData appendData:payload]; + // create a record with legacy header (saved to the file with a payload). + NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:SPTCacheRecordFileName]; + [self removeFileAtPath:filePath]; + XCTAssertTrue([[NSFileManager defaultManager] createFileAtPath:filePath contents:rawData attributes:nil]); + + // WHEN + SPTPersistentCacheRecordHeader loadedHeader; + NSError* error = SPTPersistentCacheGetHeaderFromFileWithPath(filePath, &loadedHeader); + + // THEN + XCTAssertNil(error); + XCTAssertEqual(legacyHeader.reserved1, loadedHeader.reserved1); + XCTAssertEqual(legacyHeader.reserved2, loadedHeader.reserved2); + XCTAssertEqual(legacyHeader.reserved3, loadedHeader.reserved3); + XCTAssertEqual(legacyHeader.reserved4, loadedHeader.reserved4); + XCTAssertEqual(legacyHeader.flags, loadedHeader.flags); + XCTAssertEqual(legacyHeader.magic, loadedHeader.magic); + XCTAssertEqual(legacyHeader.headerSize, loadedHeader.headerSize); + XCTAssertEqual(legacyHeader.refCount, loadedHeader.refCount); + XCTAssertEqual(legacyHeader.ttl, loadedHeader.ttl); + XCTAssertEqual(legacyHeader.payloadSizeBytes, loadedHeader.payloadSizeBytes); + XCTAssertEqual(legacyHeader.updateTimeSec, loadedHeader.updateTimeSec); + XCTAssertEqual(legacyHeader.crc, loadedHeader.crc); + + [self removeFileAtPath:filePath]; +} + +- (void)testSPTPersistentCacheSetHeaderForFileWithPath +{ + // GIVEN + NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:SPTCacheRecordFileName]; + [self removeFileAtPath:filePath]; + SPTPersistentCacheRecordHeader header = [self dummyHeader]; + + // WHEN + NSError* error = [self createRecordAtPath:filePath withHeader:&header]; + + // THEN + XCTAssertNil(error); + + [self removeFileAtPath:filePath]; +} + +- (void)testSPTPersistentCacheSetHeaderForFileWithPathFailsWithError +{ + // GIVEN + NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:SPTCacheRecordFileName]; + + // WHEN + SPTPersistentCacheRecordHeader loadedHeader; + NSError* error = SPTPersistentCacheSetHeaderForFileWithPath(filePath, &loadedHeader); + + // THEN + XCTAssertNotNil(error); +} + +#pragma mark - Private + +- (NSError*)createRecordAtPath:(NSString*)filePath withHeader:(SPTPersistentCacheRecordHeader*)header +{ + NSMutableData* data = [NSMutableData dataWithLength:header->payloadSizeBytes]; + + XCTAssertTrue([[NSFileManager defaultManager] createFileAtPath:filePath contents:data attributes:nil]); + return SPTPersistentCacheSetHeaderForFileWithPath(filePath, header); +} + +- (void)removeFileAtPath:(NSString*)filePath +{ + if ([[NSFileManager defaultManager] fileExistsAtPath:filePath]) { + NSError* error = nil; + [[NSFileManager defaultManager] removeItemAtPath:filePath error:&error]; + XCTAssertNil(error); + } +} + +- (SPTPersistentCacheRecordHeader)dummyHeader { + uint64_t ttl = 64; + uint64_t payloadSize = 400; + uint64_t updateTime = spt_uint64rint([[NSDate date] timeIntervalSince1970]); + BOOL isLocked = YES; + + return SPTPersistentCacheRecordHeaderMake(ttl, payloadSize, updateTime, isLocked); +} + @end diff --git a/Tests/SPTPersistentCacheTests.m b/Tests/SPTPersistentCacheTests.m index fdcd96d..73a0656 100644 --- a/Tests/SPTPersistentCacheTests.m +++ b/Tests/SPTPersistentCacheTests.m @@ -110,7 +110,7 @@ static NSUInteger params_GetCorruptedFilesNumber(void); static NSUInteger params_GetDefaultExpireFilesNumber(void); -static BOOL spt_test_ReadHeaderForFile(const char* path, BOOL validate, SPTPersistentCacheRecordHeader *header); +static BOOL spt_test_ReadHeaderForFile(NSString* path, BOOL validate, SPTPersistentCacheRecordHeader *header); typedef NSTimeInterval (^SPTPersistentCacheCurrentTimeSecCallback)(void); @@ -805,7 +805,7 @@ - (void)testLockedSize NSString *fileName = [self.thisBundle pathForResource:self.imageNames[i] ofType:nil]; NSData *data = [NSData dataWithContentsOfFile:fileName]; XCTAssertNotNil(data, @"Data must be valid"); - expectedSize += ([data length] + (NSUInteger)SPTPersistentCacheRecordHeaderSize); + expectedSize += [data length]; } } @@ -1092,7 +1092,7 @@ - (void)testRegularGCWithTTL NSString *path = [fileManager pathForKey:self.imageNames[i]]; SPTPersistentCacheRecordHeader header; - BOOL opened = spt_test_ReadHeaderForFile(path.UTF8String, YES, &header); + BOOL opened = spt_test_ReadHeaderForFile(path, YES, &header); if (kParams[i].locked) { ++lockedCount; XCTAssertTrue(opened, @"Locked files expected to be at place"); @@ -1139,7 +1139,7 @@ - (void)testPruneWithSizeRestriction for (unsigned i = 0; i < count && i < dropCount; ++i) { NSUInteger size = [self dataSizeForItem:self.imageNames[i]]; - expectedSize -= (size + (NSUInteger)SPTPersistentCacheRecordHeaderSize); + expectedSize -= size; [savedItems addObject:self.imageNames[i]]; } @@ -1170,7 +1170,7 @@ - (void)testPruneWithSizeRestriction NSString *path = [fileManager pathForKey:savedItems[i]]; SPTPersistentCacheRecordHeader header; - BOOL opened = spt_test_ReadHeaderForFile(path.UTF8String, YES, &header); + BOOL opened = spt_test_ReadHeaderForFile(path, YES, &header); XCTAssertTrue(opened, @"Saved files expected to in place"); } @@ -1211,7 +1211,7 @@ - (void)testSerialStoreWithLockDoesntIncrementRefCount // Check data SPTPersistentCacheRecordHeader header; - XCTAssertTrue(spt_test_ReadHeaderForFile(path.UTF8String, YES, &header), @"Expect valid record"); + XCTAssertTrue(spt_test_ReadHeaderForFile(path, YES, &header), @"Expect valid record"); XCTAssertEqual(header.ttl, kTTL1, @"TTL must match"); XCTAssertEqual(header.refCount, 1u, @"refCount must match"); @@ -1221,7 +1221,7 @@ - (void)testSerialStoreWithLockDoesntIncrementRefCount [self waitForExpectationsWithTimeout:kDefaultWaitTime handler:nil]; // Check data - XCTAssertTrue(spt_test_ReadHeaderForFile(path.UTF8String, YES, &header), @"Expect valid record"); + XCTAssertTrue(spt_test_ReadHeaderForFile(path, YES, &header), @"Expect valid record"); XCTAssertEqual(header.ttl, kTTL2, @"TTL must match"); XCTAssertEqual(header.refCount, 1u, @"refCount must match"); @@ -1231,7 +1231,7 @@ - (void)testSerialStoreWithLockDoesntIncrementRefCount [self waitForExpectationsWithTimeout:kDefaultWaitTime handler:nil]; // Check data - XCTAssertTrue(spt_test_ReadHeaderForFile(path.UTF8String, YES, &header), @"Expect valid record"); + XCTAssertTrue(spt_test_ReadHeaderForFile(path, YES, &header), @"Expect valid record"); XCTAssertEqual(header.ttl, kTTL1, @"TTL must match"); XCTAssertEqual(header.refCount, 0u, @"refCount must match"); } @@ -1834,29 +1834,10 @@ - (void)putFile:(NSString *)file - (void)corruptFile:(NSString *)filePath pdcError:(int)pdcError { - int flags = O_RDWR; - if (pdcError == SPTPersistentCacheLoadingErrorNotEnoughDataToGetHeader) { - flags |= O_TRUNC; - } - - int fd = open([filePath UTF8String], flags); - if (fd == -1) { - XCTAssert(fd != -1, @"Could not open file while trying to simulate corruption"); - return; - } - SPTPersistentCacheRecordHeader header; memset(&header, 0, (size_t)SPTPersistentCacheRecordHeaderSize); - - if (pdcError != SPTPersistentCacheLoadingErrorNotEnoughDataToGetHeader) { - - ssize_t readSize = read(fd, &header, (size_t)SPTPersistentCacheRecordHeaderSize); - if (readSize != (ssize_t)SPTPersistentCacheRecordHeaderSize) { - XCTAssert(readSize == (ssize_t)SPTPersistentCacheRecordHeaderSize, @"Header not read"); - close(fd); - return; - } - } + NSError* headerGetError = SPTPersistentCacheGetHeaderFromFileWithPath(filePath, &header); + XCTAssertNil(headerGetError); NSUInteger headerSize = (NSUInteger)SPTPersistentCacheRecordHeaderSize; @@ -1887,43 +1868,31 @@ - (void)corruptFile:(NSString *)filePath NSAssert(NO, @"Gotcha!"); break; } - - off_t ret = lseek(fd, SEEK_SET, 0); - XCTAssert(ret != -1); - - ssize_t written = write(fd, &header, headerSize); - XCTAssert(written == (ssize_t)headerSize, @"header was not written"); - fsync(fd); - close(fd); + if (pdcError == SPTPersistentCacheLoadingErrorNotEnoughDataToGetHeader) { + // this error makes sense for legacy header only. + // here we remove the cached file if any and create a new one with truncated header. + NSMutableData* data = [NSMutableData dataWithBytes:&header length:headerSize]; + NSError* error = nil; + [[NSFileManager defaultManager] removeItemAtPath:filePath error:&error]; + XCTAssertNil(error); + XCTAssertTrue([[NSFileManager defaultManager] createFileAtPath:filePath contents:data attributes:nil]); + } else { + NSError* headerSetError = SPTPersistentCacheSetHeaderForFileWithPath(filePath, &header); + XCTAssertNil(headerSetError); + } } - (void)alterUpdateTime:(uint64_t)updateTime forFileAtPath:(NSString *)filePath { - int fd = open([filePath UTF8String], O_RDWR); - if (fd == -1) { - XCTAssert(fd != -1, @"Could open file for altering"); - return; - } - SPTPersistentCacheRecordHeader header; - memset(&header, 0, (size_t)SPTPersistentCacheRecordHeaderSize); - - ssize_t readSize = read(fd, &header, (size_t)SPTPersistentCacheRecordHeaderSize); - if (readSize != (ssize_t)SPTPersistentCacheRecordHeaderSize) { - close(fd); - return; - } + NSError* headerGetError = SPTPersistentCacheGetHeaderFromFileWithPath(filePath, &header); + XCTAssertNil(headerGetError); header.updateTimeSec = updateTime; header.crc = SPTPersistentCacheCalculateHeaderCRC(&header); - off_t ret = lseek(fd, SEEK_SET, 0); - XCTAssert(ret != -1); - - ssize_t written = write(fd, &header, (size_t)SPTPersistentCacheRecordHeaderSize); - XCTAssert(written == (ssize_t)SPTPersistentCacheRecordHeaderSize, @"header was not written"); - fsync(fd); - close(fd); + NSError* headerSetError = SPTPersistentCacheSetHeaderForFileWithPath(filePath, &header); + XCTAssertNil(headerSetError); } - (SPTPersistentCacheForUnitTests *)createCacheWithTimeCallback:(SPTPersistentCacheCurrentTimeSecCallback)currentTime @@ -1972,7 +1941,7 @@ - (void)checkUpdateTimeForFileAtPath:(NSString *)path validate:(BOOL)validate re XCTAssertNotNil(path, @"Path is nil"); SPTPersistentCacheRecordHeader header; if (validate) { - XCTAssertTrue(spt_test_ReadHeaderForFile(path.UTF8String, validate, &header), @"Unable to read and validate header"); + XCTAssertTrue(spt_test_ReadHeaderForFile(path, validate, &header), @"Unable to read and validate header"); timeCheck(header.updateTimeSec); } } @@ -1993,7 +1962,7 @@ - (NSUInteger)calculateExpectedSize if (kParams[i].corruptReason == SPTPersistentCacheLoadingErrorNotEnoughDataToGetHeader) { expectedSize += kCorruptedFileSize; } else { - expectedSize += ([self dataSizeForItem:self.imageNames[i]] + (NSUInteger)SPTPersistentCacheRecordHeaderSize); + expectedSize += ([self dataSizeForItem:self.imageNames[i]]); } } @@ -2002,22 +1971,14 @@ - (NSUInteger)calculateExpectedSize @end -static BOOL spt_test_ReadHeaderForFile(const char* path, BOOL validate, SPTPersistentCacheRecordHeader *header) +static BOOL spt_test_ReadHeaderForFile(NSString* path, BOOL validate, SPTPersistentCacheRecordHeader *header) { - int fd = open(path, O_RDONLY); - if (fd == -1) { + NSError* headerGetError = SPTPersistentCacheGetHeaderFromFileWithPath(path, header); + if (headerGetError != nil) { return NO; } assert(header != NULL); - memset(header, 0, (size_t)SPTPersistentCacheRecordHeaderSize); - - ssize_t readSize = read(fd, header, (size_t)SPTPersistentCacheRecordHeaderSize); - close(fd); - - if (readSize != (ssize_t)SPTPersistentCacheRecordHeaderSize) { - return NO; - } if (validate && SPTPersistentCacheValidateHeader(header) != -1) { return NO; diff --git a/Viewer/MainWindowController.m b/Viewer/MainWindowController.m index df93f8d..07c1d94 100644 --- a/Viewer/MainWindowController.m +++ b/Viewer/MainWindowController.m @@ -139,22 +139,17 @@ - (void)tableViewSelectionDidChange:(NSNotification *)aNotification return; } - NSString *fullFilePath = [self.cacheFiles objectAtIndex:idx]; - NSData *rawData = [NSData dataWithContentsOfFile:fullFilePath]; + NSString *fullFilePath = [[self.cacheFiles objectAtIndex:idx] path]; - SPTPersistentCacheRecordHeader *h = SPTPersistentCacheGetHeaderFromData(__DECONST(void*, [rawData bytes]), [rawData length]); + SPTPersistentCacheRecordHeader header; + NSError* headerGetError = SPTPersistentCacheGetHeaderFromFileWithPath(fullFilePath, &header); - if (h == NULL) { - // TODO: error + if (headerGetError != nil) { + [NSApp presentError:headerGetError]; return; } - if (-1 != SPTPersistentCacheValidateHeader(h)) { - // TODO: error - return; - } - - memcpy(&_currHeader, h, SPTPersistentCacheRecordHeaderSize); + memcpy(&_currHeader, &header, SPTPersistentCacheRecordHeaderSize); self.magic = [NSString stringWithFormat:@"0x%X", _currHeader.magic]; self.headerSize = [NSString stringWithFormat:@"%u", _currHeader.headerSize]; @@ -167,8 +162,7 @@ - (void)tableViewSelectionDidChange:(NSNotification *)aNotification dateStyle:NSDateFormatterMediumStyle timeStyle:NSDateFormatterLongStyle]; - NSRange payloadRange = NSMakeRange(SPTPersistentCacheRecordHeaderSize, _currHeader.payloadSizeBytes); - self.payload = [rawData subdataWithRange:payloadRange]; + self.payload = [NSData dataWithContentsOfFile:fullFilePath]; self.object = [[NSImage alloc] initWithData:self.payload]; diff --git a/include/SPTPersistentCache/SPTPersistentCache.h b/include/SPTPersistentCache/SPTPersistentCache.h index cca1eb3..1b3d7b2 100644 --- a/include/SPTPersistentCache/SPTPersistentCache.h +++ b/include/SPTPersistentCache/SPTPersistentCache.h @@ -73,6 +73,10 @@ typedef NS_ENUM(NSInteger, SPTPersistentCacheLoadingError) { * Record is opened as stream and busy right now. */ SPTPersistentCacheLoadingErrorRecordIsStreamAndBusy, + /** + * The file attribute operation(read or write) failed. + */ + SPTPersistentCacheLoadingErrorFileAttributeOperationFail, /** * Something bad has happened that shouldn't. */ diff --git a/include/SPTPersistentCache/SPTPersistentCacheHeader.h b/include/SPTPersistentCache/SPTPersistentCacheHeader.h index 01dc3f6..6ced5ff 100644 --- a/include/SPTPersistentCache/SPTPersistentCacheHeader.h +++ b/include/SPTPersistentCache/SPTPersistentCacheHeader.h @@ -80,6 +80,17 @@ FOUNDATION_EXPORT SPTPersistentCacheRecordHeader SPTPersistentCacheRecordHeaderM * Function return pointer to header if there are enough data otherwise NULL. */ FOUNDATION_EXPORT SPTPersistentCacheRecordHeader *SPTPersistentCacheGetHeaderFromData(void *data, size_t size); +/** + * Function loads a header from a given file. + * @return nil if everything is ok, NSError object otherwise. + */ +FOUNDATION_EXPORT NSError* SPTPersistentCacheGetHeaderFromFileWithPath(NSString *filepath, SPTPersistentCacheRecordHeader *header); +/** + * Function saves a header to a given file. + * @return nil if everything is ok, NSError object otherwise. + */ +FOUNDATION_EXPORT NSError* SPTPersistentCacheSetHeaderForFileWithPath(NSString *filepath, const SPTPersistentCacheRecordHeader *header); + /** * Function validates header accoring to predefined rules used in production code. * @return -1 if everything is ok, otherwise one of codes from SPTPersistentCacheLoadingError.