diff --git a/.gitignore b/.gitignore index 9e5f1a97..3f01686f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ bp/BPVersion.h bluepill/src/BPTestReportHTML.h result.txt +result_bptestinspector.txt # Xcode # diff --git a/BPSampleApp/BPLogicTests-Info.plist b/BPSampleApp/BPLogicTests-Info.plist new file mode 100644 index 00000000..6c6c23c4 --- /dev/null +++ b/BPSampleApp/BPLogicTests-Info.plist @@ -0,0 +1,22 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + BNDL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + + diff --git a/BPSampleApp/BPSampleApp.xcodeproj/project.pbxproj b/BPSampleApp/BPSampleApp.xcodeproj/project.pbxproj index be65cc0a..3b429830 100644 --- a/BPSampleApp/BPSampleApp.xcodeproj/project.pbxproj +++ b/BPSampleApp/BPSampleApp.xcodeproj/project.pbxproj @@ -24,6 +24,11 @@ BAFCCA601E36DC2000E33C31 /* BPSampleAppUITests.m in Sources */ = {isa = PBXBuildFile; fileRef = BAFCCA5F1E36DC2000E33C31 /* BPSampleAppUITests.m */; }; E492360122EF61F300395D98 /* BPSampleAppMoarTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E492360022EF61F300395D98 /* BPSampleAppMoarTests.m */; }; E4F8A34326F3B1AD00FE1267 /* BPSampleAppNewTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E4F8A33B26F3B12F00FE1267 /* BPSampleAppNewTests.m */; }; + FBBBD90029A06B7B002B9115 /* BPLogicTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBBBD8F129A06B47002B9115 /* BPLogicTests.m */; }; + FBC526962B341A6400EECA73 /* libBPTestInspector.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = FBC526952B341A6400EECA73 /* libBPTestInspector.dylib */; }; + FBD772B52A33E15D0098CEFD /* BPPassingLogicTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBD772B12A33E0620098CEFD /* BPPassingLogicTests.m */; }; + FBD772C52A3483310098CEFD /* SwiftLogicTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = FBD772C32A34832D0098CEFD /* SwiftLogicTests.swift */; }; + FBD772C72A378F440098CEFD /* BPBulkLogicTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBD772C62A378F440098CEFD /* BPBulkLogicTests.m */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -115,6 +120,14 @@ E492360022EF61F300395D98 /* BPSampleAppMoarTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPSampleAppMoarTests.m; sourceTree = ""; }; E4F8A33B26F3B12F00FE1267 /* BPSampleAppNewTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPSampleAppNewTests.m; sourceTree = ""; }; E4F8A34A26F3B1AD00FE1267 /* BPSampleAppNewTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = BPSampleAppNewTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + FBBBD8F129A06B47002B9115 /* BPLogicTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPLogicTests.m; sourceTree = ""; }; + FBBBD8FE29A06B54002B9115 /* BPLogicTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = BPLogicTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + FBC526952B341A6400EECA73 /* libBPTestInspector.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; path = libBPTestInspector.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; + FBD772B12A33E0620098CEFD /* BPPassingLogicTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPPassingLogicTests.m; sourceTree = ""; }; + FBD772BC2A33E15D0098CEFD /* BPPassingLogicTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = BPPassingLogicTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + FBD772BD2A33E15D0098CEFD /* BPLogicTests-Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; name = "BPLogicTests-Info.plist"; path = "/Users/lthrockm/ios/bluepill/bluepill/BPSampleApp/BPLogicTests-Info.plist"; sourceTree = ""; }; + FBD772C32A34832D0098CEFD /* SwiftLogicTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftLogicTests.swift; sourceTree = ""; }; + FBD772C62A378F440098CEFD /* BPBulkLogicTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPBulkLogicTests.m; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -143,6 +156,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + FBC526962B341A6400EECA73 /* libBPTestInspector.dylib in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -174,6 +188,20 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + FBBBD8F929A06B54002B9115 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + FBD772B72A33E15D0098CEFD /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ @@ -210,6 +238,7 @@ isa = PBXGroup; children = ( BAB24F301DB5D45E00867756 /* BPSampleApp */, + FBBBD8F029A06B47002B9115 /* LogicTests */, BAB24F4A1DB5D45E00867756 /* BPSampleAppTests */, BAB24F5B1DB5D83C00867756 /* BPAppNegativeTests */, BAA4DA381DC3C02B00A58BCC /* BPSampleAppCrashingTests */, @@ -217,6 +246,8 @@ BA9C2DD31DD7F182007CB967 /* BPSampleAppFatalErrorTests */, BAFCCA5E1E36DC2000E33C31 /* BPSampleAppUITests */, BAB24F2F1DB5D45E00867756 /* Products */, + FBD772BD2A33E15D0098CEFD /* BPLogicTests-Info.plist */, + FBC526942B341A6400EECA73 /* Frameworks */, ); sourceTree = ""; }; @@ -231,6 +262,8 @@ BA9C2DD21DD7F182007CB967 /* BPSampleAppFatalErrorTests.xctest */, BAFCCA5D1E36DC2000E33C31 /* BPSampleAppUITests.xctest */, E4F8A34A26F3B1AD00FE1267 /* BPSampleAppNewTests.xctest */, + FBBBD8FE29A06B54002B9115 /* BPLogicTests.xctest */, + FBD772BC2A33E15D0098CEFD /* BPPassingLogicTests.xctest */, ); name = Products; sourceTree = ""; @@ -291,6 +324,41 @@ path = BPSampleAppUITests; sourceTree = ""; }; + FBBBD8F029A06B47002B9115 /* LogicTests */ = { + isa = PBXGroup; + children = ( + FBD772C82A378F6F0098CEFD /* BPLogicTests */, + FBD772C92A378F7E0098CEFD /* BPPassingLogicTests */, + ); + path = LogicTests; + sourceTree = ""; + }; + FBC526942B341A6400EECA73 /* Frameworks */ = { + isa = PBXGroup; + children = ( + FBC526952B341A6400EECA73 /* libBPTestInspector.dylib */, + ); + name = Frameworks; + sourceTree = ""; + }; + FBD772C82A378F6F0098CEFD /* BPLogicTests */ = { + isa = PBXGroup; + children = ( + FBBBD8F129A06B47002B9115 /* BPLogicTests.m */, + ); + path = BPLogicTests; + sourceTree = ""; + }; + FBD772C92A378F7E0098CEFD /* BPPassingLogicTests */ = { + isa = PBXGroup; + children = ( + FBD772B12A33E0620098CEFD /* BPPassingLogicTests.m */, + FBD772C62A378F440098CEFD /* BPBulkLogicTests.m */, + FBD772C32A34832D0098CEFD /* SwiftLogicTests.swift */, + ); + path = BPPassingLogicTests; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ @@ -437,6 +505,40 @@ productReference = E4F8A34A26F3B1AD00FE1267 /* BPSampleAppNewTests.xctest */; productType = "com.apple.product-type.bundle.unit-test"; }; + FBBBD8F229A06B54002B9115 /* BPLogicTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = FBBBD8FB29A06B54002B9115 /* Build configuration list for PBXNativeTarget "BPLogicTests" */; + buildPhases = ( + FBBBD8F529A06B54002B9115 /* Sources */, + FBBBD8F929A06B54002B9115 /* Frameworks */, + FBBBD8FA29A06B54002B9115 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = BPLogicTests; + productName = BPSampleAppTests; + productReference = FBBBD8FE29A06B54002B9115 /* BPLogicTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; + FBD772B32A33E15D0098CEFD /* BPPassingLogicTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = FBD772B92A33E15D0098CEFD /* Build configuration list for PBXNativeTarget "BPPassingLogicTests" */; + buildPhases = ( + FBD772B42A33E15D0098CEFD /* Sources */, + FBD772B72A33E15D0098CEFD /* Frameworks */, + FBD772B82A33E15D0098CEFD /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = BPPassingLogicTests; + productName = BPSampleAppTests; + productReference = FBD772BC2A33E15D0098CEFD /* BPPassingLogicTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ @@ -491,6 +593,12 @@ E4F8A33D26F3B1AD00FE1267 = { DevelopmentTeam = 57Y47U492U; }; + FBBBD8F229A06B54002B9115 = { + DevelopmentTeam = 57Y47U492U; + }; + FBD772B32A33E15D0098CEFD = { + DevelopmentTeam = 57Y47U492U; + }; }; }; buildConfigurationList = BAB24F291DB5D45E00867756 /* Build configuration list for PBXProject "BPSampleApp" */; @@ -514,6 +622,8 @@ BA9C2DD11DD7F182007CB967 /* BPSampleAppFatalErrorTests */, BAFCCA5C1E36DC2000E33C31 /* BPSampleAppUITests */, E4F8A33D26F3B1AD00FE1267 /* BPSampleAppNewTests */, + FBBBD8F229A06B54002B9115 /* BPLogicTests */, + FBD772B32A33E15D0098CEFD /* BPPassingLogicTests */, ); }; /* End PBXProject section */ @@ -579,6 +689,20 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + FBBBD8FA29A06B54002B9115 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + FBD772B82A33E15D0098CEFD /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -651,6 +775,24 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + FBBBD8F529A06B54002B9115 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + FBBBD90029A06B7B002B9115 /* BPLogicTests.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + FBD772B42A33E15D0098CEFD /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + FBD772C52A3483310098CEFD /* SwiftLogicTests.swift in Sources */, + FBD772B52A33E15D0098CEFD /* BPPassingLogicTests.m in Sources */, + FBD772C72A378F440098CEFD /* BPBulkLogicTests.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ @@ -891,6 +1033,7 @@ DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; ENABLE_NS_ASSERTIONS = NO; ENABLE_STRICT_OBJC_MSGSEND = YES; + "EXCLUDED_ARCHS[sdk=*]" = arm64; GCC_C_LANGUAGE_STANDARD = gnu99; GCC_NO_COMMON_BLOCKS = YES; GCC_WARN_64_TO_32_BIT_CONVERSION = YES; @@ -1057,6 +1200,70 @@ }; name = Release; }; + FBBBD8FC29A06B54002B9115 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + DEVELOPMENT_TEAM = 57Y47U492U; + INFOPLIST_FILE = "BPSampleAppTests-Info.plist"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = LI.BPSampleAppTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "BPSampleAppTests/BPSampleAppTests-Bridging-Header.h"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_SWIFT3_OBJC_INFERENCE = Default; + SWIFT_VERSION = 4.2; + }; + name = Debug; + }; + FBBBD8FD29A06B54002B9115 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + DEVELOPMENT_TEAM = 57Y47U492U; + INFOPLIST_FILE = "BPSampleAppTests-Info.plist"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = LI.BPSampleAppTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "BPSampleAppTests/BPSampleAppTests-Bridging-Header.h"; + SWIFT_SWIFT3_OBJC_INFERENCE = Default; + SWIFT_VERSION = 4.2; + }; + name = Release; + }; + FBD772BA2A33E15D0098CEFD /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + DEVELOPMENT_TEAM = 57Y47U492U; + "EXCLUDED_ARCHS[sdk=iphonesimulator*]" = ""; + INFOPLIST_FILE = "BPLogicTests-Info.plist"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = LI.BPSampleAppTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "BPSampleAppTests/BPSampleAppTests-Bridging-Header.h"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_SWIFT3_OBJC_INFERENCE = Default; + SWIFT_VERSION = 4.2; + }; + name = Debug; + }; + FBD772BB2A33E15D0098CEFD /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + DEVELOPMENT_TEAM = 57Y47U492U; + "EXCLUDED_ARCHS[sdk=iphonesimulator*]" = ""; + INFOPLIST_FILE = "BPLogicTests-Info.plist"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; + PRODUCT_BUNDLE_IDENTIFIER = LI.BPSampleAppTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_OBJC_BRIDGING_HEADER = "BPSampleAppTests/BPSampleAppTests-Bridging-Header.h"; + SWIFT_SWIFT3_OBJC_INFERENCE = Default; + SWIFT_VERSION = 4.2; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -1141,6 +1348,24 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + FBBBD8FB29A06B54002B9115 /* Build configuration list for PBXNativeTarget "BPLogicTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + FBBBD8FC29A06B54002B9115 /* Debug */, + FBBBD8FD29A06B54002B9115 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + FBD772B92A33E15D0098CEFD /* Build configuration list for PBXNativeTarget "BPPassingLogicTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + FBD772BA2A33E15D0098CEFD /* Debug */, + FBD772BB2A33E15D0098CEFD /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; /* End XCConfigurationList section */ }; rootObject = BAB24F261DB5D45E00867756 /* Project object */; diff --git a/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPLogicTests.xcscheme b/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPLogicTests.xcscheme new file mode 100644 index 00000000..03201efc --- /dev/null +++ b/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPLogicTests.xcscheme @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPPassingLogicTests.xcscheme b/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPPassingLogicTests.xcscheme new file mode 100644 index 00000000..5625c7da --- /dev/null +++ b/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPPassingLogicTests.xcscheme @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPSampleApp.xcscheme b/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPSampleApp.xcscheme index 6028fa6b..86fbfb16 100644 --- a/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPSampleApp.xcscheme +++ b/BPSampleApp/BPSampleApp.xcodeproj/xcshareddata/xcschemes/BPSampleApp.xcscheme @@ -104,6 +104,34 @@ ReferencedContainer = "container:BPSampleApp.xcodeproj"> + + + + + + + + + + + + + + + + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + BNDL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + + diff --git a/BPSampleApp/LogicTests/BPLogicTests/BPLogicTests.m b/BPSampleApp/LogicTests/BPLogicTests/BPLogicTests.m new file mode 100644 index 00000000..3618c788 --- /dev/null +++ b/BPSampleApp/LogicTests/BPLogicTests/BPLogicTests.m @@ -0,0 +1,81 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import + +@interface BPLogicTests : XCTestCase + +@end + +@implementation BPLogicTests + +- (void)testPassingLogicTest1 { + XCTAssert(YES); +} + +- (void)testPassingLogicTest2 { + XCTAssert(YES); +} + +- (void)testFailingLogicTest { + XCTAssert(NO); +} + +/* + This failure should be recognized as a test failure + in the xctest logs, and testing should be able to continue. + */ +- (void)testCrashTestCaseLogicTest { + NSLog(@"BPLogicTests - FORCING TEST EXECUTION CRASH."); + NSObject *unused = @[][666]; +} + +/* + This failure will cause the whole execution to fail, and + requires separate special handling. + */ +- (void)testCrashExecutionLogicTest { + NSLog(@"BPLogicTests - FORCING SIMULATOR CRASH."); + char *p = NULL; + strcpy(p, "I know this will crash my app"); +} + +- (void)testStuckLogicTest { + NSLog(@"BPLogicTests - FORCING TEST TIMEOUT"); + while(1) { + sleep(10); + } +} + +- (void)testSlowLogicTest { + NSLog(@"BPLogicTests - FORCING TEST TIMEOUT"); + while(1) { + NSLog(@"Look I'm trying, but to no avail!"); + sleep(1); + } +} + +// The below should not timeout when run in succession + +- (void)testOneSecondTest1 { + sleep(1); + XCTAssert(YES); +} + +- (void)testOneSecondTest2 { + sleep(1); + XCTAssert(YES); +} + +- (void)testOneSecondTest3 { + sleep(1); + XCTAssert(YES); +} + +@end diff --git a/BPSampleApp/LogicTests/BPPassingLogicTests/BPBulkLogicTests.m b/BPSampleApp/LogicTests/BPPassingLogicTests/BPBulkLogicTests.m new file mode 100644 index 00000000..dab12a07 --- /dev/null +++ b/BPSampleApp/LogicTests/BPPassingLogicTests/BPBulkLogicTests.m @@ -0,0 +1,219 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import + +@interface BPBulkLogicTests : XCTestCase + +@end + +@implementation BPBulkLogicTests + +- (void)testCase000 { XCTAssert(YES);} +- (void)testCase001 { XCTAssert(YES);} +- (void)testCase002 { XCTAssert(YES);} +- (void)testCase003 { XCTAssert(YES);} +- (void)testCase004 { XCTAssert(YES);} +- (void)testCase005 { XCTAssert(YES);} +- (void)testCase006 { XCTAssert(YES);} +- (void)testCase007 { XCTAssert(YES);} +- (void)testCase008 { XCTAssert(YES);} +- (void)testCase009 { XCTAssert(YES);} +- (void)testCase010 { XCTAssert(YES);} +- (void)testCase011 { XCTAssert(YES);} +- (void)testCase012 { XCTAssert(YES);} +- (void)testCase013 { XCTAssert(YES);} +- (void)testCase014 { XCTAssert(YES);} +- (void)testCase015 { XCTAssert(YES);} +- (void)testCase016 { XCTAssert(YES);} +- (void)testCase017 { XCTAssert(YES);} +- (void)testCase018 { XCTAssert(YES);} +- (void)testCase019 { XCTAssert(YES);} +- (void)testCase020 { XCTAssert(YES);} +- (void)testCase021 { XCTAssert(YES);} +- (void)testCase022 { XCTAssert(YES);} +- (void)testCase023 { XCTAssert(YES);} +- (void)testCase024 { XCTAssert(YES);} +- (void)testCase025 { XCTAssert(YES);} +- (void)testCase026 { XCTAssert(YES);} +- (void)testCase027 { XCTAssert(YES);} +- (void)testCase028 { XCTAssert(YES);} +- (void)testCase029 { XCTAssert(YES);} +- (void)testCase030 { XCTAssert(YES);} +- (void)testCase031 { XCTAssert(YES);} +- (void)testCase032 { XCTAssert(YES);} +- (void)testCase033 { XCTAssert(YES);} +- (void)testCase034 { XCTAssert(YES);} +- (void)testCase035 { XCTAssert(YES);} +- (void)testCase036 { XCTAssert(YES);} +- (void)testCase037 { XCTAssert(YES);} +- (void)testCase038 { XCTAssert(YES);} +- (void)testCase039 { XCTAssert(YES);} +- (void)testCase040 { XCTAssert(YES);} +- (void)testCase041 { XCTAssert(YES);} +- (void)testCase042 { XCTAssert(YES);} +- (void)testCase043 { XCTAssert(YES);} +- (void)testCase044 { XCTAssert(YES);} +- (void)testCase045 { XCTAssert(YES);} +- (void)testCase046 { XCTAssert(YES);} +- (void)testCase047 { XCTAssert(YES);} +- (void)testCase048 { XCTAssert(YES);} +- (void)testCase049 { XCTAssert(YES);} +- (void)testCase050 { XCTAssert(YES);} +- (void)testCase051 { XCTAssert(YES);} +- (void)testCase052 { XCTAssert(YES);} +- (void)testCase053 { XCTAssert(YES);} +- (void)testCase054 { XCTAssert(YES);} +- (void)testCase055 { XCTAssert(YES);} +- (void)testCase056 { XCTAssert(YES);} +- (void)testCase057 { XCTAssert(YES);} +- (void)testCase058 { XCTAssert(YES);} +- (void)testCase059 { XCTAssert(YES);} +- (void)testCase060 { XCTAssert(YES);} +- (void)testCase061 { XCTAssert(YES);} +- (void)testCase062 { XCTAssert(YES);} +- (void)testCase063 { XCTAssert(YES);} +- (void)testCase064 { XCTAssert(YES);} +- (void)testCase065 { XCTAssert(YES);} +- (void)testCase066 { XCTAssert(YES);} +- (void)testCase067 { XCTAssert(YES);} +- (void)testCase068 { XCTAssert(YES);} +- (void)testCase069 { XCTAssert(YES);} +- (void)testCase070 { XCTAssert(YES);} +- (void)testCase071 { XCTAssert(YES);} +- (void)testCase072 { XCTAssert(YES);} +- (void)testCase073 { XCTAssert(YES);} +- (void)testCase074 { XCTAssert(YES);} +- (void)testCase075 { XCTAssert(YES);} +- (void)testCase076 { XCTAssert(YES);} +- (void)testCase077 { XCTAssert(YES);} +- (void)testCase078 { XCTAssert(YES);} +- (void)testCase079 { XCTAssert(YES);} +- (void)testCase080 { XCTAssert(YES);} +- (void)testCase081 { XCTAssert(YES);} +- (void)testCase082 { XCTAssert(YES);} +- (void)testCase083 { XCTAssert(YES);} +- (void)testCase084 { XCTAssert(YES);} +- (void)testCase085 { XCTAssert(YES);} +- (void)testCase086 { XCTAssert(YES);} +- (void)testCase087 { XCTAssert(YES);} +- (void)testCase088 { XCTAssert(YES);} +- (void)testCase089 { XCTAssert(YES);} +- (void)testCase090 { XCTAssert(YES);} +- (void)testCase091 { XCTAssert(YES);} +- (void)testCase092 { XCTAssert(YES);} +- (void)testCase093 { XCTAssert(YES);} +- (void)testCase094 { XCTAssert(YES);} +- (void)testCase095 { XCTAssert(YES);} +- (void)testCase096 { XCTAssert(YES);} +- (void)testCase097 { XCTAssert(YES);} +- (void)testCase098 { XCTAssert(YES);} +- (void)testCase099 { XCTAssert(YES);} +- (void)testCase100 { XCTAssert(YES);} +- (void)testCase101 { XCTAssert(YES);} +- (void)testCase102 { XCTAssert(YES);} +- (void)testCase103 { XCTAssert(YES);} +- (void)testCase104 { XCTAssert(YES);} +- (void)testCase105 { XCTAssert(YES);} +- (void)testCase106 { XCTAssert(YES);} +- (void)testCase107 { XCTAssert(YES);} +- (void)testCase108 { XCTAssert(YES);} +- (void)testCase109 { XCTAssert(YES);} +- (void)testCase110 { XCTAssert(YES);} +- (void)testCase111 { XCTAssert(YES);} +- (void)testCase112 { XCTAssert(YES);} +- (void)testCase113 { XCTAssert(YES);} +- (void)testCase114 { XCTAssert(YES);} +- (void)testCase115 { XCTAssert(YES);} +- (void)testCase116 { XCTAssert(YES);} +- (void)testCase117 { XCTAssert(YES);} +- (void)testCase118 { XCTAssert(YES);} +- (void)testCase119 { XCTAssert(YES);} +- (void)testCase120 { XCTAssert(YES);} +- (void)testCase121 { XCTAssert(YES);} +- (void)testCase122 { XCTAssert(YES);} +- (void)testCase123 { XCTAssert(YES);} +- (void)testCase124 { XCTAssert(YES);} +- (void)testCase125 { XCTAssert(YES);} +- (void)testCase126 { XCTAssert(YES);} +- (void)testCase127 { XCTAssert(YES);} +- (void)testCase128 { XCTAssert(YES);} +- (void)testCase129 { XCTAssert(YES);} +- (void)testCase130 { XCTAssert(YES);} +- (void)testCase131 { XCTAssert(YES);} +- (void)testCase132 { XCTAssert(YES);} +- (void)testCase133 { XCTAssert(YES);} +- (void)testCase134 { XCTAssert(YES);} +- (void)testCase135 { XCTAssert(YES);} +- (void)testCase136 { XCTAssert(YES);} +- (void)testCase137 { XCTAssert(YES);} +- (void)testCase138 { XCTAssert(YES);} +- (void)testCase139 { XCTAssert(YES);} +- (void)testCase140 { XCTAssert(YES);} +- (void)testCase141 { XCTAssert(YES);} +- (void)testCase142 { XCTAssert(YES);} +- (void)testCase143 { XCTAssert(YES);} +- (void)testCase144 { XCTAssert(YES);} +- (void)testCase145 { XCTAssert(YES);} +- (void)testCase146 { XCTAssert(YES);} +- (void)testCase147 { XCTAssert(YES);} +- (void)testCase148 { XCTAssert(YES);} +- (void)testCase149 { XCTAssert(YES);} +- (void)testCase150 { XCTAssert(YES);} +- (void)testCase151 { XCTAssert(YES);} +- (void)testCase152 { XCTAssert(YES);} +- (void)testCase153 { XCTAssert(YES);} +- (void)testCase154 { XCTAssert(YES);} +- (void)testCase155 { XCTAssert(YES);} +- (void)testCase156 { XCTAssert(YES);} +- (void)testCase157 { XCTAssert(YES);} +- (void)testCase158 { XCTAssert(YES);} +- (void)testCase159 { XCTAssert(YES);} +- (void)testCase160 { XCTAssert(YES);} +- (void)testCase161 { XCTAssert(YES);} +- (void)testCase162 { XCTAssert(YES);} +- (void)testCase163 { XCTAssert(YES);} +- (void)testCase164 { XCTAssert(YES);} +- (void)testCase165 { XCTAssert(YES);} +- (void)testCase166 { XCTAssert(YES);} +- (void)testCase167 { XCTAssert(YES);} +- (void)testCase168 { XCTAssert(YES);} +- (void)testCase169 { XCTAssert(YES);} +- (void)testCase170 { XCTAssert(YES);} +- (void)testCase171 { XCTAssert(YES);} +- (void)testCase172 { XCTAssert(YES);} +- (void)testCase173 { XCTAssert(YES);} +- (void)testCase174 { XCTAssert(YES);} +- (void)testCase175 { XCTAssert(YES);} +- (void)testCase176 { XCTAssert(YES);} +- (void)testCase177 { XCTAssert(YES);} +- (void)testCase178 { XCTAssert(YES);} +- (void)testCase179 { XCTAssert(YES);} +- (void)testCase180 { XCTAssert(YES);} +- (void)testCase181 { XCTAssert(YES);} +- (void)testCase182 { XCTAssert(YES);} +- (void)testCase183 { XCTAssert(YES);} +- (void)testCase184 { XCTAssert(YES);} +- (void)testCase185 { XCTAssert(YES);} +- (void)testCase186 { XCTAssert(YES);} +- (void)testCase187 { XCTAssert(YES);} +- (void)testCase188 { XCTAssert(YES);} +- (void)testCase189 { XCTAssert(YES);} +- (void)testCase190 { XCTAssert(YES);} +- (void)testCase191 { XCTAssert(YES);} +- (void)testCase192 { XCTAssert(YES);} +- (void)testCase193 { XCTAssert(YES);} +- (void)testCase194 { XCTAssert(YES);} +- (void)testCase195 { XCTAssert(YES);} +- (void)testCase196 { XCTAssert(YES);} +- (void)testCase197 { XCTAssert(YES);} +- (void)testCase198 { XCTAssert(YES);} +- (void)testCase199 { XCTAssert(YES);} + +@end diff --git a/BPSampleApp/LogicTests/BPPassingLogicTests/BPPassingLogicTests.m b/BPSampleApp/LogicTests/BPPassingLogicTests/BPPassingLogicTests.m new file mode 100644 index 00000000..ba270a0a --- /dev/null +++ b/BPSampleApp/LogicTests/BPPassingLogicTests/BPPassingLogicTests.m @@ -0,0 +1,34 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import + +@interface BPPassingLogicTests : XCTestCase + +@end + +@implementation BPPassingLogicTests + +- (void)testPassingLogicTest1 { + XCTAssert(YES); +} + +- (void)testPassingLogicTest2 { + XCTAssert(YES); +} + +- (void)testPassingLogicTest3 { + XCTAssert(YES); +} + +- (void)testPassingLogicTest4 { + XCTAssert(YES); +} + +@end diff --git a/BPSampleApp/LogicTests/BPPassingLogicTests/SwiftLogicTests.swift b/BPSampleApp/LogicTests/BPPassingLogicTests/SwiftLogicTests.swift new file mode 100644 index 00000000..09779d66 --- /dev/null +++ b/BPSampleApp/LogicTests/BPPassingLogicTests/SwiftLogicTests.swift @@ -0,0 +1,26 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +import XCTest + +final class SwiftLogicTests: XCTestCase { + + func testPassingLogicTest1() { + XCTAssert(true) + } + + func testPassingLogicTest2() { + XCTAssert(true) + } + + func testPassingLogicTest3() { + XCTAssert(true) + } + +} diff --git a/BPSampleApp/LogicTests/Info.plist b/BPSampleApp/LogicTests/Info.plist new file mode 100644 index 00000000..6c6c23c4 --- /dev/null +++ b/BPSampleApp/LogicTests/Info.plist @@ -0,0 +1,22 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + BNDL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + + diff --git a/BPTestInspector/BPTestCaseInfo.h b/BPTestInspector/BPTestCaseInfo.h new file mode 100644 index 00000000..462314ff --- /dev/null +++ b/BPTestInspector/BPTestCaseInfo.h @@ -0,0 +1,42 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + This class is a basic representation of an XCTestCase, with all the information required to + add the test to an include/exclude list when specifying what tests to run in an xctest execution. + + Notably, this class conforms to NSSecureCoding so that it can be encoded, piped out to a + parent execution, and then decoded. + */ +@interface BPTestCaseInfo : NSObject + +@property (nonatomic, copy, nonnull, readonly) NSString *className; +@property (nonatomic, copy, nonnull, readonly) NSString *methodName; + +/** + The name of the test, formatted correctly for XCTest (regardless of Obj-C vs Swift) + + @example `MyTestClass/MyTestCase` in Obj-C, or `MyTestModule.MyTestClass/MyTestCase` in Swift + */ +@property (nonatomic, copy, nonnull, readonly) NSString *standardizedFullName; +/** + The name of the test, formatted according to how BP consumers expect to list the test + in the opt-in or skip lists, or how they should be displayed in the generated test results. + + @example `MyTestClass/MyTestCase` in Obj-C, or `MyTestClass/MyTestCase()` in Swift + */ +@property (nonatomic, copy, nonnull, readonly) NSString *prettifiedFullName; + +@end + +NS_ASSUME_NONNULL_END diff --git a/BPTestInspector/BPTestCaseInfo.m b/BPTestInspector/BPTestCaseInfo.m new file mode 100644 index 00000000..ab5c4c42 --- /dev/null +++ b/BPTestInspector/BPTestCaseInfo.m @@ -0,0 +1,81 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import "BPTestCaseInfo+Internal.h" +#import "XCTestCase.h" + +@implementation BPTestCaseInfo + +#pragma mark - Initializers + +- (instancetype)initWithClassName:(NSString *)className + methodName:(NSString *)methodName { + if (self = [super init]) { + _className = [className copy]; + _methodName = [methodName copy]; + } + return self; +} + ++ (instancetype)infoFromTestCase:(XCTestCase *)testCase { + NSString *className = NSStringFromClass(testCase.class); + NSString *methodName = [testCase respondsToSelector:@selector(languageAgnosticTestMethodName)] ? [testCase languageAgnosticTestMethodName] : NSStringFromSelector([testCase.invocation selector]); + return [[BPTestCaseInfo alloc] initWithClassName:className methodName:methodName]; +} + +#pragma mark - Properties + +- (NSString *)standardizedFullName { + return [NSString stringWithFormat:@"%@/%@", self.className, self.methodName]; +} + +- (NSString *)prettifiedFullName { + /* + If the class name contains a `.`, this is a Swift test case, and needs extra formatting. + Otherwise, we in Obj-C, it's unchanged from the `standardizedFullName` + */ + NSArray *classComponents = [self.className componentsSeparatedByString:@"."]; + if (classComponents.count < 2) { + return self.standardizedFullName; + } + return [NSString stringWithFormat:@"%@/%@()", classComponents[1], self.methodName]; +} + +#pragma mark - Overrides + +- (NSString *)description { + return self.standardizedFullName; +} + +- (BOOL)isEqual:(id)object { + if (!object || ![object isMemberOfClass:BPTestCaseInfo.class]) { + return NO; + } + BPTestCaseInfo *other = (BPTestCaseInfo *)object; + return [self.className isEqual:other.className] && [self.methodName isEqual:other.methodName]; +} + +#pragma mark - NSSecureCoding + ++ (BOOL)supportsSecureCoding { + return YES; +} + +- (void)encodeWithCoder:(nonnull NSCoder *)coder { + [coder encodeObject:self.className forKey:@"className"]; + [coder encodeObject:self.methodName forKey:@"methodName"]; +} + +- (nullable instancetype)initWithCoder:(nonnull NSCoder *)coder { + return [self initWithClassName:[coder decodeObjectOfClass:NSString.class forKey:@"className"] + methodName:[coder decodeObjectOfClass:NSString.class forKey:@"methodName"]]; + +} + +@end diff --git a/BPTestInspector/BPTestInspector.xcodeproj/project.pbxproj b/BPTestInspector/BPTestInspector.xcodeproj/project.pbxproj new file mode 100644 index 00000000..30305b6d --- /dev/null +++ b/BPTestInspector/BPTestInspector.xcodeproj/project.pbxproj @@ -0,0 +1,443 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 54; + objects = { + +/* Begin PBXBuildFile section */ + FB21F07F2A62286400682AC7 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = FB21F07E2A62286400682AC7 /* main.m */; }; + FB21F0862A62297F00682AC7 /* BPLoggingUtils.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0812A62297F00682AC7 /* BPLoggingUtils.h */; }; + FB21F0872A62297F00682AC7 /* BPTestCaseInfo+Internal.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0822A62297F00682AC7 /* BPTestCaseInfo+Internal.h */; settings = {ATTRIBUTES = (Public, ); }; }; + FB21F0882A62297F00682AC7 /* BPXCTestUtils.m in Sources */ = {isa = PBXBuildFile; fileRef = FB21F0832A62297F00682AC7 /* BPXCTestUtils.m */; }; + FB21F0892A62297F00682AC7 /* BPLoggingUtils.m in Sources */ = {isa = PBXBuildFile; fileRef = FB21F0842A62297F00682AC7 /* BPLoggingUtils.m */; }; + FB21F08A2A62297F00682AC7 /* BPXCTestUtils.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0852A62297F00682AC7 /* BPXCTestUtils.h */; }; + FB21F08F2A62298B00682AC7 /* BPTestCaseInfo.m in Sources */ = {isa = PBXBuildFile; fileRef = FB21F08B2A62298A00682AC7 /* BPTestCaseInfo.m */; }; + FB21F0902A62298B00682AC7 /* BPTestCaseInfo.m in Sources */ = {isa = PBXBuildFile; fileRef = FB21F08B2A62298A00682AC7 /* BPTestCaseInfo.m */; }; + FB21F0912A62298B00682AC7 /* BPTestInspectorConstants.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F08C2A62298A00682AC7 /* BPTestInspectorConstants.h */; settings = {ATTRIBUTES = (Public, ); }; }; + FB21F0922A62298B00682AC7 /* BPTestInspectorConstants.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F08C2A62298A00682AC7 /* BPTestInspectorConstants.h */; settings = {ATTRIBUTES = (Public, ); }; }; + FB21F0932A62298B00682AC7 /* BPTestCaseInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F08D2A62298B00682AC7 /* BPTestCaseInfo.h */; settings = {ATTRIBUTES = (Public, ); }; }; + FB21F0942A62298B00682AC7 /* BPTestCaseInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F08D2A62298B00682AC7 /* BPTestCaseInfo.h */; settings = {ATTRIBUTES = (Public, ); }; }; + FB21F0952A62298B00682AC7 /* BPTestInspectorConstants.m in Sources */ = {isa = PBXBuildFile; fileRef = FB21F08E2A62298B00682AC7 /* BPTestInspectorConstants.m */; }; + FB21F0962A62298B00682AC7 /* BPTestInspectorConstants.m in Sources */ = {isa = PBXBuildFile; fileRef = FB21F08E2A62298B00682AC7 /* BPTestInspectorConstants.m */; }; + FB21F0A82A622A0200682AC7 /* XCTestObserver.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0982A622A0200682AC7 /* XCTestObserver.h */; }; + FB21F0A92A622A0200682AC7 /* XCTestSuiteRun.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0992A622A0200682AC7 /* XCTestSuiteRun.h */; }; + FB21F0AA2A622A0200682AC7 /* XCTestSuite.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F09A2A622A0200682AC7 /* XCTestSuite.h */; }; + FB21F0AB2A622A0200682AC7 /* CDStructures.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F09B2A622A0200682AC7 /* CDStructures.h */; }; + FB21F0AC2A622A0200682AC7 /* XCTestLog.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F09C2A622A0200682AC7 /* XCTestLog.h */; }; + FB21F0AD2A622A0200682AC7 /* XCTest.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F09D2A622A0200682AC7 /* XCTest.h */; }; + FB21F0AE2A622A0200682AC7 /* XCTestRun.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F09E2A622A0200682AC7 /* XCTestRun.h */; }; + FB21F0AF2A622A0200682AC7 /* XCTestCase.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F09F2A622A0200682AC7 /* XCTestCase.h */; }; + FB21F0B02A622A0200682AC7 /* XCTActivity-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0A12A622A0200682AC7 /* XCTActivity-Protocol.h */; }; + FB21F0B12A622A0200682AC7 /* XCTIssueHandling-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0A22A622A0200682AC7 /* XCTIssueHandling-Protocol.h */; }; + FB21F0B22A622A0200682AC7 /* XCTestObservation-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0A32A622A0200682AC7 /* XCTestObservation-Protocol.h */; }; + FB21F0B32A622A0200682AC7 /* _XCTestObservationPrivate-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0A42A622A0200682AC7 /* _XCTestObservationPrivate-Protocol.h */; }; + FB21F0B42A622A0200682AC7 /* XCTWaiterDelegate-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0A52A622A0200682AC7 /* XCTWaiterDelegate-Protocol.h */; }; + FB21F0B52A622A0200682AC7 /* NSObject-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0A62A622A0200682AC7 /* NSObject-Protocol.h */; }; + FB21F0B62A622A0200682AC7 /* _XCTestObservationInternal-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0A72A622A0200682AC7 /* _XCTestObservationInternal-Protocol.h */; }; + FB21F0B92A622AA000682AC7 /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = FB21F0B82A622AA000682AC7 /* XCTest.framework */; }; + FB21F0DF2A65106E00682AC7 /* BPTestCaseInfo+Internal.h in Headers */ = {isa = PBXBuildFile; fileRef = FB21F0822A62297F00682AC7 /* BPTestCaseInfo+Internal.h */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + AA017F411BD7776B00F45E9D /* libBPTestInspector.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = libBPTestInspector.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; + AA56EAE21EEF08720062C2BC /* libBPMacTestInspector.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = libBPMacTestInspector.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; + EED62ED820D12209006E86E5 /* BPMacTestInspector.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = BPMacTestInspector.xcconfig; sourceTree = ""; }; + EED62ED920D12209006E86E5 /* BPTestInspector.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = BPTestInspector.xcconfig; sourceTree = ""; }; + FB21F07E2A62286400682AC7 /* main.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; + FB21F0812A62297F00682AC7 /* BPLoggingUtils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPLoggingUtils.h; sourceTree = ""; }; + FB21F0822A62297F00682AC7 /* BPTestCaseInfo+Internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "BPTestCaseInfo+Internal.h"; sourceTree = ""; }; + FB21F0832A62297F00682AC7 /* BPXCTestUtils.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPXCTestUtils.m; sourceTree = ""; }; + FB21F0842A62297F00682AC7 /* BPLoggingUtils.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPLoggingUtils.m; sourceTree = ""; }; + FB21F0852A62297F00682AC7 /* BPXCTestUtils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPXCTestUtils.h; sourceTree = ""; }; + FB21F08B2A62298A00682AC7 /* BPTestCaseInfo.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPTestCaseInfo.m; sourceTree = ""; }; + FB21F08C2A62298A00682AC7 /* BPTestInspectorConstants.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPTestInspectorConstants.h; sourceTree = ""; }; + FB21F08D2A62298B00682AC7 /* BPTestCaseInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPTestCaseInfo.h; sourceTree = ""; }; + FB21F08E2A62298B00682AC7 /* BPTestInspectorConstants.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPTestInspectorConstants.m; sourceTree = ""; }; + FB21F0982A622A0200682AC7 /* XCTestObserver.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = XCTestObserver.h; sourceTree = ""; }; + FB21F0992A622A0200682AC7 /* XCTestSuiteRun.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = XCTestSuiteRun.h; sourceTree = ""; }; + FB21F09A2A622A0200682AC7 /* XCTestSuite.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = XCTestSuite.h; sourceTree = ""; }; + FB21F09B2A622A0200682AC7 /* CDStructures.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CDStructures.h; sourceTree = ""; }; + FB21F09C2A622A0200682AC7 /* XCTestLog.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = XCTestLog.h; sourceTree = ""; }; + FB21F09D2A622A0200682AC7 /* XCTest.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = XCTest.h; sourceTree = ""; }; + FB21F09E2A622A0200682AC7 /* XCTestRun.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = XCTestRun.h; sourceTree = ""; }; + FB21F09F2A622A0200682AC7 /* XCTestCase.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = XCTestCase.h; sourceTree = ""; }; + FB21F0A12A622A0200682AC7 /* XCTActivity-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCTActivity-Protocol.h"; sourceTree = ""; }; + FB21F0A22A622A0200682AC7 /* XCTIssueHandling-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCTIssueHandling-Protocol.h"; sourceTree = ""; }; + FB21F0A32A622A0200682AC7 /* XCTestObservation-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCTestObservation-Protocol.h"; sourceTree = ""; }; + FB21F0A42A622A0200682AC7 /* _XCTestObservationPrivate-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "_XCTestObservationPrivate-Protocol.h"; sourceTree = ""; }; + FB21F0A52A622A0200682AC7 /* XCTWaiterDelegate-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCTWaiterDelegate-Protocol.h"; sourceTree = ""; }; + FB21F0A62A622A0200682AC7 /* NSObject-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "NSObject-Protocol.h"; sourceTree = ""; }; + FB21F0A72A622A0200682AC7 /* _XCTestObservationInternal-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "_XCTestObservationInternal-Protocol.h"; sourceTree = ""; }; + FB21F0B82A622AA000682AC7 /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/iPhoneOS.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + AA017F3E1BD7776B00F45E9D /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + FB21F0B92A622AA000682AC7 /* XCTest.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + AA56EADF1EEF08720062C2BC /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + AA017F381BD7776B00F45E9D = { + isa = PBXGroup; + children = ( + FB21F08D2A62298B00682AC7 /* BPTestCaseInfo.h */, + FB21F08B2A62298A00682AC7 /* BPTestCaseInfo.m */, + FB21F08C2A62298A00682AC7 /* BPTestInspectorConstants.h */, + FB21F08E2A62298B00682AC7 /* BPTestInspectorConstants.m */, + FB21F0802A62296A00682AC7 /* Internal */, + EED62ED420D1212D006E86E5 /* Configs */, + AA017F421BD7776B00F45E9D /* Products */, + FB21F0B72A622AA000682AC7 /* Frameworks */, + ); + sourceTree = ""; + }; + AA017F421BD7776B00F45E9D /* Products */ = { + isa = PBXGroup; + children = ( + AA017F411BD7776B00F45E9D /* libBPTestInspector.dylib */, + AA56EAE21EEF08720062C2BC /* libBPMacTestInspector.dylib */, + ); + name = Products; + sourceTree = ""; + }; + EED62ED420D1212D006E86E5 /* Configs */ = { + isa = PBXGroup; + children = ( + EED62ED820D12209006E86E5 /* BPMacTestInspector.xcconfig */, + EED62ED920D12209006E86E5 /* BPTestInspector.xcconfig */, + ); + path = Configs; + sourceTree = ""; + }; + FB21F0802A62296A00682AC7 /* Internal */ = { + isa = PBXGroup; + children = ( + FB21F07E2A62286400682AC7 /* main.m */, + FB21F0812A62297F00682AC7 /* BPLoggingUtils.h */, + FB21F0842A62297F00682AC7 /* BPLoggingUtils.m */, + FB21F0822A62297F00682AC7 /* BPTestCaseInfo+Internal.h */, + FB21F0852A62297F00682AC7 /* BPXCTestUtils.h */, + FB21F0832A62297F00682AC7 /* BPXCTestUtils.m */, + FB21F0972A622A0200682AC7 /* PrivateHeaders */, + ); + path = Internal; + sourceTree = ""; + }; + FB21F0972A622A0200682AC7 /* PrivateHeaders */ = { + isa = PBXGroup; + children = ( + FB21F0982A622A0200682AC7 /* XCTestObserver.h */, + FB21F0992A622A0200682AC7 /* XCTestSuiteRun.h */, + FB21F09A2A622A0200682AC7 /* XCTestSuite.h */, + FB21F09B2A622A0200682AC7 /* CDStructures.h */, + FB21F09C2A622A0200682AC7 /* XCTestLog.h */, + FB21F09D2A622A0200682AC7 /* XCTest.h */, + FB21F09E2A622A0200682AC7 /* XCTestRun.h */, + FB21F09F2A622A0200682AC7 /* XCTestCase.h */, + FB21F0A02A622A0200682AC7 /* Protocols */, + ); + path = PrivateHeaders; + sourceTree = ""; + }; + FB21F0A02A622A0200682AC7 /* Protocols */ = { + isa = PBXGroup; + children = ( + FB21F0A12A622A0200682AC7 /* XCTActivity-Protocol.h */, + FB21F0A22A622A0200682AC7 /* XCTIssueHandling-Protocol.h */, + FB21F0A32A622A0200682AC7 /* XCTestObservation-Protocol.h */, + FB21F0A42A622A0200682AC7 /* _XCTestObservationPrivate-Protocol.h */, + FB21F0A52A622A0200682AC7 /* XCTWaiterDelegate-Protocol.h */, + FB21F0A62A622A0200682AC7 /* NSObject-Protocol.h */, + FB21F0A72A622A0200682AC7 /* _XCTestObservationInternal-Protocol.h */, + ); + path = Protocols; + sourceTree = ""; + }; + FB21F0B72A622AA000682AC7 /* Frameworks */ = { + isa = PBXGroup; + children = ( + FB21F0B82A622AA000682AC7 /* XCTest.framework */, + ); + name = Frameworks; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXHeadersBuildPhase section */ + AA017F3F1BD7776B00F45E9D /* Headers */ = { + isa = PBXHeadersBuildPhase; + buildActionMask = 2147483647; + files = ( + FB21F0B12A622A0200682AC7 /* XCTIssueHandling-Protocol.h in Headers */, + FB21F0AA2A622A0200682AC7 /* XCTestSuite.h in Headers */, + FB21F0AB2A622A0200682AC7 /* CDStructures.h in Headers */, + FB21F0B42A622A0200682AC7 /* XCTWaiterDelegate-Protocol.h in Headers */, + FB21F08A2A62297F00682AC7 /* BPXCTestUtils.h in Headers */, + FB21F0A92A622A0200682AC7 /* XCTestSuiteRun.h in Headers */, + FB21F0A82A622A0200682AC7 /* XCTestObserver.h in Headers */, + FB21F0AE2A622A0200682AC7 /* XCTestRun.h in Headers */, + FB21F0862A62297F00682AC7 /* BPLoggingUtils.h in Headers */, + FB21F0B22A622A0200682AC7 /* XCTestObservation-Protocol.h in Headers */, + FB21F0AC2A622A0200682AC7 /* XCTestLog.h in Headers */, + FB21F0B52A622A0200682AC7 /* NSObject-Protocol.h in Headers */, + FB21F0B32A622A0200682AC7 /* _XCTestObservationPrivate-Protocol.h in Headers */, + FB21F0B62A622A0200682AC7 /* _XCTestObservationInternal-Protocol.h in Headers */, + FB21F0AF2A622A0200682AC7 /* XCTestCase.h in Headers */, + FB21F0932A62298B00682AC7 /* BPTestCaseInfo.h in Headers */, + FB21F0AD2A622A0200682AC7 /* XCTest.h in Headers */, + FB21F0B02A622A0200682AC7 /* XCTActivity-Protocol.h in Headers */, + FB21F0872A62297F00682AC7 /* BPTestCaseInfo+Internal.h in Headers */, + FB21F0912A62298B00682AC7 /* BPTestInspectorConstants.h in Headers */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + AA56EAE01EEF08720062C2BC /* Headers */ = { + isa = PBXHeadersBuildPhase; + buildActionMask = 2147483647; + files = ( + FB21F0942A62298B00682AC7 /* BPTestCaseInfo.h in Headers */, + FB21F0DF2A65106E00682AC7 /* BPTestCaseInfo+Internal.h in Headers */, + FB21F0922A62298B00682AC7 /* BPTestInspectorConstants.h in Headers */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXHeadersBuildPhase section */ + +/* Begin PBXNativeTarget section */ + AA017F401BD7776B00F45E9D /* BPTestInspector */ = { + isa = PBXNativeTarget; + buildConfigurationList = AA017F451BD7776B00F45E9D /* Build configuration list for PBXNativeTarget "BPTestInspector" */; + buildPhases = ( + AA017F3D1BD7776B00F45E9D /* Sources */, + AA017F3E1BD7776B00F45E9D /* Frameworks */, + AA017F3F1BD7776B00F45E9D /* Headers */, + FBD60B072B33FF48005A5642 /* Print TestInspector Path */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = BPTestInspector; + productName = Shimulator; + productReference = AA017F411BD7776B00F45E9D /* libBPTestInspector.dylib */; + productType = "com.apple.product-type.library.dynamic"; + }; + AA56EAE11EEF08720062C2BC /* BPMacTestInspector */ = { + isa = PBXNativeTarget; + buildConfigurationList = AA56EAEA1EEF08720062C2BC /* Build configuration list for PBXNativeTarget "BPMacTestInspector" */; + buildPhases = ( + AA56EADE1EEF08720062C2BC /* Sources */, + AA56EADF1EEF08720062C2BC /* Frameworks */, + AA56EAE01EEF08720062C2BC /* Headers */, + 71B9E0DC268C7A4600D40A91 /* Print MacTestInspector Path */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = BPMacTestInspector; + productName = Maculator; + productReference = AA56EAE21EEF08720062C2BC /* libBPMacTestInspector.dylib */; + productType = "com.apple.product-type.library.dynamic"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + AA017F391BD7776B00F45E9D /* Project object */ = { + isa = PBXProject; + attributes = { + LastUpgradeCheck = 0700; + ORGANIZATIONNAME = Facebook; + TargetAttributes = { + AA017F401BD7776B00F45E9D = { + CreatedOnToolsVersion = 7.0.1; + }; + AA56EAE11EEF08720062C2BC = { + CreatedOnToolsVersion = 9.0; + }; + }; + }; + buildConfigurationList = AA017F3C1BD7776B00F45E9D /* Build configuration list for PBXProject "BPTestInspector" */; + compatibilityVersion = "Xcode 3.2"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = AA017F381BD7776B00F45E9D; + productRefGroup = AA017F421BD7776B00F45E9D /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + AA017F401BD7776B00F45E9D /* BPTestInspector */, + AA56EAE11EEF08720062C2BC /* BPMacTestInspector */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXShellScriptBuildPhase section */ + 71B9E0DC268C7A4600D40A91 /* Print MacTestInspector Path */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Print MacTestInspector Path"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "echo MAC_TEST_INSPECTOR_DYLIB_PATH=$BUILT_PRODUCTS_DIR/$FULL_PRODUCT_NAME\n"; + showEnvVarsInLog = 0; + }; + FBD60B072B33FF48005A5642 /* Print TestInspector Path */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Print TestInspector Path"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "echo TEST_INSPECTOR_DYLIB_PATH=$BUILT_PRODUCTS_DIR/$FULL_PRODUCT_NAME\n"; + showEnvVarsInLog = 0; + }; +/* End PBXShellScriptBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + AA017F3D1BD7776B00F45E9D /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + FB21F0882A62297F00682AC7 /* BPXCTestUtils.m in Sources */, + FB21F08F2A62298B00682AC7 /* BPTestCaseInfo.m in Sources */, + FB21F07F2A62286400682AC7 /* main.m in Sources */, + FB21F0892A62297F00682AC7 /* BPLoggingUtils.m in Sources */, + FB21F0952A62298B00682AC7 /* BPTestInspectorConstants.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + AA56EADE1EEF08720062C2BC /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + FB21F0962A62298B00682AC7 /* BPTestInspectorConstants.m in Sources */, + FB21F0902A62298B00682AC7 /* BPTestCaseInfo.m in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + AA017F431BD7776B00F45E9D /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_IDENTITY = "-"; + GCC_OPTIMIZATION_LEVEL = 0; + }; + name = Debug; + }; + AA017F441BD7776B00F45E9D /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CLANG_ENABLE_MODULES = YES; + CODE_SIGN_IDENTITY = "-"; + GCC_OPTIMIZATION_LEVEL = 0; + }; + name = Release; + }; + AA017F461BD7776B00F45E9D /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EED62ED920D12209006E86E5 /* BPTestInspector.xcconfig */; + buildSettings = { + SUPPORTED_PLATFORMS = "iphonesimulator iphoneos"; + }; + name = Debug; + }; + AA017F471BD7776B00F45E9D /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EED62ED920D12209006E86E5 /* BPTestInspector.xcconfig */; + buildSettings = { + SUPPORTED_PLATFORMS = "iphonesimulator iphoneos"; + }; + name = Release; + }; + AA56EAE81EEF08720062C2BC /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EED62ED820D12209006E86E5 /* BPMacTestInspector.xcconfig */; + buildSettings = { + }; + name = Debug; + }; + AA56EAE91EEF08720062C2BC /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = EED62ED820D12209006E86E5 /* BPMacTestInspector.xcconfig */; + buildSettings = { + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + AA017F3C1BD7776B00F45E9D /* Build configuration list for PBXProject "BPTestInspector" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + AA017F431BD7776B00F45E9D /* Debug */, + AA017F441BD7776B00F45E9D /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + AA017F451BD7776B00F45E9D /* Build configuration list for PBXNativeTarget "BPTestInspector" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + AA017F461BD7776B00F45E9D /* Debug */, + AA017F471BD7776B00F45E9D /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + AA56EAEA1EEF08720062C2BC /* Build configuration list for PBXNativeTarget "BPMacTestInspector" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + AA56EAE81EEF08720062C2BC /* Debug */, + AA56EAE91EEF08720062C2BC /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = AA017F391BD7776B00F45E9D /* Project object */; +} diff --git a/BPTestInspector/BPTestInspector.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/BPTestInspector/BPTestInspector.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 00000000..6291fb33 --- /dev/null +++ b/BPTestInspector/BPTestInspector.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/BPTestInspector/BPTestInspector.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/BPTestInspector/BPTestInspector.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 00000000..18d98100 --- /dev/null +++ b/BPTestInspector/BPTestInspector.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/BPTestInspector/BPTestInspector.xcodeproj/xcshareddata/xcschemes/BPMacTestInspector.xcscheme b/BPTestInspector/BPTestInspector.xcodeproj/xcshareddata/xcschemes/BPMacTestInspector.xcscheme new file mode 100644 index 00000000..c56ec4cc --- /dev/null +++ b/BPTestInspector/BPTestInspector.xcodeproj/xcshareddata/xcschemes/BPMacTestInspector.xcscheme @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/BPTestInspector/BPTestInspector.xcodeproj/xcshareddata/xcschemes/BPTestInspector.xcscheme b/BPTestInspector/BPTestInspector.xcodeproj/xcshareddata/xcschemes/BPTestInspector.xcscheme new file mode 100644 index 00000000..5e481eb8 --- /dev/null +++ b/BPTestInspector/BPTestInspector.xcodeproj/xcshareddata/xcschemes/BPTestInspector.xcscheme @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/BPTestInspector/BPTestInspectorConstants.h b/BPTestInspector/BPTestInspectorConstants.h new file mode 100644 index 00000000..1c675c3f --- /dev/null +++ b/BPTestInspector/BPTestInspectorConstants.h @@ -0,0 +1,22 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface BPTestInspectorConstants : NSObject + ++ (NSString *)dylibName; ++ (NSString *)testBundleEnvironmentKey; ++ (NSString *)outputPathEnvironmentKey; + +@end + +NS_ASSUME_NONNULL_END diff --git a/BPTestInspector/BPTestInspectorConstants.m b/BPTestInspector/BPTestInspectorConstants.m new file mode 100644 index 00000000..9026f145 --- /dev/null +++ b/BPTestInspector/BPTestInspectorConstants.m @@ -0,0 +1,26 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import "BPTestInspectorConstants.h" + +@implementation BPTestInspectorConstants + ++ (NSString *)dylibName { + return @"libBPTestInspector.dylib"; +} + ++ (NSString *)testBundleEnvironmentKey { + return @"BP_XCTEST_WRAPPER__LOGIC_TEST_BUNDLE"; +} + ++ (NSString *)outputPathEnvironmentKey { + return @"BP_XCTEST_WRAPPER__TEST_CASE_OUTPUT"; +} + +@end diff --git a/BPTestInspector/Configs/BPMacTestInspector.xcconfig b/BPTestInspector/Configs/BPMacTestInspector.xcconfig new file mode 100644 index 00000000..371f806f --- /dev/null +++ b/BPTestInspector/Configs/BPMacTestInspector.xcconfig @@ -0,0 +1,8 @@ +CURRENT_PROJECT_VERSION = 1 +DYLIB_CURRENT_VERSION = 1 +DYLIB_COMPATIBILITY_VERSION = 1 +PRODUCT_NAME = BPMacTestInspector +EXECUTABLE_PREFIX = lib +SDKROOT = macosx +MACOSX_DEPLOYMENT_TARGET = 11.0 +COPY_PHASE_STRIP = NO diff --git a/BPTestInspector/Configs/BPTestInspector.xcconfig b/BPTestInspector/Configs/BPTestInspector.xcconfig new file mode 100644 index 00000000..964e63f0 --- /dev/null +++ b/BPTestInspector/Configs/BPTestInspector.xcconfig @@ -0,0 +1,9 @@ +CURRENT_PROJECT_VERSION = 1 +DYLIB_CURRENT_VERSION = 1 +DYLIB_COMPATIBILITY_VERSION = 1 +PRODUCT_NAME = BPTestInspector +EXECUTABLE_PREFIX = lib +SDKROOT = iphonesimulator +IPHONEOS_DEPLOYMENT_TARGET = 16.0 +COPY_PHASE_STRIP = NO +CODE_SIGN_IDENTITY = - // LTHROCKM - may need code signing for hosted tests diff --git a/BPTestInspector/Internal/BPLoggingUtils.h b/BPTestInspector/Internal/BPLoggingUtils.h new file mode 100644 index 00000000..65efdbb8 --- /dev/null +++ b/BPTestInspector/Internal/BPLoggingUtils.h @@ -0,0 +1,28 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + A very basic logging wrapper to simplify logging info + errors within BPTestInspector + after it's been injected into an xctest execution. + + Currently, it just prints to console, though this could be improved in the future. + */ +@interface BPLoggingUtils : NSObject + ++ (void)log:(NSString *)message; + ++ (void)logError:(NSString *)errorMessage; + +@end + +NS_ASSUME_NONNULL_END diff --git a/BPTestInspector/Internal/BPLoggingUtils.m b/BPTestInspector/Internal/BPLoggingUtils.m new file mode 100644 index 00000000..a37bc9d1 --- /dev/null +++ b/BPTestInspector/Internal/BPLoggingUtils.m @@ -0,0 +1,22 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import "BPLoggingUtils.h" + +@implementation BPLoggingUtils + ++ (void)log:(NSString *)message { + NSLog(@"[BPTestInspector] %@", message); +} + ++ (void)logError:(NSString *)errorMessage { + NSLog(@"[BPTestInspector] Error: %@", errorMessage); +} + +@end diff --git a/BPTestInspector/Internal/BPTestCaseInfo+Internal.h b/BPTestInspector/Internal/BPTestCaseInfo+Internal.h new file mode 100644 index 00000000..d4992cfc --- /dev/null +++ b/BPTestInspector/Internal/BPTestCaseInfo+Internal.h @@ -0,0 +1,27 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import + +@class XCTestCase; + +NS_ASSUME_NONNULL_BEGIN + +@interface BPTestCaseInfo (Internal) + ++ (instancetype)infoFromTestCase:(XCTestCase *)testCase; + +#pragma mark - Testing + +- (instancetype)initWithClassName:(NSString *)className + methodName:(NSString *)methodName; + +@end + +NS_ASSUME_NONNULL_END diff --git a/BPTestInspector/Internal/BPXCTestUtils.h b/BPTestInspector/Internal/BPXCTestUtils.h new file mode 100644 index 00000000..d658a615 --- /dev/null +++ b/BPTestInspector/Internal/BPXCTestUtils.h @@ -0,0 +1,28 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface BPXCTestUtils : NSObject + +/** + Given a path to an .xctest test bundle, this method will encode all of the contained test info into an output file. + This information will be saved as data encoding for an `NSArray *`, stored + at the output path. + + @param bundlePath The path of the .xctest bundle + @param outputPath The path of the output file. + */ ++ (void)logAllTestsInBundleWithPath:(NSString *)bundlePath toFile:(NSString *)outputPath; + +@end + +NS_ASSUME_NONNULL_END diff --git a/BPTestInspector/Internal/BPXCTestUtils.m b/BPTestInspector/Internal/BPXCTestUtils.m new file mode 100644 index 00000000..0df05e25 --- /dev/null +++ b/BPTestInspector/Internal/BPXCTestUtils.m @@ -0,0 +1,98 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import "BPXCTestUtils.h" + +#import +#import "XCTestSuite.h" +#import "XCTestCase.h" +#import "BPLoggingUtils.h" +#import "BPTestCaseInfo+Internal.h" + +@implementation BPXCTestUtils + ++ (void)logAllTestsInBundleWithPath:(NSString *)bundlePath toFile:(NSString *)outputPath { + NSArray *testCases = [self enumerateTestCasesInBundleWithPath:bundlePath]; + + // Encode the test data + NSError *encodingError; + NSData *data = [NSKeyedArchiver archivedDataWithRootObject:testCases requiringSecureCoding:YES error:&encodingError]; + + // Write to file. + NSFileHandle *fileHandle = [NSFileHandle fileHandleForWritingAtPath:outputPath]; + [fileHandle writeData:data]; + [fileHandle closeFile]; + + [BPLoggingUtils log:[NSString stringWithFormat:@"Wrote to file: %@.", outputPath]]; +} + ++ (NSArray *)enumerateTestCasesInBundleWithPath:(NSString *)bundlePath { + NSBundle *bundle = [NSBundle bundleWithPath:bundlePath]; + if (!bundle || !bundle.executablePath) { + [BPLoggingUtils logError:[NSString stringWithFormat:@"Unable to get executable path from bundle: %@", bundle]]; + return @[]; + } + return [self enumerateTestCasesInBundle:bundle]; +} + ++ (NSArray *)enumerateTestCasesInBundle:(NSBundle *)bundle { + /** + We need to remove the XCTest preference before continuing so that + we can open the bundle without actually executing it. If you start + to see logs resembling test output, it means this hasn't been done. + + TLDR: opening an .xctest file will normally cause the linker to load + the XCTest framework, which will trigger `XCTestSuite.initialize` + and start running the tests. + */ + + [NSUserDefaults.standardUserDefaults removeObjectForKey:@"XCTest"]; + [NSUserDefaults.standardUserDefaults synchronize]; + + /** + We must actually open the test bundle so that all of the test cases are loaded into memory. + We use `dlopen` here instead of `NSBundle.loadAndReturnError` for more informative messages on error. + */ + if (dlopen(bundle.executablePath.UTF8String, RTLD_LAZY) == NULL) { + [BPLoggingUtils logError:[NSString stringWithFormat:@"Unable to open test bundle's executable path - %@", bundle.executablePath]]; + + [BPLoggingUtils logError:@"What's the error???"]; + fprintf(stderr, "%s\n", dlerror()); + return @[]; + } + + [NSUserDefaults.standardUserDefaults setObject:@"None" forKey:@"XCTest"]; + + /** + Note that `XCTestSuite`, `XCTestCase`, etc all subclass `XCTest`, so to enumerate all tests in the current + bundle, we'll want to start `XCTestSuite.allTests`, and expand out all nested `XCTestSuite`s, adding + the suite's `.tests` to our array. + */ + NSMutableArray *testList = [NSMutableArray array]; + NSMutableArray *queue = [@[XCTestSuite.allTests] mutableCopy]; + while (queue.count) { + XCTest *test = queue.firstObject; + [queue removeObjectAtIndex:0]; + // If it's another nested XCTestSuite, keep going deeper! + // If it's an XCTestCase, we've hit a leaf and can just add it to our `testList`. + if ([test isKindOfClass:XCTestSuite.class]) { + XCTestSuite *testSuite = (XCTestSuite *)test; + [queue addObjectsFromArray:testSuite.tests]; + } else if ([test isKindOfClass:XCTestCase.class]) { + XCTestCase *testCase = (XCTestCase *)test; + [BPLoggingUtils log:[NSString stringWithFormat:@"testCase: %@", testCase]]; + [testList addObject:[BPTestCaseInfo infoFromTestCase:testCase]]; + } else { + [BPLoggingUtils logError:@"Found a currently unhandled XCTest type while enumerating tests"]; + } + } + return testList; +} + +@end diff --git a/BPTestInspector/Internal/PrivateHeaders/CDStructures.h b/BPTestInspector/Internal/PrivateHeaders/CDStructures.h new file mode 100644 index 00000000..55bc3957 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/CDStructures.h @@ -0,0 +1,57 @@ +// +// Generated by class-dump 3.5 (64 bit). +// +// class-dump is Copyright (C) 1997-1998, 2000-2001, 2004-2013 by Steve Nygard. +// + +#pragma mark Blocks + +typedef void (^CDUnknownBlockType)(void); // return type and parameters are unknown + +#pragma mark Named Structures + +struct DTXMachMessage { + struct { + struct { + unsigned int _field1; + unsigned int _field2; + unsigned int _field3; + unsigned int _field4; + unsigned int _field5; + int _field6; + } _field1; + unsigned int _field2; + } _field1; + char _field2[32672]; + char _field3[68]; +}; + +//struct DTXMessageHeader { +// unsigned int _field1; +// unsigned int _field2; +// unsigned short _field3; +// unsigned short _field4; +// unsigned int _field5; +// struct DTXMessageRoutingInfo _field6; +//}; + +struct DTXMessageRoutingInfo { + unsigned int _field1; + unsigned int _field2; + unsigned int _field3; + unsigned int :1; + unsigned int :31; +}; + +struct __va_list_tag { + unsigned int _field1; + unsigned int _field2; + void *_field3; + void *_field4; +}; + +//struct mach_timebase_info { +// unsigned int numer; +// unsigned int denom; +//}; + diff --git a/BPTestInspector/Internal/PrivateHeaders/Protocols/NSObject-Protocol.h b/BPTestInspector/Internal/PrivateHeaders/Protocols/NSObject-Protocol.h new file mode 100644 index 00000000..3320b3c9 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/Protocols/NSObject-Protocol.h @@ -0,0 +1,33 @@ +// +// Generated by class-dump 3.5 (64 bit). +// +// class-dump is Copyright (C) 1997-1998, 2000-2001, 2004-2013 by Steve Nygard. +// + +@class NSString, Protocol; + +@protocol NSObject +@property(readonly, copy) NSString *description; +@property(readonly) Class superclass; +@property(readonly) unsigned long long hash; +- (struct _NSZone *)zone; +- (unsigned long long)retainCount; +- (id)autorelease; +- (oneway void)release; +- (id)retain; +- (BOOL)respondsToSelector:(SEL)arg1; +- (BOOL)conformsToProtocol:(Protocol *)arg1; +- (BOOL)isMemberOfClass:(Class)arg1; +- (BOOL)isKindOfClass:(Class)arg1; +- (BOOL)isProxy; +- (id)performSelector:(SEL)arg1 withObject:(id)arg2 withObject:(id)arg3; +- (id)performSelector:(SEL)arg1 withObject:(id)arg2; +- (id)performSelector:(SEL)arg1; +- (id)self; +- (Class)class; +- (BOOL)isEqual:(id)arg1; + +@optional +@property(readonly, copy) NSString *debugDescription; +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTActivity-Protocol.h b/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTActivity-Protocol.h new file mode 100644 index 00000000..74becde5 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTActivity-Protocol.h @@ -0,0 +1,15 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +@import Foundation; + +@class NSString, XCTAttachment; + +@protocol XCTActivity +@property(readonly, copy) NSString *name; +- (void)addAttachment:(XCTAttachment *)arg1; +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTIssueHandling-Protocol.h b/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTIssueHandling-Protocol.h new file mode 100644 index 00000000..4dc2a6e1 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTIssueHandling-Protocol.h @@ -0,0 +1,16 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +@import Foundation; + +@class XCTExpectedFailureContext, XCTIssue; + +@protocol XCTIssueHandling +- (void)expectFailureWithContext:(XCTExpectedFailureContext *)arg1 inBlock:(void (^)(void))arg2; +- (void)expectFailureWithContext:(XCTExpectedFailureContext *)arg1; +- (void)handleIssue:(XCTIssue *)arg1; +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTWaiterDelegate-Protocol.h b/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTWaiterDelegate-Protocol.h new file mode 100644 index 00000000..ca59b949 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTWaiterDelegate-Protocol.h @@ -0,0 +1,17 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +@import Foundation; + +@class NSArray, XCTWaiter, XCTestExpectation; + +@protocol XCTWaiterDelegate +- (void)nestedWaiter:(XCTWaiter *)arg1 wasInterruptedByTimedOutWaiter:(XCTWaiter *)arg2; +- (void)waiter:(XCTWaiter *)arg1 didFulfillInvertedExpectation:(XCTestExpectation *)arg2; +- (void)waiter:(XCTWaiter *)arg1 fulfillmentDidViolateOrderingConstraintsForExpectation:(XCTestExpectation *)arg2 requiredExpectation:(XCTestExpectation *)arg3; +- (void)waiter:(XCTWaiter *)arg1 didTimeoutWithUnfulfilledExpectations:(NSArray *)arg2; +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTestObservation-Protocol.h b/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTestObservation-Protocol.h new file mode 100644 index 00000000..46528735 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/Protocols/XCTestObservation-Protocol.h @@ -0,0 +1,27 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +//#import "NSObject-Protocol.h" + +@class NSBundle, NSString, XCTExpectedFailure, XCTIssue, XCTestCase, XCTestSuite; + +@protocol XCTestObservation + +@optional +- (void)testCase:(XCTestCase *)arg1 didFailWithDescription:(NSString *)arg2 inFile:(NSString *)arg3 atLine:(unsigned long long)arg4; +- (void)testSuite:(XCTestSuite *)arg1 didFailWithDescription:(NSString *)arg2 inFile:(NSString *)arg3 atLine:(unsigned long long)arg4; +- (void)testCaseDidFinish:(XCTestCase *)arg1; +- (void)testCase:(XCTestCase *)arg1 didRecordExpectedFailure:(XCTExpectedFailure *)arg2; +- (void)testCase:(XCTestCase *)arg1 didRecordIssue:(XCTIssue *)arg2; +- (void)testCaseWillStart:(XCTestCase *)arg1; +- (void)testSuiteDidFinish:(XCTestSuite *)arg1; +- (void)testSuite:(XCTestSuite *)arg1 didRecordExpectedFailure:(XCTExpectedFailure *)arg2; +- (void)testSuite:(XCTestSuite *)arg1 didRecordIssue:(XCTIssue *)arg2; +- (void)testSuiteWillStart:(XCTestSuite *)arg1; +- (void)testBundleDidFinish:(NSBundle *)arg1; +- (void)testBundleWillStart:(NSBundle *)arg1; +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/Protocols/_XCTestObservationInternal-Protocol.h b/BPTestInspector/Internal/PrivateHeaders/Protocols/_XCTestObservationInternal-Protocol.h new file mode 100644 index 00000000..0aee3569 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/Protocols/_XCTestObservationInternal-Protocol.h @@ -0,0 +1,17 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +#import "_XCTestObservationPrivate-Protocol.h" + +@class NSArray, NSNumber, NSString, XCTSourceCodeContext, XCTestCase, XCTestRun; + +@protocol _XCTestObservationInternal <_XCTestObservationPrivate> + +@optional +- (void)_testCase:(XCTestRun *)arg1 didMeasureValues:(NSArray *)arg2 forPerformanceMetricID:(NSString *)arg3 name:(NSString *)arg4 unitsOfMeasurement:(NSString *)arg5 baselineName:(NSString *)arg6 baselineAverage:(NSNumber *)arg7 maxPercentRegression:(NSNumber *)arg8 maxPercentRelativeStandardDeviation:(NSNumber *)arg9 maxRegression:(NSNumber *)arg10 maxStandardDeviation:(NSNumber *)arg11 file:(NSString *)arg12 line:(unsigned long long)arg13 polarity:(long long)arg14; +- (void)testCase:(XCTestCase *)arg1 wasSkippedWithDescription:(NSString *)arg2 sourceCodeContext:(XCTSourceCodeContext *)arg3; +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/Protocols/_XCTestObservationPrivate-Protocol.h b/BPTestInspector/Internal/PrivateHeaders/Protocols/_XCTestObservationPrivate-Protocol.h new file mode 100644 index 00000000..3dfc04d4 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/Protocols/_XCTestObservationPrivate-Protocol.h @@ -0,0 +1,17 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +#import "XCTestObservation-Protocol.h" + +@class XCActivityRecord, XCTContext; + +@protocol _XCTestObservationPrivate + +@optional +- (void)_context:(XCTContext *)arg1 didFinishActivity:(XCActivityRecord *)arg2; +- (void)_context:(XCTContext *)arg1 willStartActivity:(XCActivityRecord *)arg2; +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/XCTest.h b/BPTestInspector/Internal/PrivateHeaders/XCTest.h new file mode 100644 index 00000000..155a225b --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/XCTest.h @@ -0,0 +1,55 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +@import Foundation; + +#import "XCTIssueHandling-Protocol.h" + +@class NSString, XCTExpectedFailureContextManager, XCTTestIdentifier, XCTestObservationCenter, XCTestRun; + +@interface XCTest : NSObject +{ + XCTExpectedFailureContextManager *_expectedFailureContextManager; + XCTestRun *_testRun; + XCTestObservationCenter *_observationCenter; +} + +- (void)expectFailureWithContext:(id)arg1 inBlock:(id)arg2; +- (void)expectFailureWithContext:(id)arg1; +- (void)_checkForExpectedFailureMatchingIssue:(id)arg1; +- (id)expectedFailureContextManager; +- (void)handleIssue:(id)arg1; +- (long long)defaultExecutionOrderCompare:(id)arg1; +@property(readonly) NSString *nameForLegacyLogging; +@property(readonly) NSString *languageAgnosticTestMethodName; +@property(readonly) NSString *languageAgnosticTestClassName; +- (_Bool)tearDownWithError:(id *)arg1; +- (void)tearDown; +- (void)setUp; +- (_Bool)setUpWithError:(id *)arg1; +- (_Bool)_shouldRerunTest; +- (void)runTest; +- (void)performTest:(id)arg1; +@property(retain) XCTestObservationCenter *observationCenter; // @synthesize observationCenter=_observationCenter; +@property(readonly) XCTestRun *testRun; // @synthesize testRun=_testRun; +@property(readonly) Class testRunClass; +@property(readonly) Class _requiredTestRunBaseClass; +@property(readonly, copy) NSString *name; +@property(readonly) unsigned long long testCaseCount; +- (id)_duplicate; +@property(readonly) NSString *_methodNameForReporting; +@property(readonly) NSString *_classNameForReporting; +- (void)removeTestsWithIdentifierInSet:(id)arg1; + +// Remaining properties +@property(readonly, copy) NSString *debugDescription; +@property(readonly, copy) NSString *description; +@property(readonly) NSUInteger hash; +@property(readonly, getter=_identifier) XCTTestIdentifier *identifier; // @dynamic identifier; +@property(readonly) Class superclass; + +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/XCTestCase.h b/BPTestInspector/Internal/PrivateHeaders/XCTestCase.h new file mode 100644 index 00000000..89c788d2 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/XCTestCase.h @@ -0,0 +1,189 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +#import "XCTest.h" + +#import "XCTActivity-Protocol.h" +#import "XCTWaiterDelegate-Protocol.h" + +@class MXMInstrument, NSArray, NSDictionary, NSInvocation, NSMutableArray, NSMutableDictionary, NSMutableSet, NSObject, NSString, NSThread, XCTAttachmentManager, XCTIssue, XCTSkippedTestContext, XCTTestIdentifier, XCTWaiter, XCTestCaseRun; +@protocol OS_dispatch_source; + +@interface XCTestCase : XCTest +{ + _Bool _continueAfterFailure; + _Bool __preciseTimeoutsEnabled; + _Bool _isMeasuringMetrics; + _Bool __didMeasureMetrics; + _Bool __didStartMeasuring; + _Bool __didStopMeasuring; + _Bool _hasDequeuedTeardownBlocks; + _Bool _hasReportedFailuresToTestCaseRun; + _Bool _canHandleInterruptions; + _Bool _shouldHaltWhenReceivesControl; + _Bool _shouldSetShouldHaltWhenReceivesControl; + _Bool _hasAttemptedToCaptureScreenshotOnFailure; + XCTTestIdentifier *_identifier; + NSInvocation *_invocation; + double _executionTimeAllowance; + NSArray *_activePerformanceMetricIDs; + unsigned long long _startWallClockTime; + struct time_value _startUserTime; + struct time_value _startSystemTime; + unsigned long long _measuringIteration; + MXMInstrument *_instrument; + long long _runLoopNestingCount; + NSMutableArray *_teardownBlocks; + NSMutableArray *_primaryThreadBlocks; + XCTAttachmentManager *_attachmentManager; + NSDictionary *_activityAggregateStatistics; + NSObject *_timeoutSource; + unsigned long long _signpostID; + NSThread *_primaryThread; + NSMutableSet *_previousIssuesAssociatedWithSwiftErrors; + NSMutableArray *_enqueuedIssues; + NSMutableArray *_expectations; + XCTWaiter *_currentWaiter; + XCTSkippedTestContext *_skippedTestContext; + XCTestCaseRun *_testCaseRun; + NSMutableDictionary *__perfMetricsForID; +} + ++ (id)_baselineDictionary; ++ (_Bool)_treatMissingBaselinesAsTestFailures; ++ (id)defaultMeasureOptions; ++ (id)defaultMetrics; ++ (id)defaultPerformanceMetrics; ++ (_Bool)_reportPerformanceFailuresForLargeImprovements; ++ (id)testInvocations; ++ (_Bool)isInheritingTestCases; ++ (id)bundle; ++ (id)testCaseWithSelector:(SEL)arg1; ++ (_Bool)_isDiscoverable; ++ (id)testCaseWithInvocation:(id)arg1; ++ (void)tearDown; ++ (void)setUp; ++ (id)defaultTestSuite; ++ (id)allTestMethodInvocations; ++ (void)_allTestMethodInvocations:(id)arg1; ++ (id)testMethodInvocations; ++ (id)allSubclassesOutsideXCTest; ++ (id)allSubclasses; ++ (id)_allSubclasses; + +@property(retain) NSMutableDictionary *_perfMetricsForID; // @synthesize _perfMetricsForID=__perfMetricsForID; +@property(retain) XCTestCaseRun *testCaseRun; // @synthesize testCaseRun=_testCaseRun; +@property(nonatomic) _Bool shouldSetShouldHaltWhenReceivesControl; // @synthesize shouldSetShouldHaltWhenReceivesControl=_shouldSetShouldHaltWhenReceivesControl; +@property(nonatomic) _Bool shouldHaltWhenReceivesControl; // @synthesize shouldHaltWhenReceivesControl=_shouldHaltWhenReceivesControl; +@property(retain) NSThread *primaryThread; // @synthesize primaryThread=_primaryThread; +@property(copy) NSDictionary *activityAggregateStatistics; // @synthesize activityAggregateStatistics=_activityAggregateStatistics; +@property long long runLoopNestingCount; // @synthesize runLoopNestingCount=_runLoopNestingCount; +@property _Bool _didStopMeasuring; // @synthesize _didStopMeasuring=__didStopMeasuring; +@property _Bool _didStartMeasuring; // @synthesize _didStartMeasuring=__didStartMeasuring; +@property _Bool _didMeasureMetrics; // @synthesize _didMeasureMetrics=__didMeasureMetrics; +@property(nonatomic) _Bool _preciseTimeoutsEnabled; // @synthesize _preciseTimeoutsEnabled=__preciseTimeoutsEnabled; +@property(readonly) double _effectiveExecutionTimeAllowance; +- (void)_resetTimer; +- (void)_stopTimeoutTimer; +- (void)_startTimeoutTimer; +- (void)_exceededExecutionTimeAllowance; +@property unsigned long long maxDurationInMinutes; +@property double executionTimeAllowance; // @synthesize executionTimeAllowance=_executionTimeAllowance; +- (void)addAttachment:(id)arg1; +- (void)runActivityNamed:(id)arg1 inScope:(id)arg2; +- (void)startActivityWithTitle:(id)arg1 block:(id)arg2; +- (void)startActivityWithTitle:(id)arg1 type:(id)arg2 block:(id)arg3; +- (void)measureMetrics:(id)arg1 automaticallyStartMeasuring:(_Bool)arg2 forBlock:(id)arg3; +- (void)registerDefaultMetrics; +- (id)baselinesDictionaryForTest; +- (void)_logAndReportPerformanceMetrics:(id)arg1 perfMetricResultsForIDs:(id)arg2 withBaselinesForTest:(id)arg3; +- (void)_logAndReportPerformanceMetrics:(id)arg1 perfMetricResultsForIDs:(id)arg2 withBaselinesForTest:(id)arg3 defaultBaselinesForPerfMetricID:(id)arg4; +- (void)registerMetricID:(id)arg1 name:(id)arg2 unitString:(id)arg3 polarity:(long long)arg4; +- (void)registerMetricID:(id)arg1 name:(id)arg2 unitString:(id)arg3; +- (void)registerMetricID:(id)arg1 name:(id)arg2 unit:(id)arg3; +- (void)reportMetric:(id)arg1 reportFailures:(_Bool)arg2; +- (void)reportMeasurements:(id)arg1 forMetricID:(id)arg2 reportFailures:(_Bool)arg3; +- (void)_recordValues:(id)arg1 forPerformanceMetricID:(id)arg2 name:(id)arg3 unitsOfMeasurement:(id)arg4 baselineName:(id)arg5 baselineAverage:(id)arg6 maxPercentRegression:(id)arg7 maxPercentRelativeStandardDeviation:(id)arg8 maxRegression:(id)arg9 maxStandardDeviation:(id)arg10 file:(id)arg11 line:(unsigned long long)arg12 polarity:(long long)arg13; +- (void)measureWithMetrics:(id)arg1 options:(id)arg2 block:(id)arg3; +- (void)measureWithMetrics:(id)arg1 block:(id)arg2; +- (void)measureWithOptions:(id)arg1 block:(id)arg2; +- (void)measureBlock:(id)arg1; +- (void)stopMeasuring; +- (void)startMeasuring; +@property(readonly) id minimumOperatingSystemVersion; +- (void)_logMemoryGraphDataFromFilePath:(id)arg1 withTitle:(id)arg2; +- (void)_logMemoryGraphData:(id)arg1 withTitle:(id)arg2; +- (unsigned long long)numberOfTestIterationsForTestWithSelector:(SEL)arg1; +- (id)addAdditionalIterationsBasedOnOptions:(id)arg1; +- (void)afterTestIteration:(unsigned long long)arg1 selector:(SEL)arg2; +- (void)beforeTestIteration:(unsigned long long)arg1 selector:(SEL)arg2; +- (void)tearDownTestWithSelector:(SEL)arg1; +- (void)setUpTestWithSelector:(SEL)arg1; +- (void)_addTeardownBlock:(id)arg1; +- (void)addTeardownBlock:(id)arg1; +- (void)_purgeTeardownBlocks; +- (void)performTest:(id)arg1; +- (_Bool)_shouldRerunTest; +- (void)_reportFailuresAtFile:(id)arg1 line:(unsigned long long)arg2 forTestAssertionsInScope:(id)arg3; +- (void)invokeTest; +- (Class)testRunClass; +- (Class)_requiredTestRunBaseClass; +- (void)recordIssue:(id)arg1; +@property _Bool continueAfterFailure; // @synthesize continueAfterFailure=_continueAfterFailure; +@property(retain) NSInvocation *invocation; // @synthesize invocation=_invocation; +- (void)dealloc; +@property(readonly, copy) NSString *description; +- (_Bool)isEqual:(id)arg1; +- (long long)defaultExecutionOrderCompare:(id)arg1; +- (id)nameForLegacyLogging; +@property(readonly, copy) NSString *name; +- (id)languageAgnosticTestMethodName; +- (id)_identifier; +- (unsigned long long)testCaseCount; +- (id)bundle; +- (id)initWithSelector:(SEL)arg1; +- (id)_duplicate; +- (id)initWithInvocation:(id)arg1; +- (id)init; +- (void)removeUIInterruptionMonitor:(id)arg1; +- (id)addUIInterruptionMonitorWithDescription:(id)arg1 handler:(id)arg2; +- (void)nestedWaiter:(id)arg1 wasInterruptedByTimedOutWaiter:(id)arg2; +- (void)waiter:(id)arg1 didFulfillInvertedExpectation:(id)arg2; +- (void)waiter:(id)arg1 fulfillmentDidViolateOrderingConstraintsForExpectation:(id)arg2 requiredExpectation:(id)arg3; +- (void)waiter:(id)arg1 didTimeoutWithUnfulfilledExpectations:(id)arg2; +- (id)expectationForPredicate:(id)arg1 evaluatedWithObject:(id)arg2 handler:(id)arg3; +- (id)expectationForNotification:(id)arg1 object:(id)arg2 notificationCenter:(id)arg3 handler:(id)arg4; +- (id)expectationForNotification:(id)arg1 object:(id)arg2 handler:(id)arg3; +- (id)keyValueObservingExpectationForObject:(id)arg1 keyPath:(id)arg2 handler:(id)arg3; +- (id)keyValueObservingExpectationForObject:(id)arg1 keyPath:(id)arg2 expectedValue:(id)arg3; +- (id)expectationWithDescription:(id)arg1; +- (void)waitForExpectations:(id)arg1 timeout:(double)arg2 enforceOrder:(_Bool)arg3; +- (void)waitForExpectations:(id)arg1 timeout:(double)arg2; +- (void)waitForExpectationsWithTimeout:(double)arg1 handler:(id)arg2; +- (id)_expectationForDistributedNotification:(id)arg1 object:(id)arg2 handler:(id)arg3; +- (id)_expectationForDarwinNotification:(id)arg1; +- (void)recordFailureWithDescription:(id)arg1 inFile:(id)arg2 atLine:(unsigned long long)arg3 expected:(_Bool)arg4; +- (void)_interruptOrMarkForLaterInterruption; +- (_Bool)_caughtUnhandledDeveloperExceptionPermittingControlFlowInterruptions:(_Bool)arg1 caughtInterruptionException:(_Bool *)arg2 whileExecutingBlock:(id)arg3; +- (_Bool)_caughtUnhandledDeveloperExceptionPermittingControlFlowInterruptions:(_Bool)arg1 whileExecutingBlock:(id)arg2; +- (id)_issueWithFailureScreenshotAttachedToIssue:(id)arg1; +- (void)_handleIssue:(id)arg1; +- (void)_dequeueIssues; +- (void)_enqueueIssue:(id)arg1; +- (void)_recordIssue:(id)arg1; +- (_Bool)_isDuplicateOfIssueAssociatedWithSameSwiftError:(id)arg1; +- (void)expectFailureWithContext:(id)arg1; +@property(copy) XCTIssue *candidateIssueForCurrentThread; +- (id)_storageKeyForCandidateIssue; +- (void)handleIssue:(id)arg1; + +// Remaining properties +@property(readonly, copy) NSString *debugDescription; +@property(readonly) NSUInteger hash; +@property(readonly) Class superclass; + +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/XCTestLog.h b/BPTestInspector/Internal/PrivateHeaders/XCTestLog.h new file mode 100644 index 00000000..f9ebe091 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/XCTestLog.h @@ -0,0 +1,59 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +#import "XCTestObserver.h" + +#import "_XCTestObservationInternal-Protocol.h" +#import "CDStructures.h" + +@class NSFileHandle, NSString, XCTestSuite, XCTestCase; + +@interface XCTestLog : XCTestObserver <_XCTestObservationInternal> +{ +} + +// Currently Handled + +- (void)testSuiteWillStart:(XCTestSuite *)suite; +- (void)testSuiteDidFinish:(XCTestCase *)testCase; +- (void)testCaseWillStart:(XCTestCase *)testCase; +- (void)testCaseDidFinish:(XCTestCase *)testCase; +- (void)testCase:(XCTestCase *)testCase +didFailWithDescription:(NSString *)description + inFile:(NSString *)file + atLine:(NSUInteger)line; +- (void)testCase:(XCTestCase *)testCase +wasSkippedWithDescription:(NSString *)description + inFile:(NSString *)file + atLine:(NSUInteger)line; + +// TODO: @lthrockm - add support + +- (void)testSuite:(id)arg1 didRecordExpectedFailure:(id)arg2; +- (void)testSuite:(id)arg1 didFailWithDescription:(id)arg2 inFile:(id)arg3 atLine:(unsigned long long)arg4; + +// Other + ++ (id)_messageForTest:(id)arg1 didMeasureValues:(id)arg2 forPerformanceMetricID:(id)arg3 name:(id)arg4 unitsOfMeasurement:(id)arg5 baselineName:(id)arg6 baselineAverage:(id)arg7 maxPercentRegression:(id)arg8 maxPercentRelativeStandardDeviation:(id)arg9 maxRegression:(id)arg10 maxStandardDeviation:(id)arg11 file:(id)arg12 line:(unsigned long long)arg13 polarity:(long long)arg14; +- (void)_context:(id)arg1 willStartActivity:(id)arg2; +- (void)testCase:(id)arg1 didRecordExpectedFailure:(id)arg2; +- (void)_testCase:(id)arg1 didMeasureValues:(id)arg2 forPerformanceMetricID:(id)arg3 name:(id)arg4 unitsOfMeasurement:(id)arg5 baselineName:(id)arg6 baselineAverage:(id)arg7 maxPercentRegression:(id)arg8 maxPercentRelativeStandardDeviation:(id)arg9 maxRegression:(id)arg10 maxStandardDeviation:(id)arg11 file:(id)arg12 line:(unsigned long long)arg13 polarity:(long long)arg14; +- (void)_testWithName:(id)arg1 didRecordExpectedFailure:(id)arg2; +- (void)_testDidFail:(id)arg1 withDescription:(id)arg2 inFile:(id)arg3 atLine:(unsigned long long)arg4; +- (void)logTestMessage:(id)arg1; +- (void)testLogWithFormat:(id)arg1 arguments:(struct __va_list_tag [1])arg2; +- (void)testLogWithFormat:(id)arg1; +@property(readonly) NSFileHandle *logFileHandle; +- (id)dateFormatter; + +// Remaining properties +@property(readonly, copy) NSString *debugDescription; +@property(readonly, copy) NSString *description; +@property(readonly) NSUInteger hash; +@property(readonly) Class superclass; + +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/XCTestObserver.h b/BPTestInspector/Internal/PrivateHeaders/XCTestObserver.h new file mode 100644 index 00000000..3e09a3c3 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/XCTestObserver.h @@ -0,0 +1,23 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +#import + +@interface XCTestObserver : NSObject +{ +} + +- (void)testCaseDidFail:(id)arg1 withDescription:(id)arg2 inFile:(id)arg3 atLine:(unsigned long long)arg4; +- (void)testCaseDidStop:(id)arg1; +- (void)testCaseDidStart:(id)arg1; +- (void)testSuiteDidFail:(id)arg1 withDescription:(id)arg2 inFile:(id)arg3 atLine:(unsigned long long)arg4; +- (void)testSuiteDidStop:(id)arg1; +- (void)testSuiteDidStart:(id)arg1; +- (void)stopObserving; +- (void)startObserving; + +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/XCTestRun.h b/BPTestInspector/Internal/PrivateHeaders/XCTestRun.h new file mode 100644 index 00000000..223c7db6 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/XCTestRun.h @@ -0,0 +1,73 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +#import + +@class NSDate, XCTIssue, XCTest, XCTestObservationCenter; + +@interface XCTestRun : NSObject +{ + _Bool _hasBeenSkipped; + _Bool _hasStarted; + _Bool _hasStopped; + _Bool _hasExpectedFailure; + XCTest *_test; + unsigned long long _executionCount; + unsigned long long _failureCount; + unsigned long long _unexpectedExceptionCount; + double _startTimeInterval; + double _stopTimeInterval; + XCTIssue *_candidateIssue; + unsigned long long __runCount; + unsigned long long _executionCountBeforeCrash; + unsigned long long _skipCountBeforeCrash; + unsigned long long _expectedFailureCountBeforeCrash; + unsigned long long _failureCountBeforeCrash; + unsigned long long _unexpectedExceptionCountBeforeCrash; +} + ++ (id)testRunWithTest:(id)arg1; +//- (void).cxx_destruct; +@property unsigned long long unexpectedExceptionCountBeforeCrash; // @synthesize unexpectedExceptionCountBeforeCrash=_unexpectedExceptionCountBeforeCrash; +@property unsigned long long failureCountBeforeCrash; // @synthesize failureCountBeforeCrash=_failureCountBeforeCrash; +@property unsigned long long expectedFailureCountBeforeCrash; // @synthesize expectedFailureCountBeforeCrash=_expectedFailureCountBeforeCrash; +@property unsigned long long skipCountBeforeCrash; // @synthesize skipCountBeforeCrash=_skipCountBeforeCrash; +@property unsigned long long executionCountBeforeCrash; // @synthesize executionCountBeforeCrash=_executionCountBeforeCrash; +@property unsigned long long _runCount; // @synthesize _runCount=__runCount; +@property(retain) XCTIssue *candidateIssue; // @synthesize candidateIssue=_candidateIssue; +@property _Bool hasExpectedFailure; // @synthesize hasExpectedFailure=_hasExpectedFailure; +@property _Bool hasStopped; // @synthesize hasStopped=_hasStopped; +@property _Bool hasStarted; // @synthesize hasStarted=_hasStarted; +@property double stopTimeInterval; // @synthesize stopTimeInterval=_stopTimeInterval; +@property double startTimeInterval; // @synthesize startTimeInterval=_startTimeInterval; +@property _Bool hasBeenSkipped; // @synthesize hasBeenSkipped=_hasBeenSkipped; +@property unsigned long long unexpectedExceptionCount; // @synthesize unexpectedExceptionCount=_unexpectedExceptionCount; +@property unsigned long long failureCount; // @synthesize failureCount=_failureCount; +@property unsigned long long executionCount; // @synthesize executionCount=_executionCount; +@property(readonly) XCTest *test; // @synthesize test=_test; +- (void)_handleIssue:(id)arg1; +- (void)_recordIssue:(id)arg1; +- (void)recordIssue:(id)arg1; +- (void)recordExpectedFailure:(id)arg1; +- (void)recordSkipWithDescription:(id)arg1 sourceCodeContext:(id)arg2; +- (unsigned long long)expectedFailureCount; +@property(readonly) unsigned long long skipCount; +@property(readonly) _Bool hasSucceeded; +@property(readonly) unsigned long long testCaseCount; +@property(readonly) unsigned long long totalFailureCount; +- (void)stop; +- (void)start; +@property(readonly, copy) NSDate *stopDate; +@property(readonly, copy) NSDate *startDate; +@property(readonly) double testDuration; +@property(readonly) double totalDuration; +@property(readonly) XCTestObservationCenter *observationCenter; +- (id)description; +- (id)initWithTest:(id)arg1; +- (void)recordFailureWithDescription:(id)arg1 inFile:(id)arg2 atLine:(unsigned long long)arg3 expected:(_Bool)arg4; + +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/XCTestSuite.h b/BPTestInspector/Internal/PrivateHeaders/XCTestSuite.h new file mode 100644 index 00000000..2cb75449 --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/XCTestSuite.h @@ -0,0 +1,70 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +#import "XCTest.h" + +typedef void (^CDUnknownBlockType)(void); +typedef void (*CDUnknownFunctionPointerType)(void); + +@class NSArray, NSDictionary, NSMutableArray, NSMutableDictionary, NSString, XCTTestIdentifier, XCTestConfiguration; + +@interface XCTestSuite : XCTest +{ + XCTTestIdentifier *_identifier; + NSString *_name; + NSMutableArray *_mutableTests; + XCTestConfiguration *_testConfiguration; + NSMutableDictionary *_mutableActivityAggregateStatistics; +} + ++ (NSArray *)allTests; + ++ (id)testClassSuitesForTestIdentifiers:(id)arg1 skippingTestIdentifiers:(id)arg2 randomNumberGenerator:(CDUnknownBlockType)arg3; ++ (id)testSuiteForTestConfiguration:(id)arg1; ++ (id)defaultTestSuite; ++ (id)testSuiteForTestCaseClass:(Class)arg1; ++ (id)testSuiteForTestCaseWithName:(id)arg1; ++ (id)testSuiteForBundlePath:(id)arg1; ++ (id)suiteForBundleCache; ++ (void)invalidateCache; ++ (id)_suiteForBundleCache; ++ (id)emptyTestSuiteNamedFromPath:(id)arg1; ++ (id)testSuiteWithName:(id)arg1; + +@property(readonly, copy) NSArray *tests; + +//- (void).cxx_destruct; +@property(readonly) NSMutableDictionary *mutableActivityAggregateStatistics; // @synthesize mutableActivityAggregateStatistics=_mutableActivityAggregateStatistics; +@property(retain) XCTestConfiguration *testConfiguration; // @synthesize testConfiguration=_testConfiguration; +@property(retain) NSMutableArray *mutableTests; // @synthesize mutableTests=_mutableTests; +@property(copy) NSString *name; // @synthesize name=_name; +- (id)_identifier; +- (id)_initWithTestConfiguration:(id)arg1; +- (void)_applyRandomExecutionOrderingWithGenerator:(CDUnknownBlockType)arg1; +- (void)_sortTestsUsingDefaultExecutionOrdering; +- (long long)defaultExecutionOrderCompare:(id)arg1; +@property(readonly) NSDictionary *activityAggregateStatistics; +- (void)_mergeActivityStatistics:(id)arg1; +- (void)runTestBasedOnRerunPolicy:(id)arg1 testRun:(id)arg2; +- (void)performTest:(id)arg1; +- (void)_performProtectedSectionForTest:(id)arg1 testSection:(CDUnknownBlockType)arg2; +- (void)_recordUnexpectedFailureForTestRun:(id)arg1 description:(id)arg2 exception:(id)arg3; +- (void)handleIssue:(id)arg1; +- (void)recordFailureWithDescription:(id)arg1 inFile:(id)arg2 atLine:(unsigned long long)arg3 expected:(_Bool)arg4; +- (Class)testRunClass; +- (Class)_requiredTestRunBaseClass; +- (_Bool)isEqual:(id)arg1; +- (unsigned long long)testCaseCount; +- (void)setTests:(id)arg1; +- (void)addTest:(id)arg1; +- (id)_testSuiteWithIdentifier:(id)arg1; +- (id)description; +- (id)initWithName:(id)arg1; +- (id)init; +- (void)removeTestsWithIdentifierInSet:(id)arg1; + +@end + diff --git a/BPTestInspector/Internal/PrivateHeaders/XCTestSuiteRun.h b/BPTestInspector/Internal/PrivateHeaders/XCTestSuiteRun.h new file mode 100644 index 00000000..56c3c18b --- /dev/null +++ b/BPTestInspector/Internal/PrivateHeaders/XCTestSuiteRun.h @@ -0,0 +1,33 @@ +// +// Generated by class-dump 3.5 (64 bit) (Debug version compiled May 11 2021 09:30:43). +// +// Copyright (C) 1997-2019 Steve Nygard. +// + +#import "XCTestRun.h" + +@class NSArray, NSMutableArray; + +@interface XCTestSuiteRun : XCTestRun +{ + NSMutableArray *_mutableTestRuns; +} + +//- (void).cxx_destruct; +@property(readonly) NSMutableArray *mutableTestRuns; // @synthesize mutableTestRuns=_mutableTestRuns; +- (void)recordExpectedFailure:(id)arg1; +- (void)_handleIssue:(id)arg1; +- (double)testDuration; +- (unsigned long long)unexpectedExceptionCount; +- (unsigned long long)failureCount; +- (unsigned long long)expectedFailureCount; +- (unsigned long long)skipCount; +- (unsigned long long)executionCount; +- (void)addTestRun:(id)arg1; +@property(readonly, copy) NSArray *testRuns; +- (void)stop; +- (void)start; +- (id)initWithTest:(id)arg1; + +@end + diff --git a/BPTestInspector/Internal/main.m b/BPTestInspector/Internal/main.m new file mode 100644 index 00000000..c9090fb2 --- /dev/null +++ b/BPTestInspector/Internal/main.m @@ -0,0 +1,55 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import +#import +#import + +#import "BPXCTestUtils.h" +#import "BPLoggingUtils.h" +#import "BPTestInspectorConstants.h" + +/** + This wrapper around XCTest hijacks the process when two key env variables are set, hooking into the xctest process to + instead get information on what tests exist in the test suite. This allows Bluepill to do two main things: + + 1) It can create an aggregate timeout based on the number of tests to run + 2) It allows us to support opting-out of tests, rather than just opting in, as xctest provides no explicit opt-out api. + + When `BP_XCTEST_WRAPPER__LOGIC_TEST_BUNDLE` and `BP_XCTEST_WRAPPER__TEST_CASE_OUTPUT` are set, + the entire process will only output an encoded file with test case info at the path specified by `BP_XCTEST_WRAPPER__TEST_CASE_OUTPUT`, + and **no tests will actually be run**. + + When they are not set, tests will be run as normal with no other side effects. + */ +__attribute__((constructor)) +static void didLoad() { + [BPLoggingUtils log:@"Booting up the wrapper."]; + + #if !TARGET_OS_IOS + [BPLoggingUtils log:@"Returning."]; + return; + #endif + + // Grab relavent info from environment + NSString *bundlePath = NSProcessInfo.processInfo.environment[BPTestInspectorConstants.testBundleEnvironmentKey]; + NSString *outputPath = NSProcessInfo.processInfo.environment[BPTestInspectorConstants.outputPathEnvironmentKey]; + // Reset DYLD_INSERT_LIBRARIES and other env variables to avoid impacting future processes + unsetenv("DYLD_INSERT_LIBRARIES"); + unsetenv(BPTestInspectorConstants.testBundleEnvironmentKey.UTF8String); + unsetenv(BPTestInspectorConstants.outputPathEnvironmentKey.UTF8String); + + if (!bundlePath || !outputPath) { + return; + } + [BPLoggingUtils log:[NSString stringWithFormat:@"Will enumerate all testCases in bundle at %@", bundlePath]]; + [BPXCTestUtils logAllTestsInBundleWithPath:bundlePath toFile:outputPath]; + // Once tests are logged, we want the process to end. + exit(0); +} diff --git a/Bluepill.xcworkspace/contents.xcworkspacedata b/Bluepill.xcworkspace/contents.xcworkspacedata index f5ad27e0..a563c0e0 100644 --- a/Bluepill.xcworkspace/contents.xcworkspacedata +++ b/Bluepill.xcworkspace/contents.xcworkspacedata @@ -1,6 +1,9 @@ + + diff --git a/Bluepill.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings b/Bluepill.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings index 54782e32..a6f6fb21 100644 --- a/Bluepill.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings +++ b/Bluepill.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings @@ -4,5 +4,7 @@ IDEWorkspaceSharedSettings_AutocreateContextsIfNeeded + PreviewsEnabled + diff --git a/Configs/BPMacTestInspector.xcconfig b/Configs/BPMacTestInspector.xcconfig new file mode 100644 index 00000000..371f806f --- /dev/null +++ b/Configs/BPMacTestInspector.xcconfig @@ -0,0 +1,8 @@ +CURRENT_PROJECT_VERSION = 1 +DYLIB_CURRENT_VERSION = 1 +DYLIB_COMPATIBILITY_VERSION = 1 +PRODUCT_NAME = BPMacTestInspector +EXECUTABLE_PREFIX = lib +SDKROOT = macosx +MACOSX_DEPLOYMENT_TARGET = 11.0 +COPY_PHASE_STRIP = NO diff --git a/Configs/BPTestInspector.xcconfig b/Configs/BPTestInspector.xcconfig new file mode 100644 index 00000000..964e63f0 --- /dev/null +++ b/Configs/BPTestInspector.xcconfig @@ -0,0 +1,9 @@ +CURRENT_PROJECT_VERSION = 1 +DYLIB_CURRENT_VERSION = 1 +DYLIB_COMPATIBILITY_VERSION = 1 +PRODUCT_NAME = BPTestInspector +EXECUTABLE_PREFIX = lib +SDKROOT = iphonesimulator +IPHONEOS_DEPLOYMENT_TARGET = 16.0 +COPY_PHASE_STRIP = NO +CODE_SIGN_IDENTITY = - // LTHROCKM - may need code signing for hosted tests diff --git a/bluepill/bluepill.xcodeproj/project.pbxproj b/bluepill/bluepill.xcodeproj/project.pbxproj index 03ef4457..794f1f4a 100644 --- a/bluepill/bluepill.xcodeproj/project.pbxproj +++ b/bluepill/bluepill.xcodeproj/project.pbxproj @@ -37,6 +37,11 @@ C4D6861A2267ABEF007D4237 /* bplib.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C4D686192267ABEF007D4237 /* bplib.framework */; }; C4FD8C581DB6E09B000ED28C /* BPPacker.m in Sources */ = {isa = PBXBuildFile; fileRef = C4FD8C571DB6E09B000ED28C /* BPPacker.m */; }; E49235FF22EA847700395D98 /* times.json in Resources */ = {isa = PBXBuildFile; fileRef = E49235FE22EA847700395D98 /* times.json */; }; + FB21F0C02A622FFE00682AC7 /* libBPMacTestInspector.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = FB21F0BE2A622FFD00682AC7 /* libBPMacTestInspector.dylib */; }; + FB940A432B3032430071F2FA /* libBPMacTestInspector.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = FB940A422B3032430071F2FA /* libBPMacTestInspector.dylib */; }; + FB940A442B3032430071F2FA /* libBPMacTestInspector.dylib in Embed Libraries */ = {isa = PBXBuildFile; fileRef = FB940A422B3032430071F2FA /* libBPMacTestInspector.dylib */; settings = {ATTRIBUTES = (CodeSignOnCopy, ); }; }; + FBA69FA82A45F889003BADBF /* BPLogicTestFixture_x86_64.xctest in Resources */ = {isa = PBXBuildFile; fileRef = FBA69FA62A45F889003BADBF /* BPLogicTestFixture_x86_64.xctest */; }; + FBC6553A2B3194130083E1BF /* BPLogicTestFixture_swift_x86_64.xctest in Resources */ = {isa = PBXBuildFile; fileRef = FBC655392B3194130083E1BF /* BPLogicTestFixture_swift_x86_64.xctest */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -49,6 +54,20 @@ }; /* End PBXContainerItemProxy section */ +/* Begin PBXCopyFilesBuildPhase section */ + FB940A452B3032430071F2FA /* Embed Libraries */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + FB940A442B3032430071F2FA /* libBPMacTestInspector.dylib in Embed Libraries */, + ); + name = "Embed Libraries"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + /* Begin PBXFileReference section */ 0173520A2366110D008BFA4E /* BPHTMLReportWriter.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = BPHTMLReportWriter.h; sourceTree = ""; }; 0173520B2366110D008BFA4E /* BPHTMLReportWriter.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPHTMLReportWriter.m; sourceTree = ""; }; @@ -85,6 +104,11 @@ C4FD8C561DB6E09B000ED28C /* BPPacker.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPPacker.h; sourceTree = ""; }; C4FD8C571DB6E09B000ED28C /* BPPacker.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPPacker.m; sourceTree = ""; }; E49235FE22EA847700395D98 /* times.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = times.json; sourceTree = ""; }; + FB21F0BE2A622FFD00682AC7 /* libBPMacTestInspector.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; path = libBPMacTestInspector.dylib; sourceTree = ""; }; + FB940A3D2B3028DD0071F2FA /* libBPTestInspector.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; path = libBPTestInspector.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; + FB940A422B3032430071F2FA /* libBPMacTestInspector.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; path = libBPMacTestInspector.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; + FBA69FA62A45F889003BADBF /* BPLogicTestFixture_x86_64.xctest */ = {isa = PBXFileReference; lastKnownFileType = wrapper; path = BPLogicTestFixture_x86_64.xctest; sourceTree = ""; }; + FBC655392B3194130083E1BF /* BPLogicTestFixture_swift_x86_64.xctest */ = {isa = PBXFileReference; lastKnownFileType = wrapper; path = BPLogicTestFixture_swift_x86_64.xctest; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -94,6 +118,7 @@ files = ( C4D6861A2267ABEF007D4237 /* bplib.framework in Frameworks */, B3109F792151F72F00B9309C /* CoreSimulator.framework in Frameworks */, + FB21F0C02A622FFE00682AC7 /* libBPMacTestInspector.dylib in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -101,6 +126,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + FB940A432B3032430071F2FA /* libBPMacTestInspector.dylib in Frameworks */, C45F9E82267E8A8800969CC3 /* bplib.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; @@ -111,6 +137,9 @@ B3380AED2150BD8600752E1B /* Frameworks */ = { isa = PBXGroup; children = ( + FB940A422B3032430071F2FA /* libBPMacTestInspector.dylib */, + FB940A3D2B3028DD0071F2FA /* libBPTestInspector.dylib */, + FB21F0BE2A622FFD00682AC7 /* libBPMacTestInspector.dylib */, C4D686192267ABEF007D4237 /* bplib.framework */, B3380AEE2150BD8700752E1B /* CoreSimulator.framework */, BA1896B821791A14000CEC36 /* XCTest.framework */, @@ -142,6 +171,8 @@ 0173521223679E87008BFA4E /* TEST-FinalReport.xml */, E49235FE22EA847700395D98 /* times.json */, BA9C2DB01DD67B66007CB967 /* testScheme.xcscheme */, + FBC655392B3194130083E1BF /* BPLogicTestFixture_swift_x86_64.xctest */, + FBA69FA62A45F889003BADBF /* BPLogicTestFixture_x86_64.xctest */, BA9C2DAE1DD674AD007CB967 /* BPSampleAppTests.xctest */, C415174C273AE3CE00646740 /* Expected-TEST-FinalReport-for-invalid-xml.xml */, ); @@ -217,6 +248,7 @@ 0137FE97237B65E100B36E69 /* Generate HTML template header */, BAEF4B301DAC539400E68294 /* Sources */, BAEF4B311DAC539400E68294 /* Frameworks */, + FB940A452B3032430071F2FA /* Embed Libraries */, ); buildRules = ( ); @@ -274,6 +306,8 @@ files = ( BA9C2DB11DD67B66007CB967 /* testScheme.xcscheme in Resources */, BA9C2DAF1DD674AD007CB967 /* BPSampleAppTests.xctest in Resources */, + FBA69FA82A45F889003BADBF /* BPLogicTestFixture_x86_64.xctest in Resources */, + FBC6553A2B3194130083E1BF /* BPLogicTestFixture_swift_x86_64.xctest in Resources */, E49235FF22EA847700395D98 /* times.json in Resources */, C415174F273AEAC400646740 /* simulator in Resources */, 0173521323679E87008BFA4E /* TEST-FinalReport.xml in Resources */, @@ -363,6 +397,10 @@ HEADER_SEARCH_PATHS = "\"$(SRCROOT)/..\""; INFOPLIST_FILE = tests/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)", + ); OTHER_LDFLAGS = ""; PRODUCT_BUNDLE_IDENTIFIER = LI.BluepillRunnerTests; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -380,6 +418,10 @@ HEADER_SEARCH_PATHS = "\"$(SRCROOT)/..\""; INFOPLIST_FILE = tests/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 12.0; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)", + ); OTHER_LDFLAGS = ""; PRODUCT_BUNDLE_IDENTIFIER = LI.BluepillRunnerTests; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -509,6 +551,10 @@ DEVELOPER_PRIVATE_FRAMEWORKS_DIR = ""; DEVELOPMENT_TEAM = 57Y47U492U; HEADER_SEARCH_PATHS = "\"$(SRCROOT)/..\""; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)", + ); MACOSX_DEPLOYMENT_TARGET = 10.15; OTHER_LDFLAGS = ( "-weak_framework", @@ -531,6 +577,10 @@ DEVELOPER_PRIVATE_FRAMEWORKS_DIR = ""; DEVELOPMENT_TEAM = 57Y47U492U; HEADER_SEARCH_PATHS = "\"$(SRCROOT)/..\""; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)", + ); MACOSX_DEPLOYMENT_TARGET = 10.15; OTHER_LDFLAGS = ( "-weak_framework", diff --git a/bluepill/bluepill.xcodeproj/xcshareddata/xcschemes/bluepill.xcscheme b/bluepill/bluepill.xcodeproj/xcshareddata/xcschemes/bluepill.xcscheme index a8b56da3..ce415051 100644 --- a/bluepill/bluepill.xcodeproj/xcshareddata/xcschemes/bluepill.xcscheme +++ b/bluepill/bluepill.xcodeproj/xcshareddata/xcschemes/bluepill.xcscheme @@ -1,7 +1,7 @@ + version = "1.8"> @@ -62,6 +62,20 @@ ReferencedContainer = "container:../bp/bp.xcodeproj"> + + + + -#import "bp/src/BPXCTestFile.h" +#import @class BPConfiguration; @interface BPApp : NSObject diff --git a/bluepill/src/BPApp.m b/bluepill/src/BPApp.m index ac134386..d3377bd0 100644 --- a/bluepill/src/BPApp.m +++ b/bluepill/src/BPApp.m @@ -8,9 +8,7 @@ // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. #import "BPApp.h" -#import "bp/src/BPConstants.h" -#import "bp/src/BPConfiguration.h" -#import "bp/src/BPUtils.h" +#import @implementation BPApp @@ -94,6 +92,20 @@ @implementation BPApp return allXCTestFiles; } ++ (NSArray *)testsFromConfig:(BPConfiguration *)config + withError:(NSError *__autoreleasing *)errPtr { + NSMutableArray *loadedTests = [[NSMutableArray alloc] initWithCapacity:config.tests.count]; + for (NSString *testName in config.tests) { + BPTestPlan *testPlan = [config.tests objectForKey:testName]; + BPXCTestFile *xcTestFile = [BPXCTestFile BPXCTestFileFromBPTestPlan:testPlan withName:testName andError:errPtr]; + if (*errPtr) { + return nil; + } + [loadedTests addObject:xcTestFile]; + } + return loadedTests; +} + + (instancetype)appWithConfig:(BPConfiguration *)config withError:(NSError *__autoreleasing *)errPtr { @@ -102,17 +114,10 @@ + (instancetype)appWithConfig:(BPConfiguration *)config if (config.tests != nil && config.tests.count != 0) { [BPUtils printInfo:INFO withString:@"Using test bundles"]; - NSMutableArray *loadedTests = [[NSMutableArray alloc] initWithCapacity:config.tests.count]; - for (NSString *testName in config.tests) { - BPTestPlan *testPlan = [config.tests objectForKey:testName]; - BPXCTestFile *xcTestFile = [BPXCTestFile BPXCTestFileFromBPTestPlan:testPlan withName:testName andError:errPtr]; - if (*errPtr) - return nil; - [loadedTests addObject:xcTestFile]; + app.testBundles = [self testsFromConfig:config withError:errPtr]; + if (*errPtr) { + return nil; } - - app.testBundles = loadedTests; - return app; } @@ -135,6 +140,12 @@ + (instancetype)appWithConfig:(BPConfiguration *)config andTestBundlePath:config.testBundlePath andUITargetAppPath:config.testRunnerAppPath withError:errPtr]]; + } else if (config.isLogicTestTarget && config.testBundlePath) { + BPXCTestFile *testFile = [BPXCTestFile BPXCTestFileFromXCTestBundle:config.testBundlePath + andHostAppBundle:nil + andUITargetAppPath:nil + withError:errPtr]; + [allXCTestFiles addObject:testFile]; } else { BP_SET_ERROR(errPtr, @"xctestrun file must be given, see usage."); return nil; diff --git a/bluepill/src/BPHTMLReportWriter.m b/bluepill/src/BPHTMLReportWriter.m index f5057474..38ed8f67 100644 --- a/bluepill/src/BPHTMLReportWriter.m +++ b/bluepill/src/BPHTMLReportWriter.m @@ -9,7 +9,7 @@ #import #import "BPTestReportHTML.h" -#import "bp/src/BPUtils.h" +#import #import "BPHTMLReportWriter.h" diff --git a/bluepill/src/BPPacker.h b/bluepill/src/BPPacker.h index 51e8e67d..b2fdbb4c 100644 --- a/bluepill/src/BPPacker.h +++ b/bluepill/src/BPPacker.h @@ -8,8 +8,7 @@ // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. #import -#import "bp/src/BPConfiguration.h" -#import "bp/src/BPXCTestFile.h" +#import @interface BPPacker : NSObject diff --git a/bluepill/src/BPPacker.m b/bluepill/src/BPPacker.m index 5ec0f259..ee10fabc 100644 --- a/bluepill/src/BPPacker.m +++ b/bluepill/src/BPPacker.m @@ -7,8 +7,7 @@ // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -#import "bp/src/BPXCTestFile.h" -#import "bp/src/BPUtils.h" +#import #import "BPPacker.h" @implementation BPPacker @@ -142,6 +141,7 @@ @implementation BPPacker andXCTestFiles:xcTestFiles]; NSMutableArray *bundles = [[NSMutableArray alloc] init]; + for (BPXCTestFile *xctFile in xcTestFiles) { NSArray *bundleTestsToRun = [[testsToRunByFilePath[xctFile.testBundlePath] allObjects] sortedArrayUsingSelector:@selector(compare:)]; NSNumber *estimatedBundleTime = testEstimatesByFilePath[xctFile.testBundlePath]; @@ -166,7 +166,11 @@ @implementation BPPacker i++; } // Make a bundle out of current xctFile - BPXCTestFile *bundle = [self makeBundle:xctFile withTests:bundleTestsToRun startAt:startIndex numTests:(i-startIndex) estimatedTime:[NSNumber numberWithDouble:splitExecTime]]; + NSInteger numTests = i - startIndex; + if (numTests <= 0) { + break; + } + BPXCTestFile *bundle = [self makeBundle:xctFile withTests:bundleTestsToRun startAt:startIndex numTests:numTests estimatedTime:[NSNumber numberWithDouble:splitExecTime]]; [bundles addObject:bundle]; } } diff --git a/bluepill/src/BPReportCollector.m b/bluepill/src/BPReportCollector.m index 211706d8..b67e0dd1 100644 --- a/bluepill/src/BPReportCollector.m +++ b/bluepill/src/BPReportCollector.m @@ -9,7 +9,7 @@ #import "BPHTMLReportWriter.h" #import "BPReportCollector.h" -#import "bp/src/BPUtils.h" +#import // Save path and mtime for reports (sort by mtime) @interface BPXMLReport:NSObject diff --git a/bluepill/src/BPRunner.h b/bluepill/src/BPRunner.h index 21d19f54..fd49e4e0 100644 --- a/bluepill/src/BPRunner.h +++ b/bluepill/src/BPRunner.h @@ -8,8 +8,7 @@ // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. #import -#import "bp/src/BPXCTestFile.h" -#import "bp/src/BPConfiguration.h" +#import @interface BPRunner : NSObject diff --git a/bluepill/src/BPRunner.m b/bluepill/src/BPRunner.m index 5fdf6162..d013a73b 100644 --- a/bluepill/src/BPRunner.m +++ b/bluepill/src/BPRunner.m @@ -8,12 +8,7 @@ // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. #import -#import "bp/src/BPCreateSimulatorHandler.h" -#import "bp/src/BPSimulator.h" -#import "bp/src/BPStats.h" -#import "bp/src/BPUtils.h" -#import "bp/src/BPWaitTimer.h" -#import "bp/src/SimulatorHelper.h" +#import #import "BPPacker.h" #import "BPRunner.h" #import "BPSwimlane.h" @@ -132,12 +127,14 @@ - (int)runWithBPXCTestFiles:(NSArray *)xcTestFiles { [BPUtils printInfo:ERROR withString:@"Packing failed: %@", [error localizedDescription]]; return 1; } + if (bundles.count < numSims) { [BPUtils printInfo:WARNING withString:@"Lowering number of parallel simulators from %lu to %lu because there aren't enough test bundles.", numSims, bundles.count]; numSims = bundles.count; } + if (self.config.cloneSimulator) { self.testHostSimTemplates = [bpSimulator createSimulatorAndInstallAppWithBundles:xcTestFiles]; if ([self.testHostSimTemplates count] == 0) { @@ -172,6 +169,7 @@ - (int)runWithBPXCTestFiles:(NSArray *)xcTestFiles { return -1; } } + while (1) { if (interrupted) { if (interrupted >=5) { @@ -222,6 +220,8 @@ - (int)runWithBPXCTestFiles:(NSArray *)xcTestFiles { [bundles removeObjectAtIndex:0]; } } + + // Resume in a second. Every 30 seconds, log status. sleep(1); if (seconds % 30 == 0) { NSString *listString; @@ -251,6 +251,7 @@ - (int)runWithBPXCTestFiles:(NSArray *)xcTestFiles { [self addCounters]; } + // Cleanup Devices for (int i = 0; i < [deviceList count]; i++) { NSTask *task = [self newTaskToDeleteDevice:[deviceList objectAtIndex:i] andNumber:i+1]; [task launch]; diff --git a/bluepill/src/BPSwimlane.h b/bluepill/src/BPSwimlane.h index 4bcfd60f..dd2f6a9e 100644 --- a/bluepill/src/BPSwimlane.h +++ b/bluepill/src/BPSwimlane.h @@ -8,8 +8,7 @@ // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. #import -#import "bp/src/BPXCTestFile.h" -#import "bp/src/BPConfiguration.h" +#import @interface BPSwimlane : NSObject diff --git a/bluepill/src/BPSwimlane.m b/bluepill/src/BPSwimlane.m index 024020fd..36ed0bf3 100644 --- a/bluepill/src/BPSwimlane.m +++ b/bluepill/src/BPSwimlane.m @@ -7,7 +7,7 @@ // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -#import "bp/src/BPUtils.h" +#import #import "BPSwimlane.h" @interface BPSwimlane() diff --git a/bluepill/src/main.m b/bluepill/src/main.m index 0bd4884e..ff897113 100644 --- a/bluepill/src/main.m +++ b/bluepill/src/main.m @@ -10,13 +10,10 @@ #import #import #import -#import "bp/src/BPConfiguration.h" -#import "bp/src/BPStats.h" -#import "bp/src/BPUtils.h" -#import "bp/src/BPWriter.h" #import "BPApp.h" #import "BPReportCollector.h" #import "BPRunner.h" +#import #include #include diff --git a/bluepill/tests/BPAppTests.m b/bluepill/tests/BPAppTests.m index 467978dc..782425e0 100644 --- a/bluepill/tests/BPAppTests.m +++ b/bluepill/tests/BPAppTests.m @@ -9,11 +9,7 @@ #import #import "bluepill/src/BPApp.h" -#import "bp/src/BPConfiguration.h" -#import "bp/src/BPXCTestFile.h" -#import "bp/src/BPUtils.h" -#import "bp/src/BPConstants.h" -#import "bp/tests/BPTestHelper.h" +#import @interface BPAppTests : XCTestCase @property (nonatomic, strong) BPConfiguration* config; diff --git a/bluepill/tests/BPCLITests.m b/bluepill/tests/BPCLITests.m index ae51d43d..b9d27aa9 100644 --- a/bluepill/tests/BPCLITests.m +++ b/bluepill/tests/BPCLITests.m @@ -8,9 +8,7 @@ // WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. #import -#import "bp/src/BPConfiguration.h" -#import "bp/src/BPUtils.h" -#import "bp/tests/BPTestHelper.h" +#import @interface BPCLITests : XCTestCase diff --git a/bluepill/tests/BPIntegrationTests.m b/bluepill/tests/BPIntegrationTests.m index 84dbe4d2..5c396f82 100644 --- a/bluepill/tests/BPIntegrationTests.m +++ b/bluepill/tests/BPIntegrationTests.m @@ -7,14 +7,12 @@ // #import -#import "bp/tests/BPTestHelper.h" -#import "bp/src/BPConfiguration.h" -#import "bp/src/BPUtils.h" #import "bluepill/src/BPRunner.h" #import "bluepill/src/BPApp.h" #import "bluepill/src/BPPacker.h" -#import "bp/src/BPXCTestFile.h" -#import "bp/src/BPConstants.h" + +#import +#import @interface BPIntegrationTests : XCTestCase @end @@ -59,6 +57,124 @@ - (void)tearDown { [super tearDown]; } +- (void)testArchitecture_x86_64 { + NSString *bundlePath = BPTestHelper.logicTestBundlePath_x86_64; + + BPConfiguration *config = [BPTestUtils makeUnhostedTestConfiguration]; + config.stuckTimeout = @(2); + config.testCaseTimeout = @(10); + // Test multiple test bundles, while skipping any failing tests so that we + // can still validate that we get a success code.. + config.testBundlePath = bundlePath; + + NSString *bpPath = [BPTestHelper bpExecutablePath]; + BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath]; + NSError *error; + BPApp *app = [BPApp appWithConfig:config withError:&error]; + + XCTAssert(runner != nil); + int rc = [runner runWithBPXCTestFiles:app.testBundles]; + XCTAssert(rc == 0, @"Wanted 0, got %d", rc); + XCTAssert([runner busySwimlaneCount] == 0); +} + +- (void)testArchitectureWithSwiftTests_x86_64 { + NSString *bundlePath = BPTestHelper.logicTestBundlePath_swift_x86_64; + + BPConfiguration *config = [BPTestUtils makeUnhostedTestConfiguration]; + config.stuckTimeout = @(2); + config.testCaseTimeout = @(10); + // Test multiple test bundles, while skipping any failing tests so that we + // can still validate that we get a success code.. + config.testBundlePath = bundlePath; + + NSString *bpPath = [BPTestHelper bpExecutablePath]; + BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath]; + NSError *error; + BPApp *app = [BPApp appWithConfig:config withError:&error]; + + XCTAssert(runner != nil); + int rc = [runner runWithBPXCTestFiles:app.testBundles]; + XCTAssert(rc == 0, @"Wanted 0, got %d", rc); + XCTAssert([runner busySwimlaneCount] == 0); +} + +// Currently having troubles generating a new arm64 fixture. +- (void)DISABLEDtestArchitecture_arm64 { + NSString *bundlePath = BPTestHelper.logicTestBundlePath_arm64; + + BPConfiguration *config = [BPTestUtils makeUnhostedTestConfiguration]; + config.stuckTimeout = @(2); + config.testCaseTimeout = @(10); + config.numSims = @(1); + // Test multiple test bundles, while skipping any failing tests so that we + // can still validate that we get a success code.. + config.testBundlePath = bundlePath; + + NSString *bpPath = [BPTestHelper bpExecutablePath]; + BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath]; + NSError *error; + BPApp *app = [BPApp appWithConfig:config withError:&error]; + + XCTAssert(runner != nil); + int rc = [runner runWithBPXCTestFiles:app.testBundles]; + XCTAssert(rc == 0, @"Wanted 0, got %d", rc); + XCTAssert([runner busySwimlaneCount] == 0); +} + +- (void)testLogicTestBundles { + BPConfiguration *config = [BPTestUtils makeUnhostedTestConfiguration]; + config.stuckTimeout = @(2); + config.testCaseTimeout = @(10); + // Test multiple test bundles, while skipping any failing tests so that we + // can still validate that we get a success code.. + config.testBundlePath = BPTestHelper.logicTestBundlePath; + config.additionalUnitTestBundles = @[BPTestHelper.logicTestBundlePath]; + config.testCasesToSkip = @[ + @"BPLogicTests/testFailingLogicTest", + @"BPLogicTests/testCrashTestCaseLogicTest", + @"BPLogicTests/testCrashExecutionLogicTest", + @"BPLogicTests/testStuckLogicTest", + @"BPLogicTests/testSlowLogicTest", + ]; + + NSString *bpPath = [BPTestHelper bpExecutablePath]; + BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath]; + NSError *error; + BPApp *app = [BPApp appWithConfig:config withError:&error]; + + XCTAssert(runner != nil); + int rc = [runner runWithBPXCTestFiles:app.testBundles]; + XCTAssert(rc == 0, @"Wanted 0, got %d", rc); + XCTAssert([runner busySwimlaneCount] == 0); +} + +- (void)testTwoBPInstancesWithLogicTestPlanJson { + [self writeLogicTestPlan]; + BPConfiguration *config = [BPTestUtils makeUnhostedTestConfiguration]; + config.numSims = @2; + config.testBundlePath = nil; + config.testRunnerAppPath = nil; + config.appBundlePath = nil; + config.testPlanPath = [BPTestHelper testPlanPath]; + + NSError *err; + [config validateConfigWithError:&err]; + BPApp *app = [BPApp appWithConfig:config withError:&err]; + NSString *bpPath = [BPTestHelper bpExecutablePath]; + BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath]; + XCTAssert(runner != nil); + int rc = [runner runWithBPXCTestFiles:app.testBundles]; + XCTAssert(rc != 0); // this runs tests that fail + XCTAssertEqual(app.testBundles.count, 2); + XCTAssertTrue([app.testBundles[0].name isEqualToString:@"BPLogicTests"]); + XCTAssertEqual(app.testBundles[0].numTests, 10); + XCTAssertEqual(app.testBundles[0].skipTestIdentifiers.count, 0); + XCTAssertTrue([app.testBundles[1].name isEqualToString:@"BPPassingLogicTests"]); + XCTAssertEqual(app.testBundles[1].numTests, 207); + XCTAssertEqual(app.testBundles[1].skipTestIdentifiers.count, 0); +} + - (void)testOneBPInstance { BPConfiguration *config = [self generateConfig]; config.numSims = @1; @@ -91,6 +207,8 @@ - (void)testTwoBPInstances { XCTAssert([runner busySwimlaneCount] == 0); } +// Note: If this is failing for you locally, try resetting all of your +// sims with `sudo rm -rf /private/tmp/com.apple.CoreSimulator.SimDevice.*` - (void)testClonedSimulators { BPConfiguration *config = [self generateConfig]; config.numSims = @2; @@ -150,13 +268,23 @@ - (void)testTwoBPInstancesWithXCTestRunFile { BPRunner *runner = [BPRunner BPRunnerWithConfig:config withBpPath:bpPath]; XCTAssert(runner != nil); int rc = [runner runWithBPXCTestFiles:app.testBundles]; + + // Set this once we learn how many tests are in the BPSampleAppTests bundle. + NSInteger sampleAppTestsRemaining = NSNotFound; for (BPXCTestFile *testBundle in app.testBundles) { + // BPSampleAppTests is a huge bundle and will be broken into multiple batches + // Across all of these batches, the NOT skipped tests should add up to the total + // test count. if ([testBundle.name isEqualToString:@"BPSampleAppTests"]) { - XCTAssertEqual(testBundle.skipTestIdentifiers.count, 8); + if (sampleAppTestsRemaining == NSNotFound) { + sampleAppTestsRemaining = testBundle.allTestCases.count; + } + sampleAppTestsRemaining -= (testBundle.allTestCases.count - testBundle.skipTestIdentifiers.count); } else { XCTAssertEqual(testBundle.skipTestIdentifiers.count, 0); } } + XCTAssertEqual(sampleAppTestsRemaining, 0); XCTAssert(rc != 0); // this runs tests that fail } @@ -224,6 +352,25 @@ - (void)writeTestPlan { [jsonData writeToFile:[BPTestHelper testPlanPath] atomically:YES]; } +- (void)writeLogicTestPlan { + NSDictionary *testPlan = @{ + @"tests": @{ + @"BPLogicTests": @{ + @"test_bundle_path": [BPTestHelper logicTestBundlePath], + @"environment": @{}, + @"arguments": @{} + }, + @"BPPassingLogicTests": @{ + @"test_bundle_path": [BPTestHelper passingLogicTestBundlePath], + @"environment": @{}, + @"arguments": @{} + } + } + }; + NSData *jsonData = [NSJSONSerialization dataWithJSONObject:testPlan options:0 error:nil]; + [jsonData writeToFile:[BPTestHelper testPlanPath] atomically:YES]; +} + // TODO: Enable this when we figure out issue #469 - (void)DISABLE_testTwoBPInstancesWithVideo { NSFileManager *fileManager = [NSFileManager defaultManager]; diff --git a/bluepill/tests/BPPackerTests.m b/bluepill/tests/BPPackerTests.m index 98826a73..ed67343d 100644 --- a/bluepill/tests/BPPackerTests.m +++ b/bluepill/tests/BPPackerTests.m @@ -10,11 +10,7 @@ #import "bluepill/src/BPRunner.h" #import "bluepill/src/BPApp.h" #import "bluepill/src/BPPacker.h" -#import "bp/tests/BPTestHelper.h" -#import "bp/src/BPConfiguration.h" -#import "bp/src/BPUtils.h" -#import "bp/src/BPXCTestFile.h" -#import "bp/src/BPConstants.h" +#import @interface BPPackerTests : XCTestCase @property (nonatomic, strong) BPConfiguration* config; diff --git a/bluepill/tests/BPRunnerTests.m b/bluepill/tests/BPRunnerTests.m index 37c4f0cc..a28489bd 100644 --- a/bluepill/tests/BPRunnerTests.m +++ b/bluepill/tests/BPRunnerTests.m @@ -9,13 +9,10 @@ #import #import "bp/tests/BPTestHelper.h" -#import "bp/src/BPConfiguration.h" -#import "bp/src/BPUtils.h" +#import #import "bluepill/src/BPRunner.h" #import "bluepill/src/BPApp.h" #import "bluepill/src/BPPacker.h" -#import "bp/src/BPXCTestFile.h" -#import "bp/src/BPConstants.h" @interface BPRunnerTests : XCTestCase @property (nonatomic, strong) BPConfiguration* config; diff --git a/bluepill/tests/Resource Files/BPLogicTestFixture_swift_x86_64.xctest/BPLogicTestFixture_swift_x86_64 b/bluepill/tests/Resource Files/BPLogicTestFixture_swift_x86_64.xctest/BPLogicTestFixture_swift_x86_64 new file mode 100755 index 00000000..06fe278b Binary files /dev/null and b/bluepill/tests/Resource Files/BPLogicTestFixture_swift_x86_64.xctest/BPLogicTestFixture_swift_x86_64 differ diff --git a/bluepill/tests/Resource Files/BPLogicTestFixture_swift_x86_64.xctest/Info.plist b/bluepill/tests/Resource Files/BPLogicTestFixture_swift_x86_64.xctest/Info.plist new file mode 100644 index 00000000..a667de3f Binary files /dev/null and b/bluepill/tests/Resource Files/BPLogicTestFixture_swift_x86_64.xctest/Info.plist differ diff --git a/bluepill/tests/Resource Files/BPLogicTestFixture_swift_x86_64.xctest/_CodeSignature/CodeResources b/bluepill/tests/Resource Files/BPLogicTestFixture_swift_x86_64.xctest/_CodeSignature/CodeResources new file mode 100644 index 00000000..a6cbde0f --- /dev/null +++ b/bluepill/tests/Resource Files/BPLogicTestFixture_swift_x86_64.xctest/_CodeSignature/CodeResources @@ -0,0 +1,101 @@ + + + + + files + + Info.plist + + DT7gNfddwFlosavksp+OY+Ga3e0= + + + files2 + + rules + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^version.plist$ + + + rules2 + + .*\.dSYM($|/) + + weight + 11 + + ^(.*/)?\.DS_Store$ + + omit + + weight + 2000 + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^Info\.plist$ + + omit + + weight + 20 + + ^PkgInfo$ + + omit + + weight + 20 + + ^embedded\.provisionprofile$ + + weight + 20 + + ^version\.plist$ + + weight + 20 + + + + diff --git a/bluepill/tests/Resource Files/BPLogicTestFixture_x86_64.xctest/BPLogicTestFixture_x86_64 b/bluepill/tests/Resource Files/BPLogicTestFixture_x86_64.xctest/BPLogicTestFixture_x86_64 new file mode 100755 index 00000000..42f14694 Binary files /dev/null and b/bluepill/tests/Resource Files/BPLogicTestFixture_x86_64.xctest/BPLogicTestFixture_x86_64 differ diff --git a/bluepill/tests/Resource Files/BPLogicTestFixture_x86_64.xctest/Info.plist b/bluepill/tests/Resource Files/BPLogicTestFixture_x86_64.xctest/Info.plist new file mode 100644 index 00000000..78869ce6 Binary files /dev/null and b/bluepill/tests/Resource Files/BPLogicTestFixture_x86_64.xctest/Info.plist differ diff --git a/bluepill/tests/Resource Files/BPLogicTestFixture_x86_64.xctest/_CodeSignature/CodeResources b/bluepill/tests/Resource Files/BPLogicTestFixture_x86_64.xctest/_CodeSignature/CodeResources new file mode 100644 index 00000000..00ca198f --- /dev/null +++ b/bluepill/tests/Resource Files/BPLogicTestFixture_x86_64.xctest/_CodeSignature/CodeResources @@ -0,0 +1,101 @@ + + + + + files + + Info.plist + + a2X62OSUA6yDTeQ0W3o6GtPmrKc= + + + files2 + + rules + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^version.plist$ + + + rules2 + + .*\.dSYM($|/) + + weight + 11 + + ^(.*/)?\.DS_Store$ + + omit + + weight + 2000 + + ^.* + + ^.*\.lproj/ + + optional + + weight + 1000 + + ^.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Base\.lproj/ + + weight + 1010 + + ^Info\.plist$ + + omit + + weight + 20 + + ^PkgInfo$ + + omit + + weight + 20 + + ^embedded\.provisionprofile$ + + weight + 20 + + ^version\.plist$ + + weight + 20 + + + + diff --git a/bp/bp.xcodeproj/project.pbxproj b/bp/bp.xcodeproj/project.pbxproj index ccbf023c..3e64e8de 100644 --- a/bp/bp.xcodeproj/project.pbxproj +++ b/bp/bp.xcodeproj/project.pbxproj @@ -69,7 +69,7 @@ BA0096FE1DCA5D810000DD45 /* testConfigRelativePath.json in Resources */ = {isa = PBXBuildFile; fileRef = BA0096FD1DCA5D810000DD45 /* testConfigRelativePath.json */; }; BA0097001DCA61210000DD45 /* BPConfigurationTests.m in Sources */ = {isa = PBXBuildFile; fileRef = BA0096FF1DCA61210000DD45 /* BPConfigurationTests.m */; }; BA0C554D1DDAE241009E1377 /* failure_retry_report.xml in Resources */ = {isa = PBXBuildFile; fileRef = BA0C554C1DDAE241009E1377 /* failure_retry_report.xml */; }; - BA1809B81DB89B5600D7D130 /* BluepillTests.m in Sources */ = {isa = PBXBuildFile; fileRef = BA1809B71DB89B5600D7D130 /* BluepillTests.m */; }; + BA1809B81DB89B5600D7D130 /* BluepillHostedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = BA1809B71DB89B5600D7D130 /* BluepillHostedTests.m */; }; BA180A091DBB00FA00D7D130 /* BPUtilsTests.m in Sources */ = {isa = PBXBuildFile; fileRef = BA180A081DBB00FA00D7D130 /* BPUtilsTests.m */; }; BA180A0C1DBB011200D7D130 /* testConfig.json in Resources */ = {isa = PBXBuildFile; fileRef = BA180A0B1DBB011200D7D130 /* testConfig.json */; }; BA180A141DBB088100D7D130 /* testScheme.xcscheme in Resources */ = {isa = PBXBuildFile; fileRef = BA180A131DBB088100D7D130 /* testScheme.xcscheme */; }; @@ -109,6 +109,19 @@ C4F08F75224C45750001AD2A /* BPExitStatus.m in Sources */ = {isa = PBXBuildFile; fileRef = 7A4D7A851DDBB156001E085D /* BPExitStatus.m */; }; C4FAC2951E5E67ED00ACC5D9 /* testConfig-busted.json in Resources */ = {isa = PBXBuildFile; fileRef = C4FAC2941E5E67ED00ACC5D9 /* testConfig-busted.json */; }; C94DE0BB4360016D3D3061D9 /* simulator-preferences.plist in Resources */ = {isa = PBXBuildFile; fileRef = C94DEF7F8BCA7AB3C9114467 /* simulator-preferences.plist */; }; + FB1C695629A6E58500BFDFB4 /* BluepillUnhostedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FB1C695529A6E58500BFDFB4 /* BluepillUnhostedTests.m */; }; + FB21F0BB2A622FF600682AC7 /* libBPMacTestInspector.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = FB21F0BA2A622FF600682AC7 /* libBPMacTestInspector.dylib */; }; + FB21F0BC2A622FF600682AC7 /* libBPMacTestInspector.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = FB21F0BA2A622FF600682AC7 /* libBPMacTestInspector.dylib */; }; + FB21F0BD2A622FF600682AC7 /* libBPMacTestInspector.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = FB21F0BA2A622FF600682AC7 /* libBPMacTestInspector.dylib */; }; + FB21F0DE2A65103F00682AC7 /* BPTestCaseInfoTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FB21F0DD2A65103F00682AC7 /* BPTestCaseInfoTests.m */; }; + FB53751F2A66288300496432 /* BPTestInspectionHandler.h in Headers */ = {isa = PBXBuildFile; fileRef = FB53751D2A66288300496432 /* BPTestInspectionHandler.h */; }; + FB5375202A66288300496432 /* BPTestInspectionHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = FB53751E2A66288300496432 /* BPTestInspectionHandler.m */; }; + FB5375212A66288300496432 /* BPTestInspectionHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = FB53751E2A66288300496432 /* BPTestInspectionHandler.m */; }; + FB5375242A66494100496432 /* libBPTestInspector.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = FB5375222A66494000496432 /* libBPTestInspector.dylib */; }; + FBBBD8EF29A06AF6002B9115 /* BPTestUtils.m in Sources */ = {isa = PBXBuildFile; fileRef = FBBBD8ED29A06AF6002B9115 /* BPTestUtils.m */; }; + FBD772BF2A33E1DA0098CEFD /* BluepillUnhostedBatchingTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBD772BE2A33E1DA0098CEFD /* BluepillUnhostedBatchingTests.m */; }; + FBD772C12A3452240098CEFD /* BPTestUtils.h in Headers */ = {isa = PBXBuildFile; fileRef = FBBBD8EE29A06AF6002B9115 /* BPTestUtils.h */; settings = {ATTRIBUTES = (Public, ); }; }; + FBD772C22A3453010098CEFD /* BPTestUtils.m in Sources */ = {isa = PBXBuildFile; fileRef = FBBBD8ED29A06AF6002B9115 /* BPTestUtils.m */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -128,6 +141,19 @@ }; /* End PBXContainerItemProxy section */ +/* Begin PBXCopyFilesBuildPhase section */ + FB9F19452A32F0EF00C894D3 /* Embed Libraries */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + ); + name = "Embed Libraries"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + /* Begin PBXFileReference section */ 018D5C1025B4FF4200B0314B /* BPIntTestCase.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = BPIntTestCase.h; sourceTree = ""; }; 018D5C1125B4FF4200B0314B /* BPIntTestCase.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPIntTestCase.m; sourceTree = ""; }; @@ -158,7 +184,6 @@ 7A7901821D5CB679004D4325 /* bp */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = bp; sourceTree = BUILT_PRODUCTS_DIR; }; 7A7901851D5CB679004D4325 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; 7A79018C1D5CF113004D4325 /* bp.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = bp.xcconfig; sourceTree = ""; }; - 7A79018E1D5D136F004D4325 /* CoreSimulator.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreSimulator.framework; path = Library/PrivateFrameworks/CoreSimulator.framework; sourceTree = DEVELOPER_DIR; }; 7A7D66001DB679820015C937 /* BPStats.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPStats.h; sourceTree = ""; }; 7A7D66011DB679820015C937 /* BPStats.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPStats.m; sourceTree = ""; }; 7A7E7BAE1DF2066B007928F3 /* BPHandler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPHandler.h; sourceTree = ""; }; @@ -193,12 +218,11 @@ BA0096FF1DCA61210000DD45 /* BPConfigurationTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPConfigurationTests.m; sourceTree = ""; }; BA0097011DCA626F0000DD45 /* BPConfiguration+Test.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "BPConfiguration+Test.h"; sourceTree = ""; }; BA0C554C1DDAE241009E1377 /* failure_retry_report.xml */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xml; path = failure_retry_report.xml; sourceTree = ""; }; - BA1809B71DB89B5600D7D130 /* BluepillTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BluepillTests.m; sourceTree = ""; }; + BA1809B71DB89B5600D7D130 /* BluepillHostedTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BluepillHostedTests.m; sourceTree = ""; }; BA1809BC1DB89B8B00D7D130 /* AppKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AppKit.framework; path = Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks/AppKit.framework; sourceTree = DEVELOPER_DIR; }; BA180A081DBB00FA00D7D130 /* BPUtilsTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPUtilsTests.m; sourceTree = ""; }; BA180A0B1DBB011200D7D130 /* testConfig.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = testConfig.json; sourceTree = ""; }; BA180A131DBB088100D7D130 /* testScheme.xcscheme */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xml; path = testScheme.xcscheme; sourceTree = ""; }; - BA1896B321791683000CEC36 /* CoreSimulator */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = CoreSimulator; path = ../../../../../Library/Developer/PrivateFrameworks/CoreSimulator.framework/Versions/A/CoreSimulator; sourceTree = ""; }; BA1896B6217916F8000CEC36 /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/MacOSX.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; }; BA1949341E4AF82F00881887 /* BPTMDRunnerConnection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPTMDRunnerConnection.h; sourceTree = ""; }; BA1949351E4AF82F00881887 /* BPTMDRunnerConnection.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPTMDRunnerConnection.m; sourceTree = ""; }; @@ -292,9 +316,6 @@ BAFCCA541E36DBA900E33C31 /* DTXTransport.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = DTXTransport.h; sourceTree = ""; }; BAFCCA581E36DBA900E33C31 /* NSURLSessionDelegate-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "NSURLSessionDelegate-Protocol.h"; sourceTree = ""; }; BAFCCA6D1E36EB5500E33C31 /* XCTestManager_IDEInterface-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCTestManager_IDEInterface-Protocol.h"; sourceTree = ""; }; - BAFCCA6E1E36EB5500E33C31 /* XCTestManager_ManagerInterface-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCTestManager_ManagerInterface-Protocol.h"; sourceTree = ""; }; - BAFCCA6F1E36EB5500E33C31 /* XCTestManager_TestsInterface-Protocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCTestManager_TestsInterface-Protocol.h"; sourceTree = ""; }; - BAFCCA701E37298B00E33C31 /* XCTestDriverInterface-Protocol.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "XCTestDriverInterface-Protocol.h"; sourceTree = ""; }; BAFCCA711E37298B00E33C31 /* XCTestManager_DaemonConnectionInterface-Protocol.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "XCTestManager_DaemonConnectionInterface-Protocol.h"; sourceTree = ""; }; C41A2C701E0B2497005D9751 /* BPXCTestFile.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPXCTestFile.h; sourceTree = ""; }; C41A2C711E0B2497005D9751 /* BPXCTestFile.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; lineEnding = 0; path = BPXCTestFile.m; sourceTree = ""; xcLanguageSpecificationIdentifier = xcode.lang.objc; }; @@ -314,6 +335,15 @@ C4AF1ADE2273649500618F0B /* BPVersion.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPVersion.h; sourceTree = ""; }; C4FAC2941E5E67ED00ACC5D9 /* testConfig-busted.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = "testConfig-busted.json"; sourceTree = ""; }; C94DEF7F8BCA7AB3C9114467 /* simulator-preferences.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist; path = "simulator-preferences.plist"; sourceTree = ""; }; + FB1C695529A6E58500BFDFB4 /* BluepillUnhostedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BluepillUnhostedTests.m; sourceTree = ""; }; + FB21F0BA2A622FF600682AC7 /* libBPMacTestInspector.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; path = libBPMacTestInspector.dylib; sourceTree = ""; }; + FB21F0DD2A65103F00682AC7 /* BPTestCaseInfoTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPTestCaseInfoTests.m; sourceTree = ""; }; + FB53751D2A66288300496432 /* BPTestInspectionHandler.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = BPTestInspectionHandler.h; sourceTree = ""; }; + FB53751E2A66288300496432 /* BPTestInspectionHandler.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BPTestInspectionHandler.m; sourceTree = ""; }; + FB5375222A66494000496432 /* libBPTestInspector.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; path = libBPTestInspector.dylib; sourceTree = ""; }; + FBBBD8ED29A06AF6002B9115 /* BPTestUtils.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = BPTestUtils.m; sourceTree = ""; }; + FBBBD8EE29A06AF6002B9115 /* BPTestUtils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BPTestUtils.h; sourceTree = ""; }; + FBD772BE2A33E1DA0098CEFD /* BluepillUnhostedBatchingTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = BluepillUnhostedBatchingTests.m; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -321,6 +351,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + FB21F0BB2A622FF600682AC7 /* libBPMacTestInspector.dylib in Frameworks */, C487CB1C226A663C00CC5BC2 /* bplib.framework in Frameworks */, B324B91D1F280AD100AAE2BC /* CoreSimulator.framework in Frameworks */, 7AB913001D5E209800621608 /* AppKit.framework in Frameworks */, @@ -332,7 +363,9 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + FB21F0BD2A622FF600682AC7 /* libBPMacTestInspector.dylib in Frameworks */, BA1896B5217916A5000CEC36 /* CoreSimulator.framework in Frameworks */, + FB5375242A66494100496432 /* libBPTestInspector.dylib in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -340,6 +373,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + FB21F0BC2A622FF600682AC7 /* libBPMacTestInspector.dylib in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -349,12 +383,10 @@ 7A4FB8CC1DF89A790073F268 /* src */ = { isa = PBXGroup; children = ( - C487CB1A226A36F500CC5BC2 /* BPVersion.h */, BA1949341E4AF82F00881887 /* BPTMDRunnerConnection.h */, BA1949351E4AF82F00881887 /* BPTMDRunnerConnection.m */, BA1949381E4AF83E00881887 /* BPTMDControlConnection.h */, BA1949391E4AF83E00881887 /* BPTMDControlConnection.m */, - BA954F551D6D1AB3007D011D /* PrivateHeaders */, BA34F5E21D6D75E30063B17F /* SimulatorHelper.h */, BA34F5E31D6D75E30063B17F /* SimulatorHelper.m */, 7A4933131DAD63A50060D54F /* SimulatorMonitor.h */, @@ -393,6 +425,8 @@ 7A7E7BAF1DF2066B007928F3 /* BPHandler.m */, 7A7E7BB11DF20F81007928F3 /* BPApplicationLaunchHandler.h */, 7A7E7BB21DF20F81007928F3 /* BPApplicationLaunchHandler.m */, + FB53751D2A66288300496432 /* BPTestInspectionHandler.h */, + FB53751E2A66288300496432 /* BPTestInspectionHandler.m */, 7A7E7BB61DF2173C007928F3 /* BPCreateSimulatorHandler.h */, 7A7E7BB71DF2173C007928F3 /* BPCreateSimulatorHandler.m */, 7A7E7BBA1DF21749007928F3 /* BPDeleteSimulatorHandler.h */, @@ -403,8 +437,10 @@ BA944BCF1D76A39A00A4BDA3 /* Bluepill.m */, 7A7E7BBE1DF22CE1007928F3 /* BPExecutionContext.h */, 7A7E7BBF1DF22CE1007928F3 /* BPExecutionContext.m */, - B368E55D213F8D2E00B4DEA3 /* Info.plist */, 7A7901851D5CB679004D4325 /* main.m */, + BA954F551D6D1AB3007D011D /* PrivateHeaders */, + C487CB1A226A36F500CC5BC2 /* BPVersion.h */, + B368E55D213F8D2E00B4DEA3 /* Info.plist */, ); path = src; sourceTree = ""; @@ -435,13 +471,13 @@ 7A79018D1D5D136F004D4325 /* Frameworks */ = { isa = PBXGroup; children = ( + FB5375222A66494000496432 /* libBPTestInspector.dylib */, + FB21F0BA2A622FF600682AC7 /* libBPMacTestInspector.dylib */, C47411F7264F4C3F000F84D1 /* XcodeKit.framework */, BA1896B6217916F8000CEC36 /* XCTest.framework */, - BA1896B321791683000CEC36 /* CoreSimulator */, B324B91C1F280AD100AAE2BC /* CoreSimulator.framework */, BA1809BC1DB89B8B00D7D130 /* AppKit.framework */, 7AB912FE1D5E209800621608 /* AppKit.framework */, - 7A79018E1D5D136F004D4325 /* CoreSimulator.framework */, 7A58D2E51D62689B002F6B6D /* DVTFoundation.framework */, 7AB912FF1D5E209800621608 /* Foundation.framework */, ); @@ -491,11 +527,8 @@ BA34F5E51D6D78120063B17F /* XCTest */ = { isa = PBXGroup; children = ( - BAFCCA701E37298B00E33C31 /* XCTestDriverInterface-Protocol.h */, BAFCCA711E37298B00E33C31 /* XCTestManager_DaemonConnectionInterface-Protocol.h */, BAFCCA6D1E36EB5500E33C31 /* XCTestManager_IDEInterface-Protocol.h */, - BAFCCA6E1E36EB5500E33C31 /* XCTestManager_ManagerInterface-Protocol.h */, - BAFCCA6F1E36EB5500E33C31 /* XCTestManager_TestsInterface-Protocol.h */, BA34F5E61D6D78120063B17F /* XCTestConfiguration.h */, ); path = XCTest; @@ -562,8 +595,10 @@ BAB24F691DB5DB2300867756 /* tests */ = { isa = PBXGroup; children = ( + FBBBD8EC29A06AE7002B9115 /* Utils */, BA180A0A1DBB011200D7D130 /* Resource Files */, - BA1809B71DB89B5600D7D130 /* BluepillTests.m */, + BA1809B71DB89B5600D7D130 /* BluepillHostedTests.m */, + FBD772C02A33E1E20098CEFD /* Unhosted Tests */, C467E5491DC930D200BC80EE /* BPCLITests.m */, BA0097011DCA626F0000DD45 /* BPConfiguration+Test.h */, BA0096FF1DCA61210000DD45 /* BPConfigurationTests.m */, @@ -618,6 +653,25 @@ path = DTXConnectionServices; sourceTree = ""; }; + FBBBD8EC29A06AE7002B9115 /* Utils */ = { + isa = PBXGroup; + children = ( + FBBBD8EE29A06AF6002B9115 /* BPTestUtils.h */, + FBBBD8ED29A06AF6002B9115 /* BPTestUtils.m */, + ); + path = Utils; + sourceTree = ""; + }; + FBD772C02A33E1E20098CEFD /* Unhosted Tests */ = { + isa = PBXGroup; + children = ( + FB21F0DD2A65103F00682AC7 /* BPTestCaseInfoTests.m */, + FB1C695529A6E58500BFDFB4 /* BluepillUnhostedTests.m */, + FBD772BE2A33E1DA0098CEFD /* BluepillUnhostedBatchingTests.m */, + ); + path = "Unhosted Tests"; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXHeadersBuildPhase section */ @@ -635,6 +689,8 @@ B3520CE2214C5DE3003DC68F /* BPDeleteSimulatorHandler.h in Headers */, B368E579213F965600B4DEA3 /* BPTestCase.h in Headers */, B368E57A213F965600B4DEA3 /* BPTestClass.h in Headers */, + FBD772C12A3452240098CEFD /* BPTestUtils.h in Headers */, + FB53751F2A66288300496432 /* BPTestInspectionHandler.h in Headers */, B368E57B213F965600B4DEA3 /* BPUtils.h in Headers */, B368E57C213F965600B4DEA3 /* BPXCTestFile.h in Headers */, B368E571213F8E8F00B4DEA3 /* BPConstants.h in Headers */, @@ -679,6 +735,7 @@ B368E556213F8D2E00B4DEA3 /* Frameworks */, B368E557213F8D2E00B4DEA3 /* Headers */, B368E558213F8D2E00B4DEA3 /* Resources */, + FB9F19452A32F0EF00C894D3 /* Embed Libraries */, ); buildRules = ( ); @@ -828,6 +885,7 @@ BA19493A1E4AF83E00881887 /* BPTMDControlConnection.m in Sources */, 7ADBB1471DCBBC0E00DC4E8D /* BPTreeAssembler.m in Sources */, 7A4D7A861DDBB156001E085D /* BPExitStatus.m in Sources */, + FB5375202A66288300496432 /* BPTestInspectionHandler.m in Sources */, BA944BD01D76A39A00A4BDA3 /* Bluepill.m in Sources */, 7A564C0E1DA817DE001BCEC2 /* BPTreeObjects.m in Sources */, 7A7901861D5CB679004D4325 /* main.m in Sources */, @@ -838,11 +896,13 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + FB5375212A66288300496432 /* BPTestInspectionHandler.m in Sources */, C4D686182267A8C9007D4237 /* BPTestHelper.m in Sources */, C47B2DB3225813C70068C5CA /* BPWriter.m in Sources */, C4F08F75224C45750001AD2A /* BPExitStatus.m in Sources */, B3DAF83E2151CB9700210286 /* BPWaitTimer.m in Sources */, B3DAF83D2151CB7000210286 /* BPHandler.m in Sources */, + FBD772C22A3453010098CEFD /* BPTestUtils.m in Sources */, B3DAF83C2151CB4100210286 /* BPDeleteSimulatorHandler.m in Sources */, B3DAF83B2151CB3300210286 /* BPCreateSimulatorHandler.m in Sources */, BA9ADEB421657004001306F5 /* BPApplicationLaunchHandler.m in Sources */, @@ -863,6 +923,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + FBD772BF2A33E1DA0098CEFD /* BluepillUnhostedBatchingTests.m in Sources */, BA19493B1E4AF83E00881887 /* BPTMDControlConnection.m in Sources */, BAB24F711DB5DBED00867756 /* SimulatorHelperTests.m in Sources */, 7A4D7A811DDA5FA1001E085D /* BPTreeParserTests.m in Sources */, @@ -874,7 +935,7 @@ 7ACE1F721DD3D27D00C0FA73 /* WaitTimerTests.m in Sources */, BA180A091DBB00FA00D7D130 /* BPUtilsTests.m in Sources */, 7A4D7A871DDBB156001E085D /* BPExitStatus.m in Sources */, - BA1809B81DB89B5600D7D130 /* BluepillTests.m in Sources */, + BA1809B81DB89B5600D7D130 /* BluepillHostedTests.m in Sources */, BA1949371E4AF82F00881887 /* BPTMDRunnerConnection.m in Sources */, 7A7E7BC11DF22CE1007928F3 /* BPExecutionContext.m in Sources */, BAD558D71DB6DCB100C9A5CD /* BPWriter.m in Sources */, @@ -884,6 +945,9 @@ BAD558D51DB6DCB100C9A5CD /* BPTreeObjects.m in Sources */, BA0097001DCA61210000DD45 /* BPConfigurationTests.m in Sources */, BAB24F741DB5DFA200867756 /* BPTestHelper.m in Sources */, + FB1C695629A6E58500BFDFB4 /* BluepillUnhostedTests.m in Sources */, + FB21F0DE2A65103F00682AC7 /* BPTestCaseInfoTests.m in Sources */, + FBBBD8EF29A06AF6002B9115 /* BPTestUtils.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1031,6 +1095,10 @@ "$(SRCROOT)/src", ); INFOPLIST_FILE = "$(SRCROOT)/src/Info.plist"; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)", + ); MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_CFLAGS = "-DBP_USE_PRIVATE_FRAMEWORKS"; OTHER_LDFLAGS = ( @@ -1070,6 +1138,10 @@ "$(SRCROOT)/src", ); INFOPLIST_FILE = "$(SRCROOT)/src/Info.plist"; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)", + ); MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_CFLAGS = "-DBP_USE_PRIVATE_FRAMEWORKS"; OTHER_LDFLAGS = ( @@ -1115,6 +1187,10 @@ "$(SRCROOT)/src", ); INFOPLIST_FILE = "$(SRCROOT)/src/Info.plist"; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)", + ); MACH_O_TYPE = staticlib; MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_LDFLAGS = ""; @@ -1152,6 +1228,10 @@ "$(SRCROOT)/src", ); INFOPLIST_FILE = "$(SRCROOT)/src/Info.plist"; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)", + ); MACH_O_TYPE = staticlib; MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_LDFLAGS = ""; @@ -1175,6 +1255,10 @@ ); INFOPLIST_FILE = tests/Info.plist; LD_RUNPATH_SEARCH_PATHS = "\"$(DEVELOPER_DIR)/Library/PrivateFrameworks\""; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)", + ); MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_CFLAGS = "-DBP_USE_PRIVATE_FRAMEWORKS"; OTHER_LDFLAGS = ( @@ -1207,6 +1291,10 @@ ); INFOPLIST_FILE = tests/Info.plist; LD_RUNPATH_SEARCH_PATHS = "\"$(DEVELOPER_DIR)/Library/PrivateFrameworks\""; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)", + ); MACOSX_DEPLOYMENT_TARGET = 11.0; OTHER_CFLAGS = "-DBP_USE_PRIVATE_FRAMEWORKS"; OTHER_LDFLAGS = ( diff --git a/bp/bp.xcodeproj/xcshareddata/xcschemes/bp.xcscheme b/bp/bp.xcodeproj/xcshareddata/xcschemes/bp.xcscheme index 0ea5c1cc..f3be982d 100644 --- a/bp/bp.xcodeproj/xcshareddata/xcschemes/bp.xcscheme +++ b/bp/bp.xcodeproj/xcshareddata/xcschemes/bp.xcscheme @@ -20,6 +20,34 @@ ReferencedContainer = "container:bp.xcodeproj"> + + + + + + + + *allTests; +@property (nonatomic, strong) NSString *xctestBinaryPath; +@property (nonatomic, strong) NSString *dyldFrameworkPath; /** Return a structure suitable for passing to `getopt()`'s long options. diff --git a/bp/src/BPConfiguration.m b/bp/src/BPConfiguration.m index 92c02ae8..f72b7899 100644 --- a/bp/src/BPConfiguration.m +++ b/bp/src/BPConfiguration.m @@ -150,6 +150,8 @@ typedef NS_OPTIONS(NSUInteger, BPOptionType) { "Directory where videos of test runs will be saved. If not provided, videos are not recorded."}, {368, "keep-passing-videos", BLUEPILL_BINARY | BP_BINARY, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "keepPassingVideos", "Whether recorded videos should be kept if the test passed. They are deleted by default."}, + {369, "logic-test", BLUEPILL_BINARY | BP_BINARY, NO, NO, no_argument, "Off", BP_VALUE | BP_BOOL, "isLogicTestTarget", + "Will run the tests without an app host"}, {0, 0, 0, 0, 0, 0, 0} }; @@ -168,9 +170,6 @@ - (BOOL)isValid:(NSError **)errPtr { if (!self.testBundlePath) { [errors addObject:@"testBundlePath field is nil"]; } - if (!self.testHost) { - [errors addObject:@"testHost field is nil"]; - } if ([errors count] > 0) { BP_SET_ERROR(errPtr, [NSString stringWithFormat:@"Invalid BPTestPlan object, %@.", @@ -614,7 +613,7 @@ - (BOOL)processOptionsWithError:(NSError **)errPtr { } // Now check we didn't miss any require options: NSMutableArray *errors = [[NSMutableArray alloc] init]; - if (!(self.appBundlePath) && !(self.xcTestRunPath) && !(self.testPlanPath) && !(self.deleteSimUDID)) { + if (!(self.appBundlePath) && !(self.xcTestRunPath) && !(self.testPlanPath) && !(self.deleteSimUDID) && !(self.isLogicTestTarget)) { [errors addObject:@"Missing required option: -a/--app OR --xctestrun-path OR --test-plan-path"]; } if ((self.program & BP_BINARY) && !(self.testBundlePath) && !(self.testPlanPath) && !(self.deleteSimUDID)) { @@ -707,7 +706,7 @@ - (BOOL)validateConfigWithError:(NSError *__autoreleasing *)errPtr { return NO; } - if (!self.appBundlePath && !self.xcTestRunDict && !self.tests) { + if (!self.appBundlePath && !self.xcTestRunDict && !self.tests && !self.isLogicTestTarget) { BP_SET_ERROR(errPtr, @"No app bundle provided."); return NO; } diff --git a/bp/src/BPHandler.h b/bp/src/BPHandler.h index 2d0ddbbc..d2a0d276 100644 --- a/bp/src/BPHandler.h +++ b/bp/src/BPHandler.h @@ -18,6 +18,8 @@ typedef void (^BasicHandlerBlock)(void); typedef void (^BasicErrorBlock)(NSError *error); +@property (nonatomic, assign, class) NSInteger timeoutErrorCode; + @property (nonatomic, strong) BPWaitTimer *timer; @property (nonatomic, copy) BasicHandlerBlock beginWith; diff --git a/bp/src/BPHandler.m b/bp/src/BPHandler.m index edeccb2b..8453c948 100644 --- a/bp/src/BPHandler.m +++ b/bp/src/BPHandler.m @@ -31,7 +31,7 @@ - (void)setup { } // call timeout block first and then execute the onError block if (__self.onError) { - NSError *error = [NSError errorWithDomain:BPErrorDomain code:-1 userInfo:nil]; + NSError *error = [NSError errorWithDomain:BPErrorDomain code:BPHandler.timeoutErrorCode userInfo:nil]; __self.onError(error); } }); @@ -63,4 +63,8 @@ - (BasicErrorBlock)defaultHandlerBlock { }; } ++ (NSInteger)timeoutErrorCode { + return 8; +} + @end diff --git a/bp/src/BPSimulator.h b/bp/src/BPSimulator.h index c1f652c9..5713fb60 100644 --- a/bp/src/BPSimulator.h +++ b/bp/src/BPSimulator.h @@ -15,6 +15,7 @@ @class BPConfiguration; @class BPTreeParser; @class SimDevice; +@class BPTestCaseInfo; @interface BPSimulator : NSObject @@ -42,6 +43,25 @@ - (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCompletion:(void (^)(NSError *, pid_t))completion; +/** + Launches a process which inspects a test bundle, and calls a completion block with a list of all tests in the bundle. + + @discussion The only way to do this is to actually launch a test execution with an injected Bluepill module. This module + then checks what test cases exist at runtime, and writes these cases to a file before ending the process (such that no tests are actually run). + We then read that file, and pass the results to the completion handler provided. + */ +- (void)collectTestSuiteInfoWithCompletion:(void (^)(NSArray *, NSError *))completionBlock; + +/** + Executes logic tests for the provided context, providing two callbacks as the execution starts and finishes. + @param parser The test log parser + @param spawnBlock Called once the process is spawned (right after it's started and the process is actually running) + @param completionBlock Called once the process completes, and all tests have either passed/failed/erred. + */ +- (void)executeLogicTestsWithParser:(BPTreeParser *)parser + onSpawn:(void (^)(NSError *, pid_t))spawnBlock + andCompletion:(void (^)(NSError *, pid_t))completionBlock; + - (void)deleteSimulatorWithCompletion:(void (^)(NSError *error, BOOL success))completion; - (void)addPhotosToSimulator; diff --git a/bp/src/BPSimulator.m b/bp/src/BPSimulator.m index b9367258..99e47d18 100644 --- a/bp/src/BPSimulator.m +++ b/bp/src/BPSimulator.m @@ -18,6 +18,8 @@ #import "BPWaitTimer.h" #import "PrivateHeaders/CoreSimulator/CoreSimulator.h" #import "SimulatorHelper.h" +#import +#import @interface BPSimulator() @@ -451,18 +453,235 @@ - (BOOL)uninstallApplicationWithError:(NSError *__autoreleasing *)errPtr { error:errPtr]; } -- (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCompletion:(void (^)(NSError *, pid_t))completion { - NSString *hostBundleId = [SimulatorHelper bundleIdForPath:self.config.appBundlePath]; - NSString *hostAppExecPath = [SimulatorHelper executablePathforPath:self.config.appBundlePath]; +/** + Logic tests are run directly on the simulator by spawning a new process with the XCTest executable, without any kind of host app. + */ +- (void)executeLogicTestsWithParser:(BPTreeParser *)parser + onSpawn:(void (^)(NSError *, pid_t))spawnBlock + andCompletion:(void (^)(NSError *, pid_t))completionBlock { + /* + Grab all test cases so that we can: + 1) create a timeout for the full test execution + 2) Support opting-out of tests, despite the fact that XCTest only provides an opt-in API. + */ + NSArray *testsToRun = [SimulatorHelper testsToRunWithConfig:self.config]; + NSString *testsToRunArg = testsToRun.count == self.config.allTestCases.count ? @"All" : [testsToRun componentsJoinedByString:@","]; + + /* + It can be useful to understand how to translate the public Breaking down CLI command `xcrun simctl spawn... `, which you'd + use to run a logic test from commandline, into the form that Bluepill must use, which is CoreSimulator's private `spawnWithPath`: + + When trying to run a command such as + ``` + xcrun simctl spawn -s AFF4165A-9A71-4860-8B6D-485B7D1BA2BC + /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/Library/Xcode/Agents/xctest + -XCTest BPLogicTests/testPassingLogicTest /Users/.../BPLogicTests.xctest + ``` + we can break down the individual components to be handled in the following: + - part of the method signature: + - xcrun simctl // these just redirect down to CoreSimulator + - spawn // this is the the spawn method :p + - AFF4165A-9A71-4860-8B6D-485B7D1BA2BC // We're calling device.spawn. No need to respecify the device ID + - path: /Applications/.../Xcode/Agents/xctest + - options: + - -s // "standalone" option + - -w // "wait_for_debugger" option + - args: + - /Applications/.../Xcode/Agents/xctest // Yes, spawn redundantly requires this both the path param + an arg :) + - -XCTest + - BPLogicTests/testPassingLogicTest // The filter for which tests to actually run. `All` is the catch-all. + - /Users/.../BPLogicTests.xctest // The path to the .xctest file w/ all the module's tests. + + From `xcrun simctl spawn help`: + `simctl spawn [-w | --wait-for-debugger] [-s | --standalone] [-a | --arch=] [ ... ]` + */ + NSString *xctestPath = self.config.testBundlePath; + NSArray *arguments = @[ + self.config.xctestBinaryPath, + @"-XCTest", + testsToRunArg, + xctestPath, + ]; - // One bp instance only run one kind of xctest file. - if (self.config.testRunnerAppPath) { - hostAppExecPath = [SimulatorHelper executablePathforPath:self.config.testRunnerAppPath]; - hostBundleId = [SimulatorHelper bundleIdForPath:self.config.testRunnerAppPath]; + // Intercept stdout, stderr and post as simulator-output events + NSString *simStdoutPath = [SimulatorHelper makeStdoutFileOnDevice:self.device]; + NSString *simStdoutRelativePath = [simStdoutPath substringFromIndex:self.device.dataPath.length]; + // Environment + NSMutableDictionary *environment = [@{ + kOptionsStdoutKey: simStdoutRelativePath, + kOptionsStderrKey: simStdoutRelativePath, + } mutableCopy]; + if (self.config.dyldFrameworkPath) { + environment[@"DYLD_FRAMEWORK_PATH"] = self.config.dyldFrameworkPath; + environment[@"DYLD_LIBRARY_PATH"] = self.config.dyldFrameworkPath; + } + [environment addEntriesFromDictionary:self.config.environmentVariables]; + + self.appOutput = [NSFileHandle fileHandleForReadingAtPath:simStdoutPath]; + + NSFileHandle *outputFileHandle = [NSFileHandle fileHandleForWritingAtPath:simStdoutPath]; + NSNumber *stdoutFileDescriptor = @(outputFileHandle.fileDescriptor); + + NSDictionary *options = @{ + kOptionsArgumentsKey: arguments, + kOptionsEnvironmentKey: environment, + @"standalone": @(1), + @"stdout": stdoutFileDescriptor, + @"stderr": stdoutFileDescriptor, + }; + + // Set up monitor + if (!self.monitor) { + self.monitor = [[SimulatorMonitor alloc] initWithConfiguration:self.config]; } - // Create the environment for the host application - NSMutableDictionary *argsAndEnv = [[NSMutableDictionary alloc] init]; - NSArray *argumentsArr = self.config.commandLineArguments ?: @[]; + self.monitor.device = self.device; + parser.delegate = self.monitor; + + self.appOutput.readabilityHandler = ^(NSFileHandle *handle) { + // This callback occurs on a background thread + NSData *chunk = [handle availableData]; + [parser handleChunkData:chunk]; + }; + + + [BPUtils printInfo:DEBUGINFO withString:@"Spawning xctest execution with options: %@", options]; + + // To see more on how to debug the expected format/inputs of the options array, + // see the in-depth documentation in SimDevice.h. + __block typeof(self) blockSelf = self; + [self.device spawnAsyncWithPath:self.config.xctestBinaryPath + options:options + terminationHandler:^(int stat_loc) { + // The naming here is confusing, but this `terminationHandler` is called once + // the xctest process COMPLETES. The `completionHandler` below is used earlier, + // once the xctest process is SPAWNED. + + // Check the location where the status code is stored; + // Handle error if there is a signal or non-zero exit code. + NSError *error = [BPSimulator errorFromStatusLocation:stat_loc]; + if (error) { + [blockSelf signalCloseToParser:parser fileHandle:outputFileHandle]; + } + [blockSelf cleanUpParser:parser]; + completionBlock(error, blockSelf.monitor.appPID); + } completionHandler:^(NSError *error, pid_t pid) { + // Again, this `completionHandler` is called once the process is done SPAWNING, + // as opposed to happening after the process itself has finished. + blockSelf.monitor.appPID = pid; + blockSelf.monitor.appState = Running; + spawnBlock(error, pid); + }]; +} + +/** + Spawns a child xctest process (which we cause to not actually run tests), injecting the BPTestInspector + to pipe out test information instead. + */ +- (void)collectTestSuiteInfoWithCompletion:(void (^)(NSArray *, NSError *))completionBlock { + NSString *xctestPath = self.config.testBundlePath; + NSArray *arguments = @[ + self.config.xctestBinaryPath, + @"-XCTest", + @"All", + xctestPath, + ]; + + NSString *testSuiteInfoOutputPath = [SimulatorHelper makeTestWrapperOutputFileOnDevice:self.device]; + // Intercept stdout, stderr and post as simulator-output events + NSString *simStdoutPath = [SimulatorHelper makeStdoutFileOnDevice:self.device]; + NSString *simStdoutRelativePath = [simStdoutPath substringFromIndex:self.device.dataPath.length]; + + // Environment + NSMutableDictionary *environment = [[SimulatorHelper logicTestEnvironmentWithConfig:self.config stdoutRelativePath:simStdoutRelativePath] mutableCopy]; + + + NSString *testInspectorPath = [BPUtils findBPTestInspectorDYLIB]; + [BPUtils printInfo:DEBUGINFO withString: @"Injecting libBPTestInspector.dylib at path: %@", testInspectorPath]; + + environment[@"DYLD_INSERT_LIBRARIES"] = testInspectorPath; + environment[BPTestInspectorConstants.outputPathEnvironmentKey] = testSuiteInfoOutputPath; + environment[BPTestInspectorConstants.testBundleEnvironmentKey] = xctestPath; + + NSFileHandle *outputFileHandle = [NSFileHandle fileHandleForWritingAtPath:simStdoutPath]; + NSNumber *stdoutFileDescriptor = @(outputFileHandle.fileDescriptor); + + NSDictionary *options = @{ + kOptionsArgumentsKey: arguments, + kOptionsEnvironmentKey: environment, + @"standalone": @(1), + }; + + // To see more on how to debug the expected format/inputs of the options array, + // see the in-depth documentation in SimDevice.h. + __block typeof(self) blockSelf = self; + [self.device spawnAsyncWithPath:self.config.xctestBinaryPath + options:options + terminationHandler:^(int stat_loc) { + NSError *error = [BPSimulator errorFromStatusLocation:stat_loc]; + if (error) { + completionBlock(nil, error); + return; + } + // Retrieve test data + [BPUtils printInfo:INFO withString: @"Test inspector execution completed successfully"]; + + NSError *unarchiveError; + + NSFileHandle *fileHandle = [NSFileHandle fileHandleForReadingAtPath:testSuiteInfoOutputPath]; + NSData *testData = [fileHandle readDataToEndOfFile]; + NSArray *testBundleInfo = [NSKeyedUnarchiver unarchivedArrayOfObjectsOfClass:BPTestCaseInfo.class + fromData:testData + error:&unarchiveError]; + [fileHandle closeFile]; + // Cleanup + Completion + [NSFileManager.defaultManager removeItemAtPath:testSuiteInfoOutputPath error:nil]; + if (unarchiveError) { + [BPUtils printInfo:ERROR withString: @"BPTestInspector failed with unarchiver error: %@", unarchiveError.userInfo]; + } else { + [BPUtils printInfo:INFO withString: @"BPTestInspector returned list of tests of size: %@", @(testBundleInfo.count)]; + } + completionBlock(testBundleInfo, unarchiveError); + } completionHandler:^(NSError *error, pid_t pid) { + if (error) { + completionBlock(nil, error); + } + }]; +} + ++ (NSError *)errorFromStatusLocation:(int)stat_loc { + if (WIFSIGNALED(stat_loc)) { + int signalCode = WTERMSIG(stat_loc); + // Ignore if the process was killed -- this occurs when we're killing + // a timed-out test, and shouldn't be treated as a crash. + if (signalCode != SIGKILL) { + [BPUtils printInfo:WARNING withString: @"Spawned XCTest execution failed with signal code: %@", @(signalCode)]; + return [BPUtils errorWithSignalCode:signalCode]; + } + } else { + // A non-zero exit code could mean a failed test or something more serious, but we can't tell the difference here. + // The best we can do is log the error code as debug info. + int exitCode = WEXITSTATUS(stat_loc); + if (exitCode) { + [BPUtils printInfo:INFO withString: @"Spawned XCTest execution failed with error code: %@", @(exitCode)]; + } + } + return nil; +} + +// Posts a APPCLOSED signal to the parser, indicating a crash/kill +- (void)signalCloseToParser:(BPTreeParser *)parser fileHandle:(NSFileHandle *)fileHandle { + [fileHandle seekToEndOfFile]; + [fileHandle writeData:[@"\nBP_APP_PROC_ENDED\n" dataUsingEncoding:NSUTF8StringEncoding]]; + [fileHandle closeFile]; +} + +- (void)cleanUpParser:(BPTreeParser *)parser { + self.monitor.appState = Completed; + [parser.delegate setParserStateCompleted]; +} + ++ (NSMutableArray *)commandLineArgsFromConfig:(BPConfiguration *)config { + NSArray *argumentsArr = config.commandLineArguments ?: @[]; NSMutableArray *commandLineArgs = [NSMutableArray array]; for (NSString *argument in argumentsArr) { NSArray *argumentsArray = [argument componentsSeparatedByString:@" "]; @@ -472,6 +691,21 @@ - (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCom } } } + return commandLineArgs; +} + +- (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCompletion:(void (^)(NSError *, pid_t))completion { + NSString *hostBundleId = [SimulatorHelper bundleIdForPath:self.config.appBundlePath]; + NSString *hostAppExecPath = [SimulatorHelper executablePathforPath:self.config.appBundlePath]; + + // One bp instance only run one kind of xctest file. + if (self.config.testRunnerAppPath) { + hostAppExecPath = [SimulatorHelper executablePathforPath:self.config.testRunnerAppPath]; + hostBundleId = [SimulatorHelper bundleIdForPath:self.config.testRunnerAppPath]; + } + // Create the environment for the host application + NSMutableArray *commandLineArgs = [BPSimulator commandLineArgsFromConfig:self.config]; + NSMutableDictionary *argsAndEnv = [[NSMutableDictionary alloc] init]; // These are appended by Xcode so we do that here. [commandLineArgs addObjectsFromArray:@[ @@ -548,13 +782,8 @@ - (void)launchApplicationAndExecuteTestsWithParser:(BPTreeParser *)parser andCom dispatch_source_cancel(source); }); dispatch_source_set_cancel_handler(source, ^{ - blockSelf.monitor.appState = Completed; - [parser.delegate setParserStateCompleted]; - // Post a APPCLOSED signal to the parser - NSFileHandle *fileHandle = [NSFileHandle fileHandleForWritingAtPath:simStdoutPath]; - [fileHandle seekToEndOfFile]; - [fileHandle writeData:[@"\nBP_APP_PROC_ENDED\n" dataUsingEncoding:NSUTF8StringEncoding]]; - [fileHandle closeFile]; + [blockSelf signalCloseToParser:parser fileHandle:[NSFileHandle fileHandleForWritingAtPath:simStdoutPath]]; + [blockSelf cleanUpParser:parser]; }); dispatch_resume(source); self.appOutput.readabilityHandler = ^(NSFileHandle *handle) { diff --git a/bp/src/BPStats.h b/bp/src/BPStats.h index 90021d05..34dc9a51 100644 --- a/bp/src/BPStats.h +++ b/bp/src/BPStats.h @@ -16,6 +16,9 @@ #define INSTALL_APPLICATION(x) [NSString stringWithFormat:@"[Attempt %lu] Install Application", (x)] #define UNINSTALL_APPLICATION(x) [NSString stringWithFormat:@"[Attempt %lu] Uninstall Application", (x)] #define LAUNCH_APPLICATION(x) [NSString stringWithFormat:@"[Attempt %lu] Launch Application", (x)] +#define TEST_INSPECTION(x) [NSString stringWithFormat:@"[Attempt %lu] Inspect Tests", (x)] +#define SPAWN_LOGIC_TEST(x) [NSString stringWithFormat:@"[Attempt %lu] Spawn Logic Tests", (x)] +#define EXECUTE_LOGIC_TEST(x) [NSString stringWithFormat:@"[Attempt %lu] Execute Logic Tests", (x)] #define DELETE_SIMULATOR(x) [NSString stringWithFormat:@"[Attempt %lu] Delete Simulator", (x)] #define DELETE_SIMULATOR_CB(x) [NSString stringWithFormat:@"[Attempt %lu] Delete Simulator due to BAD STATE", (x)] diff --git a/bp/src/BPTestInspectionHandler.h b/bp/src/BPTestInspectionHandler.h new file mode 100644 index 00000000..735a7742 --- /dev/null +++ b/bp/src/BPTestInspectionHandler.h @@ -0,0 +1,27 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + + +#import +#import + +@class BPTestCaseInfo; + +typedef void (^TestInspectionBlock)(NSArray *testBundleInfo, NSError *error); +typedef void (^TestInspectionSuccessBlock)(NSArray *testBundleInfo); + +@interface BPTestInspectionHandler : BPHandler + +@property (nonatomic, copy) TestInspectionSuccessBlock onSuccess; + +- (TestInspectionBlock)defaultHandlerBlock; + + + +@end diff --git a/bp/src/BPTestInspectionHandler.m b/bp/src/BPTestInspectionHandler.m new file mode 100644 index 00000000..0dec1708 --- /dev/null +++ b/bp/src/BPTestInspectionHandler.m @@ -0,0 +1,44 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import "BPTestInspectionHandler.h" + +#import + +@implementation BPTestInspectionHandler + +@dynamic onSuccess; + +- (TestInspectionBlock)defaultHandlerBlock { + return ^(NSArray *testBundleInfo, NSError *error) { + dispatch_once(&self->onceToken, ^{ + [self.timer cancelTimer]; + + self.error = error; + + if (self.beginWith) { + self.beginWith(); + } + + if (error) { + if (self.onError) { + self.onError(error); + } + } else if (self.onSuccess) { + self.onSuccess(testBundleInfo); + } + + if (self.endWith) { + self.endWith(); + } + }); + }; +} + +@end diff --git a/bp/src/BPUtils.h b/bp/src/BPUtils.h index 209f3428..a1defcdf 100644 --- a/bp/src/BPUtils.h +++ b/bp/src/BPUtils.h @@ -29,6 +29,7 @@ typedef NS_ENUM(int, BPKind) { }; @class BPConfiguration; +@class BPExecutionContext; @interface BPUtils : NSObject @@ -50,6 +51,8 @@ typedef NS_ENUM(int, BPKind) { */ + (BOOL)isBuildScript; ++ (NSString *)findBPTestInspectorDYLIB; + + (NSString *)findExecutablePath:(NSString *)execName; /*! @@ -71,7 +74,6 @@ typedef NS_ENUM(int, BPKind) { */ + (NSString *)mkstemp:(NSString *)pathTemplate withError:(NSError **)errPtr; - /*! @discussion print a message to stdout. @param kind one of the levels in BPKind @@ -80,8 +82,20 @@ typedef NS_ENUM(int, BPKind) { + (void)printInfo:(BPKind)kind withString:(NSString *)fmt, ... NS_FORMAT_FUNCTION(2,3); /*! - @discussion get an NSError * - This is not really meant to be called, use the BP_SET_ERROR macro below instead. + Creates an `NSError *` with BP-specific domain for a given signal code, updating the description accordingly. + @param signalCode The signal code. + */ ++ (NSError *)errorWithSignalCode:(NSInteger)signalCode; + +/*! + Creates an `NSError *` with BP-specific domain for a given exit code, updating the description accordingly. + @param exitCode The exit code. + */ ++ (NSError *)errorWithExitCode:(NSInteger)exitCode; + +/*! + @discussion get an `NSError *` + This is not really meant to be called, use the `BP_SET_ERROR` macro below instead. @param function The name of the function @param line The line number @param fmt a format string (a la printf), followed by var args. @@ -111,6 +125,18 @@ typedef NS_ENUM(int, BPKind) { + (BPConfiguration *)normalizeConfiguration:(BPConfiguration *)config withTestFiles:(NSArray *)xctTestFiles; +/*! + Returns an aggregated timeout interval for all tests to be run in an execution. While we will + still apply a per-test timeout, it's possible for things to fail in an XCTest execution when a test isn't + being run, and we want to make sure the execution still fails when this occurs. + + @discussion This timeout value is based on the timeout per test multiplied by the number of tests, + with an additional buffer per test. + @param config The fully setup configuration that will be used to calculate the aggregate timeout. + @return The aggregated timeout. + */ ++ (double)timeoutForAllTestsWithConfiguration:(BPConfiguration *)config; + /*! @discussion a function to determine if the given file name represents stdout. A file name is considered stdout if it is '-' or 'stdout'. @@ -226,4 +252,24 @@ typedef BOOL (^BPRunBlock)(void); testTimes:(NSDictionary *)testTimes andXCTestFiles:(NSArray *)xcTestFiles; +#pragma mark - Logic Test Architecture Helpers + +/** + We can isolate a single architecture out of a universal binary using the `lipo -extract` command. By doing so, we can + force an executable (such as XCTest) to always run w/ the architecture we expect. This is to avoid some funny business where + the architecture selected can be unexpected depending on multiple factors, such as Rosetta, xcode version, etc. + + @return the path of the new executable if possible + required, nil otherwise. In nil case, original executable should be used instead. + */ ++ (NSString *)lipoExecutableAtPath:(NSString *)path withContext:(BPExecutionContext *)context; + +/** + Lipo'ing the universal binary alone to isolate the desired architecture will result in errors. + Specifically, the newly lipo'ed binary won't be able to find any of the required frameworks + from within the original binary. So, we need to set up the `DYLD_FRAMEWORK_PATH` + in the environment to include the paths to these frameworks within the original universal + executable's binary. + */ ++ (NSString *)correctedDYLDFrameworkPathFromBinary:(NSString *)binaryPath; + @end diff --git a/bp/src/BPUtils.m b/bp/src/BPUtils.m index 3b8bf87b..e9541ab6 100644 --- a/bp/src/BPUtils.m +++ b/bp/src/BPUtils.m @@ -12,6 +12,11 @@ #import "BPConstants.h" #import "BPXCTestFile.h" #import "BPConfiguration.h" +#import "SimDevice.h" +#import "SimDeviceType.h" +#import "BPExecutionContext.h" +#import "BPSimulator.h" +#import @implementation BPUtils @@ -113,16 +118,6 @@ + (void)printTo:(FILE*)fd kind:(BPKind)kind withString:(NSString *)txt { fflush(fd); } -+ (NSError *)BPError:(const char *)function andLine:(int)line withFormat:(NSString *)fmt, ... { - va_list args; - va_start(args, fmt); - NSString *msg = [[NSString alloc] initWithFormat:fmt arguments:args]; - va_end(args); - return [NSError errorWithDomain:BPErrorDomain - code:-1 - userInfo:@{NSLocalizedDescriptionKey: msg}]; -} - + (NSString *)findExecutablePath:(NSString *)execName { NSString *argv0 = [[[NSProcessInfo processInfo] arguments] objectAtIndex:0]; NSString *execPath = [[argv0 stringByDeletingLastPathComponent] stringByAppendingPathComponent:execName]; @@ -132,6 +127,44 @@ + (NSString *)findExecutablePath:(NSString *)execName { return execPath; } ++ (NSString *)findBPTestInspectorDYLIB { + NSString *argv0 = [[[NSProcessInfo processInfo] arguments] objectAtIndex:0]; + // While the Bluepill binary is in a 'Release' or 'Debug' directory, the + // simulator build for the libBPTestInspector.dylib file is in a sibling + // directory 'Release-iphonesimulator' or 'Debug-iphonesimulator' + NSString *iPhoneSimDir = [[argv0 stringByDeletingLastPathComponent] stringByAppendingString:@"-iphonesimulator"]; + NSString *path = [iPhoneSimDir stringByAppendingPathComponent:BPTestInspectorConstants.dylibName]; + + [BPUtils printInfo:INFO withString: @"Searching for libBPTestInspector.dylib at path: %@", path]; + if ([[NSFileManager defaultManager] isReadableFileAtPath:path]) { + return path; + } + + // The executable may also be in derived data, accessible from the app's current working directory. + NSString *buildProductsDir = [NSFileManager.defaultManager.currentDirectoryPath stringByDeletingLastPathComponent]; + iPhoneSimDir = [buildProductsDir stringByAppendingPathComponent:@"Debug-iphonesimulator"]; + path = [iPhoneSimDir stringByAppendingPathComponent:BPTestInspectorConstants.dylibName]; + + [BPUtils printInfo:INFO withString: @"Previous search failed. Now searching for libBPTestInspector.dylib at path: %@", path]; + if ([[NSFileManager defaultManager] isReadableFileAtPath:path]) { + return path; + } + + // The executable may also be in derived data, accessible from the app's current working directory. + buildProductsDir = [NSFileManager.defaultManager.currentDirectoryPath stringByDeletingLastPathComponent]; + iPhoneSimDir = [buildProductsDir stringByAppendingPathComponent:@"Release-iphonesimulator"]; + path = [iPhoneSimDir stringByAppendingPathComponent:BPTestInspectorConstants.dylibName]; + + [BPUtils printInfo:INFO withString: @"Previous search failed. Now searching for libBPTestInspector.dylib at path: %@", path]; + + if ([[NSFileManager defaultManager] isReadableFileAtPath:path]) { + return path; + } + + [BPUtils printInfo:ERROR withString: @"Unable to find libBPTestInspector.dylib to inject into xctest process and get test data"]; + return nil; +} + + (NSString *)mkdtemp:(NSString *)template withError:(NSError **)errPtr { char *dir = strdup([[template stringByAppendingString:@"_XXXXXX"] UTF8String]); if (mkdtemp(dir) == NULL) { @@ -418,6 +451,13 @@ + (NSDictionary *)loadSimpleJsonFile:(NSString *)filePath return testsToRunByFilePath; } ++ (double)timeoutForAllTestsWithConfiguration:(BPConfiguration *)config { + // Add 1 second per test + double buffer = 1.0; + NSInteger testCount = (config.testCasesToRun.count == 0 ? config.allTestCases.count : config.testCasesToRun.count) - config.testCasesToSkip.count; + return testCount * (config.testCaseTimeout.doubleValue + buffer); +} + + (double)getTotalTimeWithConfig:(BPConfiguration *)config testTimes:(NSDictionary *)testTimes andXCTestFiles:(NSArray *)xcTestFiles { @@ -438,4 +478,202 @@ + (double)getTotalTimeWithConfig:(BPConfiguration *)config return totalTime; } +#pragma mark - Errors + ++ (NSError *)errorWithSignalCode:(NSInteger)signalCode { + NSString *description = [NSString stringWithFormat:@"Process failed signal code: %@", @(signalCode)]; + return [self errorWithCode:signalCode description:description]; +} + ++ (NSError *)errorWithExitCode:(NSInteger)exitCode { + NSString *description = [NSString stringWithFormat:@"Process failed exit code: %@", @(exitCode)]; + return [self errorWithCode:exitCode description:description]; +} + ++ (NSError *)BPError:(const char *)function andLine:(int)line withFormat:(NSString *)fmt, ... { + va_list args; + va_start(args, fmt); + NSString *msg = [[NSString alloc] initWithFormat:fmt arguments:args]; + va_end(args); + return [self errorWithCode:-1 description:msg]; +} + ++ (NSError *)errorWithCode:(NSInteger)code description:(NSString *)description { + NSDictionary *userInfo = @{ + NSLocalizedDescriptionKey: description + }; + return [NSError errorWithDomain:BPErrorDomain code:code userInfo:userInfo]; +} + +#pragma mark - Architecture Helpers + +/** + We can isolate a single architecture out of a universal binary using the `lipo -extract` command. By doing so, we can + force an executable (such as XCTest) to always run w/ the architecture we expect. This is to avoid some funny business where + the architecture selected can be unexpected depending on multiple factors, such as Rosetta, xcode version, etc. + + @return the path of the new executable if possible + required, nil otherwise. In nil case, original executable should be used instead. + */ ++ (NSString *)lipoExecutableAtPath:(NSString *)path withContext:(BPExecutionContext *)context { + [BPUtils printInfo:INFO withString:@"Preparing to extract xctest executable into required architecture."]; + // If the executable isn't a universal binary, there's nothing we can do. If we don't + // support the test bundle type, we'll let it fail later naturally. + NSArray *executableArchitectures = [self availableArchitecturesForPath:path]; + BOOL isUniversalExecutable = [executableArchitectures containsObject:self.x86_64] && [executableArchitectures containsObject:self.arm64]; + if (!isUniversalExecutable) { + [BPUtils printInfo:INFO withString:@"xctest executable was not a universal binary. No extraction possible."]; + return nil; + } + // Now, get the test bundle's architecture. + NSString *bundlePath = context.config.testBundlePath; + NSString *testBundleName = [[bundlePath pathComponents].lastObject componentsSeparatedByString:@"."][0]; + NSString *testBundleBinaryPath = [bundlePath stringByAppendingPathComponent:testBundleName]; + NSArray *testBundleArchitectures = [self availableArchitecturesForPath:testBundleBinaryPath]; + BOOL isUniversalTestBundle = [testBundleArchitectures containsObject:self.x86_64] && [testBundleArchitectures containsObject:self.arm64]; + + // If the test bundle is a univeral binary, no need to lipo... xctest (regardless of the arch it's in) + // should be able to handle it. + if (isUniversalTestBundle) { + [BPUtils printInfo:INFO withString:@"Test bundle is a universal binary -- no architecture extraction required."]; + return nil; + } + + // If the test bundle's arch isn't supported by the sim, we're in an error state + NSArray *simArchitectures = [self architecturesSupportedByDevice:context.runner.device]; + if (![simArchitectures containsObject:testBundleArchitectures.firstObject]) { + [BPUtils printInfo:ERROR withString:@"The simulator being run does not support the test bundle's arch (%@)", testBundleArchitectures.firstObject]; + return nil; + } + + // Now that we've done any error checking, we can handle our real cases, + // based on what xctest would default to vs. what we need it to do. + // Note that the universal binary will launch in the same arch as the machine, + // rather than defaulting to the arch of the parent process. + // + // 1) The current arch is x86_64 + // a) We are in Rosetta -> xctest will default to arm64 + // b) We are not in Rosetta -> xctest will default to x86 + // 2) The current arch is arm64 -> xctest will default to arm64 + // + // We handle these accordingly: + // 1a) we lipo if the test bundle is an x86_64 binary + // 1b) no-op. ... x86 will get handled automatically, and we have to fail if test bundle is arm64. + // 2) no-op. ... arm64 will get handled automatically, and we have to fail if test bundle is x86. + BOOL isRosetta = [self.currentArchitecture isEqual:self.x86_64] && isUniversalExecutable; + [BPUtils printInfo:DEBUGINFO withString:@"lipo: currentArchitecture = %@", self.currentArchitecture]; + [BPUtils printInfo:DEBUGINFO withString:@"lipo: isUniversalExecutable = %@", @(isUniversalExecutable)]; + [BPUtils printInfo:DEBUGINFO withString:@"lipo: testBundleArchitectures = %@", testBundleArchitectures]; + [BPUtils printInfo:DEBUGINFO withString:@"lipo: isRosetta = %@", @(isRosetta)]; + if (!isRosetta || ![testBundleArchitectures.firstObject isEqual:self.x86_64]) { + [BPUtils printInfo:INFO withString:@"The test bundle matches the expected xctest architecture, so no arch extraction is required"]; + return nil; + } + + // Now we lipo. + [BPUtils printInfo:INFO withString:@"We are in Rosetta, but xctest will default to arm64. Extracting xctest x86_64 binary"]; + NSError *error; + NSString *fileName = [NSString stringWithFormat:@"%@xctest", NSTemporaryDirectory()]; + NSString *thinnedExecutablePath = [BPUtils mkstemp:fileName withError:&error]; + NSString *cmd = [NSString stringWithFormat:@"/usr/bin/lipo %@ -extract %@ -output %@", path, testBundleArchitectures.firstObject, thinnedExecutablePath]; + NSString *__unused output = [BPUtils runShell:cmd]; + return thinnedExecutablePath; +} + +/** + Lipo'ing the universal binary alone to isolate the desired architecture will result in errors. + Specifically, the newly lipo'ed binary won't be able to find any of the required frameworks + from within the original binary. So, we need to set up the `DYLD_FRAMEWORK_PATH` + in the environment to include the paths to these frameworks within the original universal + executable's binary. + */ ++ (NSString *)correctedDYLDFrameworkPathFromBinary:(NSString *)binaryPath { + NSString *otoolCommand = [NSString stringWithFormat:@"/usr/bin/otool -l %@", binaryPath]; + NSString *otoolInfo = [BPUtils runShell:otoolCommand]; + + /** + Example command: + `/usr/bin/otool -l /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/Library/Xcode/Agents/xctest` + + Example output looks something like this: + + ``` + // Lots of stuff we don't care about, followed by a list of `LC_RPATH` entries + Load command 18 + cmd LC_RPATH + cmdsize 48 + path @executable_path/path/to/frameworks/ (offset 12) + // Lots more stuff we don't care about... + ``` + + We want to use a regex to extract out each of the `@executable_path/path/to/frameworks/`, + and then replace `@executable_path` with our original executable's parent directory. + */ + NSString *pattern = @"(?:^Load command \\d+\n" + "\\s*cmd LC_RPATH\n" + "\\s*cmdsize \\d+\n" + "\\s*path (@executable_path\\/.*) \\(offset \\d+\\))+"; + NSError *error; + NSRegularExpression *regex = [NSRegularExpression regularExpressionWithPattern:pattern + options:NSRegularExpressionAnchorsMatchLines + error:&error]; + + NSMutableArray *paths = [NSMutableArray array]; + NSString *parentDirectory = [[binaryPath stringByResolvingSymlinksInPath] stringByDeletingLastPathComponent]; + if (regex) { + NSArray *matches = [regex matchesInString:otoolInfo + options:0 + range:NSMakeRange(0, otoolInfo.length)]; + for (NSTextCheckingResult *match in matches) { + // Extract the substring from the input string based on the matched range + NSString *relativePath = [otoolInfo substringWithRange:[match rangeAtIndex:1]]; + NSString *path = [relativePath stringByReplacingOccurrencesOfString:@"@executable_path" withString:parentDirectory]; + [paths addObject:path]; + } + } else { + [BPUtils printInfo:ERROR withString:@"Error creating regular expression: %@", error]; + } + return [paths componentsJoinedByString:@":"]; +} + ++ (NSArray *)availableArchitecturesForPath:(NSString *)path { + NSString *cmd = [NSString stringWithFormat:@"/usr/bin/lipo -archs %@", path]; + return [[BPUtils runShell:cmd] componentsSeparatedByString:@" "]; +} + ++ (NSArray *)architecturesSupportedByDevice:(SimDevice *)device { + NSArray *simSupportedArchitectures = device.deviceType.supportedArchs; + NSMutableArray *simArchitectures = [NSMutableArray array]; + for (NSNumber *supportedArchitecture in simSupportedArchitectures) { + [simArchitectures addObject:[self architectureName:supportedArchitecture.intValue]]; + } + return [simArchitectures copy]; +} + ++ (NSString *)arm64 { + return [self architectureName:CPU_TYPE_ARM64]; +} + ++ (NSString *)x86_64 { + return [self architectureName:CPU_TYPE_X86_64]; +} + ++ (NSString *)architectureName:(integer_t)architecture { + if (architecture == CPU_TYPE_X86_64) { + return @"x86_64"; + } else if (architecture == CPU_TYPE_ARM64) { + return @"arm64"; + } + return nil; +} + ++ (NSString *)currentArchitecture { + #if TARGET_CPU_ARM64 + return [self architectureName:CPU_TYPE_ARM64]; + #elif TARGET_CPU_X86_64 + return [self architectureName:CPU_TYPE_X86_64]; + #else + return nil; + #endif +} + @end diff --git a/bp/src/Bluepill.m b/bp/src/Bluepill.m index 82a471ad..cca344e0 100644 --- a/bp/src/Bluepill.m +++ b/bp/src/Bluepill.m @@ -21,8 +21,10 @@ #import #import "BPTMDControlConnection.h" #import "BPTMDRunnerConnection.h" +#import "BPTestInspectionHandler.h" #import "BPXCTestFile.h" #import +#import // CoreSimulator #import "PrivateHeaders/CoreSimulator/SimDevice.h" @@ -220,6 +222,16 @@ - (void)createContext { NSAssert(xctTestFile != nil, @"Failed to load testcases from: %@; Error: %@", context.config.testBundlePath, [error localizedDescription]); context.config.allTestCases = [[NSArray alloc] initWithArray: xctTestFile.allTestCases]; + if (context.config.isLogicTestTarget) { + // For estimating how long this will take (and setting an appropriate timeout), we need to + // remove any skipped tests that aren't actually a part of this bundle. + NSMutableSet *allTests = [NSMutableSet setWithArray:context.config.allTestCases]; + NSMutableSet *allSkippedTests = [NSMutableSet setWithArray:self.config.testCasesToSkip]; + [allSkippedTests intersectSet:allTests]; + context.config.testCasesToSkip = allSkippedTests.allObjects; + } + + context.attemptNumber = self.retries + 1; self.context = context; // Store the context on self so that it's accessible to the interrupt handler in the loop } @@ -256,7 +268,7 @@ - (void)setupExecutionWithContext:(BPExecutionContext *)context { // Set up retry counts. self.maxCreateTries = [self.config.maxCreateTries integerValue]; self.maxInstallTries = [self.config.maxInstallTries integerValue]; - + if (context.config.deleteSimUDID) { NEXT([self deleteSimulatorOnlyTaskWithContext:context]); } else { @@ -299,13 +311,8 @@ - (void)createSimulatorWithContext:(BPExecutionContext *)context { if (self.config.scriptFilePath) { [context.runner runScriptFile:self.config.scriptFilePath]; } - if (self.config.cloneSimulator) { - // launch application directly when clone simulator - NEXT([__self launchApplicationWithContext:context]); - } else { - // Install application when test without clone - NEXT([__self installApplicationWithContext:context]); - }; + + NEXT([__self enumerateTestsWithContext:context]); }; handler.onError = ^(NSError *error) { @@ -393,6 +400,178 @@ - (void)uninstallApplicationWithContext:(BPExecutionContext *)context { } } +/** + On Apple Silicon machines, the xctest executable is likely to be a universal binary. Strangely, however, this can cause issues when + Xcode is being run in Rosetta mode with an `x86_64` test bundle. While it would seem those would be compatible, the simulator + won't naturally respect that it was launched from a Rosetta process, and will try to boot using the `arm64` binary by default. + As a result, in such a scenario we need to do some work on our side to get a modified xctest binary that will boot using x86. + */ +- (void)adaptXCTestExecutableIfRequiredWithContext:(BPExecutionContext *)context { + // First we need to make sure the archs are consistent between the xctest executable + the test bundle. + NSString *originalXCTestPath = [[NSString alloc] initWithFormat: + @"%@/Platforms/iPhoneSimulator.platform/Developer/Library/Xcode/Agents/xctest", + context.config.xcodePath]; + NSString *newXCTestPath = [BPUtils lipoExecutableAtPath:originalXCTestPath withContext:context]; + if (newXCTestPath) { + context.config.dyldFrameworkPath = [BPUtils correctedDYLDFrameworkPathFromBinary:originalXCTestPath]; + } + context.config.xctestBinaryPath = newXCTestPath ?: originalXCTestPath; +} + +/** + Will spawn a child xctest execution with an injected dylib to pipe out information about all the tests in a test bundle. + No tests will be run. + */ +- (void)enumerateTestsWithContext:(BPExecutionContext *)context { + NSString *stepName = TEST_INSPECTION(context.attemptNumber); + [BPUtils printInfo:INFO withString:@"%@", stepName]; + + // Now create handler for when the process completes + + [[BPStats sharedStats] startTimer:stepName]; + BPWaitTimer *timer = [BPWaitTimer timerWithInterval:[context.config.launchTimeout doubleValue]]; + [timer start]; + + // Set up completion handler for when we load the tests. + BPTestInspectionHandler *completionHandler = [BPTestInspectionHandler handlerWithTimer:timer]; + __weak typeof(self) __self = self; + completionHandler.onSuccess = ^(NSArray *testBundleInfo) { + // Note, we can't actually handle the tests themselves here... that will + // need to be handled in a wrapping block below. + [BPUtils printInfo:DEBUGINFO withString:@"Test bundle inspected for tests."]; + [[BPStats sharedStats] endTimer:stepName withResult:@"INFO"]; + + // Save the test info. + if (testBundleInfo) { + NSMutableDictionary *dictionary = [NSMutableDictionary dictionary]; + for (BPTestCaseInfo *info in testBundleInfo) { + dictionary[info.prettifiedFullName] = info; + } + __self.config.allTests = [dictionary copy]; + context.config.allTests = [dictionary copy]; + } + + if (self.config.isLogicTestTarget) { + NEXT([__self executeLogicTestsWithContext:context]) + } else if (self.config.cloneSimulator) { + // launch application directly when clone simulator + NEXT([__self launchApplicationWithContext:context]); + } else { + // Install application when test without clone + NEXT([__self installApplicationWithContext:context]); + }; + }; + + completionHandler.onError = ^(NSError *error) { + [[BPStats sharedStats] endTimer:stepName withResult:@"ERROR"]; + [BPUtils printInfo:ERROR withString:@"Spawned logic test execution failed: %@", [error localizedDescription]]; + + BPExitStatus exitStatus = BPExitStatusLaunchAppFailed; + NEXT([__self deleteSimulatorWithContext:context andStatus:exitStatus]); + }; + + completionHandler.onTimeout = ^{ + [[BPStats sharedStats] endTimer:stepName withResult:@"TIMEOUT"]; + [BPUtils printInfo:FAILED withString:@"Timeout: %@", stepName]; + }; + + // If test cases are already enumerated, skip straight to executing tests. + if (context.config.allTests) { + completionHandler.defaultHandlerBlock(context.config.allTests.allValues, nil); + return; + } + + // Otherwise, make sure XCTest binary is formatted correctly for current arch. Then get test info. + [self adaptXCTestExecutableIfRequiredWithContext:context]; + [context.runner collectTestSuiteInfoWithCompletion:^(NSArray *testBundleInfo, NSError *error) { + completionHandler.defaultHandlerBlock(testBundleInfo, error); + }]; +} + +- (void)executeLogicTestsWithContext:(BPExecutionContext *)context { + [self adaptXCTestExecutableIfRequiredWithContext:context]; + + // We get two callbacks after trying to spawn an execution. They happen when: + // 1) Immediately after the process is spawned. + // 2) Once the process completes. + // This method sets up those two handlers, and then passes them to the runner. + + // 1) Handle process spawn + NSString *spawnStepName = SPAWN_LOGIC_TEST(context.attemptNumber); + NSString *executeTestsStepName = EXECUTE_LOGIC_TEST(context.attemptNumber); + [BPUtils printInfo:INFO withString:@"%@", spawnStepName]; + + // Now create handler for when the process completes + + [[BPStats sharedStats] startTimer:spawnStepName]; + BPWaitTimer *spawnTimer = [BPWaitTimer timerWithInterval:[context.config.launchTimeout doubleValue]]; + __block BPWaitTimer *executionTimer = [BPWaitTimer timerWithInterval:[BPUtils timeoutForAllTestsWithConfiguration:context.config]]; + // Start spawn timer; save execution timer for the spawn handler. + [spawnTimer start]; + BPApplicationLaunchHandler *spawnHandler = [BPApplicationLaunchHandler handlerWithTimer:spawnTimer]; + + __weak typeof(self) __self = self; + __weak typeof(spawnHandler) __spawnHandler = spawnHandler; + spawnHandler.beginWith = ^{ + [BPUtils printInfo:((__spawnHandler.pid > -1) ? INFO : ERROR) withString:@"Completed: %@", spawnStepName]; + }; + spawnHandler.onSuccess = ^{ + context.pid = __spawnHandler.pid; + + // At this point, we know that the execution itself has started. + [BPUtils printInfo:DEBUGINFO withString:@"XCTest execution spawned; waiting for execution to complete."]; + [[BPStats sharedStats] endTimer:spawnStepName withResult:@"INFO"]; + + // Start timer for actual xctest execution. + [BPUtils printInfo:INFO withString:@"%@", executeTestsStepName]; + [[BPStats sharedStats] startTimer:executeTestsStepName]; + [executionTimer start]; + }; + spawnHandler.onError = ^(NSError *error) { + [[BPStats sharedStats] endTimer:spawnStepName withResult:@"ERROR"]; + [BPUtils printInfo:ERROR withString:@"Could not spawn logic test execution: %@", [error localizedDescription]]; + NEXT([__self deleteSimulatorWithContext:context andStatus:BPExitStatusLaunchAppFailed]); + }; + spawnHandler.onTimeout = ^{ + [[BPStats sharedStats] addSimulatorLaunchFailure]; + [[BPStats sharedStats] endTimer:spawnStepName withResult:@"TIMEOUT"]; + [BPUtils printInfo:FAILED withString:@"Timeout: %@", spawnStepName]; + }; + + // 2) Handle execution finished + + // Now the handler for when the execution actually completes. + BPApplicationLaunchHandler *completionHandler = [BPApplicationLaunchHandler handlerWithTimer:executionTimer]; + completionHandler.onSuccess = ^{ + [BPUtils printInfo:DEBUGINFO withString:@"XCTest execution completed. Results must now be parsed."]; + [[BPStats sharedStats] endTimer:executeTestsStepName withResult:@"INFO"]; + NEXT([__self checkUnhostedProcessWithContext:context]); + }; + + completionHandler.onError = ^(NSError *error) { + [[BPStats sharedStats] endTimer:executeTestsStepName withResult:@"ERROR"]; + [BPUtils printInfo:ERROR withString:@"Spawned logic test execution failed: %@", [error localizedDescription]]; + + BPExitStatus exitStatus = BPExitStatusAppCrashed; + BOOL didhanderTimeout = error.domain == BPErrorDomain && error.code == BPHandler.timeoutErrorCode; + BOOL wasProcessKilledAfterTimeout = error.domain == BPErrorDomain && error.code == SIGKILL; + if (didhanderTimeout || wasProcessKilledAfterTimeout) { + exitStatus = BPExitStatusTestTimeout; + } + NEXT([__self deleteSimulatorWithContext:context andStatus:exitStatus]); + }; + + completionHandler.onTimeout = ^{ + [[BPStats sharedStats] addTestRuntimeTimeout]; + [[BPStats sharedStats] endTimer:executeTestsStepName withResult:@"TIMEOUT"]; + [BPUtils printInfo:FAILED withString:@"Timeout: %@", executeTestsStepName]; + }; + + [context.runner executeLogicTestsWithParser:context.parser + onSpawn:spawnHandler.defaultHandlerBlock + andCompletion:completionHandler.defaultHandlerBlock]; +} + - (void)launchApplicationWithContext:(BPExecutionContext *)context { NSString *stepName = LAUNCH_APPLICATION(context.attemptNumber); [BPUtils printInfo:INFO withString:@"%@", stepName]; @@ -448,6 +627,7 @@ - (void)connectTestBundleAndTestDaemonWithContext:(BPExecutionContext *)context NEXT([self checkProcessWithContext:context]); } + - (void)checkProcessWithContext:(BPExecutionContext *)context { BOOL isRunning = [self isProcessRunningWithContext:context]; if (!isRunning && [context.runner isFinished]) { @@ -470,7 +650,7 @@ - (void)checkProcessWithContext:(BPExecutionContext *)context { // If it's not running and we passed the above checks (e.g., the tests are not yet completed) // then it must mean the app has crashed. // However, we have a short-circuit for tests because those may not actually run any app - if (!isRunning && context.pid > 0 && [context.runner isApplicationLaunched] && !self.config.testing_NoAppWillRun) { + if (!isRunning && context.pid > 0 && ([context.runner isApplicationLaunched] || context.config.isLogicTestTarget) && !self.config.testing_NoAppWillRun) { // The tests ended before they even got started or the process is gone for some other reason [[BPStats sharedStats] endTimer:LAUNCH_APPLICATION(context.attemptNumber) withResult:@"APP CRASHED"]; [BPUtils printInfo:ERROR withString:@"Application crashed!"]; @@ -482,6 +662,30 @@ - (void)checkProcessWithContext:(BPExecutionContext *)context { NEXT_AFTER(1, [self checkProcessWithContext:context]); } +- (void)checkUnhostedProcessWithContext:(BPExecutionContext *)context { + // Handle tests + parsing is complete + BOOL isRunning = [self isProcessRunningWithContext:context]; + if (!isRunning && [context.runner isFinished]) { + [BPUtils printInfo:INFO withString:@"Finished test execution."]; + [[BPStats sharedStats] endTimer:EXECUTE_LOGIC_TEST(context.attemptNumber) withResult:[BPExitStatusHelper stringFromExitStatus:context.exitStatus]]; + [self runnerCompletedWithContext:context]; + return; + } + // Handle Simulator Crash + if (![context.runner isSimulatorRunning]) { + [[BPStats sharedStats] endTimer:EXECUTE_LOGIC_TEST(context.attemptNumber) withResult:@"SIMULATOR CRASHED"]; + [BPUtils printInfo:ERROR withString:@"SIMULATOR CRASHED!!!"]; + context.simulatorCrashed = YES; + [[BPStats sharedStats] addSimulatorCrash]; + [self deleteSimulatorWithContext:context andStatus:BPExitStatusSimulatorCrashed]; + return; + } + + // Even though the execution has completed, the parser may not be done yet. + // If we're here, that's the case... so try again in a second. + NEXT_AFTER(1, [self checkUnhostedProcessWithContext:context]); +} + - (BOOL)isProcessRunningWithContext:(BPExecutionContext *)context { if (self.config.testing_NoAppWillRun) { return NO; @@ -520,7 +724,7 @@ - (void)runnerCompletedWithContext:(BPExecutionContext *)context { [BPUtils printInfo:INFO withString:@"Saving Diagnostics for Debugging"]; [BPUtils saveDebuggingDiagnostics:_config.outputDirectory]; } - + [self deleteSimulatorWithContext:context andStatus:[context.runner exitStatus]]; } } @@ -702,6 +906,7 @@ - (NSString *)debugDescription { } #pragma mark - BPTestBundleConnectionDelegate + - (void)_XCT_launchProcessWithPath:(NSString *)path bundleID:(NSString *)bundleID arguments:(NSArray *)arguments environmentVariables:(NSDictionary *)environment { self.context.isTestRunnerContext = YES; [self installApplicationWithContext:self.context]; diff --git a/bp/src/PrivateHeaders/CoreSimulator/SimDevice.h b/bp/src/PrivateHeaders/CoreSimulator/SimDevice.h index 6619b46d..05a082da 100644 --- a/bp/src/PrivateHeaders/CoreSimulator/SimDevice.h +++ b/bp/src/PrivateHeaders/CoreSimulator/SimDevice.h @@ -11,6 +11,36 @@ typedef void (^CDUnknownBlockType)(void); typedef void (*CDUnknownFunctionPointerType)(void); @class NSArray, NSDate, NSDictionary, NSMachPort, NSMutableArray, NSMutableDictionary, NSObject, SimDeviceIO, NSString, NSUUID, SimDeviceBootInfo, SimDeviceNotificationManager, SimDevicePasteboard, SimDeviceSet, SimDeviceType, SimRuntime; +/** + A note on how to investigate expected usage for these private, class-dumped Apple APIs: + + It is often easier to get the below commands running in their commandline, `simctl` form first, in particular because + this is a public API that is (somewhat) documented. Fortunately for us, `simctl` uses `CoreSimulator` as well + as the APIs defined here in `SimDevice.h` under the hood, which we can use to our advantage to learn more about + expected usage. + + As an example, the below steps were used to learn more about the expected usage for the various `spawn...` methods + defined in this class, leveraging lldb: + + 1. Load lldb with a process such as `xcrun simctl spawn -s booted -XCTest All <.xctest_file>` + 2. It should pause the process before starting. Then add breakpoints to all variants of `spawn` from `SimDevice.h`, including: + a) `breakpoint set --selector spawnWithPath:options:terminationQueue:terminationHandler:pid:error:` + b) `breakpoint set --selector spawnWithPath:options:terminationQueue:terminationHandler:error:` + c) `breakpoint set --selector spawnWithPath:options:terminationHandler:error:` + d) `breakpoint set --selector _spawnFromSelfWithPath:options:terminationQueue:terminationHandler:error:` + e) `breakpoint set --selector _spawnFromLaunchdWithPath:options:terminationQueue:terminationHandler:error:` + f) `breakpoint set --selector _onBootstrapQueue_spawnWithPath:options:terminationQueue:terminationHandler:erro` + g) `breakpoint set --selector spawnWithPath:options:terminationQueue:terminationHandler:pid:error:` + h) `breakpoint set --selector spawnWithPath:options:terminationQueue:terminationHandler:error:` + 3. These breakpoints should be `pending`, as CoreSimulator won't have been loaded yet. Hit `c` to continue and unpause the process, + and continue to `c` through automatically triggered pauses as additional dylibs are loaded. + 4. Eventually you'll hit one of the above selectors' breakpoint. Use `register read` to print out all the registers to find what all's in frame. + 5. Then use `po ` for the register values to see what all objc classes are stored. + 6. Eventually, you'll find the values for the `path` and `options` arguments, or whatever other parameters you want to inspect. + + With this information, you can learn more how a given `simctl` command maps onto its corresponding `SimDevice` method :) + */ + @interface SimDevice : NSObject { unsigned long long _state; @@ -192,7 +222,7 @@ typedef void (*CDUnknownFunctionPointerType)(void); - (void)triggerCloudSyncWithCompletionHandler:(CDUnknownBlockType)arg1; - (void)launchApplicationAsyncWithID:(id)arg1 options:(id)arg2 completionHandler:(void (^)(NSError *, pid_t pid))arg3; - (int)spawnWithPath:(id)arg1 options:(id)arg2 terminationHandler:(CDUnknownBlockType)arg3 error:(id *)arg4; -- (void)spawnAsyncWithPath:(id)arg1 options:(id)arg2 terminationHandler:(CDUnknownBlockType)arg3 completionHandler:(CDUnknownBlockType)arg4; +- (void)spawnAsyncWithPath:(id)arg1 options:(id)arg2 terminationHandler:(void (^)(int))arg3 completionHandler:(void (^)(NSError *, pid_t pid))arg4; - (void)restoreContentsAndSettingsAsyncFromDevice:(id)arg1 completionHandler:(CDUnknownBlockType)arg2; - (void)eraseContentsAndSettingsAsyncWithCompletionHandler:(CDUnknownBlockType)arg1; - (void)renameAsync:(id)arg1 completionHandler:(CDUnknownBlockType)arg2; diff --git a/bp/src/SimulatorHelper.h b/bp/src/SimulatorHelper.h index a17227f6..a7f1ffdf 100644 --- a/bp/src/SimulatorHelper.h +++ b/bp/src/SimulatorHelper.h @@ -20,6 +20,16 @@ */ + (BOOL)loadFrameworksWithXcodePath:(NSString *)xcodePath; +/*! + * @discussion get xctest launch environment + * @param hostBundleID the bundleID of the host app + * @param device the device to run test + * @param config the configuration object + * @return returns the app launch environment as a dictionary + */ ++ (NSDictionary *)logicTestEnvironmentWithConfig:(BPConfiguration *)config + stdoutRelativePath:(NSString *)path; + /*! * @discussion get app launch environment * @param hostBundleID the bundleID of the host app @@ -30,6 +40,13 @@ + (NSDictionary *)appLaunchEnvironmentWithBundleID:(NSString *)hostBundleID device:(SimDevice *)device config:(BPConfiguration *)config; +/*! + * @discussion Creates an array of all tests that should be run, filtering out any tests that should be skipped. + * @param config the configuration object. + * @return The list of tests to run. + */ + ++ (NSArray *)testsToRunWithConfig:(BPConfiguration *)config; /*! * @discussion get the path of the environment configuration file @@ -38,6 +55,21 @@ */ + (NSString *)testEnvironmentWithConfiguration:(BPConfiguration *)config; +/*! + @discussion Creates a stdout file on the provided device of the form `/tmp/stdout_stderr_` + @param device The device to create the file on. + @return the path of the stdout file. + */ ++ (NSString *)makeStdoutFileOnDevice:(SimDevice *)device; + +/*! + @discussion Creates an output file on the provided device of the form `/tmp/BPTestInspector_testInfo_`. + This is meant to store the test info output generated by the injected test wrapper in an xctest execution. + @param device The device to create the file on. + @return the path of the output file. + */ ++ (NSString *)makeTestWrapperOutputFileOnDevice:(SimDevice *)device; + #pragma mark - Path Helper + (NSString *)bundleIdForPath:(NSString *)path; diff --git a/bp/src/SimulatorHelper.m b/bp/src/SimulatorHelper.m index d4d5cf25..26396420 100644 --- a/bp/src/SimulatorHelper.m +++ b/bp/src/SimulatorHelper.m @@ -11,9 +11,11 @@ #import "BPConfiguration.h" #import "BPUtils.h" #import "BPXCTestFile.h" +#import "SimDevice.h" #import "PrivateHeaders/XCTest/XCTestConfiguration.h" #import "PrivateHeaders/XCTest/XCTTestIdentifier.h" #import "PrivateHeaders/XCTest/XCTTestIdentifierSet.h" +#import @implementation SimulatorHelper @@ -61,6 +63,22 @@ + (BOOL)loadFrameworksWithXcodePath:(NSString *)xcodePath { return YES; } ++ (NSDictionary *)logicTestEnvironmentWithConfig:(BPConfiguration *)config + stdoutRelativePath:(NSString *)path { + NSMutableDictionary *environment = [@{ + kOptionsStdoutKey: path, + kOptionsStderrKey: path, + } mutableCopy]; + if (config.dyldFrameworkPath) { + environment[@"DYLD_FRAMEWORK_PATH"] = config.dyldFrameworkPath; + // DYLD_LIBRARY_PATH is required specifically for swift tests, which require libXCTestSwiftSupport, + // which must be findable in the library path. + environment[@"DYLD_LIBRARY_PATH"] = config.dyldFrameworkPath; + } + [environment addEntriesFromDictionary:config.environmentVariables]; + return [environment copy]; +} + + (NSDictionary *)appLaunchEnvironmentWithBundleID:(NSString *)hostBundleID device:(SimDevice *)device config:(BPConfiguration *)config { @@ -168,6 +186,70 @@ + (NSString *)bundleIdForPath:(NSString *)path { return bundleId; } ++ (NSArray *)testsToRunWithConfig:(BPConfiguration *)config { + // First, standardize all swift test names: + NSMutableArray *allTests = [self formatTestNamesForXCTest:config.allTestCases withConfig:config]; + NSMutableArray *testsToRun = [self formatTestNamesForXCTest:config.testCasesToRun withConfig:config]; + NSArray *testsToSkip = [self formatTestNamesForXCTest:config.testCasesToSkip withConfig:config]; + + // If there's no tests to skip, we can return these back otherwise unaltered. + // Otherwise, we'll need to remove any tests from `testsToRun` that are in our skip list. + if (testsToSkip.count == 0) { + return [(testsToRun ?: allTests) copy]; + } + + // If testCasesToRun was empty/nil, we default to all tests + if (testsToRun.count == 0) { + testsToRun = allTests; + } + [testsToRun filterUsingPredicate:[NSPredicate predicateWithBlock:^BOOL(NSString *testName, NSDictionary * _Nullable bindings) { + return ![testsToSkip containsObject:testName]; + }]]; + return [testsToRun copy]; +} + ++ (nullable NSMutableArray *)formatTestNamesForXCTest:(nullable NSArray *)tests + withConfig:(BPConfiguration *)config { + if (!tests) { + return nil; + } + [BPUtils printInfo:DEBUGINFO withString:@"Formatting test names. config.allTests: %@", config.allTests]; + NSMutableArray *formattedTests = [NSMutableArray array]; + for (NSString *testName in tests) { + BPTestCaseInfo *info = config.allTests[testName]; + if (info) { + [formattedTests addObject:info.standardizedFullName]; + } else { + [BPUtils printInfo:DEBUGINFO withString:@"Omitting false positive test method from test list: %@", testName]; + } + } + return formattedTests; +} + +// Intercept stdout, stderr and post as simulator-output events ++ (NSString *)makeStdoutFileOnDevice:(SimDevice *)device { + NSString *stdout_stderr = [NSString stringWithFormat:@"%@/tmp/stdout_stderr_%@", device.dataPath, [[device UDID] UUIDString]]; + return [self createFileWithPathTemplate:stdout_stderr]; +} + ++ (NSString *)makeTestWrapperOutputFileOnDevice:(SimDevice *)device { + NSString *stdout_stderr = [NSString stringWithFormat:@"%@/tmp/BPTestInspector_testInfo_%@", device.dataPath, [[device UDID] UUIDString]]; + return [self createFileWithPathTemplate:stdout_stderr]; +} + ++ (NSString *)createFileWithPathTemplate:(NSString *)pathTemplate { + NSString *fullPath = [BPUtils mkstemp:pathTemplate withError:nil]; + assert(fullPath != nil); + + [[NSFileManager defaultManager] removeItemAtPath:fullPath error:nil]; + + // Create empty file so we can tail it and the app can write to it + [[NSFileManager defaultManager] createFileAtPath:fullPath + contents:nil + attributes:nil]; + return fullPath; +} + + (NSString *)executablePathforPath:(NSString *)path { NSDictionary *appDic = [NSDictionary dictionaryWithContentsOfFile:[path stringByAppendingPathComponent:@"Info.plist"]]; NSString *appExecutable = [appDic objectForKey:(NSString *)kCFBundleExecutableKey]; diff --git a/bp/src/SimulatorMonitor.m b/bp/src/SimulatorMonitor.m index c8a5fad7..b7eec377 100644 --- a/bp/src/SimulatorMonitor.m +++ b/bp/src/SimulatorMonitor.m @@ -86,7 +86,10 @@ - (void)onTestCaseBeganWithName:(NSString *)testName inClass:(NSString *)testCla dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(self.maxTestExecutionTime * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{ if ([__self.currentTestName isEqualToString:testName] && [__self.currentClassName isEqualToString:testClass] && __self.testsState == Running) { [BPUtils printInfo:TIMEOUT withString:@"%10.6fs %@/%@", __self.maxTestExecutionTime, testClass, testName]; - [__self stopTestsWithErrorMessage:@"Test took too long to execute and was aborted." forTestName:testName inClass:testClass]; + [__self stopTestsWithErrorMessage:@"Test took too long to execute and was aborted." + forTestName:testName + inClass:testClass + shouldRetryTest:YES]; __self.exitStatus = BPExitStatusTestTimeout; [[BPStats sharedStats] endTimer:[NSString stringWithFormat:TEST_CASE_FORMAT, [BPStats sharedStats].attemptNumber, testClass, testName] withResult:@"ERROR"]; [[BPStats sharedStats] addTestRuntimeTimeout]; @@ -100,6 +103,9 @@ - (void)onTestCasePassedWithName:(NSString *)testName inClass:(NSString *)testCl [BPUtils printInfo:PASSED withString:@"%10.6fs %@/%@", [currentTime timeIntervalSinceDate:self.lastTestCaseStartDate], testClass, testName]; + NSLog(@"%10.6fs %@/%@", + [currentTime timeIntervalSinceDate:self.lastTestCaseStartDate], + testClass, testName); // Passing or failing means that if the simulator crashes later, we shouldn't rerun this test. [self updateExecutedTestCaseList:testName inClass:testClass]; @@ -189,7 +195,7 @@ - (void)onOutputReceived:(NSString *)output { __block NSUInteger previousOutputId = self.currentOutputId; __weak typeof(self) __self = self; - // App crashed + // App or logic test's XCTest execution crashed. if ([output isEqualToString:@"BP_APP_PROC_ENDED"]) { if (__self.testsState == Running || __self.testsState == Idle) { NSString *testClass = (__self.currentClassName ?: __self.previousClassName); @@ -208,7 +214,8 @@ - (void)onOutputReceived:(NSString *)output { } [self stopTestsWithErrorMessage:@"App Crashed" forTestName:(self.currentTestName ?: self.previousTestName) - inClass:(self.currentClassName ?: self.previousClassName)]; + inClass:(self.currentClassName ?: self.previousClassName) + shouldRetryTest:self.config.retryAppCrashTests]; self.exitStatus = BPExitStatusAppCrashed; [[BPStats sharedStats] addApplicationCrash]; } @@ -231,17 +238,18 @@ - (void)onOutputReceived:(NSString *)output { __self.exitStatus = testsReallyStarted ? BPExitStatusTestTimeout : BPExitStatusSimulatorCrashed; [__self stopTestsWithErrorMessage:@"Timed out waiting for the test to produce output. Test was aborted." forTestName:testName - inClass:testClass]; + inClass:testClass + shouldRetryTest:YES]; [[BPStats sharedStats] addTestOutputTimeout]; } }); self.lastOutput = currentTime; } -- (void)stopTestsWithErrorMessage:(NSString *)message forTestName:(NSString *)testName inClass:(NSString *)testClass { +- (void)stopTestsWithErrorMessage:(NSString *)message forTestName:(NSString *)testName inClass:(NSString *)testClass shouldRetryTest:(BOOL)shouldRetryTest { // Timeout or crash on a test means we should skip it when we rerun the tests, unless we've enabled re-running failed tests - if (!self.config.onlyRetryFailed) { + if (!shouldRetryTest) { [self updateExecutedTestCaseList:testName inClass:testClass]; } if (self.appState == Running && !self.config.testing_NoAppWillRun) { diff --git a/bp/tests/BPCLITests.m b/bp/tests/BPCLITests.m index 8ef51a7b..24a80378 100644 --- a/bp/tests/BPCLITests.m +++ b/bp/tests/BPCLITests.m @@ -43,6 +43,7 @@ - (void)testListArguments { [config saveOpt:[NSNumber numberWithInt:'N'] withArg:[NSString stringWithUTF8String:"foo"]]; [config saveOpt:[NSNumber numberWithInt:'N'] withArg:[NSString stringWithUTF8String:"bar"]]; [config saveOpt:[NSNumber numberWithInt:'N'] withArg:[NSString stringWithUTF8String:"baz"]]; + [config saveOpt:[NSNumber numberWithInt:369] withArg:@"YES"]; NSError *err; BOOL result; @@ -54,6 +55,7 @@ - (void)testListArguments { XCTAssertEqualObjects(config.errorRetriesCount, @2); XCTAssertEqualObjects(config.failureTolerance, @1); XCTAssertEqualObjects(config.numSims, @5); + XCTAssert(config.isLogicTestTarget); } - (void)testIgnoringAdditionalTestBundles { diff --git a/bp/tests/BPIntTestCase.m b/bp/tests/BPIntTestCase.m index a800a44f..7cd71e86 100644 --- a/bp/tests/BPIntTestCase.m +++ b/bp/tests/BPIntTestCase.m @@ -13,6 +13,7 @@ #import "BPIntTestCase.h" #import "BPConfiguration.h" #import "BPTestHelper.h" +#import "BPTestUtils.h" #import "BPUtils.h" #import "SimDeviceType.h" #import "SimRuntime.h" @@ -24,47 +25,8 @@ - (void)setUp { [super setUp]; self.continueAfterFailure = NO; - NSString *hostApplicationPath = [BPTestHelper sampleAppPath]; - NSString *testBundlePath = [BPTestHelper sampleAppNegativeTestsBundlePath]; - self.config = [[BPConfiguration alloc] initWithProgram:BP_BINARY]; - self.config.testBundlePath = testBundlePath; - self.config.appBundlePath = hostApplicationPath; - self.config.stuckTimeout = @40; - self.config.xcodePath = [BPUtils runShell:@"/usr/bin/xcode-select -print-path"]; - self.config.runtime = @BP_DEFAULT_RUNTIME; - self.config.repeatTestsCount = @1; - self.config.errorRetriesCount = @0; - self.config.testCaseTimeout = @20; - self.config.deviceType = @BP_DEFAULT_DEVICE_TYPE; - self.config.headlessMode = YES; - self.config.videoPaths = @[[BPTestHelper sampleVideoPath]]; - self.config.testRunnerAppPath = nil; - self.config.testing_CrashAppOnLaunch = NO; - self.config.cloneSimulator = NO; [BPUtils quietMode:[BPUtils isBuildScript]]; [BPUtils enableDebugOutput:NO]; - - NSError *err; - SimServiceContext *sc = [SimServiceContext sharedServiceContextForDeveloperDir:self.config.xcodePath error:&err]; - if (!sc) { NSLog(@"Failed to initialize SimServiceContext: %@", err); } - - for (SimDeviceType *type in [sc supportedDeviceTypes]) { - if ([[type name] isEqualToString:self.config.deviceType]) { - self.config.simDeviceType = type; - break; - } - } - - XCTAssert(self.config.simDeviceType != nil); - - for (SimRuntime *runtime in [sc supportedRuntimes]) { - if ([[runtime name] containsString:self.config.runtime]) { - self.config.simRuntime = runtime; - break; - } - } - - XCTAssert(self.config.simRuntime != nil); } @end diff --git a/bp/tests/BPReportTests.m b/bp/tests/BPReportTests.m index 496eb3fd..c849ad1d 100644 --- a/bp/tests/BPReportTests.m +++ b/bp/tests/BPReportTests.m @@ -14,13 +14,17 @@ #import "BPSimulator.h" #import "BPTestHelper.h" #import "BPUtils.h" +#import "BPTestUtils.h" @interface BPReportTests : BPIntTestCase @end @implementation BPReportTests - +- (void)setUp { + [super setUp]; + self.config = [BPTestUtils makeHostedTestConfiguration]; +} - (void)testReportWithAppCrashingTestsSet { [BPUtils enableDebugOutput:NO]; diff --git a/bp/tests/BPTestHelper.h b/bp/tests/BPTestHelper.h index b4d4ffcc..aa7ee7a7 100644 --- a/bp/tests/BPTestHelper.h +++ b/bp/tests/BPTestHelper.h @@ -17,6 +17,22 @@ // Return the path to the test plan json file. The json is packed into the app bundle as resource + (NSString *)testPlanPath; +// Return the path to logic tests, that are run unhosted rather than on the Sample App ++ (NSString *)logicTestBundlePath; + +// Return the path to logic tests, that are run unhosted rather than on the Sample App +// This particular bundle will only have passing tests to make certain functionalities easier to test. ++ (NSString *)passingLogicTestBundlePath; + +// A pre-built test bundle that will always be in x86_64 ++ (NSString *)logicTestBundlePath_x86_64; + +// A pre-built test bundle containing swift tests that will always be in x86_64. ++ (NSString *)logicTestBundlePath_swift_x86_64; + +// A pre-built test bundle that will always be in arm64 ++ (NSString *)logicTestBundlePath_arm64; + // Return the path to the sample app's xctest with new test cases + (NSString *)sampleAppNewTestsBundlePath; diff --git a/bp/tests/BPTestHelper.m b/bp/tests/BPTestHelper.m index 6ff7c84a..d50da15c 100644 --- a/bp/tests/BPTestHelper.m +++ b/bp/tests/BPTestHelper.m @@ -20,6 +20,29 @@ + (NSString *)testPlanPath { return [[self sampleAppPath] stringByAppendingPathComponent:@"test_plan.json"]; } +// Return the path to logic tests, that are run unhosted rather than on the Sampple App ++ (NSString *)logicTestBundlePath { + return [[self sampleAppPath] stringByAppendingPathComponent:@"/../BPLogicTests.xctest"]; +} + +// Return the path to logic tests, that are run unhosted rather than on the Sampple App +// This particular bundle will only have passing tests to make certain functionalities easier to test. ++ (NSString *)passingLogicTestBundlePath { + return [[self sampleAppPath] stringByAppendingPathComponent:@"/../BPPassingLogicTests.xctest"]; +} + ++ (NSString *)logicTestBundlePath_x86_64 { + return [[NSBundle bundleWithIdentifier:@"LI.BluepillRunnerTests"] pathForResource:@"BPLogicTestFixture_x86_64" ofType:@"xctest"]; +} + ++ (NSString *)logicTestBundlePath_swift_x86_64 { + return [[NSBundle bundleWithIdentifier:@"LI.BluepillRunnerTests"] pathForResource:@"BPLogicTestFixture_swift_x86_64" ofType:@"xctest"]; +} + ++ (NSString *)logicTestBundlePath_arm64 { + return [[NSBundle bundleWithIdentifier:@"LI.BluepillRunnerTests"] pathForResource:@"BPLogicTestFixture_arm64" ofType:@"xctest"]; +} + // Return the path to the sample app's xctest with new test cases + (NSString *)sampleAppNewTestsBundlePath { return [[self sampleAppPath] stringByAppendingString:@"/PlugIns/BPSampleAppNewTests.xctest"]; diff --git a/bp/tests/BluepillTests.m b/bp/tests/BluepillHostedTests.m similarity index 98% rename from bp/tests/BluepillTests.m rename to bp/tests/BluepillHostedTests.m index 1c0be3f9..85988d14 100644 --- a/bp/tests/BluepillTests.m +++ b/bp/tests/BluepillHostedTests.m @@ -16,6 +16,7 @@ #import "BPTestHelper.h" #import "BPUtils.h" #import "BPSimulator.h" +#import "BPTestUtils.h" #import "SimDevice.h" @@ -24,15 +25,14 @@ * - Exit code testing * - Report validation */ -@interface BluepillTests : BPIntTestCase +@interface BluepillHostedTests : BPIntTestCase @end -@implementation BluepillTests +@implementation BluepillHostedTests - - -- (void)tearDown { - [super tearDown]; +- (void)setUp { + [super setUp]; + self.config = [BPTestUtils makeHostedTestConfiguration]; } - (void)testAppThatCrashesOnLaunch { diff --git a/bp/tests/Unhosted Tests/BPTestCaseInfoTests.m b/bp/tests/Unhosted Tests/BPTestCaseInfoTests.m new file mode 100644 index 00000000..74240146 --- /dev/null +++ b/bp/tests/Unhosted Tests/BPTestCaseInfoTests.m @@ -0,0 +1,42 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import +#import + +#import + +@interface BPTestCaseInfoTests : XCTestCase + +@end + +@implementation BPTestCaseInfoTests + +- (void)testArchiving { + // Mock data + BPTestCaseInfo *info1 = [[BPTestCaseInfo alloc] initWithClassName:@"Class" methodName:@"Method1"]; + BPTestCaseInfo *info2 = [[BPTestCaseInfo alloc] initWithClassName:@"Class" methodName:@"Method2"]; + BPTestCaseInfo *info3 = [[BPTestCaseInfo alloc] initWithClassName:@"Class" methodName:@"Method3"]; + NSArray *testCasesIn = @[info1, info2, info3]; + // Archive + NSError *error; + NSData *data = [NSKeyedArchiver archivedDataWithRootObject:testCasesIn requiringSecureCoding:NO error:&error]; + // Unarchive + NSArray *testCasesOut = [NSKeyedUnarchiver unarchivedArrayOfObjectsOfClass:BPTestCaseInfo.class + fromData:data + error:&error]; + // Validate + XCTAssertNotNil(testCasesOut); + XCTAssertEqual(testCasesIn.count, testCasesOut.count); + for (int i = 0; i < testCasesIn.count; i++) { + XCTAssertEqualObjects(testCasesIn[i], testCasesOut[i]); + } +} + +@end diff --git a/bp/tests/Unhosted Tests/BluepillUnhostedBatchingTests.m b/bp/tests/Unhosted Tests/BluepillUnhostedBatchingTests.m new file mode 100644 index 00000000..9b326f21 --- /dev/null +++ b/bp/tests/Unhosted Tests/BluepillUnhostedBatchingTests.m @@ -0,0 +1,125 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import + +#import "Bluepill.h" +#import "BPIntTestCase.h" +#import "BPConfiguration.h" +#import "BPTestHelper.h" +#import "BPUtils.h" +#import "BPTestUtils.h" + +@interface BluepillUnhostedBatchingTests : BPIntTestCase +@end + +@implementation BluepillUnhostedBatchingTests + +- (void)setUp { + [super setUp]; + self.config = [BPTestUtils makeUnhostedTestConfiguration]; + self.config.numSims = @1; + self.config.stuckTimeout = @3; + self.config.testBundlePath = [BPTestHelper passingLogicTestBundlePath]; + + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + self.config.outputDirectory = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/TestLogsTempDir", tempDir] withError:&error]; +} + +- (void)testAllTests { + // This is redundant but made explicit here for test clarity + self.config.testCasesToRun = nil; + self.config.testCasesToSkip = nil; + + // Run Tests + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + [BPTestUtils assertExitStatus:exitCode matchesExpected:BPExitStatusAllTestsPassed]; + + // Check that test is started in both sets of logs. + for (NSString *testCase in [BluepillUnhostedBatchingTests allTestCases]) { + [BPTestUtils checkIfTestCase:testCase bundleName:@"BPPassingLogicTests" wasRunInLog:[self.config.outputDirectory stringByAppendingPathComponent:@"1-simulator.log"]]; + } +} + +- (void)testOptInToObjcTests { + [self validateOptInToTests:@[ + @"BPPassingLogicTests/testPassingLogicTest2", + @"BPPassingLogicTests/testPassingLogicTest3", + ]]; +} + +- (void)testOptInToSwiftTests { + [self validateOptInToTests:@[ + @"SwiftLogicTests/testPassingLogicTest1()", + ]]; +} + +- (void)testOptOutOfObjcTests { + [self validateOptOutOfTests:@[ + @"BPPassingLogicTests/testPassingLogicTest2", + @"BPPassingLogicTests/testPassingLogicTest3", + ]]; +} + +- (void)testOptOutOfSwiftTests { + [self validateOptOutOfTests:@[ + @"SwiftLogicTests/testPassingLogicTest3()", + ]]; +} + +#pragma mark - Helpers + +- (void)validateOptInToTests:(NSArray *)tests { + self.config.testCasesToRun = tests; + [self validateExactlyTheseTestsAreExecuted:tests]; +} + +- (void)validateOptOutOfTests:(NSArray *)tests { + self.config.testCasesToSkip = tests; + NSArray *expectedTests = [BluepillUnhostedBatchingTests allTestsExcept:tests]; + [self validateExactlyTheseTestsAreExecuted:expectedTests]; +} + +- (void)validateExactlyTheseTestsAreExecuted:(NSArray *)tests { + // Run Tests + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + [BPTestUtils assertExitStatus:exitCode matchesExpected:BPExitStatusAllTestsPassed]; + + // Check that exclusively these tests were run. + for (NSString *testCase in tests) { + NSLog(@"testCase: %@", testCase); + XCTAssert([BPTestUtils checkIfTestCase:testCase bundleName:@"BPPassingLogicTests" wasRunInLog:[self.config.outputDirectory stringByAppendingPathComponent:@"1-simulator.log"]]); + } + + // Check that "skipped" tests are not run. + for (NSString *testCase in [BluepillUnhostedBatchingTests allTestsExcept:tests]) { + XCTAssertFalse([BPTestUtils checkIfTestCase:testCase bundleName:@"BPPassingLogicTests" wasRunInLog:[self.config.outputDirectory stringByAppendingPathComponent:@"1-simulator.log"]]); + } +} + ++ (NSArray *)allTestCases { + return @[ + @"BPPassingLogicTests/testPassingLogicTest1", + @"BPPassingLogicTests/testPassingLogicTest2", + @"BPPassingLogicTests/testPassingLogicTest3", + @"BPPassingLogicTests/testPassingLogicTest4", + @"SwiftLogicTests/testPassingLogicTest1()", + @"SwiftLogicTests/testPassingLogicTest2()", + @"SwiftLogicTests/testPassingLogicTest3()", + ]; +} + ++ (NSArray *)allTestsExcept:(NSArray *)omittedTests { + NSMutableArray *mutableTests = [[self allTestCases] mutableCopy]; + [mutableTests removeObjectsInArray:omittedTests]; + return [mutableTests copy]; +} + +@end diff --git a/bp/tests/Unhosted Tests/BluepillUnhostedTests.m b/bp/tests/Unhosted Tests/BluepillUnhostedTests.m new file mode 100644 index 00000000..764665d5 --- /dev/null +++ b/bp/tests/Unhosted Tests/BluepillUnhostedTests.m @@ -0,0 +1,176 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import +#import + +#import "Bluepill.h" +#import "BPIntTestCase.h" +#import "BPConfiguration.h" +#import "BPTestHelper.h" +#import "BPUtils.h" +#import "BPTestUtils.h" + +/** + * This test suite is the integration tests to make sure logic tests are being run correctly. + * It includes validation on the following: + * - Exit code testing + * - Failure/Timeout/Crash handling + * - Retry behaviors + */ +@interface BluepillUnhostedTests : BPIntTestCase +@end + +@implementation BluepillUnhostedTests + +- (void)setUp { + [super setUp]; + self.config = [BPTestUtils makeUnhostedTestConfiguration]; + self.config.numSims = @1; + self.config.stuckTimeout = @1; + + NSString *testBundlePath = [BPTestHelper logicTestBundlePath]; + self.config.testBundlePath = testBundlePath; +} + +#pragma mark - Passing Tests + +- (void)testSinglePassingLogicTests { + self.config.testCasesToRun = @[@"BPLogicTests/testPassingLogicTest1"]; + BPExitStatus exitCode = [[[Bluepill alloc] initWithConfiguration:self.config] run]; + [BPTestUtils assertExitStatus:exitCode matchesExpected:BPExitStatusAllTestsPassed]; +} + +- (void)testMultiplePassingLogicTests { + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/TestLogsTempDir", tempDir] withError:&error]; + self.config.outputDirectory = outputDir; + self.config.testCasesToRun = @[ + @"BPLogicTests/testPassingLogicTest1", + @"BPLogicTests/testPassingLogicTest2" + ]; + + // Run Tests + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + [BPTestUtils assertExitStatus:exitCode matchesExpected:BPExitStatusAllTestsPassed]; + + // Check that test is started in both sets of logs. + for (NSString *testCase in self.config.testCasesToRun) { + XCTAssert([BPTestUtils checkIfTestCase:testCase bundleName:@"BPLogicTests" wasRunInLog:[outputDir stringByAppendingPathComponent:@"1-simulator.log"]]); + } +} + +# pragma mark - Failing Tests + +- (void)testFailingLogicTest { + self.config.testCasesToRun = @[@"BPLogicTests/testFailingLogicTest"]; + + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + [BPTestUtils assertExitStatus:exitCode matchesExpected:BPExitStatusTestsFailed]; +} + +#pragma mark - Handling Crashes + +/* + A boring objective-c crash (such as index out of bounds on an NSArray) should be + handled smoothly by XCTest, and reported as such as a failed test. + */ +- (void)testCrashingTestCaseLogicTest { + self.config.testCasesToRun = @[@"BPLogicTests/testCrashTestCaseLogicTest"]; + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + [BPTestUtils assertExitStatus:exitCode matchesExpected:BPExitStatusTestsFailed]; +} + +/* + A more aggressive crash (like doing an illegal strcpy) will crash the entire XCTest + execution. + */ +- (void)testCrashingExecutionLogicTest { + self.config.testCasesToRun = @[@"BPLogicTests/testCrashExecutionLogicTest"]; + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + [BPTestUtils assertExitStatus:exitCode matchesExpected:BPExitStatusAppCrashed]; +} + +#pragma mark - Timeouts + +/* + This test validates that a test fails when the simulator has no output for over + the stuckTimeout threshold + */ +- (void)testStuckLogicTest { + self.config.testCasesToRun = @[@"BPLogicTests/testStuckLogicTest"]; + BPExitStatus exitCode = [[[Bluepill alloc] initWithConfiguration:self.config] run]; + [BPTestUtils assertExitStatus:exitCode matchesExpected:BPExitStatusTestTimeout]; +} + +/* + This test validates that a slow test will fail, even when the simulator sees + some change. + */ +- (void)testHangingLogicTest { + // `BPLogicTests/testSlowLogicTest` is designed to log a string infinitely, once a second. + // As a result, it should not "get stuck", but should eventually timeout anyway. + self.config.stuckTimeout = @1; + self.config.testCaseTimeout = @3; + self.config.testCasesToRun = @[@"BPLogicTests/testSlowLogicTest"]; + + BPExitStatus exitCode = [[[Bluepill alloc] initWithConfiguration:self.config] run]; + [BPTestUtils assertExitStatus:exitCode matchesExpected:BPExitStatusTestTimeout]; +} + +/* + The timeout should only cause a failure if an individual test exceeds the timeout, + not if the combined time sums to above the timeout. + */ +- (void)testTimeoutOnlyAppliesToTestCaseNotSuite { + // The three tests combined should exceed any timeouts, but that shouldn't be a problem. + self.config.testCaseTimeout = @2; + self.config.stuckTimeout = @2; + self.config.testCasesToRun = @[ + @"BPLogicTests/testOneSecondTest1", + @"BPLogicTests/testOneSecondTest2", + @"BPLogicTests/testOneSecondTest3", + ]; + BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + [BPTestUtils assertExitStatus:exitCode matchesExpected:BPExitStatusAllTestsPassed]; +} + +#pragma mark - Retries + +- (void)testRetriesFailure { + [self validateTestIsRetried:@"BPLogicTests/testFailingLogicTest"]; +} + +- (void)testRetriesCrash { + self.config.retryAppCrashTests = YES; + [self validateTestIsRetried:@"BPLogicTests/testCrashExecutionLogicTest"]; +} + +#pragma mark - Helpers + +- (void)validateTestIsRetried:(NSString *)testCase { + // Setup + NSString *tempDir = NSTemporaryDirectory(); + NSError *error; + NSString *outputDir = [BPUtils mkdtemp:[NSString stringWithFormat:@"%@/FailingTestsSetTempDir", tempDir] withError:&error]; + self.config.outputDirectory = outputDir; + self.config.errorRetriesCount = @1; + self.config.failureTolerance = @1; + self.config.testCasesToRun = @[testCase]; + + // Run Tests + __unused BPExitStatus exitCode = [[[Bluepill alloc ] initWithConfiguration:self.config] run]; + + // Validate + XCTAssert([BPTestUtils checkIfTestCase:testCase bundleName:@"BPLogicTests" wasRunInLog:[outputDir stringByAppendingPathComponent:@"1-simulator.log"]]); + XCTAssert([BPTestUtils checkIfTestCase:testCase bundleName:@"BPLogicTests" wasRunInLog:[outputDir stringByAppendingPathComponent:@"2-simulator.log"]]); +} + +@end diff --git a/bp/tests/Utils/BPTestUtils.h b/bp/tests/Utils/BPTestUtils.h new file mode 100644 index 00000000..45717733 --- /dev/null +++ b/bp/tests/Utils/BPTestUtils.h @@ -0,0 +1,29 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import +#import "BPExitStatus.h" + +@class BPConfiguration; + +NS_ASSUME_NONNULL_BEGIN + +@interface BPTestUtils : NSObject + ++ (nonnull BPConfiguration *)makeUnhostedTestConfiguration; + ++ (nonnull BPConfiguration *)makeHostedTestConfiguration; + ++ (void)assertExitStatus:(BPExitStatus)exitStatus matchesExpected:(BPExitStatus)expectedStatus; + ++ (BOOL)checkIfTestCase:(NSString *)testCase bundleName:(NSString *)bundleName wasRunInLog:(NSString *)logPath; + +@end + +NS_ASSUME_NONNULL_END diff --git a/bp/tests/Utils/BPTestUtils.m b/bp/tests/Utils/BPTestUtils.m new file mode 100644 index 00000000..e73b3203 --- /dev/null +++ b/bp/tests/Utils/BPTestUtils.m @@ -0,0 +1,117 @@ +// Copyright 2016 LinkedIn Corporation +// Licensed under the BSD 2-Clause License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/BSD-2-Clause +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +#import "BPTestUtils.h" + +#import + +#import "BPConfiguration.h" +#import "BPTestHelper.h" +#import "BPUtils.h" +#import "SimDeviceType.h" +#import "SimRuntime.h" +#import "SimServiceContext.h" + +@implementation BPTestUtils + ++ (nonnull BPConfiguration *)makeUnhostedTestConfiguration { + BPConfiguration *config = [self makeDefaultTestConfiguration]; + config.testBundlePath = [BPTestHelper logicTestBundlePath]; + config.isLogicTestTarget = YES; + return config; +} + ++ (nonnull BPConfiguration *)makeHostedTestConfiguration { + BPConfiguration *config = [self makeDefaultTestConfiguration]; + config.appBundlePath = [BPTestHelper sampleAppPath]; + config.testBundlePath = [BPTestHelper sampleAppNegativeTestsBundlePath]; + config.isLogicTestTarget = NO; + return config; +} + ++ (nonnull BPConfiguration *)makeDefaultTestConfiguration { + BPConfiguration *config = [[BPConfiguration alloc] initWithProgram:BP_BINARY]; + config.stuckTimeout = @40; + config.xcodePath = [BPUtils runShell:@"/usr/bin/xcode-select -print-path"]; + config.runtime = @BP_DEFAULT_RUNTIME; + config.repeatTestsCount = @1; + config.errorRetriesCount = @0; + config.testCaseTimeout = @20; + config.deviceType = @BP_DEFAULT_DEVICE_TYPE; + config.headlessMode = YES; + config.videoPaths = @[[BPTestHelper sampleVideoPath]]; + config.testRunnerAppPath = nil; + config.testing_CrashAppOnLaunch = NO; + config.cloneSimulator = NO; + config.outputDirectory = @"/Users/lthrockm/Desktop/output/"; + + // Set up simulator device + runtime + NSError *err; + SimServiceContext *sc = [SimServiceContext sharedServiceContextForDeveloperDir:config.xcodePath error:&err]; + if (!sc) { NSLog(@"Failed to initialize SimServiceContext: %@", err); } + + for (SimDeviceType *type in [sc supportedDeviceTypes]) { + if ([[type name] isEqualToString:config.deviceType]) { + config.simDeviceType = type; + break; + } + } + XCTAssert(config.simDeviceType != nil); + + for (SimRuntime *runtime in [sc supportedRuntimes]) { + if ([[runtime name] containsString:config.runtime]) { + config.simRuntime = runtime; + break; + } + } + XCTAssert(config.simRuntime != nil); + + return config; +} + ++ (void)assertExitStatus:(BPExitStatus)exitStatus matchesExpected:(BPExitStatus)expectedStatus { + XCTAssert(exitStatus == expectedStatus, + @"Expected: %@ Got: %@", + [BPExitStatusHelper stringFromExitStatus:expectedStatus], + [BPExitStatusHelper stringFromExitStatus:exitStatus]); +} + ++ (BOOL)isTestSwiftTest:(NSString *)testName { + return [testName containsString:@"."] || [testName containsString:@"()"]; +} + ++ (NSString *)formatSwiftTestForXCTest:(NSString *)testName withBundleName:(NSString *)bundleName { + NSString *formattedName = testName; + // Remove parentheses + NSRange range = [formattedName rangeOfString:@"()"]; + if (range.location != NSNotFound) { + formattedName = [formattedName substringToIndex:range.location]; + } + // Add `.` + NSString *bundlePrefix = [bundleName stringByAppendingString:@"."]; + if (![formattedName containsString:bundlePrefix]) { + formattedName = [NSString stringWithFormat:@"%@.%@", bundleName, formattedName]; + } + return formattedName; +} + ++ (BOOL)checkIfTestCase:(NSString *)testCase bundleName:(NSString *)bundleName wasRunInLog:(NSString *)logPath { + NSString *testName = testCase; + if ([self isTestSwiftTest:testName]) { + testName = [BPTestUtils formatSwiftTestForXCTest:testName withBundleName:bundleName]; + } + NSArray *testComponents = [testName componentsSeparatedByString:@"/"]; + NSString *expectedString = [NSString stringWithFormat:@"Test Case '-[%@ %@]' started.", testComponents[0], testComponents[1]]; + NSString *log = [NSString stringWithContentsOfFile:logPath encoding:NSUTF8StringEncoding error:nil]; + XCTAssertNotNil(log); + NSLog(@"log: %@", log); + return [log rangeOfString:expectedString].location != NSNotFound; +} + +@end diff --git a/bptestrunner/BUILD.bazel b/bptestrunner/BUILD.bazel index 4250458f..323d406b 100644 --- a/bptestrunner/BUILD.bazel +++ b/bptestrunner/BUILD.bazel @@ -14,6 +14,18 @@ native_binary( out = "bp", ) +native_binary( + name = "libBPTestInspector.dylib", + src = 'bin/libBPTestInspector.dylib', + out = "libBPTestInspector.dylib", +) + +native_binary( + name = "libBPMacTestInspector.dylib", + src = 'bin/libBPMacTestInspector.dylib', + out = "libBPMacTestInspector.dylib", +) + exports_files([ "bluepill_batch_test_runner.template.sh" ]) diff --git a/bptestrunner/bluepill_batch_test.bzl b/bptestrunner/bluepill_batch_test.bzl index 36203270..349a709c 100644 --- a/bptestrunner/bluepill_batch_test.bzl +++ b/bptestrunner/bluepill_batch_test.bzl @@ -5,7 +5,7 @@ load( ) def _bluepill_batch_test_impl(ctx): - runfiles = [ctx.file._bp_exec, ctx.file._bluepill_exec] + runfiles = [ctx.file._bp_exec, ctx.file._bluepill_exec, ctx.file._libBPTestInspector_dylib, ctx.file._libBPMacTestInspector_dylib] test_bundle_paths = [] test_host_paths = [] @@ -27,15 +27,23 @@ def _bluepill_batch_test_impl(ctx): test_host_paths.append("\"{}\"".format(test_host.short_path)) runfiles.append(test_host) + #test_plan - test_plan = struct( - test_host = test_host.basename.split( - "." + test_host.extension, - )[0] + ".app", - environment = test_env, - arguments = test_env, - test_bundle_path = bundle_info.bundle_name + bundle_info.bundle_extension, - ) + if test_host: + test_plan = struct( + test_host = test_host.basename.split( + "." + test_host.extension, + )[0] + ".app", + environment = test_env, + arguments = test_env, + test_bundle_path = bundle_info.bundle_name + bundle_info.bundle_extension, + ) + else: + test_plan = struct( + environment = test_env, + arguments = test_env, + test_bundle_path = bundle_info.bundle_name + bundle_info.bundle_extension, + ) test_plans[test_target.label.name] = test_plan # Write test plan json. @@ -49,12 +57,17 @@ def _bluepill_batch_test_impl(ctx): # Write the shell script. substitutions = { "test_bundle_paths": " ".join(test_bundle_paths), - "test_host_paths": " ".join(test_host_paths), "bp_test_plan": test_plan_file.short_path, "bp_path": ctx.executable._bp_exec.short_path, "bluepill_path": ctx.executable._bluepill_exec.short_path, + "testInspector_path": ctx.file._libBPTestInspector_dylib.short_path, + "macTestInspector_path": ctx.file._libBPMacTestInspector_dylib.short_path, "target_name": ctx.attr.name, } + if len(test_host_paths) > 0: + substitutions["test_host_paths"] = " ".join(test_host_paths) + + if ctx.attr.config_file: runfiles.append(ctx.file.config_file) substitutions["bp_config_file"] = ctx.file.config_file.path @@ -114,6 +127,18 @@ as possible between simulators. executable = True, cfg = "host", ), + "_libBPTestInspector_dylib": attr.label( + default = Label( + "//:libBPTestInspector.dylib", + ), + allow_single_file = True, + ), + "_libBPMacTestInspector_dylib": attr.label( + default = Label( + "//:libBPMacTestInspector.dylib", + ), + allow_single_file = True, + ), "_xcode_config": attr.label( default = configuration_field( fragment = "apple", diff --git a/bptestrunner/bluepill_batch_test_runner.template.sh b/bptestrunner/bluepill_batch_test_runner.template.sh index 012b9f38..44b9a235 100644 --- a/bptestrunner/bluepill_batch_test_runner.template.sh +++ b/bptestrunner/bluepill_batch_test_runner.template.sh @@ -18,12 +18,15 @@ BP_TEST_ESTIMATE_JSON="bp_test_time_estimates_json" BP_TEST_PLAN="bp_test_plan" BP_PATH="bp_path" BLUEPILL_PATH="bluepill_path" +TEST_INSPECTOR_PATH="testInspector_path" +MAC_TEST_INSPECTOR_PATH="macTestInspector_path" # Remove existing working folder for a clean state rm -rf $BP_WORKING_FOLDER mkdir $BP_WORKING_FOLDER # Extract test bundles + for test_bundle in ${TEST_BUNDLE_PATHS[@]}; do if [[ $test_bundle == *.zip ]]; then tar -C $BP_WORKING_FOLDER -xzf $test_bundle @@ -36,20 +39,22 @@ for test_bundle in ${TEST_BUNDLE_PATHS[@]}; do fi done -# Clone and extract test hosts -for test_host in ${TEST_HOST_PATHS[@]}; do - if [[ "$test_host" == *.ipa ]]; then - TEST_HOST_NAME=$(basename_without_extension "${test_host}") - unzip -qq -d "$BP_WORKING_FOLDER" "$test_host" - cp -cr "${BP_WORKING_FOLDER}/Payload/${TEST_HOST_NAME}.app" ${BP_WORKING_FOLDER} - elif [[ $test_host == *.app ]]; then - cp -cr $test_host $BP_WORKING_FOLDER - chmod -R ug+w "$BP_WORKING_FOLDER/$(basename "$test_host")" - else - echo "$test_host is not an ipa file or app bundle." - exit 1 - fi -done +# Clone and extract test hosts (won't be set for logic tests) +if [ ! -z ${test_host_paths+x} ]; then + for test_host in ${TEST_HOST_PATHS[@]}; do + if [[ "$test_host" == *.ipa ]]; then + TEST_HOST_NAME=$(basename_without_extension "${test_host}") + unzip -qq -d "$BP_WORKING_FOLDER" "$test_host" + cp -cr "${BP_WORKING_FOLDER}/Payload/${TEST_HOST_NAME}.app" ${BP_WORKING_FOLDER} + elif [[ $test_host == *.app ]]; then + cp -cr $test_host $BP_WORKING_FOLDER + chmod -R ug+w "$BP_WORKING_FOLDER/$(basename "$test_host")" + else + echo "$test_host is not an ipa file or app bundle." + exit 1 + fi + done +fi # Copy config file to bp working folder if [ -f "$BP_CONFIG_FILE" ]; then @@ -69,9 +74,18 @@ fi sed 's/$TEST_UNDECLARED_OUTPUTS_DIR/'"${TEST_UNDECLARED_OUTPUTS_DIR//\//\\/}"'/g' $BP_TEST_PLAN > $BP_WORKING_FOLDER/$BP_TEST_PLAN BP_TEST_PLAN_ARG="$(basename "$BP_TEST_PLAN")" -# Copy bluepill and bp executables to working folder +# Copy bluepill and bp executables to working folder, along with testInspector dylib. cp "$BP_PATH" $BP_WORKING_FOLDER cp "$BLUEPILL_PATH" $BP_WORKING_FOLDER +cp "$TEST_INSPECTOR_PATH" $BP_WORKING_FOLDER +cp "$MAC_TEST_INSPECTOR_PATH" $BP_WORKING_FOLDER + +echo "pwd" +pwd +echo "LTHROCKM DEBUG - TEST_INSPECTOR_PATH: $TEST_INSPECTOR_PATH" +echo "LTHROCKM DEBUG - BP_WORKING_FOLDER: $BP_WORKING_FOLDER" + +export "DYLD_LIBRARY_PATH=.:$BP_WORKING_FOLDER/libBPMacTestInspector.dylib" # Run bluepill # NOTE: we override output folder here and disregard the one in the config file. diff --git a/scripts/bluepill.sh b/scripts/bluepill.sh index 3ab8bc00..e3254cb2 100755 --- a/scripts/bluepill.sh +++ b/scripts/bluepill.sh @@ -36,6 +36,27 @@ mkdir -p build/ bluepill_build() { set -o pipefail + + # First build BPTestInspector, as it's a dependency. + xcodebuild \ + -workspace Bluepill.xcworkspace \ + -arch x86_64 \ + -scheme BPTestInspector \ + -sdk iphonesimulator \ + -configuration Release \ + -derivedDataPath "$DerivedDataPath" | tee result_bptestinspector.txt | $XCPRETTY + test $? == 0 || { + echo Build failed + xcodebuild -list -workspace Bluepill.xcworkspace + -scheme BPTestInspector \ + cat result_bptestinspector.txt + exit 1 + } + test -x build/Build/Products/Release-iphonesimulator/libBPTestInspector.dylib || { + echo No bp built + exit 1 + } + xcodebuild \ -workspace Bluepill.xcworkspace \ -scheme bluepill \ @@ -52,12 +73,15 @@ bluepill_build() echo No bp built exit 1 } + set +o pipefail # package bluepill TAG=$(git describe --always --tags) DST="Bluepill-$TAG" mkdir -p "build/$DST/bin" - cp build/Build/Products/Release/{bp,bluepill} "build/$DST/bin" + cp build/Build/Products/Release/{bp,bluepill,libBPMacTestInspector.dylib,libBPTestInspector.dylib} "build/$DST/bin" + cp build/Build/Products/Release-iphonesimulator/libBPTestInspector.dylib "build/$DST/bin" + ## build the man page mkdir -p "build/$DST/man/man1" /usr/bin/python scripts/man.py "build/$DST/man/man1/bluepill.1"