From 745bb1f03c2c49be8a18049fe17eb865f8f4c8e0 Mon Sep 17 00:00:00 2001 From: Daniel Boyd Date: Thu, 29 Oct 2026 12:52:30 -0500 Subject: [PATCH] Add `buildtool test` to run Xcode unit-test targets with xctest `buildtool test` builds the project, then runs each unit-test target's .xctest bundle with tools-xctest's xctest, like `xcodebuild test`. Targets come from -target, a scheme (its testables, SkippedTests and SelectedTests), or every unit-test target; xcodebuild's -only-testing and -skip-testing select tests. For test, -target and -scheme choose test targets instead of narrowing the build, so test hosts still get built. XCTEST picks the xctest program and XCTEST_FLAGS adds options. A failed build or test fails the action, and so buildtool's exit status. It works with tools-xctest as it is: -only-testing becomes an -XCTest Class[.method] selection. Skipping tests and running a target inside the application its TEST_HOST names need xctest's -skip-testing and -host, which newer tools-xctest has; buildtool uses them when xctest --help lists them, and otherwise fails that target with a message saying what is missing. Testing: Tests/run-buildtool-test-action.sh runs `buildtool test` on a new fixture project (a tool, its tests, an application and tests hosted in it): the full run, a failing test, -only-testing by method and by class, -skip-testing, a scheme's testables and skipped tests, the hosted target, and the mistakes (a non-test target, an unknown scheme). It checks which test methods ran by name, so it doesn't depend on xctest's output format, and expects the "needs an xctest with ..." failure where the xctest lacks an option. It passes against xctest from gnustep/tools-xctest master, from master plus the -only-testing and -skip-testing patch, and from a build that also has -host. Against master's buildtool it fails at the first check. AI assistance: this change, its test and this description were written with Claude (Anthropic's Claude Code). Co-Authored-By: Claude Opus 5.5 --- ChangeLog | 22 + README.md | 25 + .../Xcode/unit-tests/App/AppDelegate.h | 8 + .../Xcode/unit-tests/App/AppDelegate.m | 10 + Tests/Fixtures/Xcode/unit-tests/App/main.m | 10 + .../Xcode/unit-tests/AppTests/AppTests.m | 16 + .../unit-tests/Calc.xcodeproj/project.pbxproj | 394 +++++++++++ .../xcshareddata/xcschemes/CalcTests.xcscheme | 21 + .../Fixtures/Xcode/unit-tests/Sources/Calc.h | 5 + .../Fixtures/Xcode/unit-tests/Sources/Calc.m | 8 + .../Fixtures/Xcode/unit-tests/Sources/main.m | 8 + .../Xcode/unit-tests/Tests/CalcTests.m | 27 + .../Xcode/unit-tests/generate-project.py | 70 ++ Tests/run-buildtool-test-action.sh | 137 ++++ Tools/buildtool/ToolDelegate.m | 54 +- XCode/GNUmakefile | 2 + XCode/PBXContainer.h | 11 + XCode/PBXContainer.m | 14 + XCode/PBXProject+Testing.h | 48 ++ XCode/PBXProject+Testing.m | 618 ++++++++++++++++++ 20 files changed, 1506 insertions(+), 2 deletions(-) create mode 100644 Tests/Fixtures/Xcode/unit-tests/App/AppDelegate.h create mode 100644 Tests/Fixtures/Xcode/unit-tests/App/AppDelegate.m create mode 100644 Tests/Fixtures/Xcode/unit-tests/App/main.m create mode 100644 Tests/Fixtures/Xcode/unit-tests/AppTests/AppTests.m create mode 100644 Tests/Fixtures/Xcode/unit-tests/Calc.xcodeproj/project.pbxproj create mode 100644 Tests/Fixtures/Xcode/unit-tests/Calc.xcodeproj/xcshareddata/xcschemes/CalcTests.xcscheme create mode 100644 Tests/Fixtures/Xcode/unit-tests/Sources/Calc.h create mode 100644 Tests/Fixtures/Xcode/unit-tests/Sources/Calc.m create mode 100644 Tests/Fixtures/Xcode/unit-tests/Sources/main.m create mode 100644 Tests/Fixtures/Xcode/unit-tests/Tests/CalcTests.m create mode 100755 Tests/Fixtures/Xcode/unit-tests/generate-project.py create mode 100755 Tests/run-buildtool-test-action.sh create mode 100644 XCode/PBXProject+Testing.h create mode 100644 XCode/PBXProject+Testing.m diff --git a/ChangeLog b/ChangeLog index 6a632ee..886723d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,25 @@ +2026-10-29 Daniel Boyd + +* XCode/PBXProject+Testing.[hm]: New. `buildtool test`: build the + project, then run each unit-test target's bundle with xctest + (tools-xctest), selected with -target, a scheme's testables and + skipped/selected tests, or xcodebuild's -only-testing/-skip-testing. + A target whose TEST_HOST names an application in the project runs + inside it (xctest -host). +* XCode/PBXContainer.[hm] (setTestOptions:, test): New. +* XCode/GNUmakefile: Add PBXProject+Testing.[hm]. +* XCode/PBXProject+Testing.m: -only-testing becomes xctest's -XCTest + selection; skipped tests and TEST_HOST use xctest's -skip-testing and + -host only when its --help lists them, and fail with a message + otherwise. +* Tools/buildtool/ToolDelegate.m (testOptions:): New. Add the test + action; for it, -target and -scheme choose the test targets instead + of narrowing the build, so test hosts are still built. +* Tests/Fixtures/Xcode/unit-tests, Tests/run-buildtool-test-action.sh: + New. A project with unit tests, an application and hosted tests, and + regressions for `buildtool test`. +* README.md: Document `buildtool test`. + 2026-01-10 Gregory John Casamento * XCode/PBXNativeTarget.m: Include TEST_TYPE in bundle-style diff --git a/README.md b/README.md index a9f1973..d0e6369 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,31 @@ Supported xcodebuild-style options: `-parallelizeTargets`, `-quiet`, `-verbose`, `-list`, and `-showBuildSettings` are accepted for command-line compatibility. +`buildtool test` builds the project, like `xcodebuild test`, then runs each +unit-test target's `.xctest` bundle with `xctest` from +[tools-xctest](https://github.com/gnustep/tools-xctest). For `test`, +`-target` and `-scheme` choose the test targets (a scheme's testables and +its skipped or selected tests) rather than narrowing the build, so test +hosts are still built: + +``` +buildtool test # every unit-test target +buildtool test -target MyTests # one target +buildtool test -scheme MyApp # a scheme's testables +buildtool test -only-testing:MyTests/FooTests -skip-testing:MyTests/FooTests/testSlow +``` + +`-only-testing:` becomes an `-XCTest` selection, which any tools-xctest +accepts. Skipping tests (`-skip-testing:`, or a scheme's skipped tests) needs +an `xctest` that lists `-skip-testing` in its `--help`, and a test target +whose `TEST_HOST` names an application in the project runs inside that +application, which needs one that lists `-host` (and an X display, e.g. +`xvfb-run -a buildtool test`). With an older `xctest`, those targets fail +with a message saying so. `XCTEST` chooses the `xctest` program, and +`XCTEST_FLAGS` passes extra options to it (e.g. +`XCTEST_FLAGS="-junit-report results.xml"`). buildtool exits with status 1 +when the build or any test fails. + Buildtool exits with status 0 on success and nonzero on a failed operation, invalid project path, or exception. An explicit project path never falls back to another project in the current directory. diff --git a/Tests/Fixtures/Xcode/unit-tests/App/AppDelegate.h b/Tests/Fixtures/Xcode/unit-tests/App/AppDelegate.h new file mode 100644 index 0000000..5452984 --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/App/AppDelegate.h @@ -0,0 +1,8 @@ +#import + +@interface AppDelegate : NSObject +{ + BOOL launched; +} +@property BOOL launched; +@end diff --git a/Tests/Fixtures/Xcode/unit-tests/App/AppDelegate.m b/Tests/Fixtures/Xcode/unit-tests/App/AppDelegate.m new file mode 100644 index 0000000..de4b70b --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/App/AppDelegate.m @@ -0,0 +1,10 @@ +#import "AppDelegate.h" + +@implementation AppDelegate +@synthesize launched; + +- (void) applicationDidFinishLaunching: (NSNotification *)notification +{ + [self setLaunched: YES]; +} +@end diff --git a/Tests/Fixtures/Xcode/unit-tests/App/main.m b/Tests/Fixtures/Xcode/unit-tests/App/main.m new file mode 100644 index 0000000..6cad2fc --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/App/main.m @@ -0,0 +1,10 @@ +#import +#import "AppDelegate.h" + +int main(int argc, const char *argv[]) +{ + NSApplication *app = [NSApplication sharedApplication]; + [app setDelegate: [[AppDelegate alloc] init]]; + [app run]; + return 0; +} diff --git a/Tests/Fixtures/Xcode/unit-tests/AppTests/AppTests.m b/Tests/Fixtures/Xcode/unit-tests/AppTests/AppTests.m new file mode 100644 index 0000000..1d96e9b --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/AppTests/AppTests.m @@ -0,0 +1,16 @@ +#import +#import + +@interface AppTests : XCTestCase +@end + +@implementation AppTests + +- (void) testRunsInsideTheApp +{ + XCTAssertNotNil(NSApp); + XCTAssertEqualObjects(NSStringFromClass([[NSApp delegate] class]), @"AppDelegate"); + XCTAssertEqualObjects([(NSObject *)[NSApp delegate] valueForKey: @"launched"], [NSNumber numberWithBool: YES]); +} + +@end diff --git a/Tests/Fixtures/Xcode/unit-tests/Calc.xcodeproj/project.pbxproj b/Tests/Fixtures/Xcode/unit-tests/Calc.xcodeproj/project.pbxproj new file mode 100644 index 0000000..2194e42 --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/Calc.xcodeproj/project.pbxproj @@ -0,0 +1,394 @@ +// !$*UTF8*$! +{ + "archiveVersion" = "1"; + "classes" = { + }; + "objectVersion" = "56"; + "objects" = { + "000000000000000000000001" = { + "isa" = "PBXFileReference"; + "lastKnownFileType" = "sourcecode.c.h"; + "path" = "Sources/Calc.h"; + "sourceTree" = ""; + }; + "000000000000000000000002" = { + "isa" = "PBXFileReference"; + "lastKnownFileType" = "sourcecode.c.objc"; + "path" = "Sources/Calc.m"; + "sourceTree" = ""; + }; + "000000000000000000000003" = { + "isa" = "PBXFileReference"; + "lastKnownFileType" = "sourcecode.c.objc"; + "path" = "Sources/main.m"; + "sourceTree" = ""; + }; + "000000000000000000000004" = { + "isa" = "PBXFileReference"; + "lastKnownFileType" = "sourcecode.c.objc"; + "path" = "Tests/CalcTests.m"; + "sourceTree" = ""; + }; + "000000000000000000000005" = { + "isa" = "PBXFileReference"; + "lastKnownFileType" = "sourcecode.c.h"; + "path" = "App/AppDelegate.h"; + "sourceTree" = ""; + }; + "000000000000000000000006" = { + "isa" = "PBXFileReference"; + "lastKnownFileType" = "sourcecode.c.objc"; + "path" = "App/AppDelegate.m"; + "sourceTree" = ""; + }; + "000000000000000000000007" = { + "isa" = "PBXFileReference"; + "lastKnownFileType" = "sourcecode.c.objc"; + "path" = "App/main.m"; + "sourceTree" = ""; + }; + "000000000000000000000008" = { + "isa" = "PBXFileReference"; + "lastKnownFileType" = "sourcecode.c.objc"; + "path" = "AppTests/AppTests.m"; + "sourceTree" = ""; + }; + "000000000000000000000009" = { + "explicitFileType" = "compiled.mach-o.executable"; + "includeInIndex" = "0"; + "isa" = "PBXFileReference"; + "path" = "CalcTool"; + "sourceTree" = "BUILT_PRODUCTS_DIR"; + }; + "00000000000000000000000A" = { + "explicitFileType" = "wrapper.cfbundle"; + "includeInIndex" = "0"; + "isa" = "PBXFileReference"; + "path" = "CalcTests.xctest"; + "sourceTree" = "BUILT_PRODUCTS_DIR"; + }; + "00000000000000000000000B" = { + "explicitFileType" = "wrapper.application"; + "includeInIndex" = "0"; + "isa" = "PBXFileReference"; + "path" = "CalcApp.app"; + "sourceTree" = "BUILT_PRODUCTS_DIR"; + }; + "00000000000000000000000C" = { + "explicitFileType" = "wrapper.cfbundle"; + "includeInIndex" = "0"; + "isa" = "PBXFileReference"; + "path" = "AppTests.xctest"; + "sourceTree" = "BUILT_PRODUCTS_DIR"; + }; + "00000000000000000000000D" = { + "fileRef" = "000000000000000000000002"; + "isa" = "PBXBuildFile"; + }; + "00000000000000000000000E" = { + "fileRef" = "000000000000000000000003"; + "isa" = "PBXBuildFile"; + }; + "00000000000000000000000F" = { + "files" = ( + "00000000000000000000000D", + "00000000000000000000000E" + ); + "isa" = "PBXSourcesBuildPhase"; + }; + "000000000000000000000010" = { + "files" = ( + ); + "isa" = "PBXFrameworksBuildPhase"; + }; + "000000000000000000000011" = { + "buildSettings" = { + "PRODUCT_NAME" = "CalcTool"; + "SDKROOT" = "macosx"; + }; + "isa" = "XCBuildConfiguration"; + "name" = "Debug"; + }; + "000000000000000000000012" = { + "buildSettings" = { + "PRODUCT_NAME" = "CalcTool"; + "SDKROOT" = "macosx"; + }; + "isa" = "XCBuildConfiguration"; + "name" = "Release"; + }; + "000000000000000000000013" = { + "buildConfigurations" = ( + "000000000000000000000011", + "000000000000000000000012" + ); + "defaultConfigurationName" = "Debug"; + "isa" = "XCConfigurationList"; + }; + "000000000000000000000014" = { + "buildConfigurationList" = "000000000000000000000013"; + "buildPhases" = ( + "00000000000000000000000F", + "000000000000000000000010" + ); + "isa" = "PBXNativeTarget"; + "name" = "CalcTool"; + "productName" = "CalcTool"; + "productReference" = "000000000000000000000009"; + "productType" = "com.apple.product-type.tool"; + }; + "000000000000000000000015" = { + "files" = ( + ); + "isa" = "PBXResourcesBuildPhase"; + }; + "000000000000000000000016" = { + "fileRef" = "000000000000000000000002"; + "isa" = "PBXBuildFile"; + }; + "000000000000000000000017" = { + "fileRef" = "000000000000000000000004"; + "isa" = "PBXBuildFile"; + }; + "000000000000000000000018" = { + "files" = ( + "000000000000000000000016", + "000000000000000000000017" + ); + "isa" = "PBXSourcesBuildPhase"; + }; + "000000000000000000000019" = { + "files" = ( + ); + "isa" = "PBXFrameworksBuildPhase"; + }; + "00000000000000000000001A" = { + "buildSettings" = { + "HEADER_SEARCH_PATHS" = "Sources"; + "PRODUCT_NAME" = "CalcTests"; + "SDKROOT" = "macosx"; + }; + "isa" = "XCBuildConfiguration"; + "name" = "Debug"; + }; + "00000000000000000000001B" = { + "buildSettings" = { + "HEADER_SEARCH_PATHS" = "Sources"; + "PRODUCT_NAME" = "CalcTests"; + "SDKROOT" = "macosx"; + }; + "isa" = "XCBuildConfiguration"; + "name" = "Release"; + }; + "00000000000000000000001C" = { + "buildConfigurations" = ( + "00000000000000000000001A", + "00000000000000000000001B" + ); + "defaultConfigurationName" = "Debug"; + "isa" = "XCConfigurationList"; + }; + "00000000000000000000001D" = { + "buildConfigurationList" = "00000000000000000000001C"; + "buildPhases" = ( + "000000000000000000000018", + "000000000000000000000019", + "000000000000000000000015" + ); + "isa" = "PBXNativeTarget"; + "name" = "CalcTests"; + "productName" = "CalcTests"; + "productReference" = "00000000000000000000000A"; + "productType" = "com.apple.product-type.bundle.unit-test"; + }; + "00000000000000000000001E" = { + "files" = ( + ); + "isa" = "PBXResourcesBuildPhase"; + }; + "00000000000000000000001F" = { + "fileRef" = "000000000000000000000006"; + "isa" = "PBXBuildFile"; + }; + "000000000000000000000020" = { + "fileRef" = "000000000000000000000007"; + "isa" = "PBXBuildFile"; + }; + "000000000000000000000021" = { + "files" = ( + "00000000000000000000001F", + "000000000000000000000020" + ); + "isa" = "PBXSourcesBuildPhase"; + }; + "000000000000000000000022" = { + "files" = ( + ); + "isa" = "PBXFrameworksBuildPhase"; + }; + "000000000000000000000023" = { + "buildSettings" = { + "PRODUCT_NAME" = "CalcApp"; + "SDKROOT" = "macosx"; + }; + "isa" = "XCBuildConfiguration"; + "name" = "Debug"; + }; + "000000000000000000000024" = { + "buildSettings" = { + "PRODUCT_NAME" = "CalcApp"; + "SDKROOT" = "macosx"; + }; + "isa" = "XCBuildConfiguration"; + "name" = "Release"; + }; + "000000000000000000000025" = { + "buildConfigurations" = ( + "000000000000000000000023", + "000000000000000000000024" + ); + "defaultConfigurationName" = "Debug"; + "isa" = "XCConfigurationList"; + }; + "000000000000000000000026" = { + "buildConfigurationList" = "000000000000000000000025"; + "buildPhases" = ( + "000000000000000000000021", + "000000000000000000000022", + "00000000000000000000001E" + ); + "isa" = "PBXNativeTarget"; + "name" = "CalcApp"; + "productName" = "CalcApp"; + "productReference" = "00000000000000000000000B"; + "productType" = "com.apple.product-type.application"; + }; + "000000000000000000000027" = { + "files" = ( + ); + "isa" = "PBXResourcesBuildPhase"; + }; + "000000000000000000000028" = { + "fileRef" = "000000000000000000000008"; + "isa" = "PBXBuildFile"; + }; + "000000000000000000000029" = { + "files" = ( + "000000000000000000000028" + ); + "isa" = "PBXSourcesBuildPhase"; + }; + "00000000000000000000002A" = { + "files" = ( + ); + "isa" = "PBXFrameworksBuildPhase"; + }; + "00000000000000000000002B" = { + "buildSettings" = { + "BUNDLE_LOADER" = "$(TEST_HOST)"; + "PRODUCT_NAME" = "AppTests"; + "SDKROOT" = "macosx"; + "TEST_HOST" = "$(BUILT_PRODUCTS_DIR)/CalcApp.app/Contents/MacOS/CalcApp"; + }; + "isa" = "XCBuildConfiguration"; + "name" = "Debug"; + }; + "00000000000000000000002C" = { + "buildSettings" = { + "BUNDLE_LOADER" = "$(TEST_HOST)"; + "PRODUCT_NAME" = "AppTests"; + "SDKROOT" = "macosx"; + "TEST_HOST" = "$(BUILT_PRODUCTS_DIR)/CalcApp.app/Contents/MacOS/CalcApp"; + }; + "isa" = "XCBuildConfiguration"; + "name" = "Release"; + }; + "00000000000000000000002D" = { + "buildConfigurations" = ( + "00000000000000000000002B", + "00000000000000000000002C" + ); + "defaultConfigurationName" = "Debug"; + "isa" = "XCConfigurationList"; + }; + "00000000000000000000002E" = { + "buildConfigurationList" = "00000000000000000000002D"; + "buildPhases" = ( + "000000000000000000000029", + "00000000000000000000002A", + "000000000000000000000027" + ); + "isa" = "PBXNativeTarget"; + "name" = "AppTests"; + "productName" = "AppTests"; + "productReference" = "00000000000000000000000C"; + "productType" = "com.apple.product-type.bundle.unit-test"; + }; + "00000000000000000000002F" = { + "children" = ( + "000000000000000000000009", + "00000000000000000000000A", + "00000000000000000000000B", + "00000000000000000000000C" + ); + "isa" = "PBXGroup"; + "name" = "Products"; + "sourceTree" = ""; + }; + "000000000000000000000030" = { + "children" = ( + "000000000000000000000001", + "000000000000000000000002", + "000000000000000000000003", + "000000000000000000000004", + "000000000000000000000005", + "000000000000000000000006", + "000000000000000000000007", + "000000000000000000000008", + "00000000000000000000002F" + ); + "isa" = "PBXGroup"; + "sourceTree" = ""; + }; + "000000000000000000000031" = { + "buildSettings" = { + "SDKROOT" = "macosx"; + }; + "isa" = "XCBuildConfiguration"; + "name" = "Debug"; + }; + "000000000000000000000032" = { + "buildSettings" = { + "SDKROOT" = "macosx"; + }; + "isa" = "XCBuildConfiguration"; + "name" = "Release"; + }; + "000000000000000000000033" = { + "buildConfigurations" = ( + "000000000000000000000031", + "000000000000000000000032" + ); + "defaultConfigurationName" = "Debug"; + "isa" = "XCConfigurationList"; + }; + "000000000000000000000034" = { + "buildConfigurationList" = "000000000000000000000033"; + "compatibilityVersion" = "Xcode 15.0"; + "developmentRegion" = "en"; + "hasScannedForEncodings" = "0"; + "isa" = "PBXProject"; + "mainGroup" = "000000000000000000000030"; + "productRefGroup" = "00000000000000000000002F"; + "projectDirPath" = ""; + "projectRoot" = ""; + "targets" = ( + "000000000000000000000014", + "00000000000000000000001D", + "000000000000000000000026", + "00000000000000000000002E" + ); + }; + }; + "rootObject" = "000000000000000000000034"; +} diff --git a/Tests/Fixtures/Xcode/unit-tests/Calc.xcodeproj/xcshareddata/xcschemes/CalcTests.xcscheme b/Tests/Fixtures/Xcode/unit-tests/Calc.xcodeproj/xcshareddata/xcschemes/CalcTests.xcscheme new file mode 100644 index 0000000..72224a6 --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/Calc.xcodeproj/xcshareddata/xcschemes/CalcTests.xcscheme @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + diff --git a/Tests/Fixtures/Xcode/unit-tests/Sources/Calc.h b/Tests/Fixtures/Xcode/unit-tests/Sources/Calc.h new file mode 100644 index 0000000..3a2f1a2 --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/Sources/Calc.h @@ -0,0 +1,5 @@ +#import + +@interface Calc : NSObject ++ (NSInteger) add: (NSInteger)a to: (NSInteger)b; +@end diff --git a/Tests/Fixtures/Xcode/unit-tests/Sources/Calc.m b/Tests/Fixtures/Xcode/unit-tests/Sources/Calc.m new file mode 100644 index 0000000..87d4d65 --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/Sources/Calc.m @@ -0,0 +1,8 @@ +#import "Calc.h" + +@implementation Calc ++ (NSInteger) add: (NSInteger)a to: (NSInteger)b +{ + return a + b; +} +@end diff --git a/Tests/Fixtures/Xcode/unit-tests/Sources/main.m b/Tests/Fixtures/Xcode/unit-tests/Sources/main.m new file mode 100644 index 0000000..ec82b2d --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/Sources/main.m @@ -0,0 +1,8 @@ +#import +#import "Calc.h" + +int main(int argc, const char *argv[]) +{ + printf("%ld\n", (long)[Calc add: 2 to: 3]); + return 0; +} diff --git a/Tests/Fixtures/Xcode/unit-tests/Tests/CalcTests.m b/Tests/Fixtures/Xcode/unit-tests/Tests/CalcTests.m new file mode 100644 index 0000000..4dcd534 --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/Tests/CalcTests.m @@ -0,0 +1,27 @@ +#import +#import "Calc.h" + +@interface CalcTests : XCTestCase +@end + +@implementation CalcTests + +- (void) testAdds +{ + XCTAssertEqual([Calc add: 2 to: 3], (NSInteger)5); +} + +- (void) testAddsNegatives +{ + XCTAssertEqual([Calc add: -2 to: -3], (NSInteger)-5); +} + +- (void) testKnownBroken +{ + if (getenv("CALC_TESTS_FAIL")) + { + XCTAssertEqual([Calc add: 2 to: 2], (NSInteger)5, @"deliberately wrong"); + } +} + +@end diff --git a/Tests/Fixtures/Xcode/unit-tests/generate-project.py b/Tests/Fixtures/Xcode/unit-tests/generate-project.py new file mode 100755 index 0000000..7d37e45 --- /dev/null +++ b/Tests/Fixtures/Xcode/unit-tests/generate-project.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# Writes Calc.xcodeproj/project.pbxproj for the buildtool test regressions: +# CalcTool - a command-line tool (Sources/) +# CalcTests - unit tests for Calc (Tests/) +# CalcApp - a GNUstep application (App/) +# AppTests - unit tests hosted in CalcApp (TEST_HOST) +ids = iter(range(1, 500)) +def nid(): return "%024X" % next(ids) +objs = {} +def add(d): + i = nid(); objs[i] = d; return i +def conflist(settings): + confs = [add({"buildSettings": settings, "isa": "XCBuildConfiguration", "name": n}) for n in ("Debug", "Release")] + return add({"buildConfigurations": confs, "defaultConfigurationName": "Debug", "isa": "XCConfigurationList"}) +def fref(path, ftype): + return add({"isa": "PBXFileReference", "lastKnownFileType": ftype, "path": path, "sourceTree": ""}) +def product(path, ftype): + return add({"includeInIndex": "0", "isa": "PBXFileReference", "explicitFileType": ftype, "path": path, "sourceTree": "BUILT_PRODUCTS_DIR"}) +def bf(ref): return add({"fileRef": ref, "isa": "PBXBuildFile"}) +def phase(isa, refs=()): return add({"files": [bf(r) for r in refs], "isa": isa}) +def target(name, ptype, prod, settings, sources, extra_phases=()): + phases = [phase("PBXSourcesBuildPhase", sources), phase("PBXFrameworksBuildPhase")] + list(extra_phases) + return add({"buildConfigurationList": conflist(dict(settings, PRODUCT_NAME=name, SDKROOT="macosx")), + "buildPhases": phases, "isa": "PBXNativeTarget", "name": name, "productName": name, + "productReference": prod, "productType": ptype}) + +calc_h = fref("Sources/Calc.h", "sourcecode.c.h") +calc_m = fref("Sources/Calc.m", "sourcecode.c.objc") +main_m = fref("Sources/main.m", "sourcecode.c.objc") +tests_m = fref("Tests/CalcTests.m", "sourcecode.c.objc") +app_h = fref("App/AppDelegate.h", "sourcecode.c.h") +app_m = fref("App/AppDelegate.m", "sourcecode.c.objc") +app_main = fref("App/main.m", "sourcecode.c.objc") +apptests_m = fref("AppTests/AppTests.m", "sourcecode.c.objc") + +tool_prod = product("CalcTool", "compiled.mach-o.executable") +tests_prod = product("CalcTests.xctest", "wrapper.cfbundle") +app_prod = product("CalcApp.app", "wrapper.application") +apptests_prod = product("AppTests.xctest", "wrapper.cfbundle") + +tool = target("CalcTool", "com.apple.product-type.tool", tool_prod, {}, [calc_m, main_m]) +tests = target("CalcTests", "com.apple.product-type.bundle.unit-test", tests_prod, + {"HEADER_SEARCH_PATHS": "Sources"}, [calc_m, tests_m], [phase("PBXResourcesBuildPhase")]) +app = target("CalcApp", "com.apple.product-type.application", app_prod, {}, [app_m, app_main], + [phase("PBXResourcesBuildPhase")]) +apptests = target("AppTests", "com.apple.product-type.bundle.unit-test", apptests_prod, + {"TEST_HOST": "$(BUILT_PRODUCTS_DIR)/CalcApp.app/Contents/MacOS/CalcApp", + "BUNDLE_LOADER": "$(TEST_HOST)"}, + [apptests_m], [phase("PBXResourcesBuildPhase")]) + +products = add({"children": [tool_prod, tests_prod, app_prod, apptests_prod], "isa": "PBXGroup", "name": "Products", "sourceTree": ""}) +main_group = add({"children": [calc_h, calc_m, main_m, tests_m, app_h, app_m, app_main, apptests_m, products], + "isa": "PBXGroup", "sourceTree": ""}) +project = add({"buildConfigurationList": conflist({"SDKROOT": "macosx"}), "compatibilityVersion": "Xcode 15.0", + "developmentRegion": "en", "hasScannedForEncodings": "0", "isa": "PBXProject", "mainGroup": main_group, + "productRefGroup": products, "projectDirPath": "", "projectRoot": "", + "targets": [tool, tests, app, apptests]}) + +def fmt(v, ind): + pad = "\t" * ind + if isinstance(v, dict): + return "{\n" + "".join('%s\t"%s" = %s;\n' % (pad, k, fmt(v[k], ind + 1)) for k in sorted(v)) + pad + "}" + if isinstance(v, list): + if not v: + return "(\n%s)" % pad + return "(\n" + ",\n".join('%s\t%s' % (pad, fmt(x, ind + 1)) for x in v) + "\n" + pad + ")" + return '"%s"' % v + +root = {"archiveVersion": "1", "classes": {}, "objectVersion": "56", "objects": objs, "rootObject": project} +open("Calc.xcodeproj/project.pbxproj", "w").write("// !$*UTF8*$!\n" + fmt(root, 0) + "\n") diff --git a/Tests/run-buildtool-test-action.sh b/Tests/run-buildtool-test-action.sh new file mode 100755 index 0000000..e43086d --- /dev/null +++ b/Tests/run-buildtool-test-action.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# +# Regressions for `buildtool test`, on Tests/Fixtures/Xcode/unit-tests +# (Calc.xcodeproj: a tool, its unit tests, an application, and unit tests +# hosted in the application). Needs tools-xctest's xctest (in PATH, or +# XCTEST=...); the hosted tests also need xvfb-run or an X display. +# +# usage: Tests/run-buildtool-test-action.sh [buildtool] + +set -euo pipefail + +root=$(cd "$(dirname "$0")/.." && pwd) +buildtool=${1:-$root/Tools/buildtool/obj/buildtool} +fixture=$root/Tests/Fixtures/Xcode/unit-tests +export LD_LIBRARY_PATH="$root/XCode/XCode.framework/Versions/Current${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + +if [ -z "${XCTEST:-}" ] && ! command -v xctest >/dev/null 2>&1 && [ ! -x /usr/GNUstep/System/Tools/xctest ]; then + echo "Skipping buildtool test regressions: xctest (tools-xctest) not found." + exit 0 +fi + +# The xctest buildtool will use, and the options it lists: -skip-testing +# and -host come with newer tools-xctest, and without them buildtool fails +# those cases with a message instead. +xctest=${XCTEST:-$(command -v xctest || echo /usr/GNUstep/System/Tools/xctest)} +xctest_usage=$("$xctest" --help 2>&1 || true) +can_skip=no; [[ "$xctest_usage" == *-skip-testing* ]] && can_skip=yes +can_host=no; [[ "$xctest_usage" == *-host* ]] && can_host=yes + +if command -v xvfb-run >/dev/null 2>&1; then + display=(xvfb-run -a) +elif [ -n "${DISPLAY:-}" ]; then + display=() +else + display=(none) +fi + +fail() { + echo "$1" >&2 + echo "$output" >&2 + exit 1 +} + +# run_buildtool [args...] sets $output and $status (in the fixture). +run_buildtool() { + set +e + output=$(cd "$fixture" && "$@" 2>&1) + status=$? + set -e +} + +assert_contains() { [[ "$output" == *"$1"* ]] || fail "expected output to contain: $1"; } +assert_not_contains() { [[ "$output" != *"$1"* ]] || fail "expected output to omit: $1"; } +# Whether xctest ran a test method; its log names it in any output format. +assert_ran() { grep -Eq "(^|[^A-Za-z0-9_])$1([^A-Za-z0-9_]|\$)" <<<"$output" || fail "expected $1 to run"; } +assert_not_ran() { ! grep -Eq "(^|[^A-Za-z0-9_])$1([^A-Za-z0-9_]|\$)" <<<"$output" || fail "expected $1 not to run"; } +assert_status() { [ "$status" -eq "$1" ] || fail "expected exit status $1, got $status"; } + +rm -rf "$fixture/build" + +echo "Running buildtool test regressions..." + +# Builds, then runs the unit-test bundle with xctest. +run_buildtool "$buildtool" test -target CalcTests +assert_status 0 +assert_contains "Testing Target" +assert_ran testAdds +assert_ran testAddsNegatives +assert_ran testKnownBroken +[ -f "$fixture/build/CalcTests/Products/CalcTests.xctest/CalcTests" ] || fail "expected the test bundle" +grep -Eq 'NSExecutable = "?CalcTests"?;' "$fixture/build/CalcTests/Products/CalcTests.xctest/Resources/Info-gnustep.plist" \ + || fail "expected the bundle's Info-gnustep.plist to name its executable" + +# A failing test fails the action and buildtool's exit status. +CALC_TESTS_FAIL=1 run_buildtool "$buildtool" test -target CalcTests +assert_status 1 +assert_contains "deliberately wrong" +assert_contains "Test Failed" + +# xcodebuild-style test selection. +run_buildtool "$buildtool" test -target CalcTests -only-testing:CalcTests/CalcTests/testAdds +assert_status 0 +assert_ran testAdds +assert_not_ran testAddsNegatives +assert_not_ran testKnownBroken +run_buildtool "$buildtool" test -target CalcTests -only-testing:CalcTests/CalcTests +assert_status 0 +assert_ran testAddsNegatives +run_buildtool "$buildtool" test -target CalcTests -skip-testing:CalcTests/CalcTests/testAdds +if [ "$can_skip" = yes ]; then + assert_status 0 + assert_not_ran testAdds + assert_ran testAddsNegatives +else + assert_status 1 + assert_contains "needs an xctest with -skip-testing" +fi + +# A scheme's testables and skipped tests (AppTests is not a testable there, +# and testAddsNegatives is skipped). +run_buildtool "$buildtool" test -scheme CalcTests +if [ "$can_skip" = yes ]; then + assert_status 0 + assert_ran testAdds + assert_not_ran testAddsNegatives +else + assert_status 1 + assert_contains "needs an xctest with -skip-testing" +fi +assert_not_ran testRunsInsideTheApp + +# Mistakes are failures. +run_buildtool "$buildtool" test -target CalcTool +assert_status 1 +assert_contains "CalcTool is not a unit-test target" +run_buildtool "$buildtool" test -scheme Missing +assert_status 1 +assert_contains "Could not read scheme Missing" + +# TEST_HOST: the tests run inside the application it names. +if [ "${display[0]:-}" != none ]; then + run_buildtool "${display[@]}" "$buildtool" test -only-testing:AppTests + if [ "$can_host" = yes ]; then + assert_status 0 + assert_contains "CalcApp[" + assert_ran testRunsInsideTheApp + else + assert_status 1 + assert_contains "needs an xctest with -host" + fi + assert_not_ran testAdds +else + echo "Skipping the hosted test: no X display and no xvfb-run." +fi + +rm -rf "$fixture/build" +echo "buildtool test regressions passed." diff --git a/Tools/buildtool/ToolDelegate.m b/Tools/buildtool/ToolDelegate.m index 03d5733..65b00d9 100644 --- a/Tools/buildtool/ToolDelegate.m +++ b/Tools/buildtool/ToolDelegate.m @@ -118,7 +118,7 @@ [NSArray arrayWithObjects: @"-alltargets", @"-parallelizeTargets", @"-quiet", @"-verbose", @"-list", @"-showBuildSettings", @"-license", @"build", - @"install", @"clean", @"link", nil]; + @"install", @"clean", @"link", @"test", nil]; return [options containsObject: option]; } @@ -210,6 +210,45 @@ - (NSDictionary *) parseArguments return result; } +// -target, -scheme, and xcodebuild's -only-testing: and +// -skip-testing: (which can be repeated), for the test action. +- (NSDictionary *) testOptions: (NSDictionary *)args +{ + NSMutableDictionary *options = [NSMutableDictionary dictionary]; + NSMutableArray *only = [NSMutableArray array]; + NSMutableArray *skip = [NSMutableArray array]; + NSEnumerator *en = + [[[NSProcessInfo processInfo] arguments] objectEnumerator]; + NSString *arg = nil; + + if ([[args objectForKey: @"-target"] value] != nil) + { + [options setObject: [[args objectForKey: @"-target"] value] + forKey: @"target"]; + } + if ([[args objectForKey: @"-scheme"] value] != nil) + { + [options setObject: [[args objectForKey: @"-scheme"] value] + forKey: @"scheme"]; + } + while ((arg = [en nextObject]) != nil) + { + if ([arg hasPrefix: @"-only-testing:"]) + { + [only addObject: + [arg substringFromIndex: [@"-only-testing:" length]]]; + } + else if ([arg hasPrefix: @"-skip-testing:"]) + { + [skip addObject: + [arg substringFromIndex: [@"-skip-testing:" length]]]; + } + } + [options setObject: only forKey: @"only"]; + [options setObject: skip forKey: @"skip"]; + return options; +} + - (BOOL) applyTargetName: (NSString *)targetName toProject: (PBXProject *)project { NSMutableArray *matchingTargets = [NSMutableArray array]; @@ -336,6 +375,12 @@ - (BOOL) process function = @"clean"; } + opt = [args objectForKey: @"test"]; + if (opt != nil) + { + function = @"test"; + } + opt = [args objectForKey: @"generate"]; if (opt != nil) { @@ -384,6 +429,10 @@ - (BOOL) process coder = [[PBXCoder alloc] initWithContentsOfFile: fileName]; container = [coder unarchive]; [container setParameter: parameter]; + if ([function isEqualToString: @"test"]) + { + [container setTestOptions: [self testOptions: args]]; + } opt = [args objectForKey: @"-configuration"]; if (opt != nil) @@ -397,7 +446,8 @@ - (BOOL) process { opt = [args objectForKey: @"-scheme"]; } - if (opt != nil && [args objectForKey: @"-alltargets"] == nil && + if (opt != nil && [args objectForKey: @"-alltargets"] == nil + && ![function isEqualToString: @"test"] && [self applyTargetName: [opt value] toProject: [container rootObject]] == NO) diff --git a/XCode/GNUmakefile b/XCode/GNUmakefile index 9d60193..eb12b51 100644 --- a/XCode/GNUmakefile +++ b/XCode/GNUmakefile @@ -29,6 +29,7 @@ XCode_RESOURCE_FILES = Resources/Framework-mapping.plist \ # XCode_HEADER_FILES = \ GSXCBuildDatabase.h \ +PBXProject+Testing.h \ GSXCBuildContext.h \ GSXCCommon.h \ NSArray+Additions.h \ @@ -79,6 +80,7 @@ XCode.h \ # XCode_OBJC_FILES = \ GSXCBuildDatabase.m \ +PBXProject+Testing.m \ GSXCBuildContext.m \ NSArray+Additions.m \ NSString+PBXAdditions.m \ diff --git a/XCode/PBXContainer.h b/XCode/PBXContainer.h index 71bca3c..215db88 100644 --- a/XCode/PBXContainer.h +++ b/XCode/PBXContainer.h @@ -41,8 +41,19 @@ NSString *_workspaceLink; NSString *_workspaceLibs; NSString *_workspaceIncludes; + NSDictionary *_testOptions; } +/** + * Options for -test; see -[PBXProject(Testing) testWithOptions:]. + */ +- (void) setTestOptions: (NSDictionary *)options; + +/** + * Builds the project and runs its unit-test targets with xctest. + */ +- (BOOL) test; + /** * Initializes the container with the given root object. */ diff --git a/XCode/PBXContainer.m b/XCode/PBXContainer.m index c982ee2..7829f94 100644 --- a/XCode/PBXContainer.m +++ b/XCode/PBXContainer.m @@ -23,6 +23,7 @@ */ #import "PBXContainer.h" +#import "PBXProject+Testing.h" #import "PBXCommon.h" #import "PBXProject.h" #import "PBXFileReference.h" @@ -68,6 +69,7 @@ - (void) dealloc RELEASE (_objects); RELEASE (_filename); RELEASE (_parameter); + RELEASE (_testOptions); RELEASE (_workspaceLink); RELEASE (_workspaceLibs); RELEASE (_workspaceIncludes); @@ -240,6 +242,18 @@ - (BOOL) clean return [_rootObject clean]; } +- (void) setTestOptions: (NSDictionary *)options +{ + ASSIGN(_testOptions, options); +} + +- (BOOL) test +{ + [self collectHeaderFileReferences]; + [_rootObject setContainer: self]; + return [_rootObject testWithOptions: _testOptions]; +} + - (BOOL) install { return [_rootObject install]; diff --git a/XCode/PBXProject+Testing.h b/XCode/PBXProject+Testing.h new file mode 100644 index 0000000..787bada --- /dev/null +++ b/XCode/PBXProject+Testing.h @@ -0,0 +1,48 @@ +/* + Copyright (C) 2026 Free Software Foundation, Inc. + + Written by: Daniel Boyd + Date: 2026 + + This file is part of the GNUstep XCode Library + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free + Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110 USA. +*/ + +#import + +#import "PBXProject.h" + +/** + * Running a project's unit tests, as `buildtool test` (like `xcodebuild + * test`): builds the project, then runs each unit-test target's bundle + * with xctest (tools-xctest). + */ +@interface PBXProject (Testing) + +/** + * Builds the project, then runs its test targets. Options (all optional): + * "target" (a test target's name), "scheme" (an .xcscheme in the project: + * its testables, skipped and selected tests), "only" and "skip" (arrays of + * xcodebuild identifiers, TestTarget[/TestClass[/testMethod]]). Without a + * target or scheme, every unit-test target runs. A target whose + * TEST_HOST names an application in the project runs inside it (xctest + * -host). The XCTEST environment variable chooses the xctest program and + * XCTEST_FLAGS adds options to it. Returns YES if every test passed. + */ +- (BOOL) testWithOptions: (NSDictionary *)options; + +@end diff --git a/XCode/PBXProject+Testing.m b/XCode/PBXProject+Testing.m new file mode 100644 index 0000000..cb783fa --- /dev/null +++ b/XCode/PBXProject+Testing.m @@ -0,0 +1,618 @@ +/* + Copyright (C) 2026 Free Software Foundation, Inc. + + Written by: Daniel Boyd + Date: 2026 + + This file is part of the GNUstep XCode Library + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free + Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110 USA. +*/ + +#import "PBXCommon.h" +#import "PBXProject+Testing.h" +#import "PBXContainer.h" +#import "PBXFileReference.h" +#import "PBXNativeTarget.h" +#import "GSXCCommon.h" +#import "XCBuildConfiguration.h" +#import "XCConfigurationList.h" + +#include +#include + +/** + * The test part of an .xcscheme: the test targets it runs and, for each, + * the tests it skips or (with useTestSelectionWhitelist) only runs. + */ +@interface GSXCSchemeTests : NSObject +{ + NSMutableArray *_targets; + NSMutableDictionary *_skipped; + NSMutableDictionary *_selected; + NSString *_currentTarget; + BOOL _currentSkipped; + BOOL _currentWhitelist; + BOOL _inTestable; + BOOL _inSkipped; + BOOL _inSelected; +} ++ (GSXCSchemeTests *) testsFromSchemeAtPath: (NSString *)path; +/** Test target names, in order. */ +- (NSArray *) targets; +/** TestTarget/TestClass[/testMethod] identifiers. */ +- (NSArray *) skippedTests; +- (NSArray *) selectedTests; +@end + +@implementation GSXCSchemeTests + ++ (GSXCSchemeTests *) testsFromSchemeAtPath: (NSString *)path +{ + NSData *data = [NSData dataWithContentsOfFile: path]; + NSXMLParser *parser = nil; + GSXCSchemeTests *tests = nil; + + if (data == nil) + { + return nil; + } + + tests = AUTORELEASE([[self alloc] init]); + parser = AUTORELEASE([[NSXMLParser alloc] initWithData: data]); + [parser setDelegate: tests]; + if ([parser parse] == NO) + { + return nil; + } + return tests; +} + +- (instancetype) init +{ + self = [super init]; + if (self != nil) + { + _targets = [[NSMutableArray alloc] init]; + _skipped = [[NSMutableDictionary alloc] init]; + _selected = [[NSMutableDictionary alloc] init]; + } + return self; +} + +- (void) dealloc +{ + RELEASE(_targets); + RELEASE(_skipped); + RELEASE(_selected); + RELEASE(_currentTarget); + [super dealloc]; +} + +- (NSArray *) targets +{ + return _targets; +} + +- (NSArray *) _identifiersIn: (NSDictionary *)tests +{ + NSMutableArray *identifiers = [NSMutableArray array]; + NSEnumerator *en = [_targets objectEnumerator]; + NSString *target = nil; + + while ((target = [en nextObject]) != nil) + { + NSEnumerator *ten = [[tests objectForKey: target] objectEnumerator]; + NSString *test = nil; + + while ((test = [ten nextObject]) != nil) + { + [identifiers addObject: [NSString stringWithFormat: @"%@/%@", target, test]]; + } + } + return identifiers; +} + +- (NSArray *) skippedTests +{ + return [self _identifiersIn: _skipped]; +} + +- (NSArray *) selectedTests +{ + return [self _identifiersIn: _selected]; +} + +- (void) parser: (NSXMLParser *)parser +didStartElement: (NSString *)elementName + namespaceURI: (NSString *)namespaceURI + qualifiedName: (NSString *)qName + attributes: (NSDictionary *)attributes +{ + if ([elementName isEqualToString: @"TestableReference"]) + { + _inTestable = YES; + _currentSkipped = [[attributes objectForKey: @"skipped"] isEqualToString: @"YES"]; + _currentWhitelist = [[attributes objectForKey: @"useTestSelectionWhitelist"] isEqualToString: @"YES"]; + DESTROY(_currentTarget); + } + else if (_inTestable && [elementName isEqualToString: @"BuildableReference"]) + { + ASSIGN(_currentTarget, [attributes objectForKey: @"BlueprintName"]); + if (_currentTarget != nil && _currentSkipped == NO) + { + [_targets addObject: _currentTarget]; + [_skipped setObject: [NSMutableArray array] forKey: _currentTarget]; + if (_currentWhitelist) + { + [_selected setObject: [NSMutableArray array] forKey: _currentTarget]; + } + } + } + else if (_inTestable && [elementName isEqualToString: @"SkippedTests"]) + { + _inSkipped = YES; + } + else if (_inTestable && [elementName isEqualToString: @"SelectedTests"]) + { + _inSelected = YES; + } + else if (_inTestable && [elementName isEqualToString: @"Test"] && _currentTarget != nil) + { + NSString *identifier = [attributes objectForKey: @"Identifier"]; + + // Swift identifiers end in "()"; xctest names methods without it. + if ([identifier hasSuffix: @"()"]) + { + identifier = [identifier substringToIndex: [identifier length] - 2]; + } + if (identifier != nil && _inSkipped) + { + [[_skipped objectForKey: _currentTarget] addObject: identifier]; + } + else if (identifier != nil && _inSelected) + { + [[_selected objectForKey: _currentTarget] addObject: identifier]; + } + } +} + +- (void) parser: (NSXMLParser *)parser + didEndElement: (NSString *)elementName + namespaceURI: (NSString *)namespaceURI + qualifiedName: (NSString *)qName +{ + if ([elementName isEqualToString: @"TestableReference"]) + { + _inTestable = NO; + } + else if ([elementName isEqualToString: @"SkippedTests"]) + { + _inSkipped = NO; + } + else if ([elementName isEqualToString: @"SelectedTests"]) + { + _inSelected = NO; + } +} + +@end + +// The xctest program: $XCTEST, else the first in PATH, else GNUstep's. +static NSString *GSXCTestProgram(void) +{ + NSDictionary *env = [[NSProcessInfo processInfo] environment]; + NSString *program = [env objectForKey: @"XCTEST"]; + NSFileManager *fm = [NSFileManager defaultManager]; + NSEnumerator *en = nil; + NSString *dir = nil; + + if ([program length] > 0) + { + return program; + } + + en = [[[env objectForKey: @"PATH"] componentsSeparatedByString: @":"] objectEnumerator]; + while ((dir = [en nextObject]) != nil) + { + NSString *candidate = [dir stringByAppendingPathComponent: @"xctest"]; + + if ([fm isExecutableFileAtPath: candidate]) + { + return candidate; + } + } + + en = [[NSArray arrayWithObjects: @"/usr/GNUstep/Local/Tools/xctest", + @"/usr/GNUstep/System/Tools/xctest", nil] objectEnumerator]; + while ((program = [en nextObject]) != nil) + { + if ([fm isExecutableFileAtPath: program]) + { + return program; + } + } + return nil; +} + +// xctest's usage text. tools-xctest has gained options over time, so the +// test action passes only the ones the program lists. +static NSString *GSXCTestUsage(NSString *xctest) +{ + NSTask *task = AUTORELEASE([[NSTask alloc] init]); + NSPipe *pipe = [NSPipe pipe]; + NSData *data = nil; + + [task setLaunchPath: xctest]; + [task setArguments: [NSArray arrayWithObject: @"--help"]]; + [task setStandardOutput: pipe]; + [task setStandardError: pipe]; + NS_DURING + { + [task launch]; + data = [[pipe fileHandleForReading] readDataToEndOfFile]; + [task waitUntilExit]; + } + NS_HANDLER + { + data = nil; + } + NS_ENDHANDLER + if (data == nil) + { + return @""; + } + return AUTORELEASE([[NSString alloc] initWithData: data + encoding: NSUTF8StringEncoding]); +} + +// xctest's -XCTest selection (Class or Class.method, comma-separated) for +// the identifiers (TestTarget/Class[/method]) of one target, or nil when +// one of them names the whole target. +static NSString *GSXCSelectionForTarget(NSArray *identifiers, NSString *target) +{ + NSMutableArray *names = [NSMutableArray array]; + NSEnumerator *en = [identifiers objectEnumerator]; + NSString *identifier = nil; + + while ((identifier = [en nextObject]) != nil) + { + NSArray *components = [identifier componentsSeparatedByString: @"/"]; + + if (![[components objectAtIndex: 0] isEqualToString: target]) + { + continue; + } + if ([components count] == 1) + { + return nil; + } + if ([components count] == 2) + { + [names addObject: [components objectAtIndex: 1]]; + } + else + { + [names addObject: [NSString stringWithFormat: @"%@.%@", + [components objectAtIndex: 1], + [components objectAtIndex: 2]]]; + } + } + return [names componentsJoinedByString: @","]; +} + +// Where buildtool puts a target's product. +static NSString *GSXCProductPath(PBXNativeTarget *target) +{ + return [[[@"./build" stringByAppendingPathComponent: [target name]] + stringByAppendingPathComponent: @"Products"] + stringByAppendingPathComponent: [[target productReference] path]]; +} + +// xcodebuild identifiers (TestTarget/...) for one target, renamed for its +// bundle (xctest names the target after the bundle). +static NSArray *GSXCIdentifiersForTarget(NSArray *identifiers, NSString *target, NSString *bundleName) +{ + NSMutableArray *result = [NSMutableArray array]; + NSEnumerator *en = [identifiers objectEnumerator]; + NSString *identifier = nil; + + while ((identifier = [en nextObject]) != nil) + { + NSMutableArray *components = [NSMutableArray arrayWithArray: + [identifier componentsSeparatedByString: @"/"]]; + + if ([components count] > 0 && [[components objectAtIndex: 0] isEqualToString: target]) + { + [components replaceObjectAtIndex: 0 withObject: bundleName]; + [result addObject: [components componentsJoinedByString: @"/"]]; + } + } + return result; +} + +@implementation PBXProject (Testing) + +- (NSString *) _schemePathForName: (NSString *)name +{ + NSString *projectDir = [[[self container] filename] stringByDeletingLastPathComponent]; + NSString *fileName = [name stringByAppendingPathExtension: @"xcscheme"]; + NSFileManager *fm = [NSFileManager defaultManager]; + NSString *path = [[projectDir stringByAppendingPathComponent: @"xcshareddata/xcschemes"] + stringByAppendingPathComponent: fileName]; + NSString *userDir = [projectDir stringByAppendingPathComponent: @"xcuserdata"]; + NSEnumerator *en = nil; + NSString *user = nil; + + if ([fm fileExistsAtPath: path]) + { + return path; + } + + en = [[fm contentsOfDirectoryAtPath: userDir error: NULL] objectEnumerator]; + while ((user = [en nextObject]) != nil) + { + path = [[[userDir stringByAppendingPathComponent: user] + stringByAppendingPathComponent: @"xcschemes"] + stringByAppendingPathComponent: fileName]; + if ([fm fileExistsAtPath: path]) + { + return path; + } + } + return nil; +} + +- (PBXNativeTarget *) _targetNamed: (NSString *)name +{ + NSEnumerator *en = [[self targets] objectEnumerator]; + id target = nil; + + while ((target = [en nextObject]) != nil) + { + if ([[target name] isEqualToString: name]) + { + return target; + } + } + return nil; +} + +// The application a test target's TEST_HOST names, as built here; nil if +// it has none (or it isn't an application in this project). +- (NSString *) _hostForTestTarget: (PBXNativeTarget *)target +{ + XCBuildConfiguration *config = [[target buildConfigurationList] defaultConfiguration]; + NSString *testHost = [[config buildSettings] objectForKey: @"TEST_HOST"]; + NSEnumerator *en = nil; + NSString *component = nil; + NSString *appName = nil; + id candidate = nil; + + if ([testHost length] == 0) + { + return nil; + } + + // e.g. $(BUILT_PRODUCTS_DIR)/MyApp.app/Contents/MacOS/MyApp + en = [[testHost pathComponents] objectEnumerator]; + while ((component = [en nextObject]) != nil) + { + if ([[component pathExtension] isEqualToString: @"app"]) + { + appName = component; + break; + } + } + + en = [[self targets] objectEnumerator]; + while ((candidate = [en nextObject]) != nil) + { + if ([candidate isKindOfClass: [PBXNativeTarget class]] + && [[candidate productType] isEqualToString: APPLICATION_TYPE] + && [[[[candidate productReference] path] lastPathComponent] isEqualToString: appName]) + { + return GSXCProductPath(candidate); + } + } + + xcprintf("\t** TEST_HOST %s isn't an application built by this project; running without a host\n", + [testHost cString]); + return nil; +} + +- (BOOL) testWithOptions: (NSDictionary *)options +{ + NSString *targetName = [options objectForKey: @"target"]; + NSString *schemeName = [options objectForKey: @"scheme"]; + NSMutableArray *only = [NSMutableArray arrayWithArray: [options objectForKey: @"only"]]; + NSMutableArray *skip = [NSMutableArray arrayWithArray: [options objectForKey: @"skip"]]; + NSMutableArray *testTargets = [NSMutableArray array]; + NSString *xctest = GSXCTestProgram(); + NSArray *extraFlags = nil; + NSString *usage = nil; + BOOL canSkip = NO; + BOOL canHost = NO; + NSString *flags = [[[NSProcessInfo processInfo] environment] objectForKey: @"XCTEST_FLAGS"]; + NSEnumerator *en = nil; + id target = nil; + BOOL passed = YES; + NSUInteger ran = 0; + + // Which test targets. + if (targetName != nil) + { + target = [self _targetNamed: targetName]; + if (target == nil || ![[target productType] isEqualToString: TEST_TYPE]) + { + xcprintf("*** %s is not a unit-test target in this project\n", [targetName cString]); + return NO; + } + [testTargets addObject: target]; + } + else if (schemeName != nil) + { + NSString *path = [self _schemePathForName: schemeName]; + GSXCSchemeTests *scheme = path ? [GSXCSchemeTests testsFromSchemeAtPath: path] : nil; + NSString *name = nil; + + if (scheme == nil) + { + xcprintf("*** Could not read scheme %s\n", [schemeName cString]); + return NO; + } + en = [[scheme targets] objectEnumerator]; + while ((name = [en nextObject]) != nil) + { + target = [self _targetNamed: name]; + if (target != nil && [[target productType] isEqualToString: TEST_TYPE]) + { + [testTargets addObject: target]; + } + } + [skip addObjectsFromArray: [scheme skippedTests]]; + [only addObjectsFromArray: [scheme selectedTests]]; + } + else + { + en = [[self targets] objectEnumerator]; + while ((target = [en nextObject]) != nil) + { + if ([target isKindOfClass: [PBXNativeTarget class]] + && [[target productType] isEqualToString: TEST_TYPE]) + { + [testTargets addObject: target]; + } + } + } + + if ([testTargets count] == 0) + { + xcputs("*** No unit-test targets to run"); + return NO; + } + if (xctest == nil) + { + xcputs("*** Could not find xctest (install tools-xctest, or set XCTEST)"); + return NO; + } + + usage = GSXCTestUsage(xctest); + canSkip = [usage rangeOfString: @"-skip-testing"].location != NSNotFound; + canHost = [usage rangeOfString: @"-host"].location != NSNotFound; + + // As xcodebuild test does, build first. + if ([self build] == NO) + { + return NO; + } + + extraFlags = [flags length] > 0 + ? [[flags componentsSeparatedByCharactersInSet: [NSCharacterSet whitespaceCharacterSet]] + filteredArrayUsingPredicate: [NSPredicate predicateWithFormat: @"length > 0"]] + : [NSArray array]; + + en = [testTargets objectEnumerator]; + while ((target = [en nextObject]) != nil) + { + NSString *bundle = GSXCProductPath(target); + NSString *bundleName = [[bundle lastPathComponent] stringByDeletingPathExtension]; + NSArray *targetOnly = GSXCIdentifiersForTarget(only, [target name], bundleName); + NSArray *targetSkip = GSXCIdentifiersForTarget(skip, [target name], bundleName); + NSString *host = [self _hostForTestTarget: target]; + NSMutableArray *arguments = [NSMutableArray arrayWithObject: bundle]; + NSEnumerator *ien = nil; + NSString *identifier = nil; + NSTask *task = nil; + + // Selecting tests elsewhere leaves this target out, as in xcodebuild. + if ([only count] > 0 && [targetOnly count] == 0) + { + continue; + } + + if ([targetOnly count] > 0) + { + NSString *selection = GSXCSelectionForTarget(only, [target name]); + + if (selection != nil) + { + [arguments addObject: @"-XCTest"]; + [arguments addObject: selection]; + } + } + if ([targetSkip count] > 0) + { + if (canSkip == NO) + { + xcprintf("*** %s: skipping tests needs an xctest with -skip-testing, which %s lacks\n", + [[target name] cString], [xctest cString]); + passed = NO; + ran++; + continue; + } + ien = [targetSkip objectEnumerator]; + while ((identifier = [ien nextObject]) != nil) + { + [arguments addObject: [@"-skip-testing:" stringByAppendingString: identifier]]; + } + } + if (host != nil) + { + if (canHost == NO) + { + xcprintf("*** %s runs inside %s (TEST_HOST), which needs an xctest with -host, which %s lacks\n", + [[target name] cString], [host cString], [xctest cString]); + passed = NO; + ran++; + continue; + } + [arguments addObject: @"-host"]; + [arguments addObject: host]; + } + [arguments addObjectsFromArray: extraFlags]; + + xcprintf("=== Testing Target %s%s%s%s with %s\n", BOLD, GREEN, [[target name] cString], RESET, + [xctest cString]); + fflush(stdout); + + task = AUTORELEASE([[NSTask alloc] init]); + [task setLaunchPath: xctest]; + [task setArguments: arguments]; + [task launch]; + [task waitUntilExit]; + ran++; + + if ([task terminationStatus] != 0) + { + passed = NO; + xcprintf("=== Tests %sFAILED%s for %s\n", RED, RESET, [[target name] cString]); + } + else + { + xcprintf("=== Tests %spassed%s for %s\n", GREEN, RESET, [[target name] cString]); + } + } + + if (ran == 0) + { + xcputs("*** No test target matched the selected tests"); + return NO; + } + return passed; +} + +@end