Skip to content

Commit bdd99ed

Browse files
rr-codesrr-codes
committed
[Swift in WebKit] Integrate Swift Testing into run-api-tests (part 2)
https://bugs.webkit.org/show_bug.cgi?id=311769 rdar://174358586 Reviewed by NOBODY (OOPS!). Adds the ability for `run-api-tests` to also include and run tests written with Swift Testing. For now, this PR limits the scope to just those tests in `WKWebViewSwiftOverlayTests.swift` for validation purposes; a subsequent PR will enable this for all the existing Swift Testing tests. * Tools/Scripts/webkitpy/api_tests/manager.py: (Manager._test_list_from_output): (Manager._find_test_subset): (Manager._collect_tests): * Tools/Scripts/webkitpy/api_tests/runner.py: (_Worker._run_single_test): Modify the `run-api-tests` script to handle Swift Testing tests; specifically their different naming scheme compared to gtest tests. * Tools/Scripts/webkitpy/api_tests/manager_unittest.py: (test_test_list_from_output_mixed_gtest_and_swift): (test_find_test_subset_gtest_full_name): (test_find_test_subset_gtest_suite_and_test): (test_find_test_subset_gtest_binary_and_suite): (test_find_test_subset_gtest_suite_only): (test_find_test_subset_gtest_wildcard): (test_find_test_subset_swift_full_name): (test_find_test_subset_swift_suite_and_test): (test_find_test_subset_swift_binary_and_suite): (test_find_test_subset_swift_suite_only): (test_find_test_subset_swift_wildcard): (test_find_test_subset_parentheses_are_literal): (test_find_test_subset_no_match): Add new tests for the script. * Tools/TestWebKitAPI/Configurations/TestIPC.xcconfig: * Tools/TestWebKitAPI/Configurations/TestWGSL.xcconfig: * Tools/TestWebKitAPI/Configurations/TestWTF.xcconfig: * Tools/TestWebKitAPI/Configurations/TestWebKitAPI.xcconfig: * Tools/TestWebKitAPI/Configurations/TestWebKitAPIBase.xcconfig: Link Testing and update the LD_RUNPATH_SEARCH_PATHS as needed. * Tools/TestWebKitAPI/Runner/GoogleTestsController.swift: Drive-by cleanup: Make a function `private` * Tools/TestWebKitAPI/Runner/SwiftTestingABI.swift: Added. (__CommandLineArguments_v0.verbosity): (__CommandLineArguments_v0.listTests): (__CommandLineArguments_v0.filter): (__CommandLineArguments_v0.skip): (__CommandLineArguments_v0.repetitions): (__CommandLineArguments_v0.parallel): (ID.stringValue): (ID.encode(to:)): * Tools/TestWebKitAPI/Runner/SwiftTestsController.swift: Added. (Storage.tests): (Storage.events): (saveRecord(_:)): (writeOutputIfNeeded(_:)): (handleRecord(_:)): (run(with:)): (SwiftTestingABI.canonicalizedRepresentation): * Tools/TestWebKitAPI/Runner/TestWebKitAPI.swift: (TestWebKitAPI.run): * Tools/TestWebKitAPI/Runner/TestWebKitAPISupport.h: * Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj: Add a new `TestRunner` type that is responsible for running Swift Testing types from within the binary directly. This uses the Swift Testing ABI types and functions.
1 parent 2852545 commit bdd99ed

13 files changed

Lines changed: 551 additions & 32 deletions

File tree

‎Tools/Scripts/webkitpy/api_tests/manager.py‎

Lines changed: 42 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -52,52 +52,68 @@ def __init__(self, port, options, stream):
5252
self._stream = stream
5353

5454
@staticmethod
55-
def _test_list_from_output(output, prefix=''):
55+
def _test_list_from_output(output: str, prefix='') -> list[str]:
5656
result = []
5757
current_test_suite = None
5858
for line in output.split('\n'):
59-
line = line.split("#")[0] # Value-parametrized tests contain #.
60-
striped_line = line.lstrip().rstrip()
61-
if not striped_line:
62-
continue
63-
64-
if striped_line[-1] == '.':
65-
current_test_suite = striped_line[:-1]
59+
if prefix and line.startswith(prefix) and '/' in line:
60+
# this implies it's a Swift Test
61+
result.append(line)
6662
else:
67-
striped_line = striped_line.lstrip()
68-
if ' ' in striped_line:
63+
line = line.split("#")[0] # Value-parametrized tests contain #.
64+
striped_line = line.lstrip().rstrip()
65+
if not striped_line:
6966
continue
70-
val = f'{prefix}{current_test_suite}.{striped_line}'
71-
if val not in result:
72-
result.append(val)
67+
68+
if striped_line[-1] == '.':
69+
current_test_suite = striped_line[:-1]
70+
else:
71+
striped_line = striped_line.lstrip()
72+
if ' ' in striped_line:
73+
continue
74+
val = f'{prefix}{current_test_suite}.{striped_line}'
75+
if val not in result:
76+
result.append(val)
7377
return result
7478

7579
@staticmethod
7680
def _find_test_subset(superset, arg_filter):
7781
result = []
7882
for arg in arg_filter:
79-
# Might match <binary>.<suite>.<test> or just <suite>.<test>
80-
arg_re = re.compile(f'^{arg.replace("*", ".*")}$')
83+
escaped = re.escape(arg).replace(r'\*', '.*')
84+
arg_re = re.compile(f'^{escaped}$')
8185
for test in superset:
8286
if arg_re.match(test):
8387
result.append(test)
8488
continue
8589

86-
split_test = test.split('.')
87-
if len(split_test) == 1:
88-
continue
89-
if arg_re.match('.'.join(split_test[1:])):
90-
result.append(test)
91-
continue
92-
if arg_re.match('.'.join(split_test[:-1])):
93-
result.append(test)
90+
first_dot = test.find('.')
91+
if first_dot == -1:
9492
continue
9593

96-
if len(split_test) == 2:
97-
continue
98-
if arg_re.match('.'.join(split_test[1:-1])):
94+
# Find the last separator (either '.' or '/')
95+
last_dot = test.rfind('.')
96+
last_slash = test.rfind('/')
97+
last_sep = max(last_dot, last_slash)
98+
99+
# Without binary prefix
100+
without_binary = test[first_dot + 1:]
101+
if arg_re.match(without_binary):
99102
result.append(test)
100103
continue
104+
105+
# Without test function suffix
106+
if last_sep > first_dot:
107+
without_test = test[:last_sep]
108+
if arg_re.match(without_test):
109+
result.append(test)
110+
continue
111+
112+
# Just the suite (middle part)
113+
suite_only = test[first_dot + 1:last_sep]
114+
if arg_re.match(suite_only):
115+
result.append(test)
116+
continue
101117
return result
102118

103119
def _collect_tests(self, args):

‎Tools/Scripts/webkitpy/api_tests/manager_unittest.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,69 @@ def test_test_list_from_output(self):
5151
]
5252
got_tests = Manager._test_list_from_output(gtest_list_tests_output)
5353
self.assertEqual(expected_tests, got_tests)
54+
55+
def test_test_list_from_output_mixed_gtest_and_swift(self):
56+
mixed_output = """WKWebViewDisableSelection.
57+
DoubleClickDoesNotSelectWhenTextInteractionsAreDisabled
58+
TestWebKitAPI.WKWebViewSwiftOverlayTests/evaluateJavaScriptWithNilResponse()
59+
TestWebKitAPI.WKWebViewSwiftOverlayTests/evaluateJavaScriptYieldsExpectedResponse()
60+
"""
61+
expected_tests = [
62+
"TestWebKitAPI.WKWebViewDisableSelection.DoubleClickDoesNotSelectWhenTextInteractionsAreDisabled",
63+
"TestWebKitAPI.WKWebViewSwiftOverlayTests/evaluateJavaScriptWithNilResponse()",
64+
"TestWebKitAPI.WKWebViewSwiftOverlayTests/evaluateJavaScriptYieldsExpectedResponse()",
65+
]
66+
got_tests = Manager._test_list_from_output(mixed_output, prefix='TestWebKitAPI.')
67+
self.assertEqual(expected_tests, got_tests)
68+
69+
def test_find_test_subset_gtest_full_name(self):
70+
superset = ["TestWebKitAPI.WebKit.SomeTest"]
71+
self.assertEqual(superset, Manager._find_test_subset(superset, ["TestWebKitAPI.WebKit.SomeTest"]))
72+
73+
def test_find_test_subset_gtest_suite_and_test(self):
74+
superset = ["TestWebKitAPI.WebKit.SomeTest"]
75+
self.assertEqual(superset, Manager._find_test_subset(superset, ["WebKit.SomeTest"]))
76+
77+
def test_find_test_subset_gtest_binary_and_suite(self):
78+
superset = ["TestWebKitAPI.WebKit.SomeTest"]
79+
self.assertEqual(superset, Manager._find_test_subset(superset, ["TestWebKitAPI.WebKit"]))
80+
81+
def test_find_test_subset_gtest_suite_only(self):
82+
superset = ["TestWebKitAPI.WebKit.SomeTest"]
83+
self.assertEqual(superset, Manager._find_test_subset(superset, ["WebKit"]))
84+
85+
def test_find_test_subset_gtest_wildcard(self):
86+
superset = ["TestWebKitAPI.WebKit.SomeTest", "TestWebKitAPI.WebKit.OtherTest"]
87+
self.assertEqual(superset, Manager._find_test_subset(superset, ["WebKit.*"]))
88+
89+
def test_find_test_subset_swift_full_name(self):
90+
superset = ["TestWebKitAPI.WKWebViewSwiftOverlayTests/evaluateJavaScriptWithNilResponse()"]
91+
self.assertEqual(superset, Manager._find_test_subset(superset, ["TestWebKitAPI.WKWebViewSwiftOverlayTests/evaluateJavaScriptWithNilResponse()"]))
92+
93+
def test_find_test_subset_swift_suite_and_test(self):
94+
superset = ["TestWebKitAPI.WKWebViewSwiftOverlayTests/evaluateJavaScriptWithNilResponse()"]
95+
self.assertEqual(superset, Manager._find_test_subset(superset, ["WKWebViewSwiftOverlayTests/evaluateJavaScriptWithNilResponse()"]))
96+
97+
def test_find_test_subset_swift_binary_and_suite(self):
98+
superset = ["TestWebKitAPI.WKWebViewSwiftOverlayTests/evaluateJavaScriptWithNilResponse()"]
99+
self.assertEqual(superset, Manager._find_test_subset(superset, ["TestWebKitAPI.WKWebViewSwiftOverlayTests"]))
100+
101+
def test_find_test_subset_swift_suite_only(self):
102+
superset = ["TestWebKitAPI.WKWebViewSwiftOverlayTests/evaluateJavaScriptWithNilResponse()"]
103+
self.assertEqual(superset, Manager._find_test_subset(superset, ["WKWebViewSwiftOverlayTests"]))
104+
105+
def test_find_test_subset_swift_wildcard(self):
106+
superset = [
107+
"TestWebKitAPI.WKWebViewSwiftOverlayTests/evaluateJavaScriptWithNilResponse()",
108+
"TestWebKitAPI.WKWebViewSwiftOverlayTests/otherTest()",
109+
]
110+
self.assertEqual(superset, Manager._find_test_subset(superset, ["WKWebViewSwiftOverlayTests/*"]))
111+
112+
def test_find_test_subset_parentheses_are_literal(self):
113+
superset = ["TestWebKitAPI.Suite/testFunc()"]
114+
self.assertEqual(superset, Manager._find_test_subset(superset, ["Suite/testFunc()"]))
115+
self.assertEqual([], Manager._find_test_subset(superset, ["Suite/testFuncXY"]))
116+
117+
def test_find_test_subset_no_match(self):
118+
superset = ["TestWebKitAPI.WebKit.SomeTest"]
119+
self.assertEqual([], Manager._find_test_subset(superset, ["NonExistent"]))

‎Tools/Scripts/webkitpy/api_tests/runner.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,8 @@ def _run_single_test(self, binary_name, test):
334334
env=self._port.environment_for_api_tests())
335335

336336
status = Runner.STATUS_RUNNING
337-
if test.split('.')[1].startswith('DISABLED_') and not self._port.get_option('force'):
337+
split_test = test.split('.')
338+
if len(split_test) > 1 and split_test[1].startswith('DISABLED_') and not self._port.get_option('force'):
338339
status = Runner.STATUS_DISABLED
339340

340341
stdout_buffer = ''

‎Tools/TestWebKitAPI/Configurations/TestIPC.xcconfig‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,12 @@ WK_BROWSERENGINEKIT_LDFLAGS[sdk=appletv*] =
4444
WK_BROWSERENGINEKIT_LDFLAGS[sdk=watch*] =
4545
WK_BROWSERENGINEKIT_LDFLAGS[sdk=xr*] =
4646

47-
OTHER_LDFLAGS = $(inherited) $(WK_UIKITMACHELPER_LDFLAGS) $(WK_BROWSERENGINEKIT_LDFLAGS) $(OTHER_LDFLAGS_PLATFORM_$(WK_COCOA_TOUCH)) -framework UniformTypeIdentifiers $(WK_PGO_GEN_FLAGS);
47+
OTHER_LDFLAGS = $(inherited) $(WK_UIKITMACHELPER_LDFLAGS) $(WK_BROWSERENGINEKIT_LDFLAGS) $(OTHER_LDFLAGS_PLATFORM_$(WK_COCOA_TOUCH)) -framework UniformTypeIdentifiers -framework Testing $(WK_PGO_GEN_FLAGS);
4848
OTHER_LDFLAGS_PLATFORM_ = -framework Cocoa -framework Carbon -framework CoreServices -framework IOSurface -framework CoreVideo;
4949
OTHER_LDFLAGS_PLATFORM_cocoatouch = -framework CoreGraphics -framework MobileCoreServices -framework UIKit -framework CoreText -framework IOSurface -framework CoreVideo;
5050

51+
LD_RUNPATH_SEARCH_PATHS = "@loader_path";
52+
5153
STRIP_STYLE = debugging;
5254

5355
ENTITLEMENTS_REQUIRED = $(ENTITLEMENTS_REQUIRED_USE_INTERNAL_SDK_$(USE_INTERNAL_SDK))

‎Tools/TestWebKitAPI/Configurations/TestWGSL.xcconfig‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,12 @@ GCC_PREPROCESSOR_DEFINITIONS = $(inherited) BUILDING_TEST_WGSL GTEST_API_=
3232
WK_UIKITMACHELPER_LDFLAGS = $(WK_UIKITMACHELPER_LDFLAGS_$(WK_PLATFORM_NAME));
3333
WK_UIKITMACHELPER_LDFLAGS_maccatalyst = -framework UIKitMacHelper;
3434

35-
OTHER_LDFLAGS = $(inherited) $(WK_UIKITMACHELPER_LDFLAGS) $(OTHER_LDFLAGS_PLATFORM_$(WK_COCOA_TOUCH));
35+
OTHER_LDFLAGS = $(inherited) -framework Testing $(WK_UIKITMACHELPER_LDFLAGS) $(OTHER_LDFLAGS_PLATFORM_$(WK_COCOA_TOUCH));
3636
OTHER_LDFLAGS_PLATFORM_ = -framework Cocoa -framework Carbon;
3737
OTHER_LDFLAGS_PLATFORM_cocoatouch = -framework CoreGraphics;
3838

39+
LD_RUNPATH_SEARCH_PATHS = "@loader_path";
40+
3941
STRIP_STYLE = debugging;
4042

4143
ENTITLEMENTS_REQUIRED = $(ENTITLEMENTS_REQUIRED_USE_INTERNAL_SDK_$(USE_INTERNAL_SDK))

‎Tools/TestWebKitAPI/Configurations/TestWTF.xcconfig‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,14 @@ GCC_PREPROCESSOR_DEFINITIONS = $(inherited) BUILDING_TEST_WTF GTEST_API_=
3232
WK_UIKITMACHELPER_LDFLAGS = $(WK_UIKITMACHELPER_LDFLAGS_$(WK_PLATFORM_NAME));
3333
WK_UIKITMACHELPER_LDFLAGS_maccatalyst = -framework UIKitMacHelper;
3434

35-
OTHER_LDFLAGS = $(inherited) -lgtest -force_load $(BUILT_PRODUCTS_DIR)/libTestWTF.a $(WK_UIKITMACHELPER_LDFLAGS) $(OTHER_LDFLAGS_PLATFORM_$(WK_COCOA_TOUCH));
35+
OTHER_LDFLAGS = $(inherited) -framework Testing -lgtest -force_load $(BUILT_PRODUCTS_DIR)/libTestWTF.a $(WK_UIKITMACHELPER_LDFLAGS) $(OTHER_LDFLAGS_PLATFORM_$(WK_COCOA_TOUCH));
3636
OTHER_LDFLAGS_PLATFORM_ = -framework Cocoa -framework Carbon;
3737

3838
// FIXME: This should not be built on iOS. Instead we should create and use a TestWTF application.
3939
OTHER_LDFLAGS_PLATFORM_cocoatouch = -framework CoreGraphics;
4040

41+
LD_RUNPATH_SEARCH_PATHS = "@loader_path";
42+
4143
STRIP_STYLE = debugging;
4244

4345
ENTITLEMENTS_REQUIRED = $(ENTITLEMENTS_REQUIRED_USE_INTERNAL_SDK_$(USE_INTERNAL_SDK))

‎Tools/TestWebKitAPI/Configurations/TestWebKitAPI.xcconfig‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,4 @@ GTEST_LDFLAGS = -lgtest;
3232

3333
SWIFT_OBJC_BRIDGING_HEADER = Runner/TestWebKitAPI-Bridging-Header.h;
3434

35+
OTHER_LDFLAGS = $(inherited) -framework Testing;

‎Tools/TestWebKitAPI/Runner/GoogleTestsController.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ extension GoogleTestsController {
7373
}
7474
}
7575

76-
func withUnsafeMutableCStyleArguments<R>(
76+
private func withUnsafeMutableCStyleArguments<R>(
7777
_ arguments: some Sequence<String>,
7878
_ body: (_ argc: Int32, _ argv: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>) throws -> R
7979
) rethrows -> R {

0 commit comments

Comments
 (0)