Skip to content

Commit c8ef235

Browse files
author
Janne Härkönen
committed
statistics: calculate skip stats correctly in top suites
1 parent 6df70f6 commit c8ef235

3 files changed

Lines changed: 59 additions & 52 deletions

File tree

‎src/robot/model/stats.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ def _update_elapsed(self, test):
124124
def add_stat(self, other):
125125
self.passed += other.passed
126126
self.failed += other.failed
127-
self.skipped = other.skipped
127+
self.skipped += other.skipped
128128

129129

130130
class TagStat(Stat):

‎utest/model/test_statistics.py‎

Lines changed: 57 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,22 @@
55
from robot.result import TestCase, TestSuite
66

77

8-
def verify_stat(stat, name, passed, failed, combined=None,
9-
id=None, elapsed=0):
8+
def verify_stat(stat, name, passed, failed, skipped,
9+
combined=None, id=None, elapsed=0):
1010
assert_equal(stat.name, name, 'stat.name')
1111
assert_equal(stat.passed, passed)
1212
assert_equal(stat.failed, failed)
13-
assert_equal(stat.total, passed + failed)
13+
assert_equal(stat.skipped, skipped)
14+
assert_equal(stat.total, passed + failed + skipped)
1415
if hasattr(stat, 'combined'):
1516
assert_equal(stat.combined, combined)
1617
if hasattr(stat, 'id'):
1718
assert_equal(stat.id, id)
1819
assert_equal(stat.elapsed, elapsed)
1920

2021

21-
def verify_suite(suite, name, id, passed, failed):
22-
verify_stat(suite.stat, name, passed, failed, id=id)
22+
def verify_suite(suite, name, id, passed, failed, skipped):
23+
verify_stat(suite.stat, name, passed, failed, skipped, id=id)
2324

2425

2526
def generate_suite():
@@ -30,12 +31,15 @@ def generate_suite():
3031
s12 = s1.suites.create(name='Sub Suite 1_2')
3132
s13 = s1.suites.create(name='Sub Suite 1_3')
3233
s21 = s2.suites.create(name='Sub Suite 2_1')
34+
s22 = s2.suites.create(name='Sub Suite 3_1')
3335
s11.tests = [TestCase(status='PASS'), TestCase(status='FAIL', tags=['t1'])]
3436
s12.tests = [TestCase(status='PASS', tags=['t_1','t2',]),
3537
TestCase(status='PASS', tags=['t1','smoke']),
38+
TestCase(status='SKIP', tags=['t1','flaky']),
3639
TestCase(status='FAIL', tags=['t1','t2','t3','smoke'])]
3740
s13.tests = [TestCase(status='PASS', tags=['t1','t 2','smoke'])]
3841
s21.tests = [TestCase(status='FAIL', tags=['t3','Smoke'])]
42+
s22.tests = [TestCase(status='SKIP', tags=['flaky'])]
3943
return suite
4044

4145

@@ -44,14 +48,14 @@ class TestStatisticsSimple(unittest.TestCase):
4448
def setUp(self):
4549
suite = TestSuite(name='Hello')
4650
suite.tests = [TestCase(status='PASS'), TestCase(status='PASS'),
47-
TestCase(status='FAIL')]
51+
TestCase(status='FAIL'), TestCase(status='SKIP')]
4852
self.statistics = Statistics(suite)
4953

5054
def test_total(self):
51-
verify_stat(self.statistics.total._stat, 'All Tests', 2, 1)
55+
verify_stat(self.statistics.total._stat, 'All Tests', 2, 1, 1)
5256

5357
def test_suite(self):
54-
verify_suite(self.statistics.suite, 'Hello', 's1', 2, 1)
58+
verify_suite(self.statistics.suite, 'Hello', 's1', 2, 1, 1)
5559

5660
def test_tags(self):
5761
assert_equal(list(self.statistics.tags), [])
@@ -65,28 +69,28 @@ def setUp(self):
6569
[('t? & smoke', ''), ('none NOT t1', 'a title')])
6670

6771
def test_total(self):
68-
verify_stat(self.statistics.total._stat, 'All Tests', 4, 3)
72+
verify_stat(self.statistics.total._stat, 'All Tests', 4, 3, 2)
6973

7074
def test_suite(self):
7175
suite = self.statistics.suite
72-
verify_suite(suite, 'Root Suite', 's1', 4, 3)
76+
verify_suite(suite, 'Root Suite', 's1', 4, 3,2 )
7377
[s1, s2] = suite.suites
74-
verify_suite(s1, 'Root Suite.First Sub Suite', 's1-s1', 4, 2)
75-
verify_suite(s2, 'Root Suite.Second Sub Suite', 's1-s2', 0, 1)
78+
verify_suite(s1, 'Root Suite.First Sub Suite', 's1-s1', 4, 2, 1)
79+
verify_suite(s2, 'Root Suite.Second Sub Suite', 's1-s2', 0, 1, 1)
7680
assert_equal(len(s1.suites), 0)
7781
assert_equal(len(s2.suites), 0)
7882

7983
def test_tags(self):
8084
# Tag stats are tested more thoroughly in test_tagstatistics.py
8185
tags = self.statistics.tags
82-
verify_stat(tags.tags['smoke'], 'smoke', 2, 2)
83-
verify_stat(tags.tags['t1'], 't1', 3, 2)
84-
verify_stat(tags.tags['t2'], 't2', 2, 1)
85-
expected = [('a title', 0, 0, 'none NOT t1'),
86-
('t? & smoke', 2, 2, 't? & smoke'),
87-
('smoke', 2, 2),
88-
('t1', 3, 2),
89-
('t2', 2, 1)]
86+
verify_stat(tags.tags['smoke'], 'smoke', 2, 2, 0)
87+
verify_stat(tags.tags['t1'], 't1', 3, 2, 1)
88+
verify_stat(tags.tags['t2'], 't2', 2, 1, 0)
89+
expected = [('a title', 0, 0, 0, 'none NOT t1'),
90+
('t? & smoke', 2, 2, 0, 't? & smoke'),
91+
('smoke', 2, 2, 0),
92+
('t1', 3, 2, 1),
93+
('t2', 2, 1, 0)]
9094
assert_equal(len(list(tags)), len(expected))
9195
for t, e in zip(tags, expected):
9296
verify_stat(t, *e)
@@ -96,60 +100,63 @@ class TestSuiteStatistics(unittest.TestCase):
96100

97101
def test_all_levels(self):
98102
suite = Statistics(generate_suite()).suite
99-
verify_suite(suite, 'Root Suite', 's1', 4, 3)
103+
verify_suite(suite, 'Root Suite', 's1', 4, 3, 2)
100104
[s1, s2] = suite.suites
101-
verify_suite(s1, 'Root Suite.First Sub Suite', 's1-s1', 4, 2)
102-
verify_suite(s2, 'Root Suite.Second Sub Suite', 's1-s2', 0, 1)
105+
verify_suite(s1, 'Root Suite.First Sub Suite', 's1-s1', 4, 2, 1)
106+
verify_suite(s2, 'Root Suite.Second Sub Suite', 's1-s2', 0, 1, 1)
103107
[s11, s12, s13] = s1.suites
104-
verify_suite(s11, 'Root Suite.First Sub Suite.Sub Suite 1_1', 's1-s1-s1', 1, 1)
105-
verify_suite(s12, 'Root Suite.First Sub Suite.Sub Suite 1_2', 's1-s1-s2', 2, 1)
106-
verify_suite(s13, 'Root Suite.First Sub Suite.Sub Suite 1_3', 's1-s1-s3', 1, 0)
107-
[s21] = s2.suites
108-
verify_suite(s21, 'Root Suite.Second Sub Suite.Sub Suite 2_1', 's1-s2-s1', 0, 1)
108+
verify_suite(s11, 'Root Suite.First Sub Suite.Sub Suite 1_1', 's1-s1-s1', 1, 1, 0)
109+
verify_suite(s12, 'Root Suite.First Sub Suite.Sub Suite 1_2', 's1-s1-s2', 2, 1, 1)
110+
verify_suite(s13, 'Root Suite.First Sub Suite.Sub Suite 1_3', 's1-s1-s3', 1, 0, 0)
111+
[s21, s22] = s2.suites
112+
verify_suite(s21, 'Root Suite.Second Sub Suite.Sub Suite 2_1', 's1-s2-s1', 0, 1, 0)
113+
verify_suite(s22, 'Root Suite.Second Sub Suite.Sub Suite 3_1', 's1-s2-s2', 0, 0, 1)
109114

110115
def test_only_root_level(self):
111116
suite = Statistics(generate_suite(), suite_stat_level=1).suite
112-
verify_suite(suite, 'Root Suite', 's1', 4, 3)
117+
verify_suite(suite, 'Root Suite', 's1', 4, 3, 2)
113118
assert_equal(len(suite.suites), 0)
114119

115120
def test_deeper_level(self):
116121
PASS = TestCase(status='PASS')
117122
FAIL = TestCase(status='FAIL')
123+
SKIP = TestCase(status='SKIP')
118124
suite = TestSuite(name='1')
119125
suite.suites = [TestSuite(name='1'), TestSuite(name='2'), TestSuite(name='3')]
120126
suite.suites[0].suites = [TestSuite(name='1')]
121127
suite.suites[1].suites = [TestSuite(name='1'), TestSuite(name='2')]
122128
suite.suites[2].tests = [PASS, FAIL]
123129
suite.suites[0].suites[0].suites = [TestSuite(name='1')]
124-
suite.suites[1].suites[0].tests = [PASS, PASS, PASS, FAIL]
125-
suite.suites[1].suites[1].tests = [PASS, PASS, FAIL, FAIL]
130+
suite.suites[1].suites[0].tests = [PASS, PASS, PASS, FAIL, SKIP]
131+
suite.suites[1].suites[1].tests = [PASS, PASS, FAIL, SKIP]
126132
suite.suites[0].suites[0].suites[0].tests = [FAIL, FAIL, FAIL]
127133
s1 = Statistics(suite, suite_stat_level=3).suite
128-
verify_suite(s1, '1', 's1', 6, 7)
134+
verify_suite(s1, '1', 's1', 6, 6, 2)
129135
[s11, s12, s13] = s1.suites
130-
verify_suite(s11, '1.1', 's1-s1', 0, 3)
131-
verify_suite(s12, '1.2', 's1-s2', 5, 3)
132-
verify_suite(s13, '1.3', 's1-s3', 1, 1)
136+
verify_suite(s11, '1.1', 's1-s1', 0, 3, 0)
137+
verify_suite(s12, '1.2', 's1-s2', 5, 2, 2)
138+
verify_suite(s13, '1.3', 's1-s3', 1, 1, 0)
133139
[s111] = s11.suites
134-
verify_suite(s111, '1.1.1', 's1-s1-s1', 0, 3)
140+
verify_suite(s111, '1.1.1', 's1-s1-s1', 0, 3, 0)
135141
[s121, s122] = s12.suites
136-
verify_suite(s121, '1.2.1', 's1-s2-s1', 3, 1)
137-
verify_suite(s122, '1.2.2', 's1-s2-s2', 2, 2)
142+
verify_suite(s121, '1.2.1', 's1-s2-s1', 3, 1, 1)
143+
verify_suite(s122, '1.2.2', 's1-s2-s2', 2, 1, 1)
138144
assert_equal(len(s111.suites), 0)
139145

140146
def test_iter_only_one_level(self):
141147
[stat] = list(Statistics(generate_suite(), suite_stat_level=1).suite)
142-
verify_stat(stat, 'Root Suite', 4, 3, id='s1')
148+
verify_stat(stat, 'Root Suite', 4, 3, 2, id='s1')
143149

144150
def test_iter_also_sub_suites(self):
145151
stats = list(Statistics(generate_suite()).suite)
146-
verify_stat(stats[0], 'Root Suite', 4, 3, id='s1')
147-
verify_stat(stats[1], 'Root Suite.First Sub Suite', 4, 2, id='s1-s1')
148-
verify_stat(stats[2], 'Root Suite.First Sub Suite.Sub Suite 1_1', 1, 1, id='s1-s1-s1')
149-
verify_stat(stats[3], 'Root Suite.First Sub Suite.Sub Suite 1_2', 2, 1, id='s1-s1-s2')
150-
verify_stat(stats[4], 'Root Suite.First Sub Suite.Sub Suite 1_3', 1, 0, id='s1-s1-s3')
151-
verify_stat(stats[5], 'Root Suite.Second Sub Suite', 0, 1, id='s1-s2')
152-
verify_stat(stats[6], 'Root Suite.Second Sub Suite.Sub Suite 2_1', 0, 1, id='s1-s2-s1')
152+
verify_stat(stats[0], 'Root Suite', 4, 3, 2, id='s1')
153+
verify_stat(stats[1], 'Root Suite.First Sub Suite', 4, 2, 1, id='s1-s1')
154+
verify_stat(stats[2], 'Root Suite.First Sub Suite.Sub Suite 1_1', 1, 1, 0, id='s1-s1-s1')
155+
verify_stat(stats[3], 'Root Suite.First Sub Suite.Sub Suite 1_2', 2, 1, 1, id='s1-s1-s2')
156+
verify_stat(stats[4], 'Root Suite.First Sub Suite.Sub Suite 1_3', 1, 0, 0, id='s1-s1-s3')
157+
verify_stat(stats[5], 'Root Suite.Second Sub Suite', 0, 1, 1, id='s1-s2')
158+
verify_stat(stats[6], 'Root Suite.Second Sub Suite.Sub Suite 2_1', 0, 1, 0, id='s1-s2-s1')
159+
verify_stat(stats[7], 'Root Suite.Second Sub Suite.Sub Suite 3_1', 0, 0, 1, id='s1-s2-s2')
153160

154161

155162
class TestElapsedTime(unittest.TestCase):
@@ -175,13 +182,13 @@ def test_total_stats(self):
175182

176183
def test_tag_stats(self):
177184
t1, t2, t3 = self.stats.tags.tags.values()
178-
verify_stat(t1, 't1', 0, 3, elapsed=11001)
179-
verify_stat(t2, 't2', 0, 2, elapsed=11000)
180-
verify_stat(t3, 't3', 0, 1, elapsed=10000)
185+
verify_stat(t1, 't1', 0, 3, 0, elapsed=11001)
186+
verify_stat(t2, 't2', 0, 2, 0, elapsed=11000)
187+
verify_stat(t3, 't3', 0, 1, 0, elapsed=10000)
181188

182189
def test_combined_tag_stats(self):
183190
combined = self.stats.tags.combined[0]
184-
verify_stat(combined, 'combined', 0, 2, combined='?2', elapsed=11000)
191+
verify_stat(combined, 'combined', 0, 2, 0, combined='?2', elapsed=11000)
185192

186193
def test_suite_stats(self):
187194
assert_equal(self.stats.suite.stat.elapsed, 59999)

‎utest/reporting/test_jsmodelbuilders.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,7 @@ def test_tag_stats(self):
370370

371371
def test_suite_stats(self):
372372
root, sub1, sub2 = self._build_statistics()[2]
373-
self._verify_stat(root, 2, 2, 0, 'root', '00:00:42', name='root', id='s1')
373+
self._verify_stat(root, 2, 2, 1, 'root', '00:00:42', name='root', id='s1')
374374
self._verify_stat(sub1, 1, 1, 1, 'root.sub1', '00:00:10', name='sub1', id='s1-s1')
375375
self._verify_stat(sub2, 1, 1, 0, 'root.sub2', '00:00:30', name='sub2', id='s1-s2')
376376

0 commit comments

Comments
 (0)