kunit_parser.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867
  1. # SPDX-License-Identifier: GPL-2.0
  2. #
  3. # Parses KTAP test results from a kernel dmesg log and incrementally prints
  4. # results with reader-friendly format. Stores and returns test results in a
  5. # Test object.
  6. #
  7. # Copyright (C) 2019, Google LLC.
  8. # Author: Felix Guo <felixguoxiuping@gmail.com>
  9. # Author: Brendan Higgins <brendanhiggins@google.com>
  10. # Author: Rae Moar <rmoar@google.com>
  11. from __future__ import annotations
  12. from dataclasses import dataclass
  13. import re
  14. import textwrap
  15. from enum import Enum, auto
  16. from typing import Iterable, Iterator, List, Optional, Tuple
  17. from kunit_printer import Printer, stdout
  18. class Test:
  19. """
  20. A class to represent a test parsed from KTAP results. All KTAP
  21. results within a test log are stored in a main Test object as
  22. subtests.
  23. Attributes:
  24. status : TestStatus - status of the test
  25. name : str - name of the test
  26. expected_count : int - expected number of subtests (0 if single
  27. test case and None if unknown expected number of subtests)
  28. subtests : List[Test] - list of subtests
  29. log : List[str] - log of KTAP lines that correspond to the test
  30. counts : TestCounts - counts of the test statuses and errors of
  31. subtests or of the test itself if the test is a single
  32. test case.
  33. """
  34. def __init__(self) -> None:
  35. """Creates Test object with default attributes."""
  36. self.status = TestStatus.TEST_CRASHED
  37. self.name = ''
  38. self.expected_count = 0 # type: Optional[int]
  39. self.subtests = [] # type: List[Test]
  40. self.log = [] # type: List[str]
  41. self.counts = TestCounts()
  42. def __str__(self) -> str:
  43. """Returns string representation of a Test class object."""
  44. return (f'Test({self.status}, {self.name}, {self.expected_count}, '
  45. f'{self.subtests}, {self.log}, {self.counts})')
  46. def __repr__(self) -> str:
  47. """Returns string representation of a Test class object."""
  48. return str(self)
  49. def add_error(self, printer: Printer, error_message: str) -> None:
  50. """Records an error that occurred while parsing this test."""
  51. self.counts.errors += 1
  52. printer.print_with_timestamp(stdout.red('[ERROR]') + f' Test: {self.name}: {error_message}')
  53. def ok_status(self) -> bool:
  54. """Returns true if the status was ok, i.e. passed or skipped."""
  55. return self.status in (TestStatus.SUCCESS, TestStatus.SKIPPED)
  56. class TestStatus(Enum):
  57. """An enumeration class to represent the status of a test."""
  58. SUCCESS = auto()
  59. FAILURE = auto()
  60. SKIPPED = auto()
  61. TEST_CRASHED = auto()
  62. NO_TESTS = auto()
  63. FAILURE_TO_PARSE_TESTS = auto()
  64. @dataclass
  65. class TestCounts:
  66. """
  67. Tracks the counts of statuses of all test cases and any errors within
  68. a Test.
  69. """
  70. passed: int = 0
  71. failed: int = 0
  72. crashed: int = 0
  73. skipped: int = 0
  74. errors: int = 0
  75. def __str__(self) -> str:
  76. """Returns the string representation of a TestCounts object."""
  77. statuses = [('passed', self.passed), ('failed', self.failed),
  78. ('crashed', self.crashed), ('skipped', self.skipped),
  79. ('errors', self.errors)]
  80. return f'Ran {self.total()} tests: ' + \
  81. ', '.join(f'{s}: {n}' for s, n in statuses if n > 0)
  82. def total(self) -> int:
  83. """Returns the total number of test cases within a test
  84. object, where a test case is a test with no subtests.
  85. """
  86. return (self.passed + self.failed + self.crashed +
  87. self.skipped)
  88. def add_subtest_counts(self, counts: TestCounts) -> None:
  89. """
  90. Adds the counts of another TestCounts object to the current
  91. TestCounts object. Used to add the counts of a subtest to the
  92. parent test.
  93. Parameters:
  94. counts - a different TestCounts object whose counts
  95. will be added to the counts of the TestCounts object
  96. """
  97. self.passed += counts.passed
  98. self.failed += counts.failed
  99. self.crashed += counts.crashed
  100. self.skipped += counts.skipped
  101. self.errors += counts.errors
  102. def get_status(self) -> TestStatus:
  103. """Returns the aggregated status of a Test using test
  104. counts.
  105. """
  106. if self.total() == 0:
  107. return TestStatus.NO_TESTS
  108. if self.crashed:
  109. # Crashes should take priority.
  110. return TestStatus.TEST_CRASHED
  111. if self.failed:
  112. return TestStatus.FAILURE
  113. if self.passed:
  114. # No failures or crashes, looks good!
  115. return TestStatus.SUCCESS
  116. # We have only skipped tests.
  117. return TestStatus.SKIPPED
  118. def add_status(self, status: TestStatus) -> None:
  119. """Increments the count for `status`."""
  120. if status == TestStatus.SUCCESS:
  121. self.passed += 1
  122. elif status == TestStatus.FAILURE:
  123. self.failed += 1
  124. elif status == TestStatus.SKIPPED:
  125. self.skipped += 1
  126. elif status != TestStatus.NO_TESTS:
  127. self.crashed += 1
  128. class LineStream:
  129. """
  130. A class to represent the lines of kernel output.
  131. Provides a lazy peek()/pop() interface over an iterator of
  132. (line#, text).
  133. """
  134. _lines: Iterator[Tuple[int, str]]
  135. _next: Tuple[int, str]
  136. _need_next: bool
  137. _done: bool
  138. def __init__(self, lines: Iterator[Tuple[int, str]]):
  139. """Creates a new LineStream that wraps the given iterator."""
  140. self._lines = lines
  141. self._done = False
  142. self._need_next = True
  143. self._next = (0, '')
  144. def _get_next(self) -> None:
  145. """Advances the LineSteam to the next line, if necessary."""
  146. if not self._need_next:
  147. return
  148. try:
  149. self._next = next(self._lines)
  150. except StopIteration:
  151. self._done = True
  152. finally:
  153. self._need_next = False
  154. def peek(self) -> str:
  155. """Returns the current line, without advancing the LineStream.
  156. """
  157. self._get_next()
  158. return self._next[1]
  159. def pop(self) -> str:
  160. """Returns the current line and advances the LineStream to
  161. the next line.
  162. """
  163. s = self.peek()
  164. if self._done:
  165. raise ValueError(f'LineStream: going past EOF, last line was {s}')
  166. self._need_next = True
  167. return s
  168. def __bool__(self) -> bool:
  169. """Returns True if stream has more lines."""
  170. self._get_next()
  171. return not self._done
  172. # Only used by kunit_tool_test.py.
  173. def __iter__(self) -> Iterator[str]:
  174. """Empties all lines stored in LineStream object into
  175. Iterator object and returns the Iterator object.
  176. """
  177. while bool(self):
  178. yield self.pop()
  179. def line_number(self) -> int:
  180. """Returns the line number of the current line."""
  181. self._get_next()
  182. return self._next[0]
  183. # Parsing helper methods:
  184. KTAP_START = re.compile(r'\s*KTAP version ([0-9]+)$')
  185. TAP_START = re.compile(r'\s*TAP version ([0-9]+)$')
  186. KTAP_END = re.compile(r'\s*(List of all partitions:|'
  187. 'Kernel panic - not syncing: VFS:|reboot: System halted)')
  188. EXECUTOR_ERROR = re.compile(r'\s*kunit executor: (.*)$')
  189. def extract_tap_lines(kernel_output: Iterable[str]) -> LineStream:
  190. """Extracts KTAP lines from the kernel output."""
  191. def isolate_ktap_output(kernel_output: Iterable[str]) \
  192. -> Iterator[Tuple[int, str]]:
  193. line_num = 0
  194. started = False
  195. for line in kernel_output:
  196. line_num += 1
  197. line = line.rstrip() # remove trailing \n
  198. if not started and KTAP_START.search(line):
  199. # start extracting KTAP lines and set prefix
  200. # to number of characters before version line
  201. prefix_len = len(
  202. line.split('KTAP version')[0])
  203. started = True
  204. yield line_num, line[prefix_len:]
  205. elif not started and TAP_START.search(line):
  206. # start extracting KTAP lines and set prefix
  207. # to number of characters before version line
  208. prefix_len = len(line.split('TAP version')[0])
  209. started = True
  210. yield line_num, line[prefix_len:]
  211. elif started and KTAP_END.search(line):
  212. # stop extracting KTAP lines
  213. break
  214. elif started:
  215. # remove the prefix, if any.
  216. line = line[prefix_len:]
  217. yield line_num, line
  218. elif EXECUTOR_ERROR.search(line):
  219. yield line_num, line
  220. return LineStream(lines=isolate_ktap_output(kernel_output))
  221. KTAP_VERSIONS = [1]
  222. TAP_VERSIONS = [13, 14]
  223. def check_version(version_num: int, accepted_versions: List[int],
  224. version_type: str, test: Test, printer: Printer) -> None:
  225. """
  226. Adds error to test object if version number is too high or too
  227. low.
  228. Parameters:
  229. version_num - The inputted version number from the parsed KTAP or TAP
  230. header line
  231. accepted_version - List of accepted KTAP or TAP versions
  232. version_type - 'KTAP' or 'TAP' depending on the type of
  233. version line.
  234. test - Test object for current test being parsed
  235. printer - Printer object to output error
  236. """
  237. if version_num < min(accepted_versions):
  238. test.add_error(printer, f'{version_type} version lower than expected!')
  239. elif version_num > max(accepted_versions):
  240. test.add_error(printer, f'{version_type} version higer than expected!')
  241. def parse_ktap_header(lines: LineStream, test: Test, printer: Printer) -> bool:
  242. """
  243. Parses KTAP/TAP header line and checks version number.
  244. Returns False if fails to parse KTAP/TAP header line.
  245. Accepted formats:
  246. - 'KTAP version [version number]'
  247. - 'TAP version [version number]'
  248. Parameters:
  249. lines - LineStream of KTAP output to parse
  250. test - Test object for current test being parsed
  251. printer - Printer object to output results
  252. Return:
  253. True if successfully parsed KTAP/TAP header line
  254. """
  255. ktap_match = KTAP_START.match(lines.peek())
  256. tap_match = TAP_START.match(lines.peek())
  257. if ktap_match:
  258. version_num = int(ktap_match.group(1))
  259. check_version(version_num, KTAP_VERSIONS, 'KTAP', test, printer)
  260. elif tap_match:
  261. version_num = int(tap_match.group(1))
  262. check_version(version_num, TAP_VERSIONS, 'TAP', test, printer)
  263. else:
  264. return False
  265. lines.pop()
  266. return True
  267. TEST_HEADER = re.compile(r'^\s*# Subtest: (.*)$')
  268. def parse_test_header(lines: LineStream, test: Test) -> bool:
  269. """
  270. Parses test header and stores test name in test object.
  271. Returns False if fails to parse test header line.
  272. Accepted format:
  273. - '# Subtest: [test name]'
  274. Parameters:
  275. lines - LineStream of KTAP output to parse
  276. test - Test object for current test being parsed
  277. Return:
  278. True if successfully parsed test header line
  279. """
  280. match = TEST_HEADER.match(lines.peek())
  281. if not match:
  282. return False
  283. test.name = match.group(1)
  284. lines.pop()
  285. return True
  286. TEST_PLAN = re.compile(r'^\s*1\.\.([0-9]+)')
  287. def parse_test_plan(lines: LineStream, test: Test) -> bool:
  288. """
  289. Parses test plan line and stores the expected number of subtests in
  290. test object. Reports an error if expected count is 0.
  291. Returns False and sets expected_count to None if there is no valid test
  292. plan.
  293. Accepted format:
  294. - '1..[number of subtests]'
  295. Parameters:
  296. lines - LineStream of KTAP output to parse
  297. test - Test object for current test being parsed
  298. Return:
  299. True if successfully parsed test plan line
  300. """
  301. match = TEST_PLAN.match(lines.peek())
  302. if not match:
  303. test.expected_count = None
  304. return False
  305. expected_count = int(match.group(1))
  306. test.expected_count = expected_count
  307. lines.pop()
  308. return True
  309. TEST_RESULT = re.compile(r'^\s*(ok|not ok) ([0-9]+) ?(- )?([^#]*)( # .*)?$')
  310. TEST_RESULT_SKIP = re.compile(r'^\s*(ok|not ok) ([0-9]+) ?(- )?(.*) # SKIP ?(.*)$')
  311. def peek_test_name_match(lines: LineStream, test: Test) -> bool:
  312. """
  313. Matches current line with the format of a test result line and checks
  314. if the name matches the name of the current test.
  315. Returns False if fails to match format or name.
  316. Accepted format:
  317. - '[ok|not ok] [test number] [-] [test name] [optional skip
  318. directive]'
  319. Parameters:
  320. lines - LineStream of KTAP output to parse
  321. test - Test object for current test being parsed
  322. Return:
  323. True if matched a test result line and the name matching the
  324. expected test name
  325. """
  326. line = lines.peek()
  327. match = TEST_RESULT.match(line)
  328. if not match:
  329. return False
  330. name = match.group(4)
  331. if not name:
  332. return False
  333. return name == test.name
  334. def parse_test_result(lines: LineStream, test: Test,
  335. expected_num: int, printer: Printer) -> bool:
  336. """
  337. Parses test result line and stores the status and name in the test
  338. object. Reports an error if the test number does not match expected
  339. test number.
  340. Returns False if fails to parse test result line.
  341. Note that the SKIP directive is the only direction that causes a
  342. change in status.
  343. Accepted format:
  344. - '[ok|not ok] [test number] [-] [test name] [optional skip
  345. directive]'
  346. Parameters:
  347. lines - LineStream of KTAP output to parse
  348. test - Test object for current test being parsed
  349. expected_num - expected test number for current test
  350. printer - Printer object to output results
  351. Return:
  352. True if successfully parsed a test result line.
  353. """
  354. line = lines.peek()
  355. match = TEST_RESULT.match(line)
  356. skip_match = TEST_RESULT_SKIP.match(line)
  357. # Check if line matches test result line format
  358. if not match:
  359. return False
  360. lines.pop()
  361. # Set name of test object
  362. if skip_match:
  363. test.name = skip_match.group(4) or skip_match.group(5)
  364. else:
  365. test.name = match.group(4)
  366. # Check test num
  367. num = int(match.group(2))
  368. if num != expected_num:
  369. test.add_error(printer, f'Expected test number {expected_num} but found {num}')
  370. # Set status of test object
  371. status = match.group(1)
  372. if skip_match:
  373. test.status = TestStatus.SKIPPED
  374. elif status == 'ok':
  375. test.status = TestStatus.SUCCESS
  376. else:
  377. test.status = TestStatus.FAILURE
  378. return True
  379. def parse_diagnostic(lines: LineStream) -> List[str]:
  380. """
  381. Parse lines that do not match the format of a test result line or
  382. test header line and returns them in list.
  383. Line formats that are not parsed:
  384. - '# Subtest: [test name]'
  385. - '[ok|not ok] [test number] [-] [test name] [optional skip
  386. directive]'
  387. - 'KTAP version [version number]'
  388. Parameters:
  389. lines - LineStream of KTAP output to parse
  390. Return:
  391. Log of diagnostic lines
  392. """
  393. log = [] # type: List[str]
  394. non_diagnostic_lines = [TEST_RESULT, TEST_HEADER, KTAP_START, TAP_START, TEST_PLAN]
  395. while lines and not any(re.match(lines.peek())
  396. for re in non_diagnostic_lines):
  397. log.append(lines.pop())
  398. return log
  399. # Printing helper methods:
  400. DIVIDER = '=' * 60
  401. def format_test_divider(message: str, len_message: int) -> str:
  402. """
  403. Returns string with message centered in fixed width divider.
  404. Example:
  405. '===================== message example ====================='
  406. Parameters:
  407. message - message to be centered in divider line
  408. len_message - length of the message to be printed such that
  409. any characters of the color codes are not counted
  410. Return:
  411. String containing message centered in fixed width divider
  412. """
  413. default_count = 3 # default number of dashes
  414. len_1 = default_count
  415. len_2 = default_count
  416. difference = len(DIVIDER) - len_message - 2 # 2 spaces added
  417. if difference > 0:
  418. # calculate number of dashes for each side of the divider
  419. len_1 = int(difference / 2)
  420. len_2 = difference - len_1
  421. return ('=' * len_1) + f' {message} ' + ('=' * len_2)
  422. def print_test_header(test: Test, printer: Printer) -> None:
  423. """
  424. Prints test header with test name and optionally the expected number
  425. of subtests.
  426. Example:
  427. '=================== example (2 subtests) ==================='
  428. Parameters:
  429. test - Test object representing current test being printed
  430. printer - Printer object to output results
  431. """
  432. message = test.name
  433. if message != "":
  434. # Add a leading space before the subtest counts only if a test name
  435. # is provided using a "# Subtest" header line.
  436. message += " "
  437. if test.expected_count:
  438. if test.expected_count == 1:
  439. message += '(1 subtest)'
  440. else:
  441. message += f'({test.expected_count} subtests)'
  442. printer.print_with_timestamp(format_test_divider(message, len(message)))
  443. def print_log(log: Iterable[str], printer: Printer) -> None:
  444. """Prints all strings in saved log for test in yellow."""
  445. formatted = textwrap.dedent('\n'.join(log))
  446. for line in formatted.splitlines():
  447. printer.print_with_timestamp(printer.yellow(line))
  448. def format_test_result(test: Test, printer: Printer) -> str:
  449. """
  450. Returns string with formatted test result with colored status and test
  451. name.
  452. Example:
  453. '[PASSED] example'
  454. Parameters:
  455. test - Test object representing current test being printed
  456. printer - Printer object to output results
  457. Return:
  458. String containing formatted test result
  459. """
  460. if test.status == TestStatus.SUCCESS:
  461. return printer.green('[PASSED] ') + test.name
  462. if test.status == TestStatus.SKIPPED:
  463. return printer.yellow('[SKIPPED] ') + test.name
  464. if test.status == TestStatus.NO_TESTS:
  465. return printer.yellow('[NO TESTS RUN] ') + test.name
  466. if test.status == TestStatus.TEST_CRASHED:
  467. print_log(test.log, printer)
  468. return stdout.red('[CRASHED] ') + test.name
  469. print_log(test.log, printer)
  470. return printer.red('[FAILED] ') + test.name
  471. def print_test_result(test: Test, printer: Printer) -> None:
  472. """
  473. Prints result line with status of test.
  474. Example:
  475. '[PASSED] example'
  476. Parameters:
  477. test - Test object representing current test being printed
  478. printer - Printer object
  479. """
  480. printer.print_with_timestamp(format_test_result(test, printer))
  481. def print_test_footer(test: Test, printer: Printer) -> None:
  482. """
  483. Prints test footer with status of test.
  484. Example:
  485. '===================== [PASSED] example ====================='
  486. Parameters:
  487. test - Test object representing current test being printed
  488. printer - Printer object to output results
  489. """
  490. message = format_test_result(test, printer)
  491. printer.print_with_timestamp(format_test_divider(message,
  492. len(message) - printer.color_len()))
  493. def print_test(test: Test, failed_only: bool, printer: Printer) -> None:
  494. """
  495. Prints Test object to given printer. For a child test, the result line is
  496. printed. For a parent test, the test header, all child test results, and
  497. the test footer are all printed. If failed_only is true, only failed/crashed
  498. tests will be printed.
  499. Parameters:
  500. test - Test object to print
  501. failed_only - True if only failed/crashed tests should be printed.
  502. printer - Printer object to output results
  503. """
  504. if test.name == "main":
  505. printer.print_with_timestamp(DIVIDER)
  506. for subtest in test.subtests:
  507. print_test(subtest, failed_only, printer)
  508. printer.print_with_timestamp(DIVIDER)
  509. elif test.subtests != []:
  510. if not failed_only or not test.ok_status():
  511. print_test_header(test, printer)
  512. for subtest in test.subtests:
  513. print_test(subtest, failed_only, printer)
  514. print_test_footer(test, printer)
  515. else:
  516. if not failed_only or not test.ok_status():
  517. print_test_result(test, printer)
  518. def _summarize_failed_tests(test: Test) -> str:
  519. """Tries to summarize all the failing subtests in `test`."""
  520. def failed_names(test: Test, parent_name: str) -> List[str]:
  521. # Note: we use 'main' internally for the top-level test.
  522. if not parent_name or parent_name == 'main':
  523. full_name = test.name
  524. else:
  525. full_name = parent_name + '.' + test.name
  526. if not test.subtests: # this is a leaf node
  527. return [full_name]
  528. # If all the children failed, just say this subtest failed.
  529. # Don't summarize it down "the top-level test failed", though.
  530. failed_subtests = [sub for sub in test.subtests if not sub.ok_status()]
  531. if parent_name and len(failed_subtests) == len(test.subtests):
  532. return [full_name]
  533. all_failures = [] # type: List[str]
  534. for t in failed_subtests:
  535. all_failures.extend(failed_names(t, full_name))
  536. return all_failures
  537. failures = failed_names(test, '')
  538. # If there are too many failures, printing them out will just be noisy.
  539. if len(failures) > 10: # this is an arbitrary limit
  540. return ''
  541. return 'Failures: ' + ', '.join(failures)
  542. def print_summary_line(test: Test, printer: Printer) -> None:
  543. """
  544. Prints summary line of test object. Color of line is dependent on
  545. status of test. Color is green if test passes, yellow if test is
  546. skipped, and red if the test fails or crashes. Summary line contains
  547. counts of the statuses of the tests subtests or the test itself if it
  548. has no subtests.
  549. Example:
  550. "Testing complete. Passed: 2, Failed: 0, Crashed: 0, Skipped: 0,
  551. Errors: 0"
  552. test - Test object representing current test being printed
  553. printer - Printer object to output results
  554. """
  555. if test.status == TestStatus.SUCCESS:
  556. color = stdout.green
  557. elif test.status in (TestStatus.SKIPPED, TestStatus.NO_TESTS):
  558. color = stdout.yellow
  559. else:
  560. color = stdout.red
  561. printer.print_with_timestamp(color(f'Testing complete. {test.counts}'))
  562. # Summarize failures that might have gone off-screen since we had a lot
  563. # of tests (arbitrarily defined as >=100 for now).
  564. if test.ok_status() or test.counts.total() < 100:
  565. return
  566. summarized = _summarize_failed_tests(test)
  567. if not summarized:
  568. return
  569. printer.print_with_timestamp(color(summarized))
  570. # Other methods:
  571. def bubble_up_test_results(test: Test) -> None:
  572. """
  573. If the test has subtests, add the test counts of the subtests to the
  574. test and check if any of the tests crashed and if so set the test
  575. status to crashed. Otherwise if the test has no subtests add the
  576. status of the test to the test counts.
  577. Parameters:
  578. test - Test object for current test being parsed
  579. """
  580. subtests = test.subtests
  581. counts = test.counts
  582. status = test.status
  583. for t in subtests:
  584. counts.add_subtest_counts(t.counts)
  585. if counts.total() == 0:
  586. counts.add_status(status)
  587. elif test.counts.get_status() == TestStatus.TEST_CRASHED:
  588. test.status = TestStatus.TEST_CRASHED
  589. if status == TestStatus.FAILURE and test.counts.get_status() == TestStatus.SUCCESS:
  590. counts.add_status(status)
  591. def parse_test(lines: LineStream, expected_num: int, log: List[str], is_subtest: bool, printer: Printer) -> Test:
  592. """
  593. Finds next test to parse in LineStream, creates new Test object,
  594. parses any subtests of the test, populates Test object with all
  595. information (status, name) about the test and the Test objects for
  596. any subtests, and then returns the Test object. The method accepts
  597. three formats of tests:
  598. Accepted test formats:
  599. - Main KTAP/TAP header
  600. Example:
  601. KTAP version 1
  602. 1..4
  603. [subtests]
  604. - Subtest header (must include either the KTAP version line or
  605. "# Subtest" header line)
  606. Example (preferred format with both KTAP version line and
  607. "# Subtest" line):
  608. KTAP version 1
  609. # Subtest: name
  610. 1..3
  611. [subtests]
  612. ok 1 name
  613. Example (only "# Subtest" line):
  614. # Subtest: name
  615. 1..3
  616. [subtests]
  617. ok 1 name
  618. Example (only KTAP version line, compliant with KTAP v1 spec):
  619. KTAP version 1
  620. 1..3
  621. [subtests]
  622. ok 1 name
  623. - Test result line
  624. Example:
  625. ok 1 - test
  626. Parameters:
  627. lines - LineStream of KTAP output to parse
  628. expected_num - expected test number for test to be parsed
  629. log - list of strings containing any preceding diagnostic lines
  630. corresponding to the current test
  631. is_subtest - boolean indicating whether test is a subtest
  632. printer - Printer object to output results
  633. Return:
  634. Test object populated with characteristics and any subtests
  635. """
  636. test = Test()
  637. test.log.extend(log)
  638. # Parse any errors prior to parsing tests
  639. err_log = parse_diagnostic(lines)
  640. test.log.extend(err_log)
  641. if not is_subtest:
  642. # If parsing the main/top-level test, parse KTAP version line and
  643. # test plan
  644. test.name = "main"
  645. parse_ktap_header(lines, test, printer)
  646. test.log.extend(parse_diagnostic(lines))
  647. parse_test_plan(lines, test)
  648. parent_test = True
  649. else:
  650. # If not the main test, attempt to parse a test header containing
  651. # the KTAP version line and/or subtest header line
  652. ktap_line = parse_ktap_header(lines, test, printer)
  653. subtest_line = parse_test_header(lines, test)
  654. test.log.extend(parse_diagnostic(lines))
  655. parse_test_plan(lines, test)
  656. parent_test = (ktap_line or subtest_line)
  657. if parent_test:
  658. print_test_header(test, printer)
  659. expected_count = test.expected_count
  660. subtests = []
  661. test_num = 1
  662. while parent_test and (expected_count is None or test_num <= expected_count):
  663. # Loop to parse any subtests.
  664. # Break after parsing expected number of tests or
  665. # if expected number of tests is unknown break when test
  666. # result line with matching name to subtest header is found
  667. # or no more lines in stream.
  668. sub_log = parse_diagnostic(lines)
  669. sub_test = Test()
  670. if not lines or (peek_test_name_match(lines, test) and
  671. is_subtest):
  672. if expected_count and test_num <= expected_count:
  673. # If parser reaches end of test before
  674. # parsing expected number of subtests, print
  675. # crashed subtest and record error
  676. test.add_error(printer, 'missing expected subtest!')
  677. sub_test.log.extend(sub_log)
  678. test.counts.add_status(
  679. TestStatus.TEST_CRASHED)
  680. print_test_result(sub_test, printer)
  681. else:
  682. test.log.extend(sub_log)
  683. break
  684. else:
  685. sub_test = parse_test(lines, test_num, sub_log, True, printer)
  686. subtests.append(sub_test)
  687. test_num += 1
  688. test.subtests = subtests
  689. if is_subtest:
  690. # If not main test, look for test result line
  691. test.log.extend(parse_diagnostic(lines))
  692. if test.name != "" and not peek_test_name_match(lines, test):
  693. test.add_error(printer, 'missing subtest result line!')
  694. elif not lines:
  695. print_log(test.log, printer)
  696. test.status = TestStatus.NO_TESTS
  697. test.add_error(printer, 'No more test results!')
  698. else:
  699. parse_test_result(lines, test, expected_num, printer)
  700. # Check for there being no subtests within parent test
  701. if parent_test and len(subtests) == 0:
  702. # Don't override a bad status if this test had one reported.
  703. # Assumption: no subtests means CRASHED is from Test.__init__()
  704. if test.status in (TestStatus.TEST_CRASHED, TestStatus.SUCCESS):
  705. print_log(test.log, printer)
  706. test.status = TestStatus.NO_TESTS
  707. test.add_error(printer, '0 tests run!')
  708. # Add statuses to TestCounts attribute in Test object
  709. bubble_up_test_results(test)
  710. if parent_test and is_subtest:
  711. # If test has subtests and is not the main test object, print
  712. # footer.
  713. print_test_footer(test, printer)
  714. elif is_subtest:
  715. print_test_result(test, printer)
  716. return test
  717. def parse_run_tests(kernel_output: Iterable[str], printer: Printer) -> Test:
  718. """
  719. Using kernel output, extract KTAP lines, parse the lines for test
  720. results and print condensed test results and summary line.
  721. Parameters:
  722. kernel_output - Iterable object contains lines of kernel output
  723. printer - Printer object to output results
  724. Return:
  725. Test - the main test object with all subtests.
  726. """
  727. printer.print_with_timestamp(DIVIDER)
  728. lines = extract_tap_lines(kernel_output)
  729. test = Test()
  730. if not lines:
  731. test.name = '<missing>'
  732. test.add_error(printer, 'Could not find any KTAP output. Did any KUnit tests run?')
  733. test.status = TestStatus.FAILURE_TO_PARSE_TESTS
  734. else:
  735. test = parse_test(lines, 0, [], False, printer)
  736. if test.status != TestStatus.NO_TESTS:
  737. test.status = test.counts.get_status()
  738. printer.print_with_timestamp(DIVIDER)
  739. return test