Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Outcome analysis: reject repeated configuration names #63

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Outcome analysis: reject repeated configuration names #63

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6006333
Detect repeated configuration names
gilles-peskine-arm Jun 19, 2024
6af71b8
Improve reporting of repeated configurations
gilles-peskine-arm Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 20 additions & 3 deletions scripts/mbedtls_framework/outcome_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,16 @@ def open_outcome_file(outcome_file: str) -> typing.TextIO:
else:
return open(outcome_file, 'rt', encoding='utf-8')

def read_outcome_file(outcome_file: str) -> Outcomes:
MAKE_LIB_TARGET = re.compile(r'all\Z|lib\Z|library/')

def read_outcome_file(results: Results, outcome_file: str) -> Outcomes:
"""Parse an outcome file and return an outcome collection.
"""
outcomes = {}
seen = {} #type: typing.Dict[str, int]
with open_outcome_file(outcome_file) as input_file:
for line in input_file:
(_platform, component, suite, case, result, _cause) = line.split(';')
(platform, component, suite, case, result, _cause) = line.split(';')
# Note that `component` is not unique. If a test case passes on Linux
# and fails on FreeBSD, it'll end up in both the successes set and
# the failures set.
Expand All @@ -139,6 +142,20 @@ def read_outcome_file(outcome_file: str) -> Outcomes:
outcomes[component].successes.add(suite_case)
elif result == 'FAIL':
outcomes[component].failures.add(suite_case)
build_name = ';'.join([platform, component])
# Detect repeated make invocations to build all or part of the
# library in the same configuration on the same platform.
# This generally indicates that MBEDTLS_TEST_CONFIGURATION was
# not set to a unique value.
if suite == 'make' and MAKE_LIB_TARGET.match(case):
seen.setdefault(build_name, 0)
seen[build_name] += 1

# Complain about repeated configurations
for build_name in sorted(build_name for build_name in seen.keys()
if seen[build_name] > 1):
results.error('Repeated *{} platform;configuration for library build: {}',
seen[build_name], build_name)

return outcomes

Expand Down Expand Up @@ -380,7 +397,7 @@ def main(known_tasks: typing.Dict[str, typing.Type[Task]]) -> None:
getattr(task_class, 'DRIVER'),
options.outcomes)

outcomes = read_outcome_file(options.outcomes)
outcomes = read_outcome_file(main_results, options.outcomes)

for task_name in tasks_list:
task_constructor = known_tasks[task_name]
Expand Down