Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run TF-PSA-Crypto components without cloning Mbed TLS #190

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Run TF-PSA-Crypto components without cloning Mbed TLS #190

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e1f8283
Rename all_all_sh_components
bensze01 Dec 20, 2024
31dd5b6
Collect information about tf-psa-crypto components
bensze01 Dec 20, 2024
84f8435
Break out submodule checkout into separate methods
bensze01 Dec 20, 2024
5de7135
Test tf-psa-crypto components without Mbed TLS
bensze01 Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions src/org/mbed/tls/jenkins/BranchInfo.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ class BranchInfo {

/** Map from component name to chosen platform to run it, or to null
* if no platform has been chosen yet. */
public Map<String, String> all_all_sh_components
public Map<String, String> mbed_tls_all_sh_components

/** Map from component name to chosen platform to run it, or to null
* if no platform has been chosen yet. Empty if we are not running any tests.*/
public Map<String, String> tf_psa_crypto_all_sh_components

/** Whether scripts/min_requirements.py is available. Older branches don't
* have it, so they only get what's hard-coded in the docker files on Linux,
Expand Down Expand Up @@ -35,7 +39,8 @@ class BranchInfo {
String coverage_details

BranchInfo() {
this.all_all_sh_components = [:]
this.mbed_tls_all_sh_components = [:]
this.tf_psa_crypto_all_sh_components = [:]
this.has_min_requirements = false
this.python_requirements_override_content = ''
this.python_requirements_override_file = ''
Expand Down
49 changes: 27 additions & 22 deletions vars/checkout_repo.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,30 @@ Map<String, String> checkout_report_errors(scm_config) {
}
}

void checkout_framework_repo() {
if (env.TARGET_REPO == 'framework' && env.CHECKOUT_METHOD == 'scm') {
checkout_report_errors(scm)
} else if (env.FRAMEWORK_REPO && env.FRAMEWORK_BRANCH) {
checkout_report_errors(parametrized_repo(env.FRAMEWORK_REPO, env.FRAMEWORK_BRANCH))
} else {
echo 'Using default framework version'
}
}

void checkout_tf_psa_crypto_repo() {
if (env.TARGET_REPO == 'tf-psa-crypto' && env.CHECKOUT_METHOD == 'scm') {
checkout_report_errors(scm)
} else if (env.TF_PSA_CRYPTO_REPO && env.TF_PSA_CRYPTO_BRANCH) {
checkout_report_errors(parametrized_repo(env.TF_PSA_CRYPTO_REPO, env.TF_PSA_CRYPTO_BRANCH))
} else {
echo 'Using default tf-psa-crypto version'
}

dir('framework') {
checkout_framework_repo()
}
}

Map<String, String> checkout_tls_repo(String branch) {
def scm_config
if (env.TARGET_REPO == 'tls' && env.CHECKOUT_METHOD == 'scm') {
Expand All @@ -76,30 +100,11 @@ Map<String, String> checkout_tls_repo(String branch) {
def result = checkout_report_errors(scm_config)

dir('tf-psa-crypto') {
if (env.TARGET_REPO == 'tf-psa-crypto' && env.CHECKOUT_METHOD == 'scm') {
checkout_report_errors(scm)
} else if (env.TF_PSA_CRYPTO_REPO && env.TF_PSA_CRYPTO_BRANCH) {
checkout_report_errors(parametrized_repo(env.TF_PSA_CRYPTO_REPO, env.TF_PSA_CRYPTO_BRANCH))
} else {
echo 'Using default tf-psa-crypto version'
}
checkout_tf_psa_crypto_repo()
}

def framework_dirs = ['framework', 'tf-psa-crypto/framework']
if (env.TARGET_REPO == 'framework' && env.CHECKOUT_METHOD == 'scm') {
framework_dirs.each { framework_dir ->
dir(framework_dir) {
checkout_report_errors(scm)
}
}
} else if (env.FRAMEWORK_REPO && env.FRAMEWORK_BRANCH) {
framework_dirs.each { framework_dir ->
dir(framework_dir) {
checkout_report_errors(parametrized_repo(env.FRAMEWORK_REPO, env.FRAMEWORK_BRANCH))
}
}
} else {
echo 'Using default framework version'
dir('framework') {
checkout_framework_repo()
}

// After the clone, replicate it in the local config, so it is effective when running inside docker
Expand Down
124 changes: 77 additions & 47 deletions vars/common.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,12 @@ List<BranchInfo> get_branch_information(Collection<String> branches) {
List<BranchInfo> infos = []
Map<String, Object> jobs = [:]

def repo_roots = ['tls': '.']

if (env.RUN_TF_PSA_CRYPTO_ALL_SH == 'true') {
repo_roots['tf-psa-crypto'] = 'tf-psa-crypto'
}

branches.each { String branch ->
BranchInfo info = new BranchInfo()
info.branch = branch
Expand Down Expand Up @@ -256,26 +262,33 @@ List<BranchInfo> get_branch_information(Collection<String> branches) {

String platform = linux_platforms[0]
get_docker_image(platform)
def all_sh_help = sh(
script: docker_script(
platform, "./tests/scripts/all.sh", "--help"
),
returnStdout: true
)
if (all_sh_help.contains('list-components')) {
def all = sh(
script: docker_script(
platform, "./tests/scripts/all.sh",
"--list-all-components"
),
returnStdout: true
).trim().split('\n')
echo "All all.sh components: ${all.join(" ")}"
return all.collectEntries { element ->
return [(element): null]
return repo_roots.collectEntries { repo, root ->
dir(root) {
if (!fileExists('./tests/scripts/all.sh')) {
return [(repo): [:]]
}
def all_sh_help = sh(
script: docker_script(
platform, "./tests/scripts/all.sh", "--help"
),
returnStdout: true
)
if (all_sh_help.contains('list-components')) {
def all = sh(
script: docker_script(
platform, "./tests/scripts/all.sh",
"--list-all-components"
),
returnStdout: true
).trim().split('\n')
echo "All all.sh components: ${all.join(" ")}"
return [(repo): all.collectEntries { element ->
return [(element): null]
}]
} else {
error('Pre Test Checks failed: Base branch out of date. Please rebase')
}
}
} else {
error('Pre Test Checks failed: Base branch out of date. Please rebase')
}
} finally {
deleteDir()
Expand All @@ -292,25 +305,36 @@ List<BranchInfo> get_branch_information(Collection<String> branches) {
checkout_repo.checkout_tls_repo(branch)
}
get_docker_image(platform)
def all_sh_help = sh(
script: docker_script(
platform, "./tests/scripts/all.sh", "--help"
),
returnStdout: true
)
if (all_sh_help.contains('list-components')) {
def available = sh(
script: docker_script(
platform, "./tests/scripts/all.sh", "--list-components"
),
returnStdout: true
).trim().split('\n')
echo "Available all.sh components on ${platform}: ${available.join(" ")}"
return available.collectEntries { element ->
return [(element): platform]
return repo_roots.collectEntries { repo, root ->
// Only run tf-psa-crypto tests on the first branch
if (repo == 'tf-psa-crypto' && branch != branches[0]) {
return [(repo): [:]]
}
dir(root) {
if (!fileExists('./tests/scripts/all.sh')) {
return [(repo): [:]]
}
def all_sh_help = sh(
script: docker_script(
platform, "./tests/scripts/all.sh", "--help"
),
returnStdout: true
)
if (all_sh_help.contains('list-components')) {
def available = sh(
script: docker_script(
platform, "./tests/scripts/all.sh", "--list-components"
),
returnStdout: true
).trim().split('\n')
echo "Available all.sh components on ${platform}: ${available.join(" ")}"
return [(repo): available.collectEntries { element ->
return [(element): platform]
}]
} else {
error('Pre Test Checks failed: Base branch out of date. Please rebase')
}
}
} else {
error('Pre Test Checks failed: Base branch out of date. Please rebase')
}
} finally {
deleteDir()
Expand All @@ -321,29 +345,35 @@ List<BranchInfo> get_branch_information(Collection<String> branches) {
}

jobs.failFast = true
def results = (Map<String, Map<String, String>>) parallel(jobs)
def results = (Map<String, Map<String, Map<String, String>>>) parallel(jobs)

infos.each { BranchInfo info ->
String prefix = infos.size() > 1 ? "$info.branch-" : ''

info.all_all_sh_components = results[prefix + 'all-platforms']
linux_platforms.reverseEach { platform ->
info.all_all_sh_components << results[prefix + platform]
}
Map<String, Map<String, String>> repo_components = repo_roots.keySet().collectEntries {repo ->
def components = results[prefix + 'all-platforms'][repo]
linux_platforms.reverseEach { platform ->
components << results[prefix + platform][repo]
}

if (env.JOB_TYPE == 'PR') {
// Do not run release components in PR jobs
info.all_all_sh_components = info.all_all_sh_components.findAll {
component, platform -> !component.startsWith('release')
if (env.JOB_TYPE == 'PR') {
// Do not run release components in PR jobs
components = components.findAll {
component, platform -> !component.startsWith('release')
}
}
return [(repo): components]
}

info.mbed_tls_all_sh_components = repo_components['tls']
info.tf_psa_crypto_all_sh_components = repo_components['tf-psa-crypto'] ?: [:]
}
return infos
}

void check_every_all_sh_component_will_be_run(Collection<BranchInfo> infos) {
Map<String, Collection<String>> untested_all_sh_components = infos.collectEntries { info ->
def components = info.all_all_sh_components.findResults {
def components = info.mbed_tls_all_sh_components.findResults {
name, platform -> platform ? null : name
}
return components ? [(info.branch): components] : [:]
Expand Down
24 changes: 20 additions & 4 deletions vars/gen_jobs.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ fi
}
}

def gen_all_sh_jobs(BranchInfo info, platform, component, label_prefix='') {
def gen_all_sh_jobs(BranchInfo info, String platform, String component, String label_prefix='', String repo='tls') {
def shorthands = [
"arm-compilers-amd64": "armcc",
"ubuntu-16.04-amd64": "u16",
Expand All @@ -198,7 +198,7 @@ def gen_all_sh_jobs(BranchInfo info, platform, component, label_prefix='') {
]
/* Default to the full platform hame is a shorthand is not found */
def shortplat = shorthands.getOrDefault(platform, platform)
def job_name = "${label_prefix}all_${shortplat}-${component}"
def job_name = "${label_prefix}${repo == 'tls' ? '' : "${repo}_"}all_${shortplat}-${component}"
def outcome_file = "${job_name.replace((char) '/', (char) '_')}-outcome.csv"
def use_docker = platform_has_docker(platform)
def extra_setup_code = ''
Expand Down Expand Up @@ -245,7 +245,7 @@ echo >&2 'Note: "clang" will run /usr/bin/clang -Wno-error=c11-extensions'
'''
}

if (info.has_min_requirements) {
if (repo == 'tls' && info.has_min_requirements) {
extra_setup_code += """
scripts/min_requirements.py --user ${info.python_requirements_override_file}
"""
Expand All @@ -258,6 +258,16 @@ scripts/min_requirements.py --user ${info.python_requirements_override_file}
common.get_docker_image(platform)
}
dir('src') {
switch(repo) {
case 'tls':
checkout_repo.checkout_tls_repo(info)
break
case 'tf-psa-crypto':
checkout_repo.checkout_tf_psa_crypto_repo()
break
default:
error("Invalid repo: $repo")
}
checkout_repo.checkout_tls_repo(info)
writeFile file: 'steps.sh', text: """\
#!/bin/sh
Expand Down Expand Up @@ -654,7 +664,7 @@ def gen_release_jobs(BranchInfo info, String label_prefix='', boolean run_exampl
}

if (env.RUN_ALL_SH == "true") {
info.all_all_sh_components.each({component, platform ->
info.mbed_tls_all_sh_components.each({ component, platform ->
jobs = jobs + gen_all_sh_jobs(info, platform, component, label_prefix)
})
}
Expand All @@ -668,6 +678,12 @@ def gen_release_jobs(BranchInfo info, String label_prefix='', boolean run_exampl
}
}

if (env.RUN_TF_PSA_CRYPTO_ALL_SH == "true") {
info.tf_psa_crypto_all_sh_components.each({ component, platform ->
jobs = jobs + gen_all_sh_jobs(info, platform, component, label_prefix)
})
}

if (env.RUN_WINDOWS_TEST == "true") {
jobs = jobs + gen_windows_jobs(info, label_prefix)
}
Expand Down