feature(create_test_release_jobs): Add job metadata files

This commit adds a simple per-folder definition file (and format
documentation) format for storing additional metadata on the jenkins
jobs. The metadata is appended to the end of job description and is
planned to be consumed by Argus to determine both the job type and
subtype. Additionally, it adds a description annotation to the pipeline
file, allowing jobs to have descriptions specified

Fixes scylladb/argus#428

docs/job_defines_format.md

@@ -0,0 +1,55 @@
+# Jenkins Job Definitions
+
+Due to a growing need of having extra metadata available to jenkins consumers
+(both users and automation) a job define system was added to SCT jenkins generator.
+
+It replaces previous `_display_name` file (which used to contain just the display
+ name of the folder) with a special .yaml file called `_folder_definitions.yaml`.
+
+Example definitions file:
+
+```yaml
+# used by argus to determine argus plugin
+job-type: scylla-cluster-tests
+# used by argus to determine which parameter wizard to use on execution
+job-sub-type: longevity
+# replacement for _display_name file, sets the jenkins folder display name
+folder-name: Cluster - Longevity Tests
+folder-description: Contains all longevity tests
+# Per job (regex supported) job overrides, defines as a mapping
+overrides:
+  100gb: # regex, search for anything matching 100gb
+      job-sub-type: artifact
+  longevity-5tb-1day-gce: # specific name
+    # overrides sub-type for argus, needed for folders that contain
+    # for example both "artifact" and "artifact-offline" tests
+      job-sub-type: rolling-upgrade
+```
+
+A job description can also be provided as an annotation, like so:
+
+```js
+/** jobDescription
+    This is a simple job description.
+    Can be multi line and indented.
+    Will return the description without indent.
+*/
+```
+
+Once template is generated the defines are applied to the job description
+along with job description from the pipeline file, like so:
+
+```
+This is a simple job description.
+Can be multi line and indented.
+Will return the description without indent.
+
+### JobDefinitions
+job-sub-type: artifact
+job-type: scylla-cluster-tests
+```
+
+If a define file was not found, a previously used mechanism is used for descriptions
+```
+jenkins-pipelines/oss/longevity/longevity-cdc-100gb-4h.jenkinsfile
+```

utils/build_system/create_test_release_jobs.py

@@ -14,9 +14,11 @@
 import os
 import logging
 from pathlib import Path
+import re
 import xml.etree.ElementTree as ET
 
 import jenkins
+import yaml
 
 from sdcm.wait import wait_for
 from sdcm.utils.common import get_sct_root_path
@@ -52,9 +54,9 @@ def update_pipeline(self, new_path, xml_data):
             new_configuration.append(properties)
         self.jenkins.reconfig_job(new_path, ET.tostring(new_configuration).decode('utf-8'))
 
-    def create_directory(self, name: Path | str, display_name: str):
+    def create_directory(self, name: Path | str, display_name: str, description: str = ""):
         try:
-            dir_xml_data = DIR_TEMPLATE % dict(sct_display_name=display_name)
+            dir_xml_data = DIR_TEMPLATE % dict(sct_display_name=display_name, sct_description=description)
             new_path = str(Path(self.base_job_dir) / name)
 
             if self.jenkins.job_exists(new_path):
@@ -95,12 +97,18 @@ def create_freestyle_job(self, xml_temple: Path | str, group_name: Path | str, j
         except jenkins.JenkinsException as ex:
             self._log_jenkins_exception(ex)
 
-    def create_pipeline_job(self, jenkins_file: Path | str, group_name: Path | str, job_name: str = None, job_name_suffix="-test"):
+    def create_pipeline_job(self, jenkins_file: Path | str, group_name: Path | str, job_name: str = None, job_name_suffix="-test", defines: dict = None):
         jenkins_file = Path(jenkins_file)
         base_name = job_name or jenkins_file.stem
         sct_jenkinsfile = jenkins_file.relative_to(get_sct_root_path())
+        text_description = self.get_job_description(jenkins_file)
+        if defines:
+            description = f"{text_description}\n\n### JobDefinitions\n{yaml.safe_dump(defines)}"
+        else:
+            description = sct_jenkinsfile
+
         xml_data = JOB_TEMPLATE % dict(sct_display_name=f"{base_name}{job_name_suffix}",
-                                       sct_description=sct_jenkinsfile,
+                                       sct_description=description,
                                        sct_repo=self.sct_repo,
                                        sct_branch_name=self.sct_branch_name,
                                        sct_jenkinsfile=sct_jenkinsfile)
@@ -144,16 +152,62 @@ def _log_jenkins_exception(exc):
         else:
             LOGGER.error(exc)
 
+    def load_defines(self, root: Path) -> dict[str, str]:
+        try:
+            content = (root / "_folder_definitions.yaml").open()
+        except FileNotFoundError:
+            return {}
+        defines = yaml.safe_load(content)
+        return defines
+
+    @staticmethod
+    def locate_job_overrides(job_name: str, overrides: dict[str, str]) -> dict[str, str]:
+        if full_match := overrides.get(job_name):
+            return full_match
+
+        for key in overrides:
+            if re.search(re.compile(key, re.IGNORECASE), job_name):
+                return overrides.get(key, {})
+
+        return {}
+
+    @staticmethod
+    def get_job_description(job_path: Path) -> str:
+        if not job_path.exists():
+            return ""
+
+        file = job_path.open()
+
+        description_lines: list[str] = []
+        reading = False
+        for line in file.readlines():
+            if re.match(r"\s*/\*\*\s*jobDescription", line):
+                reading = True
+                continue
+            if re.match(r"\s*\*/", line):
+                reading = False
+                break
+
+            if reading:
+                description_lines.append(line)
+
+        description = "\n".join(line.strip() for line in description_lines)
+
+        return description
+
     def create_job_tree(self, local_path: str | Path,  # pylint: disable=too-many-arguments
                         create_freestyle_jobs: bool = True,
                         create_pipelines_jobs: bool = True,
                         template_context: dict | None = None,
                         job_name_suffix: str = '-test'):
         for root, _, job_files in os.walk(local_path):
             jenkins_path = Path(root).relative_to(local_path)
+            defines = self.load_defines(Path(root))
+            job_overrides = defines.pop("overrides", {})
 
             # get display names, if available
-            display_name = jenkins_path.name
+            display_name = defines.pop("folder-name", None) or jenkins_path.name
+            folder_description = defines.pop("folder-description", "")
             if '_display_name' in job_files:
                 display_name = (Path(root) / '_display_name').read_text().strip()
 
@@ -162,11 +216,16 @@ def create_job_tree(self, local_path: str | Path,  # pylint: disable=too-many-ar
                 display_name = self.base_job_dir.split('/')[-1]
 
             if jenkins_path and display_name:
-                self.create_directory(jenkins_path, display_name=display_name)
+                self.create_directory(jenkins_path, display_name=display_name, description=folder_description)
 
             for job_file in job_files:
                 job_file = Path(root) / job_file  # noqa: PLW2901
                 if (job_file.suffix == '.jenkinsfile') and create_pipelines_jobs:
-                    self.create_pipeline_job(job_file, group_name=jenkins_path, job_name_suffix=job_name_suffix)
+                    self.create_pipeline_job(
+                        job_file,
+                        group_name=jenkins_path,
+                        job_name_suffix=job_name_suffix,
+                        defines={**defines, **self.locate_job_overrides(job_file.stem, job_overrides)}
+                    )
                 if (job_file.suffix == '.xml') and create_freestyle_jobs:
                     self.create_freestyle_job(job_file, group_name=jenkins_path, template_context=template_context)

utils/build_system/folder-template.xml

@@ -1,6 +1,6 @@
 <?xml version='1.1' encoding='UTF-8'?>
 <com.cloudbees.hudson.plugins.folder.Folder plugin="[email protected]">
-  <description></description>
+  <description>%(sct_description)s</description>
   <displayName>%(sct_display_name)s</displayName>
   <properties>
     <org.jenkinsci.plugins.pipeline.modeldefinition.config.FolderConfig plugin="[email protected]">