obs: config for 3.2

Signed-off-by: Adrian Reber <[email protected]>

misc/obs/config.2.x

@@ -6,6 +6,7 @@ github_url = "[email protected]:openhpc/ohpc.git"
 service_template=service.file
 link_compiler_template=link_compiler
 link_mpi_template=link_mpi
+link_mpi_to_non_mpi_template=link_mpi_to_non_mpi
 
 override_templates=templates
 dry_run=True  # do not make changes to OBS, just do a dry-run
@@ -43,13 +44,29 @@ skip_aarch=["-intel\\b","lustre-client","-impi\\b","-mvapich2\\b","likwid-gnu","
 skip_x86  = ["-arm1"]
 
 # define compiler/MPI families: first entry in list is defined to be parent in OBS
-#compiler_families=["gnu13","intel","arm1"]
+#compiler_families=["gnu14","intel","arm1"]
 compiler_families=["gnu14", "intel"]
 mpi_families=["openmpi5","mpich","mvapich2","impi"]
-
-standalone = ["docs","test-suite","warewulf"]
-mpi_dependent = ["cubew","otf2","cubelib","opari2","scorep","scalasca"]
+compiler_dependent = ["openmpi", "mpich", "mvapich2", "openblas", "R", "likwid",
+                      "pdtoolkit", "gsl", "metis", "superlu", "scotch",
+                      "numpy", "plasma", "hdf5"]
+
+standalone = ["docs", "test-suite", "warewulf", "gnu-compilers", "ohpc-filesystem",
+	      "impi-devel", "meta-packages", "easybuild", "spack", "hwloc", "ucx",
+	      "lmod", "genders", "hpc-workspace", "valgrind", "slurm"]
+mpi_dependent = ["cubew", "otf2", "cubelib", "opari2", "sionlib", "fftw", "scalapack",
+		 "scorep", "scalasca", "scipy", "phdf5", "netcdf", "netcdf-fortran",
+		 "netcdf-cxx", "lmod-defaults", "geopm", "mumps", "omb",
+		 "ptscotch", "boost", "pnetcdf", "tau", "extrae", "imb",
+		 "opencoarrays", "hypre", "mpi4py", "dimemas", "adios2",
+		 "trilinos", "petsc", "slepc", "superlu_dist", "mfem"]
 skip_on_distro_openEuler_22.03 = ["-arm1","-intel","-impi","impi-devel","intel-compilers-devel","arm-compilers-devel"]
+openblas_compiler=["gnu14"]
+R_compiler=["gnu14"]
+opencoarrays_compiler=["gnu14"]
+scipy_compiler=["gnu14"]
+mpi_dependent_to_non_mpi=["netcdf","netcdf-fortran","netcdf-cxx"]
+with_ucx=["mpich"]
 
 [3.1.0]
 
@@ -162,7 +179,11 @@ skip_x86  = ["-arm1"]
 compiler_families=["gnu12","intel","arm1"]
 mpi_families=["openmpi4","mpich","mvapich2","impi"]
 
-standalone = ["intel-compilers-devel"]
+standalone = ["intel-compilers-devel", "easybuild", "gnu-compilers", "slurm", "spack", "lmod",
+	      "test-suite", "docs"]
+mpi_dependent = ["opencoarrays"]
+
+opencoarrays_compiler=["gnu12"]
 
 [2.8.0]
 

misc/obs/link_compiler

@@ -16,7 +16,7 @@ permissions and limitations under the License.
 
 <link project='!PROJECT!' package='!PACKAGE!'>
 <patches>
-  <topadd>%define compiler_family !COMPILER!</topadd>
+	<topadd>%define compiler_family !COMPILER!</topadd>
+	!REPLACE_ME!
 </patches>
 </link>
-

misc/obs/obs_config.py

@@ -148,6 +148,10 @@ def parseConfig(self, configFile=None, service_file=None):
                     'global',
                     'link_mpi_template',
                 )
+                self.linkFile_mpi_to_non_mpi = self.buildConfig.get(
+                    'global',
+                    'link_mpi_to_non_mpi_template',
+                )
                 self.overrides = self.buildConfig.get(
                     'global',
                     'override_templates',
@@ -190,14 +194,22 @@ def parseConfig(self, configFile=None, service_file=None):
             self.parentCompiler = self.compilerFamilies[0]
             self.parentMPI = self.MPIFamilies[0]
 
-            logging.info("--> (global) dry run              = %s" %
-                         self.dryRun)
-            logging.info("--> (global) service template     = %s" %
-                         self.serviceFile)
-            logging.info("--> (global) link template (comp) = %s" %
-                         self.linkFile_compiler)
-            logging.info("--> (global) link template (mpi)  = %s" %
-                         self.linkFile_mpi)
+            logging.info(
+                "--> (global) dry run" + " " * 39 + "= %s" %
+                self.dryRun)
+            logging.info(
+                "--> (global) service template" + " " * 30 + "= %s" %
+                self.serviceFile)
+            logging.info(
+                "--> (global) link template (comp)" + " " * 26 + "= %s" %
+                self.linkFile_compiler)
+            logging.info(
+                "--> (global) link template (mpi)" + " " * 27 + "= %s" %
+                self.linkFile_mpi)
+            logging.info(
+                "--> (global) link template " +
+                "(link_mpi_to_non_mpi_template)  = %s" %
+                self.linkFile_mpi)
             logging.info("\nCompiler families (%s):" % self.vip)
 
             for family in self.compilerFamilies:
@@ -275,39 +287,71 @@ def query_components(self, version="unknown"):
             components['standalone'] = ast.literal_eval(
                 self.buildConfig.get(self.vip, "standalone"))
             logging.info("Parsed components:")
-            logging.info("--> [standalone]: %s" % components['standalone'])
+            logging.info(
+                "--> [        standalone]: %s" %
+                components['standalone'])
 
             components['standalone'] = self.checkForDisabledComponents(
                 components['standalone'])
 
         if self.buildConfig.has_option(self.vip, 'compiler_dependent'):
             components['comp_dep'] = ast.literal_eval(
                 self.buildConfig.get(self.vip, "compiler_dependent"))
-            logging.info("--> [  comp_dep]: %s" % components['comp_dep'])
+            logging.info(
+                "--> [          comp_dep]: %s" %
+                components['comp_dep'])
 
             components['comp_dep'] = self.checkForDisabledComponents(
                 components['comp_dep'])
 
         if self.buildConfig.has_option(self.vip, 'mpi_dependent'):
             components['mpi_dep'] = ast.literal_eval(
                 self.buildConfig.get(self.vip, "mpi_dependent"))
-            logging.info("--> [   mpi_dep]: %s" % components['mpi_dep'])
+            logging.info(
+                "--> [           mpi_dep]: %s" %
+                components['mpi_dep'])
 
             components['mpi_dep'] = self.checkForDisabledComponents(
                 components['mpi_dep'])
 
+        if self.buildConfig.has_option(self.vip, 'mpi_dependent_to_non_mpi'):
+            components['mpi_dep_to_non_mpi'] = ast.literal_eval(
+                self.buildConfig.get(self.vip, "mpi_dependent_to_non_mpi"))
+            logging.info(
+                "--> [mpi_dep_to_non_mpi]: %s" %
+                components['mpi_dep_to_non_mpi'])
+
+            components['mpi_dep_to_non_mpi'] = self.checkForDisabledComponents(
+                components['mpi_dep_to_non_mpi'])
+
+        if self.buildConfig.has_option(self.vip, 'with_ucx'):
+            components['with_ucx'] = ast.literal_eval(
+                self.buildConfig.get(self.vip, "with_ucx"))
+            logging.info(
+                "--> [          with_ucx]: %s" %
+                components['with_ucx'])
+
+            components['with_ucx'] = self.checkForDisabledComponents(
+                components['with_ucx'])
+
         numComponents = 0
         if 'standalone' not in components:
             components['standalone'] = []
         if 'comp_dep' not in components:
             components['comp_dep'] = []
         if 'mpi_dep' not in components:
             components['mpi_dep'] = []
+        if 'mpi_dep_to_non_mpi' not in components:
+            components['mpi_dep_to_non_mpi'] = []
+        if 'with_ucx' not in components:
+            components['with_ucx'] = []
 
         numComponents = (
             len(components['standalone']) +
             len(components['comp_dep']) +
-            len(components['mpi_dep'])
+            len(components['mpi_dep']) +
+            len(components['mpi_dep_to_non_mpi']) +
+            len(components['with_ucx'])
         )
 
         logging.info("# of requested components = %i\n" % numComponents)
@@ -499,6 +543,8 @@ def addPackage(
         mpi=None,
         parentName=None,
         gitName=None,
+        isMPIDepToNonMPI=False,
+        replace=None,
     ):
         fname = inspect.stack()[0][3]
         pad = 15
@@ -710,6 +756,8 @@ def addPackage(
                 assert compiler is not None
             elif isMPIDep:
                 linkFile = self.linkFile_mpi
+            elif isMPIDepToNonMPI:
+                linkFile = self.linkFile_mpi_to_non_mpi
 
             assert parentName is not None
 
@@ -729,6 +777,11 @@ def addPackage(
             if isMPIDep:
                 contents = contents.replace('!MPI!', mpi)
 
+            if replace:
+                contents = contents.replace('!REPLACE_ME!', replace)
+            else:
+
+                contents = contents.replace('\t!REPLACE_ME!\n', '')
             fp_link = tempfile.NamedTemporaryFile(delete=True, mode='w')
             fp_link.write(contents)
             fp_link.flush()
@@ -884,10 +937,10 @@ def loglevel(debug):
     for package in components['standalone']:
         ptype = "standalone"
         if package in obsPackages:
-            logging.info("%27s (%13s): present in OBS" % (package, ptype))
+            logging.info("%34s (%13s): present in OBS" % (package, ptype))
         else:
             logging.info(
-                "%27s (%13s): *not* present in OBS, need to add" %
+                "%34s (%13s): *not* present in OBS, need to add" %
                 (package, ptype))
             obs.addPackage(package, parent=True)
 
@@ -898,10 +951,6 @@ def loglevel(debug):
         if args.package and (package != args.package):
             logging.info("skipping %s" % package)
             continue
-        else:
-            logging.info(
-                "desired override package %s is compiler dependent" %
-                package)
 
         ptype = "compiler dep"
         parent = package + '-' + obs.getParentCompiler()
@@ -910,7 +959,7 @@ def loglevel(debug):
         Defcompilers = obs.queryCompilers(package, noOverride=True)
 
         if compilers != Defcompilers:
-            pad = 15
+            pad = 22
             logging.warning(
                 " " *
                 pad +
@@ -920,10 +969,10 @@ def loglevel(debug):
 
         # check on parent first (it must exist before any children are linked)
         if parent in obsPackages:
-            logging.info("%27s (%13s): present in OBS" % (parent, ptype))
+            logging.info("%34s (%13s): present in OBS" % (parent, ptype))
         else:
             logging.info(
-                "%27s (%13s): *not* present in OBS, need to add" %
+                "%34s (%13s): *not* present in OBS, need to add" %
                 (parent, ptype))
             obs.addPackage(
                 parent,
@@ -952,10 +1001,10 @@ def loglevel(debug):
                 "checking on child compiler dependent package: %s" %
                 child)
             if child in obsPackages:
-                logging.info("%27s (%13s): present in OBS" % (child, ptype))
+                logging.info("%34s (%13s): present in OBS" % (child, ptype))
             else:
                 logging.info(
-                    "%27s (%13s): *not* present in OBS, need to add" %
+                    "%34s (%13s): *not* present in OBS, need to add" %
                     (child, ptype))
                 obs.addPackage(
                     child,
@@ -964,6 +1013,25 @@ def loglevel(debug):
                     compiler=compiler,
                     parentName=parent)
 
+        for compiler in compilers:
+            if package in components['with_ucx']:
+                child = package + '-ucx-' + compiler
+                if child in obsPackages:
+                    logging.info(
+                        "%34s (%13s): present in OBS" %
+                        (child, ptype))
+                else:
+                    logging.info(
+                        "%34s (%13s): *not* present in OBS, need to add" %
+                        (child, ptype))
+                    obs.addPackage(
+                        child,
+                        parent=False,
+                        isCompilerDep=True,
+                        compiler=compiler,
+                        parentName=parent,
+                        replace='<topadd>%define with_ucx 1</topadd>')
+
     # (3) MPI dependent packages
     for package in components['mpi_dep']:
         ptype = "MPI dep"
@@ -979,10 +1047,10 @@ def loglevel(debug):
 
         # check on parent first (it must exist before any children are linked)
         if parent in obsPackages:
-            logging.info("%27s (%13s): present in OBS" % (parent, ptype))
+            logging.info("%34s (%13s): present in OBS" % (parent, ptype))
         else:
             logging.info(
-                "%27s (%13s): *not* present in OBS, need to add" %
+                "%34s (%13s): *not* present in OBS, need to add" %
                 (parent, ptype))
             obs.addPackage(parent, parent=True, isMPIDep=True, gitName=package)
 
@@ -1000,11 +1068,11 @@ def loglevel(debug):
 
                 if child in obsPackages:
                     logging.info(
-                        "%27s (%13s): present in OBS" %
+                        "%34s (%13s): present in OBS" %
                         (child, ptype))
                 else:
                     logging.info(
-                        "%27s (%13s): *not* present in OBS, need to add" %
+                        "%34s (%13s): *not* present in OBS, need to add" %
                         (child, ptype))
                     obs.addPackage(
                         child,
@@ -1014,6 +1082,27 @@ def loglevel(debug):
                         mpi=mpi,
                         parentName=parent)
 
+        if package in components['mpi_dep_to_non_mpi']:
+            ptype = "non MPI dep"
+            for compiler in compilers:
+                child = package + '-' + compiler
+                if child in obsPackages:
+                    logging.info(
+                        "%34s (%13s): present in OBS" %
+                        (child, ptype))
+                else:
+                    logging.info(
+                        "%34s (%13s): *not* present in OBS, need to add" %
+                        (child, ptype))
+                    obs.addPackage(
+                        child,
+                        parent=False,
+                        isMPIDep=False,
+                        compiler=compiler,
+                        mpi=mpi,
+                        parentName=parent,
+                        isMPIDepToNonMPI=True)
+
     obs.cancelNewBuilds()