# IT4Innovations 2023
# JK

name = 'OpenMPI'
version = '4.0.5'

homepage = 'https://www.open-mpi.org/'
description = """The Open MPI Project is an open source MPI-3 implementation."""

toolchain = {'name': 'iccifort', 'version': '2020.4.304'}

source_urls = ['https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
sources = [SOURCELOWER_TAR_GZ]
patches = [
    'OpenMPI-4.0.1_remove-pmix-check-in-pmi-switch.patch',
    'OpenMPI-4.x_fix_pmix_discovery.patch',
    'OpenMPI-4.0.5-6-pml-ucx-datatype-memleak.patch',
]
checksums = [
    '572e777441fd47d7f06f1b8a166e7f44b8ea01b8b2e79d1e299d509725d1bd05',  # openmpi-4.0.5.tar.gz
    # OpenMPI-4.0.1_remove-pmix-check-in-pmi-switch.patch
    'a5737061eb9006e862f30019776adf092d800f681272be7f1575e74b4bfa20fb',
    '547641fff884c917237d158b0b13bdf387977cf0dddfd7e49e78d5f759a6a31b',  # OpenMPI-4.x_fix_pmix_discovery.patch
    # OpenMPI-4.0.5-6-pml-ucx-datatype-memleak.patch
    '7d8695f0d23453c82638ad33b18e41690274d5c7784291213e98335b42c54578',
]

builddependencies = [
    ('pkg-config', '0.29.2'),
]

dependencies = [
    ('zlib', '1.2.11'),
    ('hwloc', '2.2.0'),
    ('libevent', '2.1.12'),
    ('UCX', '1.9.0'),
    ('libfabric', '1.11.0'),
    ('PMIx', '3.1.5'),
]

configopts = '--enable-shared --enable-mpi-thread-multiple --with-verbs '
configopts += '--enable-mpirun-prefix-by-default '
configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
configopts += '--with-slurm ' # Enable slurm
configopts += '--enable-mpi-cxx '  # Enable building the C++ MPI bindings
configopts += '--with-ucx=$EBROOTUCX '

postinstallcmds = [
    'echo "# By default, for Open MPI 4.0 and later, infiniband ports on a device are not used by default." >> %(installdir)s/etc/openmpi-mca-params.conf',
    'echo "btl_openib_allow_ib = true" >> %(installdir)s/etc/openmpi-mca-params.conf',
]

local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
sanity_check_paths = {
    'files': [
        "bin/%s" %
        binfile for binfile in [
            "ompi_info", "opal_wrapper", "orterun"]] + [
                "lib/lib%s.%s" %
                (libfile, SHLIB_EXT) for libfile in local_libs] + [
                    "include/%s.h" %
                    x for x in [
                        "mpi-ext", "mpif-config", "mpif", "mpi", "mpi_portable_platform"]], 'dirs': [], }

import os
if os.environ.get("CLUSTERNAME") in ["BARBORA"]:
   modextravars = {'OMPI_MCA_btl_openib_if_include': 'mlx5_0',
                'OMPI_MCA_btl_tcp_if_include': '10.33.4.0/24',
                'OMPI_MCA_orte_base_help_aggregate': '0',
   }
elif os.environ.get("CLUSTERNAME") in ["KAROLINA"]:
   modextravars = {'OMPI_MCA_btl_openib_if_include': 'mlx5_0',
                'OMPI_MCA_orte_base_help_aggregate': '0',
   }
else:
   modextravars = {'OMPI_MCA_btl_openib_if_include': 'mlx4_0',
                'OMPI_MCA_oob_tcp_if_include': '10.0.0.0/8',
   }

# disable MPI1 compatibility for now, see what breaks...
# configopts = '--enable-mpi1-compatibility '

# to enable SLURM integration (site-specific)
# configopts += '--with-slurm --with-pmi=/usr/include/slurm --with-pmi-libdir=/usr'

import os
if os.environ.get("CLUSTERNAME") in ["BARBORA"]:
   modextravars = {'OMPI_MCA_btl_openib_if_include': 'mlx5_0',
                'OMPI_MCA_btl_tcp_if_include': '10.33.4.0/24',
                'OMPI_MCA_orte_base_help_aggregate': '0',
                'SLURM_MPI_TYPE': 'pmix_v4',
   }
elif os.environ.get("CLUSTERNAME") in ["KAROLINA"]:
   modextravars = {'OMPI_MCA_btl_openib_if_include': 'mlx5_0',
                'OMPI_MCA_orte_base_help_aggregate': '0',
                'SLURM_MPI_TYPE': 'pmix_v4',
   }
else:
   modextravars = {'OMPI_MCA_btl_openib_if_include': 'mlx4_0',
                'OMPI_MCA_oob_tcp_if_include': '10.0.0.0/8',
                'SLURM_MPI_TYPE': 'pmix_v4',
   }

moduleclass = 'mpi'