Host-configs

To capture (and revision control) build options, third party library paths, etc., we recommend using CMake’s initial-cache file mechanism. This feature allows you to pass a file to CMake that provides variables to bootstrap the configuration process.

You can pass initial-cache files to cmake via the -C command line option.

cmake -C config_file.cmake

We call these initial-cache files host-config files since we typically create a file for each platform or for specific hosts, if necessary.

These files use standard CMake commands. CMake set() commands need to specify CACHE as follows:

set(CMAKE_VARIABLE_NAME {VALUE} CACHE PATH "")

Here is a snippet from a host-config file that specifies compiler details for using specific gcc (version 10.3.1 in this case) on the LLNL Pascal cluster:

set(GCC_VERSION "gcc-10.3.1")
set(GCC_HOME "/usr/tce/packages/gcc/${GCC_VERSION}")

set(CMAKE_C_COMPILER "${GCC_HOME}/bin/gcc" CACHE PATH "")
set(CMAKE_CXX_COMPILER "${GCC_HOME}/bin/g++" CACHE PATH "")

# Fortran support
set(ENABLE_FORTRAN ON CACHE BOOL "")
set(CMAKE_Fortran_COMPILER "${GCC_HOME}/bin/gfortran" CACHE PATH "")

Building and Testing on Pascal

Since compute nodes on the Pascal cluster have CPUs and GPUs, here is how you can use the host-config file to configure a build of the calc_pi project with MPI and CUDA enabled on Pascal:

# create build dir
mkdir build
cd build
# configure using host-config
cmake -C ../../host-configs/llnl/toss_4_x86_64_ib/gcc@10.3.1_nvcc.cmake  ..

After building (make), you can run make test on a batch node (where the GPUs reside) to run the unit tests that are using MPI and CUDA:

bash-4.1$ salloc -A <valid bank>
bash-4.1$ make
bash-4.1$ make test

Running tests...
Test project blt/docs/tutorial/calc_pi/build
    Start 1: test_1
1/8 Test #1: test_1 ...........................   Passed    0.01 sec
    Start 2: test_2
2/8 Test #2: test_2 ...........................   Passed    2.79 sec
    Start 3: test_3
3/8 Test #3: test_3 ...........................   Passed    0.54 sec
    Start 4: blt_gtest_smoke
4/8 Test #4: blt_gtest_smoke ..................   Passed    0.01 sec
    Start 5: blt_fruit_smoke
5/8 Test #5: blt_fruit_smoke ..................   Passed    0.01 sec
    Start 6: blt_mpi_smoke
6/8 Test #6: blt_mpi_smoke ....................   Passed    2.82 sec
    Start 7: blt_cuda_smoke
7/8 Test #7: blt_cuda_smoke ...................   Passed    0.48 sec
    Start 8: blt_cuda_runtime_smoke
8/8 Test #8: blt_cuda_runtime_smoke ...........   Passed    0.11 sec

100% tests passed, 0 tests failed out of 8

Total Test time (real) =   6.80 sec

Building and Testing on Ray

Here is how you can use the host-config file to configure a build of the calc_pi project with MPI and CUDA enabled on the LLNL BlueOS Lassen cluster:

# create build dir
mkdir build
cd build
# configure using host-config
cmake -C ../../host-configs/llnl/blueos_3_ppc64le_ib_p9/clang@10.0.1_nvcc_c++17.cmake  ..

And here is how to build and test the code on Ray:

bash-4.2$ lalloc 1 -G <valid group>
bash-4.2$ make
bash-4.2$ make test

Running tests...
Test project projects/blt/docs/tutorial/calc_pi/build
    Start 1: test_1
1/7 Test #1: test_1 ...........................   Passed    0.01 sec
    Start 2: test_2
2/7 Test #2: test_2 ...........................   Passed    1.24 sec
    Start 3: test_3
3/7 Test #3: test_3 ...........................   Passed    0.17 sec
    Start 4: blt_gtest_smoke
4/7 Test #4: blt_gtest_smoke ..................   Passed    0.01 sec
    Start 5: blt_mpi_smoke
5/7 Test #5: blt_mpi_smoke ....................   Passed    0.82 sec
    Start 6: blt_cuda_smoke
6/7 Test #6: blt_cuda_smoke ...................   Passed    0.15 sec
    Start 7: blt_cuda_runtime_smoke
7/7 Test #7: blt_cuda_runtime_smoke ...........   Passed    0.04 sec

100% tests passed, 0 tests failed out of 7

Total Test time (real) =   2.47 sec

Building and Testing on Summit

Here is how you can use the host-config file to configure a build of the calc_pi project with MPI and CUDA enabled on the OLCF Summit cluster:

# load the cmake module
module load cmake
# create build dir
mkdir build
cd build
# configure using host-config
cmake -C ../../host-configs/olcf/summit/gcc@6.4.0_nvcc.cmake  ..

And here is how to build and test the code on Summit:

bash-4.2$ bsub -W 30 -nnodes 1 -P <valid project>  -Is /bin/bash
bash-4.2$ module load gcc cuda
bash-4.2$ make
bash-4.2$ make test

Running tests...
Test project /projects/blt/docs/tutorial/calc_pi/build
      Start  1: test_1
 1/11 Test  #1: test_1 ...........................   Passed    0.00 sec
      Start  2: test_2
 2/11 Test  #2: test_2 ...........................   Passed    1.03 sec
      Start  3: test_3
 3/11 Test  #3: test_3 ...........................   Passed    0.21 sec
      Start  4: blt_gtest_smoke
 4/11 Test  #4: blt_gtest_smoke ..................   Passed    0.00 sec
      Start  5: blt_fruit_smoke
 5/11 Test  #5: blt_fruit_smoke ..................   Passed    0.00 sec
      Start  6: blt_mpi_smoke
 6/11 Test  #6: blt_mpi_smoke ....................   Passed    0.76 sec
      Start  7: blt_cuda_smoke
 7/11 Test  #7: blt_cuda_smoke ...................   Passed    0.22 sec
      Start  8: blt_cuda_runtime_smoke
 8/11 Test  #8: blt_cuda_runtime_smoke ...........   Passed    0.07 sec
      Start  9: blt_cuda_version_smoke
 9/11 Test  #9: blt_cuda_version_smoke ...........   Passed    0.06 sec
      Start 10: blt_cuda_mpi_smoke
10/11 Test #10: blt_cuda_mpi_smoke ...............   Passed    0.80 sec
      Start 11: blt_cuda_gtest_smoke
11/11 Test #11: blt_cuda_gtest_smoke .............   Passed    0.21 sec

100% tests passed, 0 tests failed out of 11

Total Test time (real) =   3.39 sec

Example Host-configs

Basic TOSS3 (for example: Quartz) host-config that has C, C++, and Fortran Compilers along with MPI support:

gcc@8.3.1 host-config

 1# Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and
 2# other BLT Project Developers. See the top-level LICENSE file for details
 3# 
 4# SPDX-License-Identifier: (BSD-3-Clause)
 5
 6#------------------------------------------------------------------------------
 7# Example host-config file for a cluster on a toss4 platform (e.g. quartz) at LLNL
 8#------------------------------------------------------------------------------
 9#
10# This file provides CMake with paths / details for:
11#  C,C++, & Fortran compilers + MPI
12# 
13#------------------------------------------------------------------------------
14
15#------------------------------------------------------------------------------
16# gcc@10.3.1 compilers
17#------------------------------------------------------------------------------
18
19# _blt_tutorial_compiler_config_start
20set(GCC_VERSION "gcc-10.3.1")
21set(GCC_HOME "/usr/tce/packages/gcc/${GCC_VERSION}")
22
23set(CMAKE_C_COMPILER "${GCC_HOME}/bin/gcc" CACHE PATH "")
24
25set(CMAKE_CXX_COMPILER "${GCC_HOME}/bin/g++" CACHE PATH "")
26
27# Fortran support
28set(ENABLE_FORTRAN ON CACHE BOOL "")
29set(CMAKE_Fortran_COMPILER "${GCC_HOME}/bin/gfortran" CACHE PATH "")
30#_blt_tutorial_compiler_config_end
31
32#------------------------------------------------------------------------------
33# MPI Support
34#------------------------------------------------------------------------------
35# _blt_tutorial_mpi_config_start
36set(ENABLE_MPI ON CACHE BOOL "")
37
38set(MPI_HOME             "/usr/tce/packages/mvapich2/mvapich2-2.3.6-${GCC_VERSION}" CACHE PATH "")
39
40set(MPI_C_COMPILER       "${MPI_HOME}/bin/mpicc" CACHE PATH "")
41set(MPI_CXX_COMPILER     "${MPI_HOME}/bin/mpicxx" CACHE PATH "")
42set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpif90" CACHE PATH "")
43
44set(MPIEXEC              "/usr/bin/srun" CACHE PATH "")
45set(MPIEXEC_NUMPROC_FLAG "-n" CACHE PATH "")
46# _blt_tutorial_mpi_config_end

Here are the full example host-config files for LLNL’s Pascal, Lassen, and Quartz Clusters that uses the default compilers on the system:

gcc@10.3.1 host-config

 1# Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and
 2# other BLT Project Developers. See the top-level LICENSE file for details
 3# 
 4# SPDX-License-Identifier: (BSD-3-Clause)
 5
 6#------------------------------------------------------------------------------
 7# Example host-config file for a cluster on a toss4 platform (e.g. quartz) at LLNL
 8#------------------------------------------------------------------------------
 9#
10# This file provides CMake with paths / details for:
11#  C,C++, & Fortran compilers + MPI
12# 
13#------------------------------------------------------------------------------
14
15#------------------------------------------------------------------------------
16# gcc@10.3.1 compilers
17#------------------------------------------------------------------------------
18
19# _blt_pascal_compiler_config_start
20set(GCC_VERSION "gcc-10.3.1")
21set(GCC_HOME "/usr/tce/packages/gcc/${GCC_VERSION}")
22
23set(CMAKE_C_COMPILER "${GCC_HOME}/bin/gcc" CACHE PATH "")
24set(CMAKE_CXX_COMPILER "${GCC_HOME}/bin/g++" CACHE PATH "")
25
26# Fortran support
27set(ENABLE_FORTRAN ON CACHE BOOL "")
28set(CMAKE_Fortran_COMPILER "${GCC_HOME}/bin/gfortran" CACHE PATH "")
29# _blt_pascal_compiler_config_end
30
31#------------------------------------------------------------------------------
32# MPI Support
33#------------------------------------------------------------------------------
34set(ENABLE_MPI ON CACHE BOOL "")
35
36set(MPI_HOME             "/usr/tce/packages/mvapich2/mvapich2-2.3.6-${GCC_VERSION}" CACHE PATH "")
37
38set(MPI_C_COMPILER       "${MPI_HOME}/bin/mpicc" CACHE PATH "")
39set(MPI_CXX_COMPILER     "${MPI_HOME}/bin/mpicxx" CACHE PATH "")
40set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpif90" CACHE PATH "")
41
42set(MPIEXEC              "/usr/bin/srun" CACHE PATH "")
43set(MPIEXEC_NUMPROC_FLAG "-n" CACHE PATH "")
44
45#------------------------------------------------------------------------------
46# CUDA support
47#------------------------------------------------------------------------------
48set(ENABLE_CUDA ON CACHE BOOL "")
49
50set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-12.2.2" CACHE PATH "")
51set(CMAKE_CUDA_COMPILER "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" CACHE PATH "")
52set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE PATH "")
53
54set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
55set(CMAKE_CUDA_FLAGS "-restrict --expt-extended-lambda -G" CACHE STRING "")
56
57set(CUDA_SEPARABLE_COMPILATION ON CACHE BOOL "" )

More complicated BlueOS host-config that has C, C++, MPI, and CUDA support:

clang@upstream C++17 host-config

 1# Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and
 2# other BLT Project Developers. See the top-level LICENSE file for details
 3# 
 4# SPDX-License-Identifier: (BSD-3-Clause)
 5
 6#------------------------------------------------------------------------------
 7# Example host-config file for the blue_os cluster at LLNL
 8#------------------------------------------------------------------------------
 9#
10# This file provides CMake with paths / details for:
11#  C/C++:   Clang with GCC 8.3.1 toolchain with C++17 support
12#  Cuda
13#  MPI
14# 
15#------------------------------------------------------------------------------
16
17#------------------------------------------------------------------------------
18# Compilers
19#------------------------------------------------------------------------------
20
21set(CLANG_HOME "/usr/tce/packages/clang/clang-ibm-10.0.1-gcc-8.3.1")
22set(CMAKE_C_COMPILER   "${CLANG_HOME}/bin/clang" CACHE PATH "")
23set(CMAKE_CXX_COMPILER "${CLANG_HOME}/bin/clang++" CACHE PATH "")
24set(BLT_CXX_STD "c++17" CACHE STRING "")
25
26# Disable Fortran
27set(ENABLE_FORTRAN OFF CACHE BOOL "")
28
29#------------------------------------------------------------------------------
30# MPI Support
31#------------------------------------------------------------------------------
32set(ENABLE_MPI ON CACHE BOOL "")
33
34set(MPI_HOME               "/usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-10.0.1-gcc-8.3.1")
35set(MPI_C_COMPILER         "${MPI_HOME}/bin/mpicc"   CACHE PATH "")
36set(MPI_CXX_COMPILER       "${MPI_HOME}/bin/mpicxx"  CACHE PATH "")
37
38set(MPIEXEC                "${MPI_HOME}/bin/mpirun"  CACHE PATH "")
39set(MPIEXEC_NUMPROC_FLAG   "-np"     CACHE PATH "")
40set(BLT_MPI_COMMAND_APPEND "mpibind" CACHE PATH "")
41
42#------------------------------------------------------------------------------
43# CUDA support
44#------------------------------------------------------------------------------
45set(ENABLE_CUDA ON CACHE BOOL "")
46
47set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.2.0" CACHE PATH "")
48set(CMAKE_CUDA_COMPILER "${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc" CACHE PATH "")
49set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE PATH "")
50
51set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
52set(CMAKE_CUDA_FLAGS "-restrict --expt-extended-lambda -G" CACHE STRING "")
53
54set(CUDA_SEPARABLE_COMPILATION OFF CACHE BOOL "" )
55
56# nvcc does not like gtest's 'pthreads' flag
57set(gtest_disable_pthreads ON CACHE BOOL "")
58
59# Very specific fix for working around CMake adding implicit link directories returned by the BlueOS
60# compilers to link CUDA executables 
61set(BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE "/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/lib64" CACHE STRING "")

Here is a full example host-config file for an OSX laptop, using a set of dependencies built with Spack:

OSX clang@7.3.0 host-config

 1# Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and
 2# other BLT Project Developers. See the top-level LICENSE file for details
 3# 
 4# SPDX-License-Identifier: (BSD-3-Clause)
 5
 6########################################################################
 7# host-config for naples
 8########################################################################
 9
10########################################################################
11# Dependencies were built with spack (https://github.com/llnl/spack)
12########################################################################
13# spack install  cmake@3.8.2
14# spack install  mpich 
15# spack install  py-sphinx
16# spack activate py-sphinx
17# spack install  doxygen
18
19
20########################################################################
21# cmake path
22########################################################################
23# /Users/harrison37/Work/blt_tutorial/tpls/spack/opt/spack/darwin-elcapitan-x86_64/clang-7.3.0-apple/cmake-3.8.2-n2i4ijlet37i3jhmjfhzms2wo3b4ybcm/bin/cmake
24
25########################################################################
26# mpi from spack
27########################################################################
28set(ENABLE_MPI ON CACHE PATH "")
29
30set(MPI_BASE_DIR "/Users/harrison37/Work/blt_tutorial/tpls/spack/opt/spack/darwin-elcapitan-x86_64/clang-7.3.0-apple/mpich-3.2-yc7ipshe7e3w4ohtgjtms2agecxruavw/bin" CACHE PATH "")
31
32set(MPI_C_COMPILER   "${MPI_BASE_DIR}/mpicc" CACHE PATH "")
33set(MPI_CXX_COMPILER "${MPI_BASE_DIR}/mpicxx" CACHE PATH "")
34set(MPIEXEC          "${MPI_BASE_DIR}/mpiexec" CACHE PATH "")
35
36########################################################################
37# Cuda Support (standard osx cuda toolkit install)
38########################################################################
39set(ENABLE_CUDA ON CACHE BOOL "")
40
41set(CUDA_TOOLKIT_ROOT_DIR "/Developer/NVIDIA/CUDA-8.0/" CACHE PATH "")
42set(CUDA_BIN_DIR          "/Developer/NVIDIA/CUDA-8.0/bin/" CACHE PATH "")
43
44########################################################################
45# sphinx from spack
46########################################################################
47set(SPHINX_EXECUTABLE "/Users/harrison37/Work/blt_tutorial/tpls/spack/opt/spack/darwin-elcapitan-x86_64/clang-7.3.0-apple/python-2.7.13-jmhznopgz2j5zkmuzjygg5oyxnxtc653/bin/sphinx-build" CACHE PATH "")
48
49########################################################################
50# doxygen from spack
51########################################################################
52set(DOXYGEN_EXECUTABLE "/Users/harrison37/Work/blt_tutorial/tpls/spack/opt/spack/darwin-elcapitan-x86_64/clang-7.3.0-apple/doxygen-1.8.12-mji43fu4hxuu6js5irshpihkwwucn7rv/bin/doxygen" CACHE PATH "")