Host-configs

To capture (and revision control) build options, third party library paths, etc., we recommend using CMake’s initial-cache file mechanism. This feature allows you to pass a file to CMake that provides variables to bootstrap the configuration process.

You can pass initial-cache files to cmake via the -C command line option.

cmake -C config_file.cmake

We call these initial-cache files host-config files since we typically create a file for each platform or for specific hosts, if necessary.

These files use standard CMake commands. CMake set() commands need to specify CACHE as follows:

set(CMAKE_VARIABLE_NAME {VALUE} CACHE PATH "")

Here is a snippet from a host-config file that specifies compiler details for using specific gcc (version 10.3.1 in this case) on the LLNL Matrix cluster:

set(CMAKE_C_COMPILER "/usr/tce/packages/clang/clang-19.1.3-magic/bin/clang" CACHE PATH "")
set(CMAKE_CXX_COMPILER "/usr/tce/packages/clang/clang-19.1.3-magic/bin/clang++" CACHE PATH "")
set(CMAKE_Fortran_COMPILER "/usr/tce/packages/gcc/gcc-13.3.1/bin/gfortran" CACHE PATH "")

set(CMAKE_C_FLAGS "-fPIC" CACHE STRING "")
set(CMAKE_CXX_FLAGS "-fPIC" CACHE STRING "")
set(CMAKE_Fortran_FLAGS "-fPIC" CACHE STRING "")

set(ENABLE_FORTRAN ON CACHE BOOL "")

set(BLT_EXE_LINKER_FLAGS " -Xlinker -rpath -Xlinker /usr/tce/packages/clang/clang-19.1.3-magic/lib" CACHE STRING "Adds a missing libstdc++ rpath")

Building and Testing on Matrix

Since compute nodes on the Matrix cluster have CPUs and GPUs, here is how you can use the host-config file to configure a build of the calc_pi project with MPI and CUDA enabled on Matrix:

# create build dir
mkdir build
cd build
# configure using host-config
cmake -C ../../host-configs/llnl/toss_4_x86_64_ib/llvm@19.1.3_nvcc.cmake  ..

After building (make), you can run make test on a batch node (where the GPUs reside) to run the unit tests that are using MPI and CUDA:

bash-4.1$ salloc -A <valid bank>
bash-4.1$ make
bash-4.1$ make test

Running tests...
Test project blt/docs/tutorial/calc_pi/build
    Start 1: test_1
1/8 Test #1: test_1 ...........................   Passed    0.01 sec
    Start 2: test_2
2/8 Test #2: test_2 ...........................   Passed    2.79 sec
    Start 3: test_3
3/8 Test #3: test_3 ...........................   Passed    0.54 sec
    Start 4: blt_gtest_smoke
4/8 Test #4: blt_gtest_smoke ..................   Passed    0.01 sec
    Start 5: blt_fruit_smoke
5/8 Test #5: blt_fruit_smoke ..................   Passed    0.01 sec
    Start 6: blt_mpi_smoke
6/8 Test #6: blt_mpi_smoke ....................   Passed    2.82 sec
    Start 7: blt_cuda_smoke
7/8 Test #7: blt_cuda_smoke ...................   Passed    0.48 sec
    Start 8: blt_cuda_runtime_smoke
8/8 Test #8: blt_cuda_runtime_smoke ...........   Passed    0.11 sec

100% tests passed, 0 tests failed out of 8

Total Test time (real) =   6.80 sec

Example Host-configs

Basic TOSS4 (for example: Dane) host-config that has C, C++, and Fortran Compilers along with MPI support:

gcc@10.3.1 host-config

 1# Copyright (c) 2017-2025, Lawrence Livermore National Security, LLC and
 2# other BLT Project Developers. See the top-level LICENSE file for details
 3# 
 4# SPDX-License-Identifier: (BSD-3-Clause)
 5
 6#------------------------------------------------------------------------------
 7# Example host-config file for a cluster on a toss4 platform (e.g. dane) at LLNL
 8#------------------------------------------------------------------------------
 9#
10# This file provides CMake with paths / details for:
11#  C,C++, & Fortran compilers + MPI
12# 
13#------------------------------------------------------------------------------
14
15#------------------------------------------------------------------------------
16# gcc@10.3.1 compilers
17#------------------------------------------------------------------------------
18
19# _blt_tutorial_compiler_config_start
20set(GCC_VERSION "gcc-10.3.1")
21set(GCC_HOME "/usr/tce/packages/gcc/${GCC_VERSION}")
22
23set(CMAKE_C_COMPILER "${GCC_HOME}/bin/gcc" CACHE PATH "")
24
25set(CMAKE_CXX_COMPILER "${GCC_HOME}/bin/g++" CACHE PATH "")
26
27# Fortran support
28set(ENABLE_FORTRAN ON CACHE BOOL "")
29set(CMAKE_Fortran_COMPILER "${GCC_HOME}/bin/gfortran" CACHE PATH "")
30#_blt_tutorial_compiler_config_end
31
32#------------------------------------------------------------------------------
33# MPI Support
34#------------------------------------------------------------------------------
35# _blt_tutorial_mpi_config_start
36set(ENABLE_MPI ON CACHE BOOL "")
37
38set(MPI_HOME             "/usr/tce/packages/mvapich2/mvapich2-2.3.6-${GCC_VERSION}" CACHE PATH "")
39
40set(MPI_C_COMPILER       "${MPI_HOME}/bin/mpicc" CACHE PATH "")
41set(MPI_CXX_COMPILER     "${MPI_HOME}/bin/mpicxx" CACHE PATH "")
42set(MPI_Fortran_COMPILER "${MPI_HOME}/bin/mpif90" CACHE PATH "")
43
44set(MPIEXEC              "/usr/bin/srun" CACHE PATH "")
45set(MPIEXEC_NUMPROC_FLAG "-n" CACHE PATH "")
46# _blt_tutorial_mpi_config_end

More complicated host-config for LLNL’s Matrix that has C, C++, MPI, and CUDA support:

llvm@19.3.1_nvcc host-config

 1# Copyright (c) 2017-2025, Lawrence Livermore National Security, LLC and
 2# other BLT Project Developers. See the top-level LICENSE file for details
 3# 
 4# SPDX-License-Identifier: (BSD-3-Clause)
 5
 6#------------------------------------------------------------------------------
 7# Example host-config file for a cluster on a TOSS4 NVidia platform (e.g. matrix) at LLNL
 8#------------------------------------------------------------------------------
 9#
10# This file provides CMake with paths / details for:
11#  C,C++, & Fortran compilers + MPI + CUDA + OpenMP
12# 
13#------------------------------------------------------------------------------
14
15#------------------------------------------------------------------------------
16# Compilers
17#------------------------------------------------------------------------------
18# _blt_matrix_compiler_config_start
19set(CMAKE_C_COMPILER "/usr/tce/packages/clang/clang-19.1.3-magic/bin/clang" CACHE PATH "")
20set(CMAKE_CXX_COMPILER "/usr/tce/packages/clang/clang-19.1.3-magic/bin/clang++" CACHE PATH "")
21set(CMAKE_Fortran_COMPILER "/usr/tce/packages/gcc/gcc-13.3.1/bin/gfortran" CACHE PATH "")
22
23set(CMAKE_C_FLAGS "-fPIC" CACHE STRING "")
24set(CMAKE_CXX_FLAGS "-fPIC" CACHE STRING "")
25set(CMAKE_Fortran_FLAGS "-fPIC" CACHE STRING "")
26
27set(ENABLE_FORTRAN ON CACHE BOOL "")
28
29set(BLT_EXE_LINKER_FLAGS " -Xlinker -rpath -Xlinker /usr/tce/packages/clang/clang-19.1.3-magic/lib" CACHE STRING "Adds a missing libstdc++ rpath")
30# _blt_matrix_compiler_config_end
31
32#------------------------------------------------------------------------------
33# MPI
34#------------------------------------------------------------------------------
35
36set(MPI_C_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-19.1.3/bin/mpicc" CACHE PATH "")
37
38set(MPI_CXX_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-19.1.3/bin/mpicxx" CACHE PATH "")
39
40set(MPI_Fortran_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-19.1.3/bin/mpif90" CACHE PATH "")
41
42set(MPIEXEC_EXECUTABLE "/usr/bin/srun" CACHE PATH "")
43
44set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "")
45
46set(ENABLE_MPI ON CACHE BOOL "")
47
48#------------------------------------------------
49# Cuda
50#------------------------------------------------
51
52#_blt_tutorial_cuda_config_start
53set(ENABLE_CUDA ON CACHE BOOL "")
54
55set(CUDAToolkit_ROOT "/usr/tce/packages/cuda/cuda-13.1.1" CACHE PATH "")
56
57set(CMAKE_CUDA_COMPILER "${CUDAToolkit_ROOT}/bin/nvcc" CACHE PATH "")
58
59set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE PATH "")
60
61set(CMAKE_CUDA_ARCHITECTURES "90" CACHE STRING "")
62
63set(CMAKE_CUDA_SEPARABLE_COMPILATION ON CACHE BOOL "")
64
65set(CMAKE_CUDA_FLAGS "-Xcompiler=-fPIC" CACHE STRING "")
66
67# nvcc does not like gtest's 'pthreads' flag
68set(gtest_disable_pthreads ON CACHE BOOL "")
69#_blt_tutorial_cuda_config_end
70
71#------------------------------------------------
72# OpenMP
73#------------------------------------------------
74
75set(ENABLE_OPENMP ON CACHE BOOL "")