Using GLSLC's depfile to make included files automatically trigger recompile of SPIRV in CMake

Question

I'm trying to create a cmake function that automatically recompiles glsl to spirv upon changes to the shader files. Right now direct dependencies work, ie the shaders I use as compile arguments. However I make heavy use of #include feature that glslc provides, and by default I can't get changes in that stuff to trigger recompile. I made sure that I'm using the Ninja

Right now I have the following CMake function and arguments:

cmake -DCMAKE_BUILD_TYPE=Debug "-DCMAKE_MAKE_PROGRAM=JETBRAINSPATH/bin/ninja/win/ninja.exe" -G Ninja  "PATH_TO_CURRENT_DIRECTORY"

function

set(GLSLC "$ENV{VULKAN_SDK}/Bin/glslc")

function(target_shader_function SHADER_TARGET)
    foreach (SHADER_SOURCE_FILEPATH ${ARGN})
        get_filename_component(SHADER_SOURCE_FILENAME ${SHADER_SOURCE_FILEPATH} NAME)
        get_filename_component(SHADER_SOURCE_DIRECTORY ${SHADER_SOURCE_FILEPATH} DIRECTORY)
        set(SHADER_TARGET_NAME "${SHADER_TARGET}_${SHADER_SOURCE_FILENAME}")
        set(SHADER_BINARY_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/spirv")
        set(SHADER_FINAL_BINARY_FILEPATH "${SHADER_BINARY_DIRECTORY}/${SHADER_SOURCE_FILENAME}.spv")
        #we can use depfiles instead
        #https://stackoverflow.com/questions/60420700/cmake-invocation-of-glslc-with-respect-to-includes-dependencies
        add_custom_command(
                OUTPUT ${SHADER_FINAL_BINARY_FILEPATH}
                DEPENDS ${SHADER_SOURCE_FILEPATH}
                DEPFILE ${SHADER_SOURCE_FILEPATH}.d
                COMMAND ${CMAKE_COMMAND} -E make_directory ${SHADER_BINARY_DIRECTORY}
                COMMAND ${GLSLC} -MD -MF ${SHADER_SOURCE_FILEPATH}.d -O ${SHADER_SOURCE_FILEPATH} -o ${SHADER_FINAL_BINARY_FILEPATH} --target-env=vulkan1.2 -I ${CMAKE_SOURCE_DIR}/shaderutils
                DEPENDS ${SHADER_SOURCE_FILEPATH}
                #                BYPRODUCTS ${SHADER_FINAL_BINARY_FILEPATH} ${SHADER_SOURCE_FILEPATH}.d causes ninja to no longer work
                COMMENT "Compiling SPIRV for \nsource: \n\t${SHADER_SOURCE_FILEPATH} \nbinary: \n\t${SHADER_FINAL_BINARY_FILEPATH} \n"
        )

        add_custom_target(${SHADER_TARGET_NAME} DEPENDS ${SHADER_FINAL_BINARY_FILEPATH} ${SHADER_SOURCE_FILEPATH}.d)
        add_dependencies(${SHADER_TARGET} ${SHADER_TARGET_NAME})
    endforeach (SHADER_SOURCE_FILEPATH)
endfunction()

and I use it like this:

cmake_minimum_required(VERSION 3.21)
cmake_policy(SET CMP0116 NEW)
project(my_workspace)
add_executable(my_target main.cpp)
...
target_shader_function(my_target
        ${CMAKE_CURRENT_SOURCE_DIR}/shaders/example.comp
        )

main.cpp

#include <iostream>

int main(){
std::cout << "hello world!" << std::endl;
return 0; 
}

Again, everything works fine if I change, for example, example.comp.

However, lets say I have the following shader (lets say that this is example.comp):

#version 460
#include "fooutils.glsl"
#define WORKGROUP_SIZE 1024
layout (local_size_x = WORKGROUP_SIZE, local_size_y = 1, local_size_z = 1) in;
layout(set = 0, binding = 0) buffer MyBufferBlock{
    float data[];
}
void main(){
   uint tidx = gl_GlobalInvocationID.x;
   data[tidx] += foo(tidx); 
}

and I include the following:

#ifndef FOOUTILS_GLSL
#define FOOUTILS_GLSL

float foo(uint tidx){
    return mod(tidx, 4.51); 
}

#endif //FOOUTILS_GLSL

and I change fooutils.glsl after everything is compiled once (for example in a way that stops it from compiling),

#ifndef FOOUTILS_GLSL
#define FOOUTILS_GLSL

float foo(uint tidx){
    return x; 
    return mod(tidx, 4.51); 
}

#endif //FOOUTILS_GLSL

I don't get a recompile triggered. I had assumed that ninja would use this info to accomplish this, but I haven't seen it happen.

How do I use this depfile to force a recompile when an include dependency changes?

Answer 1

Here's my working implementation. But first, here's my terminal output so you can see it's working:

$ tree
.
├── CMakeLists.txt
├── main.cpp
├── shaders
│   └── example.comp
└── shaderutils
    └── fooutils.glsl
$ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
...
$ cmake --build build/
[1/3] Compiling SPIRV: shaders/example.comp -> spirv/example.spv
[2/3] Building CXX object CMakeFiles/my_target.dir/main.cpp.o
[3/3] Linking CXX executable my_target
$ cmake --build build/
ninja: no work to do.
$ touch shaderutils/fooutils.glsl 
$ cmake --build build/
[1/1] Compiling SPIRV: shaders/example.comp -> spirv/example.spv
$ cat build/spirv/example.d 
spirv/example.spv: /path/to/shaders/example.comp /path/to/shaderutils/fooutils.glsl
$ cat build/CMakeFiles/d/*.d 
spirv/example.spv: \
  ../shaders/example.comp \
  ../shaderutils/fooutils.glsl

Now on to the implementation

cmake_minimum_required(VERSION 3.22)
project(test)

function(target_shader_function TARGET)
    find_package(Vulkan REQUIRED)

    if (NOT TARGET Vulkan::glslc)
        message(FATAL_ERROR "Could not find glslc")
    endif ()

    foreach (source IN LISTS ARGN)
        cmake_path(ABSOLUTE_PATH source OUTPUT_VARIABLE source_abs)
        cmake_path(GET source STEM basename)

        set(depfile "spirv/${basename}.d")
        set(output "spirv/${basename}.spv")
        set(dirs "$<TARGET_PROPERTY:${TARGET},INCLUDE_DIRECTORIES>")
        set(include_flags "$<$<BOOL:${dirs}>:-I$<JOIN:${dirs},;-I>>")

        add_custom_command(
            OUTPUT "${output}"
            COMMAND "${CMAKE_COMMAND}" -E make_directory spirv
            COMMAND Vulkan::glslc -MD -MF "${depfile}" -O "${source_abs}"
                    -o "${output}" --target-env=vulkan1.2 "${include_flags}"
            DEPENDS "${source_abs}"
            BYPRODUCTS "${depfile}"
            COMMENT "Compiling SPIRV: ${source} -> ${output}"
            DEPFILE "${depfile}"
            VERBATIM
            COMMAND_EXPAND_LISTS
        )

        set(shader_target "${TARGET}_${basename}")
        add_custom_target("${shader_target}"
                          DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${output}")
        add_dependencies("${TARGET}" "${shader_target}")
    endforeach ()
endfunction()

add_executable(my_target main.cpp)
target_shader_function(my_target shaders/example.comp)
target_include_directories(
    my_target PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/shaderutils")

With a CMake minimum version of 3.20 or greater, CMP0116 will be set, which adjusts depfiles that were generated with relative paths to be relative to the top-level binary directory. You can see this in action in the last two command outputs.

For compatibility with this policy, the command to invoke glslc is careful to use only absolute paths or paths relative to ${CMAKE_CURRENT_BINARY_DIR}.

To increase the reusability of this function, I had it reuse the include paths from the TARGET rather than hard-coding shaderutils.

Also remember to always pass absolute paths to the DEPENDS arguments of add_custom_{command,target} to avoid surprising path resolution behaviors.

Finally, since CMake actually comes with a FindVulkan module that can locate glslc, we use that to get the Vulkan::glslc target. Per the documentation, it can be overridden by setting Vulkan_GLSLC_EXECUTABLE.

---

Terminal logs for VS2022 on Windows with MSVC:

> cmake -S . -B build
...
> cmake --build build --config Release
  Checking Build System
  Compiling SPIRV: shaders/example.comp -> spirv/example.spv
  Building Custom Rule D:/test/CMakeLists.txt
  Building Custom Rule D:/test/CMakeLists.txt
  main.cpp
  my_target.vcxproj -> D:\test\build\Release\my_target.exe
  Building Custom Rule D:/test/CMakeLists.txt

> cmake --build build --config Release -- -noLogo
  my_target.vcxproj -> D:\test\build\Release\my_target.exe

> notepad shaderutils\fooutils.glsl
> cmake --build build --config Release -- -noLogo
  Compiling SPIRV: shaders/example.comp -> spirv/example.spv
  my_target.vcxproj -> D:\test\build\Release\my_target.exe

> cmake --build build --config Release -- -noLogo
  my_target.vcxproj -> D:\test\build\Release\my_target.exe

and again with Ninja instead of msbuild:

> cmake -G Ninja -S . -B build -DCMAKE_BUILD_TYPE=Release ^
        -DVulkan_ROOT=C:/VulkanSDK/1.2.198.1
...
> powershell "cmake --build build | tee output.txt"
[1/3] Compiling SPIRV: shaders/example.comp -> spirv/example.spv
[2/3] Building CXX object CMakeFiles\my_target.dir\main.cpp.obj
[3/3] Linking CXX executable my_target.exe

> powershell "cmake --build build | tee output.txt"
ninja: no work to do.

> notepad shaderutils\fooutils.glsl
> powershell "cmake --build build | tee output.txt"
[1/1] Compiling SPIRV: shaders/example.comp -> spirv/example.spv

The little powershell + tee trick is just to keep the Ninja command log from overwriting itself. I could use --verbose, but then the full command lines would be printed, rather than the tidy summaries.