CMake / G++ reproducible build issue with changing build path

Question

For quite some time I have been investigating with limited success reproducible build issue I have with a software I am packaging for Debian. Reproducibility is affected when build path changes, and result is the below differences (reported using diffoscope).

The code itself is C++, build infrastructure is based upon CMake. It consists of a static library and an executable.

Thanks in advance for any hint that could help address this.

CMakeList file for the static library:

cmake_minimum_required(VERSION 3.14)

set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_BUILD_RPATH_USE_ORIGIN ON)

find_package(BZip2 REQUIRED)
find_package(PkgConfig REQUIRED)
pkg_check_modules(LIBUSB REQUIRED libusb-1.0>=1.0.16)
pkg_check_modules(LIBZIP REQUIRED libzip)
find_package(Threads)

if (STATIC)
set(OPENSSL_USE_STATIC_LIBS TRUE)
endif()

find_package(OpenSSL)

if(OPENSSL_FOUND)
set(UUUSSL "-DUUUSSL")
set(UUUOPENSLL_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR})
endif()

include_directories(${LIBUSB_INCLUDE_DIRS} ${UUUOPENSLL_INCLUDE_DIR} include)


set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g -Wall -Wstrict-aliasing -Wextra ${UUUSSL}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 ${UUUSSL}")

set(SOURCES
    error.cpp
    buffer.cpp
    cmd.cpp
    config.cpp
    notify.cpp
    sdps.cpp
    trans.cpp
    usbhotplug.cpp
    version.cpp
    sdp.cpp
    gitversion.h
    fastboot.cpp
    zip.cpp
    fat.cpp
    tar.cpp
    rominfo.cpp
    http.cpp
    hidreport.cpp
    sparse.cpp
)

set(generated_files_dir "${CMAKE_BINARY_DIR}/libuuu/gen")
set(gitversion_h "${generated_files_dir}/gitversion.h")

add_custom_command(
    OUTPUT gitversion.h
    PRE_BUILD
    COMMAND mkdir -p ${generated_files_dir}
    COMMAND sh -c 'cd ${CMAKE_CURRENT_SOURCE_DIR} && rm -f ${gitversion_h} && ./gen_ver.sh "${gitversion_h}.tmp" && mv -f "${gitversion_h}.tmp" "${gitversion_h}"'

)
include_directories(${generated_files_dir})
add_library( uuc_s STATIC ${SOURCES} )

CMakeList file for the executable:

cmake_minimum_required(VERSION 3.14)

set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_BUILD_RPATH_USE_ORIGIN ON)

find_package(PkgConfig REQUIRED)
pkg_check_modules(LIBUSB REQUIRED libusb-1.0>=1.0.16)
pkg_check_modules(LIBZIP REQUIRED libzip)
pkg_check_modules(LIBZ REQUIRED zlib)
find_package(Threads)

if (STATIC)
set(OPENSSL_USE_STATIC_LIBS TRUE)
endif()

find_package(OpenSSL)

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -O2 -Wl,--build-id=none")

if (STATIC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libstdc++ -static-libgcc")
endif()

set(LSTS
    uuu.lst
    emmc_burn_loader.lst
    emmc_burn_all.lst
    fat_write.lst
    qspi_burn_loader.lst
    sd_burn_loader.lst
    spl_boot.lst
    sd_burn_all.lst
    nand_burn_loader.lst
)

link_directories(${CMAKE_CURRENT_SOURCE_DIR}/libuuu ${LIBUSB_LIBRARY_DIRS} ${LIBZIP_LIBRARY_DIRS} ${LIBZ_LIBRARY_DIRS})

set(CLIST_EXECUTABLE ${CMAKE_CURRENT_SOURCE_DIR}/gen_txt_include.sh)
set(generated_files_dir "${CMAKE_BINARY_DIR}/uuu/gen")

function(preprocess_clst out_var)
    set(result)
    foreach(in_f ${ARGN})
        set(out_f "${generated_files_dir}/${in_f}")
        string(REPLACE ".lst" ".clst" out_f ${out_f})
        add_custom_command(OUTPUT ${out_f}
            PRE_BUILD
                COMMAND mkdir -p ${generated_files_dir}
            COMMAND ${CLIST_EXECUTABLE} ${in_f} ${out_f}
            DEPENDS ${in_f}
            WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
            COMMENT "Creating preprocessed clst file ${out_f}"
            VERBATIM
        )
        list(APPEND result ${out_f})
    endforeach()
    set(${out_var} "${result}" PARENT_SCOPE)
endfunction()

preprocess_clst(CLSTS ${LSTS})

include_directories(${generated_files_dir})

set(SOURCES
    uuu.cpp
    buildincmd.cpp
    autocomplete.cpp
    ${CLSTS}
)

add_executable(uuu ${SOURCES})
target_link_libraries(uuu uuc_s ${OPENSSL_LIBRARIES} ${LIBUSB_LIBRARIES} ${LIBZIP_LIBRARIES} ${LIBZ_LIBRARIES} dl bz2 pthread)

install(TARGETS uuu DESTINATION bin)

Note also that debian/rules set a flag that will trigger the use of -ffile-prefix-map=OLD=NEW

Differences (diffoscope output):

│ │ │ │ ├── ./usr/bin/uuu
│ │ │ │ │ ├── readelf --wide --program-header {}
│ │ │ │ │ │ @@ -4,15 +4,15 @@
│ │ │ │ │ │  There are 11 program headers, starting at offset 64
│ │ │ │ │ │  
│ │ │ │ │ │  Program Headers:
│ │ │ │ │ │    Type           Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
│ │ │ │ │ │    PHDR           0x000040 0x0000000000000040 0x0000000000000040 0x000268 0x000268 R   0x8
│ │ │ │ │ │    INTERP         0x0002a8 0x00000000000002a8 0x00000000000002a8 0x00001c 0x00001c R   0x1
│ │ │ │ │ │        [Requesting program interpreter: /lib64/ld-linux-x86-64.so.2]
│ │ │ │ │ │ -  LOAD           0x000000 0x0000000000000000 0x0000000000000000 0x0097e8 0x0097e8 R   0x1000
│ │ │ │ │ │ +  LOAD           0x000000 0x0000000000000000 0x0000000000000000 0x0097f0 0x0097f0 R   0x1000
│ │ │ │ │ │    LOAD           0x00a000 0x000000000000a000 0x000000000000a000 0x04500d 0x04500d R E 0x1000
│ │ │ │ │ │    LOAD           0x050000 0x0000000000050000 0x0000000000050000 0x01253a 0x01253a R   0x1000
│ │ │ │ │ │    LOAD           0x0627c0 0x00000000000637c0 0x00000000000637c0 0x00383c 0x004020 RW  0x1000
│ │ │ │ │ │    DYNAMIC        0x064780 0x0000000000065780 0x0000000000065780 0x000270 0x000270 RW  0x8
│ │ │ │ │ │    NOTE           0x0002c4 0x00000000000002c4 0x00000000000002c4 0x000020 0x000020 R   0x4
│ │ │ │ │ │    GNU_EH_FRAME   0x054b70 0x0000000000054b70 0x0000000000054b70 0x001944 0x001944 R   0x4
│ │ │ │ │ │    GNU_STACK      0x000000 0x0000000000000000 0x0000000000000000 0x000000 0x000000 RW  0x10
│ │ │ │ │ ├── readelf --wide --sections {}
│ │ │ │ │ │ @@ -3,19 +3,19 @@
│ │ │ │ │ │  Section Headers:
│ │ │ │ │ │    [Nr] Name              Type            Address          Off    Size   ES Flg Lk Inf Al
│ │ │ │ │ │    [ 0]                   NULL            0000000000000000 000000 000000 00      0   0  0
│ │ │ │ │ │    [ 1] .interp           PROGBITS        00000000000002a8 0002a8 00001c 00   A  0   0  1
│ │ │ │ │ │    [ 2] .note.ABI-tag     NOTE            00000000000002c4 0002c4 000020 00   A  0   0  4
│ │ │ │ │ │    [ 3] .gnu.hash         GNU_HASH        00000000000002e8 0002e8 000068 00   A  4   0  8
│ │ │ │ │ │    [ 4] .dynsym           DYNSYM          0000000000000350 000350 0013c8 18   A  5   1  8
│ │ │ │ │ │ -  [ 5] .dynstr           STRTAB          0000000000001718 001718 001891 00   A  0   0  1
│ │ │ │ │ │ -  [ 6] .gnu.version      VERSYM          0000000000002faa 002faa 0001a6 02   A  4   0  2
│ │ │ │ │ │ -  [ 7] .gnu.version_r    VERNEED         0000000000003150 003150 0001a0 00   A  5   6  8
│ │ │ │ │ │ -  [ 8] .rela.dyn         RELA            00000000000032f0 0032f0 005460 18   A  4   0  8
│ │ │ │ │ │ -  [ 9] .rela.plt         RELA            0000000000008750 008750 001098 18  AI  4  23  8
│ │ │ │ │ │ +  [ 5] .dynstr           STRTAB          0000000000001718 001718 001895 00   A  0   0  1
│ │ │ │ │ │ +  [ 6] .gnu.version      VERSYM          0000000000002fae 002fae 0001a6 02   A  4   0  2
│ │ │ │ │ │ +  [ 7] .gnu.version_r    VERNEED         0000000000003158 003158 0001a0 00   A  5   6  8
│ │ │ │ │ │ +  [ 8] .rela.dyn         RELA            00000000000032f8 0032f8 005460 18   A  4   0  8
│ │ │ │ │ │ +  [ 9] .rela.plt         RELA            0000000000008758 008758 001098 18  AI  4  23  8
│ │ │ │ │ │    [10] .init             PROGBITS        000000000000a000 00a000 000017 00  AX  0   0  4
│ │ │ │ │ │    [11] .plt              PROGBITS        000000000000a020 00a020 000b20 10  AX  0   0 16
│ │ │ │ │ │    [12] .plt.got          PROGBITS        000000000000ab40 00ab40 000010 08  AX  0   0  8
│ │ │ │ │ │    [13] .text             PROGBITS        000000000000ab50 00ab50 0444b1 00  AX  0   0 16
│ │ │ │ │ │    [14] .fini             PROGBITS        000000000004f004 04f004 000009 00  AX  0   0  4
│ │ │ │ │ │    [15] .rodata           PROGBITS        0000000000050000 050000 004b70 00   A  0   0 32
│ │ │ │ │ │    [16] .eh_frame_hdr     PROGBITS        0000000000054b70 054b70 001944 00   A  0   0  4
│ │ │ │ │ ├── readelf --wide --relocs {}
│ │ │ │ │ │ @@ -1,9 +1,9 @@
│ │ │ │ │ │  
│ │ │ │ │ │ -Relocation section '.rela.dyn' at offset 0x32f0 contains 900 entries:
│ │ │ │ │ │ +Relocation section '.rela.dyn' at offset 0x32f8 contains 900 entries:
│ │ │ │ │ │      Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
│ │ │ │ │ │  00000000000637c0  0000000000000008 R_X86_64_RELATIVE                         f030
│ │ │ │ │ │  00000000000637c8  0000000000000008 R_X86_64_RELATIVE                         e8d0
│ │ │ │ │ │  00000000000637d0  0000000000000008 R_X86_64_RELATIVE                         e990
│ │ │ │ │ │  00000000000637d8  0000000000000008 R_X86_64_RELATIVE                         e9d0
│ │ │ │ │ │  00000000000637e0  0000000000000008 R_X86_64_RELATIVE                         ea00
│ │ │ │ │ │  00000000000637e8  0000000000000008 R_X86_64_RELATIVE                         ea40
│ │ │ │ │ │ @@ -898,15 +898,15 @@
│ │ │ │ │ │  0000000000064620  000000d100000005 R_X86_64_COPY          0000000000064620 _ZTISt12bad_weak_ptr@GLIBCXX_3.4.15 + 0
│ │ │ │ │ │  0000000000064638  000000d200000005 R_X86_64_COPY          0000000000064638 _ZTVSt12bad_weak_ptr@GLIBCXX_3.4.15 + 0
│ │ │ │ │ │  0000000000067000  000000d000000005 R_X86_64_COPY          0000000000067000 _ZSt4cout@GLIBCXX_3.4 + 0
│ │ │ │ │ │  0000000000067120  000000cc00000005 R_X86_64_COPY          0000000000067120 stderr@GLIBC_2.2.5 + 0
│ │ │ │ │ │  0000000000067140  000000c800000005 R_X86_64_COPY          0000000000067140 _ZSt3cin@GLIBCXX_3.4 + 0
│ │ │ │ │ │  0000000000067260  000000c900000005 R_X86_64_COPY          0000000000067260 _ZSt4cerr@GLIBCXX_3.4 + 0
│ │ │ │ │ │  
│ │ │ │ │ │ -Relocation section '.rela.plt' at offset 0x8750 contains 177 entries:
│ │ │ │ │ │ +Relocation section '.rela.plt' at offset 0x8758 contains 177 entries:
│ │ │ │ │ │      Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
│ │ │ │ │ │  0000000000065a08  0000000100000007 R_X86_64_JUMP_SLOT     0000000000000000 libusb_free_config_descriptor + 0
│ │ │ │ │ │  0000000000065a10  0000000200000007 R_X86_64_JUMP_SLOT     0000000000000000 __printf_chk@GLIBC_2.3.4 + 0
│ │ │ │ │ │  0000000000065a18  0000000300000007 R_X86_64_JUMP_SLOT     0000000000000000 _ZNSo3putEc@GLIBCXX_3.4 + 0
│ │ │ │ │ │  0000000000065a20  0000000400000007 R_X86_64_JUMP_SLOT     0000000000000000 __errno_location@GLIBC_2.2.5 + 0
│ │ │ │ │ │  0000000000065a28  0000000500000007 R_X86_64_JUMP_SLOT     0000000000000000 BZ2_bzDecompress + 0
│ │ │ │ │ │  0000000000065a30  0000000700000007 R_X86_64_JUMP_SLOT     0000000000000000 _ZSt29_Rb_tree_insert_and_rebalancebPSt18_Rb_tree_node_baseS0_RS_@GLIBCXX_3.4 + 0
│ │ │ │ │ ├── readelf --wide --dynamic {}
│ │ │ │ │ │ @@ -14,24 +14,24 @@
│ │ │ │ │ │   0x0000000000000019 (INIT_ARRAY)         0x637c0
│ │ │ │ │ │   0x000000000000001b (INIT_ARRAYSZ)       120 (bytes)
│ │ │ │ │ │   0x000000000000001a (FINI_ARRAY)         0x63838
│ │ │ │ │ │   0x000000000000001c (FINI_ARRAYSZ)       8 (bytes)
│ │ │ │ │ │   0x000000006ffffef5 (GNU_HASH)           0x2e8
│ │ │ │ │ │   0x0000000000000005 (STRTAB)             0x1718
│ │ │ │ │ │   0x0000000000000006 (SYMTAB)             0x350
│ │ │ │ │ │ - 0x000000000000000a (STRSZ)              6289 (bytes)
│ │ │ │ │ │ + 0x000000000000000a (STRSZ)              6293 (bytes)
│ │ │ │ │ │   0x000000000000000b (SYMENT)             24 (bytes)
│ │ │ │ │ │   0x0000000000000015 (DEBUG)              0x0
│ │ │ │ │ │   0x0000000000000003 (PLTGOT)             0x659f0
│ │ │ │ │ │   0x0000000000000002 (PLTRELSZ)           4248 (bytes)
│ │ │ │ │ │   0x0000000000000014 (PLTREL)             RELA
│ │ │ │ │ │ - 0x0000000000000017 (JMPREL)             0x8750
│ │ │ │ │ │ - 0x0000000000000007 (RELA)               0x32f0
│ │ │ │ │ │ + 0x0000000000000017 (JMPREL)             0x8758
│ │ │ │ │ │ + 0x0000000000000007 (RELA)               0x32f8
│ │ │ │ │ │   0x0000000000000008 (RELASZ)             21600 (bytes)
│ │ │ │ │ │   0x0000000000000009 (RELAENT)            24 (bytes)
│ │ │ │ │ │   0x000000000000001e (FLAGS)              BIND_NOW
│ │ │ │ │ │   0x000000006ffffffb (FLAGS_1)            Flags: NOW PIE
│ │ │ │ │ │ - 0x000000006ffffffe (VERNEED)            0x3150
│ │ │ │ │ │ + 0x000000006ffffffe (VERNEED)            0x3158
│ │ │ │ │ │   0x000000006fffffff (VERNEEDNUM)         6
│ │ │ │ │ │ - 0x000000006ffffff0 (VERSYM)             0x2faa
│ │ │ │ │ │ + 0x000000006ffffff0 (VERSYM)             0x2fae
│ │ │ │ │ │   0x000000006ffffff9 (RELACOUNT)          766
│ │ │ │ │ │   0x0000000000000000 (NULL)               0x0
│ │ │ │ │ ├── readelf --wide --version-info {}
│ │ │ │ │ │ @@ -1,10 +1,10 @@
│ │ │ │ │ │  
│ │ │ │ │ │  Version symbols section '.gnu.version' contains 211 entries:
│ │ │ │ │ │ - Addr: 0x0000000000002faa  Offset: 0x002faa  Link: 4 (.dynsym)
│ │ │ │ │ │ + Addr: 0x0000000000002fae  Offset: 0x002fae  Link: 4 (.dynsym)
│ │ │ │ │ │    000:   0 (*local*)       0 (*local*)       2 (GLIBC_2.3.4)   3 (GLIBCXX_3.4)
│ │ │ │ │ │    004:   4 (GLIBC_2.2.5)   0 (*local*)       5 (GLIBCXX_3.4.22)    3 (GLIBCXX_3.4)
│ │ │ │ │ │    008:   7 (OPENSSL_1_1_0)   8 (CXXABI_1.3.5)   0 (*local*)       3 (GLIBCXX_3.4)
│ │ │ │ │ │    00c:   6 (GLIBC_2.2.5)   3 (GLIBCXX_3.4)   3 (GLIBCXX_3.4)   9 (GLIBCXX_3.4.21) 
│ │ │ │ │ │    010:   3 (GLIBCXX_3.4)   6 (GLIBC_2.2.5)   9 (GLIBCXX_3.4.21)    6 (GLIBC_2.2.5)
│ │ │ │ │ │    014:   9 (GLIBCXX_3.4.21)    6 (GLIBC_2.2.5)   4 (GLIBC_2.2.5)   3 (GLIBCXX_3.4)
│ │ │ │ │ │    018:   0 (*local*)       3 (GLIBCXX_3.4)   0 (*local*)       7 (OPENSSL_1_1_0)
│ │ │ │ │ │ @@ -52,15 +52,15 @@
│ │ │ │ │ │    0c0:   9 (GLIBCXX_3.4.21)    7 (OPENSSL_1_1_0)   3 (GLIBCXX_3.4)   3 (GLIBCXX_3.4)
│ │ │ │ │ │    0c4:   6 (GLIBC_2.2.5)   6 (GLIBC_2.2.5)   3 (GLIBCXX_3.4)   3 (GLIBCXX_3.4)
│ │ │ │ │ │    0c8:   3 (GLIBCXX_3.4)   3 (GLIBCXX_3.4)   3 (GLIBCXX_3.4)   1 (*global*)   
│ │ │ │ │ │    0cc:   6 (GLIBC_2.2.5)   3 (GLIBCXX_3.4)   3 (GLIBCXX_3.4)   1 (*global*)   
│ │ │ │ │ │    0d0:   3 (GLIBCXX_3.4)  14 (GLIBCXX_3.4.15)   14 (GLIBCXX_3.4.15) 
│ │ │ │ │ │  
│ │ │ │ │ │  Version needs section '.gnu.version_r' contains 6 entries:
│ │ │ │ │ │ - Addr: 0x0000000000003150  Offset: 0x003150  Link: 5 (.dynstr)
│ │ │ │ │ │ + Addr: 0x0000000000003158  Offset: 0x003158  Link: 5 (.dynstr)
│ │ │ │ │ │    000000: Version: 1  File: libgcc_s.so.1  Cnt: 1
│ │ │ │ │ │    0x0010:   Name: GCC_3.0  Flags: none  Version: 21
│ │ │ │ │ │    0x0020: Version: 1  File: libz.so.1  Cnt: 1
│ │ │ │ │ │    0x0030:   Name: ZLIB_1.2.3.5  Flags: none  Version: 11
│ │ │ │ │ │    0x0040: Version: 1  File: libssl.so.1.1  Cnt: 1
│ │ │ │ │ │    0x0050:   Name: OPENSSL_1_1_0  Flags: none  Version: 7
│ │ │ │ │ │    0x0060: Version: 1  File: libpthread.so.0  Cnt: 1
│ │ │ │ │ ├── readelf --wide --decompress --hex-dump=.dynstr {}
│ │ │ │ │ │ @@ -389,9 +389,9 @@
│ │ │ │ │ │    0x00002f38 474c4942 4358585f 332e3400 474c4942 GLIBCXX_3.4.GLIB
│ │ │ │ │ │    0x00002f48 435f322e 3400474c 4942435f 322e3134 C_2.4.GLIBC_2.14
│ │ │ │ │ │    0x00002f58 00474c49 42435f32 2e332e34 00000000 .GLIBC_2.3.4....
│ │ │ │ │ │    0x00002f68 00000000 00000000 00000000 00000000 ................
│ │ │ │ │ │    0x00002f78 00000000 00000000 00000000 00000000 ................
│ │ │ │ │ │    0x00002f88 00000000 00000000 00000000 00000000 ................
│ │ │ │ │ │    0x00002f98 00000000 00000000 00000000 00000000 ................
│ │ │ │ │ │ -  0x00002fa8 00                                  .
│ │ │ │ │ │ +  0x00002fa8 00000000 00                         .....
section_end:1597588268:step_script

In case parts of the code could be relevant, upstream software is here: https://github.com/NXPmicro/mfgtools/ — Nicolas, Aug 16 '20 at 14:44
There can be many reasons, and just because binary object modules compiled from different directories are not binary-identical does not mean anything. Debug data is often included in the object modules, and includes absolute pathnames. Or the build system might insert a string containing the current directory, for whatever reason, into an object module. — Sam Varshavchik, Aug 16 '20 at 14:55
@SamVarshavchik - I realize there might be plenty of reasons why there are differences. I mostly wondered if I am missing some flags in my CMakelist files which could help with reproducability. I should have made that clearer from my original post, sorry about that. — Nicolas, Aug 16 '20 at 15:02
Also - note that debug data is going into another package (dbgsym) and not this one where there are those differences. — Nicolas, Aug 16 '20 at 15:11
In the context of "creating a reproducible build", the "reproducible build" does not necessarily mean "bit-by-bit identical". It generally means: "the code can be built from source, using a canned script, with a reasonable level of confidence that the compiled code is correct". — Sam Varshavchik, Aug 16 '20 at 15:26
There is no sufficient information in the question post to reproduce your problem, so we cannot help you. And providing whole required information would just make your question "too broad". There are several ways for you to debug your problem: 1. run `make VERBOSE=1` and check actual flags passed to the compiler and linker (and other tools). 2. Examine (and compare) intermediate files for discover where the difference began. 3. Reduce a set of the files and a number of code (C++) lines and check whether the problem persist. — Tsyvarev, Aug 16 '20 at 17:22

CMake / G++ reproducible build issue with changing build path

0 Answers0