How does objcopy compute what sections for the elf file to insert into the output file?

Question

let's say I run arm-none-eabi-objcopy firmwared.elf -O ihex firmware.hex

Assume the binary was generated with the following linker script:

ENTRY(Reset_Handler)

MEMORY
{
  FLASH (RX) : ORIGIN = 0x08020000, LENGTH = 896K
  SRAM (RWX) : ORIGIN = 0x20000000, LENGTH = 512K
  BKPSRAM (RW) : ORIGIN = 0x40024000, LENGTH = 4K
}

_estack = 0x20080000;

SECTIONS
{
  .isr_vector :
  {
    . = ALIGN(4);
    _isr_vector = .;
    KEEP(*(.isr_vector))
    . = ALIGN(4);
  } > FLASH

  .firmware_header_vector :
  {
    . = ALIGN(4);
    KEEP(*(.firmware_header_vector))
    . = ALIGN(4);
  } > FLASH

  .text :
  {
    . = ALIGN(4);
    _stext = .;
    *(.Reset_Handler)
    *(.text)
    *(.text*)
    *(.rodata)
    *(.rodata*)
    *(.glue_7)
    *(.glue_7t)
    KEEP(*(.init))
    KEEP(*(.fini))
    . = ALIGN(4);
    _etext = .;

  } > FLASH

  .ARM.extab :
  {
    . = ALIGN(4);
    *(.ARM.extab)
    *(.gnu.linkonce.armextab.*)
    . = ALIGN(4);
  } > FLASH

  .exidx :
  {
    . = ALIGN(4);
    PROVIDE(__exidx_start = .);
    *(.ARM.exidx*)
    . = ALIGN(4);
    PROVIDE(__exidx_end = .);
  } > FLASH

  .preinit_array :
  {
    PROVIDE(__preinit_array_start = .);
    KEEP(*(.preinit_array*))
    PROVIDE(__preinit_array_end = .);
  } > FLASH

  .init_array :
  {
    PROVIDE(__init_array_start = .);
    KEEP(*(SORT(.init_array.*)))
    KEEP(*(.init_array*))
    PROVIDE(__init_array_end = .);
  } > FLASH

  .fini_array :
  {
    PROVIDE(__fini_array_start = .);
    KEEP(*(.fini_array*))
    KEEP(*(SORT(.fini_array.*)))
    PROVIDE(__fini_array_end = .);
  } > FLASH

  _sidata = .;
  .data_x : AT(_sidata) /* LMA address is _sidata (in FLASH) */
  {
    . = ALIGN(4);
    _sdata = .; /* data section VMA address */
    *(.data*)
    . = ALIGN(4);
    _edata = .;
  } > SRAM

  .firmware_header (_sidata + SIZEOF(.data_x)):
  {
    . = ALIGN(4);
    KEEP(*(.firmware_header))
    . = ALIGN(4);
  } > FLASH

  .eth (NOLOAD) :
  {
    . = ALIGN(4);
    KEEP(*(.RxDecripSection))
    KEEP(*(.TxDescripSection))
    KEEP(*(.RxarraySection))
    KEEP(*(.TxarraySection))
    . = ALIGN(4);
  } > SRAM

  .bss :
  {
    . = ALIGN(4);
    _sbss = .;

    PROVIDE(__bss_start__ = _sbss);
    *(.bss)
    *(.bss*)
    *(COMMON)
    . = ALIGN(4);
    _ebss = .;

    PROVIDE(__bss_end__ = _ebss);
  } > SRAM

  PROVIDE(end = .);

  .heap (NOLOAD) :
  {
    . = ALIGN(4);
    PROVIDE(__heap_start__ = .);
    KEEP(*(.heap))
    . = ALIGN(4);
    PROVIDE(__heap_end__ = .);
  } > SRAM

  .reserved_for_stack (NOLOAD) :
  {
    . = ALIGN(4);
    PROVIDE(__reserved_for_stack_start__ = .);
    KEEP(*(.reserved_for_stack))
    . = ALIGN(4);
    PROVIDE(__reserved_for_stack_end__ = .);
  } > SRAM

  .battery_backed_sram (NOLOAD) :
  {
    . = ALIGN(4);
    KEEP(*(.battery_backed_sram))
    . = ALIGN(4);
  } > BKPSRAM

  /DISCARD/ :
  {
    *(.ARM.attributes)
  }
}

This results in a readelf output of:

Section Headers:
  [Nr] Name              Type            Addr     Off    Size   ES Flg Lk Inf Al
  [ 0]                   NULL            00000000 000000 000000 00      0   0  0
  [ 1] .isr_vector       PROGBITS        08020000 010000 0001f8 00  WA  0   0  4
  [ 2] .firmware_header_ PROGBITS        080201f8 0101f8 000004 00  WA  0   0  4
  [ 3] .text             PROGBITS        08020200 010200 021b44 00  AX  0   0 64
  [ 4] .ARM.extab        PROGBITS        08041d44 042728 000000 00   W  0   0  1
  [ 5] .exidx            ARM_EXIDX       08041d44 031d44 000008 00  AL  3   0  4
  [ 6] .init_array       INIT_ARRAY      08041d4c 031d4c 000008 04  WA  0   0  4
  [ 7] .fini_array       FINI_ARRAY      08041d54 031d54 000004 04  WA  0   0  4
  [ 8] .data_x           PROGBITS        20000000 040000 0009c8 00  WA  0   0  8
  [ 9] .firmware_header  PROGBITS        08042720 042720 000008 00  WA  0   0  4
  [10] .eth              NOBITS          200009c8 0509c8 0030a0 00  WA  0   0  4
  [11] .bss              NOBITS          20003a68 0509c8 045da4 00  WA  0   0  4
  [12] .heap             PROGBITS        2004980c 042728 000000 00   W  0   0  1
  [13] .reserved_for_sta PROGBITS        2004980c 042728 000000 00   W  0   0  1
  [14] .battery_backed_s NOBITS          40024000 044000 00000c 00  WA  0   0  4
  [15] .comment          PROGBITS        00000000 042728 000075 01  MS  0   0  1
  [16] .debug_frame      PROGBITS        00000000 0427a0 00144c 00      0   0  4
  [17] .stab             PROGBITS        00000000 043bec 000084 0c     18   0  4
  [18] .stabstr          STRTAB          00000000 043c70 000117 00      0   0  1
  [19] .symtab           SYMTAB          00000000 043d88 009b00 10     20 1787  4
  [20] .strtab           STRTAB          00000000 04d888 0042bb 00      0   0  1
  [21] .shstrtab         STRTAB          00000000 051b43 0000e6 00      0   0  1
Key to Flags:
  W (write), A (alloc), X (execute), M (merge), S (strings), I (info),
  L (link order), O (extra OS processing required), G (group), T (TLS),
  C (compressed), x (unknown), o (OS specific), E (exclude),
  y (purecode), p (processor specific)

There are no section groups in this file.

Program Headers:
  Type           Offset   VirtAddr   PhysAddr   FileSiz MemSiz  Flg Align
  LOAD           0x010000 0x08020000 0x08020000 0x21d58 0x21d58 RWE 0x10000
  LOAD           0x040000 0x20000000 0x08041d58 0x009c8 0x009c8 RW  0x10000
  LOAD           0x042720 0x08042720 0x08042720 0x00008 0x00008 RW  0x10000
  LOAD           0x0509c8 0x200009c8 0x08042720 0x00000 0x48e44 RW  0x10000
  LOAD           0x044000 0x40024000 0x40024000 0x00000 0x0000c RW  0x10000

 Section to Segment mapping:
  Segment Sections...
   00     .isr_vector .firmware_header_vector .text .exidx .init_array .fini_array 
   01     .data_x 
   02     .firmware_header 
   03     .eth .bss 
   04     .battery_backed_sram

How does objcopy know not to insert sections such as .bss in to the output image? I know that it computes this on the fly, and I'm assuming this mechanism is driven through the section to segment mapping, but I cannot find any explanation as to how it actually performs the segment to section mapping. The elf file stores no information about which segments are flash, and yet somehow objcopy knows that it should not copy .bss into the output file. How?

Since it is producing a binary image, it mostly goes from the program headers. Those sections that don't get loaded into any segment will end up being ignored. Sections that are fixed zeroes (NOBITS) will likely not end up in the binary either. — Chris Dodd, Jul 18 '19 at 23:51
So that makes sense, but does that mean it has to iterate through both sections and program headers and then "join" that information to say what which will get loaded? Because going by program headers, there is no linkage to specifically what sections are covered by a given program header. As I understand, the only way to get that information would be to use the (files offset + FileSiz) and then look for blocks that occur in that range to know what sections are in that program header. IS this correct? — chris12892, Jul 19 '19 at 02:37
Further more, I am confused because .heap is marked with PROGBITS but it isn't loaded. I guess what I'm really asking for really is a simple algorithm which could be used to decide if a section is loaded or not, preferably the same way objcopy does the same. — chris12892, Jul 19 '19 at 03:13
Oh actually I just noticed that heap has a size of 0. That could be it — chris12892, Jul 19 '19 at 03:34
More or less the same as https://stackoverflow.com/questions/56501470/arm-how-does-objcopy-know-which-sections-from-an-elf-to-include-in-a-binary-or, although I don't think that got a satisfactory answer. — Clifford, Jul 19 '19 at 07:02
BSS is the uninitialized data segment, which has the working area for all uninitialized global variables. The linker just needs to know the size of the segment. BSS section is initialized to zero on loader. — SRK, Jul 19 '19 at 08:01
The segments contain a file offset and a size, that should be enough to know what to copy, without looking at the sections shouldn't it? — PaulR, Jul 19 '19 at 09:12
The segments contain an offset and a size, however There's no way to tell if a segment should be loaded into the binary with out examining the sections within that segment themselves. — chris12892, Jul 21 '19 at 06:21

score 1 · Answer 1 · answered Jul 22 '19 at 08:45

The 'A' flag in the Flg column of the readelf output, which ELF calls SHF_ALLOC, indicates a section that 'occupies memory during process execution'.

SHF_ALLOC : The section occupies memory during process execution. Some control sections do not reside in the memory image of an object file; this attribute is off for those sections.

http://refspecs.linuxbase.org/elf/elf.pdf

Typically this applies to both program and data memory, and in a 'normal' OS environment the OS loads the SHF_ALLOC sections to the indicated addresses (ignoring or using for other purposes the non-SHF_ALLOC sections as appropriate) and everything's ready to go.

In an embedded environment with ROM program memory, only SHF_ALLOC sections mapping to addresses in ROM need to be output.

(This question illustrates a case in which going by address alone without taking SHF_ALLOC into account is not enough.)

However:

Depending on how the linker produced the ELF file, initialised data sections may or may not require additional handling.

Ideally the linker produces two sections related to initialised data: one in RAM (the runtime data area), and one in ROM (the initialisation contents for the RAM data). Startup code references the addresses of these sections and copies the init data to the RAM area. In this case simply outputting SHF_ALLOC sections in the ROM address range will automatically produce correct results.

If the linker instead produces a single section as if for a normal OS, then when generating the ROM image the data section needs to be identified and emitted at an address in ROM (and the startup code needs to be able to find that address).

Most decent toolchains take the former approach if configured correctly, but I've certainly worked with the latter as well.

Tom · Answer 2 · 2022-01-25T09:29:21.847

If you want an algorithm that mimics what objcopy -O binary does you need to look at the section headers. Unlike as is often suggested, just looking at segment headers is not enough.

Quote from the objcopy man page:

objcopy can be used to generate a raw binary file by using an output target of binary (e.g., use -O binary). When objcopy generates a raw binary file, it will essentially produce a memory dump of the contents of the input object file. All symbols and relocation information will be discarded. The memory dump will start at the load address of the lowest section copied into the output file.

I think the last sentence says very deliberately 'section', as this really is what happens: objcopy goes through all the sections and discards those that do not go into a raw binary file. The remaining sections are written to the output file, sorted by ascending address. Space between sections in the output file is filled with zeros.

For an own project where I need to convert ELF files to raw binaries and where I do not want to depend on objcopy I came up with the following:

If a section is of type SHT_NULL or SHT_NOBITS, then the section is not included. This follows from the ELF specification (https://refspecs.linuxbase.org/elf/elf.pdf):
- SHT_NULL: "This value marks the section header as inactive; it does not have an associated section. Other members of the section header have undefined values."
- SHT_NOBITS: "A section of this type occupies no space in the file but otherwise resembles SHT_PROGBITS. Although this section contains no bytes, the sh_offset member contains the conceptual file offset."
If the sh_addr field of a section header is 0, then the section is not included. Quote from the ELF specification: "If the section will appear in the memory image of a process, this member gives the address at which the section's first byte should reside. Otherwise, the member contains 0."
If the sh_size field of a section header is 0, then the section is not included. Quote from the ELF specification: "This member gives the section's size in bytes. Unless the section type is SHT_NOBITS, the section occupies sh_size bytes in the file. A section of type SHT_NOBITS may have a non-zero size, but it occupies no space in the file
If the sh_flags field of a section header doesn't have the SHF_ALLOCflag set, then the section is not included. Quote from the ELF specification: "The section occupies memory during process execution. Some control sections do not reside in the memory image of an object file; this attribute is off for those sections."

Step 3 is somewhat redundant, but for me it nicely filters out sections whose size is zero, so that I do not have to deal with these during later stages.

This works for me, although I haven't thrown many ELF files at it yet. Also note that this is guesswork to some extent. I thought about reading the objcopy source code, but that leads very quickly into the bowels of libbfd.

score 0 · Answer 3 · answered Jul 21 '19 at 06:22

I have no idea if this is the real way to do this, but this is how I ended up solving this:

  # For each program header, get the sections contained by it
  # For each section, calculate the LMA the section will reside at 
  # Do NOT load a section if...
  #   Section type is SHT_NULL or NOBITS
  #   Section size = 0
  #   The LMA is outside of the isr_vector -> header region

Note that in my case, I had a section capping off the end of the image. This made it very obvious as to exactly where the image ended.

score 0 · Answer 4 · answered Jul 28 '23 at 21:29

Find all program segments:

that are non-zero file size
of type PT_LOAD

In those program segments find the sections:

that are non-zero size
that have section attribute flag SHF_ALLOC
that are not of section type SHT_NOBITS

Those are the sections that will be concatenated together into the output file (lowest address, to highest). The default gap padding is 0 (objcopy can change the gap padding with --gap-fill). The address you're interested in for each section is the LMA address. You need to look at the binutils BFD library to figure that out. See here.

In short, the following should work most of the time:

lma_addr = programHdr_physical_addr + sectionHdr_offset - programHdr_offset.

Sort the sections by that LMA address, and then concatenate them together filling in the gaps appropriately (most likely 0xFF depending on your application).

Here's some python code that uses the lief library to parse the ELF, filters out the appropriate sections using the criteria above, and calculates the LMA.

# Parse elf file
b = lief.parse(args.elf)
if b is None:
    print(f"File not found: {args.elf}")
    sys.exit(1)

segments = [
    i
    for i in filter(
        lambda x: (x.physical_size > 0)
        and (x.type == lief.ELF.SEGMENT_TYPES.LOAD),
        sorted(b.segments, key=lambda x: x.physical_address),
    )
]

sections = []
for seg in segments:
    sections_temp = [
        i
        for i in filter(
            lambda x: (x.original_size > 0)
            and (x.type != lief.ELF.SECTION_TYPES.NOBITS)
            and (lief.ELF.SECTION_FLAGS.ALLOC in x.flags_list),
            sorted(seg.sections, key=lambda x: x.virtual_address),
        )
    ]
    sections.append(sections_temp)

for idx, seg in enumerate(sections):
    for sec in seg:
        print(
            f"seg={idx},\tsec={sec.name},\tcomputed_lma={hex(segments[idx].physical_address + sec.file_offset - segments[idx].file_offset)}\tseg_paddr={hex(segments[idx].physical_address)},\tsec_vaddr={hex(sec.virtual_address)},\tsec_size={hex(sec.original_size)}"
        )

How does objcopy compute what sections for the elf file to insert into the output file?

4 Answers4