0

I'm attempting to wrap a C API in a generic C++ call method, but get different behaviour when running depending on the value of ulimit -s. I'm running this code on Linux: Debian GNU/Linux 10 (buster) x86_64, kernel 4.19.0-18-amd64, compiler is gcc/g++ (Debian 8.3.0-6) 8.3.0.

When running with the default value of 8192 I get a segfault, however if I set ulimit -s unlimited the code executes without issue. e.g.

m@devbox:~/projects/scriptCall$ ulimit -s
8192
m@devbox:~/projects/scriptCall$ ./app 
Segmentation fault
m@devbox:~/projects/scriptCall$ ulimit -s unlimited
m@devbox:~/projects/scriptCall$ ./app 
200
200
110
API function 4: arg 0 = 1
API function 4: arg 1 = 2
API function 4: arg 2 = 3
API function 4: arg 3 = 4
API function 4: arg 4 = 5
API function 4: arg 5 = 6
API function 4: arg 6 = 7
API function 4: arg 7 = 8

I think this is due to how the arguments are being handled on the stack when the function pointers are modified by the reinterpret_cast<>() calls, which is why I added the switch statement to (in theory) correctly size the result of the cast to match the API function signature (but this is just a guess).

What is causing the segfault when the stack size is limited? Is it safe/stable to run with ulimit -s unlimited or will doing that eventually result in other issues (out of memory, stack corruption, etc)?

Project code is as follows:

CMakeLists.txt

cmake_minimum_required(VERSION 3.13)
project (scriptCall)

set(CMAKE_CXX_STANDARD 11)

add_executable(app
    ScriptCall.cpp cAPI.c
    ScriptCall.hpp cAPI.h
    main.cpp)

include_directories(${CMAKE_CURRENT_SOURCE_DIR})

main.cpp

#include <iostream>
#include <cstdint>
#include <vector>

#include "cAPI.h"

#include "ScriptCall.hpp"

int main(int argc, char* argv[])
{
    scriptCall dataArgTest; 
    dataArgTest.setArg(0, argType::data, 100);
    dataArgTest.setNumArgs(1);
    dataArgTest.setFunction(reinterpret_cast<long(*)()>(apiFunc1));
    std::cout << dataArgTest.execute() << std::endl;

    scriptCall dataArgTest2;
    dataArgTest2.setArg(0, argType::data, 100);
    dataArgTest2.setNumArgs(1);
    dataArgTest2.setFunction(reinterpret_cast<long(*)()>(apiFunc1));
    std::cout << dataArgTest2.execute() << std::endl;

    scriptCall pointerArgTest;
    pointerArgTest.setArg(0, argType::data, 100);
    pointerArgTest.setArg(1, argType::pointer, 10);
    pointerArgTest.setNumArgs(2);
    pointerArgTest.setFunction(reinterpret_cast<long(*)()>(apiFunc2));
    std::cout << pointerArgTest.execute() << std::endl;

    long values[8] = {1,2,3,4,5,6,7,8};
    scriptCall pointerAccessTest;
    pointerAccessTest.setArg(0, argType::data, 8);
    pointerAccessTest.setArg(1, argType::pointer, 1);
    pointerAccessTest.setPointerArgValue(1, 8, values);
    pointerAccessTest.setNumArgs(2);
    pointerAccessTest.setFunction(reinterpret_cast<long(*)()>(apiFunc3));
    pointerAccessTest.execute();

    return 0;
}

ScriptCall.hpp

#include <vector>
#include <array>

enum class argType : long
{
    data,
    pointer
};

struct scriptArg
{
    argType type;
    std::array<long, 65536> data;
};

class scriptCall
{
private:
    int numArgs;
    long (*fnPtr)();

    std::array<scriptArg, 8> args;

public:

    scriptCall() = default;
    ~scriptCall() = default;

    long setFunction(long (*newFnPtr)());
    long setArg(long argIndex, argType t, long argValue);
    long setNumArgs(long newNumArgs);
    
    long setPointerArgValue(long argIndex, long numValues, long* src);

    long execute();
};

ScriptCall.cpp

#include <vector>
#include <cstdint>
#include <cassert>

#include "ScriptCall.hpp"

long scriptCall::setFunction(long (*newFnPtr)())
{
    this->fnPtr = newFnPtr;
    return 0;
}
long scriptCall::setNumArgs(long newNumArgs)
{
    this->numArgs = newNumArgs;
    return 0;
}

long scriptCall::setArg(long argIndex, argType t, long argValue)
{
    assert(argIndex < 8); // index must be less than max number of args
    this->args[argIndex].type = t;
    this->args[argIndex].data[0] = argValue;
    return 0;
}

long scriptCall::setPointerArgValue(long argIndex, long numValues, long* src)
{
    for (long i = 0; i < numValues; ++i)
    {
        this->args[argIndex].data[i] = *(src+i);
    } 
    return 0;
}

long scriptCall::execute()
{
    // build param array, max of 8 arguments
    long c_args[8] = { 0 };

    for(int i = 0; i < this->numArgs; ++i)
    {
        if (this->args[i].type == argType::pointer)
        {
            // if arg is long*, cast to long
            c_args[i] = reinterpret_cast<long>(this->args[i].data.begin());
        }
        else
        {
            c_args[i] = this->args[i].data[0];
        }
    }

    switch(this->numArgs)
    {
        case 0:
        return (this->fnPtr)();

        case 1:
        return reinterpret_cast<long(*)(long)>((this->fnPtr))(c_args[0]);
        
        case 2:
        return reinterpret_cast<long(*)(long, long)>((this->fnPtr))(c_args[0], c_args[1]);
        
        case 3:
        return reinterpret_cast<long(*)(long, long, long)>((this->fnPtr))(c_args[0], c_args[1], c_args[2]);
        
        case 4:
        return reinterpret_cast<long(*)(long, long, long, long)>((this->fnPtr))(c_args[0], c_args[1], c_args[2], c_args[3]);
    }
}

cAPI.h

#ifdef __cplusplus
extern "C" {
#endif

    long apiFunc1(long arg1);
    long apiFunc2(long arg1, long* arg2);
    long apiFunc3(long numArgs, long* args);

#ifdef __cplusplus
}
#endif

cAPI.c

#include "stdio.h"
#include "cAPI.h"

long apiFunc1(long arg1)
{
    return 2*arg1;
}
long apiFunc2(long arg1, long* arg2)
{
    return arg1 + *arg2;
}

long apiFunc3(long numArgs, long* args)
{
    for(long i = 0; i < numArgs; ++i)
    {
        printf("API function 4: arg %ld = %ld\n", i, *(args+i));
    }
    return 0;
}
szeneca
  • 1
  • 1
  • 2
    what about opening the core file with gdb and look at the stack trace? – OznOg Feb 09 '22 at 20:09
  • Hmm, setting `ulimit -c unlimited` dumps a core file, but the stack is corrupt? ` Core was generated by './app'. Program terminated with signal SIGSEGV, Segmentation fault. #0 0x000056152a861e96 in ?? () (gdb) bt #0 0x000056152a861e96 in ?? () Backtrace stopped: Cannot access memory at address 0x7ffc463cc900 (gdb) ` – szeneca Feb 09 '22 at 20:25
  • The stack pointer is pointing to the inaccessible address `(gdb) p $sp $1 = (void *) 0x7ffc463cc900` – szeneca Feb 09 '22 at 20:30
  • try running it in gdb directly – OznOg Feb 10 '22 at 13:59

0 Answers0