How to build python extension module with waf on windows/visual studio?

Question

I'm trying to figure out how to build a python extension module using waf but I've got stuck. Consider a simple tree such as:

foo.h

#pragma once

class Foo {
public:
  Foo();
  const int &a() const;
  int &a();

private:
  int m_a;
};

inline const int &Foo::a() const { return m_a; }

inline int &Foo::a() { return m_a; }

foo.cpp

#include "foo.h"

Foo::Foo() : m_a(0) {}

foo.i

%module foo

%{
#include "foo.h"
%}

%include "foo.h"

main.cpp

#include <iostream>

#include "foo.h"

using namespace std;
int main() {
  Foo foo;
  cout << foo.a() << endl;
  foo.a() = 10;
  cout << foo.a() << endl;
}

main.py

import sys
sys.path.append('build')

from foo import Foo

foo = Foo()
print foo.a
foo.a = 1
print foo.a

wscript

import subprocess
from pathlib import Path


def configure(conf):
    conf.env.MSVC_VERSIONS = ["msvc 15.0"]
    conf.env.MSVC_TARGETS = ["x86"]
    conf.load("msvc python swig")
    conf.check_python_version((3, 6, 8))

    # This method is not working properly on windows/virtualenv
    # conf.check_python_headers()

    # Maybe something like this could help?
    # from distutils import sysconfig
    # print(sysconfig.get_python_inc())

    conf.check_swig_version()


def build(bld):
    bld.program(source=["main.cpp", "foo.cpp"], cxxflags=["/EHsc"], target="test")

    bld.shlib(
        features="cxx",
        source="foo.cpp",
        target="foo",
        includes=".",
        export_includes=".",
        name="FOO",
    )

    # bld.program(
    #     source = 'main.cpp',
    #     target = 'main',
    #     use = ['FOO'],
    #     rpath = ['$ORIGIN'],
    # )

    # bld(
    #     features = 'cxx cxxshlib pyext',
    #     source = 'foo.i',
    #     target = '_foo',
    #     swig_flags = '-c++ -python -Wall',
    #     includes = '.',
    #     use  = 'FOO',
    #     rpath = ['$ORIGIN', ],
    #  )


def run(bld):
    root_path = Path(bld.path.abspath())
    subprocess.run(str(root_path / "build/test.exe"))

The first issue I'm already facing here is when I do waf distclean configure build, I'll get this error:

(py382_64) D:\swigtest>waf distclean configure build
'distclean' finished successfully (0.006s)
Setting top to                           : D:\swigtest
Setting out to                           : D:\swigtest\build
Checking for program 'CL'                : C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.26.28801\bin\HostX86\x86\CL.exe
Checking for program 'CL'                : C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.26.28801\bin\HostX86\x86\CL.exe
Checking for program 'LINK'              : C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.26.28801\bin\HostX86\x86\LINK.exe
Checking for program 'LIB'               : C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.26.28801\bin\HostX86\x86\LIB.exe
Checking for program 'MT'                : C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x86\MT.exe
Checking for program 'RC'                : C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x86\RC.exe
Checking for program 'python'            : d:\virtual_envs\py368_32\scripts\python.exe
Checking for program 'swig'              : D:\software\swig\swig.exe
Checking for python version >= 3.6.8     : 3.6.8
Checking for swig version                : 4.0.0
'configure' finished successfully (1.538s)
Waf: Entering directory `D:\swigtest\build'
[1/5] Compiling foo.cpp
[2/5] Compiling main.cpp
[3/5] Compiling foo.cpp
foo.cpp

foo.cpp

[4/5] Linking build\foo.dll
main.cpp

[5/5] Linking build\test.exe
Waf: Leaving directory `D:\swigtest\build'
Build failed
-> missing file: 'D:\\swigtest\\build\\foo.lib'

QUESTION: How can I fix that little issue? In any case, my main question here would be what's the proper way to generate a python extension module with swig and waf on windows?

Answer 1

You can fix your current build by changing the library type from shared (shlib) to static (stlib), i.e.

def build(bld):
    bld.program(source=["main.cpp", "foo.cpp"], cxxflags=["/EHsc"], target="test")

    bld.stlib(
        features="cxx",
        source="foo.cpp",
        target="foo",
        includes=".",
        export_includes=".",
        name="FOO",
    )

To build the python extension module, you can use this:

def build(bld):
    bld.program(source=["main.cpp", "foo.cpp"], cxxflags=["/EHsc"], target="test")

    bld.stlib(
        features="cxx",
        source="foo.cpp",
        target="foo",
        includes=".",
        export_includes=".",
        name="FOO",
    )

    bld(
        features = 'cxx cxxshlib pyext',
        source = 'foo.i',
        target = '_foo.pyd',
        swig_flags = '-c++ -python -Wall',
        includes = '.',
        use = 'FOO',
    )

This will output a dll library into a file named _foo.pyd and a python script foo.py.
I'm not claiming that this is the proper way, but it worked for me.

Also, when using the extension (e.g. from foo import Foo) do make sure that these files are on python's search path (using PYTHONPATH or sys.path.append), otherwise you might get errors such as:

• ModuleNotFoundError: No module named '_foo'
• ImportError: DLL load failed while importing _foo: The parameter is incorrect.

I got the second error when I've tried running your main.py directly from the main directory and resolved it by using an absolute (instead of relative) path in the sys.path.append call in main.py.

Edit
I was using python 3.8.5 and Visual Studio 2017
Below is the full wscript.
You'll need to change the path to the Python headers and libraries referenced in CXXFLAGS, LDFLAGS (although I guess that there's a better way to configure it)

import subprocess
from pathlib import Path

def configure(conf):
    conf.env.MSVC_VERSIONS = ["msvc 15.9"]
    conf.env.MSVC_TARGETS = ["x64"]
    conf.env.append_value('CXXFLAGS', ["/Id:\\tools\\Python38\\include"])
    conf.env.append_value('LDFLAGS', ["/LIBPATH:d:\\tools\\Python38\\libs"])
    conf.load("msvc python swig")
    conf.check_python_version((3, 6, 8))

    # This method is not working properly on windows/virtualenv
    #conf.check_python_headers()

    # Maybe something like this could help?
    # from distutils import sysconfig
    # print(sysconfig.get_python_inc())

    conf.check_swig_version()


def build(bld):
    bld.program(source=["main.cpp", "foo.cpp"], cxxflags=["/EHsc"], target="test")

    bld.stlib(
        features="cxx",
        source="foo.cpp",
        target="foo",
        includes=".",
        export_includes=".",
        name="FOO",
    )

    bld(
        features = 'cxx cxxshlib pyext',
        source = 'foo.i',
        target = '_foo.pyd',
        swig_flags = '-c++ -python -Wall',
        includes = '.',
        use = 'FOO',
    )



def run(bld):
    root_path = Path(bld.path.abspath())
    subprocess.run(str(root_path / "build/test.exe"))

Edit 2 Initially I was getting the error mentioned in the comments, but now I see that I inadvertadly worked around the bug that you've reported by moving the module declaration in foo.i from top to the bottom of the file.

%{
#include "foo.h"
%}

%include "foo.h"

%module foo