How to load a method from a file into an existing class (a 'plugin' method)

Question

Call me weird if you like, many have before, but I have a large class which I'd like to make extensible with methods loaded from a plugin directory. Essentially, I'm monkey patching the class. What I have almost works but the method loaded doesn't 'see' the globals defined in __main__. Ideally I'd like a way to tell globals() (or whatever mechanism is actually used to locate global variables) to use that existing in __main__. Here is the code I have (trimmed for the sake of brevity):

#!/usr/bin/env python3

import importlib
import os
import types

main_global = "Hi, I'm in main"

class MyClass:

    def __init__(self, plugin_dir=None):

        if plugin_dir:
            self.load_plugins(plugin_dir, ext="plugin")

    def load_plugins(self, plugin_dir, ext):
        """ Load plugins

            Plugins are files in 'plugin_dir' that have the given extension.
            The functions defined within are imported as methods of this class.
        """
        cls = self.__class__

        # First check that we're not importing the same extension twice into
        # the same class.
        try:
            plugins = getattr(cls, "_plugins")
        except AttributeError:
            plugins = set()
            setattr(cls, "_plugins", plugins)

        if ext in plugins:
            return
        plugins.add(ext)

        for file in os.listdir(plugin_dir):
            if not file.endswith(ext):
                continue
            filename = os.path.join(plugin_dir, file)

            loader = importlib.machinery.SourceFileLoader("bar", filename)
            module = types.ModuleType(loader.name)
            loader.exec_module(module)

            for name in dir(module):
                if name.startswith("__"):
                    continue

                obj = getattr(module, name)
                if callable(obj):
                    obj = obj.__get__(self, cls)
                setattr(cls, name, obj)

z = MyClass(plugin_dir="plugins")
z.foo("Hello")

And this is 'foo.plugin' from the plugins directory:

#!/usr/bin/env python3

foo_global = "I am global within foo"

def foo(self, value):
  print(f"I am foo, called with {self} and {value}")
  print(f"foo_global = {foo_global}")
  print(f"main_global = {main_global}")

The output is...

I am foo, called with <__main__.MyClass object at 0x7fd4680bfac8> and Hello
foo_global = I am global within foo
Traceback (most recent call last):
  File "./plugged", line 55, in <module>
    z.foo("Hello")
  File "plugins/foo.plugin", line 8, in foo
    print(f"main_global = {main_global}")
NameError: name 'main_global' is not defined

I know it all feels a bit 'hacky', but it's become a challenge so please don't flame me on style etc. If there's another way to achieve this aim, I'm all ears.

Thoughts, learned friends?

Answer 1

You can approach the problem with a factory function and inheritance. Assuming each of your plugins is something like this, defined in a separate importable file:

class MyPlugin:
    foo = 'bar'

    def extra_method(self):
        print(self.foo)

You can use a factory like this:

def MyClassFactory(plugin_dir):

    def search_and_import_plugins(plugin_dir):
        # Look for all possible plugins and import them
        return plugin_list  # a list of plugin classes, like [MyPlugin]

    plugin_list = search_and_import_plugins(plugin_dir):

    class MyClass(*plugin_list):
         pass

    return MyClass()

z = MyClassFactory('/home/me/plugins')

Answer 2

You can do what you want with a variation of the technique shown in @Martijn Pieters' answer to the the question: How to inject variable into scope with a decorator? tweaked to inject multiple values into a class method.

from functools import wraps
import importlib
import os
from pathlib import Path
import types


main_global = "Hi, I'm in main"

class MyClass:
    def __init__(self, plugin_dir=None):
        if plugin_dir:
            self.load_plugins(plugin_dir, ext="plugin")

    def load_plugins(self, plugin_dir, ext):
        """ Load plugins

            Plugins are files in 'plugin_dir' that have the given extension.
            The functions defined within are imported as methods of this class.
        """
        cls = self.__class__

        # First check that we're not importing the same extension twice into
        # the same class.
        try:
            plugins = getattr(cls, "_plugins")
        except AttributeError:
            plugins = set()
            setattr(cls, "_plugins", plugins)

        if ext in plugins:
            return
        plugins.add(ext)

        for file in Path(plugin_dir).glob(f'*.{ext}'):
            loader = importlib.machinery.SourceFileLoader("bar", str(file))
            module = types.ModuleType(loader.name)
            loader.exec_module(module)
            namespace = globals()

            for name in dir(module):
                if name.startswith("__"):
                    continue

                obj = getattr(module, name)
                if callable(obj):
                    obj = inject(obj.__get__(self, cls), namespace)

                setattr(cls, name, obj)


def inject(method, namespace):

    @wraps(method)
    def wrapped(*args, **kwargs):
        method_globals = method.__globals__

        # Save copies of any of method's global values replaced by the namespace.
        replaced = {key: method_globals[key] for key in namespace if key in method_globals}
        method_globals.update(namespace)

        try:
            method(*args[1:], **kwargs)
        finally:
            method_globals.update(replaced)  # Restore any replaced globals.

    return wrapped


z = MyClass(plugin_dir="plugins")
z.foo("Hello")

Example output:

I am foo, called with <__main__.MyClass object at 0x0056F670> and Hello
foo_global = I am global within foo
main_global = Hi, I'm in main