is it posible to prohibit a cast?

Question

Context:

I have been trying to use Dietmar Kühl's delegate class from this answer in a way that only the owner class could activate it, and I almost achieved it.

The code is the following:

// Example program
#include <algorithm>
#include <iostream>
#include <memory>
#include <utility>
#include <vector>

class Entity;

struct EvArgs {
    Entity* Origin;
    EvArgs(){
    }
};

template <typename Signature>
struct delegate;

template <typename Args>
struct delegate<void(Args*)>
{
    struct base {
        virtual ~base() {}
        virtual void do_call(Args* args) = 0;
    };

    template <typename T>
    struct call : base {
        T d_callback;

        template <typename S>
        call(S&& callback) : d_callback(std::forward<S>(callback)) {}

        void do_call(Args* args) {
            this->d_callback(std::forward<Args*>(args));
            return;
        }
    };

    std::vector<std::unique_ptr<base>> d_callbacks;
    std::vector<std::unique_ptr<base>> d_tmp_callbacks;

    delegate(delegate const&) = delete;
    void operator=(delegate const&) = delete;
public:
    delegate() {
        if (!std::is_base_of<EvArgs, Args>::value)
            throw "specified type is not derived class from EvArgs\n";
    }
    ~delegate() {
        d_callbacks.clear();
        d_tmp_callbacks.clear();
    }

    template <typename T>
    delegate& operator+= (T&& callback) {
        this->d_callbacks.emplace_back(new call<T>(std::forward<T>(callback)));
        return *this;
    }
    template <typename T>
    delegate& operator<< (T&& callback) {
        this->d_tmp_callbacks.emplace_back(new call<T>(std::forward<T>(callback)));
        return *this;
    }
};

template<typename Signature>
struct action_delegate;

template<typename Args>
struct action_delegate<void(Args*)> : public delegate<void(Args*)>{
    delegate<void(Args*)>& getBase(){
        return *static_cast<delegate<void(Args*)>*>(this);
    }
    
    void operator()(Args* args) {
        for (auto& callback : this->d_callbacks)        callback->do_call(args);
        for (auto& callback : this->d_tmp_callbacks)    callback->do_call(args);
        this->d_tmp_callbacks.clear();

        delete args;
    }
};

class instance{
private:
    action_delegate<void(EvArgs*)> _collision;
public:
    delegate<void(EvArgs*)>& collision = _collision.getBase();
};


int main(){
    instance i;
    i.collision << [](EvArgs* a){
        std::cout << "random calling\n";
        };

    //i want to prohibit this:
    (static_cast< action_delegate<void(EvArgs*)>& >(i.collision))(new EvArgs());
}

Problem:

As the action_delegate is a private member, only the instance class can call its activation with operator(). And as delegate is public, operator <<() and operator +=() are accessible outside the class.

The problem is that there is a way to cast delegate into action_delegate, thus it's possible to activate the delegate with operator() outside the class. I really want to avoid that.

The original creator of this concept is accessible through the link at the start of the commentary body.