Add decorator for python requests get function

Question

I was looking for a way to add some sort of decorator that applies to all instances of requests.get being used in any function.

For example,

@my_custom_decorator
def hello():
    ...
    r = requests.get('https://my-api-url')
    ...

The my_custom_decorator could then add a common param (or anything else) for all instances of requests.get. One will only need to add the decorator whereever requests.get is being used.

For now, I'm thinking of somehow checking if the original function contains presence of requests.get, but that seems not to be ideal.

Note: Also, I'm looking not to change any existing instances of requests.get..hence looking for a better way to achieve this.

Answer 1

You can use unittest.mock.patch as a decorator on functions where requests.get needs additional parameters:

import requests
from unittest.mock import patch

def get(url, params=None, orig_get=requests.get, **kwargs):
    return orig_get(url, {**(params or {}), 'extra_param': 'extra_value'}, **kwargs)

my_custom_decorator = patch('requests.get', get)

so that:

@my_custom_decorator
def hello():
    r = requests.get('https://my-api-url', {'foo': 'bar'})
    print(r.status_code)

hello()

would send a GET request to the web server at my-api-url with the following URI:

/?foo=bar&extra_param=extra_value

Answer 2

It would probably be very painful to use a decorator to solve this, since you would need to parse the code in the function (see this question for instance).

Creating your own get function would probably be more suited for your case (though it depends on what you actually want to do)

something like:

def get_with_param(url: str):
    # for instance, add a query param:
    r = requests.get(url + "?param=something")
    return r

and then you just have to replace every occurence of requests.get with get_with_param, for instance

Answer 3

Not ideal/recommended depending on the context, but in the abscence of one, you are best placed to decide this. You could also try patch to monkeypatch requests.get.

from unittest.mock import patch
from requests import get as my_requests
import requests

def my_get(*args, **kwargs):
    ''' Your version of get where you do what needs to be done'''
    return my_requests(*args, **kwargs)

with patch('requests.get', my_get): # where you provide an alternate implementation for requests.get
    requests.get('https://www.google.com')

Answer 4

In fact, you can't change "the instance" of requests.get because it returns

:return: :class:`Response <Response>` object

So, I recommend the next solution (requests's params was used for show case):

def my_custom_decorator(func):
    def wrapper(*args, **kwargs):
        params={"search":"string"}
        return func(*args, params=params, **kwargs)
    return wrapper

And, yeah, there is no way to avoid **kwargs:

@my_custom_decorator
def hello(**kwargs):
    resp = requests.get('https://mail.ru', **kwargs)

Take a look PoWs (before and after):

Answer 5

I'm looking not to change any existing instances of requests.get

Is there a specific reason for that? Temporarily overriding requests.get seems to be a feasible solution.

#!/usr/bin/env python3
import requests

def deco(f):
  def my_fun(self):
    return "requests.get was overridden!"

  def inner():
    backup_fun = requests.get 
    requests.get = my_fun
    ret = f()
    requests.get = backup_fun
    return ret

  return inner

@deco
def decorated():
  return requests.get("http://example.org")

def undecorated():
  return requests.get("http://example.org")

if __name__ == '__main__':
  print("decorated:", decorated())
  print("undecorated:", undecorated())

Will print:

decorated: requests.get was overridden!
undecorated: <Response [200]>

Answer 6

We can modify and/or add other parameters like here where I modify the url parameter for some_requests_dot_get_function via some_decorator:

url = 'google.com'#'https://www.google.com/'
def some_decorator(some_function):
    def wrapper(*args, url, **kwargs):
        some_function.url = 'https://www.' + url + '/'
        return some_function(*args, url=some_function.url, **kwargs)
    return wrapper

@some_decorator
def some_requests_dot_get_function(*args, url, **kwargs):
    return requests.get(*args, url=url, **kwargs).content.decode('utf-8')

some_requests_dot_get_function(url=url)

Not sure the exact use case you're intending, but maybe it could justify using some dict annotations for different functions to modify parameters depending on its particular item(s), that way at least you only have to modify this decorator rather than every function it decorates:

url = 'google.com'#'https://www.google.com/'
def some_decorator(some_function):
    def wrapper(*args, url, **kwargs):
        if some_function.__annotations__['return'].__getitem__('some_annotation') > 0:
            some_function.url = 'https://www.' + url + '/'
        else:
            some_function.url = url
        return some_function(*args, url=some_function.url, **kwargs)
    return wrapper

@some_decorator
def some_requests_dot_get_function(*args, url, **kwargs) -> {'some_annotation': 2}:
    return requests.get(*args, url=url, **kwargs).content.decode('utf-8')

some_requests_dot_get_function(url=url)

... or we could rely on the names of the functions to set the parameters in some_decorator:

url = 'google.com'#'https://www.google.com/'
def some_decorator(some_function):
    def wrapper(*args, url, **kwargs):
        if some_function.__name__.__contains__('some_requests'):
            some_function.url = 'https://www.' + url + '/'
        elif some_function.__name__.__contains__('some_other_requests'):
            some_function.url = 'https://www.' + url + '/?client=safari'
        else:
            some_function.url = url
        return some_function(*args, url=some_function.url, **kwargs)
    return wrapper

@some_decorator
def some_requests_dot_get_function(*args, url, **kwargs):
    return 'some: ' + requests.get(*args, url=url, **kwargs).content.decode('utf-8')

@some_decorator
def some_other_requests_dot_get_function(*args, url, **kwargs):
    return 'some other: ' + requests.get(*args, url=url, **kwargs).content.decode('utf-8')

some_requests_dot_get_function(url=url)
some_other_requests_dot_get_function(url=url)

Answer 7

The following will adjust the behavior for methods that make a call to requests.get using the decorator my_custom_decorator. Given that changes to the source code directly or injecting a new method using exec with inspect and optparse was illegal.

There is one limitation, that is that a kwarg_ball has to be used to mediate the decorator and the requests.get method, as there is no other way for injecting additional *args, **kwargs.

import inspect
from functools import wraps


def my_custom_decorator(f):
    # Filter hotkeys access if there's currently some activity going on...
    @wraps(f)
    def wrap(*args, **kwargs):
        # Have to know ahead of time if there is going to be a call made, no way around it
        source_code = inspect.getsource(f)
        if 'requests.get' in source_code:
            requests.kwarg_ball = {'DATE': 'FOO', 'VAR': 'BAR'}
            result = f(*args, **kwargs)
            requests.kwarg_ball = None
        else:
            result = f(*args, **kwargs)
        return result

    return wrap


class requests:
    kwarg_ball = None

    @staticmethod
    def get(url):
        if requests.kwarg_ball is not None:
            print("Decorator modification detected!")
            for k, v in requests.kwarg_ball.items():
                print(k, ' - ', v)
        print(f'URL {url}')

The following produce the desire result with hello demonstrating the modified behavior, and goodbye executing normally.

@an_ugly_wrapper
def hello():
    requests.get('https://my-api-url')

def goodbye():
    requests.get('https://my-api-url')

hello()
goodbye()

Additionally, the kwarg_ball may be loaded into the **kwargs for the requests.get method.

Answer 8

As you mentioned you don't want to change all occurrences, what you could do, is create a new file with name api.py (or whatever you like) and add all the function you want like this:

################## api.py ########################
import requests
def get(url, params=None):
    ..... You code goes here
    return request.get(url,params)
def post(url, params=None):
    ..... You code goes here
    return request.post(url,params)

Now, in your code files, just replace import requests with import api as requests