Write exception to log when django is unable to run properly

Question

Django Gurus, I have the following error messages on my console and I have the following logging configuration.

LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    "handlers": {
        "console": {
            "class": "logging.StreamHandler",
        },
        "file": {
            'class':'logging.FileHandler',
            "filename": str(BASE_DIR) + "/" + "logs"+ "/"  + "app.log",
            "level":"DEBUG",
        },
    },

    "loggers": {
      "django.utils.autoreload": {
        "level": "CRITICAL"
       },
       
       "django": {
         "handlers": ["file"],
         "level": "DEBUG",
         "propagate": True,
       },
    }
}

Stacktrace from my console:

System check identified no issues (0 silenced).
Exception in thread django-main-thread:
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
psycopg2.errors.InsufficientPrivilege: permission denied for table django_migrations


The above exception was the direct cause of the following exception:
    
Traceback (most recent call last):
      File "/usr/local/Cellar/python@3.8/3.8.11/Frameworks/Python.framework/Versions/3.8/lib/python3.8/threading.py", line 932, in _bootstrap_inner
        self.run()
      File "/usr/local/Cellar/python@3.8/3.8.11/Frameworks/Python.framework/Versions/3.8/lib/python3.8/threading.py", line 870, in run
        self._target(*self._args, **self._kwargs)
      File "/usr/local/lib/python3.8/site-packages/django/utils/autoreload.py", line 64, in wrapper
        fn(*args, **kwargs)
      File "/usr/local/lib/python3.8/site-packages/django/core/management/commands/runserver.py", line 121, in inner_run
        self.check_migrations()
      File "/usr/local/lib/python3.8/site-packages/django/core/management/base.py", line 486, in check_migrations
        executor = MigrationExecutor(connections[DEFAULT_DB_ALIAS])
      File "/usr/local/lib/python3.8/site-packages/django/db/migrations/executor.py", line 18, in __init__
        self.loader = MigrationLoader(self.connection)
      File "/usr/local/lib/python3.8/site-packages/django/db/migrations/loader.py", line 53, in __init__
        self.build_graph()
      File "/usr/local/lib/python3.8/site-packages/django/db/migrations/loader.py", line 220, in build_graph
        self.applied_migrations = recorder.applied_migrations()
      File "/usr/local/lib/python3.8/site-packages/django/db/migrations/recorder.py", line 78, in applied_migrations
        return {(migration.app, migration.name): migration for migration in self.migration_qs}
      File "/usr/local/lib/python3.8/site-packages/django/db/models/query.py", line 280, in __iter__
        self._fetch_all()
      File "/usr/local/lib/python3.8/site-packages/django/db/models/query.py", line 1324, in _fetch_all
        self._result_cache = list(self._iterable_class(self))
      File "/usr/local/lib/python3.8/site-packages/django/db/models/query.py", line 51, in __iter__
        results = compiler.execute_sql(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size)
      File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1169, in execute_sql
        cursor.execute(sql, params)
      File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 98, in execute
        return super().execute(sql, params)
      File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 66, in execute
        return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
      File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers
        return executor(sql, params, many, context)
      File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
        return self.cursor.execute(sql, params)
      File "/usr/local/lib/python3.8/site-packages/django/db/utils.py", line 90, in __exit__
        raise dj_exc_value.with_traceback(traceback) from exc_value
      File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute
        return self.cursor.execute(sql, params)
    django.db.utils.ProgrammingError: permission denied for table django_migrations

From the above stack trace it's obvious what the error is but I am not able to dump it to my log file. When I open app.log I always see the following:

    (0.004) 
                SELECT c.relname,
                CASE WHEN c.relispartition THEN 'p' WHEN c.relkind IN ('m', 'v') THEN 'v' ELSE 't' END
                FROM pg_catalog.pg_class c
                LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
                WHERE c.relkind IN ('f', 'm', 'p', 'r', 'v')
                    AND n.nspname NOT IN ('pg_catalog', 'pg_toast')
                    AND pg_catalog.pg_table_is_visible(c.oid)
            ; args=None
    (0.001) SELECT "django_migrations"."id", "django_migrations"."app", "django_migrations"."name", "django_migrations"."applied" FROM "django_migrations"; args=()

I want the full stack trace written in my log. When I open app.log I only able see the following information

(0.004) 
            SELECT c.relname,
            CASE WHEN c.relispartition THEN 'p' WHEN c.relkind IN ('m', 'v') THEN 'v' ELSE 't' END
            FROM pg_catalog.pg_class c
            LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
            WHERE c.relkind IN ('f', 'm', 'p', 'r', 'v')
                AND n.nspname NOT IN ('pg_catalog', 'pg_toast')
                AND pg_catalog.pg_table_is_visible(c.oid)
        ; args=None
(0.001) SELECT "django_migrations"."id", "django_migrations"."app", "django_migrations"."name", "django_migrations"."applied" FROM "django_migrations"; args=()

I have a middleware that can handle exceptions but Django stop after the above exception and I believe it did not reach to middleware to be handled.

Any help would be much appreciated.

score 0 · Answer 1 · answered May 13 '23 at 10:49

I am posting my solution.

Just to summarize, python3 manager.py run server shows the above exception in the console and there are a few lines of query that fail in my log. It means django logger works but does not do full stacktrace automatically to log file.

Middleware does not work in this scenario because no request/response. This type of exception needs to be logged and notified to take action.

I ended up implementing a custom log filer as below

import traceback

class CustomFilter(logging.Filter):

    def filter(self, log_record):
        log_record.trace = traceback.format_exc()
        return log_record

class CustomHandler(logging.FileHandler):    

    def __init__(self, *args, **kwargs):
        logging.FileHandler.__init__(self, *args, **kwargs)
        self.addFilter(CustomFilter())

My logger configuration now looks like this

LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    'formatters': {

     "verbose": {
            "format": "{levelname} {trace}",
            "style": "{",
        },
    },

    "handlers": {
        "console": {
            "class": "logging.StreamHandler",
        },
        "file": {
            'class':'master.utils.CustomHandler',
            "filename": str(BASE_DIR) + "/" + "logs"+ "/"  + "app.log",
            "level":"DEBUG",
            "formatter": "verbose"
        },
    },

    "loggers": {
      "django.utils.autoreload": {
        "level": "CRITICAL"
       },
       
       "django": {
         "handlers": ["file"],
         "level": "DEBUG",
         "propagate": True,
       },
    }
}

See the reference of master. utils.CustomHandler in file handler. I have found the implementation help from Django file logger doesn't log as expected Happy logging!

Write exception to log when django is unable to run properly

1 Answers1