In my django project I am trying to migrate my database from sqlite to postgres. I've made a dump file of the database, changed my settings file to reflect the new database and now I want to upload this data to postgres. So initially I was running the following in the terminal:
python manage.py loaddata data.json
But this took very long - it was still running for more than a day, with no errors, and I eventually I stopped it.
So then I found this answer on SO, and therefore I created a load_data.py file, like so:
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")
import django
django.setup()
from django.core.management import call_command
from django.db import transaction
DJANGO_SETTINGS_MODULE='mysite.settings'
with transaction.atomic():
call_command('loaddata', 'data.json')
And then ran it via python load_data.py. But it still takes incredibly long and the data has not loaded. My data file is 20.8MB. Is there anything that I can do to speed things up?
For reference, the model that has the most data looks like this:
class WordManager(models.Manager):
def search(self, query=None):
qs = self.get_queryset()
if query is not None:
or_lookup = (Q(target_word__icontains=query) |
Q(source_word__icontains=query)|
Q(example_sentence__icontains=query)
)
qs = qs.filter(or_lookup).distinct() # distinct() is often necessary with Q lookups
return qs
class Word(models.Model):
objects = WordManager()
target_word = models.CharField('Word in Russian', max_length=35,help_text="The word you want to add, in Russian")
source_word = models.CharField('What does it mean?',max_length=35, help_text="Write down the translation in any language", blank=False, null=True)
add_to_word_list = models.BooleanField('Would you like to create a flashcard?', default=True)
deck_name = models.CharField('Deck name', max_length=25, default="My Words")
category = models.CharField(max_length=25,blank=True,help_text="Create a custom category for this word, e.g. Food")
example_sentence = models.CharField(max_length=150,blank=True,help_text="It's important to learn words in context!")
user = models.ForeignKey(User, related_name="words",on_delete=models.CASCADE)
created_at = models.DateTimeField(auto_now=True)
def __str__(self):
return self.target_word
def get_absolute_url(self):
return reverse("vocab:detail",
kwargs={"username": self.user.username, "pk": self.pk})
class Meta:
ordering = ["target_word"]
## constrain user to add only one instance of target word
constraints = [
models.UniqueConstraint(fields=['user','target_word', 'source_word'],name='unique_word')]
from django.db.models.signals import post_save
from django.dispatch import receiver
from api.models import Flashcard, Deck
@receiver(post_save, sender=Word)
def create_deck(sender, created, instance, **kwargs):
if created:
instance._meta.model.objects.all()
deck_name = instance.deck_name
user = instance.user
flash_wanted = Word.objects.filter(add_to_word_list=True)
if flash_wanted:
deck_name, created = Deck.objects.get_or_create(owner=user, name=deck_name)
post_save.connect(create_deck, sender=Word)
@receiver(post_save, sender=Word)
def flash_from_word(sender, created, instance, **kwargs):
if created:
flash_wanted = Word.objects.filter(add_to_word_list=True)
instance._meta.model.objects.all()
target_word = instance.target_word
source_word = instance.source_word
deck_name = instance.deck_name
user = instance.user
if flash_wanted:
flash = Flashcard.objects.create(owner=user, question=target_word,answer=source_word,deck=Deck.objects.get(name=deck_name,owner=user))
post_save.connect(flash_from_word, sender=Word)
