I have a Django app in which the user can upload a tdms file. When I deploy it, this does not work for large files, which is why I want to implement chunked upload through the django-chunked-upload library. I gave it a go, but I really need help. I get this feedback in my terminal:
"GET /tdms/list_vibration_logs/ HTTP/1.1" 200 7346
<QueryDict: {'resumableChunkNumber': ['5'], 'resumableChunkSize': ['1048576'], 'resumableCurrentChunkSize': ['1339055'], 'resumableTotalSize': ['5533359'], 'resumableType': [''], 'resumableIdentifier': ['5533359-VibrationLog_2023-05
-05-084134tdms'], 'resumableFilename': ['Vibration Log_2023-05-05 - 084134.tdms'], 'resumableRelativePath': ['Vibration Log_2023-05-05 - 084134.tdms'], 'resumableTotalChunks': ['5']}>
[19/Jun/2023 14:50:44] "POST /tdms/api/chunked_upload/?resumableChunkNumber=5&resumableChunkSize=1048576&resumableCurrentChunkSize=1339055&resumableTotalSize=5533359&resumableType=&resumableIdentifier=5533359-VibrationLog_2023-05-05
-084134tdms&resumableFilename=Vibration%20Log_2023-05-05%20-%20084134.tdms&resumableRelativePath=Vibration%20Log_2023-05-05%20-%20084134.tdms&resumableTotalChunks=5 HTTP/1.1" 302 0
[19/Jun/2023 14:50:44] "GET /tdms/list_vibration_logs/ HTTP/1.1" 200 7346
[19/Jun/2023 14:50:44] "GET /tdms/list_vibration_logs/ HTTP/1.1" 200 7346
[19/Jun/2023 14:50:44] "GET /tdms/list_vibration_logs/ HTTP/1.1" 200 7346
[19/Jun/2023 14:50:44] "GET /tdms/list_vibration_logs/ HTTP/1.1" 200 7346
<QueryDict: {}>
[19/Jun/2023 14:50:44] "POST /tdms/api/chunked_upload_complete/ HTTP/1.1" 302 0
[19/Jun/2023 14:50:44] "GET /tdms/list_vibration_logs/ HTTP/1.1" 200 7346
which seems okay, but the file does not get saved anywhere.
This is my models.py:
from django.db import models
from nptdms import TdmsFile
import json
import pytdms
from chunked_upload.models import ChunkedUpload
# Create your models here.
class VibrationLogChunkedUpload(ChunkedUpload):
name = models.CharField(max_length=100, blank=True, null=True)
notes = models.TextField(blank=True, null=True)
date_created = models.DateTimeField(auto_now_add=True)
data = models.JSONField(blank=True, null=True)
sample_frequency = models.FloatField(blank=True, null=True, default=8533.33)
tdms_file = models.FileField(upload_to='tdms')
def __str__(self):
return self.name
# def save_tdms(self):
# objects, rawdata = pytdms.read(self.tdms_file.path)
# rawdata = {key.decode(): value for key, value in rawdata.items()}
# rawdata = {k: v.tolist() for k, v in rawdata.items()}
# self.data = json.dumps(rawdata)
# self.save()
class TDMSDashboard(models.Model):
notes = models.TextField(blank=True, null=True)
vibration_log = models.ForeignKey('VibrationLogChunkedUpload', on_delete=models.CASCADE)
name = models.CharField(max_length=100, blank=True, null=True)
time = models.DateTimeField(auto_now_add=True, null=True, blank=True)
saved = models.BooleanField(blank=True, null=True)
This is my views.py:
from django.shortcuts import render
from django.shortcuts import render, redirect, get_object_or_404
from django.urls import reverse, reverse_lazy
from .forms import VibrationLogModelForm
from .models import TDMSDashboard, VibrationLogChunkedUpload
from django.views.generic import View, ListView, DetailView, DeleteView, UpdateView
import pandas as pd
from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin
from django.contrib.auth.decorators import login_required, permission_required
import json
from .utils import *
from django.contrib import messages
# Create your views here.
from chunked_upload.views import ChunkedUploadView, ChunkedUploadCompleteView
from .models import VibrationLogChunkedUpload
class VibrationLogChunkedUploadView(ChunkedUploadView):
model = VibrationLogChunkedUpload
field_name = 'tdms_file'
def post(self, request):
print(request.POST)
return redirect(
reverse('tdms:list_vibration_logs'))
class VibrationLogChunkedUploadCompleteView(ChunkedUploadCompleteView):
model = VibrationLogChunkedUpload
def on_completion(self, uploaded_file, request):
print(request.POST)
def post(self, request):
print(request.POST)
return redirect(
reverse('tdms:list_vibration_logs'))
def get_response_data(self, chunked_upload, request):
return {'message': "File uploaded successfully!", 'id': chunked_upload.id}
class VibrationLogListView(LoginRequiredMixin, ListView):
model = VibrationLogChunkedUpload
context_object_name = 'vibration_log_list'
ordering = ['-date_created']
paginate_by = 20
template_name = 'tdms/VibrationLog_list.html'
control_system_obj = VibrationLogChunkedUpload.objects.all()
maps = []
def post(self, request):
if request.POST:
pk = int(dict(request.POST)['pk'][0])
vibration_log_obj = VibrationLogChunkedUpload.objects.get(pk=pk)
dashboard_obj = TDMSDashboard(vibration_log=vibration_log_obj)
dashboard_obj.save()
return redirect(
reverse('tdms:dashboard_view',
kwargs={'dashboard': dashboard_obj.pk, 'vibration_log': vibration_log_obj.pk}))
else:
return redirect(reverse('tdms:list_control_systems'))
class VibrationLogCreateView(LoginRequiredMixin, View):
model = VibrationLogChunkedUpload
form_class = VibrationLogModelForm
success_url = reverse_lazy('tdms:list_vibration_logs')
template_name = 'tdms/VibrationLog_create.html'
def get(self, request):
form = self.form_class()
context = {'form': form}
return render(request, self.template_name, context)
def post(self, request):
form = self.form_class(request.POST or None, request.FILES or None)
if form.is_valid():
vibration_log_create = form.save()
vibration_log_create.save()
else:
print(form.errors)
return redirect(self.success_url)
class VibrationLogUpdateView(LoginRequiredMixin, UpdateView):
model = VibrationLogChunkedUpload
template_name = 'tdms/VibrationLog_create.html'
fields = ['name', 'notes', 'tdms_file', 'sample_frequency']
def get_success_url(self, **kwargs):
return reverse_lazy("tdms:list_vibration_logs")
class VibrationLogDeleteView(LoginRequiredMixin, DeleteView):
# CsvCreate_confirm_delete.html
model = VibrationLogChunkedUpload
success_url = reverse_lazy('tdms:list_vibration_logs')
template_name = 'controller/ControlSystem_confirm_delete.html'
class VibrationLogDetailView(LoginRequiredMixin, DetailView):
# CsvCreate_detail.html
model = VibrationLogChunkedUpload
queryset = VibrationLogChunkedUpload.objects.order_by('name')
template_name = 'tdms/VibrationLog_detail.html'
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
pk = self.kwargs['pk']
vibration_log_obj = VibrationLogChunkedUpload.objects.get(pk=pk)
data = json.loads(vibration_log_obj.data)
df = pd.DataFrame(data)
context = {
'Dataframe': df.to_html(),
}
return context
@login_required
def dashboard_view(request, dashboard, vibration_log):
dashboard_obj = TDMSDashboard.objects.get(pk=dashboard)
vibration_log_obj = dashboard_obj.vibration_log
data = json.loads(vibration_log_obj.data)
sample_frequency = vibration_log_obj.sample_frequency
# Dataframe
try:
data.pop("/'Tach'/'Tach'")
except:
print("Key Not Found!")
df = pd.DataFrame(data)
time = np.zeros(len(df))
for i in range(len(df)):
time[i] = i / sample_frequency
df['time'] = time
# Plotting
plots = get_frequency_plots(df)
context = {
'vibration_log': vibration_log_obj,
'plots': plots,
}
return render(request, 'tdms/tdms_dashboard.html', context)
This is my urls.py:
from django.urls import path, include
from .views import dashboard_view, VibrationLogUpdateView, VibrationLogDeleteView, VibrationLogDetailView, \
VibrationLogCreateView, VibrationLogListView, VibrationLogChunkedUploadView, VibrationLogChunkedUploadCompleteView
app_name = 'tdms'
urlpatterns = [
path('create_vibration_log/', VibrationLogCreateView.as_view(), name='create_vibration_log'),
path('list_vibration_logs/', VibrationLogListView.as_view(), name='list_vibration_logs'),
path('dashboard/<int:dashboard>/<int:vibration_log>', dashboard_view, name='dashboard_view'),
path('vibration_log_update/<int:pk>', VibrationLogUpdateView.as_view(), name='update_vibration_log'),
path('delete_vibration_log/<int:pk>', VibrationLogDeleteView.as_view(), name='delete_vibration_log'),
path('vibration_log_detail/<int:pk>', VibrationLogDetailView.as_view(), name='detail_vibration_log'),
path('api/chunked_upload/', VibrationLogChunkedUploadView.as_view(), name='api_chunked_upload'),
path('api/chunked_upload_complete/', VibrationLogChunkedUploadCompleteView.as_view(), name='api_chunked_upload_complete'),
]
This is my html template (including javascript):
{% extends 'base.html' %}
{% block title %}
Create vibration log
{% endblock title %}
{% block content %}
<h1 class="d-flex justify-content-center mb-5">Create your vibration log</h1>
<div class="row">
<div class="col-3"></div>
<div class="col-6 d-flex justify-content-center">
<button id="browseButton">Select Files</button>
{% if messages %}
<ul class="messages">
{% for message in messages %}
<li> {% if message.tags %} class="{{ message.tags }}"{% endif %}>{{ message }}</li>
{% endfor %}
</ul>
{% endif %}
</div>
<div class="col-3"></div>
</div>
<script>
(javascript code from https://github.com/23/resumable.js/blob/master/resumable.js, which is too long to post here)
<script>
function getCookie(name) {
var cookieValue = null;
if (document.cookie && document.cookie !== '') {
var cookies = document.cookie.split(';');
for (var i = 0; i < cookies.length; i++) {
var cookie = cookies[i].trim();
if (cookie.substring(0, name.length + 1) === (name + '=')) {
cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
break;
}
}
}
return cookieValue;
}
var csrftoken = getCookie('csrftoken');
var r = new Resumable({
target: '/tdms/api/chunked_upload/',
chunkSize: 1*1024*1024,
simultaneousUploads: 4,
testChunks: false,
throttleProgressCallbacks: 1,
headers: {'X-CSRFToken': csrftoken},
});
r.assignBrowse(document.getElementById('browseButton'));
r.on('fileAdded', function(file, event){
console.log(file.fileName);
r.upload();
});
r.on('fileSuccess', function(file, message){
console.log(file, message);
fetch('/tdms/api/chunked_upload_complete/', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-CSRFToken': csrftoken,
},
body: JSON.stringify({
upload_id: message.upload_id,
})
})
});
</script>
{% endblock content %}
