diff --git a/portal/forms.py b/portal/forms.py index 7d364b07148695f14f1c1d3f06c087e442007965..e13c85ecdf43ad680ac7b17bef5f23e5dac02e60 100755 --- a/portal/forms.py +++ b/portal/forms.py @@ -265,6 +265,11 @@ class BatchRemoveDataFileAnnotationForm(forms.Form): class BatchDeleteDataFileForm(forms.Form): datafiles = forms.MultipleChoiceField(required=False, widget=forms.HiddenInput, label=_('Delete selected files')) + + +class BatchAddDataFileToDatasetForm(forms.Form): + datafiles = forms.MultipleChoiceField(required=True, widget=forms.HiddenInput) + dataset = forms.ModelChoiceField(required=True, label=_('Add selected files to this dataset:'), queryset=DataSet.objects.none()) class DataSetForm(forms.ModelForm): diff --git a/portal/migrations/0034_datafile_filesize.py b/portal/migrations/0034_datafile_filesize.py new file mode 100644 index 0000000000000000000000000000000000000000..9b7debb593048d3b128ce084b58676fd04b16f5a --- /dev/null +++ b/portal/migrations/0034_datafile_filesize.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.17 on 2024-10-02 13:04 + +from django.db import migrations, models + +def add_filesize(apps, schema_editor): + DataFile = apps.get_model('portal', 'DataFile') + for datafile in DataFile.objects.all(): + datafile.save() + if datafile.file and datafile.filesize is None: + datafile.filesize = datafile.file.size + datafile.save() + +def reverse_add_fize(apps, schema_editor): + pass + +class Migration(migrations.Migration): + + dependencies = [ + ('portal', '0033_auto_20230711_1634'), + ] + + operations = [ + migrations.AddField( + model_name='datafile', + name='filesize', + field=models.IntegerField(blank=True, null=True), + ), + migrations.RunPython(add_filesize, reverse_add_fize) + ] diff --git a/portal/models.py b/portal/models.py index 6a57b9346666e1345c1bed176d732a74ec8bdad7..26c79d8909098c6f47ce3fae37ba4e77b9f1ffb8 100755 --- a/portal/models.py +++ b/portal/models.py @@ -182,6 +182,7 @@ class DataFile(models.Model): upload_timestamp = models.DateTimeField(verbose_name=_('Upload Timestamp'), auto_now_add=True) filename = models.CharField(max_length=256, verbose_name=_('Filename')) + filesize = models.IntegerField(null=True, blank=True) iric_data_id = models.CharField(max_length=10, verbose_name=_('ID'), null=True, unique=True) @@ -203,8 +204,11 @@ class DataFile(models.Model): return self.file.name[4:] def save(self, *args, **kwargs): - if self.file and not self.filename: - self.filename = self.file.name + if self.file: + if not self.filename: + self.filename = self.file.name + if not self.filesize: + self.filesize = self.file.size if self.pk is None or self.iric_data_id is None: super(DataFile, self).save(*args, **kwargs) diff --git a/portal/templates/portal/secure/user/datafile_list.html b/portal/templates/portal/secure/user/datafile_list.html index dab805d259e1fab81004f5a24ed399b0daf4e087..750cc4c69a0ae07e668de7e93082a6dccf72d589 100644 --- a/portal/templates/portal/secure/user/datafile_list.html +++ b/portal/templates/portal/secure/user/datafile_list.html @@ -85,6 +85,29 @@ </div> </div> +{# Add to Dataset bulk #} +<div class="modal fade" id="addToDatasetBulkModal" tabindex="-1" role="dialog" aria-hidden="true"> + <div class="modal-dialog modal-dialog-centered" role="document"> + <div class="modal-content"> + <form id="batch_add_to_dataset" action="{% url 'user.datafile-batch-add-to-dataset' %}" method="post"> + <div class="modal-header"> + <h5 class="modal-title">{% trans 'Add to Dataset' %}</h5> + <button type="button" class="close" data-dismiss="modal" aria-label="Close"> + <span aria-hidden="true">×</span> + </button> + </div> + <div class="modal-body"> + {% csrf_token %} + {{ batch_add_datafile_to_dataset_form|crispy }} + </div> + <div class="modal-footer"> + <button type="button" class="btn btn-secondary" data-dismiss="modal">{% trans 'Cancel' %}</button> + <button type="submit" class="btn btn-primary">{% trans 'Add' %}</button> + </div> + </form> + </div> + </div> +</div> {% endblock modals %} @@ -151,15 +174,17 @@ var createDatasetClick = function() { window.location.href = "{% url 'user.dataset-create' %}?files=" + get_selected_datafiles() } - - var deleteBulkClick = function() { var datafiles = get_selected_datafiles() $("form#batch_delete_datafiles").find("#id_datafiles").val(datafiles) $("#delBulkModal").modal("show") } - +var addToDatasetClick = function() { + var datafiles = get_selected_datafiles() + $("form#batch_add_to_dataset").find("#id_datafiles").val(get_selected_datafiles()) + $("#addToDatasetBulkModal").modal("show") +} $(function() { $("#{{ dt_struct.id }}").DataTable({ @@ -200,7 +225,10 @@ $(function() { <br> <a onclick="deleteBulkClick()" id="sel-delete-bulk" href="#"><i class="fa fa-trash mr-1"></i>{% trans 'Delete selected files' %}</a> <br> - <a onclick="createDatasetClick()" id="sel-create-dataset" href="#"><i class="fa fa-upload mr-1"></i>{% trans 'Create Dataset' %}</a>`, + <a onclick="createDatasetClick()" id="sel-create-dataset" href="#"><i class="fa fa-upload mr-1"></i>{% trans 'Create Dataset' %}</a> + + <a onclick="addToDatasetClick()" id="sel-add-to-dataset" href="#"><i class="fa fa-plus-circle mr-1"></i>{% trans 'Add to Dataset' %}</a> + `, 0: "", 1: "Selected 1 row" } diff --git a/portal/urls.py b/portal/urls.py index 0dc56343c77bb28cc3ad8bd77d36e6aa41bd42dd..32514467c447ea4307ff8ec099b4a39473da6424 100644 --- a/portal/urls.py +++ b/portal/urls.py @@ -5,56 +5,91 @@ from django.urls import path from django.views.decorators.csrf import csrf_exempt from .views.public import IndexView -from .views.secure.alert import (AlertCreateView, AlertDeleteView, - AlertJSONListView, AlertListView, - AlertUpdateView) -from .views.secure.api import (AdminDataSetJSONListView, APIView, - DataFileAnnotationJSONView, - DataFileDeleteJSONView, - DataFileLookupJSONListView, - DataFileMetadataJSONView, - UserDataSetJSONListView) +from .views.secure.alert import ( + AlertCreateView, + AlertDeleteView, + AlertJSONListView, + AlertListView, + AlertUpdateView, +) +from .views.secure.api import ( + AdminDataSetJSONListView, + APIView, + DataFileAnnotationJSONView, + DataFileDeleteJSONView, + DataFileLookupJSONListView, + DataFileMetadataJSONView, + UserDataSetJSONListView, +) from .views.secure.dashboard import AdminDashboardView, DashboardView -from .views.secure.datafile import (DataFileAnnotateView, - DataFileAnnotationDownloadView, - DataFileBatchAddAnnotation, - DataFileBatchDelete, - DataFileBatchRemoveAnnotation, - DataFileDeleteView, DataFileDetailsView, - DataFileDownloadView, - DataFilesJSONListView, DataFilesView, - DataFileUpdateView, - DataFileUploadServletView, - DataFileUploadView, - SharedWithMeDataFilesJSONListView, - SharedWithMeDataFilesView, - ajax_batch_datafile_annotations_JSON) -from .views.secure.dataset import (DataSetCreateView, DataSetDeleteView, - DataSetDetailsView, - DataSetDisplayFieldsUpdateView, - DataSetDownload, DataSetFileListView, - DataSetInitDetailsView, DataSetJSONListView, - DataSetUpdateView, DataSetView) -from .views.secure.institution import (InstitutionCreateView, - InstitutionJSONListView, - InstitutionUpdateView, InstitutionView) -from .views.secure.lab import (LabCreateView, LabJSONListView, LabSwitchView, - LabUpdateView, LabDeleteView, LabView) +from .views.secure.datafile import ( + DataFileAnnotateView, + DataFileAnnotationDownloadView, + DataFileBatchAddAnnotation, + DataFileBatchAddToDataSet, + DataFileBatchDelete, + DataFileBatchRemoveAnnotation, + DataFileDeleteView, + DataFileDetailsView, + DataFileDownloadView, + DataFilesJSONListView, + DataFilesView, + DataFileUpdateView, + DataFileUploadServletView, + DataFileUploadView, + SharedWithMeDataFilesJSONListView, + SharedWithMeDataFilesView, + ajax_batch_datafile_annotations_JSON, +) +from .views.secure.dataset import ( + DataSetCreateView, + DataSetDeleteView, + DataSetDetailsView, + DataSetDisplayFieldsUpdateView, + DataSetDownload, + DataSetFileListView, + DataSetInitDetailsView, + DataSetJSONListView, + DataSetUpdateView, + DataSetView, +) +from .views.secure.institution import ( + InstitutionCreateView, + InstitutionJSONListView, + InstitutionUpdateView, + InstitutionView, +) +from .views.secure.lab import ( + LabCreateView, + LabDeleteView, + LabJSONListView, + LabSwitchView, + LabUpdateView, + LabView, +) from .views.secure.login_success import LoginSuccess from .views.secure.settings import AdminSettingsView, change_password -from .views.secure.sharegroups import (AdminShareGroupCreateView, - AdminShareGroupDeleteView, - AdminShareGroupJSONListView, - AdminShareGroupUpdateView, - AdminShareGroupView, - ShareGroupCreateView, - ShareGroupDeleteView, - ShareGroupJSONListView, - ShareGroupUpdateView, ShareGroupView) -from .views.secure.user_profile import (AdminUserView, LDAPUserCreateView, - ProfileUpdateView, UserCreateView, - UserDeleteView, UserJSONListView, - UserUpdateView) +from .views.secure.sharegroups import ( + AdminShareGroupCreateView, + AdminShareGroupDeleteView, + AdminShareGroupJSONListView, + AdminShareGroupUpdateView, + AdminShareGroupView, + ShareGroupCreateView, + ShareGroupDeleteView, + ShareGroupJSONListView, + ShareGroupUpdateView, + ShareGroupView, +) +from .views.secure.user_profile import ( + AdminUserView, + LDAPUserCreateView, + ProfileUpdateView, + UserCreateView, + UserDeleteView, + UserJSONListView, + UserUpdateView, +) urlpatterns = [ # Landing Pages @@ -112,6 +147,7 @@ urlpatterns = [ path('secure/datafiles/details/<slug:iric_data_id>', DataFileDetailsView.as_view(), name='user.datafile-details-iric-data-id'), path('secure/datafiles/batch-delete', DataFileBatchDelete.as_view(), name='user.datafile-batch-delete'), + path('secure/datafiles/batch-add-to-dataset', DataFileBatchAddToDataSet.as_view(), name='user.datafile-batch-add-to-dataset'), # Servlet path('secure/servlet/datafiles/upload', csrf_exempt(DataFileUploadServletView.as_view()), name='servlet-datafile-upload'), diff --git a/portal/views/secure/datafile.py b/portal/views/secure/datafile.py index ec6f40e048ef0f0964b238123aa73877c37dd995..2130f27954b0dd8fbfbe6920b84ee417c9f71f31 100644 --- a/portal/views/secure/datafile.py +++ b/portal/views/secure/datafile.py @@ -1,11 +1,14 @@ import json import logging +import os from django.contrib import messages from django.contrib.admin.utils import flatten from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin -from django.db.models import Q +from django.contrib.postgres.aggregates import ArrayAgg +from django.db.models import Count, Q, Value +from django.db.models.functions import Concat from django.http import JsonResponse from django.http.response import HttpResponse, HttpResponseRedirect from django.template import defaultfilters as filters @@ -18,15 +21,27 @@ from django.views.generic import DetailView, FormView, TemplateView, View from django.views.generic.edit import CreateView, DeleteView, UpdateView from private_storage.views import PrivateStorageDetailView -from ...forms import (BasicDataFileForm, BatchAddDataFileAnnotationForm, - BatchDeleteDataFileForm, - BatchRemoveDataFileAnnotationForm, - DataFileAnnotationForm, DataFileForm, - DataFileServletUploadForm) -from ...models import DataFile, Lab -from ...views import (ActivePageViewMixin, AjaxDatatableBackboneMixin, - DeleteViewMixin, JSONListView, SlugMixin, - SuccessMessageMixin, TokenLoginMixin, UpdateViewMixin) +from ...forms import ( + BasicDataFileForm, + BatchAddDataFileAnnotationForm, + BatchAddDataFileToDatasetForm, + BatchDeleteDataFileForm, + BatchRemoveDataFileAnnotationForm, + DataFileAnnotationForm, + DataFileForm, + DataFileServletUploadForm, +) +from ...models import DataFile, DataSet, Lab +from ...views import ( + ActivePageViewMixin, + AjaxDatatableBackboneMixin, + DeleteViewMixin, + JSONListView, + SlugMixin, + SuccessMessageMixin, + TokenLoginMixin, + UpdateViewMixin, +) logger = logging.getLogger('debug') @@ -52,6 +67,9 @@ class DataFilesView(LoginRequiredMixin, ActivePageViewMixin, AjaxDatatableBackbo context['batch_remove_datafile_annotation_form'] = BatchRemoveDataFileAnnotationForm() context['batch_add_datafile_annotation_form'] = BatchAddDataFileAnnotationForm() context['batch_deletebulk_datafile_form'] = BatchDeleteDataFileForm() + dataset_form = BatchAddDataFileToDatasetForm() + dataset_form.fields['dataset'].queryset = DataSet.objects.writable_by_profile(self.request.user.profile) + context['batch_add_datafile_to_dataset_form'] = dataset_form context['dt_struct']['headers'] = context['dt_struct']['headers'] + [_('Actions')] return context @@ -72,52 +90,72 @@ class DataFilesJSONListView(LoginRequiredMixin, JSONListView): model = DataFile def get_queryset(self): - labs = Lab.objects.filter(Q(pi=self.request.user.profile) | Q(data_managers=self.request.user.profile)).all() + profile = self.request.user.profile + labs = Lab.objects.filter(Q(pi=profile) | Q(data_managers=profile)).all() if labs: # User is a PI or Data Manager for lab - return super().get_queryset().filter(Q(uploaded_by=self.request.user.profile) | Q(lab__in=labs)) + qs = super().get_queryset().filter(Q(uploaded_by=profile) | Q(lab__in=labs)) + else: + qs = super().get_queryset().filter(uploaded_by=profile) - return super().get_queryset().filter(uploaded_by=self.request.user.profile) + return qs.prefetch_related('uploaded_by', 'lab') def get_rows(self): rows = [] - - for o in self.object_list: - datasets = [d.name for d in o.datasets.all()] - datasets_length = len(datasets) - datasets = ', '.join(datasets) - if 1 < datasets_length: - datasets = '<span title="{0}">{1} {2}</span>'.format( - datasets, - datasets_length, - ugettext('DataSets') - ) + objs = self.get_queryset().annotate(ds_count=Count('datasets'), ds_names=ArrayAgg('datasets__name')) + ds_token = ugettext('DataSets') + + objs = self.get_queryset().annotate( + ds_count=Count('datasets'), + ds_names=ArrayAgg('datasets__name'), + uploaded_by_fullname=Concat( + 'uploaded_by__user__first_name', + Value(' '), + 'uploaded_by__user__last_name' + ) + ).values( + 'id', 'filename', 'filesize', 'iric_data_id', 'upload_timestamp', + 'file', 'lab__name', 'ds_count', 'ds_names' ,'uploaded_by_fullname' + ) + ds_token = ugettext('DataSets') + dwnl_token = ugettext('Download File') + annot_token = ugettext('Download Annotation') + view_token = ugettext('View details') + edit_token = ugettext('Edit') + delete_token = ugettext('Delete') + + for o in objs: + ds_str = '' + if o['ds_count'] > 0 : + ds_str = ', '.join(o['ds_names']) + if o['ds_count'] > 1: + ds_str = '<span title="{0}">{1} {2}</span>'.format(ds_str, o['ds_count'], ds_token) row = { - 'id': o.id, + 'id': o['id'], 'sel': '', 'name': '<a href="{0}" class="details" title="{2}">{1}</a>'.format( - reverse('user.datafile-details-iric-data-id', args=[o.iric_data_id]), - o.filename, ugettext('View details')), - 'ext': o.filext.upper(), - 'iric_data_id': '<span class="text-monospace">{}</span>'.format(o.iric_data_id), - 'hash': '<span class="text-monospace">{}</span>'.format(o.hash[:8]), - 'size': filters.filesizeformat(o.file.size), - 'upload_date': timezone.localtime(o.upload_timestamp).strftime('%Y-%m-%d %H:%M:%S %Z'), - 'lab': str(o.lab) if o.lab else '', - 'datasets': datasets, + reverse('user.datafile-details-iric-data-id', args=[o['iric_data_id']]), + o['filename'], view_token), + 'ext': os.path.splitext(o['filename'])[-1][1:].upper(), + 'iric_data_id': '<span class="text-monospace">{}</span>'.format(o['iric_data_id']), + 'hash': '<span class="text-monospace">{}</span>'.format(o['file'][4:12]), + 'size': '' if o['filesize'] is None else filters.filesizeformat(o['filesize']), + 'upload_date': timezone.localtime(o['upload_timestamp']).strftime('%Y-%m-%d %H:%M:%S %Z'), + 'lab': '' if o['lab__name'] is None else o['lab__name'], + 'datasets': ds_str, 'action_buttons': render_to_string( 'portal/templates/portal/widgets/action_buttons.html', {'buttons': [ - (reverse('user.datafile-update-iric-data-id', args=[o.iric_data_id]), 'fas fa-pencil-alt', ugettext('Edit')), - (reverse('user.datafile-download-iric-data-id', args=[o.iric_data_id]), 'fas fa-file-download', ugettext('Download File'), 'info', 'download'), - (reverse('user.datafile-annotation-download-iric-data-id', args=[o.iric_data_id]), 'fa fa-code', ugettext('Download Annotation'), 'info', 'download'), - (reverse('user.datafile-delete-iric-data-id', args=[o.iric_data_id]), 'fas fa-trash-alt', ugettext('Delete'), 'danger') + (reverse('user.datafile-update-iric-data-id', args=[o['iric_data_id']]), 'fas fa-pencil-alt', edit_token), + (reverse('user.datafile-download-iric-data-id', args=[o['iric_data_id']]), 'fas fa-file-download', dwnl_token, 'info', 'download'), + (reverse('user.datafile-annotation-download-iric-data-id', args=[o['iric_data_id']]), 'fa fa-code', annot_token, 'info', 'download'), + (reverse('user.datafile-delete-iric-data-id', args=[o['iric_data_id']]), 'fas fa-trash-alt', delete_token, 'danger') ]}) } if self.request.user.profile.is_pi_or_data_manager: - row.update({'uploaded_by': o.uploaded_by.fullname}) + row.update({'uploaded_by': o['uploaded_by_fullname']}) rows.append(row) @@ -128,46 +166,62 @@ class SharedWithMeDataFilesJSONListView(LoginRequiredMixin, JSONListView): model = DataFile def get_queryset(self): - return DataFile.objects.accessible_to_profile(self.request.user.profile).exclude(uploaded_by=self.request.user.profile).distinct() + return DataFile.objects.accessible_to_profile(self.request.user.profile).exclude( + uploaded_by=self.request.user.profile + ).prefetch_related( + 'uploaded_by', 'lab' + ).distinct() def get_rows(self): rows = [] + + objs = self.get_queryset().annotate( + ds_count=Count('datasets'), + ds_names=ArrayAgg('datasets__name'), + uploaded_by_fullname=Concat( + 'uploaded_by__user__first_name', + Value(' '), + 'uploaded_by__user__last_name' + ) + ).values( + 'id', 'filename', 'filesize', 'iric_data_id', 'upload_timestamp', + 'file', 'lab__name', 'ds_count', 'ds_names', 'uploaded_by_fullname' + ) - for o in self.object_list: - datasets = [d.name for d in o.datasets.all()] - datasets_length = len(datasets) - datasets = ', '.join(datasets) - if 1 < datasets_length: - datasets = '<span title="{0}">{1} {2}</span>'.format( - datasets, - datasets_length, - ugettext('DataSets') - ) + ds_token = ugettext('DataSets') + dwnl_token = ugettext('Download File') + annot_token = ugettext('Download Annotation') + view_token = ugettext('View details') + + for o in objs: + ds_str = '' + if o['ds_count'] > 0 : + ds_str = ', '.join(o['ds_names']) + if o['ds_count'] > 1: + ds_str = '<span title="{0}">{1} {2}</span>'.format(ds_str, o['ds_count'], ds_token) row = { - 'id': o.id, + 'id': o['id'], 'sel': '', 'name': '<a href="{0}" class="details" title="{2}">{1}</a>'.format( - reverse('user.datafile-details-iric-data-id', args=[o.iric_data_id]), - o.filename, ugettext('View details')), - 'iric_data_id': '<span class="text-monospace">{}</span>'.format(o.iric_data_id), - 'hash': '<span class="text-monospace">{}</span>'.format(o.hash[:8]), - 'ext': o.filext.upper(), - 'size': filters.filesizeformat(o.file.size), - 'upload_date': timezone.localtime(o.upload_timestamp).strftime('%Y-%m-%d %H:%M:%S %Z'), - 'datasets': datasets, - 'uploaded_by': o.uploaded_by.fullname, - 'lab': str(o.lab) if o.lab else '', + reverse('user.datafile-details-iric-data-id', args=[o['iric_data_id']]), + o['filename'], view_token), + 'iric_data_id': '<span class="text-monospace">{}</span>'.format(o['iric_data_id']), + 'hash': '<span class="text-monospace">{}</span>'.format(o['file'][4:12]), + 'ext': os.path.splitext(o['filename'])[-1][1:].upper(), + 'size': '' if o['filesize'] is None else filters.filesizeformat(o['filesize']), + 'upload_date': timezone.localtime(o['upload_timestamp']).strftime('%Y-%m-%d %H:%M:%S %Z'), + 'datasets': ds_str, + 'uploaded_by': o['uploaded_by_fullname'], + 'lab': '' if o['lab__name'] is None else o['lab__name'], 'action_buttons': render_to_string( 'portal/templates/portal/widgets/action_buttons.html', {'buttons': [ - (reverse('user.datafile-download-iric-data-id', args=[o.iric_data_id]), 'fas fa-file-download', ugettext('Download File'), 'info', 'download'), - (reverse('user.datafile-annotation-download-iric-data-id', args=[o.iric_data_id]), 'fa fa-code', ugettext('Download Annotation'), 'info', 'download') + (reverse('user.datafile-download-iric-data-id', args=[o['iric_data_id']]), 'fas fa-file-download', dwnl_token, 'info', 'download'), + (reverse('user.datafile-annotation-download-iric-data-id', args=[o['iric_data_id']]), 'fa fa-code', annot_token, 'info', 'download') ]}) } - rows.append(row) - return rows @@ -498,3 +552,27 @@ class DataFileBatchDelete(LoginRequiredMixin, SuccessMessageMixin, View): if deleted_files: messages.success(self.request, '{0}<br />{1}'.format(_('Successfully deleted file(s) : '), '<br />'.join(deleted_files))) return HttpResponseRedirect(self.success_url) + + +class DataFileBatchAddToDataSet(LoginRequiredMixin, SuccessMessageMixin, View): + http_method_names = ['post'] + success_url = reverse_lazy("user.datafiles") + + def post(self, request, *args, **kwargs): + post_datafiles = self.request.POST.get('datafiles').split(',') + files = DataFile.objects.filter(Q(iric_data_id__in=post_datafiles) | Q(pk__in=post_datafiles)) + + post_dataset = self.request.POST.get('dataset') + dataset = DataSet.objects.filter(Q(iric_data_id__in=post_dataset) | Q(pk__in=post_dataset)).first() + + if dataset is None: + messages.error(self.request, _('No dataset selected')) + elif not DataSet.objects.writable_by_profile(self.request.user.profile).filter(pk=dataset.pk).first(): + messages.error(self.request, _('You do not have the necessary permissions to add files to dataset')) + elif files.count() == 0: + messages.error(self.request, _('No files selected')) + else: + dataset.files.add(*files) + dataset.save() + messages.success(self.request, _('Successfully added files to dataset')) + return HttpResponseRedirect(self.success_url)