import os import uuid from shutil import copyfile from django.conf import settings from django.contrib.auth.models import User from django.contrib.postgres.fields import HStoreField from django.db import models from django.utils.translation import get_language from django.utils.translation import ugettext_lazy as _ from private_storage.fields import PrivateFileField from fsutils import md5sum class ProfileManager(models.Manager): def exclude_profile(self, profile): return super().get_queryset().exclude(id=profile.id) class Profile(models.Model): """example Documentation: https://simpleisbetterthancomplex.com/tutorial/2016/07/22/how-to-extend-django-user-model.html#onetoone """ class Meta: ordering = ['user__last_name', 'user__first_name'] accountname = models.CharField(max_length=150, null=True, blank=True) unix_username = models.CharField(max_length=150, null=True, blank=True, verbose_name=_("Unix username")) user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='profile') labs = models.ManyToManyField('Lab', related_name='members') objects = ProfileManager() def __str__(self): return '{0}, {1} <{2}>'.format(self.user.last_name, self.user.first_name, self.user.email) @property def fullname(self): return self.user.get_full_name() class Institution(models.Model): class Meta: unique_together = (('abbr', 'name')) ordering = ['name'] abbr = models.CharField(max_length=64, null=True, blank=True, unique=True, verbose_name=_('Abbreviation')) name = models.CharField(max_length=256, unique=True, verbose_name=_('Name')) def __str__(self): return self.name def get_identifier(self): if self.abbr: return self.abbr else: return self.name class Lab(models.Model): class Meta: ordering = ['name'] name = models.CharField(max_length=64, verbose_name=_('Name')) ldap = models.CharField(max_length=32, null=True, blank=True, verbose_name='LDAP') pi = models.OneToOneField( Profile, on_delete=models.SET_NULL, null=True, blank=True, limit_choices_to={'user__groups__name': settings.AUTH_LDAP_PI_GROUP}, related_name='pi_of') data_managers = models.ManyToManyField(Profile, related_name='data_manager_of', verbose_name=_('Data managers')) institution = models.ForeignKey(Institution, on_delete=models.PROTECT, related_name='labs') def __str__(self): return self.name class DataFileManager(models.Manager): def limit_to_profile(self, profile): return super().get_queryset().filter(uploaded_by=profile) # File Object ######## class DataFile(models.Model): """ DataFile is the basic file object to which a number of annotations can be attached. File always belongs to 1 (and only 1) lab. Access is granted to lab PI, Lab data managers and original Uploader. To grant further access, files must be bundled in 'DataSets' for which independent access can be granted File annotations are handled trough the use of PostgreSQL's hstore datatype https://docs.djangoproject.com/en/2.2/ref/contrib/postgres/fields/#hstorefield https://docs.djangoproject.com/en/2.2/ref/contrib/postgres/operations/#create-postgresql-extensions Also of interest: https://www.citusdata.com/blog/2016/07/14/choosing-nosql-hstore-json-jsonb/ """ # Allow passing an NFS path at init? # def __init__(self, src): # self.upload_from_nfs(src) # def get_hash() file = PrivateFileField(max_length=256, null=True) annotations = HStoreField(verbose_name=_('File Annotations'), null=True) upload_ts = models.DateTimeField(verbose_name=_('Upload Timestamp'), auto_now_add=True) uploaded_by = models.ForeignKey(Profile, on_delete=models.PROTECT, verbose_name=_('Uploaded by'), related_name='uploaded_files') lab = models.ForeignKey(Lab, on_delete=models.PROTECT, related_name='files', verbose_name=_('Belongs to this lab')) objects = DataFileManager() def __str__(self): return self.file.name @property def filext(self): ext = os.path.splitext(self.file.name)[-1][1:].lower() return ext @property def name(self): return self.file.name @property def placeholder_icon(self): default = 'fas fa-file-alt' assoc = { 'pdf': 'fas fa-file-pdf', 'doc': 'fas fa-file-word', 'xls': 'fas fa-file-excel', 'ppt': 'fas fa-file-powerpoint', 'png': 'fas fa-file-image', 'jpg': 'fas fa-file-image', 'jpe': 'fas fa-file-image' } try: return assoc[self.filext[:3]] except KeyError: return default def upload_from_nfs(self, src): """ Copy to db storage partition from a local network path Return new path after copy """ # TODO: Code in this function will be moved to the Queuing system tmp_dst = settings.DATA_ROOT + "/" + str(uuid.uuid4()) copyfile(src, tmp_dst) # copy + hash at the same time? md5 = md5sum(tmp_dst).hexdigest() path = "{}/{}___{}".format(settings.DATA_ROOT, os.path.basename(src), md5) os.rename(tmp_dst, path) self.path = path self.save() class ManuallyCreatedDataSetManager(models.Manager): def get_queryset(self): return super().get_queryset().filter(name__isnull=False) class UploadCreatedDataSetManager(models.Manager): def get_queryset(self): return super().get_queryset().filter(name__isnull=True) class DataSet(models.Model): """ DataSets are collections of files for which specific access can be managed. These are akin to folders. Access can be granted to whole labs or distinctly to specific users or user groups. New data imports will also automatically generate a DataSet wrapping the uploaded files. This auto wrap will allow us to track dropzone uploaded files from incomplete uploads and mark them for deletion """ name = models.CharField(max_length=128, verbose_name=_('Name'), null=True) creation_ts = models.DateTimeField(verbose_name=_('Creation Timestamp'), auto_now_add=True) update_ts = models.DateTimeField(verbose_name=_('Last Update Timestamp'), auto_now=True) created_by = models.ForeignKey(Profile, on_delete=models.PROTECT, related_name='datasets') access_granted_to = models.ManyToManyField(Profile, blank=True, related_name='shared_datasets', verbose_name=_('Grant Access to')) open_to_labs = models.ManyToManyField(Lab, related_name='shared_datasets', verbose_name=_('Open to these labs')) files = models.ManyToManyField(DataFile, related_name='datasets', verbose_name=_('Files')) objects = models.Manager() uploads = UploadCreatedDataSetManager() created = ManuallyCreatedDataSetManager() class DataTypes(models.Model): """ DataTypes are a specialization of FSFile for which a set of annotation keys are pre-defined Management of DataTypes should be restricted to specialized staff such as Scientific Platforms personnel """ name = models.CharField(max_length=128, verbose_name=_('Data Type')) annotations = HStoreField(verbose_name=_('Suggested Annotations')) class Alert(models.Model): message_fr = models.TextField(null=True) message_en = models.TextField(null=True) start = models.DateTimeField() end = models.DateTimeField() @property def localized(self): if get_language() == 'en': return self.message_en else: return self.message_fr class AppSettings(models.Model): home_institution = models.ForeignKey(Institution, null=True, blank=True, on_delete=models.SET_NULL)