models.py

import os
import uuid
from shutil import copyfile

from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.postgres.fields import HStoreField
from django.db import models
from django.db.models import Q
from django.db.models.query import QuerySet
from django.utils.translation import get_language
from django.utils.translation import ugettext_lazy as _
from private_storage.fields import PrivateFileField
from django.template import defaultfilters

from fsutils import md5sum


class ProfileManager(models.Manager):
    def exclude_profile(self, profile):
        return super().get_queryset().exclude(id=profile.id)


class Profile(models.Model):
    """example Documentation:
    https://simpleisbetterthancomplex.com/tutorial/2016/07/22/how-to-extend-django-user-model.html#onetoone
    """
    class Meta:
        ordering = ['user__last_name', 'user__first_name']

    accountname = models.CharField(max_length=150, null=True, blank=True)
    unix_username = models.CharField(max_length=150, null=True, blank=True, verbose_name=_("Unix username"))

    user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='profile')
    labs = models.ManyToManyField('Lab', related_name='members')

    objects = ProfileManager()

    def __str__(self):
        return '{0}, {1} <{2}>'.format(self.user.last_name, self.user.first_name, self.user.email)

    @property
    def fullname(self):
        return self.user.get_full_name()

    @property
    def is_pi(self):
        return Lab.objects.filter(pi=self).exists()


class Institution(models.Model):
    class Meta:
        unique_together = (('abbr', 'name'))
        ordering = ['name']

    abbr = models.CharField(max_length=64, null=True, blank=True, unique=True, verbose_name=_('Abbreviation'))
    name = models.CharField(max_length=256, unique=True, verbose_name=_('Name'))

    def __str__(self):
        return self.name

    def get_identifier(self):
        if self.abbr:
            return self.abbr
        else:
            return self.name


class LabManager(models.Manager):
    def limit_to_profile(self, profile):
        return super().get_queryset().filter(members=profile)


class Lab(models.Model):
    class Meta:
        ordering = ['name']

    name = models.CharField(max_length=64, verbose_name=_('Name'))
    ldap = models.CharField(max_length=32, null=True, blank=True, verbose_name='LDAP')

    pi = models.OneToOneField(
        Profile,
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        limit_choices_to={'user__groups__name': settings.AUTH_LDAP_PI_GROUP},
        related_name='pi_of')

    data_managers = models.ManyToManyField(Profile, blank=True, related_name='data_manager_of', verbose_name=_('Data managers'))
    institution = models.ForeignKey(Institution, on_delete=models.PROTECT, related_name='labs')

    objects = LabManager()

    def __str__(self):
        return self.name



class DataFileManager(models.Manager):
    def limit_to_profile(self, profile):
        return super().get_queryset().filter(uploaded_by=profile)

    def shared_with_profile(self, profile):
        return super().get_queryset().filter(
            Q(datasets__share_profiles=profile)
            | Q(datasets__share_labs__members=profile)
            | Q(datasets__share_groups__profiles=profile)
        )

    def accessible_to_profile(self, profile):
        return super().get_queryset().filter(
            Q(uploaded_by=profile)
            | Q(datasets__share_profiles=profile)
            | Q(datasets__share_labs__members=profile)
            | Q(datasets__share_groups__profiles=profile)
        )


# File Object ########
class DataFile(models.Model):
    """
    DataFile is the basic file object to which a number of annotations can be attached.
    File always belongs to 1 (and only 1) lab.
    Access is granted to lab PI, Lab data managers and original Uploader.
    To grant further access, files must be bundled in 'DataSets' for which independent access can be granted
    File annotations are handled trough the use of PostgreSQL's hstore datatype
    https://docs.djangoproject.com/en/2.2/ref/contrib/postgres/fields/#hstorefield
    https://docs.djangoproject.com/en/2.2/ref/contrib/postgres/operations/#create-postgresql-extensions
    Also of interest:
    https://www.citusdata.com/blog/2016/07/14/choosing-nosql-hstore-json-jsonb/
    """
    # Allow passing an NFS path at init?
    # def __init__(self, src):
    #     self.upload_from_nfs(src)

    # def get_hash()

    file = PrivateFileField(max_length=256, null=True)
    annotations = HStoreField(verbose_name=_('File Annotations'), null=True, blank=True)
    upload_ts = models.DateTimeField(verbose_name=_('Upload Timestamp'), auto_now_add=True)

    uploaded_by = models.ForeignKey(Profile, on_delete=models.PROTECT, verbose_name=_('Uploaded by'), related_name='uploaded_files')
    lab = models.ForeignKey(Lab, null=True, blank=True, on_delete=models.PROTECT, related_name='files', verbose_name=_('Belongs to this lab'))

    objects = DataFileManager()

    def __str__(self):
        return self.file.name

    @property
    def filext(self):
        ext = os.path.splitext(self.file.name)[-1][1:].lower()
        return ext

    @property
    def name(self):
        return self.file.name

    @property
    def placeholder_icon(self):
        default = 'fas fa-file-alt'
        assoc = {
            'pdf': 'fas fa-file-pdf',
            'doc': 'fas fa-file-word',
            'xls': 'fas fa-file-excel',
            'ppt': 'fas fa-file-powerpoint',
            'png': 'fas fa-file-image',
            'jpg': 'fas fa-file-image',
            'jpe': 'fas fa-file-image'
        }
        try:
            return assoc[self.filext[:3]]
        except KeyError:
            return default

    def delete(self, using=None, keep_parents=False):
        if self.file:
            self.file.delete()
        super().delete(using, keep_parents)

    def upload_from_nfs(self, src):
        """
        Copy to db storage partition from a local network path
        Return new path after copy
        """
        # TODO: Code in this function will be moved to the Queuing system
        tmp_dst = settings.DATA_ROOT + "/" + str(uuid.uuid4())
        copyfile(src, tmp_dst)  # copy + hash at the same time?
        md5 = md5sum(tmp_dst).hexdigest()
        path = "{}/{}___{}".format(settings.DATA_ROOT, os.path.basename(src), md5)
        os.rename(tmp_dst, path)
        self.path = path
        self.save()


class ShareGroupManager(models.Manager):
    def limit_to_profile(self, profile):
        return super().get_queryset().filter(profiles=profile)


class ShareGroup(models.Model):
    class Meta:
        ordering = ['name']

    name = models.CharField(max_length=64, verbose_name=_('Name'))
    creation_ts = models.DateTimeField(verbose_name=_('Creation Timestamp'), auto_now_add=True)
    update_ts = models.DateTimeField(verbose_name=_('Last Update Timestamp'), auto_now=True)
    last_update_by = models.ForeignKey(Profile, on_delete=models.PROTECT, verbose_name=_('Last Update By'), related_name='my_sharegroups')

    profiles = models.ManyToManyField(Profile, related_name='sharegroups')

    objects = ShareGroupManager()

    def __str__(self):
        return self.name


class DataSetManager(models.Manager):
    def limit_to_profile(self, profile):
        return super().get_queryset().filter(created_by=profile)

    def shared_with_profile(self, profile):
        return super().get_queryset().filter(
            Q(share_profiles=profile)
            | Q(share_labs__members=profile)
            | Q(share_groups__profiles=profile)
        )

    def accessible_to_profile(self, profile):
        return super().get_queryset().filter(
            Q(created_by=profile)
            | Q(share_profiles=profile)
            | Q(share_labs__members=profile)
            | Q(share_groups__profiles=profile)
        )


class DataSet(models.Model):
    """
    DataSets are collections of files for which specific access can be managed.
    These are akin to folders.
    Access can be granted to whole labs or distinctly to specific users or user groups.
    """
    name = models.CharField(max_length=128, verbose_name=_('Name'), null=True)
    creation_ts = models.DateTimeField(verbose_name=_('Creation Timestamp'), auto_now_add=True)
    update_ts = models.DateTimeField(verbose_name=_('Last Update Timestamp'), auto_now=True)

    created_by = models.ForeignKey(Profile, on_delete=models.PROTECT, related_name='my_datasets')
    last_update_by = models.ForeignKey(Profile, on_delete=models.PROTECT, verbose_name=_('Last Update By'))
    share_profiles = models.ManyToManyField(Profile, blank=True, related_name='shared_datasets', verbose_name=_('Share with these users'))
    share_labs = models.ManyToManyField(Lab, blank=True, related_name='shared_datasets', verbose_name=_('Share with these labs'))
    share_groups = models.ManyToManyField(ShareGroup, blank=True, related_name='shared_datasets', verbose_name=_('Share with these groups'))
    files = models.ManyToManyField(DataFile, related_name='datasets', verbose_name=_('Files'))

    objects = DataSetManager()

    def __str__(self):
        return self.name


# class LogActionType(models.Model):
#     name = models.CharField(max_length=64, null=False, blank=False, unique=True, primary_key=True, verbose_name=_('Action'))

class Log(models.Model):
    ts = models.DateTimeField(verbose_name=_('Upload Timestamp'), auto_now_add=True)
    profile = models.ForeignKey(Profile, on_delete=models.PROTECT, related_name='logs')

    TYPE_CHOICES = (
        ('add', _('added')),
        ('remove', _('removed')),
        ('add_share', _('shared with')),
        ('remove_share', _('removed sharing with')),
        ('edit', _('edited')),
    )
    type = models.CharField(max_length=64, choices=TYPE_CHOICES, null=False)

    obj_dataset = models.ForeignKey(DataSet, null=True, on_delete=models.PROTECT, related_name='logs')
    obj_profile = models.ForeignKey(Profile, null=True, on_delete=models.PROTECT)
    obj_lab = models.ForeignKey(Lab, null=True, on_delete=models.PROTECT)
    obj_group = models.ForeignKey(ShareGroup, null=True, on_delete=models.PROTECT, verbose_name=_('Share with these groups'))
    obj_datafile = models.ForeignKey(DataFile, null=True, on_delete=models.PROTECT)
    # obj_datafile_name = models.CharField(max_length=256, null=True)
    # obj_string = models.
    label_en = models.TextField(null=True, blank=True)
    label_fr = models.TextField(null=True, blank=True)

    @property
    def m2m_obj(self):
        obj = {self.obj_profile, self.obj_lab, self.obj_group, self.obj_datafile}
        obj.remove(None)
        if len(obj) == 1:
            return obj.pop()
        return None

    def __str__(self):
        if self.type:
            action = dict(self.TYPE_CHOICES)[self.type]

            if self.obj_dataset:
                if self.type == 'edit':
                    return _("{} ago : {} {} {}").format(defaultfilters.timesince_filter(self.ts), self.profile.fullname, action, self.label_en if get_language() == 'en' else self.label_fr)

                if self.m2m_obj:
                    return _("{} ago : {} {} {} {}").format(defaultfilters.timesince_filter(self.ts), self.profile.fullname, action, self.m2m_obj.__class__.__name__, self.m2m_obj)

        return super().__str__()


class DataTypes(models.Model):
    """
    DataTypes are a specialization of FSFile for which a set of annotation keys are pre-defined
    Management of DataTypes should be restricted to specialized staff such as Scientific Platforms personnel
    """

    name = models.CharField(max_length=128, verbose_name=_('Data Type'))
    annotations = HStoreField(verbose_name=_('Suggested Annotations'))
    creation_ts = models.DateTimeField(verbose_name=_('Creation Timestamp'), auto_now_add=True)
    update_ts = models.DateTimeField(verbose_name=_('Last Update Timestamp'), auto_now=True)

    created_by = models.ForeignKey(Profile, on_delete=models.PROTECT, related_name='my_datatypes')
    last_update_by = models.ForeignKey(Profile, on_delete=models.PROTECT, verbose_name=_('Last Update By'))

    def __str__(self):
        return self.name


class Alert(models.Model):
    message_fr = models.TextField(null=True)
    message_en = models.TextField(null=True)
    start = models.DateTimeField()
    end = models.DateTimeField()

    @property
    def localized(self):
        if get_language() == 'en':
            return self.message_en
        else:
            return self.message_fr


class AppSettings(models.Model):
    home_institution = models.ForeignKey(Institution, null=True, blank=True, on_delete=models.SET_NULL)