Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • bioinfo_iric/pyiricdata
1 result
Show changes
Commits on Source (6)
Hide whitespace changes
Inline Side-by-side
pyiricdata/Client.py
View file @ 890e7c0e
......@@ -2,6 +2,7 @@
import requests
import os
from getpass import getpass
import pandas as pd
import numpy as np
import json
......@@ -13,10 +14,10 @@ from .tools import is_json
class Client:
""" python client to IRICdata API"""
def __init__(self, username, password, url='https://thepond.bioinfo.iric.ca'):
def __init__(self, username, password=None, url='https://thepond.bioinfo.iric.ca'):
self.url = url
self.user = username
self.pwd = password
self.pwd = password if not password is None else getpass()
self.datasets = pd.DataFrame()
self.labs = pd.DataFrame()
......@@ -123,44 +124,30 @@ class Client:
return df
""" Download file according to file_id """
def dwnl_file_content(self, file_id, folder_out='', path_key='external_path'):
if not folder_out or folder_out[0] != '/':
folder_out = os.path.join(os.getcwd(), folder_out)
def dwnl_file_content(self, file_id, folder_out='', filename=''):
folder_out = os.path.join(os.getcwd(), folder_out) if folder_out[0] != '/' else folder_out
os.makedirs(folder_out, exist_ok=True)
file_object = self.get_file_metadata(file_id)
if not file_object is None:
filename = file_object['filename']
filename = file_object['filename'] if not filename else filename
out_file_path = os.path.join(folder_out, filename)
path = os.path.join(self.url, 'secure/datafiles/download', str(file_id))
r = self.session.get(path, allow_redirects=True)
annotations = self.get_file_annotation(file_id)
if path_key in annotations.keys():
dirs = annotations[path_key].split('/')[0:-1]
os.makedirs(os.path.join(folder_out, '/'.join(dirs)), exist_ok=True)
out_file_path = os.path.join(folder_out, annotations[path_key])
else:
out_file_path = os.path.join(folder_out, filename)
if os.path.exists(out_file_path):
sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path)
else:
with open(out_file_path, 'wb') as outfile:
outfile.write(r.content)
""" Write file annotations json to disk """
def dwnl_file_annotation(self, file_id, folder_out='', path_key='external_path'):
if not folder_out or folder_out[0] != '/':
folder_out = os.path.join(os.getcwd(), folder_out)
def dwnl_file_annotation(self, file_id, folder_out='', filename=''):
folder_out = os.path.join(os.getcwd(), folder_out) if folder_out[0] != '/' else folder_out
os.makedirs(folder_out, exist_ok=True)
annotations = self.get_file_annotation(file_id)
file_meta = self.get_file_metadata(file_id)
if not file_meta is None:
filename = file_meta['filename']
if path_key in annotations.keys():
dirs = annotations[path_key].split('/')[0:-1]
os.makedirs(os.path.join(folder_out, '/'.join(dirs)), exist_ok=True)
out_file_path = os.path.join(folder_out, os.path.splitext(annotations[path_key])[0]+'.json')
else:
out_file_path = os.path.join(folder_out, filename+'.json')
annotations = self.get_file_annotation(file_id)
filename = file_meta['filename'] if not filename else filename
out_file_path = os.path.join(folder_out, filename + '.json')
if os.path.exists(out_file_path):
sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path)
else:
......@@ -168,16 +155,25 @@ class Client:
json.dump(annotations, outfile)
""" Download an entire dataset """
def dwnl_dataset_content(self, dataset_id, folder_out='', path_key='external_path'):
def dwnl_dataset(self, dataset_id, folder_out='', datasetname=''):
dataset = self.get_dataset_filelist(dataset_id)
datasetname = self.datasets.loc[dataset_id].dataset_name if not datasetname else datasetname
for file_id in np.unique(dataset.file_id):
self.dwnl_file_content(file_id, os.path.join(folder_out, self.datasets.loc[dataset_id].dataset_name), path_key='external_path')
self.dwnl_file_content(file_id, os.path.join(folder_out, datasetname))
self.dwnl_file_annotation(file_id, os.path.join(folder_out, datasetname))
""" Download an entire dataset annotations """
def dwnl_dataset_annotation(self, dataset_id, folder_out='', path_key='external_path'):
""" Download an entire dataset following the specified hierarchy """
def dwnl_hierarchy(self, dataset_id, hierarchy_key='hierarchy', folder_out=''):
dataset = self.get_dataset_filelist(dataset_id)
for file_id in np.unique(dataset.file_id):
self.dwnl_file_annotation(file_id, os.path.join(folder_out, self.datasets.loc[dataset_id].dataset_name), path_key='external_path')
annotations = self.get_file_annotation(file_id)
if not annotations is None and hierarchy_key in annotations.keys():
print(annotations[hierarchy_key])
loc = annotations[hierarchy_key].rsplit('/', 1)[0]
folder_path = os.path.join(folder_out, loc)
filename = annotations[hierarchy_key].rsplit('/', 1)[-1]
self.dwnl_file_content(file_id, folder_path, filename)
self.dwnl_file_annotation(file_id, folder_path, filename)
""" Check if lab is available """
def _check_lab_exists(self, lab):
......
pyiricdata/__main__.py
View file @ 890e7c0e
......@@ -8,7 +8,7 @@ from .Client import Client
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-u", "--username", help="Username", type=str, required=True)
parser.add_argument("-p", "--password", help="Password", type=str, required=True)
parser.add_argument("-p", "--password", help="Password", type=str, default=None)
parser.add_argument("--url", help="URL", type=str, default='https://thepond.bioinfo.iric.ca')
parser.add_argument("-U", "--upload", help="Upload", action='store_true')
args = parser.parse_args().__dict__
......@@ -41,8 +41,11 @@ def main():
print('Download datafile using ID or complete dataset using "all"')
dfid = input('Enter datafile ID to download: ')
if dfid == 'all':
client.dwnl_dataset_content(dsid)
client.dwnl_dataset_annotation(dsid)
hierarchy = input('Enter hierarchy ID if applicable: ')
if not hierarchy:
client.dwnl_dataset(dsid)
else:
client.dwnl_hierarchy(dsid, hierarchy)
else:
client.dwnl_file_content(dfid)
client.dwnl_file_annotation(dfid)
......