Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • bioinfo_iric/pyiricdata
1 result
Show changes
Commits on Source (6)
Hide whitespace changes
Inline Side-by-side
pyiricdata/Client.py
View file @ 890e7c0e
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
import requests import requests
import os import os
from getpass import getpass
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import json import json
...@@ -13,10 +14,10 @@ from .tools import is_json ...@@ -13,10 +14,10 @@ from .tools import is_json
class Client: class Client:
""" python client to IRICdata API""" """ python client to IRICdata API"""
def __init__(self, username, password, url='https://thepond.bioinfo.iric.ca'): def __init__(self, username, password=None, url='https://thepond.bioinfo.iric.ca'):
self.url = url self.url = url
self.user = username self.user = username
self.pwd = password self.pwd = password if not password is None else getpass()
self.datasets = pd.DataFrame() self.datasets = pd.DataFrame()
self.labs = pd.DataFrame() self.labs = pd.DataFrame()
...@@ -123,44 +124,30 @@ class Client: ...@@ -123,44 +124,30 @@ class Client:
return df return df
""" Download file according to file_id """ """ Download file according to file_id """
def dwnl_file_content(self, file_id, folder_out='', path_key='external_path'): def dwnl_file_content(self, file_id, folder_out='', filename=''):
if not folder_out or folder_out[0] != '/': folder_out = os.path.join(os.getcwd(), folder_out) if folder_out[0] != '/' else folder_out
folder_out = os.path.join(os.getcwd(), folder_out)
os.makedirs(folder_out, exist_ok=True) os.makedirs(folder_out, exist_ok=True)
file_object = self.get_file_metadata(file_id) file_object = self.get_file_metadata(file_id)
if not file_object is None: if not file_object is None:
filename = file_object['filename'] filename = file_object['filename'] if not filename else filename
out_file_path = os.path.join(folder_out, filename)
path = os.path.join(self.url, 'secure/datafiles/download', str(file_id)) path = os.path.join(self.url, 'secure/datafiles/download', str(file_id))
r = self.session.get(path, allow_redirects=True) r = self.session.get(path, allow_redirects=True)
annotations = self.get_file_annotation(file_id)
if path_key in annotations.keys():
dirs = annotations[path_key].split('/')[0:-1]
os.makedirs(os.path.join(folder_out, '/'.join(dirs)), exist_ok=True)
out_file_path = os.path.join(folder_out, annotations[path_key])
else:
out_file_path = os.path.join(folder_out, filename)
if os.path.exists(out_file_path): if os.path.exists(out_file_path):
sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path) sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path)
else: else:
with open(out_file_path, 'wb') as outfile: with open(out_file_path, 'wb') as outfile:
outfile.write(r.content) outfile.write(r.content)
""" Write file annotations json to disk """ """ Write file annotations json to disk """
def dwnl_file_annotation(self, file_id, folder_out='', path_key='external_path'): def dwnl_file_annotation(self, file_id, folder_out='', filename=''):
if not folder_out or folder_out[0] != '/': folder_out = os.path.join(os.getcwd(), folder_out) if folder_out[0] != '/' else folder_out
folder_out = os.path.join(os.getcwd(), folder_out)
os.makedirs(folder_out, exist_ok=True) os.makedirs(folder_out, exist_ok=True)
annotations = self.get_file_annotation(file_id)
file_meta = self.get_file_metadata(file_id) file_meta = self.get_file_metadata(file_id)
if not file_meta is None: if not file_meta is None:
filename = file_meta['filename'] annotations = self.get_file_annotation(file_id)
if path_key in annotations.keys(): filename = file_meta['filename'] if not filename else filename
dirs = annotations[path_key].split('/')[0:-1] out_file_path = os.path.join(folder_out, filename + '.json')
os.makedirs(os.path.join(folder_out, '/'.join(dirs)), exist_ok=True)
out_file_path = os.path.join(folder_out, os.path.splitext(annotations[path_key])[0]+'.json')
else:
out_file_path = os.path.join(folder_out, filename+'.json')
if os.path.exists(out_file_path): if os.path.exists(out_file_path):
sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path) sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path)
else: else:
...@@ -168,16 +155,25 @@ class Client: ...@@ -168,16 +155,25 @@ class Client:
json.dump(annotations, outfile) json.dump(annotations, outfile)
""" Download an entire dataset """ """ Download an entire dataset """
def dwnl_dataset_content(self, dataset_id, folder_out='', path_key='external_path'): def dwnl_dataset(self, dataset_id, folder_out='', datasetname=''):
dataset = self.get_dataset_filelist(dataset_id) dataset = self.get_dataset_filelist(dataset_id)
datasetname = self.datasets.loc[dataset_id].dataset_name if not datasetname else datasetname
for file_id in np.unique(dataset.file_id): for file_id in np.unique(dataset.file_id):
self.dwnl_file_content(file_id, os.path.join(folder_out, self.datasets.loc[dataset_id].dataset_name), path_key='external_path') self.dwnl_file_content(file_id, os.path.join(folder_out, datasetname))
self.dwnl_file_annotation(file_id, os.path.join(folder_out, datasetname))
""" Download an entire dataset annotations """ """ Download an entire dataset following the specified hierarchy """
def dwnl_dataset_annotation(self, dataset_id, folder_out='', path_key='external_path'): def dwnl_hierarchy(self, dataset_id, hierarchy_key='hierarchy', folder_out=''):
dataset = self.get_dataset_filelist(dataset_id) dataset = self.get_dataset_filelist(dataset_id)
for file_id in np.unique(dataset.file_id): for file_id in np.unique(dataset.file_id):
self.dwnl_file_annotation(file_id, os.path.join(folder_out, self.datasets.loc[dataset_id].dataset_name), path_key='external_path') annotations = self.get_file_annotation(file_id)
if not annotations is None and hierarchy_key in annotations.keys():
print(annotations[hierarchy_key])
loc = annotations[hierarchy_key].rsplit('/', 1)[0]
folder_path = os.path.join(folder_out, loc)
filename = annotations[hierarchy_key].rsplit('/', 1)[-1]
self.dwnl_file_content(file_id, folder_path, filename)
self.dwnl_file_annotation(file_id, folder_path, filename)
""" Check if lab is available """ """ Check if lab is available """
def _check_lab_exists(self, lab): def _check_lab_exists(self, lab):
......
pyiricdata/__main__.py
View file @ 890e7c0e
...@@ -8,7 +8,7 @@ from .Client import Client ...@@ -8,7 +8,7 @@ from .Client import Client
def main(): def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("-u", "--username", help="Username", type=str, required=True) parser.add_argument("-u", "--username", help="Username", type=str, required=True)
parser.add_argument("-p", "--password", help="Password", type=str, required=True) parser.add_argument("-p", "--password", help="Password", type=str, default=None)
parser.add_argument("--url", help="URL", type=str, default='https://thepond.bioinfo.iric.ca') parser.add_argument("--url", help="URL", type=str, default='https://thepond.bioinfo.iric.ca')
parser.add_argument("-U", "--upload", help="Upload", action='store_true') parser.add_argument("-U", "--upload", help="Upload", action='store_true')
args = parser.parse_args().__dict__ args = parser.parse_args().__dict__
...@@ -41,8 +41,11 @@ def main(): ...@@ -41,8 +41,11 @@ def main():
print('Download datafile using ID or complete dataset using "all"') print('Download datafile using ID or complete dataset using "all"')
dfid = input('Enter datafile ID to download: ') dfid = input('Enter datafile ID to download: ')
if dfid == 'all': if dfid == 'all':
client.dwnl_dataset_content(dsid) hierarchy = input('Enter hierarchy ID if applicable: ')
client.dwnl_dataset_annotation(dsid) if not hierarchy:
client.dwnl_dataset(dsid)
else:
client.dwnl_hierarchy(dsid, hierarchy)
else: else:
client.dwnl_file_content(dfid) client.dwnl_file_content(dfid)
client.dwnl_file_annotation(dfid) client.dwnl_file_annotation(dfid)
......