Skip to content
Snippets Groups Projects
Commit 5ee76356 authored by Albert Feghaly's avatar Albert Feghaly
Browse files

Fix bug and cleanup a bit

parent 2a4c0057
No related branches found
No related tags found
1 merge request!11Resolve "Allow to keep file content in memory"
...@@ -190,37 +190,42 @@ class Client: ...@@ -190,37 +190,42 @@ class Client:
try: try:
file_metadata = self.get_file_metadata(file_id) file_metadata = self.get_file_metadata(file_id)
file_content = self.session.get(path, allow_redirects=True).content file_content = self.session.get(path, allow_redirects=True).content
file_annotation = self.get_file_metadata(file_id) file_annotation = self.get_file_annotation(file_id)
return IDF(file_metadata, file_content, file_annotation) return IDF(file_metadata, file_content, file_annotation)
except TypeError: except TypeError:
return IDF(None, None, None) return IDF(None, None, None)
""" Download file according to file_id """ """ Download file according to file_id """
def dwnl_file_content(self, file_id, folder_out='', filename=''): def dwnl_file_content(self, file_id, folder_out=None, filename=None):
idf = self.get_file(file_id) idf = self.get_file(file_id)
if idf.data is not None: if idf.data is not None:
if folder_out and folder_out[0] == '/': if folder_out:
pass if folder_out[0] != '/':
folder_out = os.path.join(os.getcwd(), folder_out)
else: else:
folder_out = os.path.join(os.getcwd(), folder_out) folder_out = os.getcwd()
os.makedirs(folder_out, exist_ok=True) filename = idf.metadata['filename'] if filename is None
filename = idf.metadata['filename'] if not filename else filename
out_file_path = os.path.join(folder_out, filename) out_file_path = os.path.join(folder_out, filename)
if os.path.exists(out_file_path): if os.path.exists(out_file_path):
sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path) sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path)
else: else:
os.makedirs(folder_out, exist_ok=True)
with open(out_file_path, 'wb') as outfile: with open(out_file_path, 'wb') as outfile:
print('Downloading %s' % out_file_path) print('Downloading %s' % out_file_path)
outfile.write(idf.data) outfile.write(idf.data)
""" Write file annotations json to disk """ """ Write file annotations json to disk """
def dwnl_file_annotation(self, file_id, folder_out='', filename=''): def dwnl_file_annotation(self, file_id, folder_out=None, filename=None):
folder_out = folder_out if folder_out and folder_out[0] == '/' else os.path.join(os.getcwd(), folder_out) if folder_out:
if folder_out[0] != '/':
folder_out = os.path.join(os.getcwd(), folder_out)
else:
folder_out = os.getcwd()
os.makedirs(folder_out, exist_ok=True) os.makedirs(folder_out, exist_ok=True)
file_meta = self.get_file_metadata(file_id) file_meta = self.get_file_metadata(file_id)
if not file_meta is None: if not file_meta is None:
annotations = self.get_file_annotation(file_id) annotations = self.get_file_annotation(file_id)
filename = file_meta['filename'] if not filename else filename filename = file_meta['filename'] if filename is None
out_file_path = os.path.join(folder_out, filename + '.json') out_file_path = os.path.join(folder_out, filename + '.json')
if os.path.exists(out_file_path): if os.path.exists(out_file_path):
sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path) sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path)
...@@ -229,9 +234,9 @@ class Client: ...@@ -229,9 +234,9 @@ class Client:
json.dump(annotations, outfile) json.dump(annotations, outfile)
""" Download an entire dataset """ """ Download an entire dataset """
def dwnl_dataset(self, dataset_id, folder_out='', datasetname=''): def dwnl_dataset(self, dataset_id, folder_out=None, datasetname=None):
dataset = self.get_dataset_filelist(dataset_id) dataset = self.get_dataset_filelist(dataset_id)
datasetname = self.datasets.loc[dataset_id].dataset_name if not datasetname else datasetname datasetname = self.datasets.loc[dataset_id].dataset_name if datasetname is None
for file_id in np.unique(dataset.file_id): for file_id in np.unique(dataset.file_id):
self.dwnl_file_content(file_id, os.path.join(folder_out, datasetname)) self.dwnl_file_content(file_id, os.path.join(folder_out, datasetname))
self.dwnl_file_annotation(file_id, os.path.join(folder_out, datasetname)) self.dwnl_file_annotation(file_id, os.path.join(folder_out, datasetname))
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment