Skip to content
Snippets Groups Projects
Commit 5ee76356 authored by Albert Feghaly's avatar Albert Feghaly
Browse files

Fix bug and cleanup a bit

parent 2a4c0057
No related branches found
No related tags found
1 merge request!11Resolve "Allow to keep file content in memory"
......@@ -190,37 +190,42 @@ class Client:
try:
file_metadata = self.get_file_metadata(file_id)
file_content = self.session.get(path, allow_redirects=True).content
file_annotation = self.get_file_metadata(file_id)
file_annotation = self.get_file_annotation(file_id)
return IDF(file_metadata, file_content, file_annotation)
except TypeError:
return IDF(None, None, None)
""" Download file according to file_id """
def dwnl_file_content(self, file_id, folder_out='', filename=''):
def dwnl_file_content(self, file_id, folder_out=None, filename=None):
idf = self.get_file(file_id)
if idf.data is not None:
if folder_out and folder_out[0] == '/':
pass
if folder_out:
if folder_out[0] != '/':
folder_out = os.path.join(os.getcwd(), folder_out)
else:
folder_out = os.path.join(os.getcwd(), folder_out)
os.makedirs(folder_out, exist_ok=True)
filename = idf.metadata['filename'] if not filename else filename
folder_out = os.getcwd()
filename = idf.metadata['filename'] if filename is None
out_file_path = os.path.join(folder_out, filename)
if os.path.exists(out_file_path):
sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path)
else:
os.makedirs(folder_out, exist_ok=True)
with open(out_file_path, 'wb') as outfile:
print('Downloading %s' % out_file_path)
outfile.write(idf.data)
""" Write file annotations json to disk """
def dwnl_file_annotation(self, file_id, folder_out='', filename=''):
folder_out = folder_out if folder_out and folder_out[0] == '/' else os.path.join(os.getcwd(), folder_out)
def dwnl_file_annotation(self, file_id, folder_out=None, filename=None):
if folder_out:
if folder_out[0] != '/':
folder_out = os.path.join(os.getcwd(), folder_out)
else:
folder_out = os.getcwd()
os.makedirs(folder_out, exist_ok=True)
file_meta = self.get_file_metadata(file_id)
if not file_meta is None:
annotations = self.get_file_annotation(file_id)
filename = file_meta['filename'] if not filename else filename
filename = file_meta['filename'] if filename is None
out_file_path = os.path.join(folder_out, filename + '.json')
if os.path.exists(out_file_path):
sys.stderr.write('Warning: File already exists at location %s, skipping.\n' % out_file_path)
......@@ -229,9 +234,9 @@ class Client:
json.dump(annotations, outfile)
""" Download an entire dataset """
def dwnl_dataset(self, dataset_id, folder_out='', datasetname=''):
def dwnl_dataset(self, dataset_id, folder_out=None, datasetname=None):
dataset = self.get_dataset_filelist(dataset_id)
datasetname = self.datasets.loc[dataset_id].dataset_name if not datasetname else datasetname
datasetname = self.datasets.loc[dataset_id].dataset_name if datasetname is None
for file_id in np.unique(dataset.file_id):
self.dwnl_file_content(file_id, os.path.join(folder_out, datasetname))
self.dwnl_file_annotation(file_id, os.path.join(folder_out, datasetname))
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment