pyiricdata/Client.py

@@ -125,7 +125,7 @@ class Client:
 
     """ Download file according to file_id """
     def dwnl_file_content(self, file_id, folder_out='', filename=''):
-        folder_out = os.path.join(os.getcwd(), folder_out) if folder_out[0] != '/' else folder_out
+        folder_out = folder_out if folder_out and folder_out[0] == '/' else os.path.join(os.getcwd(), folder_out)
         os.makedirs(folder_out, exist_ok=True)
         file_object = self.get_file_metadata(file_id)
         if not file_object is None:
@@ -141,7 +141,7 @@ class Client:
 
     """ Write file annotations json to disk """
     def dwnl_file_annotation(self, file_id, folder_out='', filename=''):
-        folder_out = os.path.join(os.getcwd(), folder_out) if folder_out[0] != '/' else folder_out
+        folder_out = folder_out if folder_out and folder_out[0] == '/' else os.path.join(os.getcwd(), folder_out)
         os.makedirs(folder_out, exist_ok=True)
         file_meta = self.get_file_metadata(file_id)
         if not file_meta is None:
@@ -163,17 +163,30 @@ class Client:
             self.dwnl_file_annotation(file_id, os.path.join(folder_out, datasetname))
 
     """ Download an entire dataset following the specified hierarchy """
-    def dwnl_hierarchy(self, dataset_id, hierarchy_key='hierarchy', folder_out=''):
+    def dwnl_hierarchy(self, dataset_id, hierarchy_dirs=[], hierarchy_fn='', folder_out='', backup_folder=None):
+        hierarchy = hierarchy_dirs + [hierarchy_fn]
         dataset = self.get_dataset_filelist(dataset_id)
-        for file_id in np.unique(dataset.file_id):
+        for file_id in dataset.file_id:
             annotations = self.get_file_annotation(file_id)
-            if not annotations is None and hierarchy_key in annotations.keys():
-                print(annotations[hierarchy_key])
-                loc = annotations[hierarchy_key].rsplit('/', 1)[0]
-                folder_path = os.path.join(folder_out, loc)
-                filename = annotations[hierarchy_key].rsplit('/', 1)[-1]
+            assert annotations is not None  # should never fail as file_id will exist in dataset_id
+            folder_path = None
+            if all(key in annotations.keys() for key in hierarchy):
+                folder_path = os.path.join(folder_out, '/'.join([annotations[key] for key in hierarchy_dirs]))
+                filename = annotations[hierarchy_fn]
+            else:
+                sys.stderr.write('WARNING: File %s (%s) had at least one missing field in its hierarchy\n' %
+                        (filename, file_id))
+                if backup_folder is not None:
+                    folder_path = os.path.join(folder_out, backup_folder)
+                    filename = self.get_file_metadata(file_id)['filename']
+
+            if not folder_path is None:
                 self.dwnl_file_content(file_id, folder_path, filename)
                 self.dwnl_file_annotation(file_id, folder_path, filename)
+                    
+
+    #def _check_dataset_annotation(self, dataset_id, annotation=[]):
+    #    set([self.get_file_annotation[file_id].keys() for file_id in self.get_dataset_filelist(dataset_id)['file_id']])
 
     """ Check if lab is available """
     def _check_lab_exists(self, lab):