feat: Ignore files from transfer

datashuttle/datashuttle_class.py

@@ -104,7 +104,6 @@ class DataShuttle:
     """
 
     def __init__(self, project_name: str, print_startup_message: bool = True):
-
         self._error_on_base_project_name(project_name)
         self.project_name = project_name
         (
@@ -329,6 +328,7 @@ def upload_custom(
         sub_names: Union[str, list],
         ses_names: Union[str, list],
         datatype: Union[List[str], str] = "all",
+        ignore_files: Union[str, list] = "",
         overwrite_existing_files: OverwriteExistingFiles = "never",
         dry_run: bool = False,
         init_log: bool = True,
@@ -362,6 +362,11 @@ def upload_custom(
             The (broad or narrow) NeuroBlueprint datatypes to transfer.
             If "all", any broad or narrow datatype folder will be transferred.
 
+        ignore_files :
+            A list of files to ignore during transfer. This can
+            include wildcards (e.g. "*.json"). This is passed
+            to rclone as an include list.
+
         overwrite_existing_files :
             If `False`, files on central will never be overwritten
             by files transferred from local. If `True`, central files
@@ -392,7 +397,7 @@ def upload_custom(
             datatype,
             overwrite_existing_files,
             dry_run,
-            log=True,
+            "" if self._check_ignore_files(ignore_files) else ignore_files,
         )
 
         if init_log:
@@ -406,6 +411,7 @@ def download_custom(
         sub_names: Union[str, list],
         ses_names: Union[str, list],
         datatype: Union[List[str], str] = "all",
+        ignore_files: Union[str, list] = "",
         overwrite_existing_files: OverwriteExistingFiles = "never",
         dry_run: bool = False,
         init_log: bool = True,
@@ -435,6 +441,11 @@ def download_custom(
         datatype :
             see create_folders()
 
+        ignore_files :
+            A list of files to ignore during transfer. This can
+            include wildcards (e.g. "*.json"). This is passed
+            to rclone as an include list.
+
         overwrite_existing_files :
             If "never" files on target will never be overwritten by source.
             If "always" files on target will be overwritten by source if
@@ -466,7 +477,7 @@ def download_custom(
             datatype,
             overwrite_existing_files,
             dry_run,
-            log=True,
+            "" if self._check_ignore_files(ignore_files) else ignore_files,
         )
 
         if init_log:
@@ -1338,7 +1349,6 @@ def _transfer_entire_project(
                     local to central) or "download" (from central to local).
         """
         for top_level_folder in canonical_folders.get_top_level_folders():
-
             utils.log_and_message(f"Transferring `{top_level_folder}`")
 
             self._transfer_top_level_folder(
@@ -1489,8 +1499,7 @@ def _update_persistent_setting(
 
         if setting_name not in settings:
             utils.log_and_raise_error(
-                f"Setting key {setting_name} not found in "
-                f"settings dictionary",
+                f"Setting key {setting_name} not found in settings dictionary",
                 KeyError,
             )
 
@@ -1571,3 +1580,10 @@ def _check_top_level_folder(self, top_level_folder):
                 f"{canonical_top_level_folders}",
                 ValueError,
             )
+
+    def _check_ignore_files(self, ignore_files: Union[str, list]):
+        """
+        Check if there are any files or folders to be ignored.
+        """
+        test_list = [""]
+        return test_list == ignore_files

datashuttle/tui/custom_widgets.py

@@ -384,6 +384,21 @@ def get_sub_ses_names_and_datatype(
 
         return sub_names, ses_names, datatype
 
+    def get_ignore_files(self, ignore_files_input_key: str) -> List[str]:
+        """
+        Get the ignore files from the input widget.
+        Parameters
+        ----------
+        ignore_files_input_key : str
+            The textual widget id for the ignore files input (prefixed with #)
+        Returns
+            A list of ignore files.
+        -------
+        """
+        ignore_files = self.query_one(ignore_files_input_key).as_names_list()
+
+        return ignore_files
+
 
 class TopLevelFolderSelect(Select):
     """
@@ -470,7 +485,6 @@ def on_select_changed(self, event: Select.Changed) -> None:
         top_level_folder = event.value
 
         if event.value != Select.BLANK:
-
             self.interface.save_tui_settings(
                 top_level_folder, "top_level_folder_select", self.settings_key
             )

datashuttle/tui/interface.py

@@ -297,6 +297,7 @@ def transfer_custom_selection(
         sub_names: List[str],
         ses_names: List[str],
         datatype: List[str],
+        ignore_files: List[str],
         upload: bool,
     ) -> InterfaceOutput:
         """
@@ -317,6 +318,9 @@ def transfer_custom_selection(
         datatype : List[str]
             Datatypes or datatype-level canonical transfer keys to transfer.
 
+        ignore_files : List[str]
+            List of files to ignore during transfer.
+
         upload : bool
             Upload from local to central if `True`, otherwise download
             from central to remote.
@@ -332,6 +336,7 @@ def transfer_custom_selection(
                 sub_names=sub_names,
                 ses_names=ses_names,
                 datatype=datatype,
+                ignore_files=ignore_files,
                 overwrite_existing_files=self.tui_settings[
                     "overwrite_existing_files"
                 ],

datashuttle/tui/tabs/transfer.py

@@ -135,6 +135,14 @@ def compose(self) -> ComposeResult:
                 id="transfer_session_input",
                 placeholder="e.g. ses-001",
             ),
+            Label("Ignore File(s)", id="transfer_ignore_file_label"),
+            ClickableInput(
+                self.mainwindow,
+                id="transfer_ignore_file_input",
+                placeholder="e.g. *.mp4, folder/",
+                validate_on=None,
+                validators=None,
+            ),
             # These are almost identical to create tab
             Label("Datatype(s)", id="transfer_datatype_label"),
             DatatypeCheckboxes(
@@ -214,6 +222,7 @@ def on_mount(self) -> None:
             "#transfer_switch_container",
             "#transfer_subject_input",
             "#transfer_session_input",
+            "#transfer_ignore_file_input",
             "#transfer_all_checkbox",
             "#transfer_all_datatype_checkbox",
             "#transfer_all_non_datatype_checkbox",
@@ -393,11 +402,13 @@ def transfer_data(self) -> Worker[InterfaceOutput]:
                     "#transfer_subject_input", "#transfer_session_input"
                 )
             )
+            ignore_files = self.get_ignore_files("#transfer_ignore_file_input")
             success, output = self.interface.transfer_custom_selection(
                 selected_top_level_folder,
                 sub_names,
                 ses_names,
                 datatype,
+                ignore_files,
                 upload,
             )
 

datashuttle/tui/tooltips.py

@@ -201,6 +201,13 @@ def get_tooltip(id: str) -> str:
             "Use 'all_non_ses' to transfer all other folders only (i.e. that do not start with the 'ses-' prefix)."
         )
 
+    elif id == "#transfer_ignore_file_input":
+        tooltip = (
+            "A list of files or folders to ignore during transfer. "
+            "Use wildcards to match any part of a filename e.g. *.mp4, folder/.\n\n"
+            "Folders must end with '/' suffix, otherwise it will be treated as a file.\n\n"
+        )
+
     # 'all', 'all datatype', 'all non datatype'
     elif id == "#transfer_all_checkbox":
         tooltip = "Select to transfer all datatype and non-datatype folders within sessions."