From 47d7a712f5634c7fefcfbd83a94527c1a8d61931 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Tue, 18 Mar 2025 22:40:36 +0000 Subject: [PATCH 01/25] Removing type hints from all docstrings --- datashuttle/configs/canonical_configs.py | 3 +- datashuttle/configs/config_class.py | 13 +- datashuttle/configs/load_configs.py | 16 +- datashuttle/datashuttle_class.py | 166 +++++++++++--------- datashuttle/datashuttle_functions.py | 6 +- datashuttle/tui/configs.py | 6 +- datashuttle/tui/custom_widgets.py | 12 +- datashuttle/tui/interface.py | 40 ++--- datashuttle/tui/screens/datatypes.py | 18 ++- datashuttle/tui/screens/modal_dialogs.py | 8 +- datashuttle/tui/screens/new_project.py | 3 +- datashuttle/tui/screens/project_selector.py | 2 +- datashuttle/tui/tabs/create_folders.py | 7 +- datashuttle/tui/tabs/transfer.py | 12 +- datashuttle/utils/data_transfer.py | 21 +-- datashuttle/utils/ds_logger.py | 8 +- datashuttle/utils/folders.py | 82 ++++++---- datashuttle/utils/formatting.py | 26 +-- datashuttle/utils/getters.py | 29 ++-- datashuttle/utils/rclone.py | 47 +++--- datashuttle/utils/ssh.py | 40 +++-- datashuttle/utils/validation.py | 38 ++--- 22 files changed, 342 insertions(+), 261 deletions(-) diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index b65ad6c66..7213b3d88 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -76,7 +76,8 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None: Parameters ---------- - config_dict : datashuttle config UserDict + config_dict + datashuttle config UserDict """ canonical_dict = get_canonical_configs() diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 5fd70fcad..7f5a8525c 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -40,10 +40,10 @@ class Configs(UserDict): Parameters ---------- - file_path : + file_path full filepath to save the config .yaml file to. - input_dict : + input_dict a dict of config key-value pairs to input dict. This must contain all canonical_config keys """ @@ -144,9 +144,11 @@ def build_project_path( Parameters ---------- - base: "local", "central" or "datashuttle" + base + "local", "central" or "datashuttle" - sub_folders: a list (or string for 1) of + sub_folders + a list (or string for 1) of folder names to be joined into a path. If file included, must be last entry (with ext). """ @@ -177,7 +179,8 @@ def get_base_folder( Parameters ---------- - base : base path, "local", "central" or "datashuttle" + base + base path, "local", "central" or "datashuttle" """ if base == "local": diff --git a/datashuttle/configs/load_configs.py b/datashuttle/configs/load_configs.py index aac1bc93d..c15e3977d 100644 --- a/datashuttle/configs/load_configs.py +++ b/datashuttle/configs/load_configs.py @@ -25,11 +25,14 @@ def attempt_load_configs( Parameters ---------- - project_name : name of project + project_name + name of project - config_path : path to datashuttle config .yaml file + config_path + path to datashuttle config .yaml file - verbose : warnings and error messages will be printed. + verbose + warnings and error messages will be printed. """ exists = config_path.is_file() @@ -72,8 +75,11 @@ def convert_str_and_pathlib_paths( Parameters ---------- - config_dict : DataShuttle.cfg dict of configs - direction : "path_to_str" or "str_to_path" + config_dict + DataShuttle.cfg dict of configs + + direction + "path_to_str" or "str_to_path" """ for path_key in canonical_configs.keys_str_on_file_but_path_in_class(): value = config_dict[path_key] diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 173a3f46b..e58b9087f 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -90,17 +90,19 @@ class DataShuttle: Parameters ---------- - project_name : The project name to use the datashuttle - Folders containing all project files - and folders are specified in make_config_file(). - Datashuttle-related files are stored in - a .datashuttle folder in the user home - folder. Use get_datashuttle_path() to - see the path to this folder. - - print_startup_message : If `True`, a start-up message displaying the - current state of the program (e.g. persistent - settings such as the 'top-level folder') is shown. + project_name + The project name to use the datashuttle + Folders containing all project files + and folders are specified in make_config_file(). + Datashuttle-related files are stored in + a .datashuttle folder in the user home + folder. Use get_datashuttle_path() to + see the path to this folder. + + print_startup_message + If `True`, a start-up message displaying the + current state of the program (e.g. persistent + settings such as the 'top-level folder') is shown. """ def __init__(self, project_name: str, print_startup_message: bool = True): @@ -164,39 +166,39 @@ def create_folders( Parameters ---------- - top_level_folder : TopLevelFolder + top_level_folder Whether to make the folders in `rawdata` or `derivatives`. - sub_names : Union[str, List[str]] + sub_names subject name / list of subject names to make within the top-level project folder (if not already, these will be prefixed with "sub-") - ses_names : Optional[Union[str, List[str]]] + ses_names (Optional). session name / list of session names. (if not already, these will be prefixed with "ses-"). If no session is provided, no session-level folders are made. - datatype : Union[str, List[str]] + datatype The datatype to make in the sub / ses folders. (e.g. "ephys", "behav", "anat"). If "" is passed no datatype will be created. Broad or Narrow canonical NeuroBlueprint datatypes are accepted. - bypass_validation : bool + bypass_validation If `True`, folders will be created even if they are not valid to NeuroBlueprint style. - log : bool + log If `True`, details of folder creation will be logged. Returns ------- - created_paths : + created_paths A dictionary of the full filepaths made during folder creation, where the keys are the type of folder made and the values are a list of created folder paths (Path objects). If datatype were @@ -343,37 +345,37 @@ def upload_custom( Parameters ---------- - top_level_folder : + top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) to transfer files and folders within. - sub_names : + sub_names a subject name / list of subject names. These must be prefixed with "sub-", or the prefix will be automatically added. "@*@" can be used as a wildcard. "all" will search for all sub-folders in the datatype folder to upload. - ses_names : + ses_names a session name / list of session names, similar to sub_names but requiring a "ses-" prefix. - datatype : + datatype The (broad or narrow) NeuroBlueprint datatypes to transfer. If "all", any broad or narrow datatype folder will be transferred. - overwrite_existing_files : + overwrite_existing_files If `False`, files on central will never be overwritten by files transferred from local. If `True`, central files will be overwritten if there is any difference (date, size) between central and local files. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. - init_log : + init_log (Optional). Whether to handle logging. This should always be True, unless logger is handled elsewhere (e.g. in a calling function). @@ -417,37 +419,37 @@ def download_custom( Parameters ---------- - top_level_folder : + top_level_folder The top-level folder (e.g. `rawdata`) to transfer files and folders within. - sub_names : + sub_names a subject name / list of subject names. These must be prefixed with "sub-", or the prefix will be automatically added. "@*@" can be used as a wildcard. "all" will search for all sub-folders in the datatype folder to upload. - ses_names : + ses_names a session name / list of session names, similar to sub_names but requiring a "ses-" prefix. - datatype : + datatype see create_folders() - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. - init_log : + init_log (Optional). Whether to handle logging. This should always be True, unless logger is handled elsewhere (e.g. in a calling function). @@ -490,14 +492,14 @@ def upload_rawdata( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -522,14 +524,14 @@ def upload_derivatives( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -554,14 +556,14 @@ def download_rawdata( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -586,14 +588,14 @@ def download_derivatives( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -620,14 +622,14 @@ def upload_entire_project( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -653,14 +655,14 @@ def download_entire_project( Parameters ---------- - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -690,17 +692,17 @@ def upload_specific_folder_or_file( Parameters ---------- - filepath : + filepath a string containing the full filepath. - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -732,17 +734,17 @@ def download_specific_folder_or_file( Parameters ---------- - filepath : + filepath a string containing the full filepath. - overwrite_existing_files : + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. @@ -879,7 +881,7 @@ def write_public_key(self, filepath: str) -> None: Parameters ---------- - filepath : + filepath full filepath (inc filename) to write the public key to. """ @@ -920,10 +922,10 @@ def make_config_file( Parameters ---------- - local_path : + local_path path to project folder on local machine - central_path : + central_path Filepath to central project. If this is local (i.e. connection_method = "local_filesystem"), this is the full path on the local filesystem @@ -933,15 +935,15 @@ def make_config_file( include ~ home folder syntax, must contain the full path (e.g. /nfs/nhome/live/jziminski) - connection_method : + connection_method The method used to connect to the central project filesystem, e.g. "local_filesystem" (e.g. mounted drive) or "ssh" - central_host_id : + central_host_id server address for central host for ssh connection e.g. "ssh.swc.ucl.ac.uk" - central_host_username : + central_host_username username for which to log in to central host. e.g. "jziminski" """ @@ -1081,10 +1083,10 @@ def get_next_sub( Parameters ---------- - return_with_prefix : bool + return_with_prefix If `True`, return with the "sub-" prefix. - include_central : bool + include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. If in local-project mode, this flag is ignored. @@ -1122,16 +1124,16 @@ def get_next_ses( Parameters ---------- - top_level_folder: + top_level_folder "rawdata" or "derivatives" - sub: Optional[str] + sub Name of the subject to find the next session of. - return_with_prefix : bool + return_with_prefix If `True`, return with the "ses-" prefix. - include_central : bool + include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. If in local-project mode, this flag is ignored. @@ -1173,7 +1175,7 @@ def get_name_templates(self) -> Dict: Returns ------- - name_templates : Dict + name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} """ settings = self._load_persistent_settings() @@ -1189,7 +1191,7 @@ def set_name_templates(self, new_name_templates: Dict) -> None: Parameters ---------- - new_name_templates : Dict + new_name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} where "sub" or "ses" can be a regexp that subject and session names respectively are validated against. @@ -1227,20 +1229,20 @@ def validate_project( Parameters ---------- - top_level_folder : TopLevelFolder | None + top_level_folder Folder to check, either "rawdata" or "derivatives". If ``None``, will check both folders. - display_mode : DisplayMode + display_mode The validation issues are displayed as ``"error"`` (raise error) ``"warn"`` (show warning) or ``"print"`` - include_central : bool + include_central If ``False``, only the local project is validated. Otherwise, both local and central projects are validated. If in local-project mode, this flag is ignored. - strict_mode: bool + strict_mode If `True`, only allow NeuroBlueprint-formatted folders to exist in the project. By default, non-NeuroBlueprint folders (e.g. a folder called 'my_stuff' in the 'rawdata') are allowed, and only folders @@ -1293,9 +1295,10 @@ def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: Parameters ---------- - names : + names A string or list of subject or session names. - prefix: + + prefix The relevant subject or session prefix, e.g. "sub-" or "ses-" """ @@ -1328,8 +1331,9 @@ def _transfer_entire_project( Parameters ---------- - upload_or_download : direction to transfer the data, either "upload" (from - local to central) or "download" (from central to local). + upload_or_download + direction to transfer the data, either "upload" (from + local to central) or "download" (from central to local). """ for top_level_folder in canonical_folders.get_top_level_folders(): @@ -1358,12 +1362,14 @@ def _start_log( Parameters ---------- - command_name : name of the command, for the log output files. + command_name + name of the command, for the log output files. - local_vars : local_vars are passed to fancylog variables argument. - see ds_logger.wrap_variables_for_fancylog for more info + local_vars + local_vars are passed to fancylog variables argument. + see ds_logger.wrap_variables_for_fancylog for more info - store_in_temp_folder : + store_in_temp_folder if `False`, existing logging path will be used (local project .datashuttle). """ @@ -1476,8 +1482,12 @@ def _update_persistent_setting( Parameters ---------- - setting_name : dictionary key of the persistent setting to change - setting_value : value to change the persistent setting to + + setting_name + dictionary key of the persistent setting to change + + setting_value + value to change the persistent setting to """ settings = self._load_persistent_settings() diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 94868f7bc..8c220b536 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -40,15 +40,15 @@ def quick_validate_project( Path to the project to validate. Must include the project name, and hold a "rawdata" or "derivatives" folder. - top_level_folder : TopLevelFolder + top_level_folder The top-level folder ("rawdata" or "derivatives") to perform validation. If `None`, both are checked. - display_mode : DisplayMode + display_mode The validation issues are displayed as ``"error"`` (raise error) ``"warn"`` (show warning) or ``"print"``. - name_templates : Dict + name_templates A dictionary of templates for subject and session name to validate against. See ``DataShuttle.set_name_templates()`` for details. diff --git a/datashuttle/tui/configs.py b/datashuttle/tui/configs.py index 974ee08af..c16b1a425 100644 --- a/datashuttle/tui/configs.py +++ b/datashuttle/tui/configs.py @@ -299,7 +299,7 @@ def switch_ssh_widgets_display(self, display_ssh: bool) -> None: Parameters ---------- - display_ssh : bool + display_ssh If `True`, display the SSH-related widgets. """ for widget in self.config_ssh_widgets: @@ -373,11 +373,11 @@ def handle_input_fill_from_select_directory( Parameters ---------- - path_ : Union[Literal[False], Path] + path_ The path returned from `SelectDirectoryTreeScreen`. If `False`, the screen exited with no directory selected. - local_or_central : str + local_or_central The Input to fill with the path. """ if path_ is False: diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index dc7f58887..5f5559e2e 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -316,13 +316,13 @@ def handle_fill_input_from_directorytree( Parameters ---------- - sub_input_key : str + sub_input_key The textual widget id for the subject input (prefixed with #) - ses_input_key : str + ses_input_key The textual widget id for the session input (prefixed with #) - event : DirectoryTreeSpecialKeyPress + event A DirectoryTreeSpecialKeyPress event triggered from the CustomDirectoryTree. """ @@ -346,7 +346,7 @@ def insert_sub_or_ses_name_to_input( see `handle_directorytree_key_pressed` for `sub_input_key` and `ses_input_key`. - name : str + name The sub or ses name to append to the input. """ if name.startswith("sub-"): @@ -399,12 +399,12 @@ class TopLevelFolderSelect(Select): Parameters ---------- - existing_only : bool + existing_only If `True`, only top level folders that actually exist in the project are displayed. Otherwise, all possible canonical top-level-folders are displayed. - id : str + id Textualize widget id """ diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 26ee2f8ba..fd3e7b2c4 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -43,7 +43,7 @@ def select_existing_project(self, project_name: str) -> InterfaceOutput: Parameters ---------- - project_name : str + project_name The name of the datashuttle project to load. Must already exist. """ @@ -64,10 +64,10 @@ def setup_new_project( Parameters ---------- - project_name : str + project_name Name of the project to set up. - cfg_kwargs : Dict + cfg_kwargs The configurations to set the new project to. """ try: @@ -92,7 +92,7 @@ def set_configs_on_existing_project( Parameters ---------- - cfg_kwargs : Dict + cfg_kwargs The configs and new values to update. """ try: @@ -114,13 +114,13 @@ def create_folders( Parameters ---------- - sub_names : List[str] + sub_names A list of un-formatted / unvalidated subject names to create. - ses_names : List[str] + ses_names A list of un-formatted / unvalidated session names to create. - datatype : List[str] + datatype A list of canonical datatype names to create. """ top_level_folder = self.tui_settings["top_level_folder_select"][ @@ -154,10 +154,10 @@ def validate_names( Parameters ---------- - sub_names : List[str] + sub_names List of subject names to format. - ses_names : List[str] + ses_names List of session names to format. """ top_level_folder = self.tui_settings["top_level_folder_select"][ @@ -191,7 +191,7 @@ def transfer_entire_project(self, upload: bool) -> InterfaceOutput: Parameters ---------- - upload : bool + upload Upload from local to central if `True`, otherwise download from central to remote. """ @@ -222,10 +222,10 @@ def transfer_top_level_only( Parameters ---------- - selected_top_level_folder : str + selected_top_level_folder The top level folder selected in the TUI for this transfer window. - upload : bool + upload Upload from local to central if `True`, otherwise download from central to remote. @@ -272,19 +272,19 @@ def transfer_custom_selection( Parameters ---------- - selected_top_level_folder : str + selected_top_level_folder The top level folder selected in the TUI for this transfer window. - sub_names : List[str] + sub_names Subject names or subject-level canonical transfer keys to transfer. - ses_names : List[str] + ses_names Session names or session-level canonical transfer keys to transfer. - datatype : List[str] + datatype Datatypes or datatype-level canonical transfer keys to transfer. - upload : bool + upload Upload from local to central if `True`, otherwise download from central to remote. """ @@ -360,14 +360,14 @@ def save_tui_settings( Parameters ---------- - value : Any + value Value to set the `persistent_settings` tui field to - key_1 : str + key_1 First key of the tui `persistent_settings` to update e.g. "top_level_folder_select" - key_2 : str + key_2 Optionals second level of the dictionary to update. e.g. "create_tab" """ diff --git a/datashuttle/tui/screens/datatypes.py b/datashuttle/tui/screens/datatypes.py index 790cca553..aa731761d 100644 --- a/datashuttle/tui/screens/datatypes.py +++ b/datashuttle/tui/screens/datatypes.py @@ -189,18 +189,20 @@ class DatatypeCheckboxes(Static): Parameters ---------- - settings_key : 'create' if datatype checkboxes for the create tab, - 'transfer' for the transfer tab. Transfer tab includes - additional datatype options (e.g. "all"). + settings_key + 'create' if datatype checkboxes for the create tab, + 'transfer' for the transfer tab. Transfer tab includes + additional datatype options (e.g. "all"). Attributes ---------- - datatype_config : a Dictionary containing datatype as key (e.g. "ephys", "behav") - and values are `bool` indicating whether the checkbox is on / off. - If 'transfer', then transfer datatype arguments (e.g. "all") - are also included. This structure mirrors - the `persistent_settings` dictionaries. + datatype_config + a Dictionary containing datatype as key (e.g. "ephys", "behav") + and values are `bool` indicating whether the checkbox is on / off. + If 'transfer', then transfer datatype arguments (e.g. "all") + are also included. This structure mirrors + the `persistent_settings` dictionaries. Notes ----- diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index 97706db07..4d801e823 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -26,10 +26,10 @@ class MessageBox(ModalScreen): """ A screen for rendering error messages. - message : str + message The message to display in the message box - border_color : str + border_color The color to pass to the `border` style on the widget. Note that the keywords 'red' 'grey' 'green' are overridden for custom style. """ @@ -147,10 +147,10 @@ class SelectDirectoryTreeScreen(ModalScreen): Parameters ---------- - mainwindow : App + mainwindow Textual main app screen - path_ : Optional[Path] + path_ Path to use as the DirectoryTree root, if `None` set to the system user home. """ diff --git a/datashuttle/tui/screens/new_project.py b/datashuttle/tui/screens/new_project.py index 43232e17e..ea7c0826c 100644 --- a/datashuttle/tui/screens/new_project.py +++ b/datashuttle/tui/screens/new_project.py @@ -31,7 +31,8 @@ class NewProjectScreen(Screen): Parameters ---------- - mainwindow : TuiApp + mainwindow + The main TUI app """ TITLE = "Make New Project" diff --git a/datashuttle/tui/screens/project_selector.py b/datashuttle/tui/screens/project_selector.py index 43f9858ab..289990e5c 100644 --- a/datashuttle/tui/screens/project_selector.py +++ b/datashuttle/tui/screens/project_selector.py @@ -29,7 +29,7 @@ class ProjectSelectorScreen(Screen): Parameters ---------- - mainwindow : TuiApp + mainwindow The main TUI app, functions on which are used to coordinate screen display. diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 6907f684c..e596566da 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -286,10 +286,10 @@ def fill_input_with_next_sub_or_ses_template( Parameters - prefix : Prefix + prefix Whether to fill the subject or session Input - input_id : str + input_id The textual input name to update. """ top_level_folder = self.interface.tui_settings[ @@ -367,7 +367,8 @@ def run_local_validation(self, prefix: Prefix): Parameters ---------- - prefix : Prefix + prefix + Whether to run validation on the subject or session Input """ sub_names = self.query_one( "#create_folders_subject_input" diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py index cd9ba9c4b..4603c6674 100644 --- a/datashuttle/tui/tabs/transfer.py +++ b/datashuttle/tui/tabs/transfer.py @@ -52,20 +52,22 @@ class TransferTab(TreeAndInputTab): Parameters ---------- - title : str + title + The title of the tab - mainwindow : App + mainwindow + The main TUI app - interface : Interface + interface TUI-datashuttle interface object - id : str + id The textual widget id. Attributes ---------- - show_legend : bool + show_legend Convenience attribute linked to a global setting exists that turns off / on styling of directorytree nodes based on transfer status. ` diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index 21121b8e6..a39e6377f 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -24,38 +24,39 @@ class TransferData: Parameters ---------- - cfg : Configs, + cfg datashuttle configs UserDict. - upload_or_download : Literal["upload", "download"] + upload_or_download Direction to perform the transfer. - top_level_folder: TopLevelFolder + top_level_folder + The top-level folder structure where data is organized. - sub_names : Union[str, List[str]] + sub_names List of subject names or single subject to transfer. This can include transfer keywords (e.g. "all_non_sub"). - ses_names : Union[str, List[str]] + ses_names List of sessions or single session to transfer, for each subject. May include session-level transfer keywords. - datatype : Union[str, List[str]] + datatype List of datatypes to transfer, for the sessions / subjects specified. Can include datatype-level tranfser keywords. - overwrite_existing_files : OverwriteExistingFiles + overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if there is any difference in date or size. If "if_source_newer" files on target will only be overwritten by files on source with newer creation / modification datetime. - dry_run : bool, + dry_run If `True`, transfer will not actually occur but will be logged as if it did (to see what would happen for a transfer). - log : bool, + log if `True`, log and print the transfer output. """ @@ -123,7 +124,7 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: Returns ------- - include_list : List[str] + include_list A list of paths to pass to rclone's `--include` flag. """ # Find sub names to transfer diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index ca1c6bf94..3d0ac7b69 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -76,10 +76,12 @@ def log_names(list_of_headers: List[Any], list_of_names: List[Any]) -> None: Parameters ---------- - list_of_headers : a list of titles that the names - will be printed under, e.g. "sub_names", "ses_names" + list_of_headers + a list of titles that the names + will be printed under, e.g. "sub_names", "ses_names" - list_of_names : list of names to print to log + list_of_names + list of names to print to log """ for header, names in zip(list_of_headers, list_of_names): utils.log(f"{header}: {names}") diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 56852640c..03cfb38af 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -49,9 +49,11 @@ def create_folder_trees( Parameters ---------- - sub_names, ses_names, datatype : see create_folders() + sub_names, ses_names, datatype + see create_folders() - log : whether to log or not. If True, logging must + log + whether to log or not. If True, logging must already be initialised. """ datatype_passed = datatype not in [[""], ""] @@ -128,22 +130,28 @@ def make_datatype_folders( Parameters ---------- - cfg : ConfigsClass + cfg + datashuttle configs - datatype : datatype (e.g. "behav", "all") to use. Use + datatype + datatype (e.g. "behav", "all") to use. Use empty string ("") for none. - sub_or_ses_level_path : Full path to the subject + sub_or_ses_level_path + Full path to the subject or session folder where the new folder will be written. - level : The folder level that the + level + The folder level that the folder will be made at, "sub" or "ses" - save_paths : A dictionary, which will be filled + save_paths + A dictionary, which will be filled with created paths split by datatype name. - log : whether to log on or not (if True, logging must + log + whether to log on or not (if True, logging must already be initialised). """ datatype_items = cfg.get_datatype_as_dict_items(datatype) @@ -175,9 +183,11 @@ def create_folders(paths: Union[Path, List[Path]], log: bool = True) -> None: Parameters ---------- - paths : Path or list of Paths to create + paths + Path or list of Paths to create - log : if True, log all made folders. This + log + if True, log all made folders. This requires the logger to already be initialised. """ if isinstance(paths, Path): @@ -383,19 +393,24 @@ def search_for_wildcards( Parameters ---------- - project : initialised datashuttle project + project + initialised datashuttle project - base_folder : folder to search for wildcards in + base_folder + folder to search for wildcards in - local_or_central : "local" or "central" project path to + local_or_central + "local" or "central" project path to search in - all_names : list of subject or session names that + all_names + list of subject or session names that may or may not include the wildcard flag. If sub (below) is passed, it is assumed these are session names. Otherwise, it is assumed these are subject names. - sub : optional subject to search for sessions in. If not provided, + sub + optional subject to search for sessions in. If not provided, will search for subjects rather than sessions. """ @@ -448,26 +463,32 @@ def search_sub_or_ses_level( Parameters ---------- - cfg : datashuttle project cfg. Currently, this is used + cfg + datashuttle project cfg. Currently, this is used as a holder for ssh configs to avoid too many arguments, but this is not nice and breaks the general rule that these functions should operate project-agnostic. - local_or_central : search in local or central project + local_or_central + search in local or central project - sub : either a subject name (string) or None. If None, the search + sub + either a subject name (string) or None. If None, the search is performed at the top_level_folder level - ses : either a session name (string) or None, This must not + ses + either a session name (string) or None, This must not be a session name if sub is None. If provided (with sub) then the session folder is searched - str : glob-format search string to search at the + str + glob-format search string to search at the folder level. - verbose : If `True`, if a search folder cannot be found, a message - will be printed with the un-found path. + verbose + If `True`, if a search folder cannot be found, a message + will be printed with the un-found path. """ if ses and not sub: utils.log_and_raise_error( @@ -509,11 +530,18 @@ def search_for_folders( Parameters ---------- - local_or_central : "local" or "central" - search_path : full filepath to search in - search_prefix : file / folder name to search (e.g. "sub-*") - verbose : If `True`, when a search folder cannot be found, a message - will be printed with the missing path. + local_or_central + "local" or "central" + + search_path + full filepath to search in + + search_prefix + file / folder name to search (e.g. "sub-*") + + verbose + If `True`, when a search folder cannot be found, a message + will be printed with the missing path. """ if local_or_central == "central" and cfg["connection_method"] == "ssh": all_folder_names, all_filenames = ssh.search_ssh_central_for_folders( diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index 1cebf49d0..d5cee1d29 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -39,18 +39,18 @@ def check_and_format_names( Parameters ---------- - names : Union[list, str] + names str or list containing sub or ses names (e.g. to create folders) - prefix : Prefix + prefix "sub" or "ses" - this defines the prefix checks. - name_templates : Dict + name_templates A dictionary of templates to validate subject and session name against. e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} where the "sub" and "ses" may contain a regexp to validate against. - bypass_validation : Dict + bypass_validation If `True`, NeuroBlueprint validation will be performed on the passed names. """ @@ -89,9 +89,11 @@ def format_names(names: List, prefix: Prefix) -> List[str]: Parameters ----------- - names: str or list containing sub or ses names (e.g. to make folders) + names + str or list containing sub or ses names (e.g. to make folders) - prefix: "sub" or "ses" - this defines the prefix checks. + prefix + "sub" or "ses" - this defines the prefix checks. """ assert prefix in ["sub", "ses"], "`prefix` must be 'sub' or 'ses'." @@ -198,13 +200,17 @@ def make_list_of_zero_padded_names_across_range( Parameters ---------- - left_number : left (start) number from the range, e.g. "001" + left_number + left (start) number from the range, e.g. "001" - right_number : right (end) number from the range, e.g. "005" + right_number + right (end) number from the range, e.g. "005" - name_start_str : part of the name before the flag, usually "sub-" + name_start_str + part of the name before the flag, usually "sub-" - name_end_str : rest of the name after the flag, i.e. all other + name_end_str + rest of the name after the flag, i.e. all other key-value pairs. """ max_leading_zeros = max( diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index 6ef64870b..1f55465bd 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -50,29 +50,29 @@ def get_next_sub_or_ses( Parameters ---------- - cfg : Configs + cfg datashuttle configs class - top_level_folder: TopLevelFolder + top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) - sub : Optional[str] + sub subject name to search within if searching for sessions, otherwise None to search for subjects - search_str : str + search_str the string to search for within the top-level or subject-level folder ("sub-*") or ("ses-*") are suggested, respectively. - include_central : bool + include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. - return_with_prefix : bool + return_with_prefix If `True`, the next sub or ses value will include the prefix e.g. "sub-001", otherwise the value alone will be returned (e.g. "001") - default_num_value_digits : int + default_num_value_digits If no sub or ses exist in the project, the starting number is 1. Because the number of digits for the project is not accessible, the desired value can be entered here. e.g. if 3 (the default), @@ -80,7 +80,8 @@ def get_next_sub_or_ses( Returns ------- - suggested_new_num : the new suggested sub / ses. + suggested_new_num + the new suggested sub / ses. """ prefix: Prefix @@ -131,7 +132,7 @@ def get_max_sub_or_ses_num_and_value_length( Parameters ---------- - all_folders : List[str] + all_folders A list of BIDS-style formatted folder names. see `get_next_sub_or_ses()` for other arguments. @@ -139,10 +140,10 @@ def get_max_sub_or_ses_num_and_value_length( Returns ------- - max_existing_num : int + max_existing_num The largest number sub / ses value in the past list. - num_value_digits : int + num_value_digits The length of the value in all sub / ses values within the passed list. If these are not consistent, an error is raised. @@ -310,13 +311,13 @@ def get_all_sub_and_ses_paths( Parameters ---------- - cfg : Configs + cfg datashuttle Configs - top_level_folder: TopLevelFolder + top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) - include_central : bool + include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. """ diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 9644c103c..d754e47ee 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -15,9 +15,11 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: Parameters ---------- - command: Rclone command to be run + command + Rclone command to be run - pipe_std: if True, do not output anything to stdout. + pipe_std + if True, do not output anything to stdout. """ command = "rclone " + command if pipe_std: @@ -55,11 +57,12 @@ def setup_rclone_config_for_local_filesystem( Parameters ---------- - rclone_config_name : rclone config name - canonical config name, generated by - datashuttle.cfg.get_rclone_config_name() + rclone_config_name + canonical config name, generated by + datashuttle.cfg.get_rclone_config_name() - log : whether to log, if True logger must already be initialised. + log + whether to log, if True logger must already be initialised. """ call_rclone(f"config create {rclone_config_name} local", pipe_std=True) @@ -81,17 +84,19 @@ def setup_rclone_config_for_ssh( Parameters ---------- - cfg : Configs - datashuttle configs UserDict. + cfg + datashuttle configs UserDict. - rclone_config_name : rclone config name - canonical config name, generated by - datashuttle.cfg.get_rclone_config_name() + rclone_config_name + canonical config name, generated by + datashuttle.cfg.get_rclone_config_name() - ssh_key_path : path to the ssh key used for connecting to - ssh central filesystem, + ssh_key_path + path to the ssh key used for connecting to + ssh central filesystem - log : whether to log, if True logger must already be initialised. + log + whether to log, if True logger must already be initialised. """ call_rclone( f"config create " @@ -158,20 +163,20 @@ def transfer_data( Parameters ---------- - cfg: Configs + cfg datashuttle configs - upload_or_download : Literal["upload", "download"] + upload_or_download If "upload", transfer from `local_path` to `central_path`. "download" proceeds in the opposite direction. - top_level_folder: Literal["rawdata", "derivatives"] + top_level_folder The top-level-folder to transfer files within. - include_list : List[str] + include_list A list of filepaths to include in the transfer - rclone_options : Dict + rclone_options A list of options to pass to Rclone's copy function. see `cfg.make_rclone_transfer_options()`. """ @@ -222,13 +227,13 @@ def get_local_and_central_file_differences( Parameters ---------- - top_level_folders_to_check : + top_level_folders_to_check List of top-level folders to check. Returns ------- - parsed_output : Dict[str, List] + parsed_output A dictionary where the keys are the cases (e.g. "same" across local and central) and the values are lists of paths that fall into these cases. Note the paths are relative to the "rawdata" diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index 8f6de6785..eb8025bd2 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -117,15 +117,19 @@ def setup_ssh_key( Parameters ----------- - ssh_key_path : path to the ssh private key + ssh_key_path + path to the ssh private key - hostkeys_path : path to the ssh host key, once the user + hostkeys_path + path to the ssh host key, once the user has confirmed the key ID this is saved so verification is not required each time. - cfg : datashuttle config UserDict + cfg + datashuttle config UserDict - log : log if True, logger must already be initialised. + log + log if True, logger must already be initialised. """ if not sys.stdin.isatty(): proceed = input( @@ -253,14 +257,18 @@ def search_ssh_central_for_folders( Parameters ----------- - search_path : path to search for folders in + search_path + path to search for folders in - search_prefix : search prefix for folder names e.g. "sub-*" + search_prefix + search prefix for folder names e.g. "sub-*" - cfg : see connect_client_with_logging() + cfg + see connect_client_with_logging() - verbose : If `True`, if a search folder cannot be found, a message - will be printed with the un-found path. + verbose + If `True`, if a search folder cannot be found, a message + will be printed with the un-found path. """ client: paramiko.SSHClient with paramiko.SSHClient() as client: @@ -295,16 +303,20 @@ def get_list_of_folder_names_over_sftp( Parameters ---------- - stfp : connected paramiko stfp object + stfp + connected paramiko stfp object (see search_ssh_central_for_folders()) - search_path : path to search for folders in + search_path + path to search for folders in - search_prefix : prefix (can include wildcards) + search_prefix + prefix (can include wildcards) to search folder names. - verbose : If `True`, if a search folder cannot be found, a message - will be printed with the un-found path. + verbose + If `True`, if a search folder cannot be found, a message + will be printed with the un-found path. """ all_folder_names = [] all_filenames = [] diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index e85d757da..05f58f6f3 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -144,16 +144,16 @@ def validate_list_of_names( Parameters ---------- - path_or_name_list : List[Path] + path_or_name_list A list of pathlib.Path to NeuroBlueprint-formatted folders to validate - prefix: Prefix + prefix Whether these are subject (sub) or session (ses) level names - name_templates: Optional[Dict] + name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. - check_value_lengths : bool + check_value_lengths If `True`, check that the prefix- value lengths are consistent across the passed list. """ @@ -507,27 +507,27 @@ def validate_project( Parameters ----------- - cfg : Configs + cfg datashuttle Configs class. - top_level_folder_list: List[TopLevelFolder] + top_level_folder_list The top level folders to validate. - include_central : bool + include_central If `False`, only project folders in the `local_path` will be validated. Otherwise, project folders in both the `local_path` and `central_path` will be validated. - display_mode : DisplayMode + display_mode Determine whether error or warning is raised. - log : bool + log If `True`, errors or warnings are logged to "datashuttle" logger. - name_templates: Optional[Dict] + name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. - strict_mode: bool + strict_mode If `True`, only allow NeuroBlueprint-formatted folders to exist in the project. By default, non-NeuroBlueprint folders (e.g. a folder called 'my_stuff' in the 'rawdata') are allowed, and only folders @@ -616,34 +616,34 @@ def validate_names_against_project( Parameters ---------- - cfg : Configs + cfg datashuttle Configs class. - top_level_folder : TopLevelFolder + top_level_folder The top level folder to validate - sub_names : List[str] + sub_names A list of subject-level names to validate against the subject names that exist in the project. - ses_names : List[str] + ses_names A list of session-level names to validate against the session names that exist in the project. Note that duplicate checks will only be performed for sessions within the passed `sub_names`. - include_central : bool + include_central If `True`, only project folders in the `local_path` will be validated against. Otherwise, project folders in both the `local_path` and `central_path` will be validated against. - display_mode : DisplayMode + display_mode Determine whether error or warning is raised. - log : bool + log If `True`, errors or warnings are logged to "datashuttle" logger. - name_templates: Optional[Dict] + name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. """ error_messages = [] From 0fa4b7f2f54d6da89ca828a230fa296a9eb47794 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Tue, 18 Mar 2025 23:16:10 +0000 Subject: [PATCH 02/25] formatting parameter headings --- datashuttle/datashuttle_class.py | 1 + datashuttle/tui/tabs/create_folders.py | 1 + datashuttle/utils/folders.py | 1 + datashuttle/utils/formatting.py | 1 + datashuttle/utils/getters.py | 1 + datashuttle/utils/rclone.py | 11 ++++++----- 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index e58b9087f..ad757ac38 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -1191,6 +1191,7 @@ def set_name_templates(self, new_name_templates: Dict) -> None: Parameters ---------- + new_name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} where "sub" or "ses" can be a regexp that subject and session diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index e596566da..1e3ef7de8 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -285,6 +285,7 @@ def fill_input_with_next_sub_or_ses_template( will be suggested. Parameters + ---------- prefix Whether to fill the subject or session Input diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 03cfb38af..33d470088 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -130,6 +130,7 @@ def make_datatype_folders( Parameters ---------- + cfg datashuttle configs diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index d5cee1d29..12e9a6027 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -89,6 +89,7 @@ def format_names(names: List, prefix: Prefix) -> List[str]: Parameters ----------- + names str or list containing sub or ses names (e.g. to make folders) diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index 1f55465bd..8e1e8cd95 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -50,6 +50,7 @@ def get_next_sub_or_ses( Parameters ---------- + cfg datashuttle configs class diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index d754e47ee..0462af2b7 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -15,6 +15,7 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: Parameters ---------- + command Rclone command to be run @@ -77,12 +78,12 @@ def setup_rclone_config_for_ssh( log: bool = True, ): """ - RClone sets remote targets in a config file that are - used at transfer. For SSH, this must contain the central path, - username and ssh key. The relative path is supplied at transfer time. + RClone sets remote targets in a config file that are + used at transfer. For SSH, this must contain the central path, + username and ssh key. The relative path is supplied at transfer time. - Parameters - ---------- + Parameters + ---------- cfg datashuttle configs UserDict. From 6c1cf06aa3772ea75c2c776e9ff1d415f0e4074b Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 10:31:41 +0000 Subject: [PATCH 03/25] Removing black + configuring ruff using movement repo config --- .pre-commit-config.yaml | 18 ++++++++++++---- pyproject.toml | 48 ++++++++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 515c16ca8..8fbbd4031 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,23 +8,33 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: + - id: check-added-large-files - id: check-docstring-first - id: check-executables-have-shebangs + - id: check-case-conflict - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml - id: check-toml + - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending args: [--fix=lf] + - id: name-tests-test + args: ["--pytest-test-first"] - id: requirements-txt-fixer - id: trailing-whitespace + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.9.9 hooks: - id: ruff - - repo: https://github.com/psf/black - rev: 25.1.0 - hooks: - - id: black + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 997f6f406..c8ce1baca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,6 @@ dev = [ "pytest-mock", "coverage", "tox", - "black", "mypy", "pre-commit", "ruff", @@ -91,19 +90,52 @@ exclude = ["tests*", "docs*"] [tool.pytest.ini_options] addopts = "--cov=datashuttle" -[tool.black] -target-version = ['py39', 'py310', 'py311', 'py312'] -skip-string-normalization = false -line-length = 79 - [tool.ruff] line-length = 79 exclude = ["__init__.py","build",".eggs"] fix = true [tool.ruff.lint] -ignore = ["E203","E501","E731","C901","W291","W293","E402","E722"] -select = ["I", "E", "F", "TCH", "TID252"] +# See https://docs.astral.sh/ruff/rules/ +ignore = [ + "D203", # one blank line before class + "D213", # multi-line-summary second line +] +select = [ + "E", # pycodestyle errors + "F", # Pyflakes + "UP", # pyupgrade + "I", # isort + "B", # flake8 bugbear + "SIM", # flake8 simplify + "C90", # McCabe complexity + "D", # pydocstyle +] +per-file-ignores = { "tests/*" = [ + "D100", # missing docstring in public module + "D205", # missing blank line between summary and description + "D103", # missing docstring in public function +], "examples/*" = [ + "D400", # first line should end with a period. + "D415", # first line should end with a period, question mark... + "D205", # missing blank line between summary and description +] } + +# Old ruff ruleset + pydocstyle added +# Inconsistent with movement repo, but saving this here for +# now in case there are good reasons to keep these rules +#ignore = ["E203","E501","E731","C901","W291","W293","E402","E722"] +#select = [ +# "I", # isort +# "E", # pycodestyle errors +# "F", # Pyflakes +# "TC", # flake8-type-checking +# "TID252", # flake8-tidy-imports relative-imports +# "D", # pydocstyle +#] + +[tool.ruff.format] +docstring-code-format = true # Also format code in docstrings [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] From ff9d470ad8006e8eb83b204d9ffa8a3080629ebb Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 10:40:51 +0000 Subject: [PATCH 04/25] moving per-file-ignores for __init__.py to the list in tool.ruff.lint --- pyproject.toml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c8ce1baca..77052bbf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,8 +98,8 @@ fix = true [tool.ruff.lint] # See https://docs.astral.sh/ruff/rules/ ignore = [ - "D203", # one blank line before class - "D213", # multi-line-summary second line + "D203", # one blank line before class + "D213", # multi-line-summary second line ] select = [ "E", # pycodestyle errors @@ -112,13 +112,17 @@ select = [ "D", # pydocstyle ] per-file-ignores = { "tests/*" = [ - "D100", # missing docstring in public module - "D205", # missing blank line between summary and description - "D103", # missing docstring in public function + "D100", # missing docstring in public module + "D205", # missing blank line between summary and description + "D103", # missing docstring in public function ], "examples/*" = [ - "D400", # first line should end with a period. - "D415", # first line should end with a period, question mark... - "D205", # missing blank line between summary and description + "D400", # first line should end with a period. + "D415", # first line should end with a period, question mark... + "D205", # missing blank line between summary and description +], "__init__.py" = [ + # This was part of the old config + # Is this needed? __init__.py is already part of tool.ruff.exclude + "F401", # auto remove unused imports ] } # Old ruff ruleset + pydocstyle added @@ -137,9 +141,6 @@ per-file-ignores = { "tests/*" = [ [tool.ruff.format] docstring-code-format = true # Also format code in docstrings -[tool.ruff.lint.per-file-ignores] -"__init__.py" = ["F401"] - [tool.ruff.lint.mccabe] max-complexity = 18 From 0818a66597a1a3ceb7bbc540d1b6e4e039a70075 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 10:44:23 +0000 Subject: [PATCH 05/25] run pre-commit --- .pre-commit-config.yaml | 2 +- datashuttle/configs/canonical_configs.py | 47 ++--- datashuttle/configs/canonical_folders.py | 19 +- datashuttle/configs/canonical_tags.py | 3 +- datashuttle/configs/config_class.py | 41 ++-- datashuttle/configs/load_configs.py | 9 +- datashuttle/datashuttle_class.py | 197 +++++++----------- datashuttle/datashuttle_functions.py | 8 +- datashuttle/tui/app.py | 21 +- datashuttle/tui/configs.py | 115 +++++----- datashuttle/tui/custom_widgets.py | 63 ++---- datashuttle/tui/interface.py | 61 ++---- .../tui/screens/create_folder_settings.py | 25 +-- datashuttle/tui/screens/datatypes.py | 26 +-- datashuttle/tui/screens/get_help.py | 1 - datashuttle/tui/screens/modal_dialogs.py | 13 +- datashuttle/tui/screens/new_project.py | 5 +- datashuttle/tui/screens/project_manager.py | 16 +- datashuttle/tui/screens/project_selector.py | 5 +- datashuttle/tui/screens/settings.py | 3 +- datashuttle/tui/screens/setup_ssh.py | 17 +- datashuttle/tui/tabs/create_folders.py | 43 ++-- datashuttle/tui/tabs/logging.py | 2 +- datashuttle/tui/tabs/transfer.py | 24 +-- datashuttle/tui/tabs/transfer_status_tree.py | 24 +-- datashuttle/tui/tooltips.py | 3 +- datashuttle/tui/utils/tui_decorators.py | 3 +- datashuttle/tui/utils/tui_validators.py | 10 +- datashuttle/tui_launcher.py | 4 +- datashuttle/utils/data_transfer.py | 37 ++-- datashuttle/utils/decorators.py | 9 +- datashuttle/utils/ds_logger.py | 19 +- datashuttle/utils/folder_class.py | 3 +- datashuttle/utils/folders.py | 54 ++--- datashuttle/utils/formatting.py | 45 ++-- datashuttle/utils/getters.py | 32 +-- datashuttle/utils/rclone.py | 53 ++--- datashuttle/utils/ssh.py | 35 ++-- datashuttle/utils/utils.py | 39 ++-- datashuttle/utils/validation.py | 81 +++---- tests/conftest.py | 3 +- tests/ssh_test_utils.py | 14 +- tests/test_utils.py | 68 +++--- tests/tests_integration/_test_configs.py | 35 +--- tests/tests_integration/base.py | 10 +- .../tests_integration/test_create_folders.py | 37 ++-- tests/tests_integration/test_datatypes.py | 12 +- .../test_filesystem_transfer.py | 46 ++-- tests/tests_integration/test_formatting.py | 9 +- .../tests_integration/test_local_only_mode.py | 16 +- tests/tests_integration/test_logging.py | 53 ++--- tests/tests_integration/test_settings.py | 22 +- .../test_ssh_file_transfer.py | 18 +- tests/tests_integration/test_ssh_setup.py | 19 +- .../tests_integration/test_transfer_checks.py | 3 +- tests/tests_integration/test_validation.py | 55 ++--- tests/tests_tui/test_local_only_project.py | 16 +- tests/tests_tui/test_tui_configs.py | 30 +-- tests/tests_tui/test_tui_create_folders.py | 30 +-- tests/tests_tui/test_tui_datatypes.py | 6 +- tests/tests_tui/test_tui_directorytree.py | 15 +- tests/tests_tui/test_tui_get_help.py | 5 +- tests/tests_tui/test_tui_logging.py | 9 +- tests/tests_tui/test_tui_settings.py | 12 +- tests/tests_tui/test_tui_transfer.py | 13 +- .../test_tui_widgets_and_defaults.py | 47 +---- tests/tests_tui/tui_base.py | 66 ++---- tests/tests_unit/test_links.py | 3 +- tests/tests_unit/test_unit.py | 35 ++-- tests/tests_unit/test_validation_unit.py | 28 +-- 70 files changed, 654 insertions(+), 1298 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8fbbd4031..299fba1d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: rev: v0.9.9 hooks: - id: ruff - - id: ruff-format + #- id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index 7213b3d88..8d03eed6e 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -1,5 +1,4 @@ -""" -This module contains all information for the required +"""This module contains all information for the required format of the configs class. This is clearly defined as configs can be provided from file or input dynamically and so careful checks must be done. @@ -32,8 +31,7 @@ def get_canonical_configs() -> dict: - """ - The only permitted types for DataShuttle + """The only permitted types for DataShuttle config values. """ canonical_configs = { @@ -48,8 +46,7 @@ def get_canonical_configs() -> dict: def keys_str_on_file_but_path_in_class() -> list[str]: - """ - All configs which are paths are converted to pathlib.Path + """All configs which are paths are converted to pathlib.Path objects on load. This list indicates which config entries are to be converted to Path. """ @@ -65,8 +62,7 @@ def keys_str_on_file_but_path_in_class() -> list[str]: def check_dict_values_raise_on_fail(config_dict: Configs) -> None: - """ - Central function for performing checks on a + """Central function for performing checks on a DataShuttle Configs UserDict class. This should be run after any change to the configs (e.g. make_config_file, update_config_file, supply_config_file). @@ -75,9 +71,9 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None: Parameters ---------- - config_dict datashuttle config UserDict + """ canonical_dict = get_canonical_configs() @@ -144,8 +140,7 @@ def check_dict_values_raise_on_fail(config_dict: Configs) -> None: def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None: - """ - There is no circumstance where one of `central_path` and `connection_method` + """There is no circumstance where one of `central_path` and `connection_method` should be set and not the other. Either both are set ('full' project) or neither are ('local only' project). Check this assumption here. """ @@ -171,14 +166,12 @@ def raise_on_bad_path_syntax( path_name: str, path_type: str, ) -> None: - """ - Error if some common, unsupported patterns are observed + """Error if some common, unsupported patterns are observed (e.g. ~, .) for path. """ if path_name[0] == "~": utils.log_and_raise_error( - f"{path_type} must contain the full folder path " - "with no ~ syntax.", + f"{path_type} must contain the full folder path with no ~ syntax.", ConfigError, ) @@ -193,13 +186,10 @@ def raise_on_bad_path_syntax( def check_config_types(config_dict: Configs) -> None: - """ - Check the type of passed configs matches the canonical types. - """ + """Check the type of passed configs matches the canonical types.""" required_types = get_canonical_configs() for key in config_dict.keys(): - expected_type = required_types[key] try: typeguard.check_type(config_dict[key], expected_type) @@ -218,8 +208,7 @@ def check_config_types(config_dict: Configs) -> None: def get_tui_config_defaults() -> Dict: - """ - Get the default settings for the datatype checkboxes + """Get the default settings for the datatype checkboxes in the TUI. Two sets are maintained (one for creating, @@ -248,7 +237,6 @@ def get_tui_config_defaults() -> Dict: # Fill all datatype options for broad_key in get_broad_datatypes(): - settings["tui"]["create_checkboxes_on"][broad_key] = { # type: ignore "on": True, "displayed": True, @@ -276,8 +264,7 @@ def get_name_templates_defaults() -> Dict: def get_persistent_settings_defaults() -> Dict: - """ - Persistent settings are settings that are maintained + """Persistent settings are settings that are maintained across sessions. Currently, persistent settings for both the API and TUI are stored in the same place. @@ -293,8 +280,7 @@ def get_persistent_settings_defaults() -> Dict: def get_datatypes() -> List[str]: - """ - Canonical list of datatype flags based on NeuroBlueprint. + """Canonical list of datatype flags based on NeuroBlueprint. This must be kept up to date with the datatypes in the NeuroBlueprint specification. """ @@ -306,8 +292,7 @@ def get_broad_datatypes(): def get_narrow_datatypes(): - """ - Return the narrow datatype associated with each broad datatype. + """Return the narrow datatype associated with each broad datatype. The mapping between broad and narrow datatypes is required for validation. """ return { @@ -337,8 +322,7 @@ def get_narrow_datatypes(): def quick_get_narrow_datatypes(): - """ - A convenience wrapper around `get_narrow_datatypes()` + """A convenience wrapper around `get_narrow_datatypes()` to quickly get a list of all narrow datatypes. """ all_narrow_datatypes = get_narrow_datatypes() @@ -352,8 +336,7 @@ def quick_get_narrow_datatypes(): def in_place_update_settings_for_narrow_datatype(settings: dict): - """ - In versions < v0.6.0, only 'broad' datatypes were implemented + """In versions < v0.6.0, only 'broad' datatypes were implemented and available in the TUI. Since, 'narrow' datatypes are introduced and datatype tui can be set to be both on / off but also displayed / not displayed. diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index a6b77128c..104019704 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -11,8 +11,7 @@ def get_datatype_folders() -> dict: - """ - This function holds the canonical folders + """This function holds the canonical folders managed by datashuttle. Notes @@ -32,6 +31,7 @@ def get_datatype_folders() -> dict: an option for rare cases in which advanced users want to change it. level : "sub" or "ses", level to make the folder at. + """ return { datatype: Folder(name=datatype, level="ses") @@ -40,8 +40,7 @@ def get_datatype_folders() -> dict: def get_non_sub_names() -> List[str]: - """ - Get all arguments that are not allowed at the + """Get all arguments that are not allowed at the subject level for data transfer, i.e. as sub_names """ return [ @@ -53,8 +52,7 @@ def get_non_sub_names() -> List[str]: def get_non_ses_names() -> List[str]: - """ - Get all arguments that are not allowed at the + """Get all arguments that are not allowed at the session level for data transfer, i.e. as ses_names """ return [ @@ -66,8 +64,7 @@ def get_non_ses_names() -> List[str]: def canonical_reserved_keywords() -> List[str]: - """ - Key keyword arguments that are passed to `sub_names` or + """Key keyword arguments that are passed to `sub_names` or `ses_names` but that we """ return get_non_sub_names() + get_non_ses_names() @@ -78,16 +75,14 @@ def get_top_level_folders() -> List[TopLevelFolder]: def get_datashuttle_path() -> Path: - """ - Get the datashuttle path where all project + """Get the datashuttle path where all project configs are stored. """ return Path.home() / ".datashuttle" def get_project_datashuttle_path(project_name: str) -> Tuple[Path, Path]: - """ - Get the datashuttle path for the project, + """Get the datashuttle path for the project, where configuration files are stored. Also, return a temporary path in this for logging in some cases where local_path location is not clear. diff --git a/datashuttle/configs/canonical_tags.py b/datashuttle/configs/canonical_tags.py index 233350bc6..7cc274dd0 100644 --- a/datashuttle/configs/canonical_tags.py +++ b/datashuttle/configs/canonical_tags.py @@ -1,6 +1,5 @@ def tags(tag_name: str) -> str: - """ - Centralised function to get the tags used + """Centralised function to get the tags used in subject / session name processing. If changing the formatting of these tags, it is only required to change the dict values here. diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index 7f5a8525c..f1282d935 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -25,8 +25,7 @@ class Configs(UserDict): - """ - Class to hold the datashuttle configs. + """Class to hold the datashuttle configs. The configs must match exactly the standard set in canonical_configs.py. If updating these configs, @@ -39,13 +38,13 @@ class Configs(UserDict): Parameters ---------- - file_path full filepath to save the config .yaml file to. input_dict a dict of config key-value pairs to input dict. This must contain all canonical_config keys + """ def __init__( @@ -81,8 +80,7 @@ def ensure_local_and_central_path_end_in_project_name(self): self[path_type] = self[path_type] / self.project_name def check_dict_values_raise_on_fail(self) -> None: - """ - Check the values of the current dictionary are set + """Check the values of the current dictionary are set correctly and will not cause downstream errors. This will raise an error if the dictionary @@ -104,9 +102,7 @@ def values(self) -> ValuesView: # ------------------------------------------------------------------------- def dump_to_file(self) -> None: - """ - Save the dictionary to .yaml file stored in self.file_path. - """ + """Save the dictionary to .yaml file stored in self.file_path.""" cfg_to_save = copy.deepcopy(self.data) load_configs.convert_str_and_pathlib_paths(cfg_to_save, "path_to_str") @@ -114,12 +110,11 @@ def dump_to_file(self) -> None: yaml.dump(cfg_to_save, config_file, sort_keys=False) def load_from_file(self) -> None: - """ - Load a config dict saved at .yaml file. Note this will + """Load a config dict saved at .yaml file. Note this will not automatically check the configs are valid, this requires calling self.check_dict_values_raise_on_fail() """ - with open(self.file_path, "r") as config_file: + with open(self.file_path) as config_file: config_dict = yaml.full_load(config_file) load_configs.convert_str_and_pathlib_paths(config_dict, "str_to_path") @@ -136,14 +131,12 @@ def build_project_path( sub_folders: Union[str, list], top_level_folder: TopLevelFolder, ) -> Path: - """ - Function for joining relative path to base dir. + """Function for joining relative path to base dir. If path already starts with base dir, the base dir will not be joined. Parameters ---------- - base "local", "central" or "datashuttle" @@ -151,6 +144,7 @@ def build_project_path( a list (or string for 1) of folder names to be joined into a path. If file included, must be last entry (with ext). + """ if isinstance(sub_folders, list): sub_folders_str = "/".join(sub_folders) @@ -173,12 +167,10 @@ def get_base_folder( base: str, top_level_folder: TopLevelFolder, ) -> Path: - """ - Convenience function to return the full base path. + """Convenience function to return the full base path. Parameters ---------- - base base path, "local", "central" or "datashuttle" @@ -193,8 +185,7 @@ def get_base_folder( def get_rclone_config_name( self, connection_method: Optional[str] = None ) -> str: - """ - Convenience function to get the rclone config + """Convenience function to get the rclone config name (these configs are created by datashuttle but managed and stored by rclone). """ @@ -206,8 +197,7 @@ def get_rclone_config_name( def make_rclone_transfer_options( self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool ) -> Dict: - """ - This function originally collected the relevant arguments + """This function originally collected the relevant arguments from configs. Now, all are passed via function arguments However, now we fix the previously configurable arguments `show_transfer_progress` and `dry_run` here. @@ -246,8 +236,7 @@ def init_paths(self) -> None: def make_and_get_logging_path( self, ) -> Path: - """ - Build (and create if does not exist) the path where + """Build (and create if does not exist) the path where logs are stored. """ logging_path = self.project_metadata_path / "logs" @@ -257,8 +246,7 @@ def make_and_get_logging_path( def get_datatype_as_dict_items( self, datatype: Union[str, list] ) -> Union[ItemsView, zip]: - """ - Get the .items() structure of the datatype, either all of + """Get the .items() structure of the datatype, either all of the canonical datatypes or as a single item. """ if isinstance(datatype, str): @@ -279,8 +267,7 @@ def get_datatype_as_dict_items( return items def is_local_project(self): - """ - A project is 'local-only' if it has no `central_path` and `connection_method`. + """A project is 'local-only' if it has no `central_path` and `connection_method`. It can be used to make folders and validate, but not for transfer. """ canonical_configs.raise_on_bad_local_only_project_configs(self) diff --git a/datashuttle/configs/load_configs.py b/datashuttle/configs/load_configs.py index c15e3977d..edcd4b6a9 100644 --- a/datashuttle/configs/load_configs.py +++ b/datashuttle/configs/load_configs.py @@ -17,8 +17,7 @@ def attempt_load_configs( config_path: Path, verbose: bool = True, ) -> Optional[Configs]: - """ - Try to load an existing config file, that was previously + """Try to load an existing config file, that was previously saved by Datashuttle. This should always work, unless not already initialised (prompt) or these have been changed manually. @@ -33,6 +32,7 @@ def attempt_load_configs( verbose warnings and error messages will be printed. + """ exists = config_path.is_file() @@ -68,18 +68,17 @@ def attempt_load_configs( def convert_str_and_pathlib_paths( config_dict: Union["Configs", dict], direction: str ) -> None: - """ - Config paths are stored as str in the .yaml but used as Path + """Config paths are stored as str in the .yaml but used as Path in the module, so make the conversion here. Parameters ---------- - config_dict DataShuttle.cfg dict of configs direction "path_to_str" or "str_to_path" + """ for path_key in canonical_configs.keys_str_on_file_but_path_in_class(): value = config_dict[path_key] diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index ad757ac38..6cc044625 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -62,8 +62,7 @@ class DataShuttle: - """ - DataShuttle is a tool for convenient scientific + """DataShuttle is a tool for convenient scientific project management and data transfer in BIDS format. The expected organisation is a central repository @@ -89,7 +88,6 @@ class DataShuttle: Parameters ---------- - project_name The project name to use the datashuttle Folders containing all project files @@ -103,10 +101,10 @@ class DataShuttle: If `True`, a start-up message displaying the current state of the program (e.g. persistent settings such as the 'top-level folder') is shown. + """ def __init__(self, project_name: str, print_startup_message: bool = True): - self._error_on_base_project_name(project_name) self.project_name = project_name ( @@ -133,8 +131,7 @@ def __init__(self, project_name: str, print_startup_message: bool = True): rclone.prompt_rclone_download_if_does_not_exist() def _set_attributes_after_config_load(self) -> None: - """ - Once config file is loaded, update all private attributes + """Once config file is loaded, update all private attributes according to config contents. """ self.cfg.init_paths() @@ -155,8 +152,7 @@ def create_folders( bypass_validation: bool = False, log: bool = True, ) -> Dict[str, List[Path]]: - """ - Create a subject / session folder tree in the project + """Create a subject / session folder tree in the project folder. The passed subject / session names are formatted and validated. If this succeeds, fully validation against all subject / session folders in @@ -165,7 +161,6 @@ def create_folders( Parameters ---------- - top_level_folder Whether to make the folders in `rawdata` or `derivatives`. @@ -208,7 +203,6 @@ def create_folders( Notes ----- - sub_names or ses_names may contain formatting tags @TO@ @@ -227,6 +221,7 @@ def create_folders( project.create_folders("rawdata", "sub-001", datatype="behav") project.create_folders("rawdata", "sub-002@TO@005", ["ses-001", "ses-002"], ["ephys", "behav"]) + """ if log: self._start_log("create-folders", local_vars=locals()) @@ -289,8 +284,7 @@ def _format_and_validate_names( bypass_validation: bool, log: bool = True, ) -> Tuple[List[str], List[str]]: - """ - A central method for the formatting and validation of subject / session + """A central method for the formatting and validation of subject / session names for folder creation. This is called by both DataShuttle and during TUI validation. """ @@ -335,8 +329,7 @@ def upload_custom( dry_run: bool = False, init_log: bool = True, ) -> None: - """ - Upload data from a local project to the central project + """Upload data from a local project to the central project folder. In the case that a file / folder exists on the central and local, the central will not be overwritten even if the central file is an older version. Data @@ -344,7 +337,6 @@ def upload_custom( Parameters ---------- - top_level_folder The top-level folder (e.g. `"rawdata"`, `"derivatives"`) to transfer files and folders within. @@ -379,6 +371,7 @@ def upload_custom( (Optional). Whether to handle logging. This should always be True, unless logger is handled elsewhere (e.g. in a calling function). + """ if init_log: self._start_log("upload-custom", local_vars=locals()) @@ -412,13 +405,11 @@ def download_custom( dry_run: bool = False, init_log: bool = True, ) -> None: - """ - Download data from the central project folder to the + """Download data from the central project folder to the local project folder. Parameters ---------- - top_level_folder The top-level folder (e.g. `rawdata`) to transfer files and folders within. @@ -453,6 +444,7 @@ def download_custom( (Optional). Whether to handle logging. This should always be True, unless logger is handled elsewhere (e.g. in a calling function). + """ if init_log: self._start_log("download-custom", local_vars=locals()) @@ -486,12 +478,10 @@ def upload_rawdata( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """ - Upload files in the `rawdata` top level folder. + """Upload files in the `rawdata` top level folder. Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -503,6 +493,7 @@ def upload_rawdata( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._transfer_top_level_folder( "upload", @@ -518,12 +509,10 @@ def upload_derivatives( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """ - Upload files in the `derivatives` top level folder. + """Upload files in the `derivatives` top level folder. Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -535,6 +524,7 @@ def upload_derivatives( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._transfer_top_level_folder( "upload", @@ -550,12 +540,10 @@ def download_rawdata( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """ - Download files in the `rawdata` top level folder. + """Download files in the `rawdata` top level folder. Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -567,6 +555,7 @@ def download_rawdata( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._transfer_top_level_folder( "download", @@ -582,12 +571,10 @@ def download_derivatives( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ): - """ - Download files in the `derivatives` top level folder. + """Download files in the `derivatives` top level folder. Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -599,6 +586,7 @@ def download_derivatives( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._transfer_top_level_folder( "download", @@ -614,14 +602,12 @@ def upload_entire_project( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Upload the entire project (from 'local' to 'central'), + """Upload the entire project (from 'local' to 'central'), i.e. including every top level folder (e.g. 'rawdata', 'derivatives', 'code', 'analysis'). Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -633,6 +619,7 @@ def upload_entire_project( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._start_log("upload-entire-project", local_vars=locals()) self._transfer_entire_project( @@ -647,14 +634,12 @@ def download_entire_project( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Download the entire project (from 'central' to 'local'), + """Download the entire project (from 'central' to 'local'), i.e. including every top level folder (e.g. 'rawdata', 'derivatives', 'code', 'analysis'). Parameters ---------- - overwrite_existing_files If "never" files on target will never be overwritten by source. If "always" files on target will be overwritten by source if @@ -666,6 +651,7 @@ def download_entire_project( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._start_log("download-entire-project", local_vars=locals()) self._transfer_entire_project( @@ -681,8 +667,7 @@ def upload_specific_folder_or_file( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Upload a specific file or folder. If transferring + """Upload a specific file or folder. If transferring a single file, the path including the filename is required (see 'filepath' input). If a folder, wildcards "*" or "**" must be used to transfer @@ -691,7 +676,6 @@ def upload_specific_folder_or_file( Parameters ---------- - filepath a string containing the full filepath. @@ -706,6 +690,7 @@ def upload_specific_folder_or_file( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._start_log("upload-specific-folder-or-file", local_vars=locals()) @@ -723,8 +708,7 @@ def download_specific_folder_or_file( overwrite_existing_files: OverwriteExistingFiles = "never", dry_run: bool = False, ) -> None: - """ - Download a specific file or folder. If transferring + """Download a specific file or folder. If transferring a single file, the path including the filename is required (see 'filepath' input). If a folder, wildcards "*" or "**" must be used to transfer @@ -733,7 +717,6 @@ def download_specific_folder_or_file( Parameters ---------- - filepath a string containing the full filepath. @@ -748,6 +731,7 @@ def download_specific_folder_or_file( perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful to check which files will be moved on data transfer. + """ self._start_log( "download-specific-folder-or-file", local_vars=locals() @@ -767,8 +751,7 @@ def _transfer_top_level_folder( dry_run: bool = False, init_log: bool = True, ): - """ - Core function to upload / download files within a + """Core function to upload / download files within a particular top-level-folder. e.g. `upload_rawdata().` """ if init_log: @@ -798,9 +781,7 @@ def _transfer_top_level_folder( def _transfer_specific_file_or_folder( self, upload_or_download, filepath, overwrite_existing_files, dry_run ): - """ - Core function for upload/download_specific_folder_or_file(). - """ + """Core function for upload/download_specific_folder_or_file().""" if isinstance(filepath, str): filepath = Path(filepath) @@ -841,8 +822,7 @@ def _transfer_specific_file_or_folder( @requires_ssh_configs @check_is_not_local_project def setup_ssh_connection(self) -> None: - """ - Setup a connection to the central server using SSH. + """Setup a connection to the central server using SSH. Assumes the central_host_id and central_host_username are set in configs (see make_config_file() and update_config_file()) @@ -873,17 +853,16 @@ def setup_ssh_connection(self) -> None: @requires_ssh_configs @check_is_not_local_project def write_public_key(self, filepath: str) -> None: - """ - By default, the SSH private key only is stored, in + """By default, the SSH private key only is stored, in the datashuttle configs folder. Use this function to save the public key. Parameters ---------- - filepath full filepath (inc filename) to write the public key to. + """ key: paramiko.RSAKey key = paramiko.RSAKey.from_private_key_file( @@ -906,8 +885,7 @@ def make_config_file( central_host_id: Optional[str] = None, central_host_username: Optional[str] = None, ) -> None: - """ - Initialise the configurations for datashuttle to use on the + """Initialise the configurations for datashuttle to use on the local machine. Once initialised, these settings will be used each time the datashuttle is opened. This method can also be used to completely overwrite existing configs. @@ -921,7 +899,6 @@ def make_config_file( Parameters ---------- - local_path path to project folder on local machine @@ -946,6 +923,7 @@ def make_config_file( central_host_username username for which to log in to central host. e.g. "jziminski" + """ self._start_log( "make-config-file", @@ -1021,22 +999,17 @@ def update_config_file(self, **kwargs) -> None: @check_configs_set def get_local_path(self) -> Path: - """ - Get the projects local path. - """ + """Get the projects local path.""" return self.cfg["local_path"] @check_configs_set @check_is_not_local_project def get_central_path(self) -> Path: - """ - Get the project central path. - """ + """Get the project central path.""" return self.cfg["central_path"] def get_datashuttle_path(self) -> Path: - """ - Get the path to the local datashuttle + """Get the path to the local datashuttle folder where configs and other datashuttle files are stored. """ @@ -1044,9 +1017,7 @@ def get_datashuttle_path(self) -> Path: @check_configs_set def get_config_path(self) -> Path: - """ - Get the full path to the DataShuttle config file. - """ + """Get the full path to the DataShuttle config file.""" return self._config_path @check_configs_set @@ -1055,15 +1026,12 @@ def get_configs(self) -> Configs: @check_configs_set def get_logging_path(self) -> Path: - """ - Get the path where datashuttle logs are written. - """ + """Get the path where datashuttle logs are written.""" return self.cfg.logging_path @staticmethod def get_existing_projects() -> List[Path]: - """ - Get a list of existing project names found on the local machine. + """Get a list of existing project names found on the local machine. This is based on project folders in the "home / .datashuttle" folder that contain valid config.yaml files. """ @@ -1076,13 +1044,11 @@ def get_next_sub( return_with_prefix: bool = True, include_central: bool = False, ) -> str: - """ - Convenience function for get_next_sub_or_ses + """Convenience function for get_next_sub_or_ses to find the next subject number. Parameters ---------- - return_with_prefix If `True`, return with the "sub-" prefix. @@ -1090,6 +1056,7 @@ def get_next_sub( If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. If in local-project mode, this flag is ignored. + """ name_template = self.get_name_templates() name_template_regexp = ( @@ -1117,13 +1084,11 @@ def get_next_ses( return_with_prefix: bool = True, include_central: bool = False, ) -> str: - """ - Convenience function for get_next_sub_or_ses + """Convenience function for get_next_sub_or_ses to find the next session number. Parameters ---------- - top_level_folder "rawdata" or "derivatives" @@ -1137,6 +1102,7 @@ def get_next_ses( If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. If in local-project mode, this flag is ignored. + """ name_template = self.get_name_templates() name_template_regexp = ( @@ -1158,8 +1124,7 @@ def get_next_ses( @check_configs_set def is_local_project(self) -> bool: - """ - A project is 'local-only' if it has no `central_path` and `connection_method`. + """A project is 'local-only' if it has no `central_path` and `connection_method`. It can be used to make folders and validate, but not for transfer. """ return self.cfg.is_local_project() @@ -1168,22 +1133,20 @@ def is_local_project(self) -> bool: # ------------------------------------------------------------------------- def get_name_templates(self) -> Dict: - """ - Get the regexp templates used for validation. If + """Get the regexp templates used for validation. If the "on" key is set to `False`, template validation is not performed. Returns ------- - name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} + """ settings = self._load_persistent_settings() return settings["name_templates"] def set_name_templates(self, new_name_templates: Dict) -> None: - """ - Update the persistent settings with new name templates. + """Update the persistent settings with new name templates. Name templates are regexp for that, when name_templates["on"] is set to `True`, "sub" and "ses" names are validated against @@ -1191,11 +1154,11 @@ def set_name_templates(self, new_name_templates: Dict) -> None: Parameters ---------- - new_name_templates e.g. {"name_templates": {"on": False, "sub": None, "ses": None}} where "sub" or "ses" can be a regexp that subject and session names respectively are validated against. + """ self._update_persistent_setting("name_templates", new_name_templates) @@ -1205,9 +1168,7 @@ def set_name_templates(self, new_name_templates: Dict) -> None: @check_configs_set def show_configs(self) -> None: - """ - Print the current configs to the terminal. - """ + """Print the current configs to the terminal.""" utils.print_message_to_user(self._get_json_dumps_config()) # ------------------------------------------------------------------------- @@ -1222,14 +1183,12 @@ def validate_project( include_central: bool = False, strict_mode: bool = False, ) -> List[str]: - """ - Perform validation on the project. This checks the subject + """Perform validation on the project. This checks the subject and session level folders to ensure there are no NeuroBlueprint formatting issues. Parameters ---------- - top_level_folder Folder to check, either "rawdata" or "derivatives". If ``None``, will check both folders. @@ -1250,6 +1209,7 @@ def validate_project( starting with sub- or ses- prefix are checked. In `Strict Mode`, any folder not prefixed with sub-, ses- or a valid datatype will raise a validation issue. + """ utils.print_message_to_user( f"Logs of the validation will be stored in: " @@ -1285,8 +1245,7 @@ def validate_project( @staticmethod def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: - """ - Pass list of names to check how these will be auto-formatted, + """Pass list of names to check how these will be auto-formatted, for example as when passed to create_folders() or upload_custom() or download() @@ -1295,13 +1254,13 @@ def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: Parameters ---------- - names A string or list of subject or session names. prefix The relevant subject or session prefix, e.g. "sub-" or "ses-" + """ if prefix not in ["sub", "ses"]: utils.log_and_raise_error( @@ -1325,19 +1284,17 @@ def _transfer_entire_project( overwrite_existing_files: OverwriteExistingFiles, dry_run: bool, ) -> None: - """ - Transfer (i.e. upload or download) the entire project (i.e. + """Transfer (i.e. upload or download) the entire project (i.e. every 'top level folder' (e.g. 'rawdata', 'derivatives'). Parameters ---------- - upload_or_download direction to transfer the data, either "upload" (from local to central) or "download" (from central to local). + """ for top_level_folder in canonical_folders.get_top_level_folders(): - utils.log_and_message(f"Transferring `{top_level_folder}`") self._transfer_top_level_folder( @@ -1355,14 +1312,12 @@ def _start_log( store_in_temp_folder: bool = False, verbose: bool = True, ) -> None: - """ - Initialize the logger. This is typically called at + """Initialize the logger. This is typically called at the start of public methods to initialize logging for a specific function call. Parameters ---------- - command_name name of the command, for the log output files. @@ -1373,6 +1328,7 @@ def _start_log( store_in_temp_folder if `False`, existing logging path will be used (local project .datashuttle). + """ if local_vars is None: variables = None @@ -1392,8 +1348,7 @@ def _start_log( ds_logger.start(path_to_save, command_name, variables, verbose) def _move_logs_from_temp_folder(self) -> None: - """ - Logs are stored within the project folder. Although + """Logs are stored within the project folder. Although in some instances, when setting configs, we do not know what the project folder is. In this case, make the logs in a temp folder in the .datashuttle config folder, @@ -1430,8 +1385,7 @@ def _error_on_base_project_name(self, project_name): ) def _log_successful_config_change(self, message: bool = False) -> None: - """ - Log the entire config at the time of config change. + """Log the entire config at the time of config change. If messaged, just message "update successful" rather than print the entire configs as it becomes confusing. """ @@ -1443,8 +1397,7 @@ def _log_successful_config_change(self, message: bool = False) -> None: ) def _get_json_dumps_config(self) -> str: - """ - Get the config dictionary formatted as json.dumps() + """Get the config dictionary formatted as json.dumps() which allows well formatted printing. """ copy_dict = copy.deepcopy(self.cfg.data) @@ -1452,8 +1405,7 @@ def _get_json_dumps_config(self) -> str: return json.dumps(copy_dict, indent=4) def _make_project_metadata_if_does_not_exist(self) -> None: - """ - Within the project local_path is also a .datashuttle + """Within the project local_path is also a .datashuttle folder that contains additional information, e.g. logs. """ folders.create_folders(self.cfg.project_metadata_path, log=False) @@ -1477,25 +1429,23 @@ def _setup_rclone_central_local_filesystem_config(self) -> None: def _update_persistent_setting( self, setting_name: str, setting_value: Any ) -> None: - """ - Load settings that are stored persistently across datashuttle + """Load settings that are stored persistently across datashuttle sessions. These are stored in yaml dumped to dictionary. Parameters ---------- - setting_name dictionary key of the persistent setting to change setting_value value to change the persistent setting to + """ settings = self._load_persistent_settings() if setting_name not in settings: utils.log_and_raise_error( - f"Setting key {setting_name} not found in " - f"settings dictionary", + f"Setting key {setting_name} not found in settings dictionary", KeyError, ) @@ -1504,29 +1454,25 @@ def _update_persistent_setting( self._save_persistent_settings(settings) def _init_persistent_settings(self) -> None: - """ - Initialise the default persistent settings + """Initialise the default persistent settings and save to file. """ settings = canonical_configs.get_persistent_settings_defaults() self._save_persistent_settings(settings) def _save_persistent_settings(self, settings: Dict) -> None: - """ - Save the settings dict to file as .yaml - """ + """Save the settings dict to file as .yaml""" with open(self._persistent_settings_path, "w") as settings_file: yaml.dump(settings, settings_file, sort_keys=False) def _load_persistent_settings(self) -> Dict: - """ - Load settings that are stored persistently across + """Load settings that are stored persistently across datashuttle sessions. """ if not self._persistent_settings_path.is_file(): self._init_persistent_settings() - with open(self._persistent_settings_path, "r") as settings_file: + with open(self._persistent_settings_path) as settings_file: settings = yaml.full_load(settings_file) self._update_settings_with_new_canonical_keys(settings) @@ -1534,8 +1480,7 @@ def _load_persistent_settings(self) -> Dict: return settings def _update_settings_with_new_canonical_keys(self, settings: Dict): - """ - Perform a check on the keys within persistent settings. + """Perform a check on the keys within persistent settings. If they do not exist, persistent settings is from older version and the new keys need adding. If changing keys within the top level (e.g. a dict entry in @@ -1565,9 +1510,7 @@ def _update_settings_with_new_canonical_keys(self, settings: Dict): ) def _check_top_level_folder(self, top_level_folder): - """ - Raise an error if ``top_level_folder`` not correct. - """ + """Raise an error if ``top_level_folder`` not correct.""" canonical_top_level_folders = canonical_folders.get_top_level_folders() if top_level_folder not in canonical_top_level_folders: diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 8c220b536..250fd8bd6 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -28,14 +28,12 @@ def quick_validate_project( display_mode: DisplayMode = "warn", name_templates: Optional[Dict] = None, ) -> List[str]: - """ - Perform validation on the project. This checks the subject + """Perform validation on the project. This checks the subject and session level folders to ensure there are not NeuroBlueprint formatting issues. Parameters ---------- - project_path Path to the project to validate. Must include the project name, and hold a "rawdata" or "derivatives" folder. @@ -52,6 +50,7 @@ def quick_validate_project( A dictionary of templates for subject and session name to validate against. See ``DataShuttle.set_name_templates()`` for details. + """ project_path = Path(project_path) @@ -91,8 +90,7 @@ def quick_validate_project( def _format_top_level_folder( top_level_folder: TopLevelFolder | None, ) -> List[TopLevelFolder]: - """ - Take a `top_level_folder` ("rawdata" or "derivatives" str) and + """Take a `top_level_folder` ("rawdata" or "derivatives" str) and convert to list, if `None`, convert it to a list of both possible top-level folders. """ diff --git a/datashuttle/tui/app.py b/datashuttle/tui/app.py index ce00f7434..fb342c445 100644 --- a/datashuttle/tui/app.py +++ b/datashuttle/tui/app.py @@ -31,8 +31,7 @@ class TuiApp(App, inherit_bindings=False): # type: ignore - """ - The main app page for the DataShuttle TUI. + """The main app page for the DataShuttle TUI. This class acts as a base class from which all windows (select existing project, make new project, settings and @@ -67,8 +66,7 @@ def set_dark_mode(self, dark_mode: bool) -> None: self.theme = "textual-dark" if dark_mode else "textual-light" def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Raise the relevant screen after button press. `push_screen` + """Raise the relevant screen after button press. `push_screen` second argument is a callback function returned after screen closes. """ if event.button.id == "mainwindow_existing_project_button": @@ -104,8 +102,7 @@ def show_modal_error_dialog(self, message: str) -> None: self.push_screen(modal_dialogs.MessageBox(message, border_color="red")) def handle_open_filesystem_browser(self, path_: Path) -> None: - """ - Open the system file browser to the path with the `showinfm` + """Open the system file browser to the path with the `showinfm` package, performing checks that the path exists prior to opening. """ if not path_.exists(): @@ -166,8 +163,7 @@ def rename_file_or_folder(self, path_, new_name): # Global Settings --------------------------------------------------------- def load_global_settings(self) -> Dict: - """ - Load the 'global settings' for the TUI that determine + """Load the 'global settings' for the TUI that determine project-independent settings that are persistent across sessions. These are stored in the canonical .datashuttle folder (see `get_global_settings_path`). @@ -178,15 +174,13 @@ def load_global_settings(self) -> Dict: global_settings = self.get_default_global_settings() self.save_global_settings(global_settings) else: - with open(settings_path, "r") as file: + with open(settings_path) as file: global_settings = yaml.full_load(file) return global_settings def get_global_settings_path(self) -> Path: - """ - The canonical path for the TUI's global settings. - """ + """The canonical path for the TUI's global settings.""" path_ = canonical_folders.get_datashuttle_path() return path_ / "global_tui_settings.yaml" @@ -206,8 +200,7 @@ def save_global_settings(self, global_settings: Dict) -> None: yaml.dump(global_settings, file, sort_keys=False) def copy_to_clipboard(self, value): - """ - Centralized function to copy to clipboard. + """Centralized function to copy to clipboard. This may fail under some circumstances (e.g., in headless mode on an HPC). """ try: diff --git a/datashuttle/tui/configs.py b/datashuttle/tui/configs.py index c16b1a425..27a701d24 100644 --- a/datashuttle/tui/configs.py +++ b/datashuttle/tui/configs.py @@ -30,8 +30,7 @@ class ConfigsContent(Container): - """ - This screen holds widgets and logic for setting datashuttle configs. + """This screen holds widgets and logic for setting datashuttle configs. It is used in `NewProjectPage` to instantiate a new project and initialise configs, or in `TabbedContent` to update an existing project's configs. @@ -60,8 +59,7 @@ def __init__( self.config_ssh_widgets: List[Any] = [] def compose(self) -> ComposeResult: - """ - `self.config_ssh_widgets` are SSH-setup related widgets + """`self.config_ssh_widgets` are SSH-setup related widgets that are only required when the user selects the SSH connection method. These are displayed / hidden based on the `connection_method` @@ -169,8 +167,7 @@ def compose(self) -> ComposeResult: yield Container(*config_screen_widgets, id="configs_container") def on_mount(self) -> None: - """ - When we have mounted the widgets, the following logic depends on whether + """When we have mounted the widgets, the following logic depends on whether we are setting up a new project (`self.project is `None`) or have an instantiated project. @@ -185,13 +182,13 @@ def on_mount(self) -> None: if self.interface: self.fill_widgets_with_project_configs() else: - self.query_one("#configs_local_filesystem_radiobutton").value = ( - True - ) + self.query_one( + "#configs_local_filesystem_radiobutton" + ).value = True self.switch_ssh_widgets_display(display_ssh=False) - self.query_one("#configs_setup_ssh_connection_button").visible = ( - False - ) + self.query_one( + "#configs_setup_ssh_connection_button" + ).visible = False # Setup tooltips if not self.interface: @@ -222,8 +219,7 @@ def on_mount(self) -> None: self.query_one(id).tooltip = get_tooltip(id) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """ - Update the displayed SSH widgets when the `connection_method` + """Update the displayed SSH widgets when the `connection_method` radiobuttons are changed. When SSH is set, ssh config-setters are shown. Otherwise, these @@ -242,23 +238,22 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: if label == "No connection (local only)": self.query_one("#configs_central_path_input").value = "" self.query_one("#configs_central_path_input").disabled = True - self.query_one("#configs_central_path_select_button").disabled = ( - True - ) + self.query_one( + "#configs_central_path_select_button" + ).disabled = True display_ssh = False else: self.query_one("#configs_central_path_input").disabled = False - self.query_one("#configs_central_path_select_button").disabled = ( - False - ) + self.query_one( + "#configs_central_path_select_button" + ).disabled = False display_ssh = True if label == "SSH" else False self.switch_ssh_widgets_display(display_ssh) self.set_central_path_input_tooltip(display_ssh) def set_central_path_input_tooltip(self, display_ssh: bool) -> None: - """ - Use a different tooltip depending on whether connection method + """Use a different tooltip depending on whether connection method is ssh or local filesystem. """ id = "#configs_central_path_input" @@ -292,44 +287,42 @@ def get_platform_dependent_example_paths( return example_path def switch_ssh_widgets_display(self, display_ssh: bool) -> None: - """ - Show or hide SSH-related configs based on whether the current + """Show or hide SSH-related configs based on whether the current `connection_method` widget is "ssh" or "local_filesystem". Parameters ---------- - display_ssh If `True`, display the SSH-related widgets. + """ for widget in self.config_ssh_widgets: widget.display = display_ssh - self.query_one("#configs_central_path_select_button").display = ( - not display_ssh - ) + self.query_one( + "#configs_central_path_select_button" + ).display = not display_ssh if self.interface is None: - self.query_one("#configs_setup_ssh_connection_button").visible = ( - False - ) + self.query_one( + "#configs_setup_ssh_connection_button" + ).visible = False else: - self.query_one("#configs_setup_ssh_connection_button").visible = ( - display_ssh - ) + self.query_one( + "#configs_setup_ssh_connection_button" + ).visible = display_ssh if not self.query_one("#configs_central_path_input").value: if display_ssh: placeholder = f"e.g. {self.get_platform_dependent_example_paths('central', ssh=True)}" else: placeholder = f"e.g. {self.get_platform_dependent_example_paths('central', ssh=False)}" - self.query_one("#configs_central_path_input").placeholder = ( - placeholder - ) + self.query_one( + "#configs_central_path_input" + ).placeholder = placeholder def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Enables the Create Folders button to read out current input values + """Enables the Create Folders button to read out current input values and use these to call project.create_folders(). """ if event.button.id == "configs_save_configs_button": @@ -366,36 +359,33 @@ def on_button_pressed(self, event: Button.Pressed) -> None: def handle_input_fill_from_select_directory( self, path_: Path, local_or_central: Literal["local", "central"] ) -> None: - """ - Update the `local` or `central` path inputs after + """Update the `local` or `central` path inputs after `SelectDirectoryTreeScreen` returns a path. Parameters ---------- - path_ The path returned from `SelectDirectoryTreeScreen`. If `False`, the screen exited with no directory selected. local_or_central The Input to fill with the path. + """ if path_ is False: return if local_or_central == "local": - self.query_one("#configs_local_path_input").value = ( - path_.as_posix() - ) + self.query_one( + "#configs_local_path_input" + ).value = path_.as_posix() elif local_or_central == "central": - self.query_one("#configs_central_path_input").value = ( - path_.as_posix() - ) + self.query_one( + "#configs_central_path_input" + ).value = path_.as_posix() def setup_ssh_connection(self) -> None: - """ - Set up the `SetupSshScreen` screen, - """ + """Set up the `SetupSshScreen` screen,""" assert self.interface is not None, "type narrow flexible `interface`" if not self.widget_configs_match_saved_configs(): @@ -410,8 +400,7 @@ def setup_ssh_connection(self) -> None: ) def widget_configs_match_saved_configs(self): - """ - Check that the configs currently stored in the widgets + """Check that the configs currently stored in the widgets on the screen match those stored in the app. This check is to avoid user starting to set up SSH with unexpected settings. It is a little fiddly as the Input for local @@ -433,8 +422,7 @@ def widget_configs_match_saved_configs(self): return True def setup_configs_for_a_new_project(self) -> None: - """ - If a project does not exist, we are in NewProjectScreen. + """If a project does not exist, we are in NewProjectScreen. We need to instantiate a new project based on the project name, create configs based on the current widget settings, and display any errors to the user, along with confirmation and the @@ -453,17 +441,15 @@ def setup_configs_for_a_new_project(self) -> None: success, output = interface.setup_new_project(project_name, cfg_kwargs) if success: - self.interface = interface - self.query_one("#configs_go_to_project_screen_button").visible = ( - True - ) + self.query_one( + "#configs_go_to_project_screen_button" + ).visible = True # Could not find a neater way to combine the push screen # while initiating the callback in one case but not the other. if cfg_kwargs["connection_method"] == "ssh": - self.query_one( "#configs_setup_ssh_connection_button" ).visible = True @@ -495,8 +481,7 @@ def setup_configs_for_a_new_project(self) -> None: self.parent_class.mainwindow.show_modal_error_dialog(output) def setup_configs_for_an_existing_project(self) -> None: - """ - If the project already exists, we are on the TabbedContent + """If the project already exists, we are on the TabbedContent screen. We need to get the configs to set from the current widget values and display the set values (or an error if there was a problem during setup) to the user. @@ -524,8 +509,7 @@ def setup_configs_for_an_existing_project(self) -> None: self.parent_class.mainwindow.show_modal_error_dialog(output) def fill_widgets_with_project_configs(self) -> None: - """ - If a configured project already exists, we want to fill the + """If a configured project already exists, we want to fill the widgets with the current project configs. This in some instances requires recasting to a new type of changing the value. @@ -587,8 +571,7 @@ def fill_widgets_with_project_configs(self) -> None: input.value = value def get_datashuttle_inputs_from_widgets(self) -> Dict: - """ - Get the configs to pass to `make_config_file()` from + """Get the configs to pass to `make_config_file()` from the current TUI settings. """ cfg_kwargs: Dict[str, Any] = {} diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 5f5559e2e..7f2b2e12b 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -1,8 +1,8 @@ from __future__ import annotations +from collections.abc import Iterable from typing import ( TYPE_CHECKING, - Iterable, List, Optional, Tuple, @@ -38,8 +38,7 @@ # ClickableInput # -------------------------------------------------------------------------------------- class ClickableInput(Input): - """ - An input widget which emits a `ClickableInput.Clicked` + """An input widget which emits a `ClickableInput.Clicked` signal when clicked, containing the input name `input` and mouse button index `button`. """ @@ -86,10 +85,9 @@ def on_key(self, event: events.Key) -> None: class CustomDirectoryTree(DirectoryTree): - """ - Base class for directory tree with some customised additions: - - filter out top-level folders that are not canonical - - add additional keyboard shortcuts defined in `on_key`. + """Base class for directory tree with some customised additions: + - filter out top-level folders that are not canonical + - add additional keyboard shortcuts defined in `on_key`. """ @dataclass @@ -105,15 +103,13 @@ def __init__( self.mainwindow = mainwindow def filter_paths(self, paths: Iterable[Path]) -> Iterable[Path]: - """ - Filter out all hidden folders and files from DirectoryTree + """Filter out all hidden folders and files from DirectoryTree display. """ return [path for path in paths if not path.name.startswith(".")] def on_key(self, event: events.Key) -> None: - """ - Handle key presses on the CustomDirectoryTree. Depending on the keys pressed, + """Handle key presses on the CustomDirectoryTree. Depending on the keys pressed, copy the path under the cursor, refresh the directorytree or emit a DirectoryTreeSpecialKeyPress event. """ @@ -143,8 +139,7 @@ def on_key(self, event: events.Key) -> None: def _render_line( self, y: int, x1: int, x2: int, base_style: Style ) -> Strip: - """ - This function is overridden from textual's `Tree` class to stop + """This function is overridden from textual's `Tree` class to stop CSS styling on hovering and clicking which was distracting / changed the default color used for transfer status, respectively. @@ -200,6 +195,7 @@ def get_guides(style: Style) -> tuple[str, str, str, str]: Returns: Strings for space, vertical, terminator and cross. + """ lines: tuple[ Iterable[str], Iterable[str], Iterable[str], Iterable[str] @@ -287,8 +283,7 @@ def get_guides(style: Style) -> tuple[str, str, str, str]: class TreeAndInputTab(TabPane): - """ - A parent class that defined common methods for screens with + """A parent class that defined common methods for screens with a directory tree and sub / session inputs, .e. the Create tab and the Transfer tab. """ @@ -296,8 +291,7 @@ class TreeAndInputTab(TabPane): def handle_fill_input_from_directorytree( self, sub_input_key: str, ses_input_key: str, event: events.Key ) -> None: - """ - When a CustomDirectoryTree key is pressed, we typically + """When a CustomDirectoryTree key is pressed, we typically want to perform an action that involves an Input. These are coordinated here. Note that the 'copy' and 'refresh' features of the tree is handled at the level of the @@ -315,7 +309,6 @@ def handle_fill_input_from_directorytree( Parameters ---------- - sub_input_key The textual widget id for the subject input (prefixed with #) @@ -325,6 +318,7 @@ def handle_fill_input_from_directorytree( event A DirectoryTreeSpecialKeyPress event triggered from the CustomDirectoryTree. + """ if event.key == "ctrl+a": self.append_sub_or_ses_name_to_input( @@ -342,8 +336,7 @@ def handle_fill_input_from_directorytree( def insert_sub_or_ses_name_to_input( self, sub_input_key: str, ses_input_key: str, name: str ) -> None: - """ - see `handle_directorytree_key_pressed` for `sub_input_key` and + """See `handle_directorytree_key_pressed` for `sub_input_key` and `ses_input_key`. name @@ -357,9 +350,7 @@ def insert_sub_or_ses_name_to_input( def append_sub_or_ses_name_to_input( self, sub_input_key: str, ses_input_key: str, name: str ) -> None: - """ - see `insert_sub_or_ses_name_to_input`. - """ + """See `insert_sub_or_ses_name_to_input`.""" if name.startswith("sub-"): if not self.query_one(sub_input_key).value: self.query_one(sub_input_key).value = name @@ -375,9 +366,7 @@ def append_sub_or_ses_name_to_input( def get_sub_ses_names_and_datatype( self, sub_input_key: str, ses_input_key: str ) -> Tuple[List[str], List[str], List[str]]: - """ - see `handle_fill_input_from_directorytree` for parameters. - """ + """See `handle_fill_input_from_directorytree` for parameters.""" sub_names = self.query_one(sub_input_key).as_names_list() ses_names = self.query_one(ses_input_key).as_names_list() datatype = self.query_one("DatatypeCheckboxes").selected_datatypes() @@ -386,8 +375,7 @@ def get_sub_ses_names_and_datatype( class TopLevelFolderSelect(Select): - """ - A Select widget for display and updating of top-level-folders. The + """A Select widget for display and updating of top-level-folders. The Create tab and transfer tabs (custom, top-level-folder) all have top level folder selects that perform the same function. This widget unifies these in a single place. @@ -398,7 +386,6 @@ class TopLevelFolderSelect(Select): Parameters ---------- - existing_only If `True`, only top level folders that actually exist in the project are displayed. Otherwise, all possible canonical @@ -406,6 +393,7 @@ class TopLevelFolderSelect(Select): id Textualize widget id + """ def __init__(self, interface: Interface, id: str) -> None: @@ -440,8 +428,7 @@ def __init__(self, interface: Interface, id: str) -> None: ) def get_top_level_folder(self, init: bool = False) -> str: - """ - Get the top level folder from `persistent_settings`, + """Get the top level folder from `persistent_settings`, performing a confidence-check that it matches the textual display. """ top_level_folder = self.interface.tui_settings[ @@ -449,28 +436,24 @@ def get_top_level_folder(self, init: bool = False) -> str: ][self.settings_key] if not init: - assert ( - top_level_folder == self.get_displayed_top_level_folder() - ), "config and widget should never be out of sync." + assert top_level_folder == self.get_displayed_top_level_folder(), ( + "config and widget should never be out of sync." + ) return top_level_folder def get_displayed_top_level_folder(self) -> str: - """ - Get the top level folder that is currently selected + """Get the top level folder that is currently selected on the select widget. """ assert self.value in canonical_folders.get_top_level_folders() return self.value def on_select_changed(self, event: Select.Changed) -> None: - """ - When the select is changed, update the linked persistent setting. - """ + """When the select is changed, update the linked persistent setting.""" top_level_folder = event.value if event.value != Select.BLANK: - self.interface.save_tui_settings( top_level_folder, "top_level_folder_select", self.settings_key ) diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index fd3e7b2c4..8ede9cb0e 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -15,8 +15,7 @@ class Interface: - """ - An interface class between the TUI and datashuttle API. Takes input + """An interface class between the TUI and datashuttle API. Takes input to all datashuttle functions as passed from the TUI, outputs success status (True or False) and optional data, in the case of False. @@ -31,21 +30,19 @@ class Interface: """ def __init__(self) -> None: - self.project: DataShuttle self.name_templates: Dict = {} self.tui_settings: Dict = {} def select_existing_project(self, project_name: str) -> InterfaceOutput: - """ - Load an existing project into `self.project`. + """Load an existing project into `self.project`. Parameters ---------- - project_name The name of the datashuttle project to load. Must already exist. + """ try: project = DataShuttle(project_name, print_startup_message=False) @@ -58,17 +55,16 @@ def select_existing_project(self, project_name: str) -> InterfaceOutput: def setup_new_project( self, project_name: str, cfg_kwargs: Dict ) -> InterfaceOutput: - """ - Set up a new project and load into `self.project`. + """Set up a new project and load into `self.project`. Parameters ---------- - project_name Name of the project to set up. cfg_kwargs The configurations to set the new project to. + """ try: project = DataShuttle(project_name, print_startup_message=False) @@ -85,15 +81,14 @@ def setup_new_project( def set_configs_on_existing_project( self, cfg_kwargs: Dict ) -> InterfaceOutput: - """ - Update the settings on an existing project. Only the settings + """Update the settings on an existing project. Only the settings passed in `cfg_kwargs` are updated. Parameters ---------- - cfg_kwargs The configs and new values to update. + """ try: self.project.update_config_file(**cfg_kwargs) @@ -108,12 +103,10 @@ def create_folders( ses_names: Optional[List[str]], datatype: List[str], ) -> InterfaceOutput: - """ - Create folders through datashuttle. + """Create folders through datashuttle. Parameters ---------- - sub_names A list of un-formatted / unvalidated subject names to create. @@ -122,6 +115,7 @@ def create_folders( datatype A list of canonical datatype names to create. + """ top_level_folder = self.tui_settings["top_level_folder_select"][ "create_tab" @@ -144,8 +138,7 @@ def create_folders( def validate_names( self, sub_names: List[str], ses_names: Optional[List[str]] ) -> InterfaceOutput: - """ - Validate a list of subject / session names. This is used + """Validate a list of subject / session names. This is used to populate the Input tooltips with validation errors. Uses a central `_format_and_validate_names()` that is also called during folder creation itself, to ensure these a @@ -153,12 +146,12 @@ def validate_names( Parameters ---------- - sub_names List of subject names to format. ses_names List of session names to format. + """ top_level_folder = self.tui_settings["top_level_folder_select"][ "create_tab" @@ -185,15 +178,14 @@ def validate_names( # ---------------------------------------------------------------------------------- def transfer_entire_project(self, upload: bool) -> InterfaceOutput: - """ - Transfer the entire project (all canonical top-level folders). + """Transfer the entire project (all canonical top-level folders). Parameters ---------- - upload Upload from local to central if `True`, otherwise download from central to remote. + """ try: if upload: @@ -216,12 +208,10 @@ def transfer_entire_project(self, upload: bool) -> InterfaceOutput: def transfer_top_level_only( self, selected_top_level_folder: str, upload: bool ) -> InterfaceOutput: - """ - Transfer all files within a selected top level folder. + """Transfer all files within a selected top level folder. Parameters ---------- - selected_top_level_folder The top level folder selected in the TUI for this transfer window. @@ -266,12 +256,10 @@ def transfer_custom_selection( datatype: List[str], upload: bool, ) -> InterfaceOutput: - """ - Transfer a custom selection of subjects / sessions / datatypes. + """Transfer a custom selection of subjects / sessions / datatypes. Parameters ---------- - selected_top_level_folder The top level folder selected in the TUI for this transfer window. @@ -287,6 +275,7 @@ def transfer_custom_selection( upload Upload from local to central if `True`, otherwise download from central to remote. + """ try: if upload: @@ -314,8 +303,7 @@ def transfer_custom_selection( # ---------------------------------------------------------------------------------- def get_name_templates(self) -> Dict: - """ - Get the `name_templates` defining templates to validate + """Get the `name_templates` defining templates to validate against. These are stored in a variable to avoid constantly reading these values from disk where they are stored in `persistent_settings`. It is critical this variable @@ -328,8 +316,7 @@ def get_name_templates(self) -> Dict: return self.name_templates def set_name_templates(self, name_templates: Dict) -> InterfaceOutput: - """ - Set the `name_templates` here and on disk. See `get_name_templates` + """Set the `name_templates` here and on disk. See `get_name_templates` for more information. """ try: @@ -341,8 +328,7 @@ def set_name_templates(self, name_templates: Dict) -> InterfaceOutput: return False, str(e) def get_tui_settings(self) -> Dict: - """ - Get the "tui" field of `persistent_settings`. Similar to + """Get the "tui" field of `persistent_settings`. Similar to `get_name_templates`, there are held on the class to avoid constantly reading from disk. """ @@ -354,12 +340,10 @@ def get_tui_settings(self) -> Dict: def save_tui_settings( self, value: Any, key: str, key_2: Optional[str] = None ) -> None: - """ - Update the "tui" field of the `persistent_settings` on disk. + """Update the "tui" field of the `persistent_settings` on disk. Parameters ---------- - value Value to set the `persistent_settings` tui field to @@ -370,6 +354,7 @@ def save_tui_settings( key_2 Optionals second level of the dictionary to update. e.g. "create_tab" + """ if key_2 is None: self.tui_settings[key] = value @@ -388,8 +373,7 @@ def get_configs(self) -> Configs: return self.project.cfg def get_textual_compatible_project_configs(self) -> Configs: - """ - Datashuttle configs keeps paths saved as pathlib.Path + """Datashuttle configs keeps paths saved as pathlib.Path objects. In some cases textual requires str representation. This method returns datashuttle configs with all paths that are Path converted to str. @@ -414,7 +398,6 @@ def get_next_sub( def get_next_ses( self, top_level_folder: TopLevelFolder, sub: str ) -> InterfaceOutput: - try: next_ses = self.project.get_next_ses( top_level_folder, diff --git a/datashuttle/tui/screens/create_folder_settings.py b/datashuttle/tui/screens/create_folder_settings.py index b8228439f..26ae336ec 100644 --- a/datashuttle/tui/screens/create_folder_settings.py +++ b/datashuttle/tui/screens/create_folder_settings.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: - from textual.app import ComposeResult from datashuttle.tui.app import App @@ -28,8 +27,7 @@ class CreateFoldersSettingsScreen(ModalScreen): - """ - This screen handles setting datashuttle's `name_template`'s, as well + """This screen handles setting datashuttle's `name_template`'s, as well as the top-level-folder select and option to bypass all validation. Name Templates @@ -46,10 +44,10 @@ class CreateFoldersSettingsScreen(ModalScreen): Attributes ---------- - Because the Input for `name_templates` is shared between subject and session, the values are held in the `input_values` attribute. These are loaded from `persistent_settings` on init. + """ TITLE = "Create Folders Settings" @@ -164,8 +162,7 @@ def switch_template_container_disabled(self) -> None: self.query_one("#template_inner_container").disabled = not is_on def fill_input_from_template(self) -> None: - """ - Fill the `name_templates` Input, that is shared + """Fill the `name_templates` Input, that is shared between subject and session, depending on the current radioset value. """ @@ -179,8 +176,7 @@ def fill_input_from_template(self) -> None: input.value = value def on_button_pressed(self, event: Button.Pressed) -> None: - """ - On close, update the `name_templates` stored in + """On close, update the `name_templates` stored in `persistent_settings` with those set on the TUI. Setting may error if templates are turned on but @@ -208,9 +204,7 @@ def make_name_templates_from_widgets(self) -> Dict: } def on_checkbox_changed(self, event: Checkbox.Changed) -> None: - """ - Turn `name_templates` on or off and update the TUI accordingly. - """ + """Turn `name_templates` on or off and update the TUI accordingly.""" is_on = event.value if event.checkbox.id == "template_settings_validation_on_checkbox": @@ -232,13 +226,12 @@ def on_checkbox_changed(self, event: Checkbox.Changed) -> None: disable_container = not self.query_one( "#template_settings_validation_on_checkbox" ).value - self.query_one("#template_inner_container").disabled = ( - disable_container - ) + self.query_one( + "#template_inner_container" + ).disabled = disable_container def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """ - Update the displayed SSH widgets when the `connection_method` + """Update the displayed SSH widgets when the `connection_method` radiobuttons are changed. """ label = str(event.pressed.label) diff --git a/datashuttle/tui/screens/datatypes.py b/datashuttle/tui/screens/datatypes.py index aa731761d..78899342e 100644 --- a/datashuttle/tui/screens/datatypes.py +++ b/datashuttle/tui/screens/datatypes.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, List, Literal, Optional if TYPE_CHECKING: - from textual.app import ComposeResult from datashuttle.tui.interface import Interface @@ -65,8 +64,7 @@ class DisplayedDatatypesScreen(ModalScreen): - """ - Screen to select the which datatype checkboxes to show on the Create / Transfer tabs. + """Screen to select the which datatype checkboxes to show on the Create / Transfer tabs. Display a SessionList widget which all canonical broad and narrow-type datatypes. When selected, this will update DatatypeCheckboxes (coordinates @@ -83,6 +81,7 @@ class DisplayedDatatypesScreen(ModalScreen): checkboxes are so close together. Testing indicate that when writing to file after each click, syncing could get messed up and the wrong checkboxes displayed on the window. + """ def __init__( @@ -102,8 +101,7 @@ def __init__( ) def compose(self) -> ComposeResult: - """ - Collect the datatypes names and status from + """Collect the datatypes names and status from the persistent settings and display. """ selections = [] @@ -146,8 +144,7 @@ def on_mount(self): # assert False, f"{dir(self.query_one('#displayed_datatypes_selection_list'))}" def on_button_pressed(self, event): - """ - When 'Save' is pressed, the configs copied on this class + """When 'Save' is pressed, the configs copied on this class are updated back onto the interface configs, and written to disk. Otherwise, close the screen without saving. """ @@ -163,8 +160,7 @@ def on_button_pressed(self, event): def on_selection_list_selection_toggled( self, event: SelectionList.SelectionMessage.SelectionToggled ): - """ - When a selection is toggled, update the configs with + """When a selection is toggled, update the configs with the 'displayed' status and save to disk. """ datatype_name = event.selection.prompt.plain @@ -182,13 +178,11 @@ def on_selection_list_selection_toggled( class DatatypeCheckboxes(Static): - """ - Dynamically-populated checkbox widget for convenient datatype + """Dynamically-populated checkbox widget for convenient datatype selection during folder creation. Parameters ---------- - settings_key 'create' if datatype checkboxes for the create tab, 'transfer' for the transfer tab. Transfer tab includes @@ -196,7 +190,6 @@ class DatatypeCheckboxes(Static): Attributes ---------- - datatype_config a Dictionary containing datatype as key (e.g. "ephys", "behav") and values are `bool` indicating whether the checkbox is on / off. @@ -210,6 +203,7 @@ class DatatypeCheckboxes(Static): however because this screen persists through the lifetime of the app there is no clear time point in which to save the checkbox status. Therefore, the configs are updated (written to disk) on each click. + """ def __init__( @@ -242,8 +236,7 @@ def compose(self) -> ComposeResult: @on(Checkbox.Changed) def on_checkbox_changed(self) -> None: - """ - When a checkbox is changed, update the `self.datatype_config` + """When a checkbox is changed, update the `self.datatype_config` to contain new boolean values for each datatype. Also update the stored `persistent_settings`. """ @@ -266,8 +259,7 @@ def on_mount(self) -> None: ).tooltip = tooltips[datatype] def selected_datatypes(self) -> List[str]: - """ - Get the names of the datatype options for which the + """Get the names of the datatype options for which the checkboxes are switched on. """ selected_datatypes = [ diff --git a/datashuttle/tui/screens/get_help.py b/datashuttle/tui/screens/get_help.py index e177b7d42..622f9474b 100644 --- a/datashuttle/tui/screens/get_help.py +++ b/datashuttle/tui/screens/get_help.py @@ -47,7 +47,6 @@ def action_link_zulip(self): webbrowser.open(links.get_link_zulip()) def compose(self) -> ComposeResult: - yield Container( Static(self.text, id="get_help_label"), Button("Main Menu", id="all_main_menu_buttons"), diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index 4d801e823..9625c0318 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -23,8 +23,7 @@ class MessageBox(ModalScreen): - """ - A screen for rendering error messages. + """A screen for rendering error messages. message The message to display in the message box @@ -70,8 +69,7 @@ def on_button_pressed(self) -> None: class ConfirmAndAwaitTransferPopup(ModalScreen): - """ - A popup screen for confirming, awaiting and finishing a Transfer. + """A popup screen for confirming, awaiting and finishing a Transfer. When users select Transfer, this screen pops up to a) allow users to confirm transfer b) display a `LoadingIndicator` while the transfer runs in a separate worker c) indicate the transfer is finished. @@ -117,7 +115,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: async def handle_transfer_and_update_ui_when_complete(self) -> None: """Runs the data transfer worker and updates the UI on completion""" - data_transfer_worker = self.transfer_func() await data_transfer_worker.wait() success, output = data_transfer_worker.result @@ -138,21 +135,20 @@ async def handle_transfer_and_update_ui_when_complete(self) -> None: class SelectDirectoryTreeScreen(ModalScreen): - """ - A modal screen that includes a DirectoryTree to browse + """A modal screen that includes a DirectoryTree to browse and select folders. If a folder is double-clicked, the path to the folder is returned through 'dismiss' callback mechanism. Parameters ---------- - mainwindow Textual main app screen path_ Path to use as the DirectoryTree root, if `None` set to the system user home. + """ def __init__(self, mainwindow: App, path_: Optional[Path] = None) -> None: @@ -166,7 +162,6 @@ def __init__(self, mainwindow: App, path_: Optional[Path] = None) -> None: self.prev_click_time = 0 def compose(self) -> ComposeResult: - label_message = ( "Select (double click) a folder with the same name as the project.\n" "If the project folder does not exist, select the parent folder and it will be created." diff --git a/datashuttle/tui/screens/new_project.py b/datashuttle/tui/screens/new_project.py index ea7c0826c..769629ff3 100644 --- a/datashuttle/tui/screens/new_project.py +++ b/datashuttle/tui/screens/new_project.py @@ -14,8 +14,7 @@ class NewProjectScreen(Screen): - """ - Screen for setting up a new datashuttle project, by + """Screen for setting up a new datashuttle project, by inputting the desired configs. This uses the ConfigsContent window to display and set the configs. @@ -30,9 +29,9 @@ class NewProjectScreen(Screen): Parameters ---------- - mainwindow The main TUI app + """ TITLE = "Make New Project" diff --git a/datashuttle/tui/screens/project_manager.py b/datashuttle/tui/screens/project_manager.py index 7d6343632..ebe0168e9 100644 --- a/datashuttle/tui/screens/project_manager.py +++ b/datashuttle/tui/screens/project_manager.py @@ -22,8 +22,7 @@ class ProjectManagerScreen(Screen): - """ - Screen containing the Create, Transfer and Configs tabs. This is + """Screen containing the Create, Transfer and Configs tabs. This is the primary screen within which the user interacts with a pre-configured project. @@ -85,8 +84,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Dismisses the TabScreen (and returns to the main menu) once + """Dismisses the TabScreen (and returns to the main menu) once the 'Main Menu' button is pressed. """ if event.button.id == "all_main_menu_buttons": @@ -95,8 +93,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: def on_tabbed_content_tab_activated( self, event: TabbedContent.TabActivated ) -> None: - """ - Refresh the directorytree for create or transfer tabs whenever + """Refresh the directorytree for create or transfer tabs whenever the tabbedcontent is switched to one of these tabs. This is also triggered on mount, leading to it being reloaded @@ -125,8 +122,7 @@ def update_active_tab_tree(self): self.query_one(f"#{active_tab_id}").reload_directorytree() def on_configs_content_configs_saved(self) -> None: - """ - When configs are saved, we may switch between a 'full' project + """When configs are saved, we may switch between a 'full' project and a 'local only' project (no `central_path` or `connection_method` set). In such a case we need to refresh the ProjectManager screen to add / remove the transfer tab. @@ -148,7 +144,6 @@ def on_configs_content_configs_saved(self) -> None: ) if old_project_type == project_type: - if project_type == "full": self.query_one( "#tabscreen_transfer_tab" @@ -167,8 +162,7 @@ def on_configs_content_configs_saved(self) -> None: ) def wrap_dismiss(self, _): - """ - Need to wrap dismiss as cannot include it directly + """Need to wrap dismiss as cannot include it directly in push_screen callback, or even wrapped in lambda. """ self.dismiss() diff --git a/datashuttle/tui/screens/project_selector.py b/datashuttle/tui/screens/project_selector.py index 289990e5c..3bd923570 100644 --- a/datashuttle/tui/screens/project_selector.py +++ b/datashuttle/tui/screens/project_selector.py @@ -18,8 +18,7 @@ class ProjectSelectorScreen(Screen): - """ - The project selection screen. Finds and displays DataShuttle + """The project selection screen. Finds and displays DataShuttle projects present on the local system. `self.dismiss()` returns an initialised project if initialisation @@ -28,7 +27,6 @@ class ProjectSelectorScreen(Screen): Parameters ---------- - mainwindow The main TUI app, functions on which are used to coordinate screen display. @@ -55,7 +53,6 @@ def compose(self) -> ComposeResult: def on_button_pressed(self, event: Button.Pressed) -> None: if event.button.id in self.project_names: - project_name = event.button.id interface = Interface() diff --git a/datashuttle/tui/screens/settings.py b/datashuttle/tui/screens/settings.py index 9f14d9ab4..0634ea041 100644 --- a/datashuttle/tui/screens/settings.py +++ b/datashuttle/tui/screens/settings.py @@ -20,8 +20,7 @@ class SettingsScreen(ModalScreen): - """ - Screen accessible from the main window that contains + """Screen accessible from the main window that contains 'global' settings for the TUI. 'Global' settings are non-project specific settings (e.g. dark mode) and are handled independently of the main datashuttle API. diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 8d1721a13..7502caadb 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -18,8 +18,7 @@ class SetupSshScreen(ModalScreen): - """ - This dialog windows handles the TUI equivalent of API's + """This dialog windows handles the TUI equivalent of API's setup_ssh_connection(). This asks to confirm the central hostkey, and takes password to setup SSH key pair. @@ -42,7 +41,7 @@ def compose(self) -> ComposeResult: yield Container( Horizontal( Static( - "Ready to setup setup SSH. " "Press OK to proceed.", + "Ready to setup setup SSH. Press OK to proceed.", id="messagebox_message_label", ), id="messagebox_message_container", @@ -60,8 +59,7 @@ def on_mount(self) -> None: self.query_one("#setup_ssh_password_input").visible = False def on_button_pressed(self, event: Button.pressed) -> None: - """ - When each stage is successfully progressed by clicking the "ok" button, + """When each stage is successfully progressed by clicking the "ok" button, `self.stage` is iterated by 1. For saving and excepting hostkey, if there is a problem (error or user declines) the 'OK' button is frozen so it is not possible to proceed. For accepting password @@ -84,8 +82,7 @@ def on_button_pressed(self, event: Button.pressed) -> None: self.dismiss() def ask_user_to_accept_hostkeys(self) -> None: - """ - The central server is identified by a hostkey. + """The central server is identified by a hostkey. Get this hostkey and present it to user, clicking 'OK' is they are happy. If there is an error, block process (because it most likely is necessary to edit the central host id) and @@ -116,8 +113,7 @@ def ask_user_to_accept_hostkeys(self) -> None: self.stage += 1 def save_hostkeys_and_prompt_password_input(self) -> None: - """ - Once the hostkey is accepted, get the user password + """Once the hostkey is accepted, get the user password for the central server. When 'OK' is pressed we go straight to 'use_password_to_setup_ssh_key_pairs'. """ @@ -141,8 +137,7 @@ def save_hostkeys_and_prompt_password_input(self) -> None: self.stage += 1 def use_password_to_setup_ssh_key_pairs(self) -> None: - """ - Get the user password for the central server. If correct, + """Get the user password for the central server. If correct, SSH key pair is setup and 'OK' button changed to 'Finish'. Otherwise, continue allowing failed password attempts. """ diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 1e3ef7de8..7b98eb6db 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -35,9 +35,7 @@ class CreateFoldersTab(TreeAndInputTab): - """ - Create new project files formatted according to the NeuroBlueprint specification. - """ + """Create new project files formatted according to the NeuroBlueprint specification.""" def __init__(self, mainwindow: App, interface: Interface) -> None: super(CreateFoldersTab, self).__init__( @@ -110,8 +108,7 @@ def on_mount(self) -> None: self.query_one(id).tooltip = get_tooltip(id) def on_button_pressed(self, event: Button.Pressed) -> None: - """ - Enables the Create Folders button to read out current input values + """Enables the Create Folders button to read out current input values and use these to call project.create_folders(). `unused_bool` is necessary to get dismiss to call @@ -121,7 +118,6 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.create_folders() elif event.button.id == "create_folders_displayed_datatypes_button": - self.mainwindow.push_screen( DisplayedDatatypesScreen("create", self.interface), self.refresh_after_datatypes_changed, @@ -141,8 +137,7 @@ async def refresh_after_datatypes_changed(self, ignore): def on_clickable_input_clicked( self, event: ClickableInput.Clicked ) -> None: - """ - Handled a double click on the custom ClickableInput widget, + """Handled a double click on the custom ClickableInput widget, which indicates the input should be filled with a suggested value. Determine if we have the subject or session input, and @@ -161,8 +156,7 @@ def on_clickable_input_clicked( def on_custom_directory_tree_directory_tree_special_key_press( self, event: CustomDirectoryTree.DirectoryTreeSpecialKeyPress ): - """ - Handle a key press on the directory tree, which can refresh the + """Handle a key press on the directory tree, which can refresh the directorytree or fill / append subject/session folder name to the relevant input widget. """ @@ -180,8 +174,7 @@ def on_custom_directory_tree_directory_tree_special_key_press( self.mainwindow.prompt_rename_file_or_folder(event.node_path) def fill_input_with_template(self, prefix: Prefix, input_id: str) -> None: - """ - Given the `name_template`, fill the sub or ses + """Given the `name_template`, fill the sub or ses Input with the template (based on `prefix`). If `self.templates` is off, then just suggest "sub-" or "ses-". """ @@ -203,8 +196,7 @@ def templates_on(self, prefix: Prefix) -> bool: # ---------------------------------------------------------------------------------- def revalidate_inputs(self, all_prefixes: List[str]) -> None: - """ - Revalidate and style both subject and session + """Revalidate and style both subject and session inputs based on their value. """ input_names = { @@ -218,8 +210,7 @@ def revalidate_inputs(self, all_prefixes: List[str]) -> None: self.query_one(key).validate(value=value) def update_input_tooltip(self, message: List[str], prefix: Prefix) -> None: - """ - Update the value of a subject or session tooltip, which + """Update the value of a subject or session tooltip, which indicates the validation status of the input value. """ id = ( @@ -238,8 +229,7 @@ def update_input_tooltip(self, message: List[str], prefix: Prefix) -> None: # ---------------------------------------------------------------------------------- def create_folders(self) -> None: - """ - Create project folders based on current widget input + """Create project folders based on current widget input through the datashuttle API. """ ses_names: Optional[List[str]] @@ -261,8 +251,7 @@ def create_folders(self) -> None: self.mainwindow.show_modal_error_dialog(output) def reload_directorytree(self) -> None: - """ - This reloads the directorytree and also updates validation. + """This reloads the directorytree and also updates validation. Not now a good method name but done for consistency with other tab refresh methods. """ @@ -275,8 +264,7 @@ def reload_directorytree(self) -> None: def fill_input_with_next_sub_or_ses_template( self, prefix: Prefix, input_id: str ) -> None: - """ - This fills a sub / ses Input with a suggested name based on the + """This fills a sub / ses Input with a suggested name based on the next subject / session in the project (local). If `name_templates` are set, then the sub- or ses- first key @@ -286,12 +274,12 @@ def fill_input_with_next_sub_or_ses_template( Parameters ---------- - prefix Whether to fill the subject or session Input input_id The textual input name to update. + """ top_level_folder = self.interface.tui_settings[ "top_level_folder_select" @@ -345,8 +333,7 @@ def fill_input_with_next_sub_or_ses_template( input.value = fill_value def run_local_validation(self, prefix: Prefix): - """ - Run validation of the values stored in the + """Run validation of the values stored in the sub / ses Input according to the passed prefix using core datashuttle functions. @@ -367,9 +354,9 @@ def run_local_validation(self, prefix: Prefix): Parameters ---------- - prefix Whether to run validation on the subject or session Input + """ sub_names = self.query_one( "#create_folders_subject_input" @@ -397,7 +384,5 @@ def run_local_validation(self, prefix: Prefix): return True, f"Formatted names: {names}" def update_directorytree_root(self, new_root_path: Path) -> None: - """ - Will automatically refresh the tree through the reactive attribute `path`. - """ + """Will automatically refresh the tree through the reactive attribute `path`.""" self.query_one("#create_folders_directorytree").path = new_root_path diff --git a/datashuttle/tui/tabs/logging.py b/datashuttle/tui/tabs/logging.py index fb45c8269..f35c7a987 100644 --- a/datashuttle/tui/tabs/logging.py +++ b/datashuttle/tui/tabs/logging.py @@ -16,7 +16,7 @@ class RichLogScreen(ModalScreen): def __init__(self, log_file): super(RichLogScreen, self).__init__() - with open(log_file, "r") as file: + with open(log_file) as file: self.log_contents = "".join(file.readlines()) def compose(self): diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py index 4603c6674..b50a14b08 100644 --- a/datashuttle/tui/tabs/transfer.py +++ b/datashuttle/tui/tabs/transfer.py @@ -43,15 +43,13 @@ class TransferTab(TreeAndInputTab): - """ - This tab handles the upload / download of files between local + """This tab handles the upload / download of files between local and central folders. It contains a TransferDirectoryTree that displays the transfer status of the files in the local folder, and calls underlying datashuttle transfer functions. Parameters ---------- - title The title of the tab @@ -66,7 +64,6 @@ class TransferTab(TreeAndInputTab): Attributes ---------- - show_legend Convenience attribute linked to a global setting exists that turns off / on styling of directorytree nodes based on transfer status. ` @@ -76,6 +73,7 @@ class TransferTab(TreeAndInputTab): ]` When on, the legend must be hidden. + """ def __init__( @@ -210,7 +208,6 @@ def compose(self) -> ComposeResult: yield Label("⭕ Legend", id="transfer_legend") def on_mount(self) -> None: - for id in [ "#transfer_directorytree", "#transfer_switch_container", @@ -261,8 +258,7 @@ def on_checkbox_changed(self, event: Checkbox.Changed) -> None: # ---------------------------------------------------------------------------------- def switch_transfer_widgets_display(self) -> None: - """ - Show or hide transfer parameters based on whether the transfer mode + """Show or hide transfer parameters based on whether the transfer mode currently selected in `transfer_radioset`. """ for widget in self.transfer_all_widgets: @@ -279,8 +275,7 @@ def switch_transfer_widgets_display(self) -> None: ).value def on_radio_set_changed(self, event: RadioSet.Changed) -> None: - """ - Update the displayed transfer parameter widgets when the + """Update the displayed transfer parameter widgets when the `transfer_radioset` radiobuttons are changed. """ label = str(event.pressed.label) @@ -288,8 +283,7 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.switch_transfer_widgets_display() def on_button_pressed(self, event: Button.Pressed) -> None: - """ - If the Transfer button is clicked, opens a modal dialog + """If the Transfer button is clicked, opens a modal dialog to confirm that the user wishes to transfer their data (in the direction selected). If "Yes" is selected, `self.transfer_data` (see below) is run. @@ -347,8 +341,7 @@ def reload_directorytree(self) -> None: self.query_one("#transfer_directorytree").update_transfer_tree() def update_directorytree_root(self, new_root_path: Path) -> None: - """ - This will automatically refresh the tree through the + """This will automatically refresh the tree through the reactive variable `path`. """ self.query_one("#transfer_directorytree").path = new_root_path @@ -358,8 +351,7 @@ def update_directorytree_root(self, new_root_path: Path) -> None: @work(exclusive=True, thread=True) def transfer_data(self) -> Worker[InterfaceOutput]: - """ - A threaded worker to transfer data + """A threaded worker to transfer data This function transfers data based on the config provided by the radio buttons such as a) the data to be transferred (all / top-level-folders / custom) b) the @@ -375,7 +367,6 @@ def transfer_data(self) -> Worker[InterfaceOutput]: success, output = self.interface.transfer_entire_project(upload) elif self.query_one("#transfer_toplevel_radiobutton").value: - selected_top_level_folder = self.query_one( "#transfer_toplevel_select" ).get_top_level_folder() @@ -385,7 +376,6 @@ def transfer_data(self) -> Worker[InterfaceOutput]: ) elif self.query_one("#transfer_custom_radiobutton").value: - selected_top_level_folder = self.query_one( "#transfer_custom_select" ).get_top_level_folder() diff --git a/datashuttle/tui/tabs/transfer_status_tree.py b/datashuttle/tui/tabs/transfer_status_tree.py index dbcaa205c..1d4410fc7 100644 --- a/datashuttle/tui/tabs/transfer_status_tree.py +++ b/datashuttle/tui/tabs/transfer_status_tree.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING, Dict, Optional if TYPE_CHECKING: - from rich.style import Style from textual.widgets._directory_tree import DirEntry @@ -24,23 +23,21 @@ class TransferStatusTree(CustomDirectoryTree): - """ - A directorytree in which the nodes are styled depending on their + """A directorytree in which the nodes are styled depending on their transfer status. e.g. indicates whether files are changed between local or central, or appear in local only. Attributes ---------- - Keep the local path as a string, linked to project.cfg["local_path"], so that no conversion to string is necessary in `format_transfer_label` which is called many times. + """ def __init__( self, mainwindow: App, interface: Interface, id: Optional[str] = None ): - self.interface = interface self.local_path_str = self.interface.get_configs()[ "local_path" @@ -55,8 +52,7 @@ def on_mount(self) -> None: self.update_transfer_tree(init=True) def update_transfer_tree(self, init: bool = False) -> None: - """ - Updates tree styling to reflect the current TUI state + """Updates tree styling to reflect the current TUI state and project transfer status. """ self.local_path_str = self.interface.get_configs()[ @@ -72,9 +68,7 @@ def update_transfer_tree(self, init: bool = False) -> None: self.reload() def update_local_transfer_paths(self) -> None: - """ - Compiles a list of all project files and paths. - """ + """Compiles a list of all project files and paths.""" paths_list = [] for top_level_folder in canonical_folders.get_top_level_folders(): @@ -88,9 +82,7 @@ def update_local_transfer_paths(self) -> None: self.transfer_paths = paths_list def update_transfer_diffs(self) -> None: - """ - Updates the transfer diffs used to style the DirectoryTree. - """ + """Updates the transfer diffs used to style the DirectoryTree.""" self.transfer_diffs = get_local_and_central_file_differences( self.interface.get_configs(), top_level_folders_to_check=["rawdata", "derivatives"], @@ -102,8 +94,7 @@ def update_transfer_diffs(self) -> None: def render_label( self, node: TreeNode[DirEntry], base_style: Style, style: Style ) -> Text: - """ - Extends the `DirectoryTree.render_label()` method to allow + """Extends the `DirectoryTree.render_label()` method to allow custom styling of file nodes according to their transfer status. """ node_label = node._label.copy() @@ -145,8 +136,7 @@ def render_label( return text def format_transfer_label(self, node_label, node_path) -> None: - """ - Takes nodes being formatted using `render_label` and applies custom + """Takes nodes being formatted using `render_label` and applies custom formatting according to the node's transfer status. """ node_relative_path = node_path.as_posix().replace( diff --git a/datashuttle/tui/tooltips.py b/datashuttle/tui/tooltips.py index 61cb9f70d..21a6558f7 100644 --- a/datashuttle/tui/tooltips.py +++ b/datashuttle/tui/tooltips.py @@ -1,6 +1,5 @@ def get_tooltip(id: str) -> str: - """ - Master function to get tooltips for all widgets, + """Master function to get tooltips for all widgets, based on their widget (textual) id. """ # Configs diff --git a/datashuttle/tui/utils/tui_decorators.py b/datashuttle/tui/utils/tui_decorators.py index f2a96d41f..6304c9054 100644 --- a/datashuttle/tui/utils/tui_decorators.py +++ b/datashuttle/tui/utils/tui_decorators.py @@ -9,8 +9,7 @@ def require_double_click(func): - """ - A decorator that calls the decorated function + """A decorator that calls the decorated function on a double click, otherwise will not do anything. Requires the first argument (`self` on the class) to diff --git a/datashuttle/tui/utils/tui_validators.py b/datashuttle/tui/utils/tui_validators.py index 792fc1c5e..1605c63ba 100644 --- a/datashuttle/tui/utils/tui_validators.py +++ b/datashuttle/tui/utils/tui_validators.py @@ -1,6 +1,4 @@ -""" -Tools for live validation of user inputs in the DataShuttle TUI. -""" +"""Tools for live validation of user inputs in the DataShuttle TUI.""" from __future__ import annotations @@ -15,8 +13,7 @@ class NeuroBlueprintValidator(Validator): def __init__(self, prefix: Prefix, parent: CreateFoldersTab) -> None: - """ - Custom Validator() class that takes + """Custom Validator() class that takes sub / ses prefix as input. Runs validation of the name against the project and propagates any error message through the Input tooltip. @@ -26,8 +23,7 @@ def __init__(self, prefix: Prefix, parent: CreateFoldersTab) -> None: self.prefix = prefix def validate(self, name: str) -> ValidationResult: - """ - Run validation and update the tooltip with the error, + """Run validation and update the tooltip with the error, if no error then the formatted sub / ses name is displayed. This is set on an Input widget. """ diff --git a/datashuttle/tui_launcher.py b/datashuttle/tui_launcher.py index fb2e05ff9..f98f4b0be 100644 --- a/datashuttle/tui_launcher.py +++ b/datashuttle/tui_launcher.py @@ -29,9 +29,7 @@ def main() -> None: - """ - Launch the datashuttle tui. - """ + """Launch the datashuttle tui.""" args = parser.parse_args() if args.launch == "launch": diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index a39e6377f..10562f300 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -12,8 +12,7 @@ class TransferData: - """ - Class to perform data transfers. This works by first building + """Class to perform data transfers. This works by first building a large list of all files to transfer. Then, rclone is called once with this list to perform the transfer. @@ -23,7 +22,6 @@ class TransferData: Parameters ---------- - cfg datashuttle configs UserDict. @@ -58,6 +56,7 @@ class TransferData: log if `True`, log and print the transfer output. + """ def __init__( @@ -112,8 +111,7 @@ def __init__( # ------------------------------------------------------------------------- def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: - """ - Build a list of every file to transfer based on the user-passed + """Build a list of every file to transfer based on the user-passed arguments. This cycles through every subject, session and datatype and adds the outputs to three lists: @@ -123,9 +121,9 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: Returns ------- - include_list A list of paths to pass to rclone's `--include` flag. + """ # Find sub names to transfer processed_sub_names = self.get_processed_names(self.sub_names) @@ -186,8 +184,7 @@ def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: def make_include_arg( self, list_of_paths: List[str], recursive: bool = True ) -> List[str]: - """ - Format the list of paths to rclone's required + """Format the list of paths to rclone's required `--include` flag format. """ if not any(list_of_paths): @@ -212,8 +209,7 @@ def include_arg(ele: str) -> str: def update_list_with_non_sub_top_level_folders( self, extra_folder_names: List[str], extra_filenames: List[str] ) -> None: - """ - Search the subject level for all files and folders in the + """Search the subject level for all files and folders in the top-level-folder. Split the output based onto files / folders within "sub-" prefixed folders or not. """ @@ -240,8 +236,7 @@ def update_list_with_non_ses_sub_level_folders( extra_filenames: List[str], sub: str, ) -> None: - """ - For the subject, get a list of files / folders that are + """For the subject, get a list of files / folders that are not within "ses-" prefixed folders. """ sub_level_folders: List[str] @@ -277,8 +272,7 @@ def update_list_with_non_dtype_ses_level_folders( sub: str, ses: str, ) -> None: - """ - For a specific subject and session, get a list of files / folders + """For a specific subject and session, get a list of files / folders that are not in canonical datashuttle datatype folders. """ ses_level_folders: List[str] @@ -322,8 +316,7 @@ def update_list_with_dtype_paths( sub: str, ses: Optional[str] = None, ) -> None: - """ - Given a particular subject and session, get a list of all + """Given a particular subject and session, get a list of all canonical datatype folders. """ datatype = list(filter(lambda x: x != "all_non_datatype", datatype)) @@ -360,8 +353,7 @@ def to_list(self, names: Union[str, List[str]]) -> List[str]: def check_input_arguments( self, ) -> None: - """ - Check the sub / session names passed. The checking here + """Check the sub / session names passed. The checking here is stricter than for create_folders / formatting.check_and_format_names because we want to ensure that a) non-datatype arguments are not passed at the wrong input (e.g. all_non_ses as a subject name). @@ -373,8 +365,8 @@ def check_input_arguments( Parameters ---------- - see update_list_with_dtype_paths() + """ if len(self.sub_names) > 1 and any( [name in ["all", "all_sub"] for name in self.sub_names] @@ -422,8 +414,7 @@ def get_processed_names( names_checked: List[str], sub: Optional[str] = None, ) -> List[str]: - """ - Process the list of subject session names. + """Process the list of subject session names. If they are pre-defined (e.g. ["sub-001", "sub-002"]) they will be checked and formatted as per formatting.check_and_format_names() and @@ -436,7 +427,6 @@ def get_processed_names( Parameters ---------- - see transfer_sub_ses_data() """ @@ -478,8 +468,7 @@ def get_processed_names( return processed_names def transfer_non_datatype(self, datatype_checked: List[str]) -> bool: - """ - Convenience function, bool if all non-datatype folders + """Convenience function, bool if all non-datatype folders are to be transferred """ return any( diff --git a/datashuttle/utils/decorators.py b/datashuttle/utils/decorators.py index cacf54914..99dcde8ed 100644 --- a/datashuttle/utils/decorators.py +++ b/datashuttle/utils/decorators.py @@ -5,8 +5,7 @@ def requires_ssh_configs(func): - """ - Decorator to check file is loaded. Used on Mainwindow class + """Decorator to check file is loaded. Used on Mainwindow class methods only as first arg is assumed to be self (containing cfgs) """ @@ -29,8 +28,7 @@ def wrapper(*args, **kwargs): def check_configs_set(func): - """ - Check that configs have been loaded (i.e. + """Check that configs have been loaded (i.e. project.cfg is not None) before the func is run. """ @@ -50,8 +48,7 @@ def wrapper(*args, **kwargs): def check_is_not_local_project(func): - """ - Decorator to check that the project is not + """Decorator to check that the project is not a local project. If it is, raise. This decorator should be placed above methods which diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index 3d0ac7b69..353fc0886 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -38,9 +38,7 @@ def start( variables: Optional[List[Any]], verbose: bool = True, ) -> None: - """ - Call fancylog to initialise logging. - """ + """Call fancylog to initialise logging.""" filename = get_logging_filename(command_name) fancylog.start_logging( @@ -60,8 +58,7 @@ def start( def get_logging_filename(command_name: str) -> str: - """ - Get the filename to which the log will be saved. This + """Get the filename to which the log will be saved. This starts with ISO8601-formatted datetime, so logs are stored in datetime order. """ @@ -70,26 +67,24 @@ def get_logging_filename(command_name: str) -> str: def log_names(list_of_headers: List[Any], list_of_names: List[Any]) -> None: - """ - Log a list of subject or session names. + """Log a list of subject or session names. Parameters ---------- - list_of_headers a list of titles that the names will be printed under, e.g. "sub_names", "ses_names" list_of_names list of names to print to log + """ for header, names in zip(list_of_headers, list_of_names): utils.log(f"{header}: {names}") def wrap_variables_for_fancylog(local_vars: dict, cfg: Configs) -> List: - """ - Wrap the locals from the original function call to log + """Wrap the locals from the original function call to log and the datashuttle.cfg in a wrapper class with __dict__ attribute for fancylog writing. @@ -110,9 +105,7 @@ def __init__(self, local_vars_, cfg_): def close_log_filehandler() -> None: - """ - Remove handlers from all loggers. - """ + """Remove handlers from all loggers.""" logger = get_logger() logger.debug("Finished logging.") handlers = logger.handlers[:] diff --git a/datashuttle/utils/folder_class.py b/datashuttle/utils/folder_class.py index 6ada726c2..8b85856de 100644 --- a/datashuttle/utils/folder_class.py +++ b/datashuttle/utils/folder_class.py @@ -1,6 +1,5 @@ class Folder: - """ - Folder class used to contain details of canonical + """Folder class used to contain details of canonical folders in the project folder tree. see configs.canonical_folders.py for details. diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 33d470088..f7d86c4b7 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -36,8 +36,7 @@ def create_folder_trees( datatype: Union[List[str], str], log: bool = True, ) -> Dict[str, List[Path]]: - """ - Entry method to make a full folder tree. It will + """Entry method to make a full folder tree. It will iterate through all passed subjects, then sessions, then subfolders within a datatype folder. This permits flexible creation of folders (e.g. @@ -48,13 +47,13 @@ def create_folder_trees( Parameters ---------- - sub_names, ses_names, datatype see create_folders() log whether to log or not. If True, logging must already be initialised. + """ datatype_passed = datatype not in [[""], ""] @@ -123,14 +122,12 @@ def make_datatype_folders( save_paths: Dict, log: bool = True, ): - """ - Make datatype folder (e.g. behav) at the sub or ses + """Make datatype folder (e.g. behav) at the sub or ses level. Checks folder_class.Folders attributes, whether the datatype is used and at the current level. Parameters ---------- - cfg datashuttle configs @@ -154,12 +151,12 @@ def make_datatype_folders( log whether to log on or not (if True, logging must already be initialised). + """ datatype_items = cfg.get_datatype_as_dict_items(datatype) for datatype_key, datatype_folder in datatype_items: # type: ignore if datatype_folder.level == level: - datatype_name = datatype_folder.name datatype_path = sub_or_ses_level_path / datatype_name @@ -177,19 +174,18 @@ def make_datatype_folders( def create_folders(paths: Union[Path, List[Path]], log: bool = True) -> None: - """ - For path or list of paths, make them if + """For path or list of paths, make them if they do not already exist. Parameters ---------- - paths Path or list of Paths to create log if True, log all made folders. This requires the logger to already be initialised. + """ if isinstance(paths, Path): paths = [paths] @@ -217,8 +213,7 @@ def search_project_for_sub_or_ses_names( include_central: bool, return_full_path: bool = False, ) -> Dict: - """ - If sub is None, the top-level level folder will be + """If sub is None, the top-level level folder will be searched (i.e. for subjects). The search string "sub-*" is suggested in this case. Otherwise, the subject, level folder for the specified subject will be searched. The search_str "ses-*" is suggested in this case. @@ -228,7 +223,6 @@ def search_project_for_sub_or_ses_names( will be searched for on central, showing a confusing 'folder not found' message. """ - # Search local and central for folders that begin with "sub-*" local_foldernames, _ = search_sub_or_ses_level( cfg, @@ -270,15 +264,14 @@ def items_from_datatype_input( sub: str, ses: Optional[str] = None, ) -> Union[ItemsView, zip]: - """ - Get the list of datatypes to transfer, either + """Get the list of datatypes to transfer, either directly from user input, or by searching what is available if "all" is passed. Parameters ---------- - see _transfer_datatype() for parameters. + """ base_folder = cfg.get_base_folder(local_or_central, top_level_folder) @@ -310,8 +303,7 @@ def search_for_datatype_folders( sub: str, ses: Optional[str] = None, ) -> zip: - """ - Search a subject or session folder specifically + """Search a subject or session folder specifically for datatypes. First searches for all folders / files in the folder, and then returns any folders that match datatype name. @@ -323,6 +315,7 @@ def search_for_datatype_folders( ------- Find the datatype files and return in a format that mirrors dict.items() + """ search_results = search_sub_or_ses_level( cfg, base_folder, local_or_central, sub, ses @@ -339,8 +332,7 @@ def process_glob_to_find_datatype_folders( folder_names: list, datatype_folders: dict, ) -> zip: - """ - Process the results of glob on a sub or session level, + """Process the results of glob on a sub or session level, which could contain any kind of folder / file. see project.search_sub_or_ses_level() for inputs. @@ -349,6 +341,7 @@ def process_glob_to_find_datatype_folders( ------- Find the datatype files and return in a format that mirrors dict.items() + """ ses_folder_keys = [] ses_folder_values = [] @@ -377,8 +370,7 @@ def search_for_wildcards( all_names: List[str], sub: Optional[str] = None, ) -> List[str]: - """ - Handle wildcard flag in upload or download. + """Handle wildcard flag in upload or download. All names in name are searched for @*@ string, and replaced with single * for glob syntax. If sub is passed, it is @@ -393,7 +385,6 @@ def search_for_wildcards( Parameters ---------- - project initialised datashuttle project @@ -457,13 +448,11 @@ def search_sub_or_ses_level( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[str] | List[Path], List[str]]: - """ - Search project folder at the subject or session level. + """Search project folder at the subject or session level. Only returns folders Parameters ---------- - cfg datashuttle project cfg. Currently, this is used as a holder for ssh configs to avoid too many @@ -490,11 +479,11 @@ def search_sub_or_ses_level( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + """ if ses and not sub: utils.log_and_raise_error( - "cannot pass session to " - "search_sub_or_ses_level() without subject", + "cannot pass session to search_sub_or_ses_level() without subject", ValueError, ) @@ -524,13 +513,11 @@ def search_for_folders( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """ - Wrapper to determine the method used to search for search + """Wrapper to determine the method used to search for search prefix folders in the search path. Parameters ---------- - local_or_central "local" or "central" @@ -543,6 +530,7 @@ def search_for_folders( verbose If `True`, when a search folder cannot be found, a message will be printed with the missing path. + """ if local_or_central == "central" and cfg["connection_method"] == "ssh": all_folder_names, all_filenames = ssh.search_ssh_central_for_folders( @@ -570,8 +558,7 @@ def search_for_folders( def search_filesystem_path_for_folders( search_path_with_prefix: Path, return_full_path: bool = False ) -> Tuple[List[Path | str], List[Path | str]]: - """ - Use glob to search the full search path (including prefix) with glob. + """Use glob to search the full search path (including prefix) with glob. Files are filtered out of results, returning folders only. """ all_folder_names = [] @@ -581,7 +568,6 @@ def search_filesystem_path_for_folders( sorter_files_and_folders = sorted(all_files_and_folders) for file_or_folder_str in sorter_files_and_folders: - file_or_folder = Path(file_or_folder_str) if file_or_folder.is_dir(): diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index 12e9a6027..1cea80fa8 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -22,8 +22,7 @@ def check_and_format_names( name_templates: Optional[Dict] = None, bypass_validation: bool = False, ) -> List[str]: - """ - Format a list of subject or session names, e.g. + """Format a list of subject or session names, e.g. by ensuring all have sub- or ses- prefix, checking for tags, that names do not include spaces and that there are not duplicates. @@ -38,7 +37,6 @@ def check_and_format_names( Parameters ---------- - names str or list containing sub or ses names (e.g. to create folders) @@ -53,6 +51,7 @@ def check_and_format_names( bypass_validation If `True`, NeuroBlueprint validation will be performed on the passed names. + """ if isinstance(names, str): names = [names] @@ -79,8 +78,7 @@ def check_and_format_names( def format_names(names: List, prefix: Prefix) -> List[str]: - """ - Check a single or list of input session or subject names. + """Check a single or list of input session or subject names. First check the type is correct, next prepend the prefix sub- or ses- to entries that do not have the relevant prefix. @@ -88,13 +86,13 @@ def format_names(names: List, prefix: Prefix) -> List[str]: with required inputs e.g. date, time Parameters - ----------- - + ---------- names str or list containing sub or ses names (e.g. to make folders) prefix "sub" or "ses" - this defines the prefix checks. + """ assert prefix in ["sub", "ses"], "`prefix` must be 'sub' or 'ses'." @@ -117,8 +115,7 @@ def format_names(names: List, prefix: Prefix) -> List[str]: def update_names_with_range_to_flag( names: List[str], prefix: str ) -> List[str]: - """ - Given a list of names, check if they contain the @TO@ keyword. + """Given a list of names, check if they contain the @TO@ keyword. If so, expand to a range of names. Names including the @TO@ keyword must be in the form prefix-num1@num2. The maximum number of leading zeros are used to pad the output @@ -135,7 +132,9 @@ def update_names_with_range_to_flag( if tags("to") in name: check_name_with_to_tag_is_formatted_correctly(name, prefix) - prefix_tag = re.search(f"{prefix}-[0-9]+{tags('to')}[0-9]+", name)[0] # type: ignore + prefix_tag = re.search(f"{prefix}-[0-9]+{tags('to')}[0-9]+", name)[ + 0 + ] # type: ignore tag_number = prefix_tag.split(f"{prefix}-")[1] name_start_str, name_end_str = name.split(tag_number) @@ -172,8 +171,7 @@ def update_names_with_range_to_flag( def check_name_with_to_tag_is_formatted_correctly( name: str, prefix: str ) -> None: - """ - Check the input string is formatted with the @TO@ key + """Check the input string is formatted with the @TO@ key as expected. """ first_key_value_pair = name.split("_")[0] @@ -191,8 +189,7 @@ def check_name_with_to_tag_is_formatted_correctly( def make_list_of_zero_padded_names_across_range( left_number: str, right_number: str, name_start_str: str, name_end_str: str ) -> List[str]: - """ - Numbers formatted with the @TO@ keyword need to have + """Numbers formatted with the @TO@ keyword need to have standardised leading zeros on the output. Here we take the maximum number of leading zeros and apply for all numbers in the range. Note int() will strip @@ -200,7 +197,6 @@ def make_list_of_zero_padded_names_across_range( Parameters ---------- - left_number left (start) number from the range, e.g. "001" @@ -213,6 +209,7 @@ def make_list_of_zero_padded_names_across_range( name_end_str rest of the name after the flag, i.e. all other key-value pairs. + """ max_leading_zeros = max( utils.num_leading_zeros(left_number), @@ -237,8 +234,7 @@ def make_list_of_zero_padded_names_across_range( def update_names_with_datetime(names: List[str]) -> None: - """ - Replace @DATE@ and @DATETIME@ flag with date and datetime respectively. + """Replace @DATE@ and @DATETIME@ flag with date and datetime respectively. Format using key-value pair for bids, i.e. date-20221223_time- """ @@ -261,8 +257,7 @@ def replace_date_time_tags_in_name( date_with_key: str, time_with_key: str, ): - """ - For all names in the list, do the replacement of tags + """For all names in the list, do the replacement of tags with their final values. """ for i, name in enumerate(names): @@ -295,8 +290,7 @@ def format_datetime(date: str, time_: str) -> str: def add_underscore_before_after_if_not_there(string: str, key: str) -> str: - """ - If names are passed with @DATE@, @TIME@, or @DATETIME@ + """If names are passed with @DATE@, @TIME@, or @DATETIME@ but not surrounded by underscores, check and insert if required. e.g. sub-001@DATE@ becomes sub-001_@DATE@ or sub-001@DATEid-101 becomes sub-001_@DATE_id-101 @@ -307,9 +301,9 @@ def add_underscore_before_after_if_not_there(string: str, key: str) -> str: # Handle left edge if string[key_start_idx - 1] != "_": string_split = string.split(key) # assumes key only in string once - assert ( - len(string_split) == 2 - ), f"{key} must not appear in string more than once." + assert len(string_split) == 2, ( + f"{key} must not appear in string more than once." + ) string = f"{string_split[0]}_{key}{string_split[1]}" @@ -325,8 +319,7 @@ def add_underscore_before_after_if_not_there(string: str, key: str) -> str: def add_missing_prefixes_to_names( all_names: Union[List[str], str], prefix: str ) -> List[str]: - """ - Make sure all elements in the list of names are + """Make sure all elements in the list of names are prefixed with the prefix, typically "sub-" or "ses-" Use expanded list for readability diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index 8e1e8cd95..ac648f54f 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -37,8 +37,7 @@ def get_next_sub_or_ses( default_num_value_digits: int = 3, name_template_regexp: Optional[str] = None, ) -> str: - """ - Suggest the next available subject or session number. This function will + """Suggest the next available subject or session number. This function will search the local repository, and the central repository, for all subject or session folders (subject or session depending on inputs). @@ -50,7 +49,6 @@ def get_next_sub_or_ses( Parameters ---------- - cfg datashuttle configs class @@ -83,6 +81,7 @@ def get_next_sub_or_ses( ------- suggested_new_num the new suggested sub / ses. + """ prefix: Prefix @@ -123,8 +122,7 @@ def get_max_sub_or_ses_num_and_value_length( default_num_value_digits: Optional[int] = None, name_template_regexp: Optional[str] = None, ) -> Tuple[int, int]: - """ - Given a list of BIDS-style folder names, find the maximum subject or + """Given a list of BIDS-style folder names, find the maximum subject or session value (sub or ses depending on `prefix`). Also, find the number of value digits across the project, so a new suggested number can be formatted consistency. If the list is empty, set the value @@ -132,7 +130,6 @@ def get_max_sub_or_ses_num_and_value_length( Parameters ---------- - all_folders A list of BIDS-style formatted folder names. @@ -140,7 +137,6 @@ def get_max_sub_or_ses_num_and_value_length( Returns ------- - max_existing_num The largest number sub / ses value in the past list. @@ -153,10 +149,9 @@ def get_max_sub_or_ses_num_and_value_length( """ if len(all_folders) == 0: - - assert isinstance( - default_num_value_digits, int - ), "`default_num_value_digits` must be int`" + assert isinstance(default_num_value_digits, int), ( + "`default_num_value_digits` must be int`" + ) max_existing_num = 0 @@ -214,8 +209,7 @@ def get_max_sub_or_ses_num_and_value_length( def get_num_value_digits_from_project( all_values_str: List[str], prefix: Prefix ) -> int: - """ - Find the number of digits for the sub or ses key within the project. + """Find the number of digits for the sub or ses key within the project. `all_values_str` is a list of all the sub or ses values from within the project. """ @@ -235,8 +229,7 @@ def get_num_value_digits_from_project( def get_num_value_digits_from_regexp( prefix: Prefix, name_template_regexp: str ) -> Union[Literal[False], int]: - """ - Given a name template regexp, find the number of values for the + """Given a name template regexp, find the number of values for the sub or ses key. These will be fixed with "\d" (digit) or ".?" (wildcard). If there is length-unspecific wildcard (.*) in the sub key, then skip. In practice, there should never really be a .* in the sub or ses @@ -262,8 +255,7 @@ def get_num_value_digits_from_regexp( def get_existing_project_paths() -> List[Path]: - """ - Return full path and names of datashuttle projects on + """Return full path and names of datashuttle projects on this local machine. A project is determined by a project folder in the home / .datashuttle folder that contains a config.yaml file. Returns in order of most recently modified @@ -301,8 +293,7 @@ def get_all_sub_and_ses_paths( top_level_folder: TopLevelFolder, include_central: bool, ) -> Dict: - """ - Get a list of every subject and session name in the + """Get a list of every subject and session name in the local and central project folders. Local and central names are combined into a single list, separately for subject and sessions. @@ -311,7 +302,6 @@ def get_all_sub_and_ses_paths( Parameters ---------- - cfg datashuttle Configs @@ -321,6 +311,7 @@ def get_all_sub_and_ses_paths( include_central If `False, only get names from `local_path`, otherwise from `local_path` and `central_path`. + """ sub_folder_paths = folders.search_project_for_sub_or_ses_names( cfg, @@ -340,7 +331,6 @@ def get_all_sub_and_ses_paths( all_ses_folder_paths = {} for sub_path in all_sub_folder_paths: - sub = sub_path.name ses_folder_paths = folders.search_project_for_sub_or_ses_names( diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index 0462af2b7..a0bd2772c 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -9,18 +9,17 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess: - """ - Call rclone with the specified command. Current mode is double-verbose. + """Call rclone with the specified command. Current mode is double-verbose. Return the completed process from subprocess. Parameters ---------- - command Rclone command to be run pipe_std if True, do not output anything to stdout. + """ command = "rclone " + command if pipe_std: @@ -42,8 +41,7 @@ def setup_rclone_config_for_local_filesystem( rclone_config_name: str, log: bool = True, ): - """ - RClone sets remote targets in a config file that are + """RClone sets remote targets in a config file that are used at transfer. For local filesystem, this is essentially a placeholder and that is not linked to a particular filepath. It just tells rclone to use the local filesystem - then we @@ -57,13 +55,13 @@ def setup_rclone_config_for_local_filesystem( Parameters ---------- - rclone_config_name canonical config name, generated by datashuttle.cfg.get_rclone_config_name() log whether to log, if True logger must already be initialised. + """ call_rclone(f"config create {rclone_config_name} local", pipe_std=True) @@ -77,14 +75,12 @@ def setup_rclone_config_for_ssh( ssh_key_path: Path, log: bool = True, ): - """ - RClone sets remote targets in a config file that are + """RClone sets remote targets in a config file that are used at transfer. For SSH, this must contain the central path, username and ssh key. The relative path is supplied at transfer time. Parameters ---------- - cfg datashuttle configs UserDict. @@ -98,6 +94,7 @@ def setup_rclone_config_for_ssh( log whether to log, if True logger must already be initialised. + """ call_rclone( f"config create " @@ -117,15 +114,12 @@ def setup_rclone_config_for_ssh( def log_rclone_config_output(): output = call_rclone("config file", pipe_std=True) utils.log( - f"Successfully created rclone config. " - f"{output.stdout.decode('utf-8')}" + f"Successfully created rclone config. {output.stdout.decode('utf-8')}" ) def check_rclone_with_default_call() -> bool: - """ - Check to see whether rclone is installed. - """ + """Check to see whether rclone is installed.""" try: output = call_rclone("-h", pipe_std=True) except FileNotFoundError: @@ -134,8 +128,7 @@ def check_rclone_with_default_call() -> bool: def prompt_rclone_download_if_does_not_exist() -> None: - """ - Check that rclone is installed. If it does not + """Check that rclone is installed. If it does not (e.g. first time using datashuttle) then download. """ if not check_rclone_with_default_call(): @@ -158,12 +151,10 @@ def transfer_data( include_list: List[str], rclone_options: Dict, ) -> subprocess.CompletedProcess: - """ - Transfer data by making a call to Rclone. + """Transfer data by making a call to Rclone. Parameters ---------- - cfg datashuttle configs @@ -180,6 +171,7 @@ def transfer_data( rclone_options A list of options to pass to Rclone's copy function. see `cfg.make_rclone_transfer_options()`. + """ assert upload_or_download in [ "upload", @@ -217,8 +209,7 @@ def get_local_and_central_file_differences( cfg: Configs, top_level_folders_to_check: List[TopLevelFolder], ) -> Dict: - """ - Convert the output of rclone's check (with `--combine`) flag + """Convert the output of rclone's check (with `--combine`) flag to a dictionary separating each case. Rclone output comes as a list of files, separated by newlines, @@ -227,18 +218,17 @@ def get_local_and_central_file_differences( Parameters ---------- - top_level_folders_to_check List of top-level folders to check. Returns ------- - parsed_output A dictionary where the keys are the cases (e.g. "same" across local and central) and the values are lists of paths that fall into these cases. Note the paths are relative to the "rawdata" folder. + """ convert_symbols = { "=": "same", @@ -252,7 +242,6 @@ def get_local_and_central_file_differences( parsed_output = {val: [] for val in convert_symbols.values()} for top_level_folder in top_level_folders_to_check: - rclone_output = perform_rclone_check(cfg, top_level_folder) # type: ignore split_rclone_output = rclone_output.split("\n") @@ -273,8 +262,7 @@ def get_local_and_central_file_differences( def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): - """ - Ensure the output of Rclone check is as expected. Currently, the "error" + """Ensure the output of Rclone check is as expected. Currently, the "error" case is untested and a test case is required. Once the test case is obtained this should most likely be moved to tests. """ @@ -293,8 +281,7 @@ def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): def perform_rclone_check( cfg: Configs, top_level_folder: TopLevelFolder ) -> str: - """ - Use Rclone's `check` command to build a list of files that + """Use Rclone's `check` command to build a list of files that are the same ("="), different ("*"), found in local only ("+") or central only ("-"). The output is formatted as " \n". """ @@ -306,7 +293,7 @@ def perform_rclone_check( ).parent.as_posix() output = call_rclone( - f'{rclone_args("check")} ' + f"{rclone_args('check')} " f'"{local_filepath}" ' f'"{cfg.get_rclone_config_name()}:{central_filepath}"' f" --combined -", @@ -319,9 +306,7 @@ def perform_rclone_check( def handle_rclone_arguments( rclone_options: Dict, include_list: List[str] ) -> str: - """ - Construct the extra arguments to pass to RClone, - """ + """Construct the extra arguments to pass to RClone,""" extra_arguments_list = [] extra_arguments_list += ["-" + rclone_options["transfer_verbosity"]] @@ -351,9 +336,7 @@ def handle_rclone_arguments( def rclone_args(name: str) -> str: - """ - Central function to hold rclone commands - """ + """Central function to hold rclone commands""" valid_names = [ "dry_run", "copy", diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index eb8025bd2..b1cd9d2b0 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -48,9 +48,7 @@ def connect_client_core( def add_public_key_to_central_authorized_keys( cfg: Configs, password: str, log=True ) -> None: - """ - Append the public part of key to central server ~/.ssh/authorized_keys. - """ + """Append the public part of key to central server ~/.ssh/authorized_keys.""" generate_and_write_ssh_key(cfg.ssh_key_path) key = paramiko.RSAKey.from_private_key_file(cfg.ssh_key_path.as_posix()) @@ -78,8 +76,7 @@ def generate_and_write_ssh_key(ssh_key_path: Path) -> None: def get_remote_server_key(central_host_id: str): - """ - Get the remove server host key for validation before + """Get the remove server host key for validation before connection. """ transport: paramiko.Transport @@ -106,8 +103,7 @@ def setup_ssh_key( cfg: Configs, log: bool = True, ) -> None: - """ - Set up an SSH private / public key pair with + """Set up an SSH private / public key pair with central server. First, a private key is generated and saved in the .datashuttle config path. Next a connection requiring input @@ -115,8 +111,7 @@ def setup_ssh_key( added to ~/.ssh/authorized_keys. Parameters - ----------- - + ---------- ssh_key_path path to the ssh private key @@ -130,6 +125,7 @@ def setup_ssh_key( log log if True, logger must already be initialised. + """ if not sys.stdin.isatty(): proceed = input( @@ -175,8 +171,7 @@ def connect_client_with_logging( password: Optional[str] = None, message_on_sucessful_connection: bool = True, ) -> None: - """ - Connect client to central server using paramiko. + """Connect client to central server using paramiko. Accept either password or path to private key, but not both. Paramiko does not support pathlib. """ @@ -184,7 +179,7 @@ def connect_client_with_logging( connect_client_core(client, cfg, password) if message_on_sucessful_connection: utils.print_message_to_user( - f"Connection to { cfg['central_host_id']} made successfully." + f"Connection to {cfg['central_host_id']} made successfully." ) except Exception: @@ -203,8 +198,7 @@ def connect_client_with_logging( def verify_ssh_central_host( central_host_id: str, hostkeys_path: Path, log: bool = True ) -> bool: - """ - Similar to connecting with other SSH manager e.g. putty, + """Similar to connecting with other SSH manager e.g. putty, get the server key and present when connecting for manual validation. """ @@ -250,13 +244,11 @@ def search_ssh_central_for_folders( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """ - Search for the search prefix in the search path over SSH. + """Search for the search prefix in the search path over SSH. Returns the list of matching folders, files are filtered out. Parameters - ----------- - + ---------- search_path path to search for folders in @@ -269,6 +261,7 @@ def search_ssh_central_for_folders( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + """ client: paramiko.SSHClient with paramiko.SSHClient() as client: @@ -296,13 +289,11 @@ def get_list_of_folder_names_over_sftp( verbose: bool = True, return_full_path: bool = False, ) -> Tuple[List[Any], List[Any]]: - """ - Use paramiko's sftp to search a path + """Use paramiko's sftp to search a path over ssh for folders. Return the folder names. Parameters ---------- - stfp connected paramiko stfp object (see search_ssh_central_for_folders()) @@ -317,12 +308,12 @@ def get_list_of_folder_names_over_sftp( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + """ all_folder_names = [] all_filenames = [] try: for file_or_folder in sftp.listdir_attr(search_path.as_posix()): - if file_or_folder.st_mode is not None and fnmatch.fnmatch( file_or_folder.filename, search_prefix ): diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index 87a39e5c9..18b98f913 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -19,8 +19,7 @@ def log(message: str) -> None: - """ - Log the message to the main initialised + """Log the message to the main initialised logger. """ if ds_logger.logging_is_active(): @@ -29,8 +28,7 @@ def log(message: str) -> None: def log_and_message(message: str, use_rich: bool = False) -> None: - """ - Log the message and send it to user. + """Log the message and send it to user. use_rich : is True, use rich's print() function """ log(message) @@ -38,9 +36,7 @@ def log_and_message(message: str, use_rich: bool = False) -> None: def log_and_raise_error(message: str, exception: Any) -> None: - """ - Log the message before raising the same message as an error. - """ + """Log the message before raising the same message as an error.""" if ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.error(f"\n\n{' '.join(traceback.format_stack(limit=5))}") @@ -57,8 +53,7 @@ def warn(message: str, log: bool) -> None: def raise_error(message: str, exception) -> None: - """ - Centralized way to raise an error. The logger is closed + """Centralized way to raise an error. The logger is closed to ensure it is not still running if a function call raises an exception in a python environment. """ @@ -69,8 +64,7 @@ def raise_error(message: str, exception) -> None: def print_message_to_user( message: Union[str, list], use_rich: bool = False ) -> None: - """ - Centralised way to send message. + """Centralised way to send message. use_rich : use rich's print() function. """ if use_rich: @@ -80,9 +74,7 @@ def print_message_to_user( def get_user_input(message: str) -> str: - """ - Centralised way to get user input - """ + """Centralised way to get user input""" input_ = input(message) return input_ @@ -125,8 +117,7 @@ def get_values_from_bids_formatted_name( return_as_int: bool = False, sort: bool = False, ) -> Union[List[int], List[str]]: - """ - Find the values associated with a key from a list of all + """Find the values associated with a key from a list of all BIDS-formatted file / folder names. This is typically used to find sub / ses values. @@ -135,10 +126,10 @@ def get_values_from_bids_formatted_name( This function does not raise through datashuttle because we don't want to turn off logging, as some times these exceptions are caught and skipped. + """ all_values = [] for name in all_names: - if key not in name: raise NeuroBlueprintError( f"The key {key} is not found in {name}", KeyError @@ -177,8 +168,7 @@ def sub_or_ses_value_to_int(value: str) -> int: def get_value_from_key_regexp(name: str, key: str) -> List[str]: - """ - Find the value related to the key in a + """Find the value related to the key in a BIDS-style key-value pair name. e.g. sub-001_ses-312 would find 312 for key "ses". @@ -197,8 +187,7 @@ def integers_are_consecutive(list_of_ints: List[int]) -> bool: def diff(x: List) -> List: - """ - slow, custom differentiator for small inputs, to avoid + """slow, custom differentiator for small inputs, to avoid adding numpy as a dependency. """ return [x[i + 1] - x[i] for i in range(len(x) - 1)] @@ -213,14 +202,10 @@ def num_leading_zeros(string: str) -> int: def all_unique(list_: List) -> bool: - """ - Check that all values in a list are different. - """ + """Check that all values in a list are different.""" return len(list_) == len(set(list_)) def all_identical(list_: List) -> bool: - """ - Check that all values in a list are identical. - """ + """Check that all values in a list are identical.""" return len(set(list_)) == 1 diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index 05f58f6f3..99c798fb6 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -86,8 +86,7 @@ def get_datetime_error(key, name: str, strfmt: str, path_: Path | None) -> str: def get_template_error(name: str, regexp: str, path_: Path | None) -> str: - """ - The missing full-stop at the end is intentional, to avoid + """The missing full-stop at the end is intentional, to avoid confusion when reading the regexp. """ return handle_path( @@ -137,13 +136,11 @@ def validate_list_of_names( name_templates: Optional[Dict] = None, check_value_lengths: bool = True, ) -> List[str]: - """ - Validate a list of subject or session names, ensuring + """Validate a list of subject or session names, ensuring they are formatted as per NeuroBlueprint. Parameters ---------- - path_or_name_list A list of pathlib.Path to NeuroBlueprint-formatted folders to validate @@ -156,6 +153,7 @@ def validate_list_of_names( check_value_lengths If `True`, check that the prefix- value lengths are consistent across the passed list. + """ if len(path_or_name_list) == 0: return [] @@ -164,7 +162,6 @@ def validate_list_of_names( # First, just validate each name individually for path_or_name in path_or_name_list: - path_, name = get_path_and_name(path_or_name) error_messages += prefix_is_duplicate_or_has_bad_values( @@ -190,7 +187,6 @@ def validate_list_of_names( ) for path_or_name in stripped_path_or_names_list: - path_, name = get_path_and_name(path_or_name) error_messages += new_name_duplicates_existing( @@ -208,8 +204,7 @@ def validate_list_of_names( def prefix_is_duplicate_or_has_bad_values( name: str, prefix: Prefix, path_: Path | None ) -> List[str]: - """ - Check that the prefix (sub- or ses-) is found only + """Check that the prefix (sub- or ses-) is found only once in the name and that its value can be converted to integer. """ @@ -233,8 +228,7 @@ def new_name_duplicates_existing( existing_path_or_name_list: List[Path] | List[str], prefix: Prefix, ) -> List[str]: - """ - Check that a subject or session value does not duplicate + """Check that a subject or session value does not duplicate an existing value. The only case this is allowed is when the names match exactly. @@ -251,7 +245,6 @@ def new_name_duplicates_existing( error_messages = [] for exist_path_or_name in existing_path_or_name_list: - exist_path, exist_name = get_path_and_name(exist_path_or_name) exist_name_id = utils.get_values_from_bids_formatted_name( @@ -274,8 +267,7 @@ def names_dont_match_templates( prefix: Prefix, name_templates: Optional[Dict] = None, ) -> List[str]: - """ - Test a list of subject or session names against + """Test a list of subject or session names against the respective `name_templates`, a regexp template. """ if name_templates is None: @@ -298,8 +290,7 @@ def names_dont_match_templates( def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: - """ - Convenience function to get the folder name + """Convenience function to get the folder name from something that is either a Path (pointing to the folder) or a str of the folder name itself. """ @@ -310,8 +301,7 @@ def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: def replace_tags_in_regexp(regexp: str) -> str: - """ - Before validation, all tags in the names are converted to + """Before validation, all tags in the names are converted to their final values (e.g. @DATE@ -> _date-). We also want to allow template to be formatted like `sub-\d\d_@DATE@` as it is convenient for auto-completion in the TUI. @@ -336,8 +326,7 @@ def replace_tags_in_regexp(regexp: str) -> str: def name_begins_with_bad_key( name: str, prefix: Prefix, path_: Path | None ) -> List[str]: - """ - Check that a list of NeuroBlueprint names begin + """Check that a list of NeuroBlueprint names begin with the required prefix (sub- or ses-). """ if name[:4] != f"{prefix}-": @@ -349,8 +338,7 @@ def name_begins_with_bad_key( def names_include_special_characters( name: str, path_: Path | None ) -> List[str]: - """ - Check that a list of NeuroBlueprint formatted + """Check that a list of NeuroBlueprint formatted names do not contain special characters (i.e. characters that are not integers, letters, dash or underscore). """ @@ -367,8 +355,7 @@ def name_has_special_character(name: str) -> bool: def dashes_and_underscore_alternate_incorrectly( name: str, path_: Path | None ) -> List[str]: - """ - Check a list of NeuroBlueprint formatted names + """Check a list of NeuroBlueprint formatted names have the "-" and "-" ordered correctly. Names should be key-value pairs separated by underscores e.g. sub-001_ses-001. @@ -386,7 +373,7 @@ def dashes_and_underscore_alternate_incorrectly( or dashes_underscores[0] != 1 # first must be - or dashes_underscores[-1] != 1 # last must be - or underscore_dash_not_interleaved - or (name[-1] in discrim.keys()) # name cannot end with - or _ + or (name[-1] in discrim) # name cannot end with - or _ ): return [get_name_format_error(name, path_)] else: @@ -397,8 +384,7 @@ def value_lengths_are_inconsistent( path_or_names_list: List[Path] | List[str] | List[Path | str], prefix: Prefix, ) -> List[str]: - """ - Given a list of NeuroBlueprint-formatted subject or session + """Given a list of NeuroBlueprint-formatted subject or session names, determine if there are inconsistent value lengths for the sub or ses key. e.g. ["sub-01", "sub-001"] is an error. """ @@ -429,9 +415,7 @@ def datetime_are_iso_format( name: str, path_: Path | None, ) -> List[str]: - """ - Check formatting for date-, time-, or datetime- tags. - """ + """Check formatting for date-, time-, or datetime- tags.""" formats = { "datetime": "%Y%m%dT%H%M%S", "time": "%H%M%S", @@ -466,8 +450,7 @@ def datetime_are_iso_format( def raise_display_mode( message: str, display_mode: DisplayMode, log: bool ) -> None: - """ - Show a message by raising an error, displaying warning, or printing. + """Show a message by raising an error, displaying warning, or printing. Optionally log with the current datashuttle logger. """ if display_mode == "error": @@ -501,12 +484,10 @@ def validate_project( name_templates: Optional[Dict] = None, strict_mode: bool = False, ) -> List[str]: - """ - Validate all subject and session folders within a project. + """Validate all subject and session folders within a project. Parameters - ----------- - + ---------- cfg datashuttle Configs class. @@ -534,6 +515,7 @@ def validate_project( starting with sub- or ses- prefix are checked. In `Strict Mode`, any folder not prefixed with sub-, ses- or a valid datatype will raise a validation issue. + """ error_messages = [] @@ -541,7 +523,6 @@ def validate_project( error_messages += check_high_level_project_structure(cfg, include_central) for top_level_folder in top_level_folder_list: - if strict_mode: error_messages += check_strict_mode( cfg, top_level_folder, include_central @@ -568,7 +549,6 @@ def validate_project( # Check all names as well as duplicates per-subject for ses_paths in folder_paths["ses"].values(): - error_messages += validate_list_of_names( ses_paths, "ses", @@ -604,8 +584,7 @@ def validate_names_against_project( log: bool = True, name_templates: Optional[Dict] = None, ) -> None: - """ - Given a list of subject and (optionally) session names, + """Given a list of subject and (optionally) session names, check that these names are formatted consistently with the rest of the project. Used for creating folders. @@ -615,7 +594,6 @@ def validate_names_against_project( Parameters ---------- - cfg datashuttle Configs class. @@ -645,6 +623,7 @@ def validate_names_against_project( name_templates A `name_template` dictionary to validate against. See `set_name_templates()`. + """ error_messages = [] @@ -662,7 +641,6 @@ def validate_names_against_project( ) if folder_paths["sub"]: - # Strip any totally invalid names which we can't extract # the sub integer value for the following checks valid_sub_names = strip_uncheckable_names(sub_names, "sub") @@ -690,14 +668,12 @@ def validate_names_against_project( # Now we need to check the sessions. if ses_names is not None and any(ses_names): - # First, validate the list of passed session names error_messages += validate_list_of_names( ses_names, "ses", name_templates=name_templates ) if folder_paths["sub"]: - # Next, we need to check that the passed session names # do not duplicate existing session names and # that do not create inconsistent ses- lengths across the project. @@ -708,7 +684,6 @@ def validate_names_against_project( # are allowed across different subjects (but not within a single sub). for new_sub in sub_names: if new_sub in folder_paths["ses"]: - valid_ses_in_sub = strip_uncheckable_names( folder_paths["ses"][new_sub], "ses", @@ -746,8 +721,7 @@ def validate_names_against_project( def check_high_level_project_structure( cfg: Configs, include_central: bool ) -> List[str]: - """ - Perform basic validation checks on the project structure, + """Perform basic validation checks on the project structure, that the project folder name is valid, and that the project folder contains either a "rawdata" or "derivatives" folder. @@ -810,8 +784,7 @@ def check_high_level_project_structure( def check_strict_mode( cfg: Configs, top_level_folder: TopLevelFolder, include_central: bool ) -> List[str]: - """ - `strict_mode` does not allow any non-NeuroBlueprint folder to exist + """`strict_mode` does not allow any non-NeuroBlueprint folder to exist in the project outside the datatype folder. NeuroBlueprint folders are top-level folder or folder with sub-, ses- or datatype. @@ -841,7 +814,6 @@ def check_strict_mode( ) for sub_level_path in sub_level_folder_paths["local"]: - # Check all folders found in a top-level folder are # sub- prefixed folders. sub_level_name = sub_level_path.name @@ -861,7 +833,6 @@ def check_strict_mode( ) for ses_level_path in ses_level_folder_paths["local"]: - # For each sub- prefixed folder, check that all folders within # the subject folder are ses- prefixed folders. ses_level_name = ses_level_path.name @@ -884,7 +855,6 @@ def check_strict_mode( canonical_datatypes = canonical_configs.get_datatypes() for datatype_level_path in search_results: - # For each ses- prefixed folder, check that # only valid datatypes are included within it. datatype_level_name = datatype_level_path.name @@ -916,8 +886,7 @@ def strip_uncheckable_names( path_or_names_list: List[Path] | List[str], prefix: Prefix, ) -> List[Path] | List[str]: - """ - Convenience function to remove any name in which + """Convenience function to remove any name in which the `prefix` value (sub or ses typically) cannot be converted into an integer. This is necessary as some validation steps (e.g. checking duplicate names) requires @@ -927,7 +896,6 @@ def strip_uncheckable_names( new_list = [] for path_or_name in path_or_names_list: - path_, name = get_path_and_name(path_or_name) try: @@ -953,8 +921,7 @@ def strip_uncheckable_names( def check_datatypes_are_valid( datatype: Union[List[str], str], allow_all: bool = False ) -> str | None: - """ - Check a datatype of list of datatypes is a valid + """Check a datatype of list of datatypes is a valid NeuroBlueprint datatype. """ datatype_folders = canonical_folders.get_datatype_folders() diff --git a/tests/conftest.py b/tests/conftest.py index 2203b1025..50c27d849 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ -""" -Test configs, used for setting up SSH tests. +"""Test configs, used for setting up SSH tests. Before running these tests, it is necessary to setup an SSH key. This can be done through datashuttle diff --git a/tests/ssh_test_utils.py b/tests/ssh_test_utils.py index 0838669f3..bf5bf2c74 100644 --- a/tests/ssh_test_utils.py +++ b/tests/ssh_test_utils.py @@ -7,8 +7,7 @@ def setup_project_for_ssh( project, central_path, central_host_id, central_host_username ): - """ - Set up the project configs to use SSH connection + """Set up the project configs to use SSH connection to central """ project.update_config_file( @@ -26,8 +25,7 @@ def setup_project_for_ssh( def setup_mock_input(input_): - """ - This is very similar to pytest monkeypatch but + """This is very similar to pytest monkeypatch but using that was giving me very strange output, monkeypatch.setattr('builtins.input', lambda _: "n") i.e. pdb went deep into some unrelated code stack @@ -38,16 +36,12 @@ def setup_mock_input(input_): def restore_mock_input(orig_builtin): - """ - orig_builtin: the copied, original builtins.input - """ + """orig_builtin: the copied, original builtins.input""" builtins.input = orig_builtin def setup_hostkeys(project): - """ - Convenience function to verify the server hostkey. - """ + """Convenience function to verify the server hostkey.""" orig_builtin = setup_mock_input(input_="y") ssh.verify_ssh_central_host( project.cfg["central_host_id"], project.cfg.hostkeys_path, log=True diff --git a/tests/test_utils.py b/tests/test_utils.py index ad908160e..46d0b23ea 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -27,8 +27,7 @@ def setup_project_default_configs( local_path=False, central_path=False, ): - """ - Set up a fresh project to test on + """Set up a fresh project to test on local_path / central_path: provide the config paths to set """ @@ -76,8 +75,7 @@ def make_project_paths(config_dict): def glob_basenames(search_path, recursive=False, exclude=None): - """ - Use glob to search but strip the full path, including + """Use glob to search but strip the full path, including only the base name (lowest level). """ paths_ = glob.glob(search_path, recursive=recursive) @@ -131,8 +129,7 @@ def delete_project_if_it_exists(project_name): def setup_project_fixture(tmp_path, test_project_name, project_type="full"): - """ - Set up a project, either in full mode or local-only mode. This is + """Set up a project, either in full mode or local-only mode. This is very similar to the `BaseTest` fixture but is designed for use in other fixtures that require additional boilerplate e.g. logging. """ @@ -184,8 +181,7 @@ def get_test_config_arguments_dict( set_as_defaults=False, required_arguments_only=False, ): - """ - Retrieve configs, either the required configs + """Retrieve configs, either the required configs (for project.make_config_file()), all configs (default) or non-default configs. Note that default configs here are the expected default arguments in project.make_config_file(). @@ -227,8 +223,7 @@ def get_test_config_arguments_dict( def get_all_broad_folders_used(value=True): - """ - The `folders_used` construct tells the tests which + """The `folders_used` construct tells the tests which folders were used (e.g. created or transferred) and which are not. This means the expected datatypes can be checked. @@ -253,8 +248,7 @@ def get_all_broad_folders_used(value=True): def check_folder_tree_is_correct( base_folder, subs, sessions, folder_used, created_folder_dict=None ): - """ - Automated test that folders are made based + """Automated test that folders are made based on the structure specified on project itself. Cycle through all datatypes (defined in @@ -316,8 +310,7 @@ def check_folder_tree_is_correct( def check_and_cd_folder(path_): - """ - Check a folder exists and CD to it if it does. + """Check a folder exists and CD to it if it does. Use the pytest -s flag to print all tested paths """ @@ -331,8 +324,7 @@ def check_datatype_sub_ses_uploaded_correctly( subs_to_upload=None, ses_to_upload=None, ): - """ - Iterate through the project (datatype > ses > sub) and + """Iterate through the project (datatype > ses > sub) and check that the folders at each level match those that are expected (passed in datatype / sub / ses to upload). Folders are searched with wildcard glob. @@ -368,8 +360,7 @@ def check_datatype_sub_ses_uploaded_correctly( def make_and_check_local_project_folders( project, top_level_folder, subs, sessions, datatype, datatypes_used=None ): - """ - Make a local project folder tree with the specified datatype, + """Make a local project folder tree with the specified datatype, subs, sessions and check it is made successfully. Since empty folders are not transferred, it is necessary @@ -422,8 +413,7 @@ def check_project_configs( project, *kwargs, ): - """ - Core function for checking the config against + """Core function for checking the config against provided configs (kwargs). Open the config.yaml file and check the config values stored there, and in project.cfg, against the provided configs. @@ -444,7 +434,7 @@ def check_project_configs( def check_config_file(config_path, *kwargs): """""" - with open(config_path, "r") as config_file: + with open(config_path) as config_file: config_yaml = yaml.full_load(config_file) for name, value in kwargs[0].items(): @@ -461,9 +451,9 @@ def get_top_level_folder_path( ): """""" - assert ( - folder_name in canonical_folders.get_top_level_folders() - ), "folder_name must be canonical e.g. rawdata" + assert folder_name in canonical_folders.get_top_level_folders(), ( + "folder_name must be canonical e.g. rawdata" + ) if local_or_central == "local": base_path = project.cfg["local_path"] @@ -480,8 +470,7 @@ def handle_upload_or_download( top_level_folder=None, swap_last_folder_only=False, ): - """ - To keep things consistent and avoid the pain of writing + """To keep things consistent and avoid the pain of writing files over SSH, to test download just swap the central and local server (so things are still transferred from local machine to central, but using the download function). @@ -537,8 +526,7 @@ def get_transfer_func( def swap_local_and_central_paths(project, swap_last_folder_only=False): - """ - When testing upload vs. download, the most convenient way + """When testing upload vs. download, the most convenient way to test download is to swap the paths. In this case, we 'download' from local to central. It much simplifies creating the folders to transfer (which are created locally), and is fully required @@ -584,25 +572,21 @@ def swap_local_and_central_paths(project, swap_last_folder_only=False): def get_default_sub_sessions_to_test(): - """ - Canonical subs / sessions for these tests - """ + """Canonical subs / sessions for these tests""" subs = ["sub-001", "sub-002", "sub-003"] sessions = ["ses-001_datetime-20220516T135022", "ses-002", "ses-003"] return subs, sessions def move_some_keys_to_end_of_dict(config): - """ - Need to move connection method to the end + """Need to move connection method to the end so ssh opts are already set before it is changed. """ config["connection_method"] = config.pop("connection_method") def clear_capsys(capsys): - """ - read from capsys clears it, so new + """Read from capsys clears it, so new print statements are clearer to read. """ capsys.readouterr() @@ -619,14 +603,13 @@ def write_file(path_, contents="", append=False): def read_file(path_): - with open(path_, "r") as file: + with open(path_) as file: contents = file.readlines() return contents def set_datashuttle_loggers(disable): - """ - Turn off or on datashuttle logs, if these are + """Turn off or on datashuttle logs, if these are on when testing with pytest they will be propagated to pytest's output, making it difficult to read. @@ -643,8 +626,7 @@ def set_datashuttle_loggers(disable): def check_working_top_level_folder_only_exists( folder_name, base_path_to_check, subs, sessions, folders_used=None ): - """ - Check that the folder tree made in the 'folder_name' + """Check that the folder tree made in the 'folder_name' (e.g. 'rawdata') top level folder is correct. Additionally, check that no other top-level folders exist. This is to ensure that folders made / transferred from one top-level folder @@ -672,11 +654,11 @@ def read_log_file(logging_path): log_filepath = list(glob.glob(str(logging_path / "*.log"))) assert len(log_filepath) == 1, ( - f"there should only be one log " f"in log output path {logging_path}" + f"there should only be one log in log output path {logging_path}" ) log_filepath = log_filepath[0] - with open(log_filepath, "r") as file: + with open(log_filepath) as file: log = file.read() return log @@ -684,7 +666,7 @@ def read_log_file(logging_path): def delete_log_files(logging_path): ds_logger.close_log_filehandler() - for log in glob.glob((str(logging_path / "*.log"))): + for log in glob.glob(str(logging_path / "*.log")): os.remove(log) diff --git a/tests/tests_integration/_test_configs.py b/tests/tests_integration/_test_configs.py index feff2900d..3d617a072 100644 --- a/tests/tests_integration/_test_configs.py +++ b/tests/tests_integration/_test_configs.py @@ -15,25 +15,20 @@ class TestConfigs(BaseTest): @pytest.fixture(scope="function") def non_existent_path(self, tmp_path): - """ - Return a path that does not exist. - """ + """Return a path that does not exist.""" non_existent_path = tmp_path / "does_not_exist" assert not non_existent_path.is_dir() return non_existent_path @pytest.fixture(scope="function") def existent_path(self, tmp_path): - """ - Return a path that exists. - """ + """Return a path that exists.""" existent_path = tmp_path / "exists" os.makedirs(existent_path, exist_ok=True) return existent_path def test_warning_on_startup(self, no_cfg_project): - """ - When no configs have been set, a warning should be shown that + """When no configs have been set, a warning should be shown that the config has not been initialized. Need to download Rclone first to ensure input() is not called. """ @@ -60,8 +55,7 @@ def test_warning_on_startup(self, no_cfg_project): ) @pytest.mark.parametrize("path_type", ["local_path", "central_path"]) def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): - """ - "~", "." and "../" syntax is not supported because + """ "~", "." and "../" syntax is not supported because it does not work with rclone. Theoretically it could be supported by checking for "." etc. and filling in manually, but it does not seem robust. @@ -95,8 +89,7 @@ def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): assert "must contain the full folder path with no " in str(e.value) def test_no_ssh_options_set_on_make_config_file(self, no_cfg_project): - """ - Check that program will assert if not all ssh options + """Check that program will assert if not all ssh options are set on make_config_file """ with pytest.raises(ConfigError) as e: @@ -116,8 +109,7 @@ def test_no_ssh_options_set_on_make_config_file(self, no_cfg_project): # ------------------------------------------------------------- def test_required_configs(self, no_cfg_project, tmp_path): - """ - Set the required arguments of the config (local_path, central_path, + """Set the required arguments of the config (local_path, central_path, connection_method and check they are set correctly in both the no_cfg_project.cfg dict and config.yaml file. """ @@ -133,8 +125,7 @@ def test_required_configs(self, no_cfg_project, tmp_path): ) def test_config_defaults(self, no_cfg_project, tmp_path): - """ - Check the default configs are set as expected + """Check the default configs are set as expected (see get_test_config_arguments_dict()) for tested defaults. """ required_options = test_utils.get_test_config_arguments_dict( @@ -150,8 +141,7 @@ def test_config_defaults(self, no_cfg_project, tmp_path): test_utils.check_configs(no_cfg_project, default_options) def test_non_default_configs(self, no_cfg_project, tmp_path): - """ - Set the configs to non-default options, make the + """Set the configs to non-default options, make the config file and check file and no_cfg_project.cfg are set correctly. """ changed_configs = test_utils.get_test_config_arguments_dict( @@ -167,8 +157,7 @@ def test_non_default_configs(self, no_cfg_project, tmp_path): # ------------------------------------------------------------- def test_update_config_file__(self, no_cfg_project, tmp_path): - """ - Set the configs as default, and then update them to + """Set the configs as default, and then update them to new configs and check they are updated properly. Then, update only a subset (back to the defaults) and @@ -213,8 +202,7 @@ def test_update_config_file__(self, no_cfg_project, tmp_path): test_utils.check_configs(project, default_configs) def test_existing_projects(self, monkeypatch, tmp_path): - """ - Test existing projects are correctly found based on whether + """Test existing projects are correctly found based on whether they exist in the home directory and contain a config.yaml. By default, datashuttle saves project folders to @@ -266,8 +254,7 @@ def patch_get_datashuttle_path(): # -------------------------------------------------------------------------------------------------------------------- def check_config_reopen_and_check_config_again(self, project, *kwargs): - """ - Check the config file and project.cfg against provided kwargs, + """Check the config file and project.cfg against provided kwargs, delete the project and set up the project again, checking everything is loaded correctly. """ diff --git a/tests/tests_integration/base.py b/tests/tests_integration/base.py index b04816cba..c3ea76d20 100644 --- a/tests/tests_integration/base.py +++ b/tests/tests_integration/base.py @@ -10,11 +10,9 @@ class BaseTest: - @pytest.fixture(scope="function") def no_cfg_project(test): - """ - Fixture that creates an empty project. Ignore the warning + """Fixture that creates an empty project. Ignore the warning that no configs are setup yet. """ test_utils.delete_project_if_it_exists(TEST_PROJECT_NAME) @@ -27,8 +25,7 @@ def no_cfg_project(test): @pytest.fixture(scope="function") def project(self, tmp_path, request): - """ - Set up a project with default configs to use for testing. + """Set up a project with default configs to use for testing. This fixture uses indirect parameterization to test both 'full' and 'local-only' (no `central_path` or `connection_method`). The @@ -64,8 +61,7 @@ def project(self, tmp_path, request): @pytest.fixture(scope="function") def clean_project_name(self): - """ - Create an empty project, but ensure no + """Create an empty project, but ensure no configs already exists, and delete created configs after test. """ diff --git a/tests/tests_integration/test_create_folders.py b/tests/tests_integration/test_create_folders.py index e50623776..0a69b0695 100644 --- a/tests/tests_integration/test_create_folders.py +++ b/tests/tests_integration/test_create_folders.py @@ -13,11 +13,9 @@ class TestCreateFolders(BaseTest): - @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_generate_folders_default_ses(self, project): - """ - Make a subject folders with full tree. Don't specify + """Make a subject folders with full tree. Don't specify session name (it will default to no sessions). Check that the folder tree is created correctly. Pass @@ -37,8 +35,7 @@ def test_generate_folders_default_ses(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_explicitly_session_list(self, project): - """ - Perform an alternative test where the output is tested explicitly. + """Perform an alternative test where the output is tested explicitly. This is some redundancy to ensure tests are working correctly and make explicit the expected folder tree. @@ -83,8 +80,7 @@ def test_explicitly_session_list(self, project): def test_every_broad_datatype_passed( self, project, behav, ephys, funcimg, anat ): - """ - Check every combination of data type used and ensure only the + """Check every combination of data type used and ensure only the correct ones are made. NOTE: This test could be refactored to reduce code reuse. @@ -121,8 +117,7 @@ def test_every_broad_datatype_passed( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_custom_folder_names(self, project, monkeypatch): - """ - Change folder names to custom (non-default) and + """Change folder names to custom (non-default) and ensure they are made correctly. """ new_name_datafolders = canonical_folders.get_datatype_folders() @@ -178,8 +173,7 @@ def new_name_func(): ) @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_datatypes_subsection(self, project, files_to_test): - """ - Check that combinations of datatypes passed to make file folder + """Check that combinations of datatypes passed to make file folder make the correct combination of datatypes. Note this will fail when new top level folders are added, and should be @@ -207,8 +201,7 @@ def test_datatypes_subsection(self, project, files_to_test): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_date_flags_in_session(self, project): - """ - Check that @DATE@ is converted into current date + """Check that @DATE@ is converted into current date in generated folder names """ date, time_ = self.get_formatted_date_and_time() @@ -230,8 +223,7 @@ def test_date_flags_in_session(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_datetime_flag_in_session(self, project): - """ - Check that @DATETIME@ is converted to datetime + """Check that @DATETIME@ is converted to datetime in generated folder names """ date, time_ = self.get_formatted_date_and_time() @@ -257,8 +249,7 @@ def test_datetime_flag_in_session(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_created_paths_dict_sub_or_ses_only(self, project): - """ - Test that the `created_folders` dictionary returned by + """Test that the `created_folders` dictionary returned by `create_folders` correctly splits paths when only subject or session is passed. The `datatype` case is tested in `test_utils.check_folder_tree_is_correct()`. @@ -292,8 +283,7 @@ def test_created_paths_dict_sub_or_ses_only(self, project): ) @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_all_top_level_folders(self, project, top_level_folder): - """ - Check that when switching the top level folder (e.g. rawdata, derivatives) + """Check that when switching the top level folder (e.g. rawdata, derivatives) new folders are made in the correct folder. """ subs = ["sub-001", "sub-002"] @@ -319,8 +309,7 @@ def test_all_top_level_folders(self, project, top_level_folder): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("return_with_prefix", [True, False]) def test_get_next_sub(self, project, return_with_prefix, top_level_folder): - """ - Test that the next subject number is suggested correctly. + """Test that the next subject number is suggested correctly. This takes the union of subjects available in the local and central repository. As such test the case where either are empty, or when they have different subjects in. @@ -373,8 +362,7 @@ def test_get_next_sub(self, project, return_with_prefix, top_level_folder): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("return_with_prefix", [True, False]) def test_get_next_ses(self, project, return_with_prefix, top_level_folder): - """ - Almost identical to test_get_next_sub() but with calls + """Almost identical to test_get_next_sub() but with calls for searching sessions. This could be combined with above but reduces readability, so leave with some duplication. @@ -443,8 +431,7 @@ def test_get_next_ses(self, project, return_with_prefix, top_level_folder): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_get_next_sub_and_ses_name_template(self, project): - """ - In the case where a name template exists, these getters should use the + """In the case where a name template exists, these getters should use the number of digits on the template (even if these are different within the project!). """ diff --git a/tests/tests_integration/test_datatypes.py b/tests/tests_integration/test_datatypes.py index 1aec45480..10720e90b 100644 --- a/tests/tests_integration/test_datatypes.py +++ b/tests/tests_integration/test_datatypes.py @@ -8,15 +8,13 @@ class TestDatatypes(BaseTest): - """ - Tests for creating folders and transfer (very similar to other tests) + """Tests for creating folders and transfer (very similar to other tests) however which test the creation and transfer of narrow datatypes. Other tests used broad datatypes. """ def test_create_narrow_datatypes(self, project): - """ - Create all narrow datatype folders and check + """Create all narrow datatype folders and check they are created as expected. """ # Make folder tree including all narrow datatypes @@ -41,8 +39,7 @@ def test_create_narrow_datatypes(self, project): ) def get_narrow_only_datatypes_used(self, used=True): - """ - This is similar to test_utils.get_all_broad_folders_used + """This is similar to test_utils.get_all_broad_folders_used but for narrow datatypes. """ return { @@ -55,8 +52,7 @@ def test_transfer_datatypes( project, upload_or_download, ): - """ - Create a project with narrow datatypes and check these + """Create a project with narrow datatypes and check these folders are transferred as expected. """ subs, sessions = test_utils.get_default_sub_sessions_to_test() diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py index 8cbff488f..98e31a629 100644 --- a/tests/tests_integration/test_filesystem_transfer.py +++ b/tests/tests_integration/test_filesystem_transfer.py @@ -13,7 +13,6 @@ class TestFileTransfer(BaseTest): - @pytest.mark.parametrize( "top_level_folder", canonical_folders.get_top_level_folders() ) @@ -28,8 +27,7 @@ def test_transfer_empty_folder_structure( upload_or_download, transfer_method, ): - """ - First make a project (folders only) locally. + """First make a project (folders only) locally. Next upload this to the central path and check all folders are uploaded correctly. """ @@ -78,8 +76,7 @@ def test_transfer_across_top_level_folders( upload_or_download, transfer_method, ): - """ - For each possible top level folder (e.g. rawdata, derivatives) + """For each possible top level folder (e.g. rawdata, derivatives) (parametrized) create a folder tree in every top-level folder, then transfer using upload / download and upload_rawdata() / download_rawdata() that only the working top-level folder @@ -151,7 +148,6 @@ def test_transfer_all_top_level_folders(self, project, upload_or_download): transfer_function() for top_level_folder in canonical_folders.get_top_level_folders(): - test_utils.check_folder_tree_is_correct( os.path.join(base_path_to_check, top_level_folder), subs, @@ -177,8 +173,7 @@ def test_transfer_all_top_level_folders(self, project, upload_or_download): def test_transfer_empty_folder_specific_data( self, project, upload_or_download, datatype_to_transfer ): - """ - For the combination of datatype folders, make a folder + """For the combination of datatype folders, make a folder tree with all datatype folders then upload select ones, checking only the selected ones are uploaded. """ @@ -215,7 +210,7 @@ def test_transfer_empty_folder_specific_data( ["behav", "ephys", "funcimg", "anat"], ], ) - @pytest.mark.parametrize("upload_or_download", ["upload" "download"]) + @pytest.mark.parametrize("upload_or_download", ["uploaddownload"]) def test_transfer_empty_folder_specific_subs( self, project, @@ -223,8 +218,7 @@ def test_transfer_empty_folder_specific_subs( datatype_to_transfer, sub_idx_to_upload, ): - """ - Create a project folder tree with a set of subs, then + """Create a project folder tree with a set of subs, then take a subset of these subs and upload them. Check only the selected subs were uploaded. """ @@ -268,8 +262,7 @@ def test_transfer_empty_folder_specific_ses( sub_idx_to_upload, ses_idx_to_upload, ): - """ - Make a project with set subs and sessions. Then select a subset of the + """Make a project with set subs and sessions. Then select a subset of the sessions to upload. Check only the selected sessions were uploaded. """ subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -303,8 +296,7 @@ def test_transfer_empty_folder_specific_ses( def test_transfer_with_keyword_parameters( self, project, upload_or_download ): - """ - Test the @TO@ keyword is accepted properly when making a session and + """Test the @TO@ keyword is accepted properly when making a session and transferring it. First pass @TO@-formatted sub and sessions to create_folders. Then transfer the files (upload or download). @@ -350,8 +342,7 @@ def test_transfer_with_keyword_parameters( @pytest.mark.parametrize("upload_or_download", ["upload", "download"]) def test_wildcard_transfer(self, project, upload_or_download): - """ - Transfer a subset of define subject and session + """Transfer a subset of define subject and session and check only the expected folders are there. """ subs = ["sub-389", "sub-989", "sub-445"] @@ -395,8 +386,7 @@ def test_wildcard_transfer(self, project, upload_or_download): ] def test_deep_folder_structure(self, project): - """ - Just a quick test as all other tests only test files directly in the + """Just a quick test as all other tests only test files directly in the datatyp directly. Check that rlcone is setup to transfer multiple levels down from the datatype level. """ @@ -427,8 +417,7 @@ def test_rclone_options( dry_run, capsys, ): - """ - When verbosity is --vv, rclone itself will output + """When verbosity is --vv, rclone itself will output a list of all called arguments. Use this to check rclone is called with the arguments set in configs as expected. verbosity itself is tested in another method. @@ -475,8 +464,7 @@ def test_overwrite_same_size_earlier_to_later( top_level_folder, upload_or_download, ): - """ - Main test to check every parameterization for overwrite settings. + """Main test to check every parameterization for overwrite settings. It is such an important setting it is tested for all top level folder, transfer method, even though it makes for quite a confusing function. @@ -541,8 +529,7 @@ def test_overwrite_same_size_later_to_earlier( top_level_folder, upload_or_download, ): - """ - This functions is extremely similar to + """This functions is extremely similar to `test_overwrite_same_size_later_to_earlier()` but it is much easier to understand individually when they are split. @@ -588,8 +575,7 @@ def test_overwrite_same_size_later_to_earlier( def test_overwrite_different_size_different_times( self, project, overwrite_existing_files ): - """ - Quick additional test to confirm that "if_source_newer" will still + """Quick additional test to confirm that "if_source_newer" will still not transfer even if the older file is larger. This is the expected behaviour from rclone, this is confidence check on understanding. """ @@ -670,8 +656,7 @@ def setup_overwrite_file_tests( def test_dry_run( self, project, top_level_folder, transfer_method, upload_or_download ): - """ - Just do a quick functional test on dry-run that indeed nothing + """Just do a quick functional test on dry-run that indeed nothing is transferred across all top-level-folder / upload-download methods. """ @@ -704,8 +689,7 @@ def test_specific_file_or_folder( transfer_file, upload_or_download, ): - """ - Test upload_specific_folder_or_file() and download_specific_folder_or_file(). + """Test upload_specific_folder_or_file() and download_specific_folder_or_file(). Make a project with two different files (just to ensure non-target files are not transferred). Transfer diff --git a/tests/tests_integration/test_formatting.py b/tests/tests_integration/test_formatting.py index bf62c6a50..0b5db8338 100644 --- a/tests/tests_integration/test_formatting.py +++ b/tests/tests_integration/test_formatting.py @@ -11,8 +11,7 @@ class TestFormatting(BaseTest): "input", [1, {"test": "one"}, 1.0, ["1", "2", ["three"]]] ) def test_format_names_bad_input(self, input, prefix): - """ - Test that names passed in incorrect type + """Test that names passed in incorrect type (not str, list) raise appropriate error. """ with pytest.raises(TypeError) as e: @@ -22,8 +21,7 @@ def test_format_names_bad_input(self, input, prefix): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_format_names_duplicate_ele(self, prefix): - """ - Test that appropriate error is raised when duplicate name + """Test that appropriate error is raised when duplicate name is passed to format_names(). """ with pytest.raises(NeuroBlueprintError) as e: @@ -37,8 +35,7 @@ def test_format_names_duplicate_ele(self, prefix): ) def test_format_names_prefix(self): - """ - Check that format_names correctly prefixes input + """Check that format_names correctly prefixes input with default sub or ses prefix. This is less useful now that ses/sub name dash and underscore order is more strictly checked. diff --git a/tests/tests_integration/test_local_only_mode.py b/tests/tests_integration/test_local_only_mode.py index 6df59233b..0db07a503 100644 --- a/tests/tests_integration/test_local_only_mode.py +++ b/tests/tests_integration/test_local_only_mode.py @@ -13,10 +13,8 @@ class TestLocalOnlyProject(BaseTest): - def test_bad_setup(self, tmp_path): - """ - Test setup without providing both central_path and connection + """Test setup without providing both central_path and connection method (distinguishing a full vs local-only project) """ local_path = tmp_path / "test_local" @@ -41,8 +39,7 @@ def test_bad_setup(self, tmp_path): @pytest.mark.parametrize("project", ["local"], indirect=True) def test_full_to_local_project(self, project): - """ - Make a full project a local-only project, and check the transfer + """Make a full project a local-only project, and check the transfer functionality is now restricted. """ project.update_config_file(central_path=None, connection_method=None) @@ -60,8 +57,7 @@ def test_full_to_local_project(self, project): @pytest.mark.parametrize("project", ["local"], indirect=True) def test_local_project_to_full(self, tmp_path, project): - """ - Test updating a local-only project to a full one + """Test updating a local-only project to a full one by adding the required configs (both must be set together) Perform a quick check that data transfer does not error out now that the project is full, and the configs are set as expected. @@ -88,8 +84,7 @@ def test_local_project_to_full(self, tmp_path, project): @pytest.mark.parametrize("project", ["local"], indirect=True) def test_local_to_full_project(self, project): - """ - Change a project from local-only to a normal project by updating + """Change a project from local-only to a normal project by updating the relevant configs. Smoke test that general functionality is maintained and that transfers work correctly. """ @@ -126,8 +121,7 @@ def test_local_to_full_project(self, project): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("project", ["full"], indirect=True) def test_get_next_sub_and_ses(self, project, top_level_folder): - """ - Make a 'full' project with subject and session > 1 in both local + """Make a 'full' project with subject and session > 1 in both local and central projects. Then, delete the local and run get next sub / ses, and make the project local-only. Call validation requesting to also check central path, which should be ignored as we are in local-only mode. diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index 4e4c3393b..fa193c5b7 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -18,12 +18,9 @@ class TestLogging: - @pytest.fixture(scope="function") def teardown_logger(self): - """ - Ensure the logger is deleted at the end of each test. - """ + """Ensure the logger is deleted at the end of each test.""" yield if "datashuttle" in logging.root.manager.loggerDict: logging.root.manager.loggerDict.pop("datashuttle") @@ -33,14 +30,11 @@ def teardown_logger(self): # ------------------------------------------------------------------------- def test_logger_name(self): - """ - Check the canonical logger name. - """ + """Check the canonical logger name.""" assert ds_logger.get_logger_name() == "datashuttle" def test_start_logging(self, tmp_path, teardown_logger): - """ - Test that the central `start` logging function + """Test that the central `start` logging function starts the named logger with the expected handlers. """ assert ds_logger.logging_is_active() is False @@ -57,9 +51,7 @@ def test_start_logging(self, tmp_path, teardown_logger): assert isinstance(logger.handlers[0], logging.FileHandler) def test_shutdown_logger(self, tmp_path, teardown_logger): - """ - Check the log handler remover indeed removes the handles. - """ + """Check the log handler remover indeed removes the handles.""" assert ds_logger.logging_is_active() is False ds_logger.start(tmp_path, "test-command", variables=[]) @@ -72,9 +64,7 @@ def test_shutdown_logger(self, tmp_path, teardown_logger): assert ds_logger.logging_is_active() is False def test_logging_an_error(self, project, teardown_logger): - """ - Check that errors are caught and logged properly. - """ + """Check that errors are caught and logged properly.""" with pytest.raises(NeuroBlueprintError): project.create_folders("rawdata", "sob-001") @@ -89,8 +79,7 @@ def test_logging_an_error(self, project, teardown_logger): @pytest.fixture(scope="function") def clean_project_name(self): - """ - Create an empty project, but ensure no + """Create an empty project, but ensure no configs already exists, and delete created configs after test. @@ -107,8 +96,7 @@ def clean_project_name(self): @pytest.fixture(scope="function") def project(self, tmp_path, clean_project_name, request): - """ - Set `up a project with default configs to use + """Set `up a project with default configs to use for testing. This fixture is distinct from the base.py fixture as requires additional logging setup / teardown. @@ -137,16 +125,15 @@ def project(self, tmp_path, clean_project_name, request): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_log_filename(self, project): - """ - Check the log filename is formatted correctly, for + """Check the log filename is formatted correctly, for `update_config_file`, an arbitrary command """ project.update_config_file(central_host_id="test_id") log_search = list(project.cfg.logging_path.glob("*.log")) - assert ( - len(log_search) == 1 - ), "should only be 1 log in this test environment." + assert len(log_search) == 1, ( + "should only be 1 log in this test environment." + ) log_filename = log_search[0].name regex = re.compile(r"\d{8}T\d{6}_update-config-file.log") @@ -254,8 +241,7 @@ def test_create_folders(self, project): def test_logs_upload_and_download( self, project, upload_or_download, transfer_method ): - """ - Set transfer verbosity and progress settings so + """Set transfer verbosity and progress settings so maximum output is produced to test against. """ subs = ["sub-11"] @@ -311,8 +297,7 @@ def test_logs_upload_and_download( def test_logs_upload_and_download_folder_or_file( self, project, upload_or_download ): - """ - Set transfer verbosity and progress settings so + """Set transfer verbosity and progress settings so maximum output is produced to test against. """ test_utils.make_and_check_local_project_folders( @@ -353,8 +338,7 @@ def test_logs_upload_and_download_folder_or_file( def test_temp_log_folder_moved_make_config_file( self, clean_project_name, tmp_path ): - """ - Check that + """Check that logs are moved to the passed `local_path` when `make_config_file()` is passed. """ @@ -379,8 +363,7 @@ def test_temp_log_folder_moved_make_config_file( assert "make-config-file" in project_path_logs[0] def test_clear_logging_path(self, clean_project_name, tmp_path): - """ - The temporary logging path holds logs which are all + """The temporary logging path holds logs which are all transferred to a new `local_path` when configs are updated. This should only ever be the most recent log action, and not others which may @@ -455,8 +438,7 @@ def test_logs_bad_create_folders_error(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_validate_project_logging(self, project): - """ - Test that `validate_project` logs errors + """Test that `validate_project` logs errors and warnings to file. """ # Make conflicting subject folders @@ -489,8 +471,7 @@ def test_validate_project_logging(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_validate_names_against_project_logging(self, project): - """ - Implicitly test `validate_names_against_project` called when + """Implicitly test `validate_names_against_project` called when `make_project_folders` is called, that it logs errors to file. Warnings are not tested. """ diff --git a/tests/tests_integration/test_settings.py b/tests/tests_integration/test_settings.py index b996a1d02..5d5834c92 100644 --- a/tests/tests_integration/test_settings.py +++ b/tests/tests_integration/test_settings.py @@ -11,11 +11,9 @@ class TestPersistentSettings(BaseTest): - @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_name_templates(self, project): - """ - Test the 'name_templates' option that is stored in persistent + """Test the 'name_templates' option that is stored in persistent settings and adds a regexp to validate subject and session names against. @@ -120,13 +118,11 @@ def test_persistent_settings_name_templates(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_tui(self, project): - """ - Test persistent settings for the project that + """Test persistent settings for the project that determine display of the TUI. First check defaults are correct, change every one and save, then check they are correct on re-load. """ - # test all defaults settings = project._load_persistent_settings() tui_settings = settings["tui"] @@ -145,8 +141,7 @@ def test_persistent_settings_tui(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_bypass_validation(self, project): - """ - Check bypass validation which will allow folder + """Check bypass validation which will allow folder creation even when validation fails. Check it is off by default, turn on, check bad name can be created. Reload, turn off, check for error on attempting to create @@ -161,13 +156,12 @@ def test_bypass_validation(self, project): project.create_folders("rawdata", "sub-@@@") assert ( - "BAD_VALUE: The value for prefix sub in name sub-@@@ is not an integer." - == str(e.value) + str(e.value) + == "BAD_VALUE: The value for prefix sub in name sub-@@@ is not an integer." ) def get_settings_default(self): - """ - Hard-coded default settings that should mirror `canonical_configs` + """Hard-coded default settings that should mirror `canonical_configs` and should be changed whenever the canonical configs are changed. This is to protect against accidentally changing these configs. """ @@ -209,9 +203,7 @@ def get_settings_default(self): return default_settings def get_settings_changed(self): - """ - The default settings with every possible setting changed. - """ + """The default settings with every possible setting changed.""" changed_settings = { "create_checkboxes_on": {}, "transfer_checkboxes_on": { diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py index a94a5b05a..4b4babce4 100644 --- a/tests/tests_integration/test_ssh_file_transfer.py +++ b/tests/tests_integration/test_ssh_file_transfer.py @@ -29,8 +29,7 @@ class TestFileTransfer: ], ) def pathtable_and_project(self, request, tmpdir_factory): - """ - Create a project for SSH testing. Setup + """Create a project for SSH testing. Setup the project as normal, and switch configs to use SSH connection. @@ -49,7 +48,7 @@ def pathtable_and_project(self, request, tmpdir_factory): items have been transferred. This is achieved by using "class" scope. - NOTES + Notes ----- - Pytest params - The `params` key sets the `params` attribute on the pytest `request` fixture. @@ -68,6 +67,7 @@ def pathtable_and_project(self, request, tmpdir_factory): a few seconds after SSH transfer. This makes the tests run very slowly. We can get rid of this limitation on linux. + """ testing_ssh = request.param tmp_path = tmpdir_factory.mktemp("test") @@ -165,8 +165,7 @@ def test_all_data_transfer_options( datatype, upload_or_download, ): - """ - Parse the arguments to filter the pathtable, getting + """Parse the arguments to filter the pathtable, getting the files expected to be transferred passed on the arguments Note files in sub/ses/datatype folders must be handled separately to those in non-sub, non-ses, non-datatype folders @@ -245,8 +244,7 @@ def test_all_data_transfer_options( # --------------------------------------------------------------------------------------------------------------- def query_table(self, pathtable, arguments): - """ - Search the table for arguments, return empty + """Search the table for arguments, return empty if arguments empty """ if any(arguments): @@ -256,8 +254,7 @@ def query_table(self, pathtable, arguments): return folders def parse_arguments(self, pathtable, list_of_names, field): - """ - Replicate datashuttle name formatting by parsing + """Replicate datashuttle name formatting by parsing "all" arguments and turning them into a list of all names, (subject or session), taken from the pathtable. """ @@ -276,8 +273,7 @@ def parse_arguments(self, pathtable, list_of_names, field): return list_of_names def make_pathtable_search_filter(self, sub_names, ses_names, datatype): - """ - Create a string of arguments to pass to pd.query() that will + """Create a string of arguments to pass to pd.query() that will create the table of only transferred sub, ses and datatype. Two arguments must be created, one of all sub / ses / datatypes diff --git a/tests/tests_integration/test_ssh_setup.py b/tests/tests_integration/test_ssh_setup.py index 45ad0f51d..f26b1d921 100644 --- a/tests/tests_integration/test_ssh_setup.py +++ b/tests/tests_integration/test_ssh_setup.py @@ -1,5 +1,4 @@ -""" -SSH configs are set in conftest.py . The password +"""SSH configs are set in conftest.py . The password should be stored in a file called test_ssh_password.txt located in the same folder as test_ssh.py """ @@ -16,8 +15,7 @@ class TestSSH: @pytest.fixture(scope="function") def project(test, tmp_path): - """ - Make a project as per usual, but now add + """Make a project as per usual, but now add in test ssh configurations """ tmp_path = tmp_path / "test with space" @@ -45,8 +43,7 @@ def project(test, tmp_path): def test_verify_ssh_central_host_do_not_accept( self, capsys, project, input_ ): - """ - Use the main function to test this. Test the sub-function + """Use the main function to test this. Test the sub-function when accepting, because this main function will also call setup ssh key pairs which we don't want to do yet @@ -64,8 +61,7 @@ def test_verify_ssh_central_host_do_not_accept( assert "Host not accepted. No connection made.\n" in captured.out def test_verify_ssh_central_host_accept(self, capsys, project): - """ - User is asked to accept the server hostkey. Mock this here + """User is asked to accept the server hostkey. Mock this here and check hostkey is successfully accepted and written to configs. """ test_utils.clear_capsys(capsys) @@ -81,20 +77,19 @@ def test_verify_ssh_central_host_accept(self, capsys, project): captured = capsys.readouterr() assert captured.out == "Host accepted.\n" - with open(project.cfg.hostkeys_path, "r") as file: + with open(project.cfg.hostkeys_path) as file: hostkey = file.readlines()[0] assert f"{project.cfg['central_host_id']} ssh-ed25519 " in hostkey def test_generate_and_write_ssh_key(self, project): - """ - Check ssh key for passwordless connection is written + """Check ssh key for passwordless connection is written to file """ path_to_save = project.cfg["local_path"] / "test" ssh.generate_and_write_ssh_key(path_to_save) - with open(path_to_save, "r") as file: + with open(path_to_save) as file: first_line = file.readlines()[0] assert first_line == "-----BEGIN RSA PRIVATE KEY-----\n" diff --git a/tests/tests_integration/test_transfer_checks.py b/tests/tests_integration/test_transfer_checks.py index 342ed6407..7a6f631f3 100644 --- a/tests/tests_integration/test_transfer_checks.py +++ b/tests/tests_integration/test_transfer_checks.py @@ -15,8 +15,7 @@ class TestTransferChecks(BaseTest): [["rawdata", "derivatives"], ["rawdata"], ["derivatives"]], ) def test_rclone_check(self, project, top_level_folders): - """ - Test rclone.get_local_and_central_file_differences(). This function + """Test rclone.get_local_and_central_file_differences(). This function returns a dictionary where values are list of paths and keys separate based on differences between local and central projects. diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index e24a9ecaa..71fd7d658 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -14,7 +14,6 @@ class TestValidation(BaseTest): - @pytest.mark.parametrize( "sub_name", ["sub-001", "sub-999_@DATE@", "sub-001_random-tag_another-tag"], @@ -34,8 +33,7 @@ class TestValidation(BaseTest): def test_warn_on_inconsistent_sub_value_lengths( self, project, sub_name, bad_sub_name ): - """ - This test checks that inconsistent sub value lengths are properly + """This test checks that inconsistent sub value lengths are properly detected across the project. This is performed with an assortment of possible filenames and leading zero conflicts. @@ -94,8 +92,7 @@ def test_warn_on_inconsistent_sub_value_lengths( def test_warn_on_inconsistent_ses_value_lengths( self, project, ses_name, bad_ses_name ): - """ - This function is exactly the same as + """This function is exactly the same as `test_warn_on_inconsistent_sub_value_lengths()` but operates at the session level. This is extreme code duplication, but factoring the main logic out got very messy and hard to follow. @@ -139,8 +136,7 @@ def test_warn_on_inconsistent_ses_value_lengths( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_warn_on_inconsistent_sub_and_ses_value_lengths(self, project): - """ - Test that warning is shown for both subject and session when + """Test that warning is shown for both subject and session when inconsistent zeros are found in both. """ os.makedirs( @@ -173,8 +169,7 @@ def check_inconsistent_sub_or_ses_value_length_warning( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_duplicate_ses_or_sub_key_value_pair(self, project): - """ - Test the check that if a duplicate key is attempt to be made + """Test the check that if a duplicate key is attempt to be made when making a folder e.g. sub-001 exists, then make sub-001_id-123. After this check, make a folder that can be made (e.g. sub-003) just to make sure it does not raise error. @@ -207,8 +202,7 @@ def test_duplicate_ses_or_sub_key_value_pair(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_duplicate_sub_and_ses_num_leading_zeros(self, project): - """ - Very similar to test_duplicate_ses_or_sub_key_value_pair(), + """Very similar to test_duplicate_ses_or_sub_key_value_pair(), but explicitly check that error is raised if the same number is used with different number of leading zeros. """ @@ -228,8 +222,7 @@ def test_duplicate_sub_and_ses_num_leading_zeros(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_duplicate_sub_when_creating_session(self, project): - """ - Check the unique case that a duplicate subject is + """Check the unique case that a duplicate subject is introduced when the session is made. """ project.create_folders("rawdata", "sub-001") @@ -273,8 +266,7 @@ def test_duplicate_sub_when_creating_session(self, project): assert "DUPLICATE_NAME" in str(e.value) def test_duplicate_ses_across_subjects(self, project): - """ - Quick test that duplicate session folders only raise + """Quick test that duplicate session folders only raise an error when they are in the same subject. """ project.create_folders("rawdata", "sub-001", "ses-001") @@ -293,8 +285,7 @@ def test_duplicate_ses_across_subjects(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_invalid_sub_and_ses_name(self, project): - """ - This is a slightly weird case, the name is successfully + """This is a slightly weird case, the name is successfully prefixed as 'sub-sub_100` but when the value if `sub-` is extracted, it is also "sub" and so an error is raised. """ @@ -302,16 +293,16 @@ def test_invalid_sub_and_ses_name(self, project): project.create_folders("rawdata", "sub_100") assert ( - "BAD_VALUE: The value for prefix sub in name sub-sub_100 is not an integer." - == str(e.value) + str(e.value) + == "BAD_VALUE: The value for prefix sub in name sub-sub_100 is not an integer." ) with pytest.raises(NeuroBlueprintError) as e: project.create_folders("rawdata", "sub-001", "ses_100") assert ( - "BAD_VALUE: The value for prefix ses in name ses-ses_100 is not an integer." - == str(e.value) + str(e.value) + == "BAD_VALUE: The value for prefix ses in name ses-ses_100 is not an integer." ) # ------------------------------------------------------------------------- @@ -319,8 +310,7 @@ def test_invalid_sub_and_ses_name(self, project): # ------------------------------------------------------------------------- def test_validate_project(self, project): - """ - Test the `validate_project` function over all it's arguments. + """Test the `validate_project` function over all it's arguments. Note not every validation case is tested exhaustively, these are tested in `test_validation_unit.py` elsewhere here. """ @@ -452,8 +442,7 @@ def test_output_paths_are_valid(self, project): def test_validate_names_against_project_with_bad_existing_names( self, project ): - """ - When using `validate_names_against_project()` there are + """When using `validate_names_against_project()` there are three possible classes of error: 1) error in the passed names. 2) an error already exists in the project. @@ -583,8 +572,7 @@ def test_validate_names_against_project_with_bad_existing_names( @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_validate_names_against_project_interactions(self, project): - """ - Check that interactions between the list of names and existing + """Check that interactions between the list of names and existing project are caught. This includes duplicate subject / session names as well as inconsistent subject / session value lengths. """ @@ -706,8 +694,7 @@ def test_validate_names_against_project_interactions(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_tags_in_name_templates_pass_validation(self, project): - """ - It is useful to allow tags in the `name_templates` as it means + """It is useful to allow tags in the `name_templates` as it means auto-completion in the TUI can use tags for automatic name generation. Because all subject and session names are fully formatted (e.g. @DATE@ converted to actual dates) @@ -760,9 +747,7 @@ def test_tags_in_name_templates_pass_validation(self, project): assert "TEMPLATE: The name: ses-001_datex-20241212" in str(e.value) def test_name_templates_validate_project(self, project): - """ - TODO - """ + """TODO:""" name_templates = { "on": True, "sub": "sub-\d\d_id-\d.?", @@ -835,8 +820,7 @@ def test_quick_validation(self, mocker, project): assert kwargs["name_templates"] == {"on": False} def test_quick_validation_top_level_folder(self, project): - """ - Test that errors are raised as expected on + """Test that errors are raised as expected on bad project path input. """ with pytest.raises(FileNotFoundError) as e: @@ -924,8 +908,7 @@ def test_strict_mode_validation(self, project, top_level_folder): def test_check_high_level_project_structure( self, project, top_level_folder ): - """ - Check that local and central project names are properly formatted + """Check that local and central project names are properly formatted and that """ with pytest.warns(UserWarning) as w: diff --git a/tests/tests_tui/test_local_only_project.py b/tests/tests_tui/test_local_only_project.py index 0599ea144..8a06ccfa7 100644 --- a/tests/tests_tui/test_local_only_project.py +++ b/tests/tests_tui/test_local_only_project.py @@ -5,14 +5,12 @@ class TestTuiLocalOnlyProject(TuiBase): - @pytest.mark.asyncio async def test_local_only_make_project( self, empty_project_paths, ): - """ - Test a local-only project, where the only set config is `local_path`. + """Test a local-only project, where the only set config is `local_path`. Set up a local project, and check the 'Transfer' tab is disabled and set configs are propagated. """ @@ -21,7 +19,6 @@ async def test_local_only_make_project( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_and_check_local_only_project( pilot, project_name, local_path ) @@ -49,8 +46,7 @@ async def test_local_project_to_full( self, empty_project_paths, ): - """ - It is possible to switch between a 'local-only' project (`local_path` + """It is possible to switch between a 'local-only' project (`local_path` only set) and a full project with all configs set, where transfer is allowed. Here start as a local project then set configs so we become a full project. """ @@ -60,7 +56,6 @@ async def test_local_project_to_full( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up a local-only project await self.setup_and_check_local_only_project( pilot, project_name, local_path @@ -125,8 +120,7 @@ async def test_full_project_to_local( self, setup_project_paths, ): - """ - Very similar to `test_check_local_only_project_to_full()`, but + """Very similar to `test_check_local_only_project_to_full()`, but going from a full project to a local only. This still requires a refresh so the full transfer tab can be set to a placeholder. """ @@ -135,7 +129,6 @@ async def test_full_project_to_local( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Fixture generated a full project, switch to it's # project manager screen here. await self.check_and_click_onto_existing_project( @@ -187,8 +180,7 @@ async def test_full_project_to_local( async def setup_and_check_local_only_project( self, pilot, project_name, local_path ): - """ - Set up a local-only project by filling in the `local_path` and setting + """Set up a local-only project by filling in the `local_path` and setting the radio button to the no-connection option. """ # Move to configs window diff --git a/tests/tests_tui/test_tui_configs.py b/tests/tests_tui/test_tui_configs.py index da853f8bc..a9f4d8fe0 100644 --- a/tests/tests_tui/test_tui_configs.py +++ b/tests/tests_tui/test_tui_configs.py @@ -14,7 +14,6 @@ class TestTuiConfigs(TuiBase): - # ------------------------------------------------------------------------- # Test New Project Configs # ------------------------------------------------------------------------- @@ -26,8 +25,7 @@ async def test_make_new_project_configs( empty_project_paths, kwargs_set, ): - """ - Check the ConfigsContent when making a new project. This contains + """Check the ConfigsContent when making a new project. This contains many widgets shared with the ConfigsContent on the tab page, however also includes an additional information banner and input for the project name. @@ -67,7 +65,6 @@ async def test_make_new_project_configs( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select a new project, check NewProjectScreen is # displayed correctly. await self.scroll_to_click_pause( @@ -158,8 +155,7 @@ async def test_make_new_project_configs( async def test_update_config_on_project_manager_screen( self, setup_project_paths ): - """ - Test the ConfigsContent on the project manager tab screen. + """Test the ConfigsContent on the project manager tab screen. The project is set up in the fixture, navigate to the project page. Check that the default configs are displayed. Change all the configs, save, and check these are updated on the config file and on the @@ -172,7 +168,6 @@ async def test_update_config_on_project_manager_screen( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -214,7 +209,7 @@ async def test_update_config_on_project_manager_screen( "central_host_username": "random_username", } - for key in new_kwargs.keys(): + for key in new_kwargs: # The purpose is to update to completely new configs assert new_kwargs[key] != project_cfg[key] @@ -267,8 +262,7 @@ async def test_update_config_on_project_manager_screen( @pytest.mark.asyncio async def test_configs_select_path(self, monkeypatch): - """ - Test the 'Select' buttons / DirectoryTree on the ConfigsContent. + """Test the 'Select' buttons / DirectoryTree on the ConfigsContent. These are used to select folders that are filled into the Input. Open the select dialog, select a folder, check the path is filled into the Input. There is one for both local @@ -281,7 +275,6 @@ async def test_configs_select_path(self, monkeypatch): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select the page and ConfigsContent for setting up new project await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" @@ -344,10 +337,8 @@ async def test_configs_select_path(self, monkeypatch): @pytest.mark.asyncio async def test_bad_configs_screen_input(self, empty_project_paths): - app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select a new project, check NewProjectScreen is displayed correctly. await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" @@ -375,11 +366,9 @@ async def test_bad_configs_screen_input(self, empty_project_paths): async def check_configs_widgets_match_configs( self, configs_content, kwargs ): - """ - Check that the widgets of the TUI configs match those found + """Check that the widgets of the TUI configs match those found in `kwargs`. """ - # Local Path ---------------------------------------------------------- assert ( @@ -402,7 +391,6 @@ async def check_configs_widgets_match_configs( ) if kwargs["connection_method"] == "ssh": - # Central Host ID ------------------------------------------------- assert ( @@ -429,11 +417,9 @@ async def check_configs_widgets_match_configs( ) async def set_configs_content_widgets(self, pilot, kwargs): - """ - Given a dict of options that can be set on the configs TUI + """Given a dict of options that can be set on the configs TUI in kwargs, set all configs widgets according to kwargs. """ - # Local Path ---------------------------------------------------------- await self.fill_input( @@ -443,7 +429,6 @@ async def set_configs_content_widgets(self, pilot, kwargs): # Connection Method --------------------------------------------------- if kwargs["connection_method"] == "ssh": - await self.scroll_to_click_pause(pilot, "#configs_ssh_radiobutton") # Central Host ID ------------------------------------------------- @@ -471,8 +456,7 @@ async def set_configs_content_widgets(self, pilot, kwargs): async def check_new_project_configs( self, pilot, project_name, configs_content, kwargs ): - """ - Check the configs displayed on the TUI match those found in `kwargs`. + """Check the configs displayed on the TUI match those found in `kwargs`. Also, check the widgets unique to ConfigsContent on the configs selection for a new project. """ diff --git a/tests/tests_tui/test_tui_create_folders.py b/tests/tests_tui/test_tui_create_folders.py index 00a408f76..8fbc32a2a 100644 --- a/tests/tests_tui/test_tui_create_folders.py +++ b/tests/tests_tui/test_tui_create_folders.py @@ -13,7 +13,6 @@ class TestTuiCreateFolders(TuiBase): - # ------------------------------------------------------------------------- # General test Create Folders # ------------------------------------------------------------------------- @@ -23,9 +22,7 @@ class TestTuiCreateFolders(TuiBase): async def test_create_folders_sub_and_ses( self, setup_project_paths, test_multi_input ): - """ - Basic test that folders are created as expected through the TUI. - """ + """Basic test that folders are created as expected through the TUI.""" # Define folders to create if test_multi_input: sub_text = "sub-001, sub-002" @@ -42,7 +39,6 @@ async def test_create_folders_sub_and_ses( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the TUI on the 'create' tab, filling the # input with the subject and session folders to create. await self.check_and_click_onto_existing_project( @@ -87,8 +83,7 @@ async def test_create_folders_sub_and_ses( @pytest.mark.asyncio async def test_create_folders_formatted_names(self, setup_project_paths): - """ - Test preview tooltips and create folders with _@DATE@ formatting. + """Test preview tooltips and create folders with _@DATE@ formatting. The @TO@ key is not tested. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() @@ -181,15 +176,13 @@ async def test_create_folders_formatted_names(self, setup_project_paths): async def test_create_folders_bad_validation_tooltips( self, setup_project_paths ): - """ - Check that correct tooltips are displayed when + """Check that correct tooltips are displayed when various invalid subject or session names are provided. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -259,8 +252,7 @@ async def test_create_folders_bad_validation_tooltips( async def test_validation_error_and_bypass_validation( self, setup_project_paths ): - """ - Test validation and bypass validation options by + """Test validation and bypass validation options by first trying to create an invalid folder name, and checking an error displays. Next, turn on 'bypass validation' and check the folders are created despite being invalid. @@ -269,7 +261,6 @@ async def test_validation_error_and_bypass_validation( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -338,15 +329,13 @@ async def test_validation_error_and_bypass_validation( async def test_name_template_next_sub_or_ses_and_validation( self, setup_project_paths ): - """ - Test validation and double-click for next sub / ses + """Test validation and double-click for next sub / ses values when 'name templates' is set in the 'Settings' window. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -468,15 +457,13 @@ async def test_name_template_next_sub_or_ses_and_validation( @pytest.mark.asyncio async def test_get_next_sub_and_ses_no_template(self, setup_project_paths): - """ - Test the double click on Input correctly fills with the + """Test the double click on Input correctly fills with the next sub or ses (or prefix only when CTRL is pressed). """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True ) @@ -532,15 +519,13 @@ async def test_get_next_sub_and_ses_no_template(self, setup_project_paths): async def test_create_folders_settings_top_level_folder( self, setup_project_paths ): - """ - Check the folders are created in the correct top level + """Check the folders are created in the correct top level folder when this is changed in the 'Settings' screen. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open the CreateFoldersSettingsScreen await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False @@ -619,7 +604,6 @@ async def iterate_and_check_all_datatype_folders( folder_used = test_utils.get_all_broad_folders_used(value=False) for datatype in canonical_configs.get_broad_datatypes(): - await self.scroll_to_click_pause( pilot, f"#create_{datatype}_checkbox", diff --git a/tests/tests_tui/test_tui_datatypes.py b/tests/tests_tui/test_tui_datatypes.py index 508c2d06a..48bcbf27c 100644 --- a/tests/tests_tui/test_tui_datatypes.py +++ b/tests/tests_tui/test_tui_datatypes.py @@ -7,9 +7,7 @@ class TestDatatypesTUI(TuiBase): - """ - Test the datatype selection screen for the Create and Transfer tab. - """ + """Test the datatype selection screen for the Create and Transfer tab.""" @pytest.mark.asyncio async def test_select_displayed_datatypes_create( @@ -19,7 +17,6 @@ async def test_select_displayed_datatypes_create( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the TUI on the 'create' tab, filling the # input with the subject and session folders to create. await self.check_and_click_onto_existing_project( @@ -134,7 +131,6 @@ async def test_select_displayed_datatypes_transfer( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the TUI on the 'transfer' tab (custom) and # open the datatype selection screen await self.check_and_click_onto_existing_project( diff --git a/tests/tests_tui/test_tui_directorytree.py b/tests/tests_tui/test_tui_directorytree.py index bb93bb34d..11562e085 100644 --- a/tests/tests_tui/test_tui_directorytree.py +++ b/tests/tests_tui/test_tui_directorytree.py @@ -14,22 +14,19 @@ class TestTuiCreateDirectoryTree(TuiBase): - """ - Test the `Create` tab directory tree. + """Test the `Create` tab directory tree. `Transfer` """ @pytest.mark.asyncio async def test_fill_and_append_next_sub_and_ses(self, setup_project_paths): - """ - Test the CTRL+F and CTRL+A functions on the directorytree + """Test the CTRL+F and CTRL+A functions on the directorytree that fill and append subject / session name to the inputs. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open the create tab and first fill the subject # and session inputs with -001. await self.check_and_click_onto_existing_project( @@ -116,8 +113,7 @@ async def test_fill_and_append_next_sub_and_ses(self, setup_project_paths): async def test_create_folders_directorytree_clipboard( self, setup_project_paths ): - """ - Check that pressing CTRL+Q on the directorytree copies the + """Check that pressing CTRL+Q on the directorytree copies the hovered folder to the clipboard (using pyperclip). """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() @@ -159,7 +155,6 @@ async def test_failed_pyperclip_copy( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up a project and navigate to the directory tree await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True @@ -197,8 +192,7 @@ def mock_copy(_): async def test_create_folders_directorytree_open_filesystem( self, setup_project_paths, monkeypatch ): - """ - Test pressing CTRL+O on the filetree triggers the opening + """Test pressing CTRL+O on the filetree triggers the opening of a folder through the show-in-file-manager package (monkeypatched function). """ @@ -206,7 +200,6 @@ async def test_create_folders_directorytree_open_filesystem( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the 'create tab' with loaded nodes await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True diff --git a/tests/tests_tui/test_tui_get_help.py b/tests/tests_tui/test_tui_get_help.py index 55d876f35..aa8986b8c 100644 --- a/tests/tests_tui/test_tui_get_help.py +++ b/tests/tests_tui/test_tui_get_help.py @@ -5,17 +5,14 @@ class TestTuiSettings(TuiBase): - """ - Test that the 'Get Help' page from the main menu. + """Test that the 'Get Help' page from the main menu. Open it, check the expected label is displayed, close it. """ @pytest.mark.asyncio async def test_get_help(self, empty_project_paths): - app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.scroll_to_click_pause( pilot, "#mainwindow_get_help_button" ) diff --git a/tests/tests_tui/test_tui_logging.py b/tests/tests_tui/test_tui_logging.py index cb4b8417f..6a046c3e8 100644 --- a/tests/tests_tui/test_tui_logging.py +++ b/tests/tests_tui/test_tui_logging.py @@ -7,11 +7,9 @@ class TestTuiLogging(TuiBase): - @pytest.mark.asyncio async def test_logging(self, setup_project_paths): - """ - Test logging by running some commands, checking they + """Test logging by running some commands, checking they are displayed on the logging tree, that the most recent log is correct and that the log screen opens when clicked. """ @@ -19,7 +17,6 @@ async def test_logging(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Update configs and create folders to make some logs project = DataShuttle(project_name) @@ -63,7 +60,9 @@ async def test_logging(self, setup_project_paths): ) assert ( "create-folders" in widg.get_node_at_line(2).data.path.stem - ), f"ERROR MESSAGE: {widg.get_node_at_line(0).data.path}-{widg.get_node_at_line(1).data.path}-{widg.get_node_at_line(2).data.path}" + ), ( + f"ERROR MESSAGE: {widg.get_node_at_line(0).data.path}-{widg.get_node_at_line(1).data.path}-{widg.get_node_at_line(2).data.path}" + ) # Check the latest logging path is correct assert ( diff --git a/tests/tests_tui/test_tui_settings.py b/tests/tests_tui/test_tui_settings.py index 77a65b090..8b9bcaa4d 100644 --- a/tests/tests_tui/test_tui_settings.py +++ b/tests/tests_tui/test_tui_settings.py @@ -5,20 +5,16 @@ class TestTuiSettings(TuiBase): - """ - Test the 'Settings' screen accessible from the Main Menu. - """ + """Test the 'Settings' screen accessible from the Main Menu.""" @pytest.mark.asyncio async def test_light_dark_mode(self): - """ - Check the light / dark mode switch which is stored + """Check the light / dark mode switch which is stored in the global tui settings. Global refers to set across all projects not related to a specific project. """ app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.scroll_to_click_pause( pilot, "#mainwindow_settings_button" ) @@ -45,8 +41,7 @@ async def test_light_dark_mode(self): @pytest.mark.asyncio async def test_show_transfer_tree_status(self, setup_project_paths): - """ - Check the 'show transfer tree' option that turns off transfer + """Check the 'show transfer tree' option that turns off transfer tree styling by default has the intended effects. It is difficult to test whether the tree is actually styled, so here all underlying configs + the transfer tree legend @@ -56,7 +51,6 @@ async def test_show_transfer_tree_status(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # First check the show transfer tree styling is off # in the project manager tab and legend does not exist. await self.check_and_click_onto_existing_project( diff --git a/tests/tests_tui/test_tui_transfer.py b/tests/tests_tui/test_tui_transfer.py index 3e8c389f1..1fd3fe60a 100644 --- a/tests/tests_tui/test_tui_transfer.py +++ b/tests/tests_tui/test_tui_transfer.py @@ -7,8 +7,7 @@ class TestTuiTransfer(TuiBase): - """ - Test transferring through the TUI (entire project, top + """Test transferring through the TUI (entire project, top level only or custom). This class leverages the underlying test utils that check API transfers. """ @@ -24,7 +23,6 @@ async def test_transfer_entire_project( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -57,8 +55,7 @@ async def test_transfer_entire_project( await pilot.pause() async def check_persistent_settings(self, pilot): - """ - Run transfer with each overwrite setting and check it is propagated + """Run transfer with each overwrite setting and check it is propagated to datashuttle methods. """ await self.set_and_check_persistent_settings(pilot, "never", True) @@ -89,8 +86,7 @@ async def set_transfer_tab_dry_run_checkbox(self, pilot, dry_run_setting): async def set_and_check_persistent_settings( self, pilot, overwrite_setting, dry_run_setting ): - """ - Run transfer with an overwrite setting and check it is propagated + """Run transfer with an overwrite setting and check it is propagated to datashuttle methods by checking the logs. """ await self.set_overwrite_checkbox(pilot, overwrite_setting) @@ -118,7 +114,6 @@ async def test_transfer_top_level_folder( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -167,7 +162,6 @@ async def test_transfer_custom( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -228,7 +222,6 @@ async def test_transfer_custom( async def switch_top_level_folder_select( self, pilot, id, top_level_folder ): - if top_level_folder == "rawdata": assert pilot.app.screen.query_one(id).value == "rawdata" else: diff --git a/tests/tests_tui/test_tui_widgets_and_defaults.py b/tests/tests_tui/test_tui_widgets_and_defaults.py index e0b330797..897d066ce 100644 --- a/tests/tests_tui/test_tui_widgets_and_defaults.py +++ b/tests/tests_tui/test_tui_widgets_and_defaults.py @@ -14,8 +14,7 @@ class TestTuiWidgets(TuiBase): - """ - This class performs fundamental checks on the default display + """This class performs fundamental checks on the default display of widgets and that changing widgets properly change underlying configs. This does not perform any functional tests e.g. creation of configs of new files. @@ -27,12 +26,9 @@ class TestTuiWidgets(TuiBase): @pytest.mark.asyncio async def test_new_project_configs(self, empty_project_paths): - """ - Test all widgets display as expected on the New Project configs page. - """ + """Test all widgets display as expected on the New Project configs page.""" app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Select a new project, check NewProjectScreen is displayed correctly. await self.scroll_to_click_pause( pilot, "#mainwindow_new_project_button" @@ -127,7 +123,6 @@ async def test_new_project_configs(self, empty_project_paths): == "" ) if platform.system() == "Windows": - assert ( configs_content.query_one( "#configs_central_path_input" @@ -240,8 +235,7 @@ async def check_new_project_ssh_widgets( @pytest.mark.asyncio async def test_existing_project_configs(self, setup_project_paths): - """ - Because the underlying screen is shared between new and existing + """Because the underlying screen is shared between new and existing project configs, in the existing project configs just check widgets are hidden as expected. """ @@ -249,7 +243,6 @@ async def test_existing_project_configs(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -278,15 +271,13 @@ async def test_existing_project_configs(self, setup_project_paths): @pytest.mark.asyncio async def test_create_folders_widgets_display(self, setup_project_paths): - """ - Test all widgets on the 'Create' tab of the project manager screen + """Test all widgets on the 'Create' tab of the project manager screen are displayed as expected. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -370,15 +361,13 @@ async def test_create_folders_widgets_display(self, setup_project_paths): @pytest.mark.asyncio async def test_create_folder_settings_widgets(self, setup_project_paths): - """ - Test the widgets in the 'Settings' menu of the project + """Test the widgets in the 'Settings' menu of the project manager's 'Create' tab. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) @@ -487,8 +476,7 @@ async def test_create_folder_settings_widgets(self, setup_project_paths): async def test_name_templates_widgets_and_settings( self, setup_project_paths ): - """ - Check the 'Name Templates' section of the 'Create' tab 'Settings + """Check the 'Name Templates' section of the 'Create' tab 'Settings page. Here both subject and session configs share the same input, so ensure these are mapped correctly by the radiobutton setting, and that the underlying configs are set correctly. @@ -500,7 +488,6 @@ async def test_name_templates_widgets_and_settings( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) @@ -650,15 +637,13 @@ async def test_name_templates_widgets_and_settings( @pytest.mark.asyncio async def test_bypass_validation_settings(self, setup_project_paths): - """ - Test all configs that underly the 'bypass validation' + """Test all configs that underly the 'bypass validation' setting are updated correctly by the widget. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False ) @@ -717,15 +702,13 @@ async def test_bypass_validation_settings(self, setup_project_paths): @pytest.mark.asyncio async def test_all_top_level_folder_selects(self, setup_project_paths): - """ - Test all 'top level folder' selects (in Create and Transfer tabs) + """Test all 'top level folder' selects (in Create and Transfer tabs) update the underlying configs correctly. """ tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Open project, check top level folder are correct await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=False @@ -890,9 +873,7 @@ async def check_top_folder_select( expected_val, move_to_position: Union[bool, int] = False, ): - """ - If move to position is not False, must be int specifying position - """ + """If move to position is not False, must be int specifying position""" if move_to_position: await self.move_select_to_position(pilot, id, move_to_position) @@ -913,8 +894,7 @@ async def check_top_folder_select( @pytest.mark.asyncio async def test_all_checkboxes(self, setup_project_paths): - """ - Check all datatype checkboxes (Create and Transfer tab) + """Check all datatype checkboxes (Create and Transfer tab) correctly update the underlying configs. These are tested together to ensure there are no strange interaction between these as they both share stored in the project's 'tui' @@ -924,7 +904,6 @@ async def test_all_checkboxes(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - await self.check_and_click_onto_existing_project( pilot, project_name ) @@ -1041,7 +1020,6 @@ async def test_all_transfer_widgets(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -1228,7 +1206,6 @@ async def test_overwrite_existing_files(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( @@ -1280,8 +1257,7 @@ async def test_overwrite_existing_files(self, setup_project_paths): @pytest.mark.asyncio async def test_dry_run(self, setup_project_paths): - """ - Test the dry run setting. This is very similar in structure + """Test the dry run setting. This is very similar in structure to `test_overwrite_existing_files()`, merge if more persistent settings added. """ @@ -1289,7 +1265,6 @@ async def test_dry_run(self, setup_project_paths): app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Navigate to the existing project and click onto the # configs tab. await self.check_and_click_onto_existing_project( diff --git a/tests/tests_tui/tui_base.py b/tests/tests_tui/tui_base.py index 28351074a..fc20cbbc2 100644 --- a/tests/tests_tui/tui_base.py +++ b/tests/tests_tui/tui_base.py @@ -8,13 +8,10 @@ class TuiBase: - """ - Contains fixtuers and helper functions for TUI tests. - """ + """Contains fixtuers and helper functions for TUI tests.""" def tui_size(self): - """ - If the TUI screen in the test environment is not + """If the TUI screen in the test environment is not large enough, often the error `textual.pilot.OutOfBounds: Target offset is outside of currently-visible screen region.` @@ -27,8 +24,7 @@ def tui_size(self): @pytest_asyncio.fixture(scope="function") async def empty_project_paths(self, tmp_path_factory, monkeypatch): - """ - Get the paths and project name for a non-existent (i.e. not + """Get the paths and project name for a non-existent (i.e. not yet setup) project. """ project_name = "my-test-project" @@ -48,9 +44,7 @@ async def empty_project_paths(self, tmp_path_factory, monkeypatch): @pytest_asyncio.fixture(scope="function") async def setup_project_paths(self, empty_project_paths): - """ - Get the paths and project name for a setup project. - """ + """Get the paths and project name for a setup project.""" test_utils.setup_project_fixture( empty_project_paths["tmp_path"], empty_project_paths["project_name"], @@ -59,8 +53,7 @@ async def setup_project_paths(self, empty_project_paths): return empty_project_paths def monkeypatch_get_datashuttle_path(self, tmp_config_path, _monkeypatch): - """ - For these tests, store the datashuttle configs (usually stored in + """For these tests, store the datashuttle configs (usually stored in Path.home()) in the `tmp_path` provided by pytest, as it simplifies testing here. @@ -79,8 +72,7 @@ def mock_get_datashuttle_path(): ) def monkeypatch_print(self, _monkeypatch): - """ - Calls to `print` in datashuttle crash the TUI in the + """Calls to `print` in datashuttle crash the TUI in the test environment. I am not sure why. Get around this in tests by monkeypatching the datashuttle print method. """ @@ -93,9 +85,7 @@ def return_none(arg1, arg2=None): ) async def fill_input(self, pilot, id, value): - """ - Fill and input of `id` with `value`. - """ + """Fill and input of `id` with `value`.""" await self.scroll_to_click_pause(pilot, id) pilot.app.screen.query_one(id).value = "" await pilot.press(*value) @@ -104,8 +94,7 @@ async def fill_input(self, pilot, id, value): async def setup_existing_project_create_tab_filled_sub_and_ses( self, pilot, project_name, create_folders=False ): - """ - Set up an existing project and switch to the 'Create' tab + """Set up an existing project and switch to the 'Create' tab on the project manager screen. """ await self.check_and_click_onto_existing_project(pilot, project_name) @@ -123,16 +112,14 @@ async def setup_existing_project_create_tab_filled_sub_and_ses( ) async def double_click(self, pilot, id, control=False): - """ - Double-click on a widget of `id`, if `control` is `True` the + """Double-click on a widget of `id`, if `control` is `True` the control modifier key will be used. """ for _ in range(2): await self.scroll_to_click_pause(pilot, id, control=control) async def reload_tree_nodes(self, pilot, id, num_nodes): - """ - For some reason, for TUI tree nodes to register in the + """For some reason, for TUI tree nodes to register in the test environment all need to have `reload_node` called on the node. """ @@ -143,41 +130,32 @@ async def reload_tree_nodes(self, pilot, id, num_nodes): await pilot.pause() async def hover_and_press_tree(self, pilot, id, hover_line, press_string): - """ - Hover over a directorytree at a node-line and press a specific string - """ + """Hover over a directorytree at a node-line and press a specific string""" await pilot.pause() pilot.app.screen.query_one(id).hover_line = hover_line await pilot.pause() await self.press_tree(pilot, id, press_string) async def press_tree(self, pilot, id, press_string): - """ - Click on a tree to give it focus and press buttons - """ + """Click on a tree to give it focus and press buttons""" await self.scroll_to_click_pause(pilot, id) await pilot.press(press_string) await pilot.pause() async def scroll_to_and_pause(self, pilot, id): - """ - Scroll to a widget and pause. - """ + """Scroll to a widget and pause.""" widget = pilot.app.screen.query_one(id) widget.scroll_visible(animate=False) await pilot.pause() async def scroll_to_click_pause(self, pilot, id, control=False): - """ - Scroll to a widget, click it and call pause. - """ + """Scroll to a widget, click it and call pause.""" await self.scroll_to_and_pause(pilot, id) await pilot.click(id, control=control) await pilot.pause() async def check_and_click_onto_existing_project(self, pilot, project_name): - """ - From the main menu, go onto the select project page and + """From the main menu, go onto the select project page and select the project created in the test environment. Perform general TUI checks during the navigation. """ @@ -208,9 +186,7 @@ async def switch_tab(self, pilot, tab): await self.scroll_to_click_pause(pilot, f"Tab#{content_tab}") async def turn_off_all_datatype_checkboxes(self, pilot, tab="create"): - """ - Make sure all checkboxes are off to start - """ + """Make sure all checkboxes are off to start""" assert tab in ["create", "transfer"] checkbox_names = canonical_configs.get_broad_datatypes() @@ -234,8 +210,7 @@ async def turn_off_all_datatype_checkboxes(self, pilot, tab="create"): async def exit_to_main_menu_and_reeneter_project_manager( self, pilot, project_name ): - """ - Exist from the project manager screen, then re-enter back + """Exist from the project manager screen, then re-enter back into the project. This refreshes the screen and is important in testing state is preserved across re-loading. """ @@ -244,15 +219,12 @@ async def exit_to_main_menu_and_reeneter_project_manager( await self.check_and_click_onto_existing_project(pilot, project_name) async def close_messagebox(self, pilot): - """ - Close the modal_dialogs.Messagebox - """ + """Close the modal_dialogs.Messagebox""" pilot.app.screen.on_button_pressed() await pilot.pause() async def move_select_to_position(self, pilot, id, position): - """ - Move a select widget to a specific position (e.g. "rawdata" + """Move a select widget to a specific position (e.g. "rawdata" or "derivatives" select). The position can be determined by trial and error. """ diff --git a/tests/tests_unit/test_links.py b/tests/tests_unit/test_links.py index 5da0e04c2..86adc8b9f 100644 --- a/tests/tests_unit/test_links.py +++ b/tests/tests_unit/test_links.py @@ -4,8 +4,7 @@ def test_links(): - """ - Test canonical links are working. Unfortunately Zulip links cannot + """Test canonical links are working. Unfortunately Zulip links cannot be validated. """ assert validators.url(links.get_docs_link()) diff --git a/tests/tests_unit/test_unit.py b/tests/tests_unit/test_unit.py index 23257bf6f..3c55b526a 100644 --- a/tests/tests_unit/test_unit.py +++ b/tests/tests_unit/test_unit.py @@ -7,9 +7,7 @@ class TestUnit: - """ - Currently contains misc. unit tests. - """ + """Currently contains misc. unit tests.""" @pytest.mark.parametrize( "underscore_position", ["left", "right", "both", "none"] @@ -18,8 +16,7 @@ class TestUnit: "key", [tags("date"), tags("time"), tags("datetime")] ) def test_datetime_string_replacement(self, key, underscore_position): - """ - Test the function that replaces @DATE, @TIME@ or @DATETIME@ + """Test the function that replaces @DATE, @TIME@ or @DATETIME@ keywords with the date / time / datetime. Also, it will pre/append underscores to the tags if they are not already there (e.g if user input "sub-001@DATE"). @@ -42,9 +39,9 @@ def test_datetime_string_replacement(self, key, underscore_position): name_list = [name] formatting.update_names_with_datetime(name_list) - assert ( - re.search(regex, name_list[0]) is not None - ), "datetime formatting is incorrect." + assert re.search(regex, name_list[0]) is not None, ( + "datetime formatting is incorrect." + ) @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_process_to_keyword_in_sub_input(self, prefix): @@ -109,8 +106,7 @@ def test_process_to_keyword_bad_input_raises_error( ) def test_get_value_from_bids_name_regexp(self): - """ - Test the regexp that finds the value from a BIDS-name + """Test the regexp that finds the value from a BIDS-name key-value pair. """ bids_name = "sub-0123125_ses-11312_datetime-5345323_id-3asd@523" @@ -128,8 +124,7 @@ def test_get_value_from_bids_name_regexp(self): assert id == "3asd@523" def test_num_leading_zeros(self): - """ - Check num_leading_zeros handles prefixed and non-prefixed + """Check num_leading_zeros handles prefixed and non-prefixed case from -1 to -(101x 0)1. """ for i in range(101): @@ -145,8 +140,7 @@ def test_num_leading_zeros(self): def test_get_max_sub_or_ses_num_and_value_length_empty( self, prefix, default_num_value_digits ): - """ - When the list of sub or ses names is empty, the returned max number + """When the list of sub or ses names is empty, the returned max number should be zero and the `default_num_value_digits` be set to the passed default """ @@ -162,8 +156,7 @@ def test_get_max_sub_or_ses_num_and_value_length_empty( @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_get_max_sub_or_ses_num_and_value_length_error(self, prefix): - """ - An error will be shown if the sub or ses value digits are inconsistent, + """An error will be shown if the sub or ses value digits are inconsistent, because it is not possible to return the number of values required. A warning should be shown in that the number of value digits are @@ -213,8 +206,7 @@ def test_get_max_sub_or_ses_num_and_value_length_error(self, prefix): def test_get_max_sub_or_ses_num_and_value_length( self, prefix, test_max_num, test_num_digits ): - """ - Test many combinations of subject names + """Test many combinations of subject names and number of digits for a project, e.g. `names = ["sub-001", ... "sub-101"]`. """ @@ -235,9 +227,7 @@ def test_get_max_sub_or_ses_num_and_value_length( @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_get_max_sub_or_ses_num_and_value_length_edge_case(self, prefix): - """ - Test the edge case where the subject number does not start at 1. - """ + """Test the edge case where the subject number does not start at 1.""" names = [f"{prefix}-09", f"{prefix}-10", f"{prefix}-11"] max_num, num_digits = getters.get_max_sub_or_ses_num_and_value_length( @@ -252,8 +242,7 @@ def test_get_max_sub_or_ses_num_and_value_length_edge_case(self, prefix): # ------------------------------------------------------------------------- def make_name(self, key, underscore_position, start, end): - """ - Make name with / without underscore to test every + """Make name with / without underscore to test every possibility. """ if underscore_position == "left": diff --git a/tests/tests_unit/test_validation_unit.py b/tests/tests_unit/test_validation_unit.py index 473a8a638..dfd404c3a 100644 --- a/tests/tests_unit/test_validation_unit.py +++ b/tests/tests_unit/test_validation_unit.py @@ -6,8 +6,7 @@ class TestValidationUnit: @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_more_than_one_instance(self, prefix): - """ - Check that any duplicate sub or ses values are caught + """Check that any duplicate sub or ses values are caught in `validate_list_of_names()`. """ error_message = validation.validate_list_of_names( @@ -36,8 +35,7 @@ def test_more_than_one_instance(self, prefix): ], ) def test_name_does_not_begin_with_prefix(self, prefix_and_names): - """ - Check validation that names passed to `validate_list_of_names()` + """Check validation that names passed to `validate_list_of_names()` start with the prefix prefix (sub or ses). """ prefix, names = prefix_and_names @@ -48,8 +46,7 @@ def test_name_does_not_begin_with_prefix(self, prefix_and_names): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_special_characters_in_format_names(self, prefix): - """ - Check `validate_list_of_names()` catches + """Check `validate_list_of_names()` catches spaces in passed names (not all names are bad """ error_messages = validation.validate_list_of_names( @@ -88,11 +85,9 @@ def test_prefix_is_not_an_integer(self, prefix_and_names): def test_formatting_dashes_and_underscore_alternate_incorrectly( self, prefix ): - """ - Check `validate_list_of_names()` catches "-" and "_" that + """Check `validate_list_of_names()` catches "-" and "_" that are not in the correct order. """ - # Test a large range of bad names. Do not use # parametrize so we can use f"{prefix}". # There should always be two validation errors per list. @@ -144,8 +139,7 @@ def test_formatting_dashes_and_underscore_alternate_incorrectly( @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_inconsistent_value_lengths_in_list_of_names(self, prefix): - """ - Ensure a list of sub / ses names that contain inconsistent + """Ensure a list of sub / ses names that contain inconsistent leading zeros (e.g. ["sub-001", "sub-02"]) leads to an error. """ for names in [ @@ -163,8 +157,7 @@ def test_inconsistent_value_lengths_in_list_of_names(self, prefix): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_duplicate_ids_in_list_of_names(self, prefix): - """ - Ensure a list of sub / ses names that contain duplicate sub / ses + """Ensure a list of sub / ses names that contain duplicate sub / ses ids (e.g. ["sub-001", "sub-001_@DATE@"]) leads to an error. """ names = [ @@ -183,12 +176,10 @@ def test_duplicate_ids_in_list_of_names(self, prefix): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_new_name_duplicates_existing(self, prefix): - """ - Test the function `new_name_duplicates_existing()` + """Test the function `new_name_duplicates_existing()` that will throw an error if a sub / ses name matches an existing name (unless it matches exactly). """ - # Check an exactly matching case that should not raise and error new_name = f"{prefix}-002" existing_names = [f"{prefix}-001", f"{prefix}-002", f"{prefix}-003"] @@ -231,8 +222,7 @@ def test_new_name_duplicates_existing(self, prefix): ) def test_tags_autoreplace_in_regexp(self): - """ - Check the validation function `replace_tags_in_regexp()` + """Check the validation function `replace_tags_in_regexp()` correctly replaces tags in a regexp with their regexp equivalent. Test date, time and datetime with some random regexp that @@ -259,7 +249,6 @@ def test_tags_autoreplace_in_regexp(self): ) def test_handle_path(self): - output = validation.handle_path("message", None) assert output == "message" @@ -271,7 +260,6 @@ def test_handle_path(self): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_datetime_iso_format(self, prefix): - # Test dates error_messages = validation.validate_list_of_names( [ From 462726873350f16bc07f027b4cfd5a4f093e79f1 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 10:44:55 +0000 Subject: [PATCH 06/25] enabling ruff-format --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 299fba1d1..8fbbd4031 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: rev: v0.9.9 hooks: - id: ruff - #- id: ruff-format + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: From 2947de1c6d258e53978e08d1ea7a7599525d49be Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 11:23:24 +0000 Subject: [PATCH 07/25] fixing pre-commit errors for test_configs and test_validation_unit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8fbbd4031..299fba1d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: rev: v0.9.9 hooks: - id: ruff - - id: ruff-format + #- id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: From 3d42f4a5a869c06b2bd0c70d3608d20847bcde6a Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 11:27:42 +0000 Subject: [PATCH 08/25] disabling pre-commit --- pyproject.toml | 1 + .../{_test_configs.py => test_configs.py} | 16 +++++++++------- tests/tests_unit/test_validation_unit.py | 8 ++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) rename tests/tests_integration/{_test_configs.py => test_configs.py} (96%) diff --git a/pyproject.toml b/pyproject.toml index 77052bbf8..1e6aff893 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,6 +100,7 @@ fix = true ignore = [ "D203", # one blank line before class "D213", # multi-line-summary second line + "E501", # limit lines to 79 characters ] select = [ "E", # pycodestyle errors diff --git a/tests/tests_integration/_test_configs.py b/tests/tests_integration/test_configs.py similarity index 96% rename from tests/tests_integration/_test_configs.py rename to tests/tests_integration/test_configs.py index 3d617a072..32f59beec 100644 --- a/tests/tests_integration/_test_configs.py +++ b/tests/tests_integration/test_configs.py @@ -10,6 +10,8 @@ class TestConfigs(BaseTest): + """PLACEHOLDER.""" + # Test Errors # ------------------------------------------------------------- @@ -55,7 +57,7 @@ def test_warning_on_startup(self, no_cfg_project): ) @pytest.mark.parametrize("path_type", ["local_path", "central_path"]) def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): - """ "~", "." and "../" syntax is not supported because + """"~", "." and "../" syntax is not supported because it does not work with rclone. Theoretically it could be supported by checking for "." etc. and filling in manually, but it does not seem robust. @@ -90,7 +92,7 @@ def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): def test_no_ssh_options_set_on_make_config_file(self, no_cfg_project): """Check that program will assert if not all ssh options - are set on make_config_file + are set on make_config_file. """ with pytest.raises(ConfigError) as e: no_cfg_project.make_config_file( @@ -211,9 +213,9 @@ def test_existing_projects(self, monkeypatch, tmp_path): function is monkeypatched in order to point to a tmp_path. The tmp_path / "projects" is filled with a mix of project folders - with and without config, and tested against accordingly. The `local_path` - and `central_path` specified in the DataShuttle config are arbitrarily put in - `tmp_path`. + with and without config, and tested against accordingly. The + `local_path` and `central_path` specified in the DataShuttle config are + arbitrarily put in `tmp_path`. """ def patch_get_datashuttle_path(): @@ -249,9 +251,9 @@ def patch_get_datashuttle_path(): (tmp_path / "projects" / "project_3"), ] - # -------------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- # Utils - # -------------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- def check_config_reopen_and_check_config_again(self, project, *kwargs): """Check the config file and project.cfg against provided kwargs, diff --git a/tests/tests_unit/test_validation_unit.py b/tests/tests_unit/test_validation_unit.py index dfd404c3a..21c6437c0 100644 --- a/tests/tests_unit/test_validation_unit.py +++ b/tests/tests_unit/test_validation_unit.py @@ -4,6 +4,8 @@ class TestValidationUnit: + """PLACEHOLDER.""" + @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_more_than_one_instance(self, prefix): """Check that any duplicate sub or ses values are caught @@ -47,7 +49,7 @@ def test_name_does_not_begin_with_prefix(self, prefix_and_names): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_special_characters_in_format_names(self, prefix): """Check `validate_list_of_names()` catches - spaces in passed names (not all names are bad + spaces in passed names (not all names are bad. """ error_messages = validation.validate_list_of_names( [ @@ -69,7 +71,7 @@ def test_special_characters_in_format_names(self, prefix): ], ) def test_prefix_is_not_an_integer(self, prefix_and_names): - """ """ + """PLACEHOLDER.""" prefix, names = prefix_and_names error_messages = validation.validate_list_of_names(names, prefix) @@ -249,6 +251,7 @@ def test_tags_autoreplace_in_regexp(self): ) def test_handle_path(self): + """PLACEHOLDER.""" output = validation.handle_path("message", None) assert output == "message" @@ -260,6 +263,7 @@ def test_handle_path(self): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_datetime_iso_format(self, prefix): + """PLACEHOLDER.""" # Test dates error_messages = validation.validate_list_of_names( [ From ed41aaab9a7146d0987be5ea22732e0b4d5ec13b Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 12:04:50 +0000 Subject: [PATCH 09/25] Editing all test files to comply with linting rules --- pyproject.toml | 53 ++++++++++--------- tests/quick_make_project.py | 4 +- tests/ssh_test_utils.py | 8 +-- tests/test_utils.py | 24 ++++----- tests/tests_integration/base.py | 2 + .../tests_integration/test_create_folders.py | 8 ++- tests/tests_integration/test_datatypes.py | 2 +- .../test_filesystem_transfer.py | 12 +++-- tests/tests_integration/test_formatting.py | 3 ++ .../tests_integration/test_local_only_mode.py | 4 +- tests/tests_integration/test_logging.py | 12 +++-- tests/tests_integration/test_settings.py | 2 + .../test_ssh_file_transfer.py | 9 ++-- tests/tests_integration/test_ssh_setup.py | 10 ++-- .../tests_integration/test_transfer_checks.py | 3 ++ tests/tests_integration/test_validation.py | 24 ++++----- tests/tests_tui/test_local_only_project.py | 2 + tests/tests_tui/test_tui_configs.py | 3 ++ tests/tests_tui/test_tui_create_folders.py | 4 ++ tests/tests_tui/test_tui_datatypes.py | 2 + tests/tests_tui/test_tui_directorytree.py | 3 +- tests/tests_tui/test_tui_get_help.py | 1 + tests/tests_tui/test_tui_logging.py | 2 + tests/tests_tui/test_tui_transfer.py | 12 +++-- .../test_tui_widgets_and_defaults.py | 14 ++--- tests/tests_tui/tui_base.py | 11 ++-- tests/tests_unit/test_unit.py | 7 +-- 27 files changed, 148 insertions(+), 93 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1e6aff893..b7f6e1278 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,21 +97,22 @@ fix = true [tool.ruff.lint] # See https://docs.astral.sh/ruff/rules/ -ignore = [ - "D203", # one blank line before class - "D213", # multi-line-summary second line - "E501", # limit lines to 79 characters -] -select = [ - "E", # pycodestyle errors - "F", # Pyflakes - "UP", # pyupgrade - "I", # isort - "B", # flake8 bugbear - "SIM", # flake8 simplify - "C90", # McCabe complexity - "D", # pydocstyle -] +#ignore = [ +# "D203", # one blank line before class +# "D213", # multi-line-summary second line +# "D401", # first line of docstrings should be in an imperative mood +# "E501", # limit lines to 79 characters +#] +#select = [ +# "E", # pycodestyle errors +# "F", # Pyflakes +# "UP", # pyupgrade +# "I", # isort +# "B", # flake8 bugbear +# "SIM", # flake8 simplify +# "C90", # McCabe complexity +# "D", # pydocstyle +#] per-file-ignores = { "tests/*" = [ "D100", # missing docstring in public module "D205", # missing blank line between summary and description @@ -129,15 +130,19 @@ per-file-ignores = { "tests/*" = [ # Old ruff ruleset + pydocstyle added # Inconsistent with movement repo, but saving this here for # now in case there are good reasons to keep these rules -#ignore = ["E203","E501","E731","C901","W291","W293","E402","E722"] -#select = [ -# "I", # isort -# "E", # pycodestyle errors -# "F", # Pyflakes -# "TC", # flake8-type-checking -# "TID252", # flake8-tidy-imports relative-imports -# "D", # pydocstyle -#] +ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", + "D203", # one blank line before class + "D213", # multi-line-summary second line + "D401", # first line of docstrings should be in an imperative mood +] +select = [ + "I", # isort + "E", # pycodestyle errors + "F", # Pyflakes + "TC", # flake8-type-checking + "TID252", # flake8-tidy-imports relative-imports + "D", # pydocstyle +] [tool.ruff.format] docstring-code-format = true # Also format code in docstrings diff --git a/tests/quick_make_project.py b/tests/quick_make_project.py index 250e66507..cbb837a32 100644 --- a/tests/quick_make_project.py +++ b/tests/quick_make_project.py @@ -1,5 +1,5 @@ -base_path = r"C:/Users/Joe/work/git-repos/forks/yxtuix/joe" - from test_utils import quick_create_project +base_path = r"C:/Users/Joe/work/git-repos/forks/yxtuix/joe" + quick_create_project(base_path) diff --git a/tests/ssh_test_utils.py b/tests/ssh_test_utils.py index bf5bf2c74..a7af1a65c 100644 --- a/tests/ssh_test_utils.py +++ b/tests/ssh_test_utils.py @@ -8,7 +8,7 @@ def setup_project_for_ssh( project, central_path, central_host_id, central_host_username ): """Set up the project configs to use SSH connection - to central + to central. """ project.update_config_file( central_path=central_path, @@ -25,10 +25,10 @@ def setup_project_for_ssh( def setup_mock_input(input_): - """This is very similar to pytest monkeypatch but + """Very similar to pytest monkeypatch but using that was giving me very strange output, monkeypatch.setattr('builtins.input', lambda _: "n") - i.e. pdb went deep into some unrelated code stack + i.e. pdb went deep into some unrelated code stack. """ orig_builtin = copy.deepcopy(builtins.input) builtins.input = lambda _: input_ # type: ignore @@ -36,7 +36,7 @@ def setup_mock_input(input_): def restore_mock_input(orig_builtin): - """orig_builtin: the copied, original builtins.input""" + """orig_builtin: the copied, original builtins.input.""" builtins.input = orig_builtin diff --git a/tests/test_utils.py b/tests/test_utils.py index 46d0b23ea..fce9b15d5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -28,8 +28,7 @@ def setup_project_default_configs( central_path=False, ): """Set up a fresh project to test on - - local_path / central_path: provide the config paths to set + local_path / central_path: provide the config paths to set. """ delete_project_if_it_exists(project_name) @@ -90,7 +89,7 @@ def glob_basenames(search_path, recursive=False, exclude=None): def teardown_project( cwd, project ): # 99% sure these are unnecessary with pytest tmp_path but keep until SSH testing. - """""" + """PLACEHOLDER.""" os.chdir(cwd) delete_all_folders_in_project_path(project, "central") delete_all_folders_in_project_path(project, "local") @@ -104,7 +103,7 @@ def delete_all_folders_in_local_path(project): def delete_all_folders_in_project_path(project, local_or_central): - """""" + """PLACEHOLDER.""" folder = f"{local_or_central}_path" if folder == "central_path" and project.cfg[folder] is None: @@ -119,7 +118,7 @@ def delete_all_folders_in_project_path(project, local_or_central): def delete_project_if_it_exists(project_name): - """""" + """PLACEHOLDER.""" config_path, _ = canonical_folders.get_project_datashuttle_path( project_name ) @@ -158,7 +157,7 @@ def make_test_path(base_path, local_or_central, test_project_name): def create_all_pathtable_files(pathtable): - """ """ + """PLACEHOLDER.""" for i in range(pathtable.shape[0]): filepath = pathtable["base_folder"][i] / pathtable["path"][i] filepath.parents[0].mkdir(parents=True, exist_ok=True) @@ -276,7 +275,7 @@ def check_folder_tree_is_correct( key, folder, ) in canonical_folders.get_datatype_folders().items(): - assert key in folder_used.keys(), ( + assert key in folder_used, ( "Key not found in folder_used. " "Update folder used and hard-coded tests: " "test_custom_folder_names(), test_explicitly_session_list()" @@ -398,7 +397,7 @@ def make_local_folders_with_files_in( def check_configs(project, kwargs, config_path=None): - """""" + """PLACEHOLDER.""" if config_path is None: config_path = project._config_path @@ -433,7 +432,7 @@ def check_project_configs( def check_config_file(config_path, *kwargs): - """""" + """PLACEHOLDER.""" with open(config_path) as config_file: config_yaml = yaml.full_load(config_file) @@ -449,8 +448,7 @@ def check_config_file(config_path, *kwargs): def get_top_level_folder_path( project, local_or_central="local", folder_name="rawdata" ): - """""" - + """PLACEHOLDER.""" assert folder_name in canonical_folders.get_top_level_folders(), ( "folder_name must be canonical e.g. rawdata" ) @@ -496,7 +494,7 @@ def handle_upload_or_download( def get_transfer_func( project, upload_or_download, transfer_method, top_level_folder=None ): - """""" + """PLACEHOLDER.""" if transfer_method == "top_level_folder": assert top_level_folder is not None, "must pass top-level-folder" assert top_level_folder in [None, "rawdata", "derivatives"] @@ -572,7 +570,7 @@ def swap_local_and_central_paths(project, swap_last_folder_only=False): def get_default_sub_sessions_to_test(): - """Canonical subs / sessions for these tests""" + """Canonical subs / sessions for these tests.""" subs = ["sub-001", "sub-002", "sub-003"] sessions = ["ses-001_datetime-20220516T135022", "ses-002", "ses-003"] return subs, sessions diff --git a/tests/tests_integration/base.py b/tests/tests_integration/base.py index c3ea76d20..75c01adf2 100644 --- a/tests/tests_integration/base.py +++ b/tests/tests_integration/base.py @@ -10,6 +10,8 @@ class BaseTest: + """PLACEHOLDER.""" + @pytest.fixture(scope="function") def no_cfg_project(test): """Fixture that creates an empty project. Ignore the warning diff --git a/tests/tests_integration/test_create_folders.py b/tests/tests_integration/test_create_folders.py index 0a69b0695..af9f7e6d2 100644 --- a/tests/tests_integration/test_create_folders.py +++ b/tests/tests_integration/test_create_folders.py @@ -13,6 +13,8 @@ class TestCreateFolders(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_generate_folders_default_ses(self, project): """Make a subject folders with full tree. Don't specify @@ -202,7 +204,7 @@ def test_datatypes_subsection(self, project, files_to_test): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_date_flags_in_session(self, project): """Check that @DATE@ is converted into current date - in generated folder names + in generated folder names. """ date, time_ = self.get_formatted_date_and_time() @@ -224,7 +226,7 @@ def test_date_flags_in_session(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_datetime_flag_in_session(self, project): """Check that @DATETIME@ is converted to datetime - in generated folder names + in generated folder names. """ date, time_ = self.get_formatted_date_and_time() @@ -483,10 +485,12 @@ def test_get_next_sub_and_ses_name_template(self, project): # ---------------------------------------------------------------------------------- def get_formatted_date_and_time(self): + """PLACEHOLDER.""" date = str(datetime.datetime.now().date()) date = date.replace("-", "") time_ = datetime.datetime.now().time().strftime("%Hh%Mm") return date, time_ def broad_datatypes(self): + """PLACEHOLDER.""" return canonical_configs.get_broad_datatypes() diff --git a/tests/tests_integration/test_datatypes.py b/tests/tests_integration/test_datatypes.py index 10720e90b..ff08aa161 100644 --- a/tests/tests_integration/test_datatypes.py +++ b/tests/tests_integration/test_datatypes.py @@ -39,7 +39,7 @@ def test_create_narrow_datatypes(self, project): ) def get_narrow_only_datatypes_used(self, used=True): - """This is similar to test_utils.get_all_broad_folders_used + """Similar to test_utils.get_all_broad_folders_used but for narrow datatypes. """ return { diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py index 98e31a629..d35b0e84e 100644 --- a/tests/tests_integration/test_filesystem_transfer.py +++ b/tests/tests_integration/test_filesystem_transfer.py @@ -13,6 +13,8 @@ class TestFileTransfer(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize( "top_level_folder", canonical_folders.get_top_level_folders() ) @@ -57,6 +59,7 @@ def test_transfer_empty_folder_structure( ) def test_empty_folder_is_not_transferred(self, project): + """PLACEHOLDER.""" project.create_folders("rawdata", "sub-001") project.upload_rawdata() assert not ( @@ -127,7 +130,7 @@ def test_transfer_across_top_level_folders( @pytest.mark.parametrize("upload_or_download", ["upload", "download"]) def test_transfer_all_top_level_folders(self, project, upload_or_download): - """ """ + """PLACEHOLDER.""" subs, sessions = test_utils.get_default_sub_sessions_to_test() for top_level_folder in canonical_folders.get_top_level_folders(): @@ -529,7 +532,7 @@ def test_overwrite_same_size_later_to_earlier( top_level_folder, upload_or_download, ): - """This functions is extremely similar to + """Extremely similar to `test_overwrite_same_size_later_to_earlier()` but it is much easier to understand individually when they are split. @@ -609,6 +612,7 @@ def test_overwrite_different_size_different_times( assert test_utils.read_file(central_file_path) == ["file earlier"] def get_paths_to_a_local_and_central_file(self, project, top_level_folder): + """PLACEHOLDER.""" path_to_test_file = ( Path(top_level_folder) / "sub-001" @@ -625,7 +629,7 @@ def get_paths_to_a_local_and_central_file(self, project, top_level_folder): def setup_overwrite_file_tests( self, upload_or_download, top_level_folder, project ): - """""" + """PLACEHOLDER.""" local_file_path, central_file_path = ( self.get_paths_to_a_local_and_central_file( project, top_level_folder @@ -733,7 +737,7 @@ def test_specific_file_or_folder( assert transferred_files == to_test_against def setup_specific_file_or_folder_files(self, project, top_level_folder): - """ """ + """PLACEHOLDER.""" project.create_folders( top_level_folder, ["sub-001", "sub-002"], diff --git a/tests/tests_integration/test_formatting.py b/tests/tests_integration/test_formatting.py index 0b5db8338..005b7cbf7 100644 --- a/tests/tests_integration/test_formatting.py +++ b/tests/tests_integration/test_formatting.py @@ -6,6 +6,8 @@ class TestFormatting(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize("prefix", ["sub", "ses"]) @pytest.mark.parametrize( "input", [1, {"test": "one"}, 1.0, ["1", "2", ["three"]]] @@ -63,6 +65,7 @@ def test_format_names_prefix(self): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_warning_non_consecutive_numbers(self, project, top_level_folder): + """PLACEHOLDER.""" project.create_folders( top_level_folder, ["sub-01", "sub-02", "sub-04"], diff --git a/tests/tests_integration/test_local_only_mode.py b/tests/tests_integration/test_local_only_mode.py index 0db07a503..bcc42394b 100644 --- a/tests/tests_integration/test_local_only_mode.py +++ b/tests/tests_integration/test_local_only_mode.py @@ -13,9 +13,11 @@ class TestLocalOnlyProject(BaseTest): + """PLACEHOLDER.""" + def test_bad_setup(self, tmp_path): """Test setup without providing both central_path and connection - method (distinguishing a full vs local-only project) + method (distinguishing a full vs local-only project). """ local_path = tmp_path / "test_local" diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index fa193c5b7..ecf68b49e 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -18,6 +18,8 @@ class TestLogging: + """PLACEHOLDER.""" + @pytest.fixture(scope="function") def teardown_logger(self): """Ensure the logger is deleted at the end of each test.""" @@ -126,7 +128,7 @@ def project(self, tmp_path, clean_project_name, request): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_log_filename(self, project): """Check the log filename is formatted correctly, for - `update_config_file`, an arbitrary command + `update_config_file`, an arbitrary command. """ project.update_config_file(central_host_id="test_id") @@ -140,7 +142,7 @@ def test_log_filename(self, project): assert re.search(regex, log_filename) is not None def test_logs_make_config_file(self, clean_project_name, tmp_path): - """""" + """PLACEHOLDER.""" project = DataShuttle(clean_project_name) project.make_config_file( @@ -161,6 +163,7 @@ def test_logs_make_config_file(self, clean_project_name, tmp_path): assert "Update successful. New config file:" in log def test_logs_update_config_file(self, project): + """PLACEHOLDER.""" project.update_config_file(central_host_id="test_id") log = test_utils.read_log_file(project.cfg.logging_path) @@ -175,6 +178,7 @@ def test_logs_update_config_file(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_create_folders(self, project): + """PLACEHOLDER.""" subs = ["sub-111", f"sub-002{tags('to')}004"] ses = ["ses-123", "ses-101"] @@ -402,7 +406,7 @@ def test_clear_logging_path(self, clean_project_name, tmp_path): # ---------------------------------------------------------------------------------- def test_logs_check_update_config_error(self, project): - """""" + """PLACEHOLDER.""" with pytest.raises(ConfigError): project.update_config_file( connection_method="ssh", central_host_username=None @@ -421,7 +425,7 @@ def test_logs_check_update_config_error(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_logs_bad_create_folders_error(self, project): - """""" + """PLACEHOLDER.""" project.create_folders("rawdata", "sub-001", datatype="all") test_utils.delete_log_files(project.cfg.logging_path) diff --git a/tests/tests_integration/test_settings.py b/tests/tests_integration/test_settings.py index 5d5834c92..27f9eb7b6 100644 --- a/tests/tests_integration/test_settings.py +++ b/tests/tests_integration/test_settings.py @@ -11,6 +11,8 @@ class TestPersistentSettings(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_name_templates(self, project): """Test the 'name_templates' option that is stored in persistent diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py index 4b4babce4..0f6f46bcd 100644 --- a/tests/tests_integration/test_ssh_file_transfer.py +++ b/tests/tests_integration/test_ssh_file_transfer.py @@ -1,5 +1,3 @@ -""" """ - import copy import glob import shutil @@ -15,6 +13,8 @@ class TestFileTransfer: + """PLACEHOLDER.""" + @pytest.fixture( scope="class", params=[ # Set running SSH or local filesystem (see docstring). @@ -115,6 +115,7 @@ def pathtable_and_project(self, request, tmpdir_factory): # ------------------------------------------------------------------------- def central_from_local(self, path_): + """PLACEHOLDER.""" return Path(str(copy.copy(path_)).replace("local", "central")) # ------------------------------------------------------------------------- @@ -168,7 +169,7 @@ def test_all_data_transfer_options( """Parse the arguments to filter the pathtable, getting the files expected to be transferred passed on the arguments Note files in sub/ses/datatype folders must be handled - separately to those in non-sub, non-ses, non-datatype folders + separately to those in non-sub, non-ses, non-datatype folders. see test_utils.swap_local_and_central_paths() for the logic on setting up and swapping local / central paths for @@ -245,7 +246,7 @@ def test_all_data_transfer_options( def query_table(self, pathtable, arguments): """Search the table for arguments, return empty - if arguments empty + if arguments empty. """ if any(arguments): folders = pathtable.query(" | ".join(arguments)) diff --git a/tests/tests_integration/test_ssh_setup.py b/tests/tests_integration/test_ssh_setup.py index f26b1d921..5a511abc2 100644 --- a/tests/tests_integration/test_ssh_setup.py +++ b/tests/tests_integration/test_ssh_setup.py @@ -1,6 +1,6 @@ """SSH configs are set in conftest.py . The password should be stored in a file called test_ssh_password.txt located -in the same folder as test_ssh.py +in the same folder as test_ssh.py. """ import pytest @@ -13,10 +13,12 @@ @pytest.mark.skipif(ssh_config.TEST_SSH is False, reason="TEST_SSH is false") class TestSSH: + """PLACEHOLDER.""" + @pytest.fixture(scope="function") def project(test, tmp_path): """Make a project as per usual, but now add - in test ssh configurations + in test ssh configurations. """ tmp_path = tmp_path / "test with space" @@ -45,7 +47,7 @@ def test_verify_ssh_central_host_do_not_accept( ): """Use the main function to test this. Test the sub-function when accepting, because this main function will also - call setup ssh key pairs which we don't want to do yet + call setup ssh key pairs which we don't want to do yet. This should only accept for "y" so try some random strings including "n" and check they all do not make the connection. @@ -84,7 +86,7 @@ def test_verify_ssh_central_host_accept(self, capsys, project): def test_generate_and_write_ssh_key(self, project): """Check ssh key for passwordless connection is written - to file + to file. """ path_to_save = project.cfg["local_path"] / "test" ssh.generate_and_write_ssh_key(path_to_save) diff --git a/tests/tests_integration/test_transfer_checks.py b/tests/tests_integration/test_transfer_checks.py index 7a6f631f3..e443cfae8 100644 --- a/tests/tests_integration/test_transfer_checks.py +++ b/tests/tests_integration/test_transfer_checks.py @@ -10,6 +10,8 @@ class TestTransferChecks(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize( "top_level_folders", [["rawdata", "derivatives"], ["rawdata"], ["derivatives"]], @@ -90,6 +92,7 @@ def test_rclone_check(self, project, top_level_folders): assert path_ not in results_paths def get_folder_structure(self, top_level_folder): + """PLACEHOLDER.""" # fmt: off folder_structure = [ [f"{top_level_folder}/sub-001/ses-001/ephys/local_only_1.txt", "local_only"], diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index 71fd7d658..4827152df 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -14,6 +14,8 @@ class TestValidation(BaseTest): + """PLACEHOLDER.""" + @pytest.mark.parametrize( "sub_name", ["sub-001", "sub-999_@DATE@", "sub-001_random-tag_another-tag"], @@ -33,7 +35,7 @@ class TestValidation(BaseTest): def test_warn_on_inconsistent_sub_value_lengths( self, project, sub_name, bad_sub_name ): - """This test checks that inconsistent sub value lengths are properly + """Checks that inconsistent sub value lengths are properly detected across the project. This is performed with an assortment of possible filenames and leading zero conflicts. @@ -92,7 +94,7 @@ def test_warn_on_inconsistent_sub_value_lengths( def test_warn_on_inconsistent_ses_value_lengths( self, project, ses_name, bad_ses_name ): - """This function is exactly the same as + """Exactly the same as `test_warn_on_inconsistent_sub_value_lengths()` but operates at the session level. This is extreme code duplication, but factoring the main logic out got very messy and hard to follow. @@ -155,7 +157,7 @@ def test_warn_on_inconsistent_sub_and_ses_value_lengths(self, project): def check_inconsistent_sub_or_ses_value_length_warning( self, project, warn_idx=0, include_central=True ): - """""" + """PLACEHOLDER.""" with pytest.warns(UserWarning) as w: project.validate_project( "rawdata", display_mode="warn", include_central=include_central @@ -285,7 +287,7 @@ def test_duplicate_ses_across_subjects(self, project): @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_invalid_sub_and_ses_name(self, project): - """This is a slightly weird case, the name is successfully + """Slightly weird case, the name is successfully prefixed as 'sub-sub_100` but when the value if `sub-` is extracted, it is also "sub" and so an error is raised. """ @@ -380,7 +382,7 @@ def test_validate_project(self, project): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_validate_project_returned_list(self, project, prefix): - """ """ + """PLACEHOLDER.""" bad_names = [ f"{prefix}-001", f"{prefix}-001_@DATE@", @@ -409,7 +411,7 @@ def test_validate_project_returned_list(self, project, prefix): assert "VALUE_LENGTH" in concat_error def test_output_paths_are_valid(self, project): - """ """ + """PLACEHOLDER.""" sub_name = "sub-001x" ses_name = "ses-001x" project.create_folders( @@ -747,7 +749,7 @@ def test_tags_in_name_templates_pass_validation(self, project): assert "TEMPLATE: The name: ses-001_datex-20241212" in str(e.value) def test_name_templates_validate_project(self, project): - """TODO:""" + """TODO.""" name_templates = { "on": True, "sub": "sub-\d\d_id-\d.?", @@ -782,7 +784,7 @@ def test_name_templates_validate_project(self, project): # ---------------------------------------------------------------------------------- def test_quick_validation(self, mocker, project): - """ """ + """PLACEHOLDER.""" project.create_folders("rawdata", "sub-1") os.makedirs(project.cfg["local_path"] / "rawdata" / "sub-02") project.create_folders("derivatives", "sub-1") @@ -839,7 +841,7 @@ def test_quick_validation_top_level_folder(self, project): @pytest.mark.parametrize("top_level_folder", ["rawdata", "derivatives"]) def test_strict_mode_validation(self, project, top_level_folder): - """ """ + """PLACEHOLDER.""" project.create_folders( top_level_folder, ["sub-001", "sub-002"], @@ -908,9 +910,7 @@ def test_strict_mode_validation(self, project, top_level_folder): def test_check_high_level_project_structure( self, project, top_level_folder ): - """Check that local and central project names are properly formatted - and that - """ + """Check that local and central project names are properly formatted.""" with pytest.warns(UserWarning) as w: project.validate_project( top_level_folder, "warn", include_central=True diff --git a/tests/tests_tui/test_local_only_project.py b/tests/tests_tui/test_local_only_project.py index 8a06ccfa7..8802aee8c 100644 --- a/tests/tests_tui/test_local_only_project.py +++ b/tests/tests_tui/test_local_only_project.py @@ -5,6 +5,8 @@ class TestTuiLocalOnlyProject(TuiBase): + """PLACEHOLDER.""" + @pytest.mark.asyncio async def test_local_only_make_project( self, diff --git a/tests/tests_tui/test_tui_configs.py b/tests/tests_tui/test_tui_configs.py index a9f4d8fe0..1fa7802ad 100644 --- a/tests/tests_tui/test_tui_configs.py +++ b/tests/tests_tui/test_tui_configs.py @@ -14,6 +14,8 @@ class TestTuiConfigs(TuiBase): + """PLACEHOLDER.""" + # ------------------------------------------------------------------------- # Test New Project Configs # ------------------------------------------------------------------------- @@ -337,6 +339,7 @@ async def test_configs_select_path(self, monkeypatch): @pytest.mark.asyncio async def test_bad_configs_screen_input(self, empty_project_paths): + """PLACEHOLDER.""" app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: # Select a new project, check NewProjectScreen is displayed correctly. diff --git a/tests/tests_tui/test_tui_create_folders.py b/tests/tests_tui/test_tui_create_folders.py index 8fbc32a2a..736f4c63d 100644 --- a/tests/tests_tui/test_tui_create_folders.py +++ b/tests/tests_tui/test_tui_create_folders.py @@ -13,6 +13,8 @@ class TestTuiCreateFolders(TuiBase): + """PLACEHOLDER.""" + # ------------------------------------------------------------------------- # General test Create Folders # ------------------------------------------------------------------------- @@ -600,6 +602,7 @@ async def test_create_folders_settings_top_level_folder( async def iterate_and_check_all_datatype_folders( self, pilot, subs, sessions ): + """PLACEHOLDER.""" project = pilot.app.screen.interface.project folder_used = test_utils.get_all_broad_folders_used(value=False) @@ -617,6 +620,7 @@ async def iterate_and_check_all_datatype_folders( async def create_folders_and_check_output( self, pilot, project, subs, sessions, folder_used ): + """PLACEHOLDER.""" await self.scroll_to_click_pause( pilot, "#create_folders_create_folders_button", diff --git a/tests/tests_tui/test_tui_datatypes.py b/tests/tests_tui/test_tui_datatypes.py index 48bcbf27c..c3e34bd01 100644 --- a/tests/tests_tui/test_tui_datatypes.py +++ b/tests/tests_tui/test_tui_datatypes.py @@ -13,6 +13,7 @@ class TestDatatypesTUI(TuiBase): async def test_select_displayed_datatypes_create( self, setup_project_paths ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() @@ -127,6 +128,7 @@ async def test_select_displayed_datatypes_create( async def test_select_displayed_datatypes_transfer( self, setup_project_paths, mocker ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() diff --git a/tests/tests_tui/test_tui_directorytree.py b/tests/tests_tui/test_tui_directorytree.py index 11562e085..4d6e6d872 100644 --- a/tests/tests_tui/test_tui_directorytree.py +++ b/tests/tests_tui/test_tui_directorytree.py @@ -15,7 +15,7 @@ class TestTuiCreateDirectoryTree(TuiBase): """Test the `Create` tab directory tree. - `Transfer` + `Transfer`. """ @pytest.mark.asyncio @@ -151,6 +151,7 @@ async def test_create_folders_directorytree_clipboard( async def test_failed_pyperclip_copy( self, setup_project_paths, monkeypatch ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() diff --git a/tests/tests_tui/test_tui_get_help.py b/tests/tests_tui/test_tui_get_help.py index aa8986b8c..192487557 100644 --- a/tests/tests_tui/test_tui_get_help.py +++ b/tests/tests_tui/test_tui_get_help.py @@ -11,6 +11,7 @@ class TestTuiSettings(TuiBase): @pytest.mark.asyncio async def test_get_help(self, empty_project_paths): + """PLACEHOLDER.""" app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: await self.scroll_to_click_pause( diff --git a/tests/tests_tui/test_tui_logging.py b/tests/tests_tui/test_tui_logging.py index 6a046c3e8..21e288a12 100644 --- a/tests/tests_tui/test_tui_logging.py +++ b/tests/tests_tui/test_tui_logging.py @@ -7,6 +7,8 @@ class TestTuiLogging(TuiBase): + """PLACEHOLDER.""" + @pytest.mark.asyncio async def test_logging(self, setup_project_paths): """Test logging by running some commands, checking they diff --git a/tests/tests_tui/test_tui_transfer.py b/tests/tests_tui/test_tui_transfer.py index 1fd3fe60a..c273884b9 100644 --- a/tests/tests_tui/test_tui_transfer.py +++ b/tests/tests_tui/test_tui_transfer.py @@ -17,6 +17,7 @@ class TestTuiTransfer(TuiBase): async def test_transfer_entire_project( self, setup_project_paths, upload_or_download ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -67,7 +68,7 @@ async def check_persistent_settings(self, pilot): ) async def set_overwrite_checkbox(self, pilot, overwrite_setting): - """""" + """PLACEHOLDER.""" all_positions = {"never": None, "always": 5, "if_source_newer": 6} position = all_positions[overwrite_setting] @@ -77,6 +78,7 @@ async def set_overwrite_checkbox(self, pilot, overwrite_setting): ) async def set_transfer_tab_dry_run_checkbox(self, pilot, dry_run_setting): + """PLACEHOLDER.""" if ( pilot.app.screen.query_one("#transfer_tab_dry_run_checkbox") is not dry_run_setting @@ -107,7 +109,7 @@ async def set_and_check_persistent_settings( async def test_transfer_top_level_folder( self, setup_project_paths, top_level_folder, upload_or_download ): - """""" + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -153,6 +155,7 @@ async def test_transfer_top_level_folder( async def test_transfer_custom( self, setup_project_paths, top_level_folder, upload_or_download ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() subs, sessions = test_utils.get_default_sub_sessions_to_test() @@ -222,6 +225,7 @@ async def test_transfer_custom( async def switch_top_level_folder_select( self, pilot, id, top_level_folder ): + """PLACEHOLDER.""" if top_level_folder == "rawdata": assert pilot.app.screen.query_one(id).value == "rawdata" else: @@ -229,7 +233,7 @@ async def switch_top_level_folder_select( assert pilot.app.screen.query_one(id).value == "derivatives" async def run_transfer(self, pilot, upload_or_download): - """""" + """PLACEHOLDER.""" # Check assumed default is correct on the transfer switch assert pilot.app.screen.query_one("#transfer_switch").value is False @@ -246,7 +250,7 @@ def setup_project_for_data_transfer( top_level_folder_list, upload_or_download, ): - """""" + """PLACEHOLDER.""" for top_level_folder in top_level_folder_list: test_utils.make_and_check_local_project_folders( project, diff --git a/tests/tests_tui/test_tui_widgets_and_defaults.py b/tests/tests_tui/test_tui_widgets_and_defaults.py index 897d066ce..781e7473c 100644 --- a/tests/tests_tui/test_tui_widgets_and_defaults.py +++ b/tests/tests_tui/test_tui_widgets_and_defaults.py @@ -14,7 +14,7 @@ class TestTuiWidgets(TuiBase): - """This class performs fundamental checks on the default display + """Performs fundamental checks on the default display of widgets and that changing widgets properly change underlying configs. This does not perform any functional tests e.g. creation of configs of new files. @@ -208,7 +208,7 @@ async def test_new_project_configs(self, empty_project_paths): async def check_new_project_ssh_widgets( self, configs_content, ssh_on, save_pressed=False ): - """""" + """PLACEHOLDER.""" assert configs_content.query_one( "#configs_setup_ssh_connection_button" ).visible is ( @@ -873,7 +873,7 @@ async def check_top_folder_select( expected_val, move_to_position: Union[bool, int] = False, ): - """If move to position is not False, must be int specifying position""" + """If move to position is not False, must be int specifying position.""" if move_to_position: await self.move_select_to_position(pilot, id, move_to_position) @@ -992,7 +992,7 @@ async def test_all_checkboxes(self, setup_project_paths): await pilot.pause() def check_datatype_checkboxes(self, pilot, tab, expected_on): - """""" + """PLACEHOLDER.""" assert tab in ["create", "transfer"] if tab == "create": id = "#create_folders_datatype_checkboxes" @@ -1016,6 +1016,7 @@ def check_datatype_checkboxes(self, pilot, tab, expected_on): @pytest.mark.asyncio async def test_all_transfer_widgets(self, setup_project_paths): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() @@ -1201,7 +1202,7 @@ async def test_all_transfer_widgets(self, setup_project_paths): @pytest.mark.asyncio async def test_overwrite_existing_files(self, setup_project_paths): - """ """ + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() app = TuiApp() @@ -1293,6 +1294,7 @@ async def test_dry_run(self, setup_project_paths): # combine. def check_dry_run(self, pilot, project_name, value): + """PLACEHOLDER.""" assert ( pilot.app.screen.query_one("#transfer_tab_dry_run_checkbox").value == value @@ -1307,7 +1309,7 @@ def check_dry_run(self, pilot, project_name, value): def check_overwrite_existing_files_configs( self, pilot, project_name, value ): - """""" + """PLACEHOLDER.""" assert ( pilot.app.screen.query_one("#transfer_tab_overwrite_select").value == value diff --git a/tests/tests_tui/tui_base.py b/tests/tests_tui/tui_base.py index fc20cbbc2..75b872099 100644 --- a/tests/tests_tui/tui_base.py +++ b/tests/tests_tui/tui_base.py @@ -130,14 +130,14 @@ async def reload_tree_nodes(self, pilot, id, num_nodes): await pilot.pause() async def hover_and_press_tree(self, pilot, id, hover_line, press_string): - """Hover over a directorytree at a node-line and press a specific string""" + """Hover over a directorytree at a node-line and press a specific string.""" await pilot.pause() pilot.app.screen.query_one(id).hover_line = hover_line await pilot.pause() await self.press_tree(pilot, id, press_string) async def press_tree(self, pilot, id, press_string): - """Click on a tree to give it focus and press buttons""" + """Click on a tree to give it focus and press buttons.""" await self.scroll_to_click_pause(pilot, id) await pilot.press(press_string) await pilot.pause() @@ -177,16 +177,18 @@ async def check_and_click_onto_existing_project(self, pilot, project_name): ) async def change_checkbox(self, pilot, id): + """PLACEHOLDER.""" pilot.app.screen.query_one(id).toggle() await pilot.pause() async def switch_tab(self, pilot, tab): + """PLACEHOLDER.""" assert tab in ["create", "transfer", "configs", "logging"] content_tab = ContentTab.add_prefix(f"tabscreen_{tab}_tab") await self.scroll_to_click_pause(pilot, f"Tab#{content_tab}") async def turn_off_all_datatype_checkboxes(self, pilot, tab="create"): - """Make sure all checkboxes are off to start""" + """Make sure all checkboxes are off to start.""" assert tab in ["create", "transfer"] checkbox_names = canonical_configs.get_broad_datatypes() @@ -219,7 +221,7 @@ async def exit_to_main_menu_and_reeneter_project_manager( await self.check_and_click_onto_existing_project(pilot, project_name) async def close_messagebox(self, pilot): - """Close the modal_dialogs.Messagebox""" + """Close the modal_dialogs.Messagebox.""" pilot.app.screen.on_button_pressed() await pilot.pause() @@ -233,6 +235,7 @@ async def move_select_to_position(self, pilot, id, position): await pilot.pause() async def click_and_await_transfer(self, pilot): + """PLACEHOLDER.""" await self.scroll_to_click_pause(pilot, "#transfer_transfer_button") await self.scroll_to_click_pause(pilot, "#confirm_ok_button") diff --git a/tests/tests_unit/test_unit.py b/tests/tests_unit/test_unit.py index 3c55b526a..775bc089a 100644 --- a/tests/tests_unit/test_unit.py +++ b/tests/tests_unit/test_unit.py @@ -16,7 +16,7 @@ class TestUnit: "key", [tags("date"), tags("time"), tags("datetime")] ) def test_datetime_string_replacement(self, key, underscore_position): - """Test the function that replaces @DATE, @TIME@ or @DATETIME@ + r"""Test the function that replaces @DATE, @TIME@ or @DATETIME@ keywords with the date / time / datetime. Also, it will pre/append underscores to the tags if they are not already there (e.g if user input "sub-001@DATE"). @@ -45,7 +45,7 @@ def test_datetime_string_replacement(self, key, underscore_position): @pytest.mark.parametrize("prefix", ["sub", "ses"]) def test_process_to_keyword_in_sub_input(self, prefix): - """ """ + """PLACEHOLDER.""" results = formatting.update_names_with_range_to_flag( [f"{prefix}-001", f"{prefix}-01{tags('to')}123"], prefix ) @@ -94,6 +94,7 @@ def test_process_to_keyword_in_sub_input(self, prefix): def test_process_to_keyword_bad_input_raises_error( self, prefix, bad_input ): + """PLACEHOLDER.""" bad_input = bad_input.replace("prefix", prefix) with pytest.raises(ValueError) as e: @@ -142,7 +143,7 @@ def test_get_max_sub_or_ses_num_and_value_length_empty( ): """When the list of sub or ses names is empty, the returned max number should be zero and the `default_num_value_digits` - be set to the passed default + be set to the passed default. """ ( max_value, From b33d756d5ac2548e321cb1990d72bcf269faf4e6 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 12:15:12 +0000 Subject: [PATCH 10/25] ignoring rule D100 for now --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b7f6e1278..68a7a2766 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,6 +131,7 @@ per-file-ignores = { "tests/*" = [ # Inconsistent with movement repo, but saving this here for # now in case there are good reasons to keep these rules ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", + "D100", # missing docstring in public module (not enforced FOR NOW) "D203", # one blank line before class "D213", # multi-line-summary second line "D401", # first line of docstrings should be in an imperative mood From 98bb643664dbdd5c3810a0578c5cccbc299cfc0a Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 12:19:44 +0000 Subject: [PATCH 11/25] fixed datashuttle_functions.py --- datashuttle/datashuttle_functions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 250fd8bd6..5ab642570 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -28,7 +28,9 @@ def quick_validate_project( display_mode: DisplayMode = "warn", name_templates: Optional[Dict] = None, ) -> List[str]: - """Perform validation on the project. This checks the subject + """Perform validation on the project. + + This checks the subject and session level folders to ensure there are not NeuroBlueprint formatting issues. @@ -90,7 +92,9 @@ def quick_validate_project( def _format_top_level_folder( top_level_folder: TopLevelFolder | None, ) -> List[TopLevelFolder]: - """Take a `top_level_folder` ("rawdata" or "derivatives" str) and + """Format the top level folder. + + Take a `top_level_folder` ("rawdata" or "derivatives" str) and convert to list, if `None`, convert it to a list of both possible top-level folders. """ From 561903a9911c224b90953e35fdd1949170379887 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 12:32:54 +0000 Subject: [PATCH 12/25] fixing datashuttle_class --- datashuttle/datashuttle_class.py | 28 ++++++++++++++++++++++------ pyproject.toml | 1 + 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 6cc044625..5adec00b9 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -105,6 +105,7 @@ class DataShuttle: """ def __init__(self, project_name: str, print_startup_message: bool = True): + """PLACEHOLDER.""" self._error_on_base_project_name(project_name) self.project_name = project_name ( @@ -752,7 +753,7 @@ def _transfer_top_level_folder( init_log: bool = True, ): """Core function to upload / download files within a - particular top-level-folder. e.g. `upload_rawdata().` + particular top-level-folder. e.g. `upload_rawdata()`. """ if init_log: self._start_log( @@ -824,7 +825,7 @@ def _transfer_specific_file_or_folder( def setup_ssh_connection(self) -> None: """Setup a connection to the central server using SSH. Assumes the central_host_id and central_host_username - are set in configs (see make_config_file() and update_config_file()) + are set in configs (see make_config_file() and update_config_file()). First, the server key will be displayed, requiring verification of the server ID. This will store the @@ -971,7 +972,7 @@ def make_config_file( ds_logger.close_log_filehandler() def update_config_file(self, **kwargs) -> None: - """ """ + """PLACEHOLDER.""" if not self.cfg: utils.log_and_raise_error( "Must have a config loaded before updating configs.", @@ -1022,6 +1023,7 @@ def get_config_path(self) -> Path: @check_configs_set def get_configs(self) -> Configs: + """PLACEHOLDER.""" return self.cfg @check_configs_set @@ -1049,6 +1051,9 @@ def get_next_sub( Parameters ---------- + top_level_folder + "rawdata" or "derivatives" + return_with_prefix If `True`, return with the "sub-" prefix. @@ -1247,7 +1252,7 @@ def validate_project( def check_name_formatting(names: Union[str, list], prefix: Prefix) -> None: """Pass list of names to check how these will be auto-formatted, for example as when passed to create_folders() or upload_custom() - or download() + or download(). Useful for checking tags e.g. @TO@, @DATE@, @DATETIME@, @DATE@. This method will print the formatted list of names, @@ -1292,6 +1297,14 @@ def _transfer_entire_project( upload_or_download direction to transfer the data, either "upload" (from local to central) or "download" (from central to local). + + overwrite_existing_files + determines whether or not to overwrite existing files + + dry_run + perform a dry-run of transfer. This will output as if file + transfer was taking place, but no files will be moved. Useful + to check which files will be moved on data transfer. """ for top_level_folder in canonical_folders.get_top_level_folders(): @@ -1328,6 +1341,9 @@ def _start_log( store_in_temp_folder if `False`, existing logging path will be used (local project .datashuttle). + + verbose + print warnings and error messages. """ if local_vars is None: @@ -1372,7 +1388,7 @@ def _move_logs_from_temp_folder(self) -> None: ) def _clear_temp_log_path(self) -> None: - """""" + """PLACEHOLDER.""" log_files = glob.glob(str(self._temp_log_path / "*.log")) for file in log_files: os.remove(file) @@ -1461,7 +1477,7 @@ def _init_persistent_settings(self) -> None: self._save_persistent_settings(settings) def _save_persistent_settings(self, settings: Dict) -> None: - """Save the settings dict to file as .yaml""" + """Save the settings dict to file as ".yaml".""" with open(self._persistent_settings_path, "w") as settings_file: yaml.dump(settings, settings_file, sort_keys=False) diff --git a/pyproject.toml b/pyproject.toml index 68a7a2766..63d9cd3c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,6 +133,7 @@ per-file-ignores = { "tests/*" = [ ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", "D100", # missing docstring in public module (not enforced FOR NOW) "D203", # one blank line before class + "D205", # 1 blank line required between summary line and description "D213", # multi-line-summary second line "D401", # first line of docstrings should be in an imperative mood ] From 31844315e4e6500d615159bac5b26de87c162009 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:09:41 +0000 Subject: [PATCH 13/25] fixing utils --- datashuttle/configs/config_class.py | 6 ++ datashuttle/utils/custom_exceptions.py | 4 ++ datashuttle/utils/data_transfer.py | 95 +++++++++++++------------- datashuttle/utils/decorators.py | 2 +- datashuttle/utils/ds_logger.py | 3 + datashuttle/utils/folder_class.py | 1 + datashuttle/utils/folders.py | 34 +++++++-- datashuttle/utils/formatting.py | 9 ++- datashuttle/utils/getters.py | 9 ++- datashuttle/utils/rclone.py | 12 ++-- datashuttle/utils/ssh.py | 11 ++- datashuttle/utils/utils.py | 11 +-- datashuttle/utils/validation.py | 15 +++- 13 files changed, 143 insertions(+), 69 deletions(-) diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index f1282d935..b62427b7b 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -144,6 +144,9 @@ def build_project_path( a list (or string for 1) of folder names to be joined into a path. If file included, must be last entry (with ext). + + top_level_folder + either "rawdata" or "derivatives" """ if isinstance(sub_folders, list): @@ -173,6 +176,9 @@ def get_base_folder( ---------- base base path, "local", "central" or "datashuttle" + + top_level_folder + either "rawdata" or "derivatives" """ if base == "local": diff --git a/datashuttle/utils/custom_exceptions.py b/datashuttle/utils/custom_exceptions.py index 1d3d8e86a..3fe7e3368 100644 --- a/datashuttle/utils/custom_exceptions.py +++ b/datashuttle/utils/custom_exceptions.py @@ -1,6 +1,10 @@ class ConfigError(Exception): + """PLACEHOLDER.""" + pass class NeuroBlueprintError(Exception): + """PLACEHOLDER.""" + pass diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index 10562f300..bfd092bc0 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -19,44 +19,6 @@ class TransferData: The properties on this class are to be read during generation of transfer lists and should never be changed during the lifetime of the class. - - Parameters - ---------- - cfg - datashuttle configs UserDict. - - upload_or_download - Direction to perform the transfer. - - top_level_folder - The top-level folder structure where data is organized. - - sub_names - List of subject names or single subject to transfer. This - can include transfer keywords (e.g. "all_non_sub"). - - ses_names - List of sessions or single session to transfer, for each - subject. May include session-level transfer keywords. - - datatype - List of datatypes to transfer, for the sessions / subjects - specified. Can include datatype-level tranfser keywords. - - overwrite_existing_files - If "never" files on target will never be overwritten by source. - If "always" files on target will be overwritten by source if - there is any difference in date or size. - If "if_source_newer" files on target will only be overwritten - by files on source with newer creation / modification datetime. - - dry_run - If `True`, transfer will not actually occur but will be logged - as if it did (to see what would happen for a transfer). - - log - if `True`, log and print the transfer output. - """ def __init__( @@ -71,6 +33,43 @@ def __init__( dry_run: bool, log: bool, ): + """Parameters + ---------- + cfg + datashuttle configs UserDict. + + upload_or_download + Direction to perform the transfer. + + top_level_folder + The top-level folder structure where data is organized. + + sub_names + List of subject names or single subject to transfer. This + can include transfer keywords (e.g. "all_non_sub"). + + ses_names + List of sessions or single session to transfer, for each + subject. May include session-level transfer keywords. + + datatype + List of datatypes to transfer, for the sessions / subjects + specified. Can include datatype-level tranfser keywords. + + overwrite_existing_files + If "never" files on target will never be overwritten by source. + If "always" files on target will be overwritten by source if + there is any difference in date or size. + If "if_source_newer" files on target will only be overwritten + by files on source with newer creation / modification datetime. + + dry_run + If `True`, transfer will not actually occur but will be logged + as if it did (to see what would happen for a transfer). + + log + if `True`, log and print the transfer output. + """ self.__cfg = cfg self.__upload_or_download = upload_or_download self.__top_level_folder = top_level_folder @@ -112,12 +111,17 @@ def __init__( def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: """Build a list of every file to transfer based on the user-passed - arguments. This cycles through every subject, session and datatype + arguments. + + This cycles through every subject, session and datatype and adds the outputs to three lists: - `sub_ses_dtype_include` - files within datatype folders - `extra_folder_names` - folders that do not fall within datatype folders - `extra_file_names` - files that do not fall within datatype folders + `sub_ses_dtype_include` + files within datatype folders + `extra_folder_names` + folders that do not fall within datatype folders + `extra_file_names` + files that do not fall within datatype folders Returns ------- @@ -346,6 +350,7 @@ def update_list_with_dtype_paths( # ------------------------------------------------------------------------- def to_list(self, names: Union[str, List[str]]) -> List[str]: + """PLACEHOLDER.""" if isinstance(names, str): names = [names] return names @@ -425,9 +430,7 @@ def get_processed_names( will be searched to determine what files exist to transfer, and the sub / ses names list generated. - Parameters - ---------- - see transfer_sub_ses_data() + see transfer_sub_ses_data() for list of parameters. """ prefix: Prefix @@ -469,7 +472,7 @@ def get_processed_names( def transfer_non_datatype(self, datatype_checked: List[str]) -> bool: """Convenience function, bool if all non-datatype folders - are to be transferred + are to be transferred. """ return any( [name in ["all_non_datatype", "all"] for name in datatype_checked] diff --git a/datashuttle/utils/decorators.py b/datashuttle/utils/decorators.py index 99dcde8ed..19f3794b2 100644 --- a/datashuttle/utils/decorators.py +++ b/datashuttle/utils/decorators.py @@ -6,7 +6,7 @@ def requires_ssh_configs(func): """Decorator to check file is loaded. Used on Mainwindow class - methods only as first arg is assumed to be self (containing cfgs) + methods only as first arg is assumed to be self (containing cfgs). """ @wraps(func) diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index 353fc0886..4269b391d 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -18,14 +18,17 @@ def get_logger_name(): + """PLACEHOLDER.""" return "datashuttle" def get_logger(): + """PLACEHOLDER.""" return logging.getLogger(get_logger_name()) def logging_is_active(): + """PLACEHOLDER.""" logger_exists = get_logger_name() in logging.root.manager.loggerDict if logger_exists and get_logger().handlers != []: return True diff --git a/datashuttle/utils/folder_class.py b/datashuttle/utils/folder_class.py index 8b85856de..33bc16b74 100644 --- a/datashuttle/utils/folder_class.py +++ b/datashuttle/utils/folder_class.py @@ -10,5 +10,6 @@ def __init__( name: str, level: str, ): + """PLACEHOLDER.""" self.name = name self.level = level diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index f7d86c4b7..3a3dd3372 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -47,6 +47,12 @@ def create_folder_trees( Parameters ---------- + cfg + datashuttle config UserDict + + top_level_folder + either "rawdata" or "derivatives" + sub_names, ses_names, datatype see create_folders() @@ -268,10 +274,8 @@ def items_from_datatype_input( directly from user input, or by searching what is available if "all" is passed. - Parameters - ---------- - see _transfer_datatype() for parameters. - + see _transfer_datatype() for full + parameters list. """ base_folder = cfg.get_base_folder(local_or_central, top_level_folder) @@ -385,6 +389,9 @@ def search_for_wildcards( Parameters ---------- + cfg + datashuttle configs + project initialised datashuttle project @@ -449,7 +456,7 @@ def search_sub_or_ses_level( return_full_path: bool = False, ) -> Tuple[List[str] | List[Path], List[str]]: """Search project folder at the subject or session level. - Only returns folders + Only returns folders. Parameters ---------- @@ -459,26 +466,33 @@ def search_sub_or_ses_level( arguments, but this is not nice and breaks the general rule that these functions should operate project-agnostic. + + base_folder + the path to the base folder. If sub is None, the search is + performed on this folder local_or_central search in local or central project sub either a subject name (string) or None. If None, the search - is performed at the top_level_folder level + is performed at the base_folder level ses either a session name (string) or None, This must not be a session name if sub is None. If provided (with sub) then the session folder is searched - str + search_str glob-format search string to search at the folder level. verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + + return_full_path + include the search_path in the returned paths """ if ses and not sub: @@ -518,6 +532,9 @@ def search_for_folders( Parameters ---------- + cfg + datashuttle configs + local_or_central "local" or "central" @@ -531,6 +548,9 @@ def search_for_folders( If `True`, when a search folder cannot be found, a message will be printed with the missing path. + return_full_path + include the search_path in the returned paths + """ if local_or_central == "central" and cfg["connection_method"] == "ssh": all_folder_names, all_filenames = ssh.search_ssh_central_for_folders( diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index 1cea80fa8..5afdae8c2 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -120,7 +120,7 @@ def update_names_with_range_to_flag( keyword must be in the form prefix-num1@num2. The maximum number of leading zeros are used to pad the output e.g. - sub-01@003 becomes ["sub-001", "sub-002", "sub-003"] + sub-01@003 becomes ["sub-001", "sub-002", "sub-003"]. Input can also be a mixed list e.g. names = ["sub-01", "sub-02@TO@04", "sub-05@TO@10"] @@ -278,14 +278,17 @@ def replace_date_time_tags_in_name( def format_date(date: str) -> str: + """PLACEHOLDER.""" return f"date-{date}" def format_time(time_: str) -> str: + """PLACEHOLDER.""" return f"time-{time_}" def format_datetime(date: str, time_: str) -> str: + """PLACEHOLDER.""" return f"datetime-{date}T{time_}" @@ -293,7 +296,7 @@ def add_underscore_before_after_if_not_there(string: str, key: str) -> str: """If names are passed with @DATE@, @TIME@, or @DATETIME@ but not surrounded by underscores, check and insert if required. e.g. sub-001@DATE@ becomes sub-001_@DATE@ - or sub-001@DATEid-101 becomes sub-001_@DATE_id-101 + or sub-001@DATEid-101 becomes sub-001_@DATE_id-101. """ key_len = len(key) key_start_idx = string.index(key) @@ -320,7 +323,7 @@ def add_missing_prefixes_to_names( all_names: Union[List[str], str], prefix: str ) -> List[str]: """Make sure all elements in the list of names are - prefixed with the prefix, typically "sub-" or "ses-" + prefixed with the prefix, typically "sub-" or "ses-". Use expanded list for readability """ diff --git a/datashuttle/utils/getters.py b/datashuttle/utils/getters.py index ac648f54f..44e58a499 100644 --- a/datashuttle/utils/getters.py +++ b/datashuttle/utils/getters.py @@ -77,6 +77,10 @@ def get_next_sub_or_ses( the desired value can be entered here. e.g. if 3 (the default), if no subjects are found the subject returned will be "sub-001". + name_template_regexp + the name template to try and get the num digits from. + If unspecified, the number of digits will be default_num_value_digits. + Returns ------- suggested_new_num @@ -133,7 +137,8 @@ def get_max_sub_or_ses_num_and_value_length( all_folders A list of BIDS-style formatted folder names. - see `get_next_sub_or_ses()` for other arguments. + prefix, default_num_value_digits, name_template_regexp + see `get_next_sub_or_ses()`. Returns ------- @@ -229,7 +234,7 @@ def get_num_value_digits_from_project( def get_num_value_digits_from_regexp( prefix: Prefix, name_template_regexp: str ) -> Union[Literal[False], int]: - """Given a name template regexp, find the number of values for the + r"""Given a name template regexp, find the number of values for the sub or ses key. These will be fixed with "\d" (digit) or ".?" (wildcard). If there is length-unspecific wildcard (.*) in the sub key, then skip. In practice, there should never really be a .* in the sub or ses diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index a0bd2772c..d651b8f96 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -112,6 +112,7 @@ def setup_rclone_config_for_ssh( def log_rclone_config_output(): + """PLACEHOLDER.""" output = call_rclone("config file", pipe_std=True) utils.log( f"Successfully created rclone config. {output.stdout.decode('utf-8')}" @@ -218,6 +219,9 @@ def get_local_and_central_file_differences( Parameters ---------- + cfg + datashuttle configs UserDict. + top_level_folders_to_check List of top-level folders to check. @@ -281,9 +285,9 @@ def assert_rclone_check_output_is_as_expected(result, symbol, convert_symbols): def perform_rclone_check( cfg: Configs, top_level_folder: TopLevelFolder ) -> str: - """Use Rclone's `check` command to build a list of files that + r"""Use Rclone's `check` command to build a list of files that are the same ("="), different ("*"), found in local only ("+") - or central only ("-"). The output is formatted as " \n". + or central only ("-"). The output is formatted as "\ \\n". """ local_filepath = cfg.get_base_folder( "local", top_level_folder @@ -306,7 +310,7 @@ def perform_rclone_check( def handle_rclone_arguments( rclone_options: Dict, include_list: List[str] ) -> str: - """Construct the extra arguments to pass to RClone,""" + """Construct the extra arguments to pass to RClone.""" extra_arguments_list = [] extra_arguments_list += ["-" + rclone_options["transfer_verbosity"]] @@ -336,7 +340,7 @@ def handle_rclone_arguments( def rclone_args(name: str) -> str: - """Central function to hold rclone commands""" + """Central function to hold rclone commands.""" valid_names = [ "dry_run", "copy", diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index b1cd9d2b0..9b80f7d4c 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -29,6 +29,7 @@ def connect_client_core( cfg: Configs, password: Optional[str] = None, ): + """PLACEHOLDER.""" client.get_host_keys().load(cfg.hostkeys_path.as_posix()) client.set_missing_host_key_policy(paramiko.RejectPolicy()) @@ -71,6 +72,7 @@ def add_public_key_to_central_authorized_keys( def generate_and_write_ssh_key(ssh_key_path: Path) -> None: + """PLACEHOLDER.""" key = paramiko.RSAKey.generate(4096) key.write_private_key_file(ssh_key_path.as_posix()) @@ -87,6 +89,7 @@ def get_remote_server_key(central_host_id: str): def save_hostkey_locally(key, central_host_id, hostkeys_path) -> None: + """PLACEHOLDER.""" client = paramiko.SSHClient() client.get_host_keys().add(central_host_id, key.get_name(), key) client.get_host_keys().save(hostkeys_path.as_posix()) @@ -262,6 +265,9 @@ def search_ssh_central_for_folders( If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + return_full_path + include the search_path in the returned paths + """ client: paramiko.SSHClient with paramiko.SSHClient() as client: @@ -294,7 +300,7 @@ def get_list_of_folder_names_over_sftp( Parameters ---------- - stfp + sftp connected paramiko stfp object (see search_ssh_central_for_folders()) @@ -308,6 +314,9 @@ def get_list_of_folder_names_over_sftp( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. + + return_full_path + include the search_path in the returned paths """ all_folder_names = [] diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index 18b98f913..56dd43670 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -29,7 +29,7 @@ def log(message: str) -> None: def log_and_message(message: str, use_rich: bool = False) -> None: """Log the message and send it to user. - use_rich : is True, use rich's print() function + use_rich : is True, use rich's print() function. """ log(message) print_message_to_user(message, use_rich) @@ -45,7 +45,7 @@ def log_and_raise_error(message: str, exception: Any) -> None: def warn(message: str, log: bool) -> None: - """ """ + """PLACEHOLDER.""" if log and ds_logger.logging_is_active(): logger = ds_logger.get_logger() logger.warning(message) @@ -74,7 +74,7 @@ def print_message_to_user( def get_user_input(message: str) -> str: - """Centralised way to get user input""" + """Centralised way to get user input.""" input_ = input(message) return input_ @@ -85,6 +85,7 @@ def get_user_input(message: str) -> str: def path_starts_with_base_folder(base_folder: Path, path_: Path) -> bool: + """PLACEHOLDER.""" return path_.as_posix().startswith(base_folder.as_posix()) @@ -158,6 +159,7 @@ def get_values_from_bids_formatted_name( def sub_or_ses_value_to_int(value: str) -> int: + """PLACEHOLDER.""" try: int_value = int(value) except ValueError: @@ -182,6 +184,7 @@ def get_value_from_key_regexp(name: str, key: str) -> List[str]: def integers_are_consecutive(list_of_ints: List[int]) -> bool: + """PLACEHOLDER.""" diff_between_ints = diff(list_of_ints) return all([diff == 1 for diff in diff_between_ints]) @@ -194,7 +197,7 @@ def diff(x: List) -> List: def num_leading_zeros(string: str) -> int: - """int() strips leading zeros""" + """int() strips leading zeros.""" if string[:4] in ["sub-", "ses-"]: string = string[4:] diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index 99c798fb6..b4e89974d 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -34,6 +34,7 @@ def get_missing_prefix_error(name: str, prefix, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"MISSING_PREFIX: The prefix {prefix} was not found in the name: {name}", path_, @@ -41,6 +42,7 @@ def get_missing_prefix_error(name: str, prefix, path_: Path | None) -> str: def get_bad_value_error(name: str, prefix, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"BAD_VALUE: The value for prefix {prefix} in name {name} is not an integer.", path_, @@ -48,6 +50,7 @@ def get_bad_value_error(name: str, prefix, path_: Path | None) -> str: def get_duplicate_prefix_error(name: str, prefix, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"DUPLICATE_PREFIX: The name: {name} of contains more than one instance of the prefix {prefix}.", path_, @@ -55,12 +58,14 @@ def get_duplicate_prefix_error(name: str, prefix, path_: Path | None) -> str: def get_name_error(name: str, prefix: Prefix, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"BAD_NAME: The name: {name} of type: {prefix} is not valid.", path_ ) def get_special_char_error(name: str, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"SPECIAL_CHAR: The name: {name}, contains characters which are not alphanumeric, dash or underscore.", path_, @@ -68,6 +73,7 @@ def get_special_char_error(name: str, path_: Path | None) -> str: def get_name_format_error(name: str, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"NAME_FORMAT: The name {name} does not consist of key-value pairs separated by underscores.", path_, @@ -75,10 +81,12 @@ def get_name_format_error(name: str, path_: Path | None) -> str: def get_value_length_error(prefix: Prefix) -> str: + """PLACEHOLDER.""" return f"VALUE_LENGTH: Inconsistent value lengths for the prefix: {prefix} were found in the project." def get_datetime_error(key, name: str, strfmt: str, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"DATETIME: Name {name} contains an invalid {key}. It should be ISO format: {strfmt}.", path_, @@ -98,6 +106,7 @@ def get_template_error(name: str, regexp: str, path_: Path | None) -> str: def get_missing_top_level_folder_error( path_: Path | None, local_or_central: Literal["local", "central"] ) -> str: + """PLACEHOLDER.""" return handle_path( f"TOP_LEVEL_FOLDER: The {local_or_central} project must contain a 'rawdata' or 'derivatives' folder.", path_, @@ -107,6 +116,7 @@ def get_missing_top_level_folder_error( def get_duplicate_name_error( new_name: str, exist_name: str, exist_path: Path | None ) -> str: + """PLACEHOLDER.""" return handle_path( f"DUPLICATE_NAME: The prefix for {new_name} duplicates the name: {exist_name}.", exist_path, @@ -114,12 +124,14 @@ def get_duplicate_name_error( def get_datatype_error(datatype_name: str, path_: Path | None) -> str: + """PLACEHOLDER.""" return handle_path( f"DATATYPE: {datatype_name} is not a valid datatype name.", path_ ) def handle_path(message: str, path_: Path | None) -> str: + """PLACEHOLDER.""" if path_: message += f" Path: {path_.as_posix()}" return message @@ -301,7 +313,7 @@ def get_path_and_name(path_or_name: Path | str) -> Tuple[Optional[Path], str]: def replace_tags_in_regexp(regexp: str) -> str: - """Before validation, all tags in the names are converted to + r"""Before validation, all tags in the names are converted to their final values (e.g. @DATE@ -> _date-). We also want to allow template to be formatted like `sub-\d\d_@DATE@` as it is convenient for auto-completion in the TUI. @@ -349,6 +361,7 @@ def names_include_special_characters( def name_has_special_character(name: str) -> bool: + """PLACEHOLDER.""" return not re.match("^[A-Za-z0-9_-]*$", name) From 52f03ec6ded9e75d8e345803a6d581254a6acacb Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:19:42 +0000 Subject: [PATCH 14/25] fixed tui --- datashuttle/tui/app.py | 12 ++++++++++-- datashuttle/tui/configs.py | 11 +++++++---- datashuttle/tui/custom_widgets.py | 11 ++++++++++- datashuttle/tui/interface.py | 10 +++++++++- .../tui/screens/create_folder_settings.py | 10 +++++++++- datashuttle/tui/screens/datatypes.py | 8 +++++++- datashuttle/tui/screens/get_help.py | 9 ++++++++- datashuttle/tui/screens/modal_dialogs.py | 19 ++++++++++++++++--- datashuttle/tui/screens/new_project.py | 3 +++ datashuttle/tui/screens/project_manager.py | 3 +++ datashuttle/tui/screens/project_selector.py | 3 +++ datashuttle/tui/screens/settings.py | 7 ++++++- datashuttle/tui/screens/setup_ssh.py | 5 ++++- datashuttle/tui/tabs/create_folders.py | 10 +++++++--- datashuttle/tui/tabs/logging.py | 17 +++++++++++++++++ datashuttle/tui/tabs/transfer.py | 14 +++++++++++--- datashuttle/tui/tabs/transfer_status_tree.py | 2 ++ datashuttle/tui/utils/tui_validators.py | 2 ++ 18 files changed, 134 insertions(+), 22 deletions(-) diff --git a/datashuttle/tui/app.py b/datashuttle/tui/app.py index fb342c445..7fe2fc0a5 100644 --- a/datashuttle/tui/app.py +++ b/datashuttle/tui/app.py @@ -47,6 +47,7 @@ class TuiApp(App, inherit_bindings=False): # type: ignore ] def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Label("datashuttle", id="mainwindow_banner_label"), Button( @@ -60,9 +61,11 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """PLACEHOLDER.""" self.set_dark_mode(self.load_global_settings()["dark_mode"]) def set_dark_mode(self, dark_mode: bool) -> None: + """PLACEHOLDER.""" self.theme = "textual-dark" if dark_mode else "textual-light" def on_button_pressed(self, event: Button.Pressed) -> None: @@ -91,6 +94,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.push_screen(get_help.GetHelpScreen()) def load_project_page(self, interface: Interface) -> None: + """PLACEHOLDER.""" if interface: self.push_screen( project_manager.ProjectManagerScreen( @@ -99,6 +103,7 @@ def load_project_page(self, interface: Interface) -> None: ) def show_modal_error_dialog(self, message: str) -> None: + """PLACEHOLDER.""" self.push_screen(modal_dialogs.MessageBox(message, border_color="red")) def handle_open_filesystem_browser(self, path_: Path) -> None: @@ -131,14 +136,14 @@ def handle_open_filesystem_browser(self, path_: Path) -> None: self.show_modal_error_dialog(message) def prompt_rename_file_or_folder(self, path_): - """ """ + """PLACEHOLDER.""" self.push_screen( modal_dialogs.RenameFileOrFolderScreen(self, path_), lambda new_name: self.rename_file_or_folder(path_, new_name), ) def rename_file_or_folder(self, path_, new_name): - """ """ + """PLACEHOLDER.""" if new_name is False: return try: @@ -185,12 +190,14 @@ def get_global_settings_path(self) -> Path: return path_ / "global_tui_settings.yaml" def get_default_global_settings(self) -> Dict: + """PLACEHOLDER.""" return { "dark_mode": True, "show_transfer_tree_status": False, } def save_global_settings(self, global_settings: Dict) -> None: + """PLACEHOLDER.""" settings_path = self.get_global_settings_path() if not settings_path.parent.is_dir(): @@ -212,6 +219,7 @@ def copy_to_clipboard(self, value): def main(): + """PLACEHOLDER.""" TuiApp().run() diff --git a/datashuttle/tui/configs.py b/datashuttle/tui/configs.py index 27a701d24..0d50f8712 100644 --- a/datashuttle/tui/configs.py +++ b/datashuttle/tui/configs.py @@ -30,7 +30,7 @@ class ConfigsContent(Container): - """This screen holds widgets and logic for setting datashuttle configs. + """Holds widgets and logic for setting datashuttle configs. It is used in `NewProjectPage` to instantiate a new project and initialise configs, or in `TabbedContent` to update an existing project's configs. @@ -44,6 +44,8 @@ class ConfigsContent(Container): @dataclass class ConfigsSaved(Message): + """PLACEHOLDER.""" + pass def __init__( @@ -52,6 +54,7 @@ def __init__( interface: Optional[Interface], id: str, ) -> None: + """PLACEHOLDER.""" super(ConfigsContent, self).__init__(id=id) self.parent_class = parent_class @@ -62,7 +65,7 @@ def compose(self) -> ComposeResult: """`self.config_ssh_widgets` are SSH-setup related widgets that are only required when the user selects the SSH connection method. These are displayed / hidden based on the - `connection_method` + `connection_method`. `config_screen_widgets` are core config-related widgets that are always displayed. @@ -269,7 +272,7 @@ def set_central_path_input_tooltip(self, display_ssh: bool) -> None: def get_platform_dependent_example_paths( self, local_or_central: Literal["local", "central"], ssh: bool = False ) -> str: - """ """ + """PLACEHOLDER.""" assert local_or_central in ["local", "central"] # Handle the ssh central case separately @@ -385,7 +388,7 @@ def handle_input_fill_from_select_directory( ).value = path_.as_posix() def setup_ssh_connection(self) -> None: - """Set up the `SetupSshScreen` screen,""" + """Set up the `SetupSshScreen` screen.""" assert self.interface is not None, "type narrow flexible `interface`" if not self.widget_configs_match_saved_configs(): diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 7f2b2e12b..77178842b 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -45,6 +45,8 @@ class ClickableInput(Input): @dataclass class Clicked(Message): + """PLACEHOLDER.""" + input: ClickableInput ctrl: bool @@ -56,6 +58,7 @@ def __init__( validate_on: Optional[List[str]] = None, validators: Optional[List[Validator]] = None, ) -> None: + """PLACEHOLDER.""" super(ClickableInput, self).__init__( placeholder=placeholder, id=id, @@ -69,9 +72,11 @@ def _on_click(self, event: events.Click) -> None: self.post_message(self.Clicked(self, event.ctrl)) def as_names_list(self) -> List[str]: + """PLACEHOLDER.""" return self.value.replace(" ", "").split(",") def on_key(self, event: events.Key) -> None: + """PLACEHOLDER.""" if event.key == "ctrl+q": self.mainwindow.copy_to_clipboard(self.value) @@ -92,12 +97,15 @@ class CustomDirectoryTree(DirectoryTree): @dataclass class DirectoryTreeSpecialKeyPress(Message): + """PLACEHOLDER.""" + key: str node_path: Optional[Path] def __init__( self, mainwindow: App, path: Path, id: Optional[str] = None ) -> None: + """PLACEHOLDER.""" super(CustomDirectoryTree, self).__init__(path=path, id=id) self.mainwindow = mainwindow @@ -139,7 +147,7 @@ def on_key(self, event: events.Key) -> None: def _render_line( self, y: int, x1: int, x2: int, base_style: Style ) -> Strip: - """This function is overridden from textual's `Tree` class to stop + """Overridden from textual's `Tree` class to stop CSS styling on hovering and clicking which was distracting / changed the default color used for transfer status, respectively. @@ -397,6 +405,7 @@ class TopLevelFolderSelect(Select): """ def __init__(self, interface: Interface, id: str) -> None: + """PLACEHOLDER.""" self.interface = interface top_level_folders = [ diff --git a/datashuttle/tui/interface.py b/datashuttle/tui/interface.py index 8ede9cb0e..0f97b06ec 100644 --- a/datashuttle/tui/interface.py +++ b/datashuttle/tui/interface.py @@ -30,6 +30,7 @@ class Interface: """ def __init__(self) -> None: + """PLACEHOLDER.""" self.project: DataShuttle self.name_templates: Dict = {} self.tui_settings: Dict = {} @@ -347,7 +348,7 @@ def save_tui_settings( value Value to set the `persistent_settings` tui field to - key_1 + key First key of the tui `persistent_settings` to update e.g. "top_level_folder_select" @@ -367,9 +368,11 @@ def save_tui_settings( # ---------------------------------------------------------------------------------- def get_central_host_id(self) -> str: + """PLACEHOLDER.""" return self.project.cfg["central_host_id"] def get_configs(self) -> Configs: + """PLACEHOLDER.""" return self.project.cfg def get_textual_compatible_project_configs(self) -> Configs: @@ -385,6 +388,7 @@ def get_textual_compatible_project_configs(self) -> Configs: def get_next_sub( self, top_level_folder: TopLevelFolder ) -> InterfaceOutput: + """PLACEHOLDER.""" try: next_sub = self.project.get_next_sub( top_level_folder, @@ -398,6 +402,7 @@ def get_next_sub( def get_next_ses( self, top_level_folder: TopLevelFolder, sub: str ) -> InterfaceOutput: + """PLACEHOLDER.""" try: next_ses = self.project.get_next_ses( top_level_folder, @@ -410,6 +415,7 @@ def get_next_ses( return False, str(e) def get_ssh_hostkey(self) -> InterfaceOutput: + """PLACEHOLDER.""" try: key = ssh.get_remote_server_key( self.project.cfg["central_host_id"] @@ -419,6 +425,7 @@ def get_ssh_hostkey(self) -> InterfaceOutput: return False, str(e) def save_hostkey_locally(self, key: paramiko.RSAKey) -> InterfaceOutput: + """PLACEHOLDER.""" try: ssh.save_hostkey_locally( key, @@ -433,6 +440,7 @@ def save_hostkey_locally(self, key: paramiko.RSAKey) -> InterfaceOutput: def setup_key_pair_and_rclone_config( self, password: str ) -> InterfaceOutput: + """PLACEHOLDER.""" try: ssh.add_public_key_to_central_authorized_keys( self.project.cfg, password, log=False diff --git a/datashuttle/tui/screens/create_folder_settings.py b/datashuttle/tui/screens/create_folder_settings.py index 26ae336ec..fa15349ba 100644 --- a/datashuttle/tui/screens/create_folder_settings.py +++ b/datashuttle/tui/screens/create_folder_settings.py @@ -27,7 +27,7 @@ class CreateFoldersSettingsScreen(ModalScreen): - """This screen handles setting datashuttle's `name_template`'s, as well + """Handles setting datashuttle's `name_template`'s, as well as the top-level-folder select and option to bypass all validation. Name Templates @@ -53,6 +53,7 @@ class CreateFoldersSettingsScreen(ModalScreen): TITLE = "Create Folders Settings" def __init__(self, mainwindow: App, interface: Interface) -> None: + """PLACEHOLDER.""" super(CreateFoldersSettingsScreen, self).__init__() self.mainwindow = mainwindow @@ -67,9 +68,11 @@ def __init__(self, mainwindow: App, interface: Interface) -> None: } def action_link_docs(self) -> None: + """PLACEHOLDER.""" webbrowser.open(links.get_docs_link()) def compose(self) -> ComposeResult: + """PLACEHOLDER.""" sub_on = True if self.input_mode == "sub" else False ses_on = not sub_on @@ -139,6 +142,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """PLACEHOLDER.""" for id in [ "#create_folders_settings_toplevel_select", "#create_folders_settings_bypass_validation_checkbox", @@ -151,11 +155,13 @@ def on_mount(self) -> None: self.switch_template_container_disabled() def init_input_values_holding_variable(self) -> None: + """PLACEHOLDER.""" name_templates = self.interface.get_name_templates() self.input_values["sub"] = name_templates["sub"] self.input_values["ses"] = name_templates["ses"] def switch_template_container_disabled(self) -> None: + """PLACEHOLDER.""" is_on = self.query_one( "#template_settings_validation_on_checkbox" ).value @@ -195,6 +201,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.interface.save_tui_settings(False, "bypass_validation") def make_name_templates_from_widgets(self) -> Dict: + """PLACEHOLDER.""" return { "on": self.query_one( "#template_settings_validation_on_checkbox" @@ -241,6 +248,7 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.fill_input_from_template() def on_input_changed(self, message: Input.Changed) -> None: + """PLACEHOLDER.""" if message.input.id == "template_settings_input": val = None if message.value == "" else message.value self.input_values[self.input_mode] = val diff --git a/datashuttle/tui/screens/datatypes.py b/datashuttle/tui/screens/datatypes.py index 78899342e..aea5c396f 100644 --- a/datashuttle/tui/screens/datatypes.py +++ b/datashuttle/tui/screens/datatypes.py @@ -89,6 +89,7 @@ def __init__( create_or_transfer: Literal["create", "transfer"], interface: Interface, ) -> None: + """PLACEHOLDER.""" super(DisplayedDatatypesScreen, self).__init__() self.interface = interface @@ -138,6 +139,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self): + """PLACEHOLDER.""" pass # self.query_one("#display_datatypes_screen_container").action_scroll_up() @@ -212,6 +214,7 @@ def __init__( create_or_transfer: Literal["create", "transfer"] = "create", id: Optional[str] = None, ) -> None: + """PLACEHOLDER.""" super(DatatypeCheckboxes, self).__init__(id=id) self.interface = interface @@ -226,6 +229,7 @@ def __init__( ] def compose(self) -> ComposeResult: + """PLACEHOLDER.""" for datatype, setting in self.datatype_config.items(): if setting["displayed"]: yield Checkbox( @@ -251,7 +255,7 @@ def on_checkbox_changed(self) -> None: ) def on_mount(self) -> None: - """ """ + """PLACEHOLDER.""" for datatype in self.datatype_config.keys(): if self.datatype_config[datatype]["displayed"]: self.query_one( @@ -277,12 +281,14 @@ def selected_datatypes(self) -> List[str]: def get_checkbox_name( create_or_transfer: Literal["create", "transfer"], datatype ): + """PLACEHOLDER.""" return f"{create_or_transfer}_{datatype}_checkbox" def get_tui_settings_key_name( create_or_transfer: Literal["create", "transfer"], ) -> str: + """PLACEHOLDER.""" if create_or_transfer == "create": settings_key = "create_checkboxes_on" else: diff --git a/datashuttle/tui/screens/get_help.py b/datashuttle/tui/screens/get_help.py index 622f9474b..76a7ff1c6 100644 --- a/datashuttle/tui/screens/get_help.py +++ b/datashuttle/tui/screens/get_help.py @@ -18,9 +18,10 @@ class GetHelpScreen(ModalScreen): - """ """ + """PLACEHOLDER.""" def __init__(self) -> None: + """PLACEHOLDER.""" super(GetHelpScreen, self).__init__() self.text = """ @@ -35,18 +36,23 @@ def __init__(self) -> None: """ def action_link_docs(self) -> None: + """PLACEHOLDER.""" webbrowser.open(links.get_docs_link()) def action_link_github(self) -> None: + """PLACEHOLDER.""" webbrowser.open(links.get_github_link()) def action_link_github_issues(self) -> None: + """PLACEHOLDER.""" webbrowser.open(links.get_link_github_issues()) def action_link_zulip(self): + """PLACEHOLDER.""" webbrowser.open(links.get_link_zulip()) def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Static(self.text, id="get_help_label"), Button("Main Menu", id="all_main_menu_buttons"), @@ -54,5 +60,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "all_main_menu_buttons": self.dismiss() diff --git a/datashuttle/tui/screens/modal_dialogs.py b/datashuttle/tui/screens/modal_dialogs.py index 9625c0318..6acb79b91 100644 --- a/datashuttle/tui/screens/modal_dialogs.py +++ b/datashuttle/tui/screens/modal_dialogs.py @@ -34,12 +34,14 @@ class MessageBox(ModalScreen): """ def __init__(self, message: str, border_color: str) -> None: + """PLACEHOLDER.""" super(MessageBox, self).__init__() self.message = message self.border_color = border_color def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Container( Static(self.message, id="messagebox_message_label"), @@ -50,6 +52,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """PLACEHOLDER.""" if self.border_color == "red": color = "rgb(140, 12, 0)" elif self.border_color == "green": @@ -65,6 +68,7 @@ def on_mount(self) -> None: ) def on_button_pressed(self) -> None: + """PLACEHOLDER.""" self.dismiss(True) @@ -81,12 +85,14 @@ def __init__( message: str, transfer_func: Callable[[], Worker[InterfaceOutput]], ) -> None: + """PLACEHOLDER.""" super().__init__() self.transfer_func = transfer_func self.message = message def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Label(self.message, id="confirm_message_label"), Horizontal( @@ -98,6 +104,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "confirm_ok_button": self.query_one("#confirm_button_container").remove() @@ -114,7 +121,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: self.dismiss() async def handle_transfer_and_update_ui_when_complete(self) -> None: - """Runs the data transfer worker and updates the UI on completion""" + """Runs the data transfer worker and updates the UI on completion.""" data_transfer_worker = self.transfer_func() await data_transfer_worker.wait() success, output = data_transfer_worker.result @@ -152,6 +159,7 @@ class SelectDirectoryTreeScreen(ModalScreen): """ def __init__(self, mainwindow: App, path_: Optional[Path] = None) -> None: + """PLACEHOLDER.""" super(SelectDirectoryTreeScreen, self).__init__() self.mainwindow = mainwindow @@ -162,6 +170,7 @@ def __init__(self, mainwindow: App, path_: Optional[Path] = None) -> None: self.prev_click_time = 0 def compose(self) -> ComposeResult: + """PLACEHOLDER.""" label_message = ( "Select (double click) a folder with the same name as the project.\n" "If the project folder does not exist, select the parent folder and it will be created." @@ -180,26 +189,30 @@ def compose(self) -> ComposeResult: @require_double_click def on_directory_tree_directory_selected(self, node) -> None: + """PLACEHOLDER.""" if node.path.is_file(): return else: self.dismiss(node.path) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "cancel_button": self.dismiss(False) class RenameFileOrFolderScreen(ModalScreen): - """ """ + """PLACEHOLDER.""" def __init__(self, mainwindow: App, path_: Path) -> None: + """PLACEHOLDER.""" super(RenameFileOrFolderScreen, self).__init__() self.mainwindow = mainwindow self.path_ = path_ def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Label("Input the new name:", id="rename_screen_label"), Input(value=self.path_.stem, id="rename_screen_input"), @@ -212,7 +225,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: - """""" + """PLACEHOLDER.""" if event.button.id == "rename_screen_okay_button": self.dismiss(self.query_one("#rename_screen_input").value) diff --git a/datashuttle/tui/screens/new_project.py b/datashuttle/tui/screens/new_project.py index 769629ff3..d387eea39 100644 --- a/datashuttle/tui/screens/new_project.py +++ b/datashuttle/tui/screens/new_project.py @@ -37,11 +37,13 @@ class NewProjectScreen(Screen): TITLE = "Make New Project" def __init__(self, mainwindow: App) -> None: + """PLACEHOLDER.""" super(NewProjectScreen, self).__init__() self.mainwindow = mainwindow def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") yield configs.ConfigsContent( @@ -49,5 +51,6 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "all_main_menu_buttons": self.dismiss(None) diff --git a/datashuttle/tui/screens/project_manager.py b/datashuttle/tui/screens/project_manager.py index ebe0168e9..52664edee 100644 --- a/datashuttle/tui/screens/project_manager.py +++ b/datashuttle/tui/screens/project_manager.py @@ -41,6 +41,7 @@ class ProjectManagerScreen(Screen): """ def __init__(self, mainwindow: App, interface: Interface, id) -> None: + """PLACEHOLDER.""" super(ProjectManagerScreen, self).__init__(id=id) self.mainwindow = mainwindow @@ -51,6 +52,7 @@ def __init__(self, mainwindow: App, interface: Interface, id) -> None: self.tabbed_content_mount_signal = True def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Header() yield Button("Main Menu", id="all_main_menu_buttons") with TabbedContent( @@ -118,6 +120,7 @@ def on_tabbed_content_tab_activated( ).update_most_recent_label() def update_active_tab_tree(self): + """PLACEHOLDER.""" active_tab_id = self.query_one("#tabscreen_tabbed_content").active self.query_one(f"#{active_tab_id}").reload_directorytree() diff --git a/datashuttle/tui/screens/project_selector.py b/datashuttle/tui/screens/project_selector.py index 3bd923570..d952e44fe 100644 --- a/datashuttle/tui/screens/project_selector.py +++ b/datashuttle/tui/screens/project_selector.py @@ -36,6 +36,7 @@ class ProjectSelectorScreen(Screen): TITLE = "Select Project" def __init__(self, mainwindow: App) -> None: + """PLACEHOLDER.""" super(ProjectSelectorScreen, self).__init__() self.project_names = [ @@ -44,6 +45,7 @@ def __init__(self, mainwindow: App) -> None: self.mainwindow = mainwindow def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Header(id="project_select_header") yield Button("Main Menu", id="all_main_menu_buttons") yield Container( @@ -52,6 +54,7 @@ def compose(self) -> ComposeResult: ) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id in self.project_names: project_name = event.button.id diff --git a/datashuttle/tui/screens/settings.py b/datashuttle/tui/screens/settings.py index 0634ea041..ace8a1d49 100644 --- a/datashuttle/tui/screens/settings.py +++ b/datashuttle/tui/screens/settings.py @@ -27,12 +27,14 @@ class SettingsScreen(ModalScreen): """ def __init__(self, mainwindow: App) -> None: + """PLACEHOLDER.""" super(SettingsScreen, self).__init__() self.mainwindow = mainwindow self.global_settings = self.mainwindow.load_global_settings() def compose(self) -> ComposeResult: + """PLACEHOLDER.""" dark_mode = self.global_settings["dark_mode"] yield Container( RadioSet( @@ -58,11 +60,12 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """""" + """PLACEHOLDER.""" id = "#show_transfer_tree_status_checkbox" self.query_one(id).tooltip = get_tooltip(id) def on_radio_set_changed(self, event: RadioSet.Changed) -> None: + """PLACEHOLDER.""" label = str(event.pressed.label) assert label in ["Light Mode", "Dark Mode"] @@ -73,9 +76,11 @@ def on_radio_set_changed(self, event: RadioSet.Changed) -> None: self.mainwindow.save_global_settings(self.global_settings) def on_checkbox_changed(self, event: Checkbox.Changed) -> None: + """PLACEHOLDER.""" self.global_settings["show_transfer_tree_status"] = event.value self.mainwindow.save_global_settings(self.global_settings) def on_button_pressed(self, event: Button.Pressed) -> None: + """PLACEHOLDER.""" if event.button.id == "all_main_menu_buttons": self.dismiss() diff --git a/datashuttle/tui/screens/setup_ssh.py b/datashuttle/tui/screens/setup_ssh.py index 7502caadb..bf9c84ca3 100644 --- a/datashuttle/tui/screens/setup_ssh.py +++ b/datashuttle/tui/screens/setup_ssh.py @@ -18,7 +18,7 @@ class SetupSshScreen(ModalScreen): - """This dialog windows handles the TUI equivalent of API's + """Dialog window that handles the TUI equivalent of API's setup_ssh_connection(). This asks to confirm the central hostkey, and takes password to setup SSH key pair. @@ -29,6 +29,7 @@ class SetupSshScreen(ModalScreen): """ def __init__(self, interface: Interface) -> None: + """PLACEHOLDER.""" super(SetupSshScreen, self).__init__() self.interface = interface @@ -38,6 +39,7 @@ def __init__(self, interface: Interface) -> None: self.key: paramiko.RSAKey def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield Container( Horizontal( Static( @@ -56,6 +58,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: + """PLACEHOLDER.""" self.query_one("#setup_ssh_password_input").visible = False def on_button_pressed(self, event: Button.pressed) -> None: diff --git a/datashuttle/tui/tabs/create_folders.py b/datashuttle/tui/tabs/create_folders.py index 7b98eb6db..73544767c 100644 --- a/datashuttle/tui/tabs/create_folders.py +++ b/datashuttle/tui/tabs/create_folders.py @@ -38,6 +38,7 @@ class CreateFoldersTab(TreeAndInputTab): """Create new project files formatted according to the NeuroBlueprint specification.""" def __init__(self, mainwindow: App, interface: Interface) -> None: + """PLACEHOLDER.""" super(CreateFoldersTab, self).__init__( "Create", id="tabscreen_create_tab" ) @@ -47,6 +48,7 @@ def __init__(self, mainwindow: App, interface: Interface) -> None: self.prev_click_time = 0.0 def compose(self) -> ComposeResult: + """PLACEHOLDER.""" yield CustomDirectoryTree( self.mainwindow, self.interface.get_configs()["local_path"], @@ -91,7 +93,7 @@ def compose(self) -> ComposeResult: ) def on_mount(self) -> None: - """""" + """PLACEHOLDER.""" if not self.interface: self.query_one("#configs_name_input").tooltip = get_tooltip( "#configs_name_input" @@ -130,6 +132,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) async def refresh_after_datatypes_changed(self, ignore): + """PLACEHOLDER.""" await self.recompose() self.on_mount() @@ -187,6 +190,7 @@ def fill_input_with_template(self, prefix: Prefix, input_id: str) -> None: input.value = fill_value def templates_on(self, prefix: Prefix) -> bool: + """PLACEHOLDER.""" return ( self.interface.get_name_templates()["on"] and self.interface.get_name_templates()[prefix] is not None @@ -251,7 +255,7 @@ def create_folders(self) -> None: self.mainwindow.show_modal_error_dialog(output) def reload_directorytree(self) -> None: - """This reloads the directorytree and also updates validation. + """Reloads the directorytree and also updates validation. Not now a good method name but done for consistency with other tab refresh methods. """ @@ -264,7 +268,7 @@ def reload_directorytree(self) -> None: def fill_input_with_next_sub_or_ses_template( self, prefix: Prefix, input_id: str ) -> None: - """This fills a sub / ses Input with a suggested name based on the + """Fills a sub / ses Input with a suggested name based on the next subject / session in the project (local). If `name_templates` are set, then the sub- or ses- first key diff --git a/datashuttle/tui/tabs/logging.py b/datashuttle/tui/tabs/logging.py index f35c7a987..8c4f5db92 100644 --- a/datashuttle/tui/tabs/logging.py +++ b/datashuttle/tui/tabs/logging.py @@ -13,29 +13,38 @@ class RichLogScreen(ModalScreen): + """PLACEHOLDER.""" + def __init__(self, log_file): + """PLACEHOLDER.""" super(RichLogScreen, self).__init__() with open(log_file) as file: self.log_contents = "".join(file.readlines()) def compose(self): + """PLACEHOLDER.""" yield Container( RichLog(highlight=True, markup=True, id="richlog_screen_rich_log"), Button("Close", id="richlog_screen_close_button"), ) def on_mount(self): + """PLACEHOLDER.""" text_log = self.query_one(RichLog) text_log.write(self.log_contents) def on_button_pressed(self, event): + """PLACEHOLDER.""" if event.button.id == "richlog_screen_close_button": self.dismiss() class LoggingTab(TabPane): + """PLACEHOLDER.""" + def __init__(self, title, mainwindow, project, id): + """PLACEHOLDER.""" super(LoggingTab, self).__init__(title=title, id=id) self.mainwindow = mainwindow @@ -48,6 +57,7 @@ def __init__(self, title, mainwindow, project, id): self.prev_click_time = 0 def update_latest_log_path(self): + """PLACEHOLDER.""" logs = list(self.project.get_logging_path().glob("*.log")) self.latest_log_path = ( max(logs, key=os.path.getctime) @@ -56,6 +66,7 @@ def update_latest_log_path(self): ) def compose(self): + """PLACEHOLDER.""" yield Container( Label( "Double click logging file to select:", @@ -83,6 +94,7 @@ def _on_mount(self, event: events.Mount) -> None: self.update_most_recent_label() def update_most_recent_label(self): + """PLACEHOLDER.""" self.update_latest_log_path() self.query_one("#logging_most_recent_label").update( f"or open most recent: {self.latest_log_path.stem}" @@ -90,11 +102,13 @@ def update_most_recent_label(self): self.refresh() def on_button_pressed(self, event): + """PLACEHOLDER.""" if event.button.id == "logging_tab_open_most_recent_button": self.push_rich_log_screen(self.latest_log_path) @require_double_click def on_directory_tree_file_selected(self, node): + """PLACEHOLDER.""" if not node.path.is_file(): self.mainwindow.show_modal_error_dialog( "Log file no longer exists. Refresh the directory tree" @@ -105,6 +119,7 @@ def on_directory_tree_file_selected(self, node): self.push_rich_log_screen(node.path) def push_rich_log_screen(self, log_path): + """PLACEHOLDER.""" self.mainwindow.push_screen( RichLogScreen( log_path, @@ -112,7 +127,9 @@ def push_rich_log_screen(self, log_path): ) def reload_directorytree(self): + """PLACEHOLDER.""" self.query_one("#logging_tab_custom_directory_tree").reload() def on_custom_directory_tree_directory_tree_special_key_press(self): + """PLACEHOLDER.""" self.reload_directorytree() diff --git a/datashuttle/tui/tabs/transfer.py b/datashuttle/tui/tabs/transfer.py index b50a14b08..9005fba06 100644 --- a/datashuttle/tui/tabs/transfer.py +++ b/datashuttle/tui/tabs/transfer.py @@ -43,7 +43,7 @@ class TransferTab(TreeAndInputTab): - """This tab handles the upload / download of files between local + """Handles the upload / download of files between local and central folders. It contains a TransferDirectoryTree that displays the transfer status of the files in the local folder, and calls underlying datashuttle transfer functions. @@ -83,6 +83,7 @@ def __init__( interface: Interface, id: Optional[str] = None, ) -> None: + """PLACEHOLDER.""" super(TransferTab, self).__init__(title, id=id) self.mainwindow = mainwindow self.interface = interface @@ -95,6 +96,7 @@ def __init__( # ---------------------------------------------------------------------------------- def compose(self) -> ComposeResult: + """PLACEHOLDER.""" self.transfer_all_widgets = [ Label( "All data from: \n\n - Rawdata \n - Derivatives \n\nwill be transferred.", @@ -208,6 +210,7 @@ def compose(self) -> ComposeResult: yield Label("⭕ Legend", id="transfer_legend") def on_mount(self) -> None: + """PLACEHOLDER.""" for id in [ "#transfer_directorytree", "#transfer_switch_container", @@ -239,6 +242,7 @@ def on_mount(self) -> None: ) def on_select_changed(self, event: Select.Changed) -> None: + """PLACEHOLDER.""" if event.select.id == "transfer_tab_overwrite_select": assert event.select.value in ["Never", "Always", "If Source Newer"] format_select = event.select.value.lower().replace(" ", "_") @@ -248,6 +252,7 @@ def on_select_changed(self, event: Select.Changed) -> None: ) def on_checkbox_changed(self, event: Checkbox.Changed) -> None: + """PLACEHOLDER.""" if event.checkbox.id == "transfer_tab_dry_run_checkbox": self.interface.save_tui_settings( event.checkbox.value, @@ -317,6 +322,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None: ) async def refresh_after_datatype_changed(self, ignore): + """PLACEHOLDER.""" await self.recompose() self.on_mount() self.query_one("#transfer_custom_radiobutton").value = True @@ -325,6 +331,7 @@ async def refresh_after_datatype_changed(self, ignore): def on_custom_directory_tree_directory_tree_special_key_press( self, event: CustomDirectoryTree.DirectoryTreeSpecialKeyPress ) -> None: + """PLACEHOLDER.""" if event.key == "ctrl+r": self.reload_directorytree() @@ -338,10 +345,11 @@ def on_custom_directory_tree_directory_tree_special_key_press( self.reload_directorytree() def reload_directorytree(self) -> None: + """PLACEHOLDER.""" self.query_one("#transfer_directorytree").update_transfer_tree() def update_directorytree_root(self, new_root_path: Path) -> None: - """This will automatically refresh the tree through the + """Automatically refreshes the tree through the reactive variable `path`. """ self.query_one("#transfer_directorytree").path = new_root_path @@ -351,7 +359,7 @@ def update_directorytree_root(self, new_root_path: Path) -> None: @work(exclusive=True, thread=True) def transfer_data(self) -> Worker[InterfaceOutput]: - """A threaded worker to transfer data + """A threaded worker to transfer data. This function transfers data based on the config provided by the radio buttons such as a) the data to be transferred (all / top-level-folders / custom) b) the diff --git a/datashuttle/tui/tabs/transfer_status_tree.py b/datashuttle/tui/tabs/transfer_status_tree.py index 1d4410fc7..373d1034d 100644 --- a/datashuttle/tui/tabs/transfer_status_tree.py +++ b/datashuttle/tui/tabs/transfer_status_tree.py @@ -38,6 +38,7 @@ class TransferStatusTree(CustomDirectoryTree): def __init__( self, mainwindow: App, interface: Interface, id: Optional[str] = None ): + """PLACEHOLDER.""" self.interface = interface self.local_path_str = self.interface.get_configs()[ "local_path" @@ -49,6 +50,7 @@ def __init__( ) def on_mount(self) -> None: + """PLACEHOLDER.""" self.update_transfer_tree(init=True) def update_transfer_tree(self, init: bool = False) -> None: diff --git a/datashuttle/tui/utils/tui_validators.py b/datashuttle/tui/utils/tui_validators.py index 1605c63ba..47e8dbd40 100644 --- a/datashuttle/tui/utils/tui_validators.py +++ b/datashuttle/tui/utils/tui_validators.py @@ -12,6 +12,8 @@ class NeuroBlueprintValidator(Validator): + """PLACEHOLDER.""" + def __init__(self, prefix: Prefix, parent: CreateFoldersTab) -> None: """Custom Validator() class that takes sub / ses prefix as input. Runs validation of From 95296d8dc1cefe7a36ebd38e1dbfcbd7452bf14d Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:26:07 +0000 Subject: [PATCH 15/25] fixed configs --- datashuttle/configs/canonical_configs.py | 5 ++- datashuttle/configs/canonical_folders.py | 9 ++--- datashuttle/configs/config_class.py | 45 +++++++++++++----------- datashuttle/configs/links.py | 4 +++ 4 files changed, 38 insertions(+), 25 deletions(-) diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index 8d03eed6e..5fab1aaa2 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -1,4 +1,4 @@ -"""This module contains all information for the required +"""Contains all information for the required format of the configs class. This is clearly defined as configs can be provided from file or input dynamically and so careful checks must be done. @@ -156,6 +156,7 @@ def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None: def local_only_configs_are_none(config_dict: Configs) -> list[bool]: + """PLACEHOLDER.""" return [ config_dict[key] is None for key in ["central_path", "connection_method"] @@ -260,6 +261,7 @@ def get_tui_config_defaults() -> Dict: def get_name_templates_defaults() -> Dict: + """PLACEHOLDER.""" return {"name_templates": {"on": False, "sub": None, "ses": None}} @@ -288,6 +290,7 @@ def get_datatypes() -> List[str]: def get_broad_datatypes(): + """PLACEHOLDER.""" return ["ephys", "behav", "funcimg", "anat"] diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index 104019704..cc3db158d 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -11,7 +11,7 @@ def get_datatype_folders() -> dict: - """This function holds the canonical folders + """Holds the canonical folders managed by datashuttle. Notes @@ -41,7 +41,7 @@ def get_datatype_folders() -> dict: def get_non_sub_names() -> List[str]: """Get all arguments that are not allowed at the - subject level for data transfer, i.e. as sub_names + subject level for data transfer, i.e. as sub_names. """ return [ "all_ses", @@ -53,7 +53,7 @@ def get_non_sub_names() -> List[str]: def get_non_ses_names() -> List[str]: """Get all arguments that are not allowed at the - session level for data transfer, i.e. as ses_names + session level for data transfer, i.e. as ses_names. """ return [ "all_sub", @@ -65,12 +65,13 @@ def get_non_ses_names() -> List[str]: def canonical_reserved_keywords() -> List[str]: """Key keyword arguments that are passed to `sub_names` or - `ses_names` but that we + `ses_names`. """ return get_non_sub_names() + get_non_ses_names() def get_top_level_folders() -> List[TopLevelFolder]: + """PLACEHOLDER.""" return ["rawdata", "derivatives"] diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index b62427b7b..f01976fc1 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -30,26 +30,25 @@ class Configs(UserDict): The configs must match exactly the standard set in canonical_configs.py. If updating these configs, this should be done through changing canonical_configs.py - - The input dict is checked that it conforms to the - canonical standard by calling check_dict_values_raise_on_fail() - - project_name and all paths are set at runtime but not stored. - - Parameters - ---------- - file_path - full filepath to save the config .yaml file to. - - input_dict - a dict of config key-value pairs to input dict. - This must contain all canonical_config keys - """ def __init__( self, project_name: str, file_path: Path, input_dict: Union[dict, None] ) -> None: + """Parameters + ---------- + file_path + full filepath to save the config .yaml file to. + + input_dict + a dict of config key-value pairs to input dict. + This must contain all canonical_config keys + + The input dict is checked that it conforms to the + canonical standard by calling check_dict_values_raise_on_fail() + + project_name and all paths are set at runtime but not stored. + """ super(Configs, self).__init__(input_dict) self.project_name = project_name @@ -61,12 +60,13 @@ def __init__( self.project_metadata_path: Path def setup_after_load(self) -> None: + """PLACEHOLDER.""" load_configs.convert_str_and_pathlib_paths(self, "str_to_path") self.ensure_local_and_central_path_end_in_project_name() self.check_dict_values_raise_on_fail() def ensure_local_and_central_path_end_in_project_name(self): - """""" + """PLACEHOLDER.""" for path_type in ["local_path", "central_path"]: if path_type == "central_path" and self[path_type] is None: continue @@ -89,12 +89,15 @@ def check_dict_values_raise_on_fail(self) -> None: canonical_configs.check_dict_values_raise_on_fail(self) def keys(self) -> KeysView: + """D.keys() -> a set-like object providing a view on D's keys.""" return self.data.keys() def items(self) -> ItemsView: + """D.items() -> a set-like object providing a view on D's items.""" return self.data.items() def values(self) -> ValuesView: + """D.values() -> a set-like object providing a view on D's values.""" return self.data.values() # ------------------------------------------------------------------------- @@ -110,9 +113,11 @@ def dump_to_file(self) -> None: yaml.dump(cfg_to_save, config_file, sort_keys=False) def load_from_file(self) -> None: - """Load a config dict saved at .yaml file. Note this will + """Load a config dict saved at .yaml file. + + Note this will not automatically check the configs are valid, this - requires calling self.check_dict_values_raise_on_fail() + requires calling self.check_dict_values_raise_on_fail(). """ with open(self.file_path) as config_file: config_dict = yaml.full_load(config_file) @@ -203,7 +208,7 @@ def get_rclone_config_name( def make_rclone_transfer_options( self, overwrite_existing_files: OverwriteExistingFiles, dry_run: bool ) -> Dict: - """This function originally collected the relevant arguments + """Originally collected the relevant arguments from configs. Now, all are passed via function arguments However, now we fix the previously configurable arguments `show_transfer_progress` and `dry_run` here. @@ -226,7 +231,7 @@ def make_rclone_transfer_options( } def init_paths(self) -> None: - """""" + """PLACEHOLDER.""" self.project_metadata_path = self["local_path"] / ".datashuttle" datashuttle_path, _ = canonical_folders.get_project_datashuttle_path( diff --git a/datashuttle/configs/links.py b/datashuttle/configs/links.py index 0c354bc1a..e61ade29a 100644 --- a/datashuttle/configs/links.py +++ b/datashuttle/configs/links.py @@ -1,14 +1,18 @@ def get_docs_link(): + """PLACEHOLDER.""" return "https://datashuttle.neuroinformatics.dev/" def get_github_link(): + """PLACEHOLDER.""" return "https://github.com/neuroinformatics-unit/datashuttle" def get_link_github_issues(): + """PLACEHOLDER.""" return "https://github.com/neuroinformatics-unit/datashuttle/issues" def get_link_zulip(): + """PLACEHOLDER.""" return "https://neuroinformatics.zulipchat.com/#narrow/stream/405999-DataShuttle" From 2187d17ef4d16637e5f8ff8de27f0f32eb75761e Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:36:45 +0000 Subject: [PATCH 16/25] run pre-commit --- .pre-commit-config.yaml | 4 ++-- datashuttle/configs/config_class.py | 8 ++++---- datashuttle/datashuttle_class.py | 8 ++++---- datashuttle/datashuttle_functions.py | 4 ++-- datashuttle/tui/configs.py | 2 +- datashuttle/tui/custom_widgets.py | 4 ++-- datashuttle/tui/tabs/logging.py | 4 ++-- datashuttle/tui/utils/tui_validators.py | 2 +- datashuttle/utils/custom_exceptions.py | 4 ++-- datashuttle/utils/data_transfer.py | 2 +- datashuttle/utils/folders.py | 12 ++++++------ datashuttle/utils/ssh.py | 2 +- pyproject.toml | 2 +- .../{test_configs.py => _test_configs.py} | 0 tests/tests_integration/base.py | 2 +- tests/tests_integration/test_create_folders.py | 2 +- tests/tests_integration/test_filesystem_transfer.py | 2 +- tests/tests_integration/test_formatting.py | 2 +- tests/tests_integration/test_local_only_mode.py | 2 +- tests/tests_integration/test_logging.py | 2 +- tests/tests_integration/test_settings.py | 2 +- tests/tests_integration/test_ssh_file_transfer.py | 2 +- tests/tests_integration/test_transfer_checks.py | 2 +- tests/tests_integration/test_validation.py | 2 +- tests/tests_tui/test_local_only_project.py | 2 +- tests/tests_tui/test_tui_configs.py | 2 +- tests/tests_tui/test_tui_create_folders.py | 2 +- tests/tests_tui/test_tui_logging.py | 2 +- 28 files changed, 43 insertions(+), 43 deletions(-) rename tests/tests_integration/{test_configs.py => _test_configs.py} (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 299fba1d1..1dbbe5cba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,8 +20,8 @@ repos: - id: end-of-file-fixer - id: mixed-line-ending args: [--fix=lf] - - id: name-tests-test - args: ["--pytest-test-first"] + #- id: name-tests-test + # args: ["--pytest-test-first"] - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/pre-commit/pygrep-hooks diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index f01976fc1..c7b71b8cc 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -43,7 +43,7 @@ def __init__( input_dict a dict of config key-value pairs to input dict. This must contain all canonical_config keys - + The input dict is checked that it conforms to the canonical standard by calling check_dict_values_raise_on_fail() @@ -114,7 +114,7 @@ def dump_to_file(self) -> None: def load_from_file(self) -> None: """Load a config dict saved at .yaml file. - + Note this will not automatically check the configs are valid, this requires calling self.check_dict_values_raise_on_fail(). @@ -149,7 +149,7 @@ def build_project_path( a list (or string for 1) of folder names to be joined into a path. If file included, must be last entry (with ext). - + top_level_folder either "rawdata" or "derivatives" @@ -181,7 +181,7 @@ def get_base_folder( ---------- base base path, "local", "central" or "datashuttle" - + top_level_folder either "rawdata" or "derivatives" diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 5adec00b9..96b344536 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -1053,7 +1053,7 @@ def get_next_sub( ---------- top_level_folder "rawdata" or "derivatives" - + return_with_prefix If `True`, return with the "sub-" prefix. @@ -1297,10 +1297,10 @@ def _transfer_entire_project( upload_or_download direction to transfer the data, either "upload" (from local to central) or "download" (from central to local). - + overwrite_existing_files determines whether or not to overwrite existing files - + dry_run perform a dry-run of transfer. This will output as if file transfer was taking place, but no files will be moved. Useful @@ -1341,7 +1341,7 @@ def _start_log( store_in_temp_folder if `False`, existing logging path will be used (local project .datashuttle). - + verbose print warnings and error messages. diff --git a/datashuttle/datashuttle_functions.py b/datashuttle/datashuttle_functions.py index 5ab642570..b474859b2 100644 --- a/datashuttle/datashuttle_functions.py +++ b/datashuttle/datashuttle_functions.py @@ -29,7 +29,7 @@ def quick_validate_project( name_templates: Optional[Dict] = None, ) -> List[str]: """Perform validation on the project. - + This checks the subject and session level folders to ensure there are not NeuroBlueprint formatting issues. @@ -93,7 +93,7 @@ def _format_top_level_folder( top_level_folder: TopLevelFolder | None, ) -> List[TopLevelFolder]: """Format the top level folder. - + Take a `top_level_folder` ("rawdata" or "derivatives" str) and convert to list, if `None`, convert it to a list of both possible top-level folders. diff --git a/datashuttle/tui/configs.py b/datashuttle/tui/configs.py index 0d50f8712..e7f7b044a 100644 --- a/datashuttle/tui/configs.py +++ b/datashuttle/tui/configs.py @@ -45,7 +45,7 @@ class ConfigsContent(Container): @dataclass class ConfigsSaved(Message): """PLACEHOLDER.""" - + pass def __init__( diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 77178842b..9a303b0da 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -46,7 +46,7 @@ class ClickableInput(Input): @dataclass class Clicked(Message): """PLACEHOLDER.""" - + input: ClickableInput ctrl: bool @@ -98,7 +98,7 @@ class CustomDirectoryTree(DirectoryTree): @dataclass class DirectoryTreeSpecialKeyPress(Message): """PLACEHOLDER.""" - + key: str node_path: Optional[Path] diff --git a/datashuttle/tui/tabs/logging.py b/datashuttle/tui/tabs/logging.py index 8c4f5db92..21b2ddc4e 100644 --- a/datashuttle/tui/tabs/logging.py +++ b/datashuttle/tui/tabs/logging.py @@ -14,7 +14,7 @@ class RichLogScreen(ModalScreen): """PLACEHOLDER.""" - + def __init__(self, log_file): """PLACEHOLDER.""" super(RichLogScreen, self).__init__() @@ -42,7 +42,7 @@ def on_button_pressed(self, event): class LoggingTab(TabPane): """PLACEHOLDER.""" - + def __init__(self, title, mainwindow, project, id): """PLACEHOLDER.""" super(LoggingTab, self).__init__(title=title, id=id) diff --git a/datashuttle/tui/utils/tui_validators.py b/datashuttle/tui/utils/tui_validators.py index 47e8dbd40..4124e4f44 100644 --- a/datashuttle/tui/utils/tui_validators.py +++ b/datashuttle/tui/utils/tui_validators.py @@ -13,7 +13,7 @@ class NeuroBlueprintValidator(Validator): """PLACEHOLDER.""" - + def __init__(self, prefix: Prefix, parent: CreateFoldersTab) -> None: """Custom Validator() class that takes sub / ses prefix as input. Runs validation of diff --git a/datashuttle/utils/custom_exceptions.py b/datashuttle/utils/custom_exceptions.py index 3fe7e3368..47480d57b 100644 --- a/datashuttle/utils/custom_exceptions.py +++ b/datashuttle/utils/custom_exceptions.py @@ -1,10 +1,10 @@ class ConfigError(Exception): """PLACEHOLDER.""" - + pass class NeuroBlueprintError(Exception): """PLACEHOLDER.""" - + pass diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index bfd092bc0..4351834c1 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -112,7 +112,7 @@ def __init__( def build_a_list_of_all_files_and_folders_to_transfer(self) -> List[str]: """Build a list of every file to transfer based on the user-passed arguments. - + This cycles through every subject, session and datatype and adds the outputs to three lists: diff --git a/datashuttle/utils/folders.py b/datashuttle/utils/folders.py index 3a3dd3372..fb0e3e0f1 100644 --- a/datashuttle/utils/folders.py +++ b/datashuttle/utils/folders.py @@ -49,10 +49,10 @@ def create_folder_trees( ---------- cfg datashuttle config UserDict - + top_level_folder either "rawdata" or "derivatives" - + sub_names, ses_names, datatype see create_folders() @@ -391,7 +391,7 @@ def search_for_wildcards( ---------- cfg datashuttle configs - + project initialised datashuttle project @@ -466,7 +466,7 @@ def search_sub_or_ses_level( arguments, but this is not nice and breaks the general rule that these functions should operate project-agnostic. - + base_folder the path to the base folder. If sub is None, the search is performed on this folder @@ -490,7 +490,7 @@ def search_sub_or_ses_level( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. - + return_full_path include the search_path in the returned paths @@ -534,7 +534,7 @@ def search_for_folders( ---------- cfg datashuttle configs - + local_or_central "local" or "central" diff --git a/datashuttle/utils/ssh.py b/datashuttle/utils/ssh.py index 9b80f7d4c..568773438 100644 --- a/datashuttle/utils/ssh.py +++ b/datashuttle/utils/ssh.py @@ -314,7 +314,7 @@ def get_list_of_folder_names_over_sftp( verbose If `True`, if a search folder cannot be found, a message will be printed with the un-found path. - + return_full_path include the search_path in the returned paths diff --git a/pyproject.toml b/pyproject.toml index 63d9cd3c8..101f9e0e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,7 +141,7 @@ select = [ "I", # isort "E", # pycodestyle errors "F", # Pyflakes - "TC", # flake8-type-checking + "TCH", # flake8-type-checking "TID252", # flake8-tidy-imports relative-imports "D", # pydocstyle ] diff --git a/tests/tests_integration/test_configs.py b/tests/tests_integration/_test_configs.py similarity index 100% rename from tests/tests_integration/test_configs.py rename to tests/tests_integration/_test_configs.py diff --git a/tests/tests_integration/base.py b/tests/tests_integration/base.py index 75c01adf2..b28f8cec4 100644 --- a/tests/tests_integration/base.py +++ b/tests/tests_integration/base.py @@ -11,7 +11,7 @@ class BaseTest: """PLACEHOLDER.""" - + @pytest.fixture(scope="function") def no_cfg_project(test): """Fixture that creates an empty project. Ignore the warning diff --git a/tests/tests_integration/test_create_folders.py b/tests/tests_integration/test_create_folders.py index af9f7e6d2..e82815c9f 100644 --- a/tests/tests_integration/test_create_folders.py +++ b/tests/tests_integration/test_create_folders.py @@ -14,7 +14,7 @@ class TestCreateFolders(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_generate_folders_default_ses(self, project): """Make a subject folders with full tree. Don't specify diff --git a/tests/tests_integration/test_filesystem_transfer.py b/tests/tests_integration/test_filesystem_transfer.py index d35b0e84e..d06ddc4d6 100644 --- a/tests/tests_integration/test_filesystem_transfer.py +++ b/tests/tests_integration/test_filesystem_transfer.py @@ -14,7 +14,7 @@ class TestFileTransfer(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize( "top_level_folder", canonical_folders.get_top_level_folders() ) diff --git a/tests/tests_integration/test_formatting.py b/tests/tests_integration/test_formatting.py index 005b7cbf7..cf5dffafe 100644 --- a/tests/tests_integration/test_formatting.py +++ b/tests/tests_integration/test_formatting.py @@ -7,7 +7,7 @@ class TestFormatting(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize("prefix", ["sub", "ses"]) @pytest.mark.parametrize( "input", [1, {"test": "one"}, 1.0, ["1", "2", ["three"]]] diff --git a/tests/tests_integration/test_local_only_mode.py b/tests/tests_integration/test_local_only_mode.py index bcc42394b..0edc28b6f 100644 --- a/tests/tests_integration/test_local_only_mode.py +++ b/tests/tests_integration/test_local_only_mode.py @@ -14,7 +14,7 @@ class TestLocalOnlyProject(BaseTest): """PLACEHOLDER.""" - + def test_bad_setup(self, tmp_path): """Test setup without providing both central_path and connection method (distinguishing a full vs local-only project). diff --git a/tests/tests_integration/test_logging.py b/tests/tests_integration/test_logging.py index ecf68b49e..e5196c283 100644 --- a/tests/tests_integration/test_logging.py +++ b/tests/tests_integration/test_logging.py @@ -19,7 +19,7 @@ class TestLogging: """PLACEHOLDER.""" - + @pytest.fixture(scope="function") def teardown_logger(self): """Ensure the logger is deleted at the end of each test.""" diff --git a/tests/tests_integration/test_settings.py b/tests/tests_integration/test_settings.py index 27f9eb7b6..570f26dfb 100644 --- a/tests/tests_integration/test_settings.py +++ b/tests/tests_integration/test_settings.py @@ -12,7 +12,7 @@ class TestPersistentSettings(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize("project", ["local", "full"], indirect=True) def test_persistent_settings_name_templates(self, project): """Test the 'name_templates' option that is stored in persistent diff --git a/tests/tests_integration/test_ssh_file_transfer.py b/tests/tests_integration/test_ssh_file_transfer.py index 0f6f46bcd..aa8a8d20c 100644 --- a/tests/tests_integration/test_ssh_file_transfer.py +++ b/tests/tests_integration/test_ssh_file_transfer.py @@ -14,7 +14,7 @@ class TestFileTransfer: """PLACEHOLDER.""" - + @pytest.fixture( scope="class", params=[ # Set running SSH or local filesystem (see docstring). diff --git a/tests/tests_integration/test_transfer_checks.py b/tests/tests_integration/test_transfer_checks.py index e443cfae8..145ddc0f6 100644 --- a/tests/tests_integration/test_transfer_checks.py +++ b/tests/tests_integration/test_transfer_checks.py @@ -11,7 +11,7 @@ class TestTransferChecks(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize( "top_level_folders", [["rawdata", "derivatives"], ["rawdata"], ["derivatives"]], diff --git a/tests/tests_integration/test_validation.py b/tests/tests_integration/test_validation.py index 4827152df..d46fa321d 100644 --- a/tests/tests_integration/test_validation.py +++ b/tests/tests_integration/test_validation.py @@ -15,7 +15,7 @@ class TestValidation(BaseTest): """PLACEHOLDER.""" - + @pytest.mark.parametrize( "sub_name", ["sub-001", "sub-999_@DATE@", "sub-001_random-tag_another-tag"], diff --git a/tests/tests_tui/test_local_only_project.py b/tests/tests_tui/test_local_only_project.py index 8802aee8c..0ac50e587 100644 --- a/tests/tests_tui/test_local_only_project.py +++ b/tests/tests_tui/test_local_only_project.py @@ -6,7 +6,7 @@ class TestTuiLocalOnlyProject(TuiBase): """PLACEHOLDER.""" - + @pytest.mark.asyncio async def test_local_only_make_project( self, diff --git a/tests/tests_tui/test_tui_configs.py b/tests/tests_tui/test_tui_configs.py index 1fa7802ad..8e27a6701 100644 --- a/tests/tests_tui/test_tui_configs.py +++ b/tests/tests_tui/test_tui_configs.py @@ -15,7 +15,7 @@ class TestTuiConfigs(TuiBase): """PLACEHOLDER.""" - + # ------------------------------------------------------------------------- # Test New Project Configs # ------------------------------------------------------------------------- diff --git a/tests/tests_tui/test_tui_create_folders.py b/tests/tests_tui/test_tui_create_folders.py index 736f4c63d..d05615439 100644 --- a/tests/tests_tui/test_tui_create_folders.py +++ b/tests/tests_tui/test_tui_create_folders.py @@ -14,7 +14,7 @@ class TestTuiCreateFolders(TuiBase): """PLACEHOLDER.""" - + # ------------------------------------------------------------------------- # General test Create Folders # ------------------------------------------------------------------------- diff --git a/tests/tests_tui/test_tui_logging.py b/tests/tests_tui/test_tui_logging.py index 21e288a12..adce58e7c 100644 --- a/tests/tests_tui/test_tui_logging.py +++ b/tests/tests_tui/test_tui_logging.py @@ -8,7 +8,7 @@ class TestTuiLogging(TuiBase): """PLACEHOLDER.""" - + @pytest.mark.asyncio async def test_logging(self, setup_project_paths): """Test logging by running some commands, checking they From e0c4130a5568ccc6c70f86698318b5c493433882 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:40:32 +0000 Subject: [PATCH 17/25] renaming TCH to TC --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 101f9e0e1..63d9cd3c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,7 +141,7 @@ select = [ "I", # isort "E", # pycodestyle errors "F", # Pyflakes - "TCH", # flake8-type-checking + "TC", # flake8-type-checking "TID252", # flake8-tidy-imports relative-imports "D", # pydocstyle ] From 26ddc29a5ad863abf226e1b46f99e76ca11af43e Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:42:50 +0000 Subject: [PATCH 18/25] moving Iterable import into type checking block to comply with TC003 --- datashuttle/tui/custom_widgets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 9a303b0da..01f73e9e0 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -1,6 +1,5 @@ from __future__ import annotations -from collections.abc import Iterable from typing import ( TYPE_CHECKING, List, @@ -10,6 +9,8 @@ ) if TYPE_CHECKING: + from collections.abc import Iterable + from textual import events from textual.validation import Validator From c82608cca957fe1e387619191ae18913b612b995 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:47:09 +0000 Subject: [PATCH 19/25] re-arranging rules --- datashuttle/tui/custom_widgets.py | 2 +- pyproject.toml | 46 +++++++++---------------------- 2 files changed, 14 insertions(+), 34 deletions(-) diff --git a/datashuttle/tui/custom_widgets.py b/datashuttle/tui/custom_widgets.py index 01f73e9e0..5118b1f5b 100644 --- a/datashuttle/tui/custom_widgets.py +++ b/datashuttle/tui/custom_widgets.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: from collections.abc import Iterable - + from textual import events from textual.validation import Validator diff --git a/pyproject.toml b/pyproject.toml index 63d9cd3c8..3483bdda2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,39 +97,6 @@ fix = true [tool.ruff.lint] # See https://docs.astral.sh/ruff/rules/ -#ignore = [ -# "D203", # one blank line before class -# "D213", # multi-line-summary second line -# "D401", # first line of docstrings should be in an imperative mood -# "E501", # limit lines to 79 characters -#] -#select = [ -# "E", # pycodestyle errors -# "F", # Pyflakes -# "UP", # pyupgrade -# "I", # isort -# "B", # flake8 bugbear -# "SIM", # flake8 simplify -# "C90", # McCabe complexity -# "D", # pydocstyle -#] -per-file-ignores = { "tests/*" = [ - "D100", # missing docstring in public module - "D205", # missing blank line between summary and description - "D103", # missing docstring in public function -], "examples/*" = [ - "D400", # first line should end with a period. - "D415", # first line should end with a period, question mark... - "D205", # missing blank line between summary and description -], "__init__.py" = [ - # This was part of the old config - # Is this needed? __init__.py is already part of tool.ruff.exclude - "F401", # auto remove unused imports -] } - -# Old ruff ruleset + pydocstyle added -# Inconsistent with movement repo, but saving this here for -# now in case there are good reasons to keep these rules ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", "D100", # missing docstring in public module (not enforced FOR NOW) "D203", # one blank line before class @@ -145,6 +112,19 @@ select = [ "TID252", # flake8-tidy-imports relative-imports "D", # pydocstyle ] +per-file-ignores = { "tests/*" = [ + "D100", # missing docstring in public module + "D205", # missing blank line between summary and description + "D103", # missing docstring in public function +], "examples/*" = [ + "D400", # first line should end with a period. + "D415", # first line should end with a period, question mark... + "D205", # missing blank line between summary and description +], "__init__.py" = [ + # This was part of the old config + # Is this needed? __init__.py is already part of tool.ruff.exclude + "F401", # auto remove unused imports +] } [tool.ruff.format] docstring-code-format = true # Also format code in docstrings From c62c887f9f079dbf1c9509c89140c47383398ac4 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Fri, 21 Mar 2025 15:51:27 +0000 Subject: [PATCH 20/25] Adding informative comment about ruff config --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 3483bdda2..ee024ea62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,13 @@ line-length = 79 exclude = ["__init__.py","build",".eggs"] fix = true + + +# Ruff config is not exactly the same as in the movement repo, as +# currently we are only adding linting enforcement to docstrings. +# Other ruff rules that are also present in movement repo can be +# added here in a separate PR after these changes have been merged. + [tool.ruff.lint] # See https://docs.astral.sh/ruff/rules/ ignore = ["E203","E501","E731","C901","W291","W293","E402","E722", From 4f37efcdb7dff9d47d664c510764178881523a64 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Sun, 23 Mar 2025 14:25:55 +0000 Subject: [PATCH 21/25] Filling in PLACEHOLDER dosctrings in configs and utils --- datashuttle/configs/canonical_configs.py | 8 +++-- datashuttle/configs/canonical_folders.py | 10 ++++-- datashuttle/configs/config_class.py | 10 +++--- datashuttle/configs/links.py | 8 ++--- datashuttle/datashuttle_class.py | 41 ++++++++++++------------ datashuttle/utils/custom_exceptions.py | 4 +-- datashuttle/utils/data_transfer.py | 2 +- datashuttle/utils/ds_logger.py | 6 ++-- datashuttle/utils/folder_class.py | 11 ++++++- datashuttle/utils/formatting.py | 6 ++-- datashuttle/utils/rclone.py | 2 +- datashuttle/utils/utils.py | 4 +-- datashuttle/utils/validation.py | 2 +- 13 files changed, 65 insertions(+), 49 deletions(-) diff --git a/datashuttle/configs/canonical_configs.py b/datashuttle/configs/canonical_configs.py index 5fab1aaa2..61bb2d5e8 100644 --- a/datashuttle/configs/canonical_configs.py +++ b/datashuttle/configs/canonical_configs.py @@ -156,7 +156,9 @@ def raise_on_bad_local_only_project_configs(config_dict: Configs) -> None: def local_only_configs_are_none(config_dict: Configs) -> list[bool]: - """PLACEHOLDER.""" + """Check if the central_path and connection_method config options + are set to `None`. + """ return [ config_dict[key] is None for key in ["central_path", "connection_method"] @@ -261,7 +263,7 @@ def get_tui_config_defaults() -> Dict: def get_name_templates_defaults() -> Dict: - """PLACEHOLDER.""" + """Get the default values for name_templates.""" return {"name_templates": {"on": False, "sub": None, "ses": None}} @@ -290,7 +292,7 @@ def get_datatypes() -> List[str]: def get_broad_datatypes(): - """PLACEHOLDER.""" + """Return a list of broad datatypes.""" return ["ephys", "behav", "funcimg", "anat"] diff --git a/datashuttle/configs/canonical_folders.py b/datashuttle/configs/canonical_folders.py index cc3db158d..c2c90065c 100644 --- a/datashuttle/configs/canonical_folders.py +++ b/datashuttle/configs/canonical_folders.py @@ -25,12 +25,16 @@ def get_datatype_folders() -> dict: The value is a Folder() class instance with the required fields - name : The display name for the datatype, that will + Parameters + ---------- + name + The display name for the datatype, that will be used for making and transferring files in practice. This should always match the canonical name, but left as an option for rare cases in which advanced users want to change it. - level : "sub" or "ses", level to make the folder at. + level + "sub" or "ses", level to make the folder at. """ return { @@ -71,7 +75,7 @@ def canonical_reserved_keywords() -> List[str]: def get_top_level_folders() -> List[TopLevelFolder]: - """PLACEHOLDER.""" + """Return a list of the different top level folder names.""" return ["rawdata", "derivatives"] diff --git a/datashuttle/configs/config_class.py b/datashuttle/configs/config_class.py index c7b71b8cc..072297154 100644 --- a/datashuttle/configs/config_class.py +++ b/datashuttle/configs/config_class.py @@ -60,13 +60,15 @@ def __init__( self.project_metadata_path: Path def setup_after_load(self) -> None: - """PLACEHOLDER.""" + """Setup the config after loading it.""" load_configs.convert_str_and_pathlib_paths(self, "str_to_path") self.ensure_local_and_central_path_end_in_project_name() self.check_dict_values_raise_on_fail() def ensure_local_and_central_path_end_in_project_name(self): - """PLACEHOLDER.""" + """Ensure that the local and central path end in the name of + the project. + """ for path_type in ["local_path", "central_path"]: if path_type == "central_path" and self[path_type] is None: continue @@ -157,7 +159,7 @@ def build_project_path( if isinstance(sub_folders, list): sub_folders_str = "/".join(sub_folders) else: - sub_folders_str = cast(str, sub_folders) + sub_folders_str = cast("str", sub_folders) sub_folders_path = Path(sub_folders_str) @@ -231,7 +233,7 @@ def make_rclone_transfer_options( } def init_paths(self) -> None: - """PLACEHOLDER.""" + """Initiate the datashuttle paths.""" self.project_metadata_path = self["local_path"] / ".datashuttle" datashuttle_path, _ = canonical_folders.get_project_datashuttle_path( diff --git a/datashuttle/configs/links.py b/datashuttle/configs/links.py index e61ade29a..2daa03013 100644 --- a/datashuttle/configs/links.py +++ b/datashuttle/configs/links.py @@ -1,18 +1,18 @@ def get_docs_link(): - """PLACEHOLDER.""" + """Return the link to the datashuttle page.""" return "https://datashuttle.neuroinformatics.dev/" def get_github_link(): - """PLACEHOLDER.""" + """Return the link to the datashuttle repository.""" return "https://github.com/neuroinformatics-unit/datashuttle" def get_link_github_issues(): - """PLACEHOLDER.""" + """Return the link to the datashuttle repository issues page.""" return "https://github.com/neuroinformatics-unit/datashuttle/issues" def get_link_zulip(): - """PLACEHOLDER.""" + """Return the link to the datashuttle Zulip chatroom.""" return "https://neuroinformatics.zulipchat.com/#narrow/stream/405999-DataShuttle" diff --git a/datashuttle/datashuttle_class.py b/datashuttle/datashuttle_class.py index 96b344536..a07462249 100644 --- a/datashuttle/datashuttle_class.py +++ b/datashuttle/datashuttle_class.py @@ -85,27 +85,26 @@ class DataShuttle: with SSH, use setup setup_ssh_connection(). This will allow you to check the server key, add host key to profile if accepted, and setup ssh key pair. - - Parameters - ---------- - project_name - The project name to use the datashuttle - Folders containing all project files - and folders are specified in make_config_file(). - Datashuttle-related files are stored in - a .datashuttle folder in the user home - folder. Use get_datashuttle_path() to - see the path to this folder. - - print_startup_message - If `True`, a start-up message displaying the - current state of the program (e.g. persistent - settings such as the 'top-level folder') is shown. - """ def __init__(self, project_name: str, print_startup_message: bool = True): - """PLACEHOLDER.""" + """Parameters + ---------- + project_name + The project name to use the datashuttle + Folders containing all project files + and folders are specified in make_config_file(). + Datashuttle-related files are stored in + a .datashuttle folder in the user home + folder. Use get_datashuttle_path() to + see the path to this folder. + + print_startup_message + If `True`, a start-up message displaying the + current state of the program (e.g. persistent + settings such as the 'top-level folder') is shown. + + """ self._error_on_base_project_name(project_name) self.project_name = project_name ( @@ -972,7 +971,7 @@ def make_config_file( ds_logger.close_log_filehandler() def update_config_file(self, **kwargs) -> None: - """PLACEHOLDER.""" + """Update the configuration file.""" if not self.cfg: utils.log_and_raise_error( "Must have a config loaded before updating configs.", @@ -1023,7 +1022,7 @@ def get_config_path(self) -> Path: @check_configs_set def get_configs(self) -> Configs: - """PLACEHOLDER.""" + """Get the datashuttle configs.""" return self.cfg @check_configs_set @@ -1388,7 +1387,7 @@ def _move_logs_from_temp_folder(self) -> None: ) def _clear_temp_log_path(self) -> None: - """PLACEHOLDER.""" + """Delete temporary log files.""" log_files = glob.glob(str(self._temp_log_path / "*.log")) for file in log_files: os.remove(file) diff --git a/datashuttle/utils/custom_exceptions.py b/datashuttle/utils/custom_exceptions.py index 47480d57b..05be4b6d0 100644 --- a/datashuttle/utils/custom_exceptions.py +++ b/datashuttle/utils/custom_exceptions.py @@ -1,10 +1,10 @@ class ConfigError(Exception): - """PLACEHOLDER.""" + """Raise an error relating to a configuration problem.""" pass class NeuroBlueprintError(Exception): - """PLACEHOLDER.""" + """Raise an error when something doesn't conform to the NeuroBlueprint pattern.""" pass diff --git a/datashuttle/utils/data_transfer.py b/datashuttle/utils/data_transfer.py index 4351834c1..e43fd4540 100644 --- a/datashuttle/utils/data_transfer.py +++ b/datashuttle/utils/data_transfer.py @@ -350,7 +350,7 @@ def update_list_with_dtype_paths( # ------------------------------------------------------------------------- def to_list(self, names: Union[str, List[str]]) -> List[str]: - """PLACEHOLDER.""" + """Convert a name or list of names to a list.""" if isinstance(names, str): names = [names] return names diff --git a/datashuttle/utils/ds_logger.py b/datashuttle/utils/ds_logger.py index 4269b391d..140ab7c54 100644 --- a/datashuttle/utils/ds_logger.py +++ b/datashuttle/utils/ds_logger.py @@ -18,17 +18,17 @@ def get_logger_name(): - """PLACEHOLDER.""" + """Return the name of the logger.""" return "datashuttle" def get_logger(): - """PLACEHOLDER.""" + """Return the instance of the logger object.""" return logging.getLogger(get_logger_name()) def logging_is_active(): - """PLACEHOLDER.""" + """Check if the logger is active.""" logger_exists = get_logger_name() in logging.root.manager.loggerDict if logger_exists and get_logger().handlers != []: return True diff --git a/datashuttle/utils/folder_class.py b/datashuttle/utils/folder_class.py index 33bc16b74..87c036bab 100644 --- a/datashuttle/utils/folder_class.py +++ b/datashuttle/utils/folder_class.py @@ -10,6 +10,15 @@ def __init__( name: str, level: str, ): - """PLACEHOLDER.""" + """Parameters + ------------- + + name + the name of the folder. + + level + level to make the folder at. + + """ self.name = name self.level = level diff --git a/datashuttle/utils/formatting.py b/datashuttle/utils/formatting.py index 5afdae8c2..5177ff35e 100644 --- a/datashuttle/utils/formatting.py +++ b/datashuttle/utils/formatting.py @@ -278,17 +278,17 @@ def replace_date_time_tags_in_name( def format_date(date: str) -> str: - """PLACEHOLDER.""" + """Format the `date` as `date-`.""" return f"date-{date}" def format_time(time_: str) -> str: - """PLACEHOLDER.""" + """Format the `time_` as `time-`.""" return f"time-{time_}" def format_datetime(date: str, time_: str) -> str: - """PLACEHOLDER.""" + """Format the `date` and `time_` as `datetime-T`.""" return f"datetime-{date}T{time_}" diff --git a/datashuttle/utils/rclone.py b/datashuttle/utils/rclone.py index d651b8f96..e3fb8bcdb 100644 --- a/datashuttle/utils/rclone.py +++ b/datashuttle/utils/rclone.py @@ -112,7 +112,7 @@ def setup_rclone_config_for_ssh( def log_rclone_config_output(): - """PLACEHOLDER.""" + """Log the output from creating Rclone config.""" output = call_rclone("config file", pipe_std=True) utils.log( f"Successfully created rclone config. {output.stdout.decode('utf-8')}" diff --git a/datashuttle/utils/utils.py b/datashuttle/utils/utils.py index 56dd43670..8e51d9185 100644 --- a/datashuttle/utils/utils.py +++ b/datashuttle/utils/utils.py @@ -159,7 +159,7 @@ def get_values_from_bids_formatted_name( def sub_or_ses_value_to_int(value: str) -> int: - """PLACEHOLDER.""" + """Convert a subject or session value to an integer.""" try: int_value = int(value) except ValueError: @@ -184,7 +184,7 @@ def get_value_from_key_regexp(name: str, key: str) -> List[str]: def integers_are_consecutive(list_of_ints: List[int]) -> bool: - """PLACEHOLDER.""" + """Check if a list of integers is consecutive.""" diff_between_ints = diff(list_of_ints) return all([diff == 1 for diff in diff_between_ints]) diff --git a/datashuttle/utils/validation.py b/datashuttle/utils/validation.py index b4e89974d..4f6ca09f6 100644 --- a/datashuttle/utils/validation.py +++ b/datashuttle/utils/validation.py @@ -361,7 +361,7 @@ def names_include_special_characters( def name_has_special_character(name: str) -> bool: - """PLACEHOLDER.""" + """Check if the name contains special characters.""" return not re.match("^[A-Za-z0-9_-]*$", name) From 419cea635a4e10ffb8f85c792cf9ba8b817451e1 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Sun, 23 Mar 2025 14:38:12 +0000 Subject: [PATCH 22/25] syncing branch with main and adding PLACEHOLDER docstring --- tests/tests_tui/test_tui_directorytree.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests_tui/test_tui_directorytree.py b/tests/tests_tui/test_tui_directorytree.py index 8c07e4716..6618dc3b9 100644 --- a/tests/tests_tui/test_tui_directorytree.py +++ b/tests/tests_tui/test_tui_directorytree.py @@ -244,6 +244,7 @@ def set_signal_to_path(path_): async def test_create_folders_directorytree_rename( self, setup_project_paths ): + """PLACEHOLDER.""" tmp_config_path, tmp_path, project_name = setup_project_paths.values() rawdata_path = tmp_path / "local" / project_name / "rawdata" From b1b5d9d9bb83899e5ff7854f60616dd25396a168 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Sun, 23 Mar 2025 14:44:35 +0000 Subject: [PATCH 23/25] enabling ruff-format --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1dbbe5cba..fe9386ed8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: rev: v0.9.9 hooks: - id: ruff - #- id: ruff-format + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.15.0 hooks: From 4e2e70a55587ede3bb6c49c23d53209ee5d15f42 Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Sun, 23 Mar 2025 14:47:01 +0000 Subject: [PATCH 24/25] pre-commit autofix --- tests/tests_integration/_test_configs.py | 2 +- tests/tests_tui/test_tui_directorytree.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/tests_integration/_test_configs.py b/tests/tests_integration/_test_configs.py index 32f59beec..9d983acf5 100644 --- a/tests/tests_integration/_test_configs.py +++ b/tests/tests_integration/_test_configs.py @@ -57,7 +57,7 @@ def test_warning_on_startup(self, no_cfg_project): ) @pytest.mark.parametrize("path_type", ["local_path", "central_path"]) def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): - """"~", "." and "../" syntax is not supported because + """ "~", "." and "../" syntax is not supported because it does not work with rclone. Theoretically it could be supported by checking for "." etc. and filling in manually, but it does not seem robust. diff --git a/tests/tests_tui/test_tui_directorytree.py b/tests/tests_tui/test_tui_directorytree.py index 6618dc3b9..73f340d24 100644 --- a/tests/tests_tui/test_tui_directorytree.py +++ b/tests/tests_tui/test_tui_directorytree.py @@ -251,7 +251,6 @@ async def test_create_folders_directorytree_rename( app = TuiApp() async with app.run_test(size=self.tui_size()) as pilot: - # Set up the 'create tab' with loaded nodes await self.setup_existing_project_create_tab_filled_sub_and_ses( pilot, project_name, create_folders=True From 7881a786df4745968a25b447b98af8532f76621e Mon Sep 17 00:00:00 2001 From: MoffittAndrew Date: Sun, 23 Mar 2025 14:55:25 +0000 Subject: [PATCH 25/25] pre-commit autofix --- tests/tests_integration/_test_configs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tests_integration/_test_configs.py b/tests/tests_integration/_test_configs.py index 9d983acf5..d96284f28 100644 --- a/tests/tests_integration/_test_configs.py +++ b/tests/tests_integration/_test_configs.py @@ -57,9 +57,9 @@ def test_warning_on_startup(self, no_cfg_project): ) @pytest.mark.parametrize("path_type", ["local_path", "central_path"]) def test_bad_path_syntax(self, project, bad_pattern, path_type, tmp_path): - """ "~", "." and "../" syntax is not supported because + """`~`, `.` and `../` syntax is not supported because it does not work with rclone. Theoretically it - could be supported by checking for "." etc. and + could be supported by checking for `.` etc. and filling in manually, but it does not seem robust. Here check an error is raised when path contains