diff --git a/src/uiucprescon/__init__.py b/src/uiucprescon/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/uiucprescon/tripwire/gui/__init__.py b/src/uiucprescon/tripwire/gui/__init__.py index e69de29..c3d8f36 100644 --- a/src/uiucprescon/tripwire/gui/__init__.py +++ b/src/uiucprescon/tripwire/gui/__init__.py @@ -0,0 +1 @@ +"""Gui subpackage.""" diff --git a/src/uiucprescon/tripwire/gui/__main__.py b/src/uiucprescon/tripwire/gui/__main__.py index 838f98c..81b23a2 100644 --- a/src/uiucprescon/tripwire/gui/__main__.py +++ b/src/uiucprescon/tripwire/gui/__main__.py @@ -1,3 +1,5 @@ +"""Main entry point for gui.""" + from .main import main if __name__ == '__main__': diff --git a/src/uiucprescon/tripwire/main.py b/src/uiucprescon/tripwire/main.py index e466688..de1241a 100644 --- a/src/uiucprescon/tripwire/main.py +++ b/src/uiucprescon/tripwire/main.py @@ -21,7 +21,6 @@ logger = logging.getLogger(__name__) -print("hello wolrd") def capture_log( logger: logging.Logger, diff --git a/src/uiucprescon/tripwire/manifest_check.py b/src/uiucprescon/tripwire/manifest_check.py index ff7bc20..73ac726 100644 --- a/src/uiucprescon/tripwire/manifest_check.py +++ b/src/uiucprescon/tripwire/manifest_check.py @@ -6,6 +6,8 @@ import typing from typing import ( Mapping, + Callable, + Optional, Set, MutableMapping, TextIO, @@ -303,7 +305,7 @@ def __init__(self, search_path: pathlib.Path): self.cache: typing.Dict[str, PackageScanner.Cache] = {} self._scanner = PackageScanner.scanner_klass(self.search_path) - def locate(self, file_name: str) -> typing.Optional[pathlib.Path]: + def locate(self, file_name: str) -> Optional[pathlib.Path]: if file_name in self.cache: self.cache[file_name]["expected"] = True return self.cache[file_name]["location"] / file_name @@ -401,3 +403,59 @@ def locate_manifest_files( "Files found that were not included in manifest: \n%s", "\n".join(list_of_unexpected_files), ) + + +class ValidateBlockerException(StopIteration): + pass + + +def check_utf_8(file_path: pathlib.Path) -> Optional[str]: + try: + with file_path.open("r", encoding="utf-8") as fp: + fp.read() + return None + except UnicodeDecodeError: + raise ValidateBlockerException("File not utf-8 encoded") + + +def is_manifest_tsv( + file_path: pathlib.Path, test_strategy=get_manifest_type +) -> Optional[str]: + try: + with file_path.open("r", encoding="utf-8") as fp: + test_strategy(fp=fp) + return None + except (ValueError, AttributeError): + return ( + "Unable to determine TSV manifest type. A manifest file should " + "be a tsv file that contains a single row of column header." + ) + + +# The MANIFEST_VALIDATION_CHECKS is a list of callable functions that +# return a error string or raise a ValidateBlockerException containing the +# error. +# +# The function returns error string if there are additional checks after +# current one that are worth performing as well. +# The function raises ValidateBlockerException exception if no additional +# checks should be performed. +# +# For example, if the file is the wrong type of file, an exception should +# be raised because there is no reason to inspecting the contents of this file. +MANIFEST_VALIDATION_CHECKS: List[Callable[[pathlib.Path], Optional[str]]] = [ + check_utf_8, + is_manifest_tsv, +] + + +def get_issues_with_manifest_file(manifest_tsv_file: str, checks=None): + issues = set() + for check in checks if checks is not None else MANIFEST_VALIDATION_CHECKS: + try: + if issue := check(pathlib.Path(manifest_tsv_file)): + issues.add(issue) + except ValidateBlockerException as e: + issues.add(e) + return issues + return issues diff --git a/src/uiucprescon/tripwire/speedwagon_plugin/__init__.py b/src/uiucprescon/tripwire/speedwagon_plugin/__init__.py index e69de29..4883715 100644 --- a/src/uiucprescon/tripwire/speedwagon_plugin/__init__.py +++ b/src/uiucprescon/tripwire/speedwagon_plugin/__init__.py @@ -0,0 +1 @@ +"""Speedwagon plugin.""" diff --git a/src/uiucprescon/tripwire/speedwagon_plugin/workflows.py b/src/uiucprescon/tripwire/speedwagon_plugin/workflows.py index c19a743..eb2697c 100644 --- a/src/uiucprescon/tripwire/speedwagon_plugin/workflows.py +++ b/src/uiucprescon/tripwire/speedwagon_plugin/workflows.py @@ -19,6 +19,7 @@ from speedwagon import Workflow from speedwagon.tasks import Result, TaskBuilder import speedwagon.workflow +from speedwagon.exceptions import SpeedwagonException from speedwagon.validators import JobOptionsType from speedwagon.workflow import AbsOutputOptionDataType import speedwagon.validators @@ -233,10 +234,19 @@ class ManifestCheckWorkflow(Workflow): name = "manifest-check" def create_new_task(self, task_builder: TaskBuilder, job_args) -> None: - super().create_new_task(task_builder, job_args) + """Create a new task to validate manifest file.""" + task_builder.add_subtask( + locate_manifest_files_task( + manifest_tsv_file=job_args["manifest"], + search_path=job_args["search_path"], + ) + ) - # active = False - # TODO: implement manifest-check workflow + def initial_task(self, task_builder: TaskBuilder, user_args: _T) -> None: + """Validate manifest file before running full validation.""" + task_builder.add_subtask( + validate_manifest_file_task(user_args["Manifest tsv File"]) + ) def discover_task_metadata( self, @@ -254,6 +264,8 @@ def discover_task_metadata( Returns: sequence of task metadata """ + if initial_results[0].data is False: + return [] return [ { "manifest": user_args["Manifest tsv File"], @@ -274,13 +286,37 @@ def job_options(self) -> List[AbsOutputOptionDataType]: return [manifest_tsv_file, search_path] +@speedwagon.tasks.workflow_task("validate manifest", logger=logger) +def validate_manifest_file_task( + manifest_tsv_file: str, + checking_strategy=manifest_check.get_issues_with_manifest_file, +) -> bool: + if issues := checking_strategy(manifest_tsv_file): + error_report_list = "\n".join([f" * {issue}" for issue in issues]) + report = f"""Unable to continue due to issues found with manifest file: + {error_report_list} + """ + logger.error(report) + return False + return True + + @speedwagon.tasks.workflow_task( "Locate Files in Manifest", logger=manifest_check.logger ) -def locate_manifest_files_task(manifest_tsv_file: str, search_path: str): - manifest_check.locate_manifest_files( - pathlib.Path(manifest_tsv_file), pathlib.Path(search_path) - ) +def locate_manifest_files_task( + manifest_tsv_file: str, + search_path: str, + locating_strategy=manifest_check.locate_manifest_files, +): + try: + locating_strategy( + pathlib.Path(manifest_tsv_file), pathlib.Path(search_path) + ) + except UnicodeDecodeError as e: + raise SpeedwagonException( + "Check the manifest file is encoded in utf-8" + ) from e @speedwagon.tasks.workflow_task("Show Metadata", logger=logger) diff --git a/src/uiucprescon/tripwire/validation.py b/src/uiucprescon/tripwire/validation.py index fbab68d..ad7fcaa 100644 --- a/src/uiucprescon/tripwire/validation.py +++ b/src/uiucprescon/tripwire/validation.py @@ -276,7 +276,8 @@ def validate_directory_checksums_command( target_file.relative_to(path), ) if not os.path.exists(target_file): - message = f'Expected file "{target_file.relative_to(path)}" - Failed: File does not exist' + message = (f'Expected file "{target_file.relative_to(path)}" - ' + f'Failed: File does not exist') logger.error(message) errors.append(message) continue diff --git a/tests/speedwagon/test_workflows.py b/tests/speedwagon/test_workflows.py index fecb579..b9b855f 100644 --- a/tests/speedwagon/test_workflows.py +++ b/tests/speedwagon/test_workflows.py @@ -261,3 +261,25 @@ def test_create_new_task(self): } workflow.create_new_task(task_builder,job_args) task_builder.add_subtask.assert_called_once() + +@pytest.mark.parametrize("return_values,expected", [ + ([], True), + (["issue1"], False), +]) +def test_validate_manifest_file_task(return_values,expected): + task = tripwire_workflows.validate_manifest_file_task( + manifest_tsv_file="some_manifest.tsv", + checking_strategy=Mock(return_value=return_values), + ) + task.work() + assert task.task_result.data is expected + +def test_locate_manifest_files_task(): + locating_strategy= Mock() + task = tripwire_workflows.locate_manifest_files_task( + manifest_tsv_file="some_manifest.tsv", + search_path="/path/to/search", + locating_strategy=locating_strategy + ) + task.work() + locating_strategy.assert_called_once() \ No newline at end of file diff --git a/tests/test_manifest_check.py b/tests/test_manifest_check.py index 877e2d6..9d5fe0e 100644 --- a/tests/test_manifest_check.py +++ b/tests/test_manifest_check.py @@ -1,5 +1,6 @@ import io import pathlib +from os import name from unittest.mock import Mock, MagicMock import pytest @@ -219,3 +220,61 @@ def test_verify_format_type(self, data, klass, expected): def test_get_manifest_type_unknown_throws(self): with pytest.raises(ValueError): manifest_check.get_manifest_type(io.StringIO("unknown format")) + +def test_check_utf_8_passed(): + manifest_tsv = Mock(open=MagicMock()) + assert manifest_check.check_utf_8(manifest_tsv) is None + +def test_check_utf_8_failed(monkeypatch): + mocked_open = MagicMock(name="open", + __enter__=Mock( + return_value=Mock( + name="__enter__", + read=Mock( + name="read", side_effect=lambda :b'\x80'.decode("utf8") + ) + ) + ) + ) + mocked_open.read.side_effect = UnicodeDecodeError("utf-8", b"test", 0, 1, "") + manifest_tsv = Mock( + open=Mock(return_value=mocked_open) + ) + with pytest.raises(manifest_check.ValidateBlockerException): + assert manifest_check.check_utf_8(manifest_tsv) is None + +def test_is_manifest_tsv_passes(): + manifest_tsv = Mock(open=MagicMock()) + assert manifest_check.is_manifest_tsv(manifest_tsv, test_strategy=lambda *_, **__: True) is None + +def test_is_manifest_tsv_fails(): + manifest_tsv = Mock(open=MagicMock()) + assert manifest_check.is_manifest_tsv(manifest_tsv, test_strategy=Mock(side_effect=AttributeError)) is not None + +def test_get_issues_with_manifest_file_no_issues(): + assert manifest_check.get_issues_with_manifest_file("some_manifest_file", checks=[]) == set() + +def test_get_issues_with_manifest_file_that_returns(): + checks = [ + Mock(return_value="Issue found") + ] + assert len(manifest_check.get_issues_with_manifest_file("some_manifest_file", checks=checks)) > 0 + + +def test_get_issues_with_manifest_file_that_throws(): + checks = [ + Mock(side_effect=manifest_check.ValidateBlockerException("Issue found")) + ] + assert len(manifest_check.get_issues_with_manifest_file("some_manifest_file", checks=checks)) > 0 + + + +def test_get_issues_with_manifest_file_throws_stops_other_checks(): + other_check = Mock(return_value="Should not be called") + checks = [ + Mock(side_effect=manifest_check.ValidateBlockerException("Issue found")), + other_check + ] + assert len(manifest_check.get_issues_with_manifest_file("some_manifest_file", checks=checks)) > 0 + other_check.assert_not_called() +