Manifest check speedwagon

src/uiucprescon/tripwire/gui/__init__.py

@@ -0,0 +1 @@
+"""Gui subpackage."""

src/uiucprescon/tripwire/gui/__main__.py

@@ -1,3 +1,5 @@
+"""Main entry point for gui."""
+
 from .main import main
 
 if __name__ == '__main__':

src/uiucprescon/tripwire/main.py

@@ -21,7 +21,6 @@
 
 logger = logging.getLogger(__name__)
 
-print("hello wolrd")
 
 def capture_log(
     logger: logging.Logger,

src/uiucprescon/tripwire/manifest_check.py

@@ -6,6 +6,8 @@
 import typing
 from typing import (
     Mapping,
+    Callable,
+    Optional,
     Set,
     MutableMapping,
     TextIO,
@@ -303,7 +305,7 @@ def __init__(self, search_path: pathlib.Path):
         self.cache: typing.Dict[str, PackageScanner.Cache] = {}
         self._scanner = PackageScanner.scanner_klass(self.search_path)
 
-    def locate(self, file_name: str) -> typing.Optional[pathlib.Path]:
+    def locate(self, file_name: str) -> Optional[pathlib.Path]:
         if file_name in self.cache:
             self.cache[file_name]["expected"] = True
             return self.cache[file_name]["location"] / file_name
@@ -401,3 +403,59 @@ def locate_manifest_files(
             "Files found that were not included in manifest: \n%s",
             "\n".join(list_of_unexpected_files),
         )
+
+
+class ValidateBlockerException(StopIteration):
+    pass
+
+
+def check_utf_8(file_path: pathlib.Path) -> Optional[str]:
+    try:
+        with file_path.open("r", encoding="utf-8") as fp:
+            fp.read()
+            return None
+    except UnicodeDecodeError:
+        raise ValidateBlockerException("File not utf-8 encoded")
+
+
+def is_manifest_tsv(
+    file_path: pathlib.Path, test_strategy=get_manifest_type
+) -> Optional[str]:
+    try:
+        with file_path.open("r", encoding="utf-8") as fp:
+            test_strategy(fp=fp)
+        return None
+    except (ValueError, AttributeError):
+        return (
+            "Unable to determine TSV manifest type. A manifest file should "
+            "be a tsv file that contains a single row of column header."
+        )
+
+
+# The MANIFEST_VALIDATION_CHECKS is a list of callable functions that
+# return a error string or raise a ValidateBlockerException containing the
+# error.
+#
+# The function returns error string if there are additional checks after
+# current one that are worth performing as well.
+# The function raises ValidateBlockerException exception if no additional
+# checks should be performed.
+#
+# For example, if the file is the wrong type of file, an exception should
+# be raised because there is no reason to inspecting the contents of this file.
+MANIFEST_VALIDATION_CHECKS: List[Callable[[pathlib.Path], Optional[str]]] = [
+    check_utf_8,
+    is_manifest_tsv,
+]
+
+
+def get_issues_with_manifest_file(manifest_tsv_file: str, checks=None):
+    issues = set()
+    for check in checks if checks is not None else MANIFEST_VALIDATION_CHECKS:
+        try:
+            if issue := check(pathlib.Path(manifest_tsv_file)):
+                issues.add(issue)
+        except ValidateBlockerException as e:
+            issues.add(e)
+            return issues
+    return issues

src/uiucprescon/tripwire/speedwagon_plugin/__init__.py

@@ -0,0 +1 @@
+"""Speedwagon plugin."""

src/uiucprescon/tripwire/speedwagon_plugin/workflows.py

@@ -19,6 +19,7 @@
 from speedwagon import Workflow
 from speedwagon.tasks import Result, TaskBuilder
 import speedwagon.workflow
+from speedwagon.exceptions import SpeedwagonException
 from speedwagon.validators import JobOptionsType
 from speedwagon.workflow import AbsOutputOptionDataType
 import speedwagon.validators
@@ -233,10 +234,19 @@
     name = "manifest-check"
 
     def create_new_task(self, task_builder: TaskBuilder, job_args) -> None:
-        super().create_new_task(task_builder, job_args)
+        """Create a new task to validate manifest file."""
+        task_builder.add_subtask(
+            locate_manifest_files_task(
+                manifest_tsv_file=job_args["manifest"],
+                search_path=job_args["search_path"],
+            )
+        )
 
-    # active = False
-    # TODO: implement manifest-check workflow
+    def initial_task(self, task_builder: TaskBuilder, user_args: _T) -> None:
+        """Validate manifest file before running full validation."""
+        task_builder.add_subtask(
+            validate_manifest_file_task(user_args["Manifest tsv File"])
+        )
 
     def discover_task_metadata(
         self,
@@ -254,6 +264,8 @@
         Returns:
             sequence of task metadata
         """
+        if initial_results[0].data is False:
+            return []
         return [
             {
                 "manifest": user_args["Manifest tsv File"],
@@ -274,13 +286,37 @@
         return [manifest_tsv_file, search_path]
 
 
+@speedwagon.tasks.workflow_task("validate manifest", logger=logger)
+def validate_manifest_file_task(
+    manifest_tsv_file: str,
+    checking_strategy=manifest_check.get_issues_with_manifest_file,
+) -> bool:
+    if issues := checking_strategy(manifest_tsv_file):
+        error_report_list = "\n".join([f" * {issue}" for issue in issues])
+        report = f"""Unable to continue due to issues found with manifest file:
+        {error_report_list}
+        """
+        logger.error(report)
+        return False
+    return True
+
+
 @speedwagon.tasks.workflow_task(
     "Locate Files in Manifest", logger=manifest_check.logger
 )
-def locate_manifest_files_task(manifest_tsv_file: str, search_path: str):
-    manifest_check.locate_manifest_files(
-        pathlib.Path(manifest_tsv_file), pathlib.Path(search_path)
-    )
+def locate_manifest_files_task(
+    manifest_tsv_file: str,
+    search_path: str,
+    locating_strategy=manifest_check.locate_manifest_files,
+):
+    try:
+        locating_strategy(
+            pathlib.Path(manifest_tsv_file), pathlib.Path(search_path)
+        )
+    except UnicodeDecodeError as e:
+        raise SpeedwagonException(
+            "Check the manifest file is encoded in utf-8"
+        ) from e
 
 
 @speedwagon.tasks.workflow_task("Show Metadata", logger=logger)

src/uiucprescon/tripwire/validation.py

@@ -276,7 +276,8 @@ def validate_directory_checksums_command(
             target_file.relative_to(path),
         )
         if not os.path.exists(target_file):
-            message = f'Expected file "{target_file.relative_to(path)}" - Failed: File does not exist'
+            message = (f'Expected file "{target_file.relative_to(path)}" - '
+                       f'Failed: File does not exist')
             logger.error(message)
             errors.append(message)
             continue

tests/speedwagon/test_workflows.py

@@ -261,3 +261,25 @@ def test_create_new_task(self):
         }
         workflow.create_new_task(task_builder,job_args)
         task_builder.add_subtask.assert_called_once()
+
+@pytest.mark.parametrize("return_values,expected", [
+    ([], True),
+    (["issue1"], False),
+])
+def test_validate_manifest_file_task(return_values,expected):
+    task = tripwire_workflows.validate_manifest_file_task(
+        manifest_tsv_file="some_manifest.tsv",
+        checking_strategy=Mock(return_value=return_values),
+    )
+    task.work()
+    assert task.task_result.data is expected
+
+def test_locate_manifest_files_task():
+    locating_strategy= Mock()
+    task = tripwire_workflows.locate_manifest_files_task(
+        manifest_tsv_file="some_manifest.tsv",
+        search_path="/path/to/search",
+        locating_strategy=locating_strategy
+    )
+    task.work()
+    locating_strategy.assert_called_once()

tests/test_manifest_check.py

@@ -1,5 +1,6 @@
 import io
 import pathlib
+from os import name
 from unittest.mock import Mock, MagicMock
 
 import pytest
@@ -219,3 +220,61 @@ def test_verify_format_type(self, data, klass, expected):
     def test_get_manifest_type_unknown_throws(self):
         with pytest.raises(ValueError):
             manifest_check.get_manifest_type(io.StringIO("unknown format"))
+
+def test_check_utf_8_passed():
+    manifest_tsv = Mock(open=MagicMock())
+    assert manifest_check.check_utf_8(manifest_tsv) is None
+
+def test_check_utf_8_failed(monkeypatch):
+    mocked_open = MagicMock(name="open",
+            __enter__=Mock(
+                    return_value=Mock(
+                        name="__enter__",
+                        read=Mock(
+                            name="read", side_effect=lambda :b'\x80'.decode("utf8")
+                        )
+                )
+            )
+        )
+    mocked_open.read.side_effect = UnicodeDecodeError("utf-8", b"test", 0, 1, "")
+    manifest_tsv = Mock(
+        open=Mock(return_value=mocked_open)
+    )
+    with pytest.raises(manifest_check.ValidateBlockerException):
+        assert manifest_check.check_utf_8(manifest_tsv) is None
+
+def test_is_manifest_tsv_passes():
+    manifest_tsv = Mock(open=MagicMock())
+    assert manifest_check.is_manifest_tsv(manifest_tsv, test_strategy=lambda *_, **__: True) is None
+
+def test_is_manifest_tsv_fails():
+    manifest_tsv = Mock(open=MagicMock())
+    assert manifest_check.is_manifest_tsv(manifest_tsv, test_strategy=Mock(side_effect=AttributeError)) is not None
+
+def test_get_issues_with_manifest_file_no_issues():
+    assert manifest_check.get_issues_with_manifest_file("some_manifest_file", checks=[]) == set()
+
+def test_get_issues_with_manifest_file_that_returns():
+    checks = [
+        Mock(return_value="Issue found")
+    ]
+    assert len(manifest_check.get_issues_with_manifest_file("some_manifest_file", checks=checks)) > 0
+
+
+def test_get_issues_with_manifest_file_that_throws():
+    checks = [
+        Mock(side_effect=manifest_check.ValidateBlockerException("Issue found"))
+    ]
+    assert len(manifest_check.get_issues_with_manifest_file("some_manifest_file", checks=checks)) > 0
+
+
+
+def test_get_issues_with_manifest_file_throws_stops_other_checks():
+    other_check = Mock(return_value="Should not be called")
+    checks = [
+        Mock(side_effect=manifest_check.ValidateBlockerException("Issue found")),
+        other_check
+    ]
+    assert len(manifest_check.get_issues_with_manifest_file("some_manifest_file", checks=checks)) > 0
+    other_check.assert_not_called()
+