Making expand_mini function modular

docs/README.md

@@ -14,32 +14,24 @@ This enables rich output which can be helpful.
 
 ## Running the script
 ### To write the expanded output inline:
-`mini -s {path/to/source/files}`
+`weaver -s {path/to/source/directory}` or `just expand`
+
+Example:
+
+`weaver -s src/cam_source_enums/schema/`
 
 #### Re-expansion
 To rerun the expansion script on a file, remove the `permissible_values` field from the YAML file. This can either be done by deleting it manually or running the following script for each file:
 
-`mini --clear {path/to/file}`
+`weaver --clear {file_name}` or `just clear {file_name}`
 
-### To write the expanded output to a new filepath:
-`tweaver -s {path/to/source/files} -m {model_name}`
+Example:
 
-- The tool uses the model name to create the output filepath as `src/{model_name}/schema`
+`weaver --clear EnumName`
 
 
 ## Model YAML File Conventions
-The tool copies the source model YAML file to make the expanded model YAML file.
-The file is written to the output filepath location and uses the path to create the name.
-
-Example:<br>
-- output = src/enums_expanded_file<br>
-- expanded model YAML = src/enums_expanded_file/enums_expanded_file.yaml
-
-The tool uses text substituion to modify the content of the "id", "name", "title", and "description" properties. All instances of the word `source` are replaced with `expanded` in these fields.
-
 The following conventions must be used for files to be findable by the script:
-- The source model YAML file must include `_source` in the title<br>
-  - Example: enums_source_file
 - The enumeration file names must start with `Enum`<br>
   - Example: EnumDataFile
 

justfile

@@ -0,0 +1,6 @@
+expand:
+    -weaver -s src/cam_source_enums/schema/
+
+
+clear file_path:
+    weaver --clear src/cam_source_enums/schema/{{file_path}}.yaml

pyproject.toml

@@ -16,8 +16,9 @@ dependencies = [
     "linkml",
     "PyYAML",
     "jinja2",
-    "search-dragon@git+https://github.com/NIH-NCPI/search-dragon.git"
-    ]
+    "search-dragon@git+https://github.com/NIH-NCPI/search-dragon.git",
+    "car-utils",
+]
 
 dynamic = ["version"]
 
@@ -40,6 +41,8 @@ where = ["src"]  # list of folders that contain the packages (["."] by default)
 [tool.setuptools.package-data]
 "*" = ["*.yaml"]
 
+[tool.uv.sources]
+car-utils = { git = "https://github.com/carrollaboratory/car-utils.git" }
+
 [project.scripts]
-tweaver = "tweaver.weaver:exec"
-mini = "tweaver.mini:exec"
+weaver = "tweaver.weaver:exec"

src/tweaver/mini.py → src/tweaver/orig_weaver.py

@@ -85,51 +85,33 @@ def increase_indent(self, flow=False, indentless=False):
         return super().increase_indent(flow=flow, indentless=False)
 
 
-def expand_mini(
+def expand(
     local_filepath: Path,
+    model_name: str | None = None,
     iri: str | None = None,
 ):
     """Extract Enums from a monolithic LinkML model into individual YAML files
     Args:
         local_filepath: The file containing the monolithic linkml model
+        model_name: The name of the model in the output directory where the enum YAMLs are to be written
         iri: Optional iri if a specific iri is desired other than the iri derived programattically
     Returns:
         list of enum names
     """
 
-    model_filename = local_filepath.parent.name + ".yaml"
-    model_file = local_filepath / model_filename
-
+    output_filepath = Path(f"src/{model_name}/schema")
+    output_filepath.mkdir(parents=True, exist_ok=True)
     enum_count = 0
     expanded_count = 0
     enum_names = []
-
-    if not model_file.exists():
-        logger.error(f"{model_file} not found.")
-        return enum_names
-
-    model_parsed = yaml.safe_load(model_file.read_text())
-    imports = model_parsed.get("imports", [])
-
-    for imp in imports:
-        if not imp.split("/")[-1].startswith("Enum"):
-            continue
-        filename = imp.split("/")[-1]
-        matches = list(local_filepath.resolve().glob(f"**/{filename}.yaml"))
-        if not matches:
-            logger.warning(f"{imp} file not found.")
-            continue
-        enum_file = matches[0]
-        if not enum_file.exists():
-            logger.warning(f"{enum_file} not found.")
-            continue
+    for enum_file in local_filepath.glob("Enum*.yaml"):
         raw_enum = enum_file.read_text()
         parsed = yaml.safe_load(raw_enum)
 
         enums = parsed.get("enums", {})
         for name, enum in enums.items():
             enum_names.append(name)
-            expanded_enum = enum_file
+            expanded_enum = output_filepath / f"{name}.yaml"
 
             has_permissible = (
                 "permissible_values" in (enum) and enum["permissible_values"]
@@ -143,7 +125,8 @@ def expand_mini(
             endpoint = "-c" if has_direct else "-d"
 
             if has_permissible or not has_ontology:
-                logger.info(f"Skipping {name}. Does not require expansion")
+                expanded_enum.write_text(raw_enum)
+                logger.info(f"Copied {name} (does not require expansion)")
                 enum_count += 1
                 expanded_count += 1
                 continue
@@ -156,21 +139,6 @@ def expand_mini(
 
             all_permissible_values = {}
             node_failed = False
-            minus = has_reachable.get("minus", [])
-            minus_codes = set()
-            if isinstance(minus, dict):
-                minus_codes.update(minus.get("permissible_values", []))
-            else:
-                for minus_item in minus:
-                    if isinstance(minus_item, str):
-                        minus_codes.add(minus_item)
-                        continue
-                    if "permissible_values" in minus_item:
-                        minus_codes.update(minus_item["permissible_values"])
-                        continue
-                    minus_reachable = minus_item.get("reachable_from", {})
-                    minus_nodes = minus_reachable.get("source_nodes", [])
-                    minus_codes.update(minus_nodes)
 
             for node in has_nodes:
                 cmd = [
@@ -180,7 +148,7 @@ def expand_mini(
                     "-o",
                     str(ontology),
                     "-f",
-                    str(expanded_enum),
+                    str(expanded_enum.absolute()),
                     str(endpoint),
                     "-s",
                     "0",
@@ -198,20 +166,12 @@ def expand_mini(
                     logger.error(f"Failed for {name}: {result.stdout}")
                     logger.error(f"Failed for {name}: {result.stderr}")
                     node_failed = True
-                    logger.info(f"dragon_search exit code: {result.returncode}")
                 else:
                     parsed_nodes = parsed_csv(
                         expanded_enum.read_text(), endpoint, has_nodes
                     )
                     all_permissible_values.update(parsed_nodes)
-                    all_permissible_values = {
-                        k: v
-                        for k, v in all_permissible_values.items()
-                        if k not in minus_codes
-                    }
                     logger.info(f"Expanded enumeration: {name}")
-                    if minus_codes:
-                        logger.info(f"Excluding {minus_codes}")
 
                 if all_permissible_values:
                     parsed["enums"][name]["permissible_values"] = all_permissible_values
@@ -229,10 +189,43 @@ def expand_mini(
                         expanded_count += 1
 
     if expanded_count != enum_count:
-        logger.warning(f"{enum_count - expanded_count} failed to be expanded.")
+        logger.error(f"{enum_count - expanded_count} failed to be expanded.")
     return enum_names
 
 
+def copy_model(local_filepath: Path, model_name: str):
+    """Copies source model file to the same name and location as the output filepath.
+        Replaces "source" with "expanded" in the id, name, title, and description properties
+    Args:
+        local_filepath: The path containing the source model YAML file
+        model_name: The name of the model for the expanded YAML file
+    """
+    model_filepath = Path(f"src/{model_name}/schema")
+    model_filepath.mkdir(parents=True, exist_ok=True)
+
+    for file in local_filepath.glob("*_source*.yaml"):
+        orig = file.read_text()
+        parsed = yaml.safe_load(orig)
+        for key in ["id", "name", "title", "description"]:
+            if parsed.get(key):
+                parsed[key] = (
+                    parsed[key]
+                    .replace("Source", "Expanded")
+                    .replace("source", "expanded")
+                )
+        yaml_file = model_filepath / f"{model_name}.yaml"
+        yaml_file.write_text(
+            yaml.dump(
+                parsed,
+                sort_keys=False,
+                Dumper=IndentedDumper,
+                indent=2,
+                default_flow_style=False,
+                explicit_start=True,
+            )
+        )
+
+
 def restricted_chars(arg: str):
     allowed_chars = re.search(r"^[\w-]+$", arg)
     if not allowed_chars:
@@ -242,34 +235,12 @@ def restricted_chars(arg: str):
     return arg
 
 
-def clear_permissible_values(filepath: Path):
-    """Remove the permissible_values block from an enum YAML file.
-    Args:
-        filepath: The filepath to the file to remove permissible_values
-    """
-    parsed = yaml.safe_load(filepath.read_text())
-    enums = parsed.get("enums", {})
-    for name in enums:
-        if "permissible_values" in enums[name]:
-            del enums[name]["permissible_values"]
-    filepath.write_text(
-        yaml.dump(
-            parsed,
-            Dumper=IndentedDumper,
-            default_flow_style=False,
-            sort_keys=False,
-            allow_unicode=True,
-            explicit_start=True,
-        )
-    )
-    logger.info(f"Cleared permissible_values from {filepath}")
-
-
 parser = argparse.ArgumentParser(
     description="Expand enums from a monolithic LinkML model"
 )
 
 
+# This module is out of date. Please use weaver.py for the most up to date version of enum expansion.
 def exec(cli_args: list[str] | None = None):
 
     parser.add_argument(
@@ -282,11 +253,17 @@ def exec(cli_args: list[str] | None = None):
     parser.add_argument(
         "-s",
         "--source",
-        required=False,
+        required=True,
         type=Path,
         help="The source file containing the enumerations to be expanded",
     )
-
+    parser.add_argument(
+        "-m",
+        "--model",
+        required=True,
+        type=restricted_chars,
+        help="The model name used to construct the output directory where the expanded YAML files will be written (src/{model_name}/schema) ",
+    )
     parser.add_argument(
         "-i",
         "--iri",
@@ -301,26 +278,15 @@ def exec(cli_args: list[str] | None = None):
         version=f"{__version__}",
         help="Pulls the version from the __init__.py file",
     )
-    parser.add_argument(
-        "--clear",
-        required=False,
-        type=Path,
-        help="Clears permissible_values property from a speficied enum YAML file.",
-    )
 
     args = parser.parse_args(cli_args)
     # Initialize the logger with whatever the user requested
     init_logging(args.log_level)
 
-    if args.clear:
-        clear_permissible_values(args.clear)
-        return
-
-    if not args.source:
-        parser.error("-s/--source is required when not using --clear")
-    expand_mini(
+    expand(
         local_filepath=args.source,
+        model_name=args.model,
         iri=args.iri,
     )
-    logger.info("Script completed successfully")
+    copy_model(local_filepath=args.source, model_name=args.model)
     return args