nit found only in TS

Author: Andrei Bratu

src/humanloop/evals/run.py

@@ -72,6 +72,7 @@
 from humanloop.types import PromptKernelRequest as Prompt
 from humanloop.types import ToolKernelRequest as Tool
 from humanloop.types.agent_response import AgentResponse
+from humanloop.types.agent_kernel_request import AgentKernelRequest as Agent
 from humanloop.types.datapoint_response import DatapointResponse
 from humanloop.types.dataset_response import DatasetResponse
 from humanloop.types.evaluation_run_response import EvaluationRunResponse
@@ -402,7 +403,7 @@ def _get_subclient(client: "BaseHumanloop", file_config: FileEvalConfig) -> Huma
 
 
 def _safe_get_default_file_version(client: "BaseHumanloop", file_config: FileEvalConfig) -> EvaluatedFile:
-    """Get default version of a File from online workspace.
+    """Get default version of a File from remote workspace.
 
     Uses either the File path or id from the config.
 
@@ -421,13 +422,13 @@ def _safe_get_default_file_version(client: "BaseHumanloop", file_config: FileEva
             raise HumanloopRuntimeError(
                 f"File in Humanloop workspace at {path} is not of type {type}, but {hl_file.type}."
             )
-        # cast is safe, we can only fetch Files allowed by FileType
+        # cast is safe, we can only fetch Files that can be evaluated
         return typing.cast(EvaluatedFile, hl_file)
     elif file_id is not None:
         subclient = _get_subclient(client=client, file_config=file_config)
         return subclient.get(id=file_id)
     else:
-        raise HumanloopRuntimeError("You must provide a path or id in your `file` config.")
+        raise HumanloopRuntimeError("You must provide either the path or the id in your `file` config.")
 
 
 def _resolve_file(client: "BaseHumanloop", file_config: FileEvalConfig) -> tuple[EvaluatedFile, Optional[Callable]]:
@@ -447,7 +448,14 @@ def _resolve_file(client: "BaseHumanloop", file_config: FileEvalConfig) -> tuple
             "You are trying to create a new version of the File by passing the `version` argument. "
             "You must pass either the `file.path` or `file.id` argument and provider proper `file.version` for upserting the File."
         )
-    hl_file = _safe_get_default_file_version(client=client, file_config=file_config)
+    try:
+        hl_file = _safe_get_default_file_version(client=client, file_config=file_config)
+    except ApiError:
+        if not version or not path or file_id:
+            raise HumanloopRuntimeError(
+                "File does not exist on Humanloop. Please provide a `file.path` and a version to create a new version.",
+            )
+        return _upsert_file(file_config=file_config, client=client), callable or None
 
     if (version_id or environment) and (callable or version):
         raise HumanloopRuntimeError(
@@ -683,6 +691,9 @@ def _upsert_file(client: "BaseHumanloop", file_config: FileEvalConfig) -> Evalua
     elif type_ == "tool":
         # Will throw error if version is invalid
         Tool.model_validate(version)
+    elif type_ == "agent":
+        # Will throw error if version is invalid
+        Agent.model_validate(version)
     else:
         raise NotImplementedError(f"Unsupported File type: {type_}")
 

tests/integration/test_evals.py

@@ -347,3 +347,56 @@ def test_flow_eval_resolves_to_default_with_callable(
     finally:
         # Clean up test resources
         humanloop_test_client.flows.delete(id=flow_response.id)
+
+
+@pytest.mark.skip(reason="Skip until agents are in prod")
+def test_agent_eval_works_upserting(
+    humanloop_test_client: Humanloop,
+    eval_dataset: TestIdentifiers,
+    output_not_null_evaluator: TestIdentifiers,
+    sdk_test_dir: str,
+):
+    humanloop_test_client.evaluations.run(  # type: ignore [attr-defined]
+        name="test_eval_run",
+        file={
+            "path": f"{sdk_test_dir}/Test Agent",
+            "type": "agent",
+            "version": {
+                "model": "gpt-4o",
+                "template": [
+                    {
+                        "role": "system",
+                        "content": "You are a helpful assistant, offering very short answers.",
+                    },
+                    {
+                        "role": "user",
+                        "content": "{{question}}",
+                    },
+                ],
+                "provider": "openai",
+                "temperature": 0,
+                "max_iterations": 5,
+            },
+        },
+        dataset={
+            "path": eval_dataset.file_path,
+        },
+        evaluators=[
+            {
+                "path": output_not_null_evaluator.file_path,
+            }
+        ],
+    )
+    files_response = humanloop_test_client.files.list_files(page=1, size=100)
+    eval_agent = None
+    for file in files_response.records:
+        if file.path == f"{sdk_test_dir}/Test Agent":
+            eval_agent = file
+            break
+    assert eval_agent and eval_agent.type == "agent"
+    # THEN the evaluation finishes successfully
+    evaluations_response = humanloop_test_client.evaluations.list(file_id=eval_agent.id)
+    assert evaluations_response.items and len(evaluations_response.items) == 1
+    evaluation_id = evaluations_response.items[0].id
+    runs_response = humanloop_test_client.evaluations.list_runs_for_evaluation(id=evaluation_id)  # type: ignore [attr-defined, arg-type]
+    assert runs_response.runs[0].status == "completed"