Merge branch 'main' into lint-model-paths

Author: tripleaceme

.devcontainer/devcontainer.json

@@ -11,10 +11,7 @@
   "postCreateCommand": "bash .devcontainer/post-create-command.sh",
   "customizations": {
     "vscode": {
-      "extensions": [
-        "ms-python.python",
-        "ms-python.vscode-pylance"
-      ]
+      "extensions": ["ms-python.python", "ms-python.vscode-pylance"]
     }
   },
   "remoteUser": "vscode"

.github/workflows/pr.yaml

@@ -16,6 +16,7 @@ jobs:
     outputs:
       python: ${{ steps.filter.outputs.python }}
       client: ${{ steps.filter.outputs.client }}
+      vscode: ${{ steps.filter.outputs.vscode }}
       ci: ${{ steps.filter.outputs.ci }}
     steps:
       - uses: actions/checkout@v7
@@ -34,6 +35,8 @@ jobs:
               - 'pyproject.toml'
             client:
               - 'web/client/**'
+            vscode:
+              - 'vscode/**'
             ci:
               - '.github/**'
               - 'Makefile'
@@ -188,9 +191,10 @@ jobs:
 
   ui-style:
     needs: [changes]
-    if: false
-      # needs.changes.outputs.client == 'true' || needs.changes.outputs.ci ==
-      # 'true' || github.ref == 'refs/heads/main'
+    if:
+      needs.changes.outputs.client == 'true' || needs.changes.outputs.vscode ==
+      'true' || needs.changes.outputs.ci == 'true' || github.ref ==
+      'refs/heads/main'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
@@ -252,7 +256,17 @@ jobs:
       fail-fast: false
       matrix:
         engine:
-          [duckdb, postgres, mysql, mssql, trino, spark, clickhouse, risingwave, starrocks]
+          [
+            duckdb,
+            postgres,
+            mysql,
+            mssql,
+            trino,
+            spark,
+            clickhouse,
+            risingwave,
+            starrocks,
+          ]
     env:
       PYTEST_XDIST_AUTO_NUM_WORKERS: 2
       SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: '1'
@@ -393,10 +407,13 @@ jobs:
           retention-days: 7
 
   test-vscode:
+    needs: changes
+    if:
+      needs.changes.outputs.vscode == 'true' || needs.changes.outputs.ci ==
+      'true' || github.ref == 'refs/heads/main'
     env:
       PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     runs-on: ubuntu-latest
-    if: false
     steps:
       - uses: actions/checkout@v7
       - uses: actions/setup-node@v7
@@ -457,7 +474,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dbt-version: ['1.3', '1.4', '1.5', '1.6', '1.7', '1.8', '1.9', '1.10', '1.11', '1.12']
+        dbt-version:
+          [
+            '1.3',
+            '1.4',
+            '1.5',
+            '1.6',
+            '1.7',
+            '1.8',
+            '1.9',
+            '1.10',
+            '1.11',
+            '1.12',
+          ]
     steps:
       - uses: actions/checkout@v7
       - name: Set up Python

sqlmesh/core/test/definition.py

@@ -263,12 +263,9 @@ def assert_equal(
         for col, value in object_sentinel_values.items():
             try:
                 # can't use `isinstance()` here - https://stackoverflow.com/a/68743663/1707525
-                if type(value) is datetime.date:
-                    expected[col] = pd.to_datetime(expected[col]).dt.date
-                elif type(value) is datetime.time:
-                    expected[col] = pd.to_datetime(expected[col]).dt.time
-                elif type(value) is datetime.datetime:
-                    expected[col] = pd.to_datetime(expected[col]).dt.to_pydatetime()
+                value_type = type(value)
+                if value_type in (datetime.date, datetime.time, datetime.datetime):
+                    expected[col] = _parse_expected_datetime_column(expected[col], value_type)
             except Exception as e:
                 from sqlmesh.core.console import get_console
 
@@ -1014,6 +1011,34 @@ def _raise_error(msg: str, path: Path | None = None) -> None:
     raise TestError(f"Failed to run test:\n{msg}")
 
 
+def _parse_expected_datetime_column(series: pd.Series, target_type: type) -> pd.Series:
+    """Convert a series of expected values to python ``date``/``time``/``datetime``.
+
+    Falls back to microsecond resolution when pandas' default nanosecond
+    parsing overflows. SQL ``TIMESTAMP`` columns can carry values outside
+    pandas' default ``datetime64[ns]`` range (1677-09-21..2262-04-11), so
+    unit tests may compare against values like ``0001-01-01`` which are
+    valid in the database but overflow the default resolution.
+    """
+    import pandas as pd
+    from pandas.errors import OutOfBoundsDatetime
+
+    try:
+        parsed = pd.to_datetime(series)
+    except OutOfBoundsDatetime:
+        parsed = series.astype("datetime64[us]")
+
+    if target_type is datetime.date:
+        return parsed.dt.date
+    if target_type is datetime.time:
+        return parsed.dt.time
+    # `Series.dt.to_pydatetime()` returns an `ndarray` in pandas 2.x. Wrap it in a
+    # Series with ``dtype=object`` so pandas does not coerce the values back to
+    # ``pd.Timestamp`` (which would reintroduce the nanosecond overflow this
+    # function exists to avoid).
+    return pd.Series(parsed.dt.to_pydatetime(), index=parsed.index, dtype="object")
+
+
 def _normalize_df_value(value: t.Any) -> t.Any:
     """Normalize data in a pandas dataframe so ruamel and sqlglot can deal with it."""
     import numpy as np

tests/core/engine_adapter/integration/docker/_common-hive.yaml

@@ -12,22 +12,22 @@ services:
 
   # S3-style object storage
   minio:
-    image: 'minio/minio:RELEASE.2022-05-26T05-48-41Z'
+    image: 'cgr.dev/chainguard/minio:latest@sha256:039800e64ec7247d2fde7cff3697e964f6fe20b6d7d2c46aa7d82cc63355d512'
     ports:
       - '9000:9000'
       - '9001:9001'
     environment:
-      MINIO_ACCESS_KEY: minio
-      MINIO_SECRET_KEY: minio123
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: minio123
     command: server /data --console-address ":9001"
 
   # Set up minio with default buckets / paths
   mc-job:
-    image: 'minio/mc:RELEASE.2022-05-09T04-08-26Z'
+    image: 'cgr.dev/chainguard/minio-client:latest-dev@sha256:fb635b967f5f32424391150dac4985cc1aea78fb0a1ce031c01952cf661a13ec'
     entrypoint: |
       /bin/bash -c "
       sleep 5;
-      /usr/bin/mc config --quiet host add myminio http://minio:9000 minio minio123;
+      /usr/bin/mc alias --quiet set myminio http://minio:9000 minio minio123;
       /usr/bin/mc mb --quiet myminio/trino/datalake;
       /usr/bin/mc mb --quiet myminio/trino/datalake_iceberg;
       /usr/bin/mc mb --quiet myminio/trino/datalake_delta;
@@ -39,4 +39,4 @@ services:
       /usr/bin/mc mb --quiet myminio/nessie/warehouse;
       "
     depends_on:
-      - minio
+      - minio

tests/core/test_test.py

@@ -2931,6 +2931,48 @@ def test_timestamp_normalization() -> None:
     )
 
 
+def test_out_of_bounds_nanosecond_timestamp_comparison(mocker: MockerFixture) -> None:
+    # https://github.com/TobikoData/sqlmesh/issues/5929
+    # Engines like Redshift may return a TIMESTAMP column as an object-dtype
+    # series of python `datetime.datetime` instances. Values outside pandas'
+    # default `datetime64[ns]` range (1677-09-21..2262-04-11) - which SQL
+    # `TIMESTAMP` fully supports - previously raised `OutOfBoundsDatetime`
+    # while parsing the expected values, producing a "Failed to convert
+    # expected value into `datetime`" warning and a false mismatch on values
+    # whose repr survives str-coercion (the values below happen to compare
+    # equal via `str()`, so the mismatch was silent).
+    test = _create_test(
+        body=load_yaml(
+            """
+test_foo:
+  model: sushi.foo
+  outputs:
+    query:
+      - ts_col: "0001-01-01 00:00:00"
+      - ts_col: "9999-12-31 23:59:59"
+            """
+        ),
+        test_name="test_foo",
+        model=_create_model("SELECT ts_col FROM raw"),
+        context=Context(config=Config(model_defaults=ModelDefaultsConfig(dialect="duckdb"))),
+    )
+    actual = pd.DataFrame(
+        {
+            "ts_col": pd.Series(
+                [datetime.datetime(1, 1, 1), datetime.datetime(9999, 12, 31, 23, 59, 59)],
+                dtype=object,
+            )
+        }
+    )
+    # Use T separator so a string-only comparison (broken path) would mismatch
+    # against str(datetime.datetime(1, 1, 1)) == "0001-01-01 00:00:00".
+    expected = pd.DataFrame({"ts_col": ["0001-01-01T00:00:00", "9999-12-31T23:59:59"]})
+    log_warning = mocker.spy(get_console(), "log_warning")
+    test.assert_equal(expected=expected, actual=actual, sort=False)
+    for call_args in log_warning.call_args_list:
+        assert "Failed to convert expected value" not in call_args.args[0]
+
+
 @use_terminal_console
 def test_disable_test_logging_if_no_tests_found(mocker: MockerFixture, tmp_path: Path) -> None:
     init_example_project(tmp_path, engine_type="duckdb")

tests/web/test_models.py

@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import pytest
+
+from sqlmesh.core.context import Context
+from web.server.api.endpoints.models import get_models
+
+pytestmark = pytest.mark.web
+
+
+def test_get_models_multi_repo() -> None:
+    """Models of every project are serialized, not just those of the first one.
+
+    `context.path` is the first configured project, so it is not an ancestor of the models
+    defined in any of the others.
+    """
+    context = Context(paths=["examples/multi/repo_1", "examples/multi/repo_2"], gateway="memory")
+
+    paths_by_name = {model.name: model.path for model in get_models(context)}
+
+    # Each model is reported relative to the project that defines it.
+    assert paths_by_name["bronze.a"] == "models/a.sql"
+    assert paths_by_name["silver.c"] == "models/c.sql"

vscode/extension/tests/fixtures.ts

@@ -60,5 +60,5 @@ export const test = base.extend<
   ],
 })
 
-// Export expect and Page from Playwright for convenience
-export { expect, Page } from '@playwright/test'
+// Export expect and commonly used Playwright types for convenience
+export { expect, FrameLocator, Page } from '@playwright/test'

vscode/extension/tests/lineage_settings.spec.ts

@@ -1,5 +1,5 @@
 import { test, expect } from './fixtures'
-import type { FrameLocator, Page } from '@playwright/test'
+import type { FrameLocator, Page } from './fixtures'
 import fs from 'fs-extra'
 import {
   openLineageView,

web/client/src/library/components/graph/ModelNode.tsx

@@ -198,7 +198,11 @@ export default function ModelNode({
       style={{
         width: '100%',
         ...(tagColor != null
-          ? { borderColor: tagColor, backgroundColor: tagColor, color: tagColor }
+          ? {
+              borderColor: tagColor,
+              backgroundColor: tagColor,
+              color: tagColor,
+            }
           : {}),
       }}
     >
@@ -246,7 +250,10 @@ export default function ModelNode({
         />
       </div>
       {showColumns && (
-        <div ref={columnsWrapperRef} style={{ height: '10rem', overflow: 'hidden' }}>
+        <div
+          ref={columnsWrapperRef}
+          style={{ height: '10rem', overflow: 'hidden' }}
+        >
           <ModelColumns
             className="nowheel rounded-b-lg bg-theme-lighter text-xs h-full"
             nodeId={id}

web/client/src/library/components/graph/help.ts

@@ -175,7 +175,10 @@ function getNodeMap({
     const model = models.get(modelName)
     const tagsStr = model?.details?.tags
     const tags = tagsStr
-      ? tagsStr.split(',').map(t => t.trim()).filter(Boolean)
+      ? tagsStr
+          .split(',')
+          .map(t => t.trim())
+          .filter(Boolean)
       : undefined
     const node = createGraphNode(modelName, {
       label: model?.displayName ?? modelName,