Merge branch 'main' into mday/tobiko-cicd-plan-comments

Author: mday-io

sqlmesh/core/model/definition.py

@@ -1712,6 +1712,7 @@ def render(
 
     def render_seed(self) -> t.Iterator[QueryOrDF]:
         import numpy as np
+        import pandas as pd
 
         self._ensure_hydrated()
 
@@ -1752,8 +1753,6 @@ def render_seed(self) -> t.Iterator[QueryOrDF]:
 
             # convert all date/time types to native pandas timestamp
             for column in [*date_columns, *datetime_columns]:
-                import pandas as pd
-
                 df[column] = pd.to_datetime(df[column], infer_datetime_format=True, errors="ignore")  # type: ignore
 
             # extract datetime.date from pandas timestamp for DATE columns
@@ -1769,7 +1768,7 @@ def render_seed(self) -> t.Iterator[QueryOrDF]:
                     )
 
             for column in bool_columns:
-                df[column] = df[column].apply(lambda i: str_to_bool(str(i)))
+                df[column] = df[column].apply(lambda i: None if pd.isna(i) else str_to_bool(str(i)))
 
             df.loc[:, string_columns] = df[string_columns].mask(
                 cond=lambda x: x.notna(),  # type: ignore

sqlmesh/integrations/dlt.py

@@ -63,10 +63,7 @@ def generate_dlt_models_and_settings(
     if db_type == "filesystem":
         connection_config = None
     else:
-        if dlt.__version__ >= "1.10.0":
-            client = pipeline.destination_client()
-        else:
-            client = pipeline._sql_job_client(schema)  # type: ignore
+        client = pipeline.destination_client()
         config = client.config
         credentials = config.credentials
         configs = {

sqlmesh/utils/date.py

@@ -344,6 +344,8 @@ def make_exclusive(time: TimeLike) -> datetime:
 
 
 def make_ts_exclusive(time: TimeLike, dialect: DialectType) -> datetime:
+    import pandas as pd
+
     ts = to_datetime(time)
     if dialect == "tsql":
         return to_utc_timestamp(ts) - pd.Timedelta(1, unit="ns")

tests/core/test_model.py

@@ -1541,6 +1541,44 @@ def test_seed_model_custom_types(tmp_path):
     assert df["empty_date"].iloc[0] is None
 
 
+def test_seed_model_boolean_nulls_are_preserved(tmp_path):
+    model_csv_path = (tmp_path / "model.csv").absolute()
+
+    with open(model_csv_path, "w", encoding="utf-8") as fd:
+        fd.write("id,test_ind\n")
+        fd.write("1,null\n")
+        fd.write("2,false\n")
+        fd.write("3,true\n")
+        fd.write("4,null\n")
+
+    model = create_seed_model(
+        "test_db.test_model",
+        SeedKind(path=str(model_csv_path)),
+        columns={
+            "id": "int",
+            "test_ind": "boolean",
+        },
+        dialect="databricks",
+    )
+
+    df = next(model.render_seed())
+
+    assert df["test_ind"].to_list() == [None, False, True, None]
+
+    context = Context(
+        config=Config(
+            default_connection=DuckDBConnectionConfig(),
+            model_defaults=ModelDefaultsConfig(dialect="databricks"),
+        )
+    )
+    context.upsert_model(model)
+
+    rendered_sql = context.render(model).sql("databricks")
+
+    assert "CAST(NULL AS BOOLEAN)" in rendered_sql
+    assert "(4, NULL)" in rendered_sql
+
+
 def test_seed_with_special_characters_in_column(tmp_path, assert_exp_eq):
     config = Config(model_defaults=ModelDefaultsConfig(dialect="duckdb"))
     context = Context(config=config)

tests/utils/test_date.py

@@ -14,11 +14,13 @@
     is_categorical_relative_expression,
     is_relative,
     make_inclusive,
+    make_ts_exclusive,
     to_datetime,
     to_time_column,
     to_timestamp,
     to_ts,
     to_tstz,
+    to_utc_timestamp,
 )
 
 
@@ -141,6 +143,17 @@ def test_make_inclusive_tsql(start_in, end_in, start_out, end_out, dialect) -> N
     )
 
 
+def test_make_ts_exclusive() -> None:
+    # tsql subtracts 1ns from the UTC timestamp; must not raise NameError (pd not imported)
+    assert make_ts_exclusive("2020-01-01 12:00:00", dialect="tsql") == to_utc_timestamp(
+        to_datetime("2020-01-01 12:00:00")
+    ) - pd.Timedelta(1, unit="ns")
+    # non-tsql dialects add 1 microsecond
+    assert make_ts_exclusive("2020-01-01 12:00:00", dialect="duckdb") == to_datetime(
+        "2020-01-01 12:00:00.000001"
+    )
+
+
 @pytest.mark.parametrize(
     "expression, result",
     [