From c0b1e010a3fc779e74d49d2e732d9f766500ec97 Mon Sep 17 00:00:00 2001 From: tonyyuyiding Date: Mon, 5 Oct 2026 17:13:24 +0800 Subject: [PATCH 1/7] Infer column dtypes with `columns` and `schema_overrides`, without `schema` --- pyrefly/lib/alt/polars_specials.rs | 199 ++++++++++++++++++--------- pyrefly/lib/test/polars/dataframe.rs | 16 +++ 2 files changed, 150 insertions(+), 65 deletions(-) diff --git a/pyrefly/lib/alt/polars_specials.rs b/pyrefly/lib/alt/polars_specials.rs index eb6cfa1cb4..44f0911821 100644 --- a/pyrefly/lib/alt/polars_specials.rs +++ b/pyrefly/lib/alt/polars_specials.rs @@ -369,31 +369,19 @@ enum CsvColumnSelection { Indices(SmallSet), } +enum CsvSchema { + Missing, + Given(Vec<(Name, PolarsDType)>), +} + enum CsvDtypeOverrides { Unchanged, Sequence(Vec), -} - -impl CsvDtypeOverrides { - /// A complete schema changes only for positional dtype overrides. - fn parse( - expr: Option<&Expr>, - parse_dtype: impl FnMut(&Expr) -> Option, - ) -> Option { - Some(match expr { - None | Some(Expr::NoneLiteral(_) | Expr::Dict(_)) => Self::Unchanged, - Some(expr) => Self::Sequence( - literal_sequence(expr)? - .iter() - .map(parse_dtype) - .collect::>>()?, - ), - }) - } + Mapping(SmallMap), } struct PolarsCsvCommonOptions { - schema: Vec<(Name, PolarsDType)>, + schema: CsvSchema, overrides: CsvDtypeOverrides, new_columns: Vec, row_index_name: Option, @@ -1333,15 +1321,49 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { .then_some(CsvColumnSelection::Indices(indices)) } - fn polars_csv_schema(&self, expr: &Expr) -> Option> { - let (form, dict) = self.schema_literal_dict(expr)?; - if dict.items.is_empty() { - return Some(Vec::new()); - } - self.schema_dict_entries(form, dict)? - .into_iter() - .map(|(name, dtype)| Some((name, dtype?))) - .collect() + fn polars_csv_schema(&self, expr: Option<&Expr>) -> Option { + Some(match expr { + Some(expr) => { + let (form, dict) = self.schema_literal_dict(expr)?; + if dict.items.is_empty() { + CsvSchema::Given(Vec::new()) + } else { + CsvSchema::Given( + self.schema_dict_entries(form, dict)? + .into_iter() + .map(|(name, dtype)| Some((name, dtype?))) + .collect::>>()?, + ) + } + } + None => CsvSchema::Missing, + }) + } + + fn polars_csv_schema_overrides(&self, expr: Option<&Expr>) -> Option { + Some(match expr { + Some(Expr::NoneLiteral(_)) => CsvDtypeOverrides::Unchanged, + Some(Expr::Dict(dict)) => { + let mut overrides = SmallMap::with_capacity(dict.items.len()); + for item in &dict.items { + let (Some(key), value) = (&item.key, &item.value) else { + return None; + }; + overrides.insert( + self.polars_column_name(key)?, + self.polars_dtype_from_expr(value)?, + ); + } + CsvDtypeOverrides::Mapping(overrides) + } + Some(expr) => CsvDtypeOverrides::Sequence( + literal_sequence(expr)? + .iter() + .map(|expr| self.polars_dtype_from_expr(expr)) + .collect::>>()?, + ), + None => CsvDtypeOverrides::Unchanged, + }) } /// An omitted option and explicit `None` both represent semantic absence. @@ -1384,6 +1406,21 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { true } + /// Replace dtypes of the columns present in the mapping. Other columns remain unchanged. + fn apply_polars_csv_mapping_overrides( + columns: &mut [(Name, PolarsDType)], + overrides: &CsvDtypeOverrides, + ) { + let CsvDtypeOverrides::Mapping(map) = overrides else { + return; + }; + for (name, dtype) in columns.iter_mut() { + if let Some(override_dtype) = map.get(name) { + *dtype = override_dtype.clone(); + } + } + } + /// New names replace leading columns unless they exceed the width or create duplicates. fn apply_polars_csv_new_columns( columns: &mut [(Name, PolarsDType)], @@ -1462,9 +1499,8 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { return None; } - let schema = self.polars_csv_schema(schema?)?; - let overrides = - CsvDtypeOverrides::parse(overrides, |expr| self.polars_dtype_from_expr(expr))?; + let schema = self.polars_csv_schema(schema)?; + let overrides = self.polars_csv_schema_overrides(overrides)?; let new_columns = self.polars_csv_names(new_columns)?; let row_index_name = self.polars_optional_csv_name(row_index_name)?; let common = PolarsCsvCommonOptions { @@ -1478,7 +1514,13 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { PolarsCsvFunction::Read => { let selection = match Self::non_none_csv_option(selection) { None => CsvColumnSelection::All, - Some(expr) => self.polars_csv_selection(expr, common.schema.len())?, + Some(expr) => { + let width = match &common.schema { + CsvSchema::Given(columns) => columns.len(), + CsvSchema::Missing => 0, // Reject selection by index if schema is missing + }; + self.polars_csv_selection(expr, width)? + } }; PolarsCsvOptions::Read { common, selection } } @@ -1502,44 +1544,68 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { selection: CsvColumnSelection, ) -> Option> { let PolarsCsvCommonOptions { - schema: mut columns, + schema, overrides, new_columns, row_index_name, } = common; - if matches!(&overrides, CsvDtypeOverrides::Sequence(_)) - && !matches!(selection, CsvColumnSelection::All) - { - return None; - } - if !Self::apply_polars_csv_sequence_overrides(&mut columns, overrides) { - return None; - } - if !matches!(selection, CsvColumnSelection::All) { - if let CsvColumnSelection::Names(names) = &selection - && !names - .iter() - .all(|name| columns.iter().any(|(column, _)| column == name)) - { - return None; + match schema { + CsvSchema::Given(mut columns) => { + if matches!(&overrides, CsvDtypeOverrides::Sequence(_)) + && !matches!(selection, CsvColumnSelection::All) + { + return None; + } + if !Self::apply_polars_csv_sequence_overrides(&mut columns, overrides) { + return None; + } + if !matches!(selection, CsvColumnSelection::All) { + if let CsvColumnSelection::Names(names) = &selection + && !names + .iter() + .all(|name| columns.iter().any(|(column, _)| column == name)) + { + return None; + } + columns = columns + .into_iter() + .enumerate() + .filter(|(index, (name, _))| match &selection { + CsvColumnSelection::All => true, + CsvColumnSelection::Names(names) => names.contains(name), + CsvColumnSelection::Indices(indices) => indices.contains(index), + }) + .map(|(_, column)| column) + .collect(); + } + if !Self::apply_polars_csv_row_index(&mut columns, row_index_name) + || !Self::apply_polars_csv_new_columns(&mut columns, new_columns) + { + return None; + } + Some(columns) + } + CsvSchema::Missing => { + let CsvColumnSelection::Names(names) = selection else { + return None; + }; + if matches!(overrides, CsvDtypeOverrides::Unchanged) { + return None; + } + let mut columns: Vec<(Name, PolarsDType)> = names + .into_iter() + .map(|name| (name, PolarsDType::Unknown)) + .collect(); + Self::apply_polars_csv_mapping_overrides(&mut columns, &overrides); + if !Self::apply_polars_csv_sequence_overrides(&mut columns, overrides) + || !Self::apply_polars_csv_row_index(&mut columns, row_index_name) + || !Self::apply_polars_csv_new_columns(&mut columns, new_columns) + { + return None; + } + Some(columns) } - columns = columns - .into_iter() - .enumerate() - .filter(|(index, (name, _))| match &selection { - CsvColumnSelection::All => true, - CsvColumnSelection::Names(names) => names.contains(name), - CsvColumnSelection::Indices(indices) => indices.contains(index), - }) - .map(|(_, column)| column) - .collect(); - } - if !Self::apply_polars_csv_row_index(&mut columns, row_index_name) - || !Self::apply_polars_csv_new_columns(&mut columns, new_columns) - { - return None; } - Some(columns) } /// Lazy CSV inference applies overrides before inserting the row index. @@ -1550,11 +1616,14 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { include_file_paths: Option, ) -> Option> { let PolarsCsvCommonOptions { - schema: mut columns, + schema, overrides, new_columns, row_index_name, } = common; + let CsvSchema::Given(mut columns) = schema else { + return None; + }; if !Self::apply_polars_csv_sequence_overrides(&mut columns, overrides) || !Self::apply_polars_csv_row_index(&mut columns, row_index_name) || !Self::apply_polars_csv_new_columns(&mut columns, new_columns) diff --git a/pyrefly/lib/test/polars/dataframe.rs b/pyrefly/lib/test/polars/dataframe.rs index d1fdcf80fb..9ff7fbe281 100644 --- a/pyrefly/lib/test/polars/dataframe.rs +++ b/pyrefly/lib/test/polars/dataframe.rs @@ -252,6 +252,22 @@ reveal_type(pl.read_csv("data.csv", schema={"a": pl.Int64, "b": pl.Float64, "c": "#, ); +polars_testcase!( + test_read_csv_schema_overrides_without_schema, + r#" +import polars as pl +from typing import reveal_type + +reveal_type(pl.read_csv("data.csv", schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame +reveal_type(pl.read_csv("data.csv", columns=["a"], schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame[a: Int64] +reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame[a: Int64, b: Float64] +reveal_type(pl.read_csv("data.csv", columns=["a", "b", "c"], schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame[a: Int64, b: Float64, c: Unknown] +reveal_type(pl.read_csv("data.csv", schema_overrides=[pl.Int64, pl.Float64])) # E: revealed type: DataFrame +reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides=[pl.Int64, pl.Float64])) # E: revealed type: DataFrame[a: Int64, b: Float64] +reveal_type(pl.read_csv("data.csv", columns=["a", "b", "c"], schema_overrides=[pl.Int64, pl.Float64])) # E: revealed type: DataFrame[a: Int64, b: Float64, c: Unknown] +"#, +); + polars_testcase!( test_scan_csv_output_columns, r#" From fbbf63bb276b7170abc15a010a731ec50d0c4cc0 Mon Sep 17 00:00:00 2001 From: tonyyuyiding Date: Mon, 5 Oct 2026 18:08:46 +0800 Subject: [PATCH 2/7] update docs --- website/docs/dataframes.mdx | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/website/docs/dataframes.mdx b/website/docs/dataframes.mdx index c1c1b7abb0..8a81f32684 100644 --- a/website/docs/dataframes.mdx +++ b/website/docs/dataframes.mdx @@ -320,13 +320,14 @@ the plain `DataFrame` annotation that other tools accept. The previous section declares a schema for data you already hold in memory. This section covers the same idea for data read from a file. Pyrefly never -inspects file contents, so a CSV reader gets a schema only when you declare one -inline at the call site. +inspects file contents, so a CSV reader infers a schema only when enough +information is provided inline at the call site. Pyrefly infers complete schemas for `pl.read_csv` and `pl.scan_csv` when `schema=` is an inline dictionary or inline `pl.Schema`. `read_csv` produces a typed DataFrame, and `scan_csv` produces a typed LazyFrame whose schema survives -`collect()`. +`collect()`. If `schema` is not provided but `columns=` and `schema_overrides=` +are, `read_csv` will try to infer a schema from them. ```python reveal_type( @@ -337,6 +338,14 @@ reveal_type( ) ) # DataFrame[name: String] +reveal_type( + pl.read_csv( + "data.csv", + columns=["id", "name", "notes"], + schema_overrides={"id": pl.Int64, "name": pl.String}, + ) +) # DataFrame[id: Int64, name: String, notes: Unknown] + lazy = pl.scan_csv( "data.csv", schema={"id": pl.Int64, "name": pl.String}, From a96e7a505f026284ba8436a4eae62e188abfa015 Mon Sep 17 00:00:00 2001 From: tonyyuyiding Date: Wed, 18 Nov 2026 10:25:34 +0800 Subject: [PATCH 3/7] fix: schema_overrides not resolvable --- pyrefly/lib/alt/polars_specials.rs | 10 +++++++--- pyrefly/lib/test/polars/dataframe.rs | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/pyrefly/lib/alt/polars_specials.rs b/pyrefly/lib/alt/polars_specials.rs index 44f0911821..2ad99857e9 100644 --- a/pyrefly/lib/alt/polars_specials.rs +++ b/pyrefly/lib/alt/polars_specials.rs @@ -1351,7 +1351,8 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { }; overrides.insert( self.polars_column_name(key)?, - self.polars_dtype_from_expr(value)?, + self.polars_dtype_from_expr(value) + .unwrap_or(PolarsDType::Unknown), ); } CsvDtypeOverrides::Mapping(overrides) @@ -1359,8 +1360,11 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { Some(expr) => CsvDtypeOverrides::Sequence( literal_sequence(expr)? .iter() - .map(|expr| self.polars_dtype_from_expr(expr)) - .collect::>>()?, + .map(|expr| { + self.polars_dtype_from_expr(expr) + .unwrap_or(PolarsDType::Unknown) + }) + .collect(), ), None => CsvDtypeOverrides::Unchanged, }) diff --git a/pyrefly/lib/test/polars/dataframe.rs b/pyrefly/lib/test/polars/dataframe.rs index 9ff7fbe281..3940f66793 100644 --- a/pyrefly/lib/test/polars/dataframe.rs +++ b/pyrefly/lib/test/polars/dataframe.rs @@ -268,6 +268,21 @@ reveal_type(pl.read_csv("data.csv", columns=["a", "b", "c"], schema_overrides=[p "#, ); +polars_testcase!( + test_csv_unresolvable_override_dtypes_keep_schema, + r#" +import polars as pl +from typing import reveal_type + +dt: object = ... + +reveal_type(pl.read_csv("data.csv", schema={"a": pl.Int64}, schema_overrides={"a": dt})) # E: revealed type: DataFrame[a: Int64] +reveal_type(pl.read_csv("data.csv", schema={"a": pl.Int64, "b": pl.String}, schema_overrides=[dt])) # E: revealed type: DataFrame[a: Unknown, b: String] +reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides={"a": pl.Int64, "b": dt})) # E: revealed type: DataFrame[a: Int64, b: Unknown] +reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides=[pl.Int64, dt])) # E: revealed type: DataFrame[a: Int64, b: Unknown] +"#, +); + polars_testcase!( test_scan_csv_output_columns, r#" From eee0661c2d329e6b33f135cbd398aeb6d8580385 Mon Sep 17 00:00:00 2001 From: tonyyuyiding Date: Wed, 18 Nov 2026 10:49:09 +0800 Subject: [PATCH 4/7] code quality --- pyrefly/lib/alt/polars_specials.rs | 79 ++++++++++++++---------------- 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/pyrefly/lib/alt/polars_specials.rs b/pyrefly/lib/alt/polars_specials.rs index 2ad99857e9..f91fc664e4 100644 --- a/pyrefly/lib/alt/polars_specials.rs +++ b/pyrefly/lib/alt/polars_specials.rs @@ -1553,42 +1553,8 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { new_columns, row_index_name, } = common; - match schema { - CsvSchema::Given(mut columns) => { - if matches!(&overrides, CsvDtypeOverrides::Sequence(_)) - && !matches!(selection, CsvColumnSelection::All) - { - return None; - } - if !Self::apply_polars_csv_sequence_overrides(&mut columns, overrides) { - return None; - } - if !matches!(selection, CsvColumnSelection::All) { - if let CsvColumnSelection::Names(names) = &selection - && !names - .iter() - .all(|name| columns.iter().any(|(column, _)| column == name)) - { - return None; - } - columns = columns - .into_iter() - .enumerate() - .filter(|(index, (name, _))| match &selection { - CsvColumnSelection::All => true, - CsvColumnSelection::Names(names) => names.contains(name), - CsvColumnSelection::Indices(indices) => indices.contains(index), - }) - .map(|(_, column)| column) - .collect(); - } - if !Self::apply_polars_csv_row_index(&mut columns, row_index_name) - || !Self::apply_polars_csv_new_columns(&mut columns, new_columns) - { - return None; - } - Some(columns) - } + let (mut columns, selection) = match schema { + CsvSchema::Given(columns) => (columns, selection), CsvSchema::Missing => { let CsvColumnSelection::Names(names) = selection else { return None; @@ -1601,15 +1567,42 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { .map(|name| (name, PolarsDType::Unknown)) .collect(); Self::apply_polars_csv_mapping_overrides(&mut columns, &overrides); - if !Self::apply_polars_csv_sequence_overrides(&mut columns, overrides) - || !Self::apply_polars_csv_row_index(&mut columns, row_index_name) - || !Self::apply_polars_csv_new_columns(&mut columns, new_columns) - { - return None; - } - Some(columns) + (columns, CsvColumnSelection::All) } + }; + if matches!(&overrides, CsvDtypeOverrides::Sequence(_)) + && !matches!(selection, CsvColumnSelection::All) + { + return None; + } + if !Self::apply_polars_csv_sequence_overrides(&mut columns, overrides) { + return None; + } + if !matches!(selection, CsvColumnSelection::All) { + if let CsvColumnSelection::Names(names) = &selection + && !names + .iter() + .all(|name| columns.iter().any(|(column, _)| column == name)) + { + return None; + } + columns = columns + .into_iter() + .enumerate() + .filter(|(index, (name, _))| match &selection { + CsvColumnSelection::All => true, + CsvColumnSelection::Names(names) => names.contains(name), + CsvColumnSelection::Indices(indices) => indices.contains(index), + }) + .map(|(_, column)| column) + .collect(); + } + if !Self::apply_polars_csv_row_index(&mut columns, row_index_name) + || !Self::apply_polars_csv_new_columns(&mut columns, new_columns) + { + return None; } + Some(columns) } /// Lazy CSV inference applies overrides before inserting the row index. From 348cae22f7e9c395cb15653dc121dc74a44f6fe5 Mon Sep 17 00:00:00 2001 From: tonyyuyiding Date: Wed, 18 Nov 2026 10:58:46 +0800 Subject: [PATCH 5/7] add comment --- pyrefly/lib/alt/polars_specials.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pyrefly/lib/alt/polars_specials.rs b/pyrefly/lib/alt/polars_specials.rs index f91fc664e4..4a5aca6c66 100644 --- a/pyrefly/lib/alt/polars_specials.rs +++ b/pyrefly/lib/alt/polars_specials.rs @@ -1554,6 +1554,7 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { row_index_name, } = common; let (mut columns, selection) = match schema { + // A complete schema changes only for positional dtype overrides. CsvSchema::Given(columns) => (columns, selection), CsvSchema::Missing => { let CsvColumnSelection::Names(names) = selection else { From 3ae533c8e06d3e2328e6541a74e8efd9fb9865c1 Mon Sep 17 00:00:00 2001 From: tonyyuyiding Date: Thu, 19 Nov 2026 18:55:55 +0800 Subject: [PATCH 6/7] Keep columns even without schema overrides --- pyrefly/lib/alt/polars_specials.rs | 3 --- pyrefly/lib/test/polars/dataframe.rs | 12 ++++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pyrefly/lib/alt/polars_specials.rs b/pyrefly/lib/alt/polars_specials.rs index 4a5aca6c66..48d13a4fff 100644 --- a/pyrefly/lib/alt/polars_specials.rs +++ b/pyrefly/lib/alt/polars_specials.rs @@ -1560,9 +1560,6 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { let CsvColumnSelection::Names(names) = selection else { return None; }; - if matches!(overrides, CsvDtypeOverrides::Unchanged) { - return None; - } let mut columns: Vec<(Name, PolarsDType)> = names .into_iter() .map(|name| (name, PolarsDType::Unknown)) diff --git a/pyrefly/lib/test/polars/dataframe.rs b/pyrefly/lib/test/polars/dataframe.rs index 3940f66793..6783d156ba 100644 --- a/pyrefly/lib/test/polars/dataframe.rs +++ b/pyrefly/lib/test/polars/dataframe.rs @@ -268,6 +268,18 @@ reveal_type(pl.read_csv("data.csv", columns=["a", "b", "c"], schema_overrides=[p "#, ); +polars_testcase!( + test_read_csv_columns_without_valid_overrides, + r#" +import polars as pl +from typing import reveal_type + +reveal_type(pl.read_csv("data.csv", columns=["a"])) # E: revealed type: DataFrame[a: Unknown] +reveal_type(pl.read_csv("data.csv", columns=["a"], schema_overrides={})) # E: revealed type: DataFrame[a: Unknown] +reveal_type(pl.read_csv("data.csv", columns=["a"], schema_overrides={"z": pl.Int64})) # E: revealed type: DataFrame[a: Unknown] +"#, +); + polars_testcase!( test_csv_unresolvable_override_dtypes_keep_schema, r#" From ee4d6e3cc653caa9ae1ce41c5dc6e7397c5d9ae4 Mon Sep 17 00:00:00 2001 From: tonyyuyiding Date: Thu, 19 Nov 2026 19:08:35 +0800 Subject: [PATCH 7/7] update docs --- website/docs/dataframes.mdx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/docs/dataframes.mdx b/website/docs/dataframes.mdx index 8a81f32684..95feaddd5c 100644 --- a/website/docs/dataframes.mdx +++ b/website/docs/dataframes.mdx @@ -326,8 +326,9 @@ information is provided inline at the call site. Pyrefly infers complete schemas for `pl.read_csv` and `pl.scan_csv` when `schema=` is an inline dictionary or inline `pl.Schema`. `read_csv` produces a typed DataFrame, and `scan_csv` produces a typed LazyFrame whose schema survives -`collect()`. If `schema` is not provided but `columns=` and `schema_overrides=` -are, `read_csv` will try to infer a schema from them. +`collect()`. If `schema` is not provided but `columns=` is a static name list, +`read_csv` infers those columns with dtypes from `schema_overrides=` if +provided, or `Unknown` otherwise. ```python reveal_type(