diff --git a/pyrefly/lib/alt/polars_specials.rs b/pyrefly/lib/alt/polars_specials.rs index eb6cfa1cb4..48d13a4fff 100644 --- a/pyrefly/lib/alt/polars_specials.rs +++ b/pyrefly/lib/alt/polars_specials.rs @@ -369,31 +369,19 @@ enum CsvColumnSelection { Indices(SmallSet), } +enum CsvSchema { + Missing, + Given(Vec<(Name, PolarsDType)>), +} + enum CsvDtypeOverrides { Unchanged, Sequence(Vec), -} - -impl CsvDtypeOverrides { - /// A complete schema changes only for positional dtype overrides. - fn parse( - expr: Option<&Expr>, - parse_dtype: impl FnMut(&Expr) -> Option, - ) -> Option { - Some(match expr { - None | Some(Expr::NoneLiteral(_) | Expr::Dict(_)) => Self::Unchanged, - Some(expr) => Self::Sequence( - literal_sequence(expr)? - .iter() - .map(parse_dtype) - .collect::>>()?, - ), - }) - } + Mapping(SmallMap), } struct PolarsCsvCommonOptions { - schema: Vec<(Name, PolarsDType)>, + schema: CsvSchema, overrides: CsvDtypeOverrides, new_columns: Vec, row_index_name: Option, @@ -1333,15 +1321,53 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { .then_some(CsvColumnSelection::Indices(indices)) } - fn polars_csv_schema(&self, expr: &Expr) -> Option> { - let (form, dict) = self.schema_literal_dict(expr)?; - if dict.items.is_empty() { - return Some(Vec::new()); - } - self.schema_dict_entries(form, dict)? - .into_iter() - .map(|(name, dtype)| Some((name, dtype?))) - .collect() + fn polars_csv_schema(&self, expr: Option<&Expr>) -> Option { + Some(match expr { + Some(expr) => { + let (form, dict) = self.schema_literal_dict(expr)?; + if dict.items.is_empty() { + CsvSchema::Given(Vec::new()) + } else { + CsvSchema::Given( + self.schema_dict_entries(form, dict)? + .into_iter() + .map(|(name, dtype)| Some((name, dtype?))) + .collect::>>()?, + ) + } + } + None => CsvSchema::Missing, + }) + } + + fn polars_csv_schema_overrides(&self, expr: Option<&Expr>) -> Option { + Some(match expr { + Some(Expr::NoneLiteral(_)) => CsvDtypeOverrides::Unchanged, + Some(Expr::Dict(dict)) => { + let mut overrides = SmallMap::with_capacity(dict.items.len()); + for item in &dict.items { + let (Some(key), value) = (&item.key, &item.value) else { + return None; + }; + overrides.insert( + self.polars_column_name(key)?, + self.polars_dtype_from_expr(value) + .unwrap_or(PolarsDType::Unknown), + ); + } + CsvDtypeOverrides::Mapping(overrides) + } + Some(expr) => CsvDtypeOverrides::Sequence( + literal_sequence(expr)? + .iter() + .map(|expr| { + self.polars_dtype_from_expr(expr) + .unwrap_or(PolarsDType::Unknown) + }) + .collect(), + ), + None => CsvDtypeOverrides::Unchanged, + }) } /// An omitted option and explicit `None` both represent semantic absence. @@ -1384,6 +1410,21 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { true } + /// Replace dtypes of the columns present in the mapping. Other columns remain unchanged. + fn apply_polars_csv_mapping_overrides( + columns: &mut [(Name, PolarsDType)], + overrides: &CsvDtypeOverrides, + ) { + let CsvDtypeOverrides::Mapping(map) = overrides else { + return; + }; + for (name, dtype) in columns.iter_mut() { + if let Some(override_dtype) = map.get(name) { + *dtype = override_dtype.clone(); + } + } + } + /// New names replace leading columns unless they exceed the width or create duplicates. fn apply_polars_csv_new_columns( columns: &mut [(Name, PolarsDType)], @@ -1462,9 +1503,8 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { return None; } - let schema = self.polars_csv_schema(schema?)?; - let overrides = - CsvDtypeOverrides::parse(overrides, |expr| self.polars_dtype_from_expr(expr))?; + let schema = self.polars_csv_schema(schema)?; + let overrides = self.polars_csv_schema_overrides(overrides)?; let new_columns = self.polars_csv_names(new_columns)?; let row_index_name = self.polars_optional_csv_name(row_index_name)?; let common = PolarsCsvCommonOptions { @@ -1478,7 +1518,13 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { PolarsCsvFunction::Read => { let selection = match Self::non_none_csv_option(selection) { None => CsvColumnSelection::All, - Some(expr) => self.polars_csv_selection(expr, common.schema.len())?, + Some(expr) => { + let width = match &common.schema { + CsvSchema::Given(columns) => columns.len(), + CsvSchema::Missing => 0, // Reject selection by index if schema is missing + }; + self.polars_csv_selection(expr, width)? + } }; PolarsCsvOptions::Read { common, selection } } @@ -1502,11 +1548,26 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { selection: CsvColumnSelection, ) -> Option> { let PolarsCsvCommonOptions { - schema: mut columns, + schema, overrides, new_columns, row_index_name, } = common; + let (mut columns, selection) = match schema { + // A complete schema changes only for positional dtype overrides. + CsvSchema::Given(columns) => (columns, selection), + CsvSchema::Missing => { + let CsvColumnSelection::Names(names) = selection else { + return None; + }; + let mut columns: Vec<(Name, PolarsDType)> = names + .into_iter() + .map(|name| (name, PolarsDType::Unknown)) + .collect(); + Self::apply_polars_csv_mapping_overrides(&mut columns, &overrides); + (columns, CsvColumnSelection::All) + } + }; if matches!(&overrides, CsvDtypeOverrides::Sequence(_)) && !matches!(selection, CsvColumnSelection::All) { @@ -1550,11 +1611,14 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> { include_file_paths: Option, ) -> Option> { let PolarsCsvCommonOptions { - schema: mut columns, + schema, overrides, new_columns, row_index_name, } = common; + let CsvSchema::Given(mut columns) = schema else { + return None; + }; if !Self::apply_polars_csv_sequence_overrides(&mut columns, overrides) || !Self::apply_polars_csv_row_index(&mut columns, row_index_name) || !Self::apply_polars_csv_new_columns(&mut columns, new_columns) diff --git a/pyrefly/lib/test/polars/dataframe.rs b/pyrefly/lib/test/polars/dataframe.rs index d1fdcf80fb..6783d156ba 100644 --- a/pyrefly/lib/test/polars/dataframe.rs +++ b/pyrefly/lib/test/polars/dataframe.rs @@ -252,6 +252,49 @@ reveal_type(pl.read_csv("data.csv", schema={"a": pl.Int64, "b": pl.Float64, "c": "#, ); +polars_testcase!( + test_read_csv_schema_overrides_without_schema, + r#" +import polars as pl +from typing import reveal_type + +reveal_type(pl.read_csv("data.csv", schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame +reveal_type(pl.read_csv("data.csv", columns=["a"], schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame[a: Int64] +reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame[a: Int64, b: Float64] +reveal_type(pl.read_csv("data.csv", columns=["a", "b", "c"], schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame[a: Int64, b: Float64, c: Unknown] +reveal_type(pl.read_csv("data.csv", schema_overrides=[pl.Int64, pl.Float64])) # E: revealed type: DataFrame +reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides=[pl.Int64, pl.Float64])) # E: revealed type: DataFrame[a: Int64, b: Float64] +reveal_type(pl.read_csv("data.csv", columns=["a", "b", "c"], schema_overrides=[pl.Int64, pl.Float64])) # E: revealed type: DataFrame[a: Int64, b: Float64, c: Unknown] +"#, +); + +polars_testcase!( + test_read_csv_columns_without_valid_overrides, + r#" +import polars as pl +from typing import reveal_type + +reveal_type(pl.read_csv("data.csv", columns=["a"])) # E: revealed type: DataFrame[a: Unknown] +reveal_type(pl.read_csv("data.csv", columns=["a"], schema_overrides={})) # E: revealed type: DataFrame[a: Unknown] +reveal_type(pl.read_csv("data.csv", columns=["a"], schema_overrides={"z": pl.Int64})) # E: revealed type: DataFrame[a: Unknown] +"#, +); + +polars_testcase!( + test_csv_unresolvable_override_dtypes_keep_schema, + r#" +import polars as pl +from typing import reveal_type + +dt: object = ... + +reveal_type(pl.read_csv("data.csv", schema={"a": pl.Int64}, schema_overrides={"a": dt})) # E: revealed type: DataFrame[a: Int64] +reveal_type(pl.read_csv("data.csv", schema={"a": pl.Int64, "b": pl.String}, schema_overrides=[dt])) # E: revealed type: DataFrame[a: Unknown, b: String] +reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides={"a": pl.Int64, "b": dt})) # E: revealed type: DataFrame[a: Int64, b: Unknown] +reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides=[pl.Int64, dt])) # E: revealed type: DataFrame[a: Int64, b: Unknown] +"#, +); + polars_testcase!( test_scan_csv_output_columns, r#" diff --git a/website/docs/dataframes.mdx b/website/docs/dataframes.mdx index c1c1b7abb0..95feaddd5c 100644 --- a/website/docs/dataframes.mdx +++ b/website/docs/dataframes.mdx @@ -320,13 +320,15 @@ the plain `DataFrame` annotation that other tools accept. The previous section declares a schema for data you already hold in memory. This section covers the same idea for data read from a file. Pyrefly never -inspects file contents, so a CSV reader gets a schema only when you declare one -inline at the call site. +inspects file contents, so a CSV reader infers a schema only when enough +information is provided inline at the call site. Pyrefly infers complete schemas for `pl.read_csv` and `pl.scan_csv` when `schema=` is an inline dictionary or inline `pl.Schema`. `read_csv` produces a typed DataFrame, and `scan_csv` produces a typed LazyFrame whose schema survives -`collect()`. +`collect()`. If `schema` is not provided but `columns=` is a static name list, +`read_csv` infers those columns with dtypes from `schema_overrides=` if +provided, or `Unknown` otherwise. ```python reveal_type( @@ -337,6 +339,14 @@ reveal_type( ) ) # DataFrame[name: String] +reveal_type( + pl.read_csv( + "data.csv", + columns=["id", "name", "notes"], + schema_overrides={"id": pl.Int64, "name": pl.String}, + ) +) # DataFrame[id: Int64, name: String, notes: Unknown] + lazy = pl.scan_csv( "data.csv", schema={"id": pl.Int64, "name": pl.String},