Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 98 additions & 34 deletions pyrefly/lib/alt/polars_specials.rs
Original file line number Diff line number Diff line change
Expand Up @@ -369,31 +369,19 @@ enum CsvColumnSelection {
Indices(SmallSet<usize>),
}

enum CsvSchema {
Missing,
Given(Vec<(Name, PolarsDType)>),
}

enum CsvDtypeOverrides {
Unchanged,
Sequence(Vec<PolarsDType>),
}

impl CsvDtypeOverrides {
/// A complete schema changes only for positional dtype overrides.
fn parse(
expr: Option<&Expr>,
parse_dtype: impl FnMut(&Expr) -> Option<PolarsDType>,
) -> Option<Self> {
Some(match expr {
None | Some(Expr::NoneLiteral(_) | Expr::Dict(_)) => Self::Unchanged,
Some(expr) => Self::Sequence(
literal_sequence(expr)?
.iter()
.map(parse_dtype)
.collect::<Option<Vec<_>>>()?,
),
})
}
Mapping(SmallMap<Name, PolarsDType>),
}

struct PolarsCsvCommonOptions {
schema: Vec<(Name, PolarsDType)>,
schema: CsvSchema,
overrides: CsvDtypeOverrides,
new_columns: Vec<Name>,
row_index_name: Option<Name>,
Expand Down Expand Up @@ -1333,15 +1321,53 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> {
.then_some(CsvColumnSelection::Indices(indices))
}

fn polars_csv_schema(&self, expr: &Expr) -> Option<Vec<(Name, PolarsDType)>> {
let (form, dict) = self.schema_literal_dict(expr)?;
if dict.items.is_empty() {
return Some(Vec::new());
}
self.schema_dict_entries(form, dict)?
.into_iter()
.map(|(name, dtype)| Some((name, dtype?)))
.collect()
fn polars_csv_schema(&self, expr: Option<&Expr>) -> Option<CsvSchema> {
Some(match expr {
Some(expr) => {
let (form, dict) = self.schema_literal_dict(expr)?;
if dict.items.is_empty() {
CsvSchema::Given(Vec::new())
} else {
CsvSchema::Given(
self.schema_dict_entries(form, dict)?
.into_iter()
.map(|(name, dtype)| Some((name, dtype?)))
.collect::<Option<Vec<_>>>()?,
)
}
}
None => CsvSchema::Missing,
})
}

fn polars_csv_schema_overrides(&self, expr: Option<&Expr>) -> Option<CsvDtypeOverrides> {
Some(match expr {
Some(Expr::NoneLiteral(_)) => CsvDtypeOverrides::Unchanged,
Some(Expr::Dict(dict)) => {
let mut overrides = SmallMap::with_capacity(dict.items.len());
for item in &dict.items {
let (Some(key), value) = (&item.key, &item.value) else {
return None;
};
overrides.insert(
self.polars_column_name(key)?,
self.polars_dtype_from_expr(value)
.unwrap_or(PolarsDType::Unknown),
);
}
CsvDtypeOverrides::Mapping(overrides)
}
Some(expr) => CsvDtypeOverrides::Sequence(
literal_sequence(expr)?
.iter()
.map(|expr| {
self.polars_dtype_from_expr(expr)
.unwrap_or(PolarsDType::Unknown)
})
.collect(),
),
None => CsvDtypeOverrides::Unchanged,
})
}

/// An omitted option and explicit `None` both represent semantic absence.
Expand Down Expand Up @@ -1384,6 +1410,21 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> {
true
}

/// Replace dtypes of the columns present in the mapping. Other columns remain unchanged.
fn apply_polars_csv_mapping_overrides(
columns: &mut [(Name, PolarsDType)],
overrides: &CsvDtypeOverrides,
) {
let CsvDtypeOverrides::Mapping(map) = overrides else {
return;
};
for (name, dtype) in columns.iter_mut() {
if let Some(override_dtype) = map.get(name) {
*dtype = override_dtype.clone();
}
}
}

/// New names replace leading columns unless they exceed the width or create duplicates.
fn apply_polars_csv_new_columns(
columns: &mut [(Name, PolarsDType)],
Expand Down Expand Up @@ -1462,9 +1503,8 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> {
return None;
}

let schema = self.polars_csv_schema(schema?)?;
let overrides =
CsvDtypeOverrides::parse(overrides, |expr| self.polars_dtype_from_expr(expr))?;
let schema = self.polars_csv_schema(schema)?;
let overrides = self.polars_csv_schema_overrides(overrides)?;
let new_columns = self.polars_csv_names(new_columns)?;
let row_index_name = self.polars_optional_csv_name(row_index_name)?;
let common = PolarsCsvCommonOptions {
Expand All @@ -1478,7 +1518,13 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> {
PolarsCsvFunction::Read => {
let selection = match Self::non_none_csv_option(selection) {
None => CsvColumnSelection::All,
Some(expr) => self.polars_csv_selection(expr, common.schema.len())?,
Some(expr) => {
let width = match &common.schema {
CsvSchema::Given(columns) => columns.len(),
CsvSchema::Missing => 0, // Reject selection by index if schema is missing
};
self.polars_csv_selection(expr, width)?
}
};
PolarsCsvOptions::Read { common, selection }
}
Expand All @@ -1502,11 +1548,26 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> {
selection: CsvColumnSelection,
) -> Option<Vec<(Name, PolarsDType)>> {
let PolarsCsvCommonOptions {
schema: mut columns,
schema,
overrides,
new_columns,
row_index_name,
} = common;
let (mut columns, selection) = match schema {
// A complete schema changes only for positional dtype overrides.
CsvSchema::Given(columns) => (columns, selection),
CsvSchema::Missing => {
let CsvColumnSelection::Names(names) = selection else {
return None;
};
let mut columns: Vec<(Name, PolarsDType)> = names
.into_iter()
.map(|name| (name, PolarsDType::Unknown))
.collect();
Self::apply_polars_csv_mapping_overrides(&mut columns, &overrides);
(columns, CsvColumnSelection::All)
}
};
if matches!(&overrides, CsvDtypeOverrides::Sequence(_))
&& !matches!(selection, CsvColumnSelection::All)
{
Expand Down Expand Up @@ -1550,11 +1611,14 @@ impl<'ctx, 'answer, Ans: LookupAnswer> AnswersSolver<'ctx, 'answer, Ans> {
include_file_paths: Option<Name>,
) -> Option<Vec<(Name, PolarsDType)>> {
let PolarsCsvCommonOptions {
schema: mut columns,
schema,
overrides,
new_columns,
row_index_name,
} = common;
let CsvSchema::Given(mut columns) = schema else {
return None;
};
if !Self::apply_polars_csv_sequence_overrides(&mut columns, overrides)
|| !Self::apply_polars_csv_row_index(&mut columns, row_index_name)
|| !Self::apply_polars_csv_new_columns(&mut columns, new_columns)
Expand Down
43 changes: 43 additions & 0 deletions pyrefly/lib/test/polars/dataframe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,49 @@ reveal_type(pl.read_csv("data.csv", schema={"a": pl.Int64, "b": pl.Float64, "c":
"#,
);

polars_testcase!(
test_read_csv_schema_overrides_without_schema,
r#"
import polars as pl
from typing import reveal_type

reveal_type(pl.read_csv("data.csv", schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame
reveal_type(pl.read_csv("data.csv", columns=["a"], schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame[a: Int64]
reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame[a: Int64, b: Float64]
reveal_type(pl.read_csv("data.csv", columns=["a", "b", "c"], schema_overrides={"a": pl.Int64, "b": pl.Float64})) # E: revealed type: DataFrame[a: Int64, b: Float64, c: Unknown]
reveal_type(pl.read_csv("data.csv", schema_overrides=[pl.Int64, pl.Float64])) # E: revealed type: DataFrame
reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides=[pl.Int64, pl.Float64])) # E: revealed type: DataFrame[a: Int64, b: Float64]
reveal_type(pl.read_csv("data.csv", columns=["a", "b", "c"], schema_overrides=[pl.Int64, pl.Float64])) # E: revealed type: DataFrame[a: Int64, b: Float64, c: Unknown]
"#,
);

polars_testcase!(
test_read_csv_columns_without_valid_overrides,
r#"
import polars as pl
from typing import reveal_type

reveal_type(pl.read_csv("data.csv", columns=["a"])) # E: revealed type: DataFrame[a: Unknown]
reveal_type(pl.read_csv("data.csv", columns=["a"], schema_overrides={})) # E: revealed type: DataFrame[a: Unknown]
reveal_type(pl.read_csv("data.csv", columns=["a"], schema_overrides={"z": pl.Int64})) # E: revealed type: DataFrame[a: Unknown]
"#,
);

polars_testcase!(
test_csv_unresolvable_override_dtypes_keep_schema,
r#"
import polars as pl
from typing import reveal_type

dt: object = ...

reveal_type(pl.read_csv("data.csv", schema={"a": pl.Int64}, schema_overrides={"a": dt})) # E: revealed type: DataFrame[a: Int64]
reveal_type(pl.read_csv("data.csv", schema={"a": pl.Int64, "b": pl.String}, schema_overrides=[dt])) # E: revealed type: DataFrame[a: Unknown, b: String]
reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides={"a": pl.Int64, "b": dt})) # E: revealed type: DataFrame[a: Int64, b: Unknown]
reveal_type(pl.read_csv("data.csv", columns=["a", "b"], schema_overrides=[pl.Int64, dt])) # E: revealed type: DataFrame[a: Int64, b: Unknown]
"#,
);

polars_testcase!(
test_scan_csv_output_columns,
r#"
Expand Down
16 changes: 13 additions & 3 deletions website/docs/dataframes.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -320,13 +320,15 @@ the plain `DataFrame` annotation that other tools accept.

The previous section declares a schema for data you already hold in memory.
This section covers the same idea for data read from a file. Pyrefly never
inspects file contents, so a CSV reader gets a schema only when you declare one
inline at the call site.
inspects file contents, so a CSV reader infers a schema only when enough
information is provided inline at the call site.

Pyrefly infers complete schemas for `pl.read_csv` and `pl.scan_csv` when
`schema=` is an inline dictionary or inline `pl.Schema`. `read_csv` produces a
typed DataFrame, and `scan_csv` produces a typed LazyFrame whose schema survives
`collect()`.
`collect()`. If `schema` is not provided but `columns=` is a static name list,
`read_csv` infers those columns with dtypes from `schema_overrides=` if
provided, or `Unknown` otherwise.

```python
reveal_type(
Expand All @@ -337,6 +339,14 @@ reveal_type(
)
) # DataFrame[name: String]

reveal_type(
pl.read_csv(
"data.csv",
columns=["id", "name", "notes"],
schema_overrides={"id": pl.Int64, "name": pl.String},
)
) # DataFrame[id: Int64, name: String, notes: Unknown]

lazy = pl.scan_csv(
"data.csv",
schema={"id": pl.Int64, "name": pl.String},
Expand Down
Loading