Skip to content

Commit 59b1a3f

Browse files
committed
Propagate the collation exception on all paths
The failed collation returns a wrong result to SQLite. Thus, the collation exception is the cause of any error later in the step. After each step, check PyErr_Occurred() first and propagate the exception. This also prevents execute() and executemany() from calling converters, adapters and parameter iterators while the exception is set. Other callbacks can run in the same step after the collation raised. Do not call Python code from them while the exception is set. Remove the code that printed or cleared the exception.
1 parent 6733679 commit 59b1a3f

4 files changed

Lines changed: 184 additions & 88 deletions

File tree

‎Lib/test/test_sqlite3/test_hooks.py‎

Lines changed: 125 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -150,79 +150,149 @@ def test_deregister_collation(self):
150150

151151
class CollationErrorTests(MemoryDatabaseMixin, unittest.TestCase):
152152
# gh-161837: A collation callback cannot report an error to SQLite, so
153-
# the exception stays set while SQLite continues the step.
153+
# the exception stays set while SQLite continues the step. The module
154+
# must propagate that exception, and it must not call other Python code
155+
# while the exception is set.
154156

155-
# When the collation raises, the comparison is true, so abs() of the
156-
# smallest 64-bit integer causes "integer overflow".
157+
# When the collation raises, it returns 0 ("equal"). Thus the comparison
158+
# is true, and abs() of the smallest 64-bit integer makes the step fail.
157159
step_fails = """
158160
select case when y = 'q' collate mycoll then abs(x) else 0 end
159161
from t order by rowid
160162
"""
161163
step_succeeds = "select x, y = 'q' collate mycoll from t order by rowid"
162-
# The cursor reads one row ahead, so the first fetch steps to the
163-
# second row, where the collation raises.
164164

165165
def setUp(self):
166166
super().setUp()
167+
self.events = []
167168
def mycoll(x, y):
168169
if x == "boom":
170+
self.events.append("collation raised")
169171
raise KeyError("collation raised")
170172
return (x > y) - (x < y)
171173
self.con.create_collation("mycoll", mycoll)
172174
self.con.execute("create table t(x, y)")
173175
self.con.executemany("insert into t values (?, ?)",
174176
[(1, "a"), (-2**63, "boom")])
175177

176-
@with_tracebacks(KeyError, regex="collation raised",
177-
msg_regex="sqlite3 callback")
178-
def test_step_failure_in_execute(self):
179-
with self.assertRaisesRegex(sqlite.OperationalError,
180-
"integer overflow"):
181-
self.con.execute(self.step_fails + " desc")
182-
183-
@with_tracebacks(KeyError, regex="collation raised",
184-
msg_regex="sqlite3 callback")
185-
def test_step_failure_in_fetchone(self):
186-
cur = self.con.execute(self.step_fails)
187-
with self.assertRaisesRegex(sqlite.OperationalError,
188-
"integer overflow"):
189-
cur.fetchone()
190-
self.assertIsNone(cur.fetchone())
191-
192-
@with_tracebacks(KeyError, regex="collation raised",
193-
msg_regex="sqlite3 callback")
194-
def test_step_failure_in_iteration(self):
195-
with self.assertRaisesRegex(sqlite.OperationalError,
196-
"integer overflow"):
197-
list(self.con.execute(self.step_fails))
198-
199-
@with_tracebacks(KeyError, regex="collation raised",
200-
msg_regex="sqlite3 callback")
201-
def test_step_failure_in_executescript(self):
202-
with self.assertRaisesRegex(sqlite.OperationalError,
203-
"integer overflow"):
204-
self.con.executescript(self.step_fails)
205-
206-
def test_error_in_fetchone(self):
207-
cur = self.con.execute(self.step_succeeds)
208-
with self.assertRaisesRegex(KeyError, "collation raised"):
209-
cur.fetchone()
210-
self.assertIsNone(cur.fetchone())
211-
self.assertEqual(cur.execute("select 1").fetchone(), (1,))
212-
213-
def test_error_in_iteration(self):
214-
rows = []
215-
with self.assertRaisesRegex(KeyError, "collation raised"):
216-
for row in self.con.execute(self.step_succeeds):
217-
rows.append(row)
218-
self.assertEqual(rows, [])
219-
220-
def test_error_in_executescript(self):
221-
with self.assertRaisesRegex(KeyError, "collation raised"):
222-
self.con.executescript(self.step_succeeds + "; create table u(x)")
223-
# The script stops after the statement that caused the error.
224-
with self.assertRaisesRegex(sqlite.OperationalError, "no such table"):
225-
self.con.execute("select * from u")
178+
def assertCollationRaises(self):
179+
return self.assertRaisesRegex(KeyError, "collation raised")
180+
181+
def test_execute(self):
182+
for sql in self.step_fails, self.step_succeeds:
183+
with self.subTest(sql=sql), self.assertCollationRaises():
184+
self.con.execute(sql + " desc")
185+
186+
def test_fetchone(self):
187+
# The cursor reads one row ahead, so the first fetch steps to the
188+
# row that makes the collation raise.
189+
for sql in self.step_fails, self.step_succeeds:
190+
with self.subTest(sql=sql):
191+
cur = self.con.execute(sql)
192+
with self.assertCollationRaises():
193+
cur.fetchone()
194+
self.assertIsNone(cur.fetchone())
195+
self.assertEqual(cur.execute("select 1").fetchone(), (1,))
196+
197+
def test_iteration(self):
198+
for sql in self.step_fails, self.step_succeeds:
199+
with self.subTest(sql=sql):
200+
rows = []
201+
with self.assertCollationRaises():
202+
for row in self.con.execute(sql):
203+
rows.append(row)
204+
self.assertEqual(rows, [])
205+
206+
def test_executemany(self):
207+
def params():
208+
yield ("boom",)
209+
self.fail("executemany() continued after the exception")
210+
with self.assertCollationRaises():
211+
self.con.executemany(
212+
"insert into t(y) values (? = 'q' collate mycoll)", params())
213+
214+
def test_executescript(self):
215+
for sql in self.step_fails, self.step_succeeds:
216+
with self.subTest(sql=sql):
217+
with self.assertCollationRaises():
218+
self.con.executescript(sql + "; create table u(x)")
219+
# The script stops after the statement with the exception.
220+
with self.assertRaisesRegex(sqlite.OperationalError,
221+
"no such table"):
222+
self.con.execute("select * from u")
223+
224+
def test_converter_not_used(self):
225+
with memory_database(detect_types=sqlite.PARSE_COLNAMES) as con:
226+
con.create_collation("mycoll", lambda x, y: 1 / 0)
227+
with self.assertRaises(ZeroDivisionError):
228+
con.execute("select 'a' = 'b' collate mycoll as \"x [conv]\"")
229+
230+
def test_function_not_called(self):
231+
calls = []
232+
self.con.create_function("f", 1, calls.append)
233+
with self.assertCollationRaises():
234+
self.con.execute("select f(y = 'q' collate mycoll) from t "
235+
"order by rowid desc")
236+
self.assertEqual(calls, [])
237+
238+
def test_aggregate_not_called(self):
239+
calls = []
240+
class Agg:
241+
def step(self, value):
242+
calls.append(value)
243+
def finalize(self):
244+
return len(calls)
245+
self.con.create_aggregate("agg", 1, Agg)
246+
with self.assertCollationRaises():
247+
self.con.execute("select agg(y = 'q' collate mycoll) from t "
248+
"where rowid = 2")
249+
self.assertEqual(calls, [])
250+
251+
def test_window_function_not_called(self):
252+
calls = []
253+
class Window:
254+
def step(self, value):
255+
calls.append("step")
256+
def inverse(self, value):
257+
calls.append("inverse")
258+
def value(self):
259+
calls.append("value")
260+
return 0
261+
def finalize(self):
262+
return 0
263+
self.con.create_window_function("win", 1, Window)
264+
# SQLite evaluates the filter for all rows before it calls the
265+
# window function. The filter is false for the first row, so the
266+
# first call after the exception is value().
267+
with self.assertCollationRaises():
268+
self.con.execute("""
269+
select win(x) filter (where y = 'q' collate mycoll)
270+
over (order by rowid) from t
271+
""").fetchall()
272+
self.assertEqual(calls, [])
273+
274+
def test_progress_handler_not_called(self):
275+
def progress():
276+
self.events.append("progress")
277+
return 0
278+
self.con.set_progress_handler(progress, 1)
279+
with self.assertCollationRaises():
280+
self.con.execute(self.step_succeeds + " desc")
281+
self.con.set_progress_handler(None, 1)
282+
self.assertEqual(self.events[-1], "collation raised")
283+
self.assertIn("progress", self.events)
284+
285+
def test_trace_callback_not_called(self):
286+
self.con.execute("create table u(x)")
287+
self.con.execute("create trigger tr after insert on u "
288+
"begin select 1; end")
289+
traced = []
290+
self.con.set_trace_callback(traced.append)
291+
with self.assertCollationRaises():
292+
self.con.execute("insert into u select y = 'q' collate mycoll "
293+
"from t order by rowid desc")
294+
self.con.set_trace_callback(None)
295+
self.assertNotIn("-- TRIGGER tr", traced)
226296

227297

228298
class AuthorizerTests(MemoryDatabaseMixin, unittest.TestCase):
Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
1-
Fix a :exc:`SystemError` in :mod:`sqlite3` when a collation callback raises
2-
an exception. This error occurs when you fetch rows from a cursor and when you
3-
call :meth:`~sqlite3.Cursor.executescript`. Now the exception from the
4-
collation propagates. If the step also fails, the module raises
5-
:exc:`~sqlite3.OperationalError`, as :meth:`~sqlite3.Cursor.execute` does.
6-
When callback tracebacks are enabled, the module now reports the collation
7-
exception with :func:`sys.unraisablehook`, as the documentation says.
1+
Fix a :exc:`SystemError` and crashes in :mod:`sqlite3` when a collation
2+
callback raises an exception. Now the exception from the collation always
3+
propagates. Before, :meth:`~sqlite3.Cursor.execute` raised
4+
:exc:`~sqlite3.OperationalError` instead if the same step also failed. While
5+
the exception is set, the module no longer calls other callbacks, such as
6+
user-defined functions.

‎Modules/_sqlite/connection.c‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -960,10 +960,32 @@ set_sqlite_error(sqlite3_context *context, const char *msg)
960960
print_or_clear_traceback(ctx);
961961
}
962962

963+
/* If a collation callback raises, the exception stays set until
964+
* sqlite3_step() returns (see collation_callback()). Other callbacks in the
965+
* same step must not call Python code while the exception is set. Return 1
966+
* if an exception is set. If context is not NULL, also make the SQL function
967+
* fail, so that SQLite stops the step. */
968+
static int
969+
exception_is_pending(sqlite3_context *context)
970+
{
971+
if (!PyErr_Occurred()) {
972+
return 0;
973+
}
974+
if (context != NULL) {
975+
sqlite3_result_error(context,
976+
"an earlier callback raised an exception", -1);
977+
}
978+
return 1;
979+
}
980+
963981
static void
964982
func_callback(sqlite3_context *context, int argc, sqlite3_value **argv)
965983
{
966984
PyGILState_STATE threadstate = PyGILState_Ensure();
985+
if (exception_is_pending(context)) {
986+
PyGILState_Release(threadstate);
987+
return;
988+
}
967989

968990
PyObject* args;
969991
PyObject* py_retval = NULL;
@@ -994,6 +1016,10 @@ static void
9941016
step_callback(sqlite3_context *context, int argc, sqlite3_value **params)
9951017
{
9961018
PyGILState_STATE threadstate = PyGILState_Ensure();
1019+
if (exception_is_pending(context)) {
1020+
PyGILState_Release(threadstate);
1021+
return;
1022+
}
9971023

9981024
PyObject* args;
9991025
PyObject* function_result = NULL;
@@ -1306,6 +1332,10 @@ static void
13061332
value_callback(sqlite3_context *context)
13071333
{
13081334
PyGILState_STATE gilstate = PyGILState_Ensure();
1335+
if (exception_is_pending(context)) {
1336+
PyGILState_Release(gilstate);
1337+
return;
1338+
}
13091339

13101340
callback_context *ctx = (callback_context *)sqlite3_user_data(context);
13111341
incref_callback_context(ctx);
@@ -1491,6 +1521,11 @@ static int
14911521
progress_callback(void *ctx_vp)
14921522
{
14931523
PyGILState_STATE gilstate = PyGILState_Ensure();
1524+
if (exception_is_pending(NULL)) {
1525+
/* Stop the step. */
1526+
PyGILState_Release(gilstate);
1527+
return 1;
1528+
}
14941529

14951530
int rc;
14961531
PyObject *ret;
@@ -1530,6 +1565,10 @@ trace_callback(unsigned int type, void *ctx_vp, void *stmt, void *sql)
15301565
}
15311566

15321567
PyGILState_STATE gilstate = PyGILState_Ensure();
1568+
if (exception_is_pending(NULL)) {
1569+
PyGILState_Release(gilstate);
1570+
return 0;
1571+
}
15331572

15341573
callback_context *ctx = (callback_context *)ctx_vp;
15351574
incref_callback_context(ctx);

‎Modules/_sqlite/cursor.c‎

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -541,6 +541,10 @@ get_statement_from_cache(pysqlite_Cursor *self, PyObject *operation)
541541
return PyObject_Vectorcall(cache, args + 1, nargsf, NULL);
542542
}
543543

544+
/* A collation callback cannot report an error to SQLite. If it raises, the
545+
* exception stays set until sqlite3_step() returns, and SQLite continues
546+
* the step with the wrong result of the comparison. Thus, after a step, check
547+
* PyErr_Occurred() first, and propagate that exception. */
544548
static inline int
545549
stmt_step(sqlite3_stmt *statement)
546550
{
@@ -553,23 +557,6 @@ stmt_step(sqlite3_stmt *statement)
553557
return rc;
554558
}
555559

556-
/* A collation callback cannot report an error to SQLite, so it keeps the
557-
* Python exception set while SQLite continues the step. If the step then
558-
* fails, call this function to report and clear that exception. The caller
559-
* can then raise the DB-API exception for the failure. */
560-
static void
561-
report_pending_callback_error(pysqlite_state *state)
562-
{
563-
if (PyErr_Occurred()) {
564-
if (state->enable_callback_tracebacks) {
565-
PyErr_FormatUnraisable("Exception ignored on sqlite3 callback");
566-
}
567-
else {
568-
PyErr_Clear();
569-
}
570-
}
571-
}
572-
573560
static int
574561
bind_param(pysqlite_state *state, pysqlite_Statement *self, int pos,
575562
PyObject *parameter)
@@ -927,8 +914,11 @@ _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* operation
927914
}
928915

929916
rc = stmt_step(self->statement->st);
917+
if (PyErr_Occurred()) {
918+
/* A collation callback raised. */
919+
goto error;
920+
}
930921
if (rc != SQLITE_DONE && rc != SQLITE_ROW) {
931-
report_pending_callback_error(state);
932922
set_error_from_db(state, self->connection->db);
933923
goto error;
934924
}
@@ -1135,13 +1125,13 @@ pysqlite_cursor_executescript_impl(pysqlite_Cursor *self,
11351125
}
11361126
Py_END_ALLOW_THREADS
11371127

1138-
if (rc != SQLITE_OK) {
1139-
goto error;
1140-
}
11411128
if (PyErr_Occurred()) {
1142-
/* A collation callback raised, but the step did not fail. */
1129+
/* A collation callback raised. */
11431130
return NULL;
11441131
}
1132+
if (rc != SQLITE_OK) {
1133+
goto error;
1134+
}
11451135

11461136
if (*tail == (char)0) {
11471137
break;
@@ -1153,7 +1143,6 @@ pysqlite_cursor_executescript_impl(pysqlite_Cursor *self,
11531143
return Py_NewRef((PyObject *)self);
11541144

11551145
error:
1156-
report_pending_callback_error(self->connection->state);
11571146
set_error_from_db(self->connection->state, db);
11581147
return NULL;
11591148
}
@@ -1181,8 +1170,8 @@ pysqlite_cursor_iternext(PyObject *op)
11811170
return NULL;
11821171
}
11831172
int rc = stmt_step(stmt);
1184-
if ((rc == SQLITE_ROW || rc == SQLITE_DONE) && PyErr_Occurred()) {
1185-
/* A collation callback raised, but the step did not fail. */
1173+
if (PyErr_Occurred()) {
1174+
/* A collation callback raised. */
11861175
(void)stmt_reset(self->statement);
11871176
Py_CLEAR(self->statement);
11881177
Py_DECREF(row);
@@ -1199,7 +1188,6 @@ pysqlite_cursor_iternext(PyObject *op)
11991188
}
12001189
}
12011190
else if (rc != SQLITE_ROW) {
1202-
report_pending_callback_error(self->connection->state);
12031191
rc = set_error_from_db(self->connection->state, self->connection->db);
12041192
int reset_rc = stmt_reset(self->statement);
12051193
Py_CLEAR(self->statement);

0 commit comments

Comments
 (0)