From 2d2e9b2415633422cf8a1867b1254af7a97b3fc0 Mon Sep 17 00:00:00 2001 From: Adebayo Olaonipekun Date: Tue, 16 Jun 2026 22:50:45 +0100 Subject: [PATCH 1/5] BUG: improve error message for non-string dtype with DB-API connection (GH#61385) When using a raw DB-API connection (e.g. sqlite3 without SQLAlchemy), to_sql(dtype=...) requires SQL type strings. Passing a SQLAlchemy type object previously raised an unhelpful error showing only the column name and repr of the type. This adds a clearer message explaining the constraint and suggesting the use of a SQLAlchemy engine for SQLAlchemy types, per maintainer guidance on the issue. No fallback conversion is added, consistent with the maintainer's stated preference that pandas not own such conversion logic. --- doc/source/whatsnew/v3.1.0.rst | 1 + pandas/io/sql.py | 8 +++++++- pandas/tests/io/test_sql.py | 10 ++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v3.1.0.rst b/doc/source/whatsnew/v3.1.0.rst index 5fe30b4189717..97650ec302d08 100644 --- a/doc/source/whatsnew/v3.1.0.rst +++ b/doc/source/whatsnew/v3.1.0.rst @@ -346,6 +346,7 @@ MultiIndex I/O ^^^ - :func:`read_csv` with ``memory_map=True`` and an in-memory buffer (e.g. ``BytesIO``) now raises a clear ``ValueError`` instead of a cryptic ``UnsupportedOperation: fileno`` (:issue:`45630`) +- :meth:`DataFrame.to_sql` now raises a clearer ``ValueError`` when a non-string ``dtype`` is passed for a raw DB-API (e.g. sqlite3) connection, explaining that SQLAlchemy types require a SQLAlchemy engine (:issue:`61385`) - Fixed bug in :func:`read_csv` with the ``c`` engine where an embedded ``\r`` followed by a space in an unquoted field could cause an infinite re-parsing loop, producing spurious rows or a buffer overflow (:issue:`51141`) - Fixed bug in :func:`read_excel` where usage of ``skiprows`` could lead to an infinite loop (:issue:`64027`) - Fixed bug where :func:`read_html` parsed nested tables incorrectly when using ``html5lib`` or ``bs4`` flavors (:issue:`64524`) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index f79f037d908fe..abc6d9644d02b 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -2867,7 +2867,13 @@ def to_sql( for col, my_type in dtype.items(): if not isinstance(my_type, str): - raise ValueError(f"{col} ({my_type}) not a string") + raise ValueError( + f"Column '{col}' has dtype '{my_type}' which is not a " + "string. When using a DB-API connection (e.g. sqlite3), " + "dtype values must be SQL type strings (e.g. 'TEXT', " + "'FLOAT'). To use SQLAlchemy types, use a SQLAlchemy " + "engine instead." + ) table = SQLiteTable( name, diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 63feb8c12e359..0747b9bfdd610 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2648,6 +2648,16 @@ def test_sqlite_type_mapping(sqlite_buildin): assert col.split()[1] == "TIMESTAMP" +def test_sqlite_dtype_not_string_raises(sqlite_buildin): + # GH61385 to_sql with non-string dtype on a raw sqlite3 connection + # should raise a clear, actionable ValueError + conn = sqlite_buildin + df = DataFrame({"A": [1.0, 2.0], "B": [3.0, 4.0]}) + msg = "When using a DB-API connection" + with pytest.raises(ValueError, match=msg): + df.to_sql(name="test_dtype_not_string", con=conn, dtype={"A": float}) + + # ----------------------------------------------------------------------------- # -- Database flavor specific tests From bf460acbb58f74e3956a0b51cf34cc9a3c9f0d23 Mon Sep 17 00:00:00 2001 From: Adebayo Olaonipekun Date: Wed, 17 Jun 2026 14:14:16 +0100 Subject: [PATCH 2/5] TST: update test_sqlite_test_dtype to match improved dtype error message The existing test asserted on the old error message format ('B () not a string'). Updated the expected regex to match the new, more descriptive message introduced in this PR. --- pandas/tests/io/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 0747b9bfdd610..9cd8af5b79625 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -4076,7 +4076,7 @@ def test_sqlite_test_dtype(sqlite_buildin): assert get_sqlite_column_type(conn, "dtype_test", "B") == "INTEGER" assert get_sqlite_column_type(conn, "dtype_test2", "B") == "STRING" - msg = r"B \(\) not a string" + msg = r"Column 'B' has dtype '' which is not a string" with pytest.raises(ValueError, match=msg): df.to_sql(name="error", con=conn, dtype={"B": bool}) From 61124851c0aa3487d76a01145ac6d634ec94de4d Mon Sep 17 00:00:00 2001 From: Adebayo Olaonipekun Date: Sat, 20 Jun 2026 13:58:42 +0100 Subject: [PATCH 3/5] BUG: address review feedback on dtype error message (GH#61385) Per @rhshadrach's review: - Changed 'dtype' to 'type' in the error message, since the offending value is not necessarily a pandas/numpy dtype (it may be any non-string object, e.g. a SQLAlchemy type). - Removed the SQLAlchemy-specific suggestion from the message, since it's a non-sequitur when my_type isn't actually a SQLAlchemy type. - Removed test_sqlite_dtype_not_string_raises as redundant: the existing test_sqlite_test_dtype already exercises this exact ValueError path and asserts against the updated message. Verified: 616 sqlite tests passing, pre-commit clean. --- pandas/io/sql.py | 5 ++--- pandas/tests/io/test_sql.py | 12 +----------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index abc6d9644d02b..cc53a72e3e1ea 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -2868,11 +2868,10 @@ def to_sql( for col, my_type in dtype.items(): if not isinstance(my_type, str): raise ValueError( - f"Column '{col}' has dtype '{my_type}' which is not a " + f"Column '{col}' has type '{my_type}' which is not a " "string. When using a DB-API connection (e.g. sqlite3), " "dtype values must be SQL type strings (e.g. 'TEXT', " - "'FLOAT'). To use SQLAlchemy types, use a SQLAlchemy " - "engine instead." + "'FLOAT')." ) table = SQLiteTable( diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 9cd8af5b79625..48a4f5a3bcf4e 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -2648,16 +2648,6 @@ def test_sqlite_type_mapping(sqlite_buildin): assert col.split()[1] == "TIMESTAMP" -def test_sqlite_dtype_not_string_raises(sqlite_buildin): - # GH61385 to_sql with non-string dtype on a raw sqlite3 connection - # should raise a clear, actionable ValueError - conn = sqlite_buildin - df = DataFrame({"A": [1.0, 2.0], "B": [3.0, 4.0]}) - msg = "When using a DB-API connection" - with pytest.raises(ValueError, match=msg): - df.to_sql(name="test_dtype_not_string", con=conn, dtype={"A": float}) - - # ----------------------------------------------------------------------------- # -- Database flavor specific tests @@ -4076,7 +4066,7 @@ def test_sqlite_test_dtype(sqlite_buildin): assert get_sqlite_column_type(conn, "dtype_test", "B") == "INTEGER" assert get_sqlite_column_type(conn, "dtype_test2", "B") == "STRING" - msg = r"Column 'B' has dtype '' which is not a string" + msg = r"Column 'B' has type '' which is not a string" with pytest.raises(ValueError, match=msg): df.to_sql(name="error", con=conn, dtype={"B": bool}) From 0d4d60fc8f437684f1da2abe397543f557b8e4c1 Mon Sep 17 00:00:00 2001 From: Adebayo Olaonipekun Date: Sat, 20 Jun 2026 23:21:31 +0100 Subject: [PATCH 4/5] BUG: clarify dtype error message wording per review (GH#61385) Per @jbrockmendel's feedback: the previous phrasing 'Column has type X which is not a string' could be misread as describing the column's actual type in the table, rather than the invalid dtype value passed by the user. Reworded to 'Invalid type X for dtype of column Y: expected a string' to make clear it's describing the dtype argument the user passed, not the table column's type. Verified: 616 sqlite tests passing, pre-commit clean. --- pandas/io/sql.py | 8 ++++---- pandas/tests/io/test_sql.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index cc53a72e3e1ea..51120d5cec431 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -2868,10 +2868,10 @@ def to_sql( for col, my_type in dtype.items(): if not isinstance(my_type, str): raise ValueError( - f"Column '{col}' has type '{my_type}' which is not a " - "string. When using a DB-API connection (e.g. sqlite3), " - "dtype values must be SQL type strings (e.g. 'TEXT', " - "'FLOAT')." + f"Invalid type '{my_type}' for dtype of column '{col}': " + "expected a string. When using a DB-API connection " + "(e.g. sqlite3), dtype values must be SQL type " + "strings (e.g. 'TEXT', 'FLOAT')." ) table = SQLiteTable( diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 48a4f5a3bcf4e..861ca8afd5f79 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -4066,7 +4066,7 @@ def test_sqlite_test_dtype(sqlite_buildin): assert get_sqlite_column_type(conn, "dtype_test", "B") == "INTEGER" assert get_sqlite_column_type(conn, "dtype_test2", "B") == "STRING" - msg = r"Column 'B' has type '' which is not a string" + msg = r"Invalid type '' for dtype of column 'B': expected a string" with pytest.raises(ValueError, match=msg): df.to_sql(name="error", con=conn, dtype={"B": bool}) From df7ea2c4e0b2f484272aafdc655ebb75d3d7df78 Mon Sep 17 00:00:00 2001 From: Adebayo Olaonipekun Date: Sun, 21 Jun 2026 18:24:22 +0100 Subject: [PATCH 5/5] DOC: update whatsnew entry to match current error message (GH#61385) Per @jbrockmendel's question, the whatsnew entry referenced SQLAlchemy/engine guidance that was removed from the actual error message in an earlier review round. Updated to match current behavior. --- doc/source/whatsnew/v3.1.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v3.1.0.rst b/doc/source/whatsnew/v3.1.0.rst index 97650ec302d08..55884d98714fe 100644 --- a/doc/source/whatsnew/v3.1.0.rst +++ b/doc/source/whatsnew/v3.1.0.rst @@ -346,7 +346,7 @@ MultiIndex I/O ^^^ - :func:`read_csv` with ``memory_map=True`` and an in-memory buffer (e.g. ``BytesIO``) now raises a clear ``ValueError`` instead of a cryptic ``UnsupportedOperation: fileno`` (:issue:`45630`) -- :meth:`DataFrame.to_sql` now raises a clearer ``ValueError`` when a non-string ``dtype`` is passed for a raw DB-API (e.g. sqlite3) connection, explaining that SQLAlchemy types require a SQLAlchemy engine (:issue:`61385`) +- :meth:`DataFrame.to_sql` now raises a clearer ``ValueError`` when a non-string ``dtype`` is passed for a raw DB-API (e.g. sqlite3) connection (:issue:`61385`) - Fixed bug in :func:`read_csv` with the ``c`` engine where an embedded ``\r`` followed by a space in an unquoted field could cause an infinite re-parsing loop, producing spurious rows or a buffer overflow (:issue:`51141`) - Fixed bug in :func:`read_excel` where usage of ``skiprows`` could lead to an infinite loop (:issue:`64027`) - Fixed bug where :func:`read_html` parsed nested tables incorrectly when using ``html5lib`` or ``bs4`` flavors (:issue:`64524`)