FIX: gate SQL_WVARCHAR output-converter fallback to string/binary columns (#691)

mssql_python/cursor.py

@@ -1142,11 +1142,13 @@ def _build_converter_map(self):
             #    (e.g. decimal.Decimal) - the pre-existing mssql-python key style.
             if converter is None:
                 converter = self.connection.get_output_converter(desc[1])
-            # 3) last-resort WVARCHAR converter fallback (legacy behavior)
-            if converter is None:
-                from mssql_python.constants import ConstantsDDBC
-
-                converter = self.connection.get_output_converter(ConstantsDDBC.SQL_WVARCHAR.value)
+            # 3) Legacy WVARCHAR fallback: only apply it when the column's mapped type
+            #    is str/bytes, so a registered SQL_WVARCHAR converter is never used as an
+            #    unconditional catch-all for INT/DECIMAL/DATE/etc. columns (GH #691). This
+            #    mirrors the isinstance(value, (str, bytes)) gate in
+            #    Row._apply_output_converters.
+            if converter is None and desc[1] in (str, bytes):
+                converter = self.connection.get_output_converter(ddbc_sql_const.SQL_WVARCHAR.value)
 
             converter_map.append(converter)
 

tests/test_003_connection.py

@@ -1909,6 +1909,76 @@ def test_converter_integration(db_connection):
     db_connection.clear_output_converters()
 
 
+def test_output_converter_wvarchar_not_catchall_for_non_string_columns_gh691(db_connection):
+    """GH #691: a SQL_WVARCHAR converter must not be an unconditional catch-all.
+
+    A converter registered only for SQL_WVARCHAR must be invoked for string and
+    binary columns only, and never for INT / DECIMAL / DATE columns. Before the
+    fix, the WVARCHAR converter was placed on every column that lacked a direct
+    type-keyed converter.
+
+    The converter here is a spy that records every value it receives and returns
+    an observable marker for non-bytes input instead of raising. That matters:
+    the optimized apply path swallows converter exceptions, so a converter doing
+    ``value.decode(...)`` would raise-and-be-swallowed on int/Decimal/date and
+    leave the original value intact -- hiding the bug. Recording invocations (and
+    returning a marker) makes the catch-all regression detectable: the call-count
+    assertion below fails on the unfixed code (converter fired on all columns)
+    and passes once the fallback is gated to str/bytes columns.
+    """
+    import decimal
+    import datetime
+
+    sql_wvarchar = ConstantsDDBC.SQL_WVARCHAR.value
+
+    converter_calls = []
+
+    def wvarchar_spy(value):
+        # Record the invocation first, before anything that could raise, so the
+        # spy captures calls even on non-string values (int/Decimal/date).
+        converter_calls.append(value)
+        if isinstance(value, bytes):
+            return "CONV:" + value.decode("utf-16-le")
+        return "CONV_NON_STRING"
+
+    cursor = db_connection.cursor()
+    db_connection.add_output_converter(sql_wvarchar, wvarchar_spy)
+    try:
+        cursor.execute("""
+            SELECT
+                CAST(42 AS INT)                  AS int_col,
+                CAST(3.14 AS DECIMAL(10, 2))     AS dec_col,
+                CAST('2020-01-02' AS DATE)       AS date_col,
+                CAST(N'hello' AS NVARCHAR(50))   AS str_col,
+                CAST(0x41004200 AS VARBINARY(8)) AS bin_col
+            """)
+        row = cursor.fetchone()
+
+        # Non-string columns must be untouched by the WVARCHAR converter.
+        assert isinstance(row[0], int) and row[0] == 42, f"INT column mangled: {row[0]!r}"
+        assert row[1] == decimal.Decimal("3.14"), f"DECIMAL column mangled: {row[1]!r}"
+        assert row[2] == datetime.date(2020, 1, 2), f"DATE column mangled: {row[2]!r}"
+
+        # The string column must still be converted by the WVARCHAR converter.
+        assert row[3] == "CONV:hello", f"NVARCHAR column not converted: {row[3]!r}"
+
+        # The binary column is also handled by the WVARCHAR fallback -- the gate
+        # intentionally includes bytes, mirroring Row._apply_output_converters.
+        assert row[4] == "CONV:AB", f"VARBINARY column not handled by fallback: {row[4]!r}"
+
+        # Contract check that actually catches GH #691: the converter must fire
+        # for exactly the two str/bytes columns, never as a catch-all on the
+        # INT / DECIMAL / DATE columns.
+        assert len(converter_calls) == 2, (
+            "SQL_WVARCHAR converter must fire only on the NVARCHAR and VARBINARY "
+            f"columns; it was invoked {len(converter_calls)} times on "
+            f"{converter_calls!r}"
+        )
+    finally:
+        db_connection.clear_output_converters()
+        cursor.close()
+
+
 def test_output_converter_integer_sql_type_key_gh684(db_connection):
     """Integer ODBC SQL-type keys must dispatch (pyodbc-compatible). Regression for GH #684.
 
@@ -1991,6 +2061,31 @@ def test_output_converter_integer_sql_type_key_gh684(db_connection):
         cursor.close()
 
 
+def test_output_converter_wvarchar_null_string_gh691(db_connection):
+    """GH #691: a NULL string value stays None and the WVARCHAR converter is never called.
+
+    The gated fallback is placed on the NVARCHAR column (mapped type str), but the apply path
+    short-circuits on NULL, so the converter must not be invoked for a NULL value.
+    """
+    sql_wvarchar = ConstantsDDBC.SQL_WVARCHAR.value
+    calls = []
+
+    def wvarchar_spy(value):
+        calls.append(value)
+        return "CONV:" + value.decode("utf-16-le")
+
+    cursor = db_connection.cursor()
+    db_connection.add_output_converter(sql_wvarchar, wvarchar_spy)
+    try:
+        cursor.execute("SELECT CAST(NULL AS NVARCHAR(20)) AS null_str")
+        value = cursor.fetchone()[0]
+        assert value is None, f"NULL NVARCHAR must remain None, got {value!r}"
+        assert calls == [], "WVARCHAR converter must not be invoked for a NULL value"
+    finally:
+        db_connection.clear_output_converters()
+        cursor.close()
+
+
 def test_output_converter_python_type_covers_multiple_sql_types_gh684(db_connection):
     """A single Python-type converter must cover every SQL type that materializes to it.
 
@@ -2031,6 +2126,69 @@ def test_output_converter_python_type_covers_multiple_sql_types_gh684(db_connect
         cursor.close()
 
 
+def test_output_converter_wvarchar_applies_to_varchar_column_gh691(db_connection):
+    """GH #691: the gated WVARCHAR fallback still applies to VARCHAR columns (mapped type str).
+
+    VARCHAR maps to the Python type ``str`` just like NVARCHAR, so a SQL_WVARCHAR converter
+    fires on it and receives the value as UTF-16LE bytes.
+    """
+    sql_wvarchar = ConstantsDDBC.SQL_WVARCHAR.value
+    calls = []
+
+    def wvarchar_spy(value):
+        calls.append(value)
+        return "CONV:" + value.decode("utf-16-le")
+
+    cursor = db_connection.cursor()
+    db_connection.add_output_converter(sql_wvarchar, wvarchar_spy)
+    try:
+        cursor.execute("SELECT CAST('abc' AS VARCHAR(20)) AS vchar")
+        value = cursor.fetchone()[0]
+        assert value == "CONV:abc", f"VARCHAR column not converted by WVARCHAR fallback: {value!r}"
+        assert len(calls) == 1, f"WVARCHAR converter should fire once on VARCHAR, got {len(calls)}"
+    finally:
+        db_connection.clear_output_converters()
+        cursor.close()
+
+
+def test_output_converter_wvarchar_mixed_result_set_gh691(db_connection):
+    """GH #691: in a mixed row, the WVARCHAR converter fires only on the string column.
+
+    INT and DECIMAL columns (non str/bytes mapped types) must be untouched; only the NVARCHAR
+    column is converted, and the converter is invoked exactly once.
+    """
+    import decimal
+
+    sql_wvarchar = ConstantsDDBC.SQL_WVARCHAR.value
+    calls = []
+
+    def wvarchar_spy(value):
+        calls.append(value)
+        if isinstance(value, bytes):
+            return "CONV:" + value.decode("utf-16-le")
+        return "CONV_NON_STRING"
+
+    cursor = db_connection.cursor()
+    db_connection.add_output_converter(sql_wvarchar, wvarchar_spy)
+    try:
+        cursor.execute(
+            "SELECT CAST(1 AS INT) AS i, "
+            "CAST('abc' AS NVARCHAR(10)) AS s, "
+            "CAST(12.34 AS DECIMAL(10, 2)) AS d"
+        )
+        i_val, s_val, d_val = cursor.fetchone()
+        assert isinstance(i_val, int) and i_val == 1, f"INT column mangled: {i_val!r}"
+        assert s_val == "CONV:abc", f"NVARCHAR column not converted: {s_val!r}"
+        assert d_val == decimal.Decimal("12.34"), f"DECIMAL column mangled: {d_val!r}"
+        assert len(calls) == 1, (
+            "WVARCHAR converter must fire only on the NVARCHAR column; "
+            f"invoked {len(calls)} times on {calls!r}"
+        )
+    finally:
+        db_connection.clear_output_converters()
+        cursor.close()
+
+
 def test_output_converter_python_type_datetime_and_bytes_gh684(db_connection):
     """Python-type converters must work for datetime and bytes columns, not just numbers.
 
@@ -2053,6 +2211,45 @@ def test_output_converter_python_type_datetime_and_bytes_gh684(db_connection):
         cursor.close()
 
 
+def test_output_converter_wvarchar_cached_map_multiple_fetches_gh691(db_connection):
+    """GH #691: the cached converter map behaves identically across multiple fetched rows.
+
+    The gated WVARCHAR fallback is computed once per statement; every row in a multi-row
+    result must be converted consistently, and the INT column must never be touched.
+    """
+    sql_wvarchar = ConstantsDDBC.SQL_WVARCHAR.value
+    calls = []
+
+    def wvarchar_spy(value):
+        calls.append(value)
+        if isinstance(value, bytes):
+            return "CONV:" + value.decode("utf-16-le")
+        return "CONV_NON_STRING"
+
+    cursor = db_connection.cursor()
+    db_connection.add_output_converter(sql_wvarchar, wvarchar_spy)
+    try:
+        cursor.execute(
+            "SELECT CAST(10 AS INT) AS i, CAST(N'a' AS NVARCHAR(10)) AS s "
+            "UNION ALL SELECT CAST(20 AS INT), CAST(N'b' AS NVARCHAR(10)) "
+            "UNION ALL SELECT CAST(30 AS INT), CAST(N'c' AS NVARCHAR(10))"
+        )
+        rows = cursor.fetchall()
+        assert [r[0] for r in rows] == [10, 20, 30], "INT column must be untouched on every row"
+        assert [r[1] for r in rows] == [
+            "CONV:a",
+            "CONV:b",
+            "CONV:c",
+        ], "NVARCHAR column must be converted consistently on every row"
+        # Exactly one invocation per row -- the string column only, never the INT column.
+        assert (
+            len(calls) == 3
+        ), f"WVARCHAR converter must fire once per row (3 total), got {len(calls)}: {calls!r}"
+    finally:
+        db_connection.clear_output_converters()
+        cursor.close()
+
+
 def test_output_converter_null_value_skips_converter_gh684(db_connection):
     """A SQL NULL is returned as None and the converter is never invoked for it.
 
@@ -2184,6 +2381,51 @@ def test_output_converter_cached_map_across_multiple_rows_gh684(db_connection):
         cursor.close()
 
 
+def test_output_converter_int_column_converter_vs_wvarchar_gating_gh691(db_connection):
+    """An INT-column converter runs; the WVARCHAR fallback never touches a non-string column.
+
+    Locks in the GH #691 gating so it cannot regress. Positive: a converter registered against
+    the INT column (via its SQL_INTEGER type code) still fires and receives the raw integer.
+    Negative: with only a SQL_WVARCHAR converter registered, that converter must NOT sneak onto
+    the INT column -- the value passes through untouched and the converter is never invoked.
+    """
+    cursor = db_connection.cursor()
+    int_query = "SELECT CAST(42 AS INT) AS n"
+    try:
+        # Positive: an INT-column converter still runs and sees the raw integer value.
+        int_calls = []
+
+        def int_spy(value):
+            int_calls.append(value)
+            return "INT:" + str(value)
+
+        db_connection.add_output_converter(mssql_python.SQL_INTEGER, int_spy)
+        cursor.execute(int_query)
+        value = cursor.fetchone()[0]
+        assert value == "INT:42", "Registered INT-column converter did not fire"
+        assert int_calls == [42], f"INT converter must see the raw int, got {int_calls!r}"
+        db_connection.clear_output_converters()
+
+        # Negative (GH #691): a WVARCHAR converter must not sneak onto the INT column.
+        wvarchar_calls = []
+
+        def wvarchar_spy(value):
+            wvarchar_calls.append(value)
+            return "SHOULD_NOT_FIRE"
+
+        db_connection.add_output_converter(mssql_python.SQL_WVARCHAR, wvarchar_spy)
+        cursor.execute(int_query)
+        value = cursor.fetchone()[0]
+        assert value == 42, f"WVARCHAR fallback must not touch the INT column, got {value!r}"
+        assert wvarchar_calls == [], (
+            "WVARCHAR converter must never be invoked on a non-string (INT) column; "
+            f"it was called on {wvarchar_calls!r}"
+        )
+    finally:
+        db_connection.clear_output_converters()
+        cursor.close()
+
+
 def test_output_converter_with_null_values(db_connection):
     """Test that output converters handle NULL values correctly"""
     cursor = db_connection.cursor()