[SPARK-59278][SQL] Fix CHAR comparison rewrite edge cases

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ApplyCharTypePaddingHelper.scala

@@ -104,17 +104,21 @@ object ApplyCharTypePaddingHelper {
           .getRawType(attr.metadata)
           .flatMap {
             case c: CharType =>
-              val (nulls, literalChars) =
-                list.map(_.eval().asInstanceOf[UTF8String]).partition(_ == null)
-              val literalCharLengths = literalChars.map(_.numChars())
-              val targetLen = (c.length +: literalCharLengths).max
+              // Compute the length of every list element in place, so that each length stays
+              // aligned with the element it came from. NULL elements have no length: they can
+              // never match, so they are left untouched instead of being padded.
+              val literalCharLengths = list.map { lit =>
+                Option(lit.eval().asInstanceOf[UTF8String]).map(_.numChars())
+              }
+              val targetLen = (c.length +: literalCharLengths.flatten).max
               Some(
                 i.copy(
                   value = addPadding(e, c.length, targetLen, alwaysPad = padCharCol),
                   list = list.zip(literalCharLengths).map {
-                      case (lit, charLength) =>
-                        addPadding(lit, charLength, targetLen, alwaysPad = false)
-                    } ++ nulls.map(Literal.create(_, StringType))
+                    case (lit, Some(charLength)) =>
+                      addPadding(lit, charLength, targetLen, alwaysPad = false)
+                    case (lit, None) => lit
+                  }
                 )
               )
             case _ => None

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionHelper.scala

@@ -447,9 +447,13 @@ abstract class TypeCoercionHelper {
           i
         }
 
-      case i @ In(a, b) if b.exists(_.dataType != a.dataType) =>
+      case i @ In(_, _) if !haveSameType(i.children.map(_.dataType)) =>
         findWiderCommonType(i.children.map(_.dataType)) match {
-          case Some(finalDataType) => i.withNewChildren(i.children.map(Cast(_, finalDataType)))
+          // Only cast the children that are not already of the common type. A redundant Cast
+          // would hide an attribute from rules that match on it, such as the CHAR type padding
+          // in ApplyCharTypePadding.
+          case Some(finalDataType) =>
+            i.withNewChildren(i.children.map(castIfNotSameType(_, finalDataType)))
           case None => i
         }
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/CharVarcharUtils.scala

@@ -295,6 +295,11 @@ object CharVarcharUtils extends Logging with SparkCharVarcharUtils {
     val rawTypes = attrs.map(attr => getRawType(attr.metadata))
     if (rawTypes.exists(_.isEmpty)) {
       attrs
+    } else if (attrs.exists(attr => !RowOrdering.isOrderable(attr.dataType))) {
+      // Comparing a non-orderable type, such as a struct holding a MAP, is rejected later by
+      // CheckAnalysis. Leave it alone so the error names the attribute the user wrote rather
+      // than the rewritten struct.
+      attrs
     } else {
       val typeWithTargetCharLength = rawTypes.map(_.get).reduce(typeWithWiderCharLength)
       attrs.zip(rawTypes.map(_.get)).map { case (attr, rawType) =>
@@ -337,12 +342,27 @@ object CharVarcharUtils extends Logging with SparkCharVarcharUtils {
           val fieldExpr = GetStructField(expr, i, Some(field.name))
           val padded = padCharToTargetLength(
             fieldExpr, field.dataType, targets(i).dataType, alwaysPad)
-          needPadding = padded.isDefined
+          // Accumulate: any single field needing padding means the struct must be rebuilt,
+          // otherwise the padding computed for earlier fields would be discarded.
+          needPadding |= padded.isDefined
           createStructExprs += Literal(field.name)
           createStructExprs += padded.getOrElse(fieldExpr)
           i += 1
         }
-        if (needPadding) Some(CreateNamedStruct(createStructExprs.toSeq)) else None
+        if (needPadding) {
+          val struct = CreateNamedStruct(createStructExprs.toSeq)
+          // Rebuilding the struct field by field loses the nullability of the struct itself:
+          // GetStructField on a NULL struct yields NULL fields, which CreateNamedStruct would
+          // turn into a non-NULL struct of NULLs. Guard it the same way the scan-side rewrite
+          // in processStringForCharVarchar does.
+          Some(if (expr.nullable) {
+            If(IsNull(expr), Literal(null, struct.dataType), struct)
+          } else {
+            struct
+          })
+        } else {
+          None
+        }
 
       case (ArrayType(et, containsNull), ArrayType(target, _)) =>
         val param = NamedLambdaVariable("x", replaceCharVarcharWithString(et), containsNull)

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala

@@ -1747,10 +1747,10 @@ class TypeCoercionSuite extends TypeCoercionSuiteBase {
       In(Literal("test"), Seq(UnresolvedAttribute("a"), Literal(1))),
       In(Literal("test"), Seq(UnresolvedAttribute("a"), Literal(1)))
     )
+    // Only the children that are not already of the common type are cast.
     ruleTest(inConversion,
       In(Literal("a"), Seq(Literal(1), Literal("b"))),
-      In(Cast(Literal("a"), StringType),
-        Seq(Cast(Literal(1), StringType), Cast(Literal("b"), StringType)))
+      In(Literal("a"), Seq(Cast(Literal(1), StringType), Literal("b")))
     )
   }
 

sql/core/src/test/resources/sql-tests/analyzer-results/charvarchar-standard-semantics.sql.out

@@ -93,7 +93,7 @@ Project [CASE WHEN true THEN cast(cast(abcdef as varchar(2)) as varchar(4)) ELSE
 -- !query
 SELECT CAST('abcdef' AS VARCHAR(2)) IN (CAST('ab' AS VARCHAR(4)))
 -- !query analysis
-Project [cast(cast(abcdef as varchar(2)) as varchar(4)) IN (cast(cast(ab as varchar(4)) as varchar(4))) AS (CAST(abcdef AS VARCHAR(2)) IN (CAST(ab AS VARCHAR(4))))#x]
+Project [cast(cast(abcdef as varchar(2)) as varchar(4)) IN (cast(ab as varchar(4))) AS (CAST(abcdef AS VARCHAR(2)) IN (CAST(ab AS VARCHAR(4))))#x]
 +- OneRowRelation
 
 
@@ -166,15 +166,15 @@ Project [typeof(CASE WHEN true THEN cast(cast(a as char(2)) as char(4)) ELSE cas
 -- !query
 SELECT cast('a' AS CHAR(2)) IN (cast('a ' AS CHAR(2)), cast('bbb' AS VARCHAR(3)))
 -- !query analysis
-Project [cast(cast(a as char(2)) as varchar(3)) IN (cast(cast(a  as char(2)) as varchar(3)),cast(cast(bbb as varchar(3)) as varchar(3))) AS (CAST(a AS CHAR(2)) IN (CAST(a  AS CHAR(2)), CAST(bbb AS VARCHAR(3))))#x]
+Project [cast(cast(a as char(2)) as varchar(3)) IN (cast(cast(a  as char(2)) as varchar(3)),cast(bbb as varchar(3))) AS (CAST(a AS CHAR(2)) IN (CAST(a  AS CHAR(2)), CAST(bbb AS VARCHAR(3))))#x]
 +- OneRowRelation
 
 
 -- !query
 SELECT typeof(c) FROM (SELECT cast('a' AS CHAR(2)) AS c) t WHERE c IN ('a ', 'b')
 -- !query analysis
 Project [typeof(c#x) AS typeof(c)#x]
-+- Filter cast(c#x as string collate UTF8_BINARY) IN (cast(a  as string collate UTF8_BINARY),cast(b as string collate UTF8_BINARY))
++- Filter cast(c#x as string collate UTF8_BINARY) IN (a ,b)
    +- SubqueryAlias t
       +- Project [cast(a as char(2)) AS c#x]
          +- OneRowRelation
@@ -703,35 +703,35 @@ Project [(cast(cast(a as char(2) collate UTF8_BINARY_RTRIM) as char(4) collate U
 -- !query
 SELECT cast('a' AS CHAR(2)) IN (cast('a' AS CHAR(4)))
 -- !query analysis
-Project [cast(cast(a as char(2)) as char(4)) IN (cast(cast(a as char(4)) as char(4))) AS (CAST(a AS CHAR(2)) IN (CAST(a AS CHAR(4))))#x]
+Project [cast(cast(a as char(2)) as char(4)) IN (cast(a as char(4))) AS (CAST(a AS CHAR(2)) IN (CAST(a AS CHAR(4))))#x]
 +- OneRowRelation
 
 
 -- !query
 SELECT cast('a' AS CHAR(2)) IN (cast('a' AS VARCHAR(2)))
 -- !query analysis
-Project [cast(cast(a as char(2)) as varchar(2)) IN (cast(cast(a as varchar(2)) as varchar(2))) AS (CAST(a AS CHAR(2)) IN (CAST(a AS VARCHAR(2))))#x]
+Project [cast(cast(a as char(2)) as varchar(2)) IN (cast(a as varchar(2))) AS (CAST(a AS CHAR(2)) IN (CAST(a AS VARCHAR(2))))#x]
 +- OneRowRelation
 
 
 -- !query
 SELECT cast('a' AS CHAR(2)) IN (cast('a ' AS VARCHAR(2)))
 -- !query analysis
-Project [cast(cast(a as char(2)) as varchar(2)) IN (cast(cast(a  as varchar(2)) as varchar(2))) AS (CAST(a AS CHAR(2)) IN (CAST(a  AS VARCHAR(2))))#x]
+Project [cast(cast(a as char(2)) as varchar(2)) IN (cast(a  as varchar(2))) AS (CAST(a AS CHAR(2)) IN (CAST(a  AS VARCHAR(2))))#x]
 +- OneRowRelation
 
 
 -- !query
 SELECT cast('a' AS CHAR(2)) IN ('a', 'b')
 -- !query analysis
-Project [cast(cast(a as char(2)) as string collate UTF8_BINARY) IN (cast(a as string collate UTF8_BINARY),cast(b as string collate UTF8_BINARY)) AS (CAST(a AS CHAR(2)) IN (a, b))#x]
+Project [cast(cast(a as char(2)) as string collate UTF8_BINARY) IN (a,b) AS (CAST(a AS CHAR(2)) IN (a, b))#x]
 +- OneRowRelation
 
 
 -- !query
 SELECT cast('a' AS CHAR(2)) IN ('a ', 'b')
 -- !query analysis
-Project [cast(cast(a as char(2)) as string collate UTF8_BINARY) IN (cast(a  as string collate UTF8_BINARY),cast(b as string collate UTF8_BINARY)) AS (CAST(a AS CHAR(2)) IN (a , b))#x]
+Project [cast(cast(a as char(2)) as string collate UTF8_BINARY) IN (a ,b) AS (CAST(a AS CHAR(2)) IN (a , b))#x]
 +- OneRowRelation
 
 

sql/core/src/test/resources/sql-tests/analyzer-results/predicate-functions.sql.out

@@ -415,7 +415,7 @@ Project [1 IN (1,2,3) AS (1 IN (1, 2, 3))#x]
 -- !query
 select 1 in (1, 2, 3, null)
 -- !query analysis
-Project [cast(1 as int) IN (cast(1 as int),cast(2 as int),cast(3 as int),cast(null as int)) AS (1 IN (1, 2, 3, NULL))#x]
+Project [1 IN (1,2,3,cast(null as int)) AS (1 IN (1, 2, 3, NULL))#x]
 +- OneRowRelation
 
 
@@ -450,14 +450,14 @@ Project [cast(1 as bigint) IN (cast(2 as bigint),cast(3 as bigint),cast(4 as big
 -- !query
 select null in (1, 2, 3)
 -- !query analysis
-Project [cast(null as int) IN (cast(1 as int),cast(2 as int),cast(3 as int)) AS (NULL IN (1, 2, 3))#x]
+Project [cast(null as int) IN (1,2,3) AS (NULL IN (1, 2, 3))#x]
 +- OneRowRelation
 
 
 -- !query
 select null in (1, 2, null)
 -- !query analysis
-Project [cast(null as int) IN (cast(1 as int),cast(2 as int),cast(null as int)) AS (NULL IN (1, 2, NULL))#x]
+Project [cast(null as int) IN (1,2,cast(null as int)) AS (NULL IN (1, 2, NULL))#x]
 +- OneRowRelation
 
 
@@ -471,7 +471,7 @@ Project [NOT 1 IN (1,2,3) AS (NOT (1 IN (1, 2, 3)))#x]
 -- !query
 select 1 not in (1, 2, 3, null)
 -- !query analysis
-Project [NOT cast(1 as int) IN (cast(1 as int),cast(2 as int),cast(3 as int),cast(null as int)) AS (NOT (1 IN (1, 2, 3, NULL)))#x]
+Project [NOT 1 IN (1,2,3,cast(null as int)) AS (NOT (1 IN (1, 2, 3, NULL)))#x]
 +- OneRowRelation
 
 
@@ -506,14 +506,14 @@ Project [NOT cast(1 as bigint) IN (cast(2 as bigint),cast(3 as bigint),cast(4 as
 -- !query
 select null not in (1, 2, 3)
 -- !query analysis
-Project [NOT cast(null as int) IN (cast(1 as int),cast(2 as int),cast(3 as int)) AS (NOT (NULL IN (1, 2, 3)))#x]
+Project [NOT cast(null as int) IN (1,2,3) AS (NOT (NULL IN (1, 2, 3)))#x]
 +- OneRowRelation
 
 
 -- !query
 select null not in (1, 2, null)
 -- !query analysis
-Project [NOT cast(null as int) IN (cast(1 as int),cast(2 as int),cast(null as int)) AS (NOT (NULL IN (1, 2, NULL)))#x]
+Project [NOT cast(null as int) IN (1,2,cast(null as int)) AS (NOT (NULL IN (1, 2, NULL)))#x]
 +- OneRowRelation
 
 

sql/core/src/test/resources/sql-tests/analyzer-results/selectExcept.sql.out

@@ -448,7 +448,7 @@ Project [greatest(c1#x, c2#x, cast(c3#x as int), c4#x, c5#x) AS greatest(c1, c2,
 -- !query
 SELECT 5 IN (*) FROM v1
 -- !query analysis
-Project [cast(5 as int) IN (cast(c1#x as int),cast(c2#x as int),cast(c3#x as int),cast(c4#x as int),cast(c5#x as int)) AS (5 IN (c1, c2, c3, c4, c5))#x]
+Project [5 IN (c1#x,c2#x,cast(c3#x as int),c4#x,c5#x) AS (5 IN (c1, c2, c3, c4, c5))#x]
 +- SubqueryAlias v1
    +- View (`v1`, [c1#x, c2#x, c3#x, c4#x, c5#x])
       +- Project [cast(c1#x as int) AS c1#x, cast(c2#x as int) AS c2#x, cast(c3#x as void) AS c3#x, cast(c4#x as int) AS c4#x, cast(c5#x as int) AS c5#x]
@@ -492,7 +492,7 @@ Project [1 AS 1#x]
 SELECT 1 FROM v1 WHERE 4 IN (*)
 -- !query analysis
 Project [1 AS 1#x]
-+- Filter cast(4 as int) IN (cast(c1#x as int),cast(c2#x as int),cast(c3#x as int),cast(c4#x as int),cast(c5#x as int))
++- Filter 4 IN (c1#x,c2#x,cast(c3#x as int),c4#x,c5#x)
    +- SubqueryAlias v1
       +- View (`v1`, [c1#x, c2#x, c3#x, c4#x, c5#x])
          +- Project [cast(c1#x as int) AS c1#x, cast(c2#x as int) AS c2#x, cast(c3#x as void) AS c3#x, cast(c4#x as int) AS c4#x, cast(c5#x as int) AS c5#x]

sql/core/src/test/resources/sql-tests/analyzer-results/subquery/in-subquery/not-in-unit-tests-single-column-literal.sql.out

@@ -18,7 +18,7 @@ FROM   m
 WHERE  a NOT IN (null)
 -- !query analysis
 Project [a#x, b#x]
-+- Filter NOT cast(a#x as int) IN (cast(null as int))
++- Filter NOT a#x IN (cast(null as int))
    +- SubqueryAlias m
       +- View (`m`, [a#x, b#x])
          +- Project [cast(col1#x as int) AS a#x, cast(col2#x as decimal(2,1)) AS b#x]

sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out

@@ -2441,15 +2441,15 @@ Project [typeof(map(a, cast(cast(12:34:56.789 as time(3)) as time(6)), b, cast(0
 -- !query
 SELECT '01:02:03.456789' :: TIME(6) IN ('12:34:56.789' :: TIME(3), '01:02:03.456789' :: TIME(6))
 -- !query analysis
-Project [cast(cast(01:02:03.456789 as time(6)) as time(6)) IN (cast(cast(12:34:56.789 as time(3)) as time(6)),cast(cast(01:02:03.456789 as time(6)) as time(6))) AS (CAST(01:02:03.456789 AS TIME(6)) IN (CAST(12:34:56.789 AS TIME(3)), CAST(01:02:03.456789 AS TIME(6))))#x]
+Project [cast(01:02:03.456789 as time(6)) IN (cast(cast(12:34:56.789 as time(3)) as time(6)),cast(01:02:03.456789 as time(6))) AS (CAST(01:02:03.456789 AS TIME(6)) IN (CAST(12:34:56.789 AS TIME(3)), CAST(01:02:03.456789 AS TIME(6))))#x]
 +- OneRowRelation
 
 
 -- !query
 SELECT '01:02:03.456789123' :: TIME(9) IN (
   '12:34:56.789' :: TIME(3), '01:02:03.456789' :: TIME(6), '01:02:03.456789123' :: TIME(9))
 -- !query analysis
-Project [cast(cast(01:02:03.456789123 as time(9)) as time(9)) IN (cast(cast(12:34:56.789 as time(3)) as time(9)),cast(cast(01:02:03.456789 as time(6)) as time(9)),cast(cast(01:02:03.456789123 as time(9)) as time(9))) AS (CAST(01:02:03.456789123 AS TIME(9)) IN (CAST(12:34:56.789 AS TIME(3)), CAST(01:02:03.456789 AS TIME(6)), CAST(01:02:03.456789123 AS TIME(9))))#x]
+Project [cast(01:02:03.456789123 as time(9)) IN (cast(cast(12:34:56.789 as time(3)) as time(9)),cast(cast(01:02:03.456789 as time(6)) as time(9)),cast(01:02:03.456789123 as time(9))) AS (CAST(01:02:03.456789123 AS TIME(9)) IN (CAST(12:34:56.789 AS TIME(3)), CAST(01:02:03.456789 AS TIME(6)), CAST(01:02:03.456789123 AS TIME(9))))#x]
 +- OneRowRelation