Fix NaN handling in min/max in columnar aggregation pipeline

.unreleased/min-nan

@@ -0,0 +1 @@
+Fixes: #10052 Result of min/max aggregate functions in columnar aggregation pipeline possibly inconsistent with plain Postgres result

tsl/src/nodes/vector_agg/function/minmax_arithmetic_single.c

@@ -46,7 +46,7 @@ static pg_attribute_always_inline void
 FUNCTION_NAME(one)(void *restrict agg_state, const CTYPE value)
 {
 	FUNCTION_NAME(state) *state = (FUNCTION_NAME(state) *) agg_state;
-	if (!state->isvalid || PREDICATE(DATUM_TO_CTYPE(state->value), value) || isnan((double) value))
+	if (!state->isvalid || PREDICATE(DATUM_TO_CTYPE(state->value), value))
 	{
 		/*
 		 * Note that float8 Datum is by-reference on 32-bit systems, and this

tsl/test/expected/vector_agg_functions.out

@@ -7158,4 +7158,13 @@ select ss, count(*), min(f1) from edges where f1 = 65 group by 1 order by 1;
  11 |     1 |  65
  12 |     1 |  65
 
+-- NaN handling in Postgres minmax is different from other functions -- it treats
+-- NaN as bigger than any number. This test has a filter that ensures that
+-- NaNs go first in the source data, because some potentially buggy conditions
+-- can be only triggered in this case.
+select ss, min(cfloat4), max(cfloat4) from aggfns where cfloat4 = any(array['nan'::float4, 33.308792]) group by ss order by ss;
+ ss |   min   | max 
+----+---------+-----
+ 11 | 33.3088 | NaN
+
 reset max_parallel_workers_per_gather;

tsl/test/sql/vector_agg_functions.sql

@@ -198,4 +198,10 @@ select ss, count(*), min(f1) from edges where f1 = 63 group by 1 order by 1;
 select ss, count(*), min(f1) from edges where f1 = 64 group by 1 order by 1;
 select ss, count(*), min(f1) from edges where f1 = 65 group by 1 order by 1;
 
+-- NaN handling in Postgres minmax is different from other functions -- it treats
+-- NaN as bigger than any number. This test has a filter that ensures that
+-- NaNs go first in the source data, because some potentially buggy conditions
+-- can be only triggered in this case.
+select ss, min(cfloat4), max(cfloat4) from aggfns where cfloat4 = any(array['nan'::float4, 33.308792]) group by ss order by ss;
+
 reset max_parallel_workers_per_gather;