BUG: Timestamp landing on the NaT sentinel silently gave NaT (GH#66510)

A wall time whose int64 representation is exactly the NaT sentinel (-2**63)
came back as NaT, or as a Timestamp holding the sentinel that reported
pd.isna(...) as False while reading back as NaT once placed in a Series.
npy_datetimestruct_to_datetime returns the sentinel rather than raising, and
the later bounds checks are all skipped for NPY_NAT.

Also report an out-of-range shift to UTC as OutOfBoundsDatetime instead of
leaking a raw OverflowError from the int64 coercion.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: jbrockmendel

doc/source/whatsnew/v3.1.0.rst

@@ -334,6 +334,8 @@ Datetimelike
 - Bug in :class:`ArrowExtensionArray` where adding a :class:`DateOffset` to a ``date32[pyarrow]`` or ``date64[pyarrow]`` Series raised an ``ArrowTypeError`` (:issue:`57168`)
 - Bug in :class:`DatetimeIndex` constructor raising ``ValueError`` when passing equivalent but not equal frequencies (e.g. ``QS-FEB`` vs ``QS-MAY``) (:issue:`61086`)
 - Bug in :class:`DatetimeIndex` raising ``AttributeError`` when comparing against Arrow date types (date32, date64) (:issue:`62051`)
+- Bug in :class:`Timestamp` constructor and :meth:`Timestamp.replace` where a value landing exactly on the ``NaT`` sentinel — whether from a ``nanosecond`` or ``microsecond`` argument or from shifting a timezone-aware wall time to UTC — silently produced ``NaT`` (or a :class:`Timestamp` that reported itself as non-null but read back as ``NaT`` once placed in a :class:`Series`) instead of raising :class:`OutOfBoundsDatetime` (:issue:`66510`)
+- Bug in :class:`Timestamp` constructor and :meth:`Timestamp.replace` where shifting a timezone-aware value to UTC past the representable range raised a raw ``OverflowError`` instead of :class:`OutOfBoundsDatetime`, e.g. ``Timestamp.max.replace(tzinfo=timezone(timedelta(hours=-9)))`` (:issue:`66510`)
 - Bug in :class:`Timestamp` constructor where passing ``np.str_`` objects would fail in Cython string parsing (:issue:`48974`)
 - Bug in :class:`Timestamp` constructor where strings with a negative year of fewer than 4 digits (e.g. ``"-111-01-01"``) silently dropped the leading ``"-"`` and were parsed as a positive year; BC dates with 1-4 digit years now parse correctly, matching :class:`numpy.datetime64` (:issue:`55954`)
 - Bug in :class:`Timestamp` constructor, :class:`Timedelta` constructor, :func:`to_datetime`, and :func:`to_timedelta` with non-round ``float`` input and ``unit`` failing to raise when the value is just outside the representable bounds (:issue:`57366`)

pandas/_libs/tslibs/conversion.pyx

@@ -443,6 +443,13 @@ cdef class _TSObject:
         return self.value
 
 
+cdef _raise_out_of_bounds(npy_datetimestruct *dts, NPY_DATETIMEUNIT reso):
+    attrname = npy_unit_to_attrname[reso]
+    raise OutOfBoundsDatetime(
+        f"Out of bounds {attrname} timestamp: {dts_to_iso_string(dts)}"
+    )
+
+
 cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
                                    bint dayfirst, bint yearfirst, int32_t nanos=0):
     """
@@ -492,6 +499,10 @@ cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
                 obj.value = tz_localize_to_utc_single(
                     obj.value, tz, ambiguous="raise", nonexistent=None, creso=reso
                 )
+                if obj.value == NPY_NAT:
+                    # an in-bounds wall time can land on the sentinel once
+                    #  shifted to UTC
+                    _raise_out_of_bounds(&obj.dts, reso)
     elif is_integer_object(ts) or (is_float_object(ts) and ts.is_integer()):
         try:
             ts = <int64_t>ts
@@ -617,12 +628,31 @@ cdef _TSObject convert_datetime_to_tsobject(
         obj.value = npy_datetimestruct_to_datetime(reso, &obj.dts)
     except OverflowError as err:
         attrname = npy_unit_to_attrname[reso]
-        raise OutOfBoundsDatetime(f"Out of bounds {attrname} timestamp") from err
+        raise OutOfBoundsDatetime(
+            f"Out of bounds {attrname} timestamp: {dts_to_iso_string(&obj.dts)}"
+        ) from err
+
+    if obj.value == NPY_NAT:
+        # Reachable via `nanos`, which Timestamp.replace supplies.  This is the
+        #  *wall* time, not the result: it is out of bounds at this reso, and
+        #  npy_datetimestruct_to_datetime hands back the sentinel for it
+        #  instead of raising.
+        _raise_out_of_bounds(&obj.dts, reso)
 
     if obj.tzinfo is not None and not is_utc(obj.tzinfo):
         offset = get_utcoffset(obj.tzinfo, ts)
         pps = periods_per_second(reso)
-        obj.value -= int(offset.total_seconds() * pps)
+        # The shift to UTC can leave the representable range; report that as
+        #  OutOfBoundsDatetime rather than leaking a raw OverflowError from the
+        #  int64 coercion (cf. GH#65353, which fixed a silent wrap in the C
+        #  string parsers).
+        if checked_sub(obj.value, int(offset.total_seconds() * pps), &obj.value):
+            _raise_out_of_bounds(&obj.dts, reso)
+
+        if obj.value == NPY_NAT:
+            # ...and an in-bounds wall time can land on the sentinel once
+            #  shifted, which the guard above cannot see.
+            _raise_out_of_bounds(&obj.dts, reso)
 
     check_overflows(obj, reso)
     return obj

pandas/_libs/tslibs/timestamps.pyx

@@ -3691,6 +3691,14 @@ default 'raise'
                 raise OutOfBoundsDatetime(
                     f"Out of bounds timestamp: {fmt} with frequency '{self.unit}'"
                 ) from err
+            if ts.value == NPY_NAT:
+                # npy_datetimestruct_to_datetime hands back the sentinel for this
+                #  one wall time instead of raising; keeping it would build a
+                #  Timestamp that reports itself non-null but reads back as NaT.
+                fmt = dts_to_iso_string(&dts)
+                raise OutOfBoundsDatetime(
+                    f"Out of bounds timestamp: {fmt} with frequency '{self.unit}'"
+                )
             ts.dts = dts
             ts.creso = creso
             ts.fold = fold

pandas/tests/scalar/timestamp/methods/test_replace.py

@@ -1,4 +1,8 @@
-from datetime import datetime
+from datetime import (
+    datetime,
+    timedelta,
+    timezone,
+)
 import zoneinfo
 
 from dateutil.tz import gettz
@@ -31,6 +35,42 @@ def test_replace_out_of_pydatetime_bounds(self):
         assert result.year == 99_999
         assert result._value == Timestamp(np.datetime64("99999-01-01", "ms"))._value
 
+    def test_replace_microsecond_lands_on_nat_sentinel(self):
+        # GH#66510 this wall time maps to the NaT sentinel; keeping it would
+        #  build a Timestamp that reports itself non-null but reads back as NaT
+        ts = Timestamp(np.datetime64(-(2**63) + 1, "us"))
+        with pytest.raises(OutOfBoundsDatetime, match="Out of bounds"):
+            ts.replace(microsecond=224192)
+
+    @pytest.mark.parametrize(
+        "wall, tz",
+        [
+            ("1677-09-21 00:12:43.145224193", None),
+            ("1677-09-21 00:12:43.145224193", "UTC"),
+            # non-zero offset: the shift to UTC is what lands on the sentinel
+            ("1677-09-21 09:31:42.145224193", "Asia/Tokyo"),
+        ],
+    )
+    def test_replace_nanosecond_lands_on_nat_sentinel(self, wall, tz):
+        # GH#66510
+        ts = Timestamp(wall, tz=tz)
+
+        with pytest.raises(OutOfBoundsDatetime, match="Out of bounds"):
+            ts.replace(nanosecond=192)
+
+        # neighbours: below is out of bounds, above is representable
+        with pytest.raises(OutOfBoundsDatetime, match="Out of bounds"):
+            ts.replace(nanosecond=191)
+        assert ts.replace(nanosecond=194)._value == ts._value + 1
+
+    @pytest.mark.parametrize("offset_hours", [9, -9])
+    def test_replace_tzinfo_shift_out_of_bounds(self, offset_hours):
+        # GH#66510 the shift to UTC leaves the representable range; that must be
+        #  an OutOfBoundsDatetime, not a raw OverflowError from the int64 coercion
+        ts = Timestamp.max if offset_hours < 0 else Timestamp.min
+        with pytest.raises(OutOfBoundsDatetime, match="Out of bounds"):
+            ts.replace(tzinfo=timezone(timedelta(hours=offset_hours)))
+
     def test_replace_non_nano(self):
         ts = Timestamp._from_value_and_reso(
             91514880000000000, NpyDatetimeUnit.NPY_FR_us.value, None

pandas/tests/scalar/timestamp/test_constructors.py

@@ -1192,3 +1192,29 @@ def test_timestamp_iso8601_offset_out_of_bounds(tz):
     msg = "Out of bounds nanosecond timestamp"
     with pytest.raises(OutOfBoundsDatetime, match=msg):
         Timestamp("2262-04-11T20:00:00.000000000-11:00", tz=tz)
+
+
+@pytest.mark.parametrize("unit", ["ns", "us", "ms", "s"])
+def test_timestamp_constructor_nat_sentinel_tz_shift(unit):
+    # GH#66510 a wall time whose shift to UTC lands on the NaT sentinel used to
+    #  come back as NaT.  `unit` is the creso handed to tz_localize_to_utc_single;
+    #  the offset can only be positive here, since a negative one puts the wall
+    #  time itself below int64 min.
+    offset_hours = 9
+    per_second = {"ns": 10**9, "us": 10**6, "ms": 10**3, "s": 1}[unit]
+    ts_input = np.datetime64(-(2**63) + offset_hours * 3600 * per_second, unit)
+    tz = timezone(timedelta(hours=offset_hours))
+
+    with pytest.raises(OutOfBoundsDatetime, match="Out of bounds .* timestamp"):
+        Timestamp(ts_input, tz=tz)
+
+
+@pytest.mark.parametrize(
+    "ts_input, tz", [(Timestamp.max, "US/Pacific"), (Timestamp.min, "Asia/Tokyo")]
+)
+def test_timestamp_constructor_tz_shift_out_of_bounds(ts_input, tz):
+    # GH#66510 shifting the wall time to UTC leaves the representable range,
+    #  which must be OutOfBoundsDatetime rather than a raw OverflowError from
+    #  the int64 coercion
+    with pytest.raises(OutOfBoundsDatetime, match="Out of bounds nanosecond timestamp"):
+        Timestamp(ts_input, tz=tz)