BUG: raise OverflowError on datetime64 unit conversion overflow (numpy#31085)

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: jbrockmendel

numpy/_core/src/multiarray/_datetime.h

@@ -335,4 +335,45 @@ datetime_hash(PyArray_DatetimeMetaData *meta, npy_datetime dt);
 NPY_NO_EXPORT npy_hash_t
 timedelta_hash(PyArray_DatetimeMetaData *meta, npy_timedelta td);
 
+/*
+ * Scale a datetime or timedelta value by num/denom, checking for overflow.
+ *
+ * Positive values compute *dt * num / denom.
+ * Negative values compute (*dt * num - (denom - 1)) / denom to round
+ * toward negative infinity.
+ *
+ * NPY_DATETIME_NAT is NPY_MIN_INT64 (i.e. -NPY_MAX_INT64 - 1).
+ * The asymmetric neg_limit formula ensures that a valid *dt * num never
+ * produces NPY_MIN_INT64, which would be misinterpreted as NaT.
+ *
+ * NaT values pass through unchanged.
+ *
+ * Returns 0 on success, -1 on overflow (with PyExc_OverflowError set).
+ */
+static inline int
+_datetime_scale_with_overflow_check(
+        npy_int64 *dt, npy_int64 num, npy_int64 denom,
+        const char *type_name)
+{
+    if (*dt == NPY_DATETIME_NAT) {
+        return 0;
+    }
+    npy_int64 pos_limit = NPY_MAX_INT64 / num;
+    npy_int64 neg_limit = (NPY_MAX_INT64 - denom + 1) / num;
+
+    if (*dt > pos_limit || *dt < -neg_limit) {
+        PyErr_Format(PyExc_OverflowError,
+                "Overflow when converting between "
+                "%s units", type_name);
+        return -1;
+    }
+    if (*dt < 0) {
+        *dt = (*dt * num - (denom - 1)) / denom;
+    }
+    else {
+        *dt = *dt * num / denom;
+    }
+    return 0;
+}
+
 #endif  /* NUMPY_CORE_SRC_MULTIARRAY__DATETIME_H_ */

numpy/_core/src/multiarray/datetime.c

@@ -444,8 +444,16 @@ NpyDatetime_ConvertDatetime64ToDatetimeStruct(
         return -1;
     }
 
-    /* TODO: Change to a mechanism that avoids the potential overflow */
-    dt *= meta->num;
+    /* Check for overflow and apply meta->num scaling */
+    if (meta->num > 1) {
+        if (_datetime_scale_with_overflow_check(
+                &dt, (npy_int64)meta->num, 1, "datetime64") < 0) {
+            return -1;
+        }
+    }
+    else {
+        dt *= meta->num;
+    }
 
     /*
      * Note that care must be taken with the / and % operators
@@ -3178,13 +3186,11 @@ cast_timedelta_to_timedelta(PyArray_DatetimeMetaData *src_meta,
         return -1;
     }
 
-    /* Apply the scaling */
-    if (src_dt < 0) {
-        *dst_dt = (src_dt * num - (denom - 1)) / denom;
-    }
-    else {
-        *dst_dt = src_dt * num / denom;
+    /* Apply the scaling, checking for overflow */
+    if (_datetime_scale_with_overflow_check(&src_dt, num, denom, "timedelta64") < 0) {
+        return -1;
     }
+    *dst_dt = src_dt;
 
     return 0;
 }

numpy/_core/src/multiarray/dtype_transfer.c

@@ -821,14 +821,8 @@ _strided_to_strided_datetime_cast(
     while (N > 0) {
         memcpy(&dt, src, sizeof(dt));
 
-        if (dt != NPY_DATETIME_NAT) {
-            /* Apply the scaling */
-            if (dt < 0) {
-                dt = (dt * num - (denom - 1)) / denom;
-            }
-            else {
-                dt = dt * num / denom;
-            }
+        if (_datetime_scale_with_overflow_check(&dt, num, denom, "datetime64") < 0) {
+            return -1;
         }
 
         memcpy(dst, &dt, sizeof(dt));
@@ -857,14 +851,8 @@ _aligned_strided_to_strided_datetime_cast(
     while (N > 0) {
         dt = *(npy_int64 *)src;
 
-        if (dt != NPY_DATETIME_NAT) {
-            /* Apply the scaling */
-            if (dt < 0) {
-                dt = (dt * num - (denom - 1)) / denom;
-            }
-            else {
-                dt = dt * num / denom;
-            }
+        if (_datetime_scale_with_overflow_check(&dt, num, denom, "datetime64") < 0) {
+            return -1;
         }
 
         *(npy_int64 *)dst = dt;

numpy/_core/tests/test_casting_unittests.py

@@ -481,7 +481,12 @@ def test_time_to_time(self, from_dt, to_dt,
                 arr, out = self.get_data_variation(
                         orig_arr, orig_out, aligned, contig)
                 out[...] = 0
-                cast._simple_strided_call((arr, out))
+                try:
+                    cast._simple_strided_call((arr, out))
+                except OverflowError:
+                    # Extreme values (e.g. INT64_MAX) can overflow when
+                    # scaled by the unit conversion factor. gh-16352
+                    break
                 assert_array_equal(out.view("int64"), expected_out.view("int64"))
 
     def string_with_modified_length(self, dtype, change_length):

numpy/_core/tests/test_datetime.py

@@ -993,6 +993,98 @@ def cast2():
             numpy.datetime64("2014").astype("<M8[fs]")
         assert_raises(OverflowError, cast2)
 
+    def test_cast_overflow_safe_unit_conversion(self):
+        # Overflow when converting datetime64 between linear units
+        # (the fast-path cast), e.g. seconds -> nanoseconds.
+        # INT64_MAX / 1e9 ≈ 9.2e9 seconds ≈ 292 years from epoch,
+        # so dates beyond ~2262 overflow when cast to ns.
+
+        # gh-16352: upconversion to finer units overflows
+        arr = np.array(["2367-12-31 12:00:00"], dtype="datetime64[h]")
+        with pytest.raises(OverflowError, match="Overflow"):
+            arr.astype("datetime64[ns]")
+
+        # gh-16352: scalar case
+        val = np.datetime64("3000-01-01", "s")
+        with pytest.raises(OverflowError, match="Overflow"):
+            val.astype("datetime64[ns]")
+
+        # gh-22346: downconversion to coarser units overflows near INT64_MIN
+        dt = np.datetime64(np.iinfo(np.int64).min + 1, "s")
+        with pytest.raises(OverflowError, match="Overflow"):
+            dt.astype("M8[m]")
+
+        # negative overflow (far in the past)
+        val_neg = np.datetime64("0001-01-01", "s")
+        with pytest.raises(OverflowError, match="Overflow"):
+            val_neg.astype("datetime64[ns]")
+
+        # timedelta overflow (strided cast path in dtype_transfer.c)
+        td = np.timedelta64(2**62, "s")
+        with pytest.raises(OverflowError, match="Overflow"):
+            td.astype("timedelta64[ns]")
+
+        # timedelta overflow (scalar cast path in datetime.c via
+        # cast_timedelta_to_timedelta)
+        td_big = np.timedelta64(2**62, "s")
+        with pytest.raises(OverflowError, match="Overflow"):
+            np.array(td_big, dtype="timedelta64[ns]")
+
+        # timedelta exact boundary: INT64_MAX // 1e9 = 9223372036
+        td_ok = np.timedelta64(9223372036, "s")
+        result_td = td_ok.astype("timedelta64[ns]")
+        assert result_td == np.timedelta64(9223372036000000000, "ns")
+
+        td_bad = np.timedelta64(9223372037, "s")
+        with pytest.raises(OverflowError, match="Overflow"):
+            td_bad.astype("timedelta64[ns]")
+
+        # negative timedelta overflow
+        td_neg = np.timedelta64(-9223372037, "s")
+        with pytest.raises(OverflowError, match="Overflow"):
+            td_neg.astype("timedelta64[ns]")
+
+        # timedelta NaT passthrough
+        td_nat = np.timedelta64("NaT", "s")
+        result_td_nat = td_nat.astype("timedelta64[ns]")
+        assert np.isnat(result_td_nat)
+
+        # valid conversions near the boundary should still work
+        val_ok = np.datetime64("2020-01-01", "s")
+        result = val_ok.astype("datetime64[ns]")
+        assert result == np.datetime64("2020-01-01", "ns")
+
+        arr_ok = np.array(["2000-01-01", "2020-06-15"], dtype="datetime64[s]")
+        result_arr = arr_ok.astype("datetime64[ns]")
+        expected = np.array(["2000-01-01", "2020-06-15"], dtype="datetime64[ns]")
+        assert_equal(result_arr, expected)
+
+        # NaT should pass through without raising
+        arr_nat = np.array(["NaT", "2020-01-01"], dtype="datetime64[s]")
+        result_nat = arr_nat.astype("datetime64[ns]")
+        assert np.isnat(result_nat[0])
+        assert result_nat[1] == np.datetime64("2020-01-01", "ns")
+
+        # Exact boundary: INT64_MAX // 1e9 = 9223372036 seconds is OK,
+        # 9223372037 seconds overflows when cast to ns.
+        ok_boundary = np.datetime64(9223372036, "s")
+        result_boundary = ok_boundary.astype("datetime64[ns]")
+        assert result_boundary == np.datetime64(9223372036, "s")
+
+        bad_boundary = np.datetime64(9223372037, "s")
+        with pytest.raises(OverflowError, match="Overflow"):
+            bad_boundary.astype("datetime64[ns]")
+
+        # Exercise the num != 1 code path (e.g. "2s" metadata)
+        arr_2s = np.array([3], dtype="datetime64[2s]")
+        result_2s = arr_2s.astype("datetime64[s]")
+        assert result_2s[0] == np.datetime64(6, "s")
+
+        # Overflow with num != 1
+        arr_2s_big = np.array([np.iinfo(np.int64).max // 2], dtype="datetime64[2s]")
+        with pytest.raises(OverflowError, match="Overflow"):
+            arr_2s_big.astype("datetime64[ns]")
+
     def test_pyobject_roundtrip(self):
         # All datetime types should be able to roundtrip through object
         a = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0,