Skip to content

Commit e71f340

Browse files
jbrockmendelclaude
andcommitted
PERF: vectorize PeriodIndex.from_fields
Add period_ordinals_from_fields Cython function that converts arrays of year/month/day/hour/minute/second fields to period ordinals in a single C-level loop, replacing the Python-space list-append loop in _range_from_fields, and vectorize the quarter-to-calendar-month conversion with numpy ops. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent c3a779c commit e71f340

5 files changed

Lines changed: 137 additions & 11 deletions

File tree

doc/source/whatsnew/v3.1.0.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ Performance improvements
206206
- Performance improvement in :meth:`Index.join` and :meth:`Index.union` for :class:`RangeIndex` by avoiding unnecessary memory allocation in the libjoin fastpath (:issue:`54646`)
207207
- Performance improvement in :meth:`IntervalIndex.get_indexer` for monotonic non-overlapping indexes, which now uses binary search instead of the interval tree (:issue:`47614`)
208208
- Performance improvement in :meth:`NDFrame.__finalize__`, :meth:`Series.to_numpy`, :attr:`DataFrame.dtypes`, and :meth:`DataFrame.__getitem__` (:issue:`57431`)
209+
- Performance improvement in :meth:`PeriodIndex.from_fields` (:issue:`65195`)
209210
- Performance improvement in :meth:`Timedelta.total_seconds` (:issue:`65388`)
210211
- Performance improvement in :meth:`arrays.SparseArray.isna` by avoiding a dense-then-resparsify round-trip (:issue:`41023`)
211212
- Performance improvement in datetime/timedelta unit conversion (e.g. ``datetime64[s]`` to ``datetime64[ns]``) (:issue:`35025`)

pandas/_libs/tslibs/period.pyi

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,15 @@ def get_period_field_arr(
3131
arr: npt.NDArray[np.int64], # const int64_t[:]
3232
freq: int,
3333
) -> npt.NDArray[np.int64]: ...
34+
def period_ordinals_from_fields(
35+
years: npt.NDArray[np.int64],
36+
months: npt.NDArray[np.int64],
37+
days: npt.NDArray[np.int64],
38+
hours: npt.NDArray[np.int64],
39+
minutes: npt.NDArray[np.int64],
40+
seconds: npt.NDArray[np.int64],
41+
freq: int,
42+
) -> npt.NDArray[np.int64]: ...
3443
def from_calendar_ordinals(
3544
values: npt.NDArray[np.int64], # const int64_t[:]
3645
dtype: PeriodDtypeBase,

pandas/_libs/tslibs/period.pyx

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ from cpython.datetime cimport (
2727
datetime,
2828
import_datetime,
2929
)
30+
from libc.stdint cimport INT32_MAX
3031
from libc.stdlib cimport (
3132
free,
3233
malloc,
@@ -1567,6 +1568,61 @@ cdef accessor _get_accessor_func(str field):
15671568
return NULL
15681569

15691570

1571+
@cython.wraparound(False)
1572+
@cython.boundscheck(False)
1573+
def period_ordinals_from_fields(
1574+
const int64_t[:] years,
1575+
const int64_t[:] months,
1576+
const int64_t[:] days,
1577+
const int64_t[:] hours,
1578+
const int64_t[:] minutes,
1579+
const int64_t[:] seconds,
1580+
int freq,
1581+
):
1582+
"""
1583+
Vectorized version of period_ordinal: convert arrays of date/time fields
1584+
to an array of period ordinals for the given frequency.
1585+
1586+
Parameters
1587+
----------
1588+
years, months, days, hours, minutes, seconds : int64 arrays
1589+
freq : int
1590+
1591+
Returns
1592+
-------
1593+
ndarray[int64]
1594+
"""
1595+
cdef:
1596+
Py_ssize_t i, n = len(years)
1597+
int64_t[::1] result = np.empty(n, dtype="i8")
1598+
npy_datetimestruct dts
1599+
1600+
memset(&dts, 0, sizeof(npy_datetimestruct))
1601+
1602+
for i in range(n):
1603+
# month/day/hour/min/sec land in int32 npy_datetimestruct fields,
1604+
# so values outside int32 range would silently wrap; period_ordinal
1605+
# raises OverflowError for these (C int args), so match that.
1606+
if (
1607+
not (INT32_MIN <= years[i] <= INT32_MAX)
1608+
or not (INT32_MIN <= months[i] <= INT32_MAX)
1609+
or not (INT32_MIN <= days[i] <= INT32_MAX)
1610+
or not (INT32_MIN <= hours[i] <= INT32_MAX)
1611+
or not (INT32_MIN <= minutes[i] <= INT32_MAX)
1612+
or not (INT32_MIN <= seconds[i] <= INT32_MAX)
1613+
):
1614+
raise OverflowError("value too large to convert to int")
1615+
dts.year = years[i]
1616+
dts.month = months[i]
1617+
dts.day = days[i]
1618+
dts.hour = hours[i]
1619+
dts.min = minutes[i]
1620+
dts.sec = seconds[i]
1621+
result[i] = get_period_ordinal(&dts, freq)
1622+
1623+
return result.base
1624+
1625+
15701626
@cython.wraparound(False)
15711627
@cython.boundscheck(False)
15721628
def from_calendar_ordinals(const int64_t[:] values, PeriodDtypeBase dtype):

pandas/core/arrays/period.py

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
period as libperiod,
3636
to_offset,
3737
)
38+
from pandas._libs.tslibs.ccalendar import MONTH_NUMBERS
3839
from pandas._libs.tslibs.dtypes import (
3940
FreqGroup,
4041
PeriodDtypeBase,
@@ -1568,8 +1569,6 @@ def _range_from_fields(
15681569
if day is None:
15691570
day = 1
15701571

1571-
ordinals = []
1572-
15731572
if quarter is not None:
15741573
if freq is None:
15751574
freq = to_offset("Q", is_period=True)
@@ -1582,20 +1581,38 @@ def _range_from_fields(
15821581

15831582
freqstr = freq.freqstr
15841583
year, quarter = _make_field_arrays(year, quarter)
1585-
for y, q in zip(year, quarter, strict=True):
1586-
calendar_year, calendar_month = parsing.quarter_to_myear(y, q, freqstr)
1587-
val = libperiod.period_ordinal(
1588-
calendar_year, calendar_month, 1, 1, 1, 1, 0, 0, base
1589-
)
1590-
ordinals.append(val)
1584+
year = np.asarray(year, dtype=np.int64)
1585+
quarter = np.asarray(quarter, dtype=np.int64)
1586+
1587+
if (quarter < 1).any() or (quarter > 4).any():
1588+
raise ValueError("Quarter must be 1 <= q <= 4")
1589+
1590+
# Vectorized quarter_to_myear
1591+
mnum = MONTH_NUMBERS[parsing.get_rule_month(freqstr)] + 1
1592+
months = (mnum + (quarter - 1) * 3) % 12 + 1
1593+
years = np.where(months > mnum, year - 1, year)
1594+
1595+
length = len(years)
1596+
ones = np.ones(length, dtype=np.int64)
1597+
zeros = np.zeros(length, dtype=np.int64)
1598+
ordinals = libperiod.period_ordinals_from_fields(
1599+
years, months, ones, zeros, zeros, zeros, base
1600+
)
15911601
else:
15921602
freq = to_offset(freq, is_period=True)
15931603
base = libperiod.freq_to_dtype_code(freq)
15941604
arrays = _make_field_arrays(year, month, day, hour, minute, second)
1595-
for y, mth, d, h, mn, s in zip(*arrays, strict=True):
1596-
ordinals.append(libperiod.period_ordinal(y, mth, d, h, mn, s, 0, 0, base))
1605+
ordinals = libperiod.period_ordinals_from_fields(
1606+
arrays[0].astype(np.int64, copy=False),
1607+
arrays[1].astype(np.int64, copy=False),
1608+
arrays[2].astype(np.int64, copy=False),
1609+
arrays[3].astype(np.int64, copy=False),
1610+
arrays[4].astype(np.int64, copy=False),
1611+
arrays[5].astype(np.int64, copy=False),
1612+
base,
1613+
)
15971614

1598-
return np.array(ordinals, dtype=np.int64), freq
1615+
return ordinals, freq
15991616

16001617

16011618
def _make_field_arrays(*fields) -> list[np.ndarray]:

pandas/tests/indexes/period/test_constructors.py

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,49 @@ def test_constructor_invalid_quarters(self):
241241
year=range(2000, 2004), quarter=list(range(4)), freq="Q-DEC"
242242
)
243243

244+
def test_constructor_field_arrays_quarter_no_freq(self):
245+
# Quarter with no explicit freq infers Q-DEC
246+
years = np.array([2020, 2020, 2020, 2020], dtype=np.int64)
247+
quarters = np.array([1, 2, 3, 4], dtype=np.int64)
248+
result = PeriodIndex.from_fields(year=years, quarter=quarters)
249+
expected = PeriodIndex(
250+
[Period(year=2020, quarter=q, freq="Q-DEC") for q in range(1, 5)]
251+
)
252+
tm.assert_index_equal(result, expected)
253+
254+
def test_constructor_field_arrays_hourly(self):
255+
# Test non-quarter path with all 6 fields
256+
result = PeriodIndex.from_fields(
257+
year=[2020, 2020],
258+
month=[1, 6],
259+
day=[15, 20],
260+
hour=[10, 22],
261+
minute=[30, 45],
262+
second=[5, 59],
263+
freq="s",
264+
)
265+
expected = PeriodIndex(
266+
[
267+
Period("2020-01-15 10:30:05", freq="s"),
268+
Period("2020-06-20 22:45:59", freq="s"),
269+
]
270+
)
271+
tm.assert_index_equal(result, expected)
272+
273+
def test_constructor_field_arrays_empty(self):
274+
# Empty arrays should produce empty PeriodIndex
275+
result = PeriodIndex.from_fields(year=[], month=[], freq="M")
276+
expected = PeriodIndex([], dtype="period[M]")
277+
tm.assert_index_equal(result, expected)
278+
279+
@pytest.mark.parametrize("field", ["year", "month"])
280+
def test_constructor_field_arrays_outside_int32(self, field):
281+
# Fields outside int32 range must raise instead of wrapping silently
282+
fields = {"year": [2000, 2000], "month": [3, 3]}
283+
fields[field] = [fields[field][0], 2**32 + 3]
284+
with pytest.raises(OverflowError, match="value too large"):
285+
PeriodIndex.from_fields(**fields, freq="M")
286+
244287
def test_period_range_fractional_period(self):
245288
msg = "periods must be an integer, got 10.5"
246289
with pytest.raises(TypeError, match=msg):

0 commit comments

Comments
 (0)