PERF: take the read_csv token boundary from words, not word_starts

Supersedes the _token_len_checked helper from the previous commit,
which measured 11.6% slower than main on short-token string columns.

GH#66415 computed pass-1 token lengths from word_starts, a second
metadata array streamed alongside the words array coliter_next
already reads. Taking the boundary from words is identical
arithmetic over one array.

123-case csvbench sweep vs main: median 1.0018, 30 cases >=1%
faster (top 1.113x), no confirmed regression.

Also points the embedded-NUL test at GH#19886, the issue it
actually covers, and drops an unsound dtype=object oracle: the C
object path interns words by their NUL-terminated prefix, so it
is not a valid reference for NUL-bearing data.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: jbrockmendel

pandas/_libs/parsers.pyx

@@ -2570,9 +2570,9 @@ cdef _string_pyarrow_utf8(parser_t *parser, int64_t col,
                     offsets32_ptr[i + 1] = <int32_t>total_bytes
                 continue
 
-            # _token_len_checked, not strlen: an embedded NUL is a data byte
-            # here, so strlen alone would truncate the field at it (GH#66277).
-            wlen = _token_len_checked(parser, token_idx, word)
+            # Not strlen: an embedded NUL is a data byte here, so strlen would
+            # truncate the field at it (GH#66277).
+            wlen = _token_len_words(parser, token_idx, word)
 
             if not large and total_bytes + wlen > <Py_ssize_t>INT32_MAX:
                 overflow = True
@@ -3056,40 +3056,26 @@ cdef inline int64_t _token_len(parser_t *parser, int64_t token_idx) noexcept nog
     return <int64_t>parser.stream_len - parser.word_starts[token_idx] - 1
 
 
-cdef enum:
-    # Cap on the _token_len_checked scan: covers typical short string fields
-    # while keeping the worst-case scan to half a cache line.
-    _TOKEN_SCAN_CAP = 32
-
-
-cdef inline int64_t _token_len_checked(parser_t *parser, int64_t token_idx,
-                                       const char *word) noexcept nogil:
-    # Length for hot loops that already hold the token pointer.  Benchmarks
-    # (bandwidth-bound parallel string reads) show strlen beating the O(1)
-    # boundary arithmetic for short tokens: the scan's byte reads warm
-    # exactly the lines the copy pass touches next, and the boundary load's
-    # cache miss otherwise lands on the length's dependency chain.  For long
-    # tokens the scan itself becomes the bottleneck, so those keep the
-    # boundary arithmetic (the branch is uniform within a column, hence
-    # well-predicted).  The next token starts right after this one's NUL
-    # terminator, so `end == word + wlen + 1` proves no embedded NUL and the
-    # strlen result is exact; on mismatch (embedded NUL, GH#66277) the
-    # boundary arithmetic gives the true length.
-    cdef:
-        int64_t wlen
-        const char *end
+cdef inline int64_t _token_len_words(parser_t *parser, int64_t token_idx,
+                                     const char *word) noexcept nogil:
+    # Same arithmetic as _token_len, but taking the boundary from `words`
+    # rather than `word_starts`.  The tokenizer keeps the two in lockstep
+    # (words[i] == stream + word_starts[i], rebased whenever the stream
+    # reallocs), so the result is identical; what differs is which array the
+    # loop touches.  coliter_next already loaded words[token_idx] to produce
+    # `word`, so the boundary comes off a cache line the loop has in hand
+    # instead of streaming a second metadata array alongside the first.
+    # `word` must be the unmodified coliter_next result for `token_idx`;
+    # an adjusted pointer would silently yield a wrong length.
+    # Only the pyarrow string path uses this: rewriting _token_len itself to
+    # this form regressed long-token ints ~6% (GH#66277), so the numeric
+    # callers keep the word_starts version.
     if token_idx < 0:
         # missing field; word is a static "" outside the stream
         return 0
     if <uint64_t>(token_idx + 1) < parser.words_len:
-        end = parser.words[token_idx + 1]
-    else:
-        end = parser.stream + parser.stream_len
-    if <int64_t>(end - word) <= _TOKEN_SCAN_CAP:
-        wlen = <int64_t>strlen(word)
-        if end == word + wlen + 1:
-            return wlen
-    return <int64_t>(end - word) - 1
+        return <int64_t>(parser.words[token_idx + 1] - word) - 1
+    return <int64_t>(parser.stream + parser.stream_len - word) - 1
 
 
 cdef int _try_uint64_nogil(parser_t *parser, int64_t col,

pandas/tests/io/parser/test_c_parser_only.py

@@ -856,13 +856,21 @@ def test_block_lane_nrows_short_row_near_stream_capacity(c_parser_only):
 
 
 @pytest.mark.parametrize("kwargs", [{}, {"dtype_backend": "pyarrow"}])
-def test_embedded_nul_byte_roundtrip(c_parser_only, kwargs):
-    # GH#66277: the pyarrow string fast path computed token lengths with
+@pytest.mark.parametrize("prefix_len", [1, 200])
+def test_embedded_nul_byte_roundtrip(c_parser_only, kwargs, prefix_len):
+    # GH#19886: the pyarrow string fast path computed token lengths with
     # strlen, so a quoted field with an embedded NUL byte was truncated at the
-    # NUL instead of matching the object path
+    # NUL.  Column "b" is the last token in the stream, so its length comes
+    # from the stream end rather than from the next token's start.
     pytest.importorskip("pyarrow")
     parser = c_parser_only
-    result = parser.read_csv(BytesIO(b'a\n"x\x00y"\n'), **kwargs)
-    expected = parser.read_csv(BytesIO(b'a\n"x\x00y"\n'), dtype=object)
-    assert result["a"][0] == "x\x00y"
-    assert expected["a"][0] == "x\x00y"
+    value = b"x" * prefix_len + b"\x00y"
+    data = b'a,b\n"' + value + b'","' + value + b'"'
+    result = parser.read_csv(BytesIO(data), **kwargs)
+    # engine="python", not dtype=object: the C object path interns words by
+    # their NUL-terminated prefix, so it is not a sound reference for NUL data
+    expected = read_csv(BytesIO(data), engine="python")
+    assert result["a"][0] == value.decode()
+    assert result["b"][0] == value.decode()
+    assert expected["a"][0] == value.decode()
+    assert expected["b"][0] == value.decode()