Skip to content

GH-51301: Fix deprecation warnings in the tests with numpy/pandas nightly #51404

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AlenkaF wants to merge 9 commits into
base: main
Choose a base branch
from
+23 −15
Open

GH-51301: Fix deprecation warnings in the tests with numpy/pandas nightly #51404

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
1ed3c25
Fix for deprecated generic units
AlenkaF Sep 21, 2026
b2d348a
Fix dlpack warnings
AlenkaF Sep 21, 2026
180c881
Fix null_placement warnings
AlenkaF Sep 21, 2026
43de55d
Fix unit keyward warning in pd.timestamp
AlenkaF Sep 21, 2026
0baa242
Revert "Fix null_placement warnings"
AlenkaF Sep 22, 2026
84a8518
Revert the change on __dlpack__
AlenkaF Sep 22, 2026
54feb6d
Revert all dplack related changes
AlenkaF Sep 24, 2026
a148c13
Update test_does_not_mutate_timedelta_dtype
AlenkaF Sep 24, 2026
4fa9481
Add missing deprecation fixes
AlenkaF Sep 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions python/pyarrow/tests/test_array.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import pyarrow as pa
import pyarrow.tests.strategies as past
import pyarrow.compute as pc
from pyarrow.vendored.version import Version


@pytest.mark.processes
Expand Down Expand Up @@ -2679,6 +2680,8 @@ def test_array_from_list_of_timestamps(unit):

@pytest.mark.numpy
def test_array_from_timestamp_with_generic_unit():
if Version(np.__version__) >= Version("2.5.0"):
pytest.skip("generic units of timedelta64 deprecated")
n = np.datetime64('NaT')
x = np.datetime64('2017-01-01 01:01:01.111111111')
y = np.datetime64('2018-11-22 12:24:48.111111111')
Expand Down Expand Up @@ -2720,11 +2723,13 @@ def test_array_from_numpy_timedelta(dtype, type):
@pytest.mark.numpy
def test_array_from_numpy_timedelta_incorrect_unit():
# generic (no unit)
td = np.timedelta64(1)
if Version(np.__version__) < Version("2.5.0"):
# Generic units of timedelta64 deprecated in NumPy 2.5
td = np.timedelta64(1)

for data in [[td], np.array([td])]:
with pytest.raises(NotImplementedError):
pa.array(data)
for data in [[td], np.array([td])]:
with pytest.raises(NotImplementedError):
pa.array(data)

# unsupported unit
td = np.timedelta64(1, 'M')
Expand Down
8 changes: 5 additions & 3 deletions python/pyarrow/tests/test_compute.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2644,8 +2644,8 @@ def _check_datetime_components(timestamps, timezone=None):
year = ts.dt.year.astype("int64")
month = ts.dt.month.astype("int64")
day = ts.dt.day.astype("int64")
dayofweek = ts.dt.dayofweek.astype("int64")
dayofyear = ts.dt.dayofyear.astype("int64")
dayofweek = pd.DatetimeIndex(ts).day_of_week.astype("int64")
dayofyear = pd.DatetimeIndex(ts).day_of_year.astype("int64")
quarter = ts.dt.quarter.astype("int64")
hour = ts.dt.hour.astype("int64")
minute = ts.dt.minute.astype("int64")
Expand All @@ -2657,7 +2657,9 @@ def _check_datetime_components(timestamps, timezone=None):
assert pc.is_leap_year(tsa).equals(pa.array(ts.dt.is_leap_year))
assert pc.month(tsa).equals(pa.array(month))
assert pc.day(tsa).equals(pa.array(day))

assert pc.day_of_week(tsa).equals(pa.array(dayofweek))

assert pc.day_of_year(tsa).equals(pa.array(dayofyear))
assert pc.iso_year(tsa).equals(pa.array(iso_year))
assert pc.iso_week(tsa).equals(pa.array(iso_week))
Expand Down Expand Up @@ -2968,7 +2970,7 @@ def test_round_temporal(unit):
if sys.platform == "win32":
timestamps = timestamps[:3] + timestamps[5:]

ts = pd.Series([pd.Timestamp(x, unit="ns") for x in timestamps])
ts = pd.Series([pd.Timestamp(x).as_unit("ns") for x in timestamps])
_check_temporal_rounding(ts, values, unit)

timezones = ["Asia/Kolkata", "America/New_York", "Etc/GMT-4", "Etc/GMT+4",
Expand Down
2 changes: 1 addition & 1 deletion python/pyarrow/tests/test_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ def multisourcefs(request):

# create one with schema partitioning by weekday and color
mockfs.create_dir('schema')
for part, chunk in df_b.groupby([df_b.date.dt.dayofweek, df_b.color]):
for part, chunk in df_b.groupby([pd.DatetimeIndex(df_b.date).day_of_week, df_b.color]):
folder = f'schema/{part[0]}/{part[1]}'
path = f'{folder}/chunk.parquet'
mockfs.create_dir(folder)
Expand Down
15 changes: 8 additions & 7 deletions python/pyarrow/tests/test_pandas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ def _alltypes_example(size=100):

def _check_pandas_roundtrip(df, expected=None, use_threads=False,
expected_schema=None,
check_dtype=True, schema=None,
preserve_index=False,
check_dtype=True, check_freq=False,
schema=None,preserve_index=False,
as_batch=False):
klass = pa.RecordBatch if as_batch else pa.Table
table = klass.from_pandas(df, schema=schema,
Expand All @@ -125,7 +125,8 @@ def _check_pandas_roundtrip(df, expected=None, use_threads=False,
"ignore", "elementwise comparison failed", DeprecationWarning)
tm.assert_frame_equal(result, expected, check_dtype=check_dtype,
check_index_type=('equiv' if preserve_index
else False))
else False),
check_freq=check_freq)


def _check_series_roundtrip(s, type_=None, expected_pa_type=None):
Expand Down Expand Up @@ -5002,15 +5003,15 @@ def test_threaded_pandas_import():


def test_does_not_mutate_timedelta_dtype():
expected = np.dtype('m8')
expected = np.dtype('<m8[s]')

assert np.dtype(np.timedelta64) == expected
assert np.dtype(np.timedelta64(0, "s")) == expected

df = pd.DataFrame({"a": [np.timedelta64()]})
df = pd.DataFrame({"a": [np.timedelta64(0, "s")]})

@jorisvandenbossche jorisvandenbossche Sep 22, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While this makes the test run without the warning, I am not sure if then the surrounding asserts still make sense.
One thing to do is also to change expected = np.dtype('m8') to use "m8[s]", but no idea if that would then still have produced the initial bug (the reason this test was added, #13553)

@AlenkaF AlenkaF Sep 24, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, OK, sorry. Didn't try locally and assumed np.dtype(np.timedelta64) produces same result for any unit. I am guessing we want to keep testing this even for higher numpy versions. Will look first into changing expected value and what that means for the initial bug.

@AlenkaF AlenkaF Sep 24, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From testing locally it seems the initial bug is not produced even if using a time unit everywhere. Will push a commit.

@AlenkaF AlenkaF Sep 24, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a148c13

t = pa.Table.from_pandas(df)
t.to_pandas()

assert np.dtype(np.timedelta64) == expected
assert np.dtype(np.timedelta64(0, "s")) == expected


def test_does_not_mutate_timedelta_nested():
Expand Down
Loading