diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4605d608ba5e1..5740caef2bccd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -278,13 +278,13 @@ repos: name: Check for pg8000 not installed on CI for test_pg8000_sqlalchemy_passthrough_error language: pygrep entry: 'pg8000' - files: ^ci/deps - types: [yaml] + files: pixi\.toml + types: [toml] - id: validate-min-versions-in-sync name: Check minimum version of dependencies are aligned entry: python -m scripts.validate_min_versions_in_sync language: python - files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$ + files: ^(pixi\.toml|pandas/compat/_optional\.py)$ additional_dependencies: [pyyaml] pass_filenames: false - id: validate-errors-locations diff --git a/ci/deps/actions-311-minimum_versions.yaml b/ci/deps/actions-311-minimum_versions.yaml deleted file mode 100644 index 799f8e6950afa..0000000000000 --- a/ci/deps/actions-311-minimum_versions.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Minimum version of required + optional dependencies -# Aligned with getting_started/install.rst and compat/_optional.py -name: pandas-dev -channels: - - conda-forge -dependencies: - - python=3.11 - - # build dependencies - - versioneer - - cython<4.0.0a0 - - meson=1.2.3 - - meson-python=0.18.0 - - # test dependencies - - pytest>=8.3.4 - - pytest-cov - - pytest-xdist>=3.6.1 - - pytest-localserver>=0.9.0 - - pytest-qt>=4.4.0 - - boto3=1.40.46 - - # required dependencies - - python-dateutil=2.8.2 - - numpy=1.26.0 - - # optional dependencies - - adbc-driver-postgresql=1.2.0 - - adbc-driver-sqlite=1.2.0 - - beautifulsoup4=4.12.3 - - bottleneck=1.4.2 - - fastparquet=2024.11.0 - - fsspec=2024.10.0 - - html5lib=1.1 - - hypothesis=6.116.0 - - gcsfs=2024.10.0 - - jinja2=3.1.5 - - lxml=5.3.0 - - matplotlib=3.9.3 - - numba=0.60.0 - - numexpr=2.10.2 - - odfpy=1.4.1 - - qtpy=2.4.2 - - openpyxl=3.1.5 - - psycopg2=2.9.10 - - pyarrow=13.0.0 - - pyiceberg=0.8.1 - - pymysql=1.1.1 - - pyqt=5.15.9 - - pyreadstat=1.2.8 - - pytables=3.10.1 - - python-calamine=0.3.0 - - pytz=2020.1 - - pyxlsb=1.0.10 - - s3fs=2024.10.0 - - scipy=1.14.1 - - sqlalchemy=2.0.36 - - tabulate=0.9.0 - - xarray=2024.10.0 - - xlrd=2.0.1 - - xlsxwriter=3.2.0 - - zstandard=0.23.0 - - - pip: - - tzdata==2023.3 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml deleted file mode 100644 index 70b87ba685eb3..0000000000000 --- a/ci/deps/actions-311.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: pandas-dev -channels: - - conda-forge -dependencies: - - python=3.11 - - # build dependencies - - versioneer - - cython<4.0.0a0 - - meson=1.10.0 - - meson-python=0.18.0 - - # test dependencies - - pytest>=8.3.4 - - pytest-cov - - pytest-xdist>=3.6.1 - - pytest-localserver>=0.9.0 - - pytest-qt>=4.4.0 - - boto3=1.40.46 - - # required dependencies - - python-dateutil - - numpy - - # optional dependencies - - adbc-driver-postgresql>=1.2.0 - - adbc-driver-sqlite>=1.2.0 - - beautifulsoup4>=4.12.3 - - bottleneck>=1.4.2 - - fastparquet>=2024.11.0 - - fsspec>=2024.10.0 - - html5lib>=1.1 - - hypothesis>=6.116.0 - - gcsfs>=2024.10.0 - - jinja2>=3.1.5 - - lxml>=5.3.0 - - matplotlib>=3.9.3 - - numba>=0.60.0 - - numexpr>=2.10.2 - - odfpy>=1.4.1 - - qtpy>=2.4.2 - - pyqt>=5.15.9 - - openpyxl>=3.1.5 - - psycopg2>=2.9.10 - - pyarrow>=13.0.0 - - pyiceberg>=0.8.1 - - pydantic<2.12.0 # TMP pin to avoid pyiceberg/pydantic issues - - pymysql>=1.1.1 - - pyreadstat>=1.2.8 - - pytables>=3.10.1 - - python-calamine>=0.3.0 - - pytz>=2020.1 - - pyxlsb>=1.0.10 - - s3fs>=2024.10.0 - - scipy>=1.14.1 - - sqlalchemy>=2.0.36 - - tabulate>=0.9.0 - - xarray>=2024.10.0 - - xlrd>=2.0.1 - - xlsxwriter>=3.2.0 - - zstandard>=0.23.0 - - - pip: - - tzdata>=2023.3 diff --git a/ci/deps/actions-312.yaml b/ci/deps/actions-312.yaml deleted file mode 100644 index eb71afd616eea..0000000000000 --- a/ci/deps/actions-312.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: pandas-dev-312 -channels: - - conda-forge -dependencies: - - python=3.12 - - # build dependencies - - versioneer - - cython<4.0.0a0 - - meson=1.10.0 - - meson-python=0.18.0 - - # test dependencies - - pytest>=8.3.4 - - pytest-cov - - pytest-xdist>=3.6.1 - - pytest-localserver>=0.9.0 - - pytest-qt>=4.4.0 - - boto3=1.40.46 - - # required dependencies - - python-dateutil - - numpy - - # optional dependencies - - adbc-driver-postgresql>=1.2.0 - - adbc-driver-sqlite>=1.2.0 - - beautifulsoup4>=4.12.3 - - bottleneck>=1.4.2 - - fastparquet>=2024.11.0 - - fsspec>=2024.10.0 - - html5lib>=1.1 - - hypothesis>=6.116.0 - - gcsfs>=2024.10.0 - - jinja2>=3.1.5 - - lxml>=5.3.0 - - matplotlib>=3.9.3 - - numba>=0.60.0 - - numexpr>=2.10.2 - - odfpy>=1.4.1 - - qtpy>=2.4.2 - - pyqt>=5.15.9 - - openpyxl>=3.1.5 - - psycopg2>=2.9.10 - - pyarrow>=13.0.0 - - pyiceberg>=0.8.1 - - pydantic<2.12.0 # TMP pin to avoid pyiceberg/pydantic issues - - pymysql>=1.1.1 - - pyreadstat>=1.2.8 - - pytables>=3.10.1 - - python-calamine>=0.3.0 - - pytz>=2020.1 - - pyxlsb>=1.0.10 - - s3fs>=2024.10.0 - - scipy>=1.14.1 - - sqlalchemy>=2.0.36 - - tabulate>=0.9.0 - - xarray>=2024.10.0 - - xlrd>=2.0.1 - - xlsxwriter>=3.2.0 - - zstandard>=0.23.0 - - - pip: - - tzdata>=2023.3 diff --git a/ci/deps/actions-313-downstream_compat.yaml b/ci/deps/actions-313-downstream_compat.yaml deleted file mode 100644 index 4cab43c0197d3..0000000000000 --- a/ci/deps/actions-313-downstream_compat.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# Non-dependencies that pandas utilizes or has compatibility with pandas objects -name: pandas-dev -channels: - - conda-forge -dependencies: - - python=3.13 - - # build dependencies - - versioneer - - cython<4.0.0a0 - - meson=1.10.0 - - meson-python=0.18.0 - - # test dependencies - - pytest>=8.3.4 - - pytest-cov - - pytest-xdist>=3.6.1 - - pytest-localserver>=0.9.0 - - pytest-qt>=4.4.0 - - boto3=1.40.46 - - # required dependencies - - python-dateutil - - numpy - - # optional dependencies - - adbc-driver-postgresql>=1.2.0 - - beautifulsoup4>=4.12.3 - - bottleneck>=1.4.2 - - fastparquet>=2024.11.0 - - fsspec>=2024.10.0 - - html5lib>=1.1 - - hypothesis>=6.116.0 - - gcsfs>=2024.10.0 - - jinja2>=3.1.5 - - lxml>=5.3.0 - - matplotlib>=3.9.3 - - numba>=0.60.0 - - numexpr>=2.10.2 - - odfpy>=1.4.1 - - qtpy>=2.4.2 - - openpyxl>=3.1.5 - - psycopg2>=2.9.10 - - pyarrow>=13.0.0 - - pyiceberg>=0.8.1 - - pymysql>=1.1.1 - - pyqt>=5.15.9 - - pyreadstat>=1.2.8 - - pytables>=3.10.1 - - python-calamine>=0.3.0 - - pytz>=2020.1 - - pyxlsb>=1.0.10 - - s3fs>=2024.10.0 - - scipy>=1.14.1 - - sqlalchemy>=2.0.36 - - tabulate>=0.9.0 - - xarray>=2024.10.0 - - xlrd>=2.0.1 - - xlsxwriter>=3.2.0 - - zstandard>=0.23.0 - - # downstream packages - - botocore - - cftime - - dask - - ipython - - seaborn - - scikit-learn - - statsmodels - - coverage - - pandas-datareader - - pyyaml - - pip: - - tzdata>=2023.3 diff --git a/ci/deps/actions-313-freethreading.yaml b/ci/deps/actions-313-freethreading.yaml deleted file mode 100644 index d6c68b1675762..0000000000000 --- a/ci/deps/actions-313-freethreading.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: pandas-dev-313-freethreading -channels: - - conda-forge -dependencies: - - python-freethreading - - # build dependencies - - versioneer - - cython<4.0.0a0 - - meson=1.10.0 - - meson-python=0.18.0 - - # test dependencies - - pytest>=8.3.4 - - pytest-xdist>=3.6.1 - - # required dependencies - - python-dateutil - - numpy - - # optional dependencies - - hypothesis>=6.116.0 - - - pip: - # No free-threaded coveragepy (with the C-extension) on conda-forge yet - - pytest-cov - - tzdata>=2023.3 - - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" diff --git a/ci/deps/actions-313-numpydev.yaml b/ci/deps/actions-313-numpydev.yaml deleted file mode 100644 index f4706973ac5d2..0000000000000 --- a/ci/deps/actions-313-numpydev.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: pandas-dev -channels: - - conda-forge -dependencies: - - python=3.13 - - # build dependencies - - versioneer - - meson=1.10.0 - - meson-python=0.18.0 - - cython<4.0.0a0 - - # test dependencies - - pytest>=8.3.4 - - pytest-cov - - pytest-xdist>=3.6.1 - - hypothesis>=6.116.0 - - # pandas dependencies - - python-dateutil - - pip - - - pip: - - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" - - "--pre" - - "numpy" - - "tzdata>=2023.3" diff --git a/ci/deps/actions-313-pyarrownightly.yaml b/ci/deps/actions-313-pyarrownightly.yaml deleted file mode 100644 index 104757cf7e4be..0000000000000 --- a/ci/deps/actions-313-pyarrownightly.yaml +++ /dev/null @@ -1,29 +0,0 @@ -name: pandas-dev -channels: - - conda-forge -dependencies: - - python=3.13 - - # build dependencies - - versioneer - - cython<4.0.0a0 - - meson=1.10.0 - - meson-python=0.18.0 - - # test dependencies - - pytest>=8.3.4 - - pytest-cov - - pytest-xdist>=3.6.1 - - hypothesis>=6.116.0 - - # required dependencies - - python-dateutil - - numpy - - pip - - - pip: - - "tzdata>=2023.3" - - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" - - "--prefer-binary" - - "--pre" - - "pyarrow" diff --git a/ci/deps/actions-313.yaml b/ci/deps/actions-313.yaml deleted file mode 100644 index d06bc6a2a1455..0000000000000 --- a/ci/deps/actions-313.yaml +++ /dev/null @@ -1,65 +0,0 @@ -name: pandas-dev-313 -channels: - - conda-forge -dependencies: - - python=3.13 - - # build dependencies - - versioneer - - cython<4.0.0a0 - - meson=1.10.0 - - meson-python=0.18.0 - - # test dependencies - - pytest>=8.3.4 - - pytest-cov - - pytest-xdist>=3.6.1 - - pytest-localserver>=0.9.0 - - pytest-qt>=4.4.0 - - boto3=1.40.46 - - # required dependencies - - python-dateutil - - numpy - - # optional dependencies - - adbc-driver-postgresql>=1.2.0 - - adbc-driver-sqlite>=1.2.0 - - beautifulsoup4>=4.12.3 - - blosc>=1.21.3 - - bottleneck>=1.4.2 - - fastparquet>=2024.11.0 - - fsspec>=2024.10.0 - - html5lib>=1.1 - - hypothesis>=6.116.0 - - gcsfs>=2024.10.0 - - jinja2>=3.1.5 - - lxml>=5.3.0 - - matplotlib>=3.9.3 - - numba>=0.60.0 - - numexpr>=2.10.2 - - odfpy>=1.4.1 - - qtpy>=2.4.2 - - pyqt>=5.15.9 - - openpyxl>=3.1.5 - - psycopg2>=2.9.10 - - pyarrow>=13.0.0 - - pydantic<2.12.0 # TMP pin to avoid pyiceberg/pydantic issues - - pyiceberg>=0.8.1 - - pymysql>=1.1.1 - - pyreadstat>=1.2.8 - - pytables>=3.10.1 - - python-calamine>=0.3.0 - - pytz>=2020.1 - - pyxlsb>=1.0.10 - - s3fs>=2024.10.0 - - scipy>=1.14.1 - - sqlalchemy>=2.0.36 - - tabulate>=0.9.0 - - xarray>=2024.10.0 - - xlrd>=2.0.1 - - xlsxwriter>=3.2.0 - - zstandard>=0.23.0 - - - pip: - - tzdata>=2023.3 diff --git a/ci/deps/actions-314.yaml b/ci/deps/actions-314.yaml deleted file mode 100644 index 260fc6a217e87..0000000000000 --- a/ci/deps/actions-314.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: pandas-dev-314 -channels: - - conda-forge -dependencies: - - python=3.14 - - # build dependencies - - versioneer - - cython<4.0.0a0 - - meson=1.10.0 - - meson-python=0.18.0 - - # test dependencies - - pytest>=8.3.4 - - pytest-cov - - pytest-xdist>=3.6.1 - - pytest-localserver>=0.9.0 - - pytest-qt>=4.4.0 - - boto3=1.40.46 - - # required dependencies - - python-dateutil - - numpy - - # optional dependencies - - adbc-driver-postgresql>=1.2.0 - - adbc-driver-sqlite>=1.2.0 - - beautifulsoup4>=4.12.3 - - blosc>=1.21.3 - - bottleneck>=1.4.2 - - fastparquet>=2024.11.0 - - fsspec>=2024.10.0 - - html5lib>=1.1 - - hypothesis>=6.116.0 - - gcsfs>=2024.10.0 - - jinja2>=3.1.5 - - lxml>=5.3.0 - - matplotlib>=3.9.3 - - numba>=0.60.0 - - numexpr>=2.10.2 - - odfpy>=1.4.1 - - qtpy>=2.4.2 - - pyqt>=5.15.9 - - openpyxl>=3.1.5 - - psycopg2>=2.9.10 - - pyarrow>=13.0.0 - - pymysql>=1.1.1 - - pyreadstat>=1.2.8 - - pytables>=3.10.1 - - python-calamine>=0.3.0 - - pytz>=2020.1 - - pyxlsb>=1.0.10 - - s3fs>=2024.10.0 - - scipy>=1.14.1 - - sqlalchemy>=2.0.36 - - tabulate>=0.9.0 - - xarray>=2024.10.0 - - xlrd>=2.0.1 - - xlsxwriter>=3.2.0 - - zstandard>=0.23.0 - - - pip: - - tzdata>=2023.3 diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index 87a413750656d..782293f541f99 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: import types -# Update install.rst, actions-311-minimum_versions.yaml, +# Update install.rst, pixi.toml, # deps_minimum.toml & pyproject.toml when updating versions! VERSIONS = { diff --git a/scripts/tests/test_validate_min_versions_in_sync.py b/scripts/tests/test_validate_min_versions_in_sync.py index 2455e1519dfbe..89eaff9a0e268 100644 --- a/scripts/tests/test_validate_min_versions_in_sync.py +++ b/scripts/tests/test_validate_min_versions_in_sync.py @@ -1,4 +1,5 @@ import pathlib +from textwrap import dedent import tomllib import pytest @@ -6,6 +7,7 @@ from scripts.validate_min_versions_in_sync import ( get_toml_map_from, + get_versions_from_ci, get_yaml_map_from, pin_min_versions_to_yaml_file, ) @@ -56,3 +58,63 @@ def test_pin_min_versions_to_yaml_file(src_toml, src_yaml, expected_yaml) -> Non with open(expected_yaml, encoding="utf-8") as yaml_f: dummy_yaml_expected_file_1 = yaml_f.read() assert result_yaml_file == dummy_yaml_expected_file_1 + + +def test_get_versions_from_ci_parses_pixi_toml() -> None: + content = dedent( + """ + [dependencies] + # Build dependencies + meson = ">=1.2.3,<2" + pip = ">=26" + + # Required dependencies + python-dateutil = ">=2.8.2" + + [feature.numpy.dependencies] + numpy = ">=1.26.0,<3" + + [feature.numpy21.dependencies] + numpy = "2.1.*" + + [feature.test-base.dependencies] + pytest = ">=8.3.4" + pytest-xdist = ">=3.6.1" + hypothesis = ">=6.116.0" + + [feature.test-network.dependencies] + boto3 = "==1.40.46" + + [feature.test-clipboard.dependencies] + pyqt = ">=5.15.9" + qtpy = ">=2.4.2" + + [feature.pyarrow.dependencies] + pyarrow = ">=13.0.0" + + [feature.pyarrow21.dependencies] + pyarrow = "21.*" + + [feature.optional-dependencies.dependencies] + beautifulsoup4 = ">=4.12.3" + blosc = ">=1.21.3" + pytables = ">=3.10.1" + zstandard = ">=0.23.0" + """ + ).splitlines() + + required, optional = get_versions_from_ci(content) + + assert required == { + "numpy": "1.26.0", + "python-dateutil": "2.8.2", + } + assert optional == { + "beautifulsoup4": "4.12.3", + "hypothesis": "6.116.0", + "pyarrow": "13.0.0", + "pytest": "8.3.4", + "pytables": "3.10.1", + "qtpy": "2.4.2", + "zstandard": "0.23.0", + } diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index f274486868652..0f37972490f9b 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -2,9 +2,10 @@ """ Check pandas required and optional dependencies are synced across: -ci/deps/actions-.*-minimum_versions.yaml +pixi.toml pandas/compat/_optional.py -setup.cfg +pyproject.toml +environment.yml TODO: doc/source/getting_started/install.rst @@ -26,14 +27,16 @@ BASE_PATH = pathlib.Path(__file__).parents[1] DOC_PATH = (BASE_PATH / "doc/source/getting_started/install.rst").resolve() -CI_PATH = next( - (BASE_PATH / "ci/deps").absolute().glob("actions-*-minimum_versions.yaml") -) +PIXI_PATH = (BASE_PATH / "pixi.toml").resolve() CODE_PATH = (BASE_PATH / "pandas/compat/_optional.py").resolve() -SETUP_PATH = (BASE_PATH / "pyproject.toml").resolve() -YAML_PATH = BASE_PATH / "ci/deps" +PYPROJECT_PATH = (BASE_PATH / "pyproject.toml").resolve() ENV_PATH = BASE_PATH / "environment.yml" -EXCLUDE_DEPS = {"tzdata", "pyqt", "pyqt5"} +EXCLUDE_DEPS = {"blosc", "tzdata", "pyqt", "pyqt5"} +ADDITIONAL_PIXI_OPTIONAL_DEPENDENCIES = { + ("feature", "pyarrow", "dependencies"): None, + ("feature", "test-base", "dependencies"): {"hypothesis", "pytest"}, + ("feature", "test-clipboard", "dependencies"): {"qtpy"}, +} # pandas package is not available # in pre-commit environment sys.path.append(str(BASE_PATH / "pandas/compat")) @@ -46,31 +49,28 @@ import _optional -def pin_min_versions_to_ci_deps() -> int: +def pin_min_versions_to_environment_yml() -> int: """ - Pin minimum versions to CI dependencies. + Pin minimum versions to environment.yml dependencies. Pip dependencies are not pinned. """ - all_yaml_files = list(YAML_PATH.iterdir()) - all_yaml_files.append(ENV_PATH) toml_dependencies = {} - with open(SETUP_PATH, "rb") as toml_f: + with open(PYPROJECT_PATH, "rb") as toml_f: toml_dependencies = tomllib.load(toml_f) ret = 0 - for curr_file in all_yaml_files: - with open(curr_file, encoding="utf-8") as yaml_f: - yaml_start_data = yaml_f.read() - yaml_file = yaml.safe_load(yaml_start_data) - yaml_dependencies = yaml_file["dependencies"] - yaml_map = get_yaml_map_from(yaml_dependencies) - toml_map = get_toml_map_from(toml_dependencies) - yaml_result_data = pin_min_versions_to_yaml_file( - yaml_map, toml_map, yaml_start_data - ) - if yaml_result_data != yaml_start_data: - with open(curr_file, "w", encoding="utf-8") as f: - f.write(yaml_result_data) + with open(ENV_PATH, encoding="utf-8") as yaml_f: + yaml_start_data = yaml_f.read() + yaml_file = yaml.safe_load(yaml_start_data) + yaml_dependencies = yaml_file["dependencies"] + yaml_map = get_yaml_map_from(yaml_dependencies) + toml_map = get_toml_map_from(toml_dependencies) + yaml_result_data = pin_min_versions_to_yaml_file( + yaml_map, toml_map, yaml_start_data + ) + if yaml_result_data != yaml_start_data: + with open(ENV_PATH, "w", encoding="utf-8") as f: + f.write(yaml_result_data) ret |= 1 return ret @@ -191,47 +191,99 @@ def get_versions_from_code() -> dict[str, str]: def get_versions_from_ci(content: list[str]) -> tuple[dict[str, str], dict[str, str]]: - """Min versions in CI job for testing all optional dependencies.""" - # Don't parse with pyyaml because it ignores comments we're looking for - seen_required = False - seen_optional = False - seen_test = False + """Min versions in pixi.toml for testing all optional dependencies.""" + separator = "" if any(line.endswith(("\n", "\r")) for line in content) else "\n" + pixi_toml = tomllib.loads(separator.join(content)) + install_map = _optional.INSTALL_MAPPING + required_deps = _get_required_dependencies_from_pixi_content(content, pixi_toml) + optional_deps = _get_dependencies_from_pixi_table( + pixi_toml["feature"]["optional-dependencies"]["dependencies"], + install_map, + ) + required_deps.update( + _get_dependencies_from_pixi_table( + pixi_toml["feature"]["numpy"]["dependencies"], + install_map, + ) + ) + for keys, packages in ADDITIONAL_PIXI_OPTIONAL_DEPENDENCIES.items(): + dependencies = pixi_toml + for key in keys: + dependencies = dependencies[key] + optional_deps.update( + _get_dependencies_from_pixi_table(dependencies, install_map, packages) + ) + return required_deps, optional_deps + + +def _get_required_dependencies_from_pixi_content( + content: list[str], pixi_toml: dict[str, Any] +) -> dict[str, str]: + # pixi.toml keeps build and required deps in the same TOML table. required_deps = {} - optional_deps = {} + in_dependencies = False + in_required_dependencies = False for line in content: - if "# test dependencies" in line: - seen_test = True - elif seen_test and "- pytest>=" in line: - # Only grab pytest - package, version = line.strip().split(">=") - package = package[2:] - optional_deps[package.casefold()] = version - elif "# required dependencies" in line: - seen_required = True - elif "# optional dependencies" in line: - seen_optional = True - elif "- pip:" in line: + line = line.strip() + if not line: continue - elif seen_required and line.strip(): - if "==" in line: - package, version = line.strip().split("==", maxsplit=1) - else: - package, version = line.strip().split("=", maxsplit=1) - package = package.split()[-1] - if package in EXCLUDE_DEPS: - continue - if not seen_optional: - required_deps[package.casefold()] = version - else: - optional_deps[package.casefold()] = version - return required_deps, optional_deps + if line.startswith("["): + in_dependencies = line == "[dependencies]" + in_required_dependencies = False + continue + if not in_dependencies: + continue + if line.casefold() == "# required dependencies": + in_required_dependencies = True + continue + if line.startswith("#"): + in_required_dependencies = False + continue + if not in_required_dependencies: + continue + package = line.split("=", maxsplit=1)[0].strip() + version = _get_version_from_pixi_spec(pixi_toml["dependencies"][package]) + if version is not None and package not in EXCLUDE_DEPS: + required_deps[package.casefold()] = version + return required_deps + + +def _get_dependencies_from_pixi_table( + dependencies: dict[str, str | dict[str, Any]], + install_map: dict[str, str], + packages: set[str] | None = None, +) -> dict[str, str]: + versions = {} + for package, spec in dependencies.items(): + package = install_map.get(package, package).casefold() + if package in EXCLUDE_DEPS or ( + packages is not None and package not in packages + ): + continue + version = _get_version_from_pixi_spec(spec) + if version is not None: + versions[package] = version + return versions + + +def _get_version_from_pixi_spec(spec: str | dict[str, Any]) -> str | None: + if isinstance(spec, dict): + spec = spec.get("version", "") + if spec in {"", "*"}: + return None + for operator in ("==", ">=", "="): + if spec.startswith(operator): + version = spec.removeprefix(operator).split(",", maxsplit=1)[0] + if "*" not in version: + return version + return None def get_versions_from_toml() -> dict[str, str]: """Min versions in pyproject.toml for pip install pandas[extra].""" install_map = _optional.INSTALL_MAPPING optional_dependencies = {} - with open(SETUP_PATH, "rb") as pyproject_f: + with open(PYPROJECT_PATH, "rb") as pyproject_f: pyproject_toml = tomllib.load(pyproject_f) opt_deps = pyproject_toml["project"]["optional-dependencies"] dependencies = set(opt_deps["all"]) @@ -251,31 +303,31 @@ def get_versions_from_toml() -> dict[str, str]: def main() -> int: ret = 0 - ret |= pin_min_versions_to_ci_deps() - with open(CI_PATH, encoding="utf-8") as f: + ret |= pin_min_versions_to_environment_yml() + with open(PIXI_PATH, encoding="utf-8") as f: _, ci_optional = get_versions_from_ci(f.readlines()) code_optional = get_versions_from_code() - setup_optional = get_versions_from_toml() + pyproject_optional = get_versions_from_toml() - diff = (ci_optional.items() | code_optional.items() | setup_optional.items()) - ( - ci_optional.items() & code_optional.items() & setup_optional.items() - ) + diff = ( + ci_optional.items() | code_optional.items() | pyproject_optional.items() + ) - (ci_optional.items() & code_optional.items() & pyproject_optional.items()) if diff: packages = {package for package, _ in diff} out = sys.stdout out.write( f"The follow minimum version differences were found between " - f"{CI_PATH}, {CODE_PATH} AND {SETUP_PATH}. " + f"{PIXI_PATH}, {CODE_PATH} AND {PYPROJECT_PATH}. " f"Please ensure these are aligned: \n\n" ) for package in packages: out.write( f"{package}\n" - f"{CI_PATH}: {ci_optional.get(package, 'Not specified')}\n" + f"{PIXI_PATH}: {ci_optional.get(package, 'Not specified')}\n" f"{CODE_PATH}: {code_optional.get(package, 'Not specified')}\n" - f"{SETUP_PATH}: {setup_optional.get(package, 'Not specified')}\n\n" + f"{PYPROJECT_PATH}: {pyproject_optional.get(package, 'Not specified')}\n\n" # noqa: E501 ) ret |= 1 return ret