Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env python3
- from __future__ import annotations
- import itertools
- import requests
- from packaging.metadata import Metadata
- from packaging.utils import (
- InvalidWheelFilename,
- canonicalize_name,
- parse_wheel_filename,
- )
- PACKAGES = [
- "boto3",
- "botocore",
- "urllib3",
- "requests",
- "wheel",
- "certifi",
- "typing-extensions",
- "charset-normalizer",
- "idna",
- "setuptools",
- "pip",
- "python-dateutil",
- "s3transfer",
- "aiobotocore",
- "packaging",
- "six",
- "pyyaml",
- "s3fs",
- "fsspec",
- "numpy",
- "cryptography",
- "google-api-core",
- "grpcio-status",
- "importlib-metadata",
- "cffi",
- "zipp",
- "pycparser",
- "pandas",
- "attrs",
- "protobuf",
- "rsa",
- "pyasn1",
- "jmespath",
- "click",
- "pytz",
- "awscli",
- "pydantic",
- "colorama",
- "markupsafe",
- "jinja2",
- "platformdirs",
- "pyjwt",
- "tomli",
- "googleapis-common-protos",
- "google-auth",
- "cachetools",
- "filelock",
- "wrapt",
- "jsonschema",
- "virtualenv",
- "pluggy",
- "pyparsing",
- "werkzeug",
- "pyarrow",
- "docutils",
- "sqlalchemy",
- "pytest",
- "exceptiongroup",
- "flask",
- "aiohttp",
- "requests-oauthlib",
- "pyasn1-modules",
- "oauthlib",
- "isodate",
- "multidict",
- "scipy",
- "psutil",
- "yarl",
- "async-timeout",
- "soupsieve",
- "iniconfig",
- "frozenlist",
- "beautifulsoup4",
- "aiosignal",
- "grpcio",
- "greenlet",
- "tqdm",
- "pillow",
- "pygments",
- "decorator",
- "importlib-resources",
- "lxml",
- "pyopenssl",
- "requests-toolbelt",
- "openpyxl",
- "et-xmlfile",
- "azure-core",
- "tzdata",
- "asn1crypto",
- "distlib",
- "pydantic-core",
- "coverage",
- "tomlkit",
- "sniffio",
- "more-itertools",
- "pynacl",
- "pexpect",
- "h11",
- "ptyprocess",
- "google-cloud-storage",
- ]
- def check_consistency(package: str) -> None:
- name = canonicalize_name(package)
- metadata_info: dict[str, Metadata | None] = {}
- headers = {"Accept": "application/vnd.pypi.simple.v1+json"}
- rsp = requests.get(f"https://pypi.org/simple/{name}/", headers=headers, timeout=5)
- if not rsp.ok:
- print(f"Failed to get details for {name}: {rsp.text}")
- rsp.raise_for_status()
- info = rsp.json()
- version = info["versions"][-1]
- # Gather up the different metadata hashes referenced by the wheels.
- files = []
- for file in info["files"]:
- filename = file["filename"]
- if not filename.endswith(".whl"):
- continue
- try:
- parsed = parse_wheel_filename(filename)
- except InvalidWheelFilename:
- print(f"Failed to parse wheel filename {filename}")
- continue
- if str(parsed[1]) != version:
- continue
- files.append(file)
- metadata_hash = file["core-metadata"]["sha256"]
- metadata_info[metadata_hash] = None
- # If all metadata hashes are the same, we're done.
- count = len(metadata_info)
- if count < 2:
- print(f"Only {count} metadata hash found for {name} {version}")
- return
- print(f"{count} metadata hashes found for {name} {version}")
- # Download the different metadata files, one for each of the hashes.
- for file in files:
- metadata_hash = file["core-metadata"]["sha256"]
- if metadata_info[metadata_hash] is not None:
- continue
- url = file["url"] + ".metadata"
- rsp = requests.get(url, timeout=5)
- if not rsp.ok:
- print(f"Failed to get metadata from {url}: {rsp.text}")
- rsp.raise_for_status()
- metadata = Metadata.from_email(rsp.content, validate=False)
- metadata_info[metadata_hash] = metadata
- # Metadata class is a bit awkward to work with, make manual checks on the
- # interesting fields.
- inconsistencies = False
- for m1, m2 in itertools.combinations(metadata_info.values(), 2):
- assert m1 is not None
- assert m2 is not None
- if m1.requires_python != m2.requires_python:
- print("Inconsistent requires_python")
- inconsistencies = True
- m1reqs = (
- None
- if m1.requires_dist is None
- else sorted(str(r) for r in m1.requires_dist)
- )
- m2reqs = (
- None
- if m2.requires_dist is None
- else sorted(str(r) for r in m2.requires_dist)
- )
- if m1reqs != m2reqs:
- print("Inconsistent requires_dist")
- inconsistencies = True
- if not inconsistencies:
- print("No interesting inconsistencies found")
- if __name__ == "__main__":
- for package in PACKAGES:
- check_consistency(package)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement