Advertisement
Guest User

Untitled

a guest
Apr 1st, 2024
185
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 4.85 KB | None | 0 0
  1. #!/usr/bin/env python3
  2. from __future__ import annotations
  3.  
  4. import itertools
  5.  
  6. import requests
  7. from packaging.metadata import Metadata
  8. from packaging.utils import (
  9.     InvalidWheelFilename,
  10.     canonicalize_name,
  11.     parse_wheel_filename,
  12. )
  13.  
  14. PACKAGES = [
  15.     "boto3",
  16.     "botocore",
  17.     "urllib3",
  18.     "requests",
  19.     "wheel",
  20.     "certifi",
  21.     "typing-extensions",
  22.     "charset-normalizer",
  23.     "idna",
  24.     "setuptools",
  25.     "pip",
  26.     "python-dateutil",
  27.     "s3transfer",
  28.     "aiobotocore",
  29.     "packaging",
  30.     "six",
  31.     "pyyaml",
  32.     "s3fs",
  33.     "fsspec",
  34.     "numpy",
  35.     "cryptography",
  36.     "google-api-core",
  37.     "grpcio-status",
  38.     "importlib-metadata",
  39.     "cffi",
  40.     "zipp",
  41.     "pycparser",
  42.     "pandas",
  43.     "attrs",
  44.     "protobuf",
  45.     "rsa",
  46.     "pyasn1",
  47.     "jmespath",
  48.     "click",
  49.     "pytz",
  50.     "awscli",
  51.     "pydantic",
  52.     "colorama",
  53.     "markupsafe",
  54.     "jinja2",
  55.     "platformdirs",
  56.     "pyjwt",
  57.     "tomli",
  58.     "googleapis-common-protos",
  59.     "google-auth",
  60.     "cachetools",
  61.     "filelock",
  62.     "wrapt",
  63.     "jsonschema",
  64.     "virtualenv",
  65.     "pluggy",
  66.     "pyparsing",
  67.     "werkzeug",
  68.     "pyarrow",
  69.     "docutils",
  70.     "sqlalchemy",
  71.     "pytest",
  72.     "exceptiongroup",
  73.     "flask",
  74.     "aiohttp",
  75.     "requests-oauthlib",
  76.     "pyasn1-modules",
  77.     "oauthlib",
  78.     "isodate",
  79.     "multidict",
  80.     "scipy",
  81.     "psutil",
  82.     "yarl",
  83.     "async-timeout",
  84.     "soupsieve",
  85.     "iniconfig",
  86.     "frozenlist",
  87.     "beautifulsoup4",
  88.     "aiosignal",
  89.     "grpcio",
  90.     "greenlet",
  91.     "tqdm",
  92.     "pillow",
  93.     "pygments",
  94.     "decorator",
  95.     "importlib-resources",
  96.     "lxml",
  97.     "pyopenssl",
  98.     "requests-toolbelt",
  99.     "openpyxl",
  100.     "et-xmlfile",
  101.     "azure-core",
  102.     "tzdata",
  103.     "asn1crypto",
  104.     "distlib",
  105.     "pydantic-core",
  106.     "coverage",
  107.     "tomlkit",
  108.     "sniffio",
  109.     "more-itertools",
  110.     "pynacl",
  111.     "pexpect",
  112.     "h11",
  113.     "ptyprocess",
  114.     "google-cloud-storage",
  115. ]
  116.  
  117.  
  118. def check_consistency(package: str) -> None:
  119.     name = canonicalize_name(package)
  120.     metadata_info: dict[str, Metadata | None] = {}
  121.     headers = {"Accept": "application/vnd.pypi.simple.v1+json"}
  122.     rsp = requests.get(f"https://pypi.org/simple/{name}/", headers=headers, timeout=5)
  123.  
  124.     if not rsp.ok:
  125.         print(f"Failed to get details for {name}: {rsp.text}")
  126.     rsp.raise_for_status()
  127.  
  128.     info = rsp.json()
  129.     version = info["versions"][-1]
  130.  
  131.     # Gather up the different metadata hashes referenced by the wheels.
  132.     files = []
  133.     for file in info["files"]:
  134.         filename = file["filename"]
  135.         if not filename.endswith(".whl"):
  136.             continue
  137.  
  138.         try:
  139.             parsed = parse_wheel_filename(filename)
  140.         except InvalidWheelFilename:
  141.             print(f"Failed to parse wheel filename {filename}")
  142.             continue
  143.  
  144.         if str(parsed[1]) != version:
  145.             continue
  146.  
  147.         files.append(file)
  148.  
  149.         metadata_hash = file["core-metadata"]["sha256"]
  150.         metadata_info[metadata_hash] = None
  151.  
  152.     # If all metadata hashes are the same, we're done.
  153.     count = len(metadata_info)
  154.     if count < 2:
  155.         print(f"Only {count} metadata hash found for {name} {version}")
  156.         return
  157.  
  158.     print(f"{count} metadata hashes found for {name} {version}")
  159.  
  160.     # Download the different metadata files, one for each of the hashes.
  161.     for file in files:
  162.         metadata_hash = file["core-metadata"]["sha256"]
  163.         if metadata_info[metadata_hash] is not None:
  164.             continue
  165.  
  166.         url = file["url"] + ".metadata"
  167.         rsp = requests.get(url, timeout=5)
  168.  
  169.         if not rsp.ok:
  170.             print(f"Failed to get metadata from {url}: {rsp.text}")
  171.         rsp.raise_for_status()
  172.  
  173.         metadata = Metadata.from_email(rsp.content, validate=False)
  174.         metadata_info[metadata_hash] = metadata
  175.  
  176.     # Metadata class is a bit awkward to work with, make manual checks on the
  177.     # interesting fields.
  178.     inconsistencies = False
  179.     for m1, m2 in itertools.combinations(metadata_info.values(), 2):
  180.         assert m1 is not None
  181.         assert m2 is not None
  182.  
  183.         if m1.requires_python != m2.requires_python:
  184.             print("Inconsistent requires_python")
  185.             inconsistencies = True
  186.  
  187.         m1reqs = (
  188.             None
  189.             if m1.requires_dist is None
  190.             else sorted(str(r) for r in m1.requires_dist)
  191.         )
  192.         m2reqs = (
  193.             None
  194.             if m2.requires_dist is None
  195.             else sorted(str(r) for r in m2.requires_dist)
  196.         )
  197.         if m1reqs != m2reqs:
  198.             print("Inconsistent requires_dist")
  199.             inconsistencies = True
  200.  
  201.     if not inconsistencies:
  202.         print("No interesting inconsistencies found")
  203.  
  204.  
  205. if __name__ == "__main__":
  206.     for package in PACKAGES:
  207.         check_consistency(package)
  208.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement