API tools faq
paste
Advertisement
cooperlees

bandersnatch verify

cooperlees
May 12th, 2018
617
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Diff 14.65 KB | None | 0 0
raw download clone embed print report
  1. changeset:   0f1cf1154488f068fa4a8bdf36453bc5f36a94c0   (@)
  2. branch:      json_save
  3. parent:      501:4a65a57600e9
  4. user:        Cooper Lees <me@cooperlees.com>
  5. date:        Sat, 12 May 2018 15:31:47 -0400
  6.  
  7.     Add verify functionality
  8.  
  9. diff -r 4a65a57600e9 -r 0f1cf1154488 src/bandersnatch/main.py
  10. --- a/src/bandersnatch/main.py  Thu Aug 10 11:53:37 2017 +0800
  11. +++ b/src/bandersnatch/main.py  Sat May 12 15:31:47 2018 -0400
  12. @@ -61,8 +61,15 @@
  13.      p = subparsers.add_parser(
  14.          'mirror',
  15.          help='Performs a one-time synchronization with '
  16. -             'the PyPI master server.')
  17. +             'the PyPI master server.',
  18. +    )
  19.      p.set_defaults(func=mirror)
  20. +    # `verify` command
  21. +    q = subparsers.add_parser(
  22. +        'verify',
  23. +        help='Read in Metadata and check package file validity',
  24. +    )
  25. +    q.set_defaults(func=bandersnatch.utils.metadata_verify)
  26.  
  27.      args = parser.parse_args()
  28.  
  29. @@ -86,4 +93,4 @@
  30.          logging.config.fileConfig(
  31.              os.path.expanduser(config.get('mirror', 'log-config'))
  32.          )
  33. -    args.func(config)
  34. +    return args.func(config)
  35. diff -r 4a65a57600e9 -r 0f1cf1154488 src/bandersnatch/tests/json/peerme
  36. --- /dev/null   Thu Jan 01 00:00:00 1970 +0000
  37. +++ b/src/bandersnatch/tests/json/peerme    Sat May 12 15:31:47 2018 -0400
  38. @@ -0,0 +1,1 @@
  39. +{"info":{"author":"Cooper Lees","author_email":"me@cooperlees.com","bugtrack_url":null,"classifiers":["Development Status :: 3 - Alpha","Intended Audience :: System Administrators","License :: OSI Approved :: BSD License","Programming Language :: Python :: 3 :: Only","Topic :: Internet"],"description":"UNKNOWN\n\n\n","description_content_type":null,"docs_url":null,"download_url":"","downloads":{"last_day":-1,"last_month":-1,"last_week":-1},"home_page":"http://github.com/cooperlees/peerme","keywords":"","license":"BSD 2-Clause","maintainer":"","maintainer_email":"","name":"peerme","package_url":"https://pypi.org/project/peerme/","platform":"","project_url":"https://pypi.org/project/peerme/","release_url":"https://pypi.org/project/peerme/1.0.1/","requires_dist":["aiohttp","aiomysql (>=0.0.9)","click (>=5.0)","jinja2"],"requires_python":"","summary":"IX Peering DB based config discovery and generation tool","version":"1.0.1"},"last_serial":2595915,"releases":{"1.0.0":[{"comment_text":"","digests":{"md5":"58b2c358d452d32bc75437a72a8cd54a","sha256":"bc9430dae93f8bc53728773545cbb646a6b5327f98de31bdd6e1a2b2c6e805a9"},"downloads":-1,"filename":"peerme-1.0.0-py36-none-any.whl","has_sig":false,"md5_digest":"58b2c358d452d32bc75437a72a8cd54a","packagetype":"bdist_wheel","python_version":"py36","size":16911,"upload_time":"2017-01-24T16:35:11","url":"https://files.pythonhosted.org/packages/8f/1a/1aa000db9c5a799b676227e845d2b64fe725328e05e3d3b30036f50eb316/peerme-1.0.0-py36-none-any.whl"},{"comment_text":"","digests":{"md5":"0c8191a999503ee03f566ce45c3ef663","sha256":"08b8525eae7341b452005c825faffe863d6a66baf6191b022b36317fac07824e"},"downloads":-1,"filename":"peerme-1.0.0.tar.gz","has_sig":false,"md5_digest":"0c8191a999503ee03f566ce45c3ef663","packagetype":"sdist","python_version":"source","size":11616,"upload_time":"2017-01-24T16:22:03","url":"https://files.pythonhosted.org/packages/61/69/88dc7eb55a561b73bd2a88ae4eda7ad07beaa9ef18be744221e8738b9a00/peerme-1.0.0.tar.gz"}],"1.0.1":[{"comment_text":"","digests":{"md5":"4d444e5f10ac98e98b19e9aad9091a33","sha256":"b6fb0c72b1aaa16fbca1b9416da2145c1529a09be61f8c040031b02480cc3484"},"downloads":-1,"filename":"peerme-1.0.1-py36-none-any.whl","has_sig":false,"md5_digest":"4d444e5f10ac98e98b19e9aad9091a33","packagetype":"bdist_wheel","python_version":"py36","size":17685,"upload_time":"2017-01-24T19:30:37","url":"https://files.pythonhosted.org/packages/7b/51/55e4a98abf34f0377fbddfc824dd9f35349a2e2708b8828c954a78beab2c/peerme-1.0.1-py36-none-any.whl"},{"comment_text":"","digests":{"md5":"2f0d9268b2daaf4cb8c10c49b20ba2b8","sha256":"5b187e2f670a654b189b3b2b7f693d445276eeb53c702302a863f151219b7d05"},"downloads":-1,"filename":"peerme-1.0.1.tar.gz","has_sig":false,"md5_digest":"2f0d9268b2daaf4cb8c10c49b20ba2b8","packagetype":"sdist","python_version":"source","size":12134,"upload_time":"2017-01-24T19:30:38","url":"https://files.pythonhosted.org/packages/84/98/be4118af6c1bd8504e82067fa1c572077ced51be96e5b7a1d71173f51c0d/peerme-1.0.1.tar.gz"}]},"urls":[{"comment_text":"","digests":{"md5":"4d444e5f10ac98e98b19e9aad9091a33","sha256":"b6fb0c72b1aaa16fbca1b9416da2145c1529a09be61f8c040031b02480cc3484"},"downloads":-1,"filename":"peerme-1.0.1-py36-none-any.whl","has_sig":false,"md5_digest":"4d444e5f10ac98e98b19e9aad9091a33","packagetype":"bdist_wheel","python_version":"py36","size":17685,"upload_time":"2017-01-24T19:30:37","url":"https://files.pythonhosted.org/packages/7b/51/55e4a98abf34f0377fbddfc824dd9f35349a2e2708b8828c954a78beab2c/peerme-1.0.1-py36-none-any.whl"},{"comment_text":"","digests":{"md5":"2f0d9268b2daaf4cb8c10c49b20ba2b8","sha256":"5b187e2f670a654b189b3b2b7f693d445276eeb53c702302a863f151219b7d05"},"downloads":-1,"filename":"peerme-1.0.1.tar.gz","has_sig":false,"md5_digest":"2f0d9268b2daaf4cb8c10c49b20ba2b8","packagetype":"sdist","python_version":"source","size":12134,"upload_time":"2017-01-24T19:30:38","url":"https://files.pythonhosted.org/packages/84/98/be4118af6c1bd8504e82067fa1c572077ced51be96e5b7a1d71173f51c0d/peerme-1.0.1.tar.gz"}]}
  40. \ No newline at end of file
  41. diff -r 4a65a57600e9 -r 0f1cf1154488 src/bandersnatch/tests/json/pyaib
  42. --- /dev/null   Thu Jan 01 00:00:00 1970 +0000
  43. +++ b/src/bandersnatch/tests/json/pyaib Sat May 12 15:31:47 2018 -0400
  44. @@ -0,0 +1,1 @@
  45. +{"info":{"author":"Jason Fried, Facebook","author_email":"fried@fb.com","bugtrack_url":null,"classifiers":["Development Status :: 5 - Production/Stable","Intended Audience :: Developers","License :: OSI Approved :: Apache Software License","Programming Language :: Python :: 2.7","Programming Language :: Python :: 3.5","Topic :: Communications :: Chat :: Internet Relay Chat"],"description":"","description_content_type":null,"docs_url":null,"download_url":"","downloads":{"last_day":-1,"last_month":-1,"last_week":-1},"home_page":"http://github.com/facebook/pyaib","keywords":"","license":"Apache 2.0","maintainer":"","maintainer_email":"","name":"pyaib","package_url":"https://pypi.org/project/pyaib/","platform":"UNKNOWN","project_url":"https://pypi.org/project/pyaib/","release_url":"https://pypi.org/project/pyaib/2.1.0/","requires_dist":null,"requires_python":"","summary":"Python Framework for writing IRC Bots using gevent","version":"2.1.0"},"last_serial":2328239,"releases":{"1.0.2":[{"comment_text":"","digests":{"md5":"ccccfec8fe249ea4156ea57afbd29950","sha256":"b7ccd6d82dc47f6de915582e8a2469758dc108f0672a05315abe11252d2e454d"},"downloads":-1,"filename":"pyaib-1.0.2.tar.gz","has_sig":false,"md5_digest":"ccccfec8fe249ea4156ea57afbd29950","packagetype":"sdist","python_version":"source","size":20796,"upload_time":"2013-11-27T01:12:09","url":"https://files.pythonhosted.org/packages/cb/58/5670011a2715486a87a16cbcd7e6acd6c0b1e41a1b879f5f47503cfcab5b/pyaib-1.0.2.tar.gz"}],"1.0.3":[{"comment_text":"","digests":{"md5":"28f47df2e3dc671634943ba17e9fbbbf","sha256":"7370f4d07e685d302f588ec8e8ae9b272935c5a82434712cad67eace11ef901a"},"downloads":-1,"filename":"pyaib-1.0.3.tar.gz","has_sig":false,"md5_digest":"28f47df2e3dc671634943ba17e9fbbbf","packagetype":"sdist","python_version":"source","size":20818,"upload_time":"2013-11-30T04:09:04","url":"https://files.pythonhosted.org/packages/22/b9/4e2a34a3474403d96c558d2328fe036338a9a0686cedd02c95e36f205934/pyaib-1.0.3.tar.gz"}],"1.1.0":[{"comment_text":"","digests":{"md5":"21a72cf8b9a20d026c6a348fc7c8dc36","sha256":"21ce1a9ba915d41aa26b7d29da34f0177177174a257ede9855699f736a812420"},"downloads":-1,"filename":"pyaib-1.1.0.tar.gz","has_sig":false,"md5_digest":"21a72cf8b9a20d026c6a348fc7c8dc36","packagetype":"sdist","python_version":"source","size":23257,"upload_time":"2014-06-19T17:37:19","url":"https://files.pythonhosted.org/packages/9d/8a/0246f64f48422f58a6b51248a4d5c1b597dc64da4b585e5ee885c5ad40b9/pyaib-1.1.0.tar.gz"}],"2.0.0":[{"comment_text":"","digests":{"md5":"ca2a1fd4a58547aa77456e2d12696ac2","sha256":"13e9482feb79f23933a8028dfd188d3e3be453d7fd061913edf5641bf5f3c8fb"},"downloads":-1,"filename":"pyaib-2.0.0.tar.gz","has_sig":true,"md5_digest":"ca2a1fd4a58547aa77456e2d12696ac2","packagetype":"sdist","python_version":"source","size":23418,"upload_time":"2016-05-24T21:44:40","url":"https://files.pythonhosted.org/packages/14/29/38d90c491f87a8ec02364806c8fd0fe9176e8c172a8c25a98c7d0b62efe9/pyaib-2.0.0.tar.gz"}],"2.0.1":[{"comment_text":"","digests":{"md5":"7053071243b6ca79be4ff13cbafcdda0","sha256":"9a0ca13bc5b3929c1cc590b2774eb541f056c885bdb6db19fddd2c1a12908880"},"downloads":-1,"filename":"pyaib-2.0.1.tar.gz","has_sig":true,"md5_digest":"7053071243b6ca79be4ff13cbafcdda0","packagetype":"sdist","python_version":"source","size":23501,"upload_time":"2016-05-25T19:51:55","url":"https://files.pythonhosted.org/packages/94/89/9dbc5a885100c3982cd2c663d96e20eae3d7698e2510b10431fcf251abd8/pyaib-2.0.1.tar.gz"}],"2.0.2":[{"comment_text":"","digests":{"md5":"5ecb7da92046d865f05e64fee49c5b4b","sha256":"d636924e7281af45b551c0e6e0b05f01f8fc769c8a1e5d69589ed2079847c6c7"},"downloads":-1,"filename":"pyaib-2.0.2.tar.gz","has_sig":true,"md5_digest":"5ecb7da92046d865f05e64fee49c5b4b","packagetype":"sdist","python_version":"source","size":23494,"upload_time":"2016-05-25T20:25:25","url":"https://files.pythonhosted.org/packages/ff/12/4059411d182368f2600a0e7ebadc9e59b9e039e30d95e157bb90346190df/pyaib-2.0.2.tar.gz"}],"2.1.0":[{"comment_text":"","digests":{"md5":"5a348b49d53cee26925e7204632721b7","sha256":"b6114554fb312f9b0bdeaf6a7498f7da05fc17b9250c0449ed796fac9ab663e2"},"downloads":-1,"filename":"pyaib-2.1.0.tar.gz","has_sig":true,"md5_digest":"5a348b49d53cee26925e7204632721b7","packagetype":"sdist","python_version":"source","size":23554,"upload_time":"2016-09-06T21:40:56","url":"https://files.pythonhosted.org/packages/0c/af/0389466685844d95c6f1f857008d4931d14c7937ac8dba689639ccf0cc54/pyaib-2.1.0.tar.gz"}]},"urls":[{"comment_text":"","digests":{"md5":"5a348b49d53cee26925e7204632721b7","sha256":"b6114554fb312f9b0bdeaf6a7498f7da05fc17b9250c0449ed796fac9ab663e2"},"downloads":-1,"filename":"pyaib-2.1.0.tar.gz","has_sig":true,"md5_digest":"5a348b49d53cee26925e7204632721b7","packagetype":"sdist","python_version":"source","size":23554,"upload_time":"2016-09-06T21:40:56","url":"https://files.pythonhosted.org/packages/0c/af/0389466685844d95c6f1f857008d4931d14c7937ac8dba689639ccf0cc54/pyaib-2.1.0.tar.gz"}]}
  46. \ No newline at end of file
  47. diff -r 4a65a57600e9 -r 0f1cf1154488 src/bandersnatch/tests/test_verify.py
  48. --- /dev/null   Thu Jan 01 00:00:00 1970 +0000
  49. +++ b/src/bandersnatch/tests/test_verify.py Sat May 12 15:31:47 2018 -0400
  50. @@ -0,0 +1,7 @@
  51. +from bandersnatch import verify
  52. +
  53. +import pytest
  54. +
  55. +
  56. +def test_metadata_verify():
  57. +    ...
  58. diff -r 4a65a57600e9 -r 0f1cf1154488 src/bandersnatch/utils.py
  59. --- a/src/bandersnatch/utils.py Thu Aug 10 11:53:37 2017 +0800
  60. +++ b/src/bandersnatch/utils.py Sat May 12 15:31:47 2018 -0400
  61. @@ -20,6 +20,8 @@
  62.      python += ' {0}.{1}.{2}-{3}{4}'.format(*sys.version_info)
  63.      return template.format(**locals())
  64.  
  65. +
  66. +# Set a global User Agent for requests module to use
  67.  USER_AGENT = user_agent()
  68.  
  69.  
  70. diff -r 4a65a57600e9 -r 0f1cf1154488 src/bandersnatch/verify.py
  71. --- /dev/null   Thu Jan 01 00:00:00 1970 +0000
  72. +++ b/src/bandersnatch/verify.py    Sat May 12 15:31:47 2018 -0400
  73. @@ -0,0 +1,94 @@
  74. +import concurrent.futures
  75. +import logging
  76. +import json  # Possibly use a faster JSON Module
  77. +
  78. +from . import master, mirror
  79. +
  80. +
  81. +logger = logging.getLogger(__name__)
  82. +
  83. +
  84. +def _create_mirror(config):
  85. +    ''' Get a mirror object for all worker threads to use '''
  86. +    logger.debug('Generating a Mirror object')
  87. +
  88. +    master = master.Master(
  89. +        config.get('mirror', 'master'),
  90. +        config.getfloat('mirror', 'timeout'),
  91. +    )
  92. +
  93. +    return mirror.Mirror(
  94. +        config.get('mirror', 'directory'),
  95. +        master,
  96. +        stop_on_error=False,
  97. +        workers=config.getint('mirror', 'workers'),
  98. +        delete_packages=config.getboolean('mirror', 'delete-packages'),
  99. +        hash_index=config.getboolean('mirror', 'hash-index'),
  100. +        json_save=False,  # We're only looking for files we could be missing
  101. +    )
  102. +
  103. +
  104. +def _package_sync(json_file, mirror)
  105. +    ''' Create a Package object and use sync_release_files to ensure
  106. +        we have all the package files '''
  107. +    release_key = 'releases'
  108. +
  109. +    try:
  110. +        with open(json_file, r) as jf:
  111. +            json_data = json.load(jf)
  112. +    except FileExistsError as fee:
  113. +        logger.error("{} does not exist: {}. Skipping.".format(json_file, fee))
  114. +        return
  115. +
  116. +    if release_key not in json_data
  117. +        logger.error("{} does not have '{}' key".format(release_key))
  118. +        return
  119. +
  120. +    pkg_name = os.path.basename(json_file)
  121. +
  122. +    pkg = package.Package(pkg_name, None, mirror)
  123. +    pkg.releases = json_data[release_key]
  124. +    # Use Package sync_release_files to ensure they all exist!
  125. +    pkg.sync_release_files()
  126. +
  127. +
  128. +def metadata_verify(config, cli_workers=0):
  129. +    ''' Go through a PEP381 mirror and ensure all the PyPI Package files exist
  130. +        and generate the expected hash according to local JSON Metadata '''
  131. +
  132. +    mirror_base = config.get('mirror', 'directory')
  133. +    json_base = os.path.join(mirror_base, 'json')
  134. +    # TODO: Allow cli override here
  135. +    workers = cli_workers or config.getint('mirror', 'workers'),
  136. +    logger.info("Starting verify for {} with {} workers".format(
  137. +        mirror_base, workers))
  138. +    logger.debug("Listing all files in {}".format(json_base))
  139. +    try:
  140. +        json_files = os.listdir(json_base)
  141. +    except FileExistsError as fee:
  142. +        logger.error('Metadata bsae dir {} does not exist: {}'.format(
  143. +            json_base, fee))
  144. +        return 2
  145. +    if not json_files:
  146. +        logger.error("No JSON metadata files found. Can not verify")
  147. +        return 3
  148. +    logger.info("We have {} packages to verify".format(len(json_files)))
  149. +
  150. +    mirror = _create_mirror(mirror_base, workers)
  151. +    sync_futures = {}
  152. +    with concurrent.futures.ThreadPoolExecutor(
  153. +        max_workers=workers) as executor:
  154. +        for pkg in json_files:
  155. +            jfile = os.path.join(json_base, pkg)
  156. +            sync_futures[executor.submit(_package_sync, jfile, mirror)] = pkg
  157. +
  158. +        finished_packages = 0
  159. +        total_packages = len(sync_futures)
  160. +        for future in concurrent.futures.as_completed(sync_futures):
  161. +            finished_packages += 1
  162. +            logger.info("Finished verifying {} - {}/{} ({}%) finished".format(
  163. +                sync_futures[future],
  164. +                finished_packages,
  165. +                total_packages,
  166. +                int(finished_packages / total_packages * 100)
  167. +            ))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!