Advertisement
here2share

# pybind11_core.py

Apr 5th, 2021
1,160
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 15.25 KB | None | 0 0
  1. # -*- coding: utf-8 -*-
  2.  
  3. """
  4. This module provides helpers for C++11+ projects using pybind11.
  5.  
  6. LICENSE:
  7.  
  8. Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch>, All rights reserved.
  9.  
  10. Redistribution and use in source and binary forms, with or without
  11. modification, are permitted provided that the following conditions are met:
  12.  
  13. 1. Redistributions of source code must retain the above copyright notice, this
  14.   list of conditions and the following disclaimer.
  15.  
  16. 2. Redistributions in binary form must reproduce the above copyright notice,
  17.   this list of conditions and the following disclaimer in the documentation
  18.   and/or other materials provided with the distribution.
  19.  
  20. 3. Neither the name of the copyright holder nor the names of its contributors
  21.   may be used to endorse or promote products derived from this software
  22.   without specific prior written permission.
  23.  
  24. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  25. ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  26. WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  27. DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
  28. FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  29. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  30. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  31. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  32. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  33. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34. """
  35.  
  36. # IMPORTANT: If you change this file in the pybind11 repo, also review
  37. # setup_helpers.pyi for matching changes.
  38. #
  39. # If you copy this file in, you don't
  40. # need the .pyi file; it's just an interface file for static type checkers.
  41.  
  42. import contextlib
  43. import os
  44. import shutil
  45. import sys
  46. import tempfile
  47. import threading
  48. import platform
  49. import warnings
  50. import sysconfig
  51.  
  52. try:
  53.     from setuptools.command.build_ext import build_ext as _build_ext
  54.     from setuptools import Extension as _Extension
  55. except ImportError:
  56.     from distutils.command.build_ext import build_ext as _build_ext
  57.     from distutils.extension import Extension as _Extension
  58.  
  59. import distutils.errors
  60. import distutils.ccompiler
  61.  
  62.  
  63. WIN = sys.platform.startswith("win32") and sysconfig.get_platform() != "mingw"
  64. PY2 = sys.version_info[0] < 3
  65. MACOS = sys.platform.startswith("darwin")
  66. STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}"
  67.  
  68.  
  69. # It is recommended to use PEP 518 builds if using this module. However, this
  70. # file explicitly supports being copied into a user's project directory
  71. # standalone, and pulling pybind11 with the deprecated setup_requires feature.
  72. # If you copy the file, remember to add it to your MANIFEST.in, and add the current
  73. # directory into your path if it sits beside your setup.py.
  74.  
  75.  
  76. class Pybind11Extension(_Extension):
  77.     """
  78.    Build a C++11+ Extension module with pybind11. This automatically adds the
  79.    recommended flags when you init the extension and assumes C++ sources - you
  80.    can further modify the options yourself.
  81.  
  82.    The customizations are:
  83.  
  84.    * ``/EHsc`` and ``/bigobj`` on Windows
  85.    * ``stdlib=libc++`` on macOS
  86.    * ``visibility=hidden`` and ``-g0`` on Unix
  87.  
  88.    Finally, you can set ``cxx_std`` via constructor or afterwords to enable
  89.    flags for C++ std, and a few extra helper flags related to the C++ standard
  90.    level. It is _highly_ recommended you either set this, or use the provided
  91.    ``build_ext``, which will search for the highest supported extension for
  92.    you if the ``cxx_std`` property is not set. Do not set the ``cxx_std``
  93.    property more than once, as flags are added when you set it. Set the
  94.    property to None to disable the addition of C++ standard flags.
  95.  
  96.    If you want to add pybind11 headers manually, for example for an exact
  97.    git checkout, then set ``include_pybind11=False``.
  98.  
  99.    Warning: do not use property-based access to the instance on Python 2 -
  100.    this is an ugly old-style class due to Distutils.
  101.    """
  102.  
  103.     # flags are prepended, so that they can be further overridden, e.g. by
  104.     # ``extra_compile_args=["-g"]``.
  105.  
  106.     def _add_cflags(self, flags):
  107.         self.extra_compile_args[:0] = flags
  108.  
  109.     def _add_ldflags(self, flags):
  110.         self.extra_link_args[:0] = flags
  111.  
  112.     def __init__(self, *args, **kwargs):
  113.  
  114.         self._cxx_level = 0
  115.         cxx_std = kwargs.pop("cxx_std", 0)
  116.  
  117.         if "language" not in kwargs:
  118.             kwargs["language"] = "c++"
  119.  
  120.         include_pybind11 = kwargs.pop("include_pybind11", True)
  121.  
  122.         # Can't use super here because distutils has old-style classes in
  123.         # Python 2!
  124.         _Extension.__init__(self, *args, **kwargs)
  125.  
  126.         # Include the installed package pybind11 headers
  127.         if include_pybind11:
  128.             # If using setup_requires, this fails the first time - that's okay
  129.             try:
  130.                 import pybind11
  131.  
  132.                 pyinc = pybind11.get_include()
  133.  
  134.                 if pyinc not in self.include_dirs:
  135.                     self.include_dirs.append(pyinc)
  136.             except ImportError:
  137.                 pass
  138.  
  139.         # Have to use the accessor manually to support Python 2 distutils
  140.         Pybind11Extension.cxx_std.__set__(self, cxx_std)
  141.  
  142.         cflags = []
  143.         ldflags = []
  144.         if WIN:
  145.             cflags += ["/EHsc", "/bigobj"]
  146.         else:
  147.             cflags += ["-fvisibility=hidden", "-g0"]
  148.             if MACOS:
  149.                 cflags += ["-stdlib=libc++"]
  150.                 ldflags += ["-stdlib=libc++"]
  151.         self._add_cflags(cflags)
  152.         self._add_ldflags(ldflags)
  153.  
  154.     @property
  155.     def cxx_std(self):
  156.         """
  157.        The CXX standard level. If set, will add the required flags. If left
  158.        at 0, it will trigger an automatic search when pybind11's build_ext
  159.        is used. If None, will have no effect.  Besides just the flags, this
  160.        may add a register warning/error fix for Python 2 or macos-min 10.9
  161.        or 10.14.
  162.        """
  163.         return self._cxx_level
  164.  
  165.     @cxx_std.setter
  166.     def cxx_std(self, level):
  167.  
  168.         if self._cxx_level:
  169.             warnings.warn("You cannot safely change the cxx_level after setting it!")
  170.  
  171.         # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so
  172.         # force a valid flag here.
  173.         if WIN and level == 11:
  174.             level = 14
  175.  
  176.         self._cxx_level = level
  177.  
  178.         if not level:
  179.             return
  180.  
  181.         cflags = [STD_TMPL.format(level)]
  182.         ldflags = []
  183.  
  184.         if MACOS and "MACOSX_DEPLOYMENT_TARGET" not in os.environ:
  185.             # C++17 requires a higher min version of macOS. An earlier version
  186.             # (10.12 or 10.13) can be set manually via environment variable if
  187.             # you are careful in your feature usage, but 10.14 is the safest
  188.             # setting for general use. However, never set higher than the
  189.             # current macOS version!
  190.             current_macos = tuple(int(x) for x in platform.mac_ver()[0].split(".")[:2])
  191.             desired_macos = (10, 9) if level < 17 else (10, 14)
  192.             macos_string = ".".join(str(x) for x in min(current_macos, desired_macos))
  193.             macosx_min = "-mmacosx-version-min=" + macos_string
  194.             cflags += [macosx_min]
  195.             ldflags += [macosx_min]
  196.  
  197.         if PY2:
  198.             if WIN:
  199.                 # Will be ignored on MSVC 2015, where C++17 is not supported so
  200.                 # this flag is not valid.
  201.                 cflags += ["/wd5033"]
  202.             elif level >= 17:
  203.                 cflags += ["-Wno-register"]
  204.             elif level >= 14:
  205.                 cflags += ["-Wno-deprecated-register"]
  206.  
  207.         self._add_cflags(cflags)
  208.         self._add_ldflags(ldflags)
  209.  
  210.  
  211. # Just in case someone clever tries to multithread
  212. tmp_chdir_lock = threading.Lock()
  213. cpp_cache_lock = threading.Lock()
  214.  
  215.  
  216. @contextlib.contextmanager
  217. def tmp_chdir():
  218.     "Prepare and enter a temporary directory, cleanup when done"
  219.  
  220.     # Threadsafe
  221.     with tmp_chdir_lock:
  222.         olddir = os.getcwd()
  223.         try:
  224.             tmpdir = tempfile.mkdtemp()
  225.             os.chdir(tmpdir)
  226.             yield tmpdir
  227.         finally:
  228.             os.chdir(olddir)
  229.             shutil.rmtree(tmpdir)
  230.  
  231.  
  232. # cf http://bugs.python.org/issue26689
  233. def has_flag(compiler, flag):
  234.     """
  235.    Return the flag if a flag name is supported on the
  236.    specified compiler, otherwise None (can be used as a boolean).
  237.    If multiple flags are passed, return the first that matches.
  238.    """
  239.  
  240.     with tmp_chdir():
  241.         fname = "flagcheck.cpp"
  242.         with open(fname, "w") as f:
  243.             # Don't trigger -Wunused-parameter.
  244.             f.write("int main (int, char **) { return 0; }")
  245.  
  246.         try:
  247.             compiler.compile([fname], extra_postargs=[flag])
  248.         except distutils.errors.CompileError:
  249.             return False
  250.         return True
  251.  
  252.  
  253. # Every call will cache the result
  254. cpp_flag_cache = None
  255.  
  256.  
  257. def auto_cpp_level(compiler):
  258.     """
  259.    Return the max supported C++ std level (17, 14, or 11). Returns latest on Windows.
  260.    """
  261.  
  262.     if WIN:
  263.         return "latest"
  264.  
  265.     global cpp_flag_cache
  266.  
  267.     # If this has been previously calculated with the same args, return that
  268.     with cpp_cache_lock:
  269.         if cpp_flag_cache:
  270.             return cpp_flag_cache
  271.  
  272.     levels = [17, 14, 11]
  273.  
  274.     for level in levels:
  275.         if has_flag(compiler, STD_TMPL.format(level)):
  276.             with cpp_cache_lock:
  277.                 cpp_flag_cache = level
  278.             return level
  279.  
  280.     msg = "Unsupported compiler -- at least C++11 support is needed!"
  281.     raise RuntimeError(msg)
  282.  
  283.  
  284. class build_ext(_build_ext):  # noqa: N801
  285.     """
  286.    Customized build_ext that allows an auto-search for the highest supported
  287.    C++ level for Pybind11Extension. This is only needed for the auto-search
  288.    for now, and is completely optional otherwise.
  289.    """
  290.  
  291.     def build_extensions(self):
  292.         """
  293.        Build extensions, injecting C++ std for Pybind11Extension if needed.
  294.        """
  295.  
  296.         for ext in self.extensions:
  297.             if hasattr(ext, "_cxx_level") and ext._cxx_level == 0:
  298.                 # Python 2 syntax - old-style distutils class
  299.                 ext.__class__.cxx_std.__set__(ext, auto_cpp_level(self.compiler))
  300.  
  301.         # Python 2 doesn't allow super here, since distutils uses old-style
  302.         # classes!
  303.         _build_ext.build_extensions(self)
  304.  
  305.  
  306. def naive_recompile(obj, src):
  307.     """
  308.    This will recompile only if the source file changes. It does not check
  309.    header files, so a more advanced function or Ccache is better if you have
  310.    editable header files in your package.
  311.    """
  312.     return os.stat(obj).st_mtime < os.stat(src).st_mtime
  313.  
  314.  
  315. def no_recompile(obg, src):
  316.     """
  317.    This is the safest but slowest choice (and is the default) - will always
  318.    recompile sources.
  319.    """
  320.     return True
  321.  
  322.  
  323. # Optional parallel compile utility
  324. # inspired by: http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils
  325. # and: https://github.com/tbenthompson/cppimport/blob/stable/cppimport/build_module.py
  326. # and NumPy's parallel distutils module:
  327. #              https://github.com/numpy/numpy/blob/master/numpy/distutils/ccompiler.py
  328. class ParallelCompile(object):
  329.     """
  330.    Make a parallel compile function. Inspired by
  331.    numpy.distutils.ccompiler.CCompiler_compile and cppimport.
  332.  
  333.    This takes several arguments that allow you to customize the compile
  334.    function created:
  335.  
  336.    envvar:
  337.        Set an environment variable to control the compilation threads, like
  338.        NPY_NUM_BUILD_JOBS
  339.    default:
  340.        0 will automatically multithread, or 1 will only multithread if the
  341.        envvar is set.
  342.    max:
  343.        The limit for automatic multithreading if non-zero
  344.    needs_recompile:
  345.        A function of (obj, src) that returns True when recompile is needed.  No
  346.        effect in isolated mode; use ccache instead, see
  347.        https://github.com/matplotlib/matplotlib/issues/1507/
  348.  
  349.    To use::
  350.  
  351.        ParallelCompile("NPY_NUM_BUILD_JOBS").install()
  352.  
  353.    or::
  354.  
  355.        with ParallelCompile("NPY_NUM_BUILD_JOBS"):
  356.            setup(...)
  357.  
  358.    By default, this assumes all files need to be recompiled. A smarter
  359.    function can be provided via needs_recompile.  If the output has not yet
  360.    been generated, the compile will always run, and this function is not
  361.    called.
  362.    """
  363.  
  364.     __slots__ = ("envvar", "default", "max", "_old", "needs_recompile")
  365.  
  366.     def __init__(self, envvar=None, default=0, max=0, needs_recompile=no_recompile):
  367.         self.envvar = envvar
  368.         self.default = default
  369.         self.max = max
  370.         self.needs_recompile = needs_recompile
  371.         self._old = []
  372.  
  373.     def function(self):
  374.         """
  375.        Builds a function object usable as distutils.ccompiler.CCompiler.compile.
  376.        """
  377.  
  378.         def compile_function(
  379.             compiler,
  380.             sources,
  381.             output_dir=None,
  382.             macros=None,
  383.             include_dirs=None,
  384.             debug=0,
  385.             extra_preargs=None,
  386.             extra_postargs=None,
  387.             depends=None,
  388.         ):
  389.  
  390.             # These lines are directly from distutils.ccompiler.CCompiler
  391.             macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile(
  392.                 output_dir, macros, include_dirs, sources, depends, extra_postargs
  393.             )
  394.             cc_args = compiler._get_cc_args(pp_opts, debug, extra_preargs)
  395.  
  396.             # The number of threads; start with default.
  397.             threads = self.default
  398.  
  399.             # Determine the number of compilation threads, unless set by an environment variable.
  400.             if self.envvar is not None:
  401.                 threads = int(os.environ.get(self.envvar, self.default))
  402.  
  403.             def _single_compile(obj):
  404.                 try:
  405.                     src, ext = build[obj]
  406.                 except KeyError:
  407.                     return
  408.  
  409.                 if not os.path.exists(obj) or self.needs_recompile(obj, src):
  410.                     compiler._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
  411.  
  412.             try:
  413.                 import multiprocessing
  414.                 from multiprocessing.pool import ThreadPool
  415.             except ImportError:
  416.                 threads = 1
  417.  
  418.             if threads == 0:
  419.                 try:
  420.                     threads = multiprocessing.cpu_count()
  421.                     threads = self.max if self.max and self.max < threads else threads
  422.                 except NotImplementedError:
  423.                     threads = 1
  424.  
  425.             if threads > 1:
  426.                 for _ in ThreadPool(threads).imap_unordered(_single_compile, objects):
  427.                     pass
  428.             else:
  429.                 for ob in objects:
  430.                     _single_compile(ob)
  431.  
  432.             return objects
  433.  
  434.         return compile_function
  435.  
  436.     def install(self):
  437.         distutils.ccompiler.CCompiler.compile = self.function()
  438.         return self
  439.  
  440.     def __enter__(self):
  441.         self._old.append(distutils.ccompiler.CCompiler.compile)
  442.         return self.install()
  443.  
  444.     def __exit__(self, *args):
  445.         distutils.ccompiler.CCompiler.compile = self._old.pop()
  446.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement