# Copyright 2024 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This file contains repository rules and macros to support toolchain registration. """ load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY") load(":auth.bzl", "get_auth") load(":pbs_manifest.bzl", "is_astral_static_libpython_build") load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":text_util.bzl", "render") STANDALONE_INTERPRETER_FILENAME = "STANDALONE_INTERPRETER" def is_standalone_interpreter(rctx, python_interpreter_path, *, logger = None): """Query a python interpreter target for whether or not it's a rules_rust provided toolchain Args: rctx: {type}`repository_ctx` The repository rule's context object. python_interpreter_path: {type}`path` A path representing the interpreter. logger: Optional logger to use for operations. Returns: {type}`bool` Whether or not the target is from a rules_python generated toolchain. """ # Only update the location when using a hermetic toolchain. if not python_interpreter_path: return False # This is a rules_python provided toolchain. return repo_utils.execute_unchecked( rctx, op = "IsStandaloneInterpreter", arguments = [ "ls", "{}/{}".format( python_interpreter_path.dirname, STANDALONE_INTERPRETER_FILENAME, ), ], logger = logger, ).return_code == 0 def _get_pycache_root(rctx): """Calculates and creates the pycache root directory. Returns: {type}`path | None` The path to the pycache root, or None if it couldn't be created. """ res = rctx.getenv("RULES_PYTHON_PYCACHE_DIR") if res and res.lower() in ("/dev/null", "nul"): return None os_name = repo_utils.get_platforms_os_name(rctx) is_windows = os_name == "windows" # When Python uses mtime/size checking of pycs, the mtimes of files in a # runtime are fixed once the repository is created. In addition, pyc # content is a function of hash mode and optimization level, which can vary # by workspace. Keying on both the workspace and runtime archive keeps # workspaces and runtime versions from colliding on pyc files. runtime_hash = rctx.attr.sha256 or rctx.attr.python_version suffix = "{}/{}".format( rctx.name, hash("{}:{}".format(rctx.workspace_root, runtime_hash)), ) if res: return repo_utils.mkdir(rctx, rctx.path(res).get_child(suffix)) # Global locations like /tmp or TMP/TEMP are intentionally not used: # * Avoids cross-user contamination or prediction/interception of the pyc # cache. # * Temporary directories get cleaned out while the fetched runtime # repository persists, leaving dangling symlinks. res = rctx.getenv("XDG_CACHE_HOME") if not res: if is_windows: res = rctx.getenv("LOCALAPPDATA") else: res = rctx.getenv("HOME") if res: res += "/.cache" if not res: return None # The first level directory is static and documented so that it is easy to # use with e.g. --sandbox_add_mount_pair=$HOME/.cache/rules_python_pycache path = rctx.path(res).get_child("rules_python_pycache").get_child(suffix) return repo_utils.mkdir(rctx, path) def _create_pycache_symlinks(rctx, logger): """Finds all directories with a .py file and creates __pycache__ entries. Args: rctx: {type}`repository_ctx` The repository rule's context object. logger: Optional logger to use for operations. """ pycache_root = _get_pycache_root(rctx) logger.info(lambda: "pycache root: {}".format(pycache_root)) pycache_root_str = str(pycache_root) if pycache_root else None queue = [rctx.path(".")] # Starlark doesn't support recursion, use a loop with a queue. # Using a large range as a safeguard. for _ in range(1000000): if not queue: break p = queue.pop() has_py = False for child in p.readdir(): # Skip hidden files and directories if child.basename.startswith("."): continue if child.is_dir: if child.basename == "__pycache__" or str(child) == pycache_root_str: continue queue.append(child) elif child.basename.endswith(".py"): has_py = True if has_py: pycache_dir = p.get_child("__pycache__") if pycache_root: pycache_relative = repo_utils.repo_root_relative_path(rctx, pycache_dir) target_dir = pycache_root.get_child(pycache_relative) repo_utils.mkdir(rctx, target_dir) rctx.delete(pycache_dir) rctx.symlink(target_dir, pycache_dir) else: rctx.delete(pycache_dir) # The intent is to disable pyc generation, which we can't # entirely do directly. Symlinking to /dev/null makes it point # to a character device, which can act weird when read or put # into a zip. A regular file seems to make python skip local pyc # generation, so use that. rctx.file(pycache_dir, content = "", executable = False) def _python_repository_impl(rctx): if rctx.attr.distutils and rctx.attr.distutils_content: fail("Only one of (distutils, distutils_content) should be set.") if bool(rctx.attr.url) == bool(rctx.attr.urls): fail("Exactly one of (url, urls) must be set.") logger = repo_utils.logger(rctx) platform = rctx.attr.platform python_version = rctx.attr.python_version python_version_info = python_version.split(".") release_filename = rctx.attr.release_filename version_suffix = "t" if FREETHREADED in release_filename else "" python_short_version = "{0}.{1}{suffix}".format( suffix = version_suffix, *python_version_info ) urls = rctx.attr.urls or [rctx.attr.url] if rctx.attr.libpython == "include": interpreter_has_static_libpython = False elif rctx.attr.libpython == "exclude": interpreter_has_static_libpython = True else: interpreter_has_static_libpython = is_astral_static_libpython_build(urls, release_filename) auth = get_auth(rctx, urls) if INSTALL_ONLY in release_filename: rctx.download_and_extract( url = urls, sha256 = rctx.attr.sha256, stripPrefix = rctx.attr.strip_prefix, auth = auth, ) else: rctx.download_and_extract( url = urls, sha256 = rctx.attr.sha256, stripPrefix = rctx.attr.strip_prefix, auth = auth, ) # Strip the things that are not present in the INSTALL_ONLY builds # NOTE: if the dirs are not present, we will not fail here rctx.delete("python/build") rctx.delete("python/licenses") rctx.delete("python/PYTHON.json") patches = rctx.attr.patches if patches: for patch in patches: rctx.patch(patch, strip = rctx.attr.patch_strip) # Debug (`Py_DEBUG`) builds add a "d" ABI flag to the include dir and # libpython names, but not to the stdlib dir. debug = rctx.path("include/python{}d".format(python_short_version)).exists # Write distutils.cfg to the Python installation. if "windows" in platform: distutils_path = "Lib/distutils/distutils.cfg" else: distutils_path = "lib/python{}/distutils/distutils.cfg".format(python_short_version) if rctx.attr.distutils: rctx.file(distutils_path, rctx.read(rctx.attr.distutils)) elif rctx.attr.distutils_content: rctx.file(distutils_path, rctx.attr.distutils_content) if "darwin" in platform and "osx" == repo_utils.get_platforms_os_name(rctx): # Fix up the Python distribution's LC_ID_DYLIB field. # It points to a build directory local to the GitHub Actions # host machine used in the Python standalone build, which causes # dyld lookup errors. To fix, set the full path to the dylib as # it appears in the Bazel workspace as its LC_ID_DYLIB using # the `install_name_tool` bundled with macOS. dylib = "libpython{}{}.dylib".format( python_short_version, "d" if debug else "", ) repo_utils.execute_checked( rctx, op = "python_repository.FixUpDyldIdPath", arguments = [repo_utils.which_checked(rctx, "install_name_tool"), "-id", "@rpath/{}".format(dylib), "lib/{}".format(dylib)], logger = logger, ) _create_pycache_symlinks(rctx, logger) python_bin = "python.exe" if ("windows" in platform) else "bin/python3" if "linux" in platform: # Workaround around https://github.com/astral-sh/python-build-standalone/issues/231 for url in urls: head_and_release, _, _ = url.rpartition("/") _, _, release = head_and_release.rpartition("/") if not release.isdigit(): # Maybe this is some custom toolchain, so skip this break if int(release) >= 20240224: # Starting with this release the Linux toolchains have infinite symlink loop # on host platforms that are not Linux. Delete the files no # matter the host platform so that the cross-built artifacts # are the same irrespective of the host platform we are # building on. # # Link to the first affected release: # https://github.com/astral-sh/python-build-standalone/releases/tag/20240224 rctx.delete("share/terminfo") break glob_include = [] glob_exclude = [ # These pycache files are created on first use of the associated python files. # Exclude them from the glob because otherwise between the first time and second time a python toolchain is used," # the definition of this filegroup will change, and depending rules will get invalidated." # See https://github.com/bazel-contrib/rules_python/issues/1008 for unconditionally adding these to toolchains so we can stop ignoring them." # The __pycache__ directories themselves are symlinks out of the # repository. "**/__pycache__", # Also ignore everything under __pycache__, just in case pre-population # of the directories misses one and a regular directory is created. "**/__pycache__/**", ] if "windows" in platform: glob_include += [ "*.exe", "*.dll", "DLLs/**", "Lib/**", "Scripts/**", "tcl/**", ] else: glob_include.append( "lib/**", ) if "windows" in platform: coverage_tool = None else: coverage_tool = rctx.attr.coverage_tool build_content = """\ # Generated by python/private/python_repositories.bzl load("@rules_python//python/private:hermetic_runtime_repo_setup.bzl", "define_hermetic_runtime_toolchain_impl") package(default_visibility = ["//visibility:public"]) define_hermetic_runtime_toolchain_impl( name = "define_runtime", extra_files_glob_include = {extra_files_glob_include}, extra_files_glob_exclude = {extra_files_glob_exclude}, interpreter_has_static_libpython = {interpreter_has_static_libpython}, debug = {debug}, python_version = {python_version}, python_bin = {python_bin}, coverage_tool = {coverage_tool}, ) """.format( extra_files_glob_exclude = render.list(glob_exclude), extra_files_glob_include = render.list(glob_include), interpreter_has_static_libpython = str(interpreter_has_static_libpython), debug = str(debug), python_bin = render.str(python_bin), python_version = render.str(rctx.attr.python_version), coverage_tool = render.str(coverage_tool), ) rctx.delete("python") rctx.symlink(python_bin, "python") rctx.file(STANDALONE_INTERPRETER_FILENAME, "# File intentionally left blank. Indicates that this is an interpreter repo created by rules_python.") rctx.file("BUILD.bazel", build_content) attrs = { "auth_patterns": rctx.attr.auth_patterns, "coverage_tool": rctx.attr.coverage_tool, "distutils": rctx.attr.distutils, "distutils_content": rctx.attr.distutils_content, "libpython": rctx.attr.libpython, "name": rctx.attr.name, "netrc": rctx.attr.netrc, "patch_strip": rctx.attr.patch_strip, "patches": rctx.attr.patches, "platform": platform, "python_version": python_version, "release_filename": release_filename, "sha256": rctx.attr.sha256, "strip_prefix": rctx.attr.strip_prefix, } if rctx.attr.url: attrs["url"] = rctx.attr.url else: attrs["urls"] = urls # Bazel <8.3.0 lacks repository_ctx.repo_metadata if not hasattr(rctx, "repo_metadata"): return attrs reproducible = rctx.attr.sha256 != "" return rctx.repo_metadata( reproducible = reproducible, attrs_for_reproducibility = {} if reproducible else attrs, ) python_repository = repository_rule( _python_repository_impl, doc = "Fetches the external tools needed for the Python toolchain.", attrs = { "auth_patterns": attr.string_dict( doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", ), "coverage_tool": attr.string( doc = """ This is a target to use for collecting code coverage information from {rule}`py_binary` and {rule}`py_test` targets. The target is accepted as a string by the python_repository and evaluated within the context of the toolchain repository. For more information see {attr}`py_runtime.coverage_tool`. """, ), "distutils": attr.label( allow_single_file = True, doc = "A distutils.cfg file to be included in the Python installation. " + "Either distutils or distutils_content can be specified, but not both.", mandatory = False, ), "distutils_content": attr.string( doc = "A distutils.cfg file content to be included in the Python installation. " + "Either distutils or distutils_content can be specified, but not both.", mandatory = False, ), "ignore_root_user_error": attr.bool( default = True, doc = "Noop, will be removed in the next major release", mandatory = False, ), "libpython": attr.string( default = "auto", doc = "Whether to include shared libpython files: auto, include, or exclude.", values = ["auto", "include", "exclude"], ), "netrc": attr.string( doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", ), "patch_strip": attr.int( doc = """ Same as the --strip argument of Unix patch. :::{note} In the future the default value will be set to `0`, to mimic the well known function defaults (e.g. `single_version_override` for `MODULE.bazel` files. ::: :::{versionadded} 0.36.0 ::: """, default = 1, mandatory = False, ), "patches": attr.label_list( doc = "A list of patch files to apply to the unpacked interpreter", mandatory = False, ), "platform": attr.string( doc = "The platform name for the Python interpreter tarball.", mandatory = True, ), "python_version": attr.string( doc = "The Python version.", mandatory = True, ), "release_filename": attr.string( doc = "The filename of the interpreter to be downloaded", mandatory = True, ), "sha256": attr.string( doc = "The SHA256 integrity hash for the Python interpreter tarball.", mandatory = True, ), "strip_prefix": attr.string( doc = "A directory prefix to strip from the extracted files.", ), "url": attr.string( doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", ), "urls": attr.string_list( doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", ), "_rule_name": attr.string(default = "python_repository"), }, environ = [REPO_DEBUG_ENV_VAR], )