From d8ddd3d8d49ef5edefdb7c1044700d5eb16e4e32 Mon Sep 17 00:00:00 2001 From: Jeremy Volkman Date: Mon, 26 Oct 2026 01:01:24 +0000 Subject: [PATCH] feat(pypi): install action-built wheels in py_wheel_library Extends py_wheel_library and action_whl_installer to consume action-built wheels (.whldir TreeArtifacts): - Adds site_paths, distribution, and version attributes to py_wheel_library and consumes PyBuiltWheelInfo. - Exposes extracted .dist-info/entry_points.txt via OutputGroupInfo.dist_info (--entry-points-output) for action-built wheel console scripts. - Normalizes the installed .dist-info directory name via --dist-info-dir (and updates RECORD entries) so importlib.metadata works in venv mode. - Validates predicted site_paths against installed site-packages contents in action_whl_installer when venvs_site_packages=yes (--expected-site-path). --- python/private/pypi/BUILD.bazel | 5 + python/private/pypi/action_whl_installer.py | 83 +++++- .../private/pypi/py_installed_wheel_info.bzl | 10 +- python/private/pypi/py_wheel_library.bzl | 263 +++++++++++++++--- python/private/pypi/py_wheel_transform.bzl | 23 +- python/private/pypi/wheel_layout.py | 45 ++- python/private/pypi/wheel_record_rewriter.py | 29 +- tests/pypi/py_wheel_library/BUILD.bazel | 31 ++- .../py_wheel_library_bzl_tests.bzl | 36 +++ .../py_wheel_library/py_wheel_library_test.py | 6 + tests/pypi/py_wheel_library/sh_transform.sh | 4 + .../wheel_layout_and_installer_test.py | 74 +++++ 12 files changed, 533 insertions(+), 76 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 1f30cab9f7..91eff9021a 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -728,6 +728,7 @@ bzl_library( name = "py_wheel_transform", srcs = ["py_wheel_transform.bzl"], deps = [ + ":py_sdist_wheel", "//python/private:attr_builders", "//python/private:common", "//python/private:py_internal", @@ -745,10 +746,12 @@ bzl_library( "//python/private:attributes", "//python/private:builders", "//python/private:common", + "//python/private:normalize_name", "//python/private:py_info", "//python/private:reexports", "//python/private:rule_builders", "//python/private:toolchain_types", + "//python/private:version", ], ) @@ -756,7 +759,9 @@ bzl_library( name = "py_wheel_library", srcs = ["py_wheel_library.bzl"], deps = [ + ":parse_whl_name", ":py_installed_wheel_info", + ":py_sdist_wheel", "//python/private:attr_builders", "//python/private:attributes", "//python/private:builders", diff --git a/python/private/pypi/action_whl_installer.py b/python/private/pypi/action_whl_installer.py index 3c421ee5fc..a1b30b9e81 100644 --- a/python/private/pypi/action_whl_installer.py +++ b/python/private/pypi/action_whl_installer.py @@ -333,6 +333,7 @@ def _finalize_dist_info( site_packages_dir: Path, dist_info_dir: str, *, + wheel_dist_info_dir: str = "", data_exclude_rx: list[re.Pattern[str]], used_data_dir: bool, expected_data_dir: str, @@ -348,27 +349,52 @@ def _finalize_dist_info( f.write(INSTALLER_URL) record_path = dist_info_path / "RECORD" - if not (used_data_dir and record_path.is_file()): + renamed_dist_info = bool( + wheel_dist_info_dir and dist_info_dir != wheel_dist_info_dir + ) + if not ((used_data_dir or renamed_dist_info) and record_path.is_file()): return with open(record_path, encoding="utf-8") as f: lines = rewrite_record_lines( f, target_os, - expected_data_dir, + expected_data_dir if used_data_dir else "", rewritten_scripts, + wheel_dist_info_dir=wheel_dist_info_dir, + target_dist_info_dir=dist_info_dir, ) with open(record_path, "w", encoding="utf-8", newline="\n") as f: f.write("".join(lines)) +def _write_entry_points_output( + entry_points_output: Path, + site_packages_dir: Path, + dist_info_dir: str, + fallback_text: str, + touched_site_rel: set[str], +) -> None: + ep_rel = f"{dist_info_dir}/entry_points.txt" if dist_info_dir else "" + ep_path = site_packages_dir / ep_rel if ep_rel else None + if ep_path is not None and ep_path.is_file(): + text = ep_path.read_text(encoding="utf-8") + elif ep_rel and ep_rel in touched_site_rel: + text = "" + else: + text = fallback_text + entry_points_output.parent.mkdir(parents=True, exist_ok=True) + with open(entry_points_output, "w", encoding="utf-8", newline="\n") as f: + f.write(text) + + def _install_from_existing_dir( installed_dir: Path, dest_dir: Path, site_packages_dir: Path, srcs_exclude_rx: list[re.Pattern[str]], data_exclude_rx: list[re.Pattern[str]], -) -> tuple[str, set[str]]: +) -> tuple[str, set[str], str]: shutil.copytree(installed_dir, dest_dir, dirs_exist_ok=True) for p in dest_dir.rglob("*"): try: @@ -382,6 +408,16 @@ def _install_from_existing_dir( if d.is_dir() and d.name.endswith(".dist-info") ) dist_info_dir = dist_info_dirs[0] if dist_info_dirs else "" + ep_file = ( + site_packages_dir / dist_info_dir / "entry_points.txt" + if dist_info_dir + else None + ) + entry_points_text = ( + ep_file.read_text(encoding="utf-8") + if ep_file is not None and ep_file.is_file() + else "" + ) touched_site_rel: set[str] = set() for p in sorted(dest_dir.rglob("*")): @@ -403,7 +439,7 @@ def _install_from_existing_dir( touched_site_rel.add(p.relative_to(site_packages_dir).as_posix()) p.unlink() - return dist_info_dir, touched_site_rel + return dist_info_dir, touched_site_rel, entry_points_text def _extract_from_wheel( @@ -414,7 +450,8 @@ def _extract_from_wheel( target_os: str, srcs_exclude_rx: list[re.Pattern[str]], data_exclude_rx: list[re.Pattern[str]], -) -> str: + dist_info_dir_override: str = "", +) -> tuple[str, str]: if wheel_path.is_dir(): wheel_path = single_wheel_in(wheel_path) site_packages_dir.mkdir(parents=True, exist_ok=True) @@ -423,18 +460,20 @@ def _extract_from_wheel( used_data_dir = False with zipfile.ZipFile(wheel_path) as z: - dist_info_dir, _, entry_points_content = find_wheel_dist_info_and_entry_points( - z + wheel_dist_info_dir, _, entry_points_content = ( + find_wheel_dist_info_and_entry_points(z) ) - expected_data_dir = expected_data_dir_for_dist_info(dist_info_dir) + expected_data_dir = expected_data_dir_for_dist_info(wheel_dist_info_dir) data_dir_prefix = f"{expected_data_dir}/" + dist_info_dir = dist_info_dir_override or wheel_dist_info_dir for info, dest_or_script, is_shebang in plan_wheel_entries( z, - dist_info_dir, + wheel_dist_info_dir, entry_points_content, srcs_exclude_rx, data_exclude_rx, + target_dist_info_dir=dist_info_dir, ): if info.filename.startswith(data_dir_prefix): used_data_dir = True @@ -457,13 +496,14 @@ def _extract_from_wheel( _finalize_dist_info( site_packages_dir, dist_info_dir, + wheel_dist_info_dir=wheel_dist_info_dir, data_exclude_rx=data_exclude_rx, used_data_dir=used_data_dir, expected_data_dir=expected_data_dir, target_os=target_os, rewritten_scripts=rewritten_scripts, ) - return dist_info_dir + return dist_info_dir, entry_points_content or "" def _validate_installed_site_packages( @@ -555,6 +595,7 @@ def install_wheel( dest_dir: Path, *, installed_dir: Path | None = None, + entry_points_output: Path | None = None, target_os: str = "unix", srcs_exclude_globs: list[str] | None = None, data_exclude_globs: list[str] | None = None, @@ -563,6 +604,7 @@ def install_wheel( patch_strip: int = 0, expected_site_paths: list[str] | None = None, validation_source: str = "extracted_files", + dist_info_dir_override: str = "", precompile: bool = False, pyc_tag: str = "", precompile_invalidation_mode: str = "checked_hash", @@ -578,7 +620,7 @@ def install_wheel( data_exclude_rx = [glob_to_regex(g) for g in (data_exclude_globs or ())] if installed_dir is not None: - dist_info_dir, touched_site_rel = _install_from_existing_dir( + dist_info_dir, touched_site_rel, entry_points_text = _install_from_existing_dir( installed_dir, dest_dir, site_packages_dir, @@ -588,13 +630,14 @@ def install_wheel( else: if wheel_path is None: raise ValueError("Either wheel_path or installed_dir must be provided") - dist_info_dir = _extract_from_wheel( + dist_info_dir, entry_points_text = _extract_from_wheel( wheel_path, dest_dir, site_packages_dir, target_os=target_os, srcs_exclude_rx=srcs_exclude_rx, data_exclude_rx=data_exclude_rx, + dist_info_dir_override=dist_info_dir_override, ) touched_site_rel = set() @@ -613,6 +656,15 @@ def install_wheel( touched_site_rel, ) + if entry_points_output is not None: + _write_entry_points_output( + entry_points_output, + site_packages_dir, + dist_info_dir, + entry_points_text, + touched_site_rel, + ) + if create_namespace_inits_flag: create_namespace_inits( site_packages_dir, _NAMESPACE_PKG_TMPL_PATH.read_text(encoding="utf-8") @@ -656,6 +708,7 @@ def main(argv: Sequence[str] | None = None) -> None: parser.add_argument("--wheel", type=Path, default=None) parser.add_argument("--installed-dir", type=Path, default=None) parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--entry-points-output", type=Path, default=None) parser.add_argument( "--target-os", choices=["unix", "windows"], @@ -706,6 +759,10 @@ def main(argv: Sequence[str] | None = None) -> None: dest="expected_site_paths", default=[], ) + parser.add_argument( + "--dist-info-dir", + default="", + ) parser.add_argument( "--precompile", action="store_true", @@ -749,6 +806,7 @@ def main(argv: Sequence[str] | None = None) -> None: args.wheel, args.directory, installed_dir=args.installed_dir, + entry_points_output=args.entry_points_output, target_os=args.target_os, srcs_exclude_globs=args.srcs_exclude_globs, data_exclude_globs=args.data_exclude_globs, @@ -759,6 +817,7 @@ def main(argv: Sequence[str] | None = None) -> None: args.expected_site_paths if args.validate_site_packages else None ), validation_source=args.validation_source, + dist_info_dir_override=args.dist_info_dir, precompile=args.precompile, pyc_tag=args.pyc_tag, precompile_invalidation_mode=args.precompile_invalidation_mode, diff --git a/python/private/pypi/py_installed_wheel_info.bzl b/python/private/pypi/py_installed_wheel_info.bzl index dac5d2342e..b3c54d43fb 100644 --- a/python/private/pypi/py_installed_wheel_info.bzl +++ b/python/private/pypi/py_installed_wheel_info.bzl @@ -12,6 +12,11 @@ Information about an installed Python wheel TreeArtifact. :type: File The installed wheel TreeArtifact directory containing site-packages/, data/, bin/, and include/. +""", + "entry_points": """ +:type: File +The extracted `.dist-info/entry_points.txt` file (empty when the wheel has no +`entry_points.txt`), exposed via `OutputGroupInfo.dist_info`. """, "keep_source": """ :type: bool @@ -24,8 +29,9 @@ Whether `.py` files in `directory` were precompiled into `.pyc` files. "settings_key": """ :type: tuple Tuple of install settings `(should_precompile, keep_source, invalidation_mode, -optimize_level, srcs_exclude, data_exclude, create_namespace_inits)` used to -determine whether an outer `py_wheel_library` can reuse `directory`. +optimize_level, srcs_exclude, data_exclude, create_namespace_inits, +dist_info_dir)` used to determine whether an outer `py_wheel_library` can reuse +`directory`. """, "wheel": """ :type: File | None diff --git a/python/private/pypi/py_wheel_library.bzl b/python/private/pypi/py_wheel_library.bzl index 286b37ac04..3ce852730d 100644 --- a/python/private/pypi/py_wheel_library.bzl +++ b/python/private/pypi/py_wheel_library.bzl @@ -30,7 +30,12 @@ load( ) load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") -load("//python/private:py_info.bzl", "PyInfo") +load( + "//python/private:py_info.bzl", + "PyInfo", + "VenvSymlinkEntry", + "VenvSymlinkKind", +) load("//python/private:py_interpreter_program.bzl", "PyInterpreterProgramInfo") load("//python/private:py_library.bzl", "LIBRARY_ATTRS") load("//python/private:reexports.bzl", "BuiltinPyInfo") @@ -42,7 +47,9 @@ load( ) load("//python/private:venv_runfiles.bzl", "get_venv_symlinks") load("//python/private:version.bzl", "version") +load(":parse_whl_name.bzl", "parse_whl_name") load(":py_installed_wheel_info.bzl", "PyInstalledWheelInfo") +load(":py_sdist_wheel.bzl", "PyBuiltWheelInfo") PY_WHEEL_LIBRARY_ATTRS = COMMON_ATTRS | WINDOWS_CONSTRAINTS_ATTRS | { "data_exclude": lambda: attrb.StringList( @@ -52,6 +59,10 @@ Glob patterns to exclude from non-.py files when extracting the wheel. """, ), "deps": PY_SRCS_ATTRS["deps"], + "distribution": lambda: attrb.String( + default = "", + doc = "Optional Python distribution name; overrides `PyBuiltWheelInfo.distribution` and `pypi_name=` tag.", + ), "enable_implicit_namespace_pkgs": lambda: attrb.Bool( default = False, doc = """ @@ -94,12 +105,23 @@ prefixes) require `post_install_patch_strip = 1`. default = [], doc = """ Top-level script names in .data/scripts/ whose #!python shebang is rewritten. +""", + ), + "site_paths": lambda: attrb.StringList( + default = [], + doc = """ +Top-level or leaf paths under `site-packages/` when `extracted_files` is not +populated (e.g. for action-built sdists). Only used when `venvs_site_packages=yes`. """, ), "srcs_exclude": lambda: attrb.StringList( default = [], doc = "Glob patterns to exclude from .py files when extracting the wheel.", ), + "version": lambda: attrb.String( + default = "", + doc = "Optional PEP 440 package version; overrides `PyBuiltWheelInfo.version` and `pypi_version=` tag.", + ), "wheel": lambda: attrb.Label( allow_single_file = True, mandatory = True, @@ -223,62 +245,140 @@ def _add_precompile_args(ctx, args, out_short_path, exec_tools_toolchain, invali args.add("--precompiler-tool", files_to_run.executable) return [files_to_run] +def _expand_py_entry(path, keep_source, should_precompile, pyc_tag): + if not keep_source: + return [path[:-len(".py")] + ".pyc"] + if not (should_precompile and pyc_tag): + return [path] + stem = paths.basename(path)[:-len(".py")] + dirname = paths.dirname(path) + pyc_rel = ( + "{}/__pycache__/{}.{}.pyc".format(dirname, stem, pyc_tag) if dirname else "__pycache__/{}.{}.pyc".format(stem, pyc_tag) + ) + return [path, pyc_rel] + +def _resolve_meta_value(explicit_val, built_val, tag_val, whl_val, use_site_paths): + if explicit_val: + return explicit_val + if built_val: + return built_val + if use_site_paths: + return whl_val + return tag_val or whl_val + def _collect_installed_tree_files( extracted_files, shebang_scripts, *, is_windows, keep_source, + should_precompile, pyc_tag): tree_files = [] for f in extracted_files: - is_site_py = f.startswith("site-packages/") and f.endswith(".py") - if not is_site_py: + if f.startswith("site-packages/") and f.endswith(".py"): + tree_files.extend(_expand_py_entry(f, keep_source, should_precompile, pyc_tag)) + else: tree_files.append(f) - continue - if not keep_source: - tree_files.append(f[:-len(".py")] + ".pyc") - continue - tree_files.append(f) - if pyc_tag: - stem = paths.basename(f)[:-len(".py")] - tree_files.append( - "{}/__pycache__/{}.{}.pyc".format(paths.dirname(f), stem, pyc_tag), - ) for s in shebang_scripts: tree_files.append("bin/" + ((s + ".bat") if is_windows else s)) return tree_files +def _venv_symlinks_for_site_paths( + ctx, + out_files, + out_rf_path, + site_packages_rf_path, + package, + version_str, + dist_info_dir, + is_windows, + keep_source, + should_precompile = False, + pyc_tag = ""): + symlinks = [] + seen = {} + entries = [] + for sp in list(ctx.attr.site_paths) + list(ctx.attr.namespace_package_files): + if sp.endswith(".py"): + entries.extend(_expand_py_entry(sp, keep_source, should_precompile, pyc_tag)) + else: + entries.append(sp) + if dist_info_dir: + entries.append(dist_info_dir) + for sp in entries: + if not sp or sp in seen: + continue + seen[sp] = True + symlinks.append(VenvSymlinkEntry( + kind = VenvSymlinkKind.LIB, + link_to_path = paths.join(site_packages_rf_path, sp), + link_to_file = None, + package = package, + version = version_str, + venv_path = sp, + files = out_files, + )) + for s in ctx.attr.shebang_scripts: + script_name = (s + ".bat") if is_windows else s + symlinks.append(VenvSymlinkEntry( + kind = VenvSymlinkKind.BIN, + link_to_path = paths.join(out_rf_path, "bin", script_name), + link_to_file = None, + package = package, + version = version_str, + venv_path = script_name, + files = out_files, + )) + return symlinks + def _compute_wheel_venv_symlinks( ctx, out, + out_rf_path, + site_packages_rf_path, *, + package, + version_str, + dist_info_dir, is_windows, should_precompile, keep_source): - package, version_str = _get_package_and_version( - ctx.attr.extracted_files, - ctx.attr.tags, - ) pyc_tag = ( ctx.toolchains[TARGET_TOOLCHAIN_TYPE].py3_runtime.pyc_tag if should_precompile else "" ) - tree_files = _collect_installed_tree_files( - ctx.attr.extracted_files, - ctx.attr.shebang_scripts, - is_windows = is_windows, - keep_source = keep_source, - pyc_tag = pyc_tag, - ) - venv_symlinks = list(get_venv_symlinks( - ctx, - tree_files, - package, - version_str, - site_packages_root = "site-packages", - namespace_package_files = ctx.attr.namespace_package_files, - tree_artifact = out, - )) + if ctx.attr.extracted_files: + tree_files = _collect_installed_tree_files( + ctx.attr.extracted_files, + ctx.attr.shebang_scripts, + is_windows = is_windows, + keep_source = keep_source, + should_precompile = should_precompile, + pyc_tag = pyc_tag, + ) + venv_symlinks = list(get_venv_symlinks( + ctx, + tree_files, + package, + version_str, + site_packages_root = "site-packages", + namespace_package_files = ctx.attr.namespace_package_files, + tree_artifact = out, + )) + else: + venv_symlinks = _venv_symlinks_for_site_paths( + ctx, + depset([out]), + out_rf_path, + site_packages_rf_path, + package, + version_str, + dist_info_dir, + is_windows, + keep_source, + should_precompile = should_precompile, + pyc_tag = pyc_tag, + ) extra_data_files = [f for f in ctx.files.data if not f.is_directory] if extra_data_files: site_packages_root = paths.normalize(paths.join( @@ -297,7 +397,10 @@ def _compute_wheel_venv_symlinks( def _run_wheel_install_action( ctx, *, + built_info, installed_info, + dist_info_dir, + use_site_paths, effective_srcs_exclude, effective_data_exclude, is_windows, @@ -309,6 +412,8 @@ def _run_wheel_install_action( keep_source, exec_tools_toolchain): out = ctx.actions.declare_directory(ctx.attr.name) + entry_points_out = ctx.actions.declare_file(ctx.attr.name + ".dist_info/entry_points.txt") + wheel_input = built_info.directory if built_info != None else ctx.file.wheel args = ctx.actions.args() if installed_info != None and installed_info.wheel == None: @@ -323,13 +428,16 @@ def _run_wheel_install_action( underlying_wheel = None else: install_source_file = ( - installed_info.wheel if installed_info != None else ctx.file.wheel + installed_info.wheel if installed_info != None else wheel_input ) args.add_all("--wheel", [install_source_file], expand_directories = False) underlying_wheel = None if ctx.files.post_install_patches else install_source_file args.add_all("--directory", [out], expand_directories = False) + args.add("--entry-points-output", entry_points_out) args.add("--target-os", "windows" if is_windows else "unix") + if dist_info_dir: + args.add("--dist-info-dir", dist_info_dir) for pattern in effective_srcs_exclude: args.add("--srcs-exclude-glob", pattern) for pattern in effective_data_exclude: @@ -342,7 +450,11 @@ def _run_wheel_install_action( args.add("--patch", patch_file) if venvs_site_packages_enabled: args.add("--validate-site-packages") - for sp in _expected_site_paths_from_extracted(ctx.attr.extracted_files): + args.add("--validation-source", "site_paths" if use_site_paths else "extracted_files") + expected_paths = ( + ctx.attr.site_paths + ctx.attr.namespace_package_files if use_site_paths else _expected_site_paths_from_extracted(ctx.attr.extracted_files) + ) + for sp in expected_paths: args.add("--expected-site-path", sp) extra_tools = [] @@ -366,12 +478,12 @@ def _run_wheel_install_action( "PYTHONPATH": helper_files[0].dirname, }, inputs = [install_source_file] + ctx.files.post_install_patches + helper_files, - outputs = [out], + outputs = [out, entry_points_out], tools = extra_tools, mnemonic = "PyWheelInstall", progress_message = "Installing wheel {} into %{{output}}".format(install_source_file.basename), ) - return out, underlying_wheel + return out, entry_points_out, underlying_wheel def _py_wheel_library_impl(ctx): exec_tools_toolchain = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN_TYPE] @@ -395,6 +507,46 @@ def _py_wheel_library_impl(ctx): ) invalidation_mode = _resolve_invalidation_mode(ctx) if should_precompile else "" + built_info = ( + ctx.attr.wheel[PyBuiltWheelInfo] if PyBuiltWheelInfo in ctx.attr.wheel else None + ) + tag_package, tag_version = _get_package_and_version( + ctx.attr.extracted_files, + ctx.attr.tags, + ) + whl_package = None + whl_version = None + if built_info == None and ctx.file.wheel.basename.endswith(".whl") and ctx.file.wheel.basename.count("-") >= 4: + parsed_whl = parse_whl_name(ctx.file.wheel.basename) + whl_package = normalize_name(parsed_whl.distribution) + parsed_ver = version.parse(parsed_whl.version, strict = False) + whl_version = parsed_ver.string if parsed_ver else None + + use_site_paths = bool(ctx.attr.site_paths) or not ctx.attr.extracted_files + package = _resolve_meta_value( + normalize_name(ctx.attr.distribution) if ctx.attr.distribution else None, + built_info.distribution if built_info else None, + tag_package, + whl_package, + use_site_paths, + ) + version_str = _resolve_meta_value( + version.normalize(ctx.attr.version) if ctx.attr.version else None, + built_info.version if built_info else None, + tag_version, + whl_version, + use_site_paths, + ) + if (ctx.attr.site_paths or (venvs_site_packages_enabled and use_site_paths)) and (not package or not version_str): + fail( + "py_wheel_library requires distribution and version (via PyBuiltWheelInfo " + + "or the distribution and version attributes) when site_paths is used or " + + "extracted_files is empty in venvs_site_packages mode", + ) + dist_info_dir = ( + "{}-{}.dist-info".format(package, version_str) if use_site_paths and package and version_str else "" + ) + installed_info = ( ctx.attr.wheel[PyInstalledWheelInfo] if PyInstalledWheelInfo in ctx.attr.wheel else None ) @@ -416,20 +568,26 @@ def _py_wheel_library_impl(ctx): tuple(effective_srcs_exclude), tuple(effective_data_exclude), create_namespace_inits, + dist_info_dir, ) can_reuse_installed = ( installed_info != None and + installed_info.entry_points != None and not ctx.files.post_install_patches and installed_info.settings_key == settings_key ) if can_reuse_installed: out = installed_info.directory + entry_points_out = installed_info.entry_points underlying_wheel = installed_info.wheel else: - out, underlying_wheel = _run_wheel_install_action( + out, entry_points_out, underlying_wheel = _run_wheel_install_action( ctx, + built_info = built_info, installed_info = installed_info, + dist_info_dir = dist_info_dir, + use_site_paths = use_site_paths, effective_srcs_exclude = effective_srcs_exclude, effective_data_exclude = effective_data_exclude, is_windows = is_windows, @@ -450,6 +608,11 @@ def _py_wheel_library_impl(ctx): venv_symlinks = _compute_wheel_venv_symlinks( ctx, out, + out_rf_path, + site_packages_rf_path, + package = package, + version_str = version_str, + dist_info_dir = dist_info_dir, is_windows = is_windows, should_precompile = should_precompile, keep_source = keep_source, @@ -464,6 +627,13 @@ def _py_wheel_library_impl(ctx): runfiles = runfiles.build(ctx) cc_info = collect_cc_info(ctx) + has_pyi = ( + _has_pyi_files(ctx.attr.extracted_files, keep_source = keep_source) or + _has_pyi_files( + ["site-packages/" + sp.strip("/") for sp in ctx.attr.site_paths if sp.strip("/")], + keep_source = keep_source, + ) + ) py_info, builtins_py_info = create_py_info( ctx, original_sources = [out], @@ -473,7 +643,7 @@ def _py_wheel_library_impl(ctx): implicit_pyc_source_files = [], imports = imports, venv_symlinks = venv_symlinks, - pyi_files = [out] if _has_pyi_files(ctx.attr.extracted_files, keep_source = keep_source) else [], + pyi_files = [out] if has_pyi else [], srcs_version = "ALL", pyi_deps = [], ) @@ -483,6 +653,7 @@ def _py_wheel_library_impl(ctx): py_info, PyInstalledWheelInfo( directory = out, + entry_points = entry_points_out, keep_source = keep_source, precompiled = should_precompile, settings_key = settings_key, @@ -490,7 +661,10 @@ def _py_wheel_library_impl(ctx): ), create_instrumented_files_info(ctx, source_attributes = []), PyCcLinkParamsInfo(cc_info = cc_info), - create_output_group_info(py_info.transitive_sources, extra_groups = {}), + create_output_group_info( + py_info.transitive_sources, + extra_groups = {"dist_info": depset([entry_points_out])}, + ), ] if builtins_py_info: providers.append(builtins_py_info) @@ -518,10 +692,13 @@ settings (`--//tools/precompiler:execution_requirements` and `TreeArtifact` (and `PyInfo.direct_original_sources` / `transitive_original_sources` reference that sourceless directory, so static analysis and coverage tools will not see `.py` sources for the wheel). When -`.pyi` stubs or `py.typed` markers are present (`py.typed` without `.pyi` files -is ignored when sources are omitted), `PyInfo.direct_pyi_files` / +`.pyi` stubs or `py.typed` markers are present in `extracted_files` (or +explicitly listed in `site_paths` for action-built sdists; `py.typed` without +`.pyi` files is ignored when sources are omitted), `PyInfo.direct_pyi_files` / `transitive_pyi_files` contains the installed wheel `TreeArtifact` directory; -consumers must handle directory `File`s. +consumers must handle directory `File`s. For action-built sdists whose +`site_paths` only list package directory names, `pyi_files` is empty unless +`.pyi` or `py.typed` paths are included in `site_paths`. ::: """, attrs = PY_WHEEL_LIBRARY_ATTRS, diff --git a/python/private/pypi/py_wheel_transform.bzl b/python/private/pypi/py_wheel_transform.bzl index 1bece1af4c..0a2dd2beff 100644 --- a/python/private/pypi/py_wheel_transform.bzl +++ b/python/private/pypi/py_wheel_transform.bzl @@ -5,9 +5,13 @@ load("//python/private:common.bzl", "actions_run") load("//python/private:py_internal.bzl", _py_builtins = "py_internal") load("//python/private:rule_builders.bzl", "ruleb") load("//python/private:toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE") +load(":py_sdist_wheel.bzl", "PyBuiltWheelInfo") def _py_wheel_transform_impl(ctx): - wheel_input = ctx.file.wheel + built_info = ( + ctx.attr.wheel[PyBuiltWheelInfo] if PyBuiltWheelInfo in ctx.attr.wheel else None + ) + wheel_input = built_info.directory if built_info != None else ctx.file.wheel out_wheel_dir = ctx.actions.declare_directory(ctx.attr.name + ".whldir") args = ctx.actions.args() @@ -44,7 +48,16 @@ def _py_wheel_transform_impl(ctx): progress_message = "Transforming wheel %{input} into %{output}", ) - return [DefaultInfo(files = depset([out_wheel_dir]))] + providers = [DefaultInfo(files = depset([out_wheel_dir]))] + if built_info != None: + providers.append( + PyBuiltWheelInfo( + directory = out_wheel_dir, + distribution = built_info.distribution, + version = built_info.version, + ), + ) + return providers py_wheel_transform = ruleb.Rule( implementation = _py_wheel_transform_impl, @@ -79,7 +92,11 @@ wrapper's `RUNFILES_*` and `PYTHONSAFEPATH`/`PYTHONPATH`/`PYTHONHOME` launcher v "wheel": lambda: attrb.Label( mandatory = True, allow_single_file = True, - doc = "The input `.whl` file or `.whldir` TreeArtifact directory.", + doc = """ +The input `.whl` file, `.whldir` TreeArtifact directory, or `py_sdist_wheel` +target (for action-built sdists in a pip hub, pass `@pypi//:whldir` rather +than `:whl`). +""", ), "_wheel_layout": lambda: attrb.Label( allow_single_file = True, diff --git a/python/private/pypi/wheel_layout.py b/python/private/pypi/wheel_layout.py index cbab8e93cb..ec882f6504 100644 --- a/python/private/pypi/wheel_layout.py +++ b/python/private/pypi/wheel_layout.py @@ -160,18 +160,30 @@ def _resolve_wheel_entry_dest( name: str, parts: list[str], expected_data_dir: str, + *, + wheel_dist_info_dir: str = "", + target_dist_info_dir: str = "", ) -> str | None: if parts[-1] in _BAZEL_REPO_FILES: return None - if parts[0] != expected_data_dir: - return f"site-packages/{name}" - if len(parts) <= 2: - return None - dest_prefix = _DATA_CATEGORIES.get(parts[1]) - if not dest_prefix: - raise ValueError(f"Unknown .data category {parts[1]!r} in wheel entry {name!r}") - rel_path = "/".join(parts[2:]) - return f"{dest_prefix}/{rel_path}" if rel_path else None + if parts[0] == expected_data_dir: + if len(parts) <= 2: + return None + dest_prefix = _DATA_CATEGORIES.get(parts[1]) + if not dest_prefix: + raise ValueError( + f"Unknown .data category {parts[1]!r} in wheel entry {name!r}" + ) + rel_path = "/".join(parts[2:]) + return f"{dest_prefix}/{rel_path}" if rel_path else None + if ( + wheel_dist_info_dir + and target_dist_info_dir + and target_dist_info_dir != wheel_dist_info_dir + and parts[0] == wheel_dist_info_dir + ): + return f"site-packages/{target_dist_info_dir}/{'/'.join(parts[1:])}" + return f"site-packages/{name}" def _map_wheel_entry_destination( @@ -179,6 +191,9 @@ def _map_wheel_entry_destination( parts: list[str], expected_data_dir: str, seen_destinations: dict[str, str], + *, + wheel_dist_info_dir: str = "", + target_dist_info_dir: str = "", ) -> str | None: """Map a wheel archive entry to its installed relative path under the TreeArtifact root. @@ -186,7 +201,13 @@ def _map_wheel_entry_destination( distinct wheel entries map to the same destination path (checked case-insensitively across platforms). """ - dest_rel = _resolve_wheel_entry_dest(name, parts, expected_data_dir) + dest_rel = _resolve_wheel_entry_dest( + name, + parts, + expected_data_dir, + wheel_dist_info_dir=wheel_dist_info_dir, + target_dist_info_dir=target_dist_info_dir, + ) if dest_rel is None: return None @@ -211,6 +232,8 @@ def plan_wheel_entries( entry_points_content: str | None, srcs_exclude_rx: list[re.Pattern[str]], data_exclude_rx: list[re.Pattern[str]], + *, + target_dist_info_dir: str = "", ) -> Iterator[tuple[zipfile.ZipInfo, str, bool]]: """Yield `(info, dest_rel_or_script_name, is_shebang_script)` for wheel entries.""" expected_data_dir = expected_data_dir_for_dist_info(dist_info_dir) @@ -228,6 +251,8 @@ def plan_wheel_entries( parts, expected_data_dir, seen_destinations, + wheel_dist_info_dir=dist_info_dir, + target_dist_info_dir=target_dist_info_dir, ) if dest_rel is None: continue diff --git a/python/private/pypi/wheel_record_rewriter.py b/python/private/pypi/wheel_record_rewriter.py index deaad65be9..cc579f166b 100644 --- a/python/private/pypi/wheel_record_rewriter.py +++ b/python/private/pypi/wheel_record_rewriter.py @@ -19,10 +19,20 @@ def rewrite_record_lines( target_os: str, data_dir_basename: str, rewritten_scripts: Set[str], + *, + wheel_dist_info_dir: str = "", + target_dist_info_dir: str = "", ) -> list[str]: - """Rewrite `.data//...` RECORD lines to installed paths.""" - data_prefix = data_dir_basename + "/" - quoted_data_prefix = '"' + data_prefix + """Rewrite `.data//...` and renamed `.dist-info/` RECORD lines.""" + data_prefix = (data_dir_basename + "/") if data_dir_basename else "" + quoted_data_prefix = ('"' + data_prefix) if data_prefix else "" + rename_dist = bool( + wheel_dist_info_dir + and target_dist_info_dir + and wheel_dist_info_dir != target_dist_info_dir + ) + dist_prefix = (wheel_dist_info_dir + "/") if rename_dist else "" + quoted_dist_prefix = ('"' + dist_prefix) if rename_dist else "" if target_os == "windows": data_repl = "../../" @@ -41,10 +51,19 @@ def rewrite_record_lines( for raw_line in lines: line = raw_line.rstrip("\r\n") - if line.startswith(quoted_data_prefix): + if rename_dist and line.startswith(quoted_dist_prefix): + out_lines.append( + f'"{target_dist_info_dir}/{line[len(quoted_dist_prefix) :]}\n' + ) + continue + if rename_dist and line.startswith(dist_prefix): + out_lines.append(f"{target_dist_info_dir}/{line[len(dist_prefix) :]}\n") + continue + + if data_prefix and line.startswith(quoted_data_prefix): quote = '"' rest = line[len(quoted_data_prefix) :] - elif line.startswith(data_prefix): + elif data_prefix and line.startswith(data_prefix): quote = "" rest = line[len(data_prefix) :] else: diff --git a/tests/pypi/py_wheel_library/BUILD.bazel b/tests/pypi/py_wheel_library/BUILD.bazel index d4c72c9f76..67a5935a64 100644 --- a/tests/pypi/py_wheel_library/BUILD.bazel +++ b/tests/pypi/py_wheel_library/BUILD.bazel @@ -243,7 +243,7 @@ py_sdist_wheel( ":classic_bdep", ":custom_build_helper", ":custom_shadow_dep", - ":demo_wheel_lib", + ":demo_wheel_lib_venv", ], config_settings_list = { "--global-option": [ @@ -303,6 +303,26 @@ whl_filegroup( whl = ":sdist_demo-0.5.0.whldir", ) +py_wheel_transform( + name = "transformed_sdist_demo_whl", + env = {"EXPECTED_SH_ENV": "from_transform"}, + transform = ":sh_transform", + wheel = ":sdist_demo-0.5.0.whldir", +) + +py_wheel_library( + name = "sdist_demo_wheel_lib", + site_paths = ["sdist_demo"], + wheel = ":transformed_sdist_demo_whl", +) + +py_wheel_library( + name = "sdist_demo_wheel_lib_venv", + experimental_venvs_site_packages = "//python/config_settings:venvs_site_packages", + site_paths = ["sdist_demo"], + wheel = ":sdist_demo-0.5.0.whldir", +) + genrule( name = "gen_cext_sdist", outs = ["sdist_cext-0.1.0.tar.gz"], @@ -342,6 +362,10 @@ filegroup( py_wheel_library( name = "sdist_cext_wheel_lib", + site_paths = [ + "cext_demo", + "sdist_cext", + ], target_compatible_with = select({ "@platforms//os:windows": ["@platforms//:incompatible"], "//conditions:default": [], @@ -403,6 +427,7 @@ py_test( }, deps = [ ":demo_wheel_lib", + ":sdist_demo_wheel_lib", ], ) @@ -425,6 +450,7 @@ py_reconfig_test( venvs_site_packages = "yes", deps = [ ":demo_wheel_lib_venv", + ":sdist_demo_wheel_lib_venv", ], ) @@ -445,6 +471,7 @@ py_reconfig_test( precompile_flag = "enabled", deps = [ ":demo_wheel_lib", + ":sdist_demo_wheel_lib", ], ) @@ -468,6 +495,7 @@ py_reconfig_test( venvs_site_packages = "yes", deps = [ ":demo_wheel_lib_venv", + ":sdist_demo_wheel_lib_venv", ], ) @@ -522,5 +550,6 @@ py_reconfig_test( venvs_site_packages = "yes", deps = [ ":demo_wheel_lib_reuse_omit", + ":sdist_demo_wheel_lib_venv", ], ) diff --git a/tests/pypi/py_wheel_library/py_wheel_library_bzl_tests.bzl b/tests/pypi/py_wheel_library/py_wheel_library_bzl_tests.bzl index 437c318363..65767d675b 100644 --- a/tests/pypi/py_wheel_library/py_wheel_library_bzl_tests.bzl +++ b/tests/pypi/py_wheel_library/py_wheel_library_bzl_tests.bzl @@ -68,11 +68,29 @@ def _test_pyi_files_exposed_when_stubs_present(name): "site-packages/foo/__init__.py", ], ) + rt_util.helper_target( + py_wheel_library, + name = name + "_sdist_site_paths", + wheel = name + "_whl", + distribution = "foo", + version = "1.0", + site_paths = ["foo"], + ) + rt_util.helper_target( + py_wheel_library, + name = name + "_sdist_site_paths_with_pyi", + wheel = name + "_whl", + distribution = "foo", + version = "1.0", + site_paths = ["foo", "foo/__init__.pyi"], + ) analysis_test( name = name, impl = _test_pyi_files_exposed_when_stubs_present_impl, targets = { "no_stubs": name + "_no_stubs", + "sdist_site_paths": name + "_sdist_site_paths", + "sdist_site_paths_with_pyi": name + "_sdist_site_paths_with_pyi", "with_py_typed": name + "_with_py_typed", "with_py_typed_omit": name + "_with_py_typed_omit", "with_pyi": name + "_with_pyi", @@ -105,6 +123,18 @@ def _test_pyi_files_exposed_when_stubs_present_impl(env, targets): factory = py_info_subject, ) no_stubs_info.direct_pyi_files().contains_exactly([]) + sdist_site_paths_info = env.expect.that_target(targets.sdist_site_paths).provider( + PyInfo, + factory = py_info_subject, + ) + sdist_site_paths_info.direct_pyi_files().contains_exactly([]) + sdist_site_paths_with_pyi_info = env.expect.that_target(targets.sdist_site_paths_with_pyi).provider( + PyInfo, + factory = py_info_subject, + ) + sdist_site_paths_with_pyi_info.direct_pyi_files().contains_exactly([ + "{package}/" + targets.sdist_site_paths_with_pyi.label.name, + ]) _tests.append(_test_pyi_files_exposed_when_stubs_present) @@ -153,6 +183,12 @@ def _test_precompile_flags_and_providers_impl(env, targets): default_info.transitive_sources().contains_exactly([ "{package}/" + targets.default.label.name, ]) + default_t.output_group("dist_info").contains_exactly([ + "{package}/" + targets.default.label.name + ".dist_info/entry_points.txt", + ]) + default_t.action_generating("{package}/" + targets.default.label.name).argv().contains_at_least([ + "--entry-points-output", + ]) enabled_t = env.expect.that_target(targets.enabled) enabled_info = enabled_t.provider(PyInfo, factory = py_info_subject) diff --git a/tests/pypi/py_wheel_library/py_wheel_library_test.py b/tests/pypi/py_wheel_library/py_wheel_library_test.py index f6c33be20a..8041a80ec0 100644 --- a/tests/pypi/py_wheel_library/py_wheel_library_test.py +++ b/tests/pypi/py_wheel_library/py_wheel_library_test.py @@ -73,6 +73,12 @@ def test_post_install_patch_and_transform(self): self.assertTrue(copied_exec.is_file()) self.assertTrue(os.access(copied_exec, os.X_OK)) + def test_sdist_wheel_and_pre_build_patch(self): + sys.dont_write_bytecode = True + sdist_demo = importlib.import_module("sdist_demo") + self.assertEqual(sdist_demo.GREETING, "patched_before_build") + self.assertEqual(importlib.metadata.version("sdist-demo"), "0.5.0") + if __name__ == "__main__": unittest.main() diff --git a/tests/pypi/py_wheel_library/sh_transform.sh b/tests/pypi/py_wheel_library/sh_transform.sh index d72469ae01..8813044d55 100755 --- a/tests/pypi/py_wheel_library/sh_transform.sh +++ b/tests/pypi/py_wheel_library/sh_transform.sh @@ -17,4 +17,8 @@ for var in RUNFILES_DIR RUNFILES_MANIFEST_FILE PYTHONSAFEPATH PYTHONPATH PYTHONH fi done +if [[ "$(basename "${WHEEL_FILE}")" != *-*-*-*-*.whl ]]; then + echo "ERROR: WHEEL_FILE basename is not a canonical PEP 427 wheel filename: $(basename "${WHEEL_FILE}")" >&2 + exit 1 +fi cp "${WHEEL_FILE}" "${WHEEL_OUTPUT_ROOT}/$(basename "${WHEEL_FILE}")" diff --git a/tests/pypi/py_wheel_library/wheel_layout_and_installer_test.py b/tests/pypi/py_wheel_library/wheel_layout_and_installer_test.py index 51f2ff97aa..74f43ac291 100644 --- a/tests/pypi/py_wheel_library/wheel_layout_and_installer_test.py +++ b/tests/pypi/py_wheel_library/wheel_layout_and_installer_test.py @@ -851,6 +851,80 @@ def test_wheel_files_extractor(self): [("site-packages/fgpkg/missing.txt", root / "extracted/m.txt")], ) + def test_site_paths_validation_and_dist_info_rename(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + whl = _write_zip( + tmp_path / "Google.Pkg-1.0-py3-none-any.whl", + { + "Google.Pkg-1.0.dist-info/METADATA": "Name: Google.Pkg\nVersion: 1.0\n", + "Google.Pkg-1.0.dist-info/WHEEL": "Wheel-Version: 1.0\nTag: py3-none-any\n", + "Google.Pkg-1.0.dist-info/RECORD": ( + "Google.Pkg-1.0.dist-info/METADATA,sha256=abc,10\n" + "google/protobuf/__init__.py,sha256=def,6\n" + ), + "google/protobuf/__init__.py": "X = 1\n", + "google/_upb/__init__.py": "Y = 2\n", + }, + ) + with self.assertRaisesRegex(ValueError, "uncovered"): + action_whl_installer.install_wheel( + wheel_path=whl, + dest_dir=tmp_path / "out_empty", + expected_site_paths=[], + ) + with self.assertRaisesRegex(ValueError, "google/_upb/__init__.py"): + action_whl_installer.install_wheel( + wheel_path=whl, + dest_dir=tmp_path / "out_uncovered", + expected_site_paths=["google/protobuf"], + ) + with self.assertRaisesRegex(ValueError, "_missing_helper.py"): + action_whl_installer.install_wheel( + wheel_path=whl, + dest_dir=tmp_path / "out_missing", + expected_site_paths=[ + "google/protobuf", + "google/_upb", + "_missing_helper.py", + ], + ) + out_ok = tmp_path / "out_ok" + ep_out_empty = tmp_path / "out_ok.dist_info/entry_points.txt" + action_whl_installer.install_wheel( + wheel_path=whl, + dest_dir=out_ok, + entry_points_output=ep_out_empty, + expected_site_paths=["google/protobuf", "google/_upb"], + dist_info_dir_override="google_pkg-1.0.dist-info", + ) + record = ( + out_ok / "site-packages/google_pkg-1.0.dist-info/RECORD" + ).read_text(encoding="utf-8") + self.assertIn("google_pkg-1.0.dist-info/METADATA,sha256=abc,10\n", record) + self.assertNotIn("Google.Pkg-1.0.dist-info/", record) + self.assertEqual(ep_out_empty.read_text(encoding="utf-8"), "") + + whl_ep = _write_zip( + tmp_path / "Ep.Pkg-1.0-py3-none-any.whl", + { + "Ep.Pkg-1.0.dist-info/METADATA": "Name: Ep.Pkg\nVersion: 1.0\n", + "Ep.Pkg-1.0.dist-info/entry_points.txt": "[console_scripts]\nep_cli = ep_pkg:main\n", + "ep_pkg/__init__.py": "def main(): pass\n", + }, + ) + ep_out = tmp_path / "out_ep.dist_info/entry_points.txt" + action_whl_installer.install_wheel( + wheel_path=whl_ep, + dest_dir=tmp_path / "out_ep", + entry_points_output=ep_out, + dist_info_dir_override="ep_pkg-1.0.dist-info", + ) + self.assertEqual( + ep_out.read_text(encoding="utf-8"), + "[console_scripts]\nep_cli = ep_pkg:main\n", + ) + if __name__ == "__main__": unittest.main()