Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions news/4286.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
(pypi) Respect dependency-edge platform markers in `uv.lock` when selecting
packages and artifacts in {obj}`pip.parse`
([#4287](https://github.com/bazel-contrib/rules_python/issues/4287)).
7 changes: 7 additions & 0 deletions python/private/pypi/extension.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -1023,6 +1023,13 @@ the uv.lock file will be used as the primary source for package metadata.

:::{versionadded} 2.2.0
:::

:::{versionchanged} VERSION_NEXT_PATCH
Package platforms follow dependency markers from virtual and editable workspace
roots, including development dependency groups. All recorded optional extras
remain enabled, and their dependency markers are respected. Locks without
workspace roots use package-level resolution markers.
:::
""",
),
"whl_modifications": attr.label_keyed_string_dict(
Expand Down
87 changes: 76 additions & 11 deletions python/private/pypi/parse_requirements.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,79 @@ def _parse_requirements_with_uv_lock(
logger = logger,
)

def _uv_lock_package_dependencies(pkg):
"""Collect regular, optional, and development dependency edges."""
deps = list(pkg.get("dependencies", []))
for table in ["optional-dependencies", "dev-dependencies"]:
for group in pkg.get(table, {}).values():
deps.extend(group)
return deps

def _uv_lock_package_platforms(packages, all_platforms, platforms):
"""Find each lock entry's platforms by walking dependency edges from roots.

Follow all optional dependencies because uv_lock_extras_map enables every
recorded extra. Development dependency groups must also remain available.
"""
by_name = {}
roots = []
dependencies = []
for i, pkg in enumerate(packages):
by_name.setdefault(normalize_name(pkg["name"]), []).append(i)
dependencies.append(_uv_lock_package_dependencies(pkg))
source = pkg.get("source", {})
if "virtual" in source or "editable" in source:
roots.append(i)

result = [[] for _ in packages]
for p in all_platforms:
marker_env = platforms[p].env
compatible = {
i: None
for i, pkg in enumerate(packages)
if not pkg.get("resolution-markers") or evaluate(
" or ".join(pkg["resolution-markers"]),
env = marker_env,
strict = False,
) != False
}
if not roots:
# Exported/synthetic locks without workspace roots retain the
# package-level resolution-marker behavior.
for i in compatible:
result[i].append(p)
continue

visited = {i: None for i in roots if i in compatible}
queue = list(visited)

# Mark packages when enqueuing so cycles and diamonds cannot exceed
# the number of packages. Starlark has no while loops.
for _ in range(len(packages)):
if not queue:
break
i = queue.pop()
result[i].append(p)
for dep in dependencies[i]:
# Unknown values (notably `extra`) may remain as expressions;
# only a definitively false platform marker excludes an edge.
marker = dep.get("marker", "")
if marker and evaluate(marker, env = marker_env, strict = False) == False:
continue
for target in by_name.get(normalize_name(dep["name"]), []):
if target not in compatible or target in visited:
continue

# Forked entries can share a name but differ in identity.
pkg = packages[target]
if "version" in dep and dep["version"] != pkg["version"]:
continue
if "source" in dep and dep["source"] != pkg.get("source", {}):
continue
visited[target] = None
queue.append(target)
return result

def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, platforms = {}):
"""Parse uv.lock JSON and build the same return structs as parse_requirements.

Expand All @@ -168,7 +241,8 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p
if not platforms:
fail("BUG: platforms must be configured")

for pkg in uv_lock["package"]:
package_platforms = _uv_lock_package_platforms(uv_lock["package"], all_platforms, platforms)
for i, pkg in enumerate(uv_lock["package"]):
name = pkg["name"]
version = pkg["version"]
norm_name = normalize_name(name)
Expand All @@ -189,16 +263,7 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p
pkg_extras = sorted(extras_map.get(name, []))
extra_str = "[{}]".format(",".join(pkg_extras)) if pkg_extras else ""

markers = pkg.get("resolution-markers", [])
if markers:
marker_expr = " or ".join(markers)
pkg_platforms = [
p
for p in all_platforms
if evaluate(marker_expr, env = platforms[p].env)
]
else:
pkg_platforms = list(all_platforms)
pkg_platforms = package_platforms[i]

# Prepare candidates
candidates = []
Expand Down
198 changes: 195 additions & 3 deletions tests/pypi/parse_requirements/parse_requirements_tests.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ bar==0.0.1 --hash=sha256:deadb00f
"uv_lock_foo_resolution_markers_dedup": """{"package":[{"name":"foo","source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","resolution-markers":["sys_platform == 'linux'"],"wheels":[{"hash":"sha256:aaa","url":"https://files.pythonhosted.org/packages/foo-0.0.1-cp39-cp39-manylinux_2_17_x86_64.whl"},{"hash":"sha256:bbb","url":"https://files.pythonhosted.org/packages/foo-0.0.1-py3-none-any.whl"}]},{"name":"foo","source":{"registry":"https://pypi.org/simple"},"version":"0.0.2","resolution-markers":["sys_platform == 'darwin'"],"wheels":[{"hash":"sha256:ccc","url":"https://files.pythonhosted.org/packages/foo-0.0.2-cp39-cp39-macosx_11_0_arm64.whl"},{"hash":"sha256:ddd","url":"https://files.pythonhosted.org/packages/foo-0.0.2-py3-none-any.whl"}]}]}""",
"uv_lock_foo_sdist": """{"package":[{"name":"foo","sdist":{"hash":"sha256:feedcafe","url":"https://files.pythonhosted.org/packages/foo-0.0.1.tar.gz"},"source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/foo-0.0.1-py3-none-any.whl"}]}]}""",
"uv_lock_foo_sha512": """{"package":[{"name":"foo","source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha512:deadbeef","url":"https://files.pythonhosted.org/packages/foo-0.0.1-py3-none-any.whl"}]}]}""",
"uv_lock_foo_virtual": """{"package":[{"name":"foo","source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/foo-0.0.1-py3-none-any.whl"}]},{"name":"virtual-pkg","source":{"virtual":true},"version":"0.0.0"}]}""",
"uv_lock_foo_virtual": """{"package":[{"name":"foo","source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/foo-0.0.1-py3-none-any.whl"}]},{"name":"virtual-pkg","source":{"virtual":true},"version":"0.0.0","dependencies":[{"name":"foo"}]}]}""",
"uv_lock_foo_with_extras": """{"package":[{"name":"foo","provides-extras":["extra"],"source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/foo-0.0.1-py3-none-any.whl"}]}]}""",
"uv_lock_git_vcs": """{"package":[{"name":"foo","source":{"git":"https://github.com/org/foo?rev=deadbeef#deadbeef"},"version":"0.1.0"}]}""",
"uv_lock_git_vcs_subdirectory": """{"package":[{"name":"foo","source":{"git":"https://github.com/org/mono.git?subdirectory=python%2Ffoo&branch=main#0123abcd"},"version":"0.1.0+g0123abcd"}]}""",
Expand All @@ -138,11 +138,12 @@ def _make_platforms(platform_names):
)
return platforms

def parse_requirements(debug = False, **kwargs):
def parse_requirements(debug = False, ctx = None, **kwargs):
"""Get requirements by calling the original parse_requirements.

Args:
debug: If True, set verbosity to TRACE.
ctx: An optional context overriding the default test fixtures.
**kwargs: forwarded to the underlying function.

Returns:
Expand All @@ -163,7 +164,7 @@ def parse_requirements(debug = False, **kwargs):
kwargs["platforms"] = _make_platforms(["linux_x86_64"])

return _parse_requirements(
ctx = _mock_ctx(),
ctx = ctx or _mock_ctx(),
logger = repo_utils.logger(struct(
getenv = {
REPO_DEBUG_ENV_VAR: "1",
Expand Down Expand Up @@ -1745,6 +1746,197 @@ def _test_uv_lock_requires_dist_extras(env):

_tests.append(_test_uv_lock_requires_dist_extras)

def _uv_graph_package(name, version = "0.0.1", **kwargs):
return dict({
"name": name,
"source": {"registry": "https://pypi.org/simple"},
"version": version,
"wheels": [{"url": "https://example.org/{}-{}-py3-none-any.whl".format(name, version)}],
}, **kwargs)

def _parse_uv_graph(packages):
return {pkg.name: pkg for pkg in parse_requirements(
ctx = struct(read = lambda _: json.encode({"package": packages})),
uv_lock = "uv.lock",
platforms = {
"linux_x86_64": struct(
env = pep508_env(python_version = "3.11.0", os = "linux", arch = "x86_64"),
whl_abi_tags = ["none", "cp311"],
whl_platform_tags = ["any", "manylinux_*_x86_64"],
),
"osx_aarch64": struct(
env = pep508_env(python_version = "3.11.0", os = "osx", arch = "aarch64"),
whl_abi_tags = ["none", "cp311"],
whl_platform_tags = ["any", "macosx_*_arm64"],
),
},
)}

def _test_uv_lock_dependency_markers(env):
for wheels in [
[{"url": "https://example.org/uvloop-0.0.1-cp311-cp311-manylinux_2_17_x86_64.whl"}],
[{"url": "https://example.org/uvloop-0.0.1-py3-none-any.whl"}],
]:
got = _parse_uv_graph([
_uv_graph_package("root", source = {"virtual": "."}, dependencies = [
{"marker": "sys_platform == 'linux'", "name": "uvloop"},
]),
_uv_graph_package("uvloop", wheels = wheels, sdist = {"url": "https://example.org/uvloop-0.0.1.tar.gz"}),
])
env.expect.that_collection(got["uvloop"].srcs).has_size(1)
env.expect.that_collection(got["uvloop"].srcs[0].target_platforms).contains_exactly(["linux_x86_64"])

_tests.append(_test_uv_lock_dependency_markers)

def _test_uv_lock_transitive_markers(env):
got = _parse_uv_graph([
_uv_graph_package("root", source = {"editable": "."}, dependencies = [
{"marker": "sys_platform == 'linux'", "name": "parent"},
]),
_uv_graph_package("parent", dependencies = [{"name": "child"}]),
_uv_graph_package("child"),
_uv_graph_package("unreachable"),
])
for name in ["parent", "child"]:
env.expect.that_collection(got[name].srcs[0].target_platforms).contains_exactly(["linux_x86_64"])
env.expect.that_bool(got["unreachable"].is_exposed).equals(False)
env.expect.that_collection(got["unreachable"].srcs).has_size(0)

_tests.append(_test_uv_lock_transitive_markers)

def _test_uv_lock_all_extras(env):
got = _parse_uv_graph([
_uv_graph_package("root", source = {"virtual": "."}, dependencies = [
{"name": "parent"},
{"extra": ["fast"], "marker": "sys_platform == 'linux'", "name": "parent"},
]),
_uv_graph_package("parent", **{"optional-dependencies": {
"fast": [{"extra": ["nested"], "name": "child"}],
"unused": [{"name": "optional"}],
}}),
_uv_graph_package("child", **{"optional-dependencies": {
"nested": [
{"name": "leaf"},
{"marker": "sys_platform == 'linux'", "name": "linux_only"},
],
}}),
_uv_graph_package("leaf"),
_uv_graph_package("optional"),
_uv_graph_package("linux_only"),
])

# uv_lock_extras_map enables all extras, even when the edge requesting an
# extra is limited to one platform. Optional edges still honor markers.
for name in ["parent", "child", "leaf", "optional"]:
env.expect.that_collection(got[name].srcs[0].target_platforms).contains_exactly(["linux_x86_64", "osx_aarch64"])
env.expect.that_collection(got["linux_only"].srcs[0].target_platforms).contains_exactly(["linux_x86_64"])

_tests.append(_test_uv_lock_all_extras)

def _test_uv_lock_root_dev_dependencies(env):
for source in [{"virtual": "."}, {"editable": "workspace"}]:
got = _parse_uv_graph([
_uv_graph_package("root", source = source, **{"dev-dependencies": {
"dev": [{"name": "pytest"}],
"lint": [{"marker": "sys_platform == 'linux'", "name": "linter"}],
}}),
_uv_graph_package("pytest", dependencies = [{"name": "child"}]),
_uv_graph_package("child"),
_uv_graph_package("linter"),
_uv_graph_package("unreachable"),
])
for name in ["pytest", "child"]:
env.expect.that_collection(got[name].srcs).has_size(1)
env.expect.that_collection(got[name].srcs[0].target_platforms).contains_exactly(["linux_x86_64", "osx_aarch64"])
env.expect.that_collection(got["linter"].srcs[0].target_platforms).contains_exactly(["linux_x86_64"])
env.expect.that_bool(got["unreachable"].is_exposed).equals(False)

_tests.append(_test_uv_lock_root_dev_dependencies)

def _test_uv_lock_root_optional_dependencies(env):
got = _parse_uv_graph([
_uv_graph_package("root", source = {"virtual": "."}, **{"optional-dependencies": {
"fast": [{"marker": "sys_platform == 'linux'", "name": "child"}],
}}),
_uv_graph_package("child"),
])
env.expect.that_collection(got["child"].srcs[0].target_platforms).contains_exactly(["linux_x86_64"])

_tests.append(_test_uv_lock_root_optional_dependencies)

def _test_uv_lock_multiple_roots_cycles_and_diamonds(env):
# Shared nodes must be marked visited when queued: duplicate entries can
# otherwise exhaust a traversal bounded by the number of graph nodes.
got = _parse_uv_graph([
_uv_graph_package("root", source = {"virtual": "."}, dependencies = [
{"marker": "sys_platform == 'linux'", "name": "left"},
{"marker": "sys_platform == 'linux'", "name": "right"},
]),
_uv_graph_package("workspace", source = {"editable": "workspace"}, dependencies = [
{"marker": "sys_platform == 'darwin'", "name": "leaf"},
]),
_uv_graph_package("left", dependencies = [{"name": "shared"}]),
_uv_graph_package("right", dependencies = [{"name": "shared"}]),
_uv_graph_package("shared", dependencies = [{"name": "left"}, {"name": "leaf"}]),
_uv_graph_package("leaf", dependencies = [{"name": "tail"}]),
_uv_graph_package("tail"),
])
for name in ["left", "right", "shared"]:
env.expect.that_collection(got[name].srcs[0].target_platforms).contains_exactly(["linux_x86_64"])
for name in ["leaf", "tail"]:
env.expect.that_collection(got[name].srcs[0].target_platforms).contains_exactly(["linux_x86_64", "osx_aarch64"])

_tests.append(_test_uv_lock_multiple_roots_cycles_and_diamonds)

def _test_uv_lock_duplicate_dependency_edges(env):
got = _parse_uv_graph([
_uv_graph_package("root", source = {"virtual": "."}, dependencies = [
{"marker": "sys_platform == 'linux'", "name": "tail"},
{"marker": "sys_platform == 'linux'", "name": "parent"},
]),
_uv_graph_package("parent", dependencies = [{"name": "child"}] * 10),
_uv_graph_package("child"),
_uv_graph_package("tail", dependencies = [{"name": "leaf"}]),
_uv_graph_package("leaf"),
])
for name in ["parent", "child", "tail", "leaf"]:
env.expect.that_collection(got[name].srcs[0].target_platforms).contains_exactly(["linux_x86_64"])

_tests.append(_test_uv_lock_duplicate_dependency_edges)

def _test_uv_lock_dependency_identity(env):
got = _parse_uv_graph([
_uv_graph_package("root", source = {"virtual": "."}, dependencies = [
{"name": "fork", "source": {"registry": "https://pypi.org/simple"}, "version": "0.0.1"},
{"name": "scoped", "source": {"registry": "https://pypi.org/simple"}},
]),
_uv_graph_package("fork", **{"resolution-markers": ["sys_platform == 'linux'"]}),
_uv_graph_package("fork", version = "0.0.2", dependencies = [{"name": "unreachable"}]),
_uv_graph_package("scoped"),
_uv_graph_package("scoped", source = {"registry": "https://other.example/simple"}, dependencies = [{"name": "unreachable"}]),
_uv_graph_package("unreachable"),
])
env.expect.that_collection(got["fork"].srcs).has_size(1)
env.expect.that_collection(got["fork"].srcs[0].target_platforms).contains_exactly(["linux_x86_64"])
env.expect.that_collection(got["scoped"].srcs).has_size(1)
env.expect.that_bool(got["unreachable"].is_exposed).equals(False)

_tests.append(_test_uv_lock_dependency_identity)

def _test_uv_lock_partial_extra_markers(env):
got = _parse_uv_graph([
_uv_graph_package("root", source = {"virtual": "."}, dependencies = [
{"marker": "sys_platform == 'linux' and extra == 'fast'", "name": "child"},
{"marker": "sys_platform == 'linux' or extra == 'fast'", "name": "union"},
]),
_uv_graph_package("child", **{"resolution-markers": ["extra == 'fast'"]}),
_uv_graph_package("union", **{"resolution-markers": ["sys_platform == 'linux'", "extra == 'fast'"]}),
])
env.expect.that_collection(got["child"].srcs[0].target_platforms).contains_exactly(["linux_x86_64"])
env.expect.that_collection(got["union"].srcs[0].target_platforms).contains_exactly(["linux_x86_64", "osx_aarch64"])

_tests.append(_test_uv_lock_partial_extra_markers)

def parse_requirements_test_suite(name):
"""Create the test suite.

Expand Down
Loading