From c806ec3fa3cb2818a6f46da6dcd2bad6523342bb Mon Sep 17 00:00:00 2001 From: Zanie Date: Wed, 17 Jan 2024 14:53:25 -0600 Subject: [PATCH] Add `--no-binary` and `--no-binary-package ` --- Cargo.lock | 1 + crates/puffin-cli/src/commands/pip_compile.rs | 3 +- crates/puffin-cli/src/commands/pip_install.rs | 17 +- crates/puffin-cli/src/commands/pip_sync.rs | 18 +- crates/puffin-cli/src/commands/venv.rs | 2 + crates/puffin-cli/src/main.rs | 34 ++- crates/puffin-cli/tests/pip_install.rs | 197 ++++++++++++++++++ crates/puffin-cli/tests/pip_sync.rs | 41 ++++ crates/puffin-dev/src/build.rs | 2 + crates/puffin-dev/src/install_many.rs | 18 +- crates/puffin-dev/src/resolve_cli.rs | 2 + crates/puffin-dev/src/resolve_many.rs | 2 + crates/puffin-dispatch/src/lib.rs | 10 +- .../src/distribution_database.rs | 24 ++- crates/puffin-installer/src/lib.rs | 3 +- crates/puffin-installer/src/plan.rs | 25 ++- crates/puffin-resolver/src/finder.rs | 73 ++++--- crates/puffin-resolver/src/resolver/mod.rs | 1 + .../puffin-resolver/src/resolver/provider.rs | 6 +- crates/puffin-resolver/src/version_map.rs | 14 ++ crates/puffin-resolver/tests/resolver.rs | 6 +- crates/puffin-traits/Cargo.toml | 1 + crates/puffin-traits/src/lib.rs | 34 +++ 23 files changed, 485 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a3e3cdb587f2..f086c32c728b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2730,6 +2730,7 @@ dependencies = [ "pep508_rs", "puffin-cache", "puffin-interpreter", + "puffin-normalize", "tokio", ] diff --git a/crates/puffin-cli/src/commands/pip_compile.rs b/crates/puffin-cli/src/commands/pip_compile.rs index 5f3d0ead91f70..67182b8224e84 100644 --- a/crates/puffin-cli/src/commands/pip_compile.rs +++ b/crates/puffin-cli/src/commands/pip_compile.rs @@ -22,7 +22,7 @@ use platform_tags::Tags; use puffin_cache::Cache; use puffin_client::{FlatIndex, FlatIndexClient, RegistryClientBuilder}; use puffin_dispatch::BuildDispatch; -use puffin_installer::Downloader; +use puffin_installer::{Downloader, NoBinary}; use puffin_interpreter::{Interpreter, PythonVersion}; use puffin_normalize::{ExtraName, PackageName}; use puffin_resolver::{ @@ -196,6 +196,7 @@ pub(crate) async fn pip_compile( interpreter.sys_executable().to_path_buf(), setup_py, no_build, + &NoBinary::None, ) .with_options(options); diff --git a/crates/puffin-cli/src/commands/pip_install.rs b/crates/puffin-cli/src/commands/pip_install.rs index 15cdf02b924f5..a9961d19560b6 100644 --- a/crates/puffin-cli/src/commands/pip_install.rs +++ b/crates/puffin-cli/src/commands/pip_install.rs @@ -21,7 +21,7 @@ use puffin_cache::Cache; use puffin_client::{FlatIndex, FlatIndexClient, RegistryClient, RegistryClientBuilder}; use puffin_dispatch::BuildDispatch; use puffin_installer::{ - BuiltEditable, Downloader, Plan, Planner, Reinstall, ResolvedEditable, SitePackages, + BuiltEditable, Downloader, NoBinary, Plan, Planner, Reinstall, ResolvedEditable, SitePackages, }; use puffin_interpreter::{Interpreter, Virtualenv}; use puffin_normalize::PackageName; @@ -51,6 +51,7 @@ pub(crate) async fn pip_install( link_mode: LinkMode, setup_py: SetupPyStrategy, no_build: bool, + no_binary: &NoBinary, strict: bool, exclude_newer: Option>, cache: Cache, @@ -164,6 +165,7 @@ pub(crate) async fn pip_install( venv.python_executable(), setup_py, no_build, + no_binary, ) .with_options(options); @@ -240,6 +242,7 @@ pub(crate) async fn pip_install( venv.python_executable(), setup_py, no_build, + no_binary, ) }; @@ -249,6 +252,7 @@ pub(crate) async fn pip_install( editables, site_packages, reinstall, + no_binary, link_mode, &index_locations, tags, @@ -451,6 +455,7 @@ async fn install( built_editables: Vec, site_packages: SitePackages<'_>, reinstall: &Reinstall, + no_binary: &NoBinary, link_mode: LinkMode, index_urls: &IndexLocations, tags: &Tags, @@ -478,7 +483,15 @@ async fn install( extraneous: _, } = Planner::with_requirements(&requirements) .with_editable_requirements(editables) - .build(site_packages, reinstall, index_urls, cache, venv, tags) + .build( + site_packages, + reinstall, + no_binary, + index_urls, + cache, + venv, + tags, + ) .context("Failed to determine installation plan")?; // Nothing to do. diff --git a/crates/puffin-cli/src/commands/pip_sync.rs b/crates/puffin-cli/src/commands/pip_sync.rs index 5303b74f38b1b..99e921541d026 100644 --- a/crates/puffin-cli/src/commands/pip_sync.rs +++ b/crates/puffin-cli/src/commands/pip_sync.rs @@ -12,7 +12,9 @@ use platform_tags::Tags; use puffin_cache::Cache; use puffin_client::{FlatIndex, FlatIndexClient, RegistryClient, RegistryClientBuilder}; use puffin_dispatch::BuildDispatch; -use puffin_installer::{Downloader, Plan, Planner, Reinstall, ResolvedEditable, SitePackages}; +use puffin_installer::{ + Downloader, NoBinary, Plan, Planner, Reinstall, ResolvedEditable, SitePackages, +}; use puffin_interpreter::Virtualenv; use puffin_resolver::InMemoryIndex; use puffin_traits::{InFlight, SetupPyStrategy}; @@ -33,6 +35,7 @@ pub(crate) async fn pip_sync( index_locations: IndexLocations, setup_py: SetupPyStrategy, no_build: bool, + no_binary: &NoBinary, strict: bool, cache: Cache, mut printer: Printer, @@ -89,6 +92,7 @@ pub(crate) async fn pip_sync( venv.python_executable(), setup_py, no_build, + no_binary, ); // Determine the set of installed packages. @@ -121,6 +125,7 @@ pub(crate) async fn pip_sync( .build( site_packages, reinstall, + no_binary, &index_locations, &cache, &venv, @@ -163,9 +168,14 @@ pub(crate) async fn pip_sync( FlatIndex::from_entries(entries, tags) }; - let wheel_finder = - puffin_resolver::DistFinder::new(tags, &client, venv.interpreter(), &flat_index) - .with_reporter(FinderReporter::from(printer).with_length(remote.len() as u64)); + let wheel_finder = puffin_resolver::DistFinder::new( + tags, + &client, + venv.interpreter(), + &flat_index, + no_binary, + ) + .with_reporter(FinderReporter::from(printer).with_length(remote.len() as u64)); let resolution = wheel_finder.resolve(&remote).await?; let s = if resolution.len() == 1 { "" } else { "s" }; diff --git a/crates/puffin-cli/src/commands/venv.rs b/crates/puffin-cli/src/commands/venv.rs index 4860e21f24f08..7ba6e4139e514 100644 --- a/crates/puffin-cli/src/commands/venv.rs +++ b/crates/puffin-cli/src/commands/venv.rs @@ -6,6 +6,7 @@ use anyhow::Result; use fs_err as fs; use miette::{Diagnostic, IntoDiagnostic}; use owo_colors::OwoColorize; +use puffin_installer::NoBinary; use thiserror::Error; use distribution_types::{DistributionMetadata, IndexLocations, Name}; @@ -158,6 +159,7 @@ async fn venv_impl( venv.python_executable(), SetupPyStrategy::default(), true, + &NoBinary::None, ); // Resolve the seed packages. diff --git a/crates/puffin-cli/src/main.rs b/crates/puffin-cli/src/main.rs index 4d6645c0ab188..f5b4afcdf5fa6 100644 --- a/crates/puffin-cli/src/main.rs +++ b/crates/puffin-cli/src/main.rs @@ -10,7 +10,7 @@ use owo_colors::OwoColorize; use distribution_types::{FlatIndexLocation, IndexLocations, IndexUrl}; use puffin_cache::{Cache, CacheArgs}; -use puffin_installer::Reinstall; +use puffin_installer::{NoBinary, Reinstall}; use puffin_interpreter::PythonVersion; use puffin_normalize::{ExtraName, PackageName}; use puffin_resolver::{PreReleaseMode, ResolutionMode}; @@ -287,6 +287,20 @@ struct PipSyncArgs { #[clap(long)] no_build: bool, + /// Don't install pre-built wheels. + /// + /// When enabled, all installed packages will be installed from a source distribution. The resolver + /// will still use pre-built wheels for metadata. + #[clap(long)] + no_binary: bool, + + /// Don't install pre-built wheels for a specific package. + /// + /// When enabled, the specified packages will be installed from a source distribution. The resolver + /// will still use pre-built wheels for metadata. + #[clap(long)] + no_binary_package: Vec, + /// Validate the virtual environment after completing the installation, to detect packages with /// missing dependencies or other issues. #[clap(long)] @@ -397,6 +411,20 @@ struct PipInstallArgs { #[clap(long)] no_build: bool, + /// Don't install pre-built wheels. + /// + /// When enabled, all installed packages will be installed from a source distribution. The resolver + /// will still use pre-built wheels for metadata. + #[clap(long)] + no_binary: bool, + + /// Don't install pre-built wheels for a specific package. + /// + /// When enabled, the specified packages will be installed from a source distribution. The resolver + /// will still use pre-built wheels for metadata. + #[clap(long)] + no_binary_package: Vec, + /// Validate the virtual environment after completing the installation, to detect packages with /// missing dependencies or other issues. #[clap(long)] @@ -597,6 +625,7 @@ async fn inner() -> Result { .map(RequirementsSource::from) .collect::>(); let reinstall = Reinstall::from_args(args.reinstall, args.reinstall_package); + let no_binary = NoBinary::from_args(args.no_binary, args.no_binary_package); commands::pip_sync( &sources, &reinstall, @@ -608,6 +637,7 @@ async fn inner() -> Result { SetupPyStrategy::Pep517 }, args.no_build, + &no_binary, args.strict, cache, printer, @@ -648,6 +678,7 @@ async fn inner() -> Result { ExtrasSpecification::Some(&args.extra) }; let reinstall = Reinstall::from_args(args.reinstall, args.reinstall_package); + let no_binary = NoBinary::from_args(args.no_binary, args.no_binary_package); commands::pip_install( &requirements, &constraints, @@ -664,6 +695,7 @@ async fn inner() -> Result { SetupPyStrategy::Pep517 }, args.no_build, + &no_binary, args.strict, args.exclude_newer, cache, diff --git a/crates/puffin-cli/tests/pip_install.rs b/crates/puffin-cli/tests/pip_install.rs index 1ef1dac961647..dd9fa2d6cf774 100644 --- a/crates/puffin-cli/tests/pip_install.rs +++ b/crates/puffin-cli/tests/pip_install.rs @@ -743,3 +743,200 @@ fn reinstall_build_system() -> Result<()> { Ok(()) } + +/// Install a package without using pre-built wheels. +#[test] +fn install_no_binary() -> Result<()> { + let temp_dir = assert_fs::TempDir::new()?; + let cache_dir = assert_fs::TempDir::new()?; + let venv = create_venv_py312(&temp_dir, &cache_dir); + + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip") + .arg("install") + .arg("Flask") + .arg("--no-binary") + .arg("--strict") + .arg("--cache-dir") + .arg(cache_dir.path()) + .arg("--exclude-newer") + .arg(EXCLUDE_NEWER) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 7 packages in [TIME] + Downloaded 7 packages in [TIME] + Installed 7 packages in [TIME] + + blinker==1.7.0 + + click==8.1.7 + + flask==3.0.0 + + itsdangerous==2.1.2 + + jinja2==3.1.2 + + markupsafe==2.1.3 + + werkzeug==3.0.1 + "###); + }); + + assert_command(&venv, "import flask", &temp_dir).success(); + + Ok(()) +} + +/// Install a package without using pre-built wheels for a subset of packages. +#[test] +fn install_no_binary_subset() -> Result<()> { + let temp_dir = assert_fs::TempDir::new()?; + let cache_dir = assert_fs::TempDir::new()?; + let venv = create_venv_py312(&temp_dir, &cache_dir); + + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip") + .arg("install") + .arg("Flask") + .arg("--no-binary-package") + .arg("click") + .arg("--no-binary-package") + .arg("flask") + .arg("--strict") + .arg("--cache-dir") + .arg(cache_dir.path()) + .arg("--exclude-newer") + .arg(EXCLUDE_NEWER) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 7 packages in [TIME] + Downloaded 7 packages in [TIME] + Installed 7 packages in [TIME] + + blinker==1.7.0 + + click==8.1.7 + + flask==3.0.0 + + itsdangerous==2.1.2 + + jinja2==3.1.2 + + markupsafe==2.1.3 + + werkzeug==3.0.1 + "###); + }); + + assert_command(&venv, "import flask", &temp_dir).success(); + + Ok(()) +} + +/// Install a package without using pre-built wheels. +#[test] +fn reinstall_no_binary() -> Result<()> { + let temp_dir = assert_fs::TempDir::new()?; + let cache_dir = assert_fs::TempDir::new()?; + let venv = create_venv_py312(&temp_dir, &cache_dir); + + // The first installation should use a pre-built wheel + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip") + .arg("install") + .arg("Flask") + .arg("--strict") + .arg("--cache-dir") + .arg(cache_dir.path()) + .arg("--exclude-newer") + .arg(EXCLUDE_NEWER) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 7 packages in [TIME] + Downloaded 7 packages in [TIME] + Installed 7 packages in [TIME] + + blinker==1.7.0 + + click==8.1.7 + + flask==3.0.0 + + itsdangerous==2.1.2 + + jinja2==3.1.2 + + markupsafe==2.1.3 + + werkzeug==3.0.1 + "###); + }); + + assert_command(&venv, "import flask", &temp_dir).success(); + + // Running installation again with `--no-binary` should be a no-op + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip") + .arg("install") + .arg("Flask") + .arg("--no-binary") + .arg("--strict") + .arg("--cache-dir") + .arg(cache_dir.path()) + .arg("--exclude-newer") + .arg(EXCLUDE_NEWER) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Audited 1 package in [TIME] + "###); + }); + + assert_command(&venv, "import flask", &temp_dir).success(); + + // With `--reinstall`, `--no-binary` should have an affect + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip") + .arg("install") + .arg("Flask") + .arg("--no-binary") + .arg("--reinstall-package") + .arg("Flask") + .arg("--strict") + .arg("--cache-dir") + .arg(cache_dir.path()) + .arg("--exclude-newer") + .arg(EXCLUDE_NEWER) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 7 packages in [TIME] + Downloaded 1 package in [TIME] + Installed 1 package in [TIME] + - flask==3.0.0 + + flask==3.0.0 + "###); + }); + + assert_command(&venv, "import flask", &temp_dir).success(); + Ok(()) +} diff --git a/crates/puffin-cli/tests/pip_sync.rs b/crates/puffin-cli/tests/pip_sync.rs index a5230a836428f..9761272190517 100644 --- a/crates/puffin-cli/tests/pip_sync.rs +++ b/crates/puffin-cli/tests/pip_sync.rs @@ -1026,6 +1026,47 @@ fn install_numpy_py38() -> Result<()> { Ok(()) } +/// Install a package without using pre-built wheels. +#[test] +fn install_no_binary() -> Result<()> { + let temp_dir = assert_fs::TempDir::new()?; + let cache_dir = assert_fs::TempDir::new()?; + let venv = create_venv_py312(&temp_dir, &cache_dir); + + let requirements_txt = temp_dir.child("requirements.txt"); + requirements_txt.touch()?; + requirements_txt.write_str("MarkupSafe==2.1.3")?; + + insta::with_settings!({ + filters => INSTA_FILTERS.to_vec() + }, { + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .arg("pip") + .arg("sync") + .arg("requirements.txt") + .arg("--no-binary") + .arg("--strict") + .arg("--cache-dir") + .arg(cache_dir.path()) + .env("VIRTUAL_ENV", venv.as_os_str()) + .current_dir(&temp_dir), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Downloaded 1 package in [TIME] + Installed 1 package in [TIME] + + markupsafe==2.1.3 + "###); + }); + + check_command(&venv, "import markupsafe", &temp_dir); + + Ok(()) +} + #[test] fn warn_on_yanked_version() -> Result<()> { let temp_dir = assert_fs::TempDir::new()?; diff --git a/crates/puffin-dev/src/build.rs b/crates/puffin-dev/src/build.rs index 82f61fadff519..719f467a7dd62 100644 --- a/crates/puffin-dev/src/build.rs +++ b/crates/puffin-dev/src/build.rs @@ -11,6 +11,7 @@ use puffin_build::{SourceBuild, SourceBuildContext}; use puffin_cache::{Cache, CacheArgs}; use puffin_client::{FlatIndex, RegistryClientBuilder}; use puffin_dispatch::BuildDispatch; +use puffin_installer::NoBinary; use puffin_interpreter::Virtualenv; use puffin_resolver::InMemoryIndex; use puffin_traits::{BuildContext, BuildKind, InFlight, SetupPyStrategy}; @@ -72,6 +73,7 @@ pub(crate) async fn build(args: BuildArgs) -> Result { venv.python_executable(), setup_py, false, + &NoBinary::None, ); let builder = SourceBuild::setup( diff --git a/crates/puffin-dev/src/install_many.rs b/crates/puffin-dev/src/install_many.rs index fc3bb51fcaf63..5872cb47b8866 100644 --- a/crates/puffin-dev/src/install_many.rs +++ b/crates/puffin-dev/src/install_many.rs @@ -21,7 +21,7 @@ use puffin_cache::{Cache, CacheArgs}; use puffin_client::{FlatIndex, RegistryClient, RegistryClientBuilder}; use puffin_dispatch::BuildDispatch; use puffin_distribution::RegistryWheelIndex; -use puffin_installer::Downloader; +use puffin_installer::{Downloader, NoBinary}; use puffin_interpreter::Virtualenv; use puffin_normalize::PackageName; use puffin_resolver::{DistFinder, InMemoryIndex}; @@ -77,6 +77,7 @@ pub(crate) async fn install_many(args: InstallManyArgs) -> Result<()> { venv.python_executable(), setup_py, args.no_build, + &NoBinary::None, ); for (idx, requirements) in requirements.chunks(100).enumerate() { @@ -109,11 +110,16 @@ async fn install_chunk( venv: &Virtualenv, index_locations: &IndexLocations, ) -> Result<()> { - let resolution: Vec<_> = - DistFinder::new(tags, client, venv.interpreter(), &FlatIndex::default()) - .resolve_stream(requirements) - .collect() - .await; + let resolution: Vec<_> = DistFinder::new( + tags, + client, + venv.interpreter(), + &FlatIndex::default(), + &NoBinary::None, + ) + .resolve_stream(requirements) + .collect() + .await; let (resolution, failures): (FxHashMap, Vec<_>) = resolution.into_iter().partition_result(); for failure in &failures { diff --git a/crates/puffin-dev/src/resolve_cli.rs b/crates/puffin-dev/src/resolve_cli.rs index 87aacc543c718..1ad2239f2d807 100644 --- a/crates/puffin-dev/src/resolve_cli.rs +++ b/crates/puffin-dev/src/resolve_cli.rs @@ -15,6 +15,7 @@ use platform_host::Platform; use puffin_cache::{Cache, CacheArgs}; use puffin_client::{FlatIndex, FlatIndexClient, RegistryClientBuilder}; use puffin_dispatch::BuildDispatch; +use puffin_installer::NoBinary; use puffin_interpreter::Virtualenv; use puffin_resolver::{InMemoryIndex, Manifest, ResolutionOptions, Resolver}; use puffin_traits::{InFlight, SetupPyStrategy}; @@ -79,6 +80,7 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> { venv.python_executable(), SetupPyStrategy::default(), args.no_build, + &NoBinary::None, ); // Copied from `BuildDispatch` diff --git a/crates/puffin-dev/src/resolve_many.rs b/crates/puffin-dev/src/resolve_many.rs index a11968cf07ac4..5e765f0a692d3 100644 --- a/crates/puffin-dev/src/resolve_many.rs +++ b/crates/puffin-dev/src/resolve_many.rs @@ -7,6 +7,7 @@ use clap::Parser; use futures::StreamExt; use indicatif::ProgressStyle; use itertools::Itertools; +use puffin_installer::NoBinary; use tokio::time::Instant; use tracing::{info, info_span, Span}; use tracing_indicatif::span_ext::IndicatifSpanExt; @@ -91,6 +92,7 @@ pub(crate) async fn resolve_many(args: ResolveManyArgs) -> Result<()> { venv.python_executable(), setup_py, args.no_build, + &NoBinary::None, ); let build_dispatch = Arc::new(build_dispatch); diff --git a/crates/puffin-dispatch/src/lib.rs b/crates/puffin-dispatch/src/lib.rs index 9ca1e2844b1c5..20779bade6c16 100644 --- a/crates/puffin-dispatch/src/lib.rs +++ b/crates/puffin-dispatch/src/lib.rs @@ -14,7 +14,7 @@ use pep508_rs::Requirement; use puffin_build::{SourceBuild, SourceBuildContext}; use puffin_cache::Cache; use puffin_client::{FlatIndex, RegistryClient}; -use puffin_installer::{Downloader, Installer, Plan, Planner, Reinstall, SitePackages}; +use puffin_installer::{Downloader, Installer, NoBinary, Plan, Planner, Reinstall, SitePackages}; use puffin_interpreter::{Interpreter, Virtualenv}; use puffin_resolver::{InMemoryIndex, Manifest, ResolutionOptions, Resolver}; use puffin_traits::{BuildContext, BuildKind, InFlight, SetupPyStrategy}; @@ -32,6 +32,7 @@ pub struct BuildDispatch<'a> { base_python: PathBuf, setup_py: SetupPyStrategy, no_build: bool, + no_binary: &'a NoBinary, source_build_context: SourceBuildContext, options: ResolutionOptions, } @@ -49,6 +50,7 @@ impl<'a> BuildDispatch<'a> { base_python: PathBuf, setup_py: SetupPyStrategy, no_build: bool, + no_binary: &'a NoBinary, ) -> Self { Self { client, @@ -61,6 +63,7 @@ impl<'a> BuildDispatch<'a> { base_python, setup_py, no_build, + no_binary, source_build_context: SourceBuildContext::default(), options: ResolutionOptions::default(), } @@ -92,6 +95,10 @@ impl<'a> BuildContext for BuildDispatch<'a> { self.no_build } + fn no_binary(&self) -> &NoBinary { + self.no_binary + } + fn setup_py_strategy(&self) -> SetupPyStrategy { self.setup_py } @@ -157,6 +164,7 @@ impl<'a> BuildContext for BuildDispatch<'a> { } = Planner::with_requirements(&resolution.requirements()).build( site_packages, &Reinstall::None, + &NoBinary::None, self.index_locations, self.cache(), venv, diff --git a/crates/puffin-distribution/src/distribution_database.rs b/crates/puffin-distribution/src/distribution_database.rs index 21530c6d5c332..8b3d5748a9ce2 100644 --- a/crates/puffin-distribution/src/distribution_database.rs +++ b/crates/puffin-distribution/src/distribution_database.rs @@ -20,7 +20,7 @@ use puffin_cache::{Cache, CacheBucket, WheelCache}; use puffin_client::RegistryClient; use puffin_extract::unzip_no_seek; use puffin_git::GitSource; -use puffin_traits::BuildContext; +use puffin_traits::{BuildContext, NoBinary}; use pypi_types::Metadata21; use crate::download::{BuiltWheel, UnzippedWheel}; @@ -51,6 +51,8 @@ pub enum DistributionDatabaseError { Join(#[from] JoinError), #[error("Building source distributions is disabled")] NoBuild, + #[error("Using pre-built wheels is disabled")] + NoBinary, } /// A cached high-level interface to convert distributions (a requirement resolved to a location) @@ -103,13 +105,25 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> } /// Either fetch the wheel or fetch and build the source distribution + /// + /// If `no_remote_wheel` is set, the wheel will be built from a source distribution + /// even if compatible pre-built wheels are available. #[instrument(skip(self))] pub async fn get_or_build_wheel( &self, dist: Dist, ) -> Result { + let no_binary = match self.build_context.no_binary() { + NoBinary::None => false, + NoBinary::All => true, + NoBinary::Packages(packages) => packages.contains(dist.name()), + }; match &dist { Dist::Built(BuiltDist::Registry(wheel)) => { + if no_binary { + return Err(DistributionDatabaseError::NoBinary); + } + let url = match &wheel.file.url { FileLocation::RelativeUrl(base, url) => base .join_relative(url) @@ -173,6 +187,10 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> } Dist::Built(BuiltDist::DirectUrl(wheel)) => { + if no_binary { + return Err(DistributionDatabaseError::NoBinary); + } + let reader = self.client.stream_external(&wheel.url).await?; // Download and unzip the wheel to a temporary directory. @@ -200,6 +218,10 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context> } Dist::Built(BuiltDist::Path(wheel)) => { + if no_binary { + return Err(DistributionDatabaseError::NoBinary); + } + let cache_entry = self.cache.entry( CacheBucket::Wheels, WheelCache::Url(&wheel.url).remote_wheel_dir(wheel.name().as_ref()), diff --git a/crates/puffin-installer/src/lib.rs b/crates/puffin-installer/src/lib.rs index 840f4aee4aced..0d458858a8c40 100644 --- a/crates/puffin-installer/src/lib.rs +++ b/crates/puffin-installer/src/lib.rs @@ -2,9 +2,10 @@ pub use downloader::{Downloader, Reporter as DownloadReporter}; pub use editable::{BuiltEditable, ResolvedEditable}; pub use installer::{Installer, Reporter as InstallReporter}; pub use plan::{Plan, Planner, Reinstall}; +// TODO(zanieb): Just import this properly everywhere else +pub use puffin_traits::NoBinary; pub use site_packages::SitePackages; pub use uninstall::uninstall; - mod downloader; mod editable; mod installer; diff --git a/crates/puffin-installer/src/plan.rs b/crates/puffin-installer/src/plan.rs index 2b5efad45dc42..6ededa01db7ed 100644 --- a/crates/puffin-installer/src/plan.rs +++ b/crates/puffin-installer/src/plan.rs @@ -1,6 +1,7 @@ use std::hash::BuildHasherDefault; use anyhow::{bail, Result}; +use puffin_traits::NoBinary; use rustc_hash::FxHashSet; use tracing::{debug, warn}; @@ -49,6 +50,7 @@ impl<'a> Planner<'a> { self, mut site_packages: SitePackages, reinstall: &Reinstall, + no_binary: &NoBinary, index_locations: &IndexLocations, cache: &Cache, venv: &Virtualenv, @@ -127,6 +129,13 @@ impl<'a> Planner<'a> { Reinstall::Packages(packages) => packages.contains(&requirement.name), }; + // Check if installation of a binary version of the package should be allowed. + let no_binary = match no_binary { + NoBinary::None => false, + NoBinary::All => true, + NoBinary::Packages(packages) => packages.contains(&requirement.name), + }; + if reinstall { // If necessary, purge the cached distributions. debug!("Purging cached distributions for: {requirement}"); @@ -191,7 +200,7 @@ impl<'a> Planner<'a> { } Some(VersionOrUrl::Url(url)) => { match Dist::from_url(requirement.name.clone(), url.clone())? { - Dist::Built(BuiltDist::Registry(_wheel)) => { + Dist::Built(BuiltDist::Registry(_)) => { // Nothing to do. } Dist::Source(SourceDist::Registry(_)) => { @@ -205,6 +214,13 @@ impl<'a> Planner<'a> { ); } + if no_binary { + bail!( + "A URL dependency points to a wheel which conflicts with `--no-binary`: {}", + wheel.url + ); + } + // Find the exact wheel from the cache, since we know the filename in // advance. let cache_entry = cache.entry( @@ -233,6 +249,13 @@ impl<'a> Planner<'a> { ); } + if no_binary { + bail!( + "A path dependency points to a wheel which conflicts with `--no-binary`: {}", + wheel.url + ); + } + // Find the exact wheel from the cache, since we know the filename in // advance. let cache_entry = cache.entry( diff --git a/crates/puffin-resolver/src/finder.rs b/crates/puffin-resolver/src/finder.rs index 7d73ab676824e..41bf1f853c684 100644 --- a/crates/puffin-resolver/src/finder.rs +++ b/crates/puffin-resolver/src/finder.rs @@ -4,6 +4,7 @@ use anyhow::Result; use futures::{stream, Stream, StreamExt, TryStreamExt}; +use puffin_traits::NoBinary; use rustc_hash::FxHashMap; use distribution_filename::DistFilename; @@ -22,6 +23,7 @@ pub struct DistFinder<'a> { reporter: Option>, interpreter: &'a Interpreter, flat_index: &'a FlatIndex, + no_binary: &'a NoBinary, } impl<'a> DistFinder<'a> { @@ -31,6 +33,7 @@ impl<'a> DistFinder<'a> { client: &'a RegistryClient, interpreter: &'a Interpreter, flat_index: &'a FlatIndex, + no_binary: &'a NoBinary, ) -> Self { Self { tags, @@ -38,6 +41,7 @@ impl<'a> DistFinder<'a> { reporter: None, interpreter, flat_index, + no_binary, } } @@ -112,7 +116,10 @@ impl<'a> DistFinder<'a> { Ok(Resolution::new(resolution)) } - /// select a version that satisfies the requirement, preferring wheels to source distributions. + /// Select a version that satisfies the requirement. + /// + /// Wheels are preferred to source distributions unless `no_binary` excludes wheels + /// for the requirement. fn select( &self, requirement: &Requirement, @@ -120,6 +127,12 @@ impl<'a> DistFinder<'a> { index: &IndexUrl, flat_index: Option<&FlatDistributions>, ) -> Option { + let no_binary = match self.no_binary { + NoBinary::None => false, + NoBinary::All => true, + NoBinary::Packages(packages) => packages.contains(&requirement.name), + }; + // Prioritize the flat index by initializing the "best" matches with its entries. let matching_override = if let Some(flat_index) = flat_index { match &requirement.version_or_url { @@ -159,36 +172,38 @@ impl<'a> DistFinder<'a> { continue; } - // Find the most-compatible wheel - for (wheel, file) in files.wheels { - // Only add dists compatible with the python version. - // This is relevant for source dists which give no other indication of their - // compatibility and wheels which may be tagged `py3-none-any` but - // have `requires-python: ">=3.9"` - if !file - .requires_python - .as_ref() - .map_or(true, |requires_python| { - requires_python.contains(self.interpreter.version()) - }) - { - continue; - } - - best_version = Some(version.clone()); - if let Some(priority) = wheel.compatibility(self.tags) { - if best_wheel + if !no_binary { + // Find the most-compatible wheel + for (wheel, file) in files.wheels { + // Only add dists compatible with the python version. + // This is relevant for source dists which give no other indication of their + // compatibility and wheels which may be tagged `py3-none-any` but + // have `requires-python: ">=3.9"` + if !file + .requires_python .as_ref() - .map_or(true, |(.., existing)| priority > *existing) + .map_or(true, |requires_python| { + requires_python.contains(self.interpreter.version()) + }) { - best_wheel = Some(( - Dist::from_registry( - DistFilename::WheelFilename(wheel), - file, - index.clone(), - ), - priority, - )); + continue; + } + + best_version = Some(version.clone()); + if let Some(priority) = wheel.compatibility(self.tags) { + if best_wheel + .as_ref() + .map_or(true, |(.., existing)| priority > *existing) + { + best_wheel = Some(( + Dist::from_registry( + DistFilename::WheelFilename(wheel), + file, + index.clone(), + ), + priority, + )); + } } } } diff --git a/crates/puffin-resolver/src/resolver/mod.rs b/crates/puffin-resolver/src/resolver/mod.rs index cd5dcd193cfbc..afee87c936a1b 100644 --- a/crates/puffin-resolver/src/resolver/mod.rs +++ b/crates/puffin-resolver/src/resolver/mod.rs @@ -101,6 +101,7 @@ impl<'a, Context: BuildContext + Send + Sync> Resolver<'a, DefaultResolverProvid .iter() .chain(manifest.constraints.iter()) .collect(), + build_context.no_binary(), ); Self::new_custom_io( manifest, diff --git a/crates/puffin-resolver/src/resolver/provider.rs b/crates/puffin-resolver/src/resolver/provider.rs index caacbabbd30e0..4d49d682a8927 100644 --- a/crates/puffin-resolver/src/resolver/provider.rs +++ b/crates/puffin-resolver/src/resolver/provider.rs @@ -10,7 +10,7 @@ use platform_tags::Tags; use puffin_client::{FlatIndex, RegistryClient}; use puffin_distribution::{DistributionDatabase, DistributionDatabaseError}; use puffin_normalize::PackageName; -use puffin_traits::BuildContext; +use puffin_traits::{BuildContext, NoBinary}; use pypi_types::Metadata21; use crate::python_requirement::PythonRequirement; @@ -55,6 +55,7 @@ pub struct DefaultResolverProvider<'a, Context: BuildContext + Send + Sync> { python_requirement: PythonRequirement<'a>, exclude_newer: Option>, allowed_yanks: AllowedYanks, + no_binary: &'a NoBinary, } impl<'a, Context: BuildContext + Send + Sync> DefaultResolverProvider<'a, Context> { @@ -67,6 +68,7 @@ impl<'a, Context: BuildContext + Send + Sync> DefaultResolverProvider<'a, Contex python_requirement: PythonRequirement<'a>, exclude_newer: Option>, allowed_yanks: AllowedYanks, + no_binary: &'a NoBinary, ) -> Self { Self { client, @@ -76,6 +78,7 @@ impl<'a, Context: BuildContext + Send + Sync> DefaultResolverProvider<'a, Contex python_requirement, exclude_newer, allowed_yanks, + no_binary, } } } @@ -99,6 +102,7 @@ impl<'a, Context: BuildContext + Send + Sync> ResolverProvider &self.allowed_yanks, self.exclude_newer.as_ref(), self.flat_index.get(package_name).cloned(), + self.no_binary, )), Err( err @ (puffin_client::Error::PackageNotFound(_) diff --git a/crates/puffin-resolver/src/version_map.rs b/crates/puffin-resolver/src/version_map.rs index 6e87dbf943f87..08ba2338f133c 100644 --- a/crates/puffin-resolver/src/version_map.rs +++ b/crates/puffin-resolver/src/version_map.rs @@ -10,6 +10,7 @@ use pep440_rs::Version; use platform_tags::Tags; use puffin_client::{FlatDistributions, SimpleMetadata}; use puffin_normalize::PackageName; +use puffin_traits::NoBinary; use puffin_warnings::warn_user_once; use pypi_types::{Hashes, Yanked}; @@ -33,11 +34,19 @@ impl VersionMap { allowed_yanks: &AllowedYanks, exclude_newer: Option<&DateTime>, flat_index: Option, + no_binary: &NoBinary, ) -> Self { // If we have packages of the same name from find links, gives them priority, otherwise start empty let mut version_map: BTreeMap = flat_index.map(Into::into).unwrap_or_default(); + // Check if binaries are allowed for this package + let no_binary = match no_binary { + NoBinary::None => false, + NoBinary::All => true, + NoBinary::Packages(packages) => packages.contains(package_name), + }; + // Collect compatible distributions. for (version, files) in metadata { for (filename, file) in files.all() { @@ -73,6 +82,11 @@ impl VersionMap { let hash = file.hashes.clone(); match filename { DistFilename::WheelFilename(filename) => { + // If pre-built binaries are disabled, skip this wheel + if no_binary { + continue; + }; + // To be compatible, the wheel must both have compatible tags _and_ have a // compatible Python requirement. let priority = filename.compatibility(tags).filter(|_| { diff --git a/crates/puffin-resolver/tests/resolver.rs b/crates/puffin-resolver/tests/resolver.rs index 74c28fb3d5acd..55ad576478124 100644 --- a/crates/puffin-resolver/tests/resolver.rs +++ b/crates/puffin-resolver/tests/resolver.rs @@ -21,7 +21,7 @@ use puffin_resolver::{ DisplayResolutionGraph, InMemoryIndex, Manifest, PreReleaseMode, ResolutionGraph, ResolutionMode, ResolutionOptions, Resolver, }; -use puffin_traits::{BuildContext, BuildKind, SetupPyStrategy, SourceBuildTrait}; +use puffin_traits::{BuildContext, BuildKind, NoBinary, SetupPyStrategy, SourceBuildTrait}; // Exclude any packages uploaded after this date. static EXCLUDE_NEWER: Lazy> = Lazy::new(|| { @@ -54,6 +54,10 @@ impl BuildContext for DummyContext { false } + fn no_binary(&self) -> &NoBinary { + &NoBinary::None + } + fn setup_py_strategy(&self) -> SetupPyStrategy { SetupPyStrategy::default() } diff --git a/crates/puffin-traits/Cargo.toml b/crates/puffin-traits/Cargo.toml index ed80f348d8682..5bc5595a0b729 100644 --- a/crates/puffin-traits/Cargo.toml +++ b/crates/puffin-traits/Cargo.toml @@ -18,6 +18,7 @@ once-map = { path = "../once-map" } pep508_rs = { path = "../pep508-rs" } puffin-cache = { path = "../puffin-cache" } puffin-interpreter = { path = "../puffin-interpreter" } +puffin-normalize = { path = "../puffin-normalize" } anyhow = { workspace = true } tokio = { workspace = true, features = ["sync"] } diff --git a/crates/puffin-traits/src/lib.rs b/crates/puffin-traits/src/lib.rs index 0641e33b1871a..f8955dc8619b0 100644 --- a/crates/puffin-traits/src/lib.rs +++ b/crates/puffin-traits/src/lib.rs @@ -11,6 +11,7 @@ use once_map::OnceMap; use pep508_rs::Requirement; use puffin_cache::Cache; use puffin_interpreter::{Interpreter, Virtualenv}; +use puffin_normalize::PackageName; /// Avoid cyclic crate dependencies between resolver, installer and builder. /// @@ -68,6 +69,9 @@ pub trait BuildContext { /// we can't build them fn no_build(&self) -> bool; + /// Whether using pre-built wheels is disabled. + fn no_binary(&self) -> &NoBinary; + /// The strategy to use when building source distributions that lack a `pyproject.toml`. fn setup_py_strategy(&self) -> SetupPyStrategy; @@ -155,3 +159,33 @@ impl Display for BuildKind { } } } + +#[derive(Debug)] +pub enum NoBinary { + /// Allow installation of any wheel. + None, + + /// Do not allow installation from any wheels. + All, + + /// Do not allow installation from the specific wheels. + Packages(Vec), +} + +impl NoBinary { + /// Determine the binary installation strategy to use. + pub fn from_args(no_binary: bool, no_binary_package: Vec) -> Self { + if no_binary { + Self::All + } else if !no_binary_package.is_empty() { + Self::Packages(no_binary_package) + } else { + Self::None + } + } + + /// Returns `true` if no packages binaries should be excluded from installation. + pub fn is_none(&self) -> bool { + matches!(self, Self::None) + } +}