Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: registry source cleanup #12240

Merged
merged 5 commits into from
Jun 8, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions src/cargo/sources/registry/download.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,7 @@ use cargo_util::Sha256;
use crate::core::PackageId;
use crate::sources::registry::make_dep_prefix;
use crate::sources::registry::MaybeLock;
use crate::sources::registry::{
RegistryConfig, CHECKSUM_TEMPLATE, CRATE_TEMPLATE, LOWER_PREFIX_TEMPLATE, PREFIX_TEMPLATE,
VERSION_TEMPLATE,
};
use crate::sources::registry::RegistryConfig;
use crate::util::auth;
use crate::util::errors::CargoResult;
use crate::util::{Config, Filesystem};
Expand All @@ -17,6 +14,12 @@ use std::io::prelude::*;
use std::io::SeekFrom;
use std::str;

const CRATE_TEMPLATE: &str = "{crate}";
const VERSION_TEMPLATE: &str = "{version}";
const PREFIX_TEMPLATE: &str = "{prefix}";
const LOWER_PREFIX_TEMPLATE: &str = "{lowerprefix}";
const CHECKSUM_TEMPLATE: &str = "{sha256-checksum}";

pub(super) fn filename(pkg: PackageId) -> String {
format!("{}-{}.crate", pkg.name(), pkg.version())
}
Expand Down
24 changes: 13 additions & 11 deletions src/cargo/sources/registry/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
//! [`Dependency`]: crate::core::Dependency

use crate::core::{PackageId, SourceId, Summary};
use crate::sources::registry::{LoadResponse, RegistryData, RegistryPackage, INDEX_V_MAX};
use crate::sources::registry::{LoadResponse, RegistryData, RegistryPackage};
use crate::util::interning::InternedString;
use crate::util::{internal, CargoResult, Config, Filesystem, OptVersionReq, ToSemver};
use anyhow::bail;
Expand All @@ -100,6 +100,13 @@ use std::path::Path;
use std::str;
use std::task::{ready, Poll};

/// The current version of [`SummariesCache`].
const CURRENT_CACHE_VERSION: u8 = 3;

/// The maximum schema version of the `v` field in the index this version of
/// cargo understands. See [`RegistryPackage::v`] for the detail.
const INDEX_V_MAX: u32 = 2;

/// Manager for handling the on-disk index.
///
/// Different kinds of registries store the index differently:
Expand Down Expand Up @@ -202,7 +209,7 @@ pub struct IndexSummary {
///
/// ```text
/// +---------------+----------------------+--------------------+---+
/// | cache version | index format version | index file version | 0 |
/// | cache version | index schema version | index file version | 0 |
/// +---------------+----------------------+--------------------+---+
/// ```
///
Expand All @@ -219,7 +226,7 @@ pub struct IndexSummary {
/// * _cache version_ --- Intended to ensure that there's some level of
/// future compatibility against changes to this cache format so if different
/// versions of Cargo share the same cache they don't get too confused.
/// * _index format version_ --- The version of the raw index file.
/// * _index schema version_ --- The schema version of the raw index file.
epage marked this conversation as resolved.
Show resolved Hide resolved
/// See [`RegistryPackage::v`] for the detail.
/// * _index file version_ --- Tracks when a cache needs to be regenerated.
/// A cache regeneration is required whenever the index file itself updates.
Expand All @@ -231,7 +238,7 @@ pub struct IndexSummary {
/// # Changes between each cache version
///
/// * `1`: The original version.
/// * `2`: Added the "index format version" field so that if the index format
/// * `2`: Added the "index schema version" field so that if the index schema
/// changes, different versions of cargo won't get confused reading each
/// other's caches.
/// * `3`: Bumped the version to work around an issue where multiple versions of
Expand Down Expand Up @@ -697,9 +704,6 @@ impl Summaries {
}
}

/// The current version of [`SummariesCache`].
const CURRENT_CACHE_VERSION: u8 = 3;

impl<'a> SummariesCache<'a> {
/// Deserializes an on-disk cache.
fn parse(data: &'a [u8]) -> CargoResult<SummariesCache<'a>> {
Expand All @@ -712,13 +716,11 @@ impl<'a> SummariesCache<'a> {
}
let index_v_bytes = rest
.get(..4)
.ok_or_else(|| anyhow::anyhow!("cache expected 4 bytes for index version"))?;
.ok_or_else(|| anyhow::anyhow!("cache expected 4 bytes for index schema version"))?;
let index_v = u32::from_le_bytes(index_v_bytes.try_into().unwrap());
if index_v != INDEX_V_MAX {
bail!(
"index format version {} doesn't match the version I know ({})",
index_v,
INDEX_V_MAX
"index schema version {index_v} doesn't match the version I know ({INDEX_V_MAX})",
);
}
let rest = &rest[4..];
Expand Down
22 changes: 6 additions & 16 deletions src/cargo/sources/registry/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -220,15 +220,6 @@ pub const CRATES_IO_HTTP_INDEX: &str = "sparse+https://index.crates.io/";
pub const CRATES_IO_REGISTRY: &str = "crates-io";
pub const CRATES_IO_DOMAIN: &str = "crates.io";

const CRATE_TEMPLATE: &str = "{crate}";
const VERSION_TEMPLATE: &str = "{version}";
const PREFIX_TEMPLATE: &str = "{prefix}";
const LOWER_PREFIX_TEMPLATE: &str = "{lowerprefix}";
const CHECKSUM_TEMPLATE: &str = "{sha256-checksum}";

const MAX_UNPACK_SIZE: u64 = 512 * 1024 * 1024;
const MAX_COMPRESSION_RATIO: usize = 20; // 20:1

/// A [`Source`] implementation for a local or a remote registry.
///
/// This contains common functionality that is shared between each registry
Expand Down Expand Up @@ -303,10 +294,6 @@ pub struct RegistryConfig {
pub auth_required: bool,
}

/// The maximum schema version of the `v` field in the index this version of
/// cargo understands. See [`RegistryPackage::v`] for the detail.
pub(crate) const INDEX_V_MAX: u32 = 2;

/// A single line in the index representing a single version of a package.
#[derive(Deserialize)]
pub struct RegistryPackage<'a> {
Expand Down Expand Up @@ -348,10 +335,10 @@ pub struct RegistryPackage<'a> {
rust_version: Option<InternedString>,
/// The schema version for this entry.
///
/// If this is None, it defaults to version 1. Entries with unknown
/// If this is None, it defaults to version `1`. Entries with unknown
/// versions are ignored.
///
/// Version `2` format adds the `features2` field.
/// Version `2` schema adds the `features2` field.
///
/// This provides a method to safely introduce changes to index entries
/// and allow older versions of cargo to ignore newer entries it doesn't
Expand All @@ -362,7 +349,7 @@ pub struct RegistryPackage<'a> {
/// The intent is that versions older than 1.51 will work with a
/// pre-existing `Cargo.lock`, but they may not correctly process `cargo
/// update` or build a lock from scratch. In that case, cargo may
/// incorrectly select a new package that uses a new index format. A
/// incorrectly select a new package that uses a new index schema. A
/// workaround is to downgrade any packages that are incompatible with the
/// `--precise` flag of `cargo update`.
v: Option<u32>,
Expand Down Expand Up @@ -1076,6 +1063,9 @@ impl<'cfg> Source for RegistrySource<'cfg> {
fn max_unpack_size(config: &Config, size: u64) -> u64 {
const SIZE_VAR: &str = "__CARGO_TEST_MAX_UNPACK_SIZE";
const RATIO_VAR: &str = "__CARGO_TEST_MAX_UNPACK_RATIO";
const MAX_UNPACK_SIZE: u64 = 512 * 1024 * 1024; // 512 MiB
const MAX_COMPRESSION_RATIO: usize = 20; // 20:1

let max_unpack_size = if cfg!(debug_assertions) && config.get_env(SIZE_VAR).is_ok() {
// For integration test only.
config
Expand Down