Skip to content

Commit

Permalink
Slightly optimize cargo vendor
Browse files Browse the repository at this point in the history
I've noticed recently that `cargo vendor` feels really sluggish and
slow, and apparently this is primarily because we delete the registry
caches and re-extract all the tarballs. This commit implements one
possible optimization without changing this, however, which is that
currently we both copy a file and checksum it, but that ends up reading
all the contents twice. Those two functions are now folded into one,
shaving about 3s locally from Cargo's own vendor times.
  • Loading branch information
alexcrichton committed Dec 3, 2020
1 parent c7939ae commit 01f1c01
Showing 1 changed file with 25 additions and 3 deletions.
28 changes: 25 additions & 3 deletions src/cargo/ops/vendor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ use serde::Serialize;
use std::collections::HashSet;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fs;
use std::fs::File;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};

pub struct VendorOptions<'a> {
Expand Down Expand Up @@ -182,6 +184,7 @@ fn sync(
}

let mut sources = BTreeSet::new();
let mut tmp_buf = [0; 64 * 1024];
for (id, pkg) in ids.iter() {
// Next up, copy it to the vendor directory
let src = pkg
Expand Down Expand Up @@ -216,7 +219,7 @@ fn sync(
let pathsource = PathSource::new(src, id.source_id(), config);
let paths = pathsource.list_files(pkg)?;
let mut map = BTreeMap::new();
cp_sources(src, &paths, &dst, &mut map)
cp_sources(src, &paths, &dst, &mut map, &mut tmp_buf)
.chain_err(|| format!("failed to copy over vendored sources for: {}", id))?;

// Finally, emit the metadata about this package
Expand Down Expand Up @@ -299,6 +302,7 @@ fn cp_sources(
paths: &[PathBuf],
dst: &Path,
cksums: &mut BTreeMap<String, String>,
tmp_buf: &mut [u8],
) -> CargoResult<()> {
for p in paths {
let relative = p.strip_prefix(&src).unwrap();
Expand Down Expand Up @@ -334,9 +338,27 @@ fn cp_sources(

paths::create_dir_all(dst.parent().unwrap())?;

paths::copy(&p, &dst)?;
let cksum = Sha256::new().update_path(dst)?.finish_hex();
let cksum = copy_and_checksum(&p, &dst, tmp_buf)?;
cksums.insert(relative.to_str().unwrap().replace("\\", "/"), cksum);
}
Ok(())
}

fn copy_and_checksum(src_path: &Path, dst_path: &Path, buf: &mut [u8]) -> CargoResult<String> {
let mut src = File::open(src_path).chain_err(|| format!("failed to open {:?}", src_path))?;
let mut dst =
File::create(dst_path).chain_err(|| format!("failed to create {:?}", dst_path))?;
let mut cksum = Sha256::new();
loop {
let n = src
.read(buf)
.chain_err(|| format!("failed to read from {:?}", src_path))?;
if n == 0 {
break Ok(cksum.finish_hex());
}
let data = &buf[..n];
cksum.update(data);
dst.write_all(data)
.chain_err(|| format!("failed to write to {:?}", dst_path))?;
}
}

0 comments on commit 01f1c01

Please sign in to comment.