Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

sccache now tracks compiler hits/misses on a per language basis. #1895

Merged
merged 3 commits into from
Oct 17, 2023
Merged

sccache now tracks compiler hits/misses on a per language basis. #1895

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c31b376
sccache now tracks compiler hits/misses on a per language basis.
robertmaynard Oct 2, 2023
42483bf
Merge branch 'main' into bug/track_cache_hits_misses_per_language
sylvestre Oct 2, 2023
043d866
Merge branch 'main' into bug/track_cache_hits_misses_per_language
sylvestre Oct 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 5 additions & 50 deletions src/compiler/c.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
use crate::cache::FileObjectSource;
use crate::compiler::{
Cacheable, ColorMode, Compilation, CompileCommand, Compiler, CompilerArguments, CompilerHasher,
CompilerKind, HashResult,
CompilerKind, HashResult, Language,
};
#[cfg(feature = "dist-client")]
use crate::compiler::{DistPackagers, NoopOutputsRewriter};
Expand Down Expand Up @@ -62,18 +62,6 @@ where
compiler: I,
}

#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum Language {
C,
Cxx,
GenericHeader,
CHeader,
CxxHeader,
ObjectiveC,
ObjectiveCxx,
Cuda,
}

/// Artifact produced by a C/C++ compiler.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ArtifactDescriptor {
Expand Down Expand Up @@ -129,43 +117,6 @@ impl ParsedArguments {
}
}

impl Language {
pub fn from_file_name(file: &Path) -> Option<Self> {
match file.extension().and_then(|e| e.to_str()) {
// gcc: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html
Some("c") => Some(Language::C),
// Could be C or C++
Some("h") => Some(Language::GenericHeader),
// TODO i
Some("C") | Some("cc") | Some("cp") | Some("cpp") | Some("CPP") | Some("cxx")
| Some("c++") => Some(Language::Cxx),
// TODO ii
Some("H") | Some("hh") | Some("hp") | Some("hpp") | Some("HPP") | Some("hxx")
| Some("h++") | Some("tcc") => Some(Language::CxxHeader),
Some("m") => Some(Language::ObjectiveC),
// TODO mi
Some("M") | Some("mm") => Some(Language::ObjectiveCxx),
// TODO mii
Some("cu") => Some(Language::Cuda),
e => {
trace!("Unknown source extension: {}", e.unwrap_or("(None)"));
None
}
}
}

pub fn as_str(self) -> &'static str {
match self {
Language::C | Language::CHeader => "c",
Language::Cxx | Language::CxxHeader => "c++",
Language::GenericHeader => "c/c++",
Language::ObjectiveC => "objc",
Language::ObjectiveCxx => "objc++",
Language::Cuda => "cuda",
}
}
}

/// A generic implementation of the `Compilation` trait for C/C++ compilers.
struct CCompilation<I: CCompilerImpl> {
parsed_args: ParsedArguments,
Expand Down Expand Up @@ -443,6 +394,10 @@ where
fn box_clone(&self) -> Box<dyn CompilerHasher<T>> {
Box::new((*self).clone())
}

fn language(&self) -> Language {
self.parsed_args.language
}
}

impl<I: CCompilerImpl> Compilation for CCompilation<I> {
Expand Down
8 changes: 4 additions & 4 deletions src/compiler/clang.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@
#![allow(unused_imports, dead_code, unused_variables)]

use crate::compiler::args::*;
use crate::compiler::c::{
ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
};
use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
use crate::compiler::gcc::ArgData::*;
use crate::compiler::{gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments};
use crate::compiler::{
gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments, Language,
};
use crate::mock_command::{CommandCreator, CommandCreatorSync, RunCommand};
use crate::util::{run_input_output, OsStrExt};
use crate::{counted_array, dist};
Expand Down
71 changes: 66 additions & 5 deletions src/compiler/compiler.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,12 +102,71 @@ pub enum CompilerKind {
Rust,
}

impl CompilerKind {
pub fn lang_kind(&self) -> String {
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum Language {
C,
Cxx,
GenericHeader,
CHeader,
CxxHeader,
ObjectiveC,
ObjectiveCxx,
Cuda,
Rust,
}

impl Language {
pub fn from_file_name(file: &Path) -> Option<Self> {
match file.extension().and_then(|e| e.to_str()) {
// gcc: https://gcc.gnu.org/onlinedocs/gcc/Overall-Options.html
Some("c") => Some(Language::C),
// Could be C or C++
Some("h") => Some(Language::GenericHeader),
// TODO i
Some("C") | Some("cc") | Some("cp") | Some("cpp") | Some("CPP") | Some("cxx")
| Some("c++") => Some(Language::Cxx),
// TODO ii
Some("H") | Some("hh") | Some("hp") | Some("hpp") | Some("HPP") | Some("hxx")
| Some("h++") | Some("tcc") => Some(Language::CxxHeader),
Some("m") => Some(Language::ObjectiveC),
// TODO mi
Some("M") | Some("mm") => Some(Language::ObjectiveCxx),
// TODO mii
Some("cu") => Some(Language::Cuda),
// TODO cy
Some("rs") => Some(Language::Rust),
e => {
trace!("Unknown source extension: {}", e.unwrap_or("(None)"));
None
}
}
}

pub fn as_str(self) -> &'static str {
match self {
CompilerKind::C(CCompilerKind::Nvcc) => "CUDA",
CompilerKind::C(_) => "C/C++",
CompilerKind::Rust => "Rust",
Language::C | Language::CHeader => "c",
Language::Cxx | Language::CxxHeader => "c++",
Language::GenericHeader => "c/c++",
Language::ObjectiveC => "objc",
Language::ObjectiveCxx => "objc++",
Language::Cuda => "cuda",
Language::Rust => "rust",
}
}
}

impl CompilerKind {
pub fn lang_kind(&self, lang: &Language) -> String {
match lang {
Language::C
| Language::CHeader
| Language::Cxx
| Language::CxxHeader
| Language::GenericHeader
| Language::ObjectiveC
| Language::ObjectiveCxx => "C/C++",
Language::Cuda => "CUDA",
Language::Rust => "Rust",
}
.to_string()
}
Expand Down Expand Up @@ -427,6 +486,8 @@ where
fn output_pretty(&self) -> Cow<'_, str>;

fn box_clone(&self) -> Box<dyn CompilerHasher<T>>;

fn language(&self) -> Language;
}

#[cfg(not(feature = "dist-client"))]
Expand Down
6 changes: 2 additions & 4 deletions src/compiler/diab.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,8 @@ use crate::compiler::args::{
ArgDisposition, ArgInfo, ArgToStringResult, ArgsIter, Argument, FromArg, IntoArg,
NormalizedDisposition, PathTransformerFn, SearchableArgInfo,
};
use crate::compiler::c::{
ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
};
use crate::compiler::{Cacheable, ColorMode, CompileCommand, CompilerArguments};
use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
use crate::compiler::{Cacheable, ColorMode, CompileCommand, CompilerArguments, Language};
use crate::errors::*;
use crate::mock_command::{CommandCreatorSync, RunCommand};
use crate::util::{run_input_output, OsStrExt};
Expand Down
8 changes: 4 additions & 4 deletions src/compiler/gcc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,8 @@
// limitations under the License.

use crate::compiler::args::*;
use crate::compiler::c::{
ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
};
use crate::compiler::{clang, Cacheable, ColorMode, CompileCommand, CompilerArguments};
use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
use crate::compiler::{clang, Cacheable, ColorMode, CompileCommand, CompilerArguments, Language};
use crate::mock_command::{CommandCreatorSync, RunCommand};
use crate::util::{run_input_output, OsStrExt};
use crate::{counted_array, dist};
Expand Down Expand Up @@ -367,6 +365,7 @@ where
"objective-c" => Some(Language::ObjectiveC),
"objective-c++" => Some(Language::ObjectiveCxx),
"cu" => Some(Language::Cuda),
"rs" => Some(Language::Rust),
"cuda" => Some(Language::Cuda),
_ => cannot_cache!("-x"),
};
Expand Down Expand Up @@ -616,6 +615,7 @@ fn language_to_gcc_arg(lang: Language) -> Option<&'static str> {
Language::ObjectiveC => Some("objective-c"),
Language::ObjectiveCxx => Some("objective-c++"),
Language::Cuda => Some("cu"),
Language::Rust => None, // Let the compiler decide
Language::GenericHeader => None, // Let the compiler decide
}
}
Expand Down
6 changes: 2 additions & 4 deletions src/compiler/msvc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,9 @@
// limitations under the License.

use crate::compiler::args::*;
use crate::compiler::c::{
ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
};
use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
use crate::compiler::{
clang, gcc, write_temp_file, Cacheable, ColorMode, CompileCommand, CompilerArguments,
clang, gcc, write_temp_file, Cacheable, ColorMode, CompileCommand, CompilerArguments, Language,
};
use crate::mock_command::{CommandCreatorSync, RunCommand};
use crate::util::{run_input_output, OsStrExt};
Expand Down
8 changes: 4 additions & 4 deletions src/compiler/nvcc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@
#![allow(unused_imports, dead_code, unused_variables)]

use crate::compiler::args::*;
use crate::compiler::c::{
ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
};
use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
use crate::compiler::gcc::ArgData::*;
use crate::compiler::{gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments};
use crate::compiler::{
gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments, Language,
};
use crate::mock_command::{CommandCreator, CommandCreatorSync, RunCommand};
use crate::util::{run_input_output, OsStrExt};
use crate::{counted_array, dist};
Expand Down
8 changes: 4 additions & 4 deletions src/compiler/nvhpc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@
#![allow(unused_imports, dead_code, unused_variables)]

use crate::compiler::args::*;
use crate::compiler::c::{
ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments,
};
use crate::compiler::c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments};
use crate::compiler::gcc::ArgData::*;
use crate::compiler::{gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments};
use crate::compiler::{
gcc, write_temp_file, Cacheable, CompileCommand, CompilerArguments, Language,
};
use crate::mock_command::{CommandCreator, CommandCreatorSync, RunCommand};
use crate::util::{run_input_output, OsStrExt};
use crate::{counted_array, dist};
Expand Down
6 changes: 5 additions & 1 deletion src/compiler/rust.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use crate::cache::FileObjectSource;
use crate::compiler::args::*;
use crate::compiler::{
c::ArtifactDescriptor, Cacheable, ColorMode, Compilation, CompileCommand, Compiler,
CompilerArguments, CompilerHasher, CompilerKind, CompilerProxy, HashResult,
CompilerArguments, CompilerHasher, CompilerKind, CompilerProxy, HashResult, Language,
};
#[cfg(feature = "dist-client")]
use crate::compiler::{DistPackagers, OutputsRewriter};
Expand Down Expand Up @@ -1589,6 +1589,10 @@ where
fn box_clone(&self) -> Box<dyn CompilerHasher<T>> {
Box::new((*self).clone())
}

fn language(&self) -> Language {
Language::Rust
}
}

impl Compilation for RustCompilation {
Expand Down
4 changes: 2 additions & 2 deletions src/compiler/tasking_vx.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ use crate::{
ArgDisposition, ArgInfo, ArgToStringResult, ArgsIter, Argument, FromArg, IntoArg,
NormalizedDisposition, PathTransformerFn, SearchableArgInfo,
},
c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, Language, ParsedArguments},
Cacheable, ColorMode, CompileCommand, CompilerArguments,
c::{ArtifactDescriptor, CCompilerImpl, CCompilerKind, ParsedArguments},
Cacheable, ColorMode, CompileCommand, CompilerArguments, Language,
},
counted_array, dist,
errors::*,
Expand Down
21 changes: 11 additions & 10 deletions src/server.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
use crate::cache::{storage_from_config, Storage};
use crate::compiler::{
get_compiler_info, CacheControl, CompileResult, Compiler, CompilerArguments, CompilerHasher,
CompilerKind, CompilerProxy, DistType, MissType,
CompilerKind, CompilerProxy, DistType, Language, MissType,
};
#[cfg(feature = "dist-client")]
use crate::config;
Expand Down Expand Up @@ -1164,6 +1164,7 @@ where
let color_mode = hasher.color_mode();
let me = self.clone();
let kind = compiler.kind();
let lang = hasher.language();
let creator = self.creator.clone();
let storage = self.storage.clone();
let pool = self.rt.clone();
Expand Down Expand Up @@ -1199,12 +1200,12 @@ where
CompileResult::Error => {
debug!("compile result: cache error");

stats.cache_errors.increment(&kind);
stats.cache_errors.increment(&kind, &lang);
}
CompileResult::CacheHit(duration) => {
debug!("compile result: cache hit");

stats.cache_hits.increment(&kind);
stats.cache_hits.increment(&kind, &lang);
stats.cache_read_hit_duration += duration;
}
CompileResult::CacheMiss(miss_type, dist_type, duration, future) => {
Expand All @@ -1229,18 +1230,18 @@ where
stats.cache_timeouts += 1;
}
MissType::CacheReadError => {
stats.cache_errors.increment(&kind);
stats.cache_errors.increment(&kind, &lang);
}
}
stats.cache_misses.increment(&kind);
stats.cache_misses.increment(&kind, &lang);
stats.compiler_write_duration += duration;
debug!("stats after compile result: {stats:?}");
cache_write = Some(future);
}
CompileResult::NotCacheable => {
debug!("compile result: not cacheable");

stats.cache_misses.increment(&kind);
stats.cache_misses.increment(&kind, &lang);
stats.non_cacheable_compilations += 1;
}
CompileResult::CompileFailed => {
Expand Down Expand Up @@ -1298,7 +1299,7 @@ where
error!("[{:?}] \t{}", out_pretty, e);
let _ = writeln!(error, "sccache: caused by: {}", e);
}
stats.cache_errors.increment(&kind);
stats.cache_errors.increment(&kind, &lang);
//TODO: figure out a better way to communicate this?
res.retcode = Some(-2);
res.stderr = error.into_bytes();
Expand Down Expand Up @@ -1353,9 +1354,9 @@ pub struct PerLanguageCount {
}

impl PerLanguageCount {
fn increment(&mut self, kind: &CompilerKind) {
let key = kind.lang_kind();
let count = self.counts.entry(key).or_insert(0);
fn increment(&mut self, kind: &CompilerKind, lang: &Language) {
let lang_key = kind.lang_kind(lang);
let count = self.counts.entry(lang_key).or_insert(0);
*count += 1;
}

Expand Down
Loading
Loading