Skip to content

Commit

Permalink
Auto merge of #63825 - nathanwhit:check-run-results, r=Mark-Simulacrum
Browse files Browse the repository at this point in the history 
Allow checking of run-pass execution output in compiletest

Closes #63751
Adds a `check-run-results` flag to compiletest headers, which if enabled checks the output of the execution of a run-pass test's binary against expected output.
  • Loading branch information
@bors
bors committed Sep 4, 2019
2 parents 5f42f3e + 12adc39 commit 6c18a3d
Show file tree
Hide file tree
Showing 5 changed files with 123 additions and 105 deletions.
84 changes: 2 additions & 82 deletions src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
// run-pass
// ignore-cloudabi no processes
// ignore-emscripten no processes
// ignore-sgx no processes
// check-run-results

// Tests ensuring that `dbg!(expr)` has the expected run-time behavior.
// as well as some compile time properties we expect.
Expand All @@ -18,7 +16,7 @@ struct Point<T> {
#[derive(Debug, PartialEq)]
struct NoCopy(usize);

fn test() {
fn main() {
let a: Unit = dbg!(Unit);
let _: Unit = dbg!(a);
// We can move `a` because it's Copy.
Expand Down Expand Up @@ -67,81 +65,3 @@ fn test() {
assert_eq!((1u8, 2u32, "Yeah"), dbg!(1u8, 2u32,
"Yeah",));
}

fn validate_stderr(stderr: Vec<String>) {
assert_eq!(stderr, &[
":22] Unit = Unit",

":23] a = Unit",

":29] Point{x: 42, y: 24,} = Point {",
" x: 42,",
" y: 24,",
"}",

":30] b = Point {",
" x: 42,",
" y: 24,",
"}",

":38]",

":42] &a = NoCopy(",
" 1337,",
")",

":42] dbg!(& a) = NoCopy(",
" 1337,",
")",
":47] f(&42) = 42",

"before",
":52] { foo += 1; eprintln!(\"before\"); 7331 } = 7331",

":60] (\"Yeah\",) = (",
" \"Yeah\",",
")",

":63] 1 = 1",
":63] 2 = 2",

":67] 1u8 = 1",
":67] 2u32 = 2",
":67] \"Yeah\" = \"Yeah\"",
]);
}

fn main() {
// The following is a hack to deal with compiletest's inability
// to check the output (to stdout) of run-pass tests.
use std::env;
use std::process::Command;

let mut args = env::args();
let prog = args.next().unwrap();
let child = args.next();
if let Some("child") = child.as_ref().map(|s| &**s) {
// Only run the test if we've been spawned as 'child'
test()
} else {
// This essentially spawns as 'child' to run the tests
// and then it collects output of stderr and checks the output
// against what we expect.
let out = Command::new(&prog).arg("child").output().unwrap();
assert!(out.status.success());
assert!(out.stdout.is_empty());

let stderr = String::from_utf8(out.stderr).unwrap();
let stderr = stderr.lines().map(|mut s| {
if s.starts_with("[") {
// Strip `[` and file path:
s = s.trim_start_matches("[");
assert!(s.starts_with(file!()));
s = s.trim_start_matches(file!());
}
s.to_owned()
}).collect();

validate_stderr(stderr);
}
}
28 changes: 28 additions & 0 deletions src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.run.stderr
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
[$DIR/dbg-macro-expected-behavior.rs:20] Unit = Unit
[$DIR/dbg-macro-expected-behavior.rs:21] a = Unit
[$DIR/dbg-macro-expected-behavior.rs:27] Point{x: 42, y: 24,} = Point {
x: 42,
y: 24,
}
[$DIR/dbg-macro-expected-behavior.rs:28] b = Point {
x: 42,
y: 24,
}
[$DIR/dbg-macro-expected-behavior.rs:36]
[$DIR/dbg-macro-expected-behavior.rs:40] &a = NoCopy(
1337,
)
[$DIR/dbg-macro-expected-behavior.rs:40] dbg!(& a) = NoCopy(
1337,
)
[$DIR/dbg-macro-expected-behavior.rs:45] f(&42) = 42
before
[$DIR/dbg-macro-expected-behavior.rs:50] { foo += 1; eprintln!("before"); 7331 } = 7331
[$DIR/dbg-macro-expected-behavior.rs:58] ("Yeah",) = (
"Yeah",
)
[$DIR/dbg-macro-expected-behavior.rs:61] 1 = 1
[$DIR/dbg-macro-expected-behavior.rs:61] 2 = 2
[$DIR/dbg-macro-expected-behavior.rs:65] 1u8 = 1
[$DIR/dbg-macro-expected-behavior.rs:65] 2u32 = 2
[$DIR/dbg-macro-expected-behavior.rs:65] "Yeah" = "Yeah"
4 changes: 3 additions & 1 deletion src/tools/compiletest/src/common.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,10 +333,12 @@ pub fn expected_output_path(
testpaths.file.with_extension(extension)
}

pub const UI_EXTENSIONS: &[&str] = &[UI_STDERR, UI_STDOUT, UI_FIXED];
pub const UI_EXTENSIONS: &[&str] = &[UI_STDERR, UI_STDOUT, UI_FIXED, UI_RUN_STDERR, UI_RUN_STDOUT];
pub const UI_STDERR: &str = "stderr";
pub const UI_STDOUT: &str = "stdout";
pub const UI_FIXED: &str = "fixed";
pub const UI_RUN_STDERR: &str = "run.stderr";
pub const UI_RUN_STDOUT: &str = "run.stdout";

/// Absolute path to the directory where all output for all tests in the given
/// `relative_dir` group should reside. Example:
Expand Down
16 changes: 16 additions & 0 deletions src/tools/compiletest/src/header.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,11 @@ impl EarlyProps {
config.parse_needs_sanitizer_support(ln) {
props.ignore = Ignore::Ignore;
}

if config.target == "wasm32-unknown-unknown" && config.parse_check_run_results(ln) {
props.ignore = Ignore::Ignore;
}

}

if (config.mode == common::DebugInfoGdb || config.mode == common::DebugInfoGdbLldb) &&
Expand Down Expand Up @@ -326,6 +331,8 @@ pub struct TestProps {
pub force_host: bool,
// Check stdout for error-pattern output as well as stderr
pub check_stdout: bool,
// Check stdout & stderr for output of run-pass test
pub check_run_results: bool,
// For UI tests, allows compiler to generate arbitrary output to stdout
pub dont_check_compiler_stdout: bool,
// For UI tests, allows compiler to generate arbitrary output to stderr
Expand Down Expand Up @@ -388,6 +395,7 @@ impl TestProps {
build_aux_docs: false,
force_host: false,
check_stdout: false,
check_run_results: false,
dont_check_compiler_stdout: false,
dont_check_compiler_stderr: false,
no_prefer_dynamic: false,
Expand Down Expand Up @@ -468,6 +476,10 @@ impl TestProps {
self.check_stdout = config.parse_check_stdout(ln);
}

if !self.check_run_results {
self.check_run_results = config.parse_check_run_results(ln);
}

if !self.dont_check_compiler_stdout {
self.dont_check_compiler_stdout = config.parse_dont_check_compiler_stdout(ln);
}
Expand Down Expand Up @@ -712,6 +724,10 @@ impl Config {
self.parse_name_directive(line, "check-stdout")
}

fn parse_check_run_results(&self, line: &str) -> bool {
self.parse_name_directive(line, "check-run-results")
}

fn parse_dont_check_compiler_stdout(&self, line: &str) -> bool {
self.parse_name_directive(line, "dont-check-compiler-stdout")
}
Expand Down
96 changes: 74 additions & 22 deletions src/tools/compiletest/src/runtest.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

use crate::common::{CompareMode, PassMode};
use crate::common::{expected_output_path, UI_EXTENSIONS, UI_FIXED, UI_STDERR, UI_STDOUT};
use crate::common::{UI_RUN_STDERR, UI_RUN_STDOUT};
use crate::common::{output_base_dir, output_base_name, output_testname_unique};
use crate::common::{Codegen, CodegenUnits, Rustdoc};
use crate::common::{DebugInfoCdb, DebugInfoGdbLldb, DebugInfoGdb, DebugInfoLldb};
Expand Down Expand Up @@ -288,6 +289,11 @@ enum ReadFrom {
Stdin(String),
}

enum TestOutput {
Compile,
Run,
}

impl<'test> TestCx<'test> {
/// Code executed for each revision in turn (or, if there are no
/// revisions, exactly once, with revision == None).
Expand Down Expand Up @@ -2930,6 +2936,61 @@ impl<'test> TestCx<'test> {
}
}

fn load_compare_outputs(&self, proc_res: &ProcRes,
output_kind: TestOutput, explicit_format: bool) -> usize {

let (stderr_kind, stdout_kind) = match output_kind {
TestOutput::Compile => (UI_STDERR, UI_STDOUT),
TestOutput::Run => (UI_RUN_STDERR, UI_RUN_STDOUT)
};

let expected_stderr = self.load_expected_output(stderr_kind);
let expected_stdout = self.load_expected_output(stdout_kind);

let normalized_stdout = match output_kind {
TestOutput::Run if self.config.remote_test_client.is_some() => {
// When tests are run using the remote-test-client, the string
// 'uploaded "$TEST_BUILD_DIR/<test_executable>, waiting for result"'
// is printed to stdout by the client and then captured in the ProcRes,
// so it needs to be removed when comparing the run-pass test execution output
lazy_static! {
static ref REMOTE_TEST_RE: Regex = Regex::new(
"^uploaded \"\\$TEST_BUILD_DIR(/[[:alnum:]_\\-]+)+\", waiting for result\n"
).unwrap();
}
REMOTE_TEST_RE.replace(
&self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout),
""
).to_string()
}
_ => self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout)
};

let stderr = if explicit_format {
proc_res.stderr.clone()
} else {
json::extract_rendered(&proc_res.stderr)
};

let normalized_stderr = self.normalize_output(&stderr, &self.props.normalize_stderr);
let mut errors = 0;
match output_kind {
TestOutput::Compile => {
if !self.props.dont_check_compiler_stdout {
errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout);
}
if !self.props.dont_check_compiler_stderr {
errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr);
}
}
TestOutput::Run => {
errors += self.compare_output(stdout_kind, &normalized_stdout, &expected_stdout);
errors += self.compare_output(stderr_kind, &normalized_stderr, &expected_stderr);
}
}
errors
}

fn run_ui_test(&self) {
// if the user specified a format in the ui test
// print the output to the stderr file, otherwise extract
Expand All @@ -2942,32 +3003,13 @@ impl<'test> TestCx<'test> {
let proc_res = self.compile_test();
self.check_if_test_should_compile(&proc_res);

let expected_stderr = self.load_expected_output(UI_STDERR);
let expected_stdout = self.load_expected_output(UI_STDOUT);
let expected_fixed = self.load_expected_output(UI_FIXED);

let normalized_stdout =
self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout);

let stderr = if explicit {
proc_res.stderr.clone()
} else {
json::extract_rendered(&proc_res.stderr)
};

let normalized_stderr = self.normalize_output(&stderr, &self.props.normalize_stderr);

let mut errors = 0;
if !self.props.dont_check_compiler_stdout {
errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout);
}
if !self.props.dont_check_compiler_stderr {
errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr);
}

let modes_to_prune = vec![CompareMode::Nll];
self.prune_duplicate_outputs(&modes_to_prune);

let mut errors = self.load_compare_outputs(&proc_res, TestOutput::Compile, explicit);

if self.config.compare_mode.is_some() {
// don't test rustfix with nll right now
} else if self.config.rustfix_coverage {
Expand Down Expand Up @@ -3045,7 +3087,17 @@ impl<'test> TestCx<'test> {

if self.should_run_successfully() {
let proc_res = self.exec_compiled_test();

let run_output_errors = if self.props.check_run_results {
self.load_compare_outputs(&proc_res, TestOutput::Run, explicit)
} else {
0
};
if run_output_errors > 0 {
self.fatal_proc_rec(
&format!("{} errors occured comparing run output.", run_output_errors),
&proc_res,
);
}
if !proc_res.status.success() {
self.fatal_proc_rec("test run failed!", &proc_res);
}
Expand Down

0 comments on commit 6c18a3d

Please sign in to comment.