Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Revert "Produce crashreport.json and use llvm-symbolizer to create st… #81379

Merged
merged 1 commit into from
Jan 30, 2023
Merged

Revert "Produce crashreport.json and use llvm-symbolizer to create st… #81379

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions eng/pipelines/coreclr/templates/build-job.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,9 +189,8 @@ jobs:
displayName: Build and generate native prerequisites

# Build CoreCLR Runtime
# TODO: Use --keepnativesymbols only for PRs.
- ${{ if ne(parameters.osGroup, 'windows') }}:
- script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(clrRuntimePortableBuildArg) $(CoreClrPgoDataArg) --keepnativesymbols
- script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(clrRuntimePortableBuildArg) $(CoreClrPgoDataArg)
displayName: Build CoreCLR Runtime
- ${{ if eq(parameters.osGroup, 'windows') }}:
- script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg)
Expand Down
1 change: 0 additions & 1 deletion src/coreclr/debug/createdump/crashreportwriter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,6 @@ CrashReportWriter::WriteStackFrame(const StackFrame& frame)
WriteValue64("stack_pointer", frame.StackPointer());
WriteValue64("native_address", frame.InstructionPointer());
WriteValue64("native_offset", frame.NativeOffset());
WriteValue64("native_image_offset", (frame.InstructionPointer() - frame.ModuleAddress()));
if (frame.IsManaged())
{
WriteValue32("token", frame.Token());
Expand Down
310 changes: 5 additions & 305 deletions src/tests/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,6 @@
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
Expand Down Expand Up @@ -207,13 +204,12 @@ public class CoreclrTestWrapperLib
public const string COLLECT_DUMPS_ENVIRONMENT_VAR = "__CollectDumps";
public const string CRASH_DUMP_FOLDER_ENVIRONMENT_VAR = "__CrashDumpFolder";

static bool CollectCrashDump(Process process, string crashDumpPath, StreamWriter outputWriter)
static bool CollectCrashDump(Process process, string path)
{
string coreRoot = Environment.GetEnvironmentVariable("CORE_ROOT");
string createdumpPath = Path.Combine(coreRoot, "createdump");
string arguments = $"--name \"{crashDumpPath}\" {process.Id} --withheap";
string arguments = $"--name \"{path}\" {process.Id} --withheap";
Process createdump = new Process();
bool crashReportPresent = false;

if (OperatingSystem.IsWindows())
{
Expand All @@ -223,8 +219,8 @@ static bool CollectCrashDump(Process process, string crashDumpPath, StreamWriter
else if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
{
createdump.StartInfo.FileName = "sudo";
createdump.StartInfo.Arguments = $"{createdumpPath} --crashreport {arguments}";
crashReportPresent = true;
createdump.StartInfo.Arguments = $"{createdumpPath} " + arguments;
createdump.StartInfo.EnvironmentVariables.Add("DOTNET_DbgEnableElfDumpOnMacOS", "1");
}

createdump.StartInfo.UseShellExecute = false;
Expand All @@ -248,11 +244,6 @@ static bool CollectCrashDump(Process process, string crashDumpPath, StreamWriter
Console.WriteLine(output);
Console.WriteLine("createdump stderr:");
Console.WriteLine(error);

if (crashReportPresent)
{
TryPrintStackTraceFromCrashReport(crashDumpPath + ".crashreport.json", outputWriter);
}
}
else
{
Expand All @@ -262,271 +253,6 @@ static bool CollectCrashDump(Process process, string crashDumpPath, StreamWriter
return fSuccess && createdump.ExitCode == 0;
}

private static List<string> knownNativeModules = new List<string>() { "libcoreclr.so", "libclrjit.so" };
private static string TO_BE_CONTINUE_TAG = "<TO_BE_CONTINUE>";
private static string SKIP_LINE_TAG = "# <SKIP_LINE>";


static bool RunProcess(string fileName, string arguments)
{
Process proc = new Process()
{
StartInfo = new ProcessStartInfo()
{
FileName = fileName,
Arguments = arguments,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
}
};

Console.WriteLine($"Invoking: {proc.StartInfo.FileName} {proc.StartInfo.Arguments}");
proc.Start();

Task<string> stdOut = proc.StandardOutput.ReadToEndAsync();
Task<string> stdErr = proc.StandardError.ReadToEndAsync();
if(!proc.WaitForExit(DEFAULT_TIMEOUT_MS))
{
proc.Kill(true);
Console.WriteLine($"Timedout: '{fileName} {arguments}");
return false;
}

Task.WaitAll(stdOut, stdErr);
string output = stdOut.Result;
string error = stdErr.Result;
if (!string.IsNullOrWhiteSpace(output))
{
Console.WriteLine($"stdout: {output}");
}
if (!string.IsNullOrWhiteSpace(error))
{
Console.WriteLine($"stderr: {error}");
}
return true;
}

/// <summary>
/// Parse crashreport.json file, use llvm-symbolizer to extract symbols
/// and recreate the stacktrace that is printed on the console.
/// </summary>
/// <param name="crashReportJsonFile">crash dump path</param>
/// <param name="outputWriter">Stream for writing logs</param>
/// <returns>true, if we can print the stack trace, otherwise false.</returns>
static bool TryPrintStackTraceFromCrashReport(string crashReportJsonFile, StreamWriter outputWriter)
{
if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
{
if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}"))
{
return false;
}

Console.WriteLine("=========================================");
string userName = Environment.GetEnvironmentVariable("USER");
if (!string.IsNullOrEmpty(userName))
{
if (!RunProcess("sudo", $"chown {userName} {crashReportJsonFile}"))
{
return false;
}

Console.WriteLine("=========================================");
if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}"))
{
return false;
}

Console.WriteLine("=========================================");
if (!RunProcess("ls", $"-l {crashReportJsonFile}"))
{
return false;
}
}
}

if (!File.Exists(crashReportJsonFile))
{
return false;
}
outputWriter.WriteLine($"Printing stacktrace from '{crashReportJsonFile}'");

string contents = File.ReadAllText(crashReportJsonFile);
dynamic crashReport = JsonSerializer.Deserialize<JsonObject>(contents);
var threads = crashReport["payload"]["threads"];

// The logic happens in 3 steps:
// 1. Read the crashReport.json file, locate all the addresses of interest and then build
// a string that will be passed to llvm-symbolizer. It is populated so that each address
// is in its separate line along with the file name, etc. Some TAGS are added in the
// string that is used in step 2.
// 2. llvm-symbolizer is ran and above string is passed as input.
// 3. After llvm-symbolizer completes, TAGS are used to format its output to print it in
// the way it will be printed by sos.

StringBuilder addrBuilder = new StringBuilder();
string coreRoot = Environment.GetEnvironmentVariable("CORE_ROOT");
foreach (var thread in threads)
{

if (thread["native_thread_id"] == null)
{
continue;
}

addrBuilder.AppendLine();
addrBuilder.AppendLine("----------------------------------");
addrBuilder.AppendLine($"Thread Id: {thread["native_thread_id"]}");
addrBuilder.AppendLine(" Child SP IP Call Site");
var stack_frames = thread["stack_frames"];
foreach (var frame in stack_frames)
{
addrBuilder.Append($"{SKIP_LINE_TAG} {frame["stack_pointer"]} {frame["native_address"]} ");
bool isNative = (string)frame["is_managed"] == "false";

if (isNative)
{
string nativeModuleName = (string)frame["native_module"];
string unmanagedName = (string)frame["unmanaged_name"];

if ((nativeModuleName != null) && (knownNativeModules.Contains(nativeModuleName)))
{
// Need to use llvm-symbolizer (only if module_address != 0)
AppendAddress(addrBuilder, coreRoot, nativeModuleName, (string)frame["native_address"], (string)frame["module_address"]);
}
else if ((nativeModuleName != null) || (unmanagedName != null))
{
if (nativeModuleName != null)
{
addrBuilder.Append($"{nativeModuleName}!");
}
if (unmanagedName != null)
{
addrBuilder.Append($"{unmanagedName}");
}
}
}
else
{
string fileName = (string)frame["filename"];
string methodName = (string)frame["method_name"];

if ((fileName != null) || (methodName != null))
{
// found the managed method name
if (fileName != null)
{
addrBuilder.Append($"{fileName}!");
}
if (methodName != null)
{
addrBuilder.Append($"{methodName}");
}
}
else
{
addrBuilder.Append($"{frame["native_address"]}");
}
}
addrBuilder.AppendLine();

}
}

string symbolizerOutput = null;

Process llvmSymbolizer = new Process()
{
StartInfo = {
FileName = "llvm-symbolizer",
Arguments = $"--pretty-print",
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
RedirectStandardInput = true,
}
};

outputWriter.WriteLine($"Invoking {llvmSymbolizer.StartInfo.FileName} {llvmSymbolizer.StartInfo.Arguments}");

try
{
if (!llvmSymbolizer.Start())
{
outputWriter.WriteLine($"Unable to start {llvmSymbolizer.StartInfo.FileName}");
}

using (var symbolizerWriter = llvmSymbolizer.StandardInput)
{
symbolizerWriter.WriteLine(addrBuilder.ToString());
}

Task<string> stdout = llvmSymbolizer.StandardOutput.ReadToEndAsync();
Task<string> stderr = llvmSymbolizer.StandardError.ReadToEndAsync();
bool fSuccess = llvmSymbolizer.WaitForExit(DEFAULT_TIMEOUT_MS);

Task.WaitAll(stdout, stderr);

if (!fSuccess)
{
outputWriter.WriteLine("Errors while running llvm-symbolizer --pretty-print");
string output = stdout.Result;
string error = stderr.Result;

Console.WriteLine("llvm-symbolizer stdout:");
Console.WriteLine(output);
Console.WriteLine("llvm-symbolizer stderr:");
Console.WriteLine(error);

llvmSymbolizer.Kill(true);

return false;
}

symbolizerOutput = stdout.Result;

} catch (Exception e) {
outputWriter.WriteLine("Errors while running llvm-symbolizer --pretty-print");
outputWriter.WriteLine(e.ToString());
return false;
}

// Go through the output of llvm-symbolizer and strip all the markers we added initially.
string[] contentsToSantize = symbolizerOutput.Split(Environment.NewLine);
StringBuilder finalBuilder = new StringBuilder();
for (int lineNum = 0; lineNum < contentsToSantize.Length; lineNum++)
{
string line = contentsToSantize[lineNum].Replace(SKIP_LINE_TAG, string.Empty);
if (string.IsNullOrWhiteSpace(line)) continue;

if (line.EndsWith(TO_BE_CONTINUE_TAG))
{
finalBuilder.Append(line.Replace(TO_BE_CONTINUE_TAG, string.Empty));
continue;
}
finalBuilder.AppendLine(line);
}
outputWriter.WriteLine("Stack trace:");
outputWriter.WriteLine(finalBuilder.ToString());
return true;
}

private static void AppendAddress(StringBuilder sb, string coreRoot, string nativeModuleName, string native_address, string module_address)
{
if (module_address != "0x0")
{
sb.Append($"{nativeModuleName}!");
sb.Append(TO_BE_CONTINUE_TAG);
sb.AppendLine();
//addrBuilder.AppendLine(frame.native_image_offset);
ulong nativeAddress = ulong.Parse(native_address.Substring(2), System.Globalization.NumberStyles.HexNumber);
ulong moduleAddress = ulong.Parse(module_address.Substring(2), System.Globalization.NumberStyles.HexNumber);
string fullPathToModule = Path.Combine(coreRoot, nativeModuleName);
sb.AppendFormat("{0} 0x{1:x}", fullPathToModule, nativeAddress - moduleAddress);
}
}

// Finds all children processes starting with a process named childName
// The children are sorted in the order they should be dumped
static unsafe IEnumerable<Process> FindChildProcessesByName(Process process, string childName)
Expand Down Expand Up @@ -617,32 +343,6 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
exitCode = process.ExitCode;
MobileAppHandler.CheckExitCode(exitCode, testBinaryBase, category, outputWriter);
Task.WaitAll(copyOutput, copyError);

if (!OperatingSystem.IsWindows())
{
// crashreport is only for non-windows.
if (exitCode != 0)
{
// Search for dump, if created.
if (Directory.Exists(crashDumpFolder))
{
outputWriter.WriteLine($"Test failed. Trying to see if dump file was created in {crashDumpFolder} since {startTime}");
DirectoryInfo crashDumpFolderInfo = new DirectoryInfo(crashDumpFolder);
var dmpFilesInfo = crashDumpFolderInfo.GetFiles("*.crashreport.json").OrderByDescending(f => f.CreationTime);
foreach (var dmpFile in dmpFilesInfo)
{
if (dmpFile.CreationTime < startTime)
{
// No new files since test started.
outputWriter.WriteLine("Finish looking for *.crashreport.json. No new files created.");
break;
}
outputWriter.WriteLine($"Processing {dmpFile.FullName}");
TryPrintStackTraceFromCrashReport(dmpFile.FullName, outputWriter);
}
}
}
}
}
else
{
Expand Down Expand Up @@ -670,7 +370,7 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
{
string crashDumpPath = Path.Combine(Path.GetFullPath(crashDumpFolder), string.Format("crashdump_{0}.dmp", child.Id));
Console.WriteLine($"Attempting to collect crash dump: {crashDumpPath}");
if (CollectCrashDump(child, crashDumpPath, outputWriter))
if (CollectCrashDump(child, crashDumpPath))
{
Console.WriteLine("Collected crash dump: {0}", crashDumpPath);
}
Expand Down
3 changes: 1 addition & 2 deletions src/tests/Common/testenvironment.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<DOTNETVariables>
DOTNET_TieredCompilation;
DOTNET_DbgEnableMiniDump;
DOTNET_EnableCrashReport;
DOTNET_DbgEnableElfDumpOnMacOS;
DOTNET_DbgMiniDumpName;
DOTNET_EnableAES;
Expand Down Expand Up @@ -84,8 +83,8 @@
<TestEnvironment>
<TieredCompilation>0</TieredCompilation>
<DbgEnableMiniDump Condition="'$(TargetsWindows)' != 'true'">1</DbgEnableMiniDump> <!-- Enable minidumps for all scenarios -->
<DbgEnableElfDumpOnMacOS Condition="'$(TargetsOSX)' == 'true'">1</DbgEnableElfDumpOnMacOS> <!-- Enable minidumps for OSX -->
<DbgMiniDumpName Condition="'$(TargetsWindows)' != 'true'">$HELIX_DUMP_FOLDER/coredump.%d.dmp</DbgMiniDumpName>
<EnableCrashReport Condition="'$(TargetsWindows)' != 'true'">1</EnableCrashReport>
</TestEnvironment>
</ItemDefinitionGroup>

Expand Down