-
Notifications
You must be signed in to change notification settings - Fork 0
/
test_pbi_reports.cs
452 lines (380 loc) · 15.1 KB
/
test_pbi_reports.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
// Test Suite for Power BI crash tests
// Copyright © Christoph Thiede 2020.
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using HWND = System.IntPtr;
using ImageMagick;
namespace PbiCrashTests
{
/// <summary>
/// Represents a test case for a Power BI report file.
/// Purpose of the test will be to try to open and load the report
/// and detect if any errors or deadlocks occur while doing so.
/// </summary>
public class PbiReportTestCase {
/// <summary>
/// The path to the report template file.
/// </summary>
public string Report { get; }
public string Name => Path.GetFileName(Report);
/// <summary>
/// The path to the PBI Desktop executable file.
/// </summary>
public string Desktop { get; }
/// <summary>
/// The delay Power BI is granted to hang before the first window is opened. If exceeded,
/// the test will timeout.
/// </summary>
public TimeSpan PreLoadDelay { get; }
/// <summary>
/// The delay Power BI is granted to hang after all data sources have been adapted
/// before final error search starts.
/// </summary>
public TimeSpan LoadDelay { get; }
public bool HasPassed { get; private set; }
public bool HasFailed { get; private set; }
public string ResultReason { get; private set; }
protected static Dictionary<string, Bitmap> FailureIcons = Directory.GetFiles(
Path.Combine(Directory.GetCurrentDirectory(), "data/failure_icons"),
"*.bmp"
).ToDictionary(
file => Path.GetFileNameWithoutExtension(file),
file => new Bitmap(file)
);
protected Process Process { get; private set; }
protected PbiProcessSnapshot Snapshot { get; private set; }
public PbiReportTestCase(
string report, string desktop,
TimeSpan preLoadDelay, TimeSpan loadDelay) {
Report = report;
Desktop = desktop;
(PreLoadDelay, LoadDelay) = (preLoadDelay, loadDelay);
}
/// <summary>
/// Start this test.
/// </summary>
public void Start() => _logExceptions(() => {
Process = new Process {
StartInfo = {
FileName = Desktop,
Arguments = $"\"{Report}\""
}
};
Process.Start();
System.Threading.Thread.Sleep(LoadDelay);
Snapshot = new PbiProcessSnapshot(Process);
});
/// <summary>
/// Try to find indications for this test either having passed or failed.
/// If an indication is found, HasPassed or HasFailed will be set accordingly.
/// </summary>
public void Check() => _logExceptions(() => {
void handleFail(string reason) {
HasFailed = true;
ResultReason = reason;
}
_check(
handlePass: () => {
System.Threading.Thread.Sleep(LoadDelay);
_check(
handlePass: () => HasPassed = true,
handleFail: handleFail
);
},
handleFail: handleFail
);
});
/// <summary>
/// Abort this test.
/// </summary>
public void Stop() => _logExceptions(() => {
if (!Process.HasExited)
Process.Kill();
});
/// <summary>
/// Save the results of this test into <paramref name="path"/>.
/// </summary>
public void SaveResults(string path) => _logExceptions(() => {
Snapshot.SaveArtifacts(Path.Combine(path, Path.GetFileNameWithoutExtension(Report)));
});
public override string ToString() {
return $"PBI Report Test: {Name}";
}
private void _check(Action handlePass, Action<string> handleFail) {
if (Process.HasExited) {
handleFail("Power BI has unexpectedly terminated");
return;
}
Snapshot.Update();
var windows = Snapshot.Windows.ToList();
if (windows.Count == 1 && windows[0].Title.EndsWith(" - Power BI Desktop")) {
handlePass();
return;
}
if (!windows.Any(window => string.IsNullOrWhiteSpace(window.Title))) {
handleFail("Power BI did not open any valid window");
return;
}
foreach (var (failure, icon) in FailureIcons.Select(kvp => (kvp.Key, kvp.Value)))
foreach (var (window, index) in windows.Select((window, index) => (window, index))) {
Console.WriteLine($"(Checking window {index} against icon {failure})");
if (window.DisplaysIcon(icon, out var similarity)) {
handleFail($"Power BI showed an error in window {index} " +
$"while loading the report: {failure} (similarity={similarity})");
return;
}
}
}
/// <summary>
/// Helper function that catches any exception and writes the stack trace to console
/// before re-throwing the exception. This is helpful when running a C# script from
/// PowerShell.
/// </summary>
/// <param name="action">The actual action to wrap.</param>
private static void _logExceptions(Action action) {
try {
action();
} catch (Exception ex) {
Console.WriteLine(ex);
throw;
}
}
}
/// <summary>
/// Represents the state of an observed Power BI process at a certain point in time.
/// </summary>
public class PbiProcessSnapshot {
public PbiProcessSnapshot(Process process) {
Process = process;
}
public Process Process { get; }
/// <summary>
/// All windows that are currently opened by the process.
/// </summary>
public IEnumerable<PbiWindowSnapshot> Windows => _windows;
private List<PbiWindowSnapshot> _windows = new List<PbiWindowSnapshot>();
/// <summary>
/// Update this snapshot.
/// </summary>
public void Update() {
_windows = (
from window in CollectWindows()
where window.IsVisible
where !window.Bounds.Equals(default(PbiWindowSnapshot.RECT))
select window
).ToList();
}
/// <summary>
/// Save all artifacts of this snapshot into <paramref name="path"/>.
/// In detail, these are screenshots of all open windows.
/// </summary>
/// <param name="path"></param>
public void SaveArtifacts(string path) {
Directory.CreateDirectory(path);
foreach (var (window, index) in Windows.Select((window, index) => (window, index))) {
if (!window.IsVisible) continue;
var label = new StringBuilder("window");
label.AppendFormat("_{0}", index);
if (!string.IsNullOrWhiteSpace(window.Title))
label.AppendFormat("_{0}", window.Title);
label.Append(".png");
var screenshot = window.Screenshot;
screenshot?.Save(Path.Combine(path, label.ToString()));
}
}
protected IEnumerable<PbiWindowSnapshot> CollectWindows() {
foreach (var window in GetRootWindows(Process.Id))
yield return PbiWindowSnapshot.Create(window);
}
private static IEnumerable<HWND> GetRootWindows(int pid)
{
var windows = GetChildWindows(HWND.Zero);
foreach (var child in windows)
{
GetWindowThreadProcessId(child, out var lpdwProcessId);
if (lpdwProcessId == pid)
yield return child;
}
}
private static IEnumerable<HWND> GetChildWindows(HWND parent)
{
var result = new List<HWND>();
var listHandle = GCHandle.Alloc(result);
try
{
var childProc = new Win32Callback(EnumWindow);
EnumChildWindows(parent, childProc, GCHandle.ToIntPtr(listHandle));
}
finally
{
if (listHandle.IsAllocated)
listHandle.Free();
}
return result;
}
private static bool EnumWindow(IntPtr handle, IntPtr pointer)
{
var gch = GCHandle.FromIntPtr(pointer);
var list = (List<IntPtr>)gch.Target;
list.Add(handle);
return true;
}
#region DllImports
private delegate bool Win32Callback(HWND hwnd, IntPtr lParam);
[DllImport(DllImportNames.USER32)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool EnumChildWindows(HWND parentHandle, Win32Callback callback, IntPtr lParam);
[DllImport(DllImportNames.USER32)]
private static extern uint GetWindowThreadProcessId(HWND hwnd, out uint lpdwProcessId);
#endregion DllImports
}
/// <summary>
/// Represents the state of an observed Power BI window at a certain point in time.
/// </summary>
public class PbiWindowSnapshot {
[StructLayout(LayoutKind.Sequential)]
public struct RECT
{
public int Left;
public int Top;
public int Right;
public int Bottom;
}
public PbiWindowSnapshot(HWND hwnd) : this() {
Hwnd = hwnd;
}
private PbiWindowSnapshot() {
_screenshot = new Lazy<Bitmap>(RecordScreenshot);
}
/// <summary>
/// The window handle (hWnd) of this window.
/// </summary>
public HWND Hwnd { get; }
public string Title { get; private set; }
public bool IsVisible { get; private set; }
public RECT Bounds { get; private set; }
/// <summary>
/// A screenshot of this window.
/// </summary>
/// <remarks>
/// Will be generated lazilly.
/// </remarks>
public Bitmap Screenshot => _screenshot.Value;
protected static double IconSimilarityThreshold = 0.1;
private readonly Lazy<Bitmap> _screenshot;
public static PbiWindowSnapshot Create(HWND hwnd) {
var window = new PbiWindowSnapshot(hwnd);
window.Update();
return window;
}
/// <summary>
/// Update this snapshot.
/// </summary>
public void Update() {
Title = GetWindowTitle();
IsVisible = IsWindowVisible(Hwnd);
Bounds = GetWindowBounds();
}
/// <summary>
/// Tests whether this window displays the specified icon.
/// </summary>
public bool DisplaysIcon(Bitmap icon, out double similarity) {
var screenshot = Screenshot;
if (screenshot is null) {
similarity = default(double);
return false;
}
var magickIcon = CreateMagickImage(icon);
var magickScreenshot = CreateMagickImage(screenshot);
var result = magickScreenshot.SubImageSearch(magickIcon);
return (similarity = result.SimilarityMetric) <= IconSimilarityThreshold;
}
private RECT GetWindowBounds() {
if (!GetWindowRect(new HandleRef(null, Hwnd), out var rect))
throw new Win32Exception(Marshal.GetLastWin32Error());
return rect;
}
private string GetWindowTitle()
{
var length = GetWindowTextLength(Hwnd);
var title = new StringBuilder(length);
GetWindowText(Hwnd, title, length + 1);
return title.ToString();
}
private Bitmap RecordScreenshot() {
var bmp = new Bitmap(Bounds.Right - Bounds.Left, Bounds.Bottom - Bounds.Top, PixelFormat.Format32bppArgb);
using (var gfxBmp = Graphics.FromImage(bmp)) {
IntPtr hdcBitmap;
try
{
hdcBitmap = gfxBmp.GetHdc();
}
catch
{
return null;
}
bool succeeded = PrintWindow(Hwnd, hdcBitmap, 0);
gfxBmp.ReleaseHdc(hdcBitmap);
if (!succeeded)
{
return null;
}
var hRgn = CreateRectRgn(0, 0, 0, 0);
GetWindowRgn(Hwnd, hRgn);
var region = Region.FromHrgn(hRgn);
if (!region.IsEmpty(gfxBmp))
{
gfxBmp.ExcludeClip(region);
gfxBmp.Clear(Color.Transparent);
}
}
return bmp;
}
private static MagickImage CreateMagickImage(Bitmap bitmap) {
var magickImage = new MagickImage();
using (var stream = new MemoryStream())
{
bitmap.Save(stream, ImageFormat.Bmp);
stream.Position = 0;
magickImage.Read(stream);
}
const double scaleFactor = 1 / 3d; // For performance
magickImage.Resize(
(int)Math.Round(magickImage.Width * scaleFactor),
(int)Math.Round(magickImage.Height * scaleFactor)
);
return magickImage;
}
#region DllImports
[DllImport(DllImportNames.GDI32)]
private static extern IntPtr CreateRectRgn(int nLeftRect, int nTopRect, int nRightRect, int nBottomRect);
[DllImport(DllImportNames.USER32)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool GetWindowRect(HandleRef hwnd, out RECT lpRect);
[DllImport(DllImportNames.USER32)]
private static extern int GetWindowRgn(HWND hWnd, IntPtr hRgn);
[DllImport(DllImportNames.USER32, CharSet = CharSet.Auto, SetLastError = true)]
private static extern int GetWindowText(HWND hwnd, StringBuilder lpString, int nMaxCount);
[DllImport(DllImportNames.USER32, SetLastError = true, CharSet = CharSet.Auto)]
private static extern int GetWindowTextLength(HWND hwnd);
[DllImport(DllImportNames.USER32)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool IsWindowVisible(IntPtr hwnd);
[DllImport(DllImportNames.USER32, SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool PrintWindow(HWND hwnd, IntPtr hDC, uint nFlags);
#endregion DllImports
}
internal static class DllImportNames {
public const string GDI32 = "gdi32.dll";
public const string USER32 = "user32.dll";
}
}