670 lines
30 KiB
C#
670 lines
30 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Runtime.Serialization;
|
|
using System.Runtime.Serialization.Json;
|
|
using System.Text;
|
|
using System.Xml.Linq;
|
|
|
|
namespace HelixTestHelpers
|
|
{
|
|
public class TestResult
|
|
{
|
|
public TestResult()
|
|
{
|
|
Screenshots = new List<string>();
|
|
RerunResults = new List<TestResult>();
|
|
}
|
|
|
|
public string Name { get; set; }
|
|
public string SourceWttFile { get; set; }
|
|
public bool Passed { get; set; }
|
|
public bool CleanupPassed { get; set; }
|
|
public TimeSpan ExecutionTime { get; set; }
|
|
public string Details { get; set; }
|
|
|
|
public List<string> Screenshots { get; private set; }
|
|
public List<TestResult> RerunResults { get; private set; }
|
|
|
|
// Returns true if the test pass rate is sufficient to avoid being counted as a failure.
|
|
public bool PassedOrUnreliable(int requiredNumberOfPasses)
|
|
{
|
|
if(Passed)
|
|
{
|
|
return true;
|
|
}
|
|
else
|
|
{
|
|
if(RerunResults.Count == 1)
|
|
{
|
|
return RerunResults[0].Passed;
|
|
}
|
|
else
|
|
{
|
|
return RerunResults.Where(r => r.Passed).Count() >= requiredNumberOfPasses;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// Azure DevOps doesn't currently provide a way to directly report sub-results for tests that failed at least once
|
|
// that were run multiple times. To get around that limitation, we'll mark the test as "Skip" since
|
|
// that's the only non-pass/fail result we can return, and will then report the information about the
|
|
// runs in the "reason" category for the skipped test. In order to save space, we'll make the following
|
|
// optimizations for size:
|
|
//
|
|
// 1. Serialize as JSON, which is more compact than XML;
|
|
// 2. Don't serialize values that we don't need;
|
|
// 3. Store the URL prefix and suffix for the blob storage URL only once instead of
|
|
// storing every log and screenshot URL in its entirety; and
|
|
// 4. Store a list of unique error messages and then index into that instead of
|
|
// storing every error message in its entirety.
|
|
//
|
|
// #4 is motivated by the fact that if a test fails multiple times, it probably failed for the same reason
|
|
// each time, in which case we'd just be repeating ourselves if we stored every error message each time.
|
|
//
|
|
// TODO (https://github.com/dotnet/arcade/issues/2773): Once we're able to directly report things in a
|
|
// more granular fashion than just a binary pass/fail result, we should do that.
|
|
//
|
|
[DataContract]
|
|
internal class JsonSerializableTestResults
|
|
{
|
|
[DataMember]
|
|
internal string blobPrefix;
|
|
|
|
[DataMember]
|
|
internal string blobSuffix;
|
|
|
|
[DataMember]
|
|
internal string[] errors;
|
|
|
|
[DataMember]
|
|
internal JsonSerializableTestResult[] results;
|
|
}
|
|
|
|
[DataContract]
|
|
internal class JsonSerializableTestResult
|
|
{
|
|
[DataMember]
|
|
internal string outcome;
|
|
|
|
[DataMember]
|
|
internal int duration;
|
|
|
|
[DataMember(EmitDefaultValue = false)]
|
|
internal string log;
|
|
|
|
[DataMember(EmitDefaultValue = false)]
|
|
internal string[] screenshots;
|
|
|
|
[DataMember(EmitDefaultValue = false)]
|
|
internal int errorIndex;
|
|
}
|
|
|
|
public class TestPass
|
|
{
|
|
public TimeSpan TestPassExecutionTime { get; set; }
|
|
public List<TestResult> TestResults { get; set; }
|
|
|
|
public static TestPass ParseTestWttFile(string fileName, bool cleanupFailuresAreRegressions, bool truncateTestNames)
|
|
{
|
|
using (var stream = File.OpenRead(fileName))
|
|
{
|
|
var doc = XDocument.Load(stream);
|
|
var testResults = new List<TestResult>();
|
|
var testExecutionTimeMap = new Dictionary<string, List<double>>();
|
|
|
|
TestResult currentResult = null;
|
|
long frequency = 0;
|
|
long startTime = 0;
|
|
long stopTime = 0;
|
|
bool inTestCleanup = false;
|
|
|
|
bool shouldLogToTestDetails = false;
|
|
|
|
long testPassStartTime = 0;
|
|
long testPassStopTime = 0;
|
|
|
|
Func<XElement, bool> isScopeData = (elt) =>
|
|
{
|
|
return
|
|
elt.Element("Data") != null &&
|
|
elt.Element("Data").Element("WexContext") != null &&
|
|
(
|
|
elt.Element("Data").Element("WexContext").Value == "Cleanup" ||
|
|
elt.Element("Data").Element("WexContext").Value == "TestScope" ||
|
|
elt.Element("Data").Element("WexContext").Value == "TestScope" ||
|
|
elt.Element("Data").Element("WexContext").Value == "ClassScope" ||
|
|
elt.Element("Data").Element("WexContext").Value == "ModuleScope"
|
|
);
|
|
};
|
|
|
|
Func<XElement, bool> isModuleOrClassScopeStart = (elt) =>
|
|
{
|
|
return
|
|
elt.Name == "Msg" &&
|
|
elt.Element("Data") != null &&
|
|
elt.Element("Data").Element("StartGroup") != null &&
|
|
elt.Element("Data").Element("WexContext") != null &&
|
|
(elt.Element("Data").Element("WexContext").Value == "ClassScope" ||
|
|
elt.Element("Data").Element("WexContext").Value == "ModuleScope");
|
|
};
|
|
|
|
Func<XElement, bool> isModuleScopeEnd = (elt) =>
|
|
{
|
|
return
|
|
elt.Name == "Msg" &&
|
|
elt.Element("Data") != null &&
|
|
elt.Element("Data").Element("EndGroup") != null &&
|
|
elt.Element("Data").Element("WexContext") != null &&
|
|
elt.Element("Data").Element("WexContext").Value == "ModuleScope";
|
|
};
|
|
|
|
Func<XElement, bool> isClassScopeEnd = (elt) =>
|
|
{
|
|
return
|
|
elt.Name == "Msg" &&
|
|
elt.Element("Data") != null &&
|
|
elt.Element("Data").Element("EndGroup") != null &&
|
|
elt.Element("Data").Element("WexContext") != null &&
|
|
elt.Element("Data").Element("WexContext").Value == "ClassScope";
|
|
};
|
|
|
|
int testsExecuting = 0;
|
|
foreach (XElement element in doc.Root.Elements())
|
|
{
|
|
// Capturing the frequency data to record accurate
|
|
// timing data.
|
|
if (element.Name == "RTI")
|
|
{
|
|
frequency = Int64.Parse(element.Attribute("Frequency").Value);
|
|
}
|
|
|
|
// It's possible for a test to launch another test. If that happens, we won't modify the
|
|
// current result. Instead, we'll continue operating like normal and expect that we get two
|
|
// EndTests nodes before our next StartTests. We'll check that we've actually got a stop time
|
|
// before creating a new result. This will result in the two results being squashed
|
|
// into one result of the outer test that ran the inner one.
|
|
if (element.Name == "StartTest")
|
|
{
|
|
testsExecuting++;
|
|
if (testsExecuting == 1)
|
|
{
|
|
string testName = element.Attribute("Title").Value;
|
|
|
|
if (truncateTestNames)
|
|
{
|
|
const string xamlNativePrefix = "Windows::UI::Xaml::Tests::";
|
|
const string xamlManagedPrefix = "Windows.UI.Xaml.Tests.";
|
|
if (testName.StartsWith(xamlNativePrefix))
|
|
{
|
|
testName = testName.Substring(xamlNativePrefix.Length);
|
|
}
|
|
else if (testName.StartsWith(xamlManagedPrefix))
|
|
{
|
|
testName = testName.Substring(xamlManagedPrefix.Length);
|
|
}
|
|
}
|
|
|
|
currentResult = new TestResult() { Name = testName, SourceWttFile = fileName, Passed = true, CleanupPassed = true };
|
|
testResults.Add(currentResult);
|
|
startTime = Int64.Parse(element.Descendants("WexTraceInfo").First().Attribute("TimeStamp").Value);
|
|
inTestCleanup = false;
|
|
shouldLogToTestDetails = true;
|
|
stopTime = 0;
|
|
}
|
|
}
|
|
else if (currentResult != null && element.Name == "EndTest")
|
|
{
|
|
testsExecuting--;
|
|
|
|
// If any inner test fails, we'll still fail the outer
|
|
currentResult.Passed &= element.Attribute("Result").Value == "Pass";
|
|
|
|
// Only gather execution data if this is the outer test we ran initially
|
|
if (testsExecuting == 0)
|
|
{
|
|
stopTime = Int64.Parse(element.Descendants("WexTraceInfo").First().Attribute("TimeStamp").Value);
|
|
if (!testExecutionTimeMap.Keys.Contains(currentResult.Name))
|
|
testExecutionTimeMap[currentResult.Name] = new List<double>();
|
|
testExecutionTimeMap[currentResult.Name].Add((double)(stopTime - startTime) / frequency);
|
|
currentResult.ExecutionTime = TimeSpan.FromSeconds(testExecutionTimeMap[currentResult.Name].Average());
|
|
|
|
startTime = 0;
|
|
inTestCleanup = true;
|
|
}
|
|
}
|
|
else if (currentResult != null &&
|
|
(isModuleOrClassScopeStart(element) || isModuleScopeEnd(element) || isClassScopeEnd(element)))
|
|
{
|
|
shouldLogToTestDetails = false;
|
|
inTestCleanup = false;
|
|
}
|
|
|
|
// Log-appending methods.
|
|
if (currentResult != null && element.Name == "Error")
|
|
{
|
|
if (shouldLogToTestDetails)
|
|
{
|
|
currentResult.Details += "\r\n[Error]: " + element.Attribute("UserText").Value;
|
|
if (element.Attribute("File") != null && element.Attribute("File").Value != "")
|
|
{
|
|
currentResult.Details += (" [File " + element.Attribute("File").Value);
|
|
if (element.Attribute("Line") != null)
|
|
currentResult.Details += " Line: " + element.Attribute("Line").Value;
|
|
currentResult.Details += "]";
|
|
}
|
|
}
|
|
|
|
|
|
// The test cleanup errors will often come after the test claimed to have
|
|
// 'passed'. We treat them as errors as well.
|
|
if (inTestCleanup)
|
|
{
|
|
currentResult.CleanupPassed = false;
|
|
currentResult.Passed = false;
|
|
// In stress mode runs, this test will run n times before cleanup is run. If the cleanup
|
|
// fails, we want to fail every test.
|
|
if (cleanupFailuresAreRegressions)
|
|
{
|
|
foreach (var result in testResults.Where(res => res.Name == currentResult.Name))
|
|
{
|
|
result.Passed = false;
|
|
result.CleanupPassed = false;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (currentResult != null && element.Name == "Warn")
|
|
{
|
|
if (shouldLogToTestDetails)
|
|
{
|
|
currentResult.Details += "\r\n[Warn]: " + element.Attribute("UserText").Value;
|
|
}
|
|
|
|
if (element.Attribute("File") != null && element.Attribute("File").Value != "")
|
|
{
|
|
currentResult.Details += (" [File " + element.Attribute("File").Value);
|
|
if (element.Attribute("Line") != null)
|
|
currentResult.Details += " Line: " + element.Attribute("Line").Value;
|
|
currentResult.Details += "]";
|
|
}
|
|
}
|
|
|
|
if (currentResult != null && element.Name == "Msg")
|
|
{
|
|
var dataElement = element.Element("Data");
|
|
if (dataElement != null)
|
|
{
|
|
var supportingInfo = dataElement.Element("SupportingInfo");
|
|
if (supportingInfo != null)
|
|
{
|
|
var screenshots = supportingInfo.Elements("Item")
|
|
.Where(item => GetAttributeValue(item, "Name") == "Screenshot")
|
|
.Select(item => GetAttributeValue(item, "Value"));
|
|
|
|
foreach(var screenshot in screenshots)
|
|
{
|
|
string fileNameSuffix = string.Empty;
|
|
|
|
if (fileName.Contains("_rerun_multiple"))
|
|
{
|
|
fileNameSuffix = "_rerun_multiple";
|
|
}
|
|
else if (fileName.Contains("_rerun"))
|
|
{
|
|
fileNameSuffix = "_rerun";
|
|
}
|
|
|
|
currentResult.Screenshots.Add(screenshot.Replace(".jpg", fileNameSuffix + ".jpg"));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
testPassStartTime = Int64.Parse(doc.Root.Descendants("WexTraceInfo").First().Attribute("TimeStamp").Value);
|
|
testPassStopTime = Int64.Parse(doc.Root.Descendants("WexTraceInfo").Last().Attribute("TimeStamp").Value);
|
|
|
|
var testPassTime = TimeSpan.FromSeconds((double)(testPassStopTime - testPassStartTime) / frequency);
|
|
|
|
foreach (TestResult testResult in testResults)
|
|
{
|
|
if (testResult.Details != null)
|
|
{
|
|
testResult.Details = testResult.Details.Trim();
|
|
}
|
|
}
|
|
|
|
var testpass = new TestPass
|
|
{
|
|
TestPassExecutionTime = testPassTime,
|
|
TestResults = testResults
|
|
};
|
|
|
|
return testpass;
|
|
}
|
|
}
|
|
|
|
public static TestPass ParseTestWttFileWithReruns(string fileName, string singleRerunFileName, string multipleRerunFileName, bool cleanupFailuresAreRegressions, bool truncateTestNames)
|
|
{
|
|
TestPass testPass = ParseTestWttFile(fileName, cleanupFailuresAreRegressions, truncateTestNames);
|
|
TestPass singleRerunTestPass = File.Exists(singleRerunFileName) ? ParseTestWttFile(singleRerunFileName, cleanupFailuresAreRegressions, truncateTestNames) : null;
|
|
TestPass multipleRerunTestPass = File.Exists(multipleRerunFileName) ? ParseTestWttFile(multipleRerunFileName, cleanupFailuresAreRegressions, truncateTestNames) : null;
|
|
|
|
List<TestResult> rerunTestResults = new List<TestResult>();
|
|
|
|
if (singleRerunTestPass != null)
|
|
{
|
|
rerunTestResults.AddRange(singleRerunTestPass.TestResults);
|
|
}
|
|
|
|
if (multipleRerunTestPass != null)
|
|
{
|
|
rerunTestResults.AddRange(multipleRerunTestPass.TestResults);
|
|
}
|
|
|
|
// For each failed test result, we'll check to see whether the test passed at least once upon rerun.
|
|
// If so, we'll set PassedOnRerun to true to flag the fact that this is an unreliable test
|
|
// rather than a genuine test failure.
|
|
foreach (TestResult failedTestResult in testPass.TestResults.Where(r => !r.Passed))
|
|
{
|
|
failedTestResult.RerunResults.AddRange(rerunTestResults.Where(r => r.Name == failedTestResult.Name));
|
|
}
|
|
|
|
return testPass;
|
|
}
|
|
|
|
private static string GetAttributeValue(XElement element, string attributeName)
|
|
{
|
|
if(element.Attribute(attributeName) != null)
|
|
{
|
|
return element.Attribute(attributeName).Value;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static class FailedTestDetector
|
|
{
|
|
public static void OutputFailedTestQuery(string wttInputPath)
|
|
{
|
|
var testPass = TestPass.ParseTestWttFile(wttInputPath, cleanupFailuresAreRegressions: true, truncateTestNames: false);
|
|
|
|
List<string> failedTestNames = new List<string>();
|
|
|
|
foreach (var result in testPass.TestResults)
|
|
{
|
|
if (!result.Passed)
|
|
{
|
|
failedTestNames.Add(result.Name);
|
|
}
|
|
}
|
|
|
|
if (failedTestNames.Count > 0)
|
|
{
|
|
string failedTestSelectQuery = "(@Name='";
|
|
|
|
for (int i = 0; i < failedTestNames.Count; i++)
|
|
{
|
|
failedTestSelectQuery += failedTestNames[i];
|
|
|
|
if (i < failedTestNames.Count - 1)
|
|
{
|
|
failedTestSelectQuery += "' or @Name='";
|
|
}
|
|
}
|
|
|
|
failedTestSelectQuery += "')";
|
|
|
|
Console.WriteLine(failedTestSelectQuery);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("");
|
|
}
|
|
}
|
|
}
|
|
|
|
public class TestResultParser
|
|
{
|
|
private string testNamePrefix;
|
|
private string helixResultsContainerUri;
|
|
private string helixResultsContainerRsas;
|
|
|
|
public TestResultParser(string testNamePrefix, string helixResultsContainerUri, string helixResultsContainerRsas)
|
|
{
|
|
this.testNamePrefix = testNamePrefix;
|
|
this.helixResultsContainerUri = helixResultsContainerUri;
|
|
this.helixResultsContainerRsas = helixResultsContainerRsas;
|
|
}
|
|
|
|
public Dictionary<string, string> GetSubResultsJsonByMethodName(string wttInputPath, string wttSingleRerunInputPath, string wttMultipleRerunInputPath)
|
|
{
|
|
Dictionary<string, string> subResultsJsonByMethod = new Dictionary<string, string>();
|
|
TestPass testPass = TestPass.ParseTestWttFileWithReruns(wttInputPath, wttSingleRerunInputPath, wttMultipleRerunInputPath, cleanupFailuresAreRegressions: true, truncateTestNames: false);
|
|
|
|
foreach (var result in testPass.TestResults)
|
|
{
|
|
var methodName = result.Name.Substring(result.Name.LastIndexOf('.') + 1);
|
|
|
|
if (!result.Passed)
|
|
{
|
|
// If a test failed, we'll have rerun it multiple times. We'll record the results of each run
|
|
// formatted as JSON.
|
|
JsonSerializableTestResults serializableResults = new JsonSerializableTestResults();
|
|
serializableResults.blobPrefix = helixResultsContainerUri;
|
|
serializableResults.blobSuffix = helixResultsContainerRsas;
|
|
|
|
List<string> errorList = new List<string>();
|
|
errorList.Add(result.Details);
|
|
|
|
foreach (TestResult rerunResult in result.RerunResults)
|
|
{
|
|
errorList.Add(rerunResult.Details);
|
|
}
|
|
|
|
serializableResults.errors = errorList.Distinct().Where(s => s != null).ToArray();
|
|
|
|
var reason = new XElement("reason");
|
|
List<JsonSerializableTestResult> serializableResultList = new List<JsonSerializableTestResult>();
|
|
serializableResultList.Add(ConvertToSerializableResult(result, serializableResults.errors));
|
|
|
|
foreach (TestResult rerunResult in result.RerunResults)
|
|
{
|
|
serializableResultList.Add(ConvertToSerializableResult(rerunResult, serializableResults.errors));
|
|
}
|
|
|
|
serializableResults.results = serializableResultList.ToArray();
|
|
|
|
using (MemoryStream stream = new MemoryStream())
|
|
{
|
|
DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(JsonSerializableTestResults));
|
|
serializer.WriteObject(stream, serializableResults);
|
|
stream.Position = 0;
|
|
|
|
using (StreamReader streamReader = new StreamReader(stream))
|
|
{
|
|
subResultsJsonByMethod.Add(methodName, streamReader.ReadToEnd());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return subResultsJsonByMethod;
|
|
}
|
|
|
|
public void ConvertWttLogToXUnitLog(string wttInputPath, string wttSingleRerunInputPath, string wttMultipleRerunInputPath, string xunitOutputPath, int requiredPassRateThreshold)
|
|
{
|
|
TestPass testPass = TestPass.ParseTestWttFileWithReruns(wttInputPath, wttSingleRerunInputPath, wttMultipleRerunInputPath, cleanupFailuresAreRegressions: true, truncateTestNames: false);
|
|
var results = testPass.TestResults;
|
|
|
|
int resultCount = results.Count;
|
|
int passedCount = results.Where(r => r.Passed).Count();
|
|
|
|
// Since we re-run tests on failure, we'll mark every test that failed at least once as "skipped" rather than "failed".
|
|
// If the test failed sufficiently often enough for it to count as a failed test (determined by a property on the
|
|
// Azure DevOps job), we'll later mark it as failed during test results processing.
|
|
|
|
int failedCount = results.Where(r => !r.PassedOrUnreliable(requiredPassRateThreshold)).Count();
|
|
int skippedCount = results.Where(r => !r.Passed && r.PassedOrUnreliable(requiredPassRateThreshold)).Count();
|
|
|
|
var root = new XElement("assemblies");
|
|
|
|
var assembly = new XElement("assembly");
|
|
assembly.SetAttributeValue("name", "MUXControls.Test.dll");
|
|
assembly.SetAttributeValue("test-framework", "TAEF");
|
|
assembly.SetAttributeValue("run-date", DateTime.Now.ToString("yyyy-MM-dd"));
|
|
|
|
// This doesn't need to be completely accurate since it's not exposed anywhere.
|
|
// If we need accurate an start time we can probably calculate it from the te.wtl file, but for
|
|
// now this is fine.
|
|
assembly.SetAttributeValue("run-time", (DateTime.Now - testPass.TestPassExecutionTime).ToString("hh:mm:ss"));
|
|
|
|
assembly.SetAttributeValue("total", resultCount);
|
|
assembly.SetAttributeValue("passed", passedCount);
|
|
assembly.SetAttributeValue("failed", failedCount);
|
|
assembly.SetAttributeValue("skipped", skippedCount);
|
|
|
|
assembly.SetAttributeValue("time", (int)testPass.TestPassExecutionTime.TotalSeconds);
|
|
assembly.SetAttributeValue("errors", 0);
|
|
root.Add(assembly);
|
|
|
|
var collection = new XElement("collection");
|
|
collection.SetAttributeValue("total", resultCount);
|
|
collection.SetAttributeValue("passed", passedCount);
|
|
collection.SetAttributeValue("failed", failedCount);
|
|
collection.SetAttributeValue("skipped", skippedCount);
|
|
collection.SetAttributeValue("name", "Test collection");
|
|
collection.SetAttributeValue("time", (int)testPass.TestPassExecutionTime.TotalSeconds);
|
|
assembly.Add(collection);
|
|
|
|
foreach (var result in results)
|
|
{
|
|
var test = new XElement("test");
|
|
test.SetAttributeValue("name", testNamePrefix + "." + result.Name);
|
|
|
|
var className = GetTestClassName(result.Name);
|
|
var methodName = GetTestMethodName(result.Name);
|
|
test.SetAttributeValue("type", className);
|
|
test.SetAttributeValue("method", methodName);
|
|
|
|
test.SetAttributeValue("time", result.ExecutionTime.TotalSeconds);
|
|
|
|
string resultString = string.Empty;
|
|
|
|
if (result.Passed)
|
|
{
|
|
resultString = "Pass";
|
|
}
|
|
else if(result.PassedOrUnreliable(requiredPassRateThreshold))
|
|
{
|
|
resultString = "Skip";
|
|
}
|
|
else
|
|
{
|
|
resultString = "Fail";
|
|
}
|
|
|
|
|
|
test.SetAttributeValue("result", resultString);
|
|
|
|
if (!result.Passed)
|
|
{
|
|
// If a test failed, we'll have rerun it multiple times.
|
|
// We'll save the subresults to a JSON text file that we'll upload to the helix results container -
|
|
// this allows it to be as long as we want, whereas the reason field in Azure DevOps has a 4000 character limit.
|
|
string subResultsFileName = methodName + "_subresults.json";
|
|
string subResultsFilePath = Path.Combine(Path.GetDirectoryName(wttInputPath), subResultsFileName);
|
|
|
|
if (result.PassedOrUnreliable(requiredPassRateThreshold))
|
|
{
|
|
var reason = new XElement("reason");
|
|
reason.Add(new XCData(GetUploadedFileUrl(subResultsFileName, helixResultsContainerUri, helixResultsContainerRsas)));
|
|
test.Add(reason);
|
|
}
|
|
else
|
|
{
|
|
var failure = new XElement("failure");
|
|
var message = new XElement("message");
|
|
message.Add(new XCData(GetUploadedFileUrl(subResultsFileName, helixResultsContainerUri, helixResultsContainerRsas)));
|
|
failure.Add(message);
|
|
test.Add(failure);
|
|
}
|
|
}
|
|
collection.Add(test);
|
|
}
|
|
|
|
File.WriteAllText(xunitOutputPath, root.ToString());
|
|
}
|
|
|
|
private JsonSerializableTestResult ConvertToSerializableResult(TestResult rerunResult, string[] uniqueErrors)
|
|
{
|
|
var serializableResult = new JsonSerializableTestResult();
|
|
|
|
serializableResult.outcome = rerunResult.Passed ? "Passed" : "Failed";
|
|
serializableResult.duration = (int)Math.Round(rerunResult.ExecutionTime.TotalMilliseconds);
|
|
|
|
if (!rerunResult.Passed)
|
|
{
|
|
serializableResult.log = Path.GetFileName(rerunResult.SourceWttFile);
|
|
|
|
if (rerunResult.Screenshots.Any())
|
|
{
|
|
List<string> screenshots = new List<string>();
|
|
|
|
foreach (var screenshot in rerunResult.Screenshots)
|
|
{
|
|
screenshots.Add(Path.GetFileName(screenshot));
|
|
}
|
|
|
|
serializableResult.screenshots = screenshots.ToArray();
|
|
}
|
|
|
|
// To conserve space, we'll log the index of the error to index in a list of unique errors rather than
|
|
// jotting down every single error in its entirety. We'll add one to the result so we can avoid
|
|
// serializing this property when it has the default value of 0.
|
|
serializableResult.errorIndex = Array.IndexOf(uniqueErrors, rerunResult.Details) + 1;
|
|
}
|
|
|
|
return serializableResult;
|
|
}
|
|
|
|
private string GetUploadedFileUrl(string filePath, string helixResultsContainerUri, string helixResultsContainerRsas)
|
|
{
|
|
var filename = Path.GetFileName(filePath);
|
|
return string.Format("{0}/{1}{2}", helixResultsContainerUri, filename, helixResultsContainerRsas);
|
|
}
|
|
|
|
private string GetTestNameSeparator(string testname)
|
|
{
|
|
var separatorString = ".";
|
|
if (!testname.Contains(separatorString))
|
|
{
|
|
separatorString = "::";
|
|
}
|
|
return separatorString;
|
|
}
|
|
|
|
private string GetTestMethodName(string fullyQualifiedName)
|
|
{
|
|
var separatorString = GetTestNameSeparator(fullyQualifiedName);
|
|
var methodName = fullyQualifiedName.Substring(fullyQualifiedName.LastIndexOf(separatorString) + separatorString.Length);
|
|
|
|
return methodName;
|
|
}
|
|
|
|
private string GetTestClassName(string fullyQualifiedName)
|
|
{
|
|
var separatorString = GetTestNameSeparator(fullyQualifiedName);
|
|
var className = fullyQualifiedName.Substring(0, fullyQualifiedName.LastIndexOf(separatorString));
|
|
|
|
return className;
|
|
}
|
|
}
|
|
}
|