Add Are We Synapse Yet to GHA (#2321)

* Add Are We Synapse Yet to GHA

* Better output & add comments
This commit is contained in:
Till 2022-04-05 15:32:30 +02:00 committed by GitHub
parent 47be39c18e
commit c84937b852
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 21 additions and 13 deletions

View File

@ -233,7 +233,14 @@ jobs:
- name: Summarise results.tap - name: Summarise results.tap
if: ${{ always() }} if: ${{ always() }}
run: /sytest/scripts/tap_to_gha.pl /logs/results.tap run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
- name: Sytest List Maintenance
if: ${{ always() }}
run: /src/show-expected-fail-tests.sh /logs/results.tap /src/sytest-whitelist /src/sytest-blacklist
continue-on-error: true # not fatal
- name: Are We Synapse Yet?
if: ${{ always() }}
run: /src/are-we-synapse-yet.py /logs/results.tap -v
continue-on-error: true # not fatal
- name: Upload Sytest logs - name: Upload Sytest logs
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v2
if: ${{ always() }} if: ${{ always() }}

View File

@ -3,7 +3,7 @@
from __future__ import division from __future__ import division
import argparse import argparse
import re import re
import sys import os
# Usage: $ ./are-we-synapse-yet.py [-v] results.tap # Usage: $ ./are-we-synapse-yet.py [-v] results.tap
# This script scans a results.tap file from Dendrite's CI process and spits out # This script scans a results.tap file from Dendrite's CI process and spits out
@ -156,6 +156,7 @@ def parse_test_line(line):
# ✓ POST /register downcases capitals in usernames # ✓ POST /register downcases capitals in usernames
# ... # ...
def print_stats(header_name, gid_to_tests, gid_to_name, verbose): def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
ci = os.getenv("CI") # When running from GHA, this groups the subsections
subsections = [] # Registration: 100% (13/13 tests) subsections = [] # Registration: 100% (13/13 tests)
subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"] subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"]
total_passing = 0 total_passing = 0
@ -169,7 +170,7 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
for name, passing in tests.items(): for name, passing in tests.items():
if passing: if passing:
group_passing += 1 group_passing += 1
test_names_and_marks.append(f"{'' if passing else '×'} {name}") test_names_and_marks.append(f"{'' if passing else ''} {name}")
total_tests += group_total total_tests += group_total
total_passing += group_passing total_passing += group_passing
@ -186,11 +187,11 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests)) print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests))
print("-" * (len(header_name)+1)) print("-" * (len(header_name)+1))
for line in subsections: for line in subsections:
print(" %s" % (line,)) print("%s%s" % ("::group::" if ci and verbose else "", line,))
if verbose: if verbose:
for test_name_and_pass_mark in subsection_test_names[line]: for test_name_and_pass_mark in subsection_test_names[line]:
print(" %s" % (test_name_and_pass_mark,)) print(" %s" % (test_name_and_pass_mark,))
print("") print("%s" % ("::endgroup::" if ci else ""))
print("") print("")
def main(results_tap_path, verbose): def main(results_tap_path, verbose):

View File

@ -89,17 +89,17 @@ if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then
fi fi
if [ -n "${tests_to_add}" ]; then if [ -n "${tests_to_add}" ]; then
echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:" echo "::error::The following tests passed but are not present in \`$2\`. Please append them to the file:"
echo "\`\`\`" echo "::group::Passing tests"
echo -e "${tests_to_add}" echo -e "${tests_to_add}"
echo "\`\`\`" echo "::endgroup::"
fi fi
if [ -n "${already_in_whitelist}" ]; then if [ -n "${already_in_whitelist}" ]; then
echo "**WARN**: Tests in the whitelist still marked as **expected fail**:" echo "::warning::Tests in the whitelist still marked as **expected fail**:"
echo "\`\`\`" echo "::group::Still marked as expected fail"
echo -e "${already_in_whitelist}" echo -e "${already_in_whitelist}"
echo "\`\`\`" echo "::endgroup::"
fi fi
exit ${fail_build} exit ${fail_build}