mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-06 12:20:52 +01:00
debc3b3254
Previously we were downloading all of (eager311, dynamo38, dynamo311). Now we just download what's necessary. This is useful for update_failures.py because the dynamo tests finish much faster than the eager tests and it only needs the result from the dynamo tests. Pull Request resolved: https://github.com/pytorch/pytorch/pull/119027 Approved by: https://github.com/jamesjwu ghstack dependencies: #118874, #118882, #118931
102 lines
3.1 KiB
Python
102 lines
3.1 KiB
Python
import argparse
|
|
|
|
from common import (
|
|
get_excluded_testcases,
|
|
get_passed_testcases,
|
|
get_testcases,
|
|
key,
|
|
open_test_results,
|
|
)
|
|
|
|
from download_reports import download_reports
|
|
|
|
"""
|
|
Usage: passrate.py commit_sha
|
|
|
|
Parses test reports to measure the passrate. The passrate is defined as:
|
|
|
|
A) Take the number of tests that pass under eager mode, excluding
|
|
CUDA, OpInfo, and ModuleInfo tests
|
|
B) Of those tests, count the number of tests that pass under Dynamo
|
|
C) Take B/A.
|
|
|
|
You'll need to provide the commit_sha for a commit on the main branch,
|
|
from which we will pull CI test results.
|
|
|
|
This script requires the `gh` cli. You'll need to install it and then
|
|
authenticate with it via `gh auth login` before using this script.
|
|
https://docs.github.com/en/github-cli/github-cli/quickstart
|
|
"""
|
|
|
|
|
|
def testcases_by_time(xmls):
|
|
testcases = get_testcases(xmls)
|
|
testcases.sort(reverse=True, key=lambda x: float(x.attrib["time"]))
|
|
return testcases
|
|
|
|
|
|
def should_exclude(key):
|
|
test_file = key.split("::")[0]
|
|
# C++ tests
|
|
if test_file == "UNKNOWN":
|
|
return True
|
|
# Policy: "pass rate" does not include inductor, export, or dynamo tests.
|
|
if test_file.startswith("inductor/"):
|
|
return True
|
|
if test_file.startswith("export/"):
|
|
return True
|
|
if test_file.startswith("dynamo/"):
|
|
return True
|
|
return False
|
|
|
|
|
|
def compute_pass_rate(eager_dir, dynamo_dir):
|
|
print("parsing xmls")
|
|
eager_xmls = open_test_results(eager_dir)
|
|
dynamo_xmls = open_test_results(dynamo_dir)
|
|
|
|
print("computing pass rate")
|
|
eager_passed = get_passed_testcases(eager_xmls)
|
|
dynamo_passed = get_passed_testcases(dynamo_xmls)
|
|
dynamo_pass_keys = {key(testcase) for testcase in dynamo_passed}
|
|
dynamo_pass_keys = {key_ for key_ in dynamo_pass_keys if not should_exclude(key_)}
|
|
tmp_eager_pass_keys = {key(testcase) for testcase in eager_passed}
|
|
tmp_eager_pass_keys = {
|
|
key_ for key_ in tmp_eager_pass_keys if not should_exclude(key_)
|
|
}
|
|
excluded = [key(t) for t in get_excluded_testcases(dynamo_xmls)]
|
|
eager_pass_keys = tmp_eager_pass_keys - set(excluded)
|
|
|
|
subset = eager_pass_keys.intersection(dynamo_pass_keys)
|
|
total_subset = len(subset)
|
|
total_tests = len(eager_pass_keys)
|
|
print("pass rate", total_subset / total_tests, total_subset, total_tests)
|
|
|
|
dynamo_testcases = get_testcases(dynamo_xmls)
|
|
tc = {key(t): t for t in dynamo_testcases}
|
|
|
|
# Useful for debugging
|
|
not_there_keys = set()
|
|
for key_ in eager_pass_keys:
|
|
if key_ not in tc:
|
|
not_there_keys.add(key_)
|
|
|
|
fail_keys = eager_pass_keys - subset
|
|
return fail_keys
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(
|
|
prog="passrate", description="Computes the Dynamo unittest pass rate"
|
|
)
|
|
parser.add_argument(
|
|
"commit",
|
|
help=(
|
|
"The commit sha for the latest commit on a PR from which we will "
|
|
"pull CI test results, e.g. 7e5f597aeeba30c390c05f7d316829b3798064a5"
|
|
),
|
|
)
|
|
args = parser.parse_args()
|
|
dynamo311, eager311 = download_reports(args.commit, ("dynamo311", "eager311"))
|
|
compute_pass_rate(eager311, dynamo311)
|