mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-07 00:21:07 +01:00
09ce9b5877
Summary: We want to store the file names that triggers each test suite so that we can use this data for categorizing those test files. ~~After considering several solutions, this one is the most backwards compatible, and the current test cases in test_testing.py for print test stats don't break.~~ The previous plan did not work, as there are multiple Python test jobs that spawn the same suites. Instead, the new S3 format will store test files (e.g., `test_nn` and `distributed/test_distributed_fork`) which will contain the suites they spawn, which will contain the test cases run within the suite. (Currently, there is no top layer of test files.) Because of this major structural change, a lot of changes have now been made (thank you samestep!) to test_history.py and print_test_stats.py to make this new format backwards compatible. Old test plan: Make sure that the data is as expected in S3 after https://github.com/pytorch/pytorch/pull/52873 finishes. Pull Request resolved: https://github.com/pytorch/pytorch/pull/52869 Test Plan: Added tests to test_testing.py which pass, and CI. Reviewed By: samestep Differential Revision: D26672561 Pulled By: janeyx99 fbshipit-source-id: f46b91e16c1d9de5e0cb9bfa648b6448d979257e
478 lines
14 KiB
Python
Executable File
478 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import bz2
|
|
import json
|
|
import subprocess
|
|
from collections import defaultdict
|
|
from datetime import datetime
|
|
from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
|
|
|
|
import boto3 # type: ignore[import]
|
|
import botocore # type: ignore[import]
|
|
from typing_extensions import Literal, TypedDict
|
|
|
|
|
|
def get_git_commit_history(
|
|
*,
|
|
path: str,
|
|
ref: str
|
|
) -> List[Tuple[str, datetime]]:
|
|
rc = subprocess.check_output(
|
|
['git', '-C', path, 'log', '--pretty=format:%H %ct', ref],
|
|
).decode("latin-1")
|
|
return [
|
|
(x[0], datetime.fromtimestamp(int(x[1])))
|
|
for x in [line.split(" ") for line in rc.split("\n")]
|
|
]
|
|
|
|
|
|
def get_object_summaries(*, bucket: Any, sha: str) -> Dict[str, List[Any]]:
|
|
summaries = list(bucket.objects.filter(Prefix=f'test_time/{sha}/'))
|
|
by_job = defaultdict(list)
|
|
for summary in summaries:
|
|
job = summary.key.split('/')[2]
|
|
by_job[job].append(summary)
|
|
return dict(by_job)
|
|
|
|
|
|
# TODO: consolidate these typedefs with the identical ones in
|
|
# torch/testing/_internal/print_test_stats.py
|
|
|
|
Commit = str # 40-digit SHA-1 hex string
|
|
Status = Optional[Literal['errored', 'failed', 'skipped']]
|
|
|
|
|
|
class CaseMeta(TypedDict):
|
|
seconds: float
|
|
|
|
|
|
class Version1Case(CaseMeta):
|
|
name: str
|
|
errored: bool
|
|
failed: bool
|
|
skipped: bool
|
|
|
|
|
|
class Version1Suite(TypedDict):
|
|
total_seconds: float
|
|
cases: List[Version1Case]
|
|
|
|
|
|
class ReportMetaMeta(TypedDict):
|
|
build_pr: str
|
|
build_tag: str
|
|
build_sha1: Commit
|
|
build_branch: str
|
|
build_job: str
|
|
build_workflow_id: str
|
|
|
|
|
|
class ReportMeta(ReportMetaMeta):
|
|
total_seconds: float
|
|
|
|
|
|
class Version1Report(ReportMeta):
|
|
suites: Dict[str, Version1Suite]
|
|
|
|
|
|
class Version2Case(CaseMeta):
|
|
status: Status
|
|
|
|
|
|
class Version2Suite(TypedDict):
|
|
total_seconds: float
|
|
cases: Dict[str, Version2Case]
|
|
|
|
|
|
class Version2File(TypedDict):
|
|
total_seconds: float
|
|
suites: Dict[str, Version2Suite]
|
|
|
|
|
|
class VersionedReport(ReportMeta):
|
|
format_version: int
|
|
|
|
|
|
# report: Version2Report implies report['format_version'] == 2
|
|
class Version2Report(VersionedReport):
|
|
files: Dict[str, Version2File]
|
|
|
|
|
|
Report = Union[Version1Report, VersionedReport]
|
|
|
|
|
|
def get_jsons(
|
|
jobs: Optional[List[str]],
|
|
summaries: Dict[str, Any],
|
|
) -> Dict[str, Report]:
|
|
if jobs is None:
|
|
keys = sorted(summaries.keys())
|
|
else:
|
|
keys = [job for job in jobs if job in summaries]
|
|
return {
|
|
job: json.loads(bz2.decompress(summaries[job].get()['Body'].read()))
|
|
for job in keys
|
|
}
|
|
|
|
|
|
# TODO: consolidate this with the case_status function from
|
|
# torch/testing/_internal/print_test_stats.py
|
|
def case_status(case: Version1Case) -> Status:
|
|
for k in {'errored', 'failed', 'skipped'}:
|
|
if case[k]: # type: ignore[misc]
|
|
return cast(Status, k)
|
|
return None
|
|
|
|
|
|
# TODO: consolidate this with the newify_case function from
|
|
# torch/testing/_internal/print_test_stats.py
|
|
def newify_case(case: Version1Case) -> Version2Case:
|
|
return {
|
|
'seconds': case['seconds'],
|
|
'status': case_status(case),
|
|
}
|
|
|
|
|
|
# TODO: consolidate this with the simplify function from
|
|
# torch/testing/_internal/print_test_stats.py
|
|
def get_cases(
|
|
*,
|
|
data: Report,
|
|
filename: Optional[str],
|
|
suite_name: Optional[str],
|
|
test_name: str,
|
|
) -> List[Version2Case]:
|
|
cases: List[Version2Case] = []
|
|
if 'format_version' not in data: # version 1 implicitly
|
|
v1report = cast(Version1Report, data)
|
|
suites = v1report['suites']
|
|
for sname, v1suite in suites.items():
|
|
if sname == suite_name or not suite_name:
|
|
for v1case in v1suite['cases']:
|
|
if v1case['name'] == test_name:
|
|
cases.append(newify_case(v1case))
|
|
else:
|
|
v_report = cast(VersionedReport, data)
|
|
version = v_report['format_version']
|
|
if version == 2:
|
|
v2report = cast(Version2Report, v_report)
|
|
for fname, v2file in v2report['files'].items():
|
|
if fname == filename or not filename:
|
|
for sname, v2suite in v2file['suites'].items():
|
|
if sname == suite_name or not suite_name:
|
|
v2case = v2suite['cases'].get(test_name)
|
|
if v2case:
|
|
cases.append(v2case)
|
|
else:
|
|
raise RuntimeError(f'Unknown format version: {version}')
|
|
return cases
|
|
|
|
|
|
def make_column(
|
|
*,
|
|
data: Optional[Report],
|
|
filename: Optional[str],
|
|
suite_name: Optional[str],
|
|
test_name: str,
|
|
digits: int,
|
|
) -> Tuple[str, int]:
|
|
decimals = 3
|
|
num_length = digits + 1 + decimals
|
|
if data:
|
|
cases = get_cases(
|
|
data=data,
|
|
filename=filename,
|
|
suite_name=suite_name,
|
|
test_name=test_name
|
|
)
|
|
if cases:
|
|
case = cases[0]
|
|
status = case['status']
|
|
omitted = len(cases) - 1
|
|
if status:
|
|
return f'{status.rjust(num_length)} ', omitted
|
|
else:
|
|
return f'{case["seconds"]:{num_length}.{decimals}f}s', omitted
|
|
else:
|
|
return f'{"absent".rjust(num_length)} ', 0
|
|
else:
|
|
return ' ' * (num_length + 1), 0
|
|
|
|
|
|
def make_columns(
|
|
*,
|
|
jobs: List[str],
|
|
jsons: Dict[str, Report],
|
|
omitted: Dict[str, int],
|
|
filename: Optional[str],
|
|
suite_name: Optional[str],
|
|
test_name: str,
|
|
digits: int,
|
|
) -> str:
|
|
columns = []
|
|
total_omitted = 0
|
|
total_suites = 0
|
|
for job in jobs:
|
|
data = jsons.get(job)
|
|
column, omitted_suites = make_column(
|
|
data=data,
|
|
filename=filename,
|
|
suite_name=suite_name,
|
|
test_name=test_name,
|
|
digits=digits,
|
|
)
|
|
columns.append(column)
|
|
total_suites += omitted_suites
|
|
if job in omitted:
|
|
total_omitted += omitted[job]
|
|
if total_omitted > 0:
|
|
columns.append(f'({total_omitted} S3 reports omitted)')
|
|
if total_suites > 0:
|
|
columns.append(f'({total_suites}) matching suites omitted)')
|
|
return ' '.join(columns)
|
|
|
|
|
|
def make_lines(
|
|
*,
|
|
jobs: Set[str],
|
|
jsons: Dict[str, Report],
|
|
omitted: Dict[str, int],
|
|
filename: Optional[str],
|
|
suite_name: Optional[str],
|
|
test_name: str,
|
|
) -> List[str]:
|
|
lines = []
|
|
for job, data in jsons.items():
|
|
cases = get_cases(
|
|
data=data,
|
|
filename=filename,
|
|
suite_name=suite_name,
|
|
test_name=test_name,
|
|
)
|
|
if cases:
|
|
case = cases[0]
|
|
status = case['status']
|
|
line = f'{job} {case["seconds"]}s{f" {status}" if status else ""}'
|
|
if job in omitted and omitted[job] > 0:
|
|
line += f' ({omitted[job]} S3 reports omitted)'
|
|
if len(cases) > 1:
|
|
line += f' ({len(cases) - 1} matching suites omitted)'
|
|
lines.append(line)
|
|
elif job in jobs:
|
|
lines.append(f'{job} (test not found)')
|
|
if lines:
|
|
return lines
|
|
else:
|
|
return ['(no reports in S3)']
|
|
|
|
|
|
def display_history(
|
|
*,
|
|
bucket: Any,
|
|
commits: List[Tuple[str, datetime]],
|
|
jobs: Optional[List[str]],
|
|
filename: Optional[str],
|
|
suite_name: Optional[str],
|
|
test_name: str,
|
|
delta: int,
|
|
sha_length: int,
|
|
mode: str,
|
|
digits: int,
|
|
) -> None:
|
|
prev_time = datetime.now()
|
|
for sha, time in commits:
|
|
if (prev_time - time).total_seconds() < delta * 3600:
|
|
continue
|
|
prev_time = time
|
|
summaries = get_object_summaries(bucket=bucket, sha=sha)
|
|
# we assume that get_object_summaries doesn't return empty lists
|
|
jsons = get_jsons(
|
|
jobs=jobs,
|
|
summaries={job: l[0] for job, l in summaries.items()},
|
|
)
|
|
omitted = {
|
|
job: len(l) - 1
|
|
for job, l in summaries.items()
|
|
if len(l) > 1
|
|
}
|
|
if mode == 'columns':
|
|
assert jobs is not None
|
|
lines = [make_columns(
|
|
jobs=jobs,
|
|
jsons=jsons,
|
|
omitted=omitted,
|
|
filename=filename,
|
|
suite_name=suite_name,
|
|
test_name=test_name,
|
|
digits=digits,
|
|
)]
|
|
else:
|
|
assert mode == 'multiline'
|
|
lines = make_lines(
|
|
jobs=set(jobs or []),
|
|
jsons=jsons,
|
|
omitted=omitted,
|
|
filename=filename,
|
|
suite_name=suite_name,
|
|
test_name=test_name,
|
|
)
|
|
for line in lines:
|
|
print(f"{time} {sha[:sha_length]} {line}".rstrip())
|
|
|
|
|
|
class HelpFormatter(
|
|
argparse.ArgumentDefaultsHelpFormatter,
|
|
argparse.RawDescriptionHelpFormatter,
|
|
):
|
|
pass
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
__file__,
|
|
description='''
|
|
Display the history of a test.
|
|
|
|
Each line of (non-error) output starts with the timestamp and SHA1 hash
|
|
of the commit it refers to, in this format:
|
|
|
|
YYYY-MM-DD hh:mm:ss 0123456789abcdef0123456789abcdef01234567
|
|
|
|
In multiline mode, each line next includes the name of a CircleCI job,
|
|
followed by the time of the specified test in that job at that commit.
|
|
Example:
|
|
|
|
$ tools/test_history.py multiline --ref=594a66 --sha-length=8 \\
|
|
test_set_dir pytorch_linux_xenial_py3_6_gcc{5_4,7}_test
|
|
2021-02-10 03:13:34 594a66d7 pytorch_linux_xenial_py3_6_gcc5_4_test 0.36s
|
|
2021-02-10 03:13:34 594a66d7 pytorch_linux_xenial_py3_6_gcc7_test 0.573s errored
|
|
2021-02-10 02:13:25 9c0caf03 pytorch_linux_xenial_py3_6_gcc5_4_test 0.819s
|
|
2021-02-10 02:13:25 9c0caf03 pytorch_linux_xenial_py3_6_gcc7_test 0.449s
|
|
2021-02-10 02:09:14 602434bc pytorch_linux_xenial_py3_6_gcc5_4_test 0.361s
|
|
2021-02-10 02:09:14 602434bc pytorch_linux_xenial_py3_6_gcc7_test 0.454s
|
|
2021-02-10 02:09:10 2e35fe95 (no reports in S3)
|
|
2021-02-10 02:09:07 ff73be7e (no reports in S3)
|
|
2021-02-10 02:05:39 74082f0d (no reports in S3)
|
|
2021-02-09 23:42:29 0620c96f pytorch_linux_xenial_py3_6_gcc5_4_test 0.414s (1 S3 reports omitted)
|
|
2021-02-09 23:42:29 0620c96f pytorch_linux_xenial_py3_6_gcc7_test 0.377s (1 S3 reports omitted)
|
|
|
|
Another multiline example, this time with the --all flag:
|
|
|
|
$ tools/test_history.py multiline --all --ref=321b9 --delta=12 --sha-length=8 \\
|
|
test_qr_square_many_batched_complex_cuda
|
|
2021-01-07 02:04:56 321b9883 pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_test2 424.284s
|
|
2021-01-07 02:04:56 321b9883 pytorch_linux_xenial_cuda10_2_cudnn7_py3_slow_test 0.006s skipped
|
|
2021-01-07 02:04:56 321b9883 pytorch_linux_xenial_cuda11_1_cudnn8_py3_gcc7_test 402.572s
|
|
2021-01-07 02:04:56 321b9883 pytorch_linux_xenial_cuda9_2_cudnn7_py3_gcc7_test 287.164s
|
|
2021-01-06 12:58:28 fcb69d2e pytorch_linux_xenial_cuda10_2_cudnn7_py3_gcc7_test2 436.732s
|
|
2021-01-06 12:58:28 fcb69d2e pytorch_linux_xenial_cuda10_2_cudnn7_py3_slow_test 0.006s skipped
|
|
2021-01-06 12:58:28 fcb69d2e pytorch_linux_xenial_cuda11_1_cudnn8_py3_gcc7_test 407.616s
|
|
2021-01-06 12:58:28 fcb69d2e pytorch_linux_xenial_cuda9_2_cudnn7_py3_gcc7_test 287.044s
|
|
|
|
In columns mode, the name of the job isn't printed, but the order of the
|
|
columns is guaranteed to match the order of the jobs passed on the
|
|
command line. Example:
|
|
|
|
$ tools/test_history.py columns --ref=3cf783 --sha-length=8 \\
|
|
test_set_dir pytorch_linux_xenial_py3_6_gcc{5_4,7}_test
|
|
2021-02-10 04:18:50 3cf78395 0.644s 0.312s
|
|
2021-02-10 03:13:34 594a66d7 0.360s errored
|
|
2021-02-10 02:13:25 9c0caf03 0.819s 0.449s
|
|
2021-02-10 02:09:14 602434bc 0.361s 0.454s
|
|
2021-02-10 02:09:10 2e35fe95
|
|
2021-02-10 02:09:07 ff73be7e
|
|
2021-02-10 02:05:39 74082f0d
|
|
2021-02-09 23:42:29 0620c96f 0.414s 0.377s (2 S3 reports omitted)
|
|
2021-02-09 23:27:53 33afb5f1 0.381s 0.294s
|
|
|
|
Minor note: in columns mode, a blank cell means that no report was found
|
|
in S3, while the word "absent" means that a report was found but the
|
|
indicated test was not found in that report.
|
|
''',
|
|
formatter_class=HelpFormatter,
|
|
)
|
|
parser.add_argument(
|
|
'mode',
|
|
choices=['columns', 'multiline'],
|
|
help='output format',
|
|
)
|
|
parser.add_argument(
|
|
'--pytorch',
|
|
help='path to local PyTorch clone',
|
|
default='.',
|
|
)
|
|
parser.add_argument(
|
|
'--ref',
|
|
help='starting point (most recent Git ref) to display history for',
|
|
default='master',
|
|
)
|
|
parser.add_argument(
|
|
'--delta',
|
|
type=int,
|
|
help='minimum number of hours between commits',
|
|
default=0,
|
|
)
|
|
parser.add_argument(
|
|
'--sha-length',
|
|
type=int,
|
|
help='length of the prefix of the SHA1 hash to show',
|
|
default=40,
|
|
)
|
|
parser.add_argument(
|
|
'--digits',
|
|
type=int,
|
|
help='(columns) number of digits to display before the decimal point',
|
|
default=4,
|
|
)
|
|
parser.add_argument(
|
|
'--all',
|
|
action='store_true',
|
|
help='(multiline) ignore listed jobs, show all jobs for each commit',
|
|
)
|
|
parser.add_argument(
|
|
'--file',
|
|
help='name of the file containing the test',
|
|
)
|
|
parser.add_argument(
|
|
'--suite',
|
|
help='name of the suite containing the test',
|
|
)
|
|
parser.add_argument(
|
|
'test',
|
|
help='name of the test',
|
|
)
|
|
parser.add_argument(
|
|
'job',
|
|
nargs='*',
|
|
help='names of jobs to display columns for, in order',
|
|
default=[],
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
jobs = None if args.all else args.job
|
|
if jobs == []: # no jobs, and not None (which would mean all jobs)
|
|
parser.error('No jobs specified.')
|
|
|
|
commits = get_git_commit_history(path=args.pytorch, ref=args.ref)
|
|
|
|
s3 = boto3.resource("s3", config=botocore.config.Config(signature_version=botocore.UNSIGNED))
|
|
bucket = s3.Bucket('ossci-metrics')
|
|
|
|
display_history(
|
|
bucket=bucket,
|
|
commits=commits,
|
|
jobs=jobs,
|
|
filename=args.file,
|
|
suite_name=args.suite,
|
|
test_name=args.test,
|
|
delta=args.delta,
|
|
mode=args.mode,
|
|
sha_length=args.sha_length,
|
|
digits=args.digits,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|