mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-07 12:21:27 +01:00
218eeacacd
- add graph-breaks baselines - add check_graph_breaks script (message users on regress or improvement) - hook up test.sh for existing accuracy job Refactor graph-break CI check Take steps toward merging checker with existing check flow, consider merging it all the way inside the bench runner. csvs Pull Request resolved: https://github.com/pytorch/pytorch/pull/96346 Approved by: https://github.com/ezyang
84 lines
2.3 KiB
Python
84 lines
2.3 KiB
Python
import argparse
|
|
import sys
|
|
import textwrap
|
|
|
|
import pandas as pd
|
|
|
|
|
|
def get_field(csv, model_name: str, field: str, typ=float):
|
|
return typ(csv.loc[csv["name"] == model_name][field])
|
|
|
|
|
|
def check_graph_breaks(actual_csv, expected_csv, expected_filename):
|
|
|
|
failed = []
|
|
improved = []
|
|
|
|
for model in actual_csv["name"]:
|
|
|
|
graph_breaks = get_field(actual_csv, model, "graph_breaks", typ=int)
|
|
expected_graph_breaks = get_field(expected_csv, model, "graph_breaks", typ=int)
|
|
|
|
if graph_breaks == expected_graph_breaks:
|
|
status = "PASS"
|
|
elif graph_breaks > expected_graph_breaks:
|
|
status = "FAIL"
|
|
failed.append(model)
|
|
elif graph_breaks < expected_graph_breaks:
|
|
status = "IMPROVED"
|
|
improved.append(model)
|
|
print(
|
|
f"""
|
|
{model:34}:
|
|
graph_breaks={graph_breaks},
|
|
expected_graph_breaks={expected_graph_breaks},
|
|
{status}
|
|
"""
|
|
)
|
|
|
|
msg = ""
|
|
if failed or improved:
|
|
if failed:
|
|
msg += textwrap.dedent(
|
|
f"""
|
|
Error: {len(failed)} models have new dynamo graph breaks:
|
|
{' '.join(failed)}
|
|
|
|
"""
|
|
)
|
|
if improved:
|
|
msg += textwrap.dedent(
|
|
f"""
|
|
Improvement: {len(improved)} models have fixed dynamo graph breaks:
|
|
{' '.join(improved)}
|
|
|
|
"""
|
|
)
|
|
msg += textwrap.dedent(
|
|
f"""
|
|
If this change is expected, you can update `{expected_filename}` to reflect the new baseline.
|
|
This can either be done manually, or by downloading artifacts from your PR CI job.
|
|
(Search artifacts files for test-reports-test-inductor_torchbench, _timm, _huggingface)
|
|
"""
|
|
)
|
|
return failed or improved, msg
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--actual", type=str, required=True)
|
|
parser.add_argument("--expected", type=str, required=True)
|
|
args = parser.parse_args()
|
|
|
|
actual = pd.read_csv(args.actual)
|
|
expected = pd.read_csv(args.expected)
|
|
|
|
failed, msg = check_graph_breaks(actual, expected, args.expected)
|
|
if failed:
|
|
print(msg)
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|