OSSForks/pytorch
1
0
Fork 0
mirror of https://github.com/zebrajr/pytorch.git synced 2025-12-06 12:20:52 +01:00
Code Issues Actions 38 Packages Projects Releases Wiki Activity

benchmark: higher tolerance for RobertaForQuestionAnswering (#107376)

Browse Source 
Pull Request resolved: https://github.com/pytorch/pytorch/pull/107376
Approved by: https://github.com/kit1980, https://github.com/XiaobingSuper, https://github.com/jansel
ghstack dependencies: #107375
This commit is contained in:
blzheng 2023-08-17 07:09:14 -07:00 committed by PyTorch MergeBot
parent 1ea83f04d2
commit b9befc53a6
3 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
benchmarks/dynamo/ci_expected_accuracy/inductor_huggingface_dynamic_inference.csv
View File

@ -35,7 +35,7 @@ PLBartForConditionalGeneration,pass,0
PegasusForCausalLM,pass,0
PegasusForConditionalGeneration,pass,0
RobertaForCausalLM,pass,0
RobertaForQuestionAnswering,fail_accuracy,0
RobertaForQuestionAnswering,pass,0
Speech2Text2ForCausalLM,pass,0
T5ForConditionalGeneration,pass,0
T5Small,pass,0

1 name accuracy graph_breaks
35 PegasusForCausalLM pass 0
36 PegasusForConditionalGeneration pass 0
37 RobertaForCausalLM pass 0
38 RobertaForQuestionAnswering fail_accuracy pass 0
39 Speech2Text2ForCausalLM pass 0
40 T5ForConditionalGeneration pass 0
41 T5Small pass 0

2
benchmarks/dynamo/ci_expected_accuracy/inductor_huggingface_inference.csv
View File

@ -37,7 +37,7 @@ PLBartForConditionalGeneration,pass,0
PegasusForCausalLM,pass,0
PegasusForConditionalGeneration,pass,0
RobertaForCausalLM,pass,0
RobertaForQuestionAnswering,fail_accuracy,0
RobertaForQuestionAnswering,pass,0
Speech2Text2ForCausalLM,pass,0
T5ForConditionalGeneration,pass,0
T5Small,pass,0

1 name accuracy graph_breaks
37 PegasusForCausalLM pass 0
38 PegasusForConditionalGeneration pass 0
39 RobertaForCausalLM pass 0
40 RobertaForQuestionAnswering fail_accuracy pass 0
41 Speech2Text2ForCausalLM pass 0
42 T5ForConditionalGeneration pass 0
43 T5Small pass 0

10
benchmarks/dynamo/huggingface.py
View File

@ -163,12 +163,15 @@ SKIP_ACCURACY_CHECK_MODELS = {
}
REQUIRE_HIGHER_TOLERANCE = {
REQUIRE_HIGHER_TOLERANCE_TRAINING = {
"MT5ForConditionalGeneration",
# AlbertForQuestionAnswering fails in CI GCP A100 but error does not seem
# harmful.
"AlbertForQuestionAnswering",
}
REQUIRE_HIGHER_TOLERANCE_INFERENCE = {
"RobertaForQuestionAnswering",
}
SKIP_FOR_CPU = {
@ -530,10 +533,13 @@ class HuggingfaceRunner(BenchmarkRunner):
def get_tolerance_and_cosine_flag(self, is_training, current_device, name):
cosine = self.args.cosine
if is_training:
if name in REQUIRE_HIGHER_TOLERANCE:
if name in REQUIRE_HIGHER_TOLERANCE_TRAINING:
return 2e-2, cosine
else:
return 1e-2, cosine
else:
if name in REQUIRE_HIGHER_TOLERANCE_INFERENCE:
return 4e-3, cosine
return 1e-3, cosine
def compute_loss(self, pred):