[ONNX] Add sanity check in CI for onnxbench (#110178)

ONNX CI to run benchmark with `--quick` to validate the onnxbench infra.

Pull Request resolved: https://github.com/pytorch/pytorch/pull/110178
Approved by: https://github.com/thiagocrepaldi

scripts/onnx/test.sh

@@ -48,6 +48,44 @@ if [[ "$SHARD_NUMBER" == "2" ]]; then
   xdoctest torch.onnx --style=google --options="+IGNORE_WHITESPACE"
 fi
 
+if [[ "$SHARD_NUMBER" == "2" ]]; then
+  # Sanity check on torchbench w/ onnx
+  pip install pandas
+  log_folder="test/.torchbench_logs"
+  device="cpu"
+  modes=("accuracy" "performance")
+  compilers=("dynamo-onnx" "torchscript-onnx")
+  suites=("huggingface" "timm_models")
+
+  mkdir -p "${log_folder}"
+  for mode in "${modes[@]}"; do
+    for compiler in "${compilers[@]}"; do
+      for suite in "${suites[@]}"; do
+        output_file="${log_folder}/${compiler}_${suite}_float32_inference_${device}_${mode}.csv"
+        bench_file="benchmarks/dynamo/${suite}.py"
+        bench_args=("--${mode}" --float32 "-d${device}" "--output=${output_file}" "--output-directory=${top_dir}" --inference -n5 "--${compiler}" --no-skip --dashboard --batch-size 1)
+        # Run only selected model for each suite to quickly validate the benchmark suite works as expected.
+        case "$suite" in
+            "torchbench")
+                bench_args+=(-k resnet18)
+                ;;
+            "huggingface")
+                bench_args+=(-k ElectraForQuestionAnswering)
+                ;;
+            "timm_models")
+                bench_args+=(-k lcnet_050)
+                ;;
+            *)
+                echo "Unknown suite: ${suite}"
+                exit 1
+                ;;
+        esac
+        python "${top_dir}/${bench_file}" "${bench_args[@]}"
+      done
+    done
+  done
+fi
+
 # Our CI expects both coverage.xml and .coverage to be within test/
 if [ -d .coverage ]; then
   mv .coverage test/.coverage