diff --git a/.lintrunner.toml b/.lintrunner.toml index 7126745fca1..6d3aa07581b 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -53,6 +53,9 @@ include_patterns = [ 'test/cpp/tensorexpr/**/*.h', 'test/cpp/tensorexpr/**/*.cpp', ] +exclude_patterns = [ + 'torch/csrc/jit/serialization/mobile_bytecode_generated.h', +] init_command = [ 'python3', 'tools/linter/adapters/s3_init.py', @@ -74,13 +77,15 @@ command = [ code = 'MYPY' include_patterns = [ 'torch/**/*.py', + 'torch/**/*.pyi', 'caffe2/**/*.py', + 'caffe2/**/*.pyi', 'test/test_bundled_images.py', 'test/test_bundled_inputs.py', 'test/test_complex.py', 'test/test_datapipe.py', 'test/test_futures.py', - 'test/test_numpy_interop.py', + # 'test/test_numpy_interop.py', 'test/test_torch.py', 'test/test_type_hints.py', 'test/test_type_info.py', @@ -90,6 +95,17 @@ exclude_patterns = [ 'torch/include/**', 'torch/csrc/**', 'torch/distributed/elastic/agent/server/api.py', + 'torch/testing/_internal/**', + 'torch/distributed/fsdp/fully_sharded_data_parallel.py', + # TODO(suo): these exclusions were added just to get lint clean on master. + # Follow up to do more target suppressions and remove them. + 'torch/distributed/fsdp/flatten_params_wrapper.py', + 'torch/ao/quantization/fx/convert.py', + 'torch/ao/quantization/_dbr/function_fusion.py', + 'test/test_datapipe.py', + 'caffe2/contrib/fakelowp/test/test_batchmatmul_nnpi_fp16.py', + 'test/test_numpy_interop.py', + 'torch/torch_version.py', ] command = [ 'python3', @@ -108,6 +124,7 @@ init_command = [ 'mypy==0.812', 'junitparser==2.1.1', 'rich==10.9.0', + 'pyyaml==6.0', ] [[linter]] @@ -319,6 +336,7 @@ include_patterns = [ ] exclude_patterns = [ 'aten/src/ATen/native/quantized/cpu/qnnpack/**', + 'torch/csrc/jit/serialization/mobile_bytecode_generated.h', ] command = [ 'python3', diff --git a/tools/linter/adapters/flake8_linter.py b/tools/linter/adapters/flake8_linter.py index 50b257f41ff..1c9f3a2442e 100644 --- a/tools/linter/adapters/flake8_linter.py +++ b/tools/linter/adapters/flake8_linter.py @@ -1,5 +1,4 @@ import argparse -import concurrent.futures import json import logging import os @@ -244,8 +243,8 @@ def get_issue_documentation_url(code: str) -> str: return "" -def check_file( - filename: str, +def check_files( + filenames: List[str], binary: str, flake8_plugins_path: Optional[str], severities: Dict[str, LintSeverity], @@ -253,7 +252,7 @@ def check_file( ) -> List[LintMessage]: try: proc = run_command( - [binary, "--exit-zero", filename], + [binary, "--exit-zero"] + filenames, extra_env={"FLAKE8_PLUGINS_PATH": flake8_plugins_path} if flake8_plugins_path else None, @@ -262,7 +261,7 @@ def check_file( except (OSError, subprocess.CalledProcessError) as err: return [ LintMessage( - path=filename, + path=None, line=None, char=None, code="FLAKE8", @@ -369,28 +368,9 @@ def main() -> None: assert len(parts) == 2, f"invalid severity `{severity}`" severities[parts[0]] = LintSeverity(parts[1]) - with concurrent.futures.ThreadPoolExecutor( - max_workers=os.cpu_count(), - thread_name_prefix="Thread", - ) as executor: - futures = { - executor.submit( - check_file, - filename, - args.binary, - flake8_plugins_path, - severities, - args.retries, - ): filename - for filename in args.filenames - } - for future in concurrent.futures.as_completed(futures): - try: - for lint_message in future.result(): - print(json.dumps(lint_message._asdict()), flush=True) - except Exception: - logging.critical('Failed at "%s".', futures[future]) - raise + lint_messages = check_files(args.filenames, args.binary, flake8_plugins_path, severities, args.retries) + for lint_message in lint_messages: + print(json.dumps(lint_message._asdict()), flush=True) if __name__ == "__main__": diff --git a/tools/linter/adapters/mypy_linter.py b/tools/linter/adapters/mypy_linter.py index 687f8bf6806..e7b4ba46901 100644 --- a/tools/linter/adapters/mypy_linter.py +++ b/tools/linter/adapters/mypy_linter.py @@ -1,5 +1,4 @@ import argparse -import concurrent.futures import json import logging import os @@ -8,6 +7,7 @@ import subprocess import sys import time from enum import Enum +from pathlib import Path from typing import Any, Dict, List, NamedTuple, Optional, Pattern @@ -56,7 +56,6 @@ RESULTS_RE: Pattern[str] = re.compile( ) - def run_command( args: List[str], *, @@ -82,15 +81,16 @@ severities = { "note": LintSeverity.ADVICE, } -def check_file( - filename: str, + +def check_files( + filenames: List[str], config: str, binary: str, retries: int, ) -> List[LintMessage]: try: proc = run_command( - [binary, f"--config={config}", filename], + [binary, f"--config={config}"] + filenames, extra_env={}, retries=retries, ) @@ -105,9 +105,7 @@ def check_file( name="command-failed", original=None, replacement=None, - description=( - f"Failed due to {err.__class__.__name__}:\n{err}" - ), + description=(f"Failed due to {err.__class__.__name__}:\n{err}"), ) ] stdout = str(proc.stdout, "utf-8").strip() @@ -172,27 +170,26 @@ def main() -> None: stream=sys.stderr, ) - with concurrent.futures.ThreadPoolExecutor( - max_workers=os.cpu_count(), - thread_name_prefix="Thread", - ) as executor: - futures = { - executor.submit( - check_file, - filename, - args.config, - args.binary, - args.retries, - ): filename - for filename in args.filenames - } - for future in concurrent.futures.as_completed(futures): - try: - for lint_message in future.result(): - print(json.dumps(lint_message._asdict()), flush=True) - except Exception: - logging.critical('Failed at "%s".', futures[future]) - raise + # Use a dictionary here to preserve order. mypy cares about order, + # tragically, e.g. https://github.com/python/mypy/issues/2015 + filenames: Dict[str, bool] = {} + + # If a stub file exists, have mypy check it instead of the original file, in + # accordance with PEP-484 (see https://www.python.org/dev/peps/pep-0484/#stub-files) + for filename in args.filenames: + if filename.endswith(".pyi"): + filenames[filename] = True + continue + + stub_filename = filename.replace(".py", ".pyi") + if Path(stub_filename).exists(): + filenames[stub_filename] = True + else: + filenames[filename] = True + + lint_messages = check_files(list(filenames), args.config, args.binary, args.retries) + for lint_message in lint_messages: + print(json.dumps(lint_message._asdict()), flush=True) if __name__ == "__main__": diff --git a/torch/csrc/jit/runtime/decomposition_registry_util.cpp b/torch/csrc/jit/runtime/decomposition_registry_util.cpp index 9b7ef138158..e6fe44b5fdc 100644 --- a/torch/csrc/jit/runtime/decomposition_registry_util.cpp +++ b/torch/csrc/jit/runtime/decomposition_registry_util.cpp @@ -7,15 +7,14 @@ */ #include #include -#include #include +#include namespace torch { namespace jit { - const std::string decomp_funcs = -R"(def var_decomposition(input: Tensor, + R"(def var_decomposition(input: Tensor, dim: Optional[List[int]]=None, correction: Optional[int]=None, keepdim: bool=False) -> Tensor: diff --git a/torch/csrc/jit/runtime/profiling_graph_executor_impl.cpp b/torch/csrc/jit/runtime/profiling_graph_executor_impl.cpp index 7e12e91453a..bdbb80e181d 100644 --- a/torch/csrc/jit/runtime/profiling_graph_executor_impl.cpp +++ b/torch/csrc/jit/runtime/profiling_graph_executor_impl.cpp @@ -456,7 +456,9 @@ void ProfilingGraphExecutorImpl::runNoGradOptimizations( for (const auto& passPair : getCustomPostPasses()) { passPair.first(graph); } - GRAPH_DEBUG("After customPostPasses, before RemoveTensorTypeSpecializations \n", *graph); + GRAPH_DEBUG( + "After customPostPasses, before RemoveTensorTypeSpecializations \n", + *graph); RemoveTensorTypeSpecializations(graph); GRAPH_DEBUG("After RemoveTensorTypeSpecializations\n", *graph); }