diff --git a/tools/build_variables.py b/tools/build_variables.py index dd2a041ad09..646b9e1c6d3 100644 --- a/tools/build_variables.py +++ b/tools/build_variables.py @@ -138,37 +138,109 @@ libtorch_cuda_sources = [ def add_torch_libs(): r = {} - # We start torch_python_sources with all cpp files, and exclude some - # including the files already contained in the torch and cuda bindings - globbed_sources = (native.glob( - ["torch/csrc/**/*.cpp"], - exclude=[ - # remove anything that has "generic" in it"s path - "torch/csrc/**/generic/**/*.cpp", - # distributed only uses Module.cpp - # so remove all other files and just include that - "torch/csrc/distributed/**/*.cpp", - # top-level hook of extension registration lives in a separate file - "torch/csrc/stub.cpp", - # to avoid redefinitions of symbols defined in - # dynamic_library_unix.cpp - "torch/csrc/jit/fuser/cpu/dynamic_library_win.cpp", - # exclude files that are already included in ATen/core - "torch/csrc/jit/script/function_schema_parser.cpp", - "torch/csrc/jit/script/schema_type_parser.cpp", - "torch/csrc/jit/script/lexer.cpp", - "torch/csrc/jit/script/strtod.cpp", - ], - ) + [ + libtorch_python_sources = [ + ":generate-code=THCUNN.cpp", + ":generate-code=THNN.cpp", + ":generate-code=python_functions.cpp", + ":generate-code=python_nn_functions.cpp", + ":generate-code=python_torch_functions.cpp", + ":generate-code=python_variable_methods.cpp", + "torch/csrc/CudaIPCTypes.cpp", + "torch/csrc/DataLoader.cpp", + "torch/csrc/Device.cpp", + "torch/csrc/Dtype.cpp", + "torch/csrc/DynamicTypes.cpp", + "torch/csrc/Generator.cpp", + "torch/csrc/Layout.cpp", + "torch/csrc/Module.cpp", + "torch/csrc/PtrWrapper.cpp", + "torch/csrc/Size.cpp", + "torch/csrc/Storage.cpp", + "torch/csrc/TypeInfo.cpp", + "torch/csrc/api/src/cuda.cpp", + "torch/csrc/api/src/data/datasets/mnist.cpp", + "torch/csrc/api/src/data/samplers/distributed.cpp", + "torch/csrc/api/src/data/samplers/random.cpp", + "torch/csrc/api/src/data/samplers/sequential.cpp", + "torch/csrc/api/src/data/samplers/stream.cpp", + "torch/csrc/api/src/nn/init.cpp", + "torch/csrc/api/src/nn/module.cpp", + "torch/csrc/api/src/nn/modules/batchnorm.cpp", + "torch/csrc/api/src/nn/modules/conv.cpp", + "torch/csrc/api/src/nn/modules/dropout.cpp", + "torch/csrc/api/src/nn/modules/embedding.cpp", + "torch/csrc/api/src/nn/modules/functional.cpp", + "torch/csrc/api/src/nn/modules/linear.cpp", + "torch/csrc/api/src/nn/modules/named_any.cpp", + "torch/csrc/api/src/nn/modules/rnn.cpp", + "torch/csrc/api/src/optim/adagrad.cpp", + "torch/csrc/api/src/optim/adam.cpp", + "torch/csrc/api/src/optim/lbfgs.cpp", + "torch/csrc/api/src/optim/optimizer.cpp", + "torch/csrc/api/src/optim/rmsprop.cpp", + "torch/csrc/api/src/optim/serialize.cpp", + "torch/csrc/api/src/optim/sgd.cpp", + "torch/csrc/api/src/python/init.cpp", + "torch/csrc/api/src/serialize/input-archive.cpp", + "torch/csrc/api/src/serialize/output-archive.cpp", + "torch/csrc/autograd/functions/init.cpp", + "torch/csrc/autograd/init.cpp", + "torch/csrc/autograd/python_anomaly_mode.cpp", + "torch/csrc/autograd/python_cpp_function.cpp", + "torch/csrc/autograd/python_engine.cpp", + "torch/csrc/autograd/python_function.cpp", + "torch/csrc/autograd/python_hook.cpp", + "torch/csrc/autograd/python_legacy_variable.cpp", + "torch/csrc/autograd/python_variable.cpp", + "torch/csrc/autograd/python_variable_indexing.cpp", + "torch/csrc/byte_order.cpp", + "torch/csrc/cuda/Event.cpp", + "torch/csrc/cuda/Module.cpp", + "torch/csrc/cuda/Storage.cpp", + "torch/csrc/cuda/Stream.cpp", + "torch/csrc/cuda/Tensor.cpp", + "torch/csrc/cuda/python_comm.cpp", + "torch/csrc/cuda/python_nccl.cpp", + "torch/csrc/cuda/serialization.cpp", + "torch/csrc/cuda/utils.cpp", "torch/csrc/distributed/Module.cpp", - "torch/csrc/distributed/c10d/init.cpp", "torch/csrc/distributed/c10d/ddp.cpp", + "torch/csrc/distributed/c10d/init.cpp", "torch/csrc/distributed/c10d/reducer.cpp", - ] + [":generate-code=" + x for x in GENERATED_CPP]) - libtorch_python_sources = sets.to_list(sets.difference( - sets.make(globbed_sources), - sets.make(libtorch_sources + libtorch_cuda_sources), - )) + "torch/csrc/jit/init.cpp", + "torch/csrc/jit/passes/inline_fork_wait.cpp", + "torch/csrc/jit/passes/onnx.cpp", + "torch/csrc/jit/passes/onnx/constant_fold.cpp", + "torch/csrc/jit/passes/onnx/fixup_onnx_loop.cpp", + "torch/csrc/jit/passes/onnx/peephole.cpp", + "torch/csrc/jit/passes/onnx/prepare_division_for_onnx.cpp", + "torch/csrc/jit/passes/remove_inplace_ops.cpp", + "torch/csrc/jit/passes/utils/check_alias_annotation.cpp", + "torch/csrc/jit/python_arg_flatten.cpp", + "torch/csrc/jit/python_interpreter.cpp", + "torch/csrc/jit/python_ir.cpp", + "torch/csrc/jit/python_tracer.cpp", + "torch/csrc/jit/script/init.cpp", + "torch/csrc/jit/script/python_tree_views.cpp", + "torch/csrc/multiprocessing/init.cpp", + "torch/csrc/onnx/init.cpp", + "torch/csrc/serialization.cpp", + "torch/csrc/tensor/python_tensor.cpp", + "torch/csrc/utils.cpp", + "torch/csrc/utils/cuda_lazy_init.cpp", + "torch/csrc/utils/invalid_arguments.cpp", + "torch/csrc/utils/object_ptr.cpp", + "torch/csrc/utils/python_arg_parser.cpp", + "torch/csrc/utils/structseq.cpp", + "torch/csrc/utils/tensor_apply.cpp", + "torch/csrc/utils/tensor_dtypes.cpp", + "torch/csrc/utils/tensor_layouts.cpp", + "torch/csrc/utils/tensor_list.cpp", + "torch/csrc/utils/tensor_new.cpp", + "torch/csrc/utils/tensor_numpy.cpp", + "torch/csrc/utils/tensor_types.cpp", + "torch/csrc/utils/tuple_parser.cpp", + ] common_flags = { "compiler_flags": [