[BE]: Add PEP621 project section to pyproject.toml (#153055)

Follow up to @ezyang's PR #153020 , but better uses PEP621 to reduce redundant fields and pass through metadata better to uv, setuptools, poetry and other tooling.

* Enables modern tooling like uv sync and better support for tools like poetry.
* Also allows us to set project wide settings that are respected by linters and IDE (in this example we are able centralize the minimum supported python version).
* Currently most of the values are dynamically fetched from setuptools, eventually we can migrate all the statically defined values to pyproject.toml and they will be autopopulated in the setuptool arguments.
* This controls what additional metadata shows up on PyPi . Special URL Names are listed here for rendering on pypi: https://packaging.python.org/en/latest/specifications/well-known-project-urls/#well-known-labels

These also clearly shows us what fields will need to be migrated to pyproject.toml over time from setup.py per #152276. Static fields be fairly easy to migrate, the dynamically built ones like requirements are a bit more challenging.

Without this, `uv sync` complains:
```
error: No `project` table found in: `pytorch/pyproject.toml`
```

Pull Request resolved: https://github.com/pytorch/pytorch/pull/153055
Approved by: https://github.com/ezyang

pyproject.toml

@@ -1,3 +1,27 @@
+[project]
+name = "torch"
+requires-python = ">=3.9"
+license = {text = "BSD-3-Clause"}
+dynamic = [
+  "authors",
+  "classifiers",
+  "entry-points",
+  "dependencies",
+  "description",
+  "keywords",
+  "optional-dependencies",
+  "readme",
+  "scripts",
+  "version",
+]
+
+[project.urls]
+Homepage = "https://pytorch.org/"
+Documentation = "https://pytorch.org/docs/"
+Source = "https://github.com/pytorch/pytorch"
+Forum = "https://discuss.pytorch.org/"
+
+
 [build-system]
 requires = [
     # After 75.8.2 dropped dep disttools API. Please fix
@@ -18,8 +42,6 @@ build-backend = "setuptools.build_meta:__legacy__"
 
 [tool.black]
 line-length = 88
-target-version = ["py38"]
-
 
 [tool.isort]
 src_paths = ["caffe2", "torch", "torchgen", "functorch", "test"]
@@ -42,7 +64,6 @@ standard_library = ["typing_extensions"]
 
 
 [tool.ruff]
-target-version = "py39"
 line-length = 88
 src = ["caffe2", "torch", "torchgen", "functorch", "test"]
 

setup.py

@@ -1296,6 +1296,8 @@ def main():
         install_requires=install_requires,
         extras_require=extras_require,
         package_data=package_data,
+        # TODO fix later Manifest.IN file was previously ignored
+        include_package_data=False,  # defaults to True with pyproject.toml file
         url="https://pytorch.org/",
         download_url="https://github.com/pytorch/pytorch/tags",
         author="PyTorch Team",

test/test_testing.py

@@ -2352,6 +2352,11 @@ class TestImports(TestCase):
                            "torch.onnx._internal",  # depends on onnx-script
                            "torch._inductor.runtime.triton_helpers",  # depends on triton
                            "torch._inductor.codegen.cuda",  # depends on cutlass
+                           "torch.distributed.benchmarks",  # depends on RPC and DDP Optim
+                           "torch.distributed.examples",  # requires CUDA and torchvision
+                           "torch.distributed.tensor.examples",  # example scripts
+                           "torch.csrc",  # files here are devtools, not part of torch
+                           "torch.include",  # torch include files after install
                            ]
         if IS_WINDOWS or IS_MACOS or IS_JETSON:
             # Distributed should be importable on Windows(except nn.api.), but not on Mac

torch/distributed/examples/memory_tracker_example.py

@@ -1,6 +1,4 @@
 # mypy: allow-untyped-defs
-import torchvision
-
 import torch
 from torch.distributed._tools import MemoryTracker
 
@@ -30,4 +28,9 @@ def run_one_model(net: torch.nn.Module, input: torch.Tensor):
     mem_tracker.show_traces()
 
 
-run_one_model(torchvision.models.resnet34(), torch.rand(32, 3, 224, 224, device="cuda"))
+if __name__ == "__main__":
+    import torchvision
+
+    run_one_model(
+        torchvision.models.resnet34(), torch.rand(32, 3, 224, 224, device="cuda")
+    )