Merge pull request #410 from climbfuji/feature/pytorch_internal_protobuf

Add variant 'internal-protobuf', default False, for py-torch

var/spack/repos/builtin/packages/awscli-v2/package.py

@@ -32,8 +32,16 @@ class AwscliV2(PythonPackage):
 
     variant("examples", default=True, description="Install code examples")
 
+    # DH* 20240307 - temporary change until spack core updates are merged into our fork
+    # @run_after("install")
+    # @when("~examples")
+    # def post_install(self):
+    #     examples_dir = join_path(python_purelib, "awscli", "examples")
+    #     remove_directory_contents(examples_dir)
     @run_after("install")
-    @when("~examples")
     def post_install(self):
-        examples_dir = join_path(python_purelib, "awscli", "examples")
-        remove_directory_contents(examples_dir)
+        if self.spec.satisfies("~examples"):
+            examples_dir = join_path(python_purelib, "awscli", "examples")
+            remove_directory_contents(examples_dir)
+
+    # *DH

var/spack/repos/builtin/packages/py-torch/package.py

@@ -102,6 +102,11 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
         description="Enable breakpad crash dump library",
         when="@1.10:1.11",
     )
+    # py-torch has strict dependencies on old protobuf/py-protobuf versions that
+    # don't build with some of the compilers (e.g. intel) or cause problems with
+    # other packages that require newer versions of (py-)protobuf. Provide an
+    # option to use the internal/vendored protobuf instead of spack's.
+    variant("internal-protobuf", default=False, description="Use vendored protobuf/py-protobuf")
 
     conflicts("+cuda+rocm")
     conflicts("+tensorpipe", when="+rocm ^hip@:5.1", msg="TensorPipe not supported until ROCm 5.2")
@@ -172,14 +177,15 @@ class PyTorch(PythonPackage, CudaPackage, ROCmPackage):
     depends_on("py-pybind11@2.10.0", when="@1.13:1", type=("build", "link", "run"))
     depends_on("py-pybind11@2.6.2", when="@1.8:1.12", type=("build", "link", "run"))
     depends_on("py-pybind11@2.3.0", when="@:1.7", type=("build", "link", "run"))
-    depends_on("py-protobuf@3.12.2:", when="@1.10:", type=("build", "run"))
-    depends_on("py-protobuf@:3.14", when="@:1.9", type=("build", "run"))
-    depends_on("protobuf@3.12.2:", when="@1.10:")
-    depends_on("protobuf@:3.14", when="@:1.9")
-    # https://github.com/protocolbuffers/protobuf/issues/10051
-    # https://github.com/pytorch/pytorch/issues/78362
-    depends_on("py-protobuf@:3", type=("build", "run"))
-    depends_on("protobuf@:3", type=("build", "run"))
+    with when("~internal-protobuf"):
+        depends_on("py-protobuf@3.12.2:", when="@1.10:", type=("build", "run"))
+        depends_on("py-protobuf@:3.14", when="@:1.9", type=("build", "run"))
+        depends_on("protobuf@3.12.2:", when="@1.10:")
+        depends_on("protobuf@:3.14", when="@:1.9")
+        # https://github.com/protocolbuffers/protobuf/issues/10051
+        # https://github.com/pytorch/pytorch/issues/78362
+        depends_on("py-protobuf@:3", type=("build", "run"))
+        depends_on("protobuf@:3", type=("build", "run"))
     depends_on("eigen")
     # https://github.com/pytorch/pytorch/issues/60329
     # depends_on("cpuinfo@2023-01-13", when="@2.1:")
@@ -600,7 +606,10 @@ def enable_or_disable(variant, keyword="USE", var=None):
             env.set("WITH_BLAS", "generic")
 
         # Don't use vendored third-party libraries when possible
-        env.set("BUILD_CUSTOM_PROTOBUF", "OFF")
+        if self.spec.satisfies("~internal-protobuf"):
+            env.set("BUILD_CUSTOM_PROTOBUF", "OFF")
+        else:
+            env.set("BUILD_CUSTOM_PROTOBUF", "ON")
         env.set("USE_SYSTEM_NCCL", "ON")
         env.set("USE_SYSTEM_EIGEN_INSTALL", "ON")
         env.set("pybind11_DIR", self.spec["py-pybind11"].prefix)