Allow oversize protocol buffers (#3895)

Set an option to increase the memory limit on protobuf
parsing so that we can handle larger gRPC payloads.

Co-authored-by: Evan Boyle <EvanBoyle@users.noreply.github.com>

CHANGELOG.md

@@ -2,6 +2,9 @@ CHANGELOG
 =========
 
 ## HEAD (unreleased)
+- Allow oversize protocol buffers for Python SDK.
+  [#3895](https://github.com/pulumi/pulumi/pull/3895)
+
 - Avoid duplicated messages in preview/update progress display.
   [#3890](https://github.com/pulumi/pulumi/pull/3890)
 

sdk/python/lib/pulumi/runtime/invoke.py

@@ -15,6 +15,7 @@ import asyncio
 import sys
 from typing import Any, Awaitable
 import grpc
+from google.protobuf.pyext._message import SetAllowOversizeProtos  # pylint: disable-msg=E0611
 
 from ..output import Inputs
 from ..invoke import InvokeOptions
@@ -24,6 +25,14 @@ from ..runtime.proto import provider_pb2
 from . import rpc
 from .rpc_manager import RPC_MANAGER
 
+# This setting overrides a hardcoded maximum protobuf size in the python protobuf bindings. This avoids deserialization
+# exceptions on large gRPC payloads, but makes it possible to use enough memory to cause an OOM error instead [1].
+# Note: We hit the default maximum protobuf size in practice when processing Kubernetes CRDs. If this setting ends up
+# causing problems, it should be possible to work around it with more intelligent resource chunking in the k8s provider.
+#
+# [1] https://github.com/protocolbuffers/protobuf/blob/0a59054c30e4f0ba10f10acfc1d7f3814c63e1a7/python/google/protobuf/pyext/message.cc#L2017-L2024
+SetAllowOversizeProtos(True)
+
 # If we are not running on Python 3.7 or later, we need to swap the Python implementation of Task in for the C
 # implementation in order to support synchronous invokes.
 if sys.version_info[0] == 3 and sys.version_info[1] < 7:

sdk/python/mypy.ini

@@ -10,6 +10,9 @@ ignore_missing_imports = True
 [mypy-dill]
 ignore_missing_imports = True
 
+[mypy-google.protobuf.pyext._message]
+ignore_missing_imports = True
+
 # grpc generated
 [mypy-pulumi.runtime.proto.*]
 ignore_errors = True