From 3150c094b594e47a53d8a1097f854394058345f8 Mon Sep 17 00:00:00 2001
From: Ashwanth Kumar <ashwanthkumar@googlemail.com>
Date: Fri, 5 Feb 2016 13:27:55 +0530
Subject: [PATCH] Doing exponential backoff on route53 upon AWS throttling

---
 cloud/amazon/route53.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/cloud/amazon/route53.py b/cloud/amazon/route53.py
index aed53b1d319..e06b6325c5a 100644
--- a/cloud/amazon/route53.py
+++ b/cloud/amazon/route53.py
@@ -335,6 +335,21 @@ def commit(changes, retry_interval, wait, wait_timeout):
         raise TimeoutError()
     return result
 
+# Shamelessly copied over from https://git.io/vgmDG
+IGNORE_CODE = 'Throttling'
+MAX_RETRIES=5
+def invoke_with_throttling_retries(function_ref, *argv):
+    retries=0
+    while True:
+        try:
+            retval=function_ref(*argv)
+            return retval
+        except boto.exception.BotoServerError, e:
+            if e.code != IGNORE_CODE or retries==MAX_RETRIES:
+                raise e
+        time.sleep(5 * (2**retries))
+        retries += 1
+
 def main():
     argument_spec = ec2_argument_spec()
     argument_spec.update(dict(
@@ -507,7 +522,7 @@ def main():
         changes.add_change_record(command, wanted_rset)
 
     try:
-        result = commit(changes, retry_interval_in, wait_in, wait_timeout_in)
+        result = invoke_with_throttling_retries(commit, changes, retry_interval_in, wait_in, wait_timeout_in)
     except boto.route53.exception.DNSServerError, e:
         txt = e.body.split("<Message>")[1]
         txt = txt.split("</Message>")[0]