From f11bbe7f04a48eebcb446e283820d7592f76cf86 Mon Sep 17 00:00:00 2001
From: Johnson Shi <[email protected]>
Date: Thu, 25 Mar 2021 07:20:10 -0700
Subject: [PATCH 2/7] Azure helper: Ensure Azure http handler sleeps between
 retries (#842)

RH-Author: Eduardo Otubo <[email protected]>
RH-MergeRequest: 45: Add support for userdata on Azure from IMDS
RH-Commit: [2/7] e8f8bb658b629a8444bd2ba19f109952acf33311
RH-Bugzilla: 2023940
RH-Acked-by: Emanuele Giuseppe Esposito <[email protected]>
RH-Acked-by: Mohamed Gamal Morsy <[email protected]>

Ensure that the Azure helper's http handler sleeps a fixed duration
between retry failure attempts. The http handler will sleep a fixed
duration between failed attempts regardless of whether the attempt
failed due to (1) request timing out or (2) instant failure (no
timeout).

Due to certain platform issues, the http request to the Azure endpoint
may instantly fail without reaching the http timeout duration. Without
sleeping a fixed duration in between retry attempts, the http handler
will loop through the max retry attempts quickly. This causes the
communication between cloud-init and the Azure platform to be less
resilient due to the short total duration if there is no sleep in
between retries.
---
 cloudinit/sources/helpers/azure.py                   |  2 ++
 tests/unittests/test_datasource/test_azure_helper.py | 11 +++++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index d3055d08..03e7156b 100755
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -303,6 +303,7 @@ def http_with_retries(url, **kwargs) -> str:
 
     max_readurl_attempts = 240
     default_readurl_timeout = 5
+    sleep_duration_between_retries = 5
     periodic_logging_attempts = 12
 
     if 'timeout' not in kwargs:
@@ -338,6 +339,7 @@ def http_with_retries(url, **kwargs) -> str:
                     'attempt %d with exception: %s' %
                     (url, attempt, e),
                     logger_func=LOG.debug)
+            time.sleep(sleep_duration_between_retries)
 
     raise exc
 
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index b8899807..63482c6c 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -384,6 +384,7 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
 
     max_readurl_attempts = 240
     default_readurl_timeout = 5
+    sleep_duration_between_retries = 5
     periodic_logging_attempts = 12
 
     def setUp(self):
@@ -394,8 +395,8 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
         self.m_readurl = patches.enter_context(
             mock.patch.object(
                 azure_helper.url_helper, 'readurl', mock.MagicMock()))
-        patches.enter_context(
-            mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock()))
+        self.m_sleep = patches.enter_context(
+            mock.patch.object(azure_helper.time, 'sleep', autospec=True))
 
     def test_http_with_retries(self):
         self.m_readurl.return_value = 'TestResp'
@@ -438,6 +439,12 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
             self.m_readurl.call_count,
             self.periodic_logging_attempts + 1)
 
+        # Ensure that cloud-init did sleep between each failed request
+        self.assertEqual(
+            self.m_sleep.call_count,
+            self.periodic_logging_attempts)
+        self.m_sleep.assert_called_with(self.sleep_duration_between_retries)
+
     def test_http_with_retries_long_delay_logs_periodic_failure_msg(self):
         self.m_readurl.side_effect = \
             [SentinelException] * self.periodic_logging_attempts + \
-- 
2.27.0