[OSSTEST,23/82] db_retry: Make the sleeps random and increasing

Commit Message

Ian Jackson Oct. 7, 2020, 5:59 p.m. UTC

From: Ian Jackson <[e-mail]>

When there's a thundering herd, this can run out of retries.

Signed-off-by: Ian Jackson <[e-mail]>
---
 Osstest.pm | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

Patch

diff --git a/Osstest.pm b/Osstest.pm
index 734c0ef6..809194f0 100644
--- a/Osstest.pm
+++ b/Osstest.pm
@@ -311,7 +311,8 @@  sub db_retry ($$$;$$) {
     my ($pre,$body) =
         (ref $code eq 'ARRAY') ? @$code : (sub { }, $code);
 
-    my $retries= 100;
+    my $max_retries= 100;
+    my $retry_count= 0;
     my $r;
     local $db_retry_stop;
     for (;;) {
@@ -339,10 +340,12 @@  sub db_retry ($$$;$$) {
 	};
 	last if !length $@;
 	die $@ unless $mjobdb->need_retry($dbh, $committing);
-        die "$dbh $body $@ ?" unless $retries-- > 0;
+        die "$dbh $body $@ GIVING UP ?" if ++$retry_count >= $max_retries;
 	eval { $dbh->rollback(); };
-	print STDERR "DB conflict (messages above may refer); retrying...\n";
-        sleep(1);
+	my $delay = rand $retry_count;
+	print STDERR "DB conflict (messages above may refer);".
+	    " retrying after $delay...\n";
+        sleep($delay);
     }
     return $r;
 }