From 2c8825b387445fc9aaaa945433aaf94851540375 Mon Sep 17 00:00:00 2001
From: Thorsten Eckel <te@znuny.com>
Date: Tue, 19 Nov 2019 15:21:50 +0100
Subject: [PATCH] Follow up - 5ca41c83895e327c619f2f5255f87d2b26ab8d15 -
 ActiveJobLock lock update fails due to PG::TRSerializationFailure, 'Could not
 serialize access due to concurrent update' exception

It indicates that the UPDATE or DELETE statement was queued behind another UPDATE/DELETE statement on the same row. That other statement finished, and due to the guarantees of the Serializable isolation level, the statement in this session was canceled.

The PostgreSQL states that the transaction should automatically retry the UPDATE or DELETE when seeing this error message, whilst taking the updated state of the row into account.

See 'Serializable Isolation Level' section of https://www.postgresql.org/docs/10/transaction-iso.html
---
 app/jobs/concerns/has_active_job_lock.rb      |  6 +++++
 .../jobs/concerns/has_active_job_lock_spec.rb | 23 +++++++++++++++++++
 2 files changed, 29 insertions(+)

diff --git a/app/jobs/concerns/has_active_job_lock.rb b/app/jobs/concerns/has_active_job_lock.rb
index 0c09bf92f..57d784957 100644
--- a/app/jobs/concerns/has_active_job_lock.rb
+++ b/app/jobs/concerns/has_active_job_lock.rb
@@ -89,6 +89,12 @@ module HasActiveJobLock
     ActiveJobLock.transaction(isolation: :serializable) do
       yield
     end
+  # PostgeSQL prevents locking on records that are already locked
+  # for UPDATE in Serializable Isolation Level transactions,
+  # but it's safe to retry as described in the docs:
+  # https://www.postgresql.org/docs/10/transaction-iso.html
+  rescue PG::TRSerializationFailure
+    retry
   rescue ActiveRecord::RecordNotUnique
     existing_active_job_lock!
   end
diff --git a/spec/jobs/concerns/has_active_job_lock_spec.rb b/spec/jobs/concerns/has_active_job_lock_spec.rb
index e312e0274..d7354eeae 100644
--- a/spec/jobs/concerns/has_active_job_lock_spec.rb
+++ b/spec/jobs/concerns/has_active_job_lock_spec.rb
@@ -100,6 +100,29 @@ RSpec.describe HasActiveJobLock, type: :job do
         end.to have_enqueued_job(job_class).exactly(:twice)
       end
     end
+
+    if ActiveRecord::Base.connection.adapter_name == 'PostgreSQL'
+      context "when PG::TRSerializationFailure 'Could not serialize access due to concurrent update' is raised" do
+
+        it 'retries execution until succeed' do
+          allow(ActiveRecord::Base.connection).to receive(:open_transactions).and_return(0)
+          allow(ActiveJobLock).to receive(:transaction).and_call_original
+          exception_raised = false
+          allow(ActiveJobLock).to receive(:transaction).with(isolation: :serializable) do |&block|
+
+            if !exception_raised
+              exception_raised = true
+              raise PG::TRSerializationFailure, 'Could not serialize access due to concurrent update'
+            end
+
+            block.call
+          end
+
+          expect { job_class.perform_later }.to have_enqueued_job(job_class).exactly(:once)
+          expect(exception_raised).to be true
+        end
+      end
+    end
   end
 
   include_examples 'handle locking of jobs'