Testing: Complete coverage for Channel::Driver::Twitter#fetch

spec/factories/ticket/article.rb

@@ -25,6 +25,7 @@ FactoryBot.define do
       association :ticket, factory: :twitter_ticket
       message_id { '775410014383026176' }
       body { Faker::Lorem.sentence }
+      sender_name { 'Agent' }
 
       trait :reply do
         in_reply_to  { Faker::Number.number(19) }

spec/models/channel/driver/twitter_spec.rb

@@ -808,7 +808,7 @@ RSpec.describe Channel::Driver::Twitter do
 
     describe 'Twitter API activity' do
       it 'sets successful status attributes' do
-        expect { channel.fetch(true) }
+        expect { channel.fetch }
           .to change { channel.reload.attributes }
           .to hash_including(
             'status_in'    => 'ok',
@@ -818,8 +818,9 @@ RSpec.describe Channel::Driver::Twitter do
           )
       end
 
-      it 'adds tickets based on .options[:sync][:search] parameters' do
-        expect { channel.fetch(true) }
+      context 'with search term configured (at .options[:sync][:search])' do
+        it 'creates an article for each recent tweet' do
+          expect { channel.fetch }
             .to change(Ticket, :count).by(8)
 
           expect(Ticket.last.attributes).to include(
@@ -830,38 +831,168 @@ RSpec.describe Channel::Driver::Twitter do
           )
         end
 
-      it 'skips tweets more than 15 days older than channel itself'
+        it 'skips retweets' do
+          expect { channel.fetch }
+            .not_to change { Ticket.where('title LIKE ?', 'RT @%').count }.from(0)
+        end
+
+        it 'skips tweets 15+ days older than channel itself' do
+          expect { channel.fetch }
+            .not_to change { Ticket.where('title LIKE ?', 'GitHub Trending Archive, 2_ Nov 2018, Ruby. %').count }.from(0)
+        end
+
+        context 'when fetched tweets have already been imported' do
+          before do
+            tweet_ids.each { |tweet_id| create(:ticket_article, message_id: tweet_id) }
+          end
+
+          let(:tweet_ids) do
+            [1224440380881428480,
+             1224426978557800449,
+             1224427517869809666,
+             1224427776654135297,
+             1224428510225354753,
+             1223188240078909440,
+             1223273797987508227,
+             1223103807283810304,
+             1223121619561799682,
+             1222872891320143878,
+             1222881209384161283,
+             1222896407524212736,
+             1222237955588227075,
+             1222108036795334657,
+             1222126386334388225,
+             1222109934923460608]
+          end
+
+          it 'does not import duplicates' do
+            expect { channel.fetch }.not_to change(Ticket::Article, :count)
+          end
+        end
+
+        context 'for very common search terms' do
+          subject(:channel) { create(:twitter_channel, custom_options: custom_options) }
+
+          let(:custom_options) do
+            {
+              sync: {
+                search: [
+                  {
+                    term:     'coronavirus',
+                    group_id: Group.first.id
+                  }
+                ]
+              }
+            }
+          end
+
+          let(:twitter_articles) { Ticket::Article.joins(:type).where(ticket_article_types: { name: 'twitter status' }) }
+
+          it 'stops importing threads after 120 new articles' do
+            channel.fetch
+
+            expect((twitter_articles - Ticket.last.articles).count).to be < 120
+            expect(twitter_articles.count).to be >= 120
+          end
+
+          it 'refuses to import any other tweets for the next 15 minutes' do
+            channel.fetch
+            travel(14.minutes)
+
+            expect { create(:twitter_channel).fetch }
+              .not_to change(Ticket::Article, :count)
+          end
+
+          it 'resumes importing again after 15 minutes' do
+            channel.fetch
+            travel(15.minutes)
+
+            expect { create(:twitter_channel).fetch }
+              .to change(Ticket::Article, :count)
+          end
+        end
 
         context 'and "track_retweets" option' do
           subject(:channel) { create(:twitter_channel, custom_options: { sync: { track_retweets: true } }) }
 
-        it 'adds tickets based on .options[:sync][:search] parameters' do
-          expect { channel.fetch(true) }
-            .to change(Ticket, :count).by(21)
-
-          expect(Ticket.last.attributes).to include(
-            'title'       => 'RT @BarackObama: Kobe was a legend on the court and just getting started in what...',
-            'preferences' => { 'channel_id'          => channel.id,
-                               'channel_screen_name' => channel.options[:user][:screen_name] },
-            'customer_id' => User.find_by(firstname: 'Zammad', lastname: 'Ali').id
-          )
+          it 'creates an article for each recent tweet/retweet' do
+            expect { channel.fetch }
+              .to change { Ticket.where('title LIKE ?', 'RT @%').count }
+              .and change(Ticket, :count).by(21)
           end
         end
 
-      context 'and legacy "import_older_tweets" option' do
+        context 'and "import_older_tweets" option (legacy)' do
           subject(:channel) { create(:twitter_channel, :legacy) }
 
-        it 'adds tickets based on .options[:sync][:search] parameters' do
-          expect { channel.fetch(true) }
-            .to change(Ticket, :count).by(21)
+          it 'creates an article for each tweet' do
+            expect { channel.fetch }
+              .to change { Ticket.where('title LIKE ?', 'GitHub Trending Archive, 2_ Nov 2018, Ruby. %').count }
+              .and change(Ticket, :count).by(21)
+          end
+        end
 
-          expect(Ticket.last.attributes).to include(
-            'title'       => 'Wir haben unsere DMs deaktiviert. ' \
-                             'Leider können wir dank der neuen Twitter API k...',
-            'preferences' => { 'channel_id'          => channel.id,
-                               'channel_screen_name' => channel.options[:user][:screen_name] },
-            'customer_id' => User.find_by(firstname: 'Ccc', lastname: 'Event Logistics').id
-          )
+        describe 'Race condition: when #fetch finds a half-processed, outgoing tweet' do
+          subject!(:channel) { create(:twitter_channel, custom_options: custom_options) }
+
+          let(:custom_options) do
+            {
+              user: {
+                # Must match outgoing tweet author Twitter user ID
+                id: '1205290247124217856',
+              },
+              sync: {
+                search: [
+                  {
+                    term:     'zammadzammadzammad',
+                    group_id: Group.first.id
+                  }
+                ]
+              }
+            }
+          end
+
+          let!(:tweet) { create(:twitter_article, body: 'zammadzammadzammad') }
+
+          context '(i.e., after the BG job has posted the article to Twitter…' do
+            # NOTE: This context block cannot be set up programmatically.
+            # Instead, the tweet was posted, fetched, recorded into a VCR cassette,
+            # and then manually copied into the existing VCR cassette for this example.
+
+            context '…but before the BG job has "synced" article.message_id with tweet.id)' do
+              let(:twitter_job) { Delayed::Job.find_by(handler: <<~YML) }
+                --- !ruby/object:Observer::Ticket::Article::CommunicateTwitter::BackgroundJob
+                article_id: #{tweet.id}
+              YML
+
+              around do |example|
+                # This test case requires the use_vcr: :time_sensitive option
+                # to travel_to(when the VCR cassette was recorded).
+                #
+                # This ensures that #fetch doesn't ignore
+                # the "older" tweets stored in the VCR cassette,
+                # but it also freezes time,
+                # which breaks this race condition handling logic:
+                #
+                #     break if Delayed::Job.where('created_at < ?', Time.current).none?
+                #
+                # So, we unfreeze time here.
+                travel_back
+
+                # Run BG job (Why not use Scheduler.worker?
+                # It led to hangs & failures elsewhere in test suite.)
+                Thread.new do
+                  sleep 5  # simulate other bg jobs holding up the queue
+                  twitter_job.invoke_job
+                end.tap { example.run }.join
+              end
+
+              it 'does not import the duplicate tweet (waits up to 60s for BG job to finish)' do
+                expect { channel.fetch }
+                  .to not_change(Ticket::Article, :count)
+              end
+            end
+          end
         end
       end
     end

test/data/vcr_cassettes/models/channel/driver/twitter/_but_before_the_bg_job_has_synced_article_message_id_with_tweet_id_does_not_import_the_duplicate_tweet_waits_up_to_60s_for_bg_job_to_finish_.yml

@@ -0,0 +1,245 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://api.twitter.com/1.1/statuses/update.json
+    body:
+      encoding: UTF-8
+      string: in_reply_to_status_id&status=zammadzammadzammad
+    headers:
+      User-Agent:
+      - TwitterRubyGem/6.2.0
+      Authorization:
+      - OAuth oauth_consumer_key="REDACTED", oauth_nonce="bd6dd2182d97c3db65b31354e57e4898",
+        oauth_signature="CB1L6ABVnKvvX6MYA%2FgAwEO2bJk%3D", oauth_signature_method="HMAC-SHA1",
+        oauth_timestamp="1583840885", oauth_token="REDACTED",
+        oauth_version="1.0"
+      Connection:
+      - close
+      Content-Type:
+      - application/x-www-form-urlencoded
+      Host:
+      - api.twitter.com
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Cache-Control:
+      - no-cache, no-store, must-revalidate, pre-check=0, post-check=0
+      Connection:
+      - close
+      Content-Disposition:
+      - attachment; filename=json.json
+      Content-Length:
+      - '1875'
+      Content-Type:
+      - application/json;charset=utf-8
+      Date:
+      - Tue, 10 Mar 2020 11:48:05 GMT
+      Expires:
+      - Tue, 31 Mar 1981 05:00:00 GMT
+      Last-Modified:
+      - Tue, 10 Mar 2020 11:48:05 GMT
+      Pragma:
+      - no-cache
+      Server:
+      - tsa_m
+      Set-Cookie:
+      - guest_id=v1%3A158384088580519536; Max-Age=63072000; Expires=Thu, 10 Mar 2022
+        11:48:05 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None
+      - lang=en; Path=/
+      - personalization_id="v1_C0O4/lMYKslqdCLoRaa92g=="; Max-Age=63072000; Expires=Thu,
+        10 Mar 2022 11:48:05 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None
+      Status:
+      - 200 OK
+      Strict-Transport-Security:
+      - max-age=631138519
+      X-Access-Level:
+      - read-write-directmessages
+      X-Connection-Hash:
+      - 17fe1248d1e3052bdc237af4f58d515a
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Response-Time:
+      - '164'
+      X-Transaction:
+      - 00a2bc2500997791
+      X-Tsa-Request-Body-Time:
+      - '0'
+      X-Twitter-Response-Tags:
+      - BouncerCompliant
+      X-Xss-Protection:
+      - '0'
+    body:
+      encoding: UTF-8
+      string: '{"created_at":"Tue Mar 10 11:48:05 +0000 2020","id":1237344473199153152,"id_str":"1237344473199153152","text":"zammadzammadzammad","truncated":false,"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[]},"source":"\u003ca
+        href=\"https:\/\/zammad.com\/\" rel=\"nofollow\"\u003ezammad\u003c\/a\u003e","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1205290247124217856,"id_str":"1205290247124217856","name":"pennbrooke","screen_name":"pennbrooke1","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":0,"friends_count":1,"listed_count":0,"created_at":"Fri
+        Dec 13 00:56:10 +0000 2019","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":19,"lang":null,"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":0,"favorite_count":0,"favorited":false,"retweeted":false,"lang":"lv"}'
+    http_version: 
+  recorded_at: Tue, 10 Mar 2020 11:48:06 GMT
+- request:
+    method: get
+    uri: https://api.twitter.com/1.1/search/tweets.json?count=100&q=zammadzammadzammad&result_type=mixed
+    body:
+      encoding: UTF-8
+      string: ''
+    headers:
+      User-Agent:
+      - TwitterRubyGem/6.2.0
+      Authorization:
+      - OAuth oauth_consumer_key="REDACTED", oauth_nonce="37876a49fa0c474bd7732dea70083056",
+        oauth_signature="f159trPoyOipHcp%2BPlL33Kh2nO4%3D", oauth_signature_method="HMAC-SHA1",
+        oauth_timestamp="1583840887", oauth_token="REDACTED",
+        oauth_version="1.0"
+      Connection:
+      - close
+      Host:
+      - api.twitter.com
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Cache-Control:
+      - no-cache, no-store, must-revalidate, pre-check=0, post-check=0
+      Connection:
+      - close
+      Content-Disposition:
+      - attachment; filename=json.json
+      Content-Length:
+      - '2346'
+      Content-Type:
+      - application/json;charset=utf-8
+      Date:
+      - Tue, 10 Mar 2020 11:48:07 GMT
+      Expires:
+      - Tue, 31 Mar 1981 05:00:00 GMT
+      Last-Modified:
+      - Tue, 10 Mar 2020 11:48:07 GMT
+      Pragma:
+      - no-cache
+      Server:
+      - tsa_m
+      Set-Cookie:
+      - guest_id=v1%3A158384088754038008; Max-Age=63072000; Expires=Thu, 10 Mar 2022
+        11:48:07 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None
+      - lang=en; Path=/
+      - personalization_id="v1_N5oIKIuBSmkhNAVvuM7dWQ=="; Max-Age=63072000; Expires=Thu,
+        10 Mar 2022 11:48:07 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None
+      Status:
+      - 200 OK
+      Strict-Transport-Security:
+      - max-age=631138519
+      X-Access-Level:
+      - read-write-directmessages
+      X-Connection-Hash:
+      - a615db05b45fdf48368de9e967c599c2
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Rate-Limit-Limit:
+      - '180'
+      X-Rate-Limit-Remaining:
+      - '177'
+      X-Rate-Limit-Reset:
+      - '1583841131'
+      X-Response-Time:
+      - '134'
+      X-Transaction:
+      - '004926c000957504'
+      X-Twitter-Response-Tags:
+      - BouncerCompliant
+      X-Xss-Protection:
+      - '0'
+    body:
+      encoding: UTF-8
+      string: '{"statuses":[{"created_at":"Tue Mar 10 11:48:05 +0000 2020","id":1237344473199153152,"id_str":"1237344473199153152","text":"zammadzammadzammad","truncated":false,"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[]},"metadata":{"iso_language_code":"lv","result_type":"recent"},"source":"\u003ca
+        href=\"https:\/\/zammad.com\/\" rel=\"nofollow\"\u003ezammad\u003c\/a\u003e","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1205290247124217856,"id_str":"1205290247124217856","name":"pennbrooke","screen_name":"pennbrooke1","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":0,"friends_count":1,"listed_count":0,"created_at":"Fri
+        Dec 13 00:56:10 +0000 2019","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":19,"lang":null,"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":0,"favorite_count":0,"favorited":false,"retweeted":false,"lang":"lv"}],"search_metadata":{"completed_in":0.017,"max_id":1237344473199153152,"max_id_str":"1237344473199153152","next_results":"?max_id=1237344473199153151&q=zammadzammadzammad&count=100&include_entities=1&result_type=mixed","query":"zammadzammadzammad","refresh_url":"?since_id=1237344473199153152&q=zammadzammadzammad&result_type=mixed&include_entities=1","count":100,"since_id":0,"since_id_str":"0"}}'
+    http_version: 
+  recorded_at: Tue, 10 Mar 2020 11:48:07 GMT
+- request:
+    method: get
+    uri: https://api.twitter.com/1.1/search/tweets.json?count=100&include_entities=1&max_id=1237344473199153151&q=zammadzammadzammad&result_type=mixed
+    body:
+      encoding: UTF-8
+      string: ''
+    headers:
+      User-Agent:
+      - TwitterRubyGem/6.2.0
+      Authorization:
+      - OAuth oauth_consumer_key="REDACTED", oauth_nonce="75815ceef5c89fbf52da222e32a10c88",
+        oauth_signature="BVaaa8THWe1R6DZEOr%2F8FAFm2v8%3D", oauth_signature_method="HMAC-SHA1",
+        oauth_timestamp="1583840902", oauth_token="REDACTED",
+        oauth_version="1.0"
+      Connection:
+      - close
+      Host:
+      - api.twitter.com
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Cache-Control:
+      - no-cache, no-store, must-revalidate, pre-check=0, post-check=0
+      Connection:
+      - close
+      Content-Disposition:
+      - attachment; filename=json.json
+      Content-Length:
+      - '297'
+      Content-Type:
+      - application/json;charset=utf-8
+      Date:
+      - Tue, 10 Mar 2020 11:48:23 GMT
+      Expires:
+      - Tue, 31 Mar 1981 05:00:00 GMT
+      Last-Modified:
+      - Tue, 10 Mar 2020 11:48:23 GMT
+      Pragma:
+      - no-cache
+      Server:
+      - tsa_m
+      Set-Cookie:
+      - guest_id=v1%3A158384090303371642; Max-Age=63072000; Expires=Thu, 10 Mar 2022
+        11:48:23 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None
+      - lang=en; Path=/
+      - personalization_id="v1_j9baN1VbzoK0Aak9kVoDHQ=="; Max-Age=63072000; Expires=Thu,
+        10 Mar 2022 11:48:23 GMT; Path=/; Domain=.twitter.com; Secure; SameSite=None
+      Status:
+      - 200 OK
+      Strict-Transport-Security:
+      - max-age=631138519
+      X-Access-Level:
+      - read-write-directmessages
+      X-Connection-Hash:
+      - b2aecb9288e6367f9f95d89e76d342bc
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-Rate-Limit-Limit:
+      - '180'
+      X-Rate-Limit-Remaining:
+      - '176'
+      X-Rate-Limit-Reset:
+      - '1583841131'
+      X-Response-Time:
+      - '123'
+      X-Transaction:
+      - '0087d22d00c9bd19'
+      X-Twitter-Response-Tags:
+      - BouncerCompliant
+      X-Xss-Protection:
+      - '0'
+    body:
+      encoding: UTF-8
+      string: '{"statuses":[],"search_metadata":{"completed_in":0.012,"max_id":1237344473199153151,"max_id_str":"1237344473199153151","query":"zammadzammadzammad","refresh_url":"?since_id=1237344473199153151&q=zammadzammadzammad&result_type=mixed&include_entities=1","count":100,"since_id":0,"since_id_str":"0"}}'
+    http_version: 
+  recorded_at: Tue, 10 Mar 2020 11:48:23 GMT
+recorded_with: VCR 4.0.0