Fixed issue#251 - Page posts from facebook and tweets from twitter as tickets.

- new tickets from own posts / tweets will be created as closed now
- on new connected accounts only 20 old messages are imported (to prevent to import really old -initial- stuff)

app/models/channel/driver/facebook.rb

@@ -76,6 +76,9 @@ class Channel::Driver::Facebook
     return if !@sync
     return if !@sync['pages']
 
+    older_import = 0
+    older_import_max = 12
+
     @sync['pages'].each { |page_to_sync_id, page_to_sync_params|
       page = get_page(page_to_sync_id)
       next if !page
@@ -85,6 +88,14 @@ class Channel::Driver::Facebook
 
       posts = page_client.client.get_connection('me', 'feed', fields: 'id,from,to,message,created_time,comments')
       posts.each { |post|
+
+        # ignore older messages
+        if (@channel.created_at - 15.days) > Time.zone.parse(post['created_time']) || older_import >= older_import_max
+          older_import += 1
+          Rails.logger.debug "post to old: #{post['id']}/#{post['created_time']}"
+          next
+        end
+
         page_client.to_group(post, page_to_sync_params['group_id'], @channel, page)
       }
     }

app/models/channel/driver/twitter.rb

@@ -45,7 +45,7 @@ returns
 
 =end
 
-  def fetch (options, channel)
+  def fetch(options, channel)
 
     options = check_external_credential(options)
 
@@ -174,7 +174,7 @@ returns
 
 stream tweets from twitter account
 
-  stream_instance.stream
+  instance.stream
 
 returns
 
@@ -278,7 +278,16 @@ returns
       next if search[:group_id].to_s.empty?
       result_type = search[:type] || 'mixed'
       Rails.logger.debug " - searching for '#{search[:term]}'"
+      older_import = 0
+      older_import_max = 20
       @rest_client.client.search(search[:term], result_type: result_type).collect { |tweet|
+
+        # ignore older messages
+        if (@channel.created_at - 15.days) > tweet.created_at || older_import >= older_import_max
+          older_import += 1
+          Rails.logger.debug "tweet to old: #{tweet.id}/#{tweet.created_at}"
+          next
+        end
         next if Ticket::Article.find_by(message_id: tweet.id)
         break if @rest_client.tweet_limit_reached(tweet)
         @rest_client.to_group(tweet, search[:group_id], @channel)
@@ -292,7 +301,16 @@ returns
     return if !@sync[:mentions][:group_id]
     return if @sync[:mentions][:group_id].to_s.empty?
     Rails.logger.debug ' - searching for mentions'
+    older_import = 0
+    older_import_max = 20
     @rest_client.client.mentions_timeline.each { |tweet|
+
+      # ignore older messages
+      if (@channel.created_at - 15.days) > tweet.created_at || older_import >= older_import_max
+        older_import += 1
+        Rails.logger.debug "tweet to old: #{tweet.id}/#{tweet.created_at}"
+        next
+      end
       next if Ticket::Article.find_by(message_id: tweet.id)
       break if @rest_client.tweet_limit_reached(tweet)
       @rest_client.to_group(tweet, @sync[:mentions][:group_id], @channel)
@@ -305,7 +323,16 @@ returns
     return if !@sync[:direct_messages][:group_id]
     return if @sync[:direct_messages][:group_id].to_s.empty?
     Rails.logger.debug ' - searching for direct_messages'
+    older_import = 0
+    older_import_max = 20
     @rest_client.client.direct_messages.each { |tweet|
+
+      # ignore older messages
+      if (@channel.created_at - 15.days) > tweet.created_at || older_import >= older_import_max
+        older_import += 1
+        Rails.logger.debug "tweet to old: #{tweet.id}/#{tweet.created_at}"
+        next
+      end
       next if Ticket::Article.find_by(message_id: tweet.id)
       break if @rest_client.direct_message_limit_reached(tweet)
       @rest_client.to_group(tweet, @sync[:direct_messages][:group_id], @channel)

app/models/observer/ticket/article/communicate_twitter/background_job.rb

@@ -42,21 +42,41 @@ class Observer::Ticket::Article::CommunicateTwitter::BackgroundJob
       article.from = "@#{tweet.sender.screen_name}"
       article.to = "@#{tweet.recipient.screen_name}"
 
+      article.preferences['twitter'] = {
+        created_at: tweet.created_at,
+        recipient_id: tweet.recipient.id,
+        recipient_screen_name: tweet.recipient.screen_name,
+        sender_id: tweet.sender.id,
+        sender_screen_name: tweet.sender.screen_name,
+      }
+
     # regular tweet
     elsif tweet.class == Twitter::Tweet
       article.from = "@#{tweet.user.screen_name}"
       if tweet.user_mentions
         to = ''
-        twitter_mention_ids = []
+        mention_ids = []
         tweet.user_mentions.each { |user|
           if to != ''
             to += ' '
           end
           to += "@#{user.screen_name}"
-          twitter_mention_ids.push user.id
+          mention_ids.push user.id
         }
         article.to = to
-        article.preferences[:twitter_mention_ids] = twitter_mention_ids
+        article.preferences['twitter'] = {
+          mention_ids: mention_ids,
+          geo: tweet.geo,
+          retweeted: tweet.retweeted?,
+          possibly_sensitive: tweet.possibly_sensitive?,
+          in_reply_to_user_id: tweet.in_reply_to_user_id,
+          place: tweet.place,
+          retweet_count: tweet.retweet_count,
+          source: tweet.source,
+          favorited: tweet.favorited?,
+          truncated: tweet.truncated?,
+          created_at: tweet.created_at,
+        }
       end
     else
       raise "Unknown tweet type '#{tweet.class}'"
@@ -68,6 +88,14 @@ class Observer::Ticket::Article::CommunicateTwitter::BackgroundJob
     article.preferences['delivery_status_date'] = Time.zone.now
 
     article.message_id = tweet.id.to_s
+    article.preferences['links'] = [
+      {
+        url: "https://twitter.com/statuses/#{tweet.id}",
+        target: '_blank',
+        name: 'on Twitter',
+      },
+    ]
+
     article.save!
 
     Rails.logger.info "Send twitter (#{tweet.class}) to: '#{article.to}' (from #{article.from})"

lib/facebook.rb

@@ -267,7 +267,7 @@ result
       ticket_state = get_state(page, post, ticket)
       if ticket_state.name != ticket.state.name
         ticket.state = ticket_state
-        ticket.save
+        ticket.save!
       end
 
       article = {

lib/tweet_base.rb

@@ -126,11 +126,13 @@ class TweetBase
       title = "#{title[0, 80]}..."
     end
 
+    state = get_state(channel, tweet)
+
     Ticket.create(
       customer_id: user.id,
       title:       title,
       group_id:    group_id,
-      state:       Ticket::State.find_by(name: 'new'),
+      state:       state,
       priority:    Ticket::Priority.find_by(name: '2 normal'),
       preferences: {
         channel_id: channel.id,
@@ -139,7 +141,7 @@ class TweetBase
     )
   end
 
-  def to_article(tweet, user, ticket)
+  def to_article(tweet, user, ticket, channel)
 
     Rails.logger.debug 'Create article from tweet...'
     Rails.logger.debug tweet.inspect
@@ -151,13 +153,22 @@ class TweetBase
     from = nil
     article_type = nil
     in_reply_to = nil
+    preferences = {}
     if tweet.class == Twitter::DirectMessage
       article_type = 'twitter direct-message'
       to = "@#{tweet.recipient.screen_name}"
       from = "@#{tweet.sender.screen_name}"
+      preferences = {
+        created_at: tweet.created_at,
+        recipient_id: tweet.recipient.id,
+        recipient_screen_name: tweet.recipient.screen_name,
+        sender_id: tweet.sender.id,
+        sender_screen_name: tweet.sender.screen_name,
+      }
     elsif tweet.class == Twitter::Tweet
       article_type = 'twitter status'
       from = "@#{tweet.user.screen_name}"
+      mention_ids = []
       if tweet.user_mentions
         tweet.user_mentions.each { |local_user|
           if !to
@@ -166,9 +177,24 @@ class TweetBase
             to += ', '
           end
           to += "@#{local_user.screen_name}"
+          mention_ids.push local_user.id
         }
       end
       in_reply_to = tweet.in_reply_to_status_id
+
+      preferences = {
+        mention_ids: mention_ids,
+        geo: tweet.geo,
+        retweeted: tweet.retweeted?,
+        possibly_sensitive: tweet.possibly_sensitive?,
+        in_reply_to_user_id: tweet.in_reply_to_user_id,
+        place: tweet.place,
+        retweet_count: tweet.retweet_count,
+        source: tweet.source,
+        favorited: tweet.favorited?,
+        truncated: tweet.truncated?,
+      }
+
     else
       raise "Unknown tweet type '#{tweet.class}'"
     end
@@ -176,9 +202,10 @@ class TweetBase
     UserInfo.current_user_id = user.id
 
     # set ticket state to open if not new
-    if ticket.state.name != 'new'
+    ticket_state = get_state(channel, tweet, ticket)
-      ticket.state = Ticket::State.find_by(name: 'open')
+    if ticket_state.name != ticket.state.name
-      ticket.save
+      ticket.state = ticket_state
+      ticket.save!
     end
 
     Ticket::Article.create(
@@ -191,6 +218,16 @@ class TweetBase
       type_id:     Ticket::Article::Type.find_by(name: article_type).id,
       sender_id:   Ticket::Article::Sender.find_by(name: 'Customer').id,
       internal:    false,
+      preferences: {
+        twitter: preferences,
+        links: [
+          {
+            url: "https://twitter.com/statuses/#{tweet.id}",
+            target: '_blank',
+            name: 'on Twitter',
+          },
+        ],
+      }
     )
   end
 
@@ -227,7 +264,7 @@ class TweetBase
       user = to_user(tweet)
       if tweet.class == Twitter::DirectMessage
         ticket = to_ticket(tweet, user, group_id, channel)
-        to_article(tweet, user, ticket)
+        to_article(tweet, user, ticket, channel)
       elsif tweet.class == Twitter::Tweet
         if tweet.in_reply_to_status_id && tweet.in_reply_to_status_id.to_s != ''
           existing_article = Ticket::Article.find_by(message_id: tweet.in_reply_to_status_id)
@@ -246,7 +283,7 @@ class TweetBase
         if !ticket
           ticket = to_ticket(tweet, user, group_id, channel)
         end
-        to_article(tweet, user, ticket)
+        to_article(tweet, user, ticket, channel)
       else
         raise "Unknown tweet type '#{tweet.class}'"
       end
@@ -288,15 +325,34 @@ class TweetBase
     tweet
   end
 
+  def get_state(channel, tweet, ticket = nil)
+
+    tweet_user = user(tweet)
+
+    # no changes in post is from page user it self
+    if channel.options[:user][:id].to_s == tweet_user.id.to_s
+      if !ticket
+        return Ticket::State.find_by(name: 'closed') if !ticket
+      end
+      return ticket.state
+    end
+
+    state = Ticket::State.find_by(name: 'new')
+    return state if !ticket
+    return ticket.state if ticket.state.name == 'new'
+    Ticket::State.find_by(name: 'open')
+  end
+
   def tweet_limit_reached(tweet)
-    max_count = 60
+    max_count = 120
     if @connection_type == 'stream'
-      max_count = 15
+      max_count = 30
     end
     type_id = Ticket::Article::Type.lookup(name: 'twitter status').id
     created_at = Time.zone.now - 15.minutes
-    if Ticket::Article.where('created_at > ? AND type_id = ?', created_at, type_id).count > max_count
+    created_count = Ticket::Article.where('created_at > ? AND type_id = ?', created_at, type_id).count
-      Rails.logger.info "Tweet limit reached, ignored tweed id (#{tweet.id})"
+    if created_count > max_count
+      Rails.logger.info "Tweet limit of #{created_count}/#{max_count} reached, ignored tweed id (#{tweet.id})"
       return true
     end
     false
@@ -309,8 +365,9 @@ class TweetBase
     end
     type_id = Ticket::Article::Type.lookup(name: 'twitter direct-message').id
     created_at = Time.zone.now - 15.minutes
-    if Ticket::Article.where('created_at > ? AND type_id = ?', created_at, type_id).count > max_count
+    created_count = Ticket::Article.where('created_at > ? AND type_id = ?', created_at, type_id).count
-      Rails.logger.info "Tweet direct message limit reached, ignored tweed id (#{tweet.id})"
+    if created_count > max_count
+      Rails.logger.info "Tweet direct message limit reached #{created_count}/#{max_count}, ignored tweed id (#{tweet.id})"
       return true
     end
     false

test/integration/twitter_test.rb

@@ -42,6 +42,8 @@ class TwitterTest < ActiveSupport::TestCase
   system_login_without_at = system_login[1, system_login.length]
   system_token            = ENV['TWITTER_SYSTEM_TOKEN']
   system_token_secret     = ENV['TWITTER_SYSTEM_TOKEN_SECRET']
+  hash_tag1               = "#zarepl#{rand(999)}"
+  hash_tag2               = "#citheo#{rand(999)}"
 
   # me_bauer (is customer and is following armin_theo)
   if !ENV['TWITTER_CUSTOMER_LOGIN']
@@ -53,9 +55,10 @@ class TwitterTest < ActiveSupport::TestCase
   if !ENV['TWITTER_CUSTOMER_TOKEN_SECRET']
     raise "ERROR: Need CUSTOMER_TOKEN_SECRET - hint TWITTER_CUSTOMER_TOKEN_SECRET='1234'"
   end
-  customer_login        = ENV['TWITTER_CUSTOMER_LOGIN']
+  customer_login            = ENV['TWITTER_CUSTOMER_LOGIN']
-  customer_token        = ENV['TWITTER_CUSTOMER_TOKEN']
+  customer_login_without_at = customer_login[1, customer_login.length]
-  customer_token_secret = ENV['TWITTER_CUSTOMER_TOKEN_SECRET']
+  customer_token            = ENV['TWITTER_CUSTOMER_TOKEN']
+  customer_token_secret     = ENV['TWITTER_CUSTOMER_TOKEN_SECRET']
 
   # add channel
   current = Channel.where(area: 'Twitter::Account')
@@ -77,11 +80,11 @@ class TwitterTest < ActiveSupport::TestCase
       sync: {
         search: [
           {
-            term: '#citheo42',
+            term: hash_tag2,
             group_id: group.id,
           },
           {
-            term: '#zarepl24',
+            term: hash_tag1,
             group_id: 1,
           },
         ],
@@ -100,7 +103,7 @@ class TwitterTest < ActiveSupport::TestCase
 
   test 'a new outbound and reply' do
 
-    hash   = '#citheo42' + rand(999_999).to_s
+    hash   = "#{hash_tag2}#{rand(999_999)}"
     user   = User.find(2)
     text   = "Today the weather is really #{rand_word}... #{hash}"
     ticket = Ticket.create!(
@@ -133,6 +136,10 @@ class TwitterTest < ActiveSupport::TestCase
     assert_equal(system_login, article.from, 'ticket article from')
     assert_equal('', article.to, 'ticket article to')
 
+    ticket = Ticket.find(article.ticket_id)
+    ticket.state = Ticket::State.find_by(name: 'closed')
+    ticket.save!
+
     # reply by me_bauer
     client = Twitter::REST::Client.new do |config|
       config.consumer_key        = consumer_key
@@ -143,7 +150,6 @@ class TwitterTest < ActiveSupport::TestCase
 
     tweet_found = false
     client.user_timeline(system_login_without_at).each { |tweet|
-
       next if tweet.id.to_s != article.message_id.to_s
       tweet_found = true
       break
@@ -161,7 +167,7 @@ class TwitterTest < ActiveSupport::TestCase
     # fetch check system account
     sleep 10
     article = nil
-    1.times {
+    2.times {
       Channel.fetch
 
       # check if follow up article has been created
@@ -177,6 +183,8 @@ class TwitterTest < ActiveSupport::TestCase
     assert_equal(2, article.ticket.articles.count, 'ticket article inbound count')
     assert_equal(reply_text.utf8_to_3bytesutf8, ticket.articles.last.body, 'ticket article inbound body')
 
+    assert_equal('open', ticket.reload.state.name)
+
     channel = Channel.find(channel.id)
     assert_equal('', channel.last_log_out)
     assert_equal('ok', channel.status_out)
@@ -194,7 +202,7 @@ class TwitterTest < ActiveSupport::TestCase
       config.access_token_secret = customer_token_secret
     end
 
-    hash  = "#zarepl24 ##{hash_gen}"
+    hash  = "#{hash_tag1} ##{hash_gen}"
     text  = "Today #{rand_word}... #{hash}"
     tweet = client.update(
       text,
@@ -215,6 +223,7 @@ class TwitterTest < ActiveSupport::TestCase
     assert_equal(customer_login, article.from, 'ticket article from')
     assert_equal(nil, article.to, 'ticket article to')
     ticket = article.ticket
+    assert_equal('new', ticket.reload.state.name)
 
     # send reply
     reply_text = "#{customer_login} on my side #weather#{hash_gen}"
@@ -229,6 +238,7 @@ class TwitterTest < ActiveSupport::TestCase
     )
 
     Scheduler.worker(true)
+    assert_equal('open', ticket.reload.state.name)
 
     article = Ticket::Article.find(article.id)
     assert(article, "outbound article created, text: #{reply_text}")
@@ -249,6 +259,39 @@ class TwitterTest < ActiveSupport::TestCase
     assert_equal('ok', channel.status_out)
     assert_equal('', channel.last_log_in)
     assert_equal('ok', channel.status_in)
+
+    ticket = Ticket.find(article.ticket_id)
+    ticket.state = Ticket::State.find_by(name: 'closed')
+    ticket.save!
+
+    # reply with zammad user directly
+    client = Twitter::REST::Client.new do |config|
+      config.consumer_key        = consumer_key
+      config.consumer_secret     = consumer_secret
+      config.access_token        = system_token
+      config.access_token_secret = system_token_secret
+    end
+
+    hash  = "#{hash_tag1} ##{hash_gen}"
+    text  = "Today #{system_login} #{rand_word}... #{hash}"
+    tweet = client.update(
+      text,
+    )
+
+    # fetch check system account
+    sleep 20
+    article = nil
+    2.times {
+      Channel.fetch
+
+      # check if ticket and article has been created
+      article = Ticket::Article.find_by(message_id: tweet.id)
+      break if article
+      sleep 20
+    }
+
+    assert(article, "Can't find tweet id #{tweet.id}/#{text}")
+    assert_equal('closed', ticket.reload.state.name)
   end
 
   test 'c new by direct message inbound' do
@@ -406,7 +449,8 @@ class TwitterTest < ActiveSupport::TestCase
       config.access_token        = customer_token
       config.access_token_secret = customer_token_secret
     end
-    hash  = '#zarepl24 #' + hash_gen
+
+    hash  = "#{hash_tag1} ##{hash_gen}"
     text  = "Today... #{rand_word} #{hash}"
     tweet = client.update(
       text,
@@ -429,7 +473,7 @@ class TwitterTest < ActiveSupport::TestCase
       config.access_token        = customer_token
       config.access_token_secret = customer_token_secret
     end
-    hash  = '#zarepl24 #' + rand(999_999).to_s
+    hash  = "#{hash_tag1} ##{rand(999_999)}"
     text  = "Today... #{rand_word}  #{hash}"
     tweet = client.update(
       text,