From 691b63d0113c9ad1149b10bd99a7b8525117b224 Mon Sep 17 00:00:00 2001 From: Martin Edenhofer Date: Fri, 23 Jun 2017 09:37:54 +0200 Subject: [PATCH] Improved twitter length calculation. If streaming api is used, fetch parent tweet via REST api. Added initials support as article feature. --- .../ticket_zoom/article_new.coffee | 30 ++++-- .../javascripts/app/lib/app_post/utils.coffee | 87 +++++++++++------- lib/tweet_base.rb | 10 +- public/assets/tests/html_utils.js | 91 ++++++++++++++++--- 4 files changed, 161 insertions(+), 57 deletions(-) diff --git a/app/assets/javascripts/app/controllers/ticket_zoom/article_new.coffee b/app/assets/javascripts/app/controllers/ticket_zoom/article_new.coffee index 0782d2f3f..c42c5ba7a 100644 --- a/app/assets/javascripts/app/controllers/ticket_zoom/article_new.coffee +++ b/app/assets/javascripts/app/controllers/ticket_zoom/article_new.coffee @@ -85,7 +85,7 @@ class App.TicketZoomArticleNew extends App.Controller icon: 'twitter' attributes: [] internal: false, - features: ['body:limit'] + features: ['body:limit', 'body:initials'] maxTextLength: 140 warningTextLength: 30 } @@ -95,7 +95,7 @@ class App.TicketZoomArticleNew extends App.Controller icon: 'twitter' attributes: ['to'] internal: false, - features: ['body:limit'] + features: ['body:limit', 'body:initials'] maxTextLength: 10000 warningTextLength: 500 } @@ -229,7 +229,7 @@ class App.TicketZoomArticleNew extends App.Controller ) configure_attributes = [ - { name: 'customer_id', display: 'Recipients', tag: 'user_autocompletion', null: false, placeholder: 'Enter Person or Organization/Company', minLengt: 2, disableCreateUser: false }, + { name: 'customer_id', display: 'Recipients', tag: 'user_autocompletion', null: false, placeholder: 'Enter Person or Organization/Company', minLengt: 2, disableCreateObject: false }, ] controller = new App.ControllerForm( @@ -335,12 +335,12 @@ class App.TicketZoomArticleNew extends App.Controller if params.type is 'twitter status' App.Utils.htmlRemoveRichtext(@$('[data-name=body]'), false) params.content_type = 'text/plain' - params.body = "#{App.Utils.html2text(params.body, true)}\n#{@signature.text()}" + params.body = App.Utils.html2text(params.body, true) if params.type is 'twitter direct-message' App.Utils.htmlRemoveRichtext(@$('[data-name=body]'), false) params.content_type = 'text/plain' - params.body = "#{App.Utils.html2text(params.body, true)}\n#{@signature.text()}" + params.body = App.Utils.html2text(params.body, true) if params.type is 'facebook feed comment' App.Utils.htmlRemoveRichtext(@$('[data-name=body]'), false) @@ -352,6 +352,16 @@ class App.TicketZoomArticleNew extends App.Controller params.content_type = 'text/plain' params.body = App.Utils.html2text(params.body, true) + # add initals? + for articleType in @articleTypes + if articleType.name is @type + if _.contains(articleType.features, 'body:initials') + if params.content_type is 'text/html' + params.body = "#{params.body}
#{@signature.text()}" + else + params.body = "#{params.body}\n#{@signature.text()}" + break + params validate: => @@ -411,11 +421,11 @@ class App.TicketZoomArticleNew extends App.Controller return false if params.type is 'twitter status' - textLength = @maxTextLength - params.body.length + textLength = @maxTextLength - App.Utils.textLengthWithUrl(params.body) return false if textLength < 0 if params.type is 'twitter direct-message' - textLength = @maxTextLength - params.body.length + textLength = @maxTextLength - App.Utils.textLengthWithUrl(params.body) return false if textLength < 0 true @@ -534,11 +544,12 @@ class App.TicketZoomArticleNew extends App.Controller for name in articleType.features if name is 'attachment' @$('.article-attachment, .attachments').removeClass('hide') + if name is 'body:initials' + @updateInitials() if name is 'body:limit' @maxTextLength = articleType.maxTextLength @warningTextLength = articleType.warningTextLength @delay(@updateLetterCount, 600) - @updateInitials() @$('.js-textSizeLimit').removeClass('hide') @scrollToBottom() if wasScrolledToBottom @@ -557,7 +568,8 @@ class App.TicketZoomArticleNew extends App.Controller return if !@maxTextLength return if !@warningTextLength params = @params() - textLength = @maxTextLength - params.body.length + textLength = App.Utils.textLengthWithUrl(params.body) + textLength = @maxTextLength - textLength className = switch when textLength < 0 then 'label-danger' when textLength < @warningTextLength then 'label-warning' diff --git a/app/assets/javascripts/app/lib/app_post/utils.coffee b/app/assets/javascripts/app/lib/app_post/utils.coffee index 9e1265548..5b71dd5a2 100644 --- a/app/assets/javascripts/app/lib/app_post/utils.coffee +++ b/app/assets/javascripts/app/lib/app_post/utils.coffee @@ -1,6 +1,6 @@ # coffeelint: disable=no_unnecessary_double_quotes class App.Utils - @mapAttributes: + @mapTagAttributes: 'TABLE': ['align', 'bgcolor', 'border', 'cellpadding', 'cellspacing', 'frame', 'rules', 'sortable', 'summary', 'width', 'style'] 'TD': ['abbr', 'align', 'axis', 'colspan', 'headers', 'rowspan', 'valign', 'width', 'style'] 'TH': ['abbr', 'align', 'axis', 'colspan', 'headers', 'rowspan', 'scope', 'sorted', 'valign', 'width', 'style'] @@ -232,12 +232,12 @@ class App.Utils @_removeWordMarkup(html) # remove tags, keep content - html.find('div, span, p, li, ul, ol, a, b, u, i, label, small, strong, strike, pre, code, center, blockquote, form, fieldset, textarea, font, address, table, thead, tbody, tr, td, h1, h2, h3, h4, h5, h6').replaceWith( -> + html.find('div, span, p, li, ul, ol, a, b, u, i, label, small, strong, strike, pre, code, center, blockquote, form, fieldset, textarea, font, address, table, thead, tbody, tr, th, td, h1, h2, h3, h4, h5, h6').replaceWith( -> $(@).contents() ) # remove tags & content - html.find('div, span, p, li, ul, ol, a, b, u, i, label, small, strong, strike, pre, code, center, blockquote, form, fieldset, textarea, font, table, thead, tbody, tr, td, h1, h2, h3, h4, h5, h6, br, hr, img, svg, input, select, button, style, applet, embed, noframes, canvas, script, frame, iframe, meta, link, title, head').remove() + html.find('div, span, p, li, ul, ol, a, b, u, i, label, small, strong, strike, pre, code, center, blockquote, form, fieldset, textarea, font, table, thead, tbody, tr, th, td, h1, h2, h3, h4, h5, h6, br, hr, img, svg, input, select, button, style, applet, embed, noframes, canvas, script, frame, iframe, meta, link, title, head').remove() html @@ -249,20 +249,19 @@ class App.Utils # remove comments @_removeComments(html) - # remove style and class - if parent - @_removeAttributes(html) - # remove work markup @_removeWordMarkup(html) # remove tags, keep content - html.find('li, ul, ol, a, b, u, i, label, small, strong, strike, pre, code, center, blockquote, form, fieldset, textarea, font, address, table, thead, tbody, tr, td, h1, h2, h3, h4, h5, h6').replaceWith( -> + html.find('li, ul, ol, a, b, u, i, label, small, strong, strike, pre, code, center, blockquote, form, fieldset, textarea, font, address, table, thead, tbody, tr, th, td, h1, h2, h3, h4, h5, h6').replaceWith( -> $(@).contents() ) # remove tags & content - html.find('li, ul, ol, a, b, u, i, label, small, strong, strike, pre, code, center, blockquote, form, fieldset, textarea, font, address, table, thead, tbody, tr, td, h1, h2, h3, h4, h5, h6, hr, img, svg, input, select, button, style, applet, embed, noframes, canvas, script, frame, iframe, meta, link, title, head').remove() + html.find('li, ul, ol, a, b, u, i, label, small, strong, strike, pre, code, center, blockquote, form, fieldset, textarea, font, address, table, thead, tbody, tr, th, td, h1, h2, h3, h4, h5, h6, hr, img, svg, input, select, button, style, applet, embed, noframes, canvas, script, frame, iframe, meta, link, title, head').remove() + + # remove style and class + @_removeAttributes(html, parent) html @@ -274,9 +273,6 @@ class App.Utils # remove comments @_removeComments(html) - # remove style and class - @_removeAttributes(html) - # remove work markup @_removeWordMarkup(html) @@ -307,6 +303,9 @@ class App.Utils # remove tags & content html.find('font, img, svg, input, select, button, style, applet, embed, noframes, canvas, script, frame, iframe, meta, link, title, head, fieldset').remove() + # remove style and class + @_cleanAttributes(html) + html @_checkTypeOf: (item) -> @@ -327,40 +326,59 @@ class App.Utils catch err return $("
#{item}
") - @_removeAttribute: (element) -> + @_cleanAttribute: (element) -> return if !element - if @mapAttributes[element.nodeName] + if @mapTagAttributes[element.nodeName] atts = element.attributes for att in atts - if att && att.name && !_.contains(@mapAttributes[element.nodeName], att.name) - element.removeAttributeNode(att) + if att && att.name && !_.contains(@mapTagAttributes[element.nodeName], att.name) + element.removeAttribute(att.name) else - $element = $(element) - $element.removeAttr('style') - $element.removeAttr('class') - $element.removeAttr('title') - $element.removeAttr('lang') - $element.removeAttr('type') - $element.removeAttr('id') - $element.removeAttr('wrap') - $element.removeAttrs(/data-/) + @_removeAttribute(element) if @mapCss[element.nodeName] - style = element.getAttributeNode('style') - if style && style.nodeValue && style.nodeValue.split + elementStyle = element.style + styleOld = '' + for prop in elementStyle + styleOld += "#{prop}:#{elementStyle[prop]};" + + if styleOld && styleOld.split styleNew = '' - for local_pear in style.nodeValue.split(';') + for local_pear in styleOld.split(';') prop = local_pear.split(':') if prop[0] && prop[0].trim key = prop[0].trim() if _.contains(@mapCss[element.nodeName], key) styleNew += "#{local_pear};" if styleNew isnt '' - style.nodeValue = styleNew - element.setAttributeNode(style) + element.setAttribute('style', styleNew) else - element.removeAttributeNode(style) + element.removeAttribute('style') + + @_cleanAttributes: (html, parent = true) -> + if parent + html.each((index, element) => @_cleanAttribute(element) ) + html.find('*').each((index, element) => @_cleanAttribute(element) ) + html + + @_removeAttribute: (element) -> + return if !element + $element = $(element) + for att in element.attributes + if att && att.name + element.removeAttribute(att.name) + #$element.removeAttr(att.name) + + $element.removeAttr('style') + .removeAttr('class') + .removeAttr('lang') + .removeAttr('type') + .removeAttr('align') + .removeAttr('id') + .removeAttr('wrap') + .removeAttr('title') + .removeAttrs(/data-/) @_removeAttributes: (html, parent = true) -> if parent @@ -877,3 +895,10 @@ class App.Utils result = newOrderMethod(a, b, applyOrder) return false if !result applyOrder + + @textLengthWithUrl: (text, url_max_length = 23) -> + length = 0 + return length if !text + placeholder = Array(url_max_length + 1).join('X') + text = text.replace(/http(s|):\/\/[-A-Za-z0-9+&@#\/%?=~_\|!:,.;]+[-A-Za-z0-9+&@#\/%=~_|]/img, placeholder) + text.length diff --git a/lib/tweet_base.rb b/lib/tweet_base.rb index 25cae7f01..4d236f0be 100644 --- a/lib/tweet_base.rb +++ b/lib/tweet_base.rb @@ -257,11 +257,13 @@ class TweetBase begin # in case of streaming mode, get parent tweet via REST client - if !@client && @auth - @client = TweetRest.new(@auth) + if @connection_type == 'stream' + client = TweetRest.new(@auth) + parent_tweet = client.status(tweet.in_reply_to_status_id) + else + parent_tweet = @client.status(tweet.in_reply_to_status_id) end - parent_tweet = @client.status(tweet.in_reply_to_status_id) - ticket = to_group(parent_tweet, group_id, channel) + ticket = to_group(parent_tweet, group_id, channel) rescue Twitter::Error::NotFound, Twitter::Error::Forbidden => e # just ignore if tweet has already gone Rails.logger.info "Can't import tweet (#{tweet.in_reply_to_status_id}), #{e.message}" diff --git a/public/assets/tests/html_utils.js b/public/assets/tests/html_utils.js index 6ae7b9b67..a9c70a1b2 100644 --- a/public/assets/tests/html_utils.js +++ b/public/assets/tests/html_utils.js @@ -341,6 +341,11 @@ test("htmlRemoveTags", function() { result = App.Utils.htmlRemoveTags($(source)) equal(result.html(), should, source) +}); + +// htmlRemoveRichtext +test("htmlRemoveRichtext", function() { + source = "
test 123
" //should = "
test 123
" should = "test 123" @@ -367,11 +372,6 @@ test("htmlRemoveTags", function() { result = App.Utils.htmlRemoveRichtext(source) equal(result.html(), should, source) -}); - -// htmlRemoveRichtext -test("htmlRemoveRichtext", function() { - var source = "
test
" //var should = "
test
" var should = "test" @@ -466,6 +466,25 @@ test("htmlRemoveRichtext", function() { result = App.Utils.htmlRemoveRichtext(source) equal(result.html(), should, source) + var source = "
test
" + var should = "
test
" + var result = App.Utils.htmlRemoveRichtext($(source)) + equal(result.get(0).outerHTML, should, source) + + source = "
some link to somewhere" + should = "
some link to somewhere
" + result = App.Utils.htmlRemoveRichtext($(source)) + equal(result.get(0).outerHTML, should, source) + + source = "

" + should = "

" + result = App.Utils.htmlRemoveRichtext($(source)) + equal(result.get(0).outerHTML, should, source) + + source = "
111aaa
keyvalue
" + should = "
111aaakeyvalue
" + result = App.Utils.htmlRemoveRichtext(source, true) + equal(result.get(0).outerHTML, should, source) }); // htmlCleanup @@ -573,12 +592,12 @@ test("htmlCleanup", function() { equal(result.html(), should, source) source = "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n

·            \nTest 1

\n\n

·            \nTest 2

\n\n

·            \nTest 3

\n\n

·            \nTest 4

\n\n

·            \nTest5

\n\n\n\n\n" - should = "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n

· \nTest 1

\n\n

· \nTest 2

\n\n

· \nTest 3

\n\n

· \nTest 4

\n\n

· \nTest5

\n\n\n\n\n" + should = "" result = App.Utils.htmlCleanup(source) - equal(result.html(), should, source) + equal(result.html().trim(), should, source) source = "\n\n\n \n \n \n \n\n\n

1.\nGehe auf https://www.pferdiathek.ge

\n


\n\n

\n

2.\nMelde Dich mit folgende Zugangsdaten an:

\n

Benutzer:\nme@xxx.net

\n

Passwort:\nxxx.

\n\n" - should = "\n\n\n \n \n \n \n\n\n

1.\nGehe auf https://www.pferdiathek.ge

\n


\n\n

\n

2.\nMelde Dich mit folgende Zugangsdaten an:

\n

Benutzer:\nme@xxx.net

\n

Passwort:\nxxx.

\n\n" + should = "\n\n\n \n \n \n \n\n\n

1.\nGehe auf https://www.pferdiathek.ge

\n


\n\n

\n

2.\nMelde Dich mit folgende Zugangsdaten an:

\n

Benutzer:\nme@xxx.net

\n

Passwort:\nxxx.

\n\n" result = App.Utils.htmlCleanup(source) equal(result.html(), should, source) @@ -588,16 +607,17 @@ test("htmlCleanup", function() { equal(result.get(0).outerHTML, should, source) source = "
aaa
value
" - should = "
aaa
value
" + should = "
aaa
value
" result = App.Utils.htmlCleanup(source) + result.get(0).outerHTML //equal(result.get(0).outerHTML, should, source) / string order is different on browsers equal(result.first().attr('bgcolor'), 'green') - equal(result.first().attr('style'), 'color: red;') + equal(result.first().attr('style'), 'color:red;') equal(result.first().attr('aaa'), undefined) - equal(result.find('tr').first().attr('style'), 'margin-top: 10px;') + equal(result.find('tr').first().attr('style'), 'margin-top:10px;') equal(result.find('th').first().attr('colspan'), '2') equal(result.find('th').first().attr('abc'), undefined) - equal(result.find('th').first().attr('style'), 'margin-top: 12px;') + equal(result.find('th').first().attr('style'), 'margin-top:12px;') source = "
aaa
value
" should = "
aaa
value
" @@ -1645,4 +1665,49 @@ test("check diffPosition format", function() { }); -} \ No newline at end of file +// check textLengthWithUrl format +test("check textLengthWithUrl format", function() { + + var string = '123' + var result = 3 + var verify = App.Utils.textLengthWithUrl(string) + equal(verify, result) + + string = '123 http is not here' + result = 20 + verify = App.Utils.textLengthWithUrl(string) + equal(verify, result) + + string = '123 http://host is not here' + result = 39 + verify = App.Utils.textLengthWithUrl(string) + equal(verify, result) + + string = '123 http://XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX is not here' + result = 39 + verify = App.Utils.textLengthWithUrl(string) + equal(verify, result) + + string = 'http://XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + result = 23 + verify = App.Utils.textLengthWithUrl(string) + equal(verify, result) + + string = 'http://XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX, some other text' + result = 23 + 17 + verify = App.Utils.textLengthWithUrl(string) + equal(verify, result) + + string = 'some other text,http://XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + result = 23 + 16 + verify = App.Utils.textLengthWithUrl(string) + equal(verify, result) + + string = 'some other text, http://XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX?abc=123;aaa=ab+c usw' + result = 23 + 21 + verify = App.Utils.textLengthWithUrl(string) + equal(verify, result) + +}); + +}