Improved signatureIdentify() and html2text().

2015-04-03 03:11:21 +02:00 · 2015-04-03 03:11:21 +02:00 · 132e3c0362
parent c15f204b87
commit 132e3c0362
2 changed files with 106 additions and 88 deletions
--- a/app/assets/javascripts/app/lib/app_post/utils.js.coffee
+++ b/app/assets/javascripts/app/lib/app_post/utils.js.coffee
@ -6,7 +6,7 @@ class App.Utils
      .replace(/(\r\n|\n\r)/g, "\n")  # cleanup
      .replace(/\r/g, "\n")           # cleanup
      .replace(/[ ]\n/g, "\n")        # remove tailing spaces
-      .replace(/\n{3,9}/g, "\n\n")    # remove multible empty lines
+      .replace(/\n{3,20}/g, "\n\n")    # remove multible empty lines

  # htmlEscapedAndLinkified = App.Utils.text2html( rawText )
  @text2html: ( ascii ) ->
@ -22,24 +22,19 @@ class App.Utils
    # remove not needed new lines
    html = html.replace(/>\n/g, '>')

-    # convert to jquery
-    html = $('<div>' + html + '</div>')
-
    # insert new lines
-    html.find('div, p, pre, code, center, blockquote, form, textarea, address, tr').replaceWith( ->
-      content = $(@).html() + "\n"
-      content
-        .replace(/<br>/g, "\n")
-        .replace(/<br\/>/g, "\n")
-    )
-
-    # replace <br> as string, is/was not possible throuh replaceWith
-    htmlTmp = html.html().replace(/<br>/g, "\n")
+    html = html
+      .replace(/<br(|.+?)>/g, "\n")
+      .replace(/<br\/>/g, "\n")
+      .replace(/<(div)(|.+?)>/g, "")
+      .replace(/<(p|blockquote|form|textarea|address|tr)(|.+?)>/g, "\n")
+      .replace(/<\/(div|p|blockquote|form|textarea|address|tr)>/g, "\n")

    # trim and cleanup
-    $('<div>' + htmlTmp + '</div>').text().trim()
+    $('<div>' + html + '</div>').text().trim()
      .replace(/(\r\n|\n\r)/g, "\n")  # cleanup
      .replace(/\r/g, "\n")           # cleanup
+      .replace(/\n{3,20}/g, "\n\n")   # remove multible empty lines

  # htmlEscapedAndLinkified = App.Utils.linkify( rawText )
  @linkify: (ascii) ->
@ -191,56 +186,77 @@ class App.Utils
    else
      true

-  # messageWithMarker = App.Utils.signatureIdentify( message, sender )
-  @signatureIdentify: (message, sender = false) ->
+  # messageWithMarker = App.Utils.signatureIdentify( message, false )
+  @signatureIdentify: (message, test = false) ->
    textToSearch = @html2text( message )

    # count lines, if we do have lower the 10, ignore this
    textToSearchInLines = textToSearch.split("\n")
-    return message if textToSearchInLines.length < 10
+    if !test
+      return message if textToSearchInLines.length < 10

    quote = (str) ->
      (str + '').replace(/[.?*+^$[\]\\(){}|-]/g, "\\$&")

    # search for signature seperator "--\n"
-    searchForSeperator = (textToSearchInLines) ->
+    markers = []
+    searchForSeperator = (textToSearchInLines, markers) ->
+      lineCount = 0
      for line in textToSearchInLines
+        lineCount += 1
        if line && line.match( /^\s{0,10}--\s{0,10}$/ )
-          return line
-      false
-    marker = searchForSeperator(textToSearchInLines)
+          marker =
+            line:      line
+            lineCount: lineCount
+            type:      'seperator'
+          markers.push marker
+    searchForSeperator(textToSearchInLines, markers)

    # search for Apple Mail
    # On 01/04/15 10:55, Bob Smith wrote:
-    searchForAppleMail = (textToSearchInLines) ->
+    searchForAppleMail = (textToSearchInLines, markers) ->
+      lineCount = 0
      for line in textToSearchInLines
+        lineCount += 1
        if line && line.match( /^(On|Am)\s.+?\s(wrote|schrieb):/ )
-          return line
-      false
-    if !marker
-      marker = searchForAppleMail(textToSearchInLines)
+          marker =
+            line:      line
+            lineCount: lineCount
+            type:      'apple'
+          markers.push marker
+    searchForAppleMail(textToSearchInLines, markers)

    # search for otrs
    # 25.02.2015 10:26 - edv hotline schrieb:
-    searchForOtrs = (textToSearchInLines) ->
+    searchForOtrs = (textToSearchInLines, markers) ->
+      lineCount = 0
      for line in textToSearchInLines
+        lineCount += 1
        if line && line.match( /^.+?\s.+?\s-\s.+?\s(wrote|schrieb):/ )
-          return line
-      false
-    if !marker
-      markerOtrs = searchForOtrs(textToSearchInLines)
+          marker =
+            line:      line
+            lineCount: lineCount
+            type:      'Otrs'
+          markers.push marker
+    searchForOtrs(textToSearchInLines, markers)

    # search for Ms
    # From: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]
    # Send: Donnerstag, 2. April 2015 10:00
-    searchForMs = (textToSearchInLines) ->
+    searchForMs = (textToSearchInLines, markers) ->
+      lineCount = 0
      fromFound = undefined
      for line in textToSearchInLines
+        lineCount += 1

        # find Sent
        if fromFound
          if line && line.match( /^(Sent|Gesendet):\s.+?/)
-            return fromFound
+            marker =
+              line:      fromFound
+              lineCount: lineCount
+              type:      'Ms'
+            markers.push marker
          else
            fromFound = undefined

@ -248,21 +264,21 @@ class App.Utils
        else
          if line && line.match( /^(From|Von):\s.+?/ )
            fromFound = line.replace(/\s{0,5}\[.+?\]/g, '')
-      false
-    if !marker && !markerOtrs
-      markerMs = searchForMs(textToSearchInLines)
+    searchForMs(textToSearchInLines, markers)

    # if no marker is found, return
-    return message if !marker && !markerMs && !markerOtrs
+    return message if !markers || !markers[0]

    # insert marker
    markerTemplate = '<span class="js-signatureMarker"></span>'
-    if marker
-      regex = new RegExp( "\>(\s{0,10}#{quote(marker)})\s{0,10}\<" )
+
+    # get first marker
+    markers = _.sortBy(markers, 'lineCount')
+    if markers[0].type is 'seperator'
+      regex = new RegExp( "\>(\s{0,10}#{quote(markers[0].line)})\s{0,10}\<" )
      message.replace( regex, ">#{markerTemplate}\$1<" )
    else
-      marker = markerMs || markerOtrs
-      regex = new RegExp( "\>(\s{0,10}#{quote(marker)})" )
+      regex = new RegExp( "\>(\s{0,10}#{quote(markers[0].line)})" )
      message.replace( regex, ">#{markerTemplate}\$1" )

  # textReplaced = App.Utils.replaceTags( template, { user: { firstname: 'Bob', lastname: 'Smith' } } )
--- a/public/assets/tests/html-utils.js
+++ b/public/assets/tests/html-utils.js
@ -70,7 +70,7 @@ test( "text2html", function() {
 test( "html2text", function() {

  var source = "<div>Some</div><div>Value</div><div><br></div><div>Test</div>"
-  var should = "Some\nValue\n\n\nTest"
+  var should = "Some\nValue\n\nTest"
  var result = App.Utils.html2text( source )
  equal( result, should, source )

@ -95,20 +95,24 @@ test( "html2text", function() {
  equal( result, should, source )

  source = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--<br/>Bob Smith</div>"
-  should = "test 123 \n\n\n\n\n\n\n\n\n\n\n--\nBob Smith"
+  should = "test 123 \n\n--\nBob Smith"
  result = App.Utils.html2text( source )
  equal( result, should, source )

  source = "test 123 <br><br><br><br><br><br><br><br><br><br><br>--<br>Bob Smith"
-  should = "test 123 \n\n\n\n\n\n\n\n\n\n\n--\nBob Smith"
+  should = "test 123 \n\n--\nBob Smith"
  result = App.Utils.html2text( source )
  equal( result, should, source )

  source = "<div>1<br><br><br><br><br><br><br><br><br><br></div><div>Von: Martin Edenhofer via Znuny Support [<a href=\"mailto:support@znuny.inc\" title=\"mailto:support@znuny.inc\" target=\"_blank\">mailto:support@znuny.inc</a>]</div>\n<div>Gesendet: Donnerstag, 2. April 2015 11:32</div>"
-  should = "1\n\n\n\n\n\n\n\n\n\n\n\Von: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]\nGesendet: Donnerstag, 2. April 2015 11:32"
+  should = "1\n\nVon: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]\nGesendet: Donnerstag, 2. April 2015 11:32"
  result = App.Utils.html2text( source )
  equal( result, should, source )

+  source = "<div>test 123<br/>lalala<p>--</p>some test</div>"
+  should = "test 123\nlalala\n--\nsome test"
+  result = App.Utils.html2text( source )
+  equal( result, should, source )
 });

 // linkify
@ -509,81 +513,79 @@ test( "identify signature", function() {
  result  = App.Utils.signatureIdentify( message )
  equal( result, should )

-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--<br/>Bob Smith</div>"
-  should  = '<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><span class="js-signatureMarker"></span>--<br/>Bob Smith</div>'
+  message = "<div>test 123 <br/>1<br/>2<br/>3<br/>4<br/>5<br/>6<br/>7<br/>8<br/>9<br/><br/>--<br/>Bob Smith</div>"
+  should  = '<div>test 123 <br/>1<br/>2<br/>3<br/>4<br/>5<br/>6<br/>7<br/>8<br/>9<br/><br/><span class="js-signatureMarker"></span>--<br/>Bob Smith</div>'
  result  = App.Utils.signatureIdentify( message )
  equal( result, should )

-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/>--<br/>Bob Smith</div>"
-  should  = '<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><span class="js-signatureMarker"></span>--<br/>Bob Smith</div>'
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>test 123 <br/><br/>--no not match--<br/>--<br/>Bob Smith</div>"
+  should  = '<div>test 123 <br/><br/>--no not match--<br/><span class="js-signatureMarker"></span>--<br/>Bob Smith</div>'
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/> -- <br/>Bob Smith</div>"
-  should  = '<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><span class="js-signatureMarker"></span> -- <br/>Bob Smith</div>'
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>test 123 <br/><br/>--no not match--<br/> -- <br/>Bob Smith</div>"
+  should  = '<div>test 123 <br/><br/>--no not match--<br/><span class="js-signatureMarker"></span> -- <br/>Bob Smith</div>'
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--<br/>Bob Smith<br/><br/><br/><br/><br/>--<br/>Bob Smith</div>"
-  should  = '<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><br/><span class="js-signatureMarker"></span>--<br/>Bob Smith<br/><br/><br/><br/><br/>--<br/>Bob Smith</div>'
+  message = "<div>test 123 <br/><br/>--<br/>Bob Smith<br/><br/><br/><br/><br/>--<br/>Bob Smith</div>"
+  should  = '<div>test 123 <br/><br/><span class="js-signatureMarker"></span>--<br/>Bob Smith<br/><br/><br/><br/><br/>--<br/>Bob Smith</div>'
  //should  = '<div>test 123 <br><br><br><br><br><br><br><br><br><br><br><span class="js-signatureMarker"></span>--<br>Bob Smith<br/><br/><br/><br/><br/>--<br/>Bob Smith</div>'
-  result  = App.Utils.signatureIdentify( message )
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

-  message = "<div>test 123</div><div>test 123</div><div>test 123</div><div>test 123</div><div>test 123</div><div>test 123</div><div>test 123</div><div>test 123</div><div>--</div><div>Bob Smith</div>"
-  should  = "<div>test 123</div><div>test 123</div><div>test 123</div><div>test 123</div><div>test 123</div><div>test 123</div><div>test 123</div><div>test 123</div><div><span class=\"js-signatureMarker\"></span>--</div><div>Bob Smith</div>"
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>test 123</div><div>test 123</div><div>--</div><div>Bob Smith</div>"
+  should  = "<div>test 123</div><div>test 123</div><div><span class=\"js-signatureMarker\"></span>--</div><div>Bob Smith</div>"
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

-  message = "<p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>--</span></p><p><span>Bob Smith</span></p><div></div>"
-  should = "<p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span>test 123</span></p><p><span><span class=\"js-signatureMarker\"></span>--</span></p><p><span>Bob Smith</span></p><div></div>"
-  result  = App.Utils.signatureIdentify( message )
+  message = "<p><span>test 123</span></p><p><span>test 123</span></p><p><span>--</span></p><p><span>Bob Smith</span></p><div></div>"
+  should  = "<p><span>test 123</span></p><p><span>test 123</span></p><p><span><span class=\"js-signatureMarker\"></span>--</span></p><p><span>Bob Smith</span></p><div></div>"
+  result  = App.Utils.signatureIdentify( message, true )

  // apple
-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/>On 01/04/15 10:55, Bob Smith wrote:<br/>lalala</div>"
-  should  = '<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>On 01/04/15 10:55, Bob Smith wrote:<br/>lalala</div>'
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/>On 01/04/15 10:55, Bob Smith wrote:<br/>lalala<p>--</p>some test</div>"
+  should  = '<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>On 01/04/15 10:55, Bob Smith wrote:<br/>lalala<p>--</p>some test</div>'
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/>Am 01/04/15 10:55, Bob Smith schrieb:<br/>lalala</div>"
-  should  = '<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>Am 01/04/15 10:55, Bob Smith schrieb:<br/>lalala</div>'
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/>Am 01/04/15 10:55, Bob Smith schrieb:<br/>lalala</div>"
+  should  = '<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>Am 01/04/15 10:55, Bob Smith schrieb:<br/>lalala</div>'
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

  // ms
-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/>Von: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]<br/>Gesendet: Donnerstag, 2. April 2015 10:00<br/>lalala</div>"
-  should  = '<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>Von: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]<br/>Gesendet: Donnerstag, 2. April 2015 10:00<br/>lalala</div>'
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/>Von: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]<br/>Gesendet: Donnerstag, 2. April 2015 10:00<br/>lalala</div>"
+  should  = '<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>Von: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]<br/>Gesendet: Donnerstag, 2. April 2015 10:00<br/>lalala</div>'
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/>From: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]<br/>Sent: Donnerstag, 2. April 2015 10:00<br/>lalala</div>"
-  should  = '<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>From: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]<br/>Sent: Donnerstag, 2. April 2015 10:00<br/>lalala</div>'
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/>From: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]<br/>Sent: Donnerstag, 2. April 2015 10:00<br/>lalala</div>"
+  should  = '<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>From: Martin Edenhofer via Znuny Support [mailto:support@znuny.inc]<br/>Sent: Donnerstag, 2. April 2015 10:00<br/>lalala</div>'
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

-  message = "<div>1<br><br><br><br><br><br><br><br><br><br></div><div>Von: Martin Edenhofer via Znuny Support [<a href=\"mailto:support@znuny.inc\" title=\"mailto:support@znuny.inc\" target=\"_blank\">mailto:support@znuny.inc</a>]</div>\n<div>Gesendet: Donnerstag, 2. April 2015 11:32</div>"
-  should  = "<div>1<br><br><br><br><br><br><br><br><br><br></div><div><span class=\"js-signatureMarker\"></span>Von: Martin Edenhofer via Znuny Support [<a href=\"mailto:support@znuny.inc\" title=\"mailto:support@znuny.inc\" target=\"_blank\">mailto:support@znuny.inc</a>]</div>\n<div>Gesendet: Donnerstag, 2. April 2015 11:32</div>"
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>1<br><br></div><div>Von: Martin Edenhofer via Znuny Support [<a href=\"mailto:support@znuny.inc\" title=\"mailto:support@znuny.inc\" target=\"_blank\">mailto:support@znuny.inc</a>]</div>\n<div>Gesendet: Donnerstag, 2. April 2015 11:32</div>"
+  should  = "<div>1<br><br></div><div><span class=\"js-signatureMarker\"></span>Von: Martin Edenhofer via Znuny Support [<a href=\"mailto:support@znuny.inc\" title=\"mailto:support@znuny.inc\" target=\"_blank\">mailto:support@znuny.inc</a>]</div>\n<div>Gesendet: Donnerstag, 2. April 2015 11:32</div>"
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

  // otrs
-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/>01/04/15 10:55 - Bob Smith wrote:<br/>lalala</div>"
-  should  = '<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>01/04/15 10:55 - Bob Smith wrote:<br/>lalala</div>'
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/>01/04/15 10:55 - Bob Smith wrote:<br/>lalala</div>"
+  should  = '<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>01/04/15 10:55 - Bob Smith wrote:<br/>lalala</div>'
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/>01/04/15 10:55 - Bob Smith schrieb:<br/>lalala</div>"
-  should  = '<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>01/04/15 10:55 - Bob Smith schrieb:<br/>lalala</div>'
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/>01/04/15 10:55 - Bob Smith schrieb:<br/>lalala</div>"
+  should  = '<div>test 123 <br/><br/>--no not match--<br/><br/>Bob Smith<br/><span class="js-signatureMarker"></span>01/04/15 10:55 - Bob Smith schrieb:<br/>lalala</div>'
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

-  message = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/></div><div>24.02.2015 14:20 - Roy Kaldung via Znuny Sales schrieb: &nbsp;</div>"
-  should  = "<div>test 123 <br/><br/><br/><br/><br/><br/><br/><br/><br/></div><div><span class=\"js-signatureMarker\"></span>24.02.2015 14:20 - Roy Kaldung via Znuny Sales schrieb: &nbsp;</div>"
-  result  = App.Utils.signatureIdentify( message )
+  message = "<div>test 123 <br/><br/></div><div>24.02.2015 14:20 - Roy Kaldung via Znuny Sales schrieb: &nbsp;</div>"
+  should  = "<div>test 123 <br/><br/></div><div><span class=\"js-signatureMarker\"></span>24.02.2015 14:20 - Roy Kaldung via Znuny Sales schrieb: &nbsp;</div>"
+  result  = App.Utils.signatureIdentify( message, true )
  equal( result, should )

-//</div>
-
 });

 // replace tags