From 5856dd7c06bc836285ac5700e37a923087a4f9a6 Mon Sep 17 00:00:00 2001
From: Martin Edenhofer <me@edenhofer.de>
Date: Sat, 3 Dec 2016 20:37:37 +0100
Subject: [PATCH] Moved to offline translation files (improve speed of Zammad
 installation and possibility to need no network connection).

---
 .pkgr.yml                     |   3 +
 app/models/locale.rb          |  61 ++++++++++-
 app/models/translation.rb     | 196 ++++++++++++++++++++++++----------
 db/seeds.rb                   |   4 +-
 test/unit/translation_test.rb |  97 ++++++++++++++++-
 5 files changed, 296 insertions(+), 65 deletions(-)

diff --git a/.pkgr.yml b/.pkgr.yml
index a61832909..706357181 100644
--- a/.pkgr.yml
+++ b/.pkgr.yml
@@ -25,6 +25,9 @@ before:
   - env
   - "cat Gemfile.lock"
   - contrib/cleanup.sh
+after:
+  - rails r 'Locale.fetch'
+  - rails r 'Translation.fetch'
 env:
   - RAILS_ENV=production
   - PORT=3000
diff --git a/app/models/locale.rb b/app/models/locale.rb
index 0021d5668..dfc4e028b 100644
--- a/app/models/locale.rb
+++ b/app/models/locale.rb
@@ -8,7 +8,7 @@ get locals to sync
 
 all:
 
-  Locale.sync
+  Locale.to_sync
 
 returns
 
@@ -31,6 +31,21 @@ returns
 
 =begin
 
+sync locales from local if exists, otherwise from online
+
+all:
+
+  Locale.sync
+
+=end
+
+  def self.sync
+    return true if load_from_file
+    load
+  end
+
+=begin
+
 load locales from online
 
 all:
@@ -40,6 +55,39 @@ all:
 =end
 
   def self.load
+    data = fetch
+    to_database(data)
+  end
+
+=begin
+
+load locales from local
+
+all:
+
+  Locale.load_from_file
+
+=end
+
+  def self.load_from_file
+    file = Rails.root.join('config/locales.yml')
+    return false if !File.exist?(file)
+    data = YAML.load_file(file)
+    to_database(data)
+    true
+  end
+
+=begin
+
+fetch locales from remote and store them in local file system
+
+all:
+
+  Locale.fetch
+
+=end
+
+  def self.fetch
     url = 'https://i18n.zammad.com/api/v1/locales'
 
     result = UserAgent.get(
@@ -53,8 +101,16 @@ all:
     raise "Can't load locales from #{url}" if !result
     raise "Can't load locales from #{url}: #{result.error}" if !result.success?
 
+    file = Rails.root.join('config/locales.yml')
+    File.open(file, 'w') do |out|
+      YAML.dump(result.data, out)
+    end
+    result.data
+  end
+
+  private_class_method def self.to_database(data)
     ActiveRecord::Base.transaction do
-      result.data.each { |locale|
+      data.each { |locale|
         exists = Locale.find_by(locale: locale['locale'])
         if exists
           exists.update(locale.symbolize_keys!)
@@ -63,7 +119,6 @@ all:
         end
       }
     end
-    true
   end
 
 end
diff --git a/app/models/translation.rb b/app/models/translation.rb
index 7371ea8ea..ac4e3a945 100644
--- a/app/models/translation.rb
+++ b/app/models/translation.rb
@@ -8,6 +8,23 @@ class Translation < ApplicationModel
 
 =begin
 
+sync translations from local if exists, otherwise from online
+
+all:
+
+  Translation.sync
+
+  Translation.sync(locale) # e. g. 'en-us' or 'de-de'
+
+=end
+
+  def self.sync(dedicated_locale = nil)
+    return true if load_from_file(dedicated_locale)
+    load
+  end
+
+=begin
+
 load translations from online
 
 all:
@@ -21,62 +38,9 @@ dedicated:
 =end
 
   def self.load(dedicated_locale = nil)
-    locales_list = []
-    if !dedicated_locale
-      locales = Locale.to_sync
-      locales.each { |locale|
-        locales_list.push locale.locale
-      }
-    else
-      locales_list = [dedicated_locale]
-    end
-    locales_list.each { |locale|
-      url = "https://i18n.zammad.com/api/v1/translations/#{locale}"
-      if !UserInfo.current_user_id
-        UserInfo.current_user_id = 1
-      end
-      result = UserAgent.get(
-        url,
-        {},
-        {
-          json: true,
-          open_timeout: 6,
-          read_timeout: 16,
-        }
-      )
-      raise "Can't load translations from #{url}: #{result.error}" if !result.success?
-
-      translations = Translation.where(locale: locale).all
-      ActiveRecord::Base.transaction do
-        result.data.each { |translation_raw|
-
-          # handle case insensitive sql
-          translation = nil
-          translations.each { |item|
-            next if item.format != translation_raw['format']
-            next if item.source != translation_raw['source']
-            translation = item
-            break
-          }
-          if translation
-
-            # verify if update is needed
-            update_needed = false
-            translation_raw.each { |key, _value|
-              if translation_raw[key] != translation[key]
-                update_needed = true
-                break
-              end
-            }
-            if update_needed
-              translation.update_attributes(translation_raw.symbolize_keys!)
-              translation.save
-            end
-          else
-            Translation.create(translation_raw.symbolize_keys!)
-          end
-        }
-      end
+    locals_to_sync(dedicated_locale).each { |locale|
+      fetch(locale)
+      load_from_file(locale)
     }
     true
   end
@@ -246,6 +210,126 @@ translate strings in ruby context, e. g. for notifications
     string
   end
 
+=begin
+
+load locales from local
+
+all:
+
+  Translation.load_from_file
+
+  or
+
+  Translation.load_from_file(locale) # e. g. 'en-us' or 'de-de'
+
+=end
+
+  def self.load_from_file(dedicated_locale = nil)
+    directory = Rails.root.join('config/translations')
+    locals_to_sync(dedicated_locale).each { |locale|
+      file = Rails.root.join("#{directory}/#{locale}.yml")
+      return false if !File.exist?(file)
+      data = YAML.load_file(file)
+      to_database(locale, data)
+    }
+    true
+  end
+
+=begin
+
+fetch translation from remote and store them in local file system
+
+all:
+
+  Translation.fetch
+
+  or
+
+  Translation.fetch(locale) # e. g. 'en-us' or 'de-de'
+
+=end
+
+  def self.fetch(dedicated_locale = nil)
+    locals_to_sync(dedicated_locale).each { |locale|
+      url = "https://i18n.zammad.com/api/v1/translations/#{locale}"
+      if !UserInfo.current_user_id
+        UserInfo.current_user_id = 1
+      end
+      result = UserAgent.get(
+        url,
+        {},
+        {
+          json: true,
+          open_timeout: 8,
+          read_timeout: 24,
+        }
+      )
+      raise "Can't load translations from #{url}: #{result.error}" if !result.success?
+
+      directory = Rails.root.join('config/translations')
+      if !File.directory?(directory)
+        Dir.mkdir(directory, 0o755)
+      end
+      file = Rails.root.join("#{directory}/#{locale}.yml")
+      File.open(file, 'w') do |out|
+        YAML.dump(result.data, out)
+      end
+    }
+    true
+  end
+
+  private_class_method def self.to_database(locale, data)
+    translations = Translation.where(locale: locale).all
+    ActiveRecord::Base.transaction do
+      data.each { |translation_raw|
+
+        # handle case insensitive sql
+        translation = nil
+        translations.each { |item|
+          next if item.format != translation_raw['format']
+          next if item.source != translation_raw['source']
+          translation = item
+          break
+        }
+        if translation
+
+          # verify if update is needed
+          update_needed = false
+          translation_raw.each { |key, _value|
+
+            # if translation target has changes
+            next unless translation_raw[key] != translation.target
+
+            # do not update translations which are already changed by user
+            if translation.target == translation.target_initial
+              update_needed = true
+              break
+            end
+          }
+          if update_needed
+            translation.update_attributes(translation_raw.symbolize_keys!)
+            translation.save
+          end
+        else
+          Translation.create(translation_raw.symbolize_keys!)
+        end
+      }
+    end
+  end
+
+  private_class_method def self.locals_to_sync(dedicated_locale = nil)
+    locales_list = []
+    if !dedicated_locale
+      locales = Locale.to_sync
+      locales.each { |locale|
+        locales_list.push locale.locale
+      }
+    else
+      locales_list = [dedicated_locale]
+    end
+    locales_list
+  end
+
   private
 
   def set_initial
diff --git a/db/seeds.rb b/db/seeds.rb
index 4882d358f..dd709e65c 100644
--- a/db/seeds.rb
+++ b/db/seeds.rb
@@ -5455,8 +5455,8 @@ Locale.create_if_not_exists(
   alias: 'en',
   name: 'English (United States)',
 )
-Locale.load
-Translation.load
+Locale.sync
+Translation.sync
 Calendar.init_setup
 
 # install all packages in auto_install
diff --git a/test/unit/translation_test.rb b/test/unit/translation_test.rb
index 02d47043a..b5366f4df 100644
--- a/test/unit/translation_test.rb
+++ b/test/unit/translation_test.rb
@@ -3,9 +3,12 @@ require 'test_helper'
 
 class TranslationTest < ActiveSupport::TestCase
 
-  Translation.load('de-de')
+  test 'setup' do
+    Translation.reset('de-de')
+    Translation.load('de-de')
+  end
 
-  test 'translation' do
+  test 'basics' do
     tests = [
       {
         locale: 'en',
@@ -29,8 +32,94 @@ class TranslationTest < ActiveSupport::TestCase
       },
     ]
     tests.each { |test|
-      result = Translation.translate( test[:locale], test[:string] )
-      assert_equal( result, test[:result], 'verify result' )
+      result = Translation.translate(test[:locale], test[:string])
+      assert_equal(result, test[:result], 'verify result')
     }
   end
+
+  test 'own translation tests' do
+    locale = 'de-de'
+
+    # check for custom changes
+    list = Translation.lang(locale)
+    list['list'].each { |item|
+      translation = Translation.find_by(source: item[1], locale: locale)
+      assert(translation)
+      assert_equal(locale, translation.locale)
+      assert_equal(translation.target, translation.target_initial)
+    }
+
+    # add custom changes
+    translation = Translation.find_by(locale: locale, source: 'open')
+    assert_equal('offen', translation.target)
+    assert_equal('offen', translation.target_initial)
+    translation.target = 'offen2'
+    translation.save!
+
+    list = Translation.lang(locale)
+    list['list'].each { |item|
+      translation = Translation.find_by(source: item[1], locale: locale)
+      assert(translation)
+      assert_equal(locale, translation.locale)
+      if translation.source == 'open'
+        assert_equal('offen2', translation.target)
+        assert_equal('offen', translation.target_initial)
+      else
+        assert_equal(translation.target, translation.target_initial)
+      end
+    }
+
+    Translation.load(locale)
+
+    list = Translation.lang(locale)
+    list['list'].each { |item|
+      translation = Translation.find_by(source: item[1], locale: locale)
+      assert(translation)
+      assert_equal(locale, translation.locale)
+      if translation.source == 'open'
+        p translation
+        assert_equal('offen2', translation.target)
+        assert_equal('offen', translation.target_initial)
+      else
+        assert_equal(translation.target, translation.target_initial)
+      end
+    }
+
+    Translation.reset(locale)
+
+    list = Translation.lang(locale)
+    list['list'].each { |item|
+      translation = Translation.find_by(source: item[1], locale: locale)
+      assert(translation)
+      assert_equal(locale, translation.locale)
+      assert_equal(translation.target, translation.target_initial)
+    }
+
+  end
+
+  test 'file based import' do
+
+    # locales
+    directory = Rails.root.join('config')
+    file = Rails.root.join("#{directory}/locales.yml")
+    if File.exist?(file)
+      File.delete(file)
+    end
+    assert_not(File.exist?(file))
+    Locale.fetch
+    assert(File.exist?(file))
+
+    # translations
+    locale = 'de-de'
+    directory = Rails.root.join('config/translations')
+    if File.directory?(directory)
+      FileUtils.rm_rf(directory)
+    end
+    file = Rails.root.join("#{directory}/#{locale}.yml")
+    assert_not(File.exist?(file))
+    Translation.fetch(locale)
+    assert(File.exist?(file))
+
+  end
+
 end