From 15577498b0dcb20248bb67c866c347988411c63e Mon Sep 17 00:00:00 2001 From: Martin Edenhofer Date: Mon, 23 Mar 2015 01:31:30 +0100 Subject: [PATCH] Added json support to UserAgent, replaced faraday with UserAgent. --- Gemfile | 12 ---- app/models/avatar.rb | 1 + lib/geo_ip/zammad_geo_ip.rb | 4 +- lib/import/otrs2.rb | 1 + lib/search_index_backend.rb | 103 ++++++++++++++++------------ lib/user_agent.rb | 48 ++++++++++--- test/integration/user_agent_test.rb | 70 +++++++++++++++++++ test/unit/rest_test.rb | 8 +-- 8 files changed, 174 insertions(+), 73 deletions(-) diff --git a/Gemfile b/Gemfile index 000dd419f..fb3956c01 100644 --- a/Gemfile +++ b/Gemfile @@ -58,15 +58,6 @@ gem 'net-ldap' gem 'writeexcel' -# Use unicorn as the web server -# gem 'unicorn' - -# Deploy with Capistrano -# gem 'capistrano' - -# To use debugger -# gem 'ruby-debug' - # event machine gem 'eventmachine' gem 'em-websocket' @@ -86,9 +77,6 @@ group :development, :test do # UI tests w/ Selenium gem 'selenium-webdriver' - # rest client lib for unit tests - gem 'faraday' - # livereload on template changes (html, js, css) gem 'guard', '>= 2.2.2', :require => false gem 'guard-livereload', :require => false diff --git a/app/models/avatar.rb b/app/models/avatar.rb index 14dfe51d9..da6ea4a6a 100644 --- a/app/models/avatar.rb +++ b/app/models/avatar.rb @@ -117,6 +117,7 @@ add a avatar # fetch image response = UserAgent.get( data[:url], + {}, { :open_timeout => 4, :read_timeout => 6, diff --git a/lib/geo_ip/zammad_geo_ip.rb b/lib/geo_ip/zammad_geo_ip.rb index 3c7cce1ff..c167b6635 100644 --- a/lib/geo_ip/zammad_geo_ip.rb +++ b/lib/geo_ip/zammad_geo_ip.rb @@ -17,7 +17,9 @@ class GeoIp::ZammadGeoIp begin response = UserAgent.get( "#{host}#{url}", + {}, { + :json => true, :open_timeout => 2, :read_timeout => 4, }, @@ -26,7 +28,7 @@ class GeoIp::ZammadGeoIp raise "ERROR: #{response.code.to_s}/#{response.body}" end - data = JSON.parse( response.body ) + data = response.data # compat. map if data && data['country_code2'] diff --git a/lib/import/otrs2.rb b/lib/import/otrs2.rb index e96e632a6..e43c5f4f5 100644 --- a/lib/import/otrs2.rb +++ b/lib/import/otrs2.rb @@ -55,6 +55,7 @@ module Import::OTRS2 puts 'GET: ' + url response = UserAgent.get( url, + {}, { :open_timeout => 6, :read_timeout => 60, diff --git a/lib/search_index_backend.rb b/lib/search_index_backend.rb index 775604b8c..ffe90ba5c 100644 --- a/lib/search_index_backend.rb +++ b/lib/search_index_backend.rb @@ -43,18 +43,23 @@ create/update/delete index return SearchIndexBackend.remove( data[:name] ) end - puts "# curl -X PUT \"#{url}\" -d '#{data[:data].to_json}'" + puts "# curl -X PUT \"#{url}\" \\" + #puts "-d '#{data[:data].to_json}'" - conn = connection( url ) - response = conn.put do |req| - req.url url - req.headers['Content-Type'] = 'application/json' - req.body = data[:data].to_json - end - puts "# #{response.status.to_s}" + response = UserAgent.put( + url, + data[:data], + { + :json => true, + :open_timeout => 5, + :read_timeout => 14, + :user => Setting.get('es_user'), + :password => Setting.get('es_password'), + } + ) + puts "# #{response.code.to_s}" return true if response.success? - data = JSON.parse( response.body ) - raise data.inspect + raise response.body end =begin @@ -70,18 +75,23 @@ add new object to search index url = build_url( type, data['id'] ) return if !url - puts "# curl -X POST \"#{url}\" -d '#{data.to_json}'" + puts "# curl -X POST \"#{url}\" \\" + #puts "-d '#{data.to_json}'" - conn = connection( url ) - response = conn.post do |req| - req.url url - req.headers['Content-Type'] = 'application/json' - req.body = data.to_json - end - puts "# #{response.status.to_s}" + response = UserAgent.post( + url, + data, + { + :json => true, + :open_timeout => 5, + :read_timeout => 14, + :user => Setting.get('es_user'), + :password => Setting.get('es_password'), + } + ) + puts "# #{response.code.to_s}" return true if response.success? - data = JSON.parse( response.body ) - raise data.inspect + raise response.body end =begin @@ -100,13 +110,18 @@ remove whole data from index puts "# curl -X DELETE \"#{url}\"" - conn = connection( url ) - response = conn.delete( url ) - puts "# #{response.status.to_s}" - return false if !response.success? - data = JSON.parse( response.body ) -# raise data.inspect - return { :data => data, :response => response } + response = UserAgent.delete( + url, + { + :open_timeout => 5, + :read_timeout => 14, + :user => Setting.get('es_user'), + :password => Setting.get('es_password'), + } + ) + #puts "# #{response.code.to_s}" + return true if response.success? + #raise response.body end =begin @@ -177,19 +192,27 @@ return search result } data['query']['bool']['must'].push condition - puts "# curl -X POST \"#{url}\" -d '#{data.to_json}'" + puts "# curl -X POST \"#{url}\" \\" + #puts " -d'#{data.to_json}'" - conn = connection( url ) - response = conn.get do |req| - req.headers['Content-Type'] = 'application/json' - req.body = data.to_json - end - puts "# #{response.status.to_s}" - data = JSON.parse( response.body ) + response = UserAgent.get( + url, + data, + { + :json => true, + :open_timeout => 5, + :read_timeout => 14, + :user => Setting.get('es_user'), + :password => Setting.get('es_password'), + } + ) + + puts "# #{response.code.to_s}" if !response.success? return [] # raise data.inspect end + data = response.data ids = [] return ids if !data @@ -239,14 +262,4 @@ return true if backend is configured url end - def self.connection( url ) - conn = Faraday.new( :url => url, :request => { :open_timeout => 5, :timeout => 10 } ) - user = Setting.get('es_user') - pw = Setting.get('es_password') - if user && !user.empty? && pw && !pw.empty? - conn.basic_auth( user, pw ) - end - conn - end - end \ No newline at end of file diff --git a/lib/user_agent.rb b/lib/user_agent.rb index 13d0c99ad..d881f8fbd 100644 --- a/lib/user_agent.rb +++ b/lib/user_agent.rb @@ -15,6 +15,9 @@ get http/https calls result = UserAgent.get( 'http://host/some_dir/some_file?param1=123', + { + :param1 => 'some value', + }, { :open_timeout => 2, :read_timeout => 4, @@ -27,7 +30,7 @@ returns =end - def self.get(url, options = {}, count = 10) + def self.get(url, params = {}, options = {}, count = 10) uri = URI.parse(url) http = get_http(uri, options) @@ -37,10 +40,13 @@ returns # http basic auth (if needed) request = set_basic_auth(request, options) + # set params + request = set_params(request, params, options) + # start http call begin response = http.request(request) - return process(response, uri, count, options) + return process(response, uri, count, params, options) rescue Exception => e return Result.new( :error => e.inspect, @@ -80,7 +86,7 @@ returns request = Net::HTTP::Post.new( uri, {'User-Agent' => 'Zammad User Agent'} ) # set params - request.set_form_data( params ) + request = set_params(request, params, options) # http basic auth (if needed) request = set_basic_auth(request, options) @@ -88,7 +94,7 @@ returns # start http call begin response = http.request(request) - return process(response, uri, count, options) + return process(response, uri, count, params, options) rescue Exception => e return Result.new( :error => e.inspect, @@ -128,7 +134,7 @@ returns request = Net::HTTP::Put.new( uri, {'User-Agent' => 'Zammad User Agent'} ) # set params - request.set_form_data( params ) + request = set_params(request, params, options) # http basic auth (if needed) request = set_basic_auth(request, options) @@ -136,7 +142,7 @@ returns # start http call begin response = http.request(request) - return process(response, uri, count, options) + return process(response, uri, count, params, options) rescue Exception => e return Result.new( :error => e.inspect, @@ -177,7 +183,7 @@ returns # start http call begin response = http.request(request) - return process(response, uri, count, options) + return process(response, uri, count, {}, options) rescue Exception => e return Result.new( :error => e.inspect, @@ -219,7 +225,7 @@ returns when /ftp/ ftp(uri, options) when /http|https/ - get( url, options ) + get( url, {}, options ) end end @@ -249,7 +255,20 @@ returns request end - def self.process(response, uri, count, options = {}) + def self.set_params(request, params, options) + if options[:json] + request.add_field("Content-Type", "application/json") + if !params.empty? + request.body = params.to_json + end + else + if !params.empty? + request.set_form_data( params ) + end + end + request + end + def self.process(response, uri, count, params, options) if !response return Result.new( :error => "Can't connect to #{uri.to_s}, got no response!", @@ -281,10 +300,15 @@ returns when Net::HTTPRedirection raise "Too many redirections for the original URL, halting." if count <= 0 url = response['location'] - return get(url, options, count - 1) + return get(url, params, options, count - 1) when Net::HTTPOK + data = nil + if options[:json] && response.body + data = JSON.parse( response.body ) + end return Result.new( + :data => data, :body => response.body, :content_type => response['Content-Type'], :success => true, @@ -344,6 +368,7 @@ returns def initialize(options) @success = options[:success] @body = options[:body] + @data = options[:data] @code = options[:code] @content_type = options[:content_type] @error = options[:error] @@ -357,6 +382,9 @@ returns def body @body end + def data + @data + end def code @code end diff --git a/test/integration/user_agent_test.rb b/test/integration/user_agent_test.rb index 02945859d..f720e47e9 100644 --- a/test/integration/user_agent_test.rb +++ b/test/integration/user_agent_test.rb @@ -18,6 +18,7 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal(String, result.body.class) assert(result.body =~ /"get"/) assert(result.body =~ /"123"/) + assert(result.body =~ /"text\/plain"/) # get / 404 result = UserAgent.get( @@ -41,6 +42,7 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal(String, result.body.class) assert(result.body =~ /"post"/) assert(result.body =~ /"some value"/) + assert(result.body =~ /"application\/x-www-form-urlencoded"/) # post / 404 result = UserAgent.post( @@ -67,6 +69,7 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal(String, result.body.class) assert(result.body =~ /"put"/) assert(result.body =~ /"some value"/) + assert(result.body =~ /"application\/x-www-form-urlencoded"/) # put / 404 result = UserAgent.put( @@ -89,6 +92,7 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal('200', result.code) assert_equal(String, result.body.class) assert(result.body =~ /"delete"/) + assert(result.body =~ /"text\/plain"/) # delete / 404 result = UserAgent.delete( @@ -106,6 +110,7 @@ class UserAgentTest < ActiveSupport::TestCase # get / 200 result = UserAgent.get( "#{host}/test_basic_auth/get/1?submitted=123", + {}, { :user => 'basic_auth_user', :password => 'test123', @@ -117,10 +122,12 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal(String, result.body.class) assert(result.body =~ /"get"/) assert(result.body =~ /"123"/) + assert(result.body =~ /"text\/plain"/) # get / 401 result = UserAgent.get( "#{host}/test_basic_auth/get/1?submitted=123", + {}, { :user => 'basic_auth_user_not_existing', :password => 'test<>123', @@ -148,6 +155,7 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal(String, result.body.class) assert(result.body =~ /"post"/) assert(result.body =~ /"some value"/) + assert(result.body =~ /"application\/x-www-form-urlencoded"/) # post / 401 result = UserAgent.post( @@ -182,6 +190,7 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal(String, result.body.class) assert(result.body =~ /"put"/) assert(result.body =~ /"some value"/) + assert(result.body =~ /"application\/x-www-form-urlencoded"/) # put / 401 result = UserAgent.put( @@ -213,6 +222,7 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal('200', result.code) assert_equal(String, result.body.class) assert(result.body =~ /"delete"/) + assert(result.body =~ /"text\/plain"/) # delete / 401 result = UserAgent.delete( @@ -241,6 +251,7 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal(String, result.body.class) assert(result.body =~ /"get"/) assert(result.body =~ /"abc"/) + assert(result.body =~ /"text\/plain"/) # get / 301 @@ -257,6 +268,7 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal(String, result.body.class) assert(result.body =~ /"get"/) assert(result.body =~ /"abc"/) + assert(result.body =~ /"text\/plain"/) # get / 401 @@ -286,6 +298,7 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal(String, result.body.class) assert(result.body =~ /"get"/) assert(result.body =~ /"123"/) + assert(result.body =~ /"text\/plain"/) # ftp / 200 result = UserAgent.request( @@ -378,6 +391,7 @@ class UserAgentTest < ActiveSupport::TestCase # get / timeout result = UserAgent.get( "#{host}/test/get/3?submitted=123", + {}, { :open_timeout => 1, :read_timeout => 1, @@ -405,4 +419,60 @@ class UserAgentTest < ActiveSupport::TestCase assert_equal(NilClass, result.body.class) end + # check + test 'check json' do + + # get / 200 + result = UserAgent.get( + "#{host}/test/get/1", + { + :submitted => { :key => 'some value ' } + }, + { + :json => true, + } + ) + assert(result) + assert_equal(true, result.success?) + assert_equal('200', result.code) + assert_equal(String, result.body.class) + assert(result.body =~ /"content_type_requested"/) + assert(result.body =~ /"application\/json"/) + assert_equal('some value ', result.data['submitted']['key']) + + # get / 401 + result = UserAgent.get( + "#{host}/test/not_existing", + { + :submitted => { :key => 'some value ' } + }, + { + :json => true, + } + ) + assert(result) + assert_equal(false, result.success?) + assert_equal('404', result.code) + assert_equal(NilClass, result.body.class) + assert(!result.data) + + # post / 200 + result = UserAgent.post( + "#{host}/test/post/1", + { + :submitted => { :key => 'some value ' } + }, + { + :json => true, + } + ) + assert(result) + assert_equal(true, result.success?) + assert_equal('200', result.code) + assert_equal(String, result.body.class) + assert(result.body =~ /"content_type_requested"/) + assert(result.body =~ /"application\/json"/) + assert_equal('some value ', result.data['submitted']['key']) + end + end \ No newline at end of file diff --git a/test/unit/rest_test.rb b/test/unit/rest_test.rb index f02aca9eb..291220b79 100644 --- a/test/unit/rest_test.rb +++ b/test/unit/rest_test.rb @@ -231,7 +231,9 @@ class RestTest < ActiveSupport::TestCase response = UserAgent.get( "#{ENV['BROWSER_URL']}#{url}", + {}, { + :json => true, :user => user, :password => pw, } @@ -239,11 +241,7 @@ class RestTest < ActiveSupport::TestCase #puts 'URL: ' + url #puts response.code.to_s #puts response.body.to_s - data = nil - if response.body - data = JSON.parse( response.body ) - end - return { :data => data, :response => response } + return { :data => response.data, :response => response } end end