Improved issue #675 - Added web UI for import of users and organisations via csv files.

This commit is contained in:
Martin Edenhofer 2018-06-06 03:30:17 +02:00
parent 52ab76d7a7
commit b58ac7e1be
21 changed files with 394 additions and 23 deletions

View file

@ -98,6 +98,7 @@ class App.ControllerGenericIndex extends App.Controller
events:
'click [data-type=edit]': 'edit'
'click [data-type=new]': 'new'
'click [data-type=import]': 'import'
'click .js-description': 'description'
constructor: ->
@ -217,6 +218,10 @@ class App.ControllerGenericIndex extends App.Controller
large: @large
)
import: (e) ->
e.preventDefault()
@importCallback()
description: (e) =>
new App.ControllerGenericDescription(
description: App[ @genericObject ].description
@ -1275,3 +1280,123 @@ class App.ObserverActionRow extends App.ObserverController
e.preventDefault()
item.callback(object)
)
class App.Import extends App.ControllerModal
buttonClose: true
buttonCancel: true
buttonSubmit: 'Import'
head: 'Import'
large: true
templateDirectory: 'generic/object_import'
baseUrl: '/api/v1/text_modules'
content: =>
# show start dialog
content = $(App.view("#{@templateDirectory}/index")(
head: 'Import'
import_example_url: "#{@baseUrl}/import_example"
))
# check if data is processing...
if @data
result = App.view("#{@templateDirectory}/result")(
@data
)
content.find('.js-error').html(result)
if result
content.find('.js-error').removeClass('hide')
else
content.find('.js-error').addClass('hide')
content
onSubmit: (e) =>
params = new FormData($(e.currentTarget).closest('form').get(0))
params.set('try', true)
if _.isEmpty(params.get('data'))
params.delete('data')
@ajax(
id: 'csv_import'
type: 'POST'
url: "#{@baseUrl}/import"
processData: false
contentType: false
cache: false
data: params
success: (data, status, xhr) =>
if data.result is 'success'
new App.ImportTryResult(
container: @el.closest('.content')
result: data
params: params
templateDirectory: @templateDirectory
baseUrl: @baseUrl
)
@close()
return
@data = data
@update()
)
class App.ImportTryResult extends App.ControllerModal
buttonClose: true
buttonCancel: true
buttonSubmit: 'Yes, start real import.'
head: 'Import'
large: true
templateDirectory: 'generic/object_import/'
baseUrl: '/api/v1/text_modules'
content: =>
# show start dialog
content = $(App.view("#{@templateDirectory}/import_try")(
head: 'Import'
import_example_url: "#{@baseUrl}/import"
result: @result
))
content
onSubmit: (e) =>
@params.set('try', false)
@ajax(
id: 'csv_import'
type: 'POST'
url: "#{@baseUrl}/import"
processData: false
contentType: false
cache: false
data: @params
success: (data, status, xhr) =>
if data.result is 'success'
new App.ImportResult(
container: @el.closest('.content')
result: data
params: @params
templateDirectory: @templateDirectory
baseUrl: @baseUrl
)
@close()
return
@data = data
@update()
)
class App.ImportResult extends App.ControllerModal
buttonClose: true
buttonCancel: true
buttonSubmit: 'Close'
head: 'Import'
large: true
templateDirectory: 'generic/object_import/'
content: =>
content = $(App.view("#{@templateDirectory}/imported")(
head: 'Imported'
result: @result
))
content
onSubmit: (e) =>
@close()

View file

@ -8,6 +8,11 @@ class Index extends App.ControllerSubContent
el: @el
id: @id
genericObject: 'Organization'
importCallback: ->
new App.Import(
baseUrl: '/api/v1/organizations'
container: @el.closest('.content')
)
pageData:
home: 'organizations'
object: 'Organization'
@ -17,6 +22,7 @@ class Index extends App.ControllerSubContent
'Organizations are for any person in the system. Agents (Owners, Resposbiles, ...) and Customers.'
]
buttons: [
{ name: 'Import', 'data-type': 'import', class: 'btn' }
{ name: 'New Organization', 'data-type': 'new', class: 'btn--success' }
]
container: @el.closest('.content')

View file

@ -8,6 +8,11 @@ class Index extends App.ControllerSubContent
el: @el
id: @id
genericObject: 'TextModule'
importCallback: ->
new App.Import(
baseUrl: '/api/v1/text_modules'
container: @el.closest('.content')
)
pageData:
home: 'text_modules'
object: 'TextModule'
@ -17,6 +22,7 @@ class Index extends App.ControllerSubContent
'Text modules are ...'
]
buttons: [
{ name: 'Import', 'data-type': 'import', class: 'btn' }
{ name: 'New text module', 'data-type': 'new', class: 'btn--success' }
]
container: @el.closest('.content')

View file

@ -5,6 +5,7 @@ class Index extends App.ControllerSubContent
'.js-search': 'searchInput'
events:
'click [data-type=new]': 'new'
'click [data-type=import]': 'import'
constructor: ->
super
@ -14,6 +15,7 @@ class Index extends App.ControllerSubContent
@html App.view('user')(
head: 'Users'
buttons: [
{ name: 'Import', 'data-type': 'import', class: 'btn' }
{ name: 'New User', 'data-type': 'new', class: 'btn--success' }
]
roles: App.Role.all()
@ -192,4 +194,11 @@ class Index extends App.ControllerSubContent
callback: @recent
)
import: (e) ->
e.preventDefault()
new App.Import(
baseUrl: '/api/v1/users'
container: @el.closest('.content')
)
App.Config.set( 'User', { prio: 1000, name: 'Users', parent: '#manage', target: '#manage/users', controller: Index, permission: ['admin.user'] }, 'NavBarAdmin' )

View file

@ -185,7 +185,6 @@
// me - 2018-05-24
// cleanup element on hide - cleanup dom with old modal dialogs
Modal.prototype.remove = function () {
console.log('remove', this.$element)
this.$element.remove()
}

View file

@ -0,0 +1,21 @@
<div>
<p class="alert alert--danger js-error hide"></p>
<p>
<% if @result.stats: %>
<%- @T('The test run was successful.') %>
<%- @T('The following changes are made:') %>
<ul>
<% if @result.stats.deleted isnt undefined: %>
<li><%- @T('%s Object(s) are deleted.', @result.stats.deleted) %></li>
<% end %>
<% if @result.stats.created isnt undefined: %>
<li><%- @T('%s Object(s) are created.', @result.stats.created) %></li>
<% end %>
<% if @result.stats.updated isnt undefined: %>
<li><%- @T('%s Object(s) are updated.', @result.stats.updated) %></li>
<% end %>
<% end %>
</p>
</div>

View file

@ -0,0 +1,28 @@
<div>
<% if @errors: %>
<ul>
<% for error in @errors: %>
<li><%= error %>
<% end %>
<% end %>
</div>
<div>
<p class="alert alert--danger js-error hide"></p>
<p>
<% if @result.stats: %>
<%- @T('The import was successful.') %>
<%- @T('The following changes have been made:') %>
<ul>
<% if @result.stats.deleted isnt undefined: %>
<li><%- @T('%s Object(s) were deleted.', @result.stats.deleted) %></li>
<% end %>
<% if @result.stats.created isnt undefined: %>
<li><%- @T('%s Object(s) have been created.', @result.stats.created) %></li>
<% end %>
<% if @result.stats.updated isnt undefined: %>
<li><%- @T('%s Object(s) have been updated.', @result.stats.updated) %></li>
<% end %>
<% end %>
</p>
</div>

View file

@ -0,0 +1,29 @@
<div>
<p class="alert alert--danger js-error hide"></p>
<p>
<%- @T('Bulk import allows you to create and update many records at once.') %>
<%- @T('The data must be in the comma separated values (CSV) format and saved as UTF-8. You can import a CSV file or paste the data directly into the text area.') %>
</p>
<p><%- @T('Alternatively, you can use the Zammad API to import data.') %></p>
<h2><input checked="checked" disabled="disabled" type="checkbox" name="create" value="true"> <%- @T('Create new records') %></h2>
<%- @T('Records that exist in the import data (but not in Zammad) will always be created.') %>
<h2><input checked="checked" disabled="disabled" type="checkbox" name="update" value="true"> <%- @T('Update existing records') %></h2>
<%- @T('Update existing records with the attributes specified in the import data.') %>
<!--
<h2><input checked="" type="checkbox" name="delete" value="true"> <%- @T('Delete records') %></h2>
<%- @T('Delete all existigs records first.') %>
-->
<h2><%- @T('Select CSV file') %></h2>
<input name="file" type="file">
<h2><%- @T('Paste in CSV data') %></h2>
<textarea cols="25" rows="6" name="data"></textarea>
<p><%- @T('Note') %>: <a href="<%- @import_example_url %>" target="_blank"><%- @T('Example CSV file for download.') %></a></p>
</div>

View file

@ -0,0 +1,9 @@
<div>
<%- @T('Result') %>: <%= @result %>
<% if @errors: %>
<ul>
<% for error in @errors: %>
<li><%= error %>
<% end %>
<% end %>
</div>

View file

@ -347,10 +347,11 @@ curl http://localhost/api/v1/organization/{id} -v -u #{login}:#{password} -H "Co
# @response_message 401 Invalid session.
def import_start
permission_check('admin.user')
string = params[:data] || params[:file].read.force_encoding('utf-8')
result = Organization.csv_import(
string: params[:file].read.force_encoding('utf-8'),
string: string,
parse_params: {
col_sep: ';',
col_sep: params[:col_sep] || ',',
},
try: params[:try],
)

View file

@ -164,11 +164,11 @@ curl http://localhost/api/v1/text_modules.json -v -u #{login}:#{password} -H "Co
def import_example
permission_check('admin.text_module')
csv_string = TextModule.csv_example(
col_sep: ',',
col_sep: params[:col_sep] || ',',
)
send_data(
csv_string,
filename: 'example.csv',
filename: 'text_module-example.csv',
type: 'text/csv',
disposition: 'attachment'
)
@ -186,10 +186,11 @@ curl http://localhost/api/v1/text_modules.json -v -u #{login}:#{password} -H "Co
# @response_message 401 Invalid session.
def import_start
permission_check('admin.text_module')
string = params[:data] || params[:file].read.force_encoding('utf-8')
result = TextModule.csv_import(
string: params[:file].read.force_encoding('utf-8'),
string: string,
parse_params: {
col_sep: ';',
col_sep: params[:col_sep] || ',',
},
try: params[:try],
)

View file

@ -629,10 +629,11 @@ class TicketsController < ApplicationController
if Setting.get('import_mode') != true
raise 'Only can import tickets if system is in import mode.'
end
string = params[:data] || params[:file].read.force_encoding('utf-8')
result = Ticket.csv_import(
string: params[:file].read.force_encoding('utf-8'),
string: string,
parse_params: {
col_sep: ';',
col_sep: params[:col_sep] || ',',
},
try: params[:try],
)

View file

@ -1100,10 +1100,11 @@ curl http://localhost/api/v1/users/avatar -v -u #{login}:#{password} -H "Content
# @response_message 401 Invalid session.
def import_start
permission_check('admin.user')
string = params[:data] || params[:file].read.force_encoding('utf-8')
result = User.csv_import(
string: params[:file].read.force_encoding('utf-8'),
string: string,
parse_params: {
col_sep: ';',
col_sep: params[:col_sep] || ',',
},
try: params[:try],
)

View file

@ -45,6 +45,7 @@ returns
=end
def csv_import(data)
errors = []
if data[:file].present?
raise Exceptions::UnprocessableEntity, "No such file '#{data[:file]}'" if !File.exist?(data[:file])
@ -56,13 +57,25 @@ returns
end
end
if data[:string].blank?
raise Exceptions::UnprocessableEntity, 'Unable to parse empty file/string!'
errors.push "Unable to parse empty file/string for #{new.class}."
result = {
errors: errors,
try: data[:try],
result: 'failed',
}
return result
end
rows = ::CSV.parse(data[:string], data[:parse_params])
header = rows.shift
if header.blank?
raise Exceptions::UnprocessableEntity, 'Unable to parse file/string without header!'
errors.push "Unable to parse file/string without header for #{new.class}."
result = {
errors: errors,
try: data[:try],
result: 'failed',
}
return result
end
header.each do |item|
if item.respond_to?(:strip!)
@ -72,6 +85,16 @@ returns
item.downcase!
end
if rows[0].blank?
errors.push "No records found in file/string for #{new.class}."
result = {
errors: errors,
try: data[:try],
result: 'failed',
}
return result
end
# get payload based on csv
payload = []
rows.each do |row|
@ -79,6 +102,7 @@ returns
payload_last = payload.last
row.each_with_index do |item, count|
next if item.blank?
next if header[count].nil?
if payload_last[header[count].to_sym].class != Array
payload_last[header[count].to_sym] = [payload_last[header[count].to_sym]]
end
@ -110,7 +134,6 @@ returns
created: 0,
updated: 0,
}
errors = []
line_count = 0
payload.each do |attributes|
line_count += 1

View file

@ -532,7 +532,7 @@ class OrganizationControllerTest < ActionDispatch::IntegrationTest
# invalid file
csv_file = ::Rack::Test::UploadedFile.new(Rails.root.join('test', 'fixtures', 'csv', 'organization_simple_col_not_existing.csv'), 'text/csv')
post '/api/v1/organizations/import?try=true', params: { file: csv_file }, headers: { 'Authorization' => credentials }
post '/api/v1/organizations/import?try=true', params: { file: csv_file, col_sep: ';' }, headers: { 'Authorization' => credentials }
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(Hash, result.class)
@ -546,7 +546,7 @@ class OrganizationControllerTest < ActionDispatch::IntegrationTest
# valid file try
csv_file = ::Rack::Test::UploadedFile.new(Rails.root.join('test', 'fixtures', 'csv', 'organization_simple.csv'), 'text/csv')
post '/api/v1/organizations/import?try=true', params: { file: csv_file }, headers: { 'Authorization' => credentials }
post '/api/v1/organizations/import?try=true', params: { file: csv_file, col_sep: ';' }, headers: { 'Authorization' => credentials }
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(Hash, result.class)
@ -560,7 +560,7 @@ class OrganizationControllerTest < ActionDispatch::IntegrationTest
# valid file
csv_file = ::Rack::Test::UploadedFile.new(Rails.root.join('test', 'fixtures', 'csv', 'organization_simple.csv'), 'text/csv')
post '/api/v1/organizations/import', params: { file: csv_file }, headers: { 'Authorization' => credentials }
post '/api/v1/organizations/import', params: { file: csv_file, col_sep: ';' }, headers: { 'Authorization' => credentials }
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(Hash, result.class)

View file

@ -102,7 +102,7 @@ class TextModuleControllerTest < ActionDispatch::IntegrationTest
# invalid file
csv_file = ::Rack::Test::UploadedFile.new(Rails.root.join('test', 'fixtures', 'csv', 'text_module_simple_col_not_existing.csv'), 'text/csv')
post '/api/v1/text_modules/import?try=true', params: { file: csv_file }, headers: { 'Authorization' => credentials }
post '/api/v1/text_modules/import?try=true', params: { file: csv_file, col_sep: ';' }, headers: { 'Authorization' => credentials }
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(Hash, result.class)
@ -116,7 +116,7 @@ class TextModuleControllerTest < ActionDispatch::IntegrationTest
# valid file try
csv_file = ::Rack::Test::UploadedFile.new(Rails.root.join('test', 'fixtures', 'csv', 'text_module_simple.csv'), 'text/csv')
post '/api/v1/text_modules/import?try=true', params: { file: csv_file }, headers: { 'Authorization' => credentials }
post '/api/v1/text_modules/import?try=true', params: { file: csv_file, col_sep: ';' }, headers: { 'Authorization' => credentials }
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(Hash, result.class)
@ -130,7 +130,7 @@ class TextModuleControllerTest < ActionDispatch::IntegrationTest
# valid file
csv_file = ::Rack::Test::UploadedFile.new(Rails.root.join('test', 'fixtures', 'csv', 'text_module_simple.csv'), 'text/csv')
post '/api/v1/text_modules/import', params: { file: csv_file }, headers: { 'Authorization' => credentials }
post '/api/v1/text_modules/import', params: { file: csv_file, col_sep: ';' }, headers: { 'Authorization' => credentials }
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(Hash, result.class)

View file

@ -979,7 +979,7 @@ class UserControllerTest < ActionDispatch::IntegrationTest
# invalid file
csv_file = ::Rack::Test::UploadedFile.new(Rails.root.join('test', 'fixtures', 'csv', 'user_simple_col_not_existing.csv'), 'text/csv')
post '/api/v1/users/import?try=true', params: { file: csv_file }, headers: { 'Authorization' => credentials }
post '/api/v1/users/import?try=true', params: { file: csv_file, col_sep: ';' }, headers: { 'Authorization' => credentials }
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(Hash, result.class)
@ -993,7 +993,7 @@ class UserControllerTest < ActionDispatch::IntegrationTest
# valid file try
csv_file = ::Rack::Test::UploadedFile.new(Rails.root.join('test', 'fixtures', 'csv', 'user_simple.csv'), 'text/csv')
post '/api/v1/users/import?try=true', params: { file: csv_file }, headers: { 'Authorization' => credentials }
post '/api/v1/users/import?try=true', params: { file: csv_file, col_sep: ';' }, headers: { 'Authorization' => credentials }
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(Hash, result.class)
@ -1007,7 +1007,7 @@ class UserControllerTest < ActionDispatch::IntegrationTest
# valid file
csv_file = ::Rack::Test::UploadedFile.new(Rails.root.join('test', 'fixtures', 'csv', 'user_simple.csv'), 'text/csv')
post '/api/v1/users/import', params: { file: csv_file }, headers: { 'Authorization' => credentials }
post '/api/v1/users/import', params: { file: csv_file, col_sep: ';' }, headers: { 'Authorization' => credentials }
assert_response(200)
result = JSON.parse(@response.body)
assert_equal(Hash, result.class)

View file

@ -18,6 +18,34 @@ class OrganizationCsvImportTest < ActiveSupport::TestCase
assert(header.include?('members'))
end
test 'empty payload' do
csv_string = ''
result = Organization.csv_import(
string: csv_string,
parse_params: {
col_sep: ';',
},
try: true,
)
assert_equal(true, result[:try])
assert_nil(result[:records])
assert_equal('failed', result[:result])
assert_equal('Unable to parse empty file/string for Organization.', result[:errors][0])
csv_string = 'id;name;shared;domain;domain_assignment;active;'
result = Organization.csv_import(
string: csv_string,
parse_params: {
col_sep: ';',
},
try: true,
)
assert_equal(true, result[:try])
assert(result[:records].blank?)
assert_equal('failed', result[:result])
assert_equal('No records found in file/string for Organization.', result[:errors][0])
end
test 'simple import' do
csv_string = "id;name;shared;domain;domain_assignment;active;note\n;org-simple-import1;true;org-simple-import1.example.com;false;true;some note1\n;org-simple-import2;true;org-simple-import2.example.com;false;false;some note2\n"

View file

@ -22,6 +22,34 @@ class TextModuleCsvImportTest < ActiveSupport::TestCase
assert_not(header.include?('customer'))
end
test 'empty payload' do
csv_string = ''
result = TextModule.csv_import(
string: csv_string,
parse_params: {
col_sep: ';',
},
try: true,
)
assert_equal(true, result[:try])
assert_nil(result[:records])
assert_equal('failed', result[:result])
assert_equal('Unable to parse empty file/string for TextModule.', result[:errors][0])
csv_string = 'name;keywords;content;note;active;'
result = TextModule.csv_import(
string: csv_string,
parse_params: {
col_sep: ';',
},
try: true,
)
assert_equal(true, result[:try])
assert(result[:records].blank?)
assert_equal('failed', result[:result])
assert_equal('No records found in file/string for TextModule.', result[:errors][0])
end
test 'simple import' do
csv_string = "name;keywords;content;note;active;\nsome name1;keyword1;\"some\ncontent1\";-;\nsome name2;keyword2;some content<br>test123\n"

View file

@ -22,6 +22,34 @@ class TicketCsvImportTest < ActiveSupport::TestCase
end
test 'empty payload' do
csv_string = ''
result = Ticket.csv_import(
string: csv_string,
parse_params: {
col_sep: ';',
},
try: true,
)
assert_equal(true, result[:try])
assert_nil(result[:records])
assert_equal('failed', result[:result])
assert_equal('Unable to parse empty file/string for Ticket.', result[:errors][0])
csv_string = 'id;number;title;state;priority;'
result = Ticket.csv_import(
string: csv_string,
parse_params: {
col_sep: ';',
},
try: true,
)
assert_equal(true, result[:try])
assert(result[:records].blank?)
assert_equal('failed', result[:result])
assert_equal('No records found in file/string for Ticket.', result[:errors][0])
end
test 'simple import' do
csv_string = "id;number;title;state;priority;owner;customer;group;note\n;123456;some title1;new;2 normal;-;nicole.braun@zammad.org;Users;some note1\n;123457;some title2;closed;1 low;admin@example.com;nicole.braun@zammad.org;Users;some note2\n"

View file

@ -17,6 +17,34 @@ class UserCsvImportTest < ActiveSupport::TestCase
assert(header.include?('organization'))
end
test 'empty payload' do
csv_string = ''
result = User.csv_import(
string: csv_string,
parse_params: {
col_sep: ';',
},
try: true,
)
assert_equal(true, result[:try])
assert_nil(result[:records])
assert_equal('failed', result[:result])
assert_equal('Unable to parse empty file/string for User.', result[:errors][0])
csv_string = "login;firstname;lastname;email;active;\n"
result = User.csv_import(
string: csv_string,
parse_params: {
col_sep: ';',
},
try: true,
)
assert_equal(true, result[:try])
assert(result[:records].blank?)
assert_equal('failed', result[:result])
assert_equal('No records found in file/string for User.', result[:errors][0])
end
test 'simple import' do
csv_string = "login;firstname;lastname;email;active;\nuser-simple-import1;firstname-simple-import1;lastname-simple-import1;user-simple-import1@example.com;true\nuser-simple-import2;firstname-simple-import2;lastname-simple-import2;user-simple-import2@example.com;false\n"