Fixes #3068 - KB search does not allow pagination on endpoint

This commit is contained in:
Mantas Masalskis 2021-08-17 07:44:54 +02:00
parent 2fcf42dc0a
commit 310846b8b4
14 changed files with 327 additions and 113 deletions

View file

@ -0,0 +1,16 @@
# Copyright (C) 2012-2021 Zammad Foundation, http://zammad-foundation.org/
module ApplicationController::Paginates
extend ActiveSupport::Concern
def paginate_with(max: nil, default: nil)
@paginate_max = max
@paginate_default = default
end
private
def pagination
@pagination ||= ::ApplicationController::Paginates::Pagination.new(params, max: @paginate_max, default: @paginate_default)
end
end

View file

@ -0,0 +1,24 @@
# Copyright (C) 2012-2021 Zammad Foundation, http://zammad-foundation.org/
class ApplicationController::Paginates::Pagination
def initialize(params, default: nil, max: nil)
@params = params
@default = default.presence || 100
@max = max.presence || 500
end
def limit
limit = @params[:per_page] || @params[:limit] || @default
[limit.to_i, @max].min
end
def page
@params[:page]&.to_i || 1
end
def offset
(page - 1) * limit
end
end

View file

@ -3,6 +3,8 @@
module ApplicationController::RendersModels module ApplicationController::RendersModels
extend ActiveSupport::Concern extend ActiveSupport::Concern
include ApplicationController::Paginates
private private
# model helper # model helper
@ -103,16 +105,14 @@ module ApplicationController::RendersModels
end end
def model_index_render(object, params) def model_index_render(object, params)
page = (params[:page] || 1).to_i paginate_with(default: 500)
per_page = (params[:per_page] || 500).to_i
offset = (page - 1) * per_page
sql_helper = ::SqlHelper.new(object: object) sql_helper = ::SqlHelper.new(object: object)
sort_by = sql_helper.get_sort_by(params, 'id') sort_by = sql_helper.get_sort_by(params, 'id')
order_by = sql_helper.get_order_by(params, 'ASC') order_by = sql_helper.get_order_by(params, 'ASC')
order_sql = sql_helper.get_order(sort_by, order_by) order_sql = sql_helper.get_order(sort_by, order_by)
generic_objects = object.order(Arel.sql(order_sql)).offset(offset).limit(per_page) generic_objects = object.order(Arel.sql(order_sql)).offset(pagination.offset).limit(pagination.limit)
if response_expand? if response_expand?
list = [] list = []

View file

@ -6,9 +6,10 @@ class KnowledgeBase::SearchController < ApplicationController
include KnowledgeBaseHelper include KnowledgeBaseHelper
include ActionView::Helpers::SanitizeHelper include ActionView::Helpers::SanitizeHelper
include ApplicationController::Paginates
# POST /api/v1/knowledge_bases/search # POST /api/v1/knowledge_bases/search
# knowledge_base_id, locale, flavor, index, limit, include_locale # knowledge_base_id, locale, flavor, index, page, per_page, limit, include_locale
def search def search
knowledge_base = KnowledgeBase knowledge_base = KnowledgeBase
.active .active
@ -35,7 +36,7 @@ class KnowledgeBase::SearchController < ApplicationController
include_locale = params[:include_locale] && KnowledgeBase.with_multiple_locales_exists? include_locale = params[:include_locale] && KnowledgeBase.with_multiple_locales_exists?
result = search_backend.search params[:query], user: current_user result = search_backend.search params[:query], user: current_user, pagination: pagination
if (exclude_ids = params[:exclude_ids]&.map(&:to_i)) if (exclude_ids = params[:exclude_ids]&.map(&:to_i))
result.reject! { |meta| meta[:type] == params[:index] && exclude_ids.include?(meta[:id]) } result.reject! { |meta| meta[:type] == params[:index] && exclude_ids.include?(meta[:id]) }

View file

@ -4,6 +4,8 @@ class OrganizationsController < ApplicationController
prepend_before_action -> { authorize! }, except: %i[index show] prepend_before_action -> { authorize! }, except: %i[index show]
prepend_before_action { authentication_check } prepend_before_action { authentication_check }
include ApplicationController::Paginates
=begin =begin
Format: Format:
@ -176,23 +178,14 @@ curl http://localhost/api/v1/organization/{id} -v -u #{login}:#{password} -H "Co
# GET /api/v1/organizations/search # GET /api/v1/organizations/search
def search def search
per_page = params[:per_page] || params[:limit] || 100
per_page = per_page.to_i
if per_page > 500
per_page = 500
end
page = params[:page] || 1
page = page.to_i
offset = (page - 1) * per_page
query = params[:query] query = params[:query]
if query.respond_to?(:permit!) if query.respond_to?(:permit!)
query = query.permit!.to_h query = query.permit!.to_h
end end
query_params = { query_params = {
query: query, query: query,
limit: per_page, limit: pagination.limit,
offset: offset, offset: pagination.offset,
sort_by: params[:sort_by], sort_by: params[:sort_by],
order_by: params[:order_by], order_by: params[:order_by],
current_user: current_user, current_user: current_user,

View file

@ -5,28 +5,19 @@ class TicketsController < ApplicationController
include ClonesTicketArticleAttachments include ClonesTicketArticleAttachments
include ChecksUserAttributesByCurrentUserPermission include ChecksUserAttributesByCurrentUserPermission
include TicketStats include TicketStats
include ApplicationController::Paginates
prepend_before_action -> { authorize! }, only: %i[create selector import_example import_start ticket_customer ticket_history ticket_related ticket_recent ticket_merge ticket_split] prepend_before_action -> { authorize! }, only: %i[create selector import_example import_start ticket_customer ticket_history ticket_related ticket_recent ticket_merge ticket_split]
prepend_before_action :authentication_check prepend_before_action :authentication_check
# GET /api/v1/tickets # GET /api/v1/tickets
def index def index
offset = 0 paginate_with(max: 100)
per_page = 100
if params[:page] && params[:per_page]
offset = (params[:page].to_i - 1) * params[:per_page].to_i
per_page = params[:per_page].to_i
end
if per_page > 100
per_page = 100
end
tickets = TicketPolicy::ReadScope.new(current_user).resolve tickets = TicketPolicy::ReadScope.new(current_user).resolve
.order(id: :asc) .order(id: :asc)
.offset(offset) .offset(pagination.offset)
.limit(per_page) .limit(pagination.limit)
if response_expand? if response_expand?
list = [] list = []
@ -448,14 +439,7 @@ class TicketsController < ApplicationController
params.require(:condition).permit! params.require(:condition).permit!
end end
per_page = params[:per_page] || params[:limit] || 50 paginate_with(max: 200, default: 50)
per_page = per_page.to_i
if per_page > 200
per_page = 200
end
page = params[:page] || 1
page = page.to_i
offset = (page - 1) * per_page
query = params[:query] query = params[:query]
if query.respond_to?(:permit!) if query.respond_to?(:permit!)
@ -466,8 +450,8 @@ class TicketsController < ApplicationController
tickets = Ticket.search( tickets = Ticket.search(
query: query, query: query,
condition: params[:condition].to_h, condition: params[:condition].to_h,
limit: per_page, limit: pagination.limit,
offset: offset, offset: pagination.offset,
order_by: params[:order_by], order_by: params[:order_by],
sort_by: params[:sort_by], sort_by: params[:sort_by],
current_user: current_user, current_user: current_user,

View file

@ -2,6 +2,7 @@
class UsersController < ApplicationController class UsersController < ApplicationController
include ChecksUserAttributesByCurrentUserPermission include ChecksUserAttributesByCurrentUserPermission
include ApplicationController::Paginates
prepend_before_action -> { authorize! }, only: %i[import_example import_start search history unlock] prepend_before_action -> { authorize! }, only: %i[import_example import_start search history unlock]
prepend_before_action :authentication_check, except: %i[create password_reset_send password_reset_verify image email_verify email_verify_send] prepend_before_action :authentication_check, except: %i[create password_reset_send password_reset_verify image email_verify email_verify_send]
@ -17,18 +18,7 @@ class UsersController < ApplicationController
# @response_message 200 [Array<User>] List of matching User records. # @response_message 200 [Array<User>] List of matching User records.
# @response_message 403 Forbidden / Invalid session. # @response_message 403 Forbidden / Invalid session.
def index def index
offset = 0 users = policy_scope(User).order(id: :asc).offset(pagination.offset).limit(pagination.limit)
per_page = 500
if params[:page] && params[:per_page]
offset = (params[:page].to_i - 1) * params[:per_page].to_i
per_page = params[:per_page].to_i
end
if per_page > 500
per_page = 500
end
users = policy_scope(User).order(id: :asc).offset(offset).limit(per_page)
if response_expand? if response_expand?
list = [] list = []
@ -229,15 +219,6 @@ class UsersController < ApplicationController
# @response_message 200 [Array<User>] A list of User records matching the search term. # @response_message 200 [Array<User>] A list of User records matching the search term.
# @response_message 403 Forbidden / Invalid session. # @response_message 403 Forbidden / Invalid session.
def search def search
per_page = params[:per_page] || params[:limit] || 100
per_page = per_page.to_i
if per_page > 500
per_page = 500
end
page = params[:page] || 1
page = page.to_i
offset = (page - 1) * per_page
query = params[:query] query = params[:query]
if query.respond_to?(:permit!) if query.respond_to?(:permit!)
query.permit!.to_h query.permit!.to_h
@ -250,8 +231,8 @@ class UsersController < ApplicationController
query_params = { query_params = {
query: query, query: query,
limit: per_page, limit: pagination.limit,
offset: offset, offset: pagination.offset,
sort_by: params[:sort_by], sort_by: params[:sort_by],
order_by: params[:order_by], order_by: params[:order_by],
current_user: current_user, current_user: current_user,

View file

@ -337,22 +337,24 @@ remove whole data from index
end end
def self.search_by_index_sort(sort_by = nil, order_by = nil) def self.search_by_index_sort(sort_by = nil, order_by = nil)
result = [] result = (sort_by || [])
.map(&:to_s)
.each_with_object([])
.each_with_index do |(elem, memo), index|
next if elem.blank?
next if order_by&.at(index).blank?
sort_by&.each_with_index do |value, index| # for sorting values use .keyword values (no analyzer is used - plain values)
next if value.blank? if elem !~ %r{\.} && elem !~ %r{_(time|date|till|id|ids|at)$} && elem != 'id'
next if order_by&.at(index).blank? elem += '.keyword'
end
# for sorting values use .keyword values (no analyzer is used - plain values) memo.push(
if value !~ %r{\.} && value !~ %r{_(time|date|till|id|ids|at)$} elem => {
value += '.keyword' order: order_by[index],
},
)
end end
result.push(
value => {
order: order_by[index],
},
)
end
if result.blank? if result.blank?
result.push( result.push(

View file

@ -9,7 +9,9 @@ class SearchKnowledgeBaseBackend
# @option params [KnowledgeBase::Category] :scope (nil) optional search scope # @option params [KnowledgeBase::Category] :scope (nil) optional search scope
# @option params [Symbol] :flavor (:public) agent or public to indicate source and narrow down to internal or public answers accordingly # @option params [Symbol] :flavor (:public) agent or public to indicate source and narrow down to internal or public answers accordingly
# @option params [String, Array<String>] :index (nil) indexes to limit search to, searches all indexes if nil # @option params [String, Array<String>] :index (nil) indexes to limit search to, searches all indexes if nil
# @option params [Integer] :limit per page param for paginatin
# @option params [Boolean] :highlight_enabled (true) highlight matching text # @option params [Boolean] :highlight_enabled (true) highlight matching text
# @option params [Hash<String=>String>, Hash<Symbol=>Symbol>] :order_by hash with column => asc/desc
def initialize(params) def initialize(params)
@params = params.compact @params = params.compact
@ -17,23 +19,18 @@ class SearchKnowledgeBaseBackend
prepare_scope_ids prepare_scope_ids
end end
def search(query, user: nil) def search(query, user: nil, pagination: nil)
raw_results = if SearchIndexBackend.enabled? raw_results = raw_results(query, user, pagination: pagination)
SearchIndexBackend
.search(query, indexes, options)
.map do |hash|
hash[:id] = hash[:id].to_i
hash
end
else
search_fallback(query, indexes, { user: user })
end
if (limit = @params.fetch(:limit, nil)) filtered = filter_results raw_results, user
raw_results = raw_results[0, limit]
if pagination
filtered = filtered.slice pagination.offset, pagination.limit
elsif @params[:limit]
filtered = filtered.slice 0, @params[:limit]
end end
filter_results raw_results, user filtered
end end
def search_fallback(query, indexes, options) def search_fallback(query, indexes, options)
@ -47,10 +44,26 @@ class SearchKnowledgeBaseBackend
.constantize .constantize
.search_fallback("%#{query}%", @cached_scope_ids, options: options) .search_fallback("%#{query}%", @cached_scope_ids, options: options)
.where(kb_locale: kb_locales) .where(kb_locale: kb_locales)
.order(**search_fallback_order)
.pluck(:id) .pluck(:id)
.map { |id| { id: id, type: index } } .map { |id| { id: id, type: index } }
end end
def search_fallback_order
@params[:order_by].presence || { updated_at: :desc }
end
def raw_results(query, user, pagination: nil)
return search_fallback(query, indexes, { user: user }) if !SearchIndexBackend.enabled?
SearchIndexBackend
.search(query, indexes, options(pagination: pagination))
.map do |hash|
hash[:id] = hash[:id].to_i
hash
end
end
def filter_results(raw_results, user) def filter_results(raw_results, user)
raw_results raw_results
.group_by { |result| result[:type] } .group_by { |result| result[:type] }
@ -162,28 +175,56 @@ class SearchKnowledgeBaseBackend
@params.fetch(:flavor, :public).to_sym @params.fetch(:flavor, :public).to_sym
end end
def options def base_options
output = { {
query_extension: { query_extension: {
bool: { bool: {
must: [ { terms: { kb_locale_id: kb_locale_ids } } ] must: [ { terms: { kb_locale_id: kb_locale_ids } } ]
} }
} }
} }
end
if @params.fetch(:highlight_enabled, true) def options_apply_highlight(hash)
output[:highlight_fields_by_indexes] = { return if !@params.fetch(:highlight_enabled, true)
'KnowledgeBase::Answer::Translation': %w[title content.body attachment.content tags],
'KnowledgeBase::Category::Translation': %w[title], hash[:highlight_fields_by_indexes] = {
'KnowledgeBase::Translation': %w[title] 'KnowledgeBase::Answer::Translation': %w[title content.body attachment.content tags],
} 'KnowledgeBase::Category::Translation': %w[title],
'KnowledgeBase::Translation': %w[title]
}
end
def options_apply_scope(hash)
return if !@params.fetch(:scope, nil)
hash[:query_extension][:bool][:must].push({ terms: { scope_id: @cached_scope_ids } })
end
def options_apply_pagination(hash, pagination)
if @params[:from] && @params[:limit]
hash[:from] = @params[:from]
hash[:limit] = @params[:limit]
elsif pagination
hash[:from] = 0
hash[:limit] = pagination.limit * 99
end end
end
if @params.fetch(:scope, nil) def options_apply_order(hash)
scope = { terms: { scope_id: @cached_scope_ids } } return if @params[:order_by].blank?
output[:query_extension][:bool][:must].push scope hash[:sort_by] = @params[:order_by].keys
end hash[:order_by] = @params[:order_by].values
end
def options(pagination: nil)
output = base_options
options_apply_highlight(output)
options_apply_scope(output)
options_apply_pagination(output, pagination)
options_apply_order(output)
output output
end end

View file

@ -0,0 +1,81 @@
# Copyright (C) 2012-2021 Zammad Foundation, http://zammad-foundation.org/
require 'rails_helper'
RSpec.describe ApplicationController::Paginates::Pagination do
describe '#limit' do
it 'returns as set in params' do
instance = described_class.new({ per_page: 123 })
expect(instance.limit).to be 123
end
it 'ensures that per_page is an integer' do
instance = described_class.new({ per_page: '123' })
expect(instance.limit).to be 123
end
it 'when missing, returns as set in limit attribute' do
instance = described_class.new({ limit: 123 })
expect(instance.limit).to be 123
end
it 'falls back to default' do
instance = described_class.new({})
expect(instance.limit).to be 100
end
it 'falls back to custom default' do
instance = described_class.new({}, default: 222)
expect(instance.limit).to be 222
end
it 'per_page attribute preferred over limit' do
instance = described_class.new({ per_page: 123, limit: 321 })
expect(instance.limit).to be 123
end
it 'capped by limit' do
instance = described_class.new({ per_page: 9999 })
expect(instance.limit).to be 500
end
it 'capped by custom default' do
instance = described_class.new({ per_page: 9999 }, max: 10)
expect(instance.limit).to be 10
end
end
describe '#page' do
it 'returns page number' do
instance = described_class.new({ page: 123 })
expect(instance.page).to be 123
end
it 'defaults to 1 when missing' do
instance = described_class.new({})
expect(instance.page).to be 1
end
it 'ensures that page is an integer' do
instance = described_class.new({ page: '123' })
expect(instance.page).to be 123
end
end
describe '#offset' do
it 'returns 0 when no page given' do
instance = described_class.new({})
expect(instance.offset).to be 0
end
it 'returns offset for page' do
instance = described_class.new({ page: 3 })
expect(instance.offset).to be 200
end
it 'returns offset based on custom per_page value' do
instance = described_class.new({ page: 3, per_page: 15 })
expect(instance.offset).to be 30
end
end
end

View file

@ -0,0 +1,12 @@
# Copyright (C) 2012-2021 Zammad Foundation, http://zammad-foundation.org/
FactoryBot.define do
factory :'application_controller/paginates/pagination', aliases: %i[pagination] do
params { {} }
default { nil }
max { nil }
initialize_with { new(params, default: default, max: max) }
end
end

View file

@ -42,6 +42,67 @@ RSpec.describe SearchKnowledgeBaseBackend do
end end
end end
context 'with paging' do
let(:answers) do
Array.new(20) do |nth|
create(:knowledge_base_answer, :published, :with_attachment, category: category, translation_attributes: { title: "#{search_phrase} #{nth}" })
end
end
let(:search_phrase) { 'paging test' }
let(:options) do
{
knowledge_base: knowledge_base,
locale: primary_locale,
scope: nil,
order_by: { id: :desc }
}
end
shared_examples 'verify paging' do |elasticsearch:|
context "when elastic search is #{elasticsearch}", searchindex: elasticsearch do
before do
answers
configure_elasticsearch(required: true, rebuild: true) if elasticsearch
end
it 'first page is first 5 answers' do
results = instance.search(search_phrase, user: user, pagination: build(:pagination, params: { page: 1, per_page: 5 }))
first_5 = answers.reverse.slice(0, 5)
expect(results.pluck(:id)).to eq first_5.map { |answer| answer.translations.first.id }
end
it 'second page is next 5 answers' do
results = instance.search(search_phrase, user: user, pagination: build(:pagination, params: { page: 2, per_page: 5 }))
next_5 = answers.reverse.slice(5, 5)
expect(results.pluck(:id)).to eq next_5.map { |answer| answer.translations.first.id }
end
it 'last page may be partial' do
results = instance.search(search_phrase, user: user, pagination: build(:pagination, params: { page: 4, per_page: 6 }))
last_page = answers.reverse.slice(18, 6)
expect(results.pluck(:id)).to eq last_page.map { |answer| answer.translations.first.id }
end
it '5th page is empty' do
results = instance.search(search_phrase, user: user, pagination: build(:pagination, params: { page: 5, per_page: 5 }))
expect(results).to be_blank
end
end
end
include_examples 'verify paging', elasticsearch: true
include_examples 'verify paging', elasticsearch: false
end
context 'with successful API response' do context 'with successful API response' do
shared_examples 'verify response' do |elasticsearch:| shared_examples 'verify response' do |elasticsearch:|
it "ID is an Integer when ES=#{elasticsearch}", searchindex: elasticsearch do it "ID is an Integer when ES=#{elasticsearch}", searchindex: elasticsearch do

View file

@ -87,4 +87,32 @@ RSpec.describe 'Knowledge Base search with details', type: :request, searchindex
expect(json_response['details'][0]['subtitle']).to eq("#{category4.translations.first.title} > #{category5.translations.first.title}") expect(json_response['details'][0]['subtitle']).to eq("#{category4.translations.first.title} > #{category5.translations.first.title}")
end end
end end
context 'when using paging' do
let(:answers) do
Array.new(20) do |nth|
create(:knowledge_base_answer, :published, :with_attachment, category: category, translation_attributes: { title: "#{search_phrase} #{nth}" })
end
end
let(:search_phrase) { 'paging test' }
before do
configure_elasticsearch(required: true, rebuild: true) do
answers
end
end
it 'returns success' do
post endpoint, params: { query: search_phrase, per_page: 10, page: 0 }
expect(response).to have_http_status(:ok)
end
it 'returns defined amount of items' do
post endpoint, params: { query: search_phrase, per_page: 7, page: 0 }
expect(json_response['result'].count).to be 7
end
end
end end

View file

@ -42,17 +42,7 @@ RSpec.describe 'Organization', type: :request, searchindex: true do
end end
before do before do
configure_elasticsearch do configure_elasticsearch rebuild: true, required: true
travel 1.minute
rebuild_searchindex
# execute background jobs
Scheduler.worker(true)
sleep 6
end
end end
describe 'request handling' do describe 'request handling' do