From cb3cd8ee117b38b30eaae965005f368a008b140e Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 20 Jun 2016 17:13:39 +0200 Subject: [PATCH 1/5] [ongoing] paginate statistics results --- .../controllers/admin/statistics.coffee | 81 ++++++++++++++----- .../templates/admin/statistics/index.html.erb | 3 + 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/app/assets/javascripts/controllers/admin/statistics.coffee b/app/assets/javascripts/controllers/admin/statistics.coffee index 797338e17..22ecc5d77 100644 --- a/app/assets/javascripts/controllers/admin/statistics.coffee +++ b/app/assets/javascripts/controllers/admin/statistics.coffee @@ -5,6 +5,16 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", + ### PRIVATE STATIC CONSTANTS ### + + ## search window size + RESULTS_PER_PAGE = 10 + + ## keep search context for (delay in minutes) ... + ES_SCROLL_TIME = 1 + + + ### PUBLIC SCOPE ### ## ui-view transitions optimization: if true, the stats will never be refreshed @@ -19,6 +29,15 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", ## statistics data recovered from elasticSearch $scope.data = null + ## when did the search was triggered + $scope.searchDate = null + + ## id of the elastic search context + $scope.scrollId = null + + ## total number of results for the current query + $scope.totalHits = null + ## configuration of the widget allowing to pick the ages range $scope.agePicker = show: false @@ -231,6 +250,22 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", return "ID "+id + $scope.showMoreResults = -> + if moment($scope.searchDate).add(ES_SCROLL_TIME, 'minutes').isBefore(moment()) + # elastic search context has expired, so we run again the whole query + refreshStats() + else + es.search + "size": RESULTS_PER_PAGE + "scroll": ES_SCROLL_TIME+'m' + "body": $scope.scrollId + , (error, response) -> + if (error) + console.error "Error: something unexpected occurred during elasticSearch scroll query: "+error + else + $scope.data = $scope.data.concat(response.hits.hits) + + ### PRIVATE SCOPE ### @@ -273,6 +308,8 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", $scope.sumCA = 0 $scope.averageAge = 0 $scope.sumStat = 0 + $scope.totalHits = null + $scope.searchDate = new Date() custom = null if $scope.customFilter.criterion and $scope.customFilter.criterion.key and $scope.customFilter.value custom = {} @@ -283,22 +320,12 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", if (err) console.error("[statisticsController::refreshStats] Unable to refresh due to "+err) else - $scope.data = res.hits - sumCA = 0 - sumAge = 0 - sumStat = 0 - if $scope.data.length > 0 - angular.forEach $scope.data, (datum) -> - if datum._source.ca - sumCA += parseInt(datum._source.ca) - if datum._source.age - sumAge += parseInt(datum._source.age) - if datum._source.stat - sumStat += parseInt(datum._source.stat) - sumAge /= $scope.data.length - $scope.sumCA = sumCA - $scope.averageAge = Math.round(sumAge*100)/100 - $scope.sumStat = sumStat + $scope.data = res.hits.hits + $scope.totalHits = res.hits.total + $scope.sumCA = res.aggregations.total_ca.value + $scope.averageAge = Math.round(res.aggregations.average_age.value * 100) / 100 + $scope.sumStat = res.aggregations.total_stat.value + $scope.scrollId = res._scroll_id @@ -308,7 +335,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", # @param type {String} statistics type (month|year|booking|hour|user|project) # @param custom {{key:{string}, value:{string}}|null} custom filter property or null to disable this filter # @param callback {function} function be to run after results were retrieved, it will receive - # two parameters : results {Array}, error {String} (if any) + # two parameters : results {Object}, error {String} (if any) ## queryElasticStats = (index, type, custom, callback) -> # handle invalid callback @@ -320,13 +347,14 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", es.search "index": "stats" "type": index - "size": 1000000000 + "size": RESULTS_PER_PAGE + "scroll": ES_SCROLL_TIME+'m' "body": buildElasticDataQuery(type, custom, $scope.agePicker.start, $scope.agePicker.end, moment($scope.datePickerStart.selected), moment($scope.datePickerEnd.selected), $scope.sorting) , (error, response) -> if (error) - callback([], "Error: something unexpected occurred during elasticSearch query: "+error) + callback({}, "Error: something unexpected occurred during elasticSearch query: "+error) else - callback(response.hits) + callback(response) @@ -392,6 +420,19 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", if sortings q["sort"] = buildElasticSortCriteria(sortings) + + # aggregations (avg age & CA sum) + q["aggs"] = { + "total_ca": + "sum": + "field": "ca" + "average_age": + "avg": + "field": "age" + "total_stat": + "sum": + "field": "sta" + } q diff --git a/app/assets/templates/admin/statistics/index.html.erb b/app/assets/templates/admin/statistics/index.html.erb index 82743039a..552740864 100644 --- a/app/assets/templates/admin/statistics/index.html.erb +++ b/app/assets/templates/admin/statistics/index.html.erb @@ -278,6 +278,9 @@ +
+ +
From 4639a15e2f2f7c8afb836d649c54ebbe5006726d Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 21 Jun 2016 13:16:42 +0200 Subject: [PATCH 2/5] [feature] paginate statistics --- .../controllers/admin/statistics.coffee | 17 +++++++++++++---- .../templates/admin/statistics/index.html.erb | 4 ++-- app/controllers/api/statistics_controller.rb | 8 ++++++++ app/policies/statistic_policy.rb | 2 +- config/locales/app.admin.en.yml | 1 + config/locales/app.admin.fr.yml | 1 + config/routes.rb | 1 + 7 files changed, 27 insertions(+), 7 deletions(-) diff --git a/app/assets/javascripts/controllers/admin/statistics.coffee b/app/assets/javascripts/controllers/admin/statistics.coffee index 22ecc5d77..a60896555 100644 --- a/app/assets/javascripts/controllers/admin/statistics.coffee +++ b/app/assets/javascripts/controllers/admin/statistics.coffee @@ -8,7 +8,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", ### PRIVATE STATIC CONSTANTS ### ## search window size - RESULTS_PER_PAGE = 10 + RESULTS_PER_PAGE = 20 ## keep search context for (delay in minutes) ... ES_SCROLL_TIME = 1 @@ -250,19 +250,28 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", return "ID "+id + + ## + # Run a scroll query to elasticsearch to append the next packet of results to those displayed. + # If the ES search context has expired when the user ask for more results, we re-run the whole query. + ## $scope.showMoreResults = -> + # if all results were retrieved, do nothing + if $scope.data.length >= $scope.totalHits + return + if moment($scope.searchDate).add(ES_SCROLL_TIME, 'minutes').isBefore(moment()) # elastic search context has expired, so we run again the whole query refreshStats() else - es.search - "size": RESULTS_PER_PAGE + es.scroll "scroll": ES_SCROLL_TIME+'m' - "body": $scope.scrollId + "body": {scrollId: $scope.scrollId} , (error, response) -> if (error) console.error "Error: something unexpected occurred during elasticSearch scroll query: "+error else + $scope.scrollId = response._scroll_id $scope.data = $scope.data.concat(response.hits.hits) diff --git a/app/assets/templates/admin/statistics/index.html.erb b/app/assets/templates/admin/statistics/index.html.erb index 552740864..ec1b79036 100644 --- a/app/assets/templates/admin/statistics/index.html.erb +++ b/app/assets/templates/admin/statistics/index.html.erb @@ -229,7 +229,7 @@
    -
  • {{ 'entries' | translate }} {{data.length}}
    • +
  • {{ 'entries' | translate }} {{totalHits}}
  • {{ 'revenue_' | translate }} {{sumCA | currency}}
  • {{ 'average_age' | translate }} {{averageAge}} {{ 'years_old' | translate }}
  • {{ 'total' | translate }} {{type.active.label}} : {{sumStat}}
    • @@ -279,7 +279,7 @@
    - +
    diff --git a/app/controllers/api/statistics_controller.rb b/app/controllers/api/statistics_controller.rb index 56523fe45..75f92701b 100644 --- a/app/controllers/api/statistics_controller.rb +++ b/app/controllers/api/statistics_controller.rb @@ -16,4 +16,12 @@ class API::StatisticsController < API::ApiController end } end + + def scroll + authorize :statistic, :scroll? + + results = Elasticsearch::Client.new.scroll scroll: params[:scroll], scroll_id: params[:scrollId] + render json: results + end + end diff --git a/app/policies/statistic_policy.rb b/app/policies/statistic_policy.rb index 384e3781b..faec07341 100644 --- a/app/policies/statistic_policy.rb +++ b/app/policies/statistic_policy.rb @@ -1,5 +1,5 @@ class StatisticPolicy < ApplicationPolicy - ['index', 'account', 'event', 'machine', 'project', 'subscription', 'training', 'user'].each do |action| + %w(index account event machine project subscription training user scroll).each do |action| define_method "#{action}?" do user.is_admin? end diff --git a/config/locales/app.admin.en.yml b/config/locales/app.admin.en.yml index 77c179066..f5cb4f038 100644 --- a/config/locales/app.admin.en.yml +++ b/config/locales/app.admin.en.yml @@ -388,6 +388,7 @@ en: revenue: "Revenue" unknown: "Unknown" user_id: "User ID" + display_more_results: "Display more results" stats_graphs: diff --git a/config/locales/app.admin.fr.yml b/config/locales/app.admin.fr.yml index 3d3bb1a9a..f20777697 100644 --- a/config/locales/app.admin.fr.yml +++ b/config/locales/app.admin.fr.yml @@ -388,6 +388,7 @@ fr: revenue: "Chiffre d'affaires" unknown: "Inconnu" user_id: "ID Utilisateur" + display_more_results: "Afficher plus de résultats" stats_graphs: diff --git a/config/routes.rb b/config/routes.rb index 1901df414..d7e6b427e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -104,6 +104,7 @@ Rails.application.routes.draw do %w(account event machine project subscription training user).each do |path| post "/stats/#{path}/_search", to: "api/statistics##{path}" end + post '_search/scroll', to: "api/statistics#scroll" match '/project_collaborator/:valid_token', to: 'api/projects#collaborator_valid', via: :get From 24f963645b2d7be504ce4861a6d32db24f6b4205 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 21 Jun 2016 14:39:44 +0200 Subject: [PATCH 3/5] optimize usernames mapping in statistics --- .../controllers/admin/statistics.coffee | 25 ++++++------------- app/assets/javascripts/router.coffee.erb | 6 +++++ app/assets/javascripts/services/member.coffee | 3 +++ app/controllers/api/members_controller.rb | 6 +++++ app/policies/user_policy.rb | 11 +++----- app/views/api/members/mapping.json.jbuilder | 3 +++ config/routes.rb | 1 + 7 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 app/views/api/members/mapping.json.jbuilder diff --git a/app/assets/javascripts/controllers/admin/statistics.coffee b/app/assets/javascripts/controllers/admin/statistics.coffee index a60896555..a719981f2 100644 --- a/app/assets/javascripts/controllers/admin/statistics.coffee +++ b/app/assets/javascripts/controllers/admin/statistics.coffee @@ -1,7 +1,7 @@ 'use strict' -Application.Controllers.controller "StatisticsController", ["$scope", "$state", "$rootScope", "Statistics", "es", "Member", '_t' -, ($scope, $state, $rootScope, Statistics, es, Member, _t) -> +Application.Controllers.controller "StatisticsController", ["$scope", "$state", "$rootScope", "Statistics", "es", "Member", '_t', 'membersPromise', 'statisticsPromise' +, ($scope, $state, $rootScope, Statistics, es, Member, _t, membersPromise, statisticsPromise) -> @@ -21,10 +21,10 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", $scope.preventRefresh = false ## statistics structure in elasticSearch - $scope.statistics = [] + $scope.statistics = statisticsPromise ## fablab users list - $scope.members = [] + $scope.members = membersPromise ## statistics data recovered from elasticSearch $scope.data = null @@ -241,13 +241,10 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", # @param id {number} user ID ## $scope.getUserNameFromId = (id) -> - if $scope.members.length == 0 - return "ID "+id - else - for member in $scope.members - if member.id == id - return member.name - return "ID "+id + name = $scope.members[id] + if name + return name + else "ID "+id @@ -282,12 +279,6 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", # Kind of constructor: these actions will be realized first when the controller is loaded ## initialize = -> - Statistics.query (stats) -> - $scope.statistics = stats - - Member.query (members) -> - $scope.members = members - # workaround for angular-bootstrap::tabs behavior: on tab deletion, another tab will be selected # which will cause every tabs to reload, one by one, when the view is closed $rootScope.$on '$stateChangeStart', (event, toState, toParams, fromState, fromParams) -> diff --git a/app/assets/javascripts/router.coffee.erb b/app/assets/javascripts/router.coffee.erb index 7f1416510..8f023ba53 100644 --- a/app/assets/javascripts/router.coffee.erb +++ b/app/assets/javascripts/router.coffee.erb @@ -808,6 +808,12 @@ angular.module('application.router', ['ui.router']). templateUrl: '<%= asset_path "admin/statistics/index.html" %>' controller: 'StatisticsController' resolve: + membersPromise: ['Member', (Member) -> + Member.mapping().$promise + ] + statisticsPromise: ['Statistics', (Statistics)-> + Statistics.query().$promise + ] translations: [ 'Translations', (Translations) -> Translations.query('app.admin.statistics').$promise ] diff --git a/app/assets/javascripts/services/member.coffee b/app/assets/javascripts/services/member.coffee index 33779d63f..226fe730e 100644 --- a/app/assets/javascripts/services/member.coffee +++ b/app/assets/javascripts/services/member.coffee @@ -22,4 +22,7 @@ Application.Services.factory 'Member', ["$resource", ($resource)-> url: '/api/members/search/:query' params: {query: "@query"} isArray: true + mapping: + method: 'GET' + url: '/api/members/mapping' ] diff --git a/app/controllers/api/members_controller.rb b/app/controllers/api/members_controller.rb index 8950d019f..4d3c99a6d 100644 --- a/app/controllers/api/members_controller.rb +++ b/app/controllers/api/members_controller.rb @@ -204,6 +204,12 @@ class API::MembersController < API::ApiController @members end + def mapping + authorize User + + @members = User.includes(:profile) + end + private def set_member @member = User.find(params[:id]) diff --git a/app/policies/user_policy.rb b/app/policies/user_policy.rb index f56c3de95..2d8add1aa 100644 --- a/app/policies/user_policy.rb +++ b/app/policies/user_policy.rb @@ -13,10 +13,6 @@ class UserPolicy < ApplicationPolicy user.is_admin? or (record.is_allow_contact and record.is_member?) or (user.id == record.id) end - def create? - user.is_admin? - end - def update? user.is_admin? or (user.id == record.id) end @@ -29,8 +25,9 @@ class UserPolicy < ApplicationPolicy user.id == record.id end - def list? - user.is_admin? + %w(list create mapping).each do |action| + define_method "#{action}?" do + user.is_admin? + end end - end diff --git a/app/views/api/members/mapping.json.jbuilder b/app/views/api/members/mapping.json.jbuilder new file mode 100644 index 000000000..ffbc0da97 --- /dev/null +++ b/app/views/api/members/mapping.json.jbuilder @@ -0,0 +1,3 @@ +@members.each do |member| + json.set! member.id, member.profile.full_name +end \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index d7e6b427e..2d1803c84 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -42,6 +42,7 @@ Rails.application.routes.draw do put ':id/merge', action: 'merge', on: :collection post 'list', action: 'list', on: :collection get 'search/:query', action: 'search', on: :collection + get 'mapping', action: 'mapping', on: :collection end resources :reservations, only: [:show, :create, :index, :update] resources :notifications, only: [:index, :show, :update] do From 3e24d328a9201ebb3aee06ebf1b9d6a3e9808b10 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 21 Jun 2016 14:45:52 +0200 Subject: [PATCH 4/5] [feature] statistics: username link leads to admin edition --- app/assets/javascripts/controllers/admin/statistics.coffee | 4 +--- app/assets/templates/admin/statistics/index.html.erb | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/assets/javascripts/controllers/admin/statistics.coffee b/app/assets/javascripts/controllers/admin/statistics.coffee index a719981f2..454b9961f 100644 --- a/app/assets/javascripts/controllers/admin/statistics.coffee +++ b/app/assets/javascripts/controllers/admin/statistics.coffee @@ -242,9 +242,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", ## $scope.getUserNameFromId = (id) -> name = $scope.members[id] - if name - return name - else "ID "+id + return (if name then name else "ID "+id) diff --git a/app/assets/templates/admin/statistics/index.html.erb b/app/assets/templates/admin/statistics/index.html.erb index ec1b79036..9ffe47fba 100644 --- a/app/assets/templates/admin/statistics/index.html.erb +++ b/app/assets/templates/admin/statistics/index.html.erb @@ -260,7 +260,7 @@ {{formatDate(datum._source.date)}} - {{getUserNameFromId(datum._source.userId)}} + {{getUserNameFromId(datum._source.userId)}} {{formatSex(datum._source.gender)}} {{datum._source.age}} {{ 'years_old' | translate }}{{ 'unknown' }} {{formatSubtype(datum._source.subType)}} From 3138097be228b2bc78d455d08cbd4fcf97fce764 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 21 Jun 2016 15:24:25 +0200 Subject: [PATCH 5/5] updated chagelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbfebd902..2d8191ed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog Fab Manager +## Next release +- Fix a bug: field User.merged_at should not be allowed to be mapped in SSO +- Fix a bug: integration test "user reservation without plan" +- Fix a bug: can't click for some seconds in Chrome 51 +- Admin: statistics tables were paginated and optimized to improve load times. + ## v2.2.0 2016 June 16 - Built-in support for extensions plug-ins - User profile form: social networks links, personal website link, job and change profile visibility (public / private)