From 9bfaaaccd963d2c1d639d1e256d1e5f1092312b2 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 19 Dec 2016 17:08:11 +0100 Subject: [PATCH 01/14] improved numerous notifications display --- CHANGELOG.md | 4 ++++ app/assets/javascripts/controllers/application.coffee.erb | 7 ++++++- app/controllers/api/notifications_controller.rb | 6 +++++- config/locales/app.public.en.yml | 3 +++ config/locales/app.public.fr.yml | 4 ++++ 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81f6f2238..6b7279958 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog Fab Manager +## next release +- Mask new notifications alerts when more than 3 +- Fix a bug: display more than 15 unread notifications (number on the bell icon & full list) + ## v2.4.8 2016 December 15 - Added asterisks on mandatory fields in member's form diff --git a/app/assets/javascripts/controllers/application.coffee.erb b/app/assets/javascripts/controllers/application.coffee.erb index 51c42a6a3..645b6e6cb 100644 --- a/app/assets/javascripts/controllers/application.coffee.erb +++ b/app/assets/javascripts/controllers/application.coffee.erb @@ -260,7 +260,9 @@ Application.Controllers.controller 'ApplicationController', ["$rootScope", "$sco getNotifications = -> $rootScope.toCheckNotifications = true unless $rootScope.checkNotificationsIsInit or !$rootScope.currentUser - $scope.notifications = Notification.query {is_read: false} + setTimeout -> + $scope.notifications = Notification.query {is_read: false} + , 2000 $scope.$watch 'notifications', (newValue, oldValue) -> diff = [] angular.forEach newValue, (value) -> @@ -273,6 +275,9 @@ Application.Controllers.controller 'ApplicationController', ["$rootScope", "$sco unless find diff.push(value) + remain = 3 + if diff.length >= remain + diff.splice(remain, (diff.length - remain), {message: {description: _t('and_NUMBER_other_notifications', {NUMBER: diff.length - remain})}}) angular.forEach diff, (notification, key) -> growl.info(notification.message.description) diff --git a/app/controllers/api/notifications_controller.rb b/app/controllers/api/notifications_controller.rb index a9121d91e..0c242ff6c 100644 --- a/app/controllers/api/notifications_controller.rb +++ b/app/controllers/api/notifications_controller.rb @@ -4,7 +4,11 @@ class API::NotificationsController < API::ApiController def index if params[:is_read] - @notifications = current_user.notifications.where(is_read: params[:is_read] == 'true').page(params[:page]).per(15).order('created_at DESC') + if params[:is_read] == 'true' + @notifications = current_user.notifications.where(is_read: true).page(params[:page]).per(15).order('created_at DESC') + else + @notifications = current_user.notifications.where(is_read: false).order('created_at DESC') + end else @notifications = current_user.notifications.order('created_at DESC') end diff --git a/config/locales/app.public.en.yml b/config/locales/app.public.en.yml index 5a3e7d1f0..7221927ab 100644 --- a/config/locales/app.public.en.yml +++ b/config/locales/app.public.en.yml @@ -103,6 +103,9 @@ en: # Fab-manager's version version: "Version:" + # Notifications + and_NUMBER_other_notifications: "and {{NUMBER}} other notifications..." # angular interpolation + about: # about page read_the_fablab_policy: "Read the FabLab policy" diff --git a/config/locales/app.public.fr.yml b/config/locales/app.public.fr.yml index 27d7d2b5b..ad2eaeeab 100644 --- a/config/locales/app.public.fr.yml +++ b/config/locales/app.public.fr.yml @@ -102,6 +102,10 @@ fr: # Fab-manager's version version: "Version :" + + # Notifications + and_NUMBER_other_notifications: "et {{NUMBER}} autres notifications ..." # angular interpolation + about: # page à propos read_the_fablab_policy: "Consulter les règles d'utilisation du Fab Lab" From 1f0da1194bbab7435b25938b6ce9be600029de0f Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 2 Jan 2017 11:08:40 +0100 Subject: [PATCH 02/14] [bug] fix VAT amount in invoice config --- CHANGELOG.md | 1 + app/assets/templates/admin/invoices/index.html.erb | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b7279958..71a0a2d0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## next release - Mask new notifications alerts when more than 3 - Fix a bug: display more than 15 unread notifications (number on the bell icon & full list) +- Fix a bug: in invoice configuration panel, VAT amount and total excl. taxes are inverted ## v2.4.8 2016 December 15 diff --git a/app/assets/templates/admin/invoices/index.html.erb b/app/assets/templates/admin/invoices/index.html.erb index c4e47ff7d..a421d2286 100644 --- a/app/assets/templates/admin/invoices/index.html.erb +++ b/app/assets/templates/admin/invoices/index.html.erb @@ -157,11 +157,11 @@ {{ 'including_VAT' | translate }} {{invoice.VAT.rate}} % - {{30/(invoice.VAT.rate/100+1) | currency}} + {{30-(30/(invoice.VAT.rate/100+1)) | currency}} {{ 'including_total_excluding_taxes' }} - {{30-(30/(invoice.VAT.rate/100+1)) | currency}} + {{30/(invoice.VAT.rate/100+1) | currency}} {{ 'including_amount_payed_on_ordering' }} From a39be147188a981c6b87db470823477446109ea3 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 2 Jan 2017 15:56:52 +0100 Subject: [PATCH 03/14] fix number of remaining notifications --- app/assets/javascripts/controllers/application.coffee.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/controllers/application.coffee.erb b/app/assets/javascripts/controllers/application.coffee.erb index 645b6e6cb..ad23759bc 100644 --- a/app/assets/javascripts/controllers/application.coffee.erb +++ b/app/assets/javascripts/controllers/application.coffee.erb @@ -277,7 +277,7 @@ Application.Controllers.controller 'ApplicationController', ["$rootScope", "$sco remain = 3 if diff.length >= remain - diff.splice(remain, (diff.length - remain), {message: {description: _t('and_NUMBER_other_notifications', {NUMBER: diff.length - remain})}}) + diff.splice(remain, (diff.length - remain), {message: {description: _t('and_NUMBER_other_notifications', {NUMBER: diff.length - remain + 1})}}) angular.forEach diff, (notification, key) -> growl.info(notification.message.description) From 94abdb9eecfef402e3eafee113e174c050f82b3b Mon Sep 17 00:00:00 2001 From: Sylvain Date: Mon, 2 Jan 2017 17:01:52 +0100 Subject: [PATCH 04/14] revert last commit --- app/assets/javascripts/controllers/application.coffee.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/controllers/application.coffee.erb b/app/assets/javascripts/controllers/application.coffee.erb index ad23759bc..645b6e6cb 100644 --- a/app/assets/javascripts/controllers/application.coffee.erb +++ b/app/assets/javascripts/controllers/application.coffee.erb @@ -277,7 +277,7 @@ Application.Controllers.controller 'ApplicationController', ["$rootScope", "$sco remain = 3 if diff.length >= remain - diff.splice(remain, (diff.length - remain), {message: {description: _t('and_NUMBER_other_notifications', {NUMBER: diff.length - remain + 1})}}) + diff.splice(remain, (diff.length - remain), {message: {description: _t('and_NUMBER_other_notifications', {NUMBER: diff.length - remain})}}) angular.forEach diff, (notification, key) -> growl.info(notification.message.description) From d5c5e369653e4f5d8a53165c072ace5129ccc118 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 3 Jan 2017 11:24:14 +0100 Subject: [PATCH 05/14] [bug] compute age of users born on 29/02, lead to app crash on non-leap years --- CHANGELOG.md | 1 + app/models/profile.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71a0a2d0e..a3b767a4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Mask new notifications alerts when more than 3 - Fix a bug: display more than 15 unread notifications (number on the bell icon & full list) - Fix a bug: in invoice configuration panel, VAT amount and total excl. taxes are inverted +- Fix a bug: unable to compute user's age when they were born on february 29th and current year is not a leap year ## v2.4.8 2016 December 15 diff --git a/app/models/profile.rb b/app/models/profile.rb index 483fbee71..073ed582f 100644 --- a/app/models/profile.rb +++ b/app/models/profile.rb @@ -28,7 +28,7 @@ class Profile < ActiveRecord::Base def age if birthday.present? now = Time.now.utc.to_date - now.year - birthday.year - (birthday.to_date.change(:year => now.year) > now ? 1 : 0) + (now - birthday).to_f / 365.2425 else '' end From 733a552c60d5da30e3e1a651e41142be29caf8d9 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 3 Jan 2017 11:25:46 +0100 Subject: [PATCH 06/14] invalid default mime type in checking list --- config/application.yml.default | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/application.yml.default b/config/application.yml.default index 4aed18a73..fbbab825d 100644 --- a/config/application.yml.default +++ b/config/application.yml.default @@ -58,7 +58,7 @@ OPENLAB_BASE_URI: 'https://openprojects.fab-manager.com' LOG_LEVEL: 'debug' ALLOWED_EXTENSIONS: pdf ai eps cad math svg stl dxf dwg obj step iges igs 3dm 3dmf doc docx png ino scad fcad skp sldprt sldasm slddrw slddrt tex latex ps -ALLOWED_MIME_TYPES: application/pdf application/postscript application/illustrator image/x-eps image/svg+xml application/sla application/dxf application/acad application/dwg application/octet-stream application/step application/iges model/iges x-world/x-3dmf application/ application/vnd.openxmlformats-officedocument.wordprocessingml.document image/png text/x-arduino text/plain application/scad application/vnd.sketchup.skp application/x-koan application/vnd-koan koan/x-skm application/vnd.koan application/x-tex application/x-latex +ALLOWED_MIME_TYPES: application/pdf application/postscript application/illustrator image/x-eps image/svg+xml application/sla application/dxf application/acad application/dwg application/octet-stream application/step application/iges model/iges x-world/x-3dmf application/vnd.openxmlformats-officedocument.wordprocessingml.document image/png text/x-arduino text/plain application/scad application/vnd.sketchup.skp application/x-koan application/vnd-koan koan/x-skm application/vnd.koan application/x-tex application/x-latex # 10485760 = 10 megabytes MAX_IMAGE_SIZE: '10485760' From 906bb5b26ef3e6e8c45935ca7006e94529c962a8 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 3 Jan 2017 11:27:58 +0100 Subject: [PATCH 07/14] add upgrade instructions --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3b767a4a..ecff63702 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Fix a bug: display more than 15 unread notifications (number on the bell icon & full list) - Fix a bug: in invoice configuration panel, VAT amount and total excl. taxes are inverted - Fix a bug: unable to compute user's age when they were born on february 29th and current year is not a leap year +- [TODO DEPLOY] remove possible value `application/` in `ALLOWED_MIME_TYPES` list, in environment variable ## v2.4.8 2016 December 15 From 5b7e8474d49bbc7772121d3775bfdf7073551581 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 3 Jan 2017 12:07:16 +0100 Subject: [PATCH 08/14] asterisk on group select in member form --- CHANGELOG.md | 1 + app/assets/templates/admin/members/_form.html.erb | 5 ++++- app/models/availability.rb | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecff63702..1a2700dbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## next release - Mask new notifications alerts when more than 3 +- Added an asterisk on group select in admin's member form - Fix a bug: display more than 15 unread notifications (number on the bell icon & full list) - Fix a bug: in invoice configuration panel, VAT amount and total excl. taxes are inverted - Fix a bug: unable to compute user's age when they were born on february 29th and current year is not a leap year diff --git a/app/assets/templates/admin/members/_form.html.erb b/app/assets/templates/admin/members/_form.html.erb index d7c1831d0..243051576 100644 --- a/app/assets/templates/admin/members/_form.html.erb +++ b/app/assets/templates/admin/members/_form.html.erb @@ -1,5 +1,8 @@
- +
diff --git a/app/models/availability.rb b/app/models/availability.rb index 7e7dd4f4e..8e9dd393e 100644 --- a/app/models/availability.rb +++ b/app/models/availability.rb @@ -96,7 +96,7 @@ class Availability < ActiveRecord::Base def as_indexed_json json = JSON.parse(to_json) - json['hours_duration'] = (end_at - start_at) / (60*60) + json['hours_duration'] = (end_at - start_at) / (60 * 60) json.to_json end From 9edf723373b2810226811c8c4e10d8a8509e2075 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 3 Jan 2017 13:35:36 +0100 Subject: [PATCH 09/14] [bug] wrong statistics about hours available for reservation --- CHANGELOG.md | 2 ++ app/models/availability.rb | 2 ++ db/seeds.rb | 2 +- test/fixtures/statistic_custom_aggregations.yml | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a2700dbd..44759d180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ - Fix a bug: display more than 15 unread notifications (number on the bell icon & full list) - Fix a bug: in invoice configuration panel, VAT amount and total excl. taxes are inverted - Fix a bug: unable to compute user's age when they were born on february 29th and current year is not a leap year +- Fix a bug: wrong statistics about hours available for reservation. Fix requires user action (1) - [TODO DEPLOY] remove possible value `application/` in `ALLOWED_MIME_TYPES` list, in environment variable +- [TODO DEPLOY] `rails runner StatisticCustomAggregation.destroy_all`, then `rake db:seed`, then `rake fablab:es_build_availabilities_index` (1) ## v2.4.8 2016 December 15 diff --git a/app/models/availability.rb b/app/models/availability.rb index 8e9dd393e..a232dddc3 100644 --- a/app/models/availability.rb +++ b/app/models/availability.rb @@ -97,6 +97,8 @@ class Availability < ActiveRecord::Base def as_indexed_json json = JSON.parse(to_json) json['hours_duration'] = (end_at - start_at) / (60 * 60) + json['machines'] = machines_availabilities.map{|ma| ma.machine.friendly_id} + json['bookable_hours'] = json['hours_duration'] * json['machines'].length json.to_json end diff --git a/db/seeds.rb b/db/seeds.rb index c10636074..887ab461f 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -407,7 +407,7 @@ if StatisticCustomAggregation.count == 0 es_index: 'fablab', es_type: 'availabilities', field: 'available_hours', - query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"hours_duration"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}' + query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"bookable_hours"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}' }) available_hours.save! diff --git a/test/fixtures/statistic_custom_aggregations.yml b/test/fixtures/statistic_custom_aggregations.yml index 35c1e14fd..146b437fe 100644 --- a/test/fixtures/statistic_custom_aggregations.yml +++ b/test/fixtures/statistic_custom_aggregations.yml @@ -1,7 +1,7 @@ # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html one: - query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"hours_duration"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}' + query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"bookable_hours"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}' statistic_type_id: 2 field: "available_hours" es_index: "fablab" From 8119f54e4ce4fbed21152a2ed842115d029c792f Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 3 Jan 2017 17:07:23 +0100 Subject: [PATCH 10/14] Statistics about hours available for machine reservations and tickets available for training reservations, now handle custom filtering on date and type --- CHANGELOG.md | 4 +- .../controllers/admin/graphs.coffee | 1 + .../controllers/admin/statistics.coffee.erb | 32 +++++++++---- app/controllers/api/statistics_controller.rb | 11 +---- app/models/availability.rb | 12 ++++- app/services/custom_aggregation_service.rb | 45 +++++++++++++++++++ 6 files changed, 84 insertions(+), 21 deletions(-) create mode 100644 app/services/custom_aggregation_service.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 44759d180..702ebf791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,12 @@ ## next release - Mask new notifications alerts when more than 3 - Added an asterisk on group select in admin's member form +- Statistics custom aggregations can handle custom filtering +- Statistics about hours available for machine reservations and tickets available for training reservations, now handle custom filtering on date and type - Fix a bug: display more than 15 unread notifications (number on the bell icon & full list) - Fix a bug: in invoice configuration panel, VAT amount and total excl. taxes are inverted - Fix a bug: unable to compute user's age when they were born on february 29th and current year is not a leap year -- Fix a bug: wrong statistics about hours available for reservation. Fix requires user action (1) +- Fix a bug: wrong statistics about hours available for machines reservation. Fix requires user action (1) - [TODO DEPLOY] remove possible value `application/` in `ALLOWED_MIME_TYPES` list, in environment variable - [TODO DEPLOY] `rails runner StatisticCustomAggregation.destroy_all`, then `rake db:seed`, then `rake fablab:es_build_availabilities_index` (1) diff --git a/app/assets/javascripts/controllers/admin/graphs.coffee b/app/assets/javascripts/controllers/admin/graphs.coffee index 8c23e69c7..6100e20e8 100644 --- a/app/assets/javascripts/controllers/admin/graphs.coffee +++ b/app/assets/javascripts/controllers/admin/graphs.coffee @@ -363,6 +363,7 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro "type": esType "searchType": "count" "stat-type": statType + "custom-query": '' "start-date": moment($scope.datePickerStart.selected).format() "end-date": moment($scope.datePickerEnd.selected).format() "body": buildElasticAggregationsQuery(statType, $scope.display.interval, moment($scope.datePickerStart.selected), moment($scope.datePickerEnd.selected)) diff --git a/app/assets/javascripts/controllers/admin/statistics.coffee.erb b/app/assets/javascripts/controllers/admin/statistics.coffee.erb index 53d289217..b99297b21 100644 --- a/app/assets/javascripts/controllers/admin/statistics.coffee.erb +++ b/app/assets/javascripts/controllers/admin/statistics.coffee.erb @@ -380,6 +380,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", "size": RESULTS_PER_PAGE "scroll": ES_SCROLL_TIME+'m' "stat-type": type + "custom-query": if custom then JSON.stringify(Object.assign({exclude: custom.exclude}, buildElasticCustomCriterion(custom))) else '' "start-date": moment($scope.datePickerStart.selected).format() "end-date": moment($scope.datePickerEnd.selected).format() "body": buildElasticDataQuery(type, custom, $scope.agePicker.start, $scope.agePicker.end, moment($scope.datePickerStart.selected), moment($scope.datePickerEnd.selected), $scope.sorting) @@ -427,15 +428,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", "lte": ageMax # optional criterion if custom - criterion = { - "match" : {} - } - switch $scope.getCustomValueInputType($scope.customFilter.criterion) - when 'input_date' then criterion.match[custom.key] = moment(custom.value).format('YYYY-MM-DD') - when 'input_select' then criterion.match[custom.key] = custom.value.key - when 'input_list' then criterion.match[custom.key+".name"] = custom.value - else criterion.match[custom.key] = custom.value - + criterion = buildElasticCustomCriterion(custom) if (custom.exclude) q = "query": { "filtered": { @@ -470,6 +463,27 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", + ## + # Build the elasticSearch query DSL to match the selected cutom filter + # @param custom {Object} if custom is empty or undefined, an empty string will be returned + # @returns {{match:{*}}|string} + ## + buildElasticCustomCriterion = (custom) -> + if (custom) + criterion = { + "match" : {} + } + switch $scope.getCustomValueInputType($scope.customFilter.criterion) + when 'input_date' then criterion.match[custom.key] = moment(custom.value).format('YYYY-MM-DD') + when 'input_select' then criterion.match[custom.key] = custom.value.key + when 'input_list' then criterion.match[custom.key+".name"] = custom.value + else criterion.match[custom.key] = custom.value + criterion + else + '' + + + ## # Parse the provided criteria array and return the corresponding elasticSearch syntax # @param criteria {Array} array of {key_to_sort:order} diff --git a/app/controllers/api/statistics_controller.rb b/app/controllers/api/statistics_controller.rb index 356ab07ae..e82603202 100644 --- a/app/controllers/api/statistics_controller.rb +++ b/app/controllers/api/statistics_controller.rb @@ -13,6 +13,7 @@ class API::StatisticsController < API::ApiController # remove additional parameters statistic_type = request.query_parameters.delete('stat-type') + custom_query = request.query_parameters.delete('custom-query') start_date = request.query_parameters.delete('start-date') end_date = request.query_parameters.delete('end-date') @@ -21,15 +22,7 @@ class API::StatisticsController < API::ApiController results = Stats::#{path.classify}.search(query, request.query_parameters.symbolize_keys).response # run additional custom aggregations, if any - if statistic_type and start_date and end_date - stat_index = StatisticIndex.find_by(es_type_key: "#{path}") - stat_type = StatisticType.where(statistic_index_id: stat_index.id, key: statistic_type).first - client = Elasticsearch::Model.client - stat_type.statistic_custom_aggregations.each do |custom| - c_res = client.search index: custom.es_index, type:custom.es_type, body:sprintf(custom.query, {aggs_name: custom.field, start_date: start_date, end_date: end_date}) - results['aggregations'][custom.field] = c_res['aggregations'][custom.field] - end - end + CustomAggregationService.new.("#{path}", statistic_type, start_date, end_date, custom_query, results) # return result render json: results diff --git a/app/models/availability.rb b/app/models/availability.rb index a232dddc3..2b8b29c90 100644 --- a/app/models/availability.rb +++ b/app/models/availability.rb @@ -38,6 +38,7 @@ class Availability < ActiveRecord::Base settings do mappings dynamic: 'true' do indexes 'available_type', analyzer: 'simple' + indexes 'subType', index: 'not_analyzed' end end @@ -97,8 +98,15 @@ class Availability < ActiveRecord::Base def as_indexed_json json = JSON.parse(to_json) json['hours_duration'] = (end_at - start_at) / (60 * 60) - json['machines'] = machines_availabilities.map{|ma| ma.machine.friendly_id} - json['bookable_hours'] = json['hours_duration'] * json['machines'].length + if available_type == 'machines' + json['subType'] = machines_availabilities.map{|ma| ma.machine.friendly_id} + elsif available_type == 'training' + json['subType'] = trainings_availabilities.map{|ta| ta.training.friendly_id} + elsif available_type == 'event' + json['subType'] = [event.category.friendly_id] + end + json['bookable_hours'] = json['hours_duration'] * json['subType'].length + json['date'] = start_at.to_date json.to_json end diff --git a/app/services/custom_aggregation_service.rb b/app/services/custom_aggregation_service.rb new file mode 100644 index 000000000..034e237b8 --- /dev/null +++ b/app/services/custom_aggregation_service.rb @@ -0,0 +1,45 @@ +require 'json' + +class CustomAggregationService + + ## + # Run any additional custom aggregations related to the given statistic type, if any + ## + def call(statistic_index, statistic_type, start_date, end_date, custom_query, results) + if statistic_type and start_date and end_date + stat_index = StatisticIndex.find_by(es_type_key: statistic_index) + stat_type = StatisticType.find_by(statistic_index_id: stat_index.id, key: statistic_type) + client = Elasticsearch::Model.client + stat_type.statistic_custom_aggregations.each do |custom| + + query = sprintf(custom.query, {aggs_name: custom.field, start_date: start_date, end_date: end_date}) + + if custom_query and !custom_query.empty? + # Here, a custom query was provided with the original request (eg: filter by subtype) + # so we try to apply this custom filter to the current custom aggregation. + # + # The requested model mapping (ie. found in StatisticCustomAggregation.es_index > es_type) must have defined + # these fields in the indexed json, otherwise the returned value will probably not be what is expected. + # + # As an implementation exemple, you can take a look at Availability (indexed as fablab/availabilities) + # and witch will run custom filters on the fields 'date' and 'subType'. Other custom filters will return 0 + # as they are not relevant with this kind of custom aggregation. + query = JSON.parse(query) + custom_query = JSON.parse(custom_query) + + exclude = custom_query.delete('exclude') + if exclude + query = {query: { filtered: { query: query['query'], filter: { not: { term: custom_query['match'] } } } }, aggregations: query['aggregations'], size: query['size']} + else + query['query']['bool']['must'].push(custom_query) + end + query = query.to_json + end + + c_res = client.search(index: custom.es_index, type: custom.es_type, body: query) + results['aggregations'][custom.field] = c_res['aggregations'][custom.field] + end + end + results + end +end \ No newline at end of file From e656b7d7842ae7759e88850e68042111468797d6 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 4 Jan 2017 14:46:43 +0100 Subject: [PATCH 11/14] [bug] regenerate statistics does not remove all previous values --- CHANGELOG.md | 1 + app/services/statistic_service.rb | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 702ebf791..d095160c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Fix a bug: in invoice configuration panel, VAT amount and total excl. taxes are inverted - Fix a bug: unable to compute user's age when they were born on february 29th and current year is not a leap year - Fix a bug: wrong statistics about hours available for machines reservation. Fix requires user action (1) +- Fix a bug: when regenerating statistics, previous values are not fully removed (only 10 firsts), resulting in wrong statistics generation - [TODO DEPLOY] remove possible value `application/` in `ALLOWED_MIME_TYPES` list, in environment variable - [TODO DEPLOY] `rails runner StatisticCustomAggregation.destroy_all`, then `rake db:seed`, then `rake fablab:es_build_availabilities_index` (1) diff --git a/app/services/statistic_service.rb b/app/services/statistic_service.rb index b824e370e..f19736ee6 100644 --- a/app/services/statistic_service.rb +++ b/app/services/statistic_service.rb @@ -297,8 +297,10 @@ class StatisticService end def clean_stat(options = default_options) + client = Elasticsearch::Model.client %w{Account Event Machine Project Subscription Training User}.each do |o| - "Stats::#{o}".constantize.search(query: {match: {date: format_date(options[:start_date])}}).results.each(&:destroy) + model = "Stats::#{o}".constantize + client.delete_by_query(index: model.index_name, type: model.document_type, body: {query: {match: {date: format_date(options[:start_date])}}}) end end From ce7b737163e7b7b998794a599f3c57eae96dbd44 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 4 Jan 2017 14:48:32 +0100 Subject: [PATCH 12/14] possible workaround for orphan InvoiceWorkers created with nil parameter Moreover, a log was added on job creation, to help debugging if this does not fix the issue --- app/models/invoice.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/invoice.rb b/app/models/invoice.rb index 3facee497..2d699c466 100644 --- a/app/models/invoice.rb +++ b/app/models/invoice.rb @@ -13,7 +13,7 @@ class Invoice < ActiveRecord::Base has_one :avoir, class_name: 'Invoice', foreign_key: :invoice_id, dependent: :destroy after_create :update_reference - after_commit :generate_and_send_invoice, on: [:create] + after_commit :generate_and_send_invoice, on: [:create], :if => :persisted? def file dir = "invoices/#{user.id}" @@ -204,6 +204,7 @@ class Invoice < ActiveRecord::Base private def generate_and_send_invoice + puts "Creating an InvoiceWorker job to generate the following invoice: id(#{id}), invoiced_id(#{invoiced_id}), invoiced_type(#{invoiced_type}), user_id(#{user_id})" InvoiceWorker.perform_async(id) end From b5246ae1cfc8a66d644dd54a65e17b4bde005174 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 4 Jan 2017 15:20:54 +0100 Subject: [PATCH 13/14] [bug] AvailabilityIndexerWorker crash availabilities deleted just after their creation --- CHANGELOG.md | 1 + app/workers/availability_indexer_worker.rb | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d095160c5..3950ee773 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Fix a bug: unable to compute user's age when they were born on february 29th and current year is not a leap year - Fix a bug: wrong statistics about hours available for machines reservation. Fix requires user action (1) - Fix a bug: when regenerating statistics, previous values are not fully removed (only 10 firsts), resulting in wrong statistics generation +- Fix a bug: when deleting an availability just after its creation, the indexer workers crash and retries for a month - [TODO DEPLOY] remove possible value `application/` in `ALLOWED_MIME_TYPES` list, in environment variable - [TODO DEPLOY] `rails runner StatisticCustomAggregation.destroy_all`, then `rake db:seed`, then `rake fablab:es_build_availabilities_index` (1) diff --git a/app/workers/availability_indexer_worker.rb b/app/workers/availability_indexer_worker.rb index a902ed718..c26acadad 100644 --- a/app/workers/availability_indexer_worker.rb +++ b/app/workers/availability_indexer_worker.rb @@ -10,11 +10,18 @@ class AvailabilityIndexerWorker case operation.to_s when /index/ - record = Availability.find(record_id) - Client.index index: Availability.index_name, type: Availability.document_type, id: record.id, body: record.as_indexed_json - #puts record.as_indexed_json + begin + record = Availability.find(record_id) + Client.index index: Availability.index_name, type: Availability.document_type, id: record.id, body: record.as_indexed_json + rescue ActiveRecord::RecordNotFound + STDERR.puts "Availability id(#{record_id}) will not be indexed in ElasticSearch as it does not exists anymore in database" + end when /delete/ - Client.delete index: Availability.index_name, type: Availability.document_type, id: record_id + begin + Client.delete index: Availability.index_name, type: Availability.document_type, id: record_id + rescue Elasticsearch::Transport::Transport::Errors::NotFound + STDERR.puts "Availability id(#{record_id}) will not be deleted form ElasticSearch as it has not been already indexed" + end else raise ArgumentError, "Unknown operation '#{operation}'" end end From 7f98cdb76d3d6c50142de9039dbfad1eb7655690 Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 4 Jan 2017 15:30:21 +0100 Subject: [PATCH 14/14] Version 2.4.9 --- .fabmanager-version | 2 +- CHANGELOG.md | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.fabmanager-version b/.fabmanager-version index 752a79ef3..158349812 100644 --- a/.fabmanager-version +++ b/.fabmanager-version @@ -1 +1 @@ -2.4.8 \ No newline at end of file +2.4.9 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 3950ee773..f8cbe8c54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog Fab Manager -## next release +## v2.4.9 2017 January 4 + - Mask new notifications alerts when more than 3 - Added an asterisk on group select in admin's member form - Statistics custom aggregations can handle custom filtering @@ -9,10 +10,11 @@ - Fix a bug: in invoice configuration panel, VAT amount and total excl. taxes are inverted - Fix a bug: unable to compute user's age when they were born on february 29th and current year is not a leap year - Fix a bug: wrong statistics about hours available for machines reservation. Fix requires user action (1) -- Fix a bug: when regenerating statistics, previous values are not fully removed (only 10 firsts), resulting in wrong statistics generation +- Fix a bug: when regenerating statistics, previous values are not fully removed (only 10 firsts), resulting in wrong statistics generation (2) - Fix a bug: when deleting an availability just after its creation, the indexer workers crash and retries for a month - [TODO DEPLOY] remove possible value `application/` in `ALLOWED_MIME_TYPES` list, in environment variable - [TODO DEPLOY] `rails runner StatisticCustomAggregation.destroy_all`, then `rake db:seed`, then `rake fablab:es_build_availabilities_index` (1) +- [TODO DEPLOY] `fablab:generate_stats[1095]` if you already has regenerated the statistics in the past, then they are very likely corrupted. Run this task to fix (2) ## v2.4.8 2016 December 15