diff --git a/.fabmanager-version b/.fabmanager-version index 752a79ef3..158349812 100644 --- a/.fabmanager-version +++ b/.fabmanager-version @@ -1 +1 @@ -2.4.8 \ No newline at end of file +2.4.9 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 81f6f2238..f8cbe8c54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog Fab Manager +## v2.4.9 2017 January 4 + +- Mask new notifications alerts when more than 3 +- Added an asterisk on group select in admin's member form +- Statistics custom aggregations can handle custom filtering +- Statistics about hours available for machine reservations and tickets available for training reservations, now handle custom filtering on date and type +- Fix a bug: display more than 15 unread notifications (number on the bell icon & full list) +- Fix a bug: in invoice configuration panel, VAT amount and total excl. taxes are inverted +- Fix a bug: unable to compute user's age when they were born on february 29th and current year is not a leap year +- Fix a bug: wrong statistics about hours available for machines reservation. Fix requires user action (1) +- Fix a bug: when regenerating statistics, previous values are not fully removed (only 10 firsts), resulting in wrong statistics generation (2) +- Fix a bug: when deleting an availability just after its creation, the indexer workers crash and retries for a month +- [TODO DEPLOY] remove possible value `application/` in `ALLOWED_MIME_TYPES` list, in environment variable +- [TODO DEPLOY] `rails runner StatisticCustomAggregation.destroy_all`, then `rake db:seed`, then `rake fablab:es_build_availabilities_index` (1) +- [TODO DEPLOY] `fablab:generate_stats[1095]` if you already has regenerated the statistics in the past, then they are very likely corrupted. Run this task to fix (2) + ## v2.4.8 2016 December 15 - Added asterisks on mandatory fields in member's form diff --git a/app/assets/javascripts/controllers/admin/graphs.coffee b/app/assets/javascripts/controllers/admin/graphs.coffee index 8c23e69c7..6100e20e8 100644 --- a/app/assets/javascripts/controllers/admin/graphs.coffee +++ b/app/assets/javascripts/controllers/admin/graphs.coffee @@ -363,6 +363,7 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro "type": esType "searchType": "count" "stat-type": statType + "custom-query": '' "start-date": moment($scope.datePickerStart.selected).format() "end-date": moment($scope.datePickerEnd.selected).format() "body": buildElasticAggregationsQuery(statType, $scope.display.interval, moment($scope.datePickerStart.selected), moment($scope.datePickerEnd.selected)) diff --git a/app/assets/javascripts/controllers/admin/statistics.coffee.erb b/app/assets/javascripts/controllers/admin/statistics.coffee.erb index 53d289217..b99297b21 100644 --- a/app/assets/javascripts/controllers/admin/statistics.coffee.erb +++ b/app/assets/javascripts/controllers/admin/statistics.coffee.erb @@ -380,6 +380,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", "size": RESULTS_PER_PAGE "scroll": ES_SCROLL_TIME+'m' "stat-type": type + "custom-query": if custom then JSON.stringify(Object.assign({exclude: custom.exclude}, buildElasticCustomCriterion(custom))) else '' "start-date": moment($scope.datePickerStart.selected).format() "end-date": moment($scope.datePickerEnd.selected).format() "body": buildElasticDataQuery(type, custom, $scope.agePicker.start, $scope.agePicker.end, moment($scope.datePickerStart.selected), moment($scope.datePickerEnd.selected), $scope.sorting) @@ -427,15 +428,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", "lte": ageMax # optional criterion if custom - criterion = { - "match" : {} - } - switch $scope.getCustomValueInputType($scope.customFilter.criterion) - when 'input_date' then criterion.match[custom.key] = moment(custom.value).format('YYYY-MM-DD') - when 'input_select' then criterion.match[custom.key] = custom.value.key - when 'input_list' then criterion.match[custom.key+".name"] = custom.value - else criterion.match[custom.key] = custom.value - + criterion = buildElasticCustomCriterion(custom) if (custom.exclude) q = "query": { "filtered": { @@ -470,6 +463,27 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state", + ## + # Build the elasticSearch query DSL to match the selected cutom filter + # @param custom {Object} if custom is empty or undefined, an empty string will be returned + # @returns {{match:{*}}|string} + ## + buildElasticCustomCriterion = (custom) -> + if (custom) + criterion = { + "match" : {} + } + switch $scope.getCustomValueInputType($scope.customFilter.criterion) + when 'input_date' then criterion.match[custom.key] = moment(custom.value).format('YYYY-MM-DD') + when 'input_select' then criterion.match[custom.key] = custom.value.key + when 'input_list' then criterion.match[custom.key+".name"] = custom.value + else criterion.match[custom.key] = custom.value + criterion + else + '' + + + ## # Parse the provided criteria array and return the corresponding elasticSearch syntax # @param criteria {Array} array of {key_to_sort:order} diff --git a/app/assets/javascripts/controllers/application.coffee.erb b/app/assets/javascripts/controllers/application.coffee.erb index 51c42a6a3..645b6e6cb 100644 --- a/app/assets/javascripts/controllers/application.coffee.erb +++ b/app/assets/javascripts/controllers/application.coffee.erb @@ -260,7 +260,9 @@ Application.Controllers.controller 'ApplicationController', ["$rootScope", "$sco getNotifications = -> $rootScope.toCheckNotifications = true unless $rootScope.checkNotificationsIsInit or !$rootScope.currentUser - $scope.notifications = Notification.query {is_read: false} + setTimeout -> + $scope.notifications = Notification.query {is_read: false} + , 2000 $scope.$watch 'notifications', (newValue, oldValue) -> diff = [] angular.forEach newValue, (value) -> @@ -273,6 +275,9 @@ Application.Controllers.controller 'ApplicationController', ["$rootScope", "$sco unless find diff.push(value) + remain = 3 + if diff.length >= remain + diff.splice(remain, (diff.length - remain), {message: {description: _t('and_NUMBER_other_notifications', {NUMBER: diff.length - remain})}}) angular.forEach diff, (notification, key) -> growl.info(notification.message.description) diff --git a/app/assets/templates/admin/invoices/index.html.erb b/app/assets/templates/admin/invoices/index.html.erb index c4e47ff7d..a421d2286 100644 --- a/app/assets/templates/admin/invoices/index.html.erb +++ b/app/assets/templates/admin/invoices/index.html.erb @@ -157,11 +157,11 @@ {{ 'including_VAT' | translate }} {{invoice.VAT.rate}} % - {{30/(invoice.VAT.rate/100+1) | currency}} + {{30-(30/(invoice.VAT.rate/100+1)) | currency}} {{ 'including_total_excluding_taxes' }} - {{30-(30/(invoice.VAT.rate/100+1)) | currency}} + {{30/(invoice.VAT.rate/100+1) | currency}} {{ 'including_amount_payed_on_ordering' }} diff --git a/app/assets/templates/admin/members/_form.html.erb b/app/assets/templates/admin/members/_form.html.erb index d7c1831d0..243051576 100644 --- a/app/assets/templates/admin/members/_form.html.erb +++ b/app/assets/templates/admin/members/_form.html.erb @@ -1,5 +1,8 @@
- +
diff --git a/app/controllers/api/notifications_controller.rb b/app/controllers/api/notifications_controller.rb index a9121d91e..0c242ff6c 100644 --- a/app/controllers/api/notifications_controller.rb +++ b/app/controllers/api/notifications_controller.rb @@ -4,7 +4,11 @@ class API::NotificationsController < API::ApiController def index if params[:is_read] - @notifications = current_user.notifications.where(is_read: params[:is_read] == 'true').page(params[:page]).per(15).order('created_at DESC') + if params[:is_read] == 'true' + @notifications = current_user.notifications.where(is_read: true).page(params[:page]).per(15).order('created_at DESC') + else + @notifications = current_user.notifications.where(is_read: false).order('created_at DESC') + end else @notifications = current_user.notifications.order('created_at DESC') end diff --git a/app/controllers/api/statistics_controller.rb b/app/controllers/api/statistics_controller.rb index 356ab07ae..e82603202 100644 --- a/app/controllers/api/statistics_controller.rb +++ b/app/controllers/api/statistics_controller.rb @@ -13,6 +13,7 @@ class API::StatisticsController < API::ApiController # remove additional parameters statistic_type = request.query_parameters.delete('stat-type') + custom_query = request.query_parameters.delete('custom-query') start_date = request.query_parameters.delete('start-date') end_date = request.query_parameters.delete('end-date') @@ -21,15 +22,7 @@ class API::StatisticsController < API::ApiController results = Stats::#{path.classify}.search(query, request.query_parameters.symbolize_keys).response # run additional custom aggregations, if any - if statistic_type and start_date and end_date - stat_index = StatisticIndex.find_by(es_type_key: "#{path}") - stat_type = StatisticType.where(statistic_index_id: stat_index.id, key: statistic_type).first - client = Elasticsearch::Model.client - stat_type.statistic_custom_aggregations.each do |custom| - c_res = client.search index: custom.es_index, type:custom.es_type, body:sprintf(custom.query, {aggs_name: custom.field, start_date: start_date, end_date: end_date}) - results['aggregations'][custom.field] = c_res['aggregations'][custom.field] - end - end + CustomAggregationService.new.("#{path}", statistic_type, start_date, end_date, custom_query, results) # return result render json: results diff --git a/app/models/availability.rb b/app/models/availability.rb index 7e7dd4f4e..2b8b29c90 100644 --- a/app/models/availability.rb +++ b/app/models/availability.rb @@ -38,6 +38,7 @@ class Availability < ActiveRecord::Base settings do mappings dynamic: 'true' do indexes 'available_type', analyzer: 'simple' + indexes 'subType', index: 'not_analyzed' end end @@ -96,7 +97,16 @@ class Availability < ActiveRecord::Base def as_indexed_json json = JSON.parse(to_json) - json['hours_duration'] = (end_at - start_at) / (60*60) + json['hours_duration'] = (end_at - start_at) / (60 * 60) + if available_type == 'machines' + json['subType'] = machines_availabilities.map{|ma| ma.machine.friendly_id} + elsif available_type == 'training' + json['subType'] = trainings_availabilities.map{|ta| ta.training.friendly_id} + elsif available_type == 'event' + json['subType'] = [event.category.friendly_id] + end + json['bookable_hours'] = json['hours_duration'] * json['subType'].length + json['date'] = start_at.to_date json.to_json end diff --git a/app/models/invoice.rb b/app/models/invoice.rb index 3facee497..2d699c466 100644 --- a/app/models/invoice.rb +++ b/app/models/invoice.rb @@ -13,7 +13,7 @@ class Invoice < ActiveRecord::Base has_one :avoir, class_name: 'Invoice', foreign_key: :invoice_id, dependent: :destroy after_create :update_reference - after_commit :generate_and_send_invoice, on: [:create] + after_commit :generate_and_send_invoice, on: [:create], :if => :persisted? def file dir = "invoices/#{user.id}" @@ -204,6 +204,7 @@ class Invoice < ActiveRecord::Base private def generate_and_send_invoice + puts "Creating an InvoiceWorker job to generate the following invoice: id(#{id}), invoiced_id(#{invoiced_id}), invoiced_type(#{invoiced_type}), user_id(#{user_id})" InvoiceWorker.perform_async(id) end diff --git a/app/models/profile.rb b/app/models/profile.rb index 483fbee71..073ed582f 100644 --- a/app/models/profile.rb +++ b/app/models/profile.rb @@ -28,7 +28,7 @@ class Profile < ActiveRecord::Base def age if birthday.present? now = Time.now.utc.to_date - now.year - birthday.year - (birthday.to_date.change(:year => now.year) > now ? 1 : 0) + (now - birthday).to_f / 365.2425 else '' end diff --git a/app/services/custom_aggregation_service.rb b/app/services/custom_aggregation_service.rb new file mode 100644 index 000000000..034e237b8 --- /dev/null +++ b/app/services/custom_aggregation_service.rb @@ -0,0 +1,45 @@ +require 'json' + +class CustomAggregationService + + ## + # Run any additional custom aggregations related to the given statistic type, if any + ## + def call(statistic_index, statistic_type, start_date, end_date, custom_query, results) + if statistic_type and start_date and end_date + stat_index = StatisticIndex.find_by(es_type_key: statistic_index) + stat_type = StatisticType.find_by(statistic_index_id: stat_index.id, key: statistic_type) + client = Elasticsearch::Model.client + stat_type.statistic_custom_aggregations.each do |custom| + + query = sprintf(custom.query, {aggs_name: custom.field, start_date: start_date, end_date: end_date}) + + if custom_query and !custom_query.empty? + # Here, a custom query was provided with the original request (eg: filter by subtype) + # so we try to apply this custom filter to the current custom aggregation. + # + # The requested model mapping (ie. found in StatisticCustomAggregation.es_index > es_type) must have defined + # these fields in the indexed json, otherwise the returned value will probably not be what is expected. + # + # As an implementation exemple, you can take a look at Availability (indexed as fablab/availabilities) + # and witch will run custom filters on the fields 'date' and 'subType'. Other custom filters will return 0 + # as they are not relevant with this kind of custom aggregation. + query = JSON.parse(query) + custom_query = JSON.parse(custom_query) + + exclude = custom_query.delete('exclude') + if exclude + query = {query: { filtered: { query: query['query'], filter: { not: { term: custom_query['match'] } } } }, aggregations: query['aggregations'], size: query['size']} + else + query['query']['bool']['must'].push(custom_query) + end + query = query.to_json + end + + c_res = client.search(index: custom.es_index, type: custom.es_type, body: query) + results['aggregations'][custom.field] = c_res['aggregations'][custom.field] + end + end + results + end +end \ No newline at end of file diff --git a/app/services/statistic_service.rb b/app/services/statistic_service.rb index b824e370e..f19736ee6 100644 --- a/app/services/statistic_service.rb +++ b/app/services/statistic_service.rb @@ -297,8 +297,10 @@ class StatisticService end def clean_stat(options = default_options) + client = Elasticsearch::Model.client %w{Account Event Machine Project Subscription Training User}.each do |o| - "Stats::#{o}".constantize.search(query: {match: {date: format_date(options[:start_date])}}).results.each(&:destroy) + model = "Stats::#{o}".constantize + client.delete_by_query(index: model.index_name, type: model.document_type, body: {query: {match: {date: format_date(options[:start_date])}}}) end end diff --git a/app/workers/availability_indexer_worker.rb b/app/workers/availability_indexer_worker.rb index a902ed718..c26acadad 100644 --- a/app/workers/availability_indexer_worker.rb +++ b/app/workers/availability_indexer_worker.rb @@ -10,11 +10,18 @@ class AvailabilityIndexerWorker case operation.to_s when /index/ - record = Availability.find(record_id) - Client.index index: Availability.index_name, type: Availability.document_type, id: record.id, body: record.as_indexed_json - #puts record.as_indexed_json + begin + record = Availability.find(record_id) + Client.index index: Availability.index_name, type: Availability.document_type, id: record.id, body: record.as_indexed_json + rescue ActiveRecord::RecordNotFound + STDERR.puts "Availability id(#{record_id}) will not be indexed in ElasticSearch as it does not exists anymore in database" + end when /delete/ - Client.delete index: Availability.index_name, type: Availability.document_type, id: record_id + begin + Client.delete index: Availability.index_name, type: Availability.document_type, id: record_id + rescue Elasticsearch::Transport::Transport::Errors::NotFound + STDERR.puts "Availability id(#{record_id}) will not be deleted form ElasticSearch as it has not been already indexed" + end else raise ArgumentError, "Unknown operation '#{operation}'" end end diff --git a/config/application.yml.default b/config/application.yml.default index 4aed18a73..fbbab825d 100644 --- a/config/application.yml.default +++ b/config/application.yml.default @@ -58,7 +58,7 @@ OPENLAB_BASE_URI: 'https://openprojects.fab-manager.com' LOG_LEVEL: 'debug' ALLOWED_EXTENSIONS: pdf ai eps cad math svg stl dxf dwg obj step iges igs 3dm 3dmf doc docx png ino scad fcad skp sldprt sldasm slddrw slddrt tex latex ps -ALLOWED_MIME_TYPES: application/pdf application/postscript application/illustrator image/x-eps image/svg+xml application/sla application/dxf application/acad application/dwg application/octet-stream application/step application/iges model/iges x-world/x-3dmf application/ application/vnd.openxmlformats-officedocument.wordprocessingml.document image/png text/x-arduino text/plain application/scad application/vnd.sketchup.skp application/x-koan application/vnd-koan koan/x-skm application/vnd.koan application/x-tex application/x-latex +ALLOWED_MIME_TYPES: application/pdf application/postscript application/illustrator image/x-eps image/svg+xml application/sla application/dxf application/acad application/dwg application/octet-stream application/step application/iges model/iges x-world/x-3dmf application/vnd.openxmlformats-officedocument.wordprocessingml.document image/png text/x-arduino text/plain application/scad application/vnd.sketchup.skp application/x-koan application/vnd-koan koan/x-skm application/vnd.koan application/x-tex application/x-latex # 10485760 = 10 megabytes MAX_IMAGE_SIZE: '10485760' diff --git a/config/locales/app.public.en.yml b/config/locales/app.public.en.yml index 5a3e7d1f0..7221927ab 100644 --- a/config/locales/app.public.en.yml +++ b/config/locales/app.public.en.yml @@ -103,6 +103,9 @@ en: # Fab-manager's version version: "Version:" + # Notifications + and_NUMBER_other_notifications: "and {{NUMBER}} other notifications..." # angular interpolation + about: # about page read_the_fablab_policy: "Read the FabLab policy" diff --git a/config/locales/app.public.fr.yml b/config/locales/app.public.fr.yml index 27d7d2b5b..ad2eaeeab 100644 --- a/config/locales/app.public.fr.yml +++ b/config/locales/app.public.fr.yml @@ -102,6 +102,10 @@ fr: # Fab-manager's version version: "Version :" + + # Notifications + and_NUMBER_other_notifications: "et {{NUMBER}} autres notifications ..." # angular interpolation + about: # page à propos read_the_fablab_policy: "Consulter les règles d'utilisation du Fab Lab" diff --git a/db/seeds.rb b/db/seeds.rb index c10636074..887ab461f 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -407,7 +407,7 @@ if StatisticCustomAggregation.count == 0 es_index: 'fablab', es_type: 'availabilities', field: 'available_hours', - query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"hours_duration"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}' + query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"bookable_hours"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}' }) available_hours.save! diff --git a/test/fixtures/statistic_custom_aggregations.yml b/test/fixtures/statistic_custom_aggregations.yml index 35c1e14fd..146b437fe 100644 --- a/test/fixtures/statistic_custom_aggregations.yml +++ b/test/fixtures/statistic_custom_aggregations.yml @@ -1,7 +1,7 @@ # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html one: - query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"hours_duration"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}' + query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"bookable_hours"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}' statistic_type_id: 2 field: "available_hours" es_index: "fablab"