Merge branch 'dev' for release 2.4.9

.fabmanager-version

@@ -1 +1 @@
-2.4.8
+2.4.9

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog Fab Manager
 
+## v2.4.9 2017 January 4
+
+- Mask new notifications alerts when more than 3
+- Added an asterisk on group select in admin's member form
+- Statistics custom aggregations can handle custom filtering
+- Statistics about hours available for machine reservations and tickets available for training reservations, now handle custom filtering on date and type
+- Fix a bug: display more than 15 unread notifications (number on the bell icon & full list)
+- Fix a bug: in invoice configuration panel, VAT amount and total excl. taxes are inverted
+- Fix a bug: unable to compute user's age when they were born on february 29th and current year is not a leap year
+- Fix a bug: wrong statistics about hours available for machines reservation. Fix requires user action (1)
+- Fix a bug: when regenerating statistics, previous values are not fully removed (only 10 firsts), resulting in wrong statistics generation (2)
+- Fix a bug: when deleting an availability just after its creation, the indexer workers crash and retries for a month
+- [TODO DEPLOY] remove possible value `application/` in `ALLOWED_MIME_TYPES` list, in environment variable
+- [TODO DEPLOY] `rails runner StatisticCustomAggregation.destroy_all`, then `rake db:seed`, then `rake fablab:es_build_availabilities_index` (1)
+- [TODO DEPLOY] `fablab:generate_stats[1095]` if you already has regenerated the statistics in the past, then they are very likely corrupted. Run this task to fix (2)
+
 ## v2.4.8 2016 December 15
 
 - Added asterisks on mandatory fields in member's form

app/assets/javascripts/controllers/admin/graphs.coffee

@@ -363,6 +363,7 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro
       "type": esType
       "searchType": "count"
       "stat-type": statType
+      "custom-query": ''
       "start-date": moment($scope.datePickerStart.selected).format()
       "end-date": moment($scope.datePickerEnd.selected).format()
       "body": buildElasticAggregationsQuery(statType, $scope.display.interval, moment($scope.datePickerStart.selected), moment($scope.datePickerEnd.selected))

app/assets/javascripts/controllers/admin/statistics.coffee.erb

@@ -380,6 +380,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
       "size": RESULTS_PER_PAGE
       "scroll": ES_SCROLL_TIME+'m'
       "stat-type": type
+      "custom-query": if custom then JSON.stringify(Object.assign({exclude: custom.exclude}, buildElasticCustomCriterion(custom))) else ''
       "start-date": moment($scope.datePickerStart.selected).format()
       "end-date": moment($scope.datePickerEnd.selected).format()
       "body": buildElasticDataQuery(type, custom, $scope.agePicker.start, $scope.agePicker.end, moment($scope.datePickerStart.selected), moment($scope.datePickerEnd.selected), $scope.sorting)
@@ -427,15 +428,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
             "lte": ageMax
     # optional criterion
     if custom
-      criterion = {
-        "match" : {}
-      }
-      switch $scope.getCustomValueInputType($scope.customFilter.criterion)
-        when 'input_date' then criterion.match[custom.key] = moment(custom.value).format('YYYY-MM-DD')
-        when 'input_select' then criterion.match[custom.key] = custom.value.key
-        when 'input_list' then criterion.match[custom.key+".name"] = custom.value
-        else criterion.match[custom.key] = custom.value
-
+      criterion = buildElasticCustomCriterion(custom)
       if (custom.exclude)
         q = "query": {
           "filtered": {
@@ -470,6 +463,27 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
 
 
 
+  ##
+  # Build the elasticSearch query DSL to match the selected cutom filter
+  # @param custom {Object} if custom is empty or undefined, an empty string will be returned
+  # @returns {{match:{*}}|string}
+  ##
+  buildElasticCustomCriterion = (custom) ->
+    if (custom)
+      criterion = {
+        "match" : {}
+      }
+      switch $scope.getCustomValueInputType($scope.customFilter.criterion)
+        when 'input_date' then criterion.match[custom.key] = moment(custom.value).format('YYYY-MM-DD')
+        when 'input_select' then criterion.match[custom.key] = custom.value.key
+        when 'input_list' then criterion.match[custom.key+".name"] = custom.value
+        else criterion.match[custom.key] = custom.value
+      criterion
+    else
+      ''
+
+
+
   ##
   # Parse the provided criteria array and return the corresponding elasticSearch syntax
   # @param criteria {Array} array of {key_to_sort:order}

app/assets/javascripts/controllers/application.coffee.erb

@@ -260,7 +260,9 @@ Application.Controllers.controller 'ApplicationController', ["$rootScope", "$sco
   getNotifications = ->
     $rootScope.toCheckNotifications = true
     unless $rootScope.checkNotificationsIsInit or !$rootScope.currentUser
-      $scope.notifications = Notification.query {is_read: false}
+      setTimeout ->
+        $scope.notifications = Notification.query {is_read: false}
+      , 2000
       $scope.$watch 'notifications', (newValue, oldValue) ->
         diff = []
         angular.forEach newValue, (value) ->
@@ -273,6 +275,9 @@ Application.Controllers.controller 'ApplicationController', ["$rootScope", "$sco
           unless find
             diff.push(value)
 
+        remain = 3
+        if diff.length >= remain
+          diff.splice(remain, (diff.length - remain), {message: {description: _t('and_NUMBER_other_notifications', {NUMBER: diff.length - remain})}})
 
         angular.forEach diff, (notification, key) ->
           growl.info(notification.message.description)

app/assets/templates/admin/invoices/index.html.erb

@@ -157,11 +157,11 @@
 
               <tr class="invoice-vat invoice-editable vat-line italic" ng-click="openEditVAT()" ng-show="invoice.VAT.active">
                   <td>{{ 'including_VAT' | translate }} {{invoice.VAT.rate}} %</td>
-                  <td>{{30/(invoice.VAT.rate/100+1) | currency}}</td>
+                  <td>{{30-(30/(invoice.VAT.rate/100+1)) | currency}}</td>
               </tr>
               <tr class="invoice-ht vat-line italic" ng-show="invoice.VAT.active">
                   <td translate>{{ 'including_total_excluding_taxes' }}</td>
-                  <td>{{30-(30/(invoice.VAT.rate/100+1)) | currency}}</td>
+                  <td>{{30/(invoice.VAT.rate/100+1) | currency}}</td>
               </tr>
               <tr class="invoice-payed vat-line bold" ng-show="invoice.VAT.active">
                   <td translate>{{ 'including_amount_payed_on_ordering' }}</td>

app/assets/templates/admin/members/_form.html.erb

@@ -1,5 +1,8 @@
 <div class="form-group" ng-class="{'has-error': userForm['user[group_id]'].$dirty && userForm['user[group_id]'].$invalid}">
-  <label for="user_group_id" class="col-sm-3 control-label" translate>{{ 'group' }}</label>
+  <label for="user_group_id" class="col-sm-3 control-label">
+    <span translate>{{ 'group' }}</span>
+    <span class="exponent"><i class="fa fa-asterisk" aria-hidden="true"></i></span>
+  </label>
   <div class="col-sm-9">
     <select ng-model="user.group_id" ng-disabled="user.subscribed_plan" class="form-control" name="user[group_id]" id="user_group_id" ng-options="g.id as g.name for g in groups" required>
     </select>

app/controllers/api/notifications_controller.rb

@@ -4,7 +4,11 @@ class API::NotificationsController < API::ApiController
 
   def index
     if params[:is_read]
-      @notifications = current_user.notifications.where(is_read: params[:is_read] == 'true').page(params[:page]).per(15).order('created_at DESC')
+      if params[:is_read] == 'true'
+        @notifications = current_user.notifications.where(is_read: true).page(params[:page]).per(15).order('created_at DESC')
+      else
+        @notifications = current_user.notifications.where(is_read: false).order('created_at DESC')
+      end
     else
       @notifications = current_user.notifications.order('created_at DESC')
     end

app/controllers/api/statistics_controller.rb

@@ -13,6 +13,7 @@ class API::StatisticsController < API::ApiController
 
         # remove additional parameters
         statistic_type = request.query_parameters.delete('stat-type')
+        custom_query = request.query_parameters.delete('custom-query')
         start_date = request.query_parameters.delete('start-date')
         end_date = request.query_parameters.delete('end-date')
 
@@ -21,15 +22,7 @@ class API::StatisticsController < API::ApiController
         results = Stats::#{path.classify}.search(query, request.query_parameters.symbolize_keys).response
 
         # run additional custom aggregations, if any
-        if statistic_type and start_date and end_date
-          stat_index = StatisticIndex.find_by(es_type_key: "#{path}")
-          stat_type = StatisticType.where(statistic_index_id: stat_index.id, key: statistic_type).first
-          client = Elasticsearch::Model.client
-          stat_type.statistic_custom_aggregations.each do |custom|
-            c_res = client.search index: custom.es_index, type:custom.es_type, body:sprintf(custom.query, {aggs_name: custom.field, start_date: start_date, end_date: end_date})
-            results['aggregations'][custom.field] = c_res['aggregations'][custom.field]
-          end
-        end
+        CustomAggregationService.new.("#{path}", statistic_type, start_date, end_date, custom_query, results)
 
         # return result
         render json: results

app/models/availability.rb

@@ -38,6 +38,7 @@ class Availability < ActiveRecord::Base
   settings do
     mappings dynamic: 'true' do
       indexes 'available_type', analyzer: 'simple'
+      indexes 'subType', index: 'not_analyzed'
     end
   end
 
@@ -96,7 +97,16 @@ class Availability < ActiveRecord::Base
 
   def as_indexed_json
     json = JSON.parse(to_json)
-    json['hours_duration'] = (end_at - start_at) / (60*60)
+    json['hours_duration'] = (end_at - start_at) / (60 * 60)
+    if available_type == 'machines'
+      json['subType'] = machines_availabilities.map{|ma| ma.machine.friendly_id}
+    elsif available_type == 'training'
+      json['subType'] = trainings_availabilities.map{|ta| ta.training.friendly_id}
+    elsif available_type == 'event'
+      json['subType'] = [event.category.friendly_id]
+    end
+    json['bookable_hours'] = json['hours_duration'] * json['subType'].length
+    json['date'] = start_at.to_date
     json.to_json
   end
 

app/models/invoice.rb

@@ -13,7 +13,7 @@ class Invoice < ActiveRecord::Base
   has_one :avoir, class_name: 'Invoice', foreign_key: :invoice_id, dependent: :destroy
 
   after_create :update_reference
-  after_commit :generate_and_send_invoice, on: [:create]
+  after_commit :generate_and_send_invoice, on: [:create], :if => :persisted?
 
   def file
     dir = "invoices/#{user.id}"
@@ -204,6 +204,7 @@ class Invoice < ActiveRecord::Base
 
   private
   def generate_and_send_invoice
+    puts "Creating an InvoiceWorker job to generate the following invoice: id(#{id}), invoiced_id(#{invoiced_id}), invoiced_type(#{invoiced_type}), user_id(#{user_id})"
     InvoiceWorker.perform_async(id)
   end
 

app/models/profile.rb

@@ -28,7 +28,7 @@ class Profile < ActiveRecord::Base
   def age
     if birthday.present?
       now = Time.now.utc.to_date
-      now.year - birthday.year - (birthday.to_date.change(:year => now.year) > now ? 1 : 0)
+      (now - birthday).to_f / 365.2425
     else
       ''
     end

app/services/custom_aggregation_service.rb

@@ -0,0 +1,45 @@
+require 'json'
+
+class CustomAggregationService
+
+  ##
+  # Run any additional custom aggregations related to the given statistic type, if any
+  ##
+  def call(statistic_index, statistic_type, start_date, end_date, custom_query, results)
+    if statistic_type and start_date and end_date
+      stat_index = StatisticIndex.find_by(es_type_key: statistic_index)
+      stat_type = StatisticType.find_by(statistic_index_id: stat_index.id, key: statistic_type)
+      client = Elasticsearch::Model.client
+      stat_type.statistic_custom_aggregations.each do |custom|
+
+        query = sprintf(custom.query, {aggs_name: custom.field, start_date: start_date, end_date: end_date})
+
+        if custom_query and !custom_query.empty?
+          # Here, a custom query was provided with the original request (eg: filter by subtype)
+          # so we try to apply this custom filter to the current custom aggregation.
+          #
+          # The requested model mapping (ie. found in StatisticCustomAggregation.es_index > es_type) must have defined
+          # these fields in the indexed json, otherwise the returned value will probably not be what is expected.
+          #
+          # As an implementation exemple, you can take a look at Availability (indexed as fablab/availabilities)
+          # and witch will run custom filters on the fields 'date' and 'subType'. Other custom filters will return 0
+          # as they are not relevant with this kind of custom aggregation.
+          query = JSON.parse(query)
+          custom_query = JSON.parse(custom_query)
+
+          exclude = custom_query.delete('exclude')
+          if exclude
+            query = {query: { filtered: { query: query['query'], filter: { not: { term: custom_query['match'] } } } }, aggregations: query['aggregations'], size: query['size']}
+          else
+            query['query']['bool']['must'].push(custom_query)
+          end
+          query = query.to_json
+        end
+
+        c_res = client.search(index: custom.es_index, type: custom.es_type, body: query)
+        results['aggregations'][custom.field] = c_res['aggregations'][custom.field]
+      end
+    end
+    results
+  end
+end

app/services/statistic_service.rb

@@ -297,8 +297,10 @@ class StatisticService
   end
 
   def clean_stat(options = default_options)
+    client = Elasticsearch::Model.client
     %w{Account Event Machine Project Subscription Training User}.each do |o|
-      "Stats::#{o}".constantize.search(query: {match: {date: format_date(options[:start_date])}}).results.each(&:destroy)
+      model = "Stats::#{o}".constantize
+      client.delete_by_query(index: model.index_name, type: model.document_type, body: {query: {match: {date: format_date(options[:start_date])}}})
     end
   end
 

app/workers/availability_indexer_worker.rb

@@ -10,11 +10,18 @@ class AvailabilityIndexerWorker
 
     case operation.to_s
       when /index/
-        record = Availability.find(record_id)
-        Client.index  index: Availability.index_name, type: Availability.document_type, id: record.id, body: record.as_indexed_json
-        #puts record.as_indexed_json
+        begin
+          record = Availability.find(record_id)
+          Client.index  index: Availability.index_name, type: Availability.document_type, id: record.id, body: record.as_indexed_json
+        rescue ActiveRecord::RecordNotFound
+          STDERR.puts "Availability id(#{record_id}) will not be indexed in ElasticSearch as it does not exists anymore in database"
+        end
       when /delete/
-        Client.delete index: Availability.index_name, type: Availability.document_type, id: record_id
+        begin
+          Client.delete index: Availability.index_name, type: Availability.document_type, id: record_id
+        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+          STDERR.puts "Availability id(#{record_id}) will not be deleted form ElasticSearch as it has not been already indexed"
+        end
       else raise ArgumentError, "Unknown operation '#{operation}'"
     end
   end

config/application.yml.default

@@ -58,7 +58,7 @@ OPENLAB_BASE_URI: 'https://openprojects.fab-manager.com'
 LOG_LEVEL: 'debug'
 
 ALLOWED_EXTENSIONS: pdf ai eps cad math svg stl dxf dwg obj step iges igs 3dm 3dmf doc docx png ino scad fcad skp sldprt sldasm slddrw slddrt tex latex ps
-ALLOWED_MIME_TYPES: application/pdf application/postscript application/illustrator image/x-eps image/svg+xml application/sla application/dxf application/acad application/dwg application/octet-stream application/step application/iges model/iges x-world/x-3dmf application/ application/vnd.openxmlformats-officedocument.wordprocessingml.document image/png text/x-arduino text/plain application/scad application/vnd.sketchup.skp application/x-koan application/vnd-koan koan/x-skm application/vnd.koan application/x-tex application/x-latex
+ALLOWED_MIME_TYPES: application/pdf application/postscript application/illustrator image/x-eps image/svg+xml application/sla application/dxf application/acad application/dwg application/octet-stream application/step application/iges model/iges x-world/x-3dmf application/vnd.openxmlformats-officedocument.wordprocessingml.document image/png text/x-arduino text/plain application/scad application/vnd.sketchup.skp application/x-koan application/vnd-koan koan/x-skm application/vnd.koan application/x-tex application/x-latex
 
 # 10485760 = 10 megabytes
 MAX_IMAGE_SIZE: '10485760'

config/locales/app.public.en.yml

@@ -103,6 +103,9 @@ en:
         # Fab-manager's version
         version: "Version:"
 
+        # Notifications
+        and_NUMBER_other_notifications: "and {{NUMBER}} other notifications..."  # angular interpolation
+
       about:
         # about page
         read_the_fablab_policy: "Read the FabLab policy"

config/locales/app.public.fr.yml

@@ -102,6 +102,10 @@ fr:
 
         # Fab-manager's version
         version: "Version :"
+
+        # Notifications
+        and_NUMBER_other_notifications: "et {{NUMBER}} autres notifications ..."  # angular interpolation
+
       about:
         # page à propos
         read_the_fablab_policy: "Consulter les règles d'utilisation du Fab Lab"

db/seeds.rb

@@ -407,7 +407,7 @@ if StatisticCustomAggregation.count == 0
                        es_index: 'fablab',
                        es_type: 'availabilities',
                        field: 'available_hours',
-                       query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"hours_duration"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}'
+                       query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"bookable_hours"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}'
                     })
   available_hours.save!
 

test/fixtures/statistic_custom_aggregations.yml

@@ -1,7 +1,7 @@
 # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
 
 one:
-  query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"hours_duration"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}'
+  query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"bookable_hours"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}'
   statistic_type_id: 2
   field: "available_hours"
   es_index: "fablab"