Merge branch 'elastic5' into dev

CHANGELOG.md

@@ -4,12 +4,15 @@
 
 - Upgraded ElasticSearch from 1.7 to 5.6
 - Updated OmniAuth to fix Hashie warnings [omniauth#872](https://github.com/omniauth/omniauth/issues/872)
+- Fix a bug: unable to filter statistics from age 0
+- Fix a bug: events categories are not reported correctly in statistics
 - Fix a security issue: dependency loofah has a vulnerability as described in [CVE-2018-8048](https://github.com/flavorjones/loofah/issues/144)
 - Fix a security issue: rails-html-sanitizer < 1.0.3 has a security vulnerability described in [CVE-2018-3741](https://nvd.nist.gov/vuln/detail/CVE-2018-3741)
 - Ensure elasticSearch indices are started with green status on new installations
 - Refactored User.as_json to remove code duplication
-- Fixed syntax and typos in README
-- [TODO DEPLOY] **REQUIRED** Please read [elastic_upgrade.md](doc/elastic_upgrade.md) for instructions on upgrading ElasticSearch. 
+- Fixed syntax and typos in README 
+- [TODO DEPLOY] **IMPORTANT** Please read [elastic_upgrade.md](doc/elastic_upgrade.md) for instructions on upgrading ElasticSearch. 
+- [TODO DEPLOY] `rake fablab:fix:categories_slugs`
 
 ## v2.6.4 2018 March 15
 

Gemfile

@@ -122,9 +122,9 @@ gem 'recurrence'
 gem 'prawn'
 gem 'prawn-table'
 
-gem 'elasticsearch-rails'
-gem 'elasticsearch-model'
-gem 'elasticsearch-persistence'
+gem 'elasticsearch-rails', '~> 5'
+gem 'elasticsearch-model', '~> 5'
+gem 'elasticsearch-persistence', '~> 5'
 
 gem 'notify_with'
 

Gemfile.lock

@@ -154,24 +154,24 @@ GEM
     docile (1.1.5)
     domain_name (0.5.25)
       unf (>= 0.0.5, < 1.0.0)
-    elasticsearch (1.0.12)
-      elasticsearch-api (= 1.0.12)
-      elasticsearch-transport (= 1.0.12)
-    elasticsearch-api (1.0.12)
+    elasticsearch (5.0.5)
+      elasticsearch-api (= 5.0.5)
+      elasticsearch-transport (= 5.0.5)
+    elasticsearch-api (5.0.5)
       multi_json
-    elasticsearch-model (0.1.7)
+    elasticsearch-model (5.0.2)
       activesupport (> 3)
-      elasticsearch (> 0.4)
+      elasticsearch (~> 5)
       hashie
-    elasticsearch-persistence (0.1.7)
-      activemodel (> 3)
-      activesupport (> 3)
-      elasticsearch (> 0.4)
-      elasticsearch-model (>= 0.1)
+    elasticsearch-persistence (5.0.2)
+      activemodel (> 4)
+      activesupport (> 4)
+      elasticsearch (~> 5)
+      elasticsearch-model (~> 5)
       hashie
       virtus
-    elasticsearch-rails (0.1.7)
-    elasticsearch-transport (1.0.12)
+    elasticsearch-rails (5.0.2)
+    elasticsearch-transport (5.0.5)
       faraday
       multi_json
     equalizer (0.0.11)
@@ -217,7 +217,7 @@ GEM
       multi_xml (>= 0.5.2)
     i18n (0.9.1)
       concurrent-ruby (~> 1.0)
-    ice_nine (0.11.1)
+    ice_nine (0.11.2)
     jbuilder (2.5.0)
       activesupport (>= 3.0.0, < 5.1)
       multi_json (~> 1.2)
@@ -520,9 +520,9 @@ DEPENDENCIES
   database_cleaner
   devise
   devise-async
-  elasticsearch-model
-  elasticsearch-persistence
-  elasticsearch-rails
+  elasticsearch-model (~> 5)
+  elasticsearch-persistence (~> 5)
+  elasticsearch-rails (~> 5)
   faker
   figaro
   font-awesome-rails

app/assets/javascripts/controllers/admin/graphs.coffee

@@ -361,7 +361,8 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro
     es.search
       "index": "stats"
       "type": esType
-      "searchType": "count"
+      "searchType": "query_then_fetch"
+      "size": 0
       "stat-type": statType
       "custom-query": ''
       "start-date": moment($scope.datePickerStart.selected).format()
@@ -386,16 +387,16 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro
   queryElasticRanking = (esType, groupKey, sortKey, callback) ->
     # handle invalid callback
     if typeof(callback) != "function"
-      console.error('[graphsController::queryElasticRanking] Error: invalid callback provided')
-      return
+      return console.error('[graphsController::queryElasticRanking] Error: invalid callback provided')
     if !esType or !groupKey or !sortKey
-      callback([], '[graphsController::queryElasticRanking] Error: invalid parameters provided')
+      return callback([], '[graphsController::queryElasticRanking] Error: invalid parameters provided')
 
     # run query
     es.search
       "index": "stats"
       "type": esType
-      "searchType": "count"
+      "searchType": "query_then_fetch"
+      "size": 0
       "body": buildElasticAggregationsRankingQuery(groupKey, sortKey, moment($scope.datePickerStart.selected), moment($scope.datePickerEnd.selected))
     , (error, response) ->
       if (error)
@@ -458,12 +459,11 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro
 
     # scale weeks on sunday as nvd3 supports only these weeks
     if interval == 'week'
-      q.aggregations.subgroups.aggregations.intervals.date_histogram['post_offset'] = '-1d'
-      q.aggregations.subgroups.aggregations.intervals.date_histogram['pre_offset'] = '-1d'
+      q.aggregations.subgroups.aggregations.intervals.date_histogram['offset'] = '-1d'
     # scale days to UTC time
     else if interval == 'day'
       offset = moment().utcOffset()
-      q.aggregations.subgroups.aggregations.intervals.date_histogram['post_offset'] = (-offset)+'m'
+      q.aggregations.subgroups.aggregations.intervals.date_histogram['offset'] = (-offset)+'m'
     q
 
 
@@ -495,17 +495,21 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro
       "aggregations":
         "subgroups":
           "terms":
-            "field": "subType"
+            "field": groupKey
+            "size": 10
+            "order":
+              "total": "desc"
           "aggregations":
+            "top_events":
+              "top_hits":
+                "size": 1
+                "sort": [
+                  { "ca": "desc" }
+                ]
             "total":
               "sum":
                 "field": "stat"
 
-    # we group the results by the custom given key (eg. by event date)
-    q.aggregations.subgroups.terms =
-      field: groupKey
-      size: 0
-
     # results must be sorted and limited later by angular
     if sortKey != 'ca'
       angular.forEach q.query.bool.must, (must) ->

app/assets/javascripts/controllers/admin/statistics.coffee.erb

@@ -437,7 +437,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
             }
           ]
     # optional date range
-    if ageMin && ageMax
+    if typeof ageMin == 'number' && typeof ageMax == 'number'
       q.query.bool.must.push
         "range":
           "age":
@@ -447,16 +447,9 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
     if custom
       criterion = buildElasticCustomCriterion(custom)
       if (custom.exclude)
-        q = "query": {
-          "filtered": {
-            "query": q.query,
-            "filter": {
-              "not": {
-                "term": criterion.match
-              }
-            }
-          }
-        }
+        q.query.bool.must_not = [
+          "term": criterion.match
+        ]
       else
         q.query.bool.must.push(criterion)
 

app/models/project.rb

@@ -98,13 +98,10 @@ class Project < ActiveRecord::Base
   def self.build_search_query_from_context(params, current_user)
     search = {
       query: {
-        filtered: {
-          filter: {
-            bool: {
-              must: [],
-              should: []
-            }
-          }
+        bool: {
+          must: [],
+          should: [],
+          filter: [],
         }
       }
     }
@@ -112,7 +109,7 @@ class Project < ActiveRecord::Base
     if params['q'].blank? # we sort by created_at if there isn't a query
       search[:sort] = { created_at: { order: :desc } }
     else # otherwise we search for the word (q) in various fields
-      search[:query][:filtered][:query] = {
+      search[:query][:bool][:must] << {
         multi_match: {
           query: params['q'],
           type: 'most_fields',
@@ -123,25 +120,25 @@ class Project < ActiveRecord::Base
 
     params.each do |name, value| # we filter by themes, components, machines
       if name =~ /(.+_id$)/
-        search[:query][:filtered][:filter][:bool][:must] << { term: { "#{name}s" => value } } if value
+        search[:query][:bool][:filter] << { term: { "#{name}s" => value } } if value
       end
     end
 
     if current_user and params.key?('from') # if use select filter 'my project' or 'my collaborations'
       if params['from'] == 'mine'
-        search[:query][:filtered][:filter][:bool][:must] << { term: { author_id: current_user.id } }
+        search[:query][:bool][:filter] << { term: { author_id: current_user.id } }
       end
       if params['from'] == 'collaboration'
-        search[:query][:filtered][:filter][:bool][:must] << { term: { user_ids: current_user.id } }
+        search[:query][:bool][:filter] << { term: { user_ids: current_user.id } }
       end
     end
 
-    if current_user # if user is connect, also display his draft projects
-      search[:query][:filtered][:filter][:bool][:should] << { term: { state: 'published' } }
-      search[:query][:filtered][:filter][:bool][:should] << { term: { author_id: current_user.id } }
-      search[:query][:filtered][:filter][:bool][:should] << { term: { user_ids: current_user.id } }
+    if current_user # if user is connected, also display his draft projects
+      search[:query][:bool][:should] << { term: { state: 'published' } }
+      search[:query][:bool][:should] << { term: { author_id: current_user.id } }
+      search[:query][:bool][:should] << { term: { user_ids: current_user.id } }
     else # otherwise display only published projects
-      search[:query][:filtered][:filter][:bool][:must] << { term: { state: 'published' } }
+      search[:query][:bool][:must] << { term: { state: 'published' } }
     end
 
     search

app/services/custom_aggregation_service.rb

@@ -29,7 +29,7 @@ class CustomAggregationService
 
           exclude = custom_query.delete('exclude')
           if exclude
-            query = {query: { filtered: { query: query['query'], filter: { not: { term: custom_query['match'] } } } }, aggregations: query['aggregations'], size: query['size']}
+            query['query']['bool']['must_not'] = [{ term: custom_query['match'] }]
           else
             query['query']['bool']['must'].push(custom_query)
           end

app/services/statistic_service.rb

@@ -242,7 +242,7 @@ class StatisticService
           date: options[:start_date].to_date,
           reservation_id: r.id,
           event_id: r.reservable.id,
-          event_type: r.reservable.category.name,
+          event_type: r.reservable.category.slug,
           event_name: r.reservable.name,
           event_date: slot.start_at.to_date,
           event_theme: (r.reservable.event_themes.first ? r.reservable.event_themes.first.name : ''),

doc/elastic_upgrade.md

@@ -5,13 +5,14 @@ To keep using fab-manager you need to upgrade your installation with the new ver
 We've wrote a script to automate the process while keeping your data integrity, but there's some requirements to understand before running it.
 
 - You need to install *curl*, *jq* and *sudo* on your system before running the script. 
-  Usually, `sudo apt update && sudo apt install curl jq sudo` will do the trick but this may change, depending upon your system.
+  Usually, `apt update && apt install curl jq sudo`, ran as root, will do the trick but this may change, depending upon your system.
 - You'll need at least 4GB of RAM for the data migration to complete successfully.
   The script will try to add 4GB of swap memory if this requirement is detected as missing but this will consume you hard disk space (see below).
 - 1,17GB of free disk space are also required to perform the data migration.
   Please ensure that you'll have enough space, considering the point above. The script won't run otherwise.
-- This script will run on any Debian compatible OS (like Ubuntu) and on MacOS X, on any other systems you'll need to perform the upgrade yourself manually.
-- If your ElasticSearch instance uses replicas shards, you can't use this script and you must perform a manual upgrade.
+- This script will run on any Linux or Macintoch systems if you installed ElasticSearch using docker or docker-compose.
+  Otherwise, only Debian compatible OS (like Ubuntu) and MacOS X are supported for classical installations. On any other cases you'll need to perform the upgrade yourself manually.
+- If your ElasticSearch instance uses replicas shards, you can't use this script and you must perform a manual upgrade (you're not concerned if you have a standard installation).
 
 Once you've understood all the points above, you can run the migration script with the following:
 
@@ -20,3 +21,7 @@ cd /apps/fabmanager
 # do not run as root, elevation will be prompted if needed
 \curl https://raw.githubusercontent.com/LaCasemate/fab-manager/master/scripts/elastic-upgrade.sh | bash
 ```
+
+For instructions regarding a manual upgrade, please refer to the official documentation:
+- https://www.elastic.co/guide/en/elasticsearch/reference/2.4/restart-upgrade.html
+- https://www.elastic.co/guide/en/elasticsearch/reference/5.6/restart-upgrade.html

lib/tasks/fablab/fix.rake

@@ -85,5 +85,26 @@ namespace :fablab do
         puts "#{Event.where(recurrence_id: failed_ids).map(&:id)}"
       end
     end
+
+    desc 'reset slug in events categories'
+    task categories_slugs: :environment do
+      Category.all.each do |cat|
+        `curl -XPOST http://#{ENV["ELASTICSEARCH_HOST"]}:9200/stats/event/_update_by_query?conflicts=proceed&refresh&wait_for_completion -d '
+        {
+          "script": {
+            "source": "ctx._source.subType = params.slug",
+            "lang": "painless",
+            "params": {
+              "slug": "#{cat.slug}"
+            }
+          },
+          "query": {
+            "term": {
+              "subType": "#{cat.name}"
+            }
+          }
+        }';`
+      end
+    end
   end
 end