rooty/fab-manager
1
0
Fork 0
mirror of https://github.com/LaCasemate/fab-manager.git synced 2024-12-01 12:24:28 +01:00
Code Issues Releases Activity

Merge branch 'elastic5' into dev

Browse Source
This commit is contained in:
Sylvain 2018-07-12 12:50:52 +02:00
commit a7cc1711aa
10 changed files with 91 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@ -4,12 +4,15 @@
- Upgraded ElasticSearch from 1.7 to 5.6
- Updated OmniAuth to fix Hashie warnings [omniauth#872](https://github.com/omniauth/omniauth/issues/872)
- Fix a bug: unable to filter statistics from age 0
- Fix a bug: events categories are not reported correctly in statistics
- Fix a security issue: dependency loofah has a vulnerability as described in [CVE-2018-8048](https://github.com/flavorjones/loofah/issues/144)
- Fix a security issue: rails-html-sanitizer < 1.0.3 has a security vulnerability described in [CVE-2018-3741](https://nvd.nist.gov/vuln/detail/CVE-2018-3741)
- Ensure elasticSearch indices are started with green status on new installations
- Refactored User.as_json to remove code duplication
- Fixed syntax and typos in README
- [TODO DEPLOY] **REQUIRED** Please read [elastic_upgrade.md](doc/elastic_upgrade.md) for instructions on upgrading ElasticSearch.
- Fixed syntax and typos in README
- [TODO DEPLOY] **IMPORTANT** Please read [elastic_upgrade.md](doc/elastic_upgrade.md) for instructions on upgrading ElasticSearch.
- [TODO DEPLOY] `rake fablab:fix:categories_slugs`
## v2.6.4 2018 March 15

6
Gemfile
View File

@ -122,9 +122,9 @@ gem 'recurrence'
gem 'prawn'
gem 'prawn-table'
gem 'elasticsearch-rails'
gem 'elasticsearch-model'
gem 'elasticsearch-persistence'
gem 'elasticsearch-rails', '~> 5'
gem 'elasticsearch-model', '~> 5'
gem 'elasticsearch-persistence', '~> 5'
gem 'notify_with'

34
Gemfile.lock
View File

@ -154,24 +154,24 @@ GEM
docile (1.1.5)
domain_name (0.5.25)
unf (>= 0.0.5, < 1.0.0)
elasticsearch (1.0.12)
elasticsearch-api (= 1.0.12)
elasticsearch-transport (= 1.0.12)
elasticsearch-api (1.0.12)
elasticsearch (5.0.5)
elasticsearch-api (= 5.0.5)
elasticsearch-transport (= 5.0.5)
elasticsearch-api (5.0.5)
multi_json
elasticsearch-model (0.1.7)
elasticsearch-model (5.0.2)
activesupport (> 3)
elasticsearch (> 0.4)
elasticsearch (~> 5)
hashie
elasticsearch-persistence (0.1.7)
activemodel (> 3)
activesupport (> 3)
elasticsearch (> 0.4)
elasticsearch-model (>= 0.1)
elasticsearch-persistence (5.0.2)
activemodel (> 4)
activesupport (> 4)
elasticsearch (~> 5)
elasticsearch-model (~> 5)
hashie
virtus
elasticsearch-rails (0.1.7)
elasticsearch-transport (1.0.12)
elasticsearch-rails (5.0.2)
elasticsearch-transport (5.0.5)
faraday
multi_json
equalizer (0.0.11)
@ -217,7 +217,7 @@ GEM
multi_xml (>= 0.5.2)
i18n (0.9.1)
concurrent-ruby (~> 1.0)
ice_nine (0.11.1)
ice_nine (0.11.2)
jbuilder (2.5.0)
activesupport (>= 3.0.0, < 5.1)
multi_json (~> 1.2)
@ -520,9 +520,9 @@ DEPENDENCIES
database_cleaner
devise
devise-async
elasticsearch-model
elasticsearch-persistence
elasticsearch-rails
elasticsearch-model (~> 5)
elasticsearch-persistence (~> 5)
elasticsearch-rails (~> 5)
faker
figaro
font-awesome-rails

32
app/assets/javascripts/controllers/admin/graphs.coffee
View File

@ -361,7 +361,8 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro
es.search
"index": "stats"
"type": esType
"searchType": "count"
"searchType": "query_then_fetch"
"size": 0
"stat-type": statType
"custom-query": ''
"start-date": moment($scope.datePickerStart.selected).format()
@ -386,16 +387,16 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro
queryElasticRanking = (esType, groupKey, sortKey, callback) ->
# handle invalid callback
if typeof(callback) != "function"
console.error('[graphsController::queryElasticRanking] Error: invalid callback provided')
return
return console.error('[graphsController::queryElasticRanking] Error: invalid callback provided')
if !esType or !groupKey or !sortKey
callback([], '[graphsController::queryElasticRanking] Error: invalid parameters provided')
return callback([], '[graphsController::queryElasticRanking] Error: invalid parameters provided')
# run query
es.search
"index": "stats"
"type": esType
"searchType": "count"
"searchType": "query_then_fetch"
"size": 0
"body": buildElasticAggregationsRankingQuery(groupKey, sortKey, moment($scope.datePickerStart.selected), moment($scope.datePickerEnd.selected))
, (error, response) ->
if (error)
@ -458,12 +459,11 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro
# scale weeks on sunday as nvd3 supports only these weeks
if interval == 'week'
q.aggregations.subgroups.aggregations.intervals.date_histogram['post_offset'] = '-1d'
q.aggregations.subgroups.aggregations.intervals.date_histogram['pre_offset'] = '-1d'
q.aggregations.subgroups.aggregations.intervals.date_histogram['offset'] = '-1d'
# scale days to UTC time
else if interval == 'day'
offset = moment().utcOffset()
q.aggregations.subgroups.aggregations.intervals.date_histogram['post_offset'] = (-offset)+'m'
q.aggregations.subgroups.aggregations.intervals.date_histogram['offset'] = (-offset)+'m'
q
@ -495,17 +495,21 @@ Application.Controllers.controller "GraphsController", ["$scope", "$state", "$ro
"aggregations":
"subgroups":
"terms":
"field": "subType"
"field": groupKey
"size": 10
"order":
"total": "desc"
"aggregations":
"top_events":
"top_hits":
"size": 1
"sort": [
{ "ca": "desc" }
]
"total":
"sum":
"field": "stat"
# we group the results by the custom given key (eg. by event date)
q.aggregations.subgroups.terms =
field: groupKey
size: 0
# results must be sorted and limited later by angular
if sortKey != 'ca'
angular.forEach q.query.bool.must, (must) ->

15
app/assets/javascripts/controllers/admin/statistics.coffee.erb
View File

@ -437,7 +437,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
}
]
# optional date range
if ageMin && ageMax
if typeof ageMin == 'number' && typeof ageMax == 'number'
q.query.bool.must.push
"range":
"age":
@ -447,16 +447,9 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
if custom
criterion = buildElasticCustomCriterion(custom)
if (custom.exclude)
q = "query": {
"filtered": {
"query": q.query,
"filter": {
"not": {
"term": criterion.match
}
}
}
}
q.query.bool.must_not = [
"term": criterion.match
]
else
q.query.bool.must.push(criterion)

29
app/models/project.rb
View File

@ -98,13 +98,10 @@ class Project < ActiveRecord::Base
def self.build_search_query_from_context(params, current_user)
search = {
query: {
filtered: {
filter: {
bool: {
must: [],
should: []
}
}
bool: {
must: [],
should: [],
filter: [],
}
}
}
@ -112,7 +109,7 @@ class Project < ActiveRecord::Base
if params['q'].blank? # we sort by created_at if there isn't a query
search[:sort] = { created_at: { order: :desc } }
else # otherwise we search for the word (q) in various fields
search[:query][:filtered][:query] = {
search[:query][:bool][:must] << {
multi_match: {
query: params['q'],
type: 'most_fields',
@ -123,25 +120,25 @@ class Project < ActiveRecord::Base
params.each do |name, value| # we filter by themes, components, machines
if name =~ /(.+_id$)/
search[:query][:filtered][:filter][:bool][:must] << { term: { "#{name}s" => value } } if value
search[:query][:bool][:filter] << { term: { "#{name}s" => value } } if value
end
end
if current_user and params.key?('from') # if use select filter 'my project' or 'my collaborations'
if params['from'] == 'mine'
search[:query][:filtered][:filter][:bool][:must] << { term: { author_id: current_user.id } }
search[:query][:bool][:filter] << { term: { author_id: current_user.id } }
end
if params['from'] == 'collaboration'
search[:query][:filtered][:filter][:bool][:must] << { term: { user_ids: current_user.id } }
search[:query][:bool][:filter] << { term: { user_ids: current_user.id } }
end
end
if current_user # if user is connect, also display his draft projects
search[:query][:filtered][:filter][:bool][:should] << { term: { state: 'published' } }
search[:query][:filtered][:filter][:bool][:should] << { term: { author_id: current_user.id } }
search[:query][:filtered][:filter][:bool][:should] << { term: { user_ids: current_user.id } }
if current_user # if user is connected, also display his draft projects
search[:query][:bool][:should] << { term: { state: 'published' } }
search[:query][:bool][:should] << { term: { author_id: current_user.id } }
search[:query][:bool][:should] << { term: { user_ids: current_user.id } }
else # otherwise display only published projects
search[:query][:filtered][:filter][:bool][:must] << { term: { state: 'published' } }
search[:query][:bool][:must] << { term: { state: 'published' } }
end
search

2
app/services/custom_aggregation_service.rb
View File

@ -29,7 +29,7 @@ class CustomAggregationService
exclude = custom_query.delete('exclude')
if exclude
query = {query: { filtered: { query: query['query'], filter: { not: { term: custom_query['match'] } } } }, aggregations: query['aggregations'], size: query['size']}
query['query']['bool']['must_not'] = [{ term: custom_query['match'] }]
else
query['query']['bool']['must'].push(custom_query)
end

2
app/services/statistic_service.rb
View File

@ -242,7 +242,7 @@ class StatisticService
date: options[:start_date].to_date,
reservation_id: r.id,
event_id: r.reservable.id,
event_type: r.reservable.category.name,
event_type: r.reservable.category.slug,
event_name: r.reservable.name,
event_date: slot.start_at.to_date,
event_theme: (r.reservable.event_themes.first ? r.reservable.event_themes.first.name : ''),

11
doc/elastic_upgrade.md
View File

@ -5,13 +5,14 @@ To keep using fab-manager you need to upgrade your installation with the new ver
We've wrote a script to automate the process while keeping your data integrity, but there's some requirements to understand before running it.
- You need to install *curl*, *jq* and *sudo* on your system before running the script.
Usually, `sudo apt update && sudo apt install curl jq sudo` will do the trick but this may change, depending upon your system.
Usually, `apt update && apt install curl jq sudo`, ran as root, will do the trick but this may change, depending upon your system.
- You'll need at least 4GB of RAM for the data migration to complete successfully.
The script will try to add 4GB of swap memory if this requirement is detected as missing but this will consume you hard disk space (see below).
- 1,17GB of free disk space are also required to perform the data migration.
Please ensure that you'll have enough space, considering the point above. The script won't run otherwise.
- This script will run on any Debian compatible OS (like Ubuntu) and on MacOS X, on any other systems you'll need to perform the upgrade yourself manually.
- If your ElasticSearch instance uses replicas shards, you can't use this script and you must perform a manual upgrade.
- This script will run on any Linux or Macintoch systems if you installed ElasticSearch using docker or docker-compose.
Otherwise, only Debian compatible OS (like Ubuntu) and MacOS X are supported for classical installations. On any other cases you'll need to perform the upgrade yourself manually.
- If your ElasticSearch instance uses replicas shards, you can't use this script and you must perform a manual upgrade (you're not concerned if you have a standard installation).
Once you've understood all the points above, you can run the migration script with the following:
@ -20,3 +21,7 @@ cd /apps/fabmanager
# do not run as root, elevation will be prompted if needed
\curl https://raw.githubusercontent.com/LaCasemate/fab-manager/master/scripts/elastic-upgrade.sh | bash
```
For instructions regarding a manual upgrade, please refer to the official documentation:
- https://www.elastic.co/guide/en/elasticsearch/reference/2.4/restart-upgrade.html
- https://www.elastic.co/guide/en/elasticsearch/reference/5.6/restart-upgrade.html

21
lib/tasks/fablab/fix.rake
View File

@ -85,5 +85,26 @@ namespace :fablab do
puts "#{Event.where(recurrence_id: failed_ids).map(&:id)}"
end
end
desc 'reset slug in events categories'
task categories_slugs: :environment do
Category.all.each do |cat|
`curl -XPOST http://#{ENV["ELASTICSEARCH_HOST"]}:9200/stats/event/_update_by_query?conflicts=proceed&refresh&wait_for_completion -d '
{
"script": {
"source": "ctx._source.subType = params.slug",
"lang": "painless",
"params": {
"slug": "#{cat.slug}"
}
},
"query": {
"term": {
"subType": "#{cat.name}"
}
}
}';`
end
end
end
end