Merge branch 'stats' into dev

app/assets/javascripts/controllers/admin/statistics.coffee.erb

@@ -55,6 +55,9 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
   ## total of the stat field for non simple types
   $scope.sumStat = 0
 
+  ## Results of custom aggregations for the current type
+  $scope.customAggs = {}
+
   ## default: results are not sorted
   $scope.sorting =
     ca: 'none'
@@ -337,6 +340,7 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
       $scope.sumCA = 0
       $scope.averageAge = 0
       $scope.sumStat = 0
+      $scope.customAggs = {}
       $scope.totalHits = null
       $scope.searchDate = new Date()
       custom = buildCustomFilterQuery()
@@ -350,6 +354,8 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
           $scope.averageAge = Math.round(res.aggregations.average_age.value * 100) / 100
           $scope.sumStat = res.aggregations.total_stat.value
           $scope.scrollId = res._scroll_id
+          for custom in $scope.type.active.custom_aggregations
+            $scope.customAggs[custom.field] = res.aggregations[custom.field].value
 
 
 
@@ -373,6 +379,9 @@ Application.Controllers.controller "StatisticsController", ["$scope", "$state",
       "type": index
       "size": RESULTS_PER_PAGE
       "scroll": ES_SCROLL_TIME+'m'
+      "stat-type": type
+      "start-date": moment($scope.datePickerStart.selected).format()
+      "end-date": moment($scope.datePickerEnd.selected).format()
       "body": buildElasticDataQuery(type, custom, $scope.agePicker.start, $scope.agePicker.end, moment($scope.datePickerStart.selected), moment($scope.datePickerEnd.selected), $scope.sorting)
     , (error, response) ->
       if (error)

app/assets/templates/admin/statistics/index.html.erb

@@ -235,6 +235,7 @@
               <li ng-show="selectedIndex.ca">{{ 'revenue_' | translate }} {{sumCA | currency}}</li>
               <li>{{ 'average_age' | translate }} {{averageAge}} {{ 'years_old' | translate }}</li>
               <li ng-if="!type.active.simple">{{ 'total' | translate }} {{type.active.label}} : {{sumStat}}</li>
+              <li ng-repeat="custom in type.active.custom_aggregations">{{ custom.field | translate }} {{customAggs[custom.field]}}</li>
             </ul>
           </div>
 

app/controllers/api/statistics_controller.rb

@@ -10,8 +10,27 @@ class API::StatisticsController < API::ApiController
     class_eval %{
       def #{path}
         authorize :statistic, :#{path}?
+
+        # remove additional parameters
+        statistic_type = request.query_parameters.delete('stat-type')
+        start_date = request.query_parameters.delete('start-date')
+        end_date = request.query_parameters.delete('end-date')
+        puts start_date, end_date
+
+        # run main query in elasticSearch
         query = MultiJson.load(request.body.read)
         results = Stats::#{path.classify}.search(query, request.query_parameters.symbolize_keys).response
+
+        # run additional custom aggregations, if any
+        stat_index = StatisticIndex.find_by_es_type_key("#{path}")
+        stat_type = StatisticType.where(statistic_index_id: stat_index.id, key: statistic_type).first
+        client = Elasticsearch::Model.client
+        stat_type.statistic_custom_aggregations.each do |custom|
+          c_res = client.search index: custom.es_index, type:custom.es_type, body:sprintf(custom.query, {aggs_name: custom.field, start_date: start_date, end_date: end_date})
+          results['aggregations'][custom.field] = c_res['aggregations'][custom.field]
+        end
+
+        # return result
         render json: results
       end
 

app/models/availability.rb

@@ -1,4 +1,10 @@
 class Availability < ActiveRecord::Base
+
+  # elastic initialisations
+  include Elasticsearch::Model
+  index_name 'fablab'
+  document_type 'availabilities'
+
   has_many :machines_availabilities, dependent: :destroy
   has_many :machines, through: :machines_availabilities
   accepts_nested_attributes_for :machines, allow_destroy: true
@@ -24,6 +30,17 @@ class Availability < ActiveRecord::Base
   validate :length_must_be_1h_minimum, unless: proc { end_at.blank? or start_at.blank? }
   validate :should_be_associated
 
+  ## elastic callbacks
+  after_save { AvailabilityIndexerWorker.perform_async(:index, self.id) }
+  after_destroy { AvailabilityIndexerWorker.perform_async(:delete, self.id) }
+
+  # elastic mapping
+  settings do
+    mappings dynamic: 'true' do
+      indexes 'available_type', analyzer: 'simple'
+    end
+  end
+
   def safe_destroy
     if available_type == 'machines'
       reservations = Reservation.where(reservable_type: 'Machine', reservable_id: machine_ids).joins(:slots).where('slots.availability_id = ?', id)
@@ -76,6 +93,13 @@ class Availability < ActiveRecord::Base
     end
   end
 
+
+  def as_indexed_json
+    json = JSON.parse(to_json)
+    json['hours_duration'] = (end_at - start_at) / (60*60)
+    json.to_json
+  end
+
   private
   def length_must_be_1h_minimum
     if end_at < (start_at + 1.hour)

app/models/project.rb

@@ -52,8 +52,8 @@ class Project < ActiveRecord::Base
 
   ## elastic
   # callbacks
-  after_save { IndexerWorker.perform_async(:index, self.id) }
-  after_destroy { IndexerWorker.perform_async(:delete, self.id) }
+  after_save { ProjectIndexerWorker.perform_async(:index, self.id) }
+  after_destroy { ProjectIndexerWorker.perform_async(:delete, self.id) }
 
   #
   settings do

app/models/statistic_custom_aggregation.rb

@@ -0,0 +1,3 @@
+class StatisticCustomAggregation < ActiveRecord::Base
+  belongs_to :statistic_type
+end

app/models/statistic_type.rb

@@ -2,4 +2,5 @@ class StatisticType < ActiveRecord::Base
   has_one :statistic_index
   has_many :statistic_type_sub_types
   has_many :statistic_sub_types, through: :statistic_type_sub_types
+  has_many :statistic_custom_aggregations
 end

app/views/api/statistics/index.json.jbuilder

@@ -5,6 +5,9 @@ json.array!(@statistics) do |s|
   end
   json.types s.statistic_types do |t|
     json.extract! t, :id, :key, :label, :graph, :simple
+    json.custom_aggregations t.statistic_custom_aggregations do |c|
+      json.extract! c, :id, :field
+    end
     json.subtypes t.statistic_sub_types do |st|
       json.extract! st, :id, :key, :label
     end

app/workers/availability_indexer_worker.rb

@@ -0,0 +1,21 @@
+class AvailabilityIndexerWorker
+  include Sidekiq::Worker
+  sidekiq_options queue: 'elasticsearch', retry: true
+
+  Logger = Sidekiq.logger.level == Logger::DEBUG ? Sidekiq.logger : nil
+  Client = Elasticsearch::Model.client
+
+  def perform(operation, record_id)
+    logger.debug [operation, "ID: #{record_id}"]
+
+    case operation.to_s
+      when /index/
+        record = Availability.find(record_id)
+        Client.index  index: Availability.index_name, type: Availability.document_type, id: record.id, body: record.as_indexed_json
+        #puts record.as_indexed_json
+      when /delete/
+        Client.delete index: Availability.index_name, type: Availability.document_type, id: record_id
+      else raise ArgumentError, "Unknown operation '#{operation}'"
+    end
+  end
+end

app/workers/project_indexer_worker.rb

@@ -1,4 +1,4 @@
-class IndexerWorker
+class ProjectIndexerWorker
   include Sidekiq::Worker
   sidekiq_options queue: 'elasticsearch', retry: true
 
@@ -12,9 +12,8 @@ class IndexerWorker
       when /index/
         record = Project.find(record_id)
         Client.index  index: Project.index_name, type: Project.document_type, id: record.id, body: record.as_indexed_json
-        #puts record.as_indexed_json
       when /delete/
-        Client.delete index: 'fablab', type: 'projects', id: record_id
+        Client.delete index: Project.index_name, type: Project.document_type, id: record_id
       else raise ArgumentError, "Unknown operation '#{operation}'"
     end
   end

config/locales/app.admin.en.yml

@@ -445,6 +445,8 @@ en:
         average_age: "Average age:"
         years_old: "Years old"
         total: "Total"
+        available_hours: "Hours available for booking:"
+        available_tickets: "Tickets available for booking:"
         gender: "Gender"
         age: "Age"
         revenue: "Revenue"

config/locales/app.admin.fr.yml

@@ -445,6 +445,8 @@ fr:
         average_age: "Âge moyen :"
         years_old: "ans"
         total: "Total"
+        available_hours: "Heures disponibles à la réservation :"
+        available_tickets: "Places disponibles à la reservation :"
         gender: "Genre"
         age: "Âge"
         revenue: "Chiffre d'affaires"

db/migrate/20160905141858_create_statistic_custom_aggregations.rb

@@ -0,0 +1,11 @@
+class CreateStatisticCustomAggregations < ActiveRecord::Migration
+  def change
+    create_table :statistic_custom_aggregations do |t|
+      t.text :query
+      t.string :path
+      t.belongs_to :statistic_type, index: true, foreign_key: true
+
+      t.timestamps null: false
+    end
+  end
+end

db/migrate/20160905142700_add_field_to_statistic_custom_aggregation.rb

@@ -0,0 +1,5 @@
+class AddFieldToStatisticCustomAggregation < ActiveRecord::Migration
+  def change
+    add_column :statistic_custom_aggregations, :field, :string
+  end
+end

db/migrate/20160906094739_remove_path_from_statistic_custom_aggregations.rb

@@ -0,0 +1,5 @@
+class RemovePathFromStatisticCustomAggregations < ActiveRecord::Migration
+  def change
+    remove_column :statistic_custom_aggregations, :path, :string
+  end
+end

db/migrate/20160906094847_add_es_index_es_type_to_statistic_custom_aggregations.rb

@@ -0,0 +1,6 @@
+class AddEsIndexEsTypeToStatisticCustomAggregations < ActiveRecord::Migration
+  def change
+    add_column :statistic_custom_aggregations, :es_index, :string
+    add_column :statistic_custom_aggregations, :es_type, :string
+  end
+end

db/migrate/20160906145713_insert_custom_aggregations.rb

@@ -0,0 +1,29 @@
+class InsertCustomAggregations < ActiveRecord::Migration
+  def change
+    # available reservations hours for machines
+    machine = StatisticIndex.find_by_es_type_key('machine')
+    machine_hours = StatisticType.find_by(key: 'hour', statistic_index_id: machine.id)
+
+    available_hours = StatisticCustomAggregation.new({
+      statistic_type_id: machine_hours.id,
+      es_index: 'fablab',
+      es_type: 'availabilities',
+      field: 'available_hours',
+      query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"hours_duration"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}'
+    })
+    available_hours.save!
+
+    # available training tickets
+    training = StatisticIndex.find_by_es_type_key('training')
+    training_bookings = StatisticType.find_by(key: 'booking', statistic_index_id: training.id)
+
+    available_tickets = StatisticCustomAggregation.new({
+      statistic_type_id: training_bookings.id,
+      es_index: 'fablab',
+      es_type: 'availabilities',
+      field: 'available_tickets',
+      query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"nb_total_places"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"training"}}]}}}'
+    })
+    available_tickets.save!
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160831084519) do
+ActiveRecord::Schema.define(version: 20160906145713) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -541,6 +541,18 @@ ActiveRecord::Schema.define(version: 20160831084519) do
   add_index "slots", ["availability_id"], name: "index_slots_on_availability_id", using: :btree
   add_index "slots", ["reservation_id"], name: "index_slots_on_reservation_id", using: :btree
 
+  create_table "statistic_custom_aggregations", force: :cascade do |t|
+    t.text     "query"
+    t.integer  "statistic_type_id"
+    t.datetime "created_at",        null: false
+    t.datetime "updated_at",        null: false
+    t.string   "field"
+    t.string   "es_index"
+    t.string   "es_type"
+  end
+
+  add_index "statistic_custom_aggregations", ["statistic_type_id"], name: "index_statistic_custom_aggregations_on_statistic_type_id", using: :btree
+
   create_table "statistic_fields", force: :cascade do |t|
     t.integer  "statistic_index_id"
     t.string   "key",                limit: 255
@@ -805,6 +817,7 @@ ActiveRecord::Schema.define(version: 20160831084519) do
   add_foreign_key "organizations", "profiles"
   add_foreign_key "prices", "groups"
   add_foreign_key "prices", "plans"
+  add_foreign_key "statistic_custom_aggregations", "statistic_types"
   add_foreign_key "tickets", "event_price_categories"
   add_foreign_key "tickets", "reservations"
   add_foreign_key "user_tags", "tags"

lib/tasks/fablab.rake

@@ -1,42 +1,42 @@
 namespace :fablab do
-  #desc "Get all stripe plans and create in fablab app"
-  #task stripe_plan: :environment do
-    #Stripe::Plan.all.data.each do |plan|
-      #unless Plan.find_by_stp_plan_id(plan.id)
-        #group = Group.friendly.find(plan.id.split('-').first)
-        #if group
-          #Plan.create(stp_plan_id: plan.id, name: plan.name, amount: plan.amount, interval: plan.interval, group_id: group.id, skip_create_stripe_plan: true)
-        #else
-          #puts plan.name + " n'a pas été créé. [error]"
-        #end
-      #end
-    #end
+  # desc "Get all stripe plans and create in fablab app"
+  # task stripe_plan: :environment do
+  #   Stripe::Plan.all.data.each do |plan|
+  #     unless Plan.find_by_stp_plan_id(plan.id)
+  #       group = Group.friendly.find(plan.id.split('-').first)
+  #       if group
+  #         Plan.create(stp_plan_id: plan.id, name: plan.name, amount: plan.amount, interval: plan.interval, group_id: group.id, skip_create_stripe_plan: true)
+  #       else
+  #         puts plan.name + " n'a pas été créé. [error]"
+  #       end
+  #     end
+  #   end
+  #
+  #   if Plan.column_names.include? "training_credit_nb"
+  #     Plan.all.each do |p|
+  #       p.update_columns(training_credit_nb: (p.interval == 'month' ? 1 : 5))
+  #     end
+  #   end
+  # end
 
-    #if Plan.column_names.include? "training_credit_nb"
-      #Plan.all.each do |p|
-        #p.update_columns(training_credit_nb: (p.interval == 'month' ? 1 : 5))
-      #end
-    #end
-  #end
-
-  desc "Regenerate the invoices"
+  desc 'Regenerate the invoices'
   task :regenerate_invoices, [:year, :month] => :environment do |task, args|
     year = args.year || Time.now.year
     month = args.month || Time.now.month
     start_date = Time.new(year.to_i, month.to_i, 1)
     end_date = start_date.next_month
     puts "-> Start regenerate the invoices between #{I18n.l start_date, format: :long} in #{I18n.l end_date-1.minute, format: :long}"
-    invoices = Invoice.only_invoice.where("created_at >= :start_date AND created_at < :end_date", {start_date: start_date, end_date: end_date}).order(created_at: :asc)
+    invoices = Invoice.only_invoice.where('created_at >= :start_date AND created_at < :end_date', {start_date: start_date, end_date: end_date}).order(created_at: :asc)
     invoices.each(&:regenerate_invoice_pdf)
-    puts "-> Done"
+    puts '-> Done'
   end
 
-  desc "Cancel stripe subscriptions"
+  desc 'Cancel stripe subscriptions'
   task cancel_subscriptions: :environment do
-    Subscription.where("expired_at >= ?", Time.now.at_beginning_of_day).each do |s|
+    Subscription.where('expired_at >= ?', Time.now.at_beginning_of_day).each do |s|
       puts "-> Start cancel subscription of #{s.user.email}"
       s.cancel
-      puts "-> Done"
+      puts '-> Done'
     end
   end
 
@@ -96,17 +96,50 @@ namespace :fablab do
       }';`
   end
 
-  desc "sync all/one project in elastic search index"
+  desc 'sync all/one project in ElasticSearch index'
   task :es_build_projects_index, [:id] => :environment do |task, args|
-    if Project.__elasticsearch__.client.indices.exists? index: 'fablab'
-      Project.__elasticsearch__.client.indices.delete index: 'fablab'
+    client = Project.__elasticsearch__.client
+    # create index if not exists
+    unless client.indices.exists? index: Project.index_name
+      client.indices.create Project.index_name
     end
-    Project.__elasticsearch__.client.indices.create index: Project.index_name, body: { settings: Project.settings.to_hash, mappings: Project.mappings.to_hash }
+    # delete doctype if exists
+    if client.indices.exists_type? index: Project.index_name, type: Project.document_type
+      client.indices.delete_mapping index: Project.index_name, type: Project.document_type
+    end
+    # create doctype
+    client.indices.put_mapping index: Project.index_name, type: Project.document_type, body: Project.mappings.to_hash
+
+    # index requested documents
     if args.id
-      IndexerWorker.perform_async(:index, id)
+      ProjectIndexerWorker.perform_async(:index, id)
     else
       Project.pluck(:id).each do |project_id|
-        IndexerWorker.perform_async(:index, project_id)
+        ProjectIndexerWorker.perform_async(:index, project_id)
+      end
+    end
+  end
+
+  desc 'sync all/one availabilities in ElasticSearch index'
+  task :es_build_availabilities_index, [:id] => :environment do |task, args|
+    client = Availability.__elasticsearch__.client
+    # create index if not exists
+    unless client.indices.exists? index: Availability.index_name
+      client.indices.create Availability.index_name
+    end
+    # delete doctype if exists
+    if client.indices.exists_type? index: Availability.index_name, type: Availability.document_type
+      client.indices.delete_mapping index: Availability.index_name, type: Availability.document_type
+    end
+    # create doctype
+    client.indices.put_mapping index: Availability.index_name,  type: Availability.document_type, body: Availability.mappings.to_hash
+
+    # index requested documents
+    if args.id
+      AvailabilityIndexerWorker.perform_async(:index, id)
+    else
+      Availability.pluck(:id).each do |availability_id|
+        AvailabilityIndexerWorker.perform_async(:index, availability_id)
       end
     end
   end
@@ -194,7 +227,7 @@ namespace :fablab do
     puts "\nUsers successfully notified\n\n"
   end
 
-  desc "generate fixtures from db"
+  desc 'generate fixtures from db'
   task generate_fixtures: :environment do
     Rails.application.eager_load!
     ActiveRecord::Base.descendants.reject { |c| c == ActiveRecord::SchemaMigration or c == PartnerPlan }.each do |ar_base|

test/fixtures/statistic_custom_aggregations.yml

@@ -0,0 +1,8 @@
+# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+one:
+  query: '{"size":0, "aggregations":{"%{aggs_name}":{"sum":{"field":"hours_duration"}}}, "query":{"bool":{"must":[{"range":{"start_at":{"gte":"%{start_date}", "lte":"%{end_date}"}}}, {"match":{"available_type":"machines"}}]}}}'
+  statistic_type_id: 2
+  field: "available_hours"
+  es_index: "fablab"
+  es_type: "availabilities"