[feature] Async statistics export to XLSX

- fix tests due to removal of event_categories
- rake task for generating statistics

README.md

@@ -387,17 +387,11 @@ brew install homebrew/versions/elasticsearch17
 
 2. Every nights, the statistics for the day that just ended are built automatically at 01:00 (AM).
    See [schedule.yml](config/schedule.yml) to modify this behavior.
-   If the scheduled task wasn't executed for any reason (eg. you are in a dev environment and your computer was turned off at 1 AM), you can force the statistics data generation in ElasticSearch, running the following commands in a rails console.
+   If the scheduled task wasn't executed for any reason (eg. you are in a dev environment and your computer was turned off at 1 AM), you can force the statistics data generation in ElasticSearch, running the following command.
 
    ```bash
-   rails c
-   ```
-
-   ```ruby
-   # Here for the 200 last days
-   200.times.each do |i|
-      StatisticService.new.generate_statistic({start_date: i.day.ago.beginning_of_day,end_date: i.day.ago.end_of_day})
-   end
+   # Here for the 50 last days
+   rake fablab:generate_stats[50]
    ```
 
 <a name="backup-and-restore-elasticsearch"></a>

app/assets/templates/admin/statistics/export.html.erb

@@ -70,7 +70,7 @@
     <input name="_method" type="hidden" ng-value="method"/>
     <input name="type_key" type="hidden" ng-value="typeKey"/>
     <input name="body" type="hidden" ng-value="query"/>
-    <input type="submit" class="btn btn-info" value="{{ 'export' | translate }}" formtarget="_blank"/>
+    <input type="submit" class="btn btn-info" value="{{ 'export' | translate }}" formtarget="export-frame"/>
   </form>
   <button class="btn btn-default" ng-click="cancel()" translate>{{ 'cancel' }}</button>
 </div>

app/assets/templates/admin/statistics/index.html.erb

@@ -13,6 +13,7 @@
     <div class="col-xs-12 col-sm-12 col-md-3 b-t hide-b-md">
       <section class="heading-actions wrapper">
         <a class="btn btn-lg btn-default rounded m-t-sm text-sm" ng-click="exportToExcel()"><i class="fa fa-file-excel-o"></i></a>
+        <iframe name="export-frame" height="0" width="0" class="none"></iframe>
         <a class="btn btn-lg btn-warning bg-white b-2x rounded m-t-sm upper text-sm" ui-sref="app.admin.stats_graphs" role="button"><i class="fa fa-line-chart"></i> {{ 'evolution' | translate }}</a>
       </section>
     </div>

app/controllers/api/exports_controller.rb

@@ -0,0 +1,14 @@
+class API::ExportsController < API::ApiController
+  before_action :authenticate_user!
+  before_action :set_export, only: [:download]
+
+  def download
+    authorize @export
+    send_file File.join(Rails.root, @export.file), :type => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', :disposition => 'attachment'
+  end
+
+  private
+  def set_export
+    @export = Export.find(params[:id])
+  end
+end

app/controllers/api/statistics_controller.rb

@@ -18,26 +18,17 @@ class API::StatisticsController < API::ApiController
       def export_#{path}
         authorize :statistic, :export_#{path}?
 
-        query = MultiJson.load(params[:body])
-        type_key = params[:type_key]
-
-        @results = Elasticsearch::Model.client.search({index: 'stats', type: '#{path}', scroll: '30s', body: query})
-        scroll_id = @results['_scroll_id']
-        while @results['hits']['hits'].size != @results['hits']['total']
-          scroll_res = Elasticsearch::Model.client.scroll(scroll: '30s', scroll_id: scroll_id)
-          @results['hits']['hits'].concat(scroll_res['hits']['hits'])
-          scroll_id = scroll_res['_scroll_id']
+        export = Export.where({category:'statistics', export_type: '#{path}', query: params[:body], key: params[:type_key]}).last
+        if export.nil? || !FileTest.exist?(export.file)
+          @export = Export.new({category:'statistics', export_type: '#{path}', user: current_user, query: params[:body], key: params[:type_key]})
+          if @export.save
+            render json: {export_id: @export.id}, status: :ok
+          else
+            render json: @export.errors, status: :unprocessable_entity
+          end
+        else
+          send_file File.join(Rails.root, export.file), :type => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', :disposition => 'attachment'
         end
-
-        ids = @results['hits']['hits'].map { |u| u['_source']['userId'] }
-        @users = User.includes(:profile).where(:id => ids)
-
-        @index = StatisticIndex.find_by(es_type_key: '#{path}')
-        @type = StatisticType.find_by(key: type_key, statistic_index_id: @index.id)
-        @subtypes = @type.statistic_sub_types
-        @fields = @index.statistic_fields
-
-        render xlsx: 'export_current.xlsx', filename: "#{path}.xlsx"
       end
     }
   end
@@ -45,23 +36,17 @@ class API::StatisticsController < API::ApiController
   def export_global
     authorize :statistic, :export_global?
 
-    # query all stats with range arguments
-    query = MultiJson.load(params[:body])
-
-    @results = Elasticsearch::Model.client.search({index: 'stats', scroll: '30s', body: query})
-    scroll_id = @results['_scroll_id']
-    while @results['hits']['hits'].size != @results['hits']['total']
-      scroll_res = Elasticsearch::Model.client.scroll(scroll: '30s', scroll_id: scroll_id)
-      @results['hits']['hits'].concat(scroll_res['hits']['hits'])
-      scroll_id = scroll_res['_scroll_id']
+    export = Export.where({category:'statistics', export_type: 'global', query: params[:body]}).last
+    if export.nil? || !FileTest.exist?(export.file)
+      @export = Export.new({category:'statistics', export_type: 'global', user: current_user, query: params[:body]})
+      if @export.save
+        render json: {export_id: @export.id}, status: :ok
+      else
+        render json: @export.errors, status: :unprocessable_entity
+      end
+    else
+      send_file File.join(Rails.root, export.file), :type => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', :disposition => 'attachment'
     end
-
-    ids = @results['hits']['hits'].map { |u| u['_source']['userId'] }
-    @users = User.includes(:profile).where(:id => ids)
-
-    @indices = StatisticIndex.all.includes(:statistic_fields, :statistic_types => [:statistic_sub_types])
-
-    render xlsx: 'export_global.xlsx', filename: "statistics.xlsx"
   end
 
   def scroll

app/models/export.rb

@@ -0,0 +1,33 @@
+class Export < ActiveRecord::Base
+  require 'fileutils'
+
+  belongs_to :user
+
+  validates :category, presence: true
+  validates :export_type, presence: true
+  validates :user, presence: true
+
+  after_commit :generate_and_send_export, on: [:create]
+
+  def file
+    dir = "exports/#{category}/#{export_type}"
+
+    # create directories if they doesn't exists (exports & type & id)
+    FileUtils::mkdir_p dir
+    "#{dir}/#{self.filename}"
+  end
+
+  def filename
+    "#{export_type}-#{self.id}_#{self.created_at.strftime('%d%m%Y')}.xlsx"
+  end
+
+  private
+  def generate_and_send_export
+    case category
+      when 'statistics'
+        StatisticsExportWorker.perform_async(self.id)
+      else
+       raise NoMethodError, "Unknown export service for #{category}/#{export_type}"
+    end
+  end
+end

app/models/invoice.rb

@@ -173,7 +173,7 @@ class Invoice < ActiveRecord::Base
 
   ##
   # Check if the current invoice is about a training that was previously validated for the concerned user.
-  # In that case refunding the invoice must not be not allowed.
+  # In that case refunding the invoice shouldn't be allowed.
   # @return {Boolean}
   ##
   def prevent_refund?

app/models/notification_type.rb

@@ -39,5 +39,6 @@ class NotificationType
     notify_admin_invoicing_changed
     notify_user_wallet_is_credited
     notify_admin_user_wallet_is_credited
+    notify_admin_export_complete
   )
 end

app/models/user.rb

@@ -47,6 +47,8 @@ class User < ActiveRecord::Base
 
   has_one :wallet, dependent: :destroy
 
+  has_many :exports, dependent: :destroy
+
   # fix for create admin user
   before_save do
     self.email.downcase! if self.email

app/policies/export_policy.rb

@@ -1,5 +1,5 @@
 class ExportPolicy < Struct.new(:user, :export)
-  %w(export_reservations export_members export_subscriptions).each do |action|
+  %w(export_reservations export_members export_subscriptions download).each do |action|
     define_method "#{action}?" do
       user.is_admin?
     end

app/services/statistics_export_service.rb

@@ -0,0 +1,82 @@
+require 'abstract_controller'
+require 'action_controller'
+require 'action_view'
+require 'active_record'
+
+# require any helpers
+require './app/helpers/application_helper'
+
+class StatisticsExportService
+
+  def export_global(export)
+
+    # query all stats with range arguments
+    query = MultiJson.load(export.query)
+
+    @results = Elasticsearch::Model.client.search({index: 'stats', scroll: '30s', body: query})
+    scroll_id = @results['_scroll_id']
+    while @results['hits']['hits'].size != @results['hits']['total']
+      scroll_res = Elasticsearch::Model.client.scroll(scroll: '30s', scroll_id: scroll_id)
+      @results['hits']['hits'].concat(scroll_res['hits']['hits'])
+      scroll_id = scroll_res['_scroll_id']
+    end
+
+    ids = @results['hits']['hits'].map { |u| u['_source']['userId'] }
+    @users = User.includes(:profile).where(:id => ids)
+
+    @indices = StatisticIndex.all.includes(:statistic_fields, :statistic_types => [:statistic_sub_types])
+
+    ActionController::Base.prepend_view_path './app/views/'
+    # place data in view_assigns
+    view_assigns = {results: @results, users: @users, indices: @indices}
+    av = ActionView::Base.new(ActionController::Base.view_paths, view_assigns)
+    av.class_eval do
+      # include any needed helpers (for the view)
+      include ApplicationHelper
+    end
+
+    content = av.render template: 'exports/statistics_global.xlsx.axlsx'
+    # write content to file
+    File.open(export.file,"w+b") {|f| f.puts content }
+  end
+
+  %w(account event machine project subscription training).each do |path|
+    class_eval %{
+      def export_#{path}(export)
+
+        query = MultiJson.load(export.query)
+        type_key = export.key
+
+        @results = Elasticsearch::Model.client.search({index: 'stats', type: '#{path}', scroll: '30s', body: query})
+        scroll_id = @results['_scroll_id']
+        while @results['hits']['hits'].size != @results['hits']['total']
+          scroll_res = Elasticsearch::Model.client.scroll(scroll: '30s', scroll_id: scroll_id)
+          @results['hits']['hits'].concat(scroll_res['hits']['hits'])
+          scroll_id = scroll_res['_scroll_id']
+        end
+
+        ids = @results['hits']['hits'].map { |u| u['_source']['userId'] }
+        @users = User.includes(:profile).where(:id => ids)
+
+        @index = StatisticIndex.find_by(es_type_key: '#{path}')
+        @type = StatisticType.find_by(key: type_key, statistic_index_id: @index.id)
+        @subtypes = @type.statistic_sub_types
+        @fields = @index.statistic_fields
+
+        ActionController::Base.prepend_view_path './app/views/'
+        # place data in view_assigns
+        view_assigns = {results: @results, users: @users, index: @index, type: @type, subtypes: @subtypes, fields: @fields}
+        av = ActionView::Base.new(ActionController::Base.view_paths, view_assigns)
+        av.class_eval do
+          # include any needed helpers (for the view)
+          include ApplicationHelper
+        end
+
+        content = av.render template: 'exports/statistics_current.xlsx.axlsx'
+        # write content to file
+        File.open(export.file,"w+b") {|f| f.puts content }
+      end
+    }
+  end
+
+end

app/views/api/notifications/_notify_admin_export_complete.json.jbuilder

@@ -0,0 +1,6 @@
+json.title notification.notification_type
+json.description t('.export')+' '+
+                 t(".#{notification.attached_object.category}_#{notification.attached_object.export_type}")+' '+
+                 t('.is_over')+' '+
+                 link_to( t('.download_here'), "#{root_url}api/exports/#{notification.attached_object.id}/download" )+'.'
+json.url notification_url(notification, format: :json)

app/views/notifications_mailer/notify_admin_export_complete.html.erb

@@ -0,0 +1,10 @@
+<%= render 'notifications_mailer/shared/hello', recipient: @recipient %>
+
+<p>
+  <%= t('.body.you_asked_for_an_export') %>
+  <%= t(".body.#{@attached_object.category}_#{@attached_object.export_type}") %>.
+</p>
+<p>
+  <%= t('.body.click_to_download') %>
+  <%=link_to( t('.body.here'), "#{root_url}api/exports/#{@attached_object.id}/download", target: "_blank" )%>
+</p>

app/workers/statistics_export_worker.rb

@@ -0,0 +1,27 @@
+class StatisticsExportWorker
+  include Sidekiq::Worker
+
+  def perform(export_id)
+    export = Export.find(export_id)
+
+    unless export.user.is_admin?
+      raise SecurityError, 'Not allowed to export'
+    end
+
+    unless export.category == 'statistics'
+      raise KeyError, 'Wrong worker called'
+    end
+
+    service = StatisticsExportService.new
+    method_name = "export_#{export.export_type}"
+
+    if %w(account event machine project subscription training global).include?(export.export_type) and service.respond_to?(method_name)
+      service.public_send(method_name, export)
+
+      NotificationCenter.call type: :notify_admin_export_complete,
+                              receiver: export.user,
+                              attached_object: export
+    end
+
+  end
+end

config/locales/en.yml

@@ -250,6 +250,17 @@ en:
         your_wallet_is_credited: "Your wallet has been credited by administrator"
       notify_admin_user_wallet_is_credited:
         wallet_is_credited: "The wallet of member %{USER} has been credited %{AMOUNT}"
+      notify_admin_export_complete:
+        export: "The export"
+        statistics_global: "of all the statistics"
+        statistics_account: "of the registration statistics"
+        statistics_event: "of statistics about events"
+        statistics_machine: "of statistics about machine hours"
+        statistics_project: "of statistics about projects"
+        statistics_subscription: "of subscription statistics"
+        statistics_training: "of statistics about trainings"
+        is_over: "is over."
+        download_here: "Download here"
 
   statistics:
     # statistics tools for admins

config/locales/fr.yml

@@ -250,6 +250,17 @@ fr:
         your_wallet_is_credited: "Votre porte-monnaie a bien été crédité de %{AMOUNT} par l'administrateur"
       notify_admin_user_wallet_is_credited:
         wallet_is_credited: "Le porte-monnaie du membre %{USER} a bien été crédité de %{AMOUNT}"
+      notify_admin_export_complete:
+        export: "L'export"
+        statistics_global: "de toutes les statistiques"
+        statistics_account: "des statistiques d'inscriptions"
+        statistics_event: "des statistiques sur les évènements"
+        statistics_machine: "des statistiques d'heures machines"
+        statistics_project: "des statistiques sur les projets"
+        statistics_subscription: "des statistiques d'abonnements"
+        statistics_training: "des statistiques sur les formations"
+        is_over: "est terminé."
+        download_here: "Téléchargez ici"
 
   statistics:
     # outil de statistiques pour les administrateurs

config/locales/mails.en.yml

@@ -1,6 +1,6 @@
 en:
   layouts:
-    notifications_mailer: 
+    notifications_mailer:
       see_you_later: "See you soon on {GENDER, select, other{the}}"  # messageFormat interpolation
       sincerely: "Sincerely,"
       signature: "The Fab Lab team."
@@ -250,5 +250,19 @@ en:
       body:
         wallet_credit_html: "The wallet of member %{USER} has been credited %{AMOUNT} by administrator %{ADMIN}."
 
+    notify_admin_export_complete:
+      subject: "Export completed"
+      body:
+        you_asked_for_an_export: "You asked for an export"
+        statistics_global: "of all the statistics"
+        statistics_account: "of the registration statistics"
+        statistics_event: "of statistics about events"
+        statistics_machine: "of statistics about machine hours"
+        statistics_project: "of statistics about projects"
+        statistics_subscription: "of subscription statistics"
+        statistics_training: "of statistics about trainings"
+        click_to_download: "Excel file generated successfully. To download it, click"
+        here: "here"
+
     shared:
       hello: "Hello %{user_name}"

config/locales/mails.fr.yml

@@ -250,5 +250,19 @@ fr:
       body:
         wallet_credit_html: "Le porte-monnaie du membre %{USER} a bien été crédité de %{AMOUNT} par l'administrateur %{ADMIN}."
 
+    notify_admin_export_complete:
+      subject: "Export terminé"
+      body:
+        you_asked_for_an_export: "Vous avez demandé un export"
+        statistics_global: "de toutes les statistiques"
+        statistics_account: "des statistiques d'inscriptions"
+        statistics_event: "des statistiques sur les évènements"
+        statistics_machine: "des statistiques d'heures machines"
+        statistics_project: "des statistiques sur les projets"
+        statistics_subscription: "des statistiques d'abonnements"
+        statistics_training: "des statistiques sur les formations"
+        click_to_download: "La génération est terminée. Pour télécharger le fichier Excel, cliquez"
+        here: "ici"
+
     shared:
       hello: "Bonjour %{user_name}"

config/routes.rb

@@ -113,6 +113,9 @@ Rails.application.routes.draw do
 
     # i18n
     get 'translations/:locale/:state' => 'translations#show', :constraints => { :state => /[^\/]+/ } # allow dots in URL for 'state'
+
+    # XLSX exports
+    get 'exports/:id/download' => 'exports#download'
   end
 
   # open_api

db/migrate/20160726081931_create_exports.rb

@@ -0,0 +1,11 @@
+class CreateExports < ActiveRecord::Migration
+  def change
+    create_table :exports do |t|
+      t.string :category
+      t.string :type
+      t.string :query
+
+      t.timestamps null: false
+    end
+  end
+end

db/migrate/20160726111509_add_user_to_export.rb

@@ -0,0 +1,5 @@
+class AddUserToExport < ActiveRecord::Migration
+  def change
+    add_reference :exports, :user, index: true, foreign_key: true
+  end
+end

db/migrate/20160726131152_rename_type_to_stat_type_from_export.rb

@@ -0,0 +1,5 @@
+class RenameTypeToStatTypeFromExport < ActiveRecord::Migration
+  def change
+    rename_column :exports, :type, :export_type
+  end
+end

db/migrate/20160726144257_add_key_to_export.rb

@@ -0,0 +1,5 @@
+class AddKeyToExport < ActiveRecord::Migration
+  def change
+    add_column :exports, :key, :string
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160725135112) do
+ActiveRecord::Schema.define(version: 20160726144257) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -162,6 +162,18 @@ ActiveRecord::Schema.define(version: 20160725135112) do
   add_index "events_event_themes", ["event_id"], name: "index_events_event_themes_on_event_id", using: :btree
   add_index "events_event_themes", ["event_theme_id"], name: "index_events_event_themes_on_event_theme_id", using: :btree
 
+  create_table "exports", force: :cascade do |t|
+    t.string   "category"
+    t.string   "export_type"
+    t.string   "query"
+    t.datetime "created_at",  null: false
+    t.datetime "updated_at",  null: false
+    t.integer  "user_id"
+    t.string   "key"
+  end
+
+  add_index "exports", ["user_id"], name: "index_exports_on_user_id", using: :btree
+
   create_table "friendly_id_slugs", force: :cascade do |t|
     t.string   "slug",           limit: 255, null: false
     t.integer  "sluggable_id",               null: false
@@ -732,6 +744,7 @@ ActiveRecord::Schema.define(version: 20160725135112) do
   add_foreign_key "events", "categories"
   add_foreign_key "events_event_themes", "event_themes"
   add_foreign_key "events_event_themes", "events"
+  add_foreign_key "exports", "users"
   add_foreign_key "invoices", "wallet_transactions"
   add_foreign_key "o_auth2_mappings", "o_auth2_providers"
   add_foreign_key "open_api_calls_count_tracings", "open_api_clients"

lib/tasks/fablab.rake

@@ -213,4 +213,16 @@ namespace :fablab do
       File.write(cassette_file, cassette)
     end
   end
+
+  desc '(re)generate statistics in elasticsearch for the past period'
+  task :generate_stats, [:period] => :environment do |task, args|
+    unless args.period
+      fail 'FATAL ERROR: You must pass a number of days (=> past period) to generate statistics on'
+    end
+
+    days = args.period.to_i
+    days.times.each do |i|
+      StatisticService.new.generate_statistic({start_date: i.day.ago.beginning_of_day,end_date: i.day.ago.end_of_day})
+    end
+  end
 end

test/fixtures/events.yml

@@ -13,6 +13,7 @@ event_2:
   nb_free_places: 10
   recurrence_id: 1
   age_range_id: 2
+  category_id: 2
 
 event_3:
   id: 3
@@ -27,6 +28,7 @@ event_3:
   nb_total_places: 10
   nb_free_places: 10
   recurrence_id: 1
+  category_id: 2
 
 event_1:
   id: 1
@@ -41,3 +43,4 @@ event_1:
   nb_total_places: 10
   nb_free_places: 10
   recurrence_id: 1
+  category_id: 2

test/fixtures/events_categories.yml

@@ -1,84 +0,0 @@
-
-events_category_1:
-  id: 1
-  event_id: 1
-  category_id: 2
-  created_at: 2016-04-04 15:44:02.258622000 Z
-  updated_at: 2016-04-04 15:44:02.258622000 Z
-
-events_category_2:
-  id: 2
-  event_id: 2
-  category_id: 2
-  created_at: 2016-04-04 15:44:03.173253000 Z
-  updated_at: 2016-04-04 15:44:03.173253000 Z
-
-events_category_3:
-  id: 3
-  event_id: 2
-  category_id: 2
-  created_at: 2016-04-04 15:44:03.177952000 Z
-  updated_at: 2016-04-04 15:44:03.177952000 Z
-
-events_category_4:
-  id: 4
-  event_id: 3
-  category_id: 2
-  created_at: 2016-04-04 15:44:04.041364000 Z
-  updated_at: 2016-04-04 15:44:04.041364000 Z
-
-events_category_5:
-  id: 5
-  event_id: 3
-  category_id: 2
-  created_at: 2016-04-04 15:44:04.044667000 Z
-  updated_at: 2016-04-04 15:44:04.044667000 Z
-
-events_category_6:
-  id: 6
-  event_id: 1
-  category_id: 2
-  created_at: 2016-04-04 15:44:04.049543000 Z
-  updated_at: 2016-04-04 15:44:04.049543000 Z
-
-events_category_1:
-  id: 1
-  event_id: 1
-  category_id: 2
-  created_at: 2016-04-04 15:44:02.258622000 Z
-  updated_at: 2016-04-04 15:44:02.258622000 Z
-
-events_category_2:
-  id: 2
-  event_id: 2
-  category_id: 2
-  created_at: 2016-04-04 15:44:03.173253000 Z
-  updated_at: 2016-04-04 15:44:03.173253000 Z
-
-events_category_3:
-  id: 3
-  event_id: 2
-  category_id: 2
-  created_at: 2016-04-04 15:44:03.177952000 Z
-  updated_at: 2016-04-04 15:44:03.177952000 Z
-
-events_category_4:
-  id: 4
-  event_id: 3
-  category_id: 2
-  created_at: 2016-04-04 15:44:04.041364000 Z
-  updated_at: 2016-04-04 15:44:04.041364000 Z
-
-events_category_5:
-  id: 5
-  event_id: 3
-  category_id: 2
-  created_at: 2016-04-04 15:44:04.044667000 Z
-  updated_at: 2016-04-04 15:44:04.044667000 Z
-
-events_category_6:
-  id: 6
-  event_id: 1
-  category_id: 2
-  created_at: 2016-04-04 15:44:04.049543000 Z
-  updated_at: 2016-04-04 15:44:04.049543000 Z

test/fixtures/exports.yml

@@ -0,0 +1,21 @@
+# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+one:
+  id: 1
+  category: statistics
+  export_type: global
+  query: '{"query":{"bool":{"must":[{"range":{"date":{"gte":"2016-06-25T02:00:00+02:00","lte":"2016-07-25T23:59:59+02:00"}}}]}}}'
+  created_at: 2016-07-26 13:06:28.096552
+  updated_at: 2016-07-26 13:06:28.096552
+  user_id: 1
+  key:
+
+two:
+  id: 2
+  category: statistics
+  export_type: subscription
+  query: '{"query":{"bool":{"must":[{"term":{"type":"2592000"}},{"range":{"date":{"gte":"2016-06-25T02:00:00+02:00","lte":"2016-07-25T23:59:59+02:00"}}}]}},"sort":[{"date":{"order":"desc"}}],"aggs":{"total_ca":{"sum":{"field":"ca"}},"average_age":{"avg":{"field":"age"}},"total_stat":{"sum":{"field":"stat"}}}}'
+  created_at: 2016-07-26 14:59:02.624663
+  updated_at: 2016-07-26 14:59:02.624663
+  user_id: 1
+  key: '2592000'

test/models/export_test.rb

@@ -0,0 +1,18 @@
+require 'test_helper'
+
+class ExportTest < ActiveSupport::TestCase
+  test 'export must have a category' do
+    e = Export.create({export_type: 'global', user: User.first, query: '{"query":{"bool":{"must":[{"range":{"date":{"gte":"2016-06-25T02:00:00+02:00","lte":"2016-07-25T23:59:59+02:00"}}}]}}}'})
+    assert_raises ActiveRecord::RecordInvalid do
+      e.save!
+    end
+  end
+
+  test 'export generate an XLSX file' do
+    e = Export.create({category: 'statistics', export_type: 'global', user: User.first, query: '{"query":{"bool":{"must":[{"range":{"date":{"gte":"2016-06-25T02:00:00+02:00","lte":"2016-07-25T23:59:59+02:00"}}}]}}}'})
+    e.save!
+    VCR.use_cassette("export_generate_an_xlsx_file") do
+      assert_export_xlsx e
+    end
+  end
+end

test/test_helper.rb

@@ -70,6 +70,25 @@ class ActiveSupport::TestCase
 
     File.delete(invoice.file)
   end
+
+
+  # Force the statistics export generation worker to run NOW and check the resulting file generated.
+  # Delete the file afterwards.
+  # @param export {Export}
+  def assert_export_xlsx(export)
+    assert_not_nil export, 'Export was not created'
+
+    if export.category == 'statistics'
+      export_worker = StatisticsExportWorker.new
+      export_worker.perform(export.id)
+
+      assert File.exist?(export.file), 'Export XLSX was not generated'
+
+      File.delete(export.file)
+    else
+      skip('Unable to test export which is not of the category "statistics"')
+    end
+  end
 end
 
 class ActionDispatch::IntegrationTest

test/vcr_cassettes/export_generate_an_xlsx_file.yml

@@ -0,0 +1,153 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://localhost:9200/stats/_search?scroll=30s
+    body:
+      encoding: UTF-8
+      string: '{"query":{"bool":{"must":[{"range":{"date":{"gte":"2016-06-25T02:00:00+02:00","lte":"2016-07-25T23:59:59+02:00"}}}]}}}'
+    headers:
+      User-Agent:
+      - Faraday v0.9.1
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json; charset=UTF-8
+      Content-Length:
+      - '4119'
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        eyJfc2Nyb2xsX2lkIjoiY1hWbGNubFVhR1Z1Um1WMFkyZzdOVHN4TWpFNk9X
+        eENiRkowYTE5U1UxYzNOSFZLYTFFeE56Vk9VVHN4TWpJNk9XeENiRkowYTE5
+        U1UxYzNOSFZLYTFFeE56Vk9VVHN4TWpRNk9XeENiRkowYTE5U1UxYzNOSFZL
+        YTFFeE56Vk9VVHN4TWpNNk9XeENiRkowYTE5U1UxYzNOSFZLYTFFeE56Vk9V
+        VHN4TWpVNk9XeENiRkowYTE5U1UxYzNOSFZLYTFFeE56Vk9VVHN3T3c9PSIs
+        InRvb2siOjUsInRpbWVkX291dCI6ZmFsc2UsIl9zaGFyZHMiOnsidG90YWwi
+        OjUsInN1Y2Nlc3NmdWwiOjUsImZhaWxlZCI6MH0sImhpdHMiOnsidG90YWwi
+        OjE3LCJtYXhfc2NvcmUiOjEuMCwiaGl0cyI6W3siX2luZGV4Ijoic3RhdHMi
+        LCJfdHlwZSI6ImV2ZW50IiwiX2lkIjoiQVZZblM3QzdkaDZFOXVGdWxMcnki
+        LCJfc2NvcmUiOjEuMCwiX3NvdXJjZSI6eyJjcmVhdGVkX2F0IjoiMjAxNi0w
+        Ny0yNlQxMzowMToyNy4wOTcrMDA6MDAiLCJ1cGRhdGVkX2F0IjoiMjAxNi0w
+        Ny0yNlQxMzowMToyNy4wOTcrMDA6MDAiLCJ0eXBlIjoiYm9va2luZyIsInN1
+        YlR5cGUiOiJBdGVsaWVyIiwiZGF0ZSI6IjIwMTYtMDYtMjgiLCJzdGF0Ijox
+        LCJ1c2VySWQiOjE5LCJnZW5kZXIiOiJtYWxlIiwiYWdlIjozMCwiZ3JvdXAi
+        OiJzdHVkZW50IiwicmVzZXJ2YXRpb25JZCI6MzExNSwiY2EiOjI1LjAsIm5h
+        bWUiOiJST0JPVCBCUk9TU0UgIiwiZXZlbnRJZCI6MTczLCJldmVudERhdGUi
+        OiIyMDE2LTA3LTA3IiwiYWdlUmFuZ2UiOiIiLCJldmVudFRoZW1lIjoiIn19
+        LHsiX2luZGV4Ijoic3RhdHMiLCJfdHlwZSI6ImV2ZW50IiwiX2lkIjoiQVZZ
+        blM3REhkaDZFOXVGdWxMcnoiLCJfc2NvcmUiOjEuMCwiX3NvdXJjZSI6eyJj
+        cmVhdGVkX2F0IjoiMjAxNi0wNy0yNlQxMzowMToyNy4xMDkrMDA6MDAiLCJ1
+        cGRhdGVkX2F0IjoiMjAxNi0wNy0yNlQxMzowMToyNy4xMDkrMDA6MDAiLCJ0
+        eXBlIjoiaG91ciIsInN1YlR5cGUiOiJBdGVsaWVyIiwiZGF0ZSI6IjIwMTYt
+        MDYtMjgiLCJzdGF0IjozLCJ1c2VySWQiOjE5LCJnZW5kZXIiOiJtYWxlIiwi
+        YWdlIjozMCwiZ3JvdXAiOiJzdHVkZW50IiwicmVzZXJ2YXRpb25JZCI6MzEx
+        NSwiY2EiOjI1LjAsIm5hbWUiOiJST0JPVCBCUk9TU0UgIiwiZXZlbnRJZCI6
+        MTczLCJldmVudERhdGUiOiIyMDE2LTA3LTA3IiwiYWdlUmFuZ2UiOiIiLCJl
+        dmVudFRoZW1lIjoiIn19LHsiX2luZGV4Ijoic3RhdHMiLCJfdHlwZSI6ImFj
+        Y291bnQiLCJfaWQiOiJBVlluUzZGRWRoNkU5dUZ1bExybyIsIl9zY29yZSI6
+        MS4wLCJfc291cmNlIjp7ImNyZWF0ZWRfYXQiOiIyMDE2LTA3LTI2VDEzOjAx
+        OjIzLjEzOCswMDowMCIsInVwZGF0ZWRfYXQiOiIyMDE2LTA3LTI2VDEzOjAx
+        OjIzLjEzOCswMDowMCIsInR5cGUiOiJtZW1iZXIiLCJzdWJUeXBlIjoiY3Jl
+        YXRlZCIsImRhdGUiOiIyMDE2LTA3LTI1Iiwic3RhdCI6MSwidXNlcklkIjox
+        NjY2LCJnZW5kZXIiOiJtYWxlIiwiYWdlIjo2MywiZ3JvdXAiOiJzdGFuZGFy
+        ZCJ9fSx7Il9pbmRleCI6InN0YXRzIiwiX3R5cGUiOiJzdWJzY3JpcHRpb24i
+        LCJfaWQiOiJBVlluUzZEWWRoNkU5dUZ1bExybCIsIl9zY29yZSI6MS4wLCJf
+        c291cmNlIjp7ImNyZWF0ZWRfYXQiOiIyMDE2LTA3LTI2VDEzOjAxOjIzLjAz
+        MCswMDowMCIsInVwZGF0ZWRfYXQiOiIyMDE2LTA3LTI2VDEzOjAxOjIzLjAz
+        MCswMDowMCIsInR5cGUiOiI1MTg0MDAwIiwic3ViVHlwZSI6ImJpbWVuc3Vl
+        bC1zdGFuZGFyZC1tb250aC0yMDE2MDcyNTEyNDMyNCIsImRhdGUiOiIyMDE2
+        LTA3LTI1Iiwic3RhdCI6MSwidXNlcklkIjoxNjY2LCJnZW5kZXIiOiJtYWxl
+        IiwiYWdlIjo2MywiZ3JvdXAiOiJzdGFuZGFyZCIsImNhIjo1MC4wLCJwbGFu
+        SWQiOjEwLCJzdWJzY3JpcHRpb25JZCI6MzgwLCJpbnZvaWNlSXRlbUlkIjoz
+        NTk3LCJncm91cE5hbWUiOiJzdGFuZGFyZCwgYXNzb2NpYXRpb24ifX0seyJf
+        aW5kZXgiOiJzdGF0cyIsIl90eXBlIjoidXNlciIsIl9pZCI6IkFWWW5TNkdY
+        ZGg2RTl1RnVsTHJwIiwiX3Njb3JlIjoxLjAsIl9zb3VyY2UiOnsiY3JlYXRl
+        ZF9hdCI6IjIwMTYtMDctMjZUMTM6MDE6MjMuMjIxKzAwOjAwIiwidXBkYXRl
+        ZF9hdCI6IjIwMTYtMDctMjZUMTM6MDE6MjMuMjIxKzAwOjAwIiwidHlwZSI6
+        InJldmVudWUiLCJzdWJUeXBlIjoic3R1ZGVudCIsImRhdGUiOiIyMDE2LTA3
+        LTI1Iiwic3RhdCI6MjAsInVzZXJJZCI6NSwiZ2VuZGVyIjoiZmVtYWxlIiwi
+        YWdlIjowLCJncm91cCI6InN0dWRlbnQifX0seyJfaW5kZXgiOiJzdGF0cyIs
+        Il90eXBlIjoidHJhaW5pbmciLCJfaWQiOiJBVlluUzZod2RoNkU5dUZ1bExy
+        dCIsIl9zY29yZSI6MS4wLCJfc291cmNlIjp7ImNyZWF0ZWRfYXQiOiIyMDE2
+        LTA3LTI2VDEzOjAxOjI0Ljk3MyswMDowMCIsInVwZGF0ZWRfYXQiOiIyMDE2
+        LTA3LTI2VDEzOjAxOjI0Ljk3MyswMDowMCIsInR5cGUiOiJob3VyIiwic3Vi
+        VHlwZSI6ImRvbG9yLXNpdC1hbWV0IiwiZGF0ZSI6IjIwMTYtMDctMTMiLCJz
+        dGF0IjozLCJ1c2VySWQiOjUxMCwiZ2VuZGVyIjoibWFsZSIsImFnZSI6MzMs
+        Imdyb3VwIjoic3RhbmRhcmQiLCJyZXNlcnZhdGlvbklkIjozMTE2LCJjYSI6
+        MC4wLCJuYW1lIjoiTG9yZW0gaXBzdW0iLCJ0cmFpbmluZ0lkIjoxMCwidHJh
+        aW5pbmdEYXRlIjoiMjAxNi0wNy0xNSJ9fSx7Il9pbmRleCI6InN0YXRzIiwi
+        X3R5cGUiOiJ1c2VyIiwiX2lkIjoiQVZZblM2R3FkaDZFOXVGdWxMcnIiLCJf
+        c2NvcmUiOjEuMCwiX3NvdXJjZSI6eyJjcmVhdGVkX2F0IjoiMjAxNi0wNy0y
+        NlQxMzowMToyMy4yNDArMDA6MDAiLCJ1cGRhdGVkX2F0IjoiMjAxNi0wNy0y
+        NlQxMzowMToyMy4yNDArMDA6MDAiLCJ0eXBlIjoicmV2ZW51ZSIsInN1YlR5
+        cGUiOiJzdGFuZGFyZCIsImRhdGUiOiIyMDE2LTA3LTI1Iiwic3RhdCI6NTAs
+        InVzZXJJZCI6MTY2NiwiZ2VuZGVyIjoibWFsZSIsImFnZSI6NjMsImdyb3Vw
+        Ijoic3RhbmRhcmQifX0seyJfaW5kZXgiOiJzdGF0cyIsIl90eXBlIjoidHJh
+        aW5pbmciLCJfaWQiOiJBVlluUzZoLWRoNkU5dUZ1bExydSIsIl9zY29yZSI6
+        MS4wLCJfc291cmNlIjp7ImNyZWF0ZWRfYXQiOiIyMDE2LTA3LTI2VDEzOjAx
+        OjI0Ljk4NiswMDowMCIsInVwZGF0ZWRfYXQiOiIyMDE2LTA3LTI2VDEzOjAx
+        OjI0Ljk4NiswMDowMCIsInR5cGUiOiJib29raW5nIiwic3ViVHlwZSI6ImZv
+        cm1hdGlvbi1pbXByaW1hbnRlLTNkIiwiZGF0ZSI6IjIwMTYtMDctMTMiLCJz
+        dGF0IjoxLCJ1c2VySWQiOjMzMiwiZ2VuZGVyIjoiZmVtYWxlIiwiYWdlIjox
+        OSwiZ3JvdXAiOiJzdHVkZW50IiwicmVzZXJ2YXRpb25JZCI6MzExNywiY2Ei
+        OjI1LjAsIm5hbWUiOiJGb3JtYXRpb24gSW1wcmltYW50ZSAzRCIsInRyYWlu
+        aW5nSWQiOjEsInRyYWluaW5nRGF0ZSI6IjIwMTYtMDctMTQifX0seyJfaW5k
+        ZXgiOiJzdGF0cyIsIl90eXBlIjoidXNlciIsIl9pZCI6IkFWWW5TNmtLZGg2
+        RTl1RnVsTHJ3IiwiX3Njb3JlIjoxLjAsIl9zb3VyY2UiOnsiY3JlYXRlZF9h
+        dCI6IjIwMTYtMDctMjZUMTM6MDE6MjUuMTI4KzAwOjAwIiwidXBkYXRlZF9h
+        dCI6IjIwMTYtMDctMjZUMTM6MDE6MjUuMTI5KzAwOjAwIiwidHlwZSI6InJl
+        dmVudWUiLCJzdWJUeXBlIjoic3RhbmRhcmQiLCJkYXRlIjoiMjAxNi0wNy0x
+        MyIsInN0YXQiOjAsInVzZXJJZCI6NTEwLCJnZW5kZXIiOiJtYWxlIiwiYWdl
+        IjozMywiZ3JvdXAiOiJzdGFuZGFyZCJ9fSx7Il9pbmRleCI6InN0YXRzIiwi
+        X3R5cGUiOiJzdWJzY3JpcHRpb24iLCJfaWQiOiJBVlluUzZEQWRoNkU5dUZ1
+        bExyayIsIl9zY29yZSI6MS4wLCJfc291cmNlIjp7ImNyZWF0ZWRfYXQiOiIy
+        MDE2LTA3LTI2VDEzOjAxOjIzLjAwMSswMDowMCIsInVwZGF0ZWRfYXQiOiIy
+        MDE2LTA3LTI2VDEzOjAxOjIzLjAwMSswMDowMCIsInR5cGUiOiIyNTkyMDAw
+        Iiwic3ViVHlwZSI6InN0dWRlbnQtbW9udGgiLCJkYXRlIjoiMjAxNi0wNy0y
+        NSIsInN0YXQiOjEsInVzZXJJZCI6NjI2LCJnZW5kZXIiOiJtYWxlIiwiYWdl
+        IjoxOSwiZ3JvdXAiOiJzdHVkZW50IiwiY2EiOjI1LjAsInBsYW5JZCI6Mywi
+        c3Vic2NyaXB0aW9uSWQiOjM3OSwiaW52b2ljZUl0ZW1JZCI6MzU5NiwiZ3Jv
+        dXBOYW1lIjoiw6l0dWRpYW50LCAtIGRlIDI1IGFucywgZW5zZWlnbmFudCwg
+        ZGVtYW5kZXVyIGQnZW1wbG9pIn19XX19
+    http_version: 
+  recorded_at: Wed, 27 Jul 2016 09:25:19 GMT
+- request:
+    method: get
+    uri: http://localhost:9200/_search/scroll?scroll=30s
+    body:
+      encoding: UTF-8
+      string: cXVlcnlUaGVuRmV0Y2g7NTsxMjE6OWxCbFJ0a19SU1c3NHVKa1ExNzVOUTsxMjI6OWxCbFJ0a19SU1c3NHVKa1ExNzVOUTsxMjQ6OWxCbFJ0a19SU1c3NHVKa1ExNzVOUTsxMjM6OWxCbFJ0a19SU1c3NHVKa1ExNzVOUTsxMjU6OWxCbFJ0a19SU1c3NHVKa1ExNzVOUTswOw==
+    headers:
+      User-Agent:
+      - Faraday v0.9.1
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - application/json; charset=UTF-8
+      Content-Length:
+      - '2843'
+    body:
+      encoding: UTF-8
+      string: '{"_scroll_id":"cXVlcnlUaGVuRmV0Y2g7NTsxMjE6OWxCbFJ0a19SU1c3NHVKa1ExNzVOUTsxMjI6OWxCbFJ0a19SU1c3NHVKa1ExNzVOUTsxMjQ6OWxCbFJ0a19SU1c3NHVKa1ExNzVOUTsxMjM6OWxCbFJ0a19SU1c3NHVKa1ExNzVOUTsxMjU6OWxCbFJ0a19SU1c3NHVKa1ExNzVOUTswOw==","took":3,"timed_out":false,"_shards":{"total":5,"successful":5,"failed":0},"hits":{"total":17,"max_score":1.0,"hits":[{"_index":"stats","_type":"machine","_id":"AVYnS6EHdh6E9uFulLrm","_score":1.0,"_source":{"created_at":"2016-07-26T13:01:23.078+00:00","updated_at":"2016-07-26T13:01:23.078+00:00","type":"booking","subType":"petite-fraiseuse","date":"2016-07-25","stat":1,"userId":5,"gender":"female","age":0,"group":"student","reservationId":3118,"ca":20.0,"name":"Petite
+        Fraiseuse","machineId":5}},{"_index":"stats","_type":"training","_id":"AVYnS6iPdh6E9uFulLrv","_score":1.0,"_source":{"created_at":"2016-07-26T13:01:24.998+00:00","updated_at":"2016-07-26T13:01:24.998+00:00","type":"hour","subType":"formation-imprimante-3d","date":"2016-07-13","stat":4,"userId":332,"gender":"female","age":19,"group":"student","reservationId":3117,"ca":25.0,"name":"Formation
+        Imprimante 3D","trainingId":1,"trainingDate":"2016-07-14"}},{"_index":"stats","_type":"user","_id":"AVYnS7ENdh6E9uFulLr0","_score":1.0,"_source":{"created_at":"2016-07-26T13:01:27.180+00:00","updated_at":"2016-07-26T13:01:27.180+00:00","type":"revenue","subType":"student","date":"2016-06-28","stat":25,"userId":19,"gender":"male","age":30,"group":"student"}},{"_index":"stats","_type":"machine","_id":"AVYnS6EQdh6E9uFulLrn","_score":1.0,"_source":{"created_at":"2016-07-26T13:01:23.086+00:00","updated_at":"2016-07-26T13:01:23.086+00:00","type":"hour","subType":"petite-fraiseuse","date":"2016-07-25","stat":1,"userId":5,"gender":"female","age":0,"group":"student","reservationId":3118,"ca":20.0,"name":"Petite
+        Fraiseuse","machineId":5}},{"_index":"stats","_type":"user","_id":"AVYnS6Gfdh6E9uFulLrq","_score":1.0,"_source":{"created_at":"2016-07-26T13:01:23.230+00:00","updated_at":"2016-07-26T13:01:23.230+00:00","type":"revenue","subType":"student","date":"2016-07-25","stat":25,"userId":626,"gender":"male","age":19,"group":"student"}},{"_index":"stats","_type":"training","_id":"AVYnS6hldh6E9uFulLrs","_score":1.0,"_source":{"created_at":"2016-07-26T13:01:24.963+00:00","updated_at":"2016-07-26T13:01:24.963+00:00","type":"booking","subType":"dolor-sit-amet","date":"2016-07-13","stat":1,"userId":510,"gender":"male","age":33,"group":"standard","reservationId":3116,"ca":0.0,"name":"Lorem
+        ipsum","trainingId":10,"trainingDate":"2016-07-15"}},{"_index":"stats","_type":"user","_id":"AVYnS6kSdh6E9uFulLrx","_score":1.0,"_source":{"created_at":"2016-07-26T13:01:25.135+00:00","updated_at":"2016-07-26T13:01:25.135+00:00","type":"revenue","subType":"student","date":"2016-07-13","stat":25,"userId":332,"gender":"female","age":19,"group":"student"}}]}}'
+    http_version: 
+  recorded_at: Wed, 27 Jul 2016 09:25:19 GMT
+recorded_with: VCR 3.0.1