(bug) erroneous statistics

The date of the statistics data was using the date of the regenerate
command parameter. This was ok for the nightly builds but definitly not
for bulk regeneration

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog Fab-manager
 
+- Fix a bug: erroneous statistics
+- [TODO DEPLOY] `rails fablab:maintenance:regenerate_statistics[2021,6]`
+
 ## v5.4.21 2022 October 05
 
 - Ability to dismiss a user to a lower privileged role

app/services/statistics/cleaner_service.rb

@@ -12,7 +12,15 @@ class Statistics::CleanerService
         client.delete_by_query(
           index: model.index_name,
           type: model.document_type,
-          body: { query: { match: { date: format_date(options[:start_date]) } } }
+          body: {
+            query: {
+              terms: {
+                date: (to_date(options[:start_date]).to_date..to_date(options[:end_date]).to_date)
+                        .to_a
+                        .map { |d| format_date(d) }
+              }
+            }
+          }
         )
       end
     end

app/services/statistics/concerns/helpers_concern.rb

@@ -17,10 +17,14 @@ module Statistics::Concerns::HelpersConcern
     end
 
     def format_date(date)
+      to_date(date).strftime('%Y-%m-%d')
+    end
+
+    def to_date(date)
       if date.is_a?(String)
-        Date.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')
+        Date.strptime(date, '%Y%m%d')
       else
-        date.strftime('%Y-%m-%d')
+        date
       end
     end
 

app/services/statistics/fetcher_service.rb

@@ -23,7 +23,7 @@ class Statistics::FetcherService
         ca /= 100.00
         profile = sub.statistic_profile
         p = sub.plan
-        result.push({ date: options[:start_date].to_date,
+        result.push({ date: i.created_at.to_date,
                       plan: p.group.slug,
                       plan_id: p.id,
                       plan_interval: p.interval,
@@ -48,7 +48,7 @@ class Statistics::FetcherService
         next unless r.reservable
 
         profile = r.statistic_profile
-        result.push({ date: options[:start_date].to_date,
+        result.push({ date: r.created_at.to_date,
                       reservation_id: r.id,
                       machine_id: r.reservable.id,
                       machine_type: r.reservable.friendly_id,
@@ -69,7 +69,7 @@ class Statistics::FetcherService
         next unless r.reservable
 
         profile = r.statistic_profile
-        result.push({ date: options[:start_date].to_date,
+        result.push({ date: r.created_at.to_date,
                       reservation_id: r.id,
                       space_id: r.reservable.id,
                       space_name: r.reservable.name,
@@ -91,7 +91,7 @@ class Statistics::FetcherService
 
         profile = r.statistic_profile
         slot = r.slots.first
-        result.push({ date: options[:start_date].to_date,
+        result.push({ date: r.created_at.to_date,
                       reservation_id: r.id,
                       training_id: r.reservable.id,
                       training_type: r.reservable.friendly_id,
@@ -114,7 +114,7 @@ class Statistics::FetcherService
 
         profile = r.statistic_profile
         slot = r.slots.first
-        result.push({ date: options[:start_date].to_date,
+        result.push({ date: r.created_at.to_date,
                       reservation_id: r.id,
                       event_id: r.reservable.id,
                       event_type: r.reservable.category.slug,
@@ -140,7 +140,7 @@ class Statistics::FetcherService
         next unless r.reservable
 
         reservations_ca_list.push(
-          { date: options[:start_date].to_date, ca: calcul_ca(r.original_invoice) || 0 }.merge(user_info(r.statistic_profile))
+          { date: r.created_at.to_date, ca: calcul_ca(r.original_invoice) || 0 }.merge(user_info(r.statistic_profile))
         )
       end
       Avoir.where('invoices.created_at >= :start_date AND invoices.created_at <= :end_date', options)
@@ -148,12 +148,12 @@ class Statistics::FetcherService
            .each do |i|
         # the following line is a workaround for issue #196
         profile = i.statistic_profile || i.main_item.object&.wallet&.user&.statistic_profile
-        avoirs_ca_list.push({ date: options[:start_date].to_date, ca: calcul_avoir_ca(i) || 0 }.merge(user_info(profile)))
+        avoirs_ca_list.push({ date: i.created_at.to_date, ca: calcul_avoir_ca(i) || 0 }.merge(user_info(profile)))
       end
       reservations_ca_list.concat(subscriptions_ca_list).concat(avoirs_ca_list).each do |e|
         profile = StatisticProfile.find(e[:statistic_profile_id])
         u = find_or_create_user_info(profile, users_list)
-        u[:date] = options[:start_date].to_date
+        u[:date] = e[:date]
         add_ca(u, e[:ca], users_list)
       end
       users_list
@@ -167,7 +167,7 @@ class Statistics::FetcherService
                       .each do |sp|
         next if sp.user&.need_completion?
 
-        result.push({ date: options[:start_date].to_date }.merge(user_info(sp)))
+        result.push({ date: sp.created_at.to_date }.merge(user_info(sp)))
       end
       result
     end
@@ -177,7 +177,7 @@ class Statistics::FetcherService
       Project.where('projects.published_at >= :start_date AND projects.published_at <= :end_date', options)
              .eager_load(:licence, :themes, :components, :machines, :project_users, author: [:group])
              .each do |p|
-        result.push({ date: options[:start_date].to_date }.merge(user_info(p.author)).merge(project_info(p)))
+        result.push({ date: p.created_at.to_date }.merge(user_info(p.author)).merge(project_info(p)))
       end
       result
     end

test/services/statistic_service_test.rb

@@ -14,9 +14,46 @@ class StatisticServiceTest < ActionDispatch::IntegrationTest
   end
 
   test 'build stats' do
-    # Create a reservation to generate an invoice
+    # Create a reservation to generate an invoice (2 days ago)
     machine = Machine.find(1)
     slot = Availability.find(19).slots.first
+    travel_to(2.days.ago)
+    post '/api/local_payment/confirm_payment', params: {
+      customer_id: @user.id,
+      items: [
+        {
+          reservation: {
+            reservable_id: machine.id,
+            reservable_type: machine.class.name,
+            slots_reservations_attributes: [
+              {
+                slot_id: slot.id
+              }
+            ]
+          }
+        }
+      ]
+    }.to_json, headers: default_headers
+    travel_back
+
+    # Create a subscription to generate another invoice (1 day ago)
+    plan = Plan.find_by(group_id: @user.group.id, type: 'Plan')
+    travel_to(1.day.ago)
+    post '/api/local_payment/confirm_payment',
+         params: {
+           customer_id: @user.id,
+           items: [
+             {
+               subscription: {
+                 plan_id: plan.id
+               }
+             }
+           ]
+         }.to_json, headers: default_headers
+    travel_back
+
+    # Crate another machine reservation (today)
+    slot = Availability.find(19).slots.last
     post '/api/local_payment/confirm_payment', params: {
       customer_id: @user.id,
       items: [
@@ -34,26 +71,27 @@ class StatisticServiceTest < ActionDispatch::IntegrationTest
       ]
     }.to_json, headers: default_headers
 
-    # Create a subscription to generate another invoice
-    plan = Plan.find_by(group_id: @user.group.id, type: 'Plan')
-    post '/api/local_payment/confirm_payment',
-         params: {
-           customer_id: @user.id,
-           items: [
-             {
-               subscription: {
-                 plan_id: plan.id
-               }
-             }
-           ]
-         }.to_json, headers: default_headers
-
-    # Build the stats for today, we expect the above invoices (reservation+subscription) to appear in the resulting stats
-    ::Statistics::BuilderService.generate_statistic({ start_date: DateTime.current.beginning_of_day,
+    # Build the stats for the last 3 days, we expect the above invoices (reservations+subscription) to appear in the resulting stats
+    ::Statistics::BuilderService.generate_statistic({ start_date: 2.days.ago.beginning_of_day,
                                                       end_date: DateTime.current.end_of_day })
 
     Stats::Machine.refresh_index!
 
+    # first machine reservation (2 days ago)
+    stat_booking = Stats::Machine.search(query: { bool: { must: [{ term: { date: 2.days.ago.to_date.iso8601 } },
+                                                                 { term: { type: 'booking' } }] } }).first
+    assert_not_nil stat_booking
+    assert_equal machine.friendly_id, stat_booking['subType']
+    check_statistics_on_user(stat_booking)
+
+    stat_hour = Stats::Machine.search(query: { bool: { must: [{ term: { date: 2.days.ago.to_date.iso8601 } },
+                                                              { term: { type: 'hour' } }] } }).first
+
+    assert_not_nil stat_hour
+    assert_equal machine.friendly_id, stat_hour['subType']
+    check_statistics_on_user(stat_hour)
+
+    # second machine reservation (today)
     stat_booking = Stats::Machine.search(query: { bool: { must: [{ term: { date: DateTime.current.to_date.iso8601 } },
                                                                  { term: { type: 'booking' } }] } }).first
     assert_not_nil stat_booking
@@ -67,9 +105,10 @@ class StatisticServiceTest < ActionDispatch::IntegrationTest
     assert_equal machine.friendly_id, stat_hour['subType']
     check_statistics_on_user(stat_hour)
 
+    # subscription
     Stats::Subscription.refresh_index!
 
-    stat_subscription = Stats::Subscription.search(query: { bool: { must: [{ term: { date: DateTime.current.to_date.iso8601 } },
+    stat_subscription = Stats::Subscription.search(query: { bool: { must: [{ term: { date: 1.day.ago.to_date.iso8601 } },
                                                                            { term: { type: plan.find_statistic_type.key } }] } }).first
 
     assert_not_nil stat_subscription