many sidekiq fixes

Gemfile

@@ -99,6 +99,7 @@ gem 'aasm'
 gem 'sidekiq', '>= 6.0.7'
 # Recurring jobs for Sidekiq
 gem 'sidekiq-cron'
+gem 'sidekiq-unique-jobs', '~> 6.0.22'
 
 gem 'stripe', '5.1.1'
 

Gemfile.lock

@@ -389,6 +389,10 @@ GEM
     sidekiq-cron (1.1.0)
       fugit (~> 1.1)
       sidekiq (>= 4.2.1)
+    sidekiq-unique-jobs (6.0.22)
+      concurrent-ruby (~> 1.0, >= 1.0.5)
+      sidekiq (>= 4.0, < 7.0)
+      thor (~> 0)
     simplecov (0.16.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -412,7 +416,7 @@ GEM
       ffi
     term-ansicolor (1.7.1)
       tins (~> 1.0)
-    thor (1.0.1)
+    thor (0.20.3)
     thread_safe (0.3.6)
     tilt (2.0.10)
     tins (1.24.1)
@@ -517,6 +521,7 @@ DEPENDENCIES
   sha3
   sidekiq (>= 6.0.7)
   sidekiq-cron
+  sidekiq-unique-jobs (~> 6.0.22)
   spring
   spring-watcher-listen (~> 2.0.0)
   stripe (= 5.1.1)

app/assets/javascripts/controllers/admin/invoices.js.erb

@@ -24,6 +24,9 @@ Application.Controllers.controller('InvoicesController', ['$scope', '$state', 'I
     // number of invoices loaded each time we click on 'load more...'
     const INVOICES_PER_PAGE = 20;
 
+    // fake stripe secret key
+    const STRIPE_SK_HIDDEN = 'sk_test_hidden-hidden-hidden-hid';
+
     /* PUBLIC SCOPE */
 
     // default active tab
@@ -173,7 +176,7 @@ Application.Controllers.controller('InvoicesController', ['$scope', '$state', 'I
     $scope.allSettings = settings;
 
     // is the stripe private set?
-    $scope.stripeSecretKey = stripeSecretKey.isPresent;
+    $scope.stripeSecretKey = (stripeSecretKey.isPresent ? STRIPE_SK_HIDDEN : '');
 
     // Placeholding date for the invoice creation
     $scope.today = moment();
@@ -613,8 +616,8 @@ Application.Controllers.controller('InvoicesController', ['$scope', '$state', 'I
           Setting.get({ name: 'stripe_public_key' }, function (res) {
             $scope.allSettings.stripe_public_key = res.setting.value;
           })
-          Setting.isPresent({ name: 'stripe_private_key' }, function (res) {
-            $scope.stripeSecretKey = res.isPresent;
+          Setting.isPresent({ name: 'stripe_secret_key' }, function (res) {
+            $scope.stripeSecretKey = (res.isPresent ? STRIPE_SK_HIDDEN : '');
           })
         }
       })

app/assets/javascripts/router.js.erb

@@ -841,7 +841,7 @@ angular.module('application.router', ['ui.router'])
                        'accounting_Event_code', 'accounting_Event_label', 'accounting_Space_code', 'accounting_Space_label', \
                        'feature_tour_display', 'online_payment_module', 'stripe_public_key']` }).$promise;
           }],
-          stripeSecretKey: ['Setting', function (Setting) { return Setting.isPresent({ name: 'stripe_private_key' }).$promise; }],
+          stripeSecretKey: ['Setting', function (Setting) { return Setting.isPresent({ name: 'stripe_secret_key' }).$promise; }],
           invoices: [ 'Invoice', function (Invoice) {
             return Invoice.list({
               query: { number: '', customer: '', date: null, order_by: '-reference', page: 1, size: 20 }

app/assets/templates/admin/invoices/payment.html.erb

@@ -52,43 +52,45 @@
     </div>
     <div class="modal-body">
       <div class="alert alert-info" ng-bind-html="'app.admin.invoices.payment.stripe_keys_info_html' | translate"></div>
-      <div class="row m-md">
-        <label for="stripe_public_key" class="control-label">{{ 'app.admin.invoices.payment.public_key' | translate }} *</label>
-        <div class="input-group">
-          <span class="input-group-addon"><i class="fa fa-info"></i></span>
-          <input type="text"
-                 class="form-control"
-                 id="stripe_public_key"
-                 ng-model="publicKey"
-                 ng-model-options='{ debounce: 200 }'
-                 ng-change='testPublicKey()'
-                 required>
-          <span class="input-group-addon" ng-class="{'label-success': publicKeyStatus, 'label-danger text-white': !publicKeyStatus}" ng-show="publicKeyStatus !== undefined && publicKey">
-            <i class="fa fa-times" ng-show="!publicKeyStatus"></i>
-            <i class="fa fa-check" ng-show="publicKeyStatus"></i>
-          </span>
+      <form name="stripeKeysForm">
+        <div class="row m-md">
+          <label for="stripe_public_key" class="control-label">{{ 'app.admin.invoices.payment.public_key' | translate }} *</label>
+          <div class="input-group">
+            <span class="input-group-addon"><i class="fa fa-info"></i></span>
+            <input type="text"
+                   class="form-control"
+                   id="stripe_public_key"
+                   ng-model="publicKey"
+                   ng-model-options='{ debounce: 200 }'
+                   ng-change='testPublicKey()'
+                   required>
+            <span class="input-group-addon" ng-class="{'label-success': publicKeyStatus, 'label-danger text-white': !publicKeyStatus}" ng-show="publicKeyStatus !== undefined && publicKey">
+              <i class="fa fa-times" ng-show="!publicKeyStatus"></i>
+              <i class="fa fa-check" ng-show="publicKeyStatus"></i>
+            </span>
+          </div>
         </div>
-      </div>
-      <div class="row m-md">
-        <label for="stripe_secret_key" class="control-label">{{ 'app.admin.invoices.payment.secret_key' | translate }} *</label>
-        <div class="input-group">
-          <span class="input-group-addon"><i class="fa fa-key"></i></span>
-          <input type="text"
-                 class="form-control"
-                 id="stripe_secret_key"
-                 ng-model="secretKey"
-                 ng-model-options='{ debounce: 200 }'
-                 ng-change='testSecretKey()'
-                 required>
-          <span class="input-group-addon" ng-class="{'label-success': secretKeyStatus, 'label-danger text-white': !secretKeyStatus}" ng-show="secretKeyStatus !== undefined && secretKey">
-            <i class="fa fa-times" ng-show="!secretKeyStatus"></i>
-            <i class="fa fa-check" ng-show="secretKeyStatus"></i>
-          </span>
+        <div class="row m-md">
+          <label for="stripe_secret_key" class="control-label">{{ 'app.admin.invoices.payment.secret_key' | translate }} *</label>
+          <div class="input-group">
+            <span class="input-group-addon"><i class="fa fa-key"></i></span>
+            <input type="text"
+                   class="form-control"
+                   id="stripe_secret_key"
+                   ng-model="secretKey"
+                   ng-model-options='{ debounce: 200 }'
+                   ng-change='testSecretKey()'
+                   required>
+            <span class="input-group-addon" ng-class="{'label-success': secretKeyStatus, 'label-danger text-white': !secretKeyStatus}" ng-show="secretKeyStatus !== undefined && secretKey">
+              <i class="fa fa-times" ng-show="!secretKeyStatus"></i>
+              <i class="fa fa-check" ng-show="secretKeyStatus"></i>
+            </span>
+          </div>
         </div>
-      </div>
+      </form>
     </div>
     <div class="modal-footer">
-      <button class="btn btn-warning" ng-click="ok()" translate>{{ 'app.shared.buttons.confirm' }}</button>
+      <button class="btn btn-warning" ng-click="ok()" ng-disabled="stripeKeysForm.$invalid" translate>{{ 'app.shared.buttons.confirm' }}</button>
       <button class="btn btn-default" ng-click="cancel()" translate>{{ 'app.shared.buttons.cancel' }}</button>
     </div>
   </div>

app/controllers/api/settings_controller.rb

@@ -14,7 +14,7 @@ class API::SettingsController < API::ApiController
     render status: :not_modified and return if setting_params[:value] == @setting.value
 
     if @setting.save && @setting.history_values.create(value: setting_params[:value], invoicing_profile: current_user.invoicing_profile)
-      SettingService.new.after_update(@setting)
+      SettingService.after_update(@setting)
       render status: :ok
     else
       render json: @setting.errors.full_messages, status: :unprocessable_entity
@@ -30,6 +30,7 @@ class API::SettingsController < API::ApiController
 
       db_setting = Setting.find_or_initialize_by(name: setting[:name])
       db_setting.save && db_setting.history_values.create(value: setting[:value], invoicing_profile: current_user.invoicing_profile)
+      SettingService.after_update(db_setting)
       @settings.push db_setting
     end
   end

app/services/setting_service.rb

@@ -4,7 +4,7 @@
 # Due to the way the controller updates the settings, we cannot safely use ActiveRecord's callbacks (eg. after_update, after_commit...)
 # so this service provides a wrapper around these operations.
 class SettingService
-  def after_update(setting)
+  def self.after_update(setting)
     # update the stylesheet
     Stylesheet.theme&.rebuild! if %w[main_color secondary_color].include? setting.name
     Stylesheet.home_page&.rebuild! if setting.name == 'home_css'
@@ -13,6 +13,10 @@ class SettingService
     NotifyPrivacyUpdateWorker.perform_async(id) if setting.name == 'privacy_body'
 
     # sync all users on stripe
-    StripeWorker.perform_async(:sync_members) if %w[stripe_public_key stripe_secret_key].include? setting.name
+    return unless %w[stripe_public_key stripe_secret_key].include? setting.name
+
+    SyncMembersOnStripeWorker.perform_async(
+      setting.history_values.last&.invoicing_profile&.user&.id
+    )
   end
 end

app/workers/availability_indexer_worker.rb

@@ -2,7 +2,6 @@ class AvailabilityIndexerWorker
   include Sidekiq::Worker
   sidekiq_options queue: 'elasticsearch', retry: true
 
-  Logger = Sidekiq.logger.level == Logger::DEBUG ? Sidekiq.logger : nil
   Client = Elasticsearch::Model.client
 
   def perform(operation, record_id)
@@ -14,13 +13,13 @@ class AvailabilityIndexerWorker
         record = Availability.find(record_id)
         Client.index index: Availability.index_name, type: Availability.document_type, id: record.id, body: record.as_indexed_json
       rescue ActiveRecord::RecordNotFound
-        STDERR.puts "Availability id(#{record_id}) will not be indexed in ElasticSearch as it does not exists anymore in database"
+        logger.warn "Availability id(#{record_id}) will not be indexed in ElasticSearch as it does not exists anymore in database"
       end
     when /delete/
       begin
         Client.delete index: Availability.index_name, type: Availability.document_type, id: record_id
       rescue Elasticsearch::Transport::Transport::Errors::NotFound
-        STDERR.puts "Availability id(#{record_id}) will not be deleted form ElasticSearch as it has not been already indexed"
+        logger.warn "Availability id(#{record_id}) will not be deleted form ElasticSearch as it has not been already indexed"
       end
     else raise ArgumentError, "Unknown operation '#{operation}'"
     end

app/workers/openlab_worker.rb

@@ -5,8 +5,6 @@ class OpenlabWorker
   include Sidekiq::Worker
   sidekiq_options queue: 'default', retry: true
 
-  LOGGER = Sidekiq.logger.level == Logger::DEBUG ? Sidekiq.logger : nil
-
   def initialize
     client = Openlab::Client.new(app_secret: Setting.get('openlab_app_secret'))
     @projets = Openlab::Projects.new(client)
@@ -14,7 +12,7 @@ class OpenlabWorker
   end
 
   def perform(action, project_id)
-    LOGGER&.debug ['Openlab sync', action, "project ID: #{project_id}"]
+    logger.debug ['Openlab sync', action, "project ID: #{project_id}"]
 
     case action.to_s
     when /create/
@@ -29,6 +27,6 @@ class OpenlabWorker
       raise NotImplementedError
     end
 
-    LOGGER&.debug ['Openlab sync', 'RESPONSE ERROR', response.inspect] unless response.success?
+    logger.debug ['Openlab sync', 'RESPONSE ERROR', response.inspect] unless response.success?
   end
 end

app/workers/stripe_worker.rb

@@ -5,8 +5,6 @@ class StripeWorker
   include Sidekiq::Worker
   sidekiq_options queue: :stripe
 
-  LOGGER = Sidekiq.logger.level == Logger::DEBUG ? Sidekiq.logger : nil
-
   def perform(action, *params)
     send(action, *params)
   end
@@ -43,24 +41,4 @@ class StripeWorker
     cpn = Stripe::Coupon.retrieve(coupon_code)
     cpn.delete
   end
-
-  def sync_members
-    LOGGER&.debug ['StripeWorker', 'SyncMembers', 'We create all non-existing customers on stripe. This may take a while...']
-    total = User.online_payers.count
-    User.online_payers.each_with_index do |member, index|
-      LOGGER&.debug ['StripeWorker', 'SyncMembers' "#{index} / #{total}"]
-      begin
-        stp_customer = Stripe::Customer.retrieve member.stp_customer_id
-        create_stripe_customer(member.id) if stp_customer.nil? || stp_customer[:deleted]
-      rescue Stripe::InvalidRequestError
-        create_stripe_customer(member.id)
-      end
-    end
-    LOGGER&.debug ['StripeWorker', 'SyncMembers', 'Sync is done']
-    notify_user = Setting.find_by(name: 'stripe_secret_key')&.history_values&.last&.invoicing_profile&.user
-    return unless notify_user
-
-    NotificationCenter.call type: :notify_admin_members_stripe_sync,
-                            receiver: notify_user
-  end
 end

app/workers/sync_members_on_stripe_worker.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This worker perform various requests to the Stripe API (payment service)
+class SyncMembersOnStripeWorker
+  include Sidekiq::Worker
+  sidekiq_options lock: :until_executed, on_conflict: :reject, queue: :stripe
+
+  def perform(notify_user_id = nil)
+    logger.debug 'We create all non-existing customers on stripe. This may take a while...'
+    total = User.online_payers.count
+    User.online_payers.each_with_index do |member, index|
+      logger.debug "#{index} / #{total}"
+      begin
+        stp_customer = Stripe::Customer.retrieve member.stp_customer_id
+        StripeWorker.perform_async(:create_stripe_customer, member.id) if stp_customer.nil? || stp_customer[:deleted]
+      rescue Stripe::InvalidRequestError
+        StripeWorker.perform_async(:create_stripe_customer, member.id)
+      end
+    end
+    logger.debug 'Sync is done'
+    return unless notify_user_id
+
+    logger.debug "Notify user #{notify_user_id}"
+    NotificationCenter.call type: :notify_admin_members_stripe_sync,
+                            receiver: User.find(notify_user_id)
+  end
+end

app/workers/version_check_worker.rb

@@ -3,6 +3,9 @@
 # Will check the application version to ensure it is up-to-date
 class VersionCheckWorker
   include Sidekiq::Worker
+  sidekiq_options lock: :until_executed,
+                  on_conflict: :reject,
+                  queue: :system
 
   def perform
     require 'fab_hub'
@@ -10,7 +13,7 @@ class VersionCheckWorker
       res = FabHub.fab_manager_version_check
     rescue Errno::ECONNREFUSED => e
       if Rails.env.development?
-        puts "Unable to check the version, maybe FabHub is not running: #{e}"
+        logger.warn "Unable to check the version, maybe FabHub is not running: #{e}"
         return
       end
     end

config/routes.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'sidekiq/web'
+require 'sidekiq_unique_jobs/web'
 require 'sidekiq/cron/web'
 
 Rails.application.routes.draw do

lib/tasks/fablab/stripe.rake

@@ -49,7 +49,7 @@ namespace :fablab do
     desc 'sync users to the stripe database'
     task sync_members: :environment do
       puts 'We create all non-existing customers on stripe. This may take a while, please wait...'
-      StripeWorker.perform(:sync_members)
+      SyncMembersOnStripeWorker.perform
       puts 'Done'
     end