upload csv file to the server through the API and save it on the disk

.dockerignore

@@ -26,6 +26,12 @@ public/assets
 # PDF invoices
 invoices
 
+# Excel exports
+exports
+
+# CSV imports
+imports
+
 .DS_Store
 
 # Development files

.gitignore

@@ -34,6 +34,9 @@
 # XLSX exports
 /exports/*
 
+# CSV imports
+/imports/*
+
 # Archives of cLosed accounting periods
 /accounting/*
 

CHANGELOG.md

@@ -13,6 +13,8 @@
 - [TODO DEPLOY] -> (only dev) yarn install
 - [TODO DEPLOY] add `RECAPTCHA_SITE_KEY` and `RECAPTCHA_SECRET_KEY` environment variables (see [doc/environment.md](doc/environment.md) for configuration details)
 - [TODO DEPLOY] add `MAX_CAO_SIZE` environment variable (see [doc/environment.md](doc/environment.md) for configuration details)
+- [TODO DEPLOY] add `MAX_IMPORT_SIZE` environment variable (see [doc/environment.md](doc/environment.md) for configuration details)
+- [TODO DEPLOY] add `- ${PWD}/imports:/usr/src/app/imports` in the volumes list of your fabmanager service in [docker-compose.yml](docker/docker-compose.yml)
 
 ## v4.1.0 2019 September 12
 

Dockerfile

@@ -53,6 +53,7 @@ RUN mkdir -p /usr/src/app && \
     mkdir -p /usr/src/app/config && \
     mkdir -p /usr/src/app/invoices && \
     mkdir -p /usr/src/app/exports && \
+    mkdir -p /usr/src/app/imports && \
     mkdir -p /usr/src/app/log && \
     mkdir -p /usr/src/app/public/uploads && \
     mkdir -p /usr/src/app/public/assets && \
@@ -66,6 +67,7 @@ COPY . /usr/src/app
 # Volumes
 VOLUME /usr/src/app/invoices
 VOLUME /usr/src/app/exports
+VOLUME /usr/src/app/imports
 VOLUME /usr/src/app/public
 VOLUME /usr/src/app/public/uploads
 VOLUME /usr/src/app/public/assets

app/assets/templates/admin/members/import.html.erb

@@ -154,7 +154,7 @@
               <span class="input-group-addon btn btn-default btn-file"><span class="fileinput-new" translate>{{ 'members_import.select_file' }}</span>
               <span class="fileinput-exists" translate>{{ 'change' }}</span>
                 <input type="file"
-                       name="import-members"
+                       name="import_members"
                        accept="text/csv"></span>
               <a class="input-group-addon btn btn-danger fileinput-exists" data-dismiss="fileinput" ng-click="deleteFile(file)"><i class="fa fa-trash-o"></i></a>
             </div>

app/assets/templates/admin/members/index.html.erb

@@ -12,7 +12,7 @@
     </div>
     <div class="col-xs-1 col-xs-offset-1 col-md-offset-2 b-l">
       <section class="heading-actions wrapper">
-        <a role="button" class="btn btn-default b-2x rounded m-t-sm m-r-sm pull-right" ui-sref="app.admin.members_import">
+        <a role="button" class="btn btn-default b-2x rounded m-t-sm m-r-lg pull-right" ui-sref="app.admin.members_import">
           <i class="fa fa-cloud-upload"></i>
         </a>
       </section>

app/controllers/api/members_controller.rb

@@ -181,6 +181,18 @@ class API::MembersController < API::ApiController
     @members = User.includes(:profile)
   end
 
+  def import
+    authorize User
+
+    @import = Import.new(attachment: import_params, author: current_user)
+    if @import.save
+      Members::ImportService.import(@import)
+      render json: @import, status: :created
+    else
+      render json: @import.errors, status: :unprocessable_entity
+    end
+  end
+
   private
 
   def set_member
@@ -225,4 +237,8 @@ class API::MembersController < API::ApiController
   def query_params
     params.require(:query).permit(:search, :order_by, :page, :size)
   end
+
+  def import_params
+    params.require(:import_members)
+  end
 end

app/models/import.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'file_size_validator'
+
+# An Import is a file uploaded by an user that provides some data to the database.
+# Currently, this is used to import some users from a CSV file
+class Import < ActiveRecord::Base
+  mount_uploader :attachment, ImportUploader
+
+  belongs_to :author, foreign_key: :author_id, class_name: 'User'
+
+  validates :attachment, file_size: { maximum: Rails.application.secrets.max_import_size&.to_i || 5.megabytes.to_i }
+  validates :attachment, file_mime_type: { content_type: ['text/csv'] }
+end

app/models/project_cao.rb

@@ -4,12 +4,6 @@
 class ProjectCao < Asset
   mount_uploader :attachment, ProjectCaoUploader
 
-  validates :attachment, file_size: { maximum: max_size }
+  validates :attachment, file_size: { maximum: Rails.application.secrets.max_cao_size&.to_i || 5.megabytes.to_i }
   validates :attachment, file_mime_type: { content_type: ENV['ALLOWED_MIME_TYPES'].split(' ') }
-
-  private
-
-  def max_size
-    Rails.application.secrets.max_cao_size&.to_i || 5.megabytes.to_i
-  end
 end

app/policies/user_policy.rb

@@ -27,7 +27,7 @@ class UserPolicy < ApplicationPolicy
     user.id == record.id
   end
 
-  %w[list create mapping].each do |action|
+  %w[list create mapping import].each do |action|
     define_method "#{action}?" do
       user.admin?
     end

app/services/members/import_service.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Provides helper methods to bulk-import some users from a CSV file
+class Members::ImportService
+  class << self
+    def import(import)
+      puts import
+    end
+  end
+end

app/uploaders/import_uploader.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# CarrierWave uploader for import files.
+# This file defines the parameters for these uploads
+class ImportUploader < CarrierWave::Uploader::Base
+  include UploadHelper
+
+  # Choose what kind of storage to use for this uploader:
+  storage :file
+  after :remove, :delete_empty_dirs
+
+  # Override the directory where uploaded files will be stored.
+  # This is a sensible default for uploaders that are meant to be mounted:
+
+  def store_dir
+    "#{base_store_dir}/#{model.id}"
+  end
+
+  def base_store_dir
+    '../imports'
+  end
+
+  # Add a white list of extensions which are allowed to be uploaded.
+  # For images you might use something like this:
+  def extension_white_list
+    ['csv']
+  end
+end

config/application.yml.default

@@ -70,6 +70,8 @@ SUPERADMIN_EMAIL: 'admin@sleede.com'
 ALLOWED_EXTENSIONS: pdf ai eps cad math svg stl dxf dwg obj step iges igs 3dm 3dmf doc docx png ino scad fcad skp sldprt sldasm slddrw slddrt tex latex ps
 ALLOWED_MIME_TYPES: application/pdf application/postscript application/illustrator image/x-eps image/svg+xml application/sla application/dxf application/acad application/dwg application/octet-stream application/step application/iges model/iges x-world/x-3dmf application/vnd.openxmlformats-officedocument.wordprocessingml.document image/png text/x-arduino text/plain application/scad application/vnd.sketchup.skp application/x-koan application/vnd-koan koan/x-skm application/vnd.koan application/x-tex application/x-latex
 
+# 5242880 = 5 megabytes
+MAX_IMPORT_SIZE: '5242880'
 # 10485760 = 10 megabytes
 MAX_IMAGE_SIZE: '10485760'
 # 20971520 = 20 megabytes

config/routes.rb

@@ -55,6 +55,7 @@ Rails.application.routes.draw do
       post 'list', action: 'list', on: :collection
       get 'search/:query', action: 'search', on: :collection
       get 'mapping', action: 'mapping', on: :collection
+      post 'import', action: 'import', on: :collection
     end
     resources :reservations, only: %i[show create index update]
     resources :notifications, only: %i[index show update] do

config/secrets.yml

@@ -44,6 +44,7 @@ development:
   elaticsearch_host: <%= ENV["ELASTICSEARCH_HOST"] %>
   max_image_size: <%= ENV["MAX_IMAGE_SIZE"] %>
   max_cao_size: <%= ENV["MAX_CAO_SIZE"] %>
+  max_import_size: <%= ENV["MAX_IMPORT_SIZE"] %>
   disk_space_mb_alert: <%= ENV["DISK_SPACE_MB_ALERT"] %>
   superadmin_email: <%= ENV["SUPERADMIN_EMAIL"] %>
   recaptcha_site_key: <%= ENV["RECAPTCHA_SITE_KEY"] %>
@@ -83,6 +84,7 @@ test:
   elaticsearch_host: <%= ENV["ELASTICSEARCH_HOST"] %>
   max_image_size: <%= ENV["MAX_IMAGE_SIZE"] %>
   max_cao_size: <%= ENV["MAX_CAO_SIZE"] %>
+  max_import_size: <%= ENV["MAX_IMPORT_SIZE"] %>
   disk_space_mb_alert: <%= ENV["DISK_SPACE_MB_ALERT"] %>
   superadmin_email: <%= ENV["SUPERADMIN_EMAIL"] %>
   recaptcha_site_key: <%= ENV["RECAPTCHA_SITE_KEY"] %>
@@ -131,6 +133,7 @@ staging:
   elaticsearch_host: <%= ENV["ELASTICSEARCH_HOST"] %>
   max_image_size: <%= ENV["MAX_IMAGE_SIZE"] %>
   max_cao_size: <%= ENV["MAX_CAO_SIZE"] %>
+  max_import_size: <%= ENV["MAX_IMPORT_SIZE"] %>
   disk_space_mb_alert: <%= ENV["DISK_SPACE_MB_ALERT"] %>
   superadmin_email: <%= ENV["SUPERADMIN_EMAIL"] %>
   recaptcha_site_key: <%= ENV["RECAPTCHA_SITE_KEY"] %>
@@ -181,6 +184,7 @@ production:
   elaticsearch_host: <%= ENV["ELASTICSEARCH_HOST"] %>
   max_image_size: <%= ENV["MAX_IMAGE_SIZE"] %>
   max_cao_size: <%= ENV["MAX_CAO_SIZE"] %>
+  max_import_size: <%= ENV["MAX_IMPORT_SIZE"] %>
   disk_space_mb_alert: <%= ENV["DISK_SPACE_MB_ALERT"] %>
   superadmin_email: <%= ENV["SUPERADMIN_EMAIL"] %>
   recaptcha_site_key: <%= ENV["RECAPTCHA_SITE_KEY"] %>

db/migrate/20190924140726_create_imports.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# From this migration, we save the file imports into the database.
+# Currently, imports are limited to users import from a CSV file
+class CreateImports < ActiveRecord::Migration
+  def change
+    create_table :imports do |t|
+      t.integer :author_id
+      t.string :attachment
+
+      t.timestamps null: false
+    end
+  end
+end

db/schema.rb

@@ -11,12 +11,12 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20190917123631) do
+ActiveRecord::Schema.define(version: 20190924140726) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
-  enable_extension "unaccent"
   enable_extension "pg_trgm"
+  enable_extension "unaccent"
 
   create_table "abuses", force: :cascade do |t|
     t.integer  "signaled_id"
@@ -246,6 +246,13 @@ ActiveRecord::Schema.define(version: 20190917123631) do
   add_index "history_values", ["invoicing_profile_id"], name: "index_history_values_on_invoicing_profile_id", using: :btree
   add_index "history_values", ["setting_id"], name: "index_history_values_on_setting_id", using: :btree
 
+  create_table "imports", force: :cascade do |t|
+    t.integer  "author_id"
+    t.string   "attachment"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "invoice_items", force: :cascade do |t|
     t.integer  "invoice_id"
     t.string   "stp_invoice_item_id"

doc/environment.md

@@ -156,6 +156,12 @@ If this parameter is not specified the maximum size allowed will be 2MB.
     MAX_CAO_SIZE
 
 Maximum size (in bytes) allowed for CAO files uploaded on the platform, as project attachments.
+If this parameter is not specified, the maximum size allowed will be 5MB.
+
+    MAX_IMPORT_SIZE
+
+Maximum size (in bytes) allowed for import files uploaded on the platform.
+Currently, this is only used to import users from a CSV file. 
 If this parameter is not specified, the maximum size allowed will be 5MB.
 
     DISK_SPACE_MB_ALERT

docker/docker-compose.yml

@@ -12,6 +12,7 @@ services:
       - ${PWD}/public/uploads:/usr/src/app/public/uploads
       - ${PWD}/invoices:/usr/src/app/invoices
       - ${PWD}/exports:/usr/src/app/exports
+      - ${PWD}/imports:/usr/src/app/imports
       - ${PWD}/log:/var/log/supervisor
       - ${PWD}/plugins:/usr/src/app/plugins
       - ${PWD}/accounting:/usr/src/app/accounting

docker/env.example

@@ -72,6 +72,8 @@ SUPERADMIN_EMAIL='admin@sleede.com'
 ALLOWED_EXTENSIONS=pdf ai eps cad math svg stl dxf dwg obj step iges igs 3dm 3dmf doc docx png ino scad fcad skp sldprt sldasm slddrw slddrt tex latex ps
 ALLOWED_MIME_TYPES=application/pdf application/postscript application/illustrator image/x-eps image/svg+xml application/sla application/dxf application/acad application/dwg application/octet-stream application/step application/iges model/iges x-world/x-3dmf application/ application/vnd.openxmlformats-officedocument.wordprocessingml.document image/png text/x-arduino text/plain application/scad application/vnd.sketchup.skp application/x-koan application/vnd-koan koan/x-skm application/vnd.koan application/x-tex application/x-latex
 
+# 5242880 = 5 megabytes
+MAX_IMPORT_SIZE = '5242880'
 # 10485760 = 10 megabytes
 MAX_IMAGE_SIZE=10485760
 # 20971520 = 20 megabytes

test/fixtures/imports.yml

@@ -0,0 +1,8 @@
+# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+one:
+  author_id: 1
+  attachment: 'users.csv'
+  created_at: 2019-09-24 15:06:22.151882000 Z
+  updated_at: 2019-09-24 15:06:22.151882000 Z
+