From a532efd198e5c827ad6599f898b1cce58cf8233e Mon Sep 17 00:00:00 2001 From: Sylvain Date: Tue, 24 Sep 2019 17:42:50 +0200 Subject: [PATCH] upload csv file to the server through the API and save it on the disk --- .dockerignore | 6 ++++ .gitignore | 3 ++ CHANGELOG.md | 2 ++ Dockerfile | 2 ++ .../templates/admin/members/import.html.erb | 2 +- .../templates/admin/members/index.html.erb | 2 +- app/controllers/api/members_controller.rb | 16 +++++++++++ app/models/import.rb | 14 ++++++++++ app/models/project_cao.rb | 8 +----- app/policies/user_policy.rb | 2 +- app/services/members/import_service.rb | 10 +++++++ app/uploaders/import_uploader.rb | 28 +++++++++++++++++++ config/application.yml.default | 2 ++ config/routes.rb | 1 + config/secrets.yml | 4 +++ db/migrate/20190924140726_create_imports.rb | 14 ++++++++++ db/schema.rb | 11 ++++++-- doc/environment.md | 6 ++++ docker/docker-compose.yml | 1 + docker/env.example | 2 ++ test/fixtures/imports.yml | 8 ++++++ 21 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 app/models/import.rb create mode 100644 app/services/members/import_service.rb create mode 100644 app/uploaders/import_uploader.rb create mode 100644 db/migrate/20190924140726_create_imports.rb create mode 100644 test/fixtures/imports.yml diff --git a/.dockerignore b/.dockerignore index bbc584d9f..34479f9e9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -26,6 +26,12 @@ public/assets # PDF invoices invoices +# Excel exports +exports + +# CSV imports +imports + .DS_Store # Development files diff --git a/.gitignore b/.gitignore index 2e46c2324..639af2553 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,9 @@ # XLSX exports /exports/* +# CSV imports +/imports/* + # Archives of cLosed accounting periods /accounting/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 05a5fdf29..1797b9a3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - [TODO DEPLOY] -> (only dev) yarn install - [TODO DEPLOY] add `RECAPTCHA_SITE_KEY` and `RECAPTCHA_SECRET_KEY` environment variables (see [doc/environment.md](doc/environment.md) for configuration details) - [TODO DEPLOY] add `MAX_CAO_SIZE` environment variable (see [doc/environment.md](doc/environment.md) for configuration details) +- [TODO DEPLOY] add `MAX_IMPORT_SIZE` environment variable (see [doc/environment.md](doc/environment.md) for configuration details) +- [TODO DEPLOY] add `- ${PWD}/imports:/usr/src/app/imports` in the volumes list of your fabmanager service in [docker-compose.yml](docker/docker-compose.yml) ## v4.1.0 2019 September 12 diff --git a/Dockerfile b/Dockerfile index 3a15f3f78..567197a8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,6 +53,7 @@ RUN mkdir -p /usr/src/app && \ mkdir -p /usr/src/app/config && \ mkdir -p /usr/src/app/invoices && \ mkdir -p /usr/src/app/exports && \ + mkdir -p /usr/src/app/imports && \ mkdir -p /usr/src/app/log && \ mkdir -p /usr/src/app/public/uploads && \ mkdir -p /usr/src/app/public/assets && \ @@ -66,6 +67,7 @@ COPY . /usr/src/app # Volumes VOLUME /usr/src/app/invoices VOLUME /usr/src/app/exports +VOLUME /usr/src/app/imports VOLUME /usr/src/app/public VOLUME /usr/src/app/public/uploads VOLUME /usr/src/app/public/assets diff --git a/app/assets/templates/admin/members/import.html.erb b/app/assets/templates/admin/members/import.html.erb index dfc7a3ab1..3ec78f685 100644 --- a/app/assets/templates/admin/members/import.html.erb +++ b/app/assets/templates/admin/members/import.html.erb @@ -154,7 +154,7 @@ {{ 'members_import.select_file' }} {{ 'change' }} diff --git a/app/assets/templates/admin/members/index.html.erb b/app/assets/templates/admin/members/index.html.erb index d3141103f..5b58d616a 100644 --- a/app/assets/templates/admin/members/index.html.erb +++ b/app/assets/templates/admin/members/index.html.erb @@ -12,7 +12,7 @@
- +
diff --git a/app/controllers/api/members_controller.rb b/app/controllers/api/members_controller.rb index 039280418..a19f5d264 100644 --- a/app/controllers/api/members_controller.rb +++ b/app/controllers/api/members_controller.rb @@ -181,6 +181,18 @@ class API::MembersController < API::ApiController @members = User.includes(:profile) end + def import + authorize User + + @import = Import.new(attachment: import_params, author: current_user) + if @import.save + Members::ImportService.import(@import) + render json: @import, status: :created + else + render json: @import.errors, status: :unprocessable_entity + end + end + private def set_member @@ -225,4 +237,8 @@ class API::MembersController < API::ApiController def query_params params.require(:query).permit(:search, :order_by, :page, :size) end + + def import_params + params.require(:import_members) + end end diff --git a/app/models/import.rb b/app/models/import.rb new file mode 100644 index 000000000..18f087a5b --- /dev/null +++ b/app/models/import.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'file_size_validator' + +# An Import is a file uploaded by an user that provides some data to the database. +# Currently, this is used to import some users from a CSV file +class Import < ActiveRecord::Base + mount_uploader :attachment, ImportUploader + + belongs_to :author, foreign_key: :author_id, class_name: 'User' + + validates :attachment, file_size: { maximum: Rails.application.secrets.max_import_size&.to_i || 5.megabytes.to_i } + validates :attachment, file_mime_type: { content_type: ['text/csv'] } +end diff --git a/app/models/project_cao.rb b/app/models/project_cao.rb index de92c7fa7..b2fd80b0c 100644 --- a/app/models/project_cao.rb +++ b/app/models/project_cao.rb @@ -4,12 +4,6 @@ class ProjectCao < Asset mount_uploader :attachment, ProjectCaoUploader - validates :attachment, file_size: { maximum: max_size } + validates :attachment, file_size: { maximum: Rails.application.secrets.max_cao_size&.to_i || 5.megabytes.to_i } validates :attachment, file_mime_type: { content_type: ENV['ALLOWED_MIME_TYPES'].split(' ') } - - private - - def max_size - Rails.application.secrets.max_cao_size&.to_i || 5.megabytes.to_i - end end diff --git a/app/policies/user_policy.rb b/app/policies/user_policy.rb index 0b6145d3b..fd483e03c 100644 --- a/app/policies/user_policy.rb +++ b/app/policies/user_policy.rb @@ -27,7 +27,7 @@ class UserPolicy < ApplicationPolicy user.id == record.id end - %w[list create mapping].each do |action| + %w[list create mapping import].each do |action| define_method "#{action}?" do user.admin? end diff --git a/app/services/members/import_service.rb b/app/services/members/import_service.rb new file mode 100644 index 000000000..e902d8bc8 --- /dev/null +++ b/app/services/members/import_service.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Provides helper methods to bulk-import some users from a CSV file +class Members::ImportService + class << self + def import(import) + puts import + end + end +end diff --git a/app/uploaders/import_uploader.rb b/app/uploaders/import_uploader.rb new file mode 100644 index 000000000..2969411c0 --- /dev/null +++ b/app/uploaders/import_uploader.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# CarrierWave uploader for import files. +# This file defines the parameters for these uploads +class ImportUploader < CarrierWave::Uploader::Base + include UploadHelper + + # Choose what kind of storage to use for this uploader: + storage :file + after :remove, :delete_empty_dirs + + # Override the directory where uploaded files will be stored. + # This is a sensible default for uploaders that are meant to be mounted: + + def store_dir + "#{base_store_dir}/#{model.id}" + end + + def base_store_dir + '../imports' + end + + # Add a white list of extensions which are allowed to be uploaded. + # For images you might use something like this: + def extension_white_list + ['csv'] + end +end diff --git a/config/application.yml.default b/config/application.yml.default index cc321d310..70289b811 100644 --- a/config/application.yml.default +++ b/config/application.yml.default @@ -70,6 +70,8 @@ SUPERADMIN_EMAIL: 'admin@sleede.com' ALLOWED_EXTENSIONS: pdf ai eps cad math svg stl dxf dwg obj step iges igs 3dm 3dmf doc docx png ino scad fcad skp sldprt sldasm slddrw slddrt tex latex ps ALLOWED_MIME_TYPES: application/pdf application/postscript application/illustrator image/x-eps image/svg+xml application/sla application/dxf application/acad application/dwg application/octet-stream application/step application/iges model/iges x-world/x-3dmf application/vnd.openxmlformats-officedocument.wordprocessingml.document image/png text/x-arduino text/plain application/scad application/vnd.sketchup.skp application/x-koan application/vnd-koan koan/x-skm application/vnd.koan application/x-tex application/x-latex +# 5242880 = 5 megabytes +MAX_IMPORT_SIZE: '5242880' # 10485760 = 10 megabytes MAX_IMAGE_SIZE: '10485760' # 20971520 = 20 megabytes diff --git a/config/routes.rb b/config/routes.rb index 1afda02ad..88b66de79 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -55,6 +55,7 @@ Rails.application.routes.draw do post 'list', action: 'list', on: :collection get 'search/:query', action: 'search', on: :collection get 'mapping', action: 'mapping', on: :collection + post 'import', action: 'import', on: :collection end resources :reservations, only: %i[show create index update] resources :notifications, only: %i[index show update] do diff --git a/config/secrets.yml b/config/secrets.yml index 178c53c05..43e1c282e 100644 --- a/config/secrets.yml +++ b/config/secrets.yml @@ -44,6 +44,7 @@ development: elaticsearch_host: <%= ENV["ELASTICSEARCH_HOST"] %> max_image_size: <%= ENV["MAX_IMAGE_SIZE"] %> max_cao_size: <%= ENV["MAX_CAO_SIZE"] %> + max_import_size: <%= ENV["MAX_IMPORT_SIZE"] %> disk_space_mb_alert: <%= ENV["DISK_SPACE_MB_ALERT"] %> superadmin_email: <%= ENV["SUPERADMIN_EMAIL"] %> recaptcha_site_key: <%= ENV["RECAPTCHA_SITE_KEY"] %> @@ -83,6 +84,7 @@ test: elaticsearch_host: <%= ENV["ELASTICSEARCH_HOST"] %> max_image_size: <%= ENV["MAX_IMAGE_SIZE"] %> max_cao_size: <%= ENV["MAX_CAO_SIZE"] %> + max_import_size: <%= ENV["MAX_IMPORT_SIZE"] %> disk_space_mb_alert: <%= ENV["DISK_SPACE_MB_ALERT"] %> superadmin_email: <%= ENV["SUPERADMIN_EMAIL"] %> recaptcha_site_key: <%= ENV["RECAPTCHA_SITE_KEY"] %> @@ -131,6 +133,7 @@ staging: elaticsearch_host: <%= ENV["ELASTICSEARCH_HOST"] %> max_image_size: <%= ENV["MAX_IMAGE_SIZE"] %> max_cao_size: <%= ENV["MAX_CAO_SIZE"] %> + max_import_size: <%= ENV["MAX_IMPORT_SIZE"] %> disk_space_mb_alert: <%= ENV["DISK_SPACE_MB_ALERT"] %> superadmin_email: <%= ENV["SUPERADMIN_EMAIL"] %> recaptcha_site_key: <%= ENV["RECAPTCHA_SITE_KEY"] %> @@ -181,6 +184,7 @@ production: elaticsearch_host: <%= ENV["ELASTICSEARCH_HOST"] %> max_image_size: <%= ENV["MAX_IMAGE_SIZE"] %> max_cao_size: <%= ENV["MAX_CAO_SIZE"] %> + max_import_size: <%= ENV["MAX_IMPORT_SIZE"] %> disk_space_mb_alert: <%= ENV["DISK_SPACE_MB_ALERT"] %> superadmin_email: <%= ENV["SUPERADMIN_EMAIL"] %> recaptcha_site_key: <%= ENV["RECAPTCHA_SITE_KEY"] %> diff --git a/db/migrate/20190924140726_create_imports.rb b/db/migrate/20190924140726_create_imports.rb new file mode 100644 index 000000000..8de6b71c0 --- /dev/null +++ b/db/migrate/20190924140726_create_imports.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# From this migration, we save the file imports into the database. +# Currently, imports are limited to users import from a CSV file +class CreateImports < ActiveRecord::Migration + def change + create_table :imports do |t| + t.integer :author_id + t.string :attachment + + t.timestamps null: false + end + end +end diff --git a/db/schema.rb b/db/schema.rb index b8826f52f..033f25e75 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,12 +11,12 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20190917123631) do +ActiveRecord::Schema.define(version: 20190924140726) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" - enable_extension "unaccent" enable_extension "pg_trgm" + enable_extension "unaccent" create_table "abuses", force: :cascade do |t| t.integer "signaled_id" @@ -246,6 +246,13 @@ ActiveRecord::Schema.define(version: 20190917123631) do add_index "history_values", ["invoicing_profile_id"], name: "index_history_values_on_invoicing_profile_id", using: :btree add_index "history_values", ["setting_id"], name: "index_history_values_on_setting_id", using: :btree + create_table "imports", force: :cascade do |t| + t.integer "author_id" + t.string "attachment" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "invoice_items", force: :cascade do |t| t.integer "invoice_id" t.string "stp_invoice_item_id" diff --git a/doc/environment.md b/doc/environment.md index f1948c5c7..d06f64670 100644 --- a/doc/environment.md +++ b/doc/environment.md @@ -156,6 +156,12 @@ If this parameter is not specified the maximum size allowed will be 2MB. MAX_CAO_SIZE Maximum size (in bytes) allowed for CAO files uploaded on the platform, as project attachments. +If this parameter is not specified, the maximum size allowed will be 5MB. + + MAX_IMPORT_SIZE + +Maximum size (in bytes) allowed for import files uploaded on the platform. +Currently, this is only used to import users from a CSV file. If this parameter is not specified, the maximum size allowed will be 5MB. DISK_SPACE_MB_ALERT diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 6a86fba14..59d853f6d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,6 +12,7 @@ services: - ${PWD}/public/uploads:/usr/src/app/public/uploads - ${PWD}/invoices:/usr/src/app/invoices - ${PWD}/exports:/usr/src/app/exports + - ${PWD}/imports:/usr/src/app/imports - ${PWD}/log:/var/log/supervisor - ${PWD}/plugins:/usr/src/app/plugins - ${PWD}/accounting:/usr/src/app/accounting diff --git a/docker/env.example b/docker/env.example index 1b361a85f..703e26a3a 100644 --- a/docker/env.example +++ b/docker/env.example @@ -72,6 +72,8 @@ SUPERADMIN_EMAIL='admin@sleede.com' ALLOWED_EXTENSIONS=pdf ai eps cad math svg stl dxf dwg obj step iges igs 3dm 3dmf doc docx png ino scad fcad skp sldprt sldasm slddrw slddrt tex latex ps ALLOWED_MIME_TYPES=application/pdf application/postscript application/illustrator image/x-eps image/svg+xml application/sla application/dxf application/acad application/dwg application/octet-stream application/step application/iges model/iges x-world/x-3dmf application/ application/vnd.openxmlformats-officedocument.wordprocessingml.document image/png text/x-arduino text/plain application/scad application/vnd.sketchup.skp application/x-koan application/vnd-koan koan/x-skm application/vnd.koan application/x-tex application/x-latex +# 5242880 = 5 megabytes +MAX_IMPORT_SIZE = '5242880' # 10485760 = 10 megabytes MAX_IMAGE_SIZE=10485760 # 20971520 = 20 megabytes diff --git a/test/fixtures/imports.yml b/test/fixtures/imports.yml new file mode 100644 index 000000000..940213ab5 --- /dev/null +++ b/test/fixtures/imports.yml @@ -0,0 +1,8 @@ +# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html + +one: + author_id: 1 + attachment: 'users.csv' + created_at: 2019-09-24 15:06:22.151882000 Z + updated_at: 2019-09-24 15:06:22.151882000 Z +