From e95b2704338656919c6e64b0109c6e212ea441cf Mon Sep 17 00:00:00 2001 From: Sylvain Date: Wed, 24 Jun 2020 17:22:28 +0200 Subject: [PATCH] multiples fixes - improved project search - fix db schema (sql) - improved doc - removed es_language_analyzer var (deprecated) --- CHANGELOG.md | 1 + app/models/project.rb | 4 ++- config/secrets.yml | 4 --- db/structure.sql | 57 +++++++++++++++++++++++++++-------------- doc/environment.md | 10 ++------ env.example | 1 - scripts/pg-analyzers.sh | 3 +++ scripts/run-tests.sh | 1 - setup/env.example | 1 - setup/setup.sh | 2 +- 10 files changed, 48 insertions(+), 36 deletions(-) create mode 100755 scripts/pg-analyzers.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 6358d432a..a70c06fe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Fix a bug: summary of create training availability shows incorrect alert about slot splitting - Fix a security issue: updated websocket-extensions to 0.1.5 to fix [CVE-2020-7663](https://nvd.nist.gov/vuln/detail/CVE-2020-7663) - Fix a security issue: updated angular.js to 1.8 to fix [CVE-2020-7676](https://nvd.nist.gov/vuln/detail/CVE-2020-7676) +- [TODO DEPLOY] add the `POSTGRESQL_LANGUAGE_ANALYZER` environment variable (see [doc/environment.md](doc/environment.md#POSTGRESQL_LANGUAGE_ANALYZER) for configuration details) - [TODO DEPLOY] `rails fablab:setup:env_to_db` - [TODO DEPLOY] `\curl -sSL https://raw.githubusercontent.com/sleede/fab-manager/master/scripts/redis-upgrade.sh | bash` diff --git a/app/models/project.rb b/app/models/project.rb index 1961d70ca..825276eaf 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -65,10 +65,12 @@ class Project < ApplicationRecord using: { tsearch: { dictionary: Rails.application.secrets.postgresql_language_analyzer, + prefix: true, tsvector_column: 'search_vector' }, trigram: { - word_similarity: true + word_similarity: true, + threshold: 0.5 }, dmetaphone: {} }, diff --git a/config/secrets.yml b/config/secrets.yml index 56e35757e..bdf7ff39f 100644 --- a/config/secrets.yml +++ b/config/secrets.yml @@ -25,7 +25,6 @@ development: summernote_locale: <%= ENV["SUMMERNOTE_LOCALE"] %> angular_locale: <%= ENV["ANGULAR_LOCALE"] %> fullcalendar_locale: <%= ENV["FULLCALENDAR_LOCALE"] %> - elasticsearch_language_analyzer: <%= ENV["ELASTICSEARCH_LANGUAGE_ANALYZER"] %> postgresql_language_analyzer: <%= ENV.fetch("POSTGRESQL_LANGUAGE_ANALYZER", 'simple') %> openlab_base_uri: <%= ENV["OPENLAB_BASE_URI"] %> navinum_api_login: <%= ENV["NAVINUM_API_LOGIN"] %> @@ -54,7 +53,6 @@ test: summernote_locale: en-US angular_locale: en-us fullcalendar_locale: en - elasticsearch_language_analyzer: french postgresql_language_analyzer: french openlab_base_uri: navinum_api_login: @@ -91,7 +89,6 @@ staging: summernote_locale: <%= ENV["SUMMERNOTE_LOCALE"] %> angular_locale: <%= ENV["ANGULAR_LOCALE"] %> fullcalendar_locale: <%= ENV["FULLCALENDAR_LOCALE"] %> - elasticsearch_language_analyzer: <%= ENV["ELASTICSEARCH_LANGUAGE_ANALYZER"] %> postgresql_language_analyzer: <%= ENV.fetch("POSTGRESQL_LANGUAGE_ANALYZER", 'simple') %> openlab_base_uri: <%= ENV["OPENLAB_BASE_URI"] %> navinum_api_login: <%= ENV["NAVINUM_API_LOGIN"] %> @@ -131,7 +128,6 @@ production: summernote_locale: <%= ENV["SUMMERNOTE_LOCALE"] %> angular_locale: <%= ENV["ANGULAR_LOCALE"] %> fullcalendar_locale: <%= ENV["FULLCALENDAR_LOCALE"] %> - elasticsearch_language_analyzer: <%= ENV["ELASTICSEARCH_LANGUAGE_ANALYZER"] %> postgresql_language_analyzer: <%= ENV.fetch("POSTGRESQL_LANGUAGE_ANALYZER", 'simple') %> openlab_base_uri: <%= ENV["OPENLAB_BASE_URI"] %> navinum_api_login: <%= ENV["NAVINUM_API_LOGIN"] %> diff --git a/db/structure.sql b/db/structure.sql index d0715be38..1f21bf281 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -108,8 +108,8 @@ SET default_tablespace = ''; CREATE TABLE public.abuses ( id integer NOT NULL, - signaled_type character varying, signaled_id integer, + signaled_type character varying, first_name character varying, last_name character varying, email character varying, @@ -187,8 +187,8 @@ CREATE TABLE public.addresses ( locality character varying, country character varying, postal_code character varying, - placeable_type character varying, placeable_id integer, + placeable_type character varying, created_at timestamp without time zone, updated_at timestamp without time zone ); @@ -263,8 +263,8 @@ CREATE TABLE public.ar_internal_metadata ( CREATE TABLE public.assets ( id integer NOT NULL, - viewable_type character varying, viewable_id integer, + viewable_type character varying, attachment character varying, type character varying, created_at timestamp without time zone, @@ -504,8 +504,8 @@ ALTER SEQUENCE public.coupons_id_seq OWNED BY public.coupons.id; CREATE TABLE public.credits ( id integer NOT NULL, - creditable_type character varying, creditable_id integer, + creditable_type character varying, plan_id integer, hours integer, created_at timestamp without time zone, @@ -1013,8 +1013,8 @@ ALTER SEQUENCE public.invoice_items_id_seq OWNED BY public.invoice_items.id; CREATE TABLE public.invoices ( id integer NOT NULL, - invoiced_type character varying, invoiced_id integer, + invoiced_type character varying, stp_invoice_id character varying, total integer, created_at timestamp without time zone, @@ -1193,15 +1193,15 @@ ALTER SEQUENCE public.machines_id_seq OWNED BY public.machines.id; CREATE TABLE public.notifications ( id integer NOT NULL, receiver_id integer, - attached_object_type character varying, attached_object_id integer, + attached_object_type character varying, notification_type_id integer, is_read boolean DEFAULT false, created_at timestamp without time zone, updated_at timestamp without time zone, receiver_type character varying, is_send boolean DEFAULT false, - meta_data jsonb DEFAULT '"{}"'::jsonb + meta_data jsonb DEFAULT '{}'::jsonb ); @@ -1542,8 +1542,8 @@ CREATE TABLE public.prices ( id integer NOT NULL, group_id integer, plan_id integer, - priceable_type character varying, priceable_id integer, + priceable_type character varying, amount integer, created_at timestamp without time zone NOT NULL, updated_at timestamp without time zone NOT NULL @@ -1858,8 +1858,8 @@ CREATE TABLE public.reservations ( message text, created_at timestamp without time zone, updated_at timestamp without time zone, - reservable_type character varying, reservable_id integer, + reservable_type character varying, nb_reserve_places integer, statistic_profile_id integer ); @@ -1891,8 +1891,8 @@ ALTER SEQUENCE public.reservations_id_seq OWNED BY public.reservations.id; CREATE TABLE public.roles ( id integer NOT NULL, name character varying, - resource_type character varying, resource_id integer, + resource_type character varying, created_at timestamp without time zone, updated_at timestamp without time zone ); @@ -2826,8 +2826,8 @@ CREATE TABLE public.users_roles ( CREATE TABLE public.wallet_transactions ( id integer NOT NULL, wallet_id integer, - transactable_type character varying, transactable_id integer, + transactable_type character varying, transaction_type character varying, amount integer, created_at timestamp without time zone NOT NULL, @@ -3871,14 +3871,6 @@ ALTER TABLE ONLY public.roles ADD CONSTRAINT roles_pkey PRIMARY KEY (id); --- --- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.schema_migrations - ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); - - -- -- Name: settings settings_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -4894,6 +4886,29 @@ CREATE INDEX profiles_lower_unaccent_last_name_trgm_idx ON public.profiles USING CREATE INDEX projects_search_vector_idx ON public.projects USING gin (search_vector); +-- +-- Name: unique_schema_migrations; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX unique_schema_migrations ON public.schema_migrations USING btree (version); + + +-- +-- Name: accounting_periods accounting_periods_del_protect; Type: RULE; Schema: public; Owner: - +-- + +CREATE RULE accounting_periods_del_protect AS + ON DELETE TO public.accounting_periods DO INSTEAD NOTHING; + + +-- +-- Name: accounting_periods accounting_periods_upd_protect; Type: RULE; Schema: public; Owner: - +-- + +CREATE RULE accounting_periods_upd_protect AS + ON UPDATE TO public.accounting_periods DO INSTEAD NOTHING; + + -- -- Name: projects projects_search_content_trigger; Type: TRIGGER; Schema: public; Owner: - -- @@ -5380,6 +5395,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20140605125131'), ('20140605142133'), ('20140605151442'), +('20140606133116'), ('20140609092700'), ('20140609092827'), ('20140610153123'), @@ -5448,12 +5464,14 @@ INSERT INTO "schema_migrations" (version) VALUES ('20150507075620'), ('20150512123546'), ('20150520132030'), +('20150520133409'), ('20150526130729'), ('20150527153312'), ('20150529113555'), ('20150601125944'), ('20150603104502'), ('20150603104658'), +('20150603133050'), ('20150604081757'), ('20150604131525'), ('20150608142234'), @@ -5535,6 +5553,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20160905142700'), ('20160906094739'), ('20160906094847'), +('20160906145713'), ('20160915105234'), ('20161123104604'), ('20170109085345'), diff --git a/doc/environment.md b/doc/environment.md index 6fd855837..5d431bdb9 100644 --- a/doc/environment.md +++ b/doc/environment.md @@ -194,18 +194,12 @@ See [code.angularjs.org/i18n/angular-locale_*.js](https://code.angularjs.org/1.6 Configure the fullCalendar JS agenda library. See [github.com/fullcalendar/fullcalendar/lang/*.js](https://github.com/fullcalendar/fullcalendar/tree/v2.3.1/lang) for a list of available locales. Default is **en-us**. - - - ELASTICSEARCH_LANGUAGE_ANALYZER - -This configure the language analyzer for indexing and searching in projects with ElasticSearch. -See [ElasticSearch guide](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/analysis-lang-analyzer.html) for a list of available analyzers. POSTGRESQL_LANGUAGE_ANALYZER This variable configures the language analyzer for indexing and searching in projets with PostgreSQL. -List all available values with `docker-compose exec -T postgres psql -Upostgres -c \\dFd | head -n -2 | tail -n +3 | awk '{ print gensub(/([a-z]+)_stem/,"\\1","g",$3); }'` +Available values: `danish, dutch, english, finnish, french, german, hungarian, italian, norwegian, portuguese, romanian, russian, simple, spanish, swedish, turkish` TIME_ZONE @@ -258,4 +252,4 @@ Set this variable to `https://openprojects.fab-manager.com` if you want to use t In the previous versions of Fab-manager, much more settings were configurable from environment variables. Starting with Fab-manager v4.5.0, these settings can be configured from the graphical interface, when logged as an administrator. -Nevertheless, it is possible to keep the configuration in the `env` file, using a combination of `LOCKED_SETTINGS`, `rails fablab:setup:env_to_db` and the [mapping table of `env_to_db`](../lib/tasks/fablab/setup.rake#L105). +Nevertheless, it is possible to keep the configuration in the `env` file, using a combination of [`LOCKED_SETTINGS`](environment.md#LOCKED_SETTINGS), `rails fablab:setup:env_to_db` and the [mapping table of `env_to_db`](../lib/tasks/fablab/setup.rake#L105). diff --git a/env.example b/env.example index cf1582a34..ee5bd9e79 100644 --- a/env.example +++ b/env.example @@ -37,7 +37,6 @@ FULLCALENDAR_LOCALE=fr FORCE_VERSION_CHECK=false ALLOW_INSECURE_HTTP=false -ELASTICSEARCH_LANGUAGE_ANALYZER=french POSTGRESQL_LANGUAGE_ANALYZER=french TIME_ZONE=Paris diff --git a/scripts/pg-analyzers.sh b/scripts/pg-analyzers.sh new file mode 100755 index 000000000..7fac40058 --- /dev/null +++ b/scripts/pg-analyzers.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker-compose exec -T postgres psql -Upostgres -c \\dFd | head -n -2 | tail -n +3 | awk '{ print gensub(/([a-z]+)_stem/,"\\1","g",$3); }' \ No newline at end of file diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index c691b0cf9..724f38fa4 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -15,7 +15,6 @@ if [[ -z "$stripe_secret_key" ]]; then if [[ -z "$stripe_secret_key" ]]; then echo "Key was not set, exiting..."; exit 1; fi fi -RAILS_ENV='test' bin/rails db:environment:set RAILS_ENV='test' bin/rails db:drop RAILS_ENV='test' bin/rails db:create RAILS_ENV='test' bin/rails db:migrate diff --git a/setup/env.example b/setup/env.example index 8fe83e3f1..bf6195e53 100644 --- a/setup/env.example +++ b/setup/env.example @@ -27,7 +27,6 @@ SUMMERNOTE_LOCALE=fr-FR ANGULAR_LOCALE=fr-fr FULLCALENDAR_LOCALE=fr -ELASTICSEARCH_LANGUAGE_ANALYZER=french POSTGRESQL_LANGUAGE_ANALYZER=french TIME_ZONE=Paris diff --git a/setup/setup.sh b/setup/setup.sh index e985cb29c..ce16d9174 100755 --- a/setup/setup.sh +++ b/setup/setup.sh @@ -236,7 +236,7 @@ configure_env_file() doc=$(\curl -sSL https://raw.githubusercontent.com/sleede/fab-manager/master/doc/environment.md) variables=(DEFAULT_HOST DEFAULT_PROTOCOL DELIVERY_METHOD SMTP_ADDRESS SMTP_PORT SMTP_USER_NAME SMTP_PASSWORD SMTP_AUTHENTICATION \ SMTP_ENABLE_STARTTLS_AUTO SMTP_OPENSSL_VERIFY_MODE SMTP_TLS LOG_LEVEL MAX_IMAGE_SIZE MAX_CAO_SIZE MAX_IMPORT_SIZE DISK_SPACE_MB_ALERT \ - SUPERADMIN_EMAIL APP_LOCALE RAILS_LOCALE MOMENT_LOCALE SUMMERNOTE_LOCALE ANGULAR_LOCALE FULLCALENDAR_LOCALE ELASTICSEARCH_LANGUAGE_ANALYZER \ + SUPERADMIN_EMAIL APP_LOCALE RAILS_LOCALE MOMENT_LOCALE SUMMERNOTE_LOCALE ANGULAR_LOCALE FULLCALENDAR_LOCALE \ POSTGRESQL_LANGUAGE_ANALYZER TIME_ZONE WEEK_STARTING_DAY D3_DATE_FORMAT UIB_DATE_FORMAT EXCEL_DATE_FORMAT) for variable in "${variables[@]}"; do local var_doc current