From 750bd5684eb4801b46bc941b6ef357d2533b15f2 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Thu, 9 Dec 2021 17:47:17 +0100 Subject: [PATCH 01/28] ran bundle update replaced deprecated zeitwerk method --- Gemfile.lock | 93 ++++++++++++----------- config/initializers/preload_medium_sti.rb | 2 +- 2 files changed, 48 insertions(+), 47 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c640b3552..e3fe78bea 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -36,21 +36,21 @@ GIT GIT remote: https://github.com/rails/sprockets-rails - revision: 118ce60b1ffeb7a85640661b014cd2ee3c4e3e56 + revision: 4e0f1686ce108ea3a52fb0dfb24bb6272ea2df3f branch: master specs: - sprockets-rails (3.2.2) + sprockets-rails (3.4.1) actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) GIT remote: https://github.com/zdennis/activerecord-import.git - revision: 1d2a3163ee68ffb01125ec3e4fa953ff7ab24cce + revision: 213470f29f6ca63d15b2696dbff28c292809ffc6 branch: master specs: activerecord-import (1.2.0) - activerecord (>= 3.2) + activerecord (>= 4.2) GEM remote: https://rubygems.org/ @@ -112,8 +112,8 @@ GEM activerecord (6.1.4.1) activemodel (= 6.1.4.1) activesupport (= 6.1.4.1) - activerecord-nulldb-adapter (0.7.0) - activerecord (>= 5.2.0, < 6.3) + activerecord-nulldb-adapter (0.8.0) + activerecord (>= 5.2.0, < 7.1) activestorage (6.1.4.1) actionpack (= 6.1.4.1) activejob (= 6.1.4.1) @@ -131,7 +131,7 @@ GEM activerecord (>= 4.2) acts_as_tree (2.9.1) activerecord (>= 3.0.0) - acts_as_votable (0.13.1) + acts_as_votable (0.13.2) addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) afm (0.2.2) @@ -153,13 +153,13 @@ GEM parser (>= 2.4) smart_properties bindex (0.8.1) - bootsnap (1.8.1) + bootsnap (1.9.3) msgpack (~> 1.0) - bootstrap (4.6.0) + bootstrap (4.6.1) autoprefixer-rails (>= 9.1.0) popper_js (>= 1.14.3, < 2) sassc-rails (>= 2.0.0) - bootstrap_form (4.5.0) + bootstrap_form (5.0.0) actionpack (>= 5.2) activemodel (>= 5.2) builder (3.2.4) @@ -167,7 +167,7 @@ GEM cancancan (3.3.0) case_transform (0.2) activesupport - childprocess (3.0.0) + childprocess (4.1.0) choice (0.2.0) chunky_png (1.4.0) clipboard-rails (1.7.1) @@ -192,7 +192,7 @@ GEM addressable cypress-on-rails (1.11.0) rack - dalli (2.7.11) + dalli (3.1.0) database_cleaner (2.0.1) database_cleaner-active_record (~> 2.0.0) database_cleaner-active_record (2.0.1) @@ -224,7 +224,7 @@ GEM smart_properties erubi (1.10.0) erubis (2.7.0) - et-orbi (1.2.4) + et-orbi (1.2.6) tzinfo exception_handler (0.8.0.2) bundler @@ -238,7 +238,7 @@ GEM railties (>= 5.0.0) faker (2.19.0) i18n (>= 1.6, < 2) - faraday (1.7.1) + faraday (1.8.0) faraday-em_http (~> 1.0) faraday-em_synchrony (~> 1.0) faraday-excon (~> 1.1) @@ -262,13 +262,13 @@ GEM filesize (0.2.0) friendly_id (5.4.2) activerecord (>= 4.0.0) - fugit (1.5.1) + fugit (1.5.2) et-orbi (~> 1.1, >= 1.1.8) raabro (~> 1.4) fuzzy-string-match (1.0.1) RubyInline (>= 3.8.6) generator (0.0.1) - globalid (0.5.2) + globalid (1.0.0) activesupport (>= 5.0) globalize (6.0.1) activemodel (>= 4.2, < 7.0) @@ -288,7 +288,7 @@ GEM http-accept (1.7.0) http-cookie (1.0.4) domain_name (~> 0.5) - i18n (1.8.10) + i18n (1.8.11) concurrent-ruby (~> 1.0) image_processing (1.12.1) mini_magick (>= 4.9.5, < 5) @@ -296,7 +296,7 @@ GEM inline_svg (1.7.2) activesupport (>= 3.0) nokogiri (>= 1.6) - jbuilder (2.11.2) + jbuilder (2.11.3) activesupport (>= 5.0.0) jquery-rails (4.4.0) rails-dom-testing (>= 1, < 3) @@ -307,7 +307,7 @@ GEM js-routes (1.4.9) railties (>= 4) sprockets-rails - json (2.5.1) + json (2.6.1) jsonapi-renderer (0.2.2) kaminari (1.2.1) activesupport (>= 4.1.0) @@ -345,13 +345,13 @@ GEM nokogiri (>= 1.5.9) mail (2.7.1) mini_mime (>= 0.1.1) - marcel (1.0.1) + marcel (1.0.2) method_source (1.0.0) - mime-types (3.3.1) + mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.0901) + mime-types-data (3.2021.1115) mini_magick (4.11.0) - mini_mime (1.1.1) + mini_mime (1.1.2) mini_portile2 (2.6.1) minitest (5.14.4) msgpack (1.4.2) @@ -360,7 +360,7 @@ GEM mustache (1.1.1) netrc (0.11.0) nio4r (2.5.8) - nokogiri (1.12.4) + nokogiri (1.12.5) mini_portile2 (~> 2.6.1) racc (~> 1.4) onebox (1.9.30) @@ -372,10 +372,10 @@ GEM sanitize options (2.3.2) orm_adapter (0.5.0) - parallel (1.20.1) - parser (3.0.2.0) + parallel (1.21.0) + parser (3.0.3.2) ast (~> 2.4.1) - pdf-reader (2.5.0) + pdf-reader (2.6.0) Ascii85 (~> 1.0) afm (~> 0.2.1) hashery (~> 2.0) @@ -395,15 +395,15 @@ GEM progress_bar (1.3.3) highline (>= 1.6, < 3) options (~> 2.3.0) - prometheus_exporter (0.8.1) + prometheus_exporter (1.0.0) webrick public_suffix (4.0.6) - puma (4.3.8) + puma (4.3.10) nio4r (~> 2.0) pundit (2.1.1) activesupport (>= 3.0.0) raabro (1.4.0) - racc (1.5.2) + racc (1.6.0) rack (2.2.3) rack-proxy (0.7.0) rack @@ -450,8 +450,8 @@ GEM rb-fsevent (0.11.0) rb-inotify (0.10.1) ffi (~> 1.0) - redis (4.4.0) - regexp_parser (2.1.1) + redis (4.5.1) + regexp_parser (2.2.0) request_store (1.5.0) rack (>= 1.4) responders (3.0.1) @@ -490,7 +490,7 @@ GEM rspec-expectations (~> 3.10) rspec-mocks (~> 3.10) rspec-support (~> 3.10) - rspec-support (3.10.2) + rspec-support (3.10.3) rubocop (0.93.1) parallel (~> 1.10) parser (>= 2.7.1.5) @@ -500,7 +500,7 @@ GEM rubocop-ast (>= 0.6.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (1.11.0) + rubocop-ast (1.14.0) parser (>= 3.0.1.1) rubocop-packaging (0.5.1) rubocop (>= 0.89, < 2.0) @@ -515,7 +515,7 @@ GEM rexml ruby-progressbar (1.11.0) ruby-rc4 (0.1.5) - ruby-vips (2.1.3) + ruby-vips (2.1.4) ffi (~> 1.12) ruby2_keywords (0.0.5) ruby_dep (1.5.0) @@ -535,14 +535,15 @@ GEM tilt select2-rails (4.0.13) selectize-rails (0.12.6) - selenium-webdriver (3.142.7) - childprocess (>= 0.5, < 4.0) + selenium-webdriver (4.1.0) + childprocess (>= 0.5, < 5.0) + rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2) semantic_range (3.0.0) shrine (3.4.0) content_disposition (~> 1.0) down (~> 5.1) - sidekiq (6.2.2) + sidekiq (6.3.1) connection_pool (>= 2.2.2) rack (~> 2.0) redis (>= 4.2.0) @@ -558,7 +559,7 @@ GEM simplecov (~> 0.19) simplecov-html (0.12.3) simplecov_json_formatter (0.1.3) - smart_properties (1.16.2) + smart_properties (1.16.3) spring (2.1.1) spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) @@ -605,20 +606,20 @@ GEM execjs (>= 0.3.0, < 3) unf (0.1.4) unf_ext - unf_ext (0.0.7.7) - unicode-display_width (1.7.0) + unf_ext (0.0.8) + unicode-display_width (1.8.0) warden (1.2.9) rack (>= 2.0.9) - web-console (4.1.0) + web-console (4.2.0) actionview (>= 6.0.0) activemodel (>= 6.0.0) bindex (>= 0.4.0) railties (>= 6.0.0) - webdrivers (4.6.1) + webdrivers (5.0.0) nokogiri (~> 1.6) rubyzip (>= 1.3.0) - selenium-webdriver (>= 3.0, < 4.0) - webpacker (5.4.2) + selenium-webdriver (~> 4.0) + webpacker (5.4.3) activesupport (>= 5.2) rack-proxy (>= 0.6.1) railties (>= 5.2) @@ -631,7 +632,7 @@ GEM xkcd (1.1.1) google-search nokogiri (>= 1.5.0) - zeitwerk (2.4.2) + zeitwerk (2.5.1) PLATFORMS ruby diff --git a/config/initializers/preload_medium_sti.rb b/config/initializers/preload_medium_sti.rb index 052dca3e1..0e508b23a 100644 --- a/config/initializers/preload_medium_sti.rb +++ b/config/initializers/preload_medium_sti.rb @@ -5,5 +5,5 @@ sti_leaves = %w(question quiz remark) sti_leaves.each do |leaf| - autoloader.preload("#{Rails.root}/app/models/#{leaf}.rb") + autoloader.on_setup { "#{leaf}" } end \ No newline at end of file From 1080d2c2ff4842b08982b5e50f0dd5818e31caca Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Tue, 14 Dec 2021 14:16:27 +0100 Subject: [PATCH 02/28] updated ruby, but conflict with "thredded-markdown_katex" remains --- Gemfile | 4 ++-- Gemfile.lock | 39 +++++++++++++-------------------------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/Gemfile b/Gemfile index bae5cc739..c72433d09 100644 --- a/Gemfile +++ b/Gemfile @@ -1,10 +1,10 @@ source "https://rubygems.org" git_source(:github) { |repo| "https://github.com/#{repo}.git" } -ruby '2.7.2' +ruby '3.0.3' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem "rails", "~> 6.1" +gem "rails", "~> 6.1.4.1" # Use dalli for caching to memcached in production gem "dalli", ">= 2.7" # Ruby wrapper for UglifyJS JavaScript compressor diff --git a/Gemfile.lock b/Gemfile.lock index e3fe78bea..a64afa328 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -36,10 +36,10 @@ GIT GIT remote: https://github.com/rails/sprockets-rails - revision: 4e0f1686ce108ea3a52fb0dfb24bb6272ea2df3f + revision: 582e284bab79ea0663addf6b15f80b19835af351 branch: master specs: - sprockets-rails (3.4.1) + sprockets-rails (3.4.2) actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) @@ -188,11 +188,11 @@ GEM term-ansicolor thor crass (1.0.6) - css_parser (1.10.0) + css_parser (1.11.0) addressable cypress-on-rails (1.11.0) rack - dalli (3.1.0) + dalli (3.1.3) database_cleaner (2.0.1) database_cleaner-active_record (~> 2.0.0) database_cleaner-active_record (2.0.1) @@ -324,23 +324,17 @@ GEM kaminari-i18n (0.5.0) kaminari rails - katex (0.8.0) - execjs (~> 2.7) kramdown (2.3.1) rexml - kramdown-math-katex (1.0.1) - katex (~> 0.4) - kramdown (~> 2.0) kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) launchy (2.5.0) addressable (~> 2.7) lazy_priority_queue (0.1.1) - listen (3.1.5) + listen (3.0.8) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) - ruby_dep (~> 1.2) - loofah (2.12.0) + loofah (2.13.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) @@ -352,7 +346,6 @@ GEM mime-types-data (3.2021.1115) mini_magick (4.11.0) mini_mime (1.1.2) - mini_portile2 (2.6.1) minitest (5.14.4) msgpack (1.4.2) multi_json (1.15.0) @@ -360,8 +353,7 @@ GEM mustache (1.1.1) netrc (0.11.0) nio4r (2.5.8) - nokogiri (1.12.5) - mini_portile2 (~> 2.6.1) + nokogiri (1.12.5-x86_64-linux) racc (~> 1.4) onebox (1.9.30) addressable (~> 2.7.0) @@ -375,7 +367,7 @@ GEM parallel (1.21.0) parser (3.0.3.2) ast (~> 2.4.1) - pdf-reader (2.6.0) + pdf-reader (2.7.0) Ascii85 (~> 1.0) afm (~> 0.2.1) hashery (~> 2.0) @@ -500,7 +492,7 @@ GEM rubocop-ast (>= 0.6.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (1.14.0) + rubocop-ast (1.15.0) parser (>= 3.0.1.1) rubocop-packaging (0.5.1) rubocop (>= 0.89, < 2.0) @@ -518,7 +510,6 @@ GEM ruby-vips (2.1.4) ffi (~> 1.12) ruby2_keywords (0.0.5) - ruby_dep (1.5.0) rubyzip (2.3.2) sanitize (6.0.0) crass (~> 1.0.2) @@ -587,9 +578,6 @@ GEM term-ansicolor (1.7.1) tins (~> 1.0) thor (1.1.0) - thredded-markdown_katex (0.4.2) - katex (>= 0.4.3) - kramdown-math-katex tilt (2.0.10) timeago_js (3.0.2.2) tins (1.29.1) @@ -635,7 +623,7 @@ GEM zeitwerk (2.5.1) PLATFORMS - ruby + x86_64-linux DEPENDENCIES active_model_serializers @@ -690,7 +678,7 @@ DEPENDENCIES prometheus_exporter puma (~> 4.1) rack - rails (~> 6.1) + rails (~> 6.1.4.1) rails-erd rails-i18n responders @@ -719,7 +707,6 @@ DEPENDENCIES sunspot_rails sunspot_solr thredded! - thredded-markdown_katex trix-rails turbolinks (~> 5) uglifier @@ -729,7 +716,7 @@ DEPENDENCIES xkcd RUBY VERSION - ruby 2.7.2p137 + ruby 3.0.3p157 BUNDLED WITH - 2.1.4 + 2.2.32 From 53b4295ccfab0183e44ac2b573bf0ed0644f7450 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Tue, 21 Dec 2021 10:24:59 +0100 Subject: [PATCH 03/28] ruby update rails update changed thredded to custom fork (maybe needs to be changed) updated Dockerfiles --- Gemfile | 10 ++- Gemfile.lock | 93 +++++++++++++++++----------- app/validators/http_url_validator.rb | 2 +- docker/development/Dockerfile | 2 +- docker/production/Dockerfile | 2 +- docker/run_cypress_tests/Dockerfile | 2 +- docker/run_tests/Dockerfile | 2 +- 7 files changed, 69 insertions(+), 44 deletions(-) diff --git a/Gemfile b/Gemfile index c72433d09..5122aa18f 100644 --- a/Gemfile +++ b/Gemfile @@ -73,15 +73,19 @@ gem "activerecord-import", git: "https://github.com/zdennis/activerecord-import.git", branch: "master" gem "thredded", - git: "https://github.com/MaMpf-HD/thredded.git", + github: "zebleck/thredded", branch: "master" gem "kramdown-parser-gfm" -gem "thredded-markdown_katex" +gem "thredded-markdown_katex", + github: "thredded/thredded-markdown_katex", + branch: "master" gem "rails-i18n" gem "kaminari-i18n" gem "trix-rails", require: "trix" gem "xkcd" -gem "sunspot_rails" +gem "sunspot_rails", + github: 'sunspot/sunspot', + glob: 'sunspot_rails/*.gemspec' gem "sunspot_solr" gem "progress_bar" gem "barby" diff --git a/Gemfile.lock b/Gemfile.lock index a64afa328..f873044a2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,30 +1,3 @@ -GIT - remote: https://github.com/MaMpf-HD/thredded.git - revision: 7fe6b2a6d3d7c75665050cbc706363f87e40e1f1 - branch: master - specs: - thredded (0.16.16) - active_record_union (>= 1.3.0) - autoprefixer-rails - db_text_search (~> 0.3.2) - friendly_id - html-pipeline - htmlentities - inline_svg (>= 1.6.0) - kaminari - kramdown (>= 2.0.0) - kramdown-parser-gfm - nokogiri - onebox (~> 1.8, >= 1.8.99) - pundit (>= 1.1.0) - rails (>= 4.2.10, != 6.0.0.rc2) - rails_gravatar - rinku - sanitize - sassc-rails (>= 2.0.0) - sprockets-es6 - timeago_js (>= 3.0.2.2) - GIT remote: https://github.com/fosterfarrell9/commontator revision: cfc3e47c2d9b7002738f788fb463a89f318ee63a @@ -44,6 +17,24 @@ GIT activesupport (>= 5.2) sprockets (>= 3.0.0) +GIT + remote: https://github.com/sunspot/sunspot.git + revision: f2f01a6278030d086e0efb141dceefdcca8932bd + glob: sunspot_rails/*.gemspec + specs: + sunspot_rails (2.5.0) + rails (>= 3) + sunspot (= 2.5.0) + +GIT + remote: https://github.com/thredded/thredded-markdown_katex.git + revision: dbc7ead38b0ac4245bd1b1255342dd34b26faa27 + branch: master + specs: + thredded-markdown_katex (0.4.2) + katex (>= 0.4.3) + kramdown-math-katex + GIT remote: https://github.com/zdennis/activerecord-import.git revision: 213470f29f6ca63d15b2696dbff28c292809ffc6 @@ -52,6 +43,33 @@ GIT activerecord-import (1.2.0) activerecord (>= 4.2) +GIT + remote: https://github.com/zebleck/thredded.git + revision: 3cfcaa3aba383706facfad341cfaed5e6160f3fb + branch: master + specs: + thredded (0.16.16) + active_record_union (>= 1.3.0) + autoprefixer-rails + db_text_search (~> 0.3.2) + friendly_id + html-pipeline + htmlentities + inline_svg (>= 1.6.0) + kaminari + kramdown (>= 2.0.0) + kramdown-parser-gfm + nokogiri + onebox (>= 1.8.99) + pundit (>= 1.1.0) + rails (>= 4.2.10, != 6.0.0.rc2) + rails_gravatar + rinku + sanitize + sassc-rails (>= 2.0.0) + sprockets-es6 + timeago_js (>= 3.0.2.2) + GEM remote: https://rubygems.org/ specs: @@ -132,7 +150,7 @@ GEM acts_as_tree (2.9.1) activerecord (>= 3.0.0) acts_as_votable (0.13.2) - addressable (2.7.0) + addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) afm (0.2.2) ast (2.4.2) @@ -212,8 +230,8 @@ GEM docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - down (5.2.2) - addressable (~> 2.5) + down (5.2.4) + addressable (~> 2.8) erb_lint (0.1.1) activesupport better_html (~> 1.0.7) @@ -324,8 +342,13 @@ GEM kaminari-i18n (0.5.0) kaminari rails + katex (0.8.0) + execjs (~> 2.7) kramdown (2.3.1) rexml + kramdown-math-katex (1.0.1) + katex (~> 0.4) + kramdown (~> 2.0) kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) launchy (2.5.0) @@ -355,8 +378,8 @@ GEM nio4r (2.5.8) nokogiri (1.12.5-x86_64-linux) racc (~> 1.4) - onebox (1.9.30) - addressable (~> 2.7.0) + onebox (2.2.19) + addressable (~> 2.8.0) htmlentities (~> 4.3) multi_json (~> 1.11) mustache @@ -570,9 +593,6 @@ GEM sunspot (2.5.0) pr_geohash (~> 1.0) rsolr (>= 1.1.1, < 3) - sunspot_rails (2.5.0) - rails (>= 3) - sunspot (= 2.5.0) sunspot_solr (2.5.0) sync (0.5.0) term-ansicolor (1.7.1) @@ -704,9 +724,10 @@ DEPENDENCIES sprockets-rails! sqlite3 (~> 1.4) streamio-ffmpeg - sunspot_rails + sunspot_rails! sunspot_solr thredded! + thredded-markdown_katex! trix-rails turbolinks (~> 5) uglifier diff --git a/app/validators/http_url_validator.rb b/app/validators/http_url_validator.rb index c2f13d0c4..5044fa8ae 100644 --- a/app/validators/http_url_validator.rb +++ b/app/validators/http_url_validator.rb @@ -1,7 +1,7 @@ class HttpUrlValidator < ActiveModel::EachValidator def self.compliant?(value) - uri = URI.parse(URI.encode(value)) + uri = URI.parse(URI.encode_www_form_component(value)) uri.is_a?(URI::HTTP) && !uri.host.nil? rescue URI::InvalidURIError false diff --git a/docker/development/Dockerfile b/docker/development/Dockerfile index a559838d0..23b972d8f 100644 --- a/docker/development/Dockerfile +++ b/docker/development/Dockerfile @@ -7,7 +7,7 @@ COPY pdfcomprezzor/main.go . RUN GOOS=js GOARCH=wasm go build -o pdfcomprezzor.wasm # Now build the actual mampf application -FROM ruby:2.7.2 +FROM ruby:3.0.3 ENV RAILS_ENV=production EXPOSE 3000 diff --git a/docker/production/Dockerfile b/docker/production/Dockerfile index f06ee121c..67d440c2e 100644 --- a/docker/production/Dockerfile +++ b/docker/production/Dockerfile @@ -7,7 +7,7 @@ COPY pdfcomprezzor/main.go . RUN GOOS=js GOARCH=wasm go build -o pdfcomprezzor.wasm # Now build the actual mampf application -FROM ruby:2.7.2 +FROM ruby:3.0.3 ENV RAILS_ENV=production diff --git a/docker/run_cypress_tests/Dockerfile b/docker/run_cypress_tests/Dockerfile index e6c886477..a9d86c092 100644 --- a/docker/run_cypress_tests/Dockerfile +++ b/docker/run_cypress_tests/Dockerfile @@ -7,7 +7,7 @@ COPY pdfcomprezzor/main.go . RUN GOOS=js GOARCH=wasm go build -o pdfcomprezzor.wasm # Now build the actual mampf application -FROM ruby:2.7.2 +FROM ruby:3.0.3 ENV RAILS_ENV=production EXPOSE 3000 diff --git a/docker/run_tests/Dockerfile b/docker/run_tests/Dockerfile index bf9402932..3f3ee04ff 100644 --- a/docker/run_tests/Dockerfile +++ b/docker/run_tests/Dockerfile @@ -1,4 +1,4 @@ -FROM ruby:2.7.2 +FROM ruby:3.0.3 ENV RAILS_ENV=production From 4f9544e67505e39691fa08fbf2c06b704cc88b14 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Wed, 5 Jan 2022 19:25:32 +0100 Subject: [PATCH 04/28] updated all docker-compose files --- docker/development/docker-compose.yml | 4 +- .../production/docker-compose.production.yml | 68 ++++++++----------- .../docker-compose.local.yml | 4 +- docker/run_cypress_tests/docker-compose.yml | 4 +- docker/run_tests/docker-compose.yml | 4 +- 5 files changed, 37 insertions(+), 47 deletions(-) diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 4793279bd..c94157d61 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -1,11 +1,11 @@ --- -version: '3.2' +version: '3.4' services: redis: image: "redis:alpine" solr: - image: "solr:8" + image: "solr:8.11.1" ports: - "127.0.0.1:8983:8983" volumes: diff --git a/docker/production/docker-compose.production.yml b/docker/production/docker-compose.production.yml index 5ee428676..2bba7ef34 100644 --- a/docker/production/docker-compose.production.yml +++ b/docker/production/docker-compose.production.yml @@ -2,9 +2,8 @@ version: '3.4' x-mampf: &mampf build: - context: git@github.com:fosterfarrell9/mampf.git#production + context: git@github.com:MaMpf-HD/mampf.git#production dockerfile: docker/production/Dockerfile - image: mampf env_file: docker.env networks: - solr @@ -14,16 +13,16 @@ x-mampf: depends_on: - redis - cache - - solr services: master: <<: *mampf entrypoint: /usr/src/app/docker/production/entrypoint-master.sh volumes: - - "media_mampf:/private/media" + - "media:/private/media:nocopy" + - "submissions:/private/submissions:nocopy" - "public:/usr/src/app/public" - - "caches:/caches" + - "caches:/caches:nocopy" worker: <<: *mampf @@ -31,69 +30,60 @@ services: environment: - VIRTUAL_HOST=localhost volumes: - - "media_mampf:/private/media" - - "submissions:/private/submissions" - - "caches:/caches" + - "media:/private/media:nocopy" + - "submissions:/private/submissions:nocopy" + - "caches:/caches:nocopy" proxy: build: - context: git@github.com:fosterfarrell9/mampf.git#production + context: git@github.com:MaMpf-HD/mampf.git#production dockerfile: docker/production/Dockerfile.proxy - image: mampf-proxy restart: always ports: - - "127.0.0.1:3008:80" + - "127.0.0.1:3009:80" volumes: - "/var/run/docker.sock:/tmp/docker.sock" - - "submissions:/private/submissions:ro" - - "public:/public:ro,nocopy" + - "submissions:/private/submissions:ro,nocopy" + - "public:/public" + depends_on: + - master networks: - mampf - + redis: restart: always - image: redis:5-alpine + image: redis:6-alpine networks: - - default + - mampf cache: restart: always image: memcached:alpine networks: - - default - - solr: - image: solr:8.1 - container_name: solr - ports: - - "127.0.0.1:8983:8983" - volumes: - - /path/to/persistent/var_solr:/var/solr - networks: - - default - restart: always + - mampf -# Keeping the database separate from the docker setup is recommended. -# Manually create a docker network where the server can be reached and specify it here networks: postgres: external: true - mampf: solr: + external: true + mampf: + external: true volumes: - media_mampf: + media: driver_opts: type: "nfs" - o: "addr=,noatime,nolock,soft,vers=4.2,rw" - device: ":/path/to/share" + o: "addr=krikkit,noatime,nolock,soft,vers=4.2,rw" + device: ":/mampf/media/mampf" submissions: driver_opts: type: "nfs" - o: "addr=,noatime,nolock,soft,vers=4.2,rw" - device: ":/path/to/submissions/share" + o: "addr=krikkit,noatime,nolock,soft,vers=4.2,rw" + device: ":/mampf/submissions/mampf" public: - driver_opts: - type: "tmpfs" - device: "tmpfs" caches: + driver_opts: + type: "nfs" + o: "addr=krikkit,noatime,nolock,soft,vers=4.2,rw" + device: ":/mampf/caches/mampf" diff --git a/docker/run_cypress_tests/docker-compose.local.yml b/docker/run_cypress_tests/docker-compose.local.yml index f3ab3cdf8..b1692715d 100644 --- a/docker/run_cypress_tests/docker-compose.local.yml +++ b/docker/run_cypress_tests/docker-compose.local.yml @@ -1,11 +1,11 @@ --- -version: "3.2" +version: "3.4" services: redis: image: "redis:alpine" solr: - image: "solr:8" + image: "solr:8.11.1" ports: - "127.0.0.1:8983:8983" volumes: diff --git a/docker/run_cypress_tests/docker-compose.yml b/docker/run_cypress_tests/docker-compose.yml index 0c4878fe7..d192f685c 100644 --- a/docker/run_cypress_tests/docker-compose.yml +++ b/docker/run_cypress_tests/docker-compose.yml @@ -1,11 +1,11 @@ --- -version: "3.2" +version: "3.4" services: redis: image: "redis:alpine" solr: - image: "solr:8" + image: "solr:8.11.1" ports: - "127.0.0.1:8983:8983" volumes: diff --git a/docker/run_tests/docker-compose.yml b/docker/run_tests/docker-compose.yml index a59b59c8b..36d5fd7bb 100644 --- a/docker/run_tests/docker-compose.yml +++ b/docker/run_tests/docker-compose.yml @@ -1,11 +1,11 @@ --- -version: '3.2' +version: '3.4' services: redis: image: "redis:alpine" solr: - image: "solr:8" + image: "solr:8.11.1" container_name: mampf_solr_test volumes: - type: bind From ca6971b6eded799077da7dc94121ef2cd8760cc3 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Wed, 5 Jan 2022 22:29:34 +0100 Subject: [PATCH 05/28] update from rails 6.1.4.1 to 6.1.4.4 --- Gemfile | 6 +- Gemfile.lock | 233 ++++++++++++++++++++++++++------------------------- 2 files changed, 120 insertions(+), 119 deletions(-) diff --git a/Gemfile b/Gemfile index 5122aa18f..308fc0841 100644 --- a/Gemfile +++ b/Gemfile @@ -4,7 +4,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" } ruby '3.0.3' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem "rails", "~> 6.1.4.1" +gem "rails", "~> 6.1.4.4" # Use dalli for caching to memcached in production gem "dalli", ">= 2.7" # Ruby wrapper for UglifyJS JavaScript compressor @@ -73,7 +73,7 @@ gem "activerecord-import", git: "https://github.com/zdennis/activerecord-import.git", branch: "master" gem "thredded", - github: "zebleck/thredded", + github: "MaMpf-HD/thredded", branch: "master" gem "kramdown-parser-gfm" gem "thredded-markdown_katex", @@ -92,7 +92,7 @@ gem "barby" gem "rqrcode" gem "sidekiq" gem "sidekiq-cron", "~> 1.1" -gem "faraday" +gem "faraday", "~> 1.8" gem "globalize" gem "globalize-accessors" gem "commontator", diff --git a/Gemfile.lock b/Gemfile.lock index f873044a2..2b19d6c39 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,30 @@ +GIT + remote: https://github.com/MaMpf-HD/thredded.git + revision: 5ac8f75b8f52453b1964845d96b9992623363747 + branch: master + specs: + thredded (0.16.16) + active_record_union (>= 1.3.0) + autoprefixer-rails + db_text_search (~> 0.3.2) + friendly_id + html-pipeline + htmlentities + inline_svg (>= 1.6.0) + kaminari + kramdown (>= 2.0.0) + kramdown-parser-gfm + nokogiri + onebox (>= 1.8.99) + pundit (>= 1.1.0) + rails (>= 4.2.10, != 6.0.0.rc2) + rails_gravatar + rinku + sanitize + sassc-rails (>= 2.0.0) + sprockets-es6 + timeago_js (>= 3.0.2.2) + GIT remote: https://github.com/fosterfarrell9/commontator revision: cfc3e47c2d9b7002738f788fb463a89f318ee63a @@ -9,7 +36,7 @@ GIT GIT remote: https://github.com/rails/sprockets-rails - revision: 582e284bab79ea0663addf6b15f80b19835af351 + revision: 5badf679b206e4f218b9e3a42730d27779e572b2 branch: master specs: sprockets-rails (3.4.2) @@ -37,39 +64,12 @@ GIT GIT remote: https://github.com/zdennis/activerecord-import.git - revision: 213470f29f6ca63d15b2696dbff28c292809ffc6 + revision: 85d697d569062b5fadc72984ab872216dbf6ea59 branch: master specs: - activerecord-import (1.2.0) + activerecord-import (1.3.0) activerecord (>= 4.2) -GIT - remote: https://github.com/zebleck/thredded.git - revision: 3cfcaa3aba383706facfad341cfaed5e6160f3fb - branch: master - specs: - thredded (0.16.16) - active_record_union (>= 1.3.0) - autoprefixer-rails - db_text_search (~> 0.3.2) - friendly_id - html-pipeline - htmlentities - inline_svg (>= 1.6.0) - kaminari - kramdown (>= 2.0.0) - kramdown-parser-gfm - nokogiri - onebox (>= 1.8.99) - pundit (>= 1.1.0) - rails (>= 4.2.10, != 6.0.0.rc2) - rails_gravatar - rinku - sanitize - sassc-rails (>= 2.0.0) - sprockets-es6 - timeago_js (>= 3.0.2.2) - GEM remote: https://rubygems.org/ specs: @@ -77,40 +77,40 @@ GEM RubyInline (3.12.5) ZenTest (~> 4.3) ZenTest (4.12.0) - actioncable (6.1.4.1) - actionpack (= 6.1.4.1) - activesupport (= 6.1.4.1) + actioncable (6.1.4.4) + actionpack (= 6.1.4.4) + activesupport (= 6.1.4.4) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.4.1) - actionpack (= 6.1.4.1) - activejob (= 6.1.4.1) - activerecord (= 6.1.4.1) - activestorage (= 6.1.4.1) - activesupport (= 6.1.4.1) + actionmailbox (6.1.4.4) + actionpack (= 6.1.4.4) + activejob (= 6.1.4.4) + activerecord (= 6.1.4.4) + activestorage (= 6.1.4.4) + activesupport (= 6.1.4.4) mail (>= 2.7.1) - actionmailer (6.1.4.1) - actionpack (= 6.1.4.1) - actionview (= 6.1.4.1) - activejob (= 6.1.4.1) - activesupport (= 6.1.4.1) + actionmailer (6.1.4.4) + actionpack (= 6.1.4.4) + actionview (= 6.1.4.4) + activejob (= 6.1.4.4) + activesupport (= 6.1.4.4) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.4.1) - actionview (= 6.1.4.1) - activesupport (= 6.1.4.1) + actionpack (6.1.4.4) + actionview (= 6.1.4.4) + activesupport (= 6.1.4.4) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.4.1) - actionpack (= 6.1.4.1) - activerecord (= 6.1.4.1) - activestorage (= 6.1.4.1) - activesupport (= 6.1.4.1) + actiontext (6.1.4.4) + actionpack (= 6.1.4.4) + activerecord (= 6.1.4.4) + activestorage (= 6.1.4.4) + activesupport (= 6.1.4.4) nokogiri (>= 1.8.5) - actionview (6.1.4.1) - activesupport (= 6.1.4.1) + actionview (6.1.4.4) + activesupport (= 6.1.4.4) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) @@ -122,24 +122,24 @@ GEM jsonapi-renderer (>= 0.1.1.beta1, < 0.3) active_record_union (1.3.0) activerecord (>= 4.0) - activejob (6.1.4.1) - activesupport (= 6.1.4.1) + activejob (6.1.4.4) + activesupport (= 6.1.4.4) globalid (>= 0.3.6) - activemodel (6.1.4.1) - activesupport (= 6.1.4.1) - activerecord (6.1.4.1) - activemodel (= 6.1.4.1) - activesupport (= 6.1.4.1) + activemodel (6.1.4.4) + activesupport (= 6.1.4.4) + activerecord (6.1.4.4) + activemodel (= 6.1.4.4) + activesupport (= 6.1.4.4) activerecord-nulldb-adapter (0.8.0) activerecord (>= 5.2.0, < 7.1) - activestorage (6.1.4.1) - actionpack (= 6.1.4.1) - activejob (= 6.1.4.1) - activerecord (= 6.1.4.1) - activesupport (= 6.1.4.1) + activestorage (6.1.4.4) + actionpack (= 6.1.4.4) + activejob (= 6.1.4.4) + activerecord (= 6.1.4.4) + activesupport (= 6.1.4.4) marcel (~> 1.0.0) mini_mime (>= 1.1.0) - activesupport (6.1.4.1) + activesupport (6.1.4.4) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -154,7 +154,7 @@ GEM public_suffix (>= 2.0.2, < 5.0) afm (0.2.2) ast (2.4.2) - autoprefixer-rails (10.3.3.0) + autoprefixer-rails (10.4.0.0) execjs (~> 2) babel-source (5.8.35) babel-transpiler (0.7.0) @@ -208,9 +208,9 @@ GEM crass (1.0.6) css_parser (1.11.0) addressable - cypress-on-rails (1.11.0) + cypress-on-rails (1.12.0) rack - dalli (3.1.3) + dalli (3.2.0) database_cleaner (2.0.1) database_cleaner-active_record (~> 2.0.0) database_cleaner-active_record (2.0.1) @@ -219,14 +219,14 @@ GEM database_cleaner-core (2.0.1) db_text_search (0.3.2) activerecord (>= 4.1.15, < 7.0) - devise (4.8.0) + devise (4.8.1) bcrypt (~> 3.0) orm_adapter (~> 0.1) railties (>= 4.1.0) responders warden (~> 1.2.3) devise-bootstrap-views (1.1.0) - diff-lcs (1.4.4) + diff-lcs (1.5.0) docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) @@ -275,7 +275,7 @@ GEM faraday-net_http_persistent (1.2.0) faraday-patron (1.0.0) faraday-rack (1.0.0) - fastimage (2.2.5) + fastimage (2.2.6) ffi (1.15.4) filesize (0.2.0) friendly_id (5.4.2) @@ -314,7 +314,8 @@ GEM inline_svg (1.7.2) activesupport (>= 3.0) nokogiri (>= 1.6) - jbuilder (2.11.3) + jbuilder (2.11.5) + actionview (>= 5.0.0) activesupport (>= 5.0.0) jquery-rails (4.4.0) rails-dom-testing (>= 1, < 3) @@ -327,18 +328,18 @@ GEM sprockets-rails json (2.6.1) jsonapi-renderer (0.2.2) - kaminari (1.2.1) + kaminari (1.2.2) activesupport (>= 4.1.0) - kaminari-actionview (= 1.2.1) - kaminari-activerecord (= 1.2.1) - kaminari-core (= 1.2.1) - kaminari-actionview (1.2.1) + kaminari-actionview (= 1.2.2) + kaminari-activerecord (= 1.2.2) + kaminari-core (= 1.2.2) + kaminari-actionview (1.2.2) actionview - kaminari-core (= 1.2.1) - kaminari-activerecord (1.2.1) + kaminari-core (= 1.2.2) + kaminari-activerecord (1.2.2) activerecord - kaminari-core (= 1.2.1) - kaminari-core (1.2.1) + kaminari-core (= 1.2.2) + kaminari-core (1.2.2) kaminari-i18n (0.5.0) kaminari rails @@ -369,7 +370,7 @@ GEM mime-types-data (3.2021.1115) mini_magick (4.11.0) mini_mime (1.1.2) - minitest (5.14.4) + minitest (5.15.0) msgpack (1.4.2) multi_json (1.15.0) multipart-post (2.1.1) @@ -388,9 +389,9 @@ GEM options (2.3.2) orm_adapter (0.5.0) parallel (1.21.0) - parser (3.0.3.2) + parser (3.1.0.0) ast (~> 2.4.1) - pdf-reader (2.7.0) + pdf-reader (2.8.0) Ascii85 (~> 1.0) afm (~> 0.2.1) hashery (~> 2.0) @@ -410,7 +411,7 @@ GEM progress_bar (1.3.3) highline (>= 1.6, < 3) options (~> 2.3.0) - prometheus_exporter (1.0.0) + prometheus_exporter (1.0.1) webrick public_suffix (4.0.6) puma (4.3.10) @@ -420,24 +421,24 @@ GEM raabro (1.4.0) racc (1.6.0) rack (2.2.3) - rack-proxy (0.7.0) + rack-proxy (0.7.2) rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (6.1.4.1) - actioncable (= 6.1.4.1) - actionmailbox (= 6.1.4.1) - actionmailer (= 6.1.4.1) - actionpack (= 6.1.4.1) - actiontext (= 6.1.4.1) - actionview (= 6.1.4.1) - activejob (= 6.1.4.1) - activemodel (= 6.1.4.1) - activerecord (= 6.1.4.1) - activestorage (= 6.1.4.1) - activesupport (= 6.1.4.1) + rails (6.1.4.4) + actioncable (= 6.1.4.4) + actionmailbox (= 6.1.4.4) + actionmailer (= 6.1.4.4) + actionpack (= 6.1.4.4) + actiontext (= 6.1.4.4) + actionview (= 6.1.4.4) + activejob (= 6.1.4.4) + activemodel (= 6.1.4.4) + activerecord (= 6.1.4.4) + activestorage (= 6.1.4.4) + activesupport (= 6.1.4.4) bundler (>= 1.15.0) - railties (= 6.1.4.1) + railties (= 6.1.4.4) sprockets-rails (>= 2.0.0) rails-dom-testing (2.0.3) activesupport (>= 4.2.0) @@ -449,14 +450,14 @@ GEM ruby-graphviz (~> 1.2) rails-html-sanitizer (1.4.2) loofah (~> 2.3) - rails-i18n (6.0.0) + rails-i18n (7.0.1) i18n (>= 0.7, < 2) - railties (>= 6.0.0, < 7) + railties (>= 6.0.0, < 8) rails_gravatar (1.0.4) actionview - railties (6.1.4.1) - actionpack (= 6.1.4.1) - activesupport (= 6.1.4.1) + railties (6.1.4.4) + actionpack (= 6.1.4.4) + activesupport (= 6.1.4.4) method_source rake (>= 0.13) thor (~> 1.0) @@ -486,7 +487,7 @@ GEM chunky_png (~> 1.0) rqrcode_core (~> 1.0) rqrcode_core (1.2.0) - rsolr (2.3.0) + rsolr (2.4.0) builder (>= 2.1.2) faraday (>= 0.9.0) rspec-core (3.10.1) @@ -515,7 +516,7 @@ GEM rubocop-ast (>= 0.6.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (1.15.0) + rubocop-ast (1.15.1) parser (>= 3.0.1.1) rubocop-packaging (0.5.1) rubocop (>= 0.89, < 2.0) @@ -573,7 +574,7 @@ GEM simplecov (~> 0.19) simplecov-html (0.12.3) simplecov_json_formatter (0.1.3) - smart_properties (1.16.3) + smart_properties (1.17.0) spring (2.1.1) spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) @@ -597,13 +598,13 @@ GEM sync (0.5.0) term-ansicolor (1.7.1) tins (~> 1.0) - thor (1.1.0) + thor (1.2.1) tilt (2.0.10) timeago_js (3.0.2.2) - tins (1.29.1) + tins (1.30.0) sync - trix-rails (2.3.0) - rails (> 4.1, < 7.0) + trix-rails (2.4.0) + rails (> 4.1) ttfunk (1.7.0) turbolinks (5.2.1) turbolinks-source (~> 5.2) @@ -640,7 +641,7 @@ GEM xkcd (1.1.1) google-search nokogiri (>= 1.5.0) - zeitwerk (2.5.1) + zeitwerk (2.5.3) PLATFORMS x86_64-linux @@ -672,7 +673,7 @@ DEPENDENCIES exception_handler (~> 0.8.0.0) factory_bot_rails faker - faraday + faraday (~> 1.8) fastimage filesize fuzzy-string-match @@ -698,7 +699,7 @@ DEPENDENCIES prometheus_exporter puma (~> 4.1) rack - rails (~> 6.1.4.1) + rails (~> 6.1.4.4) rails-erd rails-i18n responders From c68d6a60e8e096193f272f449020766e53458e40 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Sat, 6 Nov 2021 16:58:21 +0100 Subject: [PATCH 06/28] rspec tests --- app/models/watchlist.rb | 2 +- docker/run_tests/docker-compose.yml | 3 +++ spec/factories/watchlist_entries.rb | 2 +- spec/models/watchlist_entry_spec.rb | 22 ++++++++++++++++++++- spec/models/watchlist_spec.rb | 30 ++++++++++++++++++++++++++++- 5 files changed, 55 insertions(+), 4 deletions(-) diff --git a/app/models/watchlist.rb b/app/models/watchlist.rb index 7ca27b087..6b7330174 100644 --- a/app/models/watchlist.rb +++ b/app/models/watchlist.rb @@ -1,7 +1,7 @@ class Watchlist < ApplicationRecord belongs_to :user has_many :watchlist_entries, dependent: :destroy - has_many :media, through: :watchlist_entries + has_many :media, through: :watchlist_entries validates :name, presence: true, uniqueness: { scope: :user_id } diff --git a/docker/run_tests/docker-compose.yml b/docker/run_tests/docker-compose.yml index 36d5fd7bb..072bffa7c 100644 --- a/docker/run_tests/docker-compose.yml +++ b/docker/run_tests/docker-compose.yml @@ -56,6 +56,9 @@ services: - type: bind source: ../../app/ target: /usr/src/app/app/ + - type: bind + source: ../../spec/ + target: /usr/src/app/spec/ - /usr/src/app/completed_initial_run depends_on: - db diff --git a/spec/factories/watchlist_entries.rb b/spec/factories/watchlist_entries.rb index f07477aed..706187193 100644 --- a/spec/factories/watchlist_entries.rb +++ b/spec/factories/watchlist_entries.rb @@ -5,7 +5,7 @@ medium_position { 1 } trait :with_watchlist do - watchlist { create(:watchlist, user_id: 1) } + watchlist { create(:watchlist, user: FactoryBot.create(:user)) } end trait :with_medium do diff --git a/spec/models/watchlist_entry_spec.rb b/spec/models/watchlist_entry_spec.rb index 01be81aa6..fd4b561da 100644 --- a/spec/models/watchlist_entry_spec.rb +++ b/spec/models/watchlist_entry_spec.rb @@ -1,5 +1,25 @@ require 'rails_helper' RSpec.describe WatchlistEntry, type: :model do - pending "add some examples to (or delete) #{__FILE__}" + it 'has a valid factory' do + expect(FactoryBot.build(:watchlist_entry, :with_watchlist, :with_medium)).to be_valid + end + + it 'must have a medium' do + entry = FactoryBot.build(:watchlist_entry, :with_watchlist) + entry.valid? + expect(entry.errors[:medium]).to include('muss ausgefüllt werden') + end + + it 'must have a watchlist' do + entry = FactoryBot.build(:watchlist_entry, :with_medium) + entry.valid? + expect(entry.errors[:watchlist]).to include('muss ausgefüllt werden') + end + + it 'can only be once inside a watchlist' do + entry = FactoryBot.create(:watchlist_entry, :with_watchlist, :with_medium) + second_entry = WatchlistEntry.new(watchlist: entry.watchlist, medium: entry.medium) + expect(second_entry).to be_invalid + end end diff --git a/spec/models/watchlist_spec.rb b/spec/models/watchlist_spec.rb index bb8840fdf..9c6f30145 100644 --- a/spec/models/watchlist_spec.rb +++ b/spec/models/watchlist_spec.rb @@ -1,5 +1,33 @@ require 'rails_helper' RSpec.describe Watchlist, type: :model do - pending "add some examples to (or delete) #{__FILE__}" + it 'has a valid factory' do + expect(FactoryBot.build(:watchlist, :with_user)).to be_valid + end + + it 'must have a name' do + watchlist = Watchlist.new(name: nil) + watchlist.valid? + expect(watchlist.errors[:name]).to include('muss ausgefüllt werden') + end + + it 'must have a unique name' do + first_watchlist = FactoryBot.create(:watchlist, :with_user) + second_watchlist = Watchlist.new(user_id: first_watchlist.user_id, name:first_watchlist.name) + expect(second_watchlist).to be_invalid + end + + it 'can be public' do + watchlist = FactoryBot.build(:watchlist, :with_user) + watchlist.public = true + expect(watchlist.public).to eq true + end + + it 'ownership can be tested' do + watchlist = FactoryBot.build(:watchlist, :with_user) + user = FactoryBot.build(:user) + + expect(watchlist.ownedBy(user)).to eq false + expect(watchlist.ownedBy(watchlist.user)).to eq true + end end From a20b62bcade4f08ad9dd2d4187335c184e0acca6 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Tue, 8 Feb 2022 14:33:57 +0100 Subject: [PATCH 07/28] init commit - removed protected attribute for assignments and its occurence in views - replaced it with deletion_date for assignments in model and views - started change on the submissions cleaner - integrated datepicker for deletion_date - added migration for deletion_date attribute --- app/controllers/assignments_controller.rb | 3 ++- app/models/assignment.rb | 12 +++++++++++ app/models/submission.rb | 4 ++++ app/models/submission_cleaner.rb | 21 +++++++++---------- app/models/term.rb | 19 ----------------- app/models/user.rb | 4 ++++ app/models/user_submission_join.rb | 4 ++++ app/views/assignments/_form.html.erb | 9 +++++--- app/views/assignments/_row.html.erb | 4 +++- app/views/assignments/create.coffee | 5 +++++ app/views/assignments/edit.coffee | 6 ++++++ app/views/assignments/update.coffee | 11 +++++++++- app/views/lectures/edit/_assignments.html.erb | 2 +- app/views/submissions/_card_main.html.erb | 9 ++------ app/workers/submissions_cleaner.rb | 2 +- config/locales/de.yml | 6 ++++++ config/locales/en.yml | 4 ++++ ...162730_add_deletion_date_to_assignments.rb | 8 +++++++ db/schema.rb | 4 ++-- 19 files changed, 90 insertions(+), 47 deletions(-) create mode 100644 db/migrate/20220125162730_add_deletion_date_to_assignments.rb diff --git a/app/controllers/assignments_controller.rb b/app/controllers/assignments_controller.rb index 7f83b9a2e..75beacb1d 100644 --- a/app/controllers/assignments_controller.rb +++ b/app/controllers/assignments_controller.rb @@ -65,7 +65,8 @@ def set_assignment_locale def assignment_params params.require(:assignment).permit(:title, :medium_id, :lecture_id, - :deadline, :accepted_file_type, :protected) + :deadline, :accepted_file_type, + :deletion_date) end def check_editor_status diff --git a/app/models/assignment.rb b/app/models/assignment.rb index 7e274c4b7..fc03d708a 100644 --- a/app/models/assignment.rb +++ b/app/models/assignment.rb @@ -7,6 +7,14 @@ class Assignment < ApplicationRecord validates :title, uniqueness: { scope: [:lecture_id] }, presence: true validates :deadline, presence: true + validates :deletion_date, presence: true + validate :deletion_date_cannot_be_in_the_past + + def deletion_date_cannot_be_in_the_past + return unless deletion_date.present? && deletion_date < DateTime.now + + errors.add(:deletion_date, I18n.t('activerecord.errors.models.assignment.attributes.deletion_date.in_past')) + end scope :active, -> { where('deadline >= ?', Time.now) } @@ -126,4 +134,8 @@ def accepted_for_file_input return accepted_file_type unless accepted_file_type == '.tar.gz' '.gz' end + + def self.to_be_deleted + Assignment.where('deletion_date < ?', DateTime.now) + end end diff --git a/app/models/submission.rb b/app/models/submission.rb index 9ad1dff1b..93732c1a2 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -252,6 +252,10 @@ def self.bulk_corrections!(tutorial, assignment, files) report end + def self.to_be_deleted + Submission.where(assignment: Assignment.to_be_deleted) + end + private def matching_lecture diff --git a/app/models/submission_cleaner.rb b/app/models/submission_cleaner.rb index cc2eeb4b2..e061e9582 100644 --- a/app/models/submission_cleaner.rb +++ b/app/models/submission_cleaner.rb @@ -20,8 +20,9 @@ def clean! end def set_attributes + fetch_previous_term_props + return unless @previous_term determine_actions - fetch_previous_term_props if @advance true end @@ -29,9 +30,6 @@ def set_attributes def determine_actions @advance = false - @previous_term = Term.previous_by_date(@date) - return unless @previous_term - return unless @date.in?(previous_term.submission_deletion_info_dates) if date == @previous_term.end_date + 1.day @advance = @previous_term.submission_deletion_mail.nil? @reminder = false @@ -39,17 +37,18 @@ def determine_actions @advance = @previous_term.submission_deletion_mail.present? && @previous_term.submission_deletion_reminder.nil? @reminder = true - elsif date == @previous_term.submission_deletion_date - @advance = @previous_term.submission_deletion_mail.present? && - @previous_term.submission_deletion_reminder.present? && - @previous_term.submissions_deleted_at.nil? + if @submissions_to_be_deleted.present? + @advance = true @destroy = true - end + elsif @previous_term end def fetch_previous_term_props - @submissions = @previous_term.unprotected_submissions - @submitters = @previous_term.unprotected_submitters + @previous_term = Term.previous_by_date(@date) + return unless @previous_term + @submissions = @previous_term.submissions + @submissions_to_be_deleted = @previous_term.submissions.where(deletion_date: Date.today.all_day) + @submitters = @previous_term.submitters @lectures = @previous_term.lectures_with_submissions end diff --git a/app/models/term.rb b/app/models/term.rb index 4f481056f..3e24389e1 100644 --- a/app/models/term.rb +++ b/app/models/term.rb @@ -62,18 +62,11 @@ def previous def assignments Assignment.where(lecture: lectures) end - def unprotected_assignments - Assignment.where(lecture: lectures, protected: false) - end def submissions Submission.where(assignment: assignments) end - def unprotected_submissions - Submission.where(assignment: unprotected_assignments) - end - def submitter_ids UserSubmissionJoin.where(submission: submissions).pluck(:user_id).uniq end @@ -82,18 +75,6 @@ def submitters User.where(id: submitter_ids) end - def unprotected_submitter_ids - UserSubmissionJoin.where(submission: unprotected_submissions).pluck(:user_id).uniq - end - - def unprotected_submitters - User.where(id: unprotected_submitter_ids) - end - - def submission_deletion_date - end_date + 15.days - end - def assignments_with_submissions Assignment.where(id: submissions.pluck(:assignment_id).uniq) end diff --git a/app/models/user.rb b/app/models/user.rb index 88a9cafd6..dfa8dd666 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -137,6 +137,10 @@ def self.select_editors .map { |u| [ "#{u.first} (#{u.second})", u.third] } end + def self.on_submissions_to_be_deleted + User.where(id: UserSubmissionJoin.to_be_deleted_user_ids) + end + def courses Course.where(id: lectures.pluck(:course_id).uniq) end diff --git a/app/models/user_submission_join.rb b/app/models/user_submission_join.rb index d0c6b6d01..fffde40b2 100644 --- a/app/models/user_submission_join.rb +++ b/app/models/user_submission_join.rb @@ -5,6 +5,10 @@ class UserSubmissionJoin < ApplicationRecord validate :only_one_per_assignment, on: :create validate :max_team_size, on: :create + def self.to_be_deleted_user_ids + UserSubmissionJoin.where(submission: Submission.to_be_deleted).pluck(:user_id).uniq + end + def assignment submission.assignment end diff --git a/app/views/assignments/_form.html.erb b/app/views/assignments/_form.html.erb index adfbdc6e0..5095ecfce 100644 --- a/app/views/assignments/_form.html.erb +++ b/app/views/assignments/_form.html.erb @@ -38,9 +38,12 @@ id: "assignment_filetype_id_#{assignment.id}" } %>
- <%= f.check_box :protected, - class: 'form-control', - id: "assignment_protected_#{assignment.id}"%> + <%= f.text_field :deletion_date, + class: 'form-control', + id: "assignment_deletion_date_#{assignment.id}" %> +
+
<%= f.submit t('buttons.save'), diff --git a/app/views/assignments/_row.html.erb b/app/views/assignments/_row.html.erb index e0d8b265d..616b86ab2 100644 --- a/app/views/assignments/_row.html.erb +++ b/app/views/assignments/_row.html.erb @@ -22,7 +22,9 @@ <%= assignment.accepted_file_type %>
- > + <% if assignment.deletion_date %> + <%= I18n.l(assignment.deletion_date, format: :long, locale: :'de') %> + <% end %>
<% if assignment.persisted? %> diff --git a/app/views/assignments/create.coffee b/app/views/assignments/create.coffee index 807824a6d..6e51ec2aa 100644 --- a/app/views/assignments/create.coffee +++ b/app/views/assignments/create.coffee @@ -16,6 +16,11 @@ $('#assignment-deadline-error') .append('<%= @errors[:deadline].join(" ") %>').show() $('#assignment_deadline').addClass('is-invalid') <% end %> +<% if @errors[:deletion_date].present? %> +$('#assignment-deletion-date-error') + .append('<%= @errors[:deletion_date].join(" ") %>').show() +$('#assignment_deletion_date').addClass('is-invalid') +<% end %> <% else %> $('.assignmentRow[data-id="0"') .replaceWith('<%= j render partial: "assignments/row", diff --git a/app/views/assignments/edit.coffee b/app/views/assignments/edit.coffee index d88429200..dd04ad208 100644 --- a/app/views/assignments/edit.coffee +++ b/app/views/assignments/edit.coffee @@ -14,6 +14,12 @@ $("#assignment_deadline_<%= @assignment.id %>").datetimepicker format:'d.m.Y H:i' inline:false +d = moment($("#assignment_deletion_date_<%= @assignment.id %>").val(),"YYYY-MM-DD hh:mm:ss z") +$("#assignment_deletion_date_<%= @assignment.id %>").val (d.format("DD.MM.Y H:mm")) +$("#assignment_deletion_date_<%= @assignment.id %>").datetimepicker + format:'d.m.Y H:i' + inline:false + <% unless @assignment.medium %> # make sure that no medium is preselected $('#assignment_medium_id_<%= @assignment.id %>').val(null).trigger('change') diff --git a/app/views/assignments/update.coffee b/app/views/assignments/update.coffee index 5b4cdb1c1..55c4573c4 100644 --- a/app/views/assignments/update.coffee +++ b/app/views/assignments/update.coffee @@ -9,7 +9,16 @@ $('#assignment-title-error') .append('<%= @errors[:title].join(" ") %>').show() $('#assignment_title').addClass('is-invalid') <% end %> - +<% if @errors[:deadline].present? %> +$('#assignment-deadline-error') + .append('<%= @errors[:deadline].join(" ") %>').show() +$('#assignment_deadline').addClass('is-invalid') +<% end %> +<% if @errors[:deletion_date].present? %> +$('#assignment-deletion-date-error') + .append('<%= @errors[:deletion_date].join(" ") %>').show() +$('#assignment_deletion_date').addClass('is-invalid') +<% end %> <% else %> $('.assignmentRow[data-id="<%= @assignment.id %>') .replaceWith('<%= j render partial: "assignments/row", diff --git a/app/views/lectures/edit/_assignments.html.erb b/app/views/lectures/edit/_assignments.html.erb index 8425e8c4c..55ea0775d 100644 --- a/app/views/lectures/edit/_assignments.html.erb +++ b/app/views/lectures/edit/_assignments.html.erb @@ -55,7 +55,7 @@
- <%= t('assignment.protected') %> + <%= t('assignment.deletion_date') %> <%= helpdesk(t('assignment.protected_info'), true) %>
diff --git a/app/views/submissions/_card_main.html.erb b/app/views/submissions/_card_main.html.erb index e3555a753..81184f71a 100644 --- a/app/views/submissions/_card_main.html.erb +++ b/app/views/submissions/_card_main.html.erb @@ -84,13 +84,8 @@ <%= helpdesk(t('submission.deletion_date_info'), false) %> - <% if assignment.protected %> - <%= t("assignment.protected") %> - <%= helpdesk(t('assignment.protected_info'), true) %> - <%else%> - <%= I18n.l(assignment.lecture.submission_deletion_date, - format: :long) %> - <%end%> + <%= I18n.l(assignment.deletion_date, + format: :long) %> diff --git a/app/workers/submissions_cleaner.rb b/app/workers/submissions_cleaner.rb index f1bd9ac7d..b80289e85 100644 --- a/app/workers/submissions_cleaner.rb +++ b/app/workers/submissions_cleaner.rb @@ -3,6 +3,6 @@ class SubmissionsCleaner def perform submission_cleaner = SubmissionCleaner.new(date: Date.today) - submission_cleaner.clean! + #submission_cleaner.clean! end end \ No newline at end of file diff --git a/config/locales/de.yml b/config/locales/de.yml index 288b74aff..996063b47 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -2785,6 +2785,7 @@ de: nothing_yet_in_lecture: > In dieser Vorlesung war bisher noch nichts abzugeben. protected: Geschützte Abgabe + deletion_date: Löschdatum protected_info: > "Geschützte" Abgaben werden bis zur manuellen Löschung aufbewahrt und werden nicht automatisiert nach Ablauf der Aufbewahrungsfrist (Semesterende + Karenzzeit von 14 Tagen) gelöscht. destruction_info: @@ -3456,6 +3457,11 @@ de: deadline: blank: > Es muss ein Abgabetermin angegeben werden. + deletion_date: + blank: > + Es muss ein Löschdatum angegeben werden. + in_past: > + Löschdatum darf nicht in der Vergangenheit liegen. chapter: attributes: title: diff --git a/config/locales/en.yml b/config/locales/en.yml index 914ef77fc..f82ba61a6 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -2619,6 +2619,7 @@ en: nothing_yet_in_lecture: > In this lecture nothing had yet to be submitted. protected: Protected Assignment + deletion_date: Deletion date protected_info: > "Protected" Assignments won't be deleted at the end of the semester. destruction_info: > @@ -3266,6 +3267,9 @@ en: blank: You need to give a title. deadline: blank: You need to give a due date. + deletion_date: + blank: You need to give a deletion date. + in_past: Deletion date can't lie in the past. chapter: attributes: title: diff --git a/db/migrate/20220125162730_add_deletion_date_to_assignments.rb b/db/migrate/20220125162730_add_deletion_date_to_assignments.rb new file mode 100644 index 000000000..22fd79340 --- /dev/null +++ b/db/migrate/20220125162730_add_deletion_date_to_assignments.rb @@ -0,0 +1,8 @@ +class AddDeletionDateToAssignments < ActiveRecord::Migration[6.1] + def change + add_column :assignments, + :deletion_date, + :datetime, null: false, default: (Term.active&.end_date || (Date.today + 180.days)) + 15.days + remove_column :assignments, :protected, :boolean, :default => false + end +end diff --git a/db/schema.rb b/db/schema.rb index c173fa0ef..3c71df82d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_11_19_181430) do +ActiveRecord::Schema.define(version: 2022_01_25_162730) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" @@ -45,7 +45,7 @@ t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false t.text "accepted_file_type", default: ".pdf" - t.boolean "protected", default: false + t.datetime "deletion_date", default: "2020-10-15 00:00:00", null: false t.index ["lecture_id"], name: "index_assignments_on_lecture_id" t.index ["medium_id"], name: "index_assignments_on_medium_id" end From 7a2cda46bc4e36046d3c45914dab68534ce6a9b2 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Tue, 8 Feb 2022 14:53:31 +0100 Subject: [PATCH 08/28] fixed small syntax error --- app/models/submission_cleaner.rb | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/app/models/submission_cleaner.rb b/app/models/submission_cleaner.rb index e061e9582..14e122c43 100644 --- a/app/models/submission_cleaner.rb +++ b/app/models/submission_cleaner.rb @@ -40,7 +40,7 @@ def determine_actions if @submissions_to_be_deleted.present? @advance = true @destroy = true - elsif @previous_term + end end def fetch_previous_term_props @@ -112,21 +112,21 @@ def send_info_mail_to_submitters reminder: @reminder, locale: l) .submission_deletion_email.deliver_now - end end end + end - def send_info_mail_to_editors - @lectures.each do |l| - editor_ids = l.editors.pluck(:id) + [l.teacher.id] - NotificationMailer.with(recipients: editor_ids, - term: @previous_term, - lecture: l, - deletion_date: - @previous_term.submission_deletion_date, - reminder: @reminder, - locale: l.locale) - .submission_deletion_lecture_email.deliver_now - end + def send_info_mail_to_editors + @lectures.each do |l| + editor_ids = l.editors.pluck(:id) + [l.teacher.id] + NotificationMailer.with(recipients: editor_ids, + term: @previous_term, + lecture: l, + deletion_date: + @previous_term.submission_deletion_date, + reminder: @reminder, + locale: l.locale) + .submission_deletion_lecture_email.deliver_now end + end end From 502ea39facdef7881c179e29f6244559ebc659c2 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Tue, 8 Feb 2022 18:28:39 +0100 Subject: [PATCH 09/28] -changed deletion_date type from datetime to date -reworked submissions_cleaner --- app/models/assignment.rb | 8 ++ app/models/submission_cleaner.rb | 91 ++++++++----------- ...162730_add_deletion_date_to_assignments.rb | 2 +- db/schema.rb | 2 +- 4 files changed, 47 insertions(+), 56 deletions(-) diff --git a/app/models/assignment.rb b/app/models/assignment.rb index fc03d708a..b6850dfe2 100644 --- a/app/models/assignment.rb +++ b/app/models/assignment.rb @@ -33,6 +33,14 @@ def submission(user) &.first&.submission end + def submitter_ids + UserSubmissionJoin.where(submission: submissions).pluck(:user_id).uniq + end + + def submitters + User.where(id: submitter_ids) + end + def active? Time.now <= deadline end diff --git a/app/models/submission_cleaner.rb b/app/models/submission_cleaner.rb index 14e122c43..c7674ccc2 100644 --- a/app/models/submission_cleaner.rb +++ b/app/models/submission_cleaner.rb @@ -11,64 +11,48 @@ def initialize(date:) end def clean! - set_attributes - return unless @advance - - warn_about_destruction and return unless @destroy + @previous_term = Term.previous_by_date(@date) + return unless @previous_term - destroy_and_inform + check_for_first_mail + check_for_reminder_mail + check_for_deletion end - def set_attributes - fetch_previous_term_props - return unless @previous_term - determine_actions - true + def check_for_first_mail + @deletion_date = Time.zone.today + 14.days + fetch_props + @reminder = false + + send_info_mail_to_submitters + send_info_mail_to_editors end - private + def check_for_reminder_mail + @deletion_date = Time.zone.today + 7.days + fetch_props + @reminder = true - def determine_actions - @advance = false - if date == @previous_term.end_date + 1.day - @advance = @previous_term.submission_deletion_mail.nil? - @reminder = false - elsif date == @previous_term.end_date + 8.days - @advance = @previous_term.submission_deletion_mail.present? && - @previous_term.submission_deletion_reminder.nil? - @reminder = true - if @submissions_to_be_deleted.present? - @advance = true - @destroy = true - end + send_info_mail_to_submitters + send_info_mail_to_editors end - def fetch_previous_term_props - @previous_term = Term.previous_by_date(@date) - return unless @previous_term - @submissions = @previous_term.submissions - @submissions_to_be_deleted = @previous_term.submissions.where(deletion_date: Date.today.all_day) - @submitters = @previous_term.submitters - @lectures = @previous_term.lectures_with_submissions - end + def check_for_deletion + @deletion_date = Time.zone.today + fetch_props - def destroy_and_inform - @previous_term.update(submissions_deleted_at: Time.now) @submissions.each(&:destroy) + send_destruction_mail_to_submitters send_destruction_mail_to_editors end - def warn_about_destruction - if @reminder - @previous_term.update(submission_deletion_reminder: Time.now) - else - @previous_term.update(submission_deletion_mail: Time.now) - end + private - send_info_mail_to_submitters - send_info_mail_to_editors - true + def fetch_props + @assignments = Assignments.where(deletion_date: @deletion_date) + @submitters = @assignments.submitters + @lectures = Lecture.find_by(id: @assignments.pluck(:lecture_id)) end def send_destruction_mail_to_submitters @@ -99,19 +83,19 @@ def send_destruction_mail_to_editors end def send_info_mail_to_submitters - return unless @submitters.present? + return if @submitters.blank? I18n.available_locales.each do |l| local_submitter_ids = @submitters.where(locale: l).pluck(:id) next if local_submitter_ids.empty? - local_submitter_ids.in_groups_of(200, false) do |group| - NotificationMailer.with(recipients: group, - term: @previous_term, - deletion_date: - @previous_term.submission_deletion_date, - reminder: @reminder, - locale: l) - .submission_deletion_email.deliver_now + + local_submitter_ids.in_groups_of(200, false) do |group| + NotificationMailer.with(recipients: group, + term: @previous_term, + deletion_date: @deletion_date, + reminder: @reminder, + locale: l) + .submission_deletion_email.deliver_now end end end @@ -122,8 +106,7 @@ def send_info_mail_to_editors NotificationMailer.with(recipients: editor_ids, term: @previous_term, lecture: l, - deletion_date: - @previous_term.submission_deletion_date, + deletion_date: @deletion_date, reminder: @reminder, locale: l.locale) .submission_deletion_lecture_email.deliver_now diff --git a/db/migrate/20220125162730_add_deletion_date_to_assignments.rb b/db/migrate/20220125162730_add_deletion_date_to_assignments.rb index 22fd79340..05aaa3b5b 100644 --- a/db/migrate/20220125162730_add_deletion_date_to_assignments.rb +++ b/db/migrate/20220125162730_add_deletion_date_to_assignments.rb @@ -2,7 +2,7 @@ class AddDeletionDateToAssignments < ActiveRecord::Migration[6.1] def change add_column :assignments, :deletion_date, - :datetime, null: false, default: (Term.active&.end_date || (Date.today + 180.days)) + 15.days + :date, null: false, default: (Term.active&.end_date || (Date.today + 180.days)) + 15.days remove_column :assignments, :protected, :boolean, :default => false end end diff --git a/db/schema.rb b/db/schema.rb index 3c71df82d..5228c2c6d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -45,7 +45,7 @@ t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false t.text "accepted_file_type", default: ".pdf" - t.datetime "deletion_date", default: "2020-10-15 00:00:00", null: false + t.date "deletion_date", default: "2020-10-15", null: false t.index ["lecture_id"], name: "index_assignments_on_lecture_id" t.index ["medium_id"], name: "index_assignments_on_medium_id" end From a09dfa9c6975ee6cc5325f9ce860df3d6bfa6657 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Tue, 22 Feb 2022 12:59:05 +0100 Subject: [PATCH 10/28] changed homework editing view fixed watchlist spec issue TODO: submission_cleaner spec tests --- app/models/assignment.rb | 5 +---- app/models/submission.rb | 4 ---- app/models/term.rb | 8 ++++++++ app/models/user.rb | 4 ---- app/models/user_submission_join.rb | 4 ---- app/views/assignments/_form.html.erb | 8 +++++--- app/views/assignments/edit.coffee | 6 ------ app/workers/submissions_cleaner.rb | 2 +- spec/cypress/integration/watchlists_spec.js | 2 ++ 9 files changed, 17 insertions(+), 26 deletions(-) diff --git a/app/models/assignment.rb b/app/models/assignment.rb index b6850dfe2..7ce950d7d 100644 --- a/app/models/assignment.rb +++ b/app/models/assignment.rb @@ -142,8 +142,5 @@ def accepted_for_file_input return accepted_file_type unless accepted_file_type == '.tar.gz' '.gz' end - - def self.to_be_deleted - Assignment.where('deletion_date < ?', DateTime.now) - end + end diff --git a/app/models/submission.rb b/app/models/submission.rb index 93732c1a2..9ad1dff1b 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -252,10 +252,6 @@ def self.bulk_corrections!(tutorial, assignment, files) report end - def self.to_be_deleted - Submission.where(assignment: Assignment.to_be_deleted) - end - private def matching_lecture diff --git a/app/models/term.rb b/app/models/term.rb index 3e24389e1..57e0bad5d 100644 --- a/app/models/term.rb +++ b/app/models/term.rb @@ -92,6 +92,14 @@ def submission_deletion_info_dates [end_date + 1.day, end_date + 8.days, submission_deletion_date] end + def self.possible_deletion_dates + return [Time.zone.today + 6.months] if Term.active.blank? + + [Term.active.end_date + 2.weeks, + Term.active.end_date + 2.weeks + 3.months, + Term.active.end_date + 2.weeks + 6.months] + end + # array of all terms together with their ids for use in options_for_select def self.select_terms(independent = false) return ['bla', nil] if independent diff --git a/app/models/user.rb b/app/models/user.rb index dfa8dd666..88a9cafd6 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -137,10 +137,6 @@ def self.select_editors .map { |u| [ "#{u.first} (#{u.second})", u.third] } end - def self.on_submissions_to_be_deleted - User.where(id: UserSubmissionJoin.to_be_deleted_user_ids) - end - def courses Course.where(id: lectures.pluck(:course_id).uniq) end diff --git a/app/models/user_submission_join.rb b/app/models/user_submission_join.rb index fffde40b2..d0c6b6d01 100644 --- a/app/models/user_submission_join.rb +++ b/app/models/user_submission_join.rb @@ -5,10 +5,6 @@ class UserSubmissionJoin < ApplicationRecord validate :only_one_per_assignment, on: :create validate :max_team_size, on: :create - def self.to_be_deleted_user_ids - UserSubmissionJoin.where(submission: Submission.to_be_deleted).pluck(:user_id).uniq - end - def assignment submission.assignment end diff --git a/app/views/assignments/_form.html.erb b/app/views/assignments/_form.html.erb index 5095ecfce..06122fd85 100644 --- a/app/views/assignments/_form.html.erb +++ b/app/views/assignments/_form.html.erb @@ -38,9 +38,11 @@ id: "assignment_filetype_id_#{assignment.id}" } %>
- <%= f.text_field :deletion_date, - class: 'form-control', - id: "assignment_deletion_date_#{assignment.id}" %> + <%= f.select :deletion_date, + Term.possible_deletion_dates, + {}, + class: 'form-control', + id: "assignment_deletion_date_#{assignment.id}" %>
diff --git a/app/views/assignments/edit.coffee b/app/views/assignments/edit.coffee index dd04ad208..d88429200 100644 --- a/app/views/assignments/edit.coffee +++ b/app/views/assignments/edit.coffee @@ -14,12 +14,6 @@ $("#assignment_deadline_<%= @assignment.id %>").datetimepicker format:'d.m.Y H:i' inline:false -d = moment($("#assignment_deletion_date_<%= @assignment.id %>").val(),"YYYY-MM-DD hh:mm:ss z") -$("#assignment_deletion_date_<%= @assignment.id %>").val (d.format("DD.MM.Y H:mm")) -$("#assignment_deletion_date_<%= @assignment.id %>").datetimepicker - format:'d.m.Y H:i' - inline:false - <% unless @assignment.medium %> # make sure that no medium is preselected $('#assignment_medium_id_<%= @assignment.id %>').val(null).trigger('change') diff --git a/app/workers/submissions_cleaner.rb b/app/workers/submissions_cleaner.rb index b80289e85..f1bd9ac7d 100644 --- a/app/workers/submissions_cleaner.rb +++ b/app/workers/submissions_cleaner.rb @@ -3,6 +3,6 @@ class SubmissionsCleaner def perform submission_cleaner = SubmissionCleaner.new(date: Date.today) - #submission_cleaner.clean! + submission_cleaner.clean! end end \ No newline at end of file diff --git a/spec/cypress/integration/watchlists_spec.js b/spec/cypress/integration/watchlists_spec.js index 12f79f93e..92075344f 100644 --- a/spec/cypress/integration/watchlists_spec.js +++ b/spec/cypress/integration/watchlists_spec.js @@ -29,6 +29,7 @@ describe("Watchlists", () => { cy.get('#watchlistNameField').type('Lernliste'); cy.get('#createWatchlistBtn').click(); cy.get('#watchlistEntrySubmitButton').click(); + cy.wait(100); cy.get('div.text-light > .fa-list').click(); cy.get('#watchlistEntrySubmitButton').click(); cy.get('.invalid-feedback').should('exist'); @@ -53,6 +54,7 @@ describe("Watchlists", () => { cy.wait(100); cy.get('#watchlistDescriptionField').type('Dies ist eine Lernliste.'); cy.get('#confirmChangeWatchlistButton').click(); + cy.wait(100); cy.get('#watchlistButton').contains('Lernliste'); cy.get('#descriptionButton').click(); cy.get('.card').contains('Dies ist eine Lernliste.') From 97a6e96276f8c934b17132c76da3079d871f1030 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Thu, 24 Feb 2022 11:24:48 +0100 Subject: [PATCH 11/28] submission grabbing --- app/models/submission_cleaner.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/submission_cleaner.rb b/app/models/submission_cleaner.rb index c7674ccc2..2dfa42040 100644 --- a/app/models/submission_cleaner.rb +++ b/app/models/submission_cleaner.rb @@ -41,6 +41,7 @@ def check_for_deletion @deletion_date = Time.zone.today fetch_props + @submissions = Submission.where(assignment: @assignments) @submissions.each(&:destroy) send_destruction_mail_to_submitters From 36bc1afbe9322d7bfa05f48cb8b03b2d09f69991 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Wed, 2 Mar 2022 01:49:41 +0100 Subject: [PATCH 12/28] added assignment deletion date to medium publisher changed helpdesk texts added cypress test removed submission_clenear_spec tests for now bug fixing --- app/controllers/media_controller.rb | 3 +- app/models/assignment.rb | 6 ++- app/models/medium_publisher.rb | 30 ++++++++++++-- app/models/term.rb | 4 ++ app/views/assignments/_form.html.erb | 2 +- app/views/assignments/create.coffee | 2 + app/views/lectures/edit/_assignments.html.erb | 1 - app/views/media/_publish_modal.html.erb | 17 ++++++-- app/views/media/publish.coffee | 8 ++++ config/locales/de.yml | 11 ++---- config/locales/en.yml | 10 ++--- db/schema.rb | 18 ++------- spec/cypress/integration/media_spec.js | 39 ++++++++++++++++++- spec/models/submission_cleaner_spec.rb | 2 + 14 files changed, 112 insertions(+), 41 deletions(-) diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb index 5c225dcd2..8e3e3f8a5 100644 --- a/app/controllers/media_controller.rb +++ b/app/controllers/media_controller.rb @@ -431,7 +431,8 @@ def publish_params params.require(:medium).permit(:release_now, :released, :release_date, :lock_comments, :publish_vertices, :create_assignment, :assignment_title, - :assignment_deadline, :assignment_file_type) + :assignment_deadline, :assignment_file_type, + :assignment_deletion_date) end def set_medium diff --git a/app/models/assignment.rb b/app/models/assignment.rb index 7ce950d7d..73a7f3a57 100644 --- a/app/models/assignment.rb +++ b/app/models/assignment.rb @@ -11,9 +11,11 @@ class Assignment < ApplicationRecord validate :deletion_date_cannot_be_in_the_past def deletion_date_cannot_be_in_the_past - return unless deletion_date.present? && deletion_date < DateTime.now + return unless deletion_date.present? && deletion_date < Time.zone.now.to_date - errors.add(:deletion_date, I18n.t('activerecord.errors.models.assignment.attributes.deletion_date.in_past')) + errors.add(:deletion_date, I18n.t('activerecord.errors.models.' \ + 'assignment.attributes.deletion_date.' \ + 'in_past')) end scope :active, -> { where('deadline >= ?', Time.now) } diff --git a/app/models/medium_publisher.rb b/app/models/medium_publisher.rb index cf0324440..2d8f55a14 100644 --- a/app/models/medium_publisher.rb +++ b/app/models/medium_publisher.rb @@ -4,13 +4,15 @@ class MediumPublisher attr_reader :medium_id, :user_id, :release_now, :release_for, :release_date, :lock_comments, :vertices, :create_assignment, :assignment_title, - :assignment_file_type, :assignment_deadline + :assignment_file_type, :assignment_deadline, + :assignment_deletion_date def initialize(medium_id:, user_id:, release_now:, release_for: 'all', release_date: nil, lock_comments: false, vertices: false, create_assignment: false, assignment_title: '', - assignment_file_type: '', assignment_deadline: nil) + assignment_file_type: '', assignment_deadline: nil, + assignment_deletion_date: nil) @medium_id = medium_id @user_id = user_id @release_now = release_now @@ -22,6 +24,7 @@ def initialize(medium_id:, user_id:, release_now:, @assignment_title = assignment_title @assignment_file_type = assignment_file_type @assignment_deadline = assignment_deadline + @assignment_deletion_date = assignment_deletion_date end def self.load(text) @@ -43,6 +46,11 @@ def self.parse(medium, user, params) rescue ArgumentError puts 'Argument error for medium assignment deadline' end + begin + assignment_deletion_date = Time.zone.parse(params[:assignment_deletion_date] || '') + rescue ArgumentError + puts 'Argument error for medium assignment deletion date' + end MediumPublisher.new(medium_id: medium.id, user_id: user.id, release_now: params[:release_now] == '1', release_for: params[:released], @@ -52,7 +60,8 @@ def self.parse(medium, user, params) create_assignment: params[:create_assignment] == '1', assignment_title: params[:assignment_title], assignment_file_type: params[:assignment_file_type], - assignment_deadline: assignment_deadline) + assignment_deadline: assignment_deadline, + assignment_deletion_date: assignment_deletion_date) end def publish! @@ -78,7 +87,8 @@ def assignment medium_id: @medium_id, title: @assignment_title, deadline: @assignment_deadline, - accepted_file_type: @assignment_file_type) + accepted_file_type: @assignment_file_type, + deletion_date: @assignment_deletion_date) end def errors @@ -89,6 +99,7 @@ def errors return {} unless @create_assignment add_assignment_deadline_error if invalid_assignment_deadline? + add_assignment_deletion_date_error if invalid_assignment_deletion_date? add_assignment_title_error if invalid_assignment_title? @errors end @@ -160,6 +171,10 @@ def invalid_assignment_deadline? @assignment_deadline.nil? || (@assignment_deadline < earliest_deadline) end + def invalid_assignment_deletion_date? + @assignment_deletion_date.nil? || (@assignment_deletion_date < Time.zone.today) + end + def invalid_assignment_title? @assignment_title.blank? || @assignment_title.in?(Assignment.where(lecture: medium.teachable) @@ -175,6 +190,13 @@ def add_assignment_deadline_error '.invalid_assignment_deadline') end + def add_assignment_deletion_date_error + @errors[:assignment_deletion_date] = I18n.t('activerecord.errors.' \ + 'models.assignment.' \ + 'attributes.deletion_date.' \ + 'in_past') + end + def add_assignment_title_error @errors[:assignment_title] = I18n.t('admin.medium' \ '.invalid_assignment_title') diff --git a/app/models/term.rb b/app/models/term.rb index 57e0bad5d..b08a93f72 100644 --- a/app/models/term.rb +++ b/app/models/term.rb @@ -100,6 +100,10 @@ def self.possible_deletion_dates Term.active.end_date + 2.weeks + 6.months] end + def self.possible_deletion_dates_localized + possible_deletion_dates.map { |d| d.strftime(I18n.t('date.formats.concise')) } + end + # array of all terms together with their ids for use in options_for_select def self.select_terms(independent = false) return ['bla', nil] if independent diff --git a/app/views/assignments/_form.html.erb b/app/views/assignments/_form.html.erb index 06122fd85..24ee1ed3b 100644 --- a/app/views/assignments/_form.html.erb +++ b/app/views/assignments/_form.html.erb @@ -39,7 +39,7 @@
<%= f.select :deletion_date, - Term.possible_deletion_dates, + Term.possible_deletion_dates_localized, {}, class: 'form-control', id: "assignment_deletion_date_#{assignment.id}" %> diff --git a/app/views/assignments/create.coffee b/app/views/assignments/create.coffee index 6e51ec2aa..3aab03b90 100644 --- a/app/views/assignments/create.coffee +++ b/app/views/assignments/create.coffee @@ -3,6 +3,8 @@ $('#assignment_title').removeClass('is-invalid') $('#assignment-title-error').empty() $('#assignment_deadline').removeClass('is-invalid') $('#assignment-deadline-error').empty() +$('#assignment_deletion_date').removeClass('is-invalid') +$('#assignment-deletion-date-error').empty() # display error message <% if @errors.present? %> diff --git a/app/views/lectures/edit/_assignments.html.erb b/app/views/lectures/edit/_assignments.html.erb index 55ea0775d..06206eb27 100644 --- a/app/views/lectures/edit/_assignments.html.erb +++ b/app/views/lectures/edit/_assignments.html.erb @@ -56,7 +56,6 @@
<%= t('assignment.deletion_date') %> - <%= helpdesk(t('assignment.protected_info'), true) %>
diff --git a/app/views/media/_publish_modal.html.erb b/app/views/media/_publish_modal.html.erb index 7e8b246d9..a68242a46 100644 --- a/app/views/media/_publish_modal.html.erb +++ b/app/views/media/_publish_modal.html.erb @@ -81,7 +81,7 @@
-
+
<%= f.label :assignment_title, t('basics.title') %> <%= f.text_field :assignment_title, class: 'form-control', @@ -90,7 +90,7 @@ id="assignment-title-error">
-
+
<%= f.label :assignment_deadline, t('basics.deadline') %> <%= f.text_field :assignment_deadline, class: 'form-control', @@ -103,7 +103,7 @@ id="assignment-deadline-error">
-
+
<%= f.label :assignment_file_type, t('submission.file_format') %> <%= f.select :assignment_file_type, @@ -112,6 +112,17 @@ {}, { class: 'form-control' } %>
+
+ <%= f.label :assignment_deletion_date, + t('assignment.deletion_date') %> + <%= f.select :assignment_deletion_date, + Term.possible_deletion_dates_localized, + {}, + class: 'form-control' %> +
+
+
<% end %>
diff --git a/app/views/media/publish.coffee b/app/views/media/publish.coffee index 51f3a100c..15c82305c 100644 --- a/app/views/media/publish.coffee +++ b/app/views/media/publish.coffee @@ -3,6 +3,8 @@ $('#release-date-error').empty() $('#release_date').removeClass('is-invalid') $('#assignment-deadline-error').empty() $('#medium_assignment_deadline').removeClass('is-invalid') +$('#assignment-deletion-date-error').empty() +$('#medium_assignment_deletion_date').removeClass('is-invalid') $('#assignment-title-error').empty() $('#medium_assignment_title').removeClass('is-invalid') @@ -19,6 +21,12 @@ $('#assignment-deadline-error').append('<%= @errors[:assignment_deadline] %>') $('#medium_assignment_deadline').addClass('is-invalid') <% end %> +<% if @errors[:assignment_deletion_date].present? %> +$('#assignment-deletion-date-error').append('<%= @errors[:assignment_deletion_date] %>') + .show() +$('#medium_assignment_deletion_date').addClass('is-invalid') +<% end %> + <% if @errors[:assignment_title].present? %> $('#assignment-title-error').append('<%= @errors[:assignment_title] %>') .show() diff --git a/config/locales/de.yml b/config/locales/de.yml index 996063b47..c1fde74d0 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -2761,10 +2761,10 @@ de: Beitritt oder Austritt von Teammitgliedern, Entscheidung über verspätete Abgaben und den Upload von Korrekturen durch Deine TutorIn. deletion_date_info: > - Bei der Abgabe handelt es sich um persönliche Daten, die wir aus Gründen - des Datenschutzes zwei Wochen nach Ablauf des Semesters löschen. - Du wirst darüber vorher per Mail informiert werden, so dass Du die Daten - sichern kannst. + Bei Abgaben handelt es sich um persönliche Daten, die wir aus + Datenschutzgründen im nachfolgenden Semester löschen. Das genaue Löschdatum + kannst Du der Tabelle entnehmen. Du wirst außerdem rechtzeitig per E-Mail + an die Löschung erinnert, damit Du Deine Daten sichern kannst. team_info: > Hier sieht Du Dein Abgabeteam. Du kannst Dein Team vergrößern, indem Du anderen Personen den Abgabecode weitergibst, so dass diese beitreten @@ -2784,10 +2784,7 @@ de: Es wurde noch nichts abgegeben. nothing_yet_in_lecture: > In dieser Vorlesung war bisher noch nichts abzugeben. - protected: Geschützte Abgabe deletion_date: Löschdatum - protected_info: > - "Geschützte" Abgaben werden bis zur manuellen Löschung aufbewahrt und werden nicht automatisiert nach Ablauf der Aufbewahrungsfrist (Semesterende + Karenzzeit von 14 Tagen) gelöscht. destruction_info: Hinweis. diff --git a/config/locales/en.yml b/config/locales/en.yml index f82ba61a6..1b9e96f36 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -2597,9 +2597,10 @@ en: team members, decisions about the acceptance of late submissions and upload of corrections by your tutor. deletion_date_info: > - Your submission contains personal data which we will delete two weeks - after the end of this term due to protection of data privacy. You will be - notified ahead via email so that you can save them. + The data you submit is personal data that we delete in the following semester + for data privacy reasons. You can find the exact deletion date in the table. + You will also be reminded of the deletion in advance by e-mail so that you + can back up your data. team_info: > Here you can see your submission team. You can enlarge your team by giving other persons the submission code so that they can @@ -2618,10 +2619,7 @@ en: Nothing has yet been submitted. nothing_yet_in_lecture: > In this lecture nothing had yet to be submitted. - protected: Protected Assignment deletion_date: Deletion date - protected_info: > - "Protected" Assignments won't be deleted at the end of the semester. destruction_info: > Note. diff --git a/db/schema.rb b/db/schema.rb index 5228c2c6d..48bad0ca0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -45,7 +45,7 @@ t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false t.text "accepted_file_type", default: ".pdf" - t.date "deletion_date", default: "2020-10-15", null: false + t.date "deletion_date", default: "2022-09-05", null: false t.index ["lecture_id"], name: "index_assignments_on_lecture_id" t.index ["medium_id"], name: "index_assignments_on_medium_id" end @@ -403,7 +403,7 @@ t.index ["subject_id"], name: "index_programs_on_subject_id" end - create_table "quiz_certificates", id: :uuid, default: -> { "public.gen_random_uuid()" }, force: :cascade do |t| + create_table "quiz_certificates", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.bigint "quiz_id", null: false t.bigint "user_id" t.text "code" @@ -489,7 +489,7 @@ t.datetime "updated_at", precision: 6, null: false end - create_table "submissions", id: :uuid, default: -> { "public.gen_random_uuid()" }, force: :cascade do |t| + create_table "submissions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.bigint "tutorial_id", null: false t.bigint "assignment_id", null: false t.text "token" @@ -875,16 +875,6 @@ t.datetime "updated_at", precision: 6, null: false end - create_table "watchables", force: :cascade do |t| - t.bigint "user_id", null: false - t.bigint "medium_id", null: false - t.integer "medium_position" - t.datetime "created_at", precision: 6, null: false - t.datetime "updated_at", precision: 6, null: false - t.index ["medium_id"], name: "index_watchables_on_medium_id" - t.index ["user_id"], name: "index_watchables_on_user_id" - end - create_table "watchlist_entries", force: :cascade do |t| t.bigint "watchlist_id", null: false t.bigint "medium_id", null: false @@ -946,8 +936,6 @@ add_foreign_key "user_favorite_lecture_joins", "lectures" add_foreign_key "user_favorite_lecture_joins", "users" add_foreign_key "user_submission_joins", "users" - add_foreign_key "watchables", "media" - add_foreign_key "watchables", "users" add_foreign_key "watchlist_entries", "media" add_foreign_key "watchlist_entries", "watchlists" add_foreign_key "watchlists", "users" diff --git a/spec/cypress/integration/media_spec.js b/spec/cypress/integration/media_spec.js index b00f57713..9f9066f80 100644 --- a/spec/cypress/integration/media_spec.js +++ b/spec/cypress/integration/media_spec.js @@ -134,19 +134,56 @@ describe("Media", () => { cy.contains("Media 1").should("exist"); cy.contains("Veröffentlichen").click(); cy.wait(100); - cy.contains("zum folgenden Zeitpunkt").click(); cy.contains("Hausaufgabe zu diesem Medium anlegen").click(); var date = new Date(); date.setDate(date.getDate() + 7); + cy.wait(100); console.log(date); cy.get('input[name="medium[release_date]"]').click().clear().type(date.toLocaleString("de")); + cy.wait(100); + cy.get('input[name="medium[release_date]"]').click(); date.setDate(date.getDate() + 8); cy.get('input[name="medium[assignment_deadline]"]').click().clear().type(date.toLocaleString("de")); + cy.get('select[name="medium[assignment_deletion_date]"]').should("exist"); cy.contains("Ich bestätige hiermit, dass durch die Veröffentlichung des Mediums auf der MaMpf-Plattform keine Rechte Dritter verletzt werden.").click(); cy.get("#publishMediumModal").contains("Speichern").click(); cy.contains("Dieses Medium wird planmäßig").should("exist"); }); }); + it("can create medium & release it with submission", () => { + cy.appFactories([ + ["create", + "lecture", { + "teacher_id": 1, + "released": "all" + } + ] + ]).then((lectures) => { + cy.visit(`lectures/${lectures[0].id}/edit`); + cy.contains("Medium anlegen").should("exist"); + cy.contains("Medium anlegen").click(); + cy.get('input[name="medium[description]"]').type("Media 1"); + cy.wait(10); + cy.get('select[name="medium[sort]"]').select("Übung"); + cy.contains("Speichern und bearbeiten").click(); + cy.contains("Media 1").should("exist"); + cy.contains("Veröffentlichen").click(); + cy.wait(100); + cy.contains("Hausaufgabe zu diesem Medium anlegen").click(); + var date = new Date(); + date.setDate(date.getDate() + 8); + cy.wait(100); + cy.get('input[name="medium[assignment_deadline]"]').click().clear().type(date.toLocaleString("de")); + cy.get('select[name="medium[assignment_deletion_date]"]').should("exist"); + cy.contains("Ich bestätige hiermit, dass durch die Veröffentlichung des Mediums auf der MaMpf-Plattform keine Rechte Dritter verletzt werden.").click(); + cy.get("#publishMediumModal").contains("Speichern").click(); + cy.contains("Dieses Medium wird planmäßig").should("not.exist"); + cy.wait(500); + cy.contains("zur Veranstaltung").click(); + cy.get('#assignments_heading').click(); + cy.contains("Media 1").should("exist"); + }); + }); }); }); \ No newline at end of file diff --git a/spec/models/submission_cleaner_spec.rb b/spec/models/submission_cleaner_spec.rb index 0bbd8fb93..8125b3b25 100644 --- a/spec/models/submission_cleaner_spec.rb +++ b/spec/models/submission_cleaner_spec.rb @@ -13,6 +13,7 @@ @term.destroy end +=begin it 'correctly determines wether an action is possible (example 1)' do @term = FactoryBot.create(:term, year: 2020, season: 'WS') cleaner = FactoryBot.build(:submission_cleaner, @@ -263,5 +264,6 @@ .to eq([[], [@submission21]]) end end +=end end end \ No newline at end of file From ce8bb690c7154c1d168ca823b8435c21b7d8c55c Mon Sep 17 00:00:00 2001 From: Tobias Wackenhut Date: Thu, 3 Mar 2022 20:56:39 +0100 Subject: [PATCH 13/28] Simplify docker-gen and nginx setup To simplify the setup we separate docker-gen from the actual nginx instance. From the original nginx.tmpl we only take the necessary parts. This should make it easier to implement changes. --- docker/Dockerfile.dockergen | 5 + docker/development/docker-compose.yml | 33 +++- docker/nginx.tmpl | 154 ++++++++++++++++++ docker/production/Dockerfile.proxy | 14 -- .../production/docker-compose.production.yml | 36 ++-- docker/production/proxy_location_config.txt | 3 - docker/production/proxy_server_config.txt | 19 --- 7 files changed, 212 insertions(+), 52 deletions(-) create mode 100644 docker/Dockerfile.dockergen create mode 100644 docker/nginx.tmpl delete mode 100644 docker/production/Dockerfile.proxy delete mode 100644 docker/production/proxy_location_config.txt delete mode 100644 docker/production/proxy_server_config.txt diff --git a/docker/Dockerfile.dockergen b/docker/Dockerfile.dockergen new file mode 100644 index 000000000..9f0adcb39 --- /dev/null +++ b/docker/Dockerfile.dockergen @@ -0,0 +1,5 @@ +FROM nginxproxy/docker-gen + +# Change the uid of nginx to the same as mampf-container +#RUN sed -i 's/^nginx.\+/nginx\:x\:501\:501\:nginx\:\/var\/cache\/nginx\:\/sbin\/nologin/' /etc/passwd +COPY docker/nginx.tmpl /etc/docker-gen/templates/nginx.tmpl \ No newline at end of file diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 944053cc3..db2a7444b 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -54,8 +54,6 @@ services: context: ./../.. dockerfile: docker/development/Dockerfile image: mampf:development - ports: - - "127.0.0.1:3000:3000" environment: RAILS_ENV: docker_development DEVELOPMENT_DATABASE_ADAPTER: postgresql @@ -75,6 +73,7 @@ services: URL_HOST: localhost URL_HOST_SHORT: localhost SECRET_KEY_BASE: testenvironment + VIRTUAL_HOST: workers ERDBEERE_SERVER: https://erdbeere.mathi.uni-heidelberg.de ERDBEERE_API: https://erdbeere.mathi.uni-heidelberg.de/api/v1 MUESLI_SERVER: https://muesli.mathi.uni-heidelberg.de @@ -115,9 +114,9 @@ services: context: ./../.. dockerfile: docker/development/Dockerfile environment: - - NODE_ENV=development - - RAILS_ENV=development - - WEBPACKER_DEV_SERVER_HOST=0.0.0.0 + NODE_ENV: development + RAILS_ENV: development + WEBPACKER_DEV_SERVER_HOST: 0.0.0.0 entrypoint: ["sh", "-c", "yarn && ./bin/webpack-dev-server"] volumes: - type: bind @@ -134,7 +133,31 @@ services: - redis - mailcatcher + nginx: + image: nginx + # Until https://github.com/nginx-proxy/docker-gen/pull/311 is merged, use hardcoded name + container_name: mampf-docker-nginx + ports: + - "127.0.0.1:3000:80" + volumes: + - "nginx-conf-d:/etc/nginx/conf.d" + - "submissions:/private/submissions:ro,nocopy" + - "public:/public" + + dockergen: + build: + context: ./../.. + dockerfile: docker/Dockerfile.dockergen + command: -notify-sighup mampf-docker-nginx -watch /etc/docker-gen/templates/nginx.tmpl /etc/nginx/conf.d/default.conf + restart: always + volumes: + - "nginx-conf-d:/etc/nginx/conf.d" + - "/var/run/docker.sock:/tmp/docker.sock" + volumes: + nginx-conf-d: + submissions: + public: db-data: solr-data: sprockets-cache: diff --git a/docker/nginx.tmpl b/docker/nginx.tmpl new file mode 100644 index 000000000..5ac25275a --- /dev/null +++ b/docker/nginx.tmpl @@ -0,0 +1,154 @@ +{{ $CurrentContainer := where $ "ID" .Docker.CurrentContainerID | first }} +{{ $debug_all := $.Env.DEBUG }} + +{{ define "upstream" }} + {{ $fail_timeout := "5s" }} + {{ $max_fails := "3" }} + {{ $networks := .Networks }} + {{ $debug_all := .Debug }} + upstream {{ .Upstream }} { + {{ $server_found := "false" }} + {{ range $container := .Containers }} + {{ $debug := (eq (coalesce $container.Env.DEBUG $debug_all "false") "true") }} + {{/* If only 1 port exposed, use that as a default, else 80 */}} + {{ $defaultPort := (when (eq (len $container.Addresses) 1) (first $container.Addresses) (dict "Port" "80")).Port }} + {{ $port := (coalesce $container.Env.VIRTUAL_PORT $defaultPort) }} + {{ $address := where $container.Addresses "Port" $port | first }} + {{ if $debug }} + # Exposed ports: {{ $container.Addresses }} + # Default virtual port: {{ $defaultPort }} + # VIRTUAL_PORT: {{ $container.Env.VIRTUAL_PORT }} + {{ if not $address }} + # /!\ Virtual port not exposed + {{ end }} + {{ end }} + {{ range $knownNetwork := $networks }} + {{ range $containerNetwork := $container.Networks }} + {{ if (and (ne $containerNetwork.Name "ingress") (or (eq $knownNetwork.Name $containerNetwork.Name) (eq $knownNetwork.Name "host"))) }} + ## Can be connected with "{{ $containerNetwork.Name }}" network + {{ if $address }} + {{/* If we got the containers from swarm and this container's port is published to host, use host IP:PORT */}} + {{ if and $container.Node.ID $address.HostPort }} + {{ $server_found = "true" }} + # {{ $container.Node.Name }}/{{ $container.Name }} + server {{ $container.Node.Address.IP }}:{{ $address.HostPort }} max_fails={{ $max_fails }} fail_timeout={{ $fail_timeout }}; + {{/* If there is no swarm node or the port is not published on host, use container's IP:PORT */}} + {{ else if $containerNetwork }} + {{ $server_found = "true" }} + # {{ $container.Name }} + server {{ $containerNetwork.IP }}:{{ $address.Port }} max_fails={{ $max_fails }} fail_timeout={{ $fail_timeout }}; + {{ end }} + {{ else if $containerNetwork }} + # {{ $container.Name }} + {{ if $containerNetwork.IP }} + {{ $server_found = "true" }} + server {{ $containerNetwork.IP }}:{{ $port }} max_fails={{ $max_fails }} fail_timeout={{ $fail_timeout }}; + {{ else }} + # /!\ No IP for this network! + {{ end }} + {{ end }} + {{ else }} + # Cannot connect to network '{{ $containerNetwork.Name }}' of this container + {{ end }} + {{ end }} + {{ end }} + {{ end }} + {{/* nginx-proxy/nginx-proxy#1105 */}} + {{ if (eq $server_found "false") }} + # Fallback entry + server 127.0.0.1 down; + {{ end }} + } +{{ end }} + +# If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the +# scheme used to connect to this server +map $http_x_forwarded_proto $proxy_x_forwarded_proto { + default $http_x_forwarded_proto; + '' $scheme; +} + +# If we receive X-Forwarded-Port, pass it through; otherwise, pass along the +# server port the client connected to +map $http_x_forwarded_port $proxy_x_forwarded_port { + default $http_x_forwarded_port; + '' $server_port; +} + +# If we receive Upgrade, set Connection to "upgrade"; otherwise, delete any +# Connection header that may have been passed to this server +map $http_upgrade $proxy_connection { + default upgrade; + '' close; +} + +gzip_types text/plain text/css application/javascript application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + +log_format vhost '$host [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" ' + '"$upstream_addr"'; + +error_log /dev/stderr; + +# HTTP 1.1 support +proxy_http_version 1.1; +proxy_buffering off; +proxy_set_header Host $http_host; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $proxy_connection; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; +# proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; +proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; +proxy_set_header X-Original-URI $request_uri; + +# Mitigate httpoxy attack (see README for details) +proxy_set_header Proxy ""; + +access_log /var/log/nginx/access.log vhost; + +server { + server_name _; # This is just an invalid value which will never trigger on a real hostname. + server_tokens off; + listen 80; + return 503; +} + +{{ range $host, $containers := groupByMulti $ "Env.VIRTUAL_HOST" "," }} +{{ template "upstream" (dict "Upstream" $host "Containers" $containers "Networks" $CurrentContainer.Networks "Debug" $debug_all) }} +{{ end }} + +server { + server_name localhost; + server_tokens off; + listen 80 default_server; + + root /public; + access_log off; + + # location /metrics { + # proxy_pass http://master:3000/metrics; + # } + + location / { + client_max_body_size 4G; + # try_files $uri $uri/ @rails; + try_files $uri @rails; + } + + location /__accel_redirect { + internal; + alias /private/; + add_header Content-Encoding $upstream_http_content_encoding; + gzip off; + } + + location @rails { + proxy_pass http://workers; + proxy_set_header X-Accel-Mapping /private=/__accel_redirect; + proxy_read_timeout 240s; + proxy_send_timeout 240s; + } +} diff --git a/docker/production/Dockerfile.proxy b/docker/production/Dockerfile.proxy deleted file mode 100644 index 1b786094b..000000000 --- a/docker/production/Dockerfile.proxy +++ /dev/null @@ -1,14 +0,0 @@ -FROM jwilder/nginx-proxy:alpine - -# Change the uid of nginx to the same as mampf-container -RUN sed -i 's/^nginx.\+/nginx\:x\:501\:501\:nginx\:\/var\/cache\/nginx\:\/sbin\/nologin/' /etc/passwd -# We need to manually configure the root location, so the actual proxy pass needs to be inside a special location we can -# redirect to -RUN sed -i 's/^\(\s*\)location \/ {\(\s*\)$/\1location @rails {\2/' /app/nginx.tmpl - -RUN mkdir /private /public /etc/nginx/certs - - -COPY docker/production/proxy_server_config.txt /etc/nginx/vhost.d/localhost -COPY docker/production/proxy_location_config.txt /etc/nginx/vhost.d/localhost_location -COPY docker/production/mime.types /etc/nginx/mime.types \ No newline at end of file diff --git a/docker/production/docker-compose.production.yml b/docker/production/docker-compose.production.yml index c2f1ee6b7..60da0904c 100644 --- a/docker/production/docker-compose.production.yml +++ b/docker/production/docker-compose.production.yml @@ -28,28 +28,41 @@ services: <<: *mampf entrypoint: /usr/src/app/docker/production/entrypoint-worker.sh environment: - - VIRTUAL_HOST=localhost + - VIRTUAL_HOST=workers volumes: - "media:/private/media:nocopy" - "submissions:/private/submissions:nocopy" - "caches:/caches:nocopy" - proxy: + nginx: + image: nginx + ports: + - "127.0.0.1:3000:80" + # Until https://github.com/nginx-proxy/docker-gen/pull/311 is merged, use hardcoded name + container_name: mampf-docker-proxy + volumes: + - "nginx-conf-d:/etc/nginx/conf.d" + - "submissions:/private/submissions:ro,nocopy" + - "public:/public" + depends_on: + - master + networks: + - mampf + + dockergen: build: context: git@github.com:MaMpf-HD/mampf.git#production - dockerfile: docker/production/Dockerfile.proxy + dockerfile: docker/production/Dockerfile.dockergen + command: -notify-sighup mampf-docker-proxy -watch /etc/docker-gen/templates/nginx.tmpl /etc/nginx/conf.d/default.conf restart: always - ports: - - "127.0.0.1:3009:80" volumes: + - "nginx-conf-d:/etc/nginx/conf.d" - "/var/run/docker.sock:/tmp/docker.sock" - - "submissions:/private/submissions:ro,nocopy" - - "public:/public" depends_on: - master networks: - mampf - + redis: restart: always image: redis:6-alpine @@ -85,16 +98,17 @@ volumes: media: driver_opts: type: "nfs" - o: "addr=krikkit,noatime,nolock,soft,vers=4.2,rw" + o: "addr=yourfileserverdomain,noatime,nolock,soft,vers=4.2,rw" device: ":/mampf/media/mampf" submissions: driver_opts: type: "nfs" - o: "addr=krikkit,noatime,nolock,soft,vers=4.2,rw" + o: "addr=yourfileserverdomain,noatime,nolock,soft,vers=4.2,rw" device: ":/mampf/submissions/mampf" public: + nginx-conf-d: caches: driver_opts: type: "nfs" - o: "addr=krikkit,noatime,nolock,soft,vers=4.2,rw" + o: "addr=yourfileserverdomain,noatime,nolock,soft,vers=4.2,rw" device: ":/mampf/caches/mampf" diff --git a/docker/production/proxy_location_config.txt b/docker/production/proxy_location_config.txt deleted file mode 100644 index 89a5a6ed0..000000000 --- a/docker/production/proxy_location_config.txt +++ /dev/null @@ -1,3 +0,0 @@ -proxy_set_header X-Accel-Mapping /private=/__accel_redirect; -proxy_read_timeout 240s; -proxy_send_timeout 240s; diff --git a/docker/production/proxy_server_config.txt b/docker/production/proxy_server_config.txt deleted file mode 100644 index f44152164..000000000 --- a/docker/production/proxy_server_config.txt +++ /dev/null @@ -1,19 +0,0 @@ -root /public; -access_log off; - -location /metrics { - proxy_pass http://master:3000/metrics; -} - -location / { - client_max_body_size 4G; - # try_files $uri $uri/ @rails; - try_files $uri @rails; -} - -location /__accel_redirect { - internal; - alias /private/; - add_header Content-Encoding $upstream_http_content_encoding; - gzip off; -} \ No newline at end of file From 6df89b5a21649c270e7ea70d0668bab70d699443 Mon Sep 17 00:00:00 2001 From: Tobias Wackenhut Date: Thu, 3 Mar 2022 23:14:17 +0100 Subject: [PATCH 14/28] Use openresty and make metrics call concatinate all worker metrics --- docker/development/docker-compose.yml | 3 +- docker/nginx.tmpl | 52 +++++++++++++++++++++++---- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index db2a7444b..dcbf2ea5d 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -134,7 +134,7 @@ services: - mailcatcher nginx: - image: nginx + image: openresty/openresty:alpine # Until https://github.com/nginx-proxy/docker-gen/pull/311 is merged, use hardcoded name container_name: mampf-docker-nginx ports: @@ -152,6 +152,7 @@ services: restart: always volumes: - "nginx-conf-d:/etc/nginx/conf.d" + - "../nginx.tmpl:/etc/docker-gen/templates/nginx.tmpl" - "/var/run/docker.sock:/tmp/docker.sock" volumes: diff --git a/docker/nginx.tmpl b/docker/nginx.tmpl index 5ac25275a..4fbaf18e5 100644 --- a/docker/nginx.tmpl +++ b/docker/nginx.tmpl @@ -1,6 +1,49 @@ {{ $CurrentContainer := where $ "ID" .Docker.CurrentContainerID | first }} {{ $debug_all := $.Env.DEBUG }} +{{ define "echo_subrequests" }} + {{ $networks := .Networks }} + location {{ .Endpoint }} { + types { } default_type "text/html; charset=utf-8"; + {{ $server_found := "false" }} + {{ range $container := .Containers }} + {{/* If only 1 port exposed, use that as a default, else 80 */}} + {{ $defaultPort := (when (eq (len $container.Addresses) 1) (first $container.Addresses) (dict "Port" "80")).Port }} + {{ $port := (coalesce $container.Env.VIRTUAL_PORT $defaultPort) }} + {{ $address := where $container.Addresses "Port" $port | first }} + {{ range $knownNetwork := $networks }} + {{ range $containerNetwork := $container.Networks }} + {{ if (and (ne $containerNetwork.Name "ingress") (or (eq $knownNetwork.Name $containerNetwork.Name) (eq $knownNetwork.Name "host"))) }} + {{ if $address }} + {{/* If we got the containers from swarm and this container's port is published to host, use host IP:PORT */}} + {{ if and $container.Node.ID $address.HostPort }} + {{ $server_found = "true" }} + # {{ $container.Node.Name }}/{{ $container.Name }} + echo_subrequest_async GET 'http://{{ $container.Node.Address.IP }}:{{ $address.HostPort }}/metrics'; + {{/* If there is no swarm node or the port is not published on host, use container's IP:PORT */}} + {{ else if $containerNetwork }} + {{ $server_found = "true" }} + # {{ $container.Name }} + echo_subrequest_async GET 'http://{{ $containerNetwork.IP }}:{{ $address.Port }}/metrics'; + {{ end }} + {{ else if $containerNetwork }} + # {{ $container.Name }} + {{ if $containerNetwork.IP }} + {{ $server_found = "true" }} + echo_subrequest_async GET 'http://{{ $containerNetwork.IP }}:{{ $port }}/metrics'; + {{ end }} + {{ end }} + {{ end }} + {{ end }} + {{ end }} + {{ end }} + {{ if (eq $server_found "false") }} + # Fallback entry + return 200 ''; + {{ end }} + } +{{ end }} + {{ define "upstream" }} {{ $fail_timeout := "5s" }} {{ $max_fails := "3" }} @@ -107,8 +150,6 @@ proxy_set_header X-Original-URI $request_uri; # Mitigate httpoxy attack (see README for details) proxy_set_header Proxy ""; -access_log /var/log/nginx/access.log vhost; - server { server_name _; # This is just an invalid value which will never trigger on a real hostname. server_tokens off; @@ -126,11 +167,10 @@ server { listen 80 default_server; root /public; - access_log off; - # location /metrics { - # proxy_pass http://master:3000/metrics; - # } + {{ range $host, $containers := groupByMulti $ "Env.VIRTUAL_HOST" "," }} + {{ template "echo_subrequests" (dict "Endpoint" "/metrics" "Containers" $containers "Networks" $CurrentContainer.Networks) }} + {{ end }} location / { client_max_body_size 4G; From c14c2dedf6755c4ee4370f886ed9c0d84441b85c Mon Sep 17 00:00:00 2001 From: Tobias Wackenhut Date: Fri, 4 Mar 2022 13:54:30 +0100 Subject: [PATCH 15/28] Fix metrics locations and simplify nginx.tmpl further --- docker/development/Dockerfile | 1 + docker/development/docker-compose.yml | 3 +- docker/nginx.tmpl | 126 ++++++------------ docker/production/Dockerfile | 1 + .../production/docker-compose.production.yml | 4 +- docker/production/entrypoint-master.sh | 2 +- docker/production/entrypoint-worker.sh | 1 + entrypoint.sh | 2 +- 8 files changed, 53 insertions(+), 87 deletions(-) diff --git a/docker/development/Dockerfile b/docker/development/Dockerfile index 23b972d8f..1de43b786 100644 --- a/docker/development/Dockerfile +++ b/docker/development/Dockerfile @@ -10,6 +10,7 @@ RUN GOOS=js GOARCH=wasm go build -o pdfcomprezzor.wasm FROM ruby:3.0.3 ENV RAILS_ENV=production EXPOSE 3000 +EXPOSE 9394 # https://github.com/nodesource/distributions#installation-instructions RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \ diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index dcbf2ea5d..c0c51981a 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -73,7 +73,6 @@ services: URL_HOST: localhost URL_HOST_SHORT: localhost SECRET_KEY_BASE: testenvironment - VIRTUAL_HOST: workers ERDBEERE_SERVER: https://erdbeere.mathi.uni-heidelberg.de ERDBEERE_API: https://erdbeere.mathi.uni-heidelberg.de/api/v1 MUESLI_SERVER: https://muesli.mathi.uni-heidelberg.de @@ -109,6 +108,8 @@ services: - solr - redis - mailcatcher + labels: + de.uni-heidelberg.mathi.mampf.container-type: worker webpacker: build: context: ./../.. diff --git a/docker/nginx.tmpl b/docker/nginx.tmpl index 4fbaf18e5..3bfecfc4b 100644 --- a/docker/nginx.tmpl +++ b/docker/nginx.tmpl @@ -1,38 +1,39 @@ {{ $CurrentContainer := where $ "ID" .Docker.CurrentContainerID | first }} {{ $debug_all := $.Env.DEBUG }} -{{ define "echo_subrequests" }} +{{ define "container_specific_locations" }} {{ $networks := .Networks }} + {{ $port := .Port }} + {{ $server_found := "false" }} + {{ range $container := .Containers }} + {{ range $knownNetwork := $networks }} + {{ range $containerNetwork := $container.Networks }} + {{ if (and (ne $containerNetwork.Name "ingress") (or (eq $knownNetwork.Name $containerNetwork.Name) (eq $knownNetwork.Name "host"))) }} + # {{ $container.Name }} + location /{{ $container.Name }} { + internal; + proxy_pass 'http://{{ $containerNetwork.IP }}:{{ $port }}/'; + proxy_set_header X-Accel-Mapping /private=/__accel_redirect; + } + {{ end }} + {{ end }} + {{ end }} + {{ end }} +{{ end }} + +{{ define "metric_location" }} location {{ .Endpoint }} { - types { } default_type "text/html; charset=utf-8"; + types { } default_type "text/plain; charset=utf-8"; + {{ $networks := .Networks }} + {{ $port := .Port }} {{ $server_found := "false" }} {{ range $container := .Containers }} - {{/* If only 1 port exposed, use that as a default, else 80 */}} - {{ $defaultPort := (when (eq (len $container.Addresses) 1) (first $container.Addresses) (dict "Port" "80")).Port }} - {{ $port := (coalesce $container.Env.VIRTUAL_PORT $defaultPort) }} - {{ $address := where $container.Addresses "Port" $port | first }} {{ range $knownNetwork := $networks }} {{ range $containerNetwork := $container.Networks }} {{ if (and (ne $containerNetwork.Name "ingress") (or (eq $knownNetwork.Name $containerNetwork.Name) (eq $knownNetwork.Name "host"))) }} - {{ if $address }} - {{/* If we got the containers from swarm and this container's port is published to host, use host IP:PORT */}} - {{ if and $container.Node.ID $address.HostPort }} - {{ $server_found = "true" }} - # {{ $container.Node.Name }}/{{ $container.Name }} - echo_subrequest_async GET 'http://{{ $container.Node.Address.IP }}:{{ $address.HostPort }}/metrics'; - {{/* If there is no swarm node or the port is not published on host, use container's IP:PORT */}} - {{ else if $containerNetwork }} - {{ $server_found = "true" }} + {{ $server_found = "true" }} # {{ $container.Name }} - echo_subrequest_async GET 'http://{{ $containerNetwork.IP }}:{{ $address.Port }}/metrics'; - {{ end }} - {{ else if $containerNetwork }} - # {{ $container.Name }} - {{ if $containerNetwork.IP }} - {{ $server_found = "true" }} - echo_subrequest_async GET 'http://{{ $containerNetwork.IP }}:{{ $port }}/metrics'; - {{ end }} - {{ end }} + echo_location_async '/{{ $container.Name }}/metrics'; {{ end }} {{ end }} {{ end }} @@ -45,63 +46,26 @@ {{ end }} {{ define "upstream" }} - {{ $fail_timeout := "5s" }} - {{ $max_fails := "3" }} - {{ $networks := .Networks }} - {{ $debug_all := .Debug }} upstream {{ .Upstream }} { - {{ $server_found := "false" }} - {{ range $container := .Containers }} - {{ $debug := (eq (coalesce $container.Env.DEBUG $debug_all "false") "true") }} - {{/* If only 1 port exposed, use that as a default, else 80 */}} - {{ $defaultPort := (when (eq (len $container.Addresses) 1) (first $container.Addresses) (dict "Port" "80")).Port }} - {{ $port := (coalesce $container.Env.VIRTUAL_PORT $defaultPort) }} - {{ $address := where $container.Addresses "Port" $port | first }} - {{ if $debug }} - # Exposed ports: {{ $container.Addresses }} - # Default virtual port: {{ $defaultPort }} - # VIRTUAL_PORT: {{ $container.Env.VIRTUAL_PORT }} - {{ if not $address }} - # /!\ Virtual port not exposed + {{ $networks := .Networks }} + {{ $port := .Port }} + {{ $server_found := "false" }} + {{ range $container := .Containers }} + {{ range $knownNetwork := $networks }} + {{ range $containerNetwork := $container.Networks }} + {{ if (and (ne $containerNetwork.Name "ingress") (or (eq $knownNetwork.Name $containerNetwork.Name) (eq $knownNetwork.Name "host"))) }} + {{ $server_found = "true" }} + # {{ $container.Name }} + server {{ $containerNetwork.IP }}:{{ $port }} max_fails=3 fail_timeout=10; + {{ end }} {{ end }} {{ end }} - {{ range $knownNetwork := $networks }} - {{ range $containerNetwork := $container.Networks }} - {{ if (and (ne $containerNetwork.Name "ingress") (or (eq $knownNetwork.Name $containerNetwork.Name) (eq $knownNetwork.Name "host"))) }} - ## Can be connected with "{{ $containerNetwork.Name }}" network - {{ if $address }} - {{/* If we got the containers from swarm and this container's port is published to host, use host IP:PORT */}} - {{ if and $container.Node.ID $address.HostPort }} - {{ $server_found = "true" }} - # {{ $container.Node.Name }}/{{ $container.Name }} - server {{ $container.Node.Address.IP }}:{{ $address.HostPort }} max_fails={{ $max_fails }} fail_timeout={{ $fail_timeout }}; - {{/* If there is no swarm node or the port is not published on host, use container's IP:PORT */}} - {{ else if $containerNetwork }} - {{ $server_found = "true" }} - # {{ $container.Name }} - server {{ $containerNetwork.IP }}:{{ $address.Port }} max_fails={{ $max_fails }} fail_timeout={{ $fail_timeout }}; - {{ end }} - {{ else if $containerNetwork }} - # {{ $container.Name }} - {{ if $containerNetwork.IP }} - {{ $server_found = "true" }} - server {{ $containerNetwork.IP }}:{{ $port }} max_fails={{ $max_fails }} fail_timeout={{ $fail_timeout }}; - {{ else }} - # /!\ No IP for this network! - {{ end }} - {{ end }} - {{ else }} - # Cannot connect to network '{{ $containerNetwork.Name }}' of this container - {{ end }} - {{ end }} - {{ end }} - {{ end }} - {{/* nginx-proxy/nginx-proxy#1105 */}} - {{ if (eq $server_found "false") }} + {{ end }} + {{ if (eq $server_found "false") }} # Fallback entry server 127.0.0.1 down; - {{ end }} - } + {{ end }} + } {{ end }} # If we receive X-Forwarded-Proto, pass it through; otherwise, pass along the @@ -157,9 +121,8 @@ server { return 503; } -{{ range $host, $containers := groupByMulti $ "Env.VIRTUAL_HOST" "," }} -{{ template "upstream" (dict "Upstream" $host "Containers" $containers "Networks" $CurrentContainer.Networks "Debug" $debug_all) }} -{{ end }} +{{ $worker_containers := whereLabelValueMatches $ "de.uni-heidelberg.mathi.mampf.container-type" "worker" }} +{{ template "upstream" (dict "Upstream" "workers" "Containers" $worker_containers "Networks" $CurrentContainer.Networks "Port" "3000" "Debug" $debug_all) }} server { server_name localhost; @@ -168,9 +131,8 @@ server { root /public; - {{ range $host, $containers := groupByMulti $ "Env.VIRTUAL_HOST" "," }} - {{ template "echo_subrequests" (dict "Endpoint" "/metrics" "Containers" $containers "Networks" $CurrentContainer.Networks) }} - {{ end }} + {{ template "container_specific_locations" (dict "Containers" $worker_containers "Networks" $CurrentContainer.Networks "Port" "9394") }} + {{ template "metric_location" (dict "Endpoint" "/metrics" "Containers" $worker_containers "Networks" $CurrentContainer.Networks "Port" "9394") }} location / { client_max_body_size 4G; diff --git a/docker/production/Dockerfile b/docker/production/Dockerfile index 67d440c2e..12bb2f853 100644 --- a/docker/production/Dockerfile +++ b/docker/production/Dockerfile @@ -12,6 +12,7 @@ FROM ruby:3.0.3 ENV RAILS_ENV=production EXPOSE 3000 +EXPOSE 9394 ENTRYPOINT ["/usr/src/app/docker/entrypoint-worker.sh"] diff --git a/docker/production/docker-compose.production.yml b/docker/production/docker-compose.production.yml index 60da0904c..32758cbe9 100644 --- a/docker/production/docker-compose.production.yml +++ b/docker/production/docker-compose.production.yml @@ -27,12 +27,12 @@ services: worker: <<: *mampf entrypoint: /usr/src/app/docker/production/entrypoint-worker.sh - environment: - - VIRTUAL_HOST=workers volumes: - "media:/private/media:nocopy" - "submissions:/private/submissions:nocopy" - "caches:/caches:nocopy" + labels: + de.uni-heidelberg.mathi.mampf.container-type: worker nginx: image: nginx diff --git a/docker/production/entrypoint-master.sh b/docker/production/entrypoint-master.sh index 4f45672ae..5f88fa861 100755 --- a/docker/production/entrypoint-master.sh +++ b/docker/production/entrypoint-master.sh @@ -11,5 +11,5 @@ then touch completed_initial_run fi echo "running mampf master" -prometheus_exporter -b 0.0.0.0 -p 3000 -a lib/collectors/mampf_collector.rb &> /usr/src/app/log/prometheus_exporter.log &! +prometheus_exporter --label "{\"instance\": \"${HOSTNAME}\"}" -b 0.0.0.0 -p 9394 -a lib/collectors/mampf_collector.rb &> /usr/src/app/log/prometheus_exporter.log &! exec bundle exec sidekiq &> >(tee -a /usr/src/app/log/runtime.log) \ No newline at end of file diff --git a/docker/production/entrypoint-worker.sh b/docker/production/entrypoint-worker.sh index 9fc6a99a0..9e6c4a075 100755 --- a/docker/production/entrypoint-worker.sh +++ b/docker/production/entrypoint-worker.sh @@ -3,4 +3,5 @@ cd /usr/src/app echo "Ensuring no stale server pid file is present" rm -f tmp/pids/server.pid echo "running mampf" +prometheus_exporter --label "{\"instance\": \"${HOSTNAME}\"}" -b 0.0.0.0 -p 9394 -a lib/collectors/mampf_collector.rb &> /usr/src/app/log/prometheus_exporter.log &! exec bundle exec rails s -p 3000 -b '0.0.0.0' diff --git a/entrypoint.sh b/entrypoint.sh index 7e93874db..dca4a6b9b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,5 +8,5 @@ rm -f tmp/pids/server.pid cp /pdfcomprezzor.wasm public/pdfcomprezzor/pdfcomprezzor.wasm echo "running mampf" bundle exec sidekiq & -prometheus_exporter -b 0.0.0.0 -p 9394 -a lib/collectors/mampf_collector.rb &> /usr/src/app/log/prometheus_exporter.log &! +prometheus_exporter --label "{\"instance\": \"${HOSTNAME}\"}" -b 0.0.0.0 -p 9394 -a lib/collectors/mampf_collector.rb &! exec bundle exec rails s -p 3000 -b '0.0.0.0' &> >(tee -a /usr/src/app/log/runtime.log) From 668197c2ee6585c5776409ac572eb5d0e016474b Mon Sep 17 00:00:00 2001 From: Tobias Wackenhut Date: Sat, 5 Mar 2022 12:42:38 +0100 Subject: [PATCH 16/28] Fix gzip compression bug in nginx config --- docker/nginx.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/nginx.tmpl b/docker/nginx.tmpl index 3bfecfc4b..b450130bd 100644 --- a/docker/nginx.tmpl +++ b/docker/nginx.tmpl @@ -12,6 +12,7 @@ # {{ $container.Name }} location /{{ $container.Name }} { internal; + gunzip on; proxy_pass 'http://{{ $containerNetwork.IP }}:{{ $port }}/'; proxy_set_header X-Accel-Mapping /private=/__accel_redirect; } From 152f70bdba0e704e59f917d46d052e42d84fa905 Mon Sep 17 00:00:00 2001 From: Tobias Wackenhut Date: Sun, 6 Mar 2022 13:09:19 +0100 Subject: [PATCH 17/28] Fix origin error in nginx config --- docker/nginx.tmpl | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/docker/nginx.tmpl b/docker/nginx.tmpl index b450130bd..6380f34f7 100644 --- a/docker/nginx.tmpl +++ b/docker/nginx.tmpl @@ -99,22 +99,6 @@ log_format vhost '$host [$time_local] ' error_log /dev/stderr; -# HTTP 1.1 support -proxy_http_version 1.1; -proxy_buffering off; -proxy_set_header Host $http_host; -proxy_set_header Upgrade $http_upgrade; -proxy_set_header Connection $proxy_connection; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; -# proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; -proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; -proxy_set_header X-Original-URI $request_uri; - -# Mitigate httpoxy attack (see README for details) -proxy_set_header Proxy ""; - server { server_name _; # This is just an invalid value which will never trigger on a real hostname. server_tokens off; @@ -132,7 +116,8 @@ server { root /public; - {{ template "container_specific_locations" (dict "Containers" $worker_containers "Networks" $CurrentContainer.Networks "Port" "9394") }} + {{ $contains_with_monitoring := whereLabelValueMatches $ "de.uni-heidelberg.mathi.mampf.container-type" "worker|master" }} + {{ template "container_specific_locations" (dict "Containers" $contains_with_monitoring "Networks" $CurrentContainer.Networks "Port" "9394") }} {{ template "metric_location" (dict "Endpoint" "/metrics" "Containers" $worker_containers "Networks" $CurrentContainer.Networks "Port" "9394") }} location / { @@ -153,5 +138,21 @@ server { proxy_set_header X-Accel-Mapping /private=/__accel_redirect; proxy_read_timeout 240s; proxy_send_timeout 240s; + + # HTTP 1.1 support + proxy_http_version 1.1; + proxy_buffering off; + proxy_set_header Host $http_host; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $proxy_connection; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; + # proxy_set_header X-Forwarded-Ssl $proxy_x_forwarded_ssl; + proxy_set_header X-Forwarded-Port $proxy_x_forwarded_port; + proxy_set_header X-Original-URI $request_uri; + + # Mitigate httpoxy attack (see README for details) + proxy_set_header Proxy ""; } } From f93e1e7ede992e48f24b70467d213a9ed02370f7 Mon Sep 17 00:00:00 2001 From: Tobias Wackenhut Date: Sun, 6 Mar 2022 13:25:19 +0100 Subject: [PATCH 18/28] Fix restarting of nginx container --- docker/production/docker-compose.production.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/production/docker-compose.production.yml b/docker/production/docker-compose.production.yml index 32758cbe9..a50a5aedb 100644 --- a/docker/production/docker-compose.production.yml +++ b/docker/production/docker-compose.production.yml @@ -43,7 +43,8 @@ services: volumes: - "nginx-conf-d:/etc/nginx/conf.d" - "submissions:/private/submissions:ro,nocopy" - - "public:/public" + - "public:/public:ro" + restart: always depends_on: - master networks: From 38e4c7c929469483b4345b4bf7217137567e4cea Mon Sep 17 00:00:00 2001 From: Tobias Wackenhut Date: Sun, 6 Mar 2022 16:36:23 +0100 Subject: [PATCH 19/28] Add master container to monitoring --- docker/nginx.tmpl | 3 +-- docker/production/docker-compose.production.yml | 2 +- docker/production/entrypoint-master.sh | 2 +- docker/production/entrypoint-worker.sh | 2 +- entrypoint.sh | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docker/nginx.tmpl b/docker/nginx.tmpl index 6380f34f7..0a92f680c 100644 --- a/docker/nginx.tmpl +++ b/docker/nginx.tmpl @@ -14,7 +14,6 @@ internal; gunzip on; proxy_pass 'http://{{ $containerNetwork.IP }}:{{ $port }}/'; - proxy_set_header X-Accel-Mapping /private=/__accel_redirect; } {{ end }} {{ end }} @@ -118,7 +117,7 @@ server { {{ $contains_with_monitoring := whereLabelValueMatches $ "de.uni-heidelberg.mathi.mampf.container-type" "worker|master" }} {{ template "container_specific_locations" (dict "Containers" $contains_with_monitoring "Networks" $CurrentContainer.Networks "Port" "9394") }} - {{ template "metric_location" (dict "Endpoint" "/metrics" "Containers" $worker_containers "Networks" $CurrentContainer.Networks "Port" "9394") }} + {{ template "metric_location" (dict "Endpoint" "/metrics" "Containers" $contains_with_monitoring "Networks" $CurrentContainer.Networks "Port" "9394") }} location / { client_max_body_size 4G; diff --git a/docker/production/docker-compose.production.yml b/docker/production/docker-compose.production.yml index a50a5aedb..732b11af6 100644 --- a/docker/production/docker-compose.production.yml +++ b/docker/production/docker-compose.production.yml @@ -35,7 +35,7 @@ services: de.uni-heidelberg.mathi.mampf.container-type: worker nginx: - image: nginx + image: openresty/openresty:alpine ports: - "127.0.0.1:3000:80" # Until https://github.com/nginx-proxy/docker-gen/pull/311 is merged, use hardcoded name diff --git a/docker/production/entrypoint-master.sh b/docker/production/entrypoint-master.sh index 5f88fa861..e923e1691 100755 --- a/docker/production/entrypoint-master.sh +++ b/docker/production/entrypoint-master.sh @@ -11,5 +11,5 @@ then touch completed_initial_run fi echo "running mampf master" -prometheus_exporter --label "{\"instance\": \"${HOSTNAME}\"}" -b 0.0.0.0 -p 9394 -a lib/collectors/mampf_collector.rb &> /usr/src/app/log/prometheus_exporter.log &! +prometheus_exporter --label "{\"container\": \"${HOSTNAME}\"}" -b 0.0.0.0 -p 9394 -a lib/collectors/mampf_collector.rb &> /usr/src/app/log/prometheus_exporter.log &! exec bundle exec sidekiq &> >(tee -a /usr/src/app/log/runtime.log) \ No newline at end of file diff --git a/docker/production/entrypoint-worker.sh b/docker/production/entrypoint-worker.sh index 9e6c4a075..30b443e95 100755 --- a/docker/production/entrypoint-worker.sh +++ b/docker/production/entrypoint-worker.sh @@ -3,5 +3,5 @@ cd /usr/src/app echo "Ensuring no stale server pid file is present" rm -f tmp/pids/server.pid echo "running mampf" -prometheus_exporter --label "{\"instance\": \"${HOSTNAME}\"}" -b 0.0.0.0 -p 9394 -a lib/collectors/mampf_collector.rb &> /usr/src/app/log/prometheus_exporter.log &! +prometheus_exporter --label "{\"container\": \"${HOSTNAME}\"}" -b 0.0.0.0 -p 9394 -a lib/collectors/mampf_collector.rb &> /usr/src/app/log/prometheus_exporter.log &! exec bundle exec rails s -p 3000 -b '0.0.0.0' diff --git a/entrypoint.sh b/entrypoint.sh index dca4a6b9b..69fc5a788 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,5 +8,5 @@ rm -f tmp/pids/server.pid cp /pdfcomprezzor.wasm public/pdfcomprezzor/pdfcomprezzor.wasm echo "running mampf" bundle exec sidekiq & -prometheus_exporter --label "{\"instance\": \"${HOSTNAME}\"}" -b 0.0.0.0 -p 9394 -a lib/collectors/mampf_collector.rb &! +prometheus_exporter --label "{\"container\": \"${HOSTNAME}\"}" -b 0.0.0.0 -p 9394 -a lib/collectors/mampf_collector.rb &! exec bundle exec rails s -p 3000 -b '0.0.0.0' &> >(tee -a /usr/src/app/log/runtime.log) From b5c8a6cc5fd2a46dfee0413578613b0db6b7815a Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Wed, 9 Mar 2022 13:25:37 +0100 Subject: [PATCH 20/28] added deletion date to medium publisher submission_cleaner fixing and spec tests medium_publisher spec tests changed mail text slightly --- app/mailers/notification_mailer.rb | 2 + app/models/submission_cleaner.rb | 32 +- .../submission_deletion_email.html.erb | 2 +- ...submission_deletion_lecture_email.html.erb | 2 +- app/workers/submissions_cleaner.rb | 2 +- config/environments/test.rb | 3 +- config/locales/de.yml | 14 +- config/locales/en.yml | 4 +- spec/factories/assignments.rb | 1 + spec/factories/medium_publisher.rb | 4 +- spec/factories/submission_cleaner.rb | 2 +- spec/models/medium_publisher_spec.rb | 42 ++- spec/models/submission_cleaner_spec.rb | 287 ++++-------------- 13 files changed, 135 insertions(+), 262 deletions(-) diff --git a/app/mailers/notification_mailer.rb b/app/mailers/notification_mailer.rb index 69b29e317..1b591400a 100644 --- a/app/mailers/notification_mailer.rb +++ b/app/mailers/notification_mailer.rb @@ -127,6 +127,7 @@ def submission_rejection_email def submission_deletion_email @term = params[:term] + @deletion_date = params[:deletion_date] subject = params[:reminder] ? t('basics.reminder') + ': ' : '' subject += t('mailer.submission_deletion_subject', term: @term.to_label) @@ -138,6 +139,7 @@ def submission_deletion_email def submission_deletion_lecture_email @term = params[:term] @lecture = params[:lecture] + @deletion_date = params[:deletion_date] subject = params[:reminder] ? t('basics.reminder') + ': ' : '' subject += t('mailer.submission_deletion_lecture_subject', term: @term.to_label, diff --git a/app/models/submission_cleaner.rb b/app/models/submission_cleaner.rb index 2dfa42040..2dd444801 100644 --- a/app/models/submission_cleaner.rb +++ b/app/models/submission_cleaner.rb @@ -20,29 +20,32 @@ def clean! end def check_for_first_mail - @deletion_date = Time.zone.today + 14.days + @deletion_date = @date + 14.days fetch_props - @reminder = false + return if @assignments.empty? + @reminder = false send_info_mail_to_submitters send_info_mail_to_editors end def check_for_reminder_mail - @deletion_date = Time.zone.today + 7.days + @deletion_date = @date + 7.days fetch_props - @reminder = true + return if @assignments.empty? + @reminder = true send_info_mail_to_submitters send_info_mail_to_editors end def check_for_deletion - @deletion_date = Time.zone.today + @deletion_date = @date fetch_props + return if @assignments.empty? @submissions = Submission.where(assignment: @assignments) - @submissions.each(&:destroy) + @submissions.each(&:destroy!) send_destruction_mail_to_submitters send_destruction_mail_to_editors @@ -50,14 +53,23 @@ def check_for_deletion private + def clear_props + @assignments = nil + @submitters = nil + @lectures = nil + end + def fetch_props - @assignments = Assignments.where(deletion_date: @deletion_date) - @submitters = @assignments.submitters - @lectures = Lecture.find_by(id: @assignments.pluck(:lecture_id)) + clear_props + @assignments = Assignment.where(deletion_date: @deletion_date) + return if @assignments.empty? + + @submitters = User.where(id: @assignments.flat_map(&:submitter_ids)) + @lectures = [*Lecture.find_by(id: @assignments.pluck(:lecture_id))] end def send_destruction_mail_to_submitters - return unless @submitters.present? + return if @submitters.blank? I18n.available_locales.each do |l| local_submitter_ids = @submitters.where(locale: l).pluck(:id) diff --git a/app/views/notification_mailer/submission_deletion_email.html.erb b/app/views/notification_mailer/submission_deletion_email.html.erb index 0d9fc8666..f15b38749 100644 --- a/app/views/notification_mailer/submission_deletion_email.html.erb +++ b/app/views/notification_mailer/submission_deletion_email.html.erb @@ -1,3 +1,3 @@ <%= t('mailer.submission_deletion_text_html', term: @term.to_label, - deletion_date: I18n.l(@term.submission_deletion_date)) %> \ No newline at end of file + deletion_date: I18n.l(@deletion_date)) %> \ No newline at end of file diff --git a/app/views/notification_mailer/submission_deletion_lecture_email.html.erb b/app/views/notification_mailer/submission_deletion_lecture_email.html.erb index c4348b3df..1ad1d9680 100644 --- a/app/views/notification_mailer/submission_deletion_lecture_email.html.erb +++ b/app/views/notification_mailer/submission_deletion_lecture_email.html.erb @@ -1,4 +1,4 @@ <%= t('mailer.submission_deletion_lecture_text_html', term: @term.to_label, lecture: @lecture.title_no_term, - deletion_date: I18n.l(@term.submission_deletion_date)) %> \ No newline at end of file + deletion_date: I18n.l(@deletion_date)) %> \ No newline at end of file diff --git a/app/workers/submissions_cleaner.rb b/app/workers/submissions_cleaner.rb index f1bd9ac7d..e5660eb1f 100644 --- a/app/workers/submissions_cleaner.rb +++ b/app/workers/submissions_cleaner.rb @@ -2,7 +2,7 @@ class SubmissionsCleaner include Sidekiq::Worker def perform - submission_cleaner = SubmissionCleaner.new(date: Date.today) + submission_cleaner = SubmissionCleaner.new(date: Time.zone.today) submission_cleaner.clean! end end \ No newline at end of file diff --git a/config/environments/test.rb b/config/environments/test.rb index 112143b01..f3678981d 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -39,6 +39,7 @@ # The :test delivery method accumulates sent emails in the # ActionMailer::Base.deliveries array. config.action_mailer.delivery_method = :test + config.active_job.queue_adapter = :test # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr @@ -48,8 +49,6 @@ # Don't care if the mailer can't send. config.action_mailer.perform_deliveries = true config.action_mailer.default_url_options = { host: 'localhost', port: 3000 } - config.action_mailer.delivery_method = :smtp - config.action_mailer.smtp_settings = { :address => ENV.fetch("MAILSERVER", '127.0.0.1'), :port => 1025 } config.action_mailer.raise_delivery_errors = true config.action_mailer.perform_caching = false diff --git a/config/locales/de.yml b/config/locales/de.yml index c1fde74d0..1c84ec166 100644 --- a/config/locales/de.yml +++ b/config/locales/de.yml @@ -2957,17 +2957,17 @@ de: Löschung der Abgaben und Korrekturen aus dem %{term} submission_deletion_text_html: > Aus Datenschutzgründen werden sämtliche in MaMpf von Dir gespeicherten - Abgaben und Korrekturen aus dem %{term} am %{deletion_date} - automatisch gelöscht. Bitte lade diese rechtzeitig herunter, wenn Du sie - weiterhin verwenden möchtest. + Abgaben und Korrekturen aus dem %{term} mit entsprechendem Löschdatum am + %{deletion_date} automatisch gelöscht. Bitte lade diese rechtzeitig + herunter, wenn Du sie weiterhin verwenden möchtest. submission_deletion_lecture_subject: > Löschung der Abgaben und Korrekturen aus dem %{term} [%{lecture}] submission_deletion_lecture_text_html: > Aus Datenschutzgründen werden sämtliche in MaMpf gespeicherten - Abgaben und -korrekturen der Veranstaltung %{lecture} aus dem %{term} am - %{deletion_date} automatisch gelöscht. Sollten sie anderweitig - gespeichert werden müssen, so muss dies außerhalb von MaMpf geschehen, - und die Dateien müssen rechtzeitig heruntergeladen werden. + Abgaben und Korrekturen der Veranstaltung %{lecture} aus dem %{term} mit + entsprechendem Löschdatum am %{deletion_date} automatisch gelöscht. + Sollten sie anderweitig gespeichert werden müssen, so muss dies außerhalb + von MaMpf geschehen, und die Dateien müssen rechtzeitig heruntergeladen werden. submission_destruction_subject: > Löschung der Abgaben und Korrekturen aus dem %{term} submission_destruction_text_html: > diff --git a/config/locales/en.yml b/config/locales/en.yml index 1b9e96f36..b48717782 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -2781,14 +2781,14 @@ en: Deletion of submissions and corrections from %{term} submission_deletion_text_html: > For GDPR reasons, all of your submissions and corrections from - %{term} that are stored in MaMpf will be automatically deleted at + %{term}, with the corresponding deletion date, will be automatically deleted at %{deletion_date}. If you want to use them in the future, please download them before. submission_deletion_lecture_subject: > Deletion of submissions and corrections from %{term} [%{lecture}] submission_deletion_lecture_text_html: > For GDPR reasons, all submissions and corrections of the lecture - %{lecture} in %{term} that are stored in MaMpf will be automatically + %{lecture} in %{term}, with the corresponding deletion date, will be automatically deleted at %{deletion_date}. In for some reason they have to be stored longer, this has to happen outside of MaMpf. In that case the files have to be downloaded in time. diff --git a/spec/factories/assignments.rb b/spec/factories/assignments.rb index 19a28da25..11c80ba1e 100644 --- a/spec/factories/assignments.rb +++ b/spec/factories/assignments.rb @@ -7,6 +7,7 @@ Faker::Number.between(from: 1, to: 9999).to_s end deadline { Faker::Time.forward(days: 30) } + deletion_date { Faker::Date.forward(days: 60) } accepted_file_type { '.pdf' } trait :with_lecture do diff --git a/spec/factories/medium_publisher.rb b/spec/factories/medium_publisher.rb index 9bfe94aaa..d4ffc0f89 100644 --- a/spec/factories/medium_publisher.rb +++ b/spec/factories/medium_publisher.rb @@ -16,6 +16,7 @@ assignment_title { '' } assignment_file_type { '' } assignment_deadline { nil } + assignment_deletion_date { nil } end initialize_with do @@ -26,7 +27,8 @@ create_assignment: create_assignment, assignment_title: assignment_title, assignment_file_type: assignment_file_type, - assignment_deadline: assignment_deadline) + assignment_deadline: assignment_deadline, + assignment_deletion_date: assignment_deletion_date) end end end diff --git a/spec/factories/submission_cleaner.rb b/spec/factories/submission_cleaner.rb index 81c3df793..7c3653d82 100644 --- a/spec/factories/submission_cleaner.rb +++ b/spec/factories/submission_cleaner.rb @@ -3,7 +3,7 @@ FactoryBot.define do factory :submission_cleaner do transient do - date { Faker::Date.forward(days: 365) } + date { Time.zone.today } end initialize_with do diff --git a/spec/models/medium_publisher_spec.rb b/spec/models/medium_publisher_spec.rb index f701ff3cc..91866f25c 100644 --- a/spec/models/medium_publisher_spec.rb +++ b/spec/models/medium_publisher_spec.rb @@ -38,16 +38,18 @@ release_date: '04-03-2021 13:50', lock_comments: '0', publish_vertices: '1', create_assignment: '1', assignment_title: 'Blatt 1', assignment_file_type: '.pdf', - assignment_deadline: '04-03-2021 14:50' } + assignment_deadline: '04-03-2021 14:50', + assignment_deletion_date: '04-03-2021' } publisher = MediumPublisher.parse(@medium, @user, params) comp_methods = [:release_now, :release_for, :release_date, :lock_comments, :vertices, :create_assignment, :assignment_title, - :assignment_file_type, :assignment_deadline] + :assignment_file_type, :assignment_deadline, + :assignment_deletion_date] publisher_array = comp_methods.map { |m| publisher.send(m) } expect(publisher_array) .to eq([false, 'users', Time.zone.parse('04-03-2021 13:50'), false, true, true, 'Blatt 1', '.pdf', - Time.zone.parse('04-03-2021 14:50')]) + Time.zone.parse('04-03-2021 14:50'), Time.zone.parse('04-03-2021')]) end it 'rescues argument errors for the release date' do @@ -66,6 +68,15 @@ publisher = MediumPublisher.parse(@medium, @user, params) expect(publisher.assignment_deadline).to be_nil end + + it 'rescues argument errors for the assignment deletion date' do + params = { release_now: '0', released: 'all', + release_date: '04-03-2021 13:50', lock_comments: '1', + publish_vertices: '0', create_assignment: '1', + assignment_deletion_date: 'w47qwhhhhkc4' } + publisher = MediumPublisher.parse(@medium, @user, params) + expect(publisher.assignment_deletion_date).to be_nil + end end describe '#publish!' do @@ -128,6 +139,7 @@ create_assignment: true, assignment_title: 'Blatt 1', assignment_deadline: '2025-05-02 12:00', + assignment_deletion_date: '2025-05-02', assignment_file_type: '.pdf') publisher.publish! lecture.reload @@ -162,6 +174,8 @@ assignment_title: 'Blatt 1', assignment_deadline: Time.zone.parse('2025-05-02 12:00'), + assignment_deletion_date: + Time.zone.parse('2025-05-02'), assignment_file_type: '.pdf') end @@ -194,7 +208,7 @@ expect(publisher.errors).to eq({}) end - it 'returns {} if there are no errors (example 1)' do + it 'returns {} if there are no errors (example 2)' do publisher = FactoryBot.build(:medium_publisher, medium_id: @medium.id, user_id: @user.id, @@ -203,7 +217,8 @@ create_assignment: true, assignment_title: 'Blatt 1', assignment_file_type: '.pdf', - assignment_deadline: Time.zone.now + 2.days) + assignment_deadline: Time.zone.now + 2.days, + assignment_deletion_date: Time.zone.today + 2.days) expect(publisher.errors).to eq({}) end @@ -223,7 +238,7 @@ medium_id: @medium.id, user_id: @user.id, release_now: false, - release_date: Time.zone.now - 1.days) + release_date: Time.zone.now - 1.day) expect(publisher.errors[:release_date]).not_to be_nil end @@ -234,10 +249,22 @@ user_id: @user.id, release_now: true, create_assignment: true, - assignment_deadline: Time.zone.now - 1.days) + assignment_deadline: Time.zone.now - 1.day) expect(publisher.errors[:assignment_deadline]).not_to be_nil end + it 'returns an assignment deletion date error if release is scheduled now' \ + ' and deletion date is in the past' do + publisher = FactoryBot.build(:medium_publisher, + medium_id: @medium.id, + user_id: @user.id, + release_now: true, + create_assignment: true, + assignment_deletion_date: + Time.zone.today - 1.day) + expect(publisher.errors[:assignment_deletion_date]).not_to be_nil + end + it 'returns an assignment deadline error if release is scheduled later ' \ 'and deadline is before that' do publisher = FactoryBot.build(:medium_publisher, @@ -273,6 +300,7 @@ create_assignment: true, assignment_title: 'Blatt 1', assignment_deadline: Time.zone.now + 2.days, + assignment_deletion_date: Time.zone.today + 2.days, assignment_file_type: '.pdf') expect(publisher.errors[:assignment_title]).not_to be_nil end diff --git a/spec/models/submission_cleaner_spec.rb b/spec/models/submission_cleaner_spec.rb index 8125b3b25..8675a263e 100644 --- a/spec/models/submission_cleaner_spec.rb +++ b/spec/models/submission_cleaner_spec.rb @@ -8,262 +8,91 @@ .to be_kind_of(SubmissionCleaner) end - describe '#set_attributes' do - after :each do - @term.destroy - end - -=begin - it 'correctly determines wether an action is possible (example 1)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS') - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('25.3.2020')) - cleaner.set_attributes - expect(cleaner.advance).to be false - end - - it 'correctly determines wether an action is possible (example 2a)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS') - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('1.4.2021')) - cleaner.set_attributes - expect(cleaner.advance).to be true - end - - it 'correctly determines wether an action is possible (example 2b)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS') - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('1.4.2021')) - cleaner.set_attributes - expect(cleaner.destroy).not_to be true - end - - it 'correctly determines wether an action is possible (example 3)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS') - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('3.4.2021')) - cleaner.set_attributes - expect(cleaner.advance).to be false - end - - it 'correctly determines wether an action is possible (example 4)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS') - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('8.4.2021')) - cleaner.set_attributes - expect(cleaner.advance).to be false - end - - it 'correctly determines wether an action is possible (example 5a)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS', - submission_deletion_mail: - DateTime.parse('1.4.2021 3:00')) - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('8.4.2021')) - cleaner.set_attributes - expect(cleaner.advance).to be true - end - - it 'correctly determines wether an action is possible (example 5b)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS', - submission_deletion_mail: - DateTime.parse('1.4.2021 3:00')) - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('8.4.2021')) - cleaner.set_attributes - expect(cleaner.destroy).not_to be true - end - - it 'correctly determines wether an action is possible (example 6)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS', - submission_deletion_mail: - DateTime.parse('1.4.2021 3:00'), - submission_deletion_reminder: - DateTime.parse('8.4.2021 3:00')) - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('9.4.2021')) - cleaner.set_attributes - expect(cleaner.advance).to be false - end - - it 'correctly determines wether an action is possible (example 7)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS', - submission_deletion_mail: - DateTime.parse('1.4.2021 3:00'), - submission_deletion_reminder: - DateTime.parse('8.4.2021 3:00')) - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('9.4.2021')) - cleaner.set_attributes - expect(cleaner.advance).to be false - end - - it 'correctly determines wether an action is possible (example 8a)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS', - submission_deletion_mail: - DateTime.parse('1.4.2021 3:00'), - submission_deletion_reminder: - DateTime.parse('8.4.2021 3:00')) - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('15.4.2021')) - cleaner.set_attributes - expect(cleaner.advance).to be true - end - - it 'correctly determines wether an action is possible (example 8b)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS', - submission_deletion_mail: - DateTime.parse('1.4.2021 3:00'), - submission_deletion_reminder: - DateTime.parse('8.4.2021 3:00')) - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('15.4.2021')) - cleaner.set_attributes - expect(cleaner.destroy).to be true - end - - it 'correctly determines wether an action is possible (example 8a)' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS', - submission_deletion_mail: - DateTime.parse('1.4.2021 3:00'), - submission_deletion_reminder: - nil) - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('15.4.2021')) - cleaner.set_attributes - expect(cleaner.advance).to be false - end - end - - describe '#clean!' do - after :each do - @term.destroy - end - - it 'does nothing if the date is not right' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS') - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('28.3.2021')) - cleaner.clean! - @term.reload - expect(@term.submission_deletion_mail).to be_nil - end - - it 'updates submission_deletion_mail column on first day of next term' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS') - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('1.4.2021')) - cleaner.clean! - @term.reload - expect(@term.submission_deletion_mail).not_to be_nil - end - - it 'updates submission_deletion_reminder column one week later' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS', - submission_deletion_mail: - DateTime.parse('1.4.2021 3:00')) - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('8.4.2021')) - cleaner.clean! - @term.reload - expect(@term.submission_deletion_reminder).not_to be_nil - end - - it 'updates submissions_deleted_at column two weeks later' do - @term = FactoryBot.create(:term, year: 2020, season: 'WS', - submission_deletion_mail: - DateTime.parse('1.4.2021 3:00'), - submission_deletion_reminder: - DateTime.parse('8.4.2021 3:00')) - cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('15.4.2021')) - cleaner.clean! - @term.reload - expect(@term.submissions_deleted_at).not_to be_nil - end - end - - context 'with sample submissions' do + describe 'with sample submissions' do before :all do Term.destroy_all - @term = FactoryBot.create(:term, year: 2020, season: 'WS') - @lecture = FactoryBot.create(:lecture, term: @term) + ActionMailer::Base.deliveries = [] + @term1 = FactoryBot.create(:term, year: Time.zone.today.year, season: 'SS') + @term2 = FactoryBot.create(:term, year: Time.zone.today.year - 1, season: 'WS') + @lecture = FactoryBot.create(:lecture) tutorial = FactoryBot.create(:tutorial, :with_tutors, lecture: @lecture) assignment1 = FactoryBot.create(:assignment, - lecture: @lecture, - deadline: Date.parse('31.12.2020')) + lecture: @lecture, + deletion_date: Time.zone.today + 14.days) assignment2 = FactoryBot.create(:assignment, lecture: @lecture, - deadline: Date.parse('31.1.2021')) - @user1 = FactoryBot.create(:confirmed_user) - @user2 = FactoryBot.create(:confirmed_user) - @user3 = FactoryBot.create(:confirmed_user) - @user4 = FactoryBot.create(:confirmed_user) + deletion_date: Time.zone.today + 21.days) + @user1 = FactoryBot.create(:confirmed_user, + locale: I18n.available_locales.first) + @user2 = FactoryBot.create(:confirmed_user, + locale: I18n.available_locales.first) + @user3 = FactoryBot.create(:confirmed_user, + locale: I18n.available_locales.first) @submission1 = FactoryBot.create(:submission, - tutorial: tutorial, - assignment: assignment1) + tutorial: tutorial, + assignment: assignment1) @submission1.users << @user1 @submission2 = FactoryBot.create(:submission, - tutorial: tutorial, - assignment: assignment1) + tutorial: tutorial, + assignment: assignment2) @submission2.users << [@user2, @user3] - @submission3 = FactoryBot.create(:submission, - tutorial: tutorial, - assignment: assignment2) - @submission3.users << @user4 - @term2 = FactoryBot.create(:term, year: 2021, season: 'SS') - lecture2 = FactoryBot.create(:lecture, term: @term2) - tutorial2 = FactoryBot.create(:tutorial, :with_tutors, lecture: lecture2) - assignment21 = FactoryBot.create(:assignment, - lecture: lecture2, - deadline: Date.parse('15.5.2021')) - @submission21 = FactoryBot.create(:submission, - tutorial: tutorial2, - assignment: assignment21) - user5 = FactoryBot.create(:confirmed_user) - @submission21.users << [@user1, user5] - lecture3 = FactoryBot.create(:lecture, term: @term) end describe '#set_attributes' do - it 'sets the submissions correctly' do + it 'sends info emails correctly' do cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('1.4.2021')) - cleaner.set_attributes - expect(cleaner.submissions) - .to match_array([@submission1, @submission2, @submission3]) + date: Time.zone.today) + + expect do + cleaner.clean! + end.to change { ActionMailer::Base.deliveries.count }.by(2) end - it 'sets the submitters correctly' do + it 'sends info and reminder emails correctly' do cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('1.4.2021')) - cleaner.set_attributes - expect(cleaner.submitters) - .to match_array([@user1, @user2, @user3, @user4]) + date: Time.zone.today + 7.days) + + expect do + cleaner.clean! + end.to change { ActionMailer::Base.deliveries.count }.by(4) end - it 'sets the lectures correctly' do + it 'sends deletion emails correctly (example 1)' do cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('1.4.2021')) - cleaner.set_attributes - expect(cleaner.lectures).to match_array([@lecture]) + date: Time.zone.today + 14.days) + + expect do + cleaner.clean! + end.to change { ActionMailer::Base.deliveries.count }.by(4) + end + + it 'sends deletion emails correctly (example 2)' do + cleaner = FactoryBot.build(:submission_cleaner, + date: Time.zone.today + 21.days) + + expect do + cleaner.clean! + end.to change { ActionMailer::Base.deliveries.count }.by(2) end end describe '#clean!' do - it 'destroys submissions of the relevant term two weeks later' do - @term.submission_deletion_mail = DateTime.parse('1.4.2021 3:00') - @term.submission_deletion_reminder = DateTime.parse('8.4.2021 3:00') - @term.save + it 'destroys submissions correctly' do cleaner = FactoryBot.build(:submission_cleaner, - date: Date.parse('15.4.2021')) + date: Time.zone.today + 21.days) cleaner.clean! - @term.reload - expect([@term.submissions.to_a, @term2.submissions.to_a]) - .to eq([[], [@submission21]]) + + expect(Submission.all.size).to be(1) + end + + it 'does not destroy submissions if no assignments to be deleted' do + cleaner = FactoryBot.build(:submission_cleaner, + date: Time.zone.today + 20.days) + cleaner.clean! + + expect(Submission.all.size).to be(2) end end -=end end -end \ No newline at end of file +end From c3a8b142936870e299283695de574f4ec817f802 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Wed, 9 Mar 2022 14:40:55 +0100 Subject: [PATCH 21/28] resolved merge conflicts for mampf-next --- Gemfile | 1 - app/validators/http_url_validator.rb | 2 +- docker/production/docker-compose.production.yml | 13 ++++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 308fc0841..b8ea77caa 100644 --- a/Gemfile +++ b/Gemfile @@ -82,7 +82,6 @@ gem "thredded-markdown_katex", gem "rails-i18n" gem "kaminari-i18n" gem "trix-rails", require: "trix" -gem "xkcd" gem "sunspot_rails", github: 'sunspot/sunspot', glob: 'sunspot_rails/*.gemspec' diff --git a/app/validators/http_url_validator.rb b/app/validators/http_url_validator.rb index 5044fa8ae..68806ee14 100644 --- a/app/validators/http_url_validator.rb +++ b/app/validators/http_url_validator.rb @@ -1,7 +1,7 @@ class HttpUrlValidator < ActiveModel::EachValidator def self.compliant?(value) - uri = URI.parse(URI.encode_www_form_component(value)) + uri = URI.parse(Addressable::URI.encode(value)) uri.is_a?(URI::HTTP) && !uri.host.nil? rescue URI::InvalidURIError false diff --git a/docker/production/docker-compose.production.yml b/docker/production/docker-compose.production.yml index 2bba7ef34..076751413 100644 --- a/docker/production/docker-compose.production.yml +++ b/docker/production/docker-compose.production.yml @@ -54,7 +54,18 @@ services: restart: always image: redis:6-alpine networks: - - mampf + - default + + solr: + image: solr:8.11 + container_name: solr + ports: + - "127.0.0.1:8983:8983" + volumes: + - /path/to/persistent/var_solr:/var/solr + networks: + - default + restart: always cache: restart: always From 55eaeaae4901e9121c25725e4fe7be80d574cfc8 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Wed, 9 Mar 2022 14:42:37 +0100 Subject: [PATCH 22/28] fixed merge-conflict (2) --- docker/production/docker-compose.production.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/production/docker-compose.production.yml b/docker/production/docker-compose.production.yml index 076751413..c2f1ee6b7 100644 --- a/docker/production/docker-compose.production.yml +++ b/docker/production/docker-compose.production.yml @@ -53,6 +53,12 @@ services: redis: restart: always image: redis:6-alpine + networks: + - mampf + + cache: + restart: always + image: memcached:alpine networks: - default @@ -67,12 +73,6 @@ services: - default restart: always - cache: - restart: always - image: memcached:alpine - networks: - - mampf - networks: postgres: external: true From a3fb480026ac3796f7502dd3d347485dcd5b8b65 Mon Sep 17 00:00:00 2001 From: Tobias Wackenhut Date: Wed, 9 Mar 2022 16:01:43 +0100 Subject: [PATCH 23/28] Add some documentation to nginx setup. --- INSTALL.md | 60 +++++++-------------------- docker/development/docker-compose.yml | 1 + 2 files changed, 16 insertions(+), 45 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index e378182ea..e6ae9467e 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -7,7 +7,7 @@ $ cd mampf/docker/development/ # docker-compose up ``` -NOTE: Please make sure to clone recursivly as the pdf compression feature is in an extra repository. +NOTE: Please make sure to clone recursively as the pdf compression feature is in an extra repository. If you have an already checked out version simply run: ```sh @@ -19,6 +19,8 @@ You now have the following things ready: * The MaMpf server on localhost:3000 * The mailcatcher service on localhost:1080 * The webinterface for ApacheSolr on localhost:8983 +* A test mailserver instance on Ports 1025, 10143, 10993 +* A webpacker dev server on localhost:3035 ### Database setup @@ -46,11 +48,13 @@ and put it in the `db/backups/docker_development` folder in your project directo # docker-compose exec mampf rails db:migrate ``` 4. Download the sample videos and sample manuscripts that match the data in the prepopulated - database here and extract the .zip file into the `public/` folder of your project directory. + database here and extract the .zip file into the `public/` folder of your project directory. 5. Call the MaMpf Server on localhost:3000. The prepopulated database contains data for several users that you can use to sign in: `admin@mampf.edu`, `teacher@mampf.edu`, `tutor@mampf.edu` and `student1@mampf.edu`,..., `student5@mampf.edu` (with the obvious roles). Each of these have `dockermampf` as password. 6. There you go :tada: +Instead you can also uncomment the preseed options in the docker-compose file. When in daut, just follow this guide here. + Note that in both cases, the first start of the MaMpf server can take a while, as all assets have to provided. @@ -67,61 +71,27 @@ A few common commands for `docker-compose` are: | `docker-compose exec mampf ` | run an executable in the container | -## Installation in production mode (with Docker) +## Installation in production mode 1. Install Database Server (e.g. PostgreSQL) and create Database. (Don't forget to allow access for the docker network) ``` -createuser mampf +createuser -P mampf createdb -O mampf mampf -psql -c "ALTER USER mampf PASSWORD '$PASSWORD'" -``` - 2. Create an environment file like this: -``` -RAILS_ENV=production -PRODUCTION_DATABASE_ADAPTER=postgresql -PRODUCTION_DATABASE_DATABASE=mampf -PRODUCTION_DATABASE_USERNAME=mampf -PRODUCTION_DATABASE_PASSWORD=$DATABASE_PASSWORD -PRODUCTION_DATABASE_HOST=172.17.0.1 -PRODUCTION_DATABASE_PORT=5432 -MAILSERVER=localhost -FROM_ADDRESS=mampf@localhost -URL_HOST=localhost -RAILS_MASTER_KEY=$MASTER_KEY -ERDBEERE_SERVER = your_erdbeere_server -MUESLI_SERVER = your_muesli_server -PROJECT_EMAIL = your_project_email -PROJECT_NOTIFICATION_EMAIL= your_project_notification_email -INSTANCE_PATH=mampf -USE_CAPTCHA_SERVICE=true -CAPTCHA_VERIFY_URL=your_captcha_service_verify_url -CAPTCHA_PUZZLE_URL=your_captcha_service_puzzle_url -CAPTCHA_APPLICATION_TOKEN=your_token_for_verifying_puzzle -REWRITE_ENABLED=1_if_you_want_nice_filenames -``` - 3. Execute the following commands to install and run the service: -``` -git clone -b main git@github.com:fosterfarrell9/mampf.git -docker build --label "mampf" mampf -docker create --name mampf --env-file $ENVFILE -p $OUTSIDEPORT:3000 $IMAGEID -docker run --rm --env-file $ENVFILE $IMAGEID 'rm config/credentials.yml.enc && bundle exec rails credentials:edit' -docker start mampf -docker exec mampf bundle exec rake db:migrate -docker exec mampf bundle exec rake db:seed -docker exec mampf bundle exec rake assets:precompile -docker stop mampf -docker start mampf ``` -Now you can access *mampf* via `http://localhost:$OUTSIDEPORT`. + 2. Create an environment file based on the example in docker/production/docker.env + 3. Create a docker-compose file based on docker/production/docker-compose.production.yml . We recommend serving upload caches, submissions and media files by a separate server. They communicate with Mampf using NFS as Docker volumes. See the example options in the sample file. + 4. Launch MaMpf using `docker-compose up -d` + 5. (Optional) Scale MaMpf horizontally by spawning more workers `docker-compose scale worker=5` + Now you can access *mampf* via `http://localhost:$OUTSIDEPORT`. Use the GUI to register your future admin user. Open a rails console inside the docker container. ``` -rails c +docker-compose exec master rails c ``` Give admin rights to this user: ``` User.first.update(admin: true) ``` -That's it. Everything else can be done entirely via the GUI. In a production environment you might want to delete upload caches `/usr/src/app/public/uploads/cache/*` and expired quizzes (`bundle exec rake cleanup:destroy_random_quizzes`) regularly. +That's it. Everything else can be done entirely via the GUI. diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index c0c51981a..9288b4124 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -142,6 +142,7 @@ services: - "127.0.0.1:3000:80" volumes: - "nginx-conf-d:/etc/nginx/conf.d" + # Submissions are served by nginx but are still authenticated by MaMpf. (accel_redirect) - "submissions:/private/submissions:ro,nocopy" - "public:/public" From fbe8dd084dc5bae92e575048d6bdc9b82cc1bc4f Mon Sep 17 00:00:00 2001 From: fosterfarrell9 <28628554+fosterfarrell9@users.noreply.github.com> Date: Wed, 9 Mar 2022 18:55:47 +0100 Subject: [PATCH 24/28] set up security policy --- SECURITY.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..f57d9ddf5 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security Policy + +## Supported Versions + +We only release patches for the production branch. + +## Reporting a Vulnerability + +Please report (suspected) security vulnerabilities to +mampf.security@mathi.uni-heidelberg.de. You will receive a response from us +within 48 hours. If the issue is confirmed, we will release a patch as soon +as possible depending on complexity but usually within a few days. \ No newline at end of file From 777d13df33d4f24fa574141751d700cfa3a957fa Mon Sep 17 00:00:00 2001 From: fosterfarrell9 <28628554+fosterfarrell9@users.noreply.github.com> Date: Thu, 10 Mar 2022 11:59:11 +0100 Subject: [PATCH 25/28] Update SECURITY.md --- SECURITY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index f57d9ddf5..7162e6d8c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,6 +7,6 @@ We only release patches for the production branch. ## Reporting a Vulnerability Please report (suspected) security vulnerabilities to -mampf.security@mathi.uni-heidelberg.de. You will receive a response from us +mampf-security@mathi.uni-heidelberg.de. You will receive a response from us within 48 hours. If the issue is confirmed, we will release a patch as soon -as possible depending on complexity but usually within a few days. \ No newline at end of file +as possible depending on complexity but usually within a few days. From ba099bdce9428946fa09b56751cc0c6f634274aa Mon Sep 17 00:00:00 2001 From: fosterfarrell9 <28628554+fosterfarrell9@users.noreply.github.com> Date: Thu, 10 Mar 2022 17:34:20 +0100 Subject: [PATCH 26/28] change saving policy for referrer url in interactions --- app/workers/interaction_saver.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/workers/interaction_saver.rb b/app/workers/interaction_saver.rb index e2b875f2a..62616f538 100644 --- a/app/workers/interaction_saver.rb +++ b/app/workers/interaction_saver.rb @@ -2,10 +2,13 @@ class InteractionSaver include Sidekiq::Worker def perform(session_id, full_path, referrer, study_participant) + referrer_url = if referrer.to_s.include?(ENV['URL_HOST']) + referrer.to_s.remove(ENV['URL_HOST']) + .remove('https://').remove('http://') + end Interaction.create(session_id: Digest::SHA2.hexdigest(session_id).first(10), full_path: full_path, - referrer_url: referrer&.remove(ENV['URL_HOST']) - &.remove('https://'), + referrer_url: referrer_url, study_participant: study_participant) end end \ No newline at end of file From d6a1798f1129bc2463f6d21bd4c9d3cfeddd42da Mon Sep 17 00:00:00 2001 From: fosterfarrell9 <28628554+fosterfarrell9@users.noreply.github.com> Date: Thu, 10 Mar 2022 17:57:05 +0100 Subject: [PATCH 27/28] correct indentation --- app/workers/interaction_saver.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/workers/interaction_saver.rb b/app/workers/interaction_saver.rb index 62616f538..05fa6b36f 100644 --- a/app/workers/interaction_saver.rb +++ b/app/workers/interaction_saver.rb @@ -3,8 +3,8 @@ class InteractionSaver def perform(session_id, full_path, referrer, study_participant) referrer_url = if referrer.to_s.include?(ENV['URL_HOST']) - referrer.to_s.remove(ENV['URL_HOST']) - .remove('https://').remove('http://') + referrer.to_s.remove(ENV['URL_HOST']) + .remove('https://').remove('http://') end Interaction.create(session_id: Digest::SHA2.hexdigest(session_id).first(10), full_path: full_path, From cd6adcb32587d3b103774eb6361400c0b0e7a383 Mon Sep 17 00:00:00 2001 From: Fabian Kontor Date: Tue, 15 Mar 2022 13:04:33 +0100 Subject: [PATCH 28/28] made last few migrations reversible --- db/migrate/20210923113111_add_public_to_watchlists.rb | 6 +++++- .../20211119181430_add_description_to_watchlist.rb | 6 +++++- .../20220125162730_add_deletion_date_to_assignments.rb | 9 +++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/db/migrate/20210923113111_add_public_to_watchlists.rb b/db/migrate/20210923113111_add_public_to_watchlists.rb index c661ca027..9668c108f 100644 --- a/db/migrate/20210923113111_add_public_to_watchlists.rb +++ b/db/migrate/20210923113111_add_public_to_watchlists.rb @@ -1,5 +1,9 @@ class AddPublicToWatchlists < ActiveRecord::Migration[6.1] - def change + def up add_column :watchlists, :public, :boolean, default: false end + + def down + remove_column :watchlists, :public, :boolean + end end diff --git a/db/migrate/20211119181430_add_description_to_watchlist.rb b/db/migrate/20211119181430_add_description_to_watchlist.rb index f10cdb623..0b30a2fe8 100644 --- a/db/migrate/20211119181430_add_description_to_watchlist.rb +++ b/db/migrate/20211119181430_add_description_to_watchlist.rb @@ -1,5 +1,9 @@ class AddDescriptionToWatchlist < ActiveRecord::Migration[6.1] - def change + def up add_column :watchlists, :description, :string end + + def down + remove_column :watchlists, :description, :string + end end diff --git a/db/migrate/20220125162730_add_deletion_date_to_assignments.rb b/db/migrate/20220125162730_add_deletion_date_to_assignments.rb index 05aaa3b5b..2da53ebdd 100644 --- a/db/migrate/20220125162730_add_deletion_date_to_assignments.rb +++ b/db/migrate/20220125162730_add_deletion_date_to_assignments.rb @@ -1,8 +1,13 @@ class AddDeletionDateToAssignments < ActiveRecord::Migration[6.1] - def change + def up add_column :assignments, :deletion_date, :date, null: false, default: (Term.active&.end_date || (Date.today + 180.days)) + 15.days - remove_column :assignments, :protected, :boolean, :default => false + remove_column :assignments, :protected, :boolean + end + + def down + remove_column :assignments, :deletion_date, :date + add_column :assignments, :protected, :boolean, default: false end end