detect answer titles

Platoniq · Feb 25, 2021 · e369188 · e369188
1 parent 362c54e
commit e369188
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 41 deletions.
diff --git a/README.md b/README.md
@@ -36,12 +36,11 @@ o.save
 1: Importing massive answers for proposals
 ------------------------------------------
 
-Copy the script:
+Copy the script into your `lib/tasks` of your Decidim installation:
 
 ```
-cp proposal_answers.rake ../decidim-barcelona-preprod/lib/tasks
-cp geoloc_import.rake ../decidim-barcelona-preprod/lib/tasks
-cp script_helpers.rake ../decidim-barcelona-preprod/lib/tasks
+cd lib/tasks
+wget -qO- https://github.com/Platoniq/decidim-scripts/archive/0.2.tar.gz | tar --transform 's/^decidim-scripts-0.2//' -xvz
 ```
 
 Run in local (as testing):
@@ -61,19 +60,15 @@ As there's no "copy" command to do that you can use the service https://transfer
 ```
 curl --upload-file pam-ciutat.csv https://transfer.sh/pam-ciutat.csv -H "Max-Days: 1"
 curl --upload-file pam-districtes.csv https://transfer.sh/pam-districte.csv -H "Max-Days: 1"
-curl --upload-file proposal_answers.rake https://transfer.sh/proposal_answers.rake -H "Max-Days: 1"
-curl --upload-file proposal_answers.rake https://transfer.sh/geoloc_import.rake -H "Max-Days: 1"
-curl --upload-file proposal_answers.rake https://transfer.sh/script_helpers.rb -H "Max-Days: 1"
+curl --upload-file geolocs.csv https://transfer.sh/geolocs.csv -H "Max-Days: 1"
 ```
 
 Which will return (for instance) the download addresses:
 
 ```
 https://transfer.sh/23lG7/pam-ciutat.csv
 https://transfer.sh/YwYAN/pam-districte.csv
-https://transfer.sh/x3hUa/proposal_answers.rake
-https://transfer.sh/x4hUa/geoloc_import.rake
-https://transfer.sh/x5hUa/script_helpers.rake
+https://transfer.sh/x3hUa/geolocs.csv
 ```
 
 2. Login into a shell session in heroku
@@ -87,12 +82,16 @@ heroku run bash
 ```
 wget https://transfer.sh/23lG7/pam-ciutat.csv
 wget https://transfer.sh/YwYAN/pam-districte.csv
-wget https://transfer.sh/x3hUa/proposal_answers.rake -O lib/tasks/proposal_answers.rake
-wget https://transfer.sh/x4hUa/geoloc_import.rake -O lib/tasks/proposal_answers.rake
-wget https://transfer.sh/x5hUa/script_helpers.rake -O lib/tasks/proposal_answers.rake
+wget https://transfer.sh/x3hUa/golocs.csv
 ```
 
-4. Run the script inside the shell session (2nd terminal):
+4. Download the scripts into the `lib/tasks` folder:
+
+```
+wget -qO- https://github.com/Platoniq/decidim-scripts/archive/0.2.tar.gz | tar --transform 's/^decidim-scripts-0.2//' -xvz -C lib/tasks
+```
+
+5. Run the script inside the shell session (2nd terminal):
 
 ```
 bin/rails proposals:batch:answer

diff --git a/geoloc_import.rake b/geoloc_import.rake
@@ -19,18 +19,9 @@ namespace :proposals do
 
     desc 'Geolocate proposals from a CSV'
     task :geoloc, %i[admin csv] => :environment do |_task, args|
-      process_csv(args) do |admin, table|
-        table.each_with_index do |line, index|
-          print "##{index} (#{100 * (index + 1) / table.count}%): "
-          begin
-            processor = ProposalGeolocProcessor.new(admin, normalize(line))
-            processor.process!
-          rescue UnprocessableError => e
-            show_error(e.message)
-          rescue AlreadyProcessedError => e
-            show_warning(e.message)
-          end
-        end
+      process_csv(args) do |admin, line|
+        processor = ProposalGeolocProcessor.new(admin, normalize(line))
+        processor.process!
       end
     end
 

diff --git a/proposal_answers.rake b/proposal_answers.rake
@@ -19,18 +19,9 @@ namespace :proposals do
 
     desc 'Import answers to proposals from a CSV'
     task :answer, %i[admin csv] => :environment do |_task, args|
-      process_csv(args) do |admin, table|
-        table.each_with_index do |line, index|
-          print "##{index} (#{100 * (index + 1) / table.count}%): "
-          begin
-            processor = ProposalAnswerProcessor.new(admin, normalize(line))
-            processor.process!
-          rescue UnprocessableError => e
-            show_error(e.message)
-          rescue AlreadyProcessedError => e
-            show_warning(e.message)
-          end
-        end
+      process_csv(args) do |admin, line|
+        processor = ProposalAnswerProcessor.new(admin, normalize(line))
+        processor.process!
       end
     end
 

diff --git a/script_helpers.rb b/script_helpers.rb
@@ -25,7 +25,18 @@ def process_csv(args)
 
     table = CSV.parse(File.read(args.csv), headers: true)
 
-    yield(admin, table)
+    table.each_with_index do |line, index|
+      print "##{index} (#{100 * (index + 1) / table.count}%): "
+      begin
+        yield(admin, line)
+      rescue UnprocessableError => e
+        show_error(e.message)
+      rescue ActiveRecord::RecordInvalid => e
+        show_error(e.message)            
+      rescue AlreadyProcessedError => e
+        show_warning(e.message)
+      end
+    end
   rescue ArgumentError => e
     puts
     show_error(e.message)
@@ -65,9 +76,9 @@ def normalize(line)
         values[:id] = value
       when /^estat$|^status$|^state$/i
         values[:state] = value
-      when /^text (.*)catal[à|a](.*)|^text_ca$/i
+      when /^text (.*)catal[à|a](.*)|^text_ca$|^answer\/ca$/i
         values[:answer][:ca] = value
-      when /^text (.*)castell[a|à](.*)|^text_es$/i
+      when /^text (.*)castell[a|à](.*)|^text_es$|^answer\/es$/i
         values[:answer][:es] = value
       when /^adreça$|^dirección$|^address$/i
         values[:address] = value