-
-
Notifications
You must be signed in to change notification settings - Fork 86
/
import_job.rb
133 lines (110 loc) · 3.79 KB
/
import_job.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
# rubocop:disable Metrics/ModuleLength
module ImportJob
extend ActiveSupport::Concern
included do |base|
attr_accessor :processed_file, :stats
attr_reader :job_stats, :error_details, :filename, :error_messages
base.extend ImportJobClassMethods
end
def perform(*args)
temporary_file = args[0]
@filename = args[1]
@organization = args[2]
initialize_processed_file(temporary_file, filename, @organization)
if already_processed?
fail_processed_file("Already processed a file on #{already_processed_file.first.created_at.strftime('%m/%d/%Y')} with the same name: #{filename}. Data not imported!")
elsif validate_headers(temporary_file)
if import_records(temporary_file)
complete_processed_file!
else
log("Error: #{filename} does not have valid row(s). Data not imported!", :error)
fail_processed_file("Does not have valid row(s). Data not imported!")
end
else
log("Error: #{filename} does not have valid header(s). Data not imported!", :error)
fail_processed_file("Does not have valid header(s). Data not imported!")
end
@processed_file.save
end
def validate_headers(temporary_file)
raise "No input file specified" unless temporary_file
headers = CSV.parse(
temporary_file.contents,
headers: true,
header_converters: ->(header) { CsvImporter.header_conversion(header) }
).headers.compact
valid_headers = self.class::HEADERS.map { |header| CsvImporter.header_conversion(header) }
log("Headers in file: #{headers}", :debug)
log("Valid headers for model: #{valid_headers}", :debug)
valid_headers == headers
end
def import_records(temporary_file)
raise "No input file specified" unless temporary_file
csv_importer = CsvImporter.new(
target_model,
temporary_file.contents,
@processed_file.id,
@organization
)
csv_importer.call
if csv_importer.errored?
@error_details = csv_importer.error_details
@error_messages = csv_importer.error_messages
log(error_details, :info)
return false
end
@job_stats = csv_importer.stats
log(job_stats, :info)
true
end
def target_model
self.class.target_model
end
def already_processed?
already_processed_file.count > 0
end
def already_processed_file
@already_processed_file ||= ProcessedFile.where(
status: ['Processed', 'Processed with errors'],
filename: filename
)
end
def initialize_processed_file(temporary_file, filename, organization)
@processed_file = ProcessedFile.create(
temporary_file_id: temporary_file.id,
filename: filename,
category: target_model,
organization: organization,
status: 'Running'
)
end
def complete_processed_file!
@processed_file.job_stats = job_stats
@processed_file.status = 'Processed'
end
def fail_processed_file(error)
@processed_file.status = 'Failed'
@processed_file.job_stats = error_details || {}
@processed_file.job_errors = format_errors(error)
end
def format_errors(error)
return error if error_messages.nil? || error_messages.empty?
total_lines = error_messages.keys.length
total_errors = error_messages.values.map(&:length).reduce(:+)
messages = error_messages.map { |row, row_errors| "#{row} : #{row_errors.join(', ')}" }
"#{error} #{total_errors} error(s) found on #{total_lines} row(s). #{messages.join('. ')}."
end
def logger
@logger ||= Delayed::Worker.logger
end
def log(message, level = :debug)
raise "Wrong logger level: #{level}" unless %i(debug error info).include?(level)
logger.send(level, message)
end
module ImportJobClassMethods
def target_model
@target_model ||= to_s.gsub(/Job/, '').constantize
end
end
end
# rubocop:enable Metrics/ModuleLength