Fix csv export columns

CHANGES.rst

@@ -16,7 +16,8 @@ Changelog
   [JeffersonBledsoe]
 - Added support for sending emails as a table #31
   [JeffersonBledsoe]
-
+- Fix csv export columns.
+  [cekk]
 
 2.7.0 (2023-04-03)
 ------------------

buildout.cfg

@@ -15,4 +15,6 @@ pycodestyle = 2.8.0
 multipart = 0.2.4
 pyScss = 1.4.0
 robotframework = 5.0
-plone.rest = 2.0.0a5
+zope.container = 5.1
+iw.rejectanonymous = 1.2.7
+setuptools = 

src/collective/volto/formsupport/datamanager/catalog.py

@@ -85,17 +85,13 @@ def add(self, data):
             for x in form_fields
         }
         record = Record()
-        fields_labels = {}
-        fields_order = []
         for field_data in data:
             field_id = field_data.get("field_id", "")
             value = field_data.get("value", "")
             if field_id in fields:
                 record.attrs[field_id] = value
-                fields_labels[field_id] = fields[field_id]
-                fields_order.append(field_id)
-        record.attrs["fields_labels"] = fields_labels
-        record.attrs["fields_order"] = fields_order
+        record.attrs["fields_labels"] = fields
+        record.attrs["fields_order"] = [x["field_id"] for x in form_fields]
         record.attrs["date"] = datetime.now()
         record.attrs["block_id"] = self.block_id
         return self.soup.add(record)

src/collective/volto/formsupport/restapi/services/form_data/csv.py

@@ -4,6 +4,9 @@
 from plone.restapi.services import Service
 from six import StringIO
 from zope.component import getMultiAdapter
+from zope.component import getUtility
+from plone.i18n.normalizer.interfaces import IIDNormalizer
+from datetime import datetime
 
 import csv
 import six
@@ -15,46 +18,45 @@ class FormDataExportGet(Service):
     def __init__(self, context, request):
         super().__init__(context, request)
         self.form_fields_order = []
+        self.form_fields_labels = {}
         self.form_block = {}
 
         blocks = getattr(context, "blocks", {})
         if not blocks:
             return
-        for id, block in blocks.items():
+        for block in blocks.values():
             block_type = block.get("@type", "")
             if block_type == "form":
                 self.form_block = block
 
         if self.form_block:
             for field in self.form_block.get("subblocks", []):
+                if field["field_type"] == "static_text":
+                    continue
                 field_id = field["field_id"]
                 self.form_fields_order.append(field_id)
+                # can be customized.
+                # see https://github.com/collective/collective.volto.formsupport/pull/22
+                self.form_fields_labels[field_id] = self.form_block.get(
+                    field_id, field["label"]
+                )
 
-    def get_ordered_keys(self, record):
-        """
-        We need this method because we want to maintain the fields order set in the form.
-        The form can also change during time, and each record can have different fields stored in it.
-        """
-        record_order = record.attrs.get("fields_order", [])
-        if record_order:
-            return record_order
-        order = []
-        # first add the keys that are currently in the form
-        for k in self.form_fields_order:
-            if k in record.attrs:
-                order.append(k)
-        # finally append the keys stored in the record but that are not in the form (maybe the form changed during time)
-        for k in record.attrs.keys():
-            if k not in order and k not in SKIP_ATTRS:
-                order.append(k)
-        return order
+    def get_export_filename(self):
+        title = self.form_block.get("title", "")
+        filename = ""
+        if not title:
+            filename = "export-form"
+        else:
+            normalizer = getUtility(IIDNormalizer)
+            filename = normalizer.normalize(title)
+        now = datetime.now().strftime("%Y%m%dT%H%M")
+        return f"{filename}-{now}.csv"
 
     def render(self):
         self.check_permission()
-
         self.request.response.setHeader(
             "Content-Disposition",
-            'attachment; filename="{0}.csv"'.format(self.__name__),
+            f'attachment; filename="{self.get_export_filename()}"',
         )
         self.request.response.setHeader("Content-Type", "text/comma-separated-values")
         data = self.get_data()
@@ -63,33 +65,32 @@ def render(self):
         self.request.response.write(data)
 
     def get_data(self):
+        """
+        Return only data for fields in the current form setting,
+        ignore old fields stored in some record
+        """
         store = getMultiAdapter((self.context, self.request), IFormDataStore)
         sbuf = StringIO()
         fixed_columns = ["date"]
-        columns = []
+        columns = [self.form_fields_labels.get(k, k) for k in self.form_fields_order]
+        columns.extend(fixed_columns)
 
         rows = []
         for item in store.search():
-            data = {}
-            fields_labels = item.attrs.get("fields_labels", {})
-            for k in self.get_ordered_keys(item):
-                if k in SKIP_ATTRS:
-                    continue
+            data = []
+            for k in self.form_fields_order:
                 value = item.attrs.get(k, None)
-                label = fields_labels.get(k, k)
-                if label not in columns and label not in fixed_columns:
-                    columns.append(label)
-                data[label] = json_compatible(value)
+                data.append(json_compatible(value))
             for k in fixed_columns:
                 # add fixed columns values
                 value = item.attrs.get(k, None)
-                data[k] = json_compatible(value)
+                data.append(json_compatible(value))
             rows.append(data)
-        columns.extend(fixed_columns)
-        writer = csv.DictWriter(sbuf, fieldnames=columns, quoting=csv.QUOTE_ALL)
-        writer.writeheader()
-        for row in rows:
-            writer.writerow(row)
+        writer = csv.writer(sbuf, quoting=csv.QUOTE_ALL)
+        # header
+        writer.writerow(columns)
+        # data
+        writer.writerows(rows)
         res = sbuf.getvalue()
         sbuf.close()
         return res

src/collective/volto/formsupport/tests/test_send_action_form.py

@@ -553,7 +553,7 @@ def test_send_attachment_validate_size(
         with open(filename, "rb") as f:
             file_str = f.read()
         # increase file dimension
-        file_str = file_str * 100
+        file_str = file_str * 25
         response = self.submit_form(
             data={
                 "data": [
@@ -568,7 +568,7 @@ def test_send_attachment_validate_size(
         transaction.commit()
         self.assertEqual(response.status_code, 400)
         self.assertIn(
-            "Attachments too big. You uploaded 7.1 MB, but limit is 1 MB",
+            "Attachments too big. You uploaded 1.77 MB, but limit is 1 MB",
             response.json()["message"],
         )
         self.assertEqual(len(self.mailhost.messages), 0)

src/collective/volto/formsupport/tests/test_store_action_form.py

@@ -190,7 +190,7 @@ def test_store_data(self):
         response = self.export_csv()
         data = [*csv.reader(StringIO(response.text), delimiter=",")]
         self.assertEqual(len(data), 1)
-        self.assertEqual(data[0], ["date"])
+        self.assertEqual(data[0], ["Message", "Name", "date"])
 
     def test_export_csv(self):
         self.document.blocks = {