Fixed UnicodeDecodeError when vocabulary value is a non-ascii string.

This could happen in selection and multi selection fields.

CHANGES.txt

@@ -4,6 +4,10 @@ Change History
 1.8.4 (unreleased)
 ------------------
 
+- Fixed UnicodeDecodeError when vocabulary value is a non-ascii string.
+  This could happen in selection and multi selection fields.
+  [maurits]
+
 - change captcha verification message for #186
   [tkimnguyen]
 

Products/PloneFormGen/content/fields.py

@@ -779,7 +779,10 @@ def htmlValue(self, REQUEST):
         vocabulary = self.fgField.Vocabulary(self)
         v = vocabulary.getValue(vu) or vu
 
-        return cgi.escape(v.encode(charset))
+        # v might be unicode or string.  We need string.
+        if isinstance(v, unicode):
+            v = v.encode(charset)
+        return cgi.escape(v)
 
 
 registerATCT(FGSelectionField, PROJECTNAME)
@@ -888,11 +891,14 @@ def htmlValue(self, REQUEST):
                 # so decode the key before lookup
                 ku = k.decode(charset)
                 v = vocabulary.getValue(ku) or ku
+                # v might be unicode or string.  We need string.
+                if isinstance(v, unicode):
+                    v = v.encode(charset)
                 result.append(v)
 
-        value = u', '.join(result)
+        value = ', '.join(result)
 
-        return cgi.escape(value.encode(charset))
+        return cgi.escape(value)
 
 
 registerATCT(FGMultiSelectField, PROJECTNAME)

Products/PloneFormGen/tests/testFunctions.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 #
 # Integration tests. See other test modules for specific components.
 #
@@ -199,7 +200,9 @@ def testHtmlValueSelectionField(self):
         """ Test field htmlValue method of selection field """
 
         self.ff1.invokeFactory('FormSelectionField', 'fsf')
-        self.ff1.fsf.fgVocabulary = ('1|one', '2|two', '3|three',)
+        # Let's mix ascii and non-ascii strings and unicode.
+        self.ff1.fsf.fgVocabulary = (
+            '1|one', '2|two', '3|three', u'4|fo\xfcr', '5|f\xc3\xacve')
 
         # first test inside the vocabulary
         request = self.fakeRequest(fsf = '2')
@@ -217,12 +220,23 @@ def testHtmlValueSelectionField(self):
         val = self.ff1['fsf'].htmlValue(request)
         self.assertEqual( val, '&')
 
+        # Now test unicode
+        request = self.fakeRequest(fsf = '4')
+        val = self.ff1['fsf'].htmlValue(request)
+        self.assertEqual(val, u'fo\xfcr'.encode('utf-8'))
+
+        # And test a non-ascii string
+        request = self.fakeRequest(fsf = '5')
+        val = self.ff1['fsf'].htmlValue(request)
+        self.assertEqual(val, 'f\xc3\xacve')
 
     def testHtmlValueMultiSelectionField(self):
         """ Test field htmlValue method of multi-selection field """
 
         self.ff1.invokeFactory('FormMultiSelectionField', 'fsf')
-        self.ff1.fsf.fgVocabulary = ('1|one', '2|two', '3|three',)
+        # Let's mix ascii and non-ascii strings and unicode.
+        self.ff1.fsf.fgVocabulary = (
+            '1|one', '2|two', '3|three', u'4|fo\xfcr', '5|f\xc3\xacve')
 
         # first test inside the vocabulary
         request = self.fakeRequest(fsf = ['2', '3', ''])
@@ -245,6 +259,11 @@ def testHtmlValueMultiSelectionField(self):
         val = self.ff1['fsf'].htmlValue(request)
         self.assertEqual( val, 'one, 7')
 
+        # Now test ascii and non-ascii strings and unicode.
+        request = self.fakeRequest(fsf = ['1', '4', '5'])
+        val = self.ff1['fsf'].htmlValue(request)
+        self.assertEqual(val, 'one, fo\xc3\xbcr, f\xc3\xacve')
+        self.assertEqual(val.decode('utf-8'), u'one, fo\xfcr, f\xecve')
 
     def testHtmlValueDateField(self):
         """ Test field htmlValue method of date field """