Repairing more bugs in character handling in Markdown and plain text,…

… with unit testing: usnistgov#452

build/metaschema/json/md-oscal-converter.xsl

@@ -334,8 +334,8 @@
             <replace match="&amp;"  >&amp;amp;</replace>
             <replace match="&lt;"   >&amp;lt;</replace>
             <!-- next, explicit escape sequences -->
-            <replace match="\\&#34;">&amp;quot;</replace>
-            <replace match="\\&#39;">&amp;apos;</replace>
+            <replace match="\\&quot;">&amp;quot;</replace>
+            <!--<replace match="\\&#39;">&amp;apos;</replace>-->
             <replace match="\\\*"   >&amp;#x2A;</replace>
             <replace match="\\`"    >&amp;#x60;</replace>
             <replace match="\\~"    >&amp;#x7E;</replace>

build/metaschema/unit-testing/char-handling/charstrings-test-json-converter.xsl

@@ -364,11 +364,11 @@
          <replace match="&lt;">&amp;lt;</replace>
          <!-- next, explicit escape sequences -->
          <replace match="\\&#34;">&amp;quot;</replace>
-         <replace match="\\'">&amp;apos;</replace>
-         <replace match="\\\*">&amp;#2A;</replace>
-         <replace match="\\`">&amp;#60;</replace>
-         <replace match="\\~">&amp;#7E;</replace>
-         <replace match="\\^">&amp;#5E;</replace>
+         <!--<replace match="\\&#39;">&amp;apos;</replace>-->
+         <replace match="\\\*">&amp;#x2A;</replace>
+         <replace match="\\`">&amp;#x60;</replace>
+         <replace match="\\~">&amp;#x7E;</replace>
+         <replace match="\\^">&amp;#x5E;</replace>
          <!-- then, replacements based on $tag-specification -->
          <xsl:for-each select="$tag-specification/*">
             <xsl:variable name="match-expr">

build/metaschema/unit-testing/char-handling/charstrings-test-xml-converter.xsl

@@ -16,8 +16,8 @@
    <xsl:param name="json-indent" as="xs:string">no</xsl:param>
    <!-- Pass $diagnostic as 'rough' for first pass, 'rectified' for second pass -->
    <xsl:param name="diagnostic" as="xs:string">no</xsl:param>
-   <xsl:template match="text()" mode="md #default">
-      <xsl:value-of select="replace(., '([`~\^\*''&#34;])', '\\$1')"/>
+   <xsl:template match="text()" mode="md">
+      <xsl:value-of select="replace(., '([`~\^\*&#34;])', '\\$1')"/>
    </xsl:template>
    <xsl:variable name="write-options" as="map(*)" expand-text="true">
       <xsl:map>
@@ -246,7 +246,7 @@
    </xsl:template>
    <xsl:template match="charstring" mode="xml2json">
       <string key="charstring">
-         <xsl:apply-templates mode="md"/>
+         <xsl:apply-templates mode="#current"/>
       </string>
    </xsl:template>
    <xsl:template match="markupline" mode="xml2json">

build/metaschema/unit-testing/char-handling/charstrings-test_okay-PASS-back.xml

@@ -1,9 +1,9 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <examples xmlns="http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-dateTime">
-   <charstring>Here\'s a string including punctuation \' \" \ . $</charstring>
-   <charstring>Here\'s a string with an ampersand &amp;</charstring>
+   <charstring>Here's a string including punctuation ' " \ . $</charstring>
+   <charstring>Here's a string with an ampersand &amp;</charstring>
    <charstring>Less-than and greater-than, as &lt;delimiters&gt; ...</charstring>
-   <charstring>The star aka asterisk \* may be magical and maybe also the tick \`</charstring>
+   <charstring>The star aka asterisk * may be magical and maybe also the tick `</charstring>
    <markupline>Also, occasionally we have <a href="w3.org">links</a> and <em>other markup</em>.</markupline>
-   <markupline>And *sometimes*, we have \*very confusing stuff\* as well.</markupline>
+   <markupline>And <em>sometimes</em>, we have *very confusing stuff* as well.</markupline>
 </examples>

build/metaschema/unit-testing/char-handling/charstrings-test_okay-PASS.json

@@ -1,10 +1,10 @@
 
   { "examples" : 
     { "character-strings" : 
-      [ "Here\\'s a string including punctuation \\' \\\" \\ . $",
-        "Here\\'s a string with an ampersand &",
+      [ "Here's a string including punctuation ' \" \\ . $",
+        "Here's a string with an ampersand &",
         "Less-than and greater-than, as <delimiters> ...",
-        "The star aka asterisk \\* may be magical and maybe also the tick \\`" ],
+        "The star aka asterisk * may be magical and maybe also the tick `" ],
       "markup-lines" : 
       [ "Also, occasionally we have [links](w3.org) and *other markup*.",
         "And *sometimes*, we have \\*very confusing stuff\\* as well." ] } }

build/metaschema/unit-testing/char-handling/charstrings-test_okay-PASS.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<examples xmlns="http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-dateTime"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-dateTime file:/C:/Users/wap1/Documents/OSCAL/build/metaschema/unit-testing/char-encoding/charstrings-test-schema.xsd">
+<examples xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-dateTime charstrings-test-schema.xsd"
+ xmlns="http://csrc.nist.gov/ns/metaschema/unit-test/datatypes-dateTime">
   <charstring>Here's a string including punctuation ' " \ . $</charstring>
   <charstring>Here's a string with an ampersand &amp;</charstring>
   <charstring>Less-than and greater-than, as &lt;delimiters> ...</charstring>

build/metaschema/xml/produce-xml-converter.xsl

@@ -63,7 +63,7 @@
             <!-- Note that Saxon's JSON serializer already escapes characters per
                  https://www.w3.org/XML/Group/qtspecs/specifications/xpath-functions-31/html/Overview.html#func-xml-to-json 
             -->
-            <XSLT:template match="text()" mode="md #default">
+            <XSLT:template match="text()" mode="md">
                 <!-- Escapes go here       -->
                 <!-- prefixes ` ~ ^ * with char E0000 from Unicode PUA -->
                 <!--<XSLT:value-of select="replace(., '([`~\^\*''&quot;])', '&#xE0000;$1')"/>-->
@@ -154,15 +154,24 @@
         </XSLT:template>
     </xsl:template>
 
-    <xsl:template match="define-field[empty(flag)]" priority="2">
+    <xsl:template match="define-field[@as-type=('markup-line','markup-multiline')][empty(flag)]" priority="3">
         <XSLT:template match="{@name}" mode="xml2json">
             <string key="{@name}">
-                <!-- When the input has no markup, no markdown  will be produced. -->
+                <!-- Mode 'md' will escape characters for Markdown. -->
                 <XSLT:apply-templates mode="md"/>
             </string>
         </XSLT:template>
     </xsl:template>
-
+
+    <xsl:template match="define-field[empty(flag)]" priority="2">
+        <XSLT:template match="{@name}" mode="xml2json">
+            <string key="{@name}">
+                <!-- Not escaping any characters this time. -->
+                <XSLT:apply-templates mode="#current"/>
+            </string>
+        </XSLT:template>
+    </xsl:template>
+
     <xsl:template match="define-field">
         <XSLT:template match="{@name}" mode="xml2json">
             <XSLT:variable name="text-key">