Round trip fixes (#5)

* Small bug fixes in scripts * Getting closer to round tripping catalogs
david-waltermire · Jun 11, 2019 · e15bce8 · e15bce8
1 parent 3803f96
commit e15bce8
Show file tree

Hide file tree

Showing 6 changed files with 108 additions and 35 deletions.
diff --git a/.gitignore b/.gitignore
@@ -32,3 +32,5 @@ svrl
 !content/**/*.md
 !xml/**/*.md
 !json/**/*.md
+/roundtrip
+__pycache__
diff --git a/build/ci-cd/python/xmlComparison.py b/build/ci-cd/python/xmlComparison.py
@@ -76,5 +76,5 @@ def compareElements(e1, e2, xpath, position):
 root2 = doc2.getroot()
 
 if not compare(root1, root2):
-    print "compare failed"
+    print ("compare failed")
     sys.exit(1)
diff --git a/build/ci-cd/validate-round-trips.sh b/build/ci-cd/validate-round-trips.sh
@@ -107,7 +107,7 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
 
         # compare the XML files to see if there is data loss
         echo "${P_INFO}Checking XML->JSON->XML conversion for '${P_END}${file}${P_ERROR}'.${P_END}\n"
-        python ${OSCALDIR}/build/ci-cd/python/xmlComparison.py "$file" "$back_to_xml"
+        python ${OSCALDIR}/build/ci-cd/python/xmlComparison.py "$back_to_xml" "$file"
         cmd_exitcode=$?
         if [ $cmd_exitcode != 0 ]; then
             echo "${P_ERROR}XML round-trip comparison failed.${P_END}"

diff --git a/build/metaschema/json/md-oscal-converter.xsl b/build/metaschema/json/md-oscal-converter.xsl
@@ -9,7 +9,16 @@
     <xsl:output indent="yes"/>
 
     <xsl:param name="target-ns" as="xs:string?">http://csrc.nist.gov/ns/oscal/1.0</xsl:param>
-    <!-- TO DO: Test numbered and mixed lists; implement tables -->
+
+    <xsl:template name="xsl:initial-template" match="/">
+        <!--<xsl:copy-of select="$tag-replacements"/>-->
+        <!--<xsl:call-template name="parse">
+            <xsl:with-param name="str" select="string($examples)"/>
+        </xsl:call-template>-->
+        <xsl:for-each select="$line-example">
+            <xsl:apply-templates select="text()" mode="infer-inlines"/>
+        </xsl:for-each>
+    </xsl:template>
 
     <!-- Markdown pseudoparser in XSLT  -->
 
@@ -268,13 +277,13 @@
     </xsl:template>
 
     <xsl:template match="text()" mode="infer-inlines">
-        <xsl:variable name="markup" expand-text="true">
+        <xsl:variable name="markup">
             <xsl:apply-templates select="$tag-replacements/m:rules">
                 <xsl:with-param name="original" tunnel="yes" as="text()" select="."/>
             </xsl:apply-templates>
         </xsl:variable>
         <xsl:try select="parse-xml-fragment($markup)">
-            <xsl:catch expand-text="yes" select="."/>
+            <xsl:catch select="."/>
         </xsl:try>
     </xsl:template>
 
@@ -352,7 +361,9 @@
             <!-- Note that text contents are regex notation for matching so * must be \* -->
             <q>"<text/>"</q>
 
-            <img alt="!\[{{$text}}\]" src="\({{$text}}\)"/>
+            <img         alt="!\[{{$text}}\]" src="\({{$text}}\)"/>
+            <insert param-id="\{{{{$nws}}\}}"/>
+
             <a href="\[{{$text}}\]">\(<text/>\)</a>
             <code>`<text/>`</code>
             <strong>
@@ -371,7 +382,7 @@
         <xsl:text>&lt;</xsl:text>
         <xsl:value-of select="local-name()"/>
         <!-- coercing the order to ensure correct formation of regegex       -->
-        <xsl:apply-templates mode="#current" select="@href, @alt, @src"/>
+        <xsl:apply-templates mode="#current" select="@*"/>
         <xsl:text>&gt;</xsl:text>
 
         <xsl:apply-templates mode="#current" select="*"/>
@@ -389,10 +400,20 @@
         <xsl:value-of select="replace(., '\{\$text\}', '(.*)?')"/>
     </xsl:template>
 
+    <xsl:template match="@*[matches(., '\{\$nws\}')]" mode="write-match">
+        <!--<xsl:value-of select="."/>-->
+        <!--<xsl:value-of select="replace(., '\{\$nws\}', '(\S*)?')"/>-->
+        <xsl:value-of select="replace(., '\{\$nws\}', '\\s*(\\S+)?\\s*')"/>
+    </xsl:template>
+
     <xsl:template match="m:text" mode="write-replace">
         <xsl:text>$1</xsl:text>
     </xsl:template>
 
+    <xsl:template match="m:insert/@param-id" mode="write-replace">
+        <xsl:text> param-id="$1"</xsl:text>
+    </xsl:template>
+
     <xsl:template match="m:a/@href" mode="write-replace">
         <xsl:text> href="$2"</xsl:text>
         <!--<xsl:value-of select="replace(.,'\{\$insert\}','\$2')"/>-->
@@ -412,16 +433,24 @@
         <xsl:text>(.*?)</xsl:text>
     </xsl:template>
 
-    <!--<xsl:variable name="examples" xml:space="preserve">
+    <xsl:variable name="line-example" xml:space="preserve"> { insertion } </xsl:variable>
+
+     <xsl:variable name="examples" xml:space="preserve">
         <p>**Markdown**</p>
         <p>
 ## My test file!            
-            
+
+
+ { ac-4.4_prm_2 } 
+
+
 Extra long x
             y and z
 
-            
-            
+
+Here's a text with a parameter insertion: { insert }
+
+{insert-me}            
 
 And interesting.
 
@@ -465,6 +494,6 @@ And stuff.
         <p>`code` may occasionally turn up `in the middle`.</p>
         <p>Here's a ***really interesting*** markdown string.</p>
         <p>Some paragraphs might have [links elsewhere](https://link.org).</p>
-    </xsl:variable>-->
+    </xsl:variable>
 
 </xsl:stylesheet>
diff --git a/build/metaschema/json/produce-json-converter.xsl b/build/metaschema/json/produce-json-converter.xsl
@@ -53,7 +53,7 @@
             <!-- copying in templates from md-oscal-converter.xsl - everything but top-level parameters -->
             <xsl:comment> 00000000000000000000000000000000000000000000000000000000000000 </xsl:comment>
             <xsl:comment> Markdown converter</xsl:comment>
-            <xsl:copy-of select="document('md-oscal-converter.xsl')/xsl:*/(xsl:* except xsl:param)"/>
+            <xsl:copy-of select="document('md-oscal-converter.xsl')/xsl:*/(xsl:* except (xsl:param|xsl:variable[@name='examples']|xsl:template[@name='xsl:initial-template']))"/>
         </XSLT:stylesheet>
     </xsl:template>
 
@@ -83,10 +83,12 @@
             </XSLT:attribute>
         </XSLT:template>
     </xsl:template>
-        
+
     <xsl:template match="define-flag/* | define-field/* | define-assembly/*"/>
 
-
+    <!-- 'any' keyword not handled  -->
+    <xsl:template match="any" priority="10"/>
+
     <xsl:template match="model | choice" priority="2">
         <xsl:apply-templates/>
     </xsl:template>
@@ -108,21 +110,35 @@
                 <xsl:for-each select="@address">
                     <XSLT:attribute name="{.}" select="../@key"/>
                 </xsl:for-each>
-                <XSLT:apply-templates mode="as-attribute"/>
-                <xsl:apply-templates/>
-                <XSLT:apply-templates mode="json2xml" select="*[@key='{ if (@as='mixed') then $markdown-value-label else $string-value-label }']"/>
+                <XSLT:apply-templates select="*" mode="as-attribute"/>
+                <xsl:apply-templates select="." mode="field-text"/>
             </XSLT:element>
         </XSLT:template>
         <xsl:call-template name="drop-address"/>
     </xsl:template>
 
+    <xsl:template match="define-field" mode="field-text">
+        <XSLT:apply-templates mode="json2xml"/>
+    </xsl:template>
+
+    <xsl:template match="define-field[@as='mixed']" mode="field-text">
+        <XSLT:variable name="markup">
+            <XSLT:apply-templates mode="infer-inlines"/>
+        </XSLT:variable>
+        <XSLT:apply-templates mode="cast-ns" select="$markup"/>
+    </xsl:template>
+
+    <xsl:template priority="2" match="define-field[exists(flag)]" mode="field-text">
+        <XSLT:apply-templates mode="json2xml" select="string[@key=('{$string-value-label}','{$markdown-value-label}')]"/>
+    </xsl:template>
+
     <xsl:template match="define-assembly" expand-text="true">
         <xsl:variable name="assembly-match" as="xs:string">*[@key='{@name}']{ @group-as/(' | *[@key=''' || . || ''']/*') }</xsl:variable>
         <xsl:comment> 000 Handling assembly "{ @name }" 000 </xsl:comment>
         <XSLT:template match="{$assembly-match}" priority="2" mode="json2xml">
             <XSLT:element name="{@name}" namespace="{$target-namespace}">
                 <xsl:for-each select="@address">
-                    <XSLT:attribute name="{.}" select="../@key"/>
+                    <XSLT:attribute name="{.}" select="@key"/>
                 </xsl:for-each>
                 <XSLT:apply-templates mode="as-attribute"/>
                 <xsl:apply-templates/>
@@ -152,8 +168,8 @@
         <XSLT:variable name="json-xml" select="unparsed-text($json-file) ! json-to-xml(.)"/>
 
         <XSLT:template name="xsl:initial-template" match="/">
-            <XSLT:choose>
-                <XSLT:when test="exists($json-xml/map)">
+                <XSLT:choose>
+                <XSLT:when test="matches($json-file,'\S') and exists($json-xml/map)">
                     <XSLT:apply-templates select="$json-xml" mode="json2xml"/>
                 </XSLT:when>
                 <XSLT:otherwise>
@@ -172,17 +188,21 @@
             <XSLT:apply-templates mode="#current"/>
         </XSLT:template>
 
-        <XSLT:template match="array[@key='prose']/*" priority="5" mode="json2xml">
-            <XSLT:element name="p" namespace="{$target-namespace}">
-                <XSLT:variable name="text-contents" select="string-join(string,'&#xA;')"/>
-                <XSLT:call-template name="parse">
-                    <XSLT:with-param name="str" select="$text-contents"/>
-                </XSLT:call-template>
-            </XSLT:element>
+        <XSLT:template match="array[@key='prose']" priority="5" mode="json2xml">
+            <XSLT:variable name="text-contents" select="string-join(string,'&#xA;')"/>
+            <XSLT:call-template name="parse">
+                <XSLT:with-param name="str" select="$text-contents"/>
+            </XSLT:call-template>
         </XSLT:template>
 
-        <XSLT:template match="string[@key='{$markdown-value-label}']" mode="json2xml">
-            <XSLT:call-template name="parse"/>
+        <!-- drops all strings except those expecting the inline markup -->
+        <XSLT:template match="string" mode="handle-inlines"/>
+
+        <XSLT:template match="string[@key='{$markdown-value-label}']" mode="json2xml handle-inlines">
+            <XSLT:variable name="markup">
+                <XSLT:apply-templates mode="infer-inlines"/>
+            </XSLT:variable>
+            <XSLT:apply-templates mode="cast-ns" select="$markup"/>
         </XSLT:template>
 
         <XSLT:template match="string[@key='{$string-value-label}']" mode="json2xml">
@@ -191,9 +211,9 @@
 
         <XSLT:template mode="as-attribute" match="*"/>
 
-        <XSLT:template mode="as-attribute" match="map">
+        <!--<XSLT:template mode="as-attribute" match="map">
             <XSLT:apply-templates mode="#current"/>
-        </XSLT:template>
+        </XSLT:template>-->
 
         <XSLT:template mode="as-attribute"  match="string[@key='id']" priority="0.4">
             <XSLT:attribute name="id">

diff --git a/build/metaschema/xml/produce-xml-converter.xsl b/build/metaschema/xml/produce-xml-converter.xsl
@@ -142,7 +142,7 @@
     <!-- no special provision made for addressing by @id that happens at the other end -->
     <xsl:template match="define-assembly">
         <XSLT:template match="{@name}" mode="xml2json">
-            <map  key="{@name}">
+            <map key="{@name}">
                 <xsl:for-each select="@address">
                     <xsl:attribute name="key">{@<xsl:value-of select="."/>}</xsl:attribute>
                 </xsl:for-each>
@@ -154,11 +154,14 @@
         </XSLT:template>
     </xsl:template>
 
+    <!-- dropping a flag that has been used as an address -->
+    <xsl:template match="flag[@name=../@address]"/>
+
     <xsl:template match="flag">
         <!-- no datatyping support yet -->
         <XSLT:apply-templates mode="as-string" select="@{@name}"/>
     </xsl:template>
-        
+
     <xsl:template match="model">
         <xsl:apply-templates/>
     </xsl:template>
@@ -251,13 +254,23 @@
 
         <!--<XSLT:key name="parameters" match="param" use="@id"/>-->
 
+        <XSLT:template name="conditional-lf">
+            <XSLT:variable name="predecessor"
+                select="preceding-sibling::p | preceding-sibling::ul | preceding-sibling::ol | preceding-sibling::table | preceding-sibling::pre"/>
+            <XSLT:if test="exists($predecessor)">
+                <string/>
+            </XSLT:if>
+        </XSLT:template>
+
         <XSLT:template mode="md" match="p | link | part/*">
+            <XSLT:call-template name="conditional-lf"/>
             <string>
                 <XSLT:apply-templates mode="md"/>
             </string>
         </XSLT:template>
 
         <XSLT:template mode="md" match="h1 | h2 | h3 | h4 | h5 | h6">
+            <XSLT:call-template name="conditional-lf"/>
             <string>
                 <XSLT:apply-templates select="." mode="mark"/>
                 <XSLT:apply-templates mode="md"/>
@@ -272,6 +285,7 @@
         <XSLT:template mode="mark" match="h6">###### </XSLT:template>
 
         <XSLT:template mode="md" match="table">
+            <XSLT:call-template name="conditional-lf"/>
             <XSLT:apply-templates  select="*"  mode="md"/>
         </XSLT:template>
 
@@ -297,6 +311,7 @@
         </XSLT:template>
 
         <XSLT:template mode="md" priority="1" match="pre">
+            <XSLT:call-template name="conditional-lf"/>
             <string>```</string>
             <string>
                 <XSLT:apply-templates mode="md"/>
@@ -305,7 +320,7 @@
         </XSLT:template>
 
         <XSLT:template mode="md" priority="1" match="ul | ol">
-            <string/>
+            <XSLT:call-template name="conditional-lf"/>
             <XSLT:apply-templates mode="md"/>
             <string/>
         </XSLT:template>
@@ -361,6 +376,13 @@
             <XSLT:text>"</XSLT:text>
         </XSLT:template>
 
+        <!-- <insert param-id="ac-1_prm_1"/> -->
+        <XSLT:template mode="md" match="insert">
+            <XSLT:text>{ </XSLT:text>
+            <XSLT:value-of select="@param-id"/>
+            <XSLT:text> }</XSLT:text>
+        </XSLT:template>
+
         <XSLT:key name="element-by-id" match="*[exists(@id)]" use="@id"/>
 
         <XSLT:template mode="md" match="a">