Issue 468 364 478 integration

.circleci/config.yml

@@ -256,6 +256,7 @@ jobs: # a collection of
       - install-maven-dependencies
       - install-jq
       - install-prettyjson
+      - install-xmllint
       - run:
           name: Generate OSCAL converters
           command: |

build/ci-cd/config/content

@@ -1,4 +1,5 @@
 # path to source|format of source|model of source|format(s) to convert to
+src/content/ssp-example/ssp-example.json|json|ssp|xml
 src/content/fedramp.gov/xml/*catalog.xml|xml|catalog|json
 src/content/fedramp.gov/xml/*profile.xml|xml|profile|json
 src/content/nist.gov/SP800-53/rev4/xml/*catalog.xml|xml|catalog|json

build/ci-cd/config/metaschema

@@ -3,5 +3,3 @@ src/metaschema/oscal_catalog_metaschema.xml|xml,json|xml,json|xml,json
 src/metaschema/oscal_profile_metaschema.xml|xml,json|xml,json|xml,json
 src/metaschema/oscal_component_metaschema.xml|xml,json|xml,json|xml,json
 src/metaschema/oscal_ssp_metaschema.xml|xml,json|xml,json|xml,json
-
-

build/ci-cd/config/release-content

@@ -5,6 +5,7 @@ content/fedramp.gov/**/*.yaml
 content/nist.gov/**/*.xml
 content/nist.gov/**/*.json
 content/nist.gov/**/*.yaml
+content/ssp-example/ssp-example.json
 xml/convert/*.xsl
 xml/schema/*.xsd
 json/convert/*.xsl

build/ci-cd/copy-and-convert-content.sh

@@ -57,18 +57,18 @@ done
 
 OTHER_ARGS=$@ # save the remaining args
 
-echo ""
-echo "${P_INFO}Copying and Converting Content${P_END}"
-echo "${P_INFO}==============================${P_END}"
+echo -e ""
+echo -e "${P_INFO}Copying and Converting Content${P_END}"
+echo -e "${P_INFO}==============================${P_END}"
 
 if [ "$VERBOSE" = "true" ]; then
-  echo "${P_INFO}Using working directory:${P_END} ${WORKING_DIR}"
+  echo -e "${P_INFO}Using working directory:${P_END} ${WORKING_DIR}"
 fi
 
 # check for perl
 result=$(which perl 2>&1)
 if [ $? -ne 0 ]; then
-  echo "${P_ERROR}Perl is not installed. Perl is needed by this script.${P_END}"
+  echo -e "${P_ERROR}Perl is not installed. Perl is needed by this script.${P_END}"
   exit 1
 fi
 
@@ -92,16 +92,15 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
       file_relative=$(realpath --relative-to="${OSCALDIR}" "$file")
       dest="$WORKING_DIR/${file/$OSCALDIR\/src\//}"
       dest_dir=${dest%/*} # remove filename
-      dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest")
-
       mkdir -p "$dest_dir"
+      dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest")
       result=$(cp "$file" "$dest" 2>&1)
       cmd_exitcode=$?
       if [ $cmd_exitcode -ne 0 ]; then
-        echo "${P_ERROR}Unable to copy '${P_END}${file_relative}${P_ERROR}' to '${P_END}${dest_relative}${P_ERROR}'.${P_END}"
-        echo "${P_ERROR}${result}${P_END}"
+        echo -e "${P_ERROR}Unable to copy '${P_END}${file_relative}${P_ERROR}' to '${P_END}${dest_relative}${P_ERROR}'.${P_END}"
+        echo -e "${P_ERROR}${result}${P_END}"
       else
-        echo "${P_OK}Copied '${P_END}${file_relative}${P_OK}' to '${P_END}${dest_relative}${P_OK}'.${P_END}"
+        echo -e "${P_OK}Copied '${P_END}${file_relative}${P_OK}' to '${P_END}${dest_relative}${P_OK}'.${P_END}"
       fi
 
       IFS=","
@@ -110,32 +109,48 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
         newpath="${newpath/\/$format\///$altformat/}" # change path from old to new format dir
         newpath="${newpath%.*}" # strip extension
 
-        dest="$WORKING_DIR/${newpath}-min.${altformat}"
-        dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest")
         converter="$WORKING_DIR/$altformat/convert/oscal_${model}_${format}-to-${altformat}-converter.xsl"
         converter_relative=$(realpath --relative-to="${WORKING_DIR}" "$converter")
 
-        if [ "$VERBOSE" = "true" ]; then
-          echo "${P_INFO}Generating ${altformat^^} file '${P_END}${dest_relative}${P_INFO}' from '${P_END}${file_relative}${P_INFO}' using converter '${P_END}${converter_relative}${P_INFO}'.${P_END}"
-        fi
-        result=$(xsl_transform "$converter" "$file" "$dest" 2>&1)
+
+        case $format in
+        xml)
+          dest="$WORKING_DIR/${newpath}-min.${altformat}"
+          dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest")
+          if [ "$VERBOSE" = "true" ]; then
+            echo -e "${P_INFO}Generating ${altformat^^} file '${P_END}${dest_relative}${P_INFO}' from '${P_END}${file_relative}${P_INFO}' using converter '${P_END}${converter_relative}${P_INFO}'.${P_END}"
+          fi
+          result=$(xsl_transform "$converter" "$file" "$dest" 2>&1)
+          ;;
+        json)
+          dest="$WORKING_DIR/${newpath}.${altformat}"
+          dest_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest")
+          if [ "$VERBOSE" = "true" ]; then
+            echo -e "${P_INFO}Generating ${altformat^^} file '${P_END}${dest_relative}${P_INFO}' from '${P_END}${file_relative}${P_INFO}' using converter '${P_END}${converter_relative}${P_INFO}'.${P_END}"
+          fi
+          result=$(xsl_transform "$converter" "" "$dest" "-it" "json-file=${file}" 2>&1)
+          ;;
+        *)
+          echo -e "${P_WARN}Conversion from '${format} to '${altformat^^}' is unsupported for '${P_END}${file_relative}${P_OK}'.${P_END}"
+          continue;
+        esac
         cmd_exitcode=$?
         if [ $cmd_exitcode -ne 0 ]; then
-          echo "${P_ERROR}Content conversion to ${altformat^^} failed for '${P_END}${file_relative}${P_ERROR}' using converter '${P_END}${converter_relative}${P_ERROR}'.${P_END}"
-          echo "${P_ERROR}${result}${P_END}"
+          echo -e "${P_ERROR}Content conversion to ${altformat^^} failed for '${P_END}${file_relative}${P_ERROR}' using converter '${P_END}${converter_relative}${P_ERROR}'.${P_END}"
+          echo -e "${P_ERROR}${result}${P_END}"
           exitcode=1
           continue
         else
           if [ "$VERBOSE" = "true" ]; then
-            echo "${P_OK}Content conversion to ${altformat^^} succeeded for '${P_END}${file_relative}${P_OK}'.${P_END}"
+            echo -e "${P_OK}Content conversion to ${altformat^^} succeeded for '${P_END}${file_relative}${P_OK}'.${P_END}"
           fi
         fi
 
         # Format specific post-processing
         case $altformat in
         json)
           # Remove extra slashes
-          perl -pi -e 's,\\/,/,g' "${dest}"
+          # perl -pi -e 's,\\/,/,g' "${dest}"
           # translate OSCAL mime types
           perl -pi -e 's,(application/oscal\.[a-z]+\+)xml\",\1json\",g' "${dest}"
           # relative content paths
@@ -152,21 +167,21 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
           result=$(validate_json "$schema" "$dest")
           cmd_exitcode=$?
           if [ $cmd_exitcode -ne 0 ]; then
-            echo "${P_ERROR}JSON Schema validation failed for '${P_END}${dest_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
-            echo "${P_ERROR}${result}${P_END}"
+            echo -e "${P_ERROR}JSON Schema validation failed for '${P_END}${dest_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
+            echo -e "${P_ERROR}${result}${P_END}"
             exitcode=1
             continue
           else
-            echo "${P_OK}JSON Schema validation passed for '${P_END}${dest_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}"
+            echo -e "${P_OK}JSON Schema validation passed for '${P_END}${dest_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}"
           fi
 
           # produce pretty JSON
           dest_pretty="$WORKING_DIR/${newpath}.${altformat}"
           dest_pretty_relative=$(realpath --relative-to="${WORKING_DIR}" "$dest_pretty")
           result=$(jq . "$dest" > "$dest_pretty" 2>&1)
           if [ $? -ne 0 ]; then
-            echo "${P_ERROR}Unable to execute jq on '${P_END}${dest_pretty_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
-            echo "${P_ERROR}${result}${P_END}"
+            echo -e "${P_ERROR}Unable to execute jq on '${P_END}${dest_pretty_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
+            echo -e "${P_ERROR}${result}${P_END}"
             exitcode=1
             continue
           fi
@@ -176,12 +191,12 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
           result=$(validate_json "$schema" "$dest_pretty" 2>&1)
           cmd_exitcode=$?
           if [ $cmd_exitcode -ne 0 ]; then
-            echo "${P_ERROR}JSON Schema validation failed for '${P_END}${dest_pretty_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
-            echo "${P_ERROR}${result}${P_END}"
+            echo -e "${P_ERROR}JSON Schema validation failed for '${P_END}${dest_pretty_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
+            echo -e "${P_ERROR}${result}${P_END}"
             exitcode=1
             continue
           else
-            echo "${P_OK}JSON Schema validation passed for '${P_END}${dest_pretty_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}"
+            echo -e "${P_OK}JSON Schema validation passed for '${P_END}${dest_pretty_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}"
           fi
 
           # produce yaml
@@ -198,15 +213,15 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do
           result=$(xmllint --noout --schema "$schema" "$dest" 2>&1)
           cmd_exitcode=$?
           if [ $cmd_exitcode -ne 0 ]; then
-            echo "${P_ERROR}XML Schema validation failed for '${P_END}${dest_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
-            echo "${P_ERROR}${result}${P_END}"
+            echo -e "${P_ERROR}XML Schema validation failed for '${P_END}${dest_relative}${P_ERROR}' using schema '${P_END}${schema_relative}${P_ERROR}'.${P_END}"
+            echo -e "${P_ERROR}${result}${P_END}"
             exitcode=1
           else
-            echo "${P_OK}XML Schema validation passed for '${P_END}${dest_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}"
+            echo -e "${P_OK}XML Schema validation passed for '${P_END}${dest_relative}${P_OK}' using schema '${P_END}${schema_relative}${P_OK}'.${P_END}"
           fi
           ;;
         *)
-          echo "${P_WARN}Post processing of '${altformat^^}' is unsupported for '${P_END}${dest_relative}${P_OK}'.${P_END}"
+          echo -e "${P_WARN}Post processing of '${altformat^^}' is unsupported for '${P_END}${dest_relative}${P_OK}'.${P_END}"
           continue;
         esac
       done

build/ci-cd/generate-content-converters.sh

@@ -13,7 +13,7 @@ HELP=false
 
 usage() {                                      # Function: Print a help message.
   cat << EOF
-Usage: $0 [options]
+Usage: $0 [options] [metaschema paths]
 
 -h, --help                        Display help
 -w DIR, --working-dir DIR         Generate artifacts in DIR
@@ -54,81 +54,94 @@ done
 
 OTHER_ARGS=$@ # save the remaining args
 
-echo ""
-echo "${P_INFO}Generating XML <-> JSON Content Converters${P_END}"
-echo "${P_INFO}==========================================${P_END}"
+echo -e ""
+echo -e "${P_INFO}Generating XML <-> JSON Content Converters${P_END}"
+echo -e "${P_INFO}==========================================${P_END}"
 
 if [ "$VERBOSE" = "true" ]; then
-  echo "${P_INFO}Using working directory:${P_END} ${WORKING_DIR}"
+  echo -e "${P_INFO}Using working directory:${P_END} ${WORKING_DIR}"
 fi
 
-exitcode=0
-shopt -s nullglob
-shopt -s globstar
-while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; do
-  shopt -s extglob
-  [[ "$path" =~ ^[[:space:]]*# ]] && continue
-  # remove leading space
-  path="${path##+([[:space:]])}"
-  # remove trailing space
-  gen_docs="${gen_docs%%+([[:space:]])}"
-  shopt -u extglob
-
-  ([ -z "$path" ] || [ -z "$gen_converter" ]) && continue;
-
-  files_to_process="$OSCALDIR"/"$path"
-
-  IFS= # disable word splitting
-  for metaschema in $files_to_process
-  do
-    filename=$(basename -- "$metaschema")
-    extension="${filename##*.}"
-    filename="${filename%.*}"
-    model="${filename/_metaschema/}"
-    metaschema_relative=$(realpath --relative-to="${OSCALDIR}" "$metaschema")
-
-    #split on commas
-    IFS=, read -a formats <<< "$gen_converter"
-    for target_format in "${formats[@]}"; do
-      if [ -z "$target_format" ]; then
-        # skip blanks
-        continue;
-      fi
-
-      # Run the XSL template for the format
-      case $target_format in
-      xml)
-        source_format="json"
-        ;;
-      json)
-        source_format="xml"
-        ;;
-      *)
-        echo "${P_WARN}Generating converter from '${source_format^^}' to '${target_format^^}' is unsupported for '${P_END}${metaschema_relative}${P_WARN}'.${P_END}"
-        continue;
-        ;;
-      esac
-
-      converter="$WORKING_DIR/${target_format}/convert/${model}_${source_format}-to-${target_format}-converter.xsl"
-      converter_relative=$(realpath --relative-to="${WORKING_DIR}" "$converter")
-
-      if [ "$VERBOSE" = "true" ]; then
-        echo "${P_INFO}Generating ${source_format^^} to ${target_format^^} converter for '${P_END}${metaschema_relative}${P_INFO}' as '${P_END}${converter_relative}${P_INFO}'.${P_END}"
-      fi
-
-      result=$(xsl_transform "$OSCALDIR/build/metaschema/$source_format/produce-${source_format}-converter.xsl" "$metaschema" "$converter" 2>&1)
-      cmd_exitcode=$?
-      if [ $cmd_exitcode -ne 0 ]; then
-        echo "${P_ERROR}Generating ${source_format^^} to ${target_format^^} converter failed for '${P_END}${metaschema_relative}${P_ERROR}'.${P_END}"
-        echo "${P_ERROR}${result}${P_END}"
-        exitcode=1
-      else
-        echo "${P_OK}Generating ${source_format^^} to ${target_format^^} converter passed for '${P_END}${metaschema_relative}${P_OK}'.${P_END}"
-      fi
+declare -a paths
+declare -a formats
+if [ "$#" -ne 0 ]; then
+  paths=("$@")
+  for i in "${!paths[@]}"; do
+    formats[$i]="xml,json"
+  done
+else
+  while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; do
+    [[ "$path" =~ ^[[:space:]]*# ]] && continue
+    # remove leading space
+    path="${path##+([[:space:]])}"
+
+    ([ -z "$path" ] || [ -z "$gen_converter" ]) && continue;
+
+    path_absolute="$OSCALDIR"/"$path"
+
+    IFS_OLD=$IFS
+    IFS= # disable word splitting
+    for metaschema in $path_absolute
+    do
+      paths+=("$metaschema")
+      formats+=("$gen_converter")
     done
+    IFS=$IFS_OLD
+  done < "$OSCALDIR/build/ci-cd/config/metaschema"
+fi
+
+exitcode=0
+for i in "${!paths[@]}"; do
+  metaschema="${paths[$i]}"
+  gen_converter="${formats[$i]}"
+
+  filename=$(basename -- "$metaschema")
+  extension="${filename##*.}"
+  filename="${filename%.*}"
+  model="${filename/_metaschema/}"
+  metaschema_relative=$(realpath --relative-to="${OSCALDIR}" "$metaschema")
+
+  #split on commas
+  IFS_OLD=$IFS
+  IFS=, #read -a formats <<< "$gen_schema"
+  for target_format in ${gen_converter}; do
+    if [ -z "$target_format" ]; then
+      # skip blanks
+      continue;
+    fi
+
+    # Run the XSL template for the format
+    case $target_format in
+    xml)
+      source_format="json"
+      ;;
+    json)
+      source_format="xml"
+      ;;
+    *)
+      echo -e "${P_WARN}Generating converter from '${source_format^^}' to '${target_format^^}' is unsupported for '${P_END}${metaschema_relative}${P_WARN}'.${P_END}"
+      continue;
+      ;;
+    esac
+
+    converter="$WORKING_DIR/${target_format}/convert/${model}_${source_format}-to-${target_format}-converter.xsl"
+    converter_relative=$(realpath --relative-to="${WORKING_DIR}" "$converter")
+
+    if [ "$VERBOSE" = "true" ]; then
+      echo -e "${P_INFO}Generating ${source_format^^} to ${target_format^^} converter for '${P_END}${metaschema_relative}${P_INFO}' as '${P_END}${converter_relative}${P_INFO}'.${P_END}"
+    fi
+
+    result=$(xsl_transform "$OSCALDIR/build/metaschema/$source_format/produce-${source_format}-converter.xsl" "$metaschema" "$converter" 2>&1)
+    cmd_exitcode=$?
+    if [ $cmd_exitcode -ne 0 ]; then
+      echo -e "${P_ERROR}Generating ${source_format^^} to ${target_format^^} converter failed for '${P_END}${metaschema_relative}${P_ERROR}'.${P_END}"
+      echo -e "${P_ERROR}${result}${P_END}"
+      exitcode=1
+    else
+      echo -e "${P_OK}Generating ${source_format^^} to ${target_format^^} converter passed for '${P_END}${metaschema_relative}${P_OK}'.${P_END}"
+    fi
   done
-done < $OSCALDIR/build/ci-cd/config/metaschema
-shopt -u nullglob
-shopt -u globstar
+  IFS=$IFS_OLD
+done
 
 exit $exitcode