Hotfix to disable STALLED in CI as an error (#2523)

This **_hotfix_** to the last CI updates disables the STALLED feature by
not flagging it as an error until a more succinct algorithm is
determined (i.e. the STALLED state will be at par with RUNNING).

A second minor bug is in the event that no error log is produced when a
Case fails, the error of the fail is still reported to the general user
in GitHub. (Jenkins enabled users can always see the full comprehensive
state of system by following the links at the bottom of the PR).

ci/Jenkinsfile

@@ -19,6 +19,7 @@ pipeline {
     stages { // This initial stage is used to get the Machine name from the GitHub labels on the PR
              // which is used to designate the Nodes in the Jenkins Controler by the agent label
              // Each Jenknis Node is connected to said machine via an JAVA agent via an ssh tunnel
+             // no op 2
 
         stage('Get Machine') {
             agent { label 'built-in' }
@@ -111,7 +112,7 @@ pipeline {
                                                     try {
                                                         sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
                                                         gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
-                                                        sh(script:  """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`${error_logs_message}\n\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
+                                                        sh(script:  """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
                                                     } catch (Exception error_comment) {
                                                         echo "Failed to comment on PR: ${error_comment.getMessage()}"
                                                     }
@@ -215,6 +216,7 @@ pipeline {
                                         STATUS = 'Failed'
                                         try {
                                             sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "CI-${Machine}-Running" --add-label "CI-${Machine}-${STATUS}" """, returnStatus: true)
+                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${Case} **FAILED** on ${Machine}\nin\\`${HOME}/RUNTESTS/${pslot}\\`" """)
                                         } catch (Exception e) {
                                             echo "Failed to update label from Running to ${STATUS}: ${e.getMessage()}"
                                         }

ci/scripts/utils/rocotostat.py

@@ -95,7 +95,11 @@ def rocoto_statcount():
         error_return = rocoto_status['UNKNOWN']
         rocoto_state = 'UNKNOWN'
     elif rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0:
-        error_return = -3
+    #
+    #  TODO for now a STALLED state will be just a warning as it can
+    #  produce a false negative if there is a timestamp on a file dependency.
+    #
+    #   error_return = -3
         rocoto_state = 'STALLED'
     else:
         rocoto_state = 'RUNNING'