Merge pull request #92 from icgc-argo/RC/2.2.0

RC/2.2.0

pom.xml

@@ -10,7 +10,7 @@
     </parent>
     <groupId>org.icgc_argo</groupId>
     <artifactId>workflow-search</artifactId>
-    <version>2.1.0</version>
+    <version>2.2.0</version>
     <name>workflow-search</name>
     <description>Demo project for Spring Boot</description>
 

src/main/java/org/icgc_argo/workflow/search/index/model/TaskDocument.java

@@ -75,15 +75,39 @@ public class TaskDocument {
   /** Task filesystem working directory */
   @NonNull private String workdir;
 
-  /** Task cpu usage */
+  /** The cpus number request for the task execution */
   private Integer cpus;
 
-  /** Task memory usage */
-  private Integer memory;
+  /** The memory request for the task execution */
+  private Long memory;
 
-  /** Task duration (ms) */
-  private Integer duration;
+  /** Time elapsed to complete since the submission */
+  private Long duration;
 
-  /** Task real execution time (ms) */
-  private Integer realtime;
+  /** Task execution time i.e. delta between completion and start timestamp */
+  private Long realtime;
+
+  /** Real memory (resident set) size of the process. Equivalent to `ps -o rss` */
+  private Long rss;
+
+  /** Peak of real memory. This data is read from field `VmHWM` in `/proc/$pid/status file` */
+  private Long peakRss;
+
+  /** Virtual memory size of the process. Equivalent to `ps -o vsize` */
+  private Long vmem;
+
+  /** Peak of virtual memory. This data is read from field `VmPeak` in `/proc/$pid/status file` */
+  private Long peakVmem;
+
+  /**
+   * Number of bytes the process directly read from disk. This data is read from file
+   * `/proc/$pid/io`
+   */
+  private Long readBytes;
+
+  /**
+   * Number of bytes the process originally dirtied in the page-cache (assuming they will go to disk
+   * later). This data is read from file `/proc/$pid/io`
+   */
+  private Long writeBytes;
 }

src/main/java/org/icgc_argo/workflow/search/index/model/WorkflowDocument.java

@@ -64,7 +64,7 @@ public class WorkflowDocument {
   private String errorReport;
 
   /** Workflow duration */
-  private Integer duration;
+  private Long duration;
 
   /** The command line that was executed */
   @NonNull private String commandLine;

src/main/java/org/icgc_argo/workflow/search/model/graphql/Run.java

@@ -54,7 +54,7 @@ public class Run {
 
   private String errorReport;
 
-  private String duration;
+  private Long duration;
 
   private String commandLine;
 

src/main/java/org/icgc_argo/workflow/search/model/graphql/Task.java

@@ -61,13 +61,41 @@ public class Task {
 
   private String workdir;
 
+  /** The cpus number request for the task execution */
   private Integer cpus;
 
-  private Integer memory;
+  /** The memory request for the task execution */
+  private Long memory;
 
-  private Integer duration;
+  /** Time elapsed to complete since the submission */
+  private Long duration;
 
-  private Integer realtime;
+  /** Task execution time i.e. delta between completion and start timestamp */
+  private Long realtime;
+
+  /** Real memory (resident set) size of the process. Equivalent to `ps -o rss` */
+  private Long rss;
+
+  /** Peak of real memory. This data is read from field `VmHWM` in `/proc/$pid/status file` */
+  private Long peakRss;
+
+  /** Virtual memory size of the process. Equivalent to `ps -o vsize` */
+  private Long vmem;
+
+  /** Peak of virtual memory. This data is read from field `VmPeak` in `/proc/$pid/status file` */
+  private Long peakVmem;
+
+  /**
+   * Number of bytes the process directly read from disk. This data is read from file
+   * `/proc/$pid/io`
+   */
+  private Long readBytes;
+
+  /**
+   * Number of bytes the process originally dirtied in the page-cache (assuming they will go to disk
+   * later). This data is read from file `/proc/$pid/io`
+   */
+  private Long writeBytes;
 
   @SneakyThrows
   public static Task parse(@NonNull Map<String, Object> sourceMap) {

src/main/java/org/icgc_argo/workflow/search/model/wes/RunLog.java

@@ -21,10 +21,11 @@
 import com.fasterxml.jackson.databind.PropertyNamingStrategy;
 import com.fasterxml.jackson.databind.annotation.JsonNaming;
 import io.swagger.annotations.ApiModel;
-import java.util.List;
-import javax.validation.Valid;
 import lombok.*;
 
+import javax.validation.Valid;
+import java.util.List;
+
 /** Log and other info */
 @ApiModel(description = "Log and other info")
 @Data
@@ -52,5 +53,5 @@ public class RunLog {
 
   private Boolean success;
 
-  private Integer duration;
+  private Long duration;
 }

src/main/java/org/icgc_argo/workflow/search/model/wes/TaskLog.java

@@ -21,10 +21,11 @@
 import com.fasterxml.jackson.databind.PropertyNamingStrategy;
 import com.fasterxml.jackson.databind.annotation.JsonNaming;
 import io.swagger.annotations.ApiModel;
-import java.util.List;
-import javax.validation.Valid;
 import lombok.*;
 
+import javax.validation.Valid;
+import java.util.List;
+
 /** Log and other info */
 @ApiModel(description = "Log and other info")
 @Data
@@ -66,11 +67,39 @@ public class TaskLog {
 
   private String workdir;
 
+  /** The cpus number request for the task execution */
   private Integer cpus;
 
-  private Integer memory;
+  /** The memory request for the task execution */
+  private Long memory;
+
+  /** Time elapsed to complete since the submission */
+  private Long duration;
+
+  /** Task execution time i.e. delta between completion and start timestamp */
+  private Long realtime;
+
+  /** Real memory (resident set) size of the process. Equivalent to `ps -o rss` */
+  private Long rss;
+
+  /** Peak of real memory. This data is read from field `VmHWM` in `/proc/$pid/status file` */
+  private Long peakRss;
+
+  /** Virtual memory size of the process. Equivalent to `ps -o vsize` */
+  private Long vmem;
+
+  /** Peak of virtual memory. This data is read from field `VmPeak` in `/proc/$pid/status file` */
+  private Long peakVmem;
 
-  private Integer duration;
+  /**
+   * Number of bytes the process directly read from disk. This data is read from file
+   * `/proc/$pid/io`
+   */
+  private Long readBytes;
 
-  private Integer realtime;
+  /**
+   * Number of bytes the process originally dirtied in the page-cache (assuming they will go to disk
+   * later). This data is read from file `/proc/$pid/io`
+   */
+  private Long writeBytes;
 }

src/main/java/org/icgc_argo/workflow/search/util/Converter.java

@@ -18,17 +18,9 @@
 
 package org.icgc_argo.workflow.search.util;
 
-import static org.icgc_argo.workflow.search.model.SearchFields.RUN_ID;
-import static org.icgc_argo.workflow.search.model.SearchFields.STATE;
-import static org.icgc_argo.workflow.search.model.wes.State.fromValue;
-
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import java.time.Instant;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
 import lombok.NonNull;
 import lombok.SneakyThrows;
 import lombok.experimental.UtilityClass;
@@ -39,6 +31,15 @@
 import org.icgc_argo.workflow.search.index.model.WorkflowDocument;
 import org.icgc_argo.workflow.search.model.wes.*;
 
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.icgc_argo.workflow.search.model.SearchFields.RUN_ID;
+import static org.icgc_argo.workflow.search.model.SearchFields.STATE;
+import static org.icgc_argo.workflow.search.model.wes.State.fromValue;
+
 @Slf4j
 @UtilityClass
 public class Converter {
@@ -73,7 +74,12 @@ public static TaskLog taskDocumentToLog(@NonNull TaskDocument task) {
         .memory(task.getMemory())
         .duration(task.getDuration())
         .realtime(task.getRealtime())
-        // todo ticket #24 workflow-relay
+        .rss(task.getRss())
+        .peakRss(task.getPeakRss())
+        .vmem(task.getVmem())
+        .peakVmem(task.getPeakVmem())
+        .readBytes(task.getReadBytes())
+        .writeBytes(task.getWriteBytes())
         .stderr("")
         .stdout("")
         .build();

src/main/resources/schema.graphqls

@@ -1,4 +1,5 @@
 scalar JSON
+scalar Long
 
 type Run @key(fields: "runId") {
     runId: ID!
@@ -11,7 +12,7 @@ type Run @key(fields: "runId") {
     success: Boolean
     exitStatus: Int
     errorReport: String
-    duration: String
+    duration: Long
     commandLine: String
     engineParameters: EngineParameters
     tasks(taskId: String, state: String, tag: String): [Task]
@@ -42,9 +43,15 @@ type Task {
     script: String
     workdir: String
     cpus: Int
-    memory: Int
-    duration: Int
-    realtime: Int
+    memory: Long
+    duration: Long
+    realtime: Long
+    rss: Long
+    peakRss: Long
+    vmem: Long
+    peakVmem: Long
+    readBytes: Long
+    writeBytes: Long
     run: Run
 }
 

src/test/java/org/icgc_argo/workflow/search/service/ConverterTest.java

@@ -54,9 +54,17 @@ public class ConverterTest {
   private static final Integer TASK_ATTEMPT = 1;
   private static final String TASK_WORKDIR = "/this/is/dir";
   private static final Integer TASK_CPUS = 4;
-  private static final Integer TASK_MEM = 1024;
-  private static final Integer TASK_DURATION = 2000;
-  private static final Integer TASK_REALTIME = 1098;
+  private static final Long TASK_MEM = 1024L;
+  private static final Long TASK_DURATION = 2000L;
+  private static final Long TASK_REALTIME = 1098L;
+  private static final Long TASK_RSS = 1234L;
+  private static final Long TASK_PEAK_RSS = 4567L;
+  private static final Long TASK_VMEM = 8901L;
+  private static final Long TASK_PEAK_VMEM = 2345L;
+  private static final Long TASK_READ_BYTES = 6789L;
+  private static final Long TASK_WRITE_BYTES = 10123L;
+
+
 
   @Test
   public void TestConvertSourceMapToRunStatus() {
@@ -90,6 +98,12 @@ public void testTaskDocumentToLog() {
             .memory(TASK_MEM)
             .duration(TASK_DURATION)
             .realtime(TASK_REALTIME)
+            .rss(TASK_RSS)
+            .peakRss(TASK_PEAK_RSS)
+            .vmem(TASK_VMEM)
+            .peakVmem(TASK_PEAK_VMEM)
+            .readBytes(TASK_READ_BYTES)
+            .writeBytes(TASK_WRITE_BYTES)
             .build();
 
     val log = Converter.taskDocumentToLog(taskDocument);
@@ -113,6 +127,12 @@ public void testTaskDocumentToLog() {
     assertEquals(log.getMemory(), taskDocument.getMemory());
     assertEquals(log.getDuration(), taskDocument.getDuration());
     assertEquals(log.getRealtime(), taskDocument.getRealtime());
+    assertEquals(log.getRss(), taskDocument.getRss());
+    assertEquals(log.getPeakRss(), taskDocument.getPeakRss());
+    assertEquals(log.getVmem(), taskDocument.getVmem());
+    assertEquals(log.getPeakVmem(), taskDocument.getPeakVmem());
+    assertEquals(log.getReadBytes(), taskDocument.getReadBytes());
+    assertEquals(log.getWriteBytes(), taskDocument.getWriteBytes());
   }
 
   @Test
@@ -167,7 +187,7 @@ private WorkflowDocument buildWorkflowDoc() {
         .parameters(params)
         .engineParameters(engineParams)
         .success(true)
-        .duration(1000)
+        .duration(1000L)
         .build();
   }
 }