Merge pull request #116 from icgc-argo/rc/2.6.0

Rc/2.6.0
icgc-argo · Jan 18, 2021 · c4499b5 · c4499b5
2 parents e3127f7 + 29b080c
commit c4499b5
Show file tree

Hide file tree

Showing 18 changed files with 487 additions and 57 deletions.
diff --git a/pom.xml b/pom.xml
@@ -10,7 +10,7 @@
     </parent>
     <groupId>org.icgc_argo</groupId>
     <artifactId>workflow-search</artifactId>
-    <version>2.5.0</version>
+    <version>2.6.0</version>
     <name>workflow-search</name>
     <description>Demo project for Spring Boot</description>
 
@@ -124,13 +124,6 @@
             <version>${elasticsearch.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-
         <!-- Graphql java dependencies -->
         <dependency>
             <groupId>com.graphql-java</groupId>

diff --git a/src/main/java/org/icgc_argo/workflow/search/graphql/GraphQLProvider.java b/src/main/java/org/icgc_argo/workflow/search/graphql/GraphQLProvider.java
@@ -110,7 +110,9 @@ private RuntimeWiring buildWiring() {
     return RuntimeWiring.newRuntimeWiring()
         .scalar(ExtendedScalars.Json)
         .type(newTypeWiring("Query").dataFetcher("runs", runDataFetchers.getRunsDataFetcher()))
+        .type(newTypeWiring("Query").dataFetcher("aggregateRuns", runDataFetchers.getRunsDataFetcher()))
         .type(newTypeWiring("Query").dataFetcher("tasks", taskDataFetchers.getTasksDataFetcher()))
+        .type(newTypeWiring("Query").dataFetcher("aggregateTasks", taskDataFetchers.getAggregateTasksDataFetcher()))
         .type(
             newTypeWiring("Run").dataFetcher("tasks", taskDataFetchers.getNestedTaskDataFetcher()))
         .type(newTypeWiring("Task").dataFetcher("run", runDataFetchers.getNestedRunDataFetcher()))

diff --git a/src/main/java/org/icgc_argo/workflow/search/graphql/RunDataFetchers.java b/src/main/java/org/icgc_argo/workflow/search/graphql/RunDataFetchers.java
@@ -18,18 +18,21 @@
 
 package org.icgc_argo.workflow.search.graphql;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import graphql.schema.DataFetcher;
 import java.util.List;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
-import org.icgc_argo.workflow.search.model.graphql.Run;
-import org.icgc_argo.workflow.search.model.graphql.Task;
+import org.icgc_argo.workflow.search.model.graphql.*;
 import org.icgc_argo.workflow.search.service.graphql.RunService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import static java.util.stream.Collectors.toUnmodifiableList;
+import static org.icgc_argo.workflow.search.util.JacksonUtils.convertValue;
+
 @Slf4j
 @Component
 public class RunDataFetchers {
@@ -43,18 +46,40 @@ public RunDataFetchers(RunService runService) {
   }
 
   @SuppressWarnings("unchecked")
-  public DataFetcher<List<Run>> getRunsDataFetcher() {
+  public DataFetcher<SearchResult<Run>> getRunsDataFetcher() {
     return environment -> {
       val args = environment.getArguments();
 
       val filter = ImmutableMap.<String, Object>builder();
       val page = ImmutableMap.<String, Integer>builder();
+      val sorts = ImmutableList.<Sort>builder();
 
       if (args != null) {
         if (args.get("filter") != null) filter.putAll((Map<String, Object>) args.get("filter"));
         if (args.get("page") != null) page.putAll((Map<String, Integer>) args.get("page"));
+        if (args.get("sorts") != null) {
+          val rawSorts = (List<Object>) args.get("sorts");
+          sorts.addAll(
+                  rawSorts.stream()
+                          .map(sort -> convertValue(sort, Sort.class))
+                          .collect(toUnmodifiableList()));
+        }
+      }
+      return runService.searchRuns(filter.build(), page.build(), sorts.build());
+    };
+  }
+
+  @SuppressWarnings("unchecked")
+  public DataFetcher<AggregationResult> getAggregateAnalysesDataFetcher() {
+    return environment -> {
+      val args = environment.getArguments();
+
+      val filter = ImmutableMap.<String, Object>builder();
+
+      if (args != null) {
+        if (args.get("filter") != null) filter.putAll((Map<String, Object>) args.get("filter"));
       }
-      return runService.getRuns(filter.build(), page.build());
+      return runService.aggregateRuns(filter.build());
     };
   }
 

diff --git a/src/main/java/org/icgc_argo/workflow/search/graphql/TaskDataFetchers.java b/src/main/java/org/icgc_argo/workflow/search/graphql/TaskDataFetchers.java
@@ -18,19 +18,22 @@
 
 package org.icgc_argo.workflow.search.graphql;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import graphql.schema.DataFetcher;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
-import org.icgc_argo.workflow.search.model.graphql.Run;
-import org.icgc_argo.workflow.search.model.graphql.Task;
+import org.icgc_argo.workflow.search.model.graphql.*;
 import org.icgc_argo.workflow.search.service.graphql.TaskService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
 import java.util.Map;
 
+import static java.util.stream.Collectors.toUnmodifiableList;
+import static org.icgc_argo.workflow.search.util.JacksonUtils.convertValue;
+
 @Slf4j
 @Component
 public class TaskDataFetchers {
@@ -44,19 +47,40 @@ public TaskDataFetchers(TaskService taskService) {
   }
 
   @SuppressWarnings("unchecked")
-  public DataFetcher<List<Task>> getTasksDataFetcher() {
+  public DataFetcher<SearchResult<Task>> getTasksDataFetcher() {
     return environment -> {
       val args = environment.getArguments();
 
       val filter = ImmutableMap.<String, Object>builder();
       val page = ImmutableMap.<String, Integer>builder();
+      val sorts = ImmutableList.<Sort>builder();
 
       if (args != null) {
         if (args.get("filter") != null) filter.putAll((Map<String, Object>) args.get("filter"));
         if (args.get("page") != null) page.putAll((Map<String, Integer>) args.get("page"));
+        if (args.get("sorts") != null) {
+          val rawSorts = (List<Object>) args.get("sorts");
+          sorts.addAll(
+                  rawSorts.stream()
+                          .map(sort -> convertValue(sort, Sort.class))
+                          .collect(toUnmodifiableList()));
+        }
       }
+      return taskService.searchRuns(filter.build(), page.build(), sorts.build());
+    };
+  }
+
+  @SuppressWarnings("unchecked")
+  public DataFetcher<AggregationResult> getAggregateTasksDataFetcher() {
+    return environment -> {
+      val args = environment.getArguments();
+
+      val filter = ImmutableMap.<String, Object>builder();
 
-      return taskService.getTasks(null, filter.build(), page.build());
+      if (args != null) {
+        if (args.get("filter") != null) filter.putAll((Map<String, Object>) args.get("filter"));
+      }
+      return taskService.aggregateTasks(filter.build());
     };
   }
 

diff --git a/src/main/java/org/icgc_argo/workflow/search/model/EsDefaults.java b/src/main/java/org/icgc_argo/workflow/search/model/EsDefaults.java
@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright (c) 2020 The Ontario Institute for Cancer Research. All rights reserved
+ *
+ * This program and the accompanying materials are made available under the terms of the GNU Affero General Public License v3.0.
+ * You should have received a copy of the GNU Affero General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *
+ */
+
+package org.icgc_argo.workflow.search.model;
+
+import lombok.NoArgsConstructor;
+
+import static lombok.AccessLevel.PRIVATE;
+
+@NoArgsConstructor(access = PRIVATE)
+public class EsDefaults {
+  // Default values from ES pagination:
+  // https://www.elastic.co/guide/en/elasticsearch/reference/7.x/paginate-search-results.html
+  public static final Integer ES_PAGE_DEFAULT_SIZE = 10;
+  public static final Integer ES_PAGE_DEFAULT_FROM = 0;
+}
diff --git a/src/main/java/org/icgc_argo/workflow/search/model/SearchFields.java b/src/main/java/org/icgc_argo/workflow/search/model/SearchFields.java
@@ -38,4 +38,6 @@ public class SearchFields {
   public static final String TAG = "tag";
   public static final String WORK_DIR = "workDir";
   public static final String ANALYSIS_ID = "analysisId";
+  public static final String CPUS = "cpus";
+  public static final String MEMORY = "memory";
 }
diff --git a/src/main/java/org/icgc_argo/workflow/search/model/graphql/AggregationResult.java b/src/main/java/org/icgc_argo/workflow/search/model/graphql/AggregationResult.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2020 The Ontario Institute for Cancer Research. All rights reserved
+ *
+ * This program and the accompanying materials are made available under the terms of the GNU Affero General Public License v3.0.
+ * You should have received a copy of the GNU Affero General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.icgc_argo.workflow.search.model.graphql;
+
+import lombok.Value;
+
+@Value
+public class AggregationResult {
+  Long totalHits;
+}
diff --git a/src/main/java/org/icgc_argo/workflow/search/model/graphql/SearchResult.java b/src/main/java/org/icgc_argo/workflow/search/model/graphql/SearchResult.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2020 The Ontario Institute for Cancer Research. All rights reserved
+ *
+ * This program and the accompanying materials are made available under the terms of the GNU Affero General Public License v3.0.
+ * You should have received a copy of the GNU Affero General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.icgc_argo.workflow.search.model.graphql;
+
+import lombok.Value;
+
+import java.util.List;
+
+@Value
+public class SearchResult<T> {
+  List<T> content;
+  Info info;
+
+  public SearchResult(List<T> content, Boolean hasNextFrom, Long totalHits) {
+    this.content = content;
+    this.info = new Info(hasNextFrom, totalHits, content.size());
+  }
+
+  @Value
+  public static class Info {
+    Boolean hasNextFrom;
+    Long totalHits;
+    Integer contentCount;
+  }
+}
diff --git a/src/main/java/org/icgc_argo/workflow/search/model/graphql/Sort.java b/src/main/java/org/icgc_argo/workflow/search/model/graphql/Sort.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2020 The Ontario Institute for Cancer Research. All rights reserved
+ *
+ * This program and the accompanying materials are made available under the terms of the GNU Affero General Public License v3.0.
+ * You should have received a copy of the GNU Affero General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.icgc_argo.workflow.search.model.graphql;
+
+import lombok.Data;
+
+@Data
+public class Sort {
+  String fieldName;
+  String order;
+}
diff --git a/src/main/java/org/icgc_argo/workflow/search/repository/RunRepository.java b/src/main/java/org/icgc_argo/workflow/search/repository/RunRepository.java
@@ -29,8 +29,10 @@
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.index.query.*;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
 import org.icgc_argo.workflow.search.config.ElasticsearchProperties;
+import org.icgc_argo.workflow.search.model.graphql.Sort;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -39,7 +41,10 @@
 import java.util.function.Function;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.search.sort.SortOrder.DESC;
 import static org.icgc_argo.workflow.search.model.SearchFields.*;
+import static org.icgc_argo.workflow.search.util.ElasticsearchQueryUtils.queryFromArgs;
+import static org.icgc_argo.workflow.search.util.ElasticsearchQueryUtils.sortsToEsSortBuilders;
 
 @Slf4j
 @Component
@@ -52,6 +57,9 @@ public class RunRepository {
           "parameters.tumour_aln_analysis_id");
   private static final Map<String, Function<String, AbstractQueryBuilder<?>>> QUERY_RESOLVER =
       argumentPathMap();
+
+  private static final Map<String, FieldSortBuilder> SORT_BUILDER_RESOLVER = sortPathMap();
+
   private final RestHighLevelClient client;
   private final String workflowIndex;
 
@@ -63,19 +71,6 @@ public RunRepository(
     this.workflowIndex = elasticsearchProperties.getWorkflowIndex();
   }
 
-  /**
-   * For each argument, find its query producer function and apply the argument value ANDing it in a
-   * bool query
-   *
-   * @param args Argument Map from GraphQL
-   * @return Elasticsearch Bool Query containing ANDed (MUSTed) term queries
-   */
-  private static BoolQueryBuilder queryFromArgs(Map<String, Object> args) {
-    val bool = QueryBuilders.boolQuery();
-    args.forEach((key, value) -> bool.must(QUERY_RESOLVER.get(key).apply(value.toString())));
-    return bool;
-  }
-
   private static Map<String, Function<String, AbstractQueryBuilder<?>>> argumentPathMap() {
     return ImmutableMap.<String, Function<String, AbstractQueryBuilder<?>>>builder()
         .put(RUN_ID, value -> new TermQueryBuilder("runId", value))
@@ -100,12 +95,34 @@ private static Map<String, Function<String, AbstractQueryBuilder<?>>> argumentPa
         .build();
   }
 
+  private static Map<String, FieldSortBuilder> sortPathMap() {
+    return ImmutableMap.<String, FieldSortBuilder>builder()
+                   .put(RUN_ID, SortBuilders.fieldSort("runId"))
+                   .put(SESSION_ID, SortBuilders.fieldSort("sessionId"))
+                   .put(STATE, SortBuilders.fieldSort("state"))
+                   .put(START_TIME, SortBuilders.fieldSort("startTime"))
+                   .put(COMPLETE_TIME, SortBuilders.fieldSort("completeTime"))
+                   .put(REPOSITORY, SortBuilders.fieldSort("repository"))
+                   .build();
+  }
+
   public SearchResponse getRuns(Map<String, Object> filter, Map<String, Integer> page) {
+    return getRuns(filter, page, List.of());
+  }
+
+  public SearchResponse getRuns(Map<String, Object> filter, Map<String, Integer> page, List<Sort> sorts) {
     final AbstractQueryBuilder<?> query =
-        (filter == null || filter.size() == 0) ? matchAllQuery() : queryFromArgs(filter);
+        (filter == null || filter.size() == 0) ? matchAllQuery() : queryFromArgs(QUERY_RESOLVER, filter);
 
     val searchSourceBuilder = new SearchSourceBuilder();
-    searchSourceBuilder.sort(START_TIME, SortOrder.DESC);
+
+    if (sorts.isEmpty()) {
+      searchSourceBuilder.sort(SORT_BUILDER_RESOLVER.get(START_TIME).order(DESC));
+    } else {
+      val sortBuilders = sortsToEsSortBuilders(SORT_BUILDER_RESOLVER, sorts);
+      sortBuilders.forEach(searchSourceBuilder::sort);
+    }
+
     searchSourceBuilder.query(query);
 
     if (page != null && page.size() != 0) {