Skip to content

Commit

Permalink
#133 - added processing of file into StagingS3Create lambda
Browse files Browse the repository at this point in the history
  • Loading branch information
@mfriesen
mfriesen committed Jul 28, 2023
1 parent d3082d1 commit d6b2771
Show file tree
Hide file tree
Showing 12 changed files with 638 additions and 167 deletions.
13 changes: 13 additions & 0 deletions dynamodb-documents/src/main/java/com/formkiq/stacks/dynamodb/DocumentSearchService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.formkiq.aws.dynamodb.PaginationResults;
import com.formkiq.aws.dynamodb.model.DynamicDocumentItem;
import com.formkiq.aws.dynamodb.model.SearchQuery;
import com.formkiq.aws.dynamodb.model.SearchTagCriteria;

/**
*
Expand Down Expand Up @@ -58,4 +59,16 @@ PaginationResults<DynamicDocumentItem> findInFolder(String siteId, String indexK
*/
PaginationResults<DynamicDocumentItem> search(String siteId, SearchQuery search,
PaginationMapToken token, int maxresults);

/**
* Search for Document Ids.
*
* @param siteId Optional Grouping siteId
* @param criteria {@link SearchTagCriteria}
* @param token {@link PaginationMapToken}
* @param maxresults int
* @return {@link PaginationResults} {@link String}
*/
PaginationResults<String> searchForDocumentIds(String siteId, SearchTagCriteria criteria,
PaginationMapToken token, int maxresults);
}
163 changes: 108 additions & 55 deletions dynamodb-documents/src/main/java/com/formkiq/stacks/dynamodb/DocumentSearchServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,13 +117,14 @@ public DocumentSearchServiceImpl(final DynamoDbConnectionBuilder connection,

private QueryRequest createQueryRequest(final String index, final String expression,
final Map<String, AttributeValue> values, final PaginationMapToken token,
final int maxresults, final Boolean scanIndexForward) {
final int maxresults, final Boolean scanIndexForward, final String projectionExpression) {
Map<String, AttributeValue> startkey = new PaginationToAttributeValue().apply(token);

QueryRequest q = QueryRequest.builder().tableName(this.documentTableName).indexName(index)
.keyConditionExpression(expression).expressionAttributeValues(values)
.exclusiveStartKey(startkey).scanIndexForward(scanIndexForward)
.limit(Integer.valueOf(maxresults)).build();
.projectionExpression(projectionExpression).exclusiveStartKey(startkey)
.scanIndexForward(scanIndexForward).limit(Integer.valueOf(maxresults)).build();

return q;
}

Expand Down Expand Up @@ -251,11 +252,12 @@ private Map<String, Map<String, AttributeValue>> findDocumentsTags(final String
* @param value {@link String}
* @param token {@link PaginationMapToken}
* @param maxresults int
* @param projectionExpression {@link String}
* @return {@link PaginationResults}
*/
private PaginationResults<DynamicDocumentItem> findDocumentsTagStartWith(final String siteId,
final SearchQuery query, final String key, final String value, final PaginationMapToken token,
final int maxresults) {
final int maxresults, final String projectionExpression) {

String expression = GSI2_PK + " = :pk and begins_with(" + GSI2_SK + ", :sk)";

Expand All @@ -264,7 +266,8 @@ private PaginationResults<DynamicDocumentItem> findDocumentsTagStartWith(final S
AttributeValue.builder().s(createDatabaseKey(siteId, PREFIX_TAG + key)).build());
values.put(":sk", AttributeValue.builder().s(value).build());

QueryRequest q = createQueryRequest(GSI2, expression, values, token, maxresults, Boolean.FALSE);
QueryRequest q = createQueryRequest(GSI2, expression, values, token, maxresults, Boolean.FALSE,
projectionExpression);

return searchForDocuments(q, siteId, query);
}
Expand All @@ -278,19 +281,21 @@ private PaginationResults<DynamicDocumentItem> findDocumentsTagStartWith(final S
* @param value {@link String}
* @param token {@link PaginationMapToken}
* @param maxresults int
* @param projectionExpression {@link String}
* @return {@link PaginationResults}
*/
private PaginationResults<DynamicDocumentItem> findDocumentsWithTag(final String siteId,
final SearchQuery query, final String key, final String value, final PaginationMapToken token,
final int maxresults) {
final int maxresults, final String projectionExpression) {

String expression = GSI2_PK + " = :pk";

Map<String, AttributeValue> values = new HashMap<String, AttributeValue>();
values.put(":pk",
AttributeValue.builder().s(createDatabaseKey(siteId, PREFIX_TAG + key)).build());

QueryRequest q = createQueryRequest(GSI2, expression, values, token, maxresults, Boolean.FALSE);
QueryRequest q = createQueryRequest(GSI2, expression, values, token, maxresults, Boolean.FALSE,
projectionExpression);
return searchForDocuments(q, siteId, query);
}

Expand All @@ -303,18 +308,20 @@ private PaginationResults<DynamicDocumentItem> findDocumentsWithTag(final String
* @param value {@link String}
* @param token {@link PaginationMapToken}
* @param maxresults int
* @param projectionExpression {@link String}
* @return {@link PaginationResults}
*/
private PaginationResults<DynamicDocumentItem> findDocumentsWithTagAndValue(final String siteId,
final SearchQuery query, final String key, final String value, final PaginationMapToken token,
final int maxresults) {
final int maxresults, final String projectionExpression) {

String expression = GSI1_PK + " = :pk";

Map<String, AttributeValue> values = new HashMap<String, AttributeValue>();
values.put(":pk", AttributeValue.builder()
.s(createDatabaseKey(siteId, PREFIX_TAG + key + TAG_DELIMINATOR + value)).build());
QueryRequest q = createQueryRequest(GSI1, expression, values, token, maxresults, Boolean.FALSE);
QueryRequest q = createQueryRequest(GSI1, expression, values, token, maxresults, Boolean.FALSE,
projectionExpression);
return searchForDocuments(q, siteId, query);
}

Expand All @@ -326,16 +333,17 @@ private PaginationResults<DynamicDocumentItem> findDocumentsWithTagAndValue(fina
* @param key {@link String}
* @param eqOr {@link Collection} {@link String}
* @param maxresults int
* @param projectionExpression {@link String}
* @return {@link PaginationResults}
*/
private PaginationResults<DynamicDocumentItem> findDocumentsWithTagAndValues(final String siteId,
final SearchQuery query, final String key, final Collection<String> eqOr,
final int maxresults) {
final int maxresults, final String projectionExpression) {

List<DynamicDocumentItem> list = new ArrayList<>();
for (String eq : eqOr) {
PaginationResults<DynamicDocumentItem> result =
findDocumentsWithTagAndValue(siteId, query, key, eq, null, maxresults);
PaginationResults<DynamicDocumentItem> result = findDocumentsWithTagAndValue(siteId, query,
key, eq, null, maxresults, projectionExpression);
list.addAll(result.getResults());
}

Expand Down Expand Up @@ -443,7 +451,7 @@ public PaginationResults<DynamicDocumentItem> search(final String siteId, final

} else {
SearchTagCriteria search = query.tag();
results = searchByTag(siteId, query, search, token, maxresults);
results = searchByTag(siteId, query, search, token, maxresults, null);
}

return results;
Expand Down Expand Up @@ -485,7 +493,7 @@ private PaginationResults<DynamicDocumentItem> searchByMeta(final String siteId,
}

QueryRequest q =
createQueryRequest(null, expression, values, token, maxresults, Boolean.TRUE);
createQueryRequest(null, expression, values, token, maxresults, Boolean.TRUE, null);

result = searchForMetaDocuments(q, siteId);

Expand All @@ -498,7 +506,7 @@ private PaginationResults<DynamicDocumentItem> searchByMeta(final String siteId,

private PaginationResults<DynamicDocumentItem> searchByTag(final String siteId,
final SearchQuery query, final SearchTagCriteria csearch, final PaginationMapToken token,
final int maxresults) {
final int maxresults, final String projectionExpression) {

SearchTagCriteria search = csearch;

Expand Down Expand Up @@ -537,20 +545,38 @@ private PaginationResults<DynamicDocumentItem> searchByTag(final String siteId,
} else {

if (!Objects.notNull(search.eqOr()).isEmpty()) {
result = findDocumentsWithTagAndValues(siteId, query, key, search.eqOr(), maxresults);
result = findDocumentsWithTagAndValues(siteId, query, key, search.eqOr(), maxresults,
projectionExpression);
} else if (search.eq() != null) {
result = findDocumentsWithTagAndValue(siteId, query, key, search.eq(), token, maxresults);
result = findDocumentsWithTagAndValue(siteId, query, key, search.eq(), token, maxresults,
projectionExpression);
} else if (search.beginsWith() != null) {
result =
findDocumentsTagStartWith(siteId, query, key, search.beginsWith(), token, maxresults);
result = findDocumentsTagStartWith(siteId, query, key, search.beginsWith(), token,
maxresults, projectionExpression);
} else {
result = findDocumentsWithTag(siteId, query, key, null, token, maxresults);
result =
findDocumentsWithTag(siteId, query, key, null, token, maxresults, projectionExpression);
}
}

return result;
}

@Override
public PaginationResults<String> searchForDocumentIds(final String siteId,
final SearchTagCriteria criteria, final PaginationMapToken token, final int maxresults) {

SearchQuery query = new SearchQuery();
String projectionExpression = "documentId";
PaginationResults<DynamicDocumentItem> searchByTag =
searchByTag(siteId, query, criteria, token, maxresults, projectionExpression);

List<String> documentIds = searchByTag.getResults().stream().map(m -> m.getString("documentId"))
.collect(Collectors.toList());

return new PaginationResults<>(documentIds, searchByTag.getToken());
}

/**
* Search for Documents.
*
Expand All @@ -562,56 +588,44 @@ private PaginationResults<DynamicDocumentItem> searchByTag(final String siteId,
private PaginationResults<DynamicDocumentItem> searchForDocuments(final QueryRequest q,
final String siteId, final SearchQuery query) {

String projectionExpression = q.projectionExpression();
QueryResponse result = this.dbClient.query(q);

Map<String, DocumentTag> tags = new HashMap<>();
result.items().forEach(s -> {
Map<String, DocumentTag> tags = transformToDocumentTagMap(result);

if (s.containsKey("documentId")) {
String documentId = s.get("documentId").s();

String tagKey = s.containsKey("tagKey") ? s.get("tagKey").s() : null;
String tagValue = s.containsKey("tagValue") ? s.get("tagValue").s() : "";
PaginationResults<DynamicDocumentItem> ret = null;

DocumentTag tag = tags.containsKey(documentId) ? tags.get(documentId)
: new DocumentTag().setKey(tagKey).setValue(tagValue)
.setType(DocumentTagType.USERDEFINED);
List<String> documentIds = new ArrayList<>(tags.keySet());

if (tags.containsKey(documentId)) {
if (projectionExpression == null || !"documentId".equals(projectionExpression)) {

if (tag.getValues() == null) {
tag.setValues(new ArrayList<>());
tag.getValues().add(tag.getValue());
tag.setValue(null);
}
List<DocumentItem> list = this.docService.findDocuments(siteId, documentIds);

tag.getValues().add(tagValue);
List<DynamicDocumentItem> results =
list != null ? list.stream().map(l -> new DocumentItemToDynamicDocumentItem().apply(l))
.collect(Collectors.toList()) : Collections.emptyList();

} else {
tags.put(documentId, tag);
}
}
});
results.forEach(r -> {

List<String> documentIds = new ArrayList<>(tags.keySet());
DocumentTag tag = tags.get(r.getDocumentId());
r.put("matchedTag", new DocumentTagToDynamicDocumentTag().apply(tag));

List<DocumentItem> list = this.docService.findDocuments(siteId, documentIds);
if (!notNull(query.tags()).isEmpty()) {
updateToMatchedTags(query, r);
}
});

List<DynamicDocumentItem> results =
list != null ? list.stream().map(l -> new DocumentItemToDynamicDocumentItem().apply(l))
.collect(Collectors.toList()) : Collections.emptyList();
ret = new PaginationResults<>(results, new QueryResponseToPagination().apply(result));

results.forEach(r -> {
} else {

DocumentTag tag = tags.get(r.getDocumentId());
r.put("matchedTag", new DocumentTagToDynamicDocumentTag().apply(tag));
List<DynamicDocumentItem> results = documentIds.stream()
.map(d -> new DynamicDocumentItem(Map.of("documentId", d))).collect(Collectors.toList());

if (!notNull(query.tags()).isEmpty()) {
updateToMatchedTags(query, r);
}
});
ret = new PaginationResults<>(results, new QueryResponseToPagination().apply(result));
}

return new PaginationResults<>(results, new QueryResponseToPagination().apply(result));
return ret;
}

/**
Expand Down Expand Up @@ -651,6 +665,45 @@ private PaginationResults<DynamicDocumentItem> searchForMetaDocuments(final Quer
return new PaginationResults<>(results, new QueryResponseToPagination().apply(result));
}

/**
* Transform {@link QueryResponse} to {@link DocumentTag} {@link Map}.
*
* @param result {@link QueryResponse}
* @return {@link Map} {@link DocumentTag}
*/
private Map<String, DocumentTag> transformToDocumentTagMap(final QueryResponse result) {

Map<String, DocumentTag> tags = new HashMap<>();
result.items().forEach(s -> {

if (s.containsKey("documentId")) {
String documentId = s.get("documentId").s();

String tagKey = s.containsKey("tagKey") ? s.get("tagKey").s() : null;
String tagValue = s.containsKey("tagValue") ? s.get("tagValue").s() : "";

DocumentTag tag = tags.containsKey(documentId) ? tags.get(documentId)
: new DocumentTag().setKey(tagKey).setValue(tagValue)
.setType(DocumentTagType.USERDEFINED);

if (tags.containsKey(documentId)) {

if (tag.getValues() == null) {
tag.setValues(new ArrayList<>());
tag.getValues().add(tag.getValue());
tag.setValue(null);
}

tag.getValues().add(tagValue);

} else {
tags.put(documentId, tag);
}
}
});
return tags;
}

private void updateFolderMetaData(final SearchMetaCriteria meta) {

String folder = meta.folder();
Expand Down
9 changes: 9 additions & 0 deletions dynamodb-documents/src/main/java/com/formkiq/stacks/dynamodb/DocumentService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,15 @@ public interface DocumentService extends DocumentTagLoader {
*/
void addTags(String siteId, String documentId, Collection<DocumentTag> tags, String timeToLive);

/**
* Add Tags to {@link Collection} of Documents.
*
* @param siteId Optional Grouping siteId
* @param tags {@link Map} {@link Collection} {@link DocumentTag}
* @param timeToLive {@link String}
*/
void addTags(String siteId, Map<String, Collection<DocumentTag>> tags, String timeToLive);

/**
* Delete Document.
*
Expand Down
Loading

0 comments on commit d6b2771

Please sign in to comment.