Skip to content

Commit

Permalink
Changed determining Document Version to S3 metadata rather than etag (#…
Browse files Browse the repository at this point in the history 
…94)

Co-authored-by: Mike Friesen - FormKiQ <mike@formkiq.com>
  • Loading branch information
@mfriesen @formkiqMike
mfriesen and formkiqMike committed Dec 21, 2022
1 parent 981ebd4 commit b980ffd
Show file tree
Hide file tree
Showing 11 changed files with 80 additions and 30 deletions.
1 change: 1 addition & 0 deletions aws-s3/config/checkstyle/import-control.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

<allow pkg="software.amazon.awssdk.auth.credentials" />
<allow pkg="software.amazon.awssdk.auth.signer" />
<allow pkg="software.amazon.awssdk.awscore" />
<allow pkg="software.amazon.awssdk.core" />
<allow pkg="software.amazon.awssdk.http" />
<allow pkg="software.amazon.awssdk.regions" />
Expand Down
35 changes: 30 additions & 5 deletions aws-s3/src/main/java/com/formkiq/aws/s3/S3Service.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import java.util.Collection;
import java.util.Map;
import java.util.Optional;
import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
Expand All @@ -58,6 +59,7 @@
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsRequest.Builder;
import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
import software.amazon.awssdk.services.s3.model.MetadataDirective;
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.PutBucketVersioningRequest;
Expand Down Expand Up @@ -149,17 +151,23 @@ public S3Service(final S3ConnectionBuilder s3connectionBuilder) {
* @param destinationBucket {@link String}
* @param destinationKey {@link String}
* @param contentType {@link String}
* @param metadata {@link Map}
* @return {@link CopyObjectResponse}
*/
public CopyObjectResponse copyObject(final String sourcebucket, final String sourcekey,
final String destinationBucket, final String destinationKey, final String contentType) {
final String destinationBucket, final String destinationKey, final String contentType,
final Map<String, String> metadata) {
CopyObjectRequest.Builder req = CopyObjectRequest.builder().sourceBucket(sourcebucket)
.sourceKey(sourcekey).destinationBucket(destinationBucket).destinationKey(destinationKey);

if (contentType != null) {
req = req.contentType(contentType);
}

if (metadata != null) {
req = req.metadata(metadata).metadataDirective(MetadataDirective.REPLACE);
}

return this.s3Client.copyObject(req.build());
}

Expand Down Expand Up @@ -428,17 +436,34 @@ public URL presignPostUrl(final String bucket, final String key, final Duration
* @param bucket {@link String}
* @param key {@link String}
* @param duration {@link Duration}
* @param contentLength {@link Optional} {@link Long}
* @param metadata {@link Map}
* @return {@link URL}
*/
public URL presignPutUrl(final String bucket, final String key, final Duration duration) {
public URL presignPutUrl(final String bucket, final String key, final Duration duration,
final Optional<Long> contentLength, final Map<String, String> metadata) {

try (S3Presigner signer = this.builder.buildPresigner()) {

PutObjectRequest putObjectRequest =
PutObjectRequest.builder().bucket(bucket).key(key).build();
PutObjectRequest.Builder putObjectRequest =
PutObjectRequest.builder().bucket(bucket).key(key);

if (contentLength.isPresent()) {
putObjectRequest = putObjectRequest.contentLength(contentLength.get());
}

if (metadata != null) {
AwsRequestOverrideConfiguration.Builder override =
AwsRequestOverrideConfiguration.builder();

for (Map.Entry<String, String> e : metadata.entrySet()) {
override = override.putRawQueryParameter(e.getKey(), e.getValue());
}
putObjectRequest = putObjectRequest.overrideConfiguration(override.build());
}

PutObjectPresignRequest putRequest = PutObjectPresignRequest.builder()
.signatureDuration(duration).putObjectRequest(putObjectRequest).build();
.signatureDuration(duration).putObjectRequest(putObjectRequest.build()).build();

PresignedPutObjectRequest req = signer.presignPutObject(putRequest);
URL url = req.url();
Expand Down
2 changes: 1 addition & 1 deletion console/src/main/java/com/formkiq/stacks/console/ConsoleInstallHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ private void deleteConsole(final LambdaLogger logger) {
String destinationBucket = this.environmentMap.get("CONSOLE_BUCKET");
logger.log("deleting console from: " + destinationBucket);
this.s3.deleteAllFiles(destinationBucket);

String cognitoConfigBucket = this.environmentMap.get("COGNITO_CONFIG_BUCKET");
logger.log("deleting cognito config from: " + cognitoConfigBucket);
this.s3.deleteAllFiles(cognitoConfigBucket);
Expand Down
2 changes: 1 addition & 1 deletion console/src/test/java/com/formkiq/stacks/console/awstest/AwsResourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ public void testLogin() throws URISyntaxException, IOException, InterruptedExcep
Map<String, Object> map = new GsonBuilder().create().fromJson(response.body(), Map.class);
String userAuthentication = map.get("userAuthentication").toString();

LaunchOptions options = new LaunchOptions().setHeadless(false);
LaunchOptions options = new LaunchOptions().setHeadless(true);

try (Playwright playwright = Playwright.create()) {
try (Browser browser = playwright.chromium().launch(options)) {
Expand Down
6 changes: 4 additions & 2 deletions lambda-api/src/main/java/com/formkiq/stacks/api/handler/DocumentIdRequestHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,8 +226,10 @@ private String generateUploadUrl(final AwsServiceCache awsservice, final String
Duration duration = Duration.ofHours(DEFAULT_DURATION_HOURS);
String key = createS3Key(siteId, documentId);
S3Service s3Service = awsservice.getExtension(S3Service.class);
url = s3Service.presignPostUrl(awsservice.environment("DOCUMENTS_S3_BUCKET"), key, duration,
Optional.empty()).toString();

Map<String, String> map = Map.of("checksum", UUID.randomUUID().toString());
url = s3Service.presignPutUrl(awsservice.environment("DOCUMENTS_S3_BUCKET"), key, duration,
Optional.empty(), map).toString();
}

return url;
Expand Down
6 changes: 4 additions & 2 deletions lambda-api/src/main/java/com/formkiq/stacks/api/handler/DocumentsIdUploadRequestHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,10 @@ private String generatePresignedUrl(final AwsServiceCache awsservice, final Stri
Duration duration = caculateDuration(query);
Optional<Long> contentLength = calculateContentLength(awsservice, query, siteId);
S3Service s3Service = awsservice.getExtension(S3Service.class);
URL url = s3Service.presignPostUrl(awsservice.environment("DOCUMENTS_S3_BUCKET"), key, duration,
contentLength);

Map<String, String> map = Map.of("checksum", UUID.randomUUID().toString());
URL url = s3Service.presignPutUrl(awsservice.environment("DOCUMENTS_S3_BUCKET"), key, duration,
contentLength, map);

String urlstring = url.toString();
return urlstring;
Expand Down
5 changes: 3 additions & 2 deletions lambda-api/src/main/java/com/formkiq/stacks/api/handler/DocumentsUploadRequestHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,9 @@ private String generatePresignedUrl(final ApiGatewayRequestEvent event,

S3Service s3Service = awsservice.getExtension(S3Service.class);

URL url = s3Service.presignPostUrl(awsservice.environment("DOCUMENTS_S3_BUCKET"), key, duration,
contentLength);
Map<String, String> map = Map.of("checksum", UUID.randomUUID().toString());
URL url = s3Service.presignPutUrl(awsservice.environment("DOCUMENTS_S3_BUCKET"), key, duration,
contentLength, map);

String urlstring = url.toString();
return urlstring;
Expand Down
9 changes: 8 additions & 1 deletion lambda-s3/src/main/java/com/formkiq/stacks/lambda/s3/DocumentsS3Update.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,8 @@ public Void handleRequest(final Map<String, Object> map, final Context context)
}

private boolean isChecksumChanged(final S3ObjectMetadata resp, final DocumentItem doc) {
return doc.getChecksum() != null && !resp.getEtag().contains(doc.getChecksum());
String s3Checksum = resp.getMetadata().getOrDefault("checksum", "");
return doc.getChecksum() != null && !s3Checksum.contains(doc.getChecksum());
}

/**
Expand Down Expand Up @@ -515,6 +516,12 @@ private void processS3File(final LambdaLogger logger, final boolean create, fina
Map<String, AttributeValue> attributes = new HashMap<>();

boolean isChecksumChanged = isChecksumChanged(resp, item);

if (debug) {
logger.log("metadata: " + resp.getMetadata());
logger.log("item checksum: " + item.getChecksum());
}

if (isChecksumChanged) {
attributes.put("lastModifiedDate", AttributeValue.fromS(this.df.format(new Date())));
}
Expand Down
15 changes: 8 additions & 7 deletions lambda-s3/src/main/java/com/formkiq/stacks/lambda/s3/StagingS3Create.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,8 @@ private void writeS3Document(final LambdaLogger logger, final String bucket, fin
final String siteId, final DynamicDocumentItem item, final Map<String, String> contentMap,
final Map<String, String> contentTypeMap) {

Map<String, String> map = Map.of("checksum", item.getChecksum());

if (s3Key.endsWith(FORMKIQ_B64_EXT)) {

for (Map.Entry<String, String> e : contentMap.entrySet()) {
Expand All @@ -579,10 +581,7 @@ private void writeS3Document(final LambdaLogger logger, final String bucket, fin
logger.log(String.format("Inserted %s into bucket %s as %s", item.getPath(),
this.documentsBucket, createDatabaseKey(siteId, item.getDocumentId())));

this.s3.putObject(this.documentsBucket, key, bytes, contentType);
// attributes.put("checksum", AttributeValue.fromS(resp.eTag()));
// item.setChecksum(response.eTag());
// item.setContentLength(Long.valueOf(bytes.length));
this.s3.putObject(this.documentsBucket, key, bytes, contentType, map);
}

} else {
Expand All @@ -594,9 +593,8 @@ private void writeS3Document(final LambdaLogger logger, final String bucket, fin
logger.log(String.format("Copying %s from bucket %s to %s in bucket %s.", s3Key, bucket,
destKey, this.documentsBucket));

this.s3.copyObject(bucket, s3Key, this.documentsBucket, destKey, metadata.getContentType());
// attributes.put("checksum", AttributeValue.fromS(response.responseMetadata()));
// this.d
this.s3.copyObject(bucket, s3Key, this.documentsBucket, destKey, metadata.getContentType(),
map);
}
}

Expand All @@ -614,6 +612,9 @@ private DynamicDocumentItem createDocument(final LambdaLogger logger, final Stri

if (hasContent) {
doc.setContentLength(null);
doc.setChecksum(UUID.randomUUID().toString());
} else if (doc.getChecksum() == null) {
doc.setChecksum(UUID.randomUUID().toString());
}

if (isEmpty(doc.getPath())) {
Expand Down
25 changes: 17 additions & 8 deletions lambda-s3/src/test/java/com/formkiq/stacks/lambda/s3/StagingS3CreateTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@
import com.formkiq.aws.dynamodb.model.SearchQuery;
import com.formkiq.aws.dynamodb.model.SearchTagCriteria;
import com.formkiq.aws.s3.S3ConnectionBuilder;
import com.formkiq.aws.s3.S3ObjectMetadata;
import com.formkiq.aws.s3.S3Service;
import com.formkiq.aws.sns.SnsConnectionBuilder;
import com.formkiq.aws.sns.SnsService;
Expand Down Expand Up @@ -520,7 +521,7 @@ private void processFkB64File(final String siteId, final DynamicDocumentItem doc
}
assertEquals("plain/text", item.getContentType());
assertNotNull(item.getInsertedDate());
assertNull(item.getChecksum());
assertNotNull(item.getChecksum());
assertEquals("test.txt", item.getPath());
assertEquals("joe", item.getUserId());

Expand Down Expand Up @@ -597,11 +598,9 @@ private void testCopyFile(final String siteId, final String path, final String e

DocumentItem item = service.findDocument(siteId, destDocumentId);
assertNotNull(item);
// assertEquals("8", item.getContentLength().toString());
// assertEquals("application/pdf", item.getContentType());
assertNull(item.getContentLength());
assertNull(item.getContentType());
assertNull(item.getChecksum());
assertNotNull(item.getChecksum());
assertNotNull(item.getInsertedDate());
assertEquals(item.getInsertedDate(), item.getLastModifiedDate());

Expand All @@ -625,6 +624,8 @@ private void testCopyFile(final String siteId, final String path, final String e

assertEquals("untagged", tags.get(i).getKey());
assertEquals(DocumentTagType.SYSTEMDEFINED, tags.get(i++).getType());

verifyS3Metadata(siteId, item);
}

/**
Expand Down Expand Up @@ -879,7 +880,7 @@ public void testFkB64Extension07() throws IOException {
assertNull(item.getContentLength());
assertEquals("text/plain", item.getContentType());
assertNotNull(item.getInsertedDate());
assertNull(item.getChecksum());
assertNotNull(item.getChecksum());
// assertTrue(item.getChecksum() != null && item.getInsertedDate() != null);
assertEquals(item.getDocumentId(), item.getPath());
assertEquals("joesmith", item.getUserId());
Expand Down Expand Up @@ -945,7 +946,7 @@ public void testFkB64Extension08() throws IOException {
assertNull(item.getContentLength());
assertEquals("text/plain", item.getContentType());
assertNotNull(item.getInsertedDate());
assertNull(item.getChecksum());
assertNotNull(item.getChecksum());
assertEquals(item.getDocumentId(), item.getPath());
assertEquals("joesmith", item.getUserId());

Expand Down Expand Up @@ -1005,7 +1006,7 @@ public void testFkB64Extension09() throws IOException {
DocumentItem item = service.findDocument(siteId, documentId);
assertNull(item.getContentLength());
assertEquals("text/plain", item.getContentType());
assertNull(item.getChecksum());
assertNotNull(item.getChecksum());
assertNotNull(item.getInsertedDate());
assertEquals(item.getDocumentId(), item.getPath());
assertEquals("joesmith", item.getUserId());
Expand Down Expand Up @@ -1146,7 +1147,7 @@ public void testFkB64Extension11() throws IOException {
DocumentItem item = service.findDocument(siteId, documentId);
assertNull(item.getContentLength());
assertEquals("text/plain", item.getContentType());
assertNull(item.getChecksum());
assertNotNull(item.getChecksum());
assertNotNull(item.getInsertedDate());
assertEquals(path, item.getPath());
assertEquals("joesmith", item.getUserId());
Expand Down Expand Up @@ -1372,6 +1373,14 @@ private void verifyBelongsToDocument(final DocumentItem item, final String docum
}
}

private void verifyS3Metadata(final String siteId, final DocumentItem item) {
String key = createDatabaseKey(siteId, item.getDocumentId());
S3ObjectMetadata objectMetadata = s3.getObjectMetadata(DOCUMENTS_BUCKET, key);
assertTrue(objectMetadata.isObjectExists());

assertNotNull(objectMetadata.getMetadata().get("checksum"));
}

/**
* Verify SQS Messages.
*
Expand Down
4 changes: 3 additions & 1 deletion lambda-s3/src/test/java/com/formkiq/stacks/lambda/s3/awstest/AwsResourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import org.junit.Test;
Expand Down Expand Up @@ -291,7 +292,8 @@ public void testAddDeleteFile03() throws Exception {
getDocumentService().saveDocumentItemWithTag(null, doc);

// when
URL url = getS3Service().presignPutUrl(getDocumentsbucketname(), key, Duration.ofHours(1));
URL url = getS3Service().presignPutUrl(getDocumentsbucketname(), key, Duration.ofHours(1),
Optional.empty(), null);
HttpResponse<String> put =
http.send(
HttpRequest.newBuilder(url.toURI()).header("Content-Type", contentType)
Expand Down

0 comments on commit b980ffd

Please sign in to comment.