From 1c161725a0e436d331b8e4d5e292c3cd20db41bb Mon Sep 17 00:00:00 2001 From: Mike Friesen Date: Mon, 17 Jul 2023 21:25:38 -0500 Subject: [PATCH] v1.11.1 (#153) * #148 - POST /documents/{documentId}/actions fails to run new actions * #149 - adjusting chatgpt response * Updated console to v3.2.2 * GET /configuration - Adding mask keys --- .../formkiq/aws/dynamodb/objects/Strings.java | 20 +++ .../aws/dynamodb/objects/StringsTest.java | 18 ++ build.gradle | 2 +- .../console/awstest/AwsResourceTest.java | 2 +- .../resources/cloudformation/template.yaml | 2 +- docs/openapi/openapi-iam.yaml | 4 +- docs/openapi/openapi-jwt.yaml | 4 +- docs/openapi/openapi-key.yaml | 4 +- .../module/documentevents/DocumentEvent.java | 9 + lambda-api/build.gradle | 1 + .../config/checkstyle/import-control.xml | 1 + .../stacks/api/CoreRequestHandler.java | 3 +- .../handler/ConfigurationRequestHandler.java | 22 ++- .../api/handler/SitesRequestHandler.java | 12 +- .../resources/cloudformation/api-apikey.yaml | 3 +- .../resources/cloudformation/api-iam.yaml | 3 +- .../main/resources/cloudformation/api.yaml | 3 +- .../stacks/api/ConfigurationRequestTest.java | 23 ++- .../formkiq/stacks/api/SitesRequestTest.java | 10 +- .../lambda/s3/DocumentTaggingAction.java | 96 ++++++++++- .../s3/DocumentActionsProcessorTest.java | 154 +++++++++++++++++- .../src/test/resources/chatgpt/response4.json | 22 +++ .../src/test/resources/chatgpt/response5.json | 21 +++ 23 files changed, 387 insertions(+), 52 deletions(-) create mode 100644 lambda-s3/src/test/resources/chatgpt/response4.json create mode 100644 lambda-s3/src/test/resources/chatgpt/response5.json diff --git a/aws-dynamodb/src/main/java/com/formkiq/aws/dynamodb/objects/Strings.java b/aws-dynamodb/src/main/java/com/formkiq/aws/dynamodb/objects/Strings.java index 0cc2a54ab..7304a783e 100644 --- a/aws-dynamodb/src/main/java/com/formkiq/aws/dynamodb/objects/Strings.java +++ b/aws-dynamodb/src/main/java/com/formkiq/aws/dynamodb/objects/Strings.java @@ -62,4 +62,24 @@ public static String getFilename(final String path) { public static boolean isEmpty(final CharSequence cs) { return cs == null || cs.length() == 0; } + + /** + * Remove single/double quotes from {@link String}. + * + * @param s {@link String} + * @return {@link String} + */ + public static String removeQuotes(final String s) { + return s.replaceAll("^['\"]|['\"]$", ""); + } + + /** + * Remove single/double quotes from {@link String}. + * + * @param s {@link String} + * @return {@link String} + */ + public static String removeEndingPunctuation(final String s) { + return s.replaceAll("[!\\.,?]$", ""); + } } diff --git a/aws-dynamodb/src/test/java/com/formkiq/aws/dynamodb/objects/StringsTest.java b/aws-dynamodb/src/test/java/com/formkiq/aws/dynamodb/objects/StringsTest.java index 9f682002a..d74ace763 100644 --- a/aws-dynamodb/src/test/java/com/formkiq/aws/dynamodb/objects/StringsTest.java +++ b/aws-dynamodb/src/test/java/com/formkiq/aws/dynamodb/objects/StringsTest.java @@ -47,4 +47,22 @@ void testFilename() { assertEquals("test (something).txt", Strings.getFilename("/bleh/something/test (something).txt")); } + + @Test + void replaceQuotes() { + assertEquals("text", Strings.removeQuotes("text")); + assertEquals("text", Strings.removeQuotes("\"text\"")); + assertEquals("text", Strings.removeQuotes("\"text")); + assertEquals("text", Strings.removeQuotes("'text'")); + assertEquals("text", Strings.removeQuotes("\"text'")); + } + + @Test + void removeEndingPunctuation() { + assertEquals("text", Strings.removeEndingPunctuation("text")); + assertEquals("\"text\"", Strings.removeEndingPunctuation("\"text\",")); + assertEquals("\"text", Strings.removeEndingPunctuation("\"text!")); + assertEquals("'text?'", Strings.removeEndingPunctuation("'text?'")); + assertEquals("\"text'", Strings.removeEndingPunctuation("\"text'")); + } } diff --git a/build.gradle b/build.gradle index d90974e82..d25c5a81d 100644 --- a/build.gradle +++ b/build.gradle @@ -24,7 +24,7 @@ def getCmdParam() { repositories { mavenCentral() } allprojects { - version = '1.11.0' + version = '1.11.1' ext.awsCognitoVersion = '1.5.0' group = 'com.formkiq.stacks' diff --git a/console/src/integration/java/com/formkiq/stacks/console/awstest/AwsResourceTest.java b/console/src/integration/java/com/formkiq/stacks/console/awstest/AwsResourceTest.java index 59da33242..2f27ee353 100644 --- a/console/src/integration/java/com/formkiq/stacks/console/awstest/AwsResourceTest.java +++ b/console/src/integration/java/com/formkiq/stacks/console/awstest/AwsResourceTest.java @@ -201,7 +201,7 @@ public void testS3Buckets() { */ @Test public void testSsmParameters() { - assertEquals("v3.1.0", + assertEquals("v3.2.2", ssmService.getParameterValue("/formkiq/" + appenvironment + "/console/version")); assertTrue(ssmService.getParameterValue("/formkiq/" + appenvironment + "/s3/Console") .contains(appenvironment + "-console-")); diff --git a/console/src/main/resources/cloudformation/template.yaml b/console/src/main/resources/cloudformation/template.yaml index f8c2462dc..a7ff9f14c 100644 --- a/console/src/main/resources/cloudformation/template.yaml +++ b/console/src/main/resources/cloudformation/template.yaml @@ -17,7 +17,7 @@ Parameters: ConsoleVersion: Type: String Description: Version of FormKiQ console to deploy - Default: v3.1.0 + Default: v3.2.2 FormKiQType: Description: The type of FormKiQ installation diff --git a/docs/openapi/openapi-iam.yaml b/docs/openapi/openapi-iam.yaml index 71a9b63e3..5869e00ab 100644 --- a/docs/openapi/openapi-iam.yaml +++ b/docs/openapi/openapi-iam.yaml @@ -12,8 +12,8 @@ license: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0.html - description: FormKiQ IAM API - version: 1.11.0 + description: 'Formkiq API: Document Management Platform API using AWS IAM Authentication' + version: 1.11.1 paths: /version: get: diff --git a/docs/openapi/openapi-jwt.yaml b/docs/openapi/openapi-jwt.yaml index 609569355..5add22dbf 100644 --- a/docs/openapi/openapi-jwt.yaml +++ b/docs/openapi/openapi-jwt.yaml @@ -12,8 +12,8 @@ license: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0.html - description: FormKiQ HTTP API - version: 1.11.0 + description: 'Formkiq API: Document Management Platform API using JWT Authentication' + version: 1.11.1 paths: /version: get: diff --git a/docs/openapi/openapi-key.yaml b/docs/openapi/openapi-key.yaml index 37c0d1469..4098f1451 100644 --- a/docs/openapi/openapi-key.yaml +++ b/docs/openapi/openapi-key.yaml @@ -12,8 +12,8 @@ license: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0.html - description: FormKiQ Api Key - version: 1.11.0 + description: 'Formkiq API: Document Management Platform API using API Key(s) Authentication' + version: 1.11.1 paths: /version: get: diff --git a/document-events/src/main/java/com/formkiq/module/documentevents/DocumentEvent.java b/document-events/src/main/java/com/formkiq/module/documentevents/DocumentEvent.java index f81f0b5ed..f6b9b4937 100644 --- a/document-events/src/main/java/com/formkiq/module/documentevents/DocumentEvent.java +++ b/document-events/src/main/java/com/formkiq/module/documentevents/DocumentEvent.java @@ -33,22 +33,31 @@ public class DocumentEvent { /** Document SiteId. */ + @Reflectable private String siteId; /** Document Id. */ + @Reflectable private String documentId; /** S3 Key. */ + @Reflectable private String s3key; /** S3 Bucket. */ + @Reflectable private String s3bucket; /** Document Type. */ + @Reflectable private String type; /** User Id. */ + @Reflectable private String userId; /** Document Content. */ + @Reflectable private String content; /** Document Content Type. */ + @Reflectable private String contentType; /** Docuemnt Path. */ + @Reflectable private String path; /** diff --git a/lambda-api/build.gradle b/lambda-api/build.gradle index c0ce81c62..bb0cbcda1 100644 --- a/lambda-api/build.gradle +++ b/lambda-api/build.gradle @@ -29,6 +29,7 @@ dependencies { annotationProcessor group: 'com.formkiq', name: 'graalvm-annotations-processor', version: '1.3.0' + implementation project(':document-events') implementation project(':fkq-lambda-services') implementation project(':fkq-lambda-core') implementation project(':fkq-validation') diff --git a/lambda-api/config/checkstyle/import-control.xml b/lambda-api/config/checkstyle/import-control.xml index 8b003360d..8cc92c09c 100644 --- a/lambda-api/config/checkstyle/import-control.xml +++ b/lambda-api/config/checkstyle/import-control.xml @@ -53,6 +53,7 @@ + diff --git a/lambda-api/src/main/java/com/formkiq/stacks/api/CoreRequestHandler.java b/lambda-api/src/main/java/com/formkiq/stacks/api/CoreRequestHandler.java index 8e799d50f..49571ae79 100644 --- a/lambda-api/src/main/java/com/formkiq/stacks/api/CoreRequestHandler.java +++ b/lambda-api/src/main/java/com/formkiq/stacks/api/CoreRequestHandler.java @@ -42,6 +42,7 @@ import com.formkiq.graalvm.annotations.Reflectable; import com.formkiq.graalvm.annotations.ReflectableImport; import com.formkiq.module.actions.Action; +import com.formkiq.module.documentevents.DocumentEvent; import com.formkiq.plugins.tagschema.DocumentTagSchemaPluginEmpty; import com.formkiq.stacks.dynamodb.DocumentItemDynamoDb; import com.formkiq.stacks.dynamodb.DocumentTags; @@ -58,7 +59,7 @@ @ReflectableImport(classes = {DocumentItemDynamoDb.class, DocumentTagType.class, DocumentTag.class, DocumentMetadata.class, DocumentTags.class, PaginationMapToken.class, SearchQuery.class, SearchTagCriteria.class, SearchMetaCriteria.class, SearchResponseFields.class, PresetTag.class, - Preset.class, ApiGatewayRequestEvent.class, ApiMapResponse.class, + Preset.class, ApiGatewayRequestEvent.class, ApiMapResponse.class, DocumentEvent.class, ApiGatewayRequestContext.class, ApiMessageResponse.class, ApiResponseError.class, ApiPagination.class, Action.class, ValidationErrorImpl.class, DocumentVersionServiceDynamoDb.class, DocumentVersionServiceNoVersioning.class}) diff --git a/lambda-api/src/main/java/com/formkiq/stacks/api/handler/ConfigurationRequestHandler.java b/lambda-api/src/main/java/com/formkiq/stacks/api/handler/ConfigurationRequestHandler.java index 7aca34aed..9519f6ae6 100644 --- a/lambda-api/src/main/java/com/formkiq/stacks/api/handler/ConfigurationRequestHandler.java +++ b/lambda-api/src/main/java/com/formkiq/stacks/api/handler/ConfigurationRequestHandler.java @@ -23,6 +23,7 @@ */ package com.formkiq.stacks.api.handler; +import static com.formkiq.aws.dynamodb.objects.Strings.isEmpty; import static com.formkiq.aws.services.lambda.ApiResponseStatus.SC_OK; import static com.formkiq.stacks.dynamodb.ConfigService.CHATGPT_API_KEY; import static com.formkiq.stacks.dynamodb.ConfigService.MAX_DOCUMENTS; @@ -47,18 +48,15 @@ public class ConfigurationRequestHandler implements ApiGatewayRequestHandler, ApiGatewayRequestEventUtil { + /** Mask Value, must be even number. */ + private static final int MASK = 4; + /** * constructor. * */ public ConfigurationRequestHandler() {} - @Override - public void beforeGet(final LambdaLogger logger, final ApiGatewayRequestEvent event, - final ApiAuthorizer authorizer, final AwsServiceCache awsServices) throws Exception { - checkPermissions(authorizer); - } - @Override public void beforePatch(final LambdaLogger logger, final ApiGatewayRequestEvent event, final ApiAuthorizer authorizer, final AwsServiceCache awsServices) throws Exception { @@ -71,6 +69,16 @@ private void checkPermissions(final ApiAuthorizer authorizer) throws Unauthorize } } + /** + * Mask {@link String}. + * + * @param s {@link String} + * @return {@link String} + */ + private String mask(final String s) { + return !isEmpty(s) ? s.subSequence(0, MASK) + "*******" + s.substring(s.length() - MASK) : s; + } + @Override public ApiRequestHandlerResponse get(final LambdaLogger logger, final ApiGatewayRequestEvent event, final ApiAuthorizer authorizer, @@ -81,7 +89,7 @@ public ApiRequestHandlerResponse get(final LambdaLogger logger, DynamicObject obj = configService.get(siteId); Map map = new HashMap<>(); - map.put("chatGptApiKey", obj.getOrDefault(CHATGPT_API_KEY, "")); + map.put("chatGptApiKey", mask(obj.getOrDefault(CHATGPT_API_KEY, "").toString())); map.put("maxContentLengthBytes", obj.getOrDefault(MAX_DOCUMENT_SIZE_BYTES, "")); map.put("maxDocuments", obj.getOrDefault(MAX_DOCUMENTS, "")); map.put("maxWebhooks", obj.getOrDefault(MAX_WEBHOOKS, "")); diff --git a/lambda-api/src/main/java/com/formkiq/stacks/api/handler/SitesRequestHandler.java b/lambda-api/src/main/java/com/formkiq/stacks/api/handler/SitesRequestHandler.java index b9853aab7..0f1b6135d 100644 --- a/lambda-api/src/main/java/com/formkiq/stacks/api/handler/SitesRequestHandler.java +++ b/lambda-api/src/main/java/com/formkiq/stacks/api/handler/SitesRequestHandler.java @@ -25,6 +25,7 @@ import static com.formkiq.aws.dynamodb.SiteIdKeyGenerator.DEFAULT_SITE_ID; import static com.formkiq.aws.services.lambda.ApiResponseStatus.SC_OK; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -40,7 +41,6 @@ import com.formkiq.aws.ssm.SsmConnectionBuilder; import com.formkiq.aws.ssm.SsmService; import com.formkiq.module.lambdaservices.AwsServiceCache; -import com.formkiq.stacks.dynamodb.ConfigService; import software.amazon.awssdk.services.ssm.SsmClient; /** {@link ApiGatewayRequestHandler} for "/sites". */ @@ -80,13 +80,12 @@ public ApiRequestHandlerResponse get(final LambdaLogger logger, final ApiGatewayRequestEvent event, final ApiAuthorizer authorizer, final AwsServiceCache awsservice) throws Exception { - ConfigService configService = awsservice.getExtension(ConfigService.class); - SsmConnectionBuilder ssm = awsservice.getExtension(SsmConnectionBuilder.class); try (SsmClient ssmClient = ssm.build()) { List sites = authorizer.getSiteIds().stream().map(siteId -> { - DynamicObject config = configService.get(siteId); + + DynamicObject config = new DynamicObject(new HashMap<>()); config.put("siteId", siteId != null ? siteId : DEFAULT_SITE_ID); boolean write = authorizer.getWriteSiteIds().contains(siteId); @@ -95,11 +94,6 @@ public ApiRequestHandlerResponse get(final LambdaLogger logger, return config; }).collect(Collectors.toList()); - sites.forEach(ob -> { - ob.remove("PK"); - ob.remove("SK"); - }); - updateUploadEmail(logger, awsservice, authorizer, sites); return new ApiRequestHandlerResponse(SC_OK, new ApiMapResponse(Map.of("sites", sites))); diff --git a/lambda-api/src/main/resources/cloudformation/api-apikey.yaml b/lambda-api/src/main/resources/cloudformation/api-apikey.yaml index 834d905b5..0401fe1db 100644 --- a/lambda-api/src/main/resources/cloudformation/api-apikey.yaml +++ b/lambda-api/src/main/resources/cloudformation/api-apikey.yaml @@ -25,7 +25,8 @@ Resources: license: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0.html - description: "FormKiQ Api Key" + description: |- + Formkiq API: Document Management Platform API using API Key(s) Authentication version: #@ data.values.version or assert.fail("missing version") paths: /version: diff --git a/lambda-api/src/main/resources/cloudformation/api-iam.yaml b/lambda-api/src/main/resources/cloudformation/api-iam.yaml index e18ccfca4..2dda0aae5 100644 --- a/lambda-api/src/main/resources/cloudformation/api-iam.yaml +++ b/lambda-api/src/main/resources/cloudformation/api-iam.yaml @@ -25,7 +25,8 @@ Resources: license: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0.html - description: "FormKiQ IAM API" + description: |- + Formkiq API: Document Management Platform API using AWS IAM Authentication version: #@ data.values.version or assert.fail("missing version") paths: /version: diff --git a/lambda-api/src/main/resources/cloudformation/api.yaml b/lambda-api/src/main/resources/cloudformation/api.yaml index 73c4bb7d1..87195e00a 100644 --- a/lambda-api/src/main/resources/cloudformation/api.yaml +++ b/lambda-api/src/main/resources/cloudformation/api.yaml @@ -25,7 +25,8 @@ Resources: license: name: Apache 2.0 url: https://www.apache.org/licenses/LICENSE-2.0.html - description: "FormKiQ HTTP API" + description: |- + Formkiq API: Document Management Platform API using JWT Authentication version: #@ data.values.version or assert.fail("missing version") paths: /version: diff --git a/lambda-api/src/test/java/com/formkiq/stacks/api/ConfigurationRequestTest.java b/lambda-api/src/test/java/com/formkiq/stacks/api/ConfigurationRequestTest.java index cbf003790..83f7c1d19 100644 --- a/lambda-api/src/test/java/com/formkiq/stacks/api/ConfigurationRequestTest.java +++ b/lambda-api/src/test/java/com/formkiq/stacks/api/ConfigurationRequestTest.java @@ -107,7 +107,7 @@ public void testHandleGetSites01() throws Exception { final int expected = 4; DynamicObject resp = new DynamicObject(fromJson(mget.get("body"), Map.class)); assertEquals(expected, resp.size()); - assertEquals("anothervalue", resp.getString("chatGptApiKey")); + assertEquals("anot*******alue", resp.getString("chatGptApiKey")); assertEquals("", resp.getString("maxContentLengthBytes")); assertEquals("", resp.getString("maxDocuments")); assertEquals("", resp.getString("maxWebhooks")); @@ -124,6 +124,8 @@ public void testHandleGetSites02() throws Exception { // given String siteId = null; String group = "default"; + ConfigService config = getAwsServices().getExtension(ConfigService.class); + config.save(siteId, new DynamicObject(Map.of(CHATGPT_API_KEY, "somevalue"))); ApiGatewayRequestEvent event = getRequest(siteId, group); @@ -132,12 +134,15 @@ public void testHandleGetSites02() throws Exception { // then Map m = GsonUtil.getInstance().fromJson(response, Map.class); + verifyResponse(m); - final int mapsize = 3; - assertEquals(mapsize, m.size()); - assertEquals("401.0", String.valueOf(m.get("statusCode"))); - assertEquals(getHeaders(), "\"headers\":" + GsonUtil.getInstance().toJson(m.get("headers"))); - assertEquals("{\"message\":\"user is unauthorized\"}", String.valueOf(m.get("body"))); + final int expected = 4; + DynamicObject resp = new DynamicObject(fromJson(m.get("body"), Map.class)); + assertEquals(expected, resp.size()); + assertEquals("some*******alue", resp.getString("chatGptApiKey")); + assertEquals("", resp.getString("maxContentLengthBytes")); + assertEquals("", resp.getString("maxDocuments")); + assertEquals("", resp.getString("maxWebhooks")); } /** @@ -168,7 +173,7 @@ public void testHandleGetSites03() throws Exception { final int expected = 4; DynamicObject resp = new DynamicObject(fromJson(m.get("body"), Map.class)); assertEquals(expected, resp.size()); - assertEquals("somevalue", resp.getString("chatGptApiKey")); + assertEquals("some*******alue", resp.getString("chatGptApiKey")); assertEquals("", resp.getString("maxContentLengthBytes")); assertEquals("", resp.getString("maxDocuments")); assertEquals("", resp.getString("maxWebhooks")); @@ -203,7 +208,7 @@ public void testHandleGetSites04() throws Exception { final int expected = 4; DynamicObject resp = new DynamicObject(fromJson(m.get("body"), Map.class)); assertEquals(expected, resp.size()); - assertEquals("anothervalue", resp.getString("chatGptApiKey")); + assertEquals("anot*******alue", resp.getString("chatGptApiKey")); assertEquals("", resp.getString("maxContentLengthBytes")); assertEquals("", resp.getString("maxDocuments")); assertEquals("", resp.getString("maxWebhooks")); @@ -241,7 +246,7 @@ public void testHandlePutSites01() throws Exception { final int expected = 4; DynamicObject resp = new DynamicObject(fromJson(mget.get("body"), Map.class)); assertEquals(expected, resp.size()); - assertEquals("anotherkey", resp.getString("chatGptApiKey")); + assertEquals("anot*******rkey", resp.getString("chatGptApiKey")); assertEquals("1000000", resp.getString("maxContentLengthBytes")); assertEquals("1000", resp.getString("maxDocuments")); assertEquals("5", resp.getString("maxWebhooks")); diff --git a/lambda-api/src/test/java/com/formkiq/stacks/api/SitesRequestTest.java b/lambda-api/src/test/java/com/formkiq/stacks/api/SitesRequestTest.java index 8435649d9..7a702cd12 100644 --- a/lambda-api/src/test/java/com/formkiq/stacks/api/SitesRequestTest.java +++ b/lambda-api/src/test/java/com/formkiq/stacks/api/SitesRequestTest.java @@ -75,8 +75,12 @@ private ApiGatewayRequestEvent getRequest(final String siteId, final String grou @Test public void testHandleGetSites01() throws Exception { // given + String siteId = null; + ConfigService config = getAwsServices().getExtension(ConfigService.class); + config.save(siteId, new DynamicObject(Map.of("chatGptApiKey", "somevalue"))); + putSsmParameter("/formkiq/" + FORMKIQ_APP_ENVIRONMENT + "/maildomain", "tryformkiq.com"); - ApiGatewayRequestEvent event = getRequest(null, "default Admins finance"); + ApiGatewayRequestEvent event = getRequest(siteId, "default Admins finance"); // when String response = handleRequest(event); @@ -96,6 +100,8 @@ public void testHandleGetSites01() throws Exception { List sites = resp.getList("sites"); assertEquals(2, sites.size()); + final int expected = 3; + assertEquals(expected, sites.get(0).size()); assertEquals(DEFAULT_SITE_ID, sites.get(0).get("siteId")); assertEquals("READ_WRITE", sites.get(0).get("permission")); assertNotNull(sites.get(0).get("uploadEmail")); @@ -246,7 +252,5 @@ public void testHandleGetSites04() throws Exception { assertEquals(1, resp.getList("sites").size()); assertEquals(siteId, resp.getList("sites").get(0).getString("siteId")); assertEquals("READ_WRITE", resp.getList("sites").get(0).getString("permission")); - assertEquals("5", resp.getList("sites").get(0).getString(MAX_DOCUMENTS)); - assertEquals("10", resp.getList("sites").get(0).getString(MAX_WEBHOOKS)); } } diff --git a/lambda-s3/src/main/java/com/formkiq/stacks/lambda/s3/DocumentTaggingAction.java b/lambda-s3/src/main/java/com/formkiq/stacks/lambda/s3/DocumentTaggingAction.java index 63c55367f..401d7c62f 100644 --- a/lambda-s3/src/main/java/com/formkiq/stacks/lambda/s3/DocumentTaggingAction.java +++ b/lambda-s3/src/main/java/com/formkiq/stacks/lambda/s3/DocumentTaggingAction.java @@ -23,6 +23,8 @@ */ package com.formkiq.stacks.lambda.s3; +import static com.formkiq.aws.dynamodb.objects.Strings.removeEndingPunctuation; +import static com.formkiq.aws.dynamodb.objects.Strings.removeQuotes; import static com.formkiq.module.http.HttpResponseStatus.is2XX; import static com.formkiq.stacks.dynamodb.ConfigService.CHATGPT_API_KEY; import java.io.IOException; @@ -36,8 +38,8 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.stream.Collectors; import java.util.Optional; +import java.util.stream.Collectors; import com.amazonaws.services.lambda.runtime.LambdaLogger; import com.formkiq.aws.dynamodb.DynamicObject; import com.formkiq.aws.dynamodb.model.DocumentItem; @@ -97,6 +99,22 @@ public DocumentTaggingAction(final AwsServiceCache services) { this.documentService = services.getExtension(DocumentService.class); } + /** + * Adjust Tag Key. + * + * @param tags {@link List} {@link String} + * @param key {@link String} + * @return {@link String} + */ + private String adjustKeyFromTags(final List tags, final String key) { + String s = removeQuotes(key); + + Optional o = tags.stream().filter(t -> t.toLowerCase().replaceAll("[^A-Za-z0-9]", "") + .equals(key.toLowerCase().replaceAll("[^A-Za-z0-9]", ""))).findAny(); + + return o.isPresent() ? o.get() : s; + } + private String createChatGptPrompt(final String siteId, final String documentId, final Action action) throws IOException { @@ -205,10 +223,19 @@ private Map parseGptText(final List tags, final String t if (pos > -1) { String key = s.substring(0, pos).trim().toLowerCase(); - String value = s.substring(pos + 1).trim(); + String value = removeQuotes(removeEndingPunctuation(s.substring(pos + 1).trim())); if (!key.isEmpty() && !value.isEmpty()) { - data.put(key, Arrays.asList(value)); + + List list = getObjectAsJsonList(value); + if (list != null) { + + List slist = transformToStringList(list); + data.put(key, slist); + + } else { + data.put(key, Arrays.asList(value)); + } } } } @@ -220,7 +247,68 @@ private Map parseGptText(final List tags, final String t data.remove(e.getKey()); } - return data; + Map values = + data.entrySet().stream().filter(d -> d.getKey() != null && d.getValue() != null) + .collect(Collectors.toMap(d -> adjustKeyFromTags(tags, d.getKey()), + d -> removeQuotesFromObject(d.getValue()))); + + return tags.stream().filter(t -> values.containsKey(t)) + .collect(Collectors.toMap(t -> t, t -> values.get(t))); + } + + @SuppressWarnings("unchecked") + private List transformToStringList(final List objs) { + + List list = new ArrayList<>(); + + for (Object ob : objs) { + + if (ob instanceof Map) { + + for (Map.Entry e : ((Map) ob).entrySet()) { + list.add(e.getKey() + ": " + e.getValue()); + } + + } else { + list.add(ob.toString()); + } + } + + return list; + } + + @SuppressWarnings("unchecked") + private List getObjectAsJsonList(final String s) { + + List list; + + try { + list = this.gson.fromJson(s, List.class); + } catch (JsonSyntaxException e) { + list = null; + } + + return list; + } + + @SuppressWarnings("unchecked") + private Object removeQuotesFromObject(final Object o) { + Object oo = o; + if (oo instanceof String) { + oo = removeQuotes((String) oo); + } else if (oo instanceof Collection) { + + Collection list = new ArrayList<>(); + for (Object obj : (Collection) oo) { + if (obj instanceof String) { + list.add(removeQuotes((String) obj)); + } + } + + oo = list; + } + + return oo; } @Override diff --git a/lambda-s3/src/test/java/com/formkiq/stacks/lambda/s3/DocumentActionsProcessorTest.java b/lambda-s3/src/test/java/com/formkiq/stacks/lambda/s3/DocumentActionsProcessorTest.java index a41156c12..c6eaa164a 100644 --- a/lambda-s3/src/test/java/com/formkiq/stacks/lambda/s3/DocumentActionsProcessorTest.java +++ b/lambda-s3/src/test/java/com/formkiq/stacks/lambda/s3/DocumentActionsProcessorTest.java @@ -90,6 +90,7 @@ import com.formkiq.testutils.aws.TypeSenseExtension; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import joptsimple.internal.Strings; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; @@ -185,7 +186,7 @@ private static void createMockServer() throws IOException { final int status = 200; - for (String item : Arrays.asList("1", "2", "3")) { + for (String item : Arrays.asList("1", "2", "3", "4", "5")) { String text = FileUtils.loadFile(mockServer, "/chatgpt/response" + item + ".json"); mockServer.when(request().withMethod("POST").withPath("/chatgpt" + item)).respond( org.mockserver.model.HttpResponse.response(text).withStatusCode(Integer.valueOf(status))); @@ -305,20 +306,20 @@ public void testDocumentTaggingAction02() throws Exception { assertEquals(expectedSize, tags.getResults().size()); int i = 0; - assertEquals("Document Type", tags.getResults().get(i).getKey()); + assertEquals("document type", tags.getResults().get(i).getKey()); assertEquals("Memorandum", tags.getResults().get(i++).getValue()); - assertEquals("Location", tags.getResults().get(i).getKey()); + assertEquals("location", tags.getResults().get(i).getKey()); assertEquals("YellowBelly Brewery Pub, St. Johns, NL", tags.getResults().get(i++).getValue()); - assertEquals("Organization", tags.getResults().get(i).getKey()); + assertEquals("organization", tags.getResults().get(i).getKey()); assertEquals("Great Auk Enterprises", tags.getResults().get(i++).getValue()); - assertEquals("Person", tags.getResults().get(i).getKey()); + assertEquals("person", tags.getResults().get(i).getKey()); assertEquals("Thomas Bewick,Ketill Ketilsson,Farley Mowat,Aaron Thomas", String.join(",", tags.getResults().get(i++).getValues())); - assertEquals("Subject", tags.getResults().get(i).getKey()); + assertEquals("subject", tags.getResults().get(i).getKey()); assertEquals("MINUTES OF A MEETING OF DIRECTORS", tags.getResults().get(i++).getValue()); } } @@ -402,7 +403,7 @@ public void testDocumentTaggingAction04() throws Exception { int i = 0; assertEquals("Organization", tags.getResults().get(i).getKey()); - assertEquals("East Repair Inc.", tags.getResults().get(i++).getValue()); + assertEquals("East Repair Inc", tags.getResults().get(i++).getValue()); assertEquals("document type", tags.getResults().get(i).getKey()); assertEquals("Receipt", tags.getResults().get(i++).getValue()); @@ -491,6 +492,145 @@ public void testDocumentTaggingAction05() throws Exception { } } + /** + * Handle documentTagging ChatApt Action extra quotes. + * + * @throws Exception Exception + */ + @Test + public void testDocumentTaggingAction06() throws Exception { + + initProcessor("fulltext", "chatgpt4"); + + for (String siteId : Arrays.asList(null, UUID.randomUUID().toString())) { + // given + configService.save(siteId, new DynamicObject(Map.of(CHATGPT_API_KEY, "asd"))); + + String documentId = UUID.randomUUID().toString(); + + DocumentItem item = new DocumentItemDynamoDb(documentId, new Date(), "joe"); + item.setContentType("text/plain"); + + String s3Key = SiteIdKeyGenerator.createS3Key(siteId, documentId); + String content = "this is some data"; + s3Service.putObject(BUCKET_NAME, s3Key, content.getBytes(StandardCharsets.UTF_8), + "text/plain"); + + documentService.saveDocument(siteId, item, null); + documentService.addTags(siteId, documentId, Arrays.asList(new DocumentTag(documentId, + "untagged", "", new Date(), "joe", DocumentTagType.SYSTEMDEFINED)), null); + + List actions = + Arrays.asList(new Action().type(ActionType.DOCUMENTTAGGING).parameters(Map.of("engine", + "chatgpt", "tags", "organization,location,person,subject,sentiment,document type"))); + actionsService.saveActions(siteId, documentId, actions); + + Map map = + loadFileAsMap(this, "/actions-event01.json", "c2695f67-d95e-4db0-985e-574168b12e57", + documentId, "default", siteId != null ? siteId : "default"); + + // when + processor.handleRequest(map, this.context); + + // then + final int expectedSize = 5; + assertEquals(ActionStatus.COMPLETE, + actionsService.getActions(siteId, documentId).get(0).status()); + + PaginationResults tags = + documentService.findDocumentTags(siteId, documentId, null, MAX_RESULTS); + assertEquals(expectedSize, tags.getResults().size()); + + int i = 0; + assertEquals("document type", tags.getResults().get(i).getKey()); + assertEquals("Memorandum", tags.getResults().get(i++).getValue()); + + assertEquals("location", tags.getResults().get(i).getKey()); + assertEquals("YellowBelly Brewery Pub, St. Johns, NL", tags.getResults().get(i++).getValue()); + + assertEquals("organization", tags.getResults().get(i).getKey()); + assertEquals("Great Auk Enterprises", tags.getResults().get(i++).getValue()); + + assertEquals("person", tags.getResults().get(i).getKey()); + assertEquals("Thomas Bewick,Ketill Ketilsson,Farley Mowat,Aaron Thomas", + String.join(",", tags.getResults().get(i++).getValues())); + + assertEquals("subject", tags.getResults().get(i).getKey()); + assertEquals("MINUTES OF A MEETING OF DIRECTORS", tags.getResults().get(i++).getValue()); + } + } + + /** + * Handle documentTagging ChatApt Action extra quotes. + * + * @throws Exception Exception + */ + @Test + public void testDocumentTaggingAction07() throws Exception { + + initProcessor("fulltext", "chatgpt5"); + + for (String siteId : Arrays.asList(null, UUID.randomUUID().toString())) { + // given + configService.save(siteId, new DynamicObject(Map.of(CHATGPT_API_KEY, "asd"))); + + String documentId = UUID.randomUUID().toString(); + + DocumentItem item = new DocumentItemDynamoDb(documentId, new Date(), "joe"); + item.setContentType("text/plain"); + + String s3Key = SiteIdKeyGenerator.createS3Key(siteId, documentId); + String content = "this is some data"; + s3Service.putObject(BUCKET_NAME, s3Key, content.getBytes(StandardCharsets.UTF_8), + "text/plain"); + + documentService.saveDocument(siteId, item, null); + documentService.addTags(siteId, documentId, Arrays.asList(new DocumentTag(documentId, + "untagged", "", new Date(), "joe", DocumentTagType.SYSTEMDEFINED)), null); + + List actions = Arrays.asList(new Action().type(ActionType.DOCUMENTTAGGING) + .parameters(Map.of("engine", "chatgpt", "tags", + "document type,meeting date,chairperson,secretary,board members,resolutions"))); + actionsService.saveActions(siteId, documentId, actions); + + Map map = + loadFileAsMap(this, "/actions-event01.json", "c2695f67-d95e-4db0-985e-574168b12e57", + documentId, "default", siteId != null ? siteId : "default"); + + // when + processor.handleRequest(map, this.context); + + // then + final int expectedSize = 6; + assertEquals(ActionStatus.COMPLETE, + actionsService.getActions(siteId, documentId).get(0).status()); + + PaginationResults tags = + documentService.findDocumentTags(siteId, documentId, null, MAX_RESULTS); + assertEquals(expectedSize, tags.getResults().size()); + + int i = 0; + assertEquals("board members", tags.getResults().get(i).getKey()); + assertEquals("Thomas Bewick,Ketill Ketilsson,Farley Mowat", + Strings.join(tags.getResults().get(i++).getValues(), ",")); + + assertEquals("chairperson", tags.getResults().get(i).getKey()); + assertEquals("Thomas Bewick", tags.getResults().get(i++).getValue()); + + assertEquals("document type", tags.getResults().get(i).getKey()); + assertEquals("Minutes of the Director's Meeting", tags.getResults().get(i++).getValue()); + + assertEquals("meeting date", tags.getResults().get(i).getKey()); + assertEquals("21st day of April, 2023", tags.getResults().get(i++).getValue()); + + assertEquals("resolutions", tags.getResults().get(i).getKey()); + assertEquals("individualAppointed: Thomas Bewick", tags.getResults().get(i++).getValue()); + + assertEquals("secretary", tags.getResults().get(i).getKey()); + assertEquals("Aaron Thomas", tags.getResults().get(i++).getValue()); + } + } + /** * Test converting Ocr Parse Types. */ diff --git a/lambda-s3/src/test/resources/chatgpt/response4.json b/lambda-s3/src/test/resources/chatgpt/response4.json new file mode 100644 index 000000000..05f57c9c6 --- /dev/null +++ b/lambda-s3/src/test/resources/chatgpt/response4.json @@ -0,0 +1,22 @@ +{ + "id": "cmpl-7HOmfBlUrif1MEDOLEFEtJ44A9iGq", + "object": "text_completion", + "created": 1684381201, + "model": "text-davinci-003", + "choices": + [ + { + "text": "\n\n{\n \"Organization\": [\"'Great Auk Enterprises'\"],\n \"Location\": [\"YellowBelly Brewery Pub, St. Johns, NL\"],\n \"Person\": [\"Thomas Bewick\", \"Ketill Ketilsson\", \"Farley Mowat\", \"Aaron Thomas\"], \n \"Subject\": [\"MINUTES OF A MEETING OF DIRECTORS\"], \n \"Sentiment\": [], \n \"Document_Type\": [\"Memorandum\"] \n}", + "index": 0, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": + { + "prompt_tokens": 255, + "completion_tokens": 106, + "total_tokens": 361 + } +} + diff --git a/lambda-s3/src/test/resources/chatgpt/response5.json b/lambda-s3/src/test/resources/chatgpt/response5.json new file mode 100644 index 000000000..40e86b31f --- /dev/null +++ b/lambda-s3/src/test/resources/chatgpt/response5.json @@ -0,0 +1,21 @@ +{ + "id": "cmpl-7bwGRyeJOxasYu6ucPx5tP0pH1ohf", + "object": "text_completion", + "created": 1689276459, + "model": "text-davinci-003", + "choices": + [ + { + "text": " board: Thomas Bewick, Ketill Ketilsson, and Farley Mowat.\n\n{\n \"documentType\": \"Minutes of the Director's Meeting\",\n \"meetingDate\": \"21st day of April, 2023\", \n \"chairperson\": \"Thomas Bewick\", \n \"secretary\": \"Aaron Thomas\", \n \"boardMembers\": [\"Thomas Bewick\",\"Ketill Ketilsson\",\"Farley Mowat\"], \n \"resolutions\": [{\"individualAppointed\":\"Thomas Bewick\"}] \n}", + "index": 0, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": + { + "prompt_tokens": 477, + "completion_tokens": 127, + "total_tokens": 604 + } +} \ No newline at end of file