diff --git a/README.md b/README.md index 1c5e2e73..fff6be40 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,24 @@ The xAPI Java Client has a Spring AutoConfiguration bean which picks up the foll | xapi.client.password | Password for basic authorization header | | xapi.client.authorization | Authorization header (has precedence over the username and password properties) | -Properties can be set using any [external configuration](https://docs.spring.io/spring-boot/docs/3.0.4/reference/htmlsingle/#features.external-config.files) method supported by Spring Boot. +Properties can be set using any [external configuration](https://docs.spring.io/spring-boot/docs/3.0.6/reference/htmlsingle/#features.external-config.files) method supported by Spring Boot. If you need more specific customization (eg. your LRS needs specific headers, or you want to set the authorization header dynamically) you can create a custom configurer by implementing the `XapiClientConfigurer` interface. +### Advanced Configuration + +The xAPI Java Client uses the Spring WebClient. Spring WebClient has default memory limit of 256KB for buffering data. If this limit is exceeded then a DataBufferLimitException will be thrown. + +The default memory limit of 256KB for buffering data could be exceeded if the LRS returns a large number of Statements or if the Statements contain attachments. + +It is possible to set the memory limit for buffering data with the `spring.codec.max-in-memory-size` property. + +Example: + +``` +spring.codec.max-in-memory-size=1MB +``` + ### Statement Resource The xAPI Java Client allows applications to store and fetch xAPI [Statements](https://github.com/adlnet/xAPI-Spec/blob/master/xAPI-Data.md#statements). @@ -101,6 +115,20 @@ StatementResult moreStatementResult = moreResponse.getBody(); Statement[] statements = moreStatementResult.getStatements(); ``` +### Getting Statements as Iterator (and processing them as a Stream) + +`getStatementIterator()` is convenient method a which combines the functionality of `getStatments()` and `getMoreStatements()`. In most cases it is preferable to use getStatementIterator() instead of `getStatments()` and `getMoreStatements()`. + +Example: + +```java +var statements = client.getStatementIterator().block(); + +// process the first 100 Statements +statements.toStream().limit(100).forEach(s -> { + // add logic here... + }); +``` ### Posting a Statement diff --git a/samples/get-statement-iterator/pom.xml b/samples/get-statement-iterator/pom.xml new file mode 100644 index 00000000..4b3fed5d --- /dev/null +++ b/samples/get-statement-iterator/pom.xml @@ -0,0 +1,22 @@ + + + 4.0.0 + + dev.learning.xapi.samples + xapi-samples-build + 1.1.5-SNAPSHOT + + get-statement-iterator + Get xAPI StatementIterator Sample + Get xAPI StatementIterator + + + dev.learning.xapi + xapi-client + + + dev.learning.xapi.samples + core + + + diff --git a/samples/get-statement-iterator/src/main/java/dev/learning/xapi/samples/getstatements/GetStatementIteratorApplication.java b/samples/get-statement-iterator/src/main/java/dev/learning/xapi/samples/getstatements/GetStatementIteratorApplication.java new file mode 100644 index 00000000..35395be4 --- /dev/null +++ b/samples/get-statement-iterator/src/main/java/dev/learning/xapi/samples/getstatements/GetStatementIteratorApplication.java @@ -0,0 +1,51 @@ +/* + * Copyright 2016-2023 Berry Cloud Ltd. All rights reserved. + */ + +package dev.learning.xapi.samples.getstatements; + +import dev.learning.xapi.client.XapiClient; +import dev.learning.xapi.model.Verb; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +/** + * Sample using xAPI client to get multiple statements as StatementIterator. + * + * @author Thomas Turrell-Croft + * @author István Rátkai (Selindek) + */ +@SpringBootApplication +public class GetStatementIteratorApplication implements CommandLineRunner { + + /** + * Default xAPI client. Properties are picked automatically from application.properties. + */ + @Autowired + private XapiClient client; + + public static void main(String[] args) { + SpringApplication.run(GetStatementIteratorApplication.class, args).close(); + } + + @Override + public void run(String... args) throws Exception { + + // Get Statements as StatementIterator + var statements = client.getStatementIterator().block(); + + // Print the returned statements to the console + statements.toStream().forEach(s -> System.out.println(s)); + + // Get Statements with Verb filter as StatementIterator + var filteredStatements = + client.getStatementIterator(r -> r.verb(Verb.ATTEMPTED)).block(); + + // Print the returned statements to the console + filteredStatements.toStream().forEach(s -> System.out.println(s)); + + } + +} diff --git a/samples/get-statement-iterator/src/main/resources/application.properties b/samples/get-statement-iterator/src/main/resources/application.properties new file mode 100644 index 00000000..379e502a --- /dev/null +++ b/samples/get-statement-iterator/src/main/resources/application.properties @@ -0,0 +1,5 @@ +xapi.client.username = admin +xapi.client.password = password +xapi.client.baseUrl = https://example.com/xapi/ + +spring.codec.max-in-memory-size=1MB diff --git a/samples/get-statement-with-attachment/src/main/resources/application.properties b/samples/get-statement-with-attachment/src/main/resources/application.properties index de20217a..379e502a 100644 --- a/samples/get-statement-with-attachment/src/main/resources/application.properties +++ b/samples/get-statement-with-attachment/src/main/resources/application.properties @@ -1,3 +1,5 @@ xapi.client.username = admin xapi.client.password = password xapi.client.baseUrl = https://example.com/xapi/ + +spring.codec.max-in-memory-size=1MB diff --git a/samples/get-statements/src/main/java/dev/learning/xapi/samples/getstatements/GetStatementsApplication.java b/samples/get-statements/src/main/java/dev/learning/xapi/samples/getstatements/GetStatementsApplication.java index e04a6de7..48b32a49 100644 --- a/samples/get-statements/src/main/java/dev/learning/xapi/samples/getstatements/GetStatementsApplication.java +++ b/samples/get-statements/src/main/java/dev/learning/xapi/samples/getstatements/GetStatementsApplication.java @@ -7,7 +7,6 @@ import dev.learning.xapi.client.XapiClient; import dev.learning.xapi.model.StatementResult; import dev.learning.xapi.model.Verb; -import java.util.Arrays; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; @@ -39,7 +38,7 @@ public void run(String... args) throws Exception { ResponseEntity response = client.getStatements().block(); // Print the returned statements to the console - Arrays.asList(response.getBody().getStatements()).forEach(s -> System.out.println(s)); + response.getBody().getStatements().forEach(s -> System.out.println(s)); @@ -48,7 +47,7 @@ public void run(String... args) throws Exception { client.getStatements(r -> r.verb(Verb.ATTEMPTED.getId())).block(); // Print the returned statements to the console - Arrays.asList(filteredResponse.getBody().getStatements()).forEach(s -> System.out.println(s)); + filteredResponse.getBody().getStatements().forEach(s -> System.out.println(s)); } diff --git a/samples/get-statements/src/main/resources/application.properties b/samples/get-statements/src/main/resources/application.properties index de20217a..379e502a 100644 --- a/samples/get-statements/src/main/resources/application.properties +++ b/samples/get-statements/src/main/resources/application.properties @@ -1,3 +1,5 @@ xapi.client.username = admin xapi.client.password = password xapi.client.baseUrl = https://example.com/xapi/ + +spring.codec.max-in-memory-size=1MB diff --git a/samples/pom.xml b/samples/pom.xml index 66a96cc3..e90caf89 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -35,6 +35,7 @@ core get-statement + get-statement-iterator get-statement-with-attachment post-statement post-signed-statement diff --git a/xapi-client/src/main/java/dev/learning/xapi/client/GetStatementsRequest.java b/xapi-client/src/main/java/dev/learning/xapi/client/GetStatementsRequest.java index 4b5bd133..b8aba64a 100644 --- a/xapi-client/src/main/java/dev/learning/xapi/client/GetStatementsRequest.java +++ b/xapi-client/src/main/java/dev/learning/xapi/client/GetStatementsRequest.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.learning.xapi.model.Agent; import dev.learning.xapi.model.StatementFormat; +import dev.learning.xapi.model.Verb; import java.net.URI; import java.time.Instant; import java.util.Map; @@ -179,6 +180,20 @@ public Builder verb(String verb) { return this; } + /** + * Sets the verb. + * + * @param verb The verb of the GetStatementRequest. + * + * @return This builder + * + * @see GetStatementsRequest#verb + */ + public Builder verb(Verb verb) { + this.verb = verb.getId(); + return this; + } + /** * Sets the activity. * diff --git a/xapi-client/src/main/java/dev/learning/xapi/client/XapiClient.java b/xapi-client/src/main/java/dev/learning/xapi/client/XapiClient.java index 0de6facd..7f7466a3 100644 --- a/xapi-client/src/main/java/dev/learning/xapi/client/XapiClient.java +++ b/xapi-client/src/main/java/dev/learning/xapi/client/XapiClient.java @@ -6,16 +6,26 @@ import dev.learning.xapi.model.About; import dev.learning.xapi.model.Activity; +import dev.learning.xapi.model.Actor; import dev.learning.xapi.model.Person; import dev.learning.xapi.model.Statement; import dev.learning.xapi.model.StatementResult; +import dev.learning.xapi.model.Verb; +import java.net.URI; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.UUID; import java.util.function.Consumer; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import lombok.RequiredArgsConstructor; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.ResponseEntity; +import org.springframework.util.Assert; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; @@ -1257,4 +1267,191 @@ public Mono> getAbout() { } + // Enhanced features + + /** + * Gets a list of Statements as a {@link StatementIterator}. + *

+ * This method loads ALL of Statements which fullfills the request filters from the LRS + * dynamically. (It sends additional + * {@link XapiClient#getMoreStatements(java.util.function.Consumer)} request if all the previously + * loaded Statements were processed from the iterator.) + *

+ * + * @param request The parameters of the get statements request + * + * @return a {@link StatementIterator} object as a {@link Mono}. + */ + public Mono getStatementIterator(GetStatementsRequest request) { + + return getStatements(request).map(result -> new StatementIterator(result)); + + } + + /** + * Gets a list of Statements as a {@link StatementIterator}. + *

+ * This method loads ALL of Statements which fullfills the request filters from the LRS + * dynamically. (It sends additional + * {@link XapiClient#getMoreStatements(java.util.function.Consumer)} request if all the previously + * loaded Statements were processed from the iterator.) + *

+ * + * @param request The parameters of the get statements request + * + * @return a {@link StatementIterator} object as a {@link Mono}. + */ + public Mono getStatementIterator( + Consumer request) { + + final var builder = GetStatementsRequest.builder(); + + request.accept(builder); + + return getStatementIterator(builder.build()); + + } + + /** + * Gets all of the Statements as a {@link StatementIterator}. + *

+ * This method loads ALL of Statements which fullfills the request filters from the LRS + * dynamically. (It sends additional + * {@link XapiClient#getMoreStatements(java.util.function.Consumer)} request if all the previously + * loaded Statements were processed from the iterator.) + *

+ * + * @return a {@link StatementIterator} object as a {@link Mono}. + */ + public Mono getStatementIterator() { + + return getStatementIterator(r -> { + }); + + } + + /** + *

+ * Voids a {@link Statement}. + *

+ * The Actor of the voiding statement will be the same as the Actor of the target Statement. + *

+ * The returned ResponseEntity contains the response headers and the Statement identifier of the + * generated voiding Statement. + *

+ * + * @param targetStatement The {@link Statement} to be voided + * + * @return the ResponseEntity + */ + public Mono> voidStatement(Statement targetStatement) { + return voidStatement(targetStatement, targetStatement.getActor()); + } + + /** + * Voids a {@link Statement}. + *

+ * The returned ResponseEntity contains the response headers and the Statement identifier of the + * generated voiding Statement. + *

+ * + * @param targetStatement The {@link Statement} to be voided + * @param actor the Actor of the voiding Statement + * + * @return the ResponseEntity + */ + public Mono> voidStatement(Statement targetStatement, Actor actor) { + return voidStatement(targetStatement.getId(), actor); + } + + /** + * Voids a {@link Statement}. + *

+ * The returned ResponseEntity contains the response headers and the Statement identifier of the + * generated voiding Statement. + *

+ * + * @param targetStatementId The id of the {@link Statement} to be voided + * @param actor the Actor of the voiding Statement + * + * @return the ResponseEntity + */ + public Mono> voidStatement(UUID targetStatementId, Actor actor) { + Assert.notNull(targetStatementId, "Target Statement id cannot be null"); + Assert.notNull(actor, "Actor cannot be null"); + + return postStatement(r -> r + + .statement(s -> s + + .actor(actor) + + .verb(Verb.VOIDED) + + .statementReferenceObject(o -> o + + .id(targetStatementId) + + ) + + ) + + ); + + } + + /** + *

+ * StatementIterator. + *

+ * Iterates through the Statements of the result of a + * {@link XapiClient#getStatements(GetStatementsRequest)}. If more Statements are available it + * automatically loads them from the server. + * + * @author István Rátkai (Selindek) + */ + @RequiredArgsConstructor + public class StatementIterator implements Iterator { + + private URI more; + private Iterator statements; + + private StatementIterator(ResponseEntity response) { + init(response); + } + + /** + * Convenient method for transforming this StatementIterator to a {@link Stream}. + * + * @return a {@link Stream} of {@link Statement}s + */ + public Stream toStream() { + final Iterable iterable = () -> this; + return StreamSupport.stream(iterable.spliterator(), false); + } + + private void init(ResponseEntity response) { + final var statementResult = response.getBody(); + more = statementResult.hasMore() ? statementResult.getMore() : null; + final var s = statementResult.getStatements(); + statements = s == null ? Collections.emptyIterator() : s.iterator(); + } + + @Override + public boolean hasNext() { + return statements.hasNext() || more != null; + } + + @Override + public Statement next() { + if (!statements.hasNext()) { + if (more == null) { + throw new NoSuchElementException(); + } + init(getMoreStatements(r -> r.more(more)).block()); + } + return statements.next(); + } + + } } diff --git a/xapi-client/src/test/java/dev/learning/xapi/client/XapiClientTests.java b/xapi-client/src/test/java/dev/learning/xapi/client/XapiClientTests.java index dcc9ebff..85f4252d 100644 --- a/xapi-client/src/test/java/dev/learning/xapi/client/XapiClientTests.java +++ b/xapi-client/src/test/java/dev/learning/xapi/client/XapiClientTests.java @@ -3,6 +3,7 @@ */ package dev.learning.xapi.client; +import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsInstanceOf.instanceOf; @@ -21,7 +22,9 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.NoSuchElementException; import java.util.UUID; +import java.util.concurrent.TimeUnit; import lombok.Getter; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -515,7 +518,7 @@ void whenGettingStatementsWithAllParametersThenPathIsExpected() throws Interrupt .agent(a -> a.name("A N Other").mbox("mailto:another@example.com")) - .verb("http://adlnet.gov/expapi/verbs/answered") + .verb(Verb.ANSWERED) .activity("https://example.com/activity/1") @@ -2243,6 +2246,270 @@ void whenGettingActivityProfilesWithoutSinceParameterThenPathIsExpected() is("/activities/profile?activityId=https%3A%2F%2Fexample.com%2Factivity%2F1")); } + @Test + void whenGettingStatementIteratorViaMultipeResponsesThenResultIsExpected() + throws InterruptedException { + final var body1 = """ + { + "statements" : [ + { + "id" : "c0aaea0b-252b-4d9d-b7ad-46c541572570" + } + ], + "more" : "/statements/more/1" + } + """; + final var body2 = """ + { + "statements" : [ + { + "id" : "4ed0209a-f50f-4f57-8602-ba5f981d211a" + } + ], + "more" : "" + } + """; + + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body1) + .addHeader("Content-Type", "application/json; charset=utf-8")); + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body2) + .addHeader("Content-Type", "application/json; charset=utf-8")); + + // When Getting StatementIterator Via Multipe Responses + final var iterator = client.getStatementIterator().block(); + + // Then Result Is Expected + assertThat(iterator.hasNext(), is(true)); + assertThat(iterator.next().getId(), + is(UUID.fromString("c0aaea0b-252b-4d9d-b7ad-46c541572570"))); + assertThat(iterator.hasNext(), is(true)); + assertThat(iterator.next().getId(), + is(UUID.fromString("4ed0209a-f50f-4f57-8602-ba5f981d211a"))); + assertThat(iterator.hasNext(), is(false)); + + } + + @Test + void whenGettingStatementIteratorViaMultipeResponsesThenRequestsAreExpected() + throws InterruptedException { + final var body1 = """ + { + "statements" : [ + { + "id" : "c0aaea0b-252b-4d9d-b7ad-46c541572570" + } + ], + "more" : "/statements/more/1" + } + """; + final var body2 = """ + { + "statements" : [ + { + "id" : "4ed0209a-f50f-4f57-8602-ba5f981d211a" + } + ], + "more" : "" + } + """; + + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body1) + .addHeader("Content-Type", "application/json; charset=utf-8")); + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body2) + .addHeader("Content-Type", "application/json; charset=utf-8")); + + // When Getting StatementIterator Via Multipe Responses + final var iterator = client.getStatementIterator().block(); + iterator.next(); + iterator.next(); + + // Then Requests Are Expected + assertThat(mockWebServer.takeRequest().getPath(), is("/statements")); + assertThat(mockWebServer.takeRequest().getPath(), is("/statements/more/1")); + + } + + @Test + void whenGettingStatementIteratorThenRequestsAreExpected() throws InterruptedException { + final var body1 = """ + { + "statements" : [ + { + "id" : "c0aaea0b-252b-4d9d-b7ad-46c541572570" + } + ], + "more" : "/statements/more/1" + } + """; + final var body2 = """ + { + "statements" : [ + { + "id" : "4ed0209a-f50f-4f57-8602-ba5f981d211a" + } + ], + "more" : "" + } + """; + + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body1) + .addHeader("Content-Type", "application/json; charset=utf-8")); + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body2) + .addHeader("Content-Type", "application/json; charset=utf-8")); + + // When Getting StatementIterator + client.getStatementIterator().block(); + + // Then Requests Are Expected + assertThat(mockWebServer.takeRequest().getPath(), is("/statements")); + assertThat(mockWebServer.takeRequest(1, TimeUnit.SECONDS), is(nullValue())); + + } + + @Test + void whenGettingStatementIteratorAndProcessingItAsStreamThenRequestsAreExpected() + throws InterruptedException { + final var body1 = """ + { + "statements" : [ + { + "id" : "c0aaea0b-252b-4d9d-b7ad-46c541572570" + }, + { + "id" : "940a3f5c-1f31-47c7-82fc-5979e2786c02" + } + ], + "more" : "/statements/more/1" + } + """; + final var body2 = """ + { + "statements" : [ + { + "id" : "4ed0209a-f50f-4f57-8602-ba5f981d211a" + } + ], + "more" : "" + } + """; + + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body1) + .addHeader("Content-Type", "application/json; charset=utf-8")); + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body2) + .addHeader("Content-Type", "application/json; charset=utf-8")); + + // When Getting StatementIterator + final var iterator = client.getStatementIterator().block(); + + // And Processing it As Stream + iterator.toStream().limit(1).forEach(s -> { + }); + + // Then Requests Are Expected + assertThat(mockWebServer.takeRequest().getPath(), is("/statements")); + assertThat(mockWebServer.takeRequest(1, TimeUnit.SECONDS), is(nullValue())); + + } + + @Test + void givenEmptyStatementResultWhenGettingStatementIteratorThenHasNextIsFalse() + throws InterruptedException { + + // Given Empty StatementResult + final var body = """ + { + "statements" : [ + ], + "more" : "" + } + """; + + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body) + .addHeader("Content-Type", "application/json; charset=utf-8")); + + // When Getting StatementIterator + final var iterator = client.getStatementIterator().block(); + + // Then HasNext Is False + assertThat(iterator.hasNext(), is(false)); + + } + + @Test + void givenEmptyResponseWhenGettingStatementIteratorThenHasNextIsFalse() + throws InterruptedException { + + // Given Empty Response + // This response is technically invalid by the xAPI specification, but we cannot assume + // conformance. + // conformance of the commercial LRSs. + final var body = "{}"; + + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body) + .addHeader("Content-Type", "application/json; charset=utf-8")); + + // When Getting StatementIterator + final var iterator = client.getStatementIterator().block(); + + // Then HasNext Is False + assertThat(iterator.hasNext(), is(false)); + + } + + @Test + void givenEmptyResponseWhenGettingStatementIteratorThenNextThrowsAnException() + throws InterruptedException { + + // Given Empty Response + final var body = """ + { + "statements" : [ + ], + "more" : "" + } + """; + + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK").setBody(body) + .addHeader("Content-Type", "application/json; charset=utf-8")); + + // When Getting StatementIterator + final var iterator = client.getStatementIterator().block(); + + // Then Next Throws An Exception + assertThrows(NoSuchElementException.class, () -> iterator.next()); + + } + + @Test + void whenVoidingStatementThenBodyIsExpected() throws InterruptedException { + + mockWebServer.enqueue(new MockResponse().setStatus("HTTP/1.1 200 OK") + .setBody("[\"2eb84e56-441a-492c-9d7b-f8e9ddd3e15d\"]") + .addHeader("Content-Type", "application/json")); + + final var attemptedStatement = Statement.builder() + + .id(UUID.fromString("175c9264-692f-4108-9b7d-0ba64bd59ac3")) + + .agentActor(a -> a.name("A N Other").mbox("mailto:another@example.com")) + + .verb(Verb.ATTEMPTED) + + .activityObject(o -> o.id("https://example.com/activity/simplestatement") + .definition(d -> d.addName(Locale.ENGLISH, "Simple Statement"))) + + .build(); + + // When Voiding Statement + client.voidStatement(attemptedStatement).block(); + + final var recordedRequest = mockWebServer.takeRequest(); + + // Then Body Is Expected + assertThat(recordedRequest.getBody().readUtf8(), is( + "{\"actor\":{\"objectType\":\"Agent\",\"name\":\"A N Other\",\"mbox\":\"mailto:another@example.com\"},\"verb\":{\"id\":\"http://adlnet.gov/expapi/verbs/voided\",\"display\":{\"und\":\"voided\"}},\"object\":{\"objectType\":\"StatementRef\",\"id\":\"175c9264-692f-4108-9b7d-0ba64bd59ac3\"}}")); + } + @Getter private static class SamplePerson {