feat: retries for translations build (#667)

src/main/java/com/crowdin/cli/client/CrowdinClientCore.java

@@ -23,7 +23,7 @@
 
 abstract class CrowdinClientCore {
 
-    private static final long millisToRetry = 100;
+    private static final long defaultMillisToRetry = 100;
 
     private static final Map<BiPredicate<String, String>, RuntimeException> standardErrorHandlers =
         new LinkedHashMap<BiPredicate<String, String>, RuntimeException>() {{
@@ -67,32 +67,23 @@ protected static <T> List<T> executeRequestFullList(BiFunction<Integer, Integer,
         return directories;
     }
 
-    protected static <T> T executeRequestWithPossibleRetry(BiPredicate<String, String> expectedError, Supplier<T> request) {
-        Map<BiPredicate<String, String>, RepeatException> errorHandler = new LinkedHashMap<BiPredicate<String, String>, RepeatException>() {{
-                put(expectedError, new RepeatException());
-            }};
-        try {
-            return executeRequest(errorHandler, request);
-        } catch (RepeatException e) {
-            try {
-                Thread.sleep(millisToRetry);
-            } catch (InterruptedException ie) {
-//                    ignore
-            }
-            return executeRequest(request);
-        }
+    protected static <T> T executeRequestWithPossibleRetry(Map<BiPredicate<String, String>, ResponseException> errorHandlers, Supplier<T> request) throws ResponseException {
+        return executeRequestWithPossibleRetries(errorHandlers, request, 2, defaultMillisToRetry);
     }
 
-    protected static <T> T executeRequestWithPossibleRetry(Map<BiPredicate<String, String>, ResponseException> errorHandlers, Supplier<T> request) throws ResponseException {
+    protected static <T> T executeRequestWithPossibleRetries(Map<BiPredicate<String, String>, ResponseException> errorHandlers, Supplier<T> request, int maxAttempts, long millisToRetry) throws ResponseException {
+        if (maxAttempts < 1) {
+            throw new MaxNumberOfRetriesException();
+        }
         try {
             return executeRequest(errorHandlers, request);
         } catch (RepeatException e) {
             try {
                 Thread.sleep(millisToRetry);
             } catch (InterruptedException ie) {
-//                    ignore
+//              ignore
             }
-            return executeRequest(request);
+            return executeRequestWithPossibleRetries(errorHandlers, request, maxAttempts - 1, millisToRetry);
         }
     }
 

src/main/java/com/crowdin/cli/client/CrowdinProjectClient.java

@@ -253,10 +253,17 @@ public void uploadTranslations(String languageId, UploadTranslationsRequest requ
     }
 
     @Override
-    public ProjectBuild startBuildingTranslation(BuildProjectTranslationRequest request) {
-        return executeRequest(() -> this.client.getTranslationsApi()
-            .buildProjectTranslation(this.projectId, request)
-            .getData());
+    public ProjectBuild startBuildingTranslation(BuildProjectTranslationRequest request) throws ResponseException {
+        Map<BiPredicate<String, String>, ResponseException> errorHandler = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
+            put((code, message) -> code.equals("409") && message.contains("Another build is currently in progress"),
+                new RepeatException());
+        }};
+        return executeRequestWithPossibleRetries(
+            errorHandler,
+            () -> this.client.getTranslationsApi().buildProjectTranslation(this.projectId, request).getData(),
+            3,
+            60 * 100
+        );
     }
 
     @Override

src/main/java/com/crowdin/cli/client/MaxNumberOfRetriesException.java

@@ -0,0 +1,4 @@
+package com.crowdin.cli.client;
+
+public class MaxNumberOfRetriesException extends ResponseException {
+}

src/main/java/com/crowdin/cli/client/ProjectClient.java

@@ -61,7 +61,7 @@ default CrowdinProjectFull downloadFullProject() {
 
     void uploadTranslations(String languageId, UploadTranslationsRequest request) throws ResponseException;
 
-    ProjectBuild startBuildingTranslation(BuildProjectTranslationRequest request);
+    ProjectBuild startBuildingTranslation(BuildProjectTranslationRequest request) throws ResponseException;
 
     ProjectBuild checkBuildingTranslation(Long buildId);
 

src/main/java/com/crowdin/cli/commands/actions/DownloadAction.java

@@ -1,8 +1,6 @@
 package com.crowdin.cli.commands.actions;
 
-import com.crowdin.cli.client.CrowdinProjectFull;
-import com.crowdin.cli.client.LanguageMapping;
-import com.crowdin.cli.client.ProjectClient;
+import com.crowdin.cli.client.*;
 import com.crowdin.cli.commands.NewAction;
 import com.crowdin.cli.commands.Outputter;
 import com.crowdin.cli.commands.functionality.FilesInterface;
@@ -327,6 +325,8 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
                     out.println(EMPTY.withIcon(RESOURCE_BUNDLE.getString("message.faq_link")));
                 }
             }
+        } catch (ProjectBuildFailedException e) {
+            out.println(WARNING.withIcon(RESOURCE_BUNDLE.getString("message.translations_build_unsuccessful")));
         } finally {
             try {
                 for (File tempDir : tempDirs.keySet()) {
@@ -348,6 +348,9 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
      */
     private Pair<File, List<String>> download(BuildProjectTranslationRequest request, ProjectClient client, String basePath, Boolean keepArchive) {
         ProjectBuild projectBuild = buildTranslation(client, request);
+        if (projectBuild == null) {
+            throw new ProjectBuildFailedException();
+        }
         String randomHash = RandomStringUtils.random(11, false, true);
         File baseTempDir =
             new File(StringUtils.removeEnd(
@@ -418,23 +421,27 @@ private ProjectBuild buildTranslation(ProjectClient client, BuildProjectTranslat
             this.noProgress,
             this.plainView,
             () -> {
-                ProjectBuild build = client.startBuildingTranslation(request);
+                ProjectBuild build = null;
+                try {
+                    build = client.startBuildingTranslation(request);
 
-                while (!build.getStatus().equalsIgnoreCase("finished")) {
-                    ConsoleSpinner.update(
-                        String.format(RESOURCE_BUNDLE.getString("message.building_translation"),
-                            Math.toIntExact(build.getProgress())));
+                    while (!build.getStatus().equalsIgnoreCase("finished")) {
+                        ConsoleSpinner.update(
+                            String.format(RESOURCE_BUNDLE.getString("message.building_translation"),
+                                Math.toIntExact(build.getProgress())));
 
-                    Thread.sleep(sleepTime.getAndUpdate(val -> val < CHECK_WAITING_TIME_MAX ? val + CHECK_WAITING_TIME_INCREMENT : CHECK_WAITING_TIME_MAX));
+                        Thread.sleep(sleepTime.getAndUpdate(val -> val < CHECK_WAITING_TIME_MAX ? val + CHECK_WAITING_TIME_INCREMENT : CHECK_WAITING_TIME_MAX));
 
-                    build = client.checkBuildingTranslation(build.getId());
+                        build = client.checkBuildingTranslation(build.getId());
 
-                    if (build.getStatus().equalsIgnoreCase("failed")) {
-                        throw new RuntimeException(RESOURCE_BUNDLE.getString("message.spinner.build_has_failed"));
+                        if (build.getStatus().equalsIgnoreCase("failed")) {
+                            throw new RuntimeException(RESOURCE_BUNDLE.getString("message.spinner.build_has_failed"));
+                        }
                     }
+                    ConsoleSpinner.update(String.format(RESOURCE_BUNDLE.getString("message.building_translation"), 100));
+                } catch (MaxNumberOfRetriesException e) {
+                    ConsoleSpinner.stop(WARNING, RESOURCE_BUNDLE.getString("message.warning.another_build_in_progress"));
                 }
-
-                ConsoleSpinner.update(String.format(RESOURCE_BUNDLE.getString("message.building_translation"), 100));
                 return build;
             }
         );
@@ -555,4 +562,6 @@ private Set<Pair<String, String>> flattenInnerMap(Collection<Map<String, String>
         }
         return result;
     }
+
+    private static class ProjectBuildFailedException extends RuntimeException { }
 }

src/main/resources/messages/messages.properties

@@ -721,6 +721,7 @@ message.warning.file_not_uploaded_cause_of_language=Translation file @|yellow,bo
 message.warning.auto_approve_option_with_mt='--auto-approve-option' is used only for the TM Pre-Translation method
 message.warning.no_file_to_download=Couldn't find any file to download
 message.warning.no_file_to_download_skipuntranslated=Couldn't find any file to download. As you are using the 'Skip untranslated files' option, please make sure you have fully translated files
+message.warning.another_build_in_progress=Another build is currently in progress. Please wait until it's finished
 message.spinner.fetching_project_info=Fetching project info
 message.spinner.building_translation=Building translations
 message.spinner.building_reviewed_sources=Building reviewed sources
@@ -743,6 +744,7 @@ message.spinner.pre_translate_done=Pre-translation is finished @|bold (%d%%)|@
 message.spinner.build_has_failed=The build has failed
 
 message.faq_link=Visit the @|cyan https://crowdin.github.io/crowdin-cli/faq|@ for more details
+message.translations_build_unsuccessful=Didn't manage to build translations
 
 message.tree.elem=@|cyan \u251C\u2500\u0020|@
 message.tree.last_elem=@|cyan \u2570\u2500\u0020|@

src/test/java/com/crowdin/cli/client/CrowdinProjectClientTest.java

@@ -437,7 +437,7 @@ public void testUploadTranslationsWithRepeat() throws ResponseException {
     }
 
     @Test
-    public void testStartBuildingTranslation() {
+    public void testStartBuildingTranslation() throws ResponseException {
         ProjectBuildResponseObject response = new ProjectBuildResponseObject() {{
                 setData(new ProjectBuild());
             }};

src/test/java/com/crowdin/cli/commands/actions/DownloadActionTest.java

@@ -1,9 +1,7 @@
 package com.crowdin.cli.commands.actions;
 
 import com.crowdin.cli.MockitoUtils;
-import com.crowdin.cli.client.ProjectClient;
-import com.crowdin.cli.client.ProjectBuilder;
-import com.crowdin.cli.client.ResponseException;
+import com.crowdin.cli.client.*;
 import com.crowdin.cli.commands.NewAction;
 import com.crowdin.cli.commands.Outputter;
 import com.crowdin.cli.commands.functionality.FilesInterface;
@@ -58,7 +56,7 @@ public static ProjectBuild buildProjectBuild(Long buildId, Long projectId, Strin
     }
 
     @Test
-    public void testEmptyProject() throws IOException {
+    public void testEmptyProject() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
             .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
             .setBasePath(project.getBasePath());
@@ -103,7 +101,7 @@ public void testEmptyProject() throws IOException {
     }
 
     @Test
-    public void testProjectOneFittingFile() throws IOException {
+    public void testProjectOneFittingFile() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
                 .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
                 .setBasePath(project.getBasePath());
@@ -159,7 +157,7 @@ public void testProjectOneFittingFile() throws IOException {
     }
 
     @Test
-    public void testProjectOneFittingFile_WithExportApprovedOnly_WithSkipUntranslatedFiles() throws IOException {
+    public void testProjectOneFittingFile_WithExportApprovedOnly_WithSkipUntranslatedFiles() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
                 .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
                 .setBasePath(project.getBasePath());
@@ -220,7 +218,7 @@ public void testProjectOneFittingFile_WithExportApprovedOnly_WithSkipUntranslate
     }
 
     @Test
-    public void testProjectDownloadWithKeepArchive() throws IOException {
+    public void testProjectDownloadWithKeepArchive() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
                 .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
                 .setBasePath(project.getBasePath());
@@ -275,7 +273,7 @@ public void testProjectDownloadWithKeepArchive() throws IOException {
     }
 
     @Test
-    public void testProjectOneFittingFile_LongBuild() throws IOException {
+    public void testProjectOneFittingFile_LongBuild() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
             .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
             .setBasePath(project.getBasePath());
@@ -336,7 +334,7 @@ public void testProjectOneFittingFile_LongBuild() throws IOException {
     }
 
     @Test
-    public void testProjectOneFittingOneUnfittingFile_LongBuild() throws IOException {
+    public void testProjectOneFittingOneUnfittingFile_LongBuild() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
                 .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
                 .setBasePath(project.getBasePath());
@@ -396,7 +394,7 @@ public void testProjectOneFittingOneUnfittingFile_LongBuild() throws IOException
     }
 
     @Test
-    public void testProjectOneFittingOneUnfittingOneWithUnfoundSourceFile_LongBuild() throws IOException {
+    public void testProjectOneFittingOneUnfittingOneWithUnfoundSourceFile_LongBuild() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
             .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
             .setBasePath(project.getBasePath());
@@ -457,7 +455,7 @@ public void testProjectOneFittingOneUnfittingOneWithUnfoundSourceFile_LongBuild(
     }
 
     @Test
-    public void testProjectOneFittingFile_WithLanguageMapping() throws IOException {
+    public void testProjectOneFittingFile_WithLanguageMapping() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
             .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
             .setBasePath(project.getBasePath());
@@ -519,7 +517,7 @@ public void testProjectOneFittingFile_WithLanguageMapping() throws IOException {
     }
 
     @Test
-    public void testProjectOneFittingFile_UploadedWithoutHierarchy() throws IOException {
+    public void testProjectOneFittingFile_UploadedWithoutHierarchy() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
                 .minimalBuiltPropertiesBean("files" + Utils.PATH_SEPARATOR + "*", Utils.PATH_SEPARATOR + "%original_path%" + Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
                 .setBasePath(project.getBasePath());
@@ -575,7 +573,7 @@ public void testProjectOneFittingFile_UploadedWithoutHierarchy() throws IOExcept
     }
 
     @Test
-    public void testProjectOneFittingFile_FailBuild() {
+    public void testProjectOneFittingFile_FailBuild() throws ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
             .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
             .setBasePath(project.getBasePath());
@@ -608,6 +606,40 @@ public void testProjectOneFittingFile_FailBuild() {
         verifyNoMoreInteractions(files);
     }
 
+    @Test
+    public void testProjectOneFittingFile_FailBuildInProgress() throws ResponseException {
+        NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
+                .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
+                .setBasePath(project.getBasePath());
+        PropertiesWithFiles pb = pbBuilder.build();
+
+        project.addFile("first.po");
+
+        ProjectClient client = mock(ProjectClient.class);
+        when(client.downloadFullProject(null))
+                .thenReturn(ProjectBuilder.emptyProject(Long.parseLong(pb.getProjectId()))
+                        .addFile("first.po", "gettext", 101L, null, null, "/%original_file_name%-CR-%locale%").build());
+        CrowdinTranslationCreateProjectBuildForm buildProjectTranslationRequest = new CrowdinTranslationCreateProjectBuildForm();
+        long buildId = 42L;
+        when(client.startBuildingTranslation(eq(buildProjectTranslationRequest)))
+                .thenThrow(new MaxNumberOfRetriesException());
+        URL urlMock = MockitoUtils.getMockUrl(getClass());
+        when(client.downloadBuild(eq(buildId)))
+                .thenReturn(urlMock);
+
+        FilesInterface files = mock(FilesInterface.class);
+
+        NewAction<PropertiesWithFiles, ProjectClient> action =
+                new DownloadAction(files, false, null, null, false, null, false, false, false, false, false);
+        action.act(Outputter.getDefault(), pb, client);
+
+        verify(client).downloadFullProject(null);
+        verify(client).startBuildingTranslation(eq(buildProjectTranslationRequest));
+        verifyNoMoreInteractions(client);
+
+        verifyNoMoreInteractions(files);
+    }
+
     @Test
     public void testProjectOneFittingFile_failDownloadProject() {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
@@ -634,7 +666,7 @@ public void testProjectOneFittingFile_failDownloadProject() {
     }
 
     @Test
-    public void testProjectOneFittingFile_failDeleteFile() throws IOException {
+    public void testProjectOneFittingFile_failDeleteFile() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
             .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
             .setBasePath(project.getBasePath());
@@ -695,7 +727,7 @@ public void testProjectOneFittingFile_failDeleteFile() throws IOException {
     }
 
     @Test
-    public void testProjectOneFittingFile_failDownloadingException() {
+    public void testProjectOneFittingFile_failDownloadingException() throws ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
                 .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
                 .setBasePath(project.getBasePath());
@@ -729,7 +761,7 @@ public void testProjectOneFittingFile_failDownloadingException() {
     }
 
     @Test
-    public void testProjectOneFittingFile_failWritingFile() throws IOException {
+    public void testProjectOneFittingFile_failWritingFile() throws IOException, ResponseException {
         NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
             .minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
             .setBasePath(project.getBasePath());