reactivate ARTE.FR

ARD: update topics search depth BR: migrate to ARD crawler
mediathekview · Sep 23, 2023 · 0eead7b · 0eead7b
2 parents 0158f50 + 44c112c
commit 0eead7b
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 21 deletions.
diff --git a/build.gradle b/build.gradle
@@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
 targetCompatibility = JavaVersion.VERSION_17
 group = 'de.mediathekview'
 archivesBaseName = "MServer"
-version = '3.1.219'
+version = '3.1.220'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'

diff --git a/src/main/java/mServer/crawler/sender/arte/ArteCategoryFilmListDeserializer.java b/src/main/java/mServer/crawler/sender/arte/ArteCategoryFilmListDeserializer.java
@@ -41,17 +41,19 @@ public ArteCategoryFilmsDTO deserialize(JsonElement aJsonElement, Type aType, Js
         final JsonObject contentObject = jsonElement.getAsJsonObject().get(JSON_ELEMENT_CONTENT).getAsJsonObject();
         if (contentObject.has(JSON_ELEMENT_DATA)) {
           for(JsonElement dataElement : contentObject.get(JSON_ELEMENT_DATA).getAsJsonArray()) {
-            String programId = dataElement.getAsJsonObject().get(JSON_ELEMENT_PROGRAMID).getAsString();
-            if (programId != null) {
-              if (programId.startsWith("RC-")) {
-                try {
-                  long collectionId = Long.parseLong(programId.replace("RC-", ""));
-                  dto.addCollection(String.format("RC-%06d", collectionId));
-                } catch (NumberFormatException e) {
-                  Log.errorLog(12834939, "Invalid collection id: " + programId);
+            if (!dataElement.getAsJsonObject().get(JSON_ELEMENT_PROGRAMID).isJsonNull()) {
+              String programId = dataElement.getAsJsonObject().get(JSON_ELEMENT_PROGRAMID).getAsString();
+              if (programId != null) {
+                if (programId.startsWith("RC-")) {
+                  try {
+                    long collectionId = Long.parseLong(programId.replace("RC-", ""));
+                    dto.addCollection(String.format("RC-%06d", collectionId));
+                  } catch (NumberFormatException e) {
+                    Log.errorLog(12834939, "Invalid collection id: " + programId);
+                  }
+                } else {
+                  dto.addProgramId(programId);
                 }
-              } else {
-                dto.addProgramId(programId);
               }
             }
           }

diff --git a/src/main/java/mServer/crawler/sender/arte/ArteCategoryFilmsDTO.java b/src/main/java/mServer/crawler/sender/arte/ArteCategoryFilmsDTO.java
@@ -1,11 +1,12 @@
 package mServer.crawler.sender.arte;
 
-import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
 
 public class ArteCategoryFilmsDTO {
 
-  private final ArrayList<String> programIds = new ArrayList<>();
-  private final ArrayList<String> collectionIds = new ArrayList<>();
+  private final Set<String> programIds = new HashSet<>();
+  private final Set<String> collectionIds = new HashSet<>();
 
   private boolean hasNextPage;
 
@@ -16,10 +17,10 @@ public void addCollection(String aCollectionId) {
     collectionIds.add(aCollectionId);
   }
 
-  public ArrayList<String> getProgramIds() {
+  public Set<String> getProgramIds() {
     return programIds;
   }
-  public ArrayList<String> getCollectionIds() {
+  public Set<String> getCollectionIds() {
     return collectionIds;
   }
 

diff --git a/src/main/java/mServer/crawler/sender/arte/MediathekArte.java b/src/main/java/mServer/crawler/sender/arte/MediathekArte.java
@@ -84,7 +84,7 @@ protected synchronized void meldungStart() {
     super.meldungStart();
 
     senderLanguages.put(Const.ARTE_DE, "de");
-  //  senderLanguages.put(Const.ARTE_FR, "fr");
+    senderLanguages.put(Const.ARTE_FR, "fr");
     /*if (LocalDate.now().getDayOfYear() % 2 == 0) {
       senderLanguages.put(ARTE_EN, "en");
       senderLanguages.put(ARTE_ES, "es");
@@ -233,11 +233,11 @@ private void loadCategory(String sender, String langCode, String aCategory, Stri
       ArteCategoryFilmsDTO dto = loadSubCategoryPage(gson, sender, aUrl);
       if (dto != null) {
         loadCollections(sender, langCode, gsonCollectionParent, gsonCollectionChild, dto);
-        Log.sysLog(String.format("%s: %d, %d", aCategory, dto.getProgramIds().size(), dto.getCollectionIds().size()));
+        Log.sysLog(String.format("%s: category %s: %d programs, %d collections", sender, aCategory, dto.getProgramIds().size(), dto.getCollectionIds().size()));
         // alle programIds verarbeiten
         ListeFilme loadedFilme = loadPrograms(sender, langCode, dto);
         loadedFilme.forEach(film -> addFilm(film));
-        Log.sysLog(String.format("%s: Subcategory %s: %d Filme", sender, aCategory, loadedFilme.size()));
+        Log.sysLog(String.format("%s: category %s: %d Filme", sender, aCategory, loadedFilme.size()));
       }
     }
 

diff --git a/...st/developTest/java/mServer/crawler/sender/arte/ArteCategoryFilmListDeserializerTest.java b/...st/developTest/java/mServer/crawler/sender/arte/ArteCategoryFilmListDeserializerTest.java
@@ -7,9 +7,9 @@
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Set;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
@@ -45,7 +45,7 @@ public void testDeserialize() {
 
     assertThat(actual, notNullValue());
     assertThat(actual.hasNextPage(), equalTo(expectedHasNextPage));
-    ArrayList<String> actualProgramIds = actual.getProgramIds();
+    Set<String> actualProgramIds = actual.getProgramIds();
     assertThat(actualProgramIds, Matchers.containsInAnyOrder(expectedProgramIds));
   }
 }