Merge branch 'master' into feature/upgradeMasterDependencies

.ci/notify_github.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Installiere curl
+apt-get update -qq && apt-get install -y -qq curl > /dev/null
+
+GITHUB_ORG="mediathekview"
+GITHUB_REPO="MServer"
+GITHUB_SHA="${CI_COMMIT_SHA}"
+cat << EOF > headers.curl
+Accept: application/vnd.github+json
+Authorization: token ${GITHUB_API_TOKEN}
+EOF
+cat << EOF > success.json
+{
+  "state" : "success",
+  "target_url" : "${CI_PIPELINE_URL}",
+  "description" : "CI runs at ElaonDE systems successful"
+}
+EOF
+cat << EOF > failure.json
+{
+  "state" : "failure",
+  "target_url" : "${CI_PIPELINE_URL}",
+  "description" : "CI runs at ElaonDE systems failed"
+}
+EOF
+cat << EOF > pending.json
+{
+  "state" : "pending",
+  "target_url" : "${CI_PIPELINE_URL}",
+  "description" : "CI runs at ElaonDE systems pending"
+}
+EOF
+GITHUB_API_URL="https://github.com/gitapi/repos/${GITHUB_ORG}/${GITHUB_REPO}/statuses/${GITHUB_SHA}"
+if [ "$1" == "success" ]; then
+curl -s -X POST -H @headers.curl "${GITHUB_API_URL}" -d @success.json
+elif [ "$1" == "failure" ]; then
+curl -s -X POST -H @headers.curl "${GITHUB_API_URL}" -d @failure.json
+elif [ "$1" == "pending" ]; then
+  curl -s -X POST -H @headers.curl "${GITHUB_API_URL}" -d @pending.json
+fi 

.gitlab-ci.yml

@@ -0,0 +1,68 @@
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+
+#image: maven:3.9-eclipse-temurin-8
+image: bellsoft/liberica-openjdk-debian:8u
+
+cache:
+  paths:
+    - ./.m2
+
+
+#default: 
+#  before_script:
+
+
+stages:
+  - .prenotify
+#  - test
+  - release
+  - .postnotify
+
+notify-github-pending:
+  stage: .prenotify
+  when: always
+  script:
+    - .ci/notify_github.sh pending
+
+#test:
+#  stage: test
+#  script:
+#    - mvn -B package
+#  rules:
+#    - if: '$CI_COMMIT_BRANCH == "develop"'
+
+Build and Deploy:
+  stage: release
+  script:
+#    - apt-get update -qq
+#    - apt-get install -y -qq ssh lib32ncurses6 lib32z1 wget tar file gnupg2 git-lfs > /dev/null
+#    - source /private/MServerENVS
+    - whoami
+#    - mkdir ~/.ssh
+#    - ssh-keyscan -p 60002 dw2.mvorg.de >> ~/.ssh/known_hosts
+#    - ssh-keyscan -p 60002 148.251.176.136 >> ~/.ssh/known_hosts
+#    - chmod 644 ~/.ssh/known_hosts
+    - java -version
+#    - mvn -v
+#    - mvn clean
+#    - mvn -B package
+    - ./gradlew build
+    - ls -l
+  tags:
+    - mvweb1
+#  rules:
+#    - if: $CI_COMMIT_TAG
+
+
+notify-github-success:
+  stage: .postnotify
+  when: on_success
+  script:
+    - .ci/notify_github.sh success
+
+notify-github-failure:
+  stage: .postnotify
+  when: on_failure
+  script:
+    - .ci/notify_github.sh failure

README.md

@@ -45,7 +45,7 @@ java -jar MServer.jar
 | ARD|ARD-Mediathek|Alpha, BR, Das Erste, HR, MDR, NDR, ONE, Radio Bremen, RBB, SR, SWR, WDR, tagesschau24|ARD, HR, MDR, NDR, Radio Bremen, RBB, SWR, WDR| x|
 | ARTE|ARTE-Mediathek|ARTE in DE, FR, EN, ES, PL, IT|ARTE.DE, ARTE.FR||
 | BR|BR-Mediathek|BR|BR||
-| DW|DW-Mediathek|DW|DW||
+| DW|DW-Mediathek|DW|DW|x|
 | FUNK | FUNK-Webseite | FUNK |FUNK |x|
 | KIKA|KIKA-Mediathek|KIKA|KIKA|x|
 | ORF|ORF-Mediathek|ORF1, ORF2, ORF3, ORFSport|ORF|x|

build.gradle

@@ -27,29 +27,29 @@ sourceCompatibility = JavaVersion.VERSION_17
 targetCompatibility = JavaVersion.VERSION_17
 group = 'de.mediathekview'
 archivesBaseName = "MServer"
-version = '3.1.199'
+version = '3.1.213'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'
 
 dependencies {
-    implementation 'de.mediathekview:MLib:3.0.12'
-    implementation 'commons-net:commons-net:3.8.0'
-    implementation 'org.apache.commons:commons-compress:1.21'
+    implementation 'de.mediathekview:MLib:3.0.13'
+    implementation 'commons-net:commons-net:3.9.0'
+    implementation 'org.apache.commons:commons-compress:1.22'
     implementation 'org.apache.commons:commons-text:1.10.0'
     implementation 'org.tukaani:xz:1.9'
-    implementation 'com.google.code.gson:gson:2.9.0'
+    implementation 'com.google.code.gson:gson:2.10.1'
     implementation 'org.glassfish.jersey.core:jersey-client:3.0.8'
     implementation 'org.glassfish.jersey.inject:jersey-hk2:3.0.8'
-    implementation 'org.jsoup:jsoup:1.15.3'
+    implementation 'org.jsoup:jsoup:1.15.4'
     implementation 'javax.activation:activation:1.1.1'
     implementation 'com.google.guava:guava:31.1-jre'
-    implementation 'org.apache.logging.log4j:log4j-core:2.19.0'
+    implementation 'org.apache.logging.log4j:log4j-core:2.20.0'
     implementation 'fm.void.jetm:jetm:1.2.3'
 
 
     testImplementation 'junit:junit:4.13.2'
-    testImplementation 'org.mockito:mockito-core:4.8.0'
+    testImplementation 'org.mockito:mockito-core:4.11.0'
     testImplementation 'com.squareup.okhttp3:mockwebserver:4.10.0'
     testImplementation "org.hamcrest:hamcrest-all:1.3"
     testImplementation 'org.assertj:assertj-core:3.23.1'

src/main/java/mServer/crawler/AddToFilmlist.java

@@ -120,6 +120,18 @@ private void performInitialCleanup() {
     updateAudioDescriptionOrf(listeEinsortieren);
     updateAudioDescriptionSrf(listeEinsortieren);
     updateArdWebsite(listeEinsortieren);
+    updateFunkMissingHost(listeEinsortieren);
+  }
+
+  private void updateFunkMissingHost(ListeFilme listeEinsortieren) {
+    final List<DatenFilm> list = listeEinsortieren.parallelStream()
+            .filter(film -> film.arr[DatenFilm.FILM_SENDER].equals("Funk.net") && film.arr[DatenFilm.FILM_URL].matches("https:\\/\\/[0-9]*\\/.*"))
+            .collect(Collectors.toList());
+    Log.sysLog("FUNK: add missing host für " + list.size() + " Einträge.");
+
+    list.forEach(film -> film.arr[DatenFilm.FILM_URL] = film.arr[DatenFilm.FILM_URL].replace("https://", "https://funk-02.akamaized.net/").trim());
+    list.forEach(film -> film.arr[DatenFilm.FILM_URL_KLEIN] = film.arr[DatenFilm.FILM_URL_KLEIN].replace("https://", "https://funk-02.akamaized.net/").trim());
+    list.forEach(film -> film.arr[DatenFilm.FILM_URL_HD] = film.arr[DatenFilm.FILM_URL_HD].replace("https://", "https://funk-02.akamaized.net/").trim());
   }
 
   private void updateArdWebsite(ListeFilme listeEinsortieren) {
@@ -337,9 +349,10 @@ public void run() {
             } else {
               Log.sysLog("film removed: code: " + response.code() + ": " + url);
             }
-          } catch (SocketTimeoutException ignored) {
-          } catch (IOException ex) {
-            ex.printStackTrace();
+          } catch (Exception ex) {
+            Log.errorLog(12834738, ex, "exception online check film: " + url);
+            // add film to list, because online check failed
+            addOld(film);
           }
         } else {
           if (Long.parseLong(film.arr[DatenFilm.FILM_GROESSE]) > MIN_SIZE_ADD_OLD) {

src/main/java/mServer/crawler/BannedFilmFilter.java

@@ -11,6 +11,18 @@ public static boolean isBanned(final DatenFilm film) {
     if (film.arr[DatenFilm.FILM_TITEL].equalsIgnoreCase("Wir haben genug - Wirtschaft ohne Wachstum")) {
       return true;
     }
+    // if (film.arr[DatenFilm.FILM_TITEL].equalsIgnoreCase("Auslegung der Wirklichkeit - Georg Stefan Troller - Dokumentarfilm von Ruth Rieser, Österreich 2021")) {
+    //   return true;
+    // }
+    final String title = film.arr[DatenFilm.FILM_TITEL].toLowerCase();
+    if (title.contains("auslegung der wirklichkeit") 
+            && title.contains("georg stefan troller") 
+            && title.contains("dokumentarfilm von ruth rieser") 
+            && title.contains("österreich 2021"))
+    {
+      return true;
+    }
+
 
     return false;
   }

src/main/java/mServer/crawler/FilmeSuchen.java

@@ -30,8 +30,9 @@
 import mServer.crawler.sender.arte.MediathekArte;
 import mServer.crawler.sender.br.BrCrawler;
 import mServer.crawler.sender.dreisat.DreiSatCrawler;
+import mServer.crawler.sender.dw.DwCrawler;
 import mServer.crawler.sender.funk.FunkCrawler;
-import mServer.crawler.sender.kika.KikaCrawler;
+import mServer.crawler.sender.kika.KikaApiCrawler;
 import mServer.crawler.sender.orf.OrfCrawler;
 import mServer.crawler.sender.phoenix.PhoenixCrawler;
 import mServer.crawler.sender.sr.SrCrawler;
@@ -75,8 +76,8 @@ public FilmeSuchen() {
     mediathekListe.add(new ZdfCrawler(this, 0));
     mediathekListe.add(new MediathekArte(this, 0));
     mediathekListe.add(new DreiSatCrawler(this, 1));
-    mediathekListe.add(new KikaCrawler(this, 0));
-    mediathekListe.add(new MediathekDw(this, 0));
+    mediathekListe.add(new DwCrawler(this, 0));
+    mediathekListe.add(new KikaApiCrawler(this, 0));
     mediathekListe.add(new FunkCrawler(this, 0));
     // Spalte 2
     mediathekListe.add(new SrCrawler(this, 1));
@@ -107,7 +108,7 @@ public synchronized void filmeBeimSenderLaden(ListeFilme listeFilme) {
     mrStarten(0);
     if (!Config.getStop()) {
       // waren und wenn Suchlauf noch nicht abgebrochen weiter mit dem Rest
-      mrWarten();
+      mrWarten(4*60);//4*60);
       mrStarten(1);
       allStarted = true;
     }
@@ -276,10 +277,10 @@ private synchronized void mrClear() {
     mediathekListe.forEach(MediathekReader::clear);
   }
 
-  private synchronized void mrWarten() {
+  private synchronized void mrWarten(int seconds) {
     // 4 Minuten warten, alle 10 Sekunden auf STOP prüfen
     try {
-      for (int i = 0; i < 4 * 60; ++i) {
+      for (int i = 0; i < seconds; ++i) {
         if (Config.getStop()) {
           break;
         }