From 6631e31fe1cba3cf26caffcc408546aba70c77d0 Mon Sep 17 00:00:00 2001 From: CodingPF Date: Fri, 24 May 2024 21:16:08 +0200 Subject: [PATCH] fix 992-991-990-989 --- .../sender/ard/json/ArdFilmDeserializer.java | 76 +++++++++++++++---- .../crawler/sender/base/Qualities.java | 13 ++-- .../zdf/json/ZdfFilmDetailDeserializer.java | 3 + 3 files changed, 68 insertions(+), 24 deletions(-) diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java index c03ecf28c..c3877c385 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java @@ -17,6 +17,7 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.ArrayList; +import java.util.Comparator; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; @@ -24,6 +25,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import mServer.crawler.CrawlerTool; @@ -32,6 +34,8 @@ import mServer.crawler.sender.ard.ArdFilmInfoDto; import mServer.crawler.sender.base.JsonUtils; import mServer.crawler.sender.base.Qualities; +import mServer.crawler.sender.base.UrlUtils; + import org.apache.logging.log4j.LogManager; public class ArdFilmDeserializer implements JsonDeserializer> { @@ -66,6 +70,7 @@ public class ArdFilmDeserializer implements JsonDeserializer> { private static final String ATTRIBUTE_RESOLUTION_H = "maxHResolutionPx"; private static final String ATTRIBUTE_MIME = "mimeType"; private static final String ATTRIBUTE_KIND = "kind"; + private static final String ATTRIBUTE_ADUIO_LANG = "languageCode"; private static final String MARKER_VIDEO_MP4 = "video/mp4"; private static final String MARKER_VIDEO_STANDARD = "standard"; @@ -73,6 +78,8 @@ public class ArdFilmDeserializer implements JsonDeserializer> { private static final String MARKER_VIDEO_CATEGORY_MPEG = "application/vnd.apple.mpegurl"; private static final String MARKER_VIDEO_AD = "audio-description"; private static final String MARKER_VIDEO_DGS = "sign-language"; + private static final String MARKER_VIDEO_OV = "OV"; + private static final String MARKER_VIDEO_DE = "deu"; private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("dd.MM.yyyy"); @@ -222,10 +229,11 @@ public List deserialize( final Optional date = parseDate(itemObject); final Optional duration = parseDuration(itemObject); final Optional partner = parsePartner(itemObject); - final Optional> videoInfoStandard = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4); - final Optional> videoInfoAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG); - final Optional> videoInfoAD = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4); - final Optional> videoInfoDGS = parseVideoUrls(itemObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4); + final Optional> videoInfoStandard = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE); + final Optional> videoInfoAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE); + final Optional> videoInfoAD = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE); + final Optional> videoInfoDGS = parseVideoUrls(itemObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE); + final Optional> videoInfoOV = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_OV); final Optional subtitles = prepareSubtitleUrl(itemObject); if (topic.isEmpty() || title.isEmpty() || partner.isEmpty() || ADDITIONAL_SENDER.get(partner.get()) == null) { @@ -235,6 +243,24 @@ public List deserialize( return films; } + + + if ((title.get().toUpperCase().contains("(OV)") || title.get().toUpperCase().contains("(ORIGINALVERSION)")) && videoInfoOV.isPresent()) { + final ArdFilmDto filmDto + = new ArdFilmDto( + createFilm( + ADDITIONAL_SENDER.get(partner.get()), + topic.get(), + title.get(), + description, + date, + duration, + videoInfoOV.get(), + subtitles)); + films.add(filmDto); + return films; + } + if(videoInfoStandard.isEmpty() && videoInfoAD.isEmpty() && videoInfoDGS.isEmpty() && videoInfoAdaptive.isPresent()) { // UUAAAARRGGGG - SAD Map qualitiesUrls = videoInfoAdaptive.get().entrySet().stream() @@ -310,7 +336,7 @@ public List deserialize( } // if (videoInfoDGS.isPresent() && videoInfoDGS.get().size() > 0) { - // add film standard + // add film dgs final ArdFilmDto filmDto = new ArdFilmDto( createFilm( @@ -382,11 +408,7 @@ private DatenFilm createFilm( String dateValue = time.format(DATE_FORMAT); String timeValue = time.format(TIME_FORMAT); - String baseUrl = videoUrls.get(Qualities.NORMAL); - baseUrl = baseUrl != null ? baseUrl : videoUrls.get(Qualities.SMALL); - baseUrl = baseUrl != null ? baseUrl : videoUrls.get(Qualities.HD); - DatenFilm film = new DatenFilm(sender, topic, "", title, baseUrl, "", dateValue, timeValue, duration.orElse(Duration.ZERO).getSeconds(), description.orElse("")); if (videoUrls.containsKey(Qualities.SMALL)) { @@ -402,8 +424,32 @@ private DatenFilm createFilm( return film; } + private Optional> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) { + Optional> urls = parseVideoUrlMap(playerPageObject, streamType, aduioType, mimeType, language); + if (urls.isEmpty()) { + return Optional.empty(); + } + Map videoInfo = new EnumMap<>(Qualities.class); + for (Map.Entry entry : urls.get().entrySet()) { + Qualities resolution = Qualities.getResolutionFromWidth(entry.getKey()); + if(!videoInfo.containsKey(resolution)) { + videoInfo.put(resolution, entry.getValue()); + } + } + // issue if we do not have normal res + if (!videoInfo.containsKey(Qualities.NORMAL)) { + if (videoInfo.containsKey(Qualities.HD)) { + videoInfo.put(Qualities.NORMAL, videoInfo.get(Qualities.HD)); + videoInfo.remove(Qualities.HD); + } else { + videoInfo.put(Qualities.NORMAL, videoInfo.get(Qualities.SMALL)); + videoInfo.remove(Qualities.SMALL); + } + } + return Optional.of(videoInfo); + } - private Optional> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType) { + private Optional> parseVideoUrlMap(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) { final Optional mediaCollectionObject = getMediaCollectionObject(playerPageObject); if (mediaCollectionObject.isEmpty()) return Optional.empty(); @@ -411,7 +457,7 @@ private Optional> parseVideoUrls(final JsonObject playerP if (streams.isEmpty() || !streams.get().isJsonArray() || (streams.get().getAsJsonArray().size() == 0)) return Optional.empty(); // - Map videoInfo = new EnumMap<>(Qualities.class); + Map videoInfo = new TreeMap<>(Comparator.reverseOrder()); for (JsonElement streamsCategory : streams.get().getAsJsonArray()) { final Optional streamKind = JsonUtils.getElementValueAsString(streamsCategory, ATTRIBUTE_KIND); final Optional media = JsonUtils.getElement(streamsCategory, ELEMENT_MEDIA); @@ -426,11 +472,9 @@ private Optional> parseVideoUrls(final JsonObject playerP Optional kind = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_KIND); Optional resh = JsonUtils.getElementValueAsString(video, ATTRIBUTE_RESOLUTION_H); Optional url = JsonUtils.getElementValueAsString(video, ATTRIBUTE_URL); - if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType)) { - Qualities resolution = Qualities.getResolutionFromWidth(Integer.parseInt(resh.get())); - if(!videoInfo.containsKey(resolution)) { // do not overwrite 1920 with 1280 res - videoInfo.put(resolution, url.get()); - } + Optional languageCode = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_ADUIO_LANG); + if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType) && languageCode.orElse("").equalsIgnoreCase(language)) { + videoInfo.put(Integer.parseInt(resh.get()), UrlUtils.removeParameters(url.get())); } } } diff --git a/src/main/java/mServer/crawler/sender/base/Qualities.java b/src/main/java/mServer/crawler/sender/base/Qualities.java index 74e3af9a5..907022c85 100644 --- a/src/main/java/mServer/crawler/sender/base/Qualities.java +++ b/src/main/java/mServer/crawler/sender/base/Qualities.java @@ -17,15 +17,12 @@ public String getDescription() } public static Qualities getResolutionFromWidth(final int width) { - if (width >= 2160) { - return Qualities.UHD; - } - if (width >= 1280) { + if (width > 1280) { return Qualities.HD; - } - if (width >= 720) { + } else if (width > 640) { return Qualities.NORMAL; + } else { + return Qualities.SMALL; } - return Qualities.SMALL; - } + } } diff --git a/src/main/java/mServer/crawler/sender/zdf/json/ZdfFilmDetailDeserializer.java b/src/main/java/mServer/crawler/sender/zdf/json/ZdfFilmDetailDeserializer.java index 3c63ec63e..a1fda0b33 100644 --- a/src/main/java/mServer/crawler/sender/zdf/json/ZdfFilmDetailDeserializer.java +++ b/src/main/java/mServer/crawler/sender/zdf/json/ZdfFilmDetailDeserializer.java @@ -259,6 +259,9 @@ private Optional formatEpisodeTitle(Optional season, Optional