Skip to content

Commit

Permalink
fix 992-991-990-989
Browse files Browse the repository at this point in the history
  • Loading branch information
@codingPF
codingPF committed May 24, 2024
1 parent 354921e commit 6631e31
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 24 deletions.
76 changes: 60 additions & 16 deletions src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;

import mServer.crawler.CrawlerTool;
Expand All @@ -32,6 +34,8 @@
import mServer.crawler.sender.ard.ArdFilmInfoDto;
import mServer.crawler.sender.base.JsonUtils;
import mServer.crawler.sender.base.Qualities;
import mServer.crawler.sender.base.UrlUtils;

import org.apache.logging.log4j.LogManager;

public class ArdFilmDeserializer implements JsonDeserializer<List<ArdFilmDto>> {
Expand Down Expand Up @@ -66,13 +70,16 @@ public class ArdFilmDeserializer implements JsonDeserializer<List<ArdFilmDto>> {
private static final String ATTRIBUTE_RESOLUTION_H = "maxHResolutionPx";
private static final String ATTRIBUTE_MIME = "mimeType";
private static final String ATTRIBUTE_KIND = "kind";
private static final String ATTRIBUTE_ADUIO_LANG = "languageCode";

private static final String MARKER_VIDEO_MP4 = "video/mp4";
private static final String MARKER_VIDEO_STANDARD = "standard";
private static final String MARKER_VIDEO_CATEGORY_MAIN = "main";
private static final String MARKER_VIDEO_CATEGORY_MPEG = "application/vnd.apple.mpegurl";
private static final String MARKER_VIDEO_AD = "audio-description";
private static final String MARKER_VIDEO_DGS = "sign-language";
private static final String MARKER_VIDEO_OV = "OV";
private static final String MARKER_VIDEO_DE = "deu";

private static final DateTimeFormatter DATE_FORMAT
= DateTimeFormatter.ofPattern("dd.MM.yyyy");
Expand Down Expand Up @@ -222,10 +229,11 @@ public List<ArdFilmDto> deserialize(
final Optional<LocalDateTime> date = parseDate(itemObject);
final Optional<Duration> duration = parseDuration(itemObject);
final Optional<String> partner = parsePartner(itemObject);
final Optional<Map<Qualities, String>> videoInfoStandard = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4);
final Optional<Map<Qualities, String>> videoInfoAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG);
final Optional<Map<Qualities, String>> videoInfoAD = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4);
final Optional<Map<Qualities, String>> videoInfoDGS = parseVideoUrls(itemObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4);
final Optional<Map<Qualities, String>> videoInfoStandard = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Qualities, String>> videoInfoAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE);
final Optional<Map<Qualities, String>> videoInfoAD = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Qualities, String>> videoInfoDGS = parseVideoUrls(itemObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Qualities, String>> videoInfoOV = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_OV);
final Optional<String> subtitles = prepareSubtitleUrl(itemObject);

if (topic.isEmpty() || title.isEmpty() || partner.isEmpty() || ADDITIONAL_SENDER.get(partner.get()) == null) {
Expand All @@ -235,6 +243,24 @@ public List<ArdFilmDto> deserialize(
return films;
}



if ((title.get().toUpperCase().contains("(OV)") || title.get().toUpperCase().contains("(ORIGINALVERSION)")) && videoInfoOV.isPresent()) {
final ArdFilmDto filmDto
= new ArdFilmDto(
createFilm(
ADDITIONAL_SENDER.get(partner.get()),
topic.get(),
title.get(),
description,
date,
duration,
videoInfoOV.get(),
subtitles));
films.add(filmDto);
return films;
}

if(videoInfoStandard.isEmpty() && videoInfoAD.isEmpty() && videoInfoDGS.isEmpty() && videoInfoAdaptive.isPresent()) {
// UUAAAARRGGGG - SAD
Map<Qualities, URL> qualitiesUrls = videoInfoAdaptive.get().entrySet().stream()
Expand Down Expand Up @@ -310,7 +336,7 @@ public List<ArdFilmDto> deserialize(
}
//
if (videoInfoDGS.isPresent() && videoInfoDGS.get().size() > 0) {
// add film standard
// add film dgs
final ArdFilmDto filmDto
= new ArdFilmDto(
createFilm(
Expand Down Expand Up @@ -382,11 +408,7 @@ private DatenFilm createFilm(

String dateValue = time.format(DATE_FORMAT);
String timeValue = time.format(TIME_FORMAT);

String baseUrl = videoUrls.get(Qualities.NORMAL);
baseUrl = baseUrl != null ? baseUrl : videoUrls.get(Qualities.SMALL);
baseUrl = baseUrl != null ? baseUrl : videoUrls.get(Qualities.HD);

DatenFilm film = new DatenFilm(sender, topic, "", title, baseUrl, "",
dateValue, timeValue, duration.orElse(Duration.ZERO).getSeconds(), description.orElse(""));
if (videoUrls.containsKey(Qualities.SMALL)) {
Expand All @@ -402,16 +424,40 @@ private DatenFilm createFilm(
return film;
}

private Optional<Map<Qualities, String>> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) {
Optional<Map<Integer, String>> urls = parseVideoUrlMap(playerPageObject, streamType, aduioType, mimeType, language);
if (urls.isEmpty()) {
return Optional.empty();
}
Map<Qualities, String> videoInfo = new EnumMap<>(Qualities.class);
for (Map.Entry<Integer, String> entry : urls.get().entrySet()) {
Qualities resolution = Qualities.getResolutionFromWidth(entry.getKey());
if(!videoInfo.containsKey(resolution)) {
videoInfo.put(resolution, entry.getValue());
}
}
// issue if we do not have normal res
if (!videoInfo.containsKey(Qualities.NORMAL)) {
if (videoInfo.containsKey(Qualities.HD)) {
videoInfo.put(Qualities.NORMAL, videoInfo.get(Qualities.HD));
videoInfo.remove(Qualities.HD);
} else {
videoInfo.put(Qualities.NORMAL, videoInfo.get(Qualities.SMALL));
videoInfo.remove(Qualities.SMALL);
}
}
return Optional.of(videoInfo);
}

private Optional<Map<Qualities, String>> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType) {
private Optional<Map<Integer, String>> parseVideoUrlMap(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) {
final Optional<JsonObject> mediaCollectionObject = getMediaCollectionObject(playerPageObject);
if (mediaCollectionObject.isEmpty())
return Optional.empty();
final Optional<JsonElement> streams = JsonUtils.getElement(mediaCollectionObject.get(), ELEMENT_STREAMS);
if (streams.isEmpty() || !streams.get().isJsonArray() || (streams.get().getAsJsonArray().size() == 0))
return Optional.empty();
//
Map<Qualities, String> videoInfo = new EnumMap<>(Qualities.class);
Map<Integer, String> videoInfo = new TreeMap<>(Comparator.reverseOrder());
for (JsonElement streamsCategory : streams.get().getAsJsonArray()) {
final Optional<String> streamKind = JsonUtils.getElementValueAsString(streamsCategory, ATTRIBUTE_KIND);
final Optional<JsonElement> media = JsonUtils.getElement(streamsCategory, ELEMENT_MEDIA);
Expand All @@ -426,11 +472,9 @@ private Optional<Map<Qualities, String>> parseVideoUrls(final JsonObject playerP
Optional<String> kind = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_KIND);
Optional<String> resh = JsonUtils.getElementValueAsString(video, ATTRIBUTE_RESOLUTION_H);
Optional<String> url = JsonUtils.getElementValueAsString(video, ATTRIBUTE_URL);
if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType)) {
Qualities resolution = Qualities.getResolutionFromWidth(Integer.parseInt(resh.get()));
if(!videoInfo.containsKey(resolution)) { // do not overwrite 1920 with 1280 res
videoInfo.put(resolution, url.get());
}
Optional<String> languageCode = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_ADUIO_LANG);
if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType) && languageCode.orElse("").equalsIgnoreCase(language)) {
videoInfo.put(Integer.parseInt(resh.get()), UrlUtils.removeParameters(url.get()));
}
}
}
Expand Down
13 changes: 5 additions & 8 deletions src/main/java/mServer/crawler/sender/base/Qualities.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,12 @@ public String getDescription()
}

public static Qualities getResolutionFromWidth(final int width) {
if (width >= 2160) {
return Qualities.UHD;
}
if (width >= 1280) {
if (width > 1280) {
return Qualities.HD;
}
if (width >= 720) {
} else if (width > 640) {
return Qualities.NORMAL;
} else {
return Qualities.SMALL;
}
return Qualities.SMALL;
}
}
}
3 changes: 3 additions & 0 deletions src/main/java/mServer/crawler/sender/zdf/json/ZdfFilmDetailDeserializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,9 @@ private Optional<String> formatEpisodeTitle(Optional<Integer> season, Optional<I
if (season.isPresent()) {
result += String.format("S%02d", season.get());
}
if (season.isPresent() && episode.isPresent()) {
result += "/";
}
if (episode.isPresent()) {
result += String.format("E%02d", episode.get());
}
Expand Down

0 comments on commit 6631e31

Please sign in to comment.