Skip to content

cleanups #1137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alex1702 merged 3 commits into from
Apr 13, 2026
Merged

cleanups #1137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
66fa6eb
remove unnecessary cleanup functions
pidoubleyou Apr 12, 2026
f24f935
remove srfpod code, keep it for local testing
pidoubleyou Apr 12, 2026
bc4e4df
Merge branch 'master' into hotfix/cleanup
pidoubleyou Apr 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 0 additions & 98 deletions src/main/java/mServer/crawler/AddToFilmlist.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,6 @@
this.bannedFilmFilter = new BannedFilmFilter();
}

private static String cutOutSrfParameterInUrl(String url) {
int startIndex = url.indexOf("/hdntl=exp");
int endIndex = url.indexOf("/index-f");
if (endIndex > -1 && startIndex < endIndex) {
url = url.substring(0, startIndex) + url.substring(endIndex);
}
return url;
}

public synchronized void addLiveStream() {
if (listeEinsortieren.size() <= 0) {
return;
Expand Down Expand Up @@ -134,16 +125,10 @@
});
removeTimeFromMdrAktuell(listeEinsortieren);
removeTimeFromOrf(listeEinsortieren);
updateAudioDescriptionOrf(listeEinsortieren);
updateAudioDescriptionSrf(listeEinsortieren);
updateAudioDescriptionZdf(listeEinsortieren);
updateThema(listeEinsortieren);
updateTitle(listeEinsortieren);
updateArdWebsite(listeEinsortieren);
updateSenderTagesschau24(listeEinsortieren);
updateFunkMissingHost(listeEinsortieren);
updateGeoForNdrProgressiveGeo(listeEinsortieren);
removeSrfUrlParameter(listeEinsortieren);
}

private void updateGeoForNdrProgressiveGeo(ListeFilme listeEinsortieren) {
Expand All @@ -165,36 +150,6 @@
|| url.startsWith("https://pmdonlinekika-a.akamaihd.net");
}

// check https://github.com/mediathekview/MServer/issues/904 for examples and more information
private void removeSrfUrlParameter(ListeFilme listeEinsortieren) {
final List<DatenFilm> list = listeEinsortieren.parallelStream()
.filter(film -> film.arr[DatenFilm.FILM_SENDER].equals(Const.SRF) && film.arr[DatenFilm.FILM_URL].contains("/hdntl=exp"))
.collect(Collectors.toList());
Log.sysLog("SRF: remove url parameter für " + list.size() + " Einträge von " + listeEinsortieren.size());

list.forEach(film -> {
String url = film.arr[DatenFilm.FILM_URL];
String urlKlein = film.arr[DatenFilm.FILM_URL_KLEIN] == null || film.arr[DatenFilm.FILM_URL_KLEIN].isEmpty() ? "" : film.getUrlFuerAufloesung(DatenFilm.AUFLOESUNG_KLEIN);
String urlGross = film.arr[DatenFilm.FILM_URL_HD] == null || film.arr[DatenFilm.FILM_URL_HD].isEmpty() ? "" : film.getUrlFuerAufloesung(DatenFilm.AUFLOESUNG_HD);
film.arr[DatenFilm.FILM_URL] = cutOutSrfParameterInUrl(UrlUtils.removeParameters(url));
CrawlerTool.addUrlKlein(film, cutOutSrfParameterInUrl(UrlUtils.removeParameters(urlKlein)));
CrawlerTool.addUrlHd(film, cutOutSrfParameterInUrl(UrlUtils.removeParameters(urlGross)));
});


}

private void updateFunkMissingHost(ListeFilme listeEinsortieren) {
final List<DatenFilm> list = listeEinsortieren.parallelStream()
.filter(film -> film.arr[DatenFilm.FILM_SENDER].equals("Funk.net") && film.arr[DatenFilm.FILM_URL].matches("https:\\/\\/[0-9]*\\/.*"))
.collect(Collectors.toList());
Log.sysLog("FUNK: add missing host für " + list.size() + " Einträge.");
// ich glaube dies macht die urls (klein/dh) kaputt. Diese liegen in index Form (123|abc.mp4) vor und der index verschiebt sich wenn man die FILM_URL ändert
list.forEach(film -> film.arr[DatenFilm.FILM_URL] = film.arr[DatenFilm.FILM_URL].replace("https://", "https://funk-02.akamaized.net/").trim());
list.forEach(film -> film.arr[DatenFilm.FILM_URL_KLEIN] = film.arr[DatenFilm.FILM_URL_KLEIN].replace("https://", "https://funk-02.akamaized.net/").trim());
list.forEach(film -> film.arr[DatenFilm.FILM_URL_HD] = film.arr[DatenFilm.FILM_URL_HD].replace("https://", "https://funk-02.akamaized.net/").trim());
}

private void updateThema(ListeFilme listeEinsortieren) {
listeEinsortieren.parallelStream().forEach(film -> {
final String thema = film.arr[DatenFilm.FILM_THEMA].trim();
Expand All @@ -217,64 +172,11 @@
final List<DatenFilm> list = listeEinsortieren.parallelStream()
.filter(film -> film.arr[DatenFilm.FILM_SENDER].equals(Const.ARD) && !film.arr[DatenFilm.FILM_WEBSEITE].startsWith("https://www.ardmediathek.de/video/"))
.collect(Collectors.toList());
Log.sysLog("ARD: update webseite für " + list.size() + " Einträge.");

Check failure on line 175 in src/main/java/mServer/crawler/AddToFilmlist.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Define a constant instead of duplicating this literal " Einträge." 3 times. 


See more on https://sonarcloud.io/project/issues?id=mediathekview_MServer&issues=AZ2DSDWpt-zbsGOGdAQa&open=AZ2DSDWpt-zbsGOGdAQa&pullRequest=1137

list.forEach(film -> film.arr[DatenFilm.FILM_WEBSEITE] = film.arr[DatenFilm.FILM_WEBSEITE].replace("/ard/player/", "/video/").trim());
}

private void updateSenderTagesschau24(ListeFilme listeEinsortieren) {
final List<String> whiteListTagesschau24 = Arrays.stream(new String[] { "Ulrich Timm im Gespräch", "tagesschau in Einfacher Sprache", "tagesschau in 100 Sekunden", "tagesschau24" }).toList();
final List<DatenFilm> list = listeEinsortieren.parallelStream()
.filter(film -> film.arr[DatenFilm.FILM_SENDER].equals(Const.ARD) && whiteListTagesschau24.contains(film.arr[DatenFilm.FILM_THEMA]))
.collect(Collectors.toList());
Log.sysLog("ARD: set sender tagesschau24 für " + list.size() + " Einträge.");

list.forEach(film -> film.arr[DatenFilm.FILM_SENDER] = Const.TAGESSCHAU24);
}

private void updateAudioDescriptionOrf(ListeFilme listeEinsortieren) {
final List<DatenFilm> list = listeEinsortieren.parallelStream()
.filter(
film -> film.arr[DatenFilm.FILM_SENDER].equals(Const.ORF) && film.arr[DatenFilm.FILM_THEMA]
.startsWith("AD |"))
.collect(Collectors.toList());
Log.sysLog("ORF: update Thema/Titel für " + list.size() + " Einträge mit Audiodeskription.");
if (!list.isEmpty()) {
list.forEach(film -> {
film.arr[DatenFilm.FILM_THEMA] = film.arr[DatenFilm.FILM_THEMA].replace("AD |", "").trim();
film.arr[DatenFilm.FILM_TITEL] = film.arr[DatenFilm.FILM_TITEL].replace("AD |", "").trim() + " (Audiodeskription)";
});
}
}

private void updateAudioDescriptionSrf(ListeFilme listeEinsortieren) {
final List<DatenFilm> list = listeEinsortieren.parallelStream()
.filter(
film -> film.arr[DatenFilm.FILM_SENDER].equals(Const.SRF) && film.arr[DatenFilm.FILM_THEMA]
.contains("mit Audiodeskription"))
.collect(Collectors.toList());
Log.sysLog("SRF: update Thema/Titel für " + list.size() + " Einträge mit Audiodeskription.");
if (!list.isEmpty()) {
list.forEach(film -> {
film.arr[DatenFilm.FILM_THEMA] = film.arr[DatenFilm.FILM_THEMA].replace("mit Audiodeskription", "").trim();
film.arr[DatenFilm.FILM_TITEL] = film.arr[DatenFilm.FILM_TITEL].replace(" mit Audiodeskription", "").trim() + " (Audiodeskription)";
});
}
}

private void updateAudioDescriptionZdf(ListeFilme listeEinsortieren) {
final List<DatenFilm> list = listeEinsortieren.parallelStream()
.filter(
film -> film.arr[DatenFilm.FILM_TITEL].contains("(deu-ad)"))
.toList();
Log.sysLog("ZDF: erstetze (deu-ad) mit " + list.size() + " (Audiodeskription).");
if (!list.isEmpty()) {
list.forEach(film -> {
film.arr[DatenFilm.FILM_TITEL] = film.arr[DatenFilm.FILM_TITEL].replace("(deu-ad)", " (Audiodeskription)").trim();
});
}
}

private void removeTimeFromOrf(ListeFilme listeEinsortieren) {
final List<DatenFilm> list = listeEinsortieren.parallelStream()
.filter(
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/mServer/crawler/FilmeSuchen.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public FilmeSuchen() {
mediathekListe.add(new SrfCrawler(this, 1));
}
if (crawlerList.contains("SRFPOD")) {
mediathekListe.add(new MediathekSrfPod(this, 1));
mediathekListe.add(new MediathekDummySrfPod(this, 1));
}
if (crawlerList.contains("ORF")) {
mediathekListe.add(new OrfOnCrawler(this, 1));
Expand Down
19 changes: 19 additions & 0 deletions src/main/java/mServer/crawler/sender/MediathekDummySrfPod.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package mServer.crawler.sender;

Check warning on line 1 in src/main/java/mServer/crawler/sender/MediathekDummySrfPod.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Rename this package name to match the regular expression '^[a-z_]+(\.[a-z_][a-z0-9_]*)*$'. 


See more on https://sonarcloud.io/project/issues?id=mediathekview_MServer&issues=AZ2DSDV6t-zbsGOGdAQY&open=AZ2DSDV6t-zbsGOGdAQY&pullRequest=1137

import de.mediathekview.mlib.Const;
import mServer.crawler.FilmeSuchen;

public class MediathekDummySrfPod extends MediathekReader implements Runnable {

public final static String SENDERNAME = Const.SRF_PODCAST;

Check warning on line 8 in src/main/java/mServer/crawler/sender/MediathekDummySrfPod.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Reorder the modifiers to comply with the Java Language Specification. 


See more on https://sonarcloud.io/project/issues?id=mediathekview_MServer&issues=AZ2DSDV6t-zbsGOGdAQZ&open=AZ2DSDV6t-zbsGOGdAQZ&pullRequest=1137

public MediathekDummySrfPod(FilmeSuchen ssearch, int startPrio) {
super(ssearch, SENDERNAME,/* threads */ 2, /* urlWarten */ 200, startPrio);
}

@Override
protected void addToList() {
// empty because dummy crawler
meldungThreadUndFertig();
}
}
Loading
Loading