Merge branch 'master' into hotfix/cleanup

Author: pidoubleyou

build.gradle

@@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
 targetCompatibility = JavaVersion.VERSION_17
 group = 'de.mediathekview'
 archivesBaseName = "MServer"
-version = '3.1.281'
+version = '3.1.282'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'

src/main/java/mServer/crawler/AddToFilmlist.java

@@ -128,6 +128,20 @@ private void performInitialCleanup() {
     updateThema(listeEinsortieren);
     updateTitle(listeEinsortieren);
     updateArdWebsite(listeEinsortieren);
+    updateGeoForNdrProgressiveGeo(listeEinsortieren);
+  }
+
+  private void updateGeoForNdrProgressiveGeo(ListeFilme listeEinsortieren) {
+    final List<DatenFilm> list = listeEinsortieren.parallelStream()
+            .filter(film -> film.arr[DatenFilm.FILM_SENDER].equals(Const.NDR)
+                    && film.arr[DatenFilm.FILM_URL] != null
+                    && film.arr[DatenFilm.FILM_URL].contains("progressive_geo"))
+            .filter(film -> film.arr[DatenFilm.FILM_GEO] == null || film.arr[DatenFilm.FILM_GEO].isEmpty())
+            .toList();
+    Log.sysLog("NDR: set GEO=DE for " + list.size() + " entries with progressive_geo (out of " + list.size() + ").");
+    if (!list.isEmpty()) {
+      list.forEach(film -> film.arr[DatenFilm.FILM_GEO] = DatenFilm.GEO_DE);
+    }
   }
 
   private boolean isArdUrlToRemove(final String url) {

src/main/java/mServer/crawler/sender/MediathekReader.java

@@ -245,14 +245,10 @@ private void processArd(DatenFilm film) {
     if (film.arr[DatenFilm.FILM_URL].startsWith("http://mvideos-geo.daserste.de/")
             || film.arr[DatenFilm.FILM_URL].contains("/geoblocking/")
             || film.arr[DatenFilm.FILM_URL].contains("/de/")
-            || film.arr[DatenFilm.FILM_URL].startsWith("http://media.ndr.de/progressive_geo/")
-            || film.arr[DatenFilm.FILM_URL].startsWith("http://mediandr-a.akamaihd.net//progressive_geo/")
-            || film.arr[DatenFilm.FILM_URL].startsWith("https://mediandr-a.akamaihd.net//progressive_geo/")
+            || film.arr[DatenFilm.FILM_URL].contains("/progressive_geo/")
             || film.arr[DatenFilm.FILM_URL].startsWith("https://pdodswr-a.akamaihd.net/swr/geo/de/")
             || film.arr[DatenFilm.FILM_URL].startsWith("https://pdodswr-a.akamaihd.net/swrfernsehen/geo/de/")
             || film.arr[DatenFilm.FILM_URL].startsWith("https://pdodswr-a.akamaihd.net/kindernetz/geo/de/")
-            || film.arr[DatenFilm.FILM_URL].startsWith("http://mediandr-a.akamaihd.net/progressive_geo")
-            || film.arr[DatenFilm.FILM_URL].startsWith("https://mediandr-a.akamaihd.net/progressive_geo")
             || film.arr[DatenFilm.FILM_URL].startsWith("http://cdn-storage.br.de/geo/")
             || film.arr[DatenFilm.FILM_URL].startsWith("http://cdn-sotschi.br.de/geo/b7/")
             || film.arr[DatenFilm.FILM_URL].startsWith("https://cdn-storage.br.de/geo/")

src/test/developTest/java/mServer/crawler/AddToFilmlistTest.java

@@ -56,6 +56,7 @@ public MockResponse dispatch(RecordedRequest request) throws InterruptedExceptio
 
         switch (request.getPath()) {
           case "/" + FILM_NAME_ONLINE:
+          case "/progressive_geo/" + FILM_NAME_ONLINE:
           case "/" + FILM_NAME_ONLINE2:
           case "/" + FILM_NAME_ARTE_EXTRAIT:
           case "/world/hls/10vor10/2023/07/10vor10_20230731_215000_19875207_v_webcast_h264_,q40,q10,q20,q30,q50,q60,.mp4.csmil/index-f1-v1-a1.m3u8":
@@ -465,6 +466,37 @@ public void testArdTagesschau24EntriesNotAdded() {
     assertEquals(Const.ARD, testFilmArdOk.arr[DatenFilm.FILM_SENDER]);
   }
 
+  @Test
+  public void testSetGeoForNdrProgressiveGeoWhenEmpty() {
+    final DatenFilm ndrFilm1 = createTestFilm(Const.NDR, "NDR Topic", "NDR Title", "progressive_geo/" + FILM_NAME_ONLINE);
+    final DatenFilm ndrFilm2 = createTestFilm(Const.NDR, "NDR Topic", "NDR Title 1", FILM_NAME_ONLINE);
+    listToAdd.add(ndrFilm1);
+    listToAdd.add(ndrFilm2);
+
+    AddToFilmlist target = new AddToFilmlist(list, listToAdd);
+    target.addOldList();
+
+    assertEquals(list.size(), 4);
+    // GEO should be set to DE
+    assertEquals(DatenFilm.GEO_DE, ndrFilm1.arr[DatenFilm.FILM_GEO]);
+    assertEquals("", ndrFilm2.arr[DatenFilm.FILM_GEO]);
+  }
+
+  @Test
+  public void testDoNotOverrideExistingGeoForNdrProgressiveGeo() {
+    final DatenFilm ndrFilm = createTestFilm(Const.NDR, "NDR Topic", "NDR Title", "progressive_geo/" + FILM_NAME_ONLINE);
+    // preset GEO to AT (should not be overridden)
+    ndrFilm.arr[DatenFilm.FILM_GEO] = DatenFilm.GEO_AT;
+    listToAdd.add(ndrFilm);
+
+    AddToFilmlist target = new AddToFilmlist(list, listToAdd);
+    target.addOldList();
+
+    assertEquals(list.size(), 3);
+    // GEO should remain AT
+    assertEquals(DatenFilm.GEO_AT, ndrFilm.arr[DatenFilm.FILM_GEO]);
+  }
+
   private static DatenFilm createTestFilm(String sender, String topic, String title,
       String filmUrl) {
     DatenFilm film = new DatenFilm(sender, topic, "url", title, baseUrl + filmUrl, "", "", "", 12,