add compilation subpage handling

src/main/java/mServer/crawler/sender/ard/ArdConstants.java

@@ -11,11 +11,13 @@ public class ArdConstants {
 
   public static final String TOPICS_URL = API_URL + "/page-gateway/pages/%s/editorial/experiment-a-z?embedded=false";
   public static final String TOPICS_COMPILATION_URL = API_URL + "/page-gateway/widgets/%s/editorials/%s?pageNumber=0&pageSize=%s";
+  public static final String TOPIC_COMPILATION_PAGE_URL = API_URL + "/page-gateway/widgets/%s/compilation/%s?pageNumber=%d&pageSize=%d";
   public static final String TOPIC_URL = API_URL + "/page-gateway/widgets/ard/asset/%s?pageSize=%d";
   public static final String DAY_PAGE_URL = "https://programm-api.ard.de/program/api/program?day=%s&channelIds=%s&mode=channel";
 
   public static final int TOPICS_COMPILATION_PAGE_SIZE = 100;
   public static final int TOPIC_PAGE_SIZE = 50;
+  public static final int MAX_COMPILATION_SUB_PAGES = 4;
 
   public static final String DEFAULT_CLIENT = "ard";
 

src/main/java/mServer/crawler/sender/ard/ArdTopicInfoDto.java

@@ -6,53 +6,77 @@
 public class ArdTopicInfoDto {
   private final Set<ArdFilmInfoDto> filmInfos;
   private String id;
-  private int subPageNumber;
+  private int pageNumber;
+  private int pageSize;
+  private int totalElements;
   private int maxSubPageNumber;
 
+
   public ArdTopicInfoDto(final Set<ArdFilmInfoDto> filmInfos) {
     this.filmInfos = filmInfos;
-    subPageNumber = 0;
-    maxSubPageNumber = 0;
+    setPageNumber(0);
+    setPageSize(0);
+    setTotalElements(0);
+    setMaxSubPageNumber(0);
   }
 
   public Set<ArdFilmInfoDto> getFilmInfos() {
     return filmInfos;
   }
 
-  public String getId() {
-    return id;
+  @Override
+  public boolean equals(final Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof final ArdTopicInfoDto that)) {
+      return false;
+    }
+    return getId() == that.getId()
+            && getPageNumber() == that.getPageNumber()
+            && getPageSize() == that.getPageSize()
+            && getTotalElements() == that.getTotalElements()
+            && Objects.equals(filmInfos, that.filmInfos);
   }
 
-  public void setId(String id) {
-    this.id = id;
+  @Override
+  public int hashCode() {
+    return Objects.hash(filmInfos, getId(), getPageNumber(), getPageSize(), getTotalElements());
   }
 
-  public int getSubPageNumber() {
-    return subPageNumber;
+  public String getId() { return id; }
+
+  public int getPageNumber() {
+    return pageNumber;
   }
 
-  public void setSubPageNumber(final int subPageNumber) {
-    this.subPageNumber = subPageNumber;
+  public void setPageNumber(int pageNumber) {
+    this.pageNumber = pageNumber;
   }
 
-  public int getMaxSubPageNumber() {
-    return maxSubPageNumber;
+  public int getPageSize() {
+    return pageSize;
   }
 
-  public void setMaxSubPageNumber(final int maxSubPageNumber) {
-    this.maxSubPageNumber = maxSubPageNumber;
+  public void setPageSize(int pageSize) {
+    this.pageSize = pageSize;
   }
 
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
-    ArdTopicInfoDto that = (ArdTopicInfoDto) o;
-    return Objects.equals(id, that.id) && subPageNumber == that.subPageNumber && maxSubPageNumber == that.maxSubPageNumber && Objects.equals(filmInfos, that.filmInfos);
+  public int getTotalElements() {
+    return totalElements;
   }
 
-  @Override
-  public int hashCode() {
-    return Objects.hash(id, filmInfos, subPageNumber, maxSubPageNumber);
+  public void setTotalElements(int totalElements) {
+    this.totalElements = totalElements;
+  }
+
+  public void setId(String id) { this.id = id; }
+
+  public int getMaxSubPageNumber() {
+    return maxSubPageNumber;
+  }
+
+  public void setMaxSubPageNumber(final int maxSubPageNumber) {
+    this.maxSubPageNumber = maxSubPageNumber;
   }
 }

src/main/java/mServer/crawler/sender/ard/json/ArdTopicCompilationDeserializer.java

@@ -33,19 +33,39 @@ public ArdTopicInfoDto deserialize(
           results.addAll(parseTeasers(teasers));
         }
       });
+    } else if (showPageObject.getAsJsonObject().has(ELEMENT_TEASERS)) {
+      final JsonArray teasers = showPageObject.getAsJsonObject().get(ELEMENT_TEASERS).getAsJsonArray();
+      results.addAll(parseTeasers(teasers));
     }
 
-    final JsonElement paginationElement = showPageObject.get(ELEMENT_PAGINATION);
+    final JsonElement paginationElement = findPaginationElement(showPageObject);
     final int pageNumber = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER);
     final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS);
+    ardTopicInfoDto.setTotalElements(totalElements);
     final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE);
+    ardTopicInfoDto.setPageSize(pageSize);
     ardTopicInfoDto.setMaxSubPageNumber(pageSize == 0 ? 0 :
             (totalElements + pageSize - 1) / pageSize);
-    ardTopicInfoDto.setSubPageNumber(pageNumber);
+    ardTopicInfoDto.setPageNumber(pageNumber);
 
     return ardTopicInfoDto;
   }
 
+  private JsonElement findPaginationElement(JsonObject showPageObject) {
+    if (showPageObject.has(ELEMENT_WIDGETS)) {
+      final JsonElement widgetElement = showPageObject.get(ELEMENT_WIDGETS);
+      if (widgetElement.isJsonArray()) {
+        final JsonArray widgetArray = widgetElement.getAsJsonArray();
+        if (!widgetArray.isEmpty()) {
+          return widgetArray.get(0).getAsJsonObject().get(ELEMENT_PAGINATION);
+        }
+      }
+    } else if (showPageObject.has(ELEMENT_PAGINATION)) {
+      return showPageObject.get(ELEMENT_PAGINATION);
+    }
+    return null;
+  }
+
   private int getChildElementAsIntOrNullIfNotExist(
           final JsonElement parentElement, final String childElementName) {
     if (parentElement == null || parentElement.isJsonNull()) {

src/main/java/mServer/crawler/sender/ard/json/ArdTopicPageDeserializer.java

@@ -35,10 +35,12 @@ public ArdTopicInfoDto deserialize(
     id.ifPresent(ardTopicInfoDto::setId);
 
     final JsonElement paginationElement = showPageObject.get(ELEMENT_PAGINATION);
-    ardTopicInfoDto.setSubPageNumber(
+    ardTopicInfoDto.setPageNumber(
             getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER));
     final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS);
+    ardTopicInfoDto.setTotalElements(totalElements);
     final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE);
+    ardTopicInfoDto.setPageSize(pageSize);
     ardTopicInfoDto.setMaxSubPageNumber(pageSize == 0 ? 0 :
             (totalElements + pageSize - 1) / pageSize);
 

src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicCompilationTask.java

@@ -3,19 +3,28 @@
 import com.google.gson.reflect.TypeToken;
 import jakarta.ws.rs.client.WebTarget;
 import mServer.crawler.sender.MediathekReader;
+import mServer.crawler.sender.ard.ArdConstants;
 import mServer.crawler.sender.ard.ArdFilmInfoDto;
 import mServer.crawler.sender.ard.ArdTopicInfoDto;
 import mServer.crawler.sender.ard.json.ArdTopicCompilationDeserializer;
 import mServer.crawler.sender.base.AbstractRecursivConverterTask;
 import mServer.crawler.sender.base.CrawlerUrlDTO;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.lang.reflect.Type;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class ArdTopicCompilationTask extends ArdTaskBase<ArdFilmInfoDto, CrawlerUrlDTO> {
+  private static final Logger LOG = LogManager.getLogger(ArdTopicCompilationTask.class);
+  private static final Pattern UUID_PATTERN = Pattern.compile("/compilation/([^/?]+)([/?])");
+  private static final Pattern WIDGETS_SENDER_PATTERN = Pattern.compile("/pages/([^/]+)/"); // captures the segment after /widgets/ (e.g. "ard")
 
   private static final Type ARDTOPICINFODTO_TYPE_TOKEN =
-          new TypeToken<ArdTopicInfoDto>() {}.getType();
+          new TypeToken<ArdTopicInfoDto>() {
+          }.getType();
 
   public ArdTopicCompilationTask(
           final MediathekReader aCrawler, final ConcurrentLinkedQueue<CrawlerUrlDTO> aUrlToCrawlDtos) {
@@ -31,7 +40,52 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg
             && topicInfo.getFilmInfos() != null
             && !topicInfo.getFilmInfos().isEmpty()) {
       taskResults.addAll(topicInfo.getFilmInfos());
+
+      final ConcurrentLinkedQueue<CrawlerUrlDTO> subpages = createSubPageUrls(aDTO, topicInfo);
+      if (!subpages.isEmpty()) {
+        taskResults.addAll(createNewOwnInstance(subpages).fork().join());
+      }
+    }
+  }
+
+  private ConcurrentLinkedQueue<CrawlerUrlDTO> createSubPageUrls(CrawlerUrlDTO urlDto, ArdTopicInfoDto topicInfo) {
+    final ConcurrentLinkedQueue<CrawlerUrlDTO> subpages = new ConcurrentLinkedQueue<>();
+    if (topicInfo.getTotalElements() < topicInfo.getPageSize() || topicInfo.getPageNumber() > 0) {
+      return subpages;
+    }
+    //
+    final String sender = extractSender(urlDto.getUrl());
+    final String id = extractCompilation(urlDto.getUrl());
+
+    int index = 0;
+    while ((topicInfo.getPageSize() + (index * topicInfo.getPageSize())) < topicInfo.getTotalElements()) {
+      subpages.add(new CrawlerUrlDTO(String.format(ArdConstants.TOPIC_COMPILATION_PAGE_URL, sender, id, index + 1, topicInfo.getPageSize())));
+      index++;
+      if (index >= ArdConstants.MAX_COMPILATION_SUB_PAGES) {
+        LOG.debug("ignore more subpage due to limit of {} pages but found {}", ArdConstants.MAX_COMPILATION_SUB_PAGES, Integer.valueOf(topicInfo.getTotalElements() / topicInfo.getPageSize()));
+        break;
+      }
+    }
+    return subpages;
+  }
+
+  private String extractCompilation(String url) {
+    final Matcher m = UUID_PATTERN.matcher(url);
+    if (m.find()) {
+      return m.group(1);
+    }
+
+    LOG.warn("no compilation pattern found in url {}", url);
+    return "";
+  }
+
+  private String extractSender(String url) {
+    final Matcher m = WIDGETS_SENDER_PATTERN.matcher(url);
+    if (m.find()) {
+      return m.group(1);
     }
+    LOG.warn("no sender pattern found in url {}", url);
+    return "";
   }
 
   @Override

src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java

@@ -34,6 +34,20 @@ public class ArdTopicPageTask extends ArdTaskBase<ArdFilmInfoDto, CrawlerUrlDTO>
   private static final Set<String> TOPICS_LOAD_ALL_PAGES = new HashSet<>();
 
   static {
+    // Retro
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3JhZGlvYnJlbWVuLmRlL2IwYTJlZWFlLWI2NjAtNDI5Yi05ZTE3LTM5YzlkZDhmNTc4Ng");
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2JyLmRlL2Jyb2FkY2FzdFNlcmllcy8yOGEwMzU4Yi00N2ViLTQ0MDktOGFmZi02ZjVkMDE5NDA2NDc");
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3dkci5kZS93ZHJyZXRybw");
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL25kci5kZS80NTg4");
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3JiYi1vbmxpbmUuZGUvYmVybGluZXItYWJlbmRzY2hhdQ");
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2hyLW9ubGluZS8zODIyMDExOA");
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3N3ci5kZS8yNDEwMzE1Ng");
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2hyLW9ubGluZS8zODIyMDA5Ng");
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3NyLW9ubGluZS5kZS9SRVRSTy1IRA");
+    // NDR Nordstory
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL25kci5kZS8xNDk1");
+    // SWR Eisenbahn-Romantik
+    TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL3N3ci5kZS8yMjUwOTg2");
     // example: TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL3Nwb3J0c2NoYXU");
   }
 
@@ -68,7 +82,7 @@ private ConcurrentLinkedQueue<CrawlerUrlDTO> createSubPageUrls(
           final WebTarget aTarget, final ArdTopicInfoDto topicInfo) {
     final ConcurrentLinkedQueue<CrawlerUrlDTO> subpages = new ConcurrentLinkedQueue<>();
 
-    final int actualSubPageNumber = topicInfo.getSubPageNumber();
+    final int actualSubPageNumber = topicInfo.getPageNumber();
     final int maximumAllowedSubpages = getMaximumSubpages(topicInfo.getId());
     if (actualSubPageNumber != 0) {
       LOG.debug("Sub page {} is already the maximum allowed sub page.", actualSubPageNumber);