Handle GVL deletions

docs/config-app.md

@@ -457,6 +457,9 @@ If not defined in config all other Health Checkers would be disabled and endpoin
 - `gdpr.special-features.sfN.vendor-exceptions[]` - bidder names that will be treated opposite to `sfN.enforce` value.
 - `gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow.
 - `gdpr.vendorlist.default-timeout-ms` - default operation timeout for obtaining new vendor list.
+- `gdpr.vendorlist.live.url` - URL of the latest TCF GVL used to detect vendors with a past `deletedDate`. Default `https://vendor-list.consensu.org/v3/vendor-list.json`.
+- `gdpr.vendorlist.live.refresh-period-ms` - how often to refresh the live GVL deleted-vendor set, in milliseconds. Default `86400000` (24 hours).
+- `gdpr.vendorlist.live.startup-cache-dir` - directory for local storage vendor list cache. At startup server will attempt to read latest GVL from this location.
 - `gdpr.vendorlist.v2.http-endpoint-template` - template string for vendor list url version 2.
 - `gdpr.vendorlist.v2.refresh-missing-list-period-ms` - time to wait between attempts to fetch vendor list version that previously was reported to be missing by origin. Default `3600000` (one hour).
 - `gdpr.vendorlist.v2.fallback-vendor-list-path` - location on the file system of the fallback vendor list that will be used in place of missing vendor list versions. Optional.

docs/metrics.md

@@ -131,6 +131,7 @@ Following metrics are collected and submitted if account is configured with `det
 - `privacy.tcf.(v1,v2).in-geo` - number of requests received from TCF-concerned geo region with consent string of particular version 
 - `privacy.tcf.(v1,v2).out-geo` - number of requests received outside of TCF-concerned geo region with consent string of particular version
 - `privacy.tcf.(v1,v2).vendorlist.(missing|ok|err|fallback)` - number of processed vendor lists of particular version
+- `privacy.tcf.vendorlist.live.(ok|err)` - number of successful or failed refreshes of the live GVL used for deleted-vendor detection
 - `privacy.usp.specified` - number of requests with a valid US Privacy string (CCPA)
 - `privacy.usp.opt-out` - number of requests that required privacy enforcement according to CCPA rules
 - `privacy.lmt` - number of requests that required privacy enforcement according to LMT flag

sample/001_banner/configs/config.yaml

@@ -25,6 +25,8 @@ gdpr:
       cache-dir: /var/tmp/vendor2
     v3:
       cache-dir: /var/tmp/vendor3
+    live:
+      startup-cache-dir: /var/tmp/vendor3
 admin-endpoints:
   logging-changelevel:
     enabled: true

sample/configs/localdev-config.yaml

@@ -24,3 +24,5 @@ gdpr:
       cache-dir: /var/tmp/vendor2
     v3:
       cache-dir: /var/tmp/vendor3
+    live:
+      startup-cache-dir: /var/tmp/vendor3

sample/configs/prebid-config-db.yaml

@@ -43,6 +43,8 @@ gdpr:
       cache-dir: /var/tmp/vendor2
     v3:
       cache-dir: /var/tmp/vendor3
+    live:
+      startup-cache-dir: /var/tmp/vendor3
 admin-endpoints:
   logging-changelevel:
     enabled: true

sample/configs/prebid-config-s3.yaml

@@ -51,6 +51,8 @@ gdpr:
       cache-dir: /var/tmp/vendor2
     v3:
       cache-dir: /var/tmp/vendor3
+    live:
+      startup-cache-dir: /var/tmp/vendor3
 
 admin-endpoints:
   logging-changelevel:

sample/configs/prebid-config-with-51d-dd.yaml

@@ -33,6 +33,8 @@ gdpr:
       cache-dir: /var/tmp/vendor2
     v3:
       cache-dir: /var/tmp/vendor3
+    live:
+      startup-cache-dir: /var/tmp/vendor3
 admin-endpoints:
   logging-changelevel:
     enabled: true

sample/configs/prebid-config-with-module.yaml

@@ -33,6 +33,8 @@ gdpr:
       cache-dir: /var/tmp/vendor2
     v3:
       cache-dir: /var/tmp/vendor3
+    live:
+      startup-cache-dir: /var/tmp/vendor3
 admin-endpoints:
   logging-changelevel:
     enabled: true

sample/configs/prebid-config-with-optable.yaml

@@ -39,6 +39,8 @@ gdpr:
       cache-dir: /var/tmp/vendor2
     v3:
       cache-dir: /var/tmp/vendor3
+    live:
+      startup-cache-dir: /var/tmp/vendor3
 admin-endpoints:
   logging-changelevel:
     enabled: true

sample/configs/prebid-config-with-wurfl.yaml

@@ -34,6 +34,8 @@ gdpr:
       cache-dir: /var/tmp/vendor2
     v3:
       cache-dir: /var/tmp/vendor3
+    live:
+      startup-cache-dir: /var/tmp/vendor3
 admin-endpoints:
   logging-changelevel:
     enabled: true

sample/configs/prebid-config.yaml

@@ -34,6 +34,8 @@ gdpr:
       cache-dir: /var/tmp/vendor2
     v3:
       cache-dir: /var/tmp/vendor3
+    live:
+      startup-cache-dir: /var/tmp/vendor3
 admin-endpoints:
   logging-changelevel:
     enabled: true

src/main/docker/application.yaml

@@ -4,3 +4,5 @@ gdpr:
       cache-dir: /app/prebid-server/data/vendorlist-v2
     v3:
       cache-dir: /app/prebid-server/data/vendorlist-v3
+    live:
+      startup-cache-dir: /app/prebid-server/data/vendorlist-v3

src/main/java/org/prebid/server/metric/Metrics.java

@@ -554,6 +554,14 @@ public void updatePrivacyTcfVendorListFallbackMetric(int version) {
         updatePrivacyTcfVendorListMetric(version, MetricName.fallback);
     }
 
+    public void updatePrivacyTcfLiveVendorListOkMetric() {
+        privacy().tcf().liveVendorList().incCounter(MetricName.ok);
+    }
+
+    public void updatePrivacyTcfLiveVendorListErrorMetric() {
+        privacy().tcf().liveVendorList().incCounter(MetricName.err);
+    }
+
     private void updatePrivacyTcfVendorListMetric(int version, MetricName metricName) {
         final TcfMetrics tcfMetrics = privacy().tcf();
         tcfMetrics.fromVersion(version).vendorList().incCounter(metricName);

src/main/java/org/prebid/server/metric/TcfMetrics.java

@@ -16,6 +16,7 @@ class TcfMetrics extends UpdatableMetrics {
 
     private final TcfVersionMetrics tcfVersion1Metrics;
     private final TcfVersionMetrics tcfVersion2Metrics;
+    private final LiveVendorListMetrics liveVendorListMetrics;
 
     TcfMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
         super(
@@ -25,6 +26,7 @@ class TcfMetrics extends UpdatableMetrics {
 
         tcfVersion1Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v1");
         tcfVersion2Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v2");
+        liveVendorListMetrics = new LiveVendorListMetrics(metricRegistry, counterType, createTcfPrefix(prefix));
     }
 
     TcfVersionMetrics fromVersion(int version) {
@@ -35,6 +37,10 @@ TcfVersionMetrics fromVersion(int version) {
         };
     }
 
+    LiveVendorListMetrics liveVendorList() {
+        return liveVendorListMetrics;
+    }
+
     private static String createTcfPrefix(String prefix) {
         return prefix + ".tcf";
     }
@@ -87,4 +93,22 @@ private static Function<MetricName, String> nameCreator(String prefix) {
             return metricName -> "%s.%s".formatted(prefix, metricName);
         }
     }
+
+    static class LiveVendorListMetrics extends UpdatableMetrics {
+
+        LiveVendorListMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
+            super(
+                    metricRegistry,
+                    counterType,
+                    nameCreator(createLatestPrefix(prefix)));
+        }
+
+        private static String createLatestPrefix(String prefix) {
+            return prefix + ".vendorlist.live";
+        }
+
+        private static Function<MetricName, String> nameCreator(String prefix) {
+            return metricName -> "%s.%s".formatted(prefix, metricName);
+        }
+    }
 }

src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java

@@ -12,6 +12,7 @@
 import org.prebid.server.privacy.gdpr.model.VendorPermissionWithGvl;
 import org.prebid.server.privacy.gdpr.tcfstrategies.purpose.PurposeStrategy;
 import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy;
+import org.prebid.server.privacy.gdpr.vendorlist.VendorListWrapper;
 import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService;
 import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode;
 import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor;
@@ -114,7 +115,7 @@ private Future<Collection<VendorPermission>> permissionsForInternal(Collection<V
                                 mergedPurposeOneTreatmentInterpretation),
                         ignored -> processDowngradedSupportedPurposeStrategies(
                                 tcfConsent,
-                                wrapWithGVL(vendorPermissionsByType, Collections.emptyMap()),
+                                wrapWithGVL(vendorPermissionsByType, VendorListWrapper.EMPTY),
                                 mergedPurposes,
                                 mergedPurposeOneTreatmentInterpretation))
                 .map(ignored -> enforcePurpose4IfRequired(mergedPurposes, vendorPermissionsByType))
@@ -146,7 +147,7 @@ private static VendorPermissionsByType<VendorPermission> toVendorPermissionsByTy
 
     private static VendorPermissionsByType<VendorPermissionWithGvl> wrapWithGVL(
             VendorPermissionsByType<VendorPermission> vendorPermissionsByType,
-            Map<Integer, Vendor> vendorGvlPermissions) {
+            VendorListWrapper vendorGvlPermissions) {
 
         final List<VendorPermissionWithGvl> weakPermissions = vendorPermissionsByType.getWeakPermissions().stream()
                 .map(vendorPermission -> wrapWithGVL(vendorPermission, vendorGvlPermissions))
@@ -161,7 +162,7 @@ private static VendorPermissionsByType<VendorPermissionWithGvl> wrapWithGVL(
     }
 
     private static VendorPermissionWithGvl wrapWithGVL(VendorPermission vendorPermission,
-                                                       Map<Integer, Vendor> vendorGvlPermissions) {
+                                                       VendorListWrapper vendorGvlPermissions) {
 
         final Integer vendorId = vendorPermission.getVendorId();
         final Vendor vendorGvlByVendorId = Optional.ofNullable(vendorId)

src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java

@@ -0,0 +1,127 @@
+package org.prebid.server.privacy.gdpr.vendorlist;
+
+import io.vertx.core.Future;
+import io.vertx.core.Vertx;
+import lombok.Getter;
+import org.prebid.server.exception.PreBidException;
+import org.prebid.server.json.JacksonMapper;
+import org.prebid.server.log.Logger;
+import org.prebid.server.log.LoggerFactory;
+import org.prebid.server.metric.Metrics;
+import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor;
+import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList;
+import org.prebid.server.util.HttpUtil;
+import org.prebid.server.vertx.Initializable;
+import org.prebid.server.vertx.httpclient.HttpClient;
+import org.prebid.server.vertx.httpclient.model.HttpClientResponse;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class LiveVendorListService implements Initializable {
+
+    private static final Logger logger = LoggerFactory.getLogger(LiveVendorListService.class);
+
+    private final String cacheDir;
+    private final String liveGvlUrl;
+    private final long refreshPeriodMs;
+    private final int defaultTimeoutMs;
+    private final Vertx vertx;
+    private final HttpClient httpClient;
+    private final VendorListFileStore vendorListFileStore;
+    private final Metrics metrics;
+    private final JacksonMapper mapper;
+    private final Clock clock;
+
+    @Getter
+    private volatile Set<Integer> deletedVendorIds = Set.of();
+
+    public LiveVendorListService(String cacheDir,
+                                 String liveGvlUrl,
+                                 long refreshPeriodMs,
+                                 int defaultTimeoutMs,
+                                 Vertx vertx,
+                                 HttpClient httpClient,
+                                 VendorListFileStore vendorListFileStore,
+                                 Metrics metrics,
+                                 JacksonMapper mapper,
+                                 Clock clock) {
+
+        this.cacheDir = Objects.requireNonNull(cacheDir);
+        this.liveGvlUrl = HttpUtil.validateUrl(Objects.requireNonNull(liveGvlUrl));
+        this.refreshPeriodMs = refreshPeriodMs;
+        this.defaultTimeoutMs = defaultTimeoutMs;
+        this.vertx = Objects.requireNonNull(vertx);
+        this.httpClient = Objects.requireNonNull(httpClient);
+        this.vendorListFileStore = Objects.requireNonNull(vendorListFileStore);
+        this.metrics = Objects.requireNonNull(metrics);
+        this.mapper = Objects.requireNonNull(mapper);
+        this.clock = Objects.requireNonNull(clock);
+    }
+
+    @Override
+    public Future<Void> initialize() {
+        initializeWithLatestCachedVersion();
+        vertx.setPeriodic(0, refreshPeriodMs, ignored -> refresh());
+        return Future.succeededFuture();
+    }
+
+    private void initializeWithLatestCachedVersion() {
+        vendorListFileStore.getLatestVendorListFromCache(cacheDir).ifPresent(vendorList -> {
+            saveDeletedVendorsFromVendorList(vendorList);
+            logger.info("Initialized live GVL from cache with version %d".formatted(vendorList.getVendorListVersion()));
+        });
+    }
+
+    void refresh() {
+        httpClient.get(liveGvlUrl, defaultTimeoutMs)
+                .map(this::processResponse)
+                .map(this::saveDeletedVendorsFromVendorList)
+                .otherwise(this::handleError);
+    }
+
+    private Void saveDeletedVendorsFromVendorList(VendorList vendorList) {
+        updateDeletedVendorIds(extractDeletedVendorIds(vendorList));
+        return null;
+    }
+
+    private VendorList processResponse(HttpClientResponse response) {
+        final int statusCode = response.getStatusCode();
+        if (statusCode != 200) {
+            throw new PreBidException("HTTP status code " + statusCode);
+        }
+
+        final String body = response.getBody();
+        final VendorList vendorList = VendorListUtil.parseVendorList(body, mapper);
+
+        if (!VendorListUtil.vendorListIsValid(vendorList)) {
+            throw new PreBidException("Fetched vendor list parsed but has invalid data: " + body);
+        }
+
+        return vendorList;
+    }
+
+    Set<Integer> extractDeletedVendorIds(VendorList vendorList) {
+        final Instant now = clock.instant();
+        return vendorList.getVendors().values().stream()
+                .filter(vendor -> VendorListUtil.vendorIsDeletedAt(vendor, now))
+                .map(Vendor::getId)
+                .filter(Objects::nonNull)
+                .collect(Collectors.toUnmodifiableSet());
+    }
+
+    private Void updateDeletedVendorIds(Set<Integer> ids) {
+        deletedVendorIds = ids;
+        metrics.updatePrivacyTcfLiveVendorListOkMetric();
+        return null;
+    }
+
+    private Void handleError(Throwable exception) {
+        logger.warn("Error occurred while fetching live GVL", exception);
+        metrics.updatePrivacyTcfLiveVendorListErrorMetric();
+        return null;
+    }
+}