From 206c088e048a6098310baed6382842e249f69807 Mon Sep 17 00:00:00 2001 From: Viktor Kryshtal Date: Thu, 28 May 2026 14:48:05 +0300 Subject: [PATCH 01/11] Implemented handling of gvl deletions --- docs/config-app.md | 2 + docs/metrics.md | 1 + .../org/prebid/server/metric/Metrics.java | 8 + .../org/prebid/server/metric/TcfMetrics.java | 24 ++ .../vendorlist/LiveVendorListService.java | 122 ++++++++++ .../VersionedVendorListService.java | 18 +- .../privacy/gdpr/vendorlist/proto/Vendor.java | 4 + .../config/PrivacyServiceConfiguration.java | 29 ++- src/main/resources/application.yaml | 2 + .../metrics-config/prometheus-labels.yaml | 4 + .../org/prebid/server/metric/MetricsTest.java | 18 ++ .../vendorlist/LiveVendorListServiceTest.java | 229 ++++++++++++++++++ .../VersionedVendorListServiceTest.java | 47 +++- 13 files changed, 500 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java create mode 100644 src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java diff --git a/docs/config-app.md b/docs/config-app.md index a661f5a74a2..5c5fbaa95e2 100644 --- a/docs/config-app.md +++ b/docs/config-app.md @@ -457,6 +457,8 @@ If not defined in config all other Health Checkers would be disabled and endpoin - `gdpr.special-features.sfN.vendor-exceptions[]` - bidder names that will be treated opposite to `sfN.enforce` value. - `gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow. - `gdpr.vendorlist.default-timeout-ms` - default operation timeout for obtaining new vendor list. +- `gdpr.vendorlist.live-gvl-url` - URL of the latest TCF GVL used to detect vendors with a past `deletedDate`. Default `https://vendor-list.consensu.org/v3/vendor-list.json`. +- `gdpr.vendorlist.live-gvl-refresh-period-ms` - how often to refresh the live GVL deleted-vendor set, in milliseconds. Default `86400000` (24 hours). - `gdpr.vendorlist.v2.http-endpoint-template` - template string for vendor list url version 2. - `gdpr.vendorlist.v2.refresh-missing-list-period-ms` - time to wait between attempts to fetch vendor list version that previously was reported to be missing by origin. Default `3600000` (one hour). - `gdpr.vendorlist.v2.fallback-vendor-list-path` - location on the file system of the fallback vendor list that will be used in place of missing vendor list versions. Optional. diff --git a/docs/metrics.md b/docs/metrics.md index c07e0660598..e5844413032 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -131,6 +131,7 @@ Following metrics are collected and submitted if account is configured with `det - `privacy.tcf.(v1,v2).in-geo` - number of requests received from TCF-concerned geo region with consent string of particular version - `privacy.tcf.(v1,v2).out-geo` - number of requests received outside of TCF-concerned geo region with consent string of particular version - `privacy.tcf.(v1,v2).vendorlist.(missing|ok|err|fallback)` - number of processed vendor lists of particular version +- `privacy.tcf.vendorlist.latest.(ok|err)` - number of successful or failed refreshes of the live GVL used for deleted-vendor detection - `privacy.usp.specified` - number of requests with a valid US Privacy string (CCPA) - `privacy.usp.opt-out` - number of requests that required privacy enforcement according to CCPA rules - `privacy.lmt` - number of requests that required privacy enforcement according to LMT flag diff --git a/src/main/java/org/prebid/server/metric/Metrics.java b/src/main/java/org/prebid/server/metric/Metrics.java index 517fbb36bd5..e533ba53636 100644 --- a/src/main/java/org/prebid/server/metric/Metrics.java +++ b/src/main/java/org/prebid/server/metric/Metrics.java @@ -554,6 +554,14 @@ public void updatePrivacyTcfVendorListFallbackMetric(int version) { updatePrivacyTcfVendorListMetric(version, MetricName.fallback); } + public void updatePrivacyTcfVendorListLatestOkMetric() { + privacy().tcf().vendorListLatest().incCounter(MetricName.ok); + } + + public void updatePrivacyTcfVendorListLatestErrorMetric() { + privacy().tcf().vendorListLatest().incCounter(MetricName.err); + } + private void updatePrivacyTcfVendorListMetric(int version, MetricName metricName) { final TcfMetrics tcfMetrics = privacy().tcf(); tcfMetrics.fromVersion(version).vendorList().incCounter(metricName); diff --git a/src/main/java/org/prebid/server/metric/TcfMetrics.java b/src/main/java/org/prebid/server/metric/TcfMetrics.java index 9fd5a811562..ba5d8ed86f3 100644 --- a/src/main/java/org/prebid/server/metric/TcfMetrics.java +++ b/src/main/java/org/prebid/server/metric/TcfMetrics.java @@ -16,6 +16,7 @@ class TcfMetrics extends UpdatableMetrics { private final TcfVersionMetrics tcfVersion1Metrics; private final TcfVersionMetrics tcfVersion2Metrics; + private final VendorListLatestMetrics vendorListLatestMetrics; TcfMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { super( @@ -25,6 +26,7 @@ class TcfMetrics extends UpdatableMetrics { tcfVersion1Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v1"); tcfVersion2Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v2"); + vendorListLatestMetrics = new VendorListLatestMetrics(metricRegistry, counterType, createTcfPrefix(prefix)); } TcfVersionMetrics fromVersion(int version) { @@ -35,6 +37,10 @@ TcfVersionMetrics fromVersion(int version) { }; } + VendorListLatestMetrics vendorListLatest() { + return vendorListLatestMetrics; + } + private static String createTcfPrefix(String prefix) { return prefix + ".tcf"; } @@ -87,4 +93,22 @@ private static Function nameCreator(String prefix) { return metricName -> "%s.%s".formatted(prefix, metricName); } } + + static class VendorListLatestMetrics extends UpdatableMetrics { + + VendorListLatestMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { + super( + metricRegistry, + counterType, + nameCreator(createLatestPrefix(prefix))); + } + + private static String createLatestPrefix(String prefix) { + return prefix + ".vendorlist.latest"; + } + + private static Function nameCreator(String prefix) { + return metricName -> "%s.%s".formatted(prefix, metricName); + } + } } diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java new file mode 100644 index 00000000000..dc3c8b62ed2 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java @@ -0,0 +1,122 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import io.vertx.core.Promise; +import io.vertx.core.Vertx; +import org.apache.commons.collections4.CollectionUtils; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; +import org.prebid.server.metric.Metrics; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; +import org.prebid.server.util.HttpUtil; +import org.prebid.server.vertx.Initializable; +import org.prebid.server.vertx.httpclient.HttpClient; +import org.prebid.server.vertx.httpclient.model.HttpClientResponse; + +import java.io.IOException; +import java.time.Clock; +import java.time.Instant; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class LiveVendorListService implements Initializable { + + private static final Logger logger = LoggerFactory.getLogger(LiveVendorListService.class); + + private final String liveGvlUrl; + private final long refreshPeriodMs; + private final int defaultTimeoutMs; + private final Vertx vertx; + private final HttpClient httpClient; + private final JacksonMapper mapper; + private final Metrics metrics; + private final Clock clock; + + private volatile Set deletedVendorIds = Set.of(); + + public LiveVendorListService(String liveGvlUrl, + long refreshPeriodMs, + int defaultTimeoutMs, + Vertx vertx, + HttpClient httpClient, + JacksonMapper mapper, + Metrics metrics, + Clock clock) { + + this.liveGvlUrl = HttpUtil.validateUrl(Objects.requireNonNull(liveGvlUrl)); + this.refreshPeriodMs = refreshPeriodMs; + this.defaultTimeoutMs = defaultTimeoutMs; + this.vertx = Objects.requireNonNull(vertx); + this.httpClient = Objects.requireNonNull(httpClient); + this.mapper = Objects.requireNonNull(mapper); + this.metrics = Objects.requireNonNull(metrics); + this.clock = Objects.requireNonNull(clock); + } + + public boolean isDeleted(Integer id) { + final Set ids = deletedVendorIds; + return !ids.isEmpty() && ids.contains(id); + } + + @Override + public void initialize(Promise initializePromise) { + vertx.setPeriodic(0, refreshPeriodMs, ignored -> refresh()); + + initializePromise.tryComplete(); + } + + void refresh() { + httpClient.get(liveGvlUrl, defaultTimeoutMs) + .map(this::processResponse) + .map(this::extractDeletedVendorIds) + .map(this::updateDeletedVendorIds) + .otherwise(this::handleError); + } + + private VendorList processResponse(HttpClientResponse response) { + final int statusCode = response.getStatusCode(); + if (statusCode != 200) { + throw new PreBidException("HTTP status code " + statusCode); + } + + final String body = response.getBody(); + try { + return mapper.mapper().readValue(body, VendorList.class); + } catch (IOException e) { + throw new PreBidException("Cannot parse live vendor list: " + body, e); + } + } + + Set extractDeletedVendorIds(VendorList vendorList) { + final Instant now = clock.instant(); + return vendorList.getVendors().values().stream() + .filter(vendor -> isDeletedAt(vendor, now)) + .map(Vendor::getId) + .filter(Objects::nonNull) + .collect(Collectors.toUnmodifiableSet()); + } + + private static boolean isDeletedAt(Vendor vendor, Instant now) { + final Instant deletedDate = vendor.getDeletedDate(); + return deletedDate != null && deletedDate.isBefore(now); + } + + private Void updateDeletedVendorIds(Set ids) { + if (CollectionUtils.isEmpty(ids)) { + throw new PreBidException("Live GVL response has no deleted vendors"); + } + + deletedVendorIds = ids; + metrics.updatePrivacyTcfVendorListLatestOkMetric(); + return null; + } + + private Void handleError(Throwable exception) { + logger.warn("Error occurred while fetching live GVL", exception); + metrics.updatePrivacyTcfVendorListLatestErrorMetric(); + return null; + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java index 5e261d9b6b4..dada968604d 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java @@ -6,23 +6,37 @@ import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; public class VersionedVendorListService { private final VendorListService vendorListServiceV2; private final VendorListService vendorListServiceV3; + private final LiveVendorListService liveVendorListService; + + public VersionedVendorListService(VendorListService vendorListServiceV2, + VendorListService vendorListServiceV3, + LiveVendorListService liveVendorListService) { - public VersionedVendorListService(VendorListService vendorListServiceV2, VendorListService vendorListServiceV3) { this.vendorListServiceV2 = Objects.requireNonNull(vendorListServiceV2); this.vendorListServiceV3 = Objects.requireNonNull(vendorListServiceV3); + this.liveVendorListService = Objects.requireNonNull(liveVendorListService); } public Future> forConsent(TCString consent) { final int tcfPolicyVersion = consent.getTcfPolicyVersion(); final int vendorListVersion = consent.getVendorListVersion(); - return tcfPolicyVersion < 4 + final Future> vendorListFuture = tcfPolicyVersion < 4 ? vendorListServiceV2.forVersion(vendorListVersion) : vendorListServiceV3.forVersion(vendorListVersion); + + return vendorListFuture.map(this::filterDeletedVendors); + } + + private Map filterDeletedVendors(Map vendors) { + return vendors.entrySet().stream() + .filter(entry -> !liveVendorListService.isDeleted(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } } diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java index 6bb2be9dddb..336a5d4cdae 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/proto/Vendor.java @@ -6,6 +6,7 @@ import lombok.Data; import lombok.NoArgsConstructor; +import java.time.Instant; import java.util.EnumSet; @AllArgsConstructor @@ -16,6 +17,9 @@ public class Vendor { Integer id; + @JsonProperty("deletedDate") + Instant deletedDate; + EnumSet purposes; @JsonProperty("legIntPurposes") diff --git a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java index b8175828c3b..70d5b672cab 100644 --- a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java @@ -37,6 +37,7 @@ import org.prebid.server.privacy.gdpr.tcfstrategies.purpose.typestrategies.PurposeTwoBasicEnforcePurposeStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesOneStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy; +import org.prebid.server.privacy.gdpr.vendorlist.LiveVendorListService; import org.prebid.server.privacy.gdpr.vendorlist.VendorListFetchThrottler; import org.prebid.server.privacy.gdpr.vendorlist.VendorListService; import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService; @@ -135,11 +136,35 @@ VendorListServiceConfigurationProperties vendorListServiceV3Properties() { return new VendorListServiceConfigurationProperties(); } + @Bean + LiveVendorListService liveVendorListService( + @Value("${gdpr.vendorlist.live-gvl-url}") String liveGvlUrl, + @Value("${gdpr.vendorlist.live-gvl-refresh-period-ms}") long refreshPeriodMs, + @Value("${gdpr.vendorlist.default-timeout-ms}") int defaultTimeoutMs, + Vertx vertx, + HttpClient httpClient, + JacksonMapper mapper, + Metrics metrics, + Clock clock) { + + return new LiveVendorListService( + liveGvlUrl, + refreshPeriodMs, + defaultTimeoutMs, + vertx, + httpClient, + mapper, + metrics, + clock); + } + @Bean VersionedVendorListService versionedVendorListService(VendorListService vendorListServiceV2, - VendorListService vendorListServiceV3) { + VendorListService vendorListServiceV3, + LiveVendorListService liveVendorListService) { - return new VersionedVendorListService(vendorListServiceV2, vendorListServiceV3); + return new VersionedVendorListService( + vendorListServiceV2, vendorListServiceV3, liveVendorListService); } @Bean diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 55822047954..28fc07808f8 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -207,6 +207,8 @@ gdpr: eea-countries: at,bg,be,cy,cz,dk,ee,fi,fr,de,gr,hu,ie,it,lv,lt,lu,mt,nl,pl,pt,ro,sk,si,es,se,gb,is,no,li,ai,aw,pt,bm,aq,io,vg,ic,ky,fk,re,mw,gp,gf,yt,pf,tf,gl,pt,ms,an,bq,cw,sx,nc,pn,sh,pm,gs,tc,uk,wf vendorlist: default-timeout-ms: 2000 + live-gvl-refresh-period-ms: 86400000 + live-gvl-url: https://vendor-list.consensu.org/v3/vendor-list.json v2: http-endpoint-template: https://vendor-list.consensu.org/v2/archives/vendor-list-v{VERSION}.json refresh-missing-list-period-ms: 3600000 diff --git a/src/main/resources/metrics-config/prometheus-labels.yaml b/src/main/resources/metrics-config/prometheus-labels.yaml index b40139d6a8a..6941b1d7cb0 100644 --- a/src/main/resources/metrics-config/prometheus-labels.yaml +++ b/src/main/resources/metrics-config/prometheus-labels.yaml @@ -85,6 +85,10 @@ mappers: labels: tcf: ${0} status: ${1} + - match: privacy.tcf.vendorlist.latest.* + name: privacy.tcf.vendorlist.latest + labels: + status: ${0} - match: privacy.tcf.*.* name: privacy.tcf.${1} labels: diff --git a/src/test/java/org/prebid/server/metric/MetricsTest.java b/src/test/java/org/prebid/server/metric/MetricsTest.java index 8a56e279d30..1df98cedd68 100644 --- a/src/test/java/org/prebid/server/metric/MetricsTest.java +++ b/src/test/java/org/prebid/server/metric/MetricsTest.java @@ -1004,6 +1004,24 @@ public void updatePrivacyTcfVendorListFallbackMetricShouldIncrementMetric() { assertThat(metricRegistry.counter("privacy.tcf.v1.vendorlist.fallback").getCount()).isEqualTo(1); } + @Test + public void updatePrivacyTcfVendorListLatestOkMetricShouldIncrementMetric() { + // when + metrics.updatePrivacyTcfVendorListLatestOkMetric(); + + // then + assertThat(metricRegistry.counter("privacy.tcf.vendorlist.latest.ok").getCount()).isOne(); + } + + @Test + public void updatePrivacyTcfVendorListLatestErrorMetricShouldIncrementMetric() { + // when + metrics.updatePrivacyTcfVendorListLatestErrorMetric(); + + // then + assertThat(metricRegistry.counter("privacy.tcf.vendorlist.latest.err").getCount()).isOne(); + } + @Test public void shouldNotUpdateAccountMetricsIfVerbosityIsNone() { // given diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java new file mode 100644 index 00000000000..f2f05c8b88b --- /dev/null +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java @@ -0,0 +1,229 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.VertxTest; +import org.prebid.server.metric.Metrics; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; +import org.prebid.server.vertx.httpclient.HttpClient; +import org.prebid.server.vertx.httpclient.model.HttpClientResponse; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +@ExtendWith(MockitoExtension.class) +public class LiveVendorListServiceTest extends VertxTest { + + private static final Instant NOW = Instant.parse("2024-06-01T12:00:00Z"); + private static final String LIVE_GVL_URL = "https://example.com"; + + @Mock + private Vertx vertx; + @Mock + private HttpClient httpClient; + @Mock + private Metrics metrics; + + private LiveVendorListService target; + + @BeforeEach + public void setUp() { + target = new LiveVendorListService( + LIVE_GVL_URL, + 0, + 1000, + vertx, + httpClient, + jacksonMapper, + metrics, + Clock.fixed(NOW, ZoneOffset.UTC)); + } + + @Test + public void isDeletedShouldReturnFalseWhenFetchNeverSucceeded() { + // when and then + assertThat(target.isDeleted(1)).isFalse(); + assertThat(target.isDeleted(null)).isFalse(); + } + + @Test + public void isDeletedShouldReturnTrueWhenVendorIsDeletedInLiveVendorList() throws JsonProcessingException { + // given + final String responseBody = givenLiveGvlJson(Map.of(42, "2024-01-01T00:00:00Z")); + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.succeededFuture(HttpClientResponse.of(200, null, responseBody))); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(42)).isTrue(); + assertThat(target.isDeleted(99)).isFalse(); + } + + @Test + public void extractDeletedVendorIdsShouldReturnOnlyVendorsWithPastDeletedDate() { + // given + final VendorList vendorList = givenVendorList(Map.of( + 1, givenVendor(1, "2024-01-01T00:00:00Z"), + 2, givenVendor(2, null), + 3, givenVendor(3, "2025-01-01T00:00:00Z"), + 4, givenVendor(4, "2024-06-01T12:00:00Z"))); + + // when + final var deletedIds = target.extractDeletedVendorIds(vendorList); + + // then + assertThat(deletedIds).containsExactly(1); + } + + @Test + public void refreshShouldUpdateDeletedVendorIdsAndIncrementOkMetric() throws JsonProcessingException { + // given + givenHttpClientReturnsResponse(200, givenLiveGvlJson(Map.of(1, "2024-01-01T00:00:00Z"))); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isTrue(); + verify(metrics).updatePrivacyTcfVendorListLatestOkMetric(); + verify(metrics, never()).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + @Test + public void refreshShouldReplaceDeletedVendorIdsOnSubsequentSuccessfulFetch() throws JsonProcessingException { + // given + given(httpClient.get(anyString(), anyLong())) + .willReturn( + Future.succeededFuture(HttpClientResponse.of( + 200, null, givenLiveGvlJson(Map.of(1, "2024-01-01T00:00:00Z")))), + Future.succeededFuture(HttpClientResponse.of( + 200, null, givenLiveGvlJson(Map.of(2, "2024-02-01T00:00:00Z"))))); + + // when + target.refresh(); + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + assertThat(target.isDeleted(2)).isTrue(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnHttpFailure() { + // given + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.failedFuture(new RuntimeException("connection failed"))); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + verify(metrics, never()).updatePrivacyTcfVendorListLatestOkMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnNonOkStatus() { + // given + givenHttpClientReturnsResponse(503, "{}"); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricOnInvalidJson() { + // given + givenHttpClientReturnsResponse(200, "invalid-json"); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + @Test + public void refreshShouldIncrementErrorMetricWhenNoDeletedVendorsInResponse() throws JsonProcessingException { + // given + givenHttpClientReturnsResponse(200, givenLiveGvlJson(emptyMap())); + + // when + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isFalse(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + verify(metrics, never()).updatePrivacyTcfVendorListLatestOkMetric(); + } + + @Test + public void refreshShouldKeepLastGoodSetOnFailureAfterSuccessfulFetch() throws JsonProcessingException { + // given + given(httpClient.get(anyString(), anyLong())) + .willReturn( + Future.succeededFuture(HttpClientResponse.of( + 200, null, givenLiveGvlJson(Map.of(1, "2024-01-01T00:00:00Z")))), + Future.failedFuture(new RuntimeException("connection failed"))); + + // when + target.refresh(); + target.refresh(); + + // then + assertThat(target.isDeleted(1)).isTrue(); + verify(metrics).updatePrivacyTcfVendorListLatestOkMetric(); + verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + } + + private void givenHttpClientReturnsResponse(int statusCode, String response) { + given(httpClient.get(anyString(), anyLong())) + .willReturn(Future.succeededFuture(HttpClientResponse.of(statusCode, null, response))); + } + + private String givenLiveGvlJson(Map vendorIdToDeletedDate) throws JsonProcessingException { + final Map vendors = new HashMap<>(); + for (Map.Entry entry : vendorIdToDeletedDate.entrySet()) { + vendors.put(entry.getKey(), Map.of("id", entry.getKey(), "deletedDate", entry.getValue())); + } + return mapper.writeValueAsString(Map.of("vendors", vendors)); + } + + private static Vendor givenVendor(int id, String deletedDate) { + return Vendor.builder() + .id(id) + .deletedDate(deletedDate != null ? Instant.parse(deletedDate) : null) + .build(); + } + + private static VendorList givenVendorList(Map vendors) { + return VendorList.of(1, Date.from(Instant.parse("2020-08-20T16:05:24Z")), vendors); + } +} diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java index 8437698b9f0..3ec04ec5602 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java @@ -2,29 +2,40 @@ import com.iabtcf.decoder.TCString; import com.iabtcf.encoder.TCStringEncoder; +import io.vertx.core.Future; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import java.util.Map; import java.util.concurrent.ThreadLocalRandom; +import static java.util.Collections.emptyMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.verify; +import static org.prebid.server.assertion.FutureAssertion.assertThat; @ExtendWith(MockitoExtension.class) public class VersionedVendorListServiceTest { - private VersionedVendorListService versionedVendorListService; + private VersionedVendorListService target; @Mock private VendorListService vendorListServiceV2; @Mock private VendorListService vendorListServiceV3; + @Mock + private LiveVendorListService liveVendorListService; @BeforeEach public void setUp() { - versionedVendorListService = new VersionedVendorListService(vendorListServiceV2, vendorListServiceV3); + target = new VersionedVendorListService( + vendorListServiceV2, vendorListServiceV3, liveVendorListService); } @Test @@ -36,9 +47,10 @@ public void versionedVendorListServiceShouldTreatTcfPolicyLessThanFourAsVendorLi .tcfPolicyVersion(tcfPolicyVersion) .vendorListVersion(12) .toTCString(); + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(emptyMap())); // when - versionedVendorListService.forConsent(consent); + target.forConsent(consent); // then verify(vendorListServiceV2).forVersion(12); @@ -53,11 +65,38 @@ public void versionedVendorListServiceShouldTreatTcfPolicyGreaterOrEqualFourAsVe .tcfPolicyVersion(tcfPolicyVersion) .vendorListVersion(12) .toTCString(); + given(vendorListServiceV3.forVersion(anyInt())).willReturn(Future.succeededFuture(emptyMap())); // when - versionedVendorListService.forConsent(consent); + target.forConsent(consent); // then verify(vendorListServiceV3).forVersion(12); } + + @Test + public void forConsentShouldRemoveVendorsMarkedDeletedInLiveGvl() { + // given + final Vendor deletedVendor = Vendor.empty(1); + final Vendor activeVendor = Vendor.empty(52); + final Map vendorList = Map.of(1, deletedVendor, 52, activeVendor); + final TCString consent = TCStringEncoder.newBuilder() + .version(2) + .tcfPolicyVersion(3) + .vendorListVersion(12) + .toTCString(); + + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(vendorList)); + given(liveVendorListService.isDeleted(1)).willReturn(true); + given(liveVendorListService.isDeleted(52)).willReturn(false); + + // when and then + assertThat(target.forConsent(consent)) + .isSucceeded() + .unwrap() + .satisfies(result -> { + assertThat(result).containsOnlyKeys(52); + assertThat(result.get(52)).isSameAs(activeVendor); + }); + } } From 8aa7fdc3e9ad8a9cc6aab08cfcea100e912f438d Mon Sep 17 00:00:00 2001 From: Viktor Kryshtal Date: Tue, 9 Jun 2026 15:25:36 +0300 Subject: [PATCH 02/11] Refactor: separated disk related functionality of VendorListService --- .../gdpr/vendorlist/VendorListFileStore.java | 116 ++++++ .../gdpr/vendorlist/VendorListResult.java | 14 + .../gdpr/vendorlist/VendorListService.java | 186 +--------- .../gdpr/vendorlist/VendorListUtil.java | 50 +++ .../config/PrivacyServiceConfiguration.java | 26 +- .../vendorlist/VendorListFileStoreTest.java | 334 ++++++++++++++++++ .../vendorlist/VendorListServiceTest.java | 263 ++++++-------- .../gdpr/vendorlist/VendorListUtilTest.java | 192 ++++++++++ 8 files changed, 843 insertions(+), 338 deletions(-) create mode 100644 src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java create mode 100644 src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java create mode 100644 src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java create mode 100644 src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java create mode 100644 src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtilTest.java diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java new file mode 100644 index 00000000000..09bb9a98b57 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java @@ -0,0 +1,116 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.github.benmanes.caffeine.cache.Caffeine; +import io.vertx.core.Future; +import io.vertx.core.Promise; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.file.FileProps; +import io.vertx.core.file.FileSystem; +import io.vertx.core.file.FileSystemException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.ConditionalLogger; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class VendorListFileStore { + + private static final Logger logger = LoggerFactory.getLogger(VendorListFileStore.class); + private static final ConditionalLogger conditionalLogger = new ConditionalLogger(logger); + + private static final String JSON_SUFFIX = ".json"; + + private final double logSamplingRate; + private final FileSystem fileSystem; + private final JacksonMapper mapper; + + public VendorListFileStore(double logSamplingRate, + FileSystem fileSystem, + JacksonMapper mapper) { + + this.logSamplingRate = logSamplingRate; + this.fileSystem = Objects.requireNonNull(fileSystem); + this.mapper = Objects.requireNonNull(mapper); + } + + Map> createCacheFromDisk(String cacheDir) { + createAndCheckWritePermissionsForCacheDir(cacheDir); + final Map versionToFileContent = readFileSystemCache(cacheDir); + + final Map> cache = Caffeine.newBuilder() + .>build() + .asMap(); + + for (Map.Entry versionAndFileContent : versionToFileContent.entrySet()) { + final VendorList vendorList = VendorListUtil.parseVendorList(versionAndFileContent.getValue(), mapper); + + cache.put(Integer.valueOf(versionAndFileContent.getKey()), vendorList.getVendors()); + } + return cache; + } + + private void createAndCheckWritePermissionsForCacheDir(String cacheDir) { + final FileProps props = fileSystem.existsBlocking(cacheDir) ? fileSystem.propsBlocking(cacheDir) : null; + if (props == null || !props.isDirectory()) { + try { + fileSystem.mkdirsBlocking(cacheDir); + } catch (FileSystemException e) { + throw new PreBidException("Cannot create directory: " + cacheDir, e); + } + } else if (!Files.isWritable(Paths.get(cacheDir))) { + throw new PreBidException("No write permissions for directory: " + cacheDir); + } + } + + private Map readFileSystemCache(String cacheDir) { + return fileSystem.readDirBlocking(cacheDir).stream() + .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) + .collect(Collectors.toMap(filepath -> StringUtils.removeEnd(new File(filepath).getName(), JSON_SUFFIX), + filename -> fileSystem.readFileBlocking(filename).toString())); + } + + Future saveToFile(VendorListResult vendorListResult, String cacheDir, String generationVersion) { + final Promise promise = Promise.promise(); + final int version = vendorListResult.getVersion(); + final String filepath = new File(cacheDir, version + JSON_SUFFIX).getPath(); + + fileSystem.writeFile(filepath, Buffer.buffer(vendorListResult.getVendorListAsString()), result -> { + if (result.succeeded()) { + promise.complete(vendorListResult); + } else { + conditionalLogger.error( + "Could not create new vendor list for version %s.%s, file: %s, trace: %s".formatted( + generationVersion, version, filepath, ExceptionUtils.getStackTrace(result.cause())), + logSamplingRate); + promise.fail(result.cause()); + } + }); + + return promise.future(); + } + + Map readFallbackVendorList(String fallbackVendorListPath) { + if (StringUtils.isBlank(fallbackVendorListPath)) { + return null; + } + + final String vendorListContent = fileSystem.readFileBlocking(fallbackVendorListPath).toString(); + final VendorList vendorList = VendorListUtil.parseVendorList(vendorListContent, mapper); + if (!VendorListUtil.vendorListIsValid(vendorList)) { + throw new PreBidException("Fallback vendor list parsed but has invalid data: " + vendorListContent); + } + + return vendorList.getVendors(); + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java new file mode 100644 index 00000000000..827300f90c2 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java @@ -0,0 +1,14 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import lombok.Value; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +@Value(staticConstructor = "of") +class VendorListResult { + + int version; + + String vendorListAsString; + + VendorList vendorList; +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java index 51c9b3e9252..694ab0ed01c 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListService.java @@ -1,17 +1,8 @@ package org.prebid.server.privacy.gdpr.vendorlist; -import com.github.benmanes.caffeine.cache.Caffeine; import io.netty.handler.codec.http.HttpResponseStatus; import io.vertx.core.Future; -import io.vertx.core.Promise; import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.file.FileProps; -import io.vertx.core.file.FileSystem; -import io.vertx.core.file.FileSystemException; -import lombok.Value; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.prebid.server.exception.PreBidException; import org.prebid.server.json.JacksonMapper; @@ -24,16 +15,10 @@ import org.prebid.server.vertx.httpclient.HttpClient; import org.prebid.server.vertx.httpclient.model.HttpClientResponse; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Collection; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; /** * Works with GDPR Vendor List. @@ -54,7 +39,6 @@ public class VendorListService { private static final int TCF_VERSION = 2; - private static final String JSON_SUFFIX = ".json"; private static final String VERSION_PLACEHOLDER = "{VERSION}"; private final double logSamplingRate; @@ -64,21 +48,18 @@ public class VendorListService { private final long refreshMissingListPeriodMs; private final boolean deprecated; private final Vertx vertx; - private final FileSystem fileSystem; private final HttpClient httpClient; private final Metrics metrics; private final String generationVersion; - protected final JacksonMapper mapper; + private final VendorListFetchThrottler fetchThrottler; + private final VendorListFileStore vendorListFileStore; + private final JacksonMapper mapper; - /** - * This is memory/performance optimized model slice: - * map of vendor list version -> map of vendor ID -> Vendors - */ + // Map of vendor list version -> map of vendor ID -> Vendors private final Map> cache; private final Map fallbackVendorList; private final Set versionsToFallback; - private final VendorListFetchThrottler fetchThrottler; public VendorListService(double logSamplingRate, String cacheDir, @@ -88,12 +69,12 @@ public VendorListService(double logSamplingRate, boolean deprecated, String fallbackVendorListPath, Vertx vertx, - FileSystem fileSystem, HttpClient httpClient, Metrics metrics, String generationVersion, - JacksonMapper mapper, - VendorListFetchThrottler fetchThrottler) { + VendorListFetchThrottler fetchThrottler, + VendorListFileStore vendorListFileStore, + JacksonMapper mapper) { this.logSamplingRate = logSamplingRate; this.cacheDir = Objects.requireNonNull(cacheDir); @@ -103,17 +84,15 @@ public VendorListService(double logSamplingRate, this.deprecated = deprecated; this.generationVersion = generationVersion; this.vertx = Objects.requireNonNull(vertx); - this.fileSystem = Objects.requireNonNull(fileSystem); this.httpClient = Objects.requireNonNull(httpClient); this.metrics = Objects.requireNonNull(metrics); - this.mapper = Objects.requireNonNull(mapper); this.fetchThrottler = Objects.requireNonNull(fetchThrottler); + this.vendorListFileStore = Objects.requireNonNull(vendorListFileStore); + this.mapper = Objects.requireNonNull(mapper); - createAndCheckWritePermissionsFor(fileSystem, cacheDir); - cache = Objects.requireNonNull(createCache(fileSystem, cacheDir)); + cache = Objects.requireNonNull(vendorListFileStore.createCacheFromDisk(cacheDir)); - fallbackVendorList = StringUtils.isNotBlank(fallbackVendorListPath) - ? readFallbackVendorList(fallbackVendorListPath) : null; + fallbackVendorList = vendorListFileStore.readFallbackVendorList(fallbackVendorListPath); if (deprecated) { validateFallbackVendorListIfDeprecatedVersion(); } @@ -161,50 +140,6 @@ public Future> forVersion(int version) { .formatted(tcf, generationVersion, version)); } - /** - * Creates vendorList object from string content or throw {@link PreBidException}. - */ - private VendorList toVendorList(String content) { - try { - return mapper.mapper().readValue(content, VendorList.class); - } catch (IOException e) { - final String message = "Cannot parse vendor list from: " + content; - - logger.error(message, e); - throw new PreBidException(message, e); - } - } - - /** - * Returns a Map of vendor id to Vendors. - */ - private Map filterVendorIdToVendors(VendorList vendorList) { - return vendorList.getVendors().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - /** - * Verifies all significant fields of given {@link VendorList} object. - */ - private boolean isValid(VendorList vendorList) { - return vendorList.getVendorListVersion() != null - && vendorList.getLastUpdated() != null - && MapUtils.isNotEmpty(vendorList.getVendors()) - && isValidVendors(vendorList.getVendors().values()); - } - - private static boolean isValidVendors(Collection vendors) { - return vendors.stream() - .allMatch(vendor -> vendor != null - && vendor.getId() != null - && vendor.getPurposes() != null - && vendor.getLegIntPurposes() != null - && vendor.getFlexiblePurposes() != null - && vendor.getSpecialPurposes() != null - && vendor.getFeatures() != null - && vendor.getSpecialFeatures() != null); - } - /** * Returns the version of TCF which {@link VendorListService} implementation deals with. */ @@ -212,62 +147,6 @@ private int getTcfVersion() { return TCF_VERSION; } - /** - * Creates if doesn't exists and checks write permissions for the given directory. - */ - private static void createAndCheckWritePermissionsFor(FileSystem fileSystem, String dir) { - final FileProps props = fileSystem.existsBlocking(dir) ? fileSystem.propsBlocking(dir) : null; - if (props == null || !props.isDirectory()) { - try { - fileSystem.mkdirsBlocking(dir); - } catch (FileSystemException e) { - throw new PreBidException("Cannot create directory: " + dir, e); - } - } else if (!Files.isWritable(Paths.get(dir))) { - throw new PreBidException("No write permissions for directory: " + dir); - } - } - - /** - * Creates the cache from previously downloaded vendor lists. - */ - private Map> createCache(FileSystem fileSystem, String cacheDir) { - final Map versionToFileContent = readFileSystemCache(fileSystem, cacheDir); - - final Map> cache = Caffeine.newBuilder() - .>build() - .asMap(); - - for (Map.Entry versionAndFileContent : versionToFileContent.entrySet()) { - final VendorList vendorList = toVendorList(versionAndFileContent.getValue()); - final Map vendorIdToVendors = filterVendorIdToVendors(vendorList); - - cache.put(Integer.valueOf(versionAndFileContent.getKey()), vendorIdToVendors); - } - return cache; - } - - /** - * Reads files with .json extension in configured directory and - * returns a {@link Map} where key is a file name without .json extension and value is file content. - */ - private Map readFileSystemCache(FileSystem fileSystem, String dir) { - return fileSystem.readDirBlocking(dir).stream() - .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) - .collect(Collectors.toMap(filepath -> StringUtils.removeEnd(new File(filepath).getName(), JSON_SUFFIX), - filename -> fileSystem.readFileBlocking(filename).toString())); - } - - private Map readFallbackVendorList(String fallbackVendorListPath) { - final String vendorListContent = fileSystem.readFileBlocking(fallbackVendorListPath).toString(); - final VendorList vendorList = toVendorList(vendorListContent); - if (!isValid(vendorList)) { - throw new PreBidException("Fallback vendor list parsed but has invalid data: " + vendorListContent); - } - - return filterVendorIdToVendors(vendorList); - } - private boolean shouldFallback(int version) { return deprecated || (versionsToFallback != null && versionsToFallback.contains(version)); } @@ -290,7 +169,7 @@ private void fetchNewVendorListFor(int version) { * and creates {@link Future} with {@link VendorListResult} from body content * or throws {@link PreBidException} in case of errors. */ - private VendorListResult processResponse(HttpClientResponse response, int version) { + private VendorListResult processResponse(HttpClientResponse response, int version) { final int statusCode = response.getStatusCode(); if (statusCode == HttpResponseStatus.NOT_FOUND.code()) { @@ -301,11 +180,11 @@ private VendorListResult processResponse(HttpClientResponse response } final String body = response.getBody(); - final VendorList vendorList = toVendorList(body); + final VendorList vendorList = VendorListUtil.parseVendorList(body, mapper); // we should care on obtained vendor list, because it'll be saved and never be downloaded again // while application is running - if (!isValid(vendorList)) { + if (!VendorListUtil.vendorListIsValid(vendorList)) { throw new PreBidException("Fetched vendor list parsed but has invalid data: " + body); } @@ -313,33 +192,14 @@ private VendorListResult processResponse(HttpClientResponse response return VendorListResult.of(version, body, vendorList); } - /** - * Saves given vendor list on file system. - */ - private Future> saveToFile(VendorListResult vendorListResult) { - final Promise> promise = Promise.promise(); - final int version = vendorListResult.getVersion(); - final String filepath = new File(cacheDir, version + JSON_SUFFIX).getPath(); - - fileSystem.writeFile(filepath, Buffer.buffer(vendorListResult.getVendorListAsString()), result -> { - if (result.succeeded()) { - promise.complete(vendorListResult); - } else { - conditionalLogger.error( - "Could not create new vendor list for version %s.%s, file: %s, trace: %s".formatted( - generationVersion, version, filepath, ExceptionUtils.getStackTrace(result.cause())), - logSamplingRate); - promise.fail(result.cause()); - } - }); - - return promise.future(); + private Future saveToFile(VendorListResult vendorListResult) { + return vendorListFileStore.saveToFile(vendorListResult, cacheDir, generationVersion); } - private Void updateCache(VendorListResult vendorListResult) { + private Void updateCache(VendorListResult vendorListResult) { final int version = vendorListResult.getVersion(); - cache.put(version, filterVendorIdToVendors(vendorListResult.getVendorList())); + cache.put(version, vendorListResult.getVendorList().getVendors()); final int tcf = getTcfVersion(); @@ -395,16 +255,6 @@ private void stopUsingFallbackForVersion(int version) { versionsToFallback.remove(version); } - @Value(staticConstructor = "of") - private static class VendorListResult { - - int version; - - String vendorListAsString; - - T vendorList; - } - private static class MissingVendorListException extends RuntimeException { MissingVendorListException(String message) { diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java new file mode 100644 index 00000000000..cf8a8b7dd10 --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java @@ -0,0 +1,50 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import org.apache.commons.collections4.MapUtils; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.io.IOException; +import java.util.Collection; + +public class VendorListUtil { + + private static final Logger logger = LoggerFactory.getLogger(VendorListUtil.class); + + private VendorListUtil() { + } + + public static VendorList parseVendorList(String content, JacksonMapper mapper) { + try { + return mapper.mapper().readValue(content, VendorList.class); + } catch (IOException e) { + final String message = "Cannot parse vendor list from: " + content; + + logger.error(message, e); + throw new PreBidException(message, e); + } + } + + public static boolean vendorListIsValid(VendorList vendorList) { + return vendorList.getVendorListVersion() != null + && vendorList.getLastUpdated() != null + && MapUtils.isNotEmpty(vendorList.getVendors()) + && vendorsAreValid(vendorList.getVendors().values()); + } + + private static boolean vendorsAreValid(Collection vendors) { + return vendors.stream() + .allMatch(vendor -> vendor != null + && vendor.getId() != null + && vendor.getPurposes() != null + && vendor.getLegIntPurposes() != null + && vendor.getFlexiblePurposes() != null + && vendor.getSpecialPurposes() != null + && vendor.getFeatures() != null + && vendor.getSpecialFeatures() != null); + } +} diff --git a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java index 70d5b672cab..deb6d751d70 100644 --- a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java @@ -39,6 +39,7 @@ import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy; import org.prebid.server.privacy.gdpr.vendorlist.LiveVendorListService; import org.prebid.server.privacy.gdpr.vendorlist.VendorListFetchThrottler; +import org.prebid.server.privacy.gdpr.vendorlist.VendorListFileStore; import org.prebid.server.privacy.gdpr.vendorlist.VendorListService; import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService; import org.prebid.server.settings.model.GdprConfig; @@ -66,6 +67,15 @@ @Configuration public class PrivacyServiceConfiguration { + @Bean + VendorListFileStore vendorListFileStore( + @Value("${logging.sampling-rate:0.01}") double logSamplingRate, + FileSystem fileSystem, + JacksonMapper mapper) { + + return new VendorListFileStore(logSamplingRate, fileSystem, mapper); + } + @Bean VendorListService vendorListServiceV2( @Value("${logging.sampling-rate:0.01}") double logSamplingRate, @@ -73,9 +83,9 @@ VendorListService vendorListServiceV2( VendorListServiceConfigurationProperties vendorListServiceV2Properties, Vertx vertx, Clock clock, - FileSystem fileSystem, HttpClient httpClient, Metrics metrics, + VendorListFileStore vendorListFileStore, JacksonMapper mapper) { return new VendorListService( @@ -87,12 +97,12 @@ VendorListService vendorListServiceV2( vendorListServiceV2Properties.getDeprecated(), vendorListServiceV2Properties.getFallbackVendorListPath(), vertx, - fileSystem, httpClient, metrics, "v2", - mapper, - new VendorListFetchThrottler(vendorListServiceV2Properties.getRetryPolicy().toPolicy(), clock)); + new VendorListFetchThrottler(vendorListServiceV2Properties.getRetryPolicy().toPolicy(), clock), + vendorListFileStore, + mapper); } @Bean @@ -108,9 +118,9 @@ VendorListService vendorListServiceV3( VendorListServiceConfigurationProperties vendorListServiceV3Properties, Vertx vertx, Clock clock, - FileSystem fileSystem, HttpClient httpClient, Metrics metrics, + VendorListFileStore vendorListFileStore, JacksonMapper mapper) { return new VendorListService( @@ -122,12 +132,12 @@ VendorListService vendorListServiceV3( vendorListServiceV3Properties.getDeprecated(), vendorListServiceV3Properties.getFallbackVendorListPath(), vertx, - fileSystem, httpClient, metrics, "v3", - mapper, - new VendorListFetchThrottler(vendorListServiceV3Properties.getRetryPolicy().toPolicy(), clock)); + new VendorListFetchThrottler(vendorListServiceV3Properties.getRetryPolicy().toPolicy(), clock), + vendorListFileStore, + mapper); } @Bean diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java new file mode 100644 index 00000000000..845e4e85db0 --- /dev/null +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java @@ -0,0 +1,334 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.vertx.core.AsyncResult; +import io.vertx.core.Future; +import io.vertx.core.Handler; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.file.FileProps; +import io.vertx.core.file.FileSystem; +import io.vertx.core.file.FileSystemException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.prebid.server.VertxTest; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Feature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialFeature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialPurpose; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.io.File; +import java.nio.file.Path; +import java.util.Date; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO; + +@ExtendWith(MockitoExtension.class) +public class VendorListFileStoreTest extends VertxTest { + + private static final String CACHE_DIR = "/cache/dir"; + private static final String FALLBACK_VENDOR_LIST_PATH = "fallback.json"; + private static final String GENERATION_VERSION = "v0"; + + @Mock + private FileSystem fileSystem; + + @TempDir + Path tempDir; + + private VendorListFileStore target; + + @BeforeEach + public void setUp() { + target = new VendorListFileStore(0, fileSystem, jacksonMapper); + } + + @Test + public void createCacheFromDiskShouldCreateCacheDirWhenItDoesNotExist() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + target.createCacheFromDisk(CACHE_DIR); + + // then + verify(fileSystem).mkdirsBlocking(eq(CACHE_DIR)); + } + + @Test + public void createCacheFromDiskShouldCreateCacheDirWhenPathExistsButIsNotADirectory() { + // given + final FileProps fileProps = mock(FileProps.class); + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(true); + given(fileSystem.propsBlocking(eq(CACHE_DIR))).willReturn(fileProps); + given(fileProps.isDirectory()).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + target.createCacheFromDisk(CACHE_DIR); + + // then + verify(fileSystem).mkdirsBlocking(eq(CACHE_DIR)); + } + + @Test + public void createCacheFromDiskShouldFailIfCannotCreateCacheDir() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.mkdirsBlocking(eq(CACHE_DIR))) + .willThrow(new FileSystemException("dir creation error")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot create directory: " + CACHE_DIR); + } + + @Test + public void createCacheFromDiskShouldFailIfNoWritePermissionsForCacheDir() { + // given + final String cacheDir = tempDir.toString(); + tempDir.toFile().setWritable(false, false); + final FileProps fileProps = mock(FileProps.class); + given(fileSystem.existsBlocking(eq(cacheDir))).willReturn(true); + given(fileSystem.propsBlocking(eq(cacheDir))).willReturn(fileProps); + given(fileProps.isDirectory()).willReturn(true); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(cacheDir)) + .isInstanceOf(PreBidException.class) + .hasMessage("No write permissions for directory: " + cacheDir); + } + + @Test + public void createCacheFromDiskShouldReturnCacheLoadedFromJsonFiles() throws JsonProcessingException { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("/cache/dir/1.json")); + given(fileSystem.readFileBlocking(eq("/cache/dir/1.json"))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(givenVendorList()))); + + // when + final Map> cache = target.createCacheFromDisk(CACHE_DIR); + + // then + assertThat(cache).isEqualTo(singletonMap(1, givenVendorMap())); + } + + @Test + public void createCacheFromDiskShouldReturnEmptyCacheWhenCacheDirHasNoJsonFiles() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + final Map> cache = target.createCacheFromDisk(CACHE_DIR); + + // then + assertThat(cache).isEmpty(); + } + + @Test + public void createCacheFromDiskShouldFailIfCannotReadCacheDir() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void createCacheFromDiskShouldFailIfCannotReadAtLeastOneVendorListFile() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("1.json")); + given(fileSystem.readFileBlocking(eq("1.json"))).willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void createCacheFromDiskShouldFailIfAtLeastOneVendorListFileCannotBeParsed() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("1.json")); + given(fileSystem.readFileBlocking(eq("1.json"))).willReturn(Buffer.buffer("invalid")); + + // when and then + assertThatThrownBy(() -> target.createCacheFromDisk(CACHE_DIR)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void saveToFileShouldCompleteWithVendorListResultWhenWriteSucceeds() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); + final VendorListResult vendorListResult = VendorListResult.of(1, vendorListAsString, vendorList); + givenWriteFileSucceeds(); + + // when + final Future future = target.saveToFile(vendorListResult, CACHE_DIR, GENERATION_VERSION); + + // then + org.prebid.server.assertion.FutureAssertion.assertThat(future).succeededWith(vendorListResult); + } + + @Test + public void saveToFileShouldWriteToExpectedPathWithExpectedContent() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); + final VendorListResult vendorListResult = VendorListResult.of(1, vendorListAsString, vendorList); + givenWriteFileSucceeds(); + + // when + target.saveToFile(vendorListResult, CACHE_DIR, GENERATION_VERSION); + + // then + final ArgumentCaptor pathCaptor = ArgumentCaptor.forClass(String.class); + final ArgumentCaptor bufferCaptor = ArgumentCaptor.forClass(Buffer.class); + verify(fileSystem).writeFile(pathCaptor.capture(), bufferCaptor.capture(), any()); + assertThat(pathCaptor.getValue()).isEqualTo(new File(CACHE_DIR, "1.json").getPath()); + assertThat(bufferCaptor.getValue().toString()).isEqualTo(vendorListAsString); + } + + @Test + public void saveToFileShouldFailWhenWriteFails() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); + final VendorListResult vendorListResult = VendorListResult.of(1, vendorListAsString, vendorList); + final RuntimeException exception = new RuntimeException("write error"); + givenWriteFileFails(exception); + + // when + final Future future = target.saveToFile(vendorListResult, CACHE_DIR, GENERATION_VERSION); + + // then + org.prebid.server.assertion.FutureAssertion.assertThat(future) + .isFailed() + .isInstanceOf(RuntimeException.class) + .hasMessage("write error"); + } + + @Test + public void readFallbackVendorListShouldReturnNullWhenPathIsBlank() { + // when and then + assertThat(target.readFallbackVendorList(null)).isNull(); + assertThat(target.readFallbackVendorList("")).isNull(); + assertThat(target.readFallbackVendorList(" ")).isNull(); + } + + @Test + public void readFallbackVendorListShouldReturnVendorsWhenPathIsValid() throws JsonProcessingException { + // given + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(givenVendorList()))); + + // when + final Map vendors = target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH); + + // then + assertThat(vendors).isEqualTo(givenVendorMap()); + } + + @Test + public void readFallbackVendorListShouldFailIfCannotReadFallbackFile() { + // given + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void readFallbackVendorListShouldFailIfFallbackCannotBeParsed() { + // given + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willReturn(Buffer.buffer("invalid")); + + // when and then + assertThatThrownBy(() -> target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void readFallbackVendorListShouldFailIfFallbackHasInvalidData() throws JsonProcessingException { + // given + final VendorList invalidVendorList = VendorList.of(1, new Date(), emptyMap()); + given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(invalidVendorList))); + + // when and then + assertThatThrownBy(() -> target.readFallbackVendorList(FALLBACK_VENDOR_LIST_PATH)) + .isInstanceOf(PreBidException.class) + .hasMessageStartingWith("Fallback vendor list parsed but has invalid data:"); + } + + private void givenWriteFileSucceeds() { + given(fileSystem.writeFile(anyString(), any(Buffer.class), any())).willAnswer(invocation -> { + final Handler> handler = invocation.getArgument(2); + handler.handle(Future.succeededFuture()); + return null; + }); + } + + private void givenWriteFileFails(Throwable throwable) { + given(fileSystem.writeFile(anyString(), any(Buffer.class), any())).willAnswer(invocation -> { + final Handler> handler = invocation.getArgument(2); + handler.handle(Future.failedFuture(throwable)); + return null; + }); + } + + private static VendorList givenVendorList() { + return VendorList.of(1, new Date(), givenVendorMap()); + } + + private static Map givenVendorMap() { + final Vendor vendor = Vendor.builder() + .id(52) + .purposes(EnumSet.of(ONE)) + .legIntPurposes(EnumSet.of(TWO)) + .flexiblePurposes(EnumSet.noneOf(PurposeCode.class)) + .specialPurposes(EnumSet.noneOf(SpecialPurpose.class)) + .features(EnumSet.noneOf(Feature.class)) + .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) + .build(); + return singletonMap(52, vendor); + } +} diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java index 82ef85987d2..9be7e32d546 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListServiceTest.java @@ -2,17 +2,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import io.vertx.core.Future; -import io.vertx.core.Handler; import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.file.FileSystem; import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.stubbing.Answer; import org.prebid.server.VertxTest; import org.prebid.server.bidder.BidderCatalog; import org.prebid.server.exception.PreBidException; @@ -26,7 +22,6 @@ import org.prebid.server.vertx.httpclient.HttpClient; import org.prebid.server.vertx.httpclient.model.HttpClientResponse; -import java.io.File; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; @@ -34,7 +29,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; -import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -42,9 +36,11 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.BDDMockito.given; import static org.mockito.Mock.Strictness.LENIENT; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.prebid.server.assertion.FutureAssertion.assertThat; @@ -61,8 +57,6 @@ public class VendorListServiceTest extends VertxTest { @Mock private Vertx vertx; - @Mock - private FileSystem fileSystem; @Mock(strictness = LENIENT) private HttpClient httpClient; @Mock @@ -71,19 +65,17 @@ public class VendorListServiceTest extends VertxTest { private BidderCatalog bidderCatalog; @Mock(strictness = LENIENT) private VendorListFetchThrottler fetchThrottler; + @Mock(strictness = LENIENT) + private VendorListFileStore vendorListFileStore; private VendorListService target; @BeforeEach public void setUp() throws JsonProcessingException { - given(fileSystem.existsBlocking(anyString())).willReturn(false); // always create cache dir - given(bidderCatalog.knownVendorIds()).willReturn(singleton(52)); - - given(fileSystem.readFileBlocking(eq(FALLBACK_VENDOR_LIST_PATH))) - .willReturn(Buffer.buffer(mapper.writeValueAsString(givenVendorList()))); - given(fetchThrottler.registerFetchAttempt(anyInt())).willReturn(true); + given(vendorListFileStore.readFallbackVendorList(isNull())).willReturn(null); + given(vendorListFileStore.readFallbackVendorList(eq(FALLBACK_VENDOR_LIST_PATH))).willReturn(givenVendorMap()); target = new VendorListService( 0, @@ -94,44 +86,72 @@ public void setUp() throws JsonProcessingException { false, FALLBACK_VENDOR_LIST_PATH, vertx, - fileSystem, httpClient, metrics, GENERATION_VERSION, - jacksonMapper, - fetchThrottler); + fetchThrottler, + vendorListFileStore, + jacksonMapper); } // Creation related tests @Test - public void creationShouldFailsIfCannotCreateCacheDir() { - // given - given(fileSystem.mkdirsBlocking(anyString())).willThrow(new RuntimeException("dir creation error")); + public void creationShouldCreateCacheFromDisk() { + reset(vendorListFileStore); + + // given and when + new VendorListService( + 0, + CACHE_DIR, + "http://vendorlist/%s", + 0, + REFRESH_MISSING_LIST_PERIOD_MS, + false, + FALLBACK_VENDOR_LIST_PATH, + vertx, + httpClient, + metrics, + GENERATION_VERSION, + fetchThrottler, + vendorListFileStore, + jacksonMapper); // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/%s", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .hasMessage("dir creation error"); + verify(vendorListFileStore).createCacheFromDisk(eq(CACHE_DIR)); + } + + @Test + public void creationShouldFailIfCannotCreateCache() { + // given + given(vendorListFileStore.createCacheFromDisk(anyString())) + .willThrow(new RuntimeException("error creating cache")); + + // when and then + assertThatThrownBy(() -> new VendorListService( + 0, + CACHE_DIR, + "http://vendorlist/{VERSION}", + 0, + REFRESH_MISSING_LIST_PERIOD_MS, + true, + null, + vertx, + httpClient, + metrics, + GENERATION_VERSION, + fetchThrottler, + vendorListFileStore, + jacksonMapper)) + .isInstanceOf(RuntimeException.class) + .hasMessage("error creating cache"); } @Test public void shouldStartUsingFallbackVersionIfDeprecatedIsTrue() { // given + given(vendorListFileStore.readFallbackVendorList(anyString())).willReturn(givenVendorMap()); + target = new VendorListService( 0, CACHE_DIR, @@ -141,12 +161,12 @@ public void shouldStartUsingFallbackVersionIfDeprecatedIsTrue() { true, FALLBACK_VENDOR_LIST_PATH, vertx, - fileSystem, httpClient, metrics, GENERATION_VERSION, - jacksonMapper, - fetchThrottler); + fetchThrottler, + vendorListFileStore, + jacksonMapper); // when final Future> future = target.forVersion(1); @@ -177,96 +197,16 @@ public void shouldThrowExceptionIfVersionIsDeprecatedAndNoFallbackPresent() { true, null, vertx, - fileSystem, httpClient, metrics, GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) + fetchThrottler, + vendorListFileStore, + jacksonMapper)) .isInstanceOf(PreBidException.class) .hasMessage("No fallback vendorList for deprecated version present"); } - @Test - public void creationShouldFailsIfCannotReadFiles() { - // given - given(fileSystem.readDirBlocking(anyString())).willThrow(new RuntimeException("read error")); - - // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/%s", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .isInstanceOf(RuntimeException.class) - .hasMessage("read error"); - } - - @Test - public void creationShouldFailsIfCannotReadAtLeastOneVendorListFile() { - // given - given(fileSystem.readDirBlocking(anyString())).willReturn(singletonList("1.json")); - given(fileSystem.readFileBlocking(anyString())).willThrow(new RuntimeException("read error")); - - // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/%s", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .isInstanceOf(RuntimeException.class) - .hasMessage("read error"); - } - - @Test - public void creationShouldFailsIfAtLeastOneVendorListFileCannotBeParsed() { - // given - given(fileSystem.readDirBlocking(anyString())).willReturn(singletonList("1.json")); - given(fileSystem.readFileBlocking(anyString())).willReturn(Buffer.buffer("invalid")); - - // then - assertThatThrownBy( - () -> new VendorListService( - 0, - CACHE_DIR, - "http://vendorlist/%s", - 0, - REFRESH_MISSING_LIST_PERIOD_MS, - false, - FALLBACK_VENDOR_LIST_PATH, - vertx, - fileSystem, - httpClient, - metrics, - GENERATION_VERSION, - jacksonMapper, - fetchThrottler)) - .isInstanceOf(PreBidException.class) - .hasMessage("Cannot parse vendor list from: invalid"); - } - // Http related tests @Test @@ -293,11 +233,11 @@ public void shouldNotPerformHttpRequestIfVendorListNotFoundAndFetchNotAllowed() // then verify(httpClient, never()).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfReadingHttpResponseFails() { + public void shouldNotTryToSaveFileIfReadingHttpResponseFails() { // given givenHttpClientProducesException(new RuntimeException("Response exception")); @@ -306,11 +246,11 @@ public void shouldNotAskToSaveFileIfReadingHttpResponseFails() { // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfResponseCodeIsNot200() { + public void shouldNotTryToSaveFileIfResponseCodeIsNot200() { // given givenHttpClientReturnsResponse(503, null); @@ -319,11 +259,11 @@ public void shouldNotAskToSaveFileIfResponseCodeIsNot200() { // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfResponseBodyCouldNotBeParsed() { + public void shouldNotTryToSaveFileIfResponseBodyCouldNotBeParsed() { // given givenHttpClientReturnsResponse(200, "response"); @@ -332,11 +272,11 @@ public void shouldNotAskToSaveFileIfResponseBodyCouldNotBeParsed() { // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidVendorListVersion() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasInvalidVendorListVersion() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(null, null, null); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -346,11 +286,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidVendorListVersion // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidLastUpdated() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasInvalidLastUpdated() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, null, null); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -360,11 +300,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasInvalidLastUpdated() thr // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasNoVendors() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasNoVendors() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, new Date(), null); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -374,11 +314,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasNoVendors() throws JsonP // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasEmptyVendors() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasEmptyVendors() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, new Date(), emptyMap()); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -388,11 +328,11 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasEmptyVendors() throws Js // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } @Test - public void shouldNotAskToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor() throws JsonProcessingException { + public void shouldNotTryToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor() throws JsonProcessingException { // given final VendorList vendorList = VendorList.of(1, new Date(), singletonMap(1, Vendor.builder().build())); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); @@ -402,7 +342,7 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor( // then verify(httpClient).get(anyString(), anyLong()); - verify(fileSystem, never()).writeFile(any(), any(), any()); + verify(vendorListFileStore, never()).saveToFile(any(), anyString(), anyString()); } // File system related tests @@ -410,16 +350,20 @@ public void shouldNotAskToSaveFileIfFetchedVendorListHasAtLeastOneInvalidVendor( @Test public void shouldSaveFileWithExpectedPathAndContentIfVendorListNotFound() throws JsonProcessingException { // given - final String vendorListAsString = mapper.writeValueAsString(givenVendorList()); + final VendorList vendorList = givenVendorList(); + final String vendorListAsString = mapper.writeValueAsString(vendorList); givenHttpClientReturnsResponse(200, vendorListAsString); - // generate file path to avoid conflicts with path separators in different OS - final String filePath = new File("/cache/dir/1.json").getPath(); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); // then - verify(fileSystem).writeFile(eq(filePath), eq(Buffer.buffer(vendorListAsString)), any()); + verify(vendorListFileStore).saveToFile( + eq(VendorListResult.of(1, vendorListAsString, vendorList)), + eq(CACHE_DIR), + eq(GENERATION_VERSION)); } // In-memory cache related tests @@ -456,8 +400,8 @@ public void shouldReturnVendorListFromCache() throws JsonProcessingException { // given givenHttpClientReturnsResponse(200, mapper.writeValueAsString(givenVendorList())); - given(fileSystem.writeFile(anyString(), any(), any())) - .willAnswer(withSelfAndPassObjectToHandler(Future.succeededFuture())); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); // populate cache @@ -504,8 +448,8 @@ public void shouldKeepPurposesForAllVendors() throws JsonProcessingException { final VendorList vendorList = VendorList.of(1, new Date(), idToVendor); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(vendorList)); - given(fileSystem.writeFile(anyString(), any(), any())) - .willAnswer(withSelfAndPassObjectToHandler(Future.succeededFuture())); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); // populate cache @@ -572,8 +516,8 @@ public void shouldIncrementVendorListErrorMetricWhenFileIsNotSaved() throws Json // given givenHttpClientReturnsResponse(200, mapper.writeValueAsString(givenVendorList())); - given(fileSystem.writeFile(anyString(), any(), any())) - .willAnswer(withSelfAndPassObjectToHandler(Future.failedFuture("error"))); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willReturn(Future.failedFuture("error")); // when target.forVersion(1); @@ -587,8 +531,8 @@ public void shouldIncrementVendorListOkMetric() throws JsonProcessingException { // given givenHttpClientReturnsResponse(200, mapper.writeValueAsString(givenVendorList())); - given(fileSystem.writeFile(anyString(), any(), any())) - .willAnswer(withSelfAndPassObjectToHandler(Future.succeededFuture())); + given(vendorListFileStore.saveToFile(any(), anyString(), anyString())) + .willAnswer(inv -> Future.succeededFuture(inv.getArgument(0))); // when target.forVersion(1); @@ -614,6 +558,10 @@ public void shouldIncrementVendorListFallbackMetric() { } private static VendorList givenVendorList() { + return VendorList.of(1, new Date(), givenVendorMap()); + } + + private static Map givenVendorMap() { final Vendor vendor = Vendor.builder() .id(52) .purposes(EnumSet.of(ONE)) @@ -623,7 +571,7 @@ private static VendorList givenVendorList() { .features(EnumSet.noneOf(Feature.class)) .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) .build(); - return VendorList.of(1, new Date(), singletonMap(52, vendor)); + return singletonMap(52, vendor); } private void givenHttpClientReturnsResponse(int statusCode, String response) { @@ -635,13 +583,4 @@ private void givenHttpClientProducesException(Throwable throwable) { given(httpClient.get(anyString(), anyLong())) .willReturn(Future.failedFuture(throwable)); } - - @SuppressWarnings("unchecked") - private static Answer withSelfAndPassObjectToHandler(T obj) { - return inv -> { - // invoking handler right away passing mock to it - ((Handler) inv.getArgument(2)).handle(obj); - return inv.getMock(); - }; - } } diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtilTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtilTest.java new file mode 100644 index 00000000000..8ff016e924b --- /dev/null +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtilTest.java @@ -0,0 +1,192 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.jupiter.api.Test; +import org.prebid.server.VertxTest; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Feature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialFeature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialPurpose; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; + +import java.util.Date; +import java.util.EnumSet; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO; + +public class VendorListUtilTest extends VertxTest { + + @Test + public void parseVendorListShouldReturnVendorListWhenContentIsValid() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(); + final String content = mapper.writeValueAsString(vendorList); + + // when + final VendorList result = VendorListUtil.parseVendorList(content, jacksonMapper); + + // then + assertThat(result).isEqualTo(vendorList); + } + + @Test + public void parseVendorListShouldThrowExceptionWhenContentCannotBeParsed() { + // when and then + assertThatThrownBy(() -> VendorListUtil.parseVendorList("invalid", jacksonMapper)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + + @Test + public void vendorListIsValidShouldReturnTrueWhenVendorListIsValid() { + // when and then + assertThat(VendorListUtil.vendorListIsValid(givenVendorList())).isTrue(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorListVersionIsNull() { + // given + final VendorList vendorList = VendorList.of(null, new Date(), givenVendorMap()); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenLastUpdatedIsNull() { + // given + final VendorList vendorList = VendorList.of(1, null, givenVendorMap()); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorsIsNull() { + // given + final VendorList vendorList = VendorList.of(1, new Date(), null); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorsIsEmpty() { + // given + final VendorList vendorList = VendorList.of(1, new Date(), emptyMap()); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorIsNull() { + // given + final VendorList vendorList = VendorList.of(1, new Date(), singletonMap(1, null)); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorIdIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().id(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorPurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().purposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorLegIntPurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().legIntPurposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorFlexiblePurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().flexiblePurposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorSpecialPurposesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().specialPurposes(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorFeaturesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().features(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + @Test + public void vendorListIsValidShouldReturnFalseWhenVendorSpecialFeaturesIsNull() { + // given + final Vendor vendor = givenVendor().toBuilder().specialFeatures(null).build(); + final VendorList vendorList = givenVendorListWithVendor(vendor); + + // when and then + assertThat(VendorListUtil.vendorListIsValid(vendorList)).isFalse(); + } + + private static VendorList givenVendorList() { + return VendorList.of(1, new Date(), givenVendorMap()); + } + + private static VendorList givenVendorListWithVendor(Vendor vendor) { + return VendorList.of(1, new Date(), singletonMap(vendor.getId() != null ? vendor.getId() : 1, vendor)); + } + + private static Map givenVendorMap() { + return singletonMap(52, givenVendor()); + } + + private static Vendor givenVendor() { + return Vendor.builder() + .id(52) + .purposes(EnumSet.of(ONE)) + .legIntPurposes(EnumSet.of(TWO)) + .flexiblePurposes(EnumSet.noneOf(PurposeCode.class)) + .specialPurposes(EnumSet.noneOf(SpecialPurpose.class)) + .features(EnumSet.noneOf(Feature.class)) + .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) + .build(); + } +} From 844a3cbefd968441ba33acf26989c7f609a899bb Mon Sep 17 00:00:00 2001 From: Viktor Kryshtal Date: Tue, 9 Jun 2026 15:52:46 +0300 Subject: [PATCH 03/11] Added validation of live vendor list and removed exception on empty deleted vendors --- .../vendorlist/LiveVendorListService.java | 16 ++-- .../vendorlist/LiveVendorListServiceTest.java | 74 +++++++++++-------- 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java index dc3c8b62ed2..ea3bed35049 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java @@ -2,7 +2,6 @@ import io.vertx.core.Promise; import io.vertx.core.Vertx; -import org.apache.commons.collections4.CollectionUtils; import org.prebid.server.exception.PreBidException; import org.prebid.server.json.JacksonMapper; import org.prebid.server.log.Logger; @@ -15,7 +14,6 @@ import org.prebid.server.vertx.httpclient.HttpClient; import org.prebid.server.vertx.httpclient.model.HttpClientResponse; -import java.io.IOException; import java.time.Clock; import java.time.Instant; import java.util.Objects; @@ -83,11 +81,13 @@ private VendorList processResponse(HttpClientResponse response) { } final String body = response.getBody(); - try { - return mapper.mapper().readValue(body, VendorList.class); - } catch (IOException e) { - throw new PreBidException("Cannot parse live vendor list: " + body, e); + final VendorList vendorList = VendorListUtil.parseVendorList(body, mapper); + + if (!VendorListUtil.vendorListIsValid(vendorList)) { + throw new PreBidException("Fetched vendor list parsed but has invalid data: " + body); } + + return vendorList; } Set extractDeletedVendorIds(VendorList vendorList) { @@ -105,10 +105,6 @@ private static boolean isDeletedAt(Vendor vendor, Instant now) { } private Void updateDeletedVendorIds(Set ids) { - if (CollectionUtils.isEmpty(ids)) { - throw new PreBidException("Live GVL response has no deleted vendors"); - } - deletedVendorIds = ids; metrics.updatePrivacyTcfVendorListLatestOkMetric(); return null; diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java index f2f05c8b88b..68953b15120 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java @@ -10,6 +10,10 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.prebid.server.VertxTest; import org.prebid.server.metric.Metrics; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Feature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialFeature; +import org.prebid.server.privacy.gdpr.vendorlist.proto.SpecialPurpose; import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; import org.prebid.server.vertx.httpclient.HttpClient; @@ -18,17 +22,20 @@ import java.time.Clock; import java.time.Instant; import java.time.ZoneOffset; +import java.util.Arrays; import java.util.Date; -import java.util.HashMap; -import java.util.Map; +import java.util.EnumSet; +import java.util.function.Function; +import java.util.stream.Collectors; -import static java.util.Collections.emptyMap; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE; +import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO; @ExtendWith(MockitoExtension.class) public class LiveVendorListServiceTest extends VertxTest { @@ -68,7 +75,8 @@ public void isDeletedShouldReturnFalseWhenFetchNeverSucceeded() { @Test public void isDeletedShouldReturnTrueWhenVendorIsDeletedInLiveVendorList() throws JsonProcessingException { // given - final String responseBody = givenLiveGvlJson(Map.of(42, "2024-01-01T00:00:00Z")); + final Vendor vendor = givenVendor(42, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); given(httpClient.get(anyString(), anyLong())) .willReturn(Future.succeededFuture(HttpClientResponse.of(200, null, responseBody))); @@ -83,11 +91,11 @@ public void isDeletedShouldReturnTrueWhenVendorIsDeletedInLiveVendorList() throw @Test public void extractDeletedVendorIdsShouldReturnOnlyVendorsWithPastDeletedDate() { // given - final VendorList vendorList = givenVendorList(Map.of( - 1, givenVendor(1, "2024-01-01T00:00:00Z"), - 2, givenVendor(2, null), - 3, givenVendor(3, "2025-01-01T00:00:00Z"), - 4, givenVendor(4, "2024-06-01T12:00:00Z"))); + final VendorList vendorList = givenVendorList( + givenVendor(1, "2024-01-01T00:00:00Z"), + givenVendor(2, null), + givenVendor(3, "2025-01-01T00:00:00Z"), + givenVendor(4, "2024-06-01T12:00:00Z")); // when final var deletedIds = target.extractDeletedVendorIds(vendorList); @@ -99,7 +107,9 @@ public void extractDeletedVendorIdsShouldReturnOnlyVendorsWithPastDeletedDate() @Test public void refreshShouldUpdateDeletedVendorIdsAndIncrementOkMetric() throws JsonProcessingException { // given - givenHttpClientReturnsResponse(200, givenLiveGvlJson(Map.of(1, "2024-01-01T00:00:00Z"))); + final Vendor vendor = givenVendor(1, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + givenHttpClientReturnsResponse(200, responseBody); // when target.refresh(); @@ -113,12 +123,14 @@ public void refreshShouldUpdateDeletedVendorIdsAndIncrementOkMetric() throws Jso @Test public void refreshShouldReplaceDeletedVendorIdsOnSubsequentSuccessfulFetch() throws JsonProcessingException { // given + final Vendor vendor = givenVendor(1, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + final Vendor vendor2 = givenVendor(2, "2024-02-01T00:00:00Z"); + final String responseBody2 = mapper.writeValueAsString(givenVendorList(vendor2)); given(httpClient.get(anyString(), anyLong())) .willReturn( - Future.succeededFuture(HttpClientResponse.of( - 200, null, givenLiveGvlJson(Map.of(1, "2024-01-01T00:00:00Z")))), - Future.succeededFuture(HttpClientResponse.of( - 200, null, givenLiveGvlJson(Map.of(2, "2024-02-01T00:00:00Z"))))); + Future.succeededFuture(HttpClientResponse.of(200, null, responseBody)), + Future.succeededFuture(HttpClientResponse.of(200, null, responseBody2))); // when target.refresh(); @@ -171,9 +183,12 @@ public void refreshShouldIncrementErrorMetricOnInvalidJson() { } @Test - public void refreshShouldIncrementErrorMetricWhenNoDeletedVendorsInResponse() throws JsonProcessingException { + public void refreshShouldIncrementErrorMetricOnInvalidVendorList() throws JsonProcessingException { // given - givenHttpClientReturnsResponse(200, givenLiveGvlJson(emptyMap())); + final Vendor vendor = givenVendor(42, "2024-01-01T00:00:00Z") + .toBuilder().features(null).build(); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); + givenHttpClientReturnsResponse(200, responseBody); // when target.refresh(); @@ -181,16 +196,16 @@ public void refreshShouldIncrementErrorMetricWhenNoDeletedVendorsInResponse() th // then assertThat(target.isDeleted(1)).isFalse(); verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); - verify(metrics, never()).updatePrivacyTcfVendorListLatestOkMetric(); } @Test public void refreshShouldKeepLastGoodSetOnFailureAfterSuccessfulFetch() throws JsonProcessingException { // given + final Vendor vendor = givenVendor(1, "2024-01-01T00:00:00Z"); + final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); given(httpClient.get(anyString(), anyLong())) .willReturn( - Future.succeededFuture(HttpClientResponse.of( - 200, null, givenLiveGvlJson(Map.of(1, "2024-01-01T00:00:00Z")))), + Future.succeededFuture(HttpClientResponse.of(200, null, responseBody)), Future.failedFuture(new RuntimeException("connection failed"))); // when @@ -208,22 +223,23 @@ private void givenHttpClientReturnsResponse(int statusCode, String response) { .willReturn(Future.succeededFuture(HttpClientResponse.of(statusCode, null, response))); } - private String givenLiveGvlJson(Map vendorIdToDeletedDate) throws JsonProcessingException { - final Map vendors = new HashMap<>(); - for (Map.Entry entry : vendorIdToDeletedDate.entrySet()) { - vendors.put(entry.getKey(), Map.of("id", entry.getKey(), "deletedDate", entry.getValue())); - } - return mapper.writeValueAsString(Map.of("vendors", vendors)); - } - private static Vendor givenVendor(int id, String deletedDate) { return Vendor.builder() .id(id) .deletedDate(deletedDate != null ? Instant.parse(deletedDate) : null) + .purposes(EnumSet.of(ONE)) + .legIntPurposes(EnumSet.of(TWO)) + .flexiblePurposes(EnumSet.noneOf(PurposeCode.class)) + .specialPurposes(EnumSet.noneOf(SpecialPurpose.class)) + .features(EnumSet.noneOf(Feature.class)) + .specialFeatures(EnumSet.noneOf(SpecialFeature.class)) .build(); } - private static VendorList givenVendorList(Map vendors) { - return VendorList.of(1, Date.from(Instant.parse("2020-08-20T16:05:24Z")), vendors); + private static VendorList givenVendorList(Vendor... vendors) { + return VendorList.of( + 1, + Date.from(Instant.parse("2020-08-20T16:05:24Z")), + Arrays.stream(vendors).collect(Collectors.toMap(Vendor::getId, Function.identity()))); } } From 6c750b43854a7ad4f35f4b4651f350bc5ba385d7 Mon Sep 17 00:00:00 2001 From: Viktor Kryshtal Date: Tue, 9 Jun 2026 16:07:23 +0300 Subject: [PATCH 04/11] Added validation that vendor list is not deleted in requested GVL --- .../vendorlist/LiveVendorListService.java | 7 +--- .../gdpr/vendorlist/VendorListUtil.java | 6 ++++ .../VersionedVendorListService.java | 9 ++++- .../config/PrivacyServiceConfiguration.java | 5 +-- .../VersionedVendorListServiceTest.java | 34 ++++++++++++++++++- 5 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java index ea3bed35049..3d9f3c338eb 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java @@ -93,17 +93,12 @@ private VendorList processResponse(HttpClientResponse response) { Set extractDeletedVendorIds(VendorList vendorList) { final Instant now = clock.instant(); return vendorList.getVendors().values().stream() - .filter(vendor -> isDeletedAt(vendor, now)) + .filter(vendor -> VendorListUtil.vendorIsDeletedAt(vendor, now)) .map(Vendor::getId) .filter(Objects::nonNull) .collect(Collectors.toUnmodifiableSet()); } - private static boolean isDeletedAt(Vendor vendor, Instant now) { - final Instant deletedDate = vendor.getDeletedDate(); - return deletedDate != null && deletedDate.isBefore(now); - } - private Void updateDeletedVendorIds(Set ids) { deletedVendorIds = ids; metrics.updatePrivacyTcfVendorListLatestOkMetric(); diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java index cf8a8b7dd10..371a90e1c55 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListUtil.java @@ -9,6 +9,7 @@ import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList; import java.io.IOException; +import java.time.Instant; import java.util.Collection; public class VendorListUtil { @@ -47,4 +48,9 @@ private static boolean vendorsAreValid(Collection vendors) { && vendor.getFeatures() != null && vendor.getSpecialFeatures() != null); } + + public static boolean vendorIsDeletedAt(Vendor vendor, Instant now) { + final Instant deletedDate = vendor.getDeletedDate(); + return deletedDate != null && deletedDate.isBefore(now); + } } diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java index dada968604d..7b0ad653e30 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java @@ -4,6 +4,8 @@ import io.vertx.core.Future; import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import java.time.Clock; +import java.time.Instant; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -13,14 +15,17 @@ public class VersionedVendorListService { private final VendorListService vendorListServiceV2; private final VendorListService vendorListServiceV3; private final LiveVendorListService liveVendorListService; + private final Clock clock; public VersionedVendorListService(VendorListService vendorListServiceV2, VendorListService vendorListServiceV3, - LiveVendorListService liveVendorListService) { + LiveVendorListService liveVendorListService, + Clock clock) { this.vendorListServiceV2 = Objects.requireNonNull(vendorListServiceV2); this.vendorListServiceV3 = Objects.requireNonNull(vendorListServiceV3); this.liveVendorListService = Objects.requireNonNull(liveVendorListService); + this.clock = Objects.requireNonNull(clock); } public Future> forConsent(TCString consent) { @@ -35,7 +40,9 @@ public Future> forConsent(TCString consent) { } private Map filterDeletedVendors(Map vendors) { + Instant now = clock.instant(); return vendors.entrySet().stream() + .filter(entry -> !VendorListUtil.vendorIsDeletedAt(entry.getValue(), now)) .filter(entry -> !liveVendorListService.isDeleted(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } diff --git a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java index deb6d751d70..dd7d5c0077d 100644 --- a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java @@ -171,10 +171,11 @@ LiveVendorListService liveVendorListService( @Bean VersionedVendorListService versionedVendorListService(VendorListService vendorListServiceV2, VendorListService vendorListServiceV3, - LiveVendorListService liveVendorListService) { + LiveVendorListService liveVendorListService, + Clock clock) { return new VersionedVendorListService( - vendorListServiceV2, vendorListServiceV3, liveVendorListService); + vendorListServiceV2, vendorListServiceV3, liveVendorListService, clock); } @Bean diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java index 3ec04ec5602..2f20041192f 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java @@ -10,6 +10,9 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.Map; import java.util.concurrent.ThreadLocalRandom; @@ -23,6 +26,8 @@ @ExtendWith(MockitoExtension.class) public class VersionedVendorListServiceTest { + private static final Instant NOW = Instant.parse("2024-06-01T12:00:00Z"); + private VersionedVendorListService target; @Mock @@ -35,7 +40,7 @@ public class VersionedVendorListServiceTest { @BeforeEach public void setUp() { target = new VersionedVendorListService( - vendorListServiceV2, vendorListServiceV3, liveVendorListService); + vendorListServiceV2, vendorListServiceV3, liveVendorListService, Clock.fixed(NOW, ZoneOffset.UTC)); } @Test @@ -74,6 +79,33 @@ public void versionedVendorListServiceShouldTreatTcfPolicyGreaterOrEqualFourAsVe verify(vendorListServiceV3).forVersion(12); } + @Test + public void forConsentShouldRemoveVendorsMarkedDeletedInRequestedGvl() { + // given + final Vendor deletedVendor = Vendor.empty(1).toBuilder() + .deletedDate(Instant.parse("2024-01-01T00:00:00Z")) + .build(); + final Vendor activeVendor = Vendor.empty(52); + final Map vendorList = Map.of(1, deletedVendor, 52, activeVendor); + final TCString consent = TCStringEncoder.newBuilder() + .version(2) + .tcfPolicyVersion(3) + .vendorListVersion(12) + .toTCString(); + + given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(vendorList)); + given(liveVendorListService.isDeleted(anyInt())).willReturn(false); + + // when and then + assertThat(target.forConsent(consent)) + .isSucceeded() + .unwrap() + .satisfies(result -> { + assertThat(result).containsOnlyKeys(52); + assertThat(result.get(52)).isSameAs(activeVendor); + }); + } + @Test public void forConsentShouldRemoveVendorsMarkedDeletedInLiveGvl() { // given From 22555dda8ab3318e483bcf7a304cf95b876c34f9 Mon Sep 17 00:00:00 2001 From: Viktor Kryshtal Date: Wed, 10 Jun 2026 14:28:10 +0300 Subject: [PATCH 05/11] Added loading of live vendor list from disk cache on startup --- .../vendorlist/LiveVendorListService.java | 30 +++++-- .../gdpr/vendorlist/VendorListFileStore.java | 28 +++++-- .../VersionedVendorListService.java | 2 +- .../config/PrivacyServiceConfiguration.java | 8 +- .../vendorlist/LiveVendorListServiceTest.java | 53 ++++++++++++- .../vendorlist/VendorListFileStoreTest.java | 78 ++++++++++++++++++- 6 files changed, 182 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java index 3d9f3c338eb..c0d388dfac2 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java @@ -24,33 +24,39 @@ public class LiveVendorListService implements Initializable { private static final Logger logger = LoggerFactory.getLogger(LiveVendorListService.class); + private final String cacheDir; private final String liveGvlUrl; private final long refreshPeriodMs; private final int defaultTimeoutMs; private final Vertx vertx; private final HttpClient httpClient; - private final JacksonMapper mapper; + private final VendorListFileStore vendorListFileStore; private final Metrics metrics; + private final JacksonMapper mapper; private final Clock clock; private volatile Set deletedVendorIds = Set.of(); - public LiveVendorListService(String liveGvlUrl, + public LiveVendorListService(String cacheDir, + String liveGvlUrl, long refreshPeriodMs, int defaultTimeoutMs, Vertx vertx, HttpClient httpClient, - JacksonMapper mapper, + VendorListFileStore vendorListFileStore, Metrics metrics, + JacksonMapper mapper, Clock clock) { + this.cacheDir = Objects.requireNonNull(cacheDir); this.liveGvlUrl = HttpUtil.validateUrl(Objects.requireNonNull(liveGvlUrl)); this.refreshPeriodMs = refreshPeriodMs; this.defaultTimeoutMs = defaultTimeoutMs; this.vertx = Objects.requireNonNull(vertx); this.httpClient = Objects.requireNonNull(httpClient); - this.mapper = Objects.requireNonNull(mapper); + this.vendorListFileStore = Objects.requireNonNull(vendorListFileStore); this.metrics = Objects.requireNonNull(metrics); + this.mapper = Objects.requireNonNull(mapper); this.clock = Objects.requireNonNull(clock); } @@ -61,19 +67,31 @@ public boolean isDeleted(Integer id) { @Override public void initialize(Promise initializePromise) { + initializeWithLatestCachedVersion(); vertx.setPeriodic(0, refreshPeriodMs, ignored -> refresh()); initializePromise.tryComplete(); } + private void initializeWithLatestCachedVersion() { + vendorListFileStore.getLatestVendorListFromCache(cacheDir).ifPresent(vendorList -> { + saveDeletedVendorsFromVendorList(vendorList); + logger.info("Initialized live GVL from cache with version %d".formatted(vendorList.getVendorListVersion())); + }); + } + void refresh() { httpClient.get(liveGvlUrl, defaultTimeoutMs) .map(this::processResponse) - .map(this::extractDeletedVendorIds) - .map(this::updateDeletedVendorIds) + .map(this::saveDeletedVendorsFromVendorList) .otherwise(this::handleError); } + private Void saveDeletedVendorsFromVendorList(VendorList vendorList) { + updateDeletedVendorIds(extractDeletedVendorIds(vendorList)); + return null; + } + private VendorList processResponse(HttpClientResponse response) { final int statusCode = response.getStatusCode(); if (statusCode != 200) { diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java index 09bb9a98b57..acf3ffb11ce 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java @@ -20,8 +20,10 @@ import java.io.File; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Comparator; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; public class VendorListFileStore { @@ -46,16 +48,16 @@ public VendorListFileStore(double logSamplingRate, Map> createCacheFromDisk(String cacheDir) { createAndCheckWritePermissionsForCacheDir(cacheDir); - final Map versionToFileContent = readFileSystemCache(cacheDir); + final Map versionToFileContent = readFileSystemCache(cacheDir); final Map> cache = Caffeine.newBuilder() .>build() .asMap(); - for (Map.Entry versionAndFileContent : versionToFileContent.entrySet()) { + for (Map.Entry versionAndFileContent : versionToFileContent.entrySet()) { final VendorList vendorList = VendorListUtil.parseVendorList(versionAndFileContent.getValue(), mapper); - cache.put(Integer.valueOf(versionAndFileContent.getKey()), vendorList.getVendors()); + cache.put(versionAndFileContent.getKey(), vendorList.getVendors()); } return cache; } @@ -73,13 +75,29 @@ private void createAndCheckWritePermissionsForCacheDir(String cacheDir) { } } - private Map readFileSystemCache(String cacheDir) { + private Map readFileSystemCache(String cacheDir) { return fileSystem.readDirBlocking(cacheDir).stream() .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) - .collect(Collectors.toMap(filepath -> StringUtils.removeEnd(new File(filepath).getName(), JSON_SUFFIX), + .collect(Collectors.toMap(VendorListFileStore::parseCachedFileVersion, filename -> fileSystem.readFileBlocking(filename).toString())); } + Optional getLatestVendorListFromCache(String cacheDir) { + createAndCheckWritePermissionsForCacheDir(cacheDir); + return fileSystem.readDirBlocking(cacheDir).stream() + .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) + .max(Comparator.comparing(VendorListFileStore::parseCachedFileVersion)) + .map(fileSystem::readFileBlocking) + .map(Buffer::toString) + .map(content -> VendorListUtil.parseVendorList(content, mapper)); + } + + private static Integer parseCachedFileVersion(String filepath) { + final String filename = new File(filepath).getName(); + final String filenameWithoutExtension = StringUtils.removeEnd(filename, JSON_SUFFIX); + return Integer.valueOf(filenameWithoutExtension); + } + Future saveToFile(VendorListResult vendorListResult, String cacheDir, String generationVersion) { final Promise promise = Promise.promise(); final int version = vendorListResult.getVersion(); diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java index 7b0ad653e30..8f6487ff60e 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java @@ -40,7 +40,7 @@ public Future> forConsent(TCString consent) { } private Map filterDeletedVendors(Map vendors) { - Instant now = clock.instant(); + final Instant now = clock.instant(); return vendors.entrySet().stream() .filter(entry -> !VendorListUtil.vendorIsDeletedAt(entry.getValue(), now)) .filter(entry -> !liveVendorListService.isDeleted(entry.getKey())) diff --git a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java index dd7d5c0077d..9ba7da3d839 100644 --- a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java @@ -148,23 +148,27 @@ VendorListServiceConfigurationProperties vendorListServiceV3Properties() { @Bean LiveVendorListService liveVendorListService( + VendorListServiceConfigurationProperties vendorListServiceV3Properties, @Value("${gdpr.vendorlist.live-gvl-url}") String liveGvlUrl, @Value("${gdpr.vendorlist.live-gvl-refresh-period-ms}") long refreshPeriodMs, @Value("${gdpr.vendorlist.default-timeout-ms}") int defaultTimeoutMs, Vertx vertx, HttpClient httpClient, - JacksonMapper mapper, + VendorListFileStore vendorListFileStore, Metrics metrics, + JacksonMapper mapper, Clock clock) { return new LiveVendorListService( + vendorListServiceV3Properties.getCacheDir(), liveGvlUrl, refreshPeriodMs, defaultTimeoutMs, vertx, httpClient, - mapper, + vendorListFileStore, metrics, + mapper, clock); } diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java index 68953b15120..66676ed51ca 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import io.vertx.core.Future; +import io.vertx.core.Promise; import io.vertx.core.Vertx; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -25,12 +26,15 @@ import java.util.Arrays; import java.util.Date; import java.util.EnumSet; +import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -41,13 +45,17 @@ public class LiveVendorListServiceTest extends VertxTest { private static final Instant NOW = Instant.parse("2024-06-01T12:00:00Z"); + private static final String CACHE_DIR = "/cache/dir"; private static final String LIVE_GVL_URL = "https://example.com"; + private static final long REFRESH_PERIOD_MS = 1000; @Mock private Vertx vertx; @Mock private HttpClient httpClient; @Mock + private VendorListFileStore vendorListFileStore; + @Mock private Metrics metrics; private LiveVendorListService target; @@ -55,13 +63,15 @@ public class LiveVendorListServiceTest extends VertxTest { @BeforeEach public void setUp() { target = new LiveVendorListService( + CACHE_DIR, LIVE_GVL_URL, - 0, + REFRESH_PERIOD_MS, 1000, vertx, httpClient, - jacksonMapper, + vendorListFileStore, metrics, + jacksonMapper, Clock.fixed(NOW, ZoneOffset.UTC)); } @@ -198,6 +208,45 @@ public void refreshShouldIncrementErrorMetricOnInvalidVendorList() throws JsonPr verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); } + @Test + public void initializeShouldLoadDeletedVendorsFromCachedVendorList() { + // given + final VendorList vendorList = givenVendorList(givenVendor(42, "2024-01-01T00:00:00Z")); + given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.of(vendorList)); + + // when + target.initialize(Promise.promise()); + + // then + assertThat(target.isDeleted(42)).isTrue(); + assertThat(target.isDeleted(99)).isFalse(); + } + + @Test + public void initializeShouldSchedulePeriodicRefresh() { + // given + given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.empty()); + + // when + target.initialize(Promise.promise()); + + // then + verify(vertx).setPeriodic(eq(0L), eq(REFRESH_PERIOD_MS), any()); + } + + @Test + public void initializeShouldCompleteInitializePromise() { + // given + given(vendorListFileStore.getLatestVendorListFromCache(eq(CACHE_DIR))).willReturn(Optional.empty()); + final Promise promise = Promise.promise(); + + // when + target.initialize(promise); + + // then + assertThat(promise.future().succeeded()).isTrue(); + } + @Test public void refreshShouldKeepLastGoodSetOnFailureAfterSuccessfulFetch() throws JsonProcessingException { // given diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java index 845e4e85db0..4a96f8f6dd3 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java @@ -30,6 +30,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Map; +import java.util.Optional; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -40,6 +41,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.ONE; import static org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode.TWO; @@ -286,6 +288,76 @@ public void readFallbackVendorListShouldFailIfFallbackCannotBeParsed() { .hasMessage("Cannot parse vendor list from: invalid"); } + @Test + public void getLatestVendorListFromCacheShouldReturnEmptyWhenCacheDirHasNoJsonFiles() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + final Optional result = target.getLatestVendorListFromCache(CACHE_DIR); + + // then + assertThat(result).isEmpty(); + } + + @Test + public void getLatestVendorListFromCacheShouldReturnVendorListWithHighestVersion() throws JsonProcessingException { + // given + final VendorList vendorList = givenVendorList(10); + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of( + "/cache/dir/2.json", + "/cache/dir/10.json")); + given(fileSystem.readFileBlocking(eq("/cache/dir/10.json"))) + .willReturn(Buffer.buffer(mapper.writeValueAsString(vendorList))); + + // when + final Optional result = target.getLatestVendorListFromCache(CACHE_DIR); + + // then + assertThat(result).hasValue(vendorList); + verify(fileSystem, never()).readFileBlocking(eq("/cache/dir/2.json")); + } + + @Test + public void getLatestVendorListFromCacheShouldCreateCacheDirWhenItDoesNotExist() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of()); + + // when + target.getLatestVendorListFromCache(CACHE_DIR); + + // then + verify(fileSystem).mkdirsBlocking(eq(CACHE_DIR)); + } + + @Test + public void getLatestVendorListFromCacheShouldFailIfCannotReadCacheDir() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willThrow(new RuntimeException("read error")); + + // when and then + assertThatThrownBy(() -> target.getLatestVendorListFromCache(CACHE_DIR)) + .isInstanceOf(RuntimeException.class) + .hasMessage("read error"); + } + + @Test + public void getLatestVendorListFromCacheShouldFailIfLatestVendorListFileCannotBeParsed() { + // given + given(fileSystem.existsBlocking(eq(CACHE_DIR))).willReturn(false); + given(fileSystem.readDirBlocking(eq(CACHE_DIR))).willReturn(List.of("/cache/dir/1.json")); + given(fileSystem.readFileBlocking(eq("/cache/dir/1.json"))).willReturn(Buffer.buffer("invalid")); + + // when and then + assertThatThrownBy(() -> target.getLatestVendorListFromCache(CACHE_DIR)) + .isInstanceOf(PreBidException.class) + .hasMessage("Cannot parse vendor list from: invalid"); + } + @Test public void readFallbackVendorListShouldFailIfFallbackHasInvalidData() throws JsonProcessingException { // given @@ -316,7 +388,11 @@ private void givenWriteFileFails(Throwable throwable) { } private static VendorList givenVendorList() { - return VendorList.of(1, new Date(), givenVendorMap()); + return givenVendorList(1); + } + + private static VendorList givenVendorList(int version) { + return VendorList.of(version, new Date(), givenVendorMap()); } private static Map givenVendorMap() { From 0dbfefdcb4ccea3cef4fa0b1117211cdfe92da8c Mon Sep 17 00:00:00 2001 From: osulzhenko <125548596+osulzhenko@users.noreply.github.com> Date: Sun, 5 Jul 2026 19:19:55 +0300 Subject: [PATCH 06/11] Tests: Handle GVL deletions (#4470) --- .../vendorlist/VendorListResponse.groovy | 5 +- .../service/PrebidServerService.groovy | 9 + .../scaffolding/VendorList.groovy | 40 ++- .../tests/privacy/PrivacyBaseSpec.groovy | 13 +- .../tests/privacy/gdpr/GdprAuctionSpec.groovy | 282 +++++++++++++++++- .../functional/util/privacy/TcfConsent.groovy | 6 +- 6 files changed, 325 insertions(+), 30 deletions(-) diff --git a/src/test/groovy/org/prebid/server/functional/model/mock/services/vendorlist/VendorListResponse.groovy b/src/test/groovy/org/prebid/server/functional/model/mock/services/vendorlist/VendorListResponse.groovy index d44755978e4..9e54f4ce7e6 100644 --- a/src/test/groovy/org/prebid/server/functional/model/mock/services/vendorlist/VendorListResponse.groovy +++ b/src/test/groovy/org/prebid/server/functional/model/mock/services/vendorlist/VendorListResponse.groovy @@ -34,6 +34,7 @@ class VendorListResponse { List features List specialFeatures String policyUrl + ZonedDateTime deletedDate Overflow overflow String cookieMaxAgeSeconds Boolean usesCookies @@ -41,9 +42,9 @@ class VendorListResponse { Boolean usesNonCookieAccess Boolean deviceStorageDisclosureUrl - static Vendor getDefaultVendor(int id) { + static Vendor getDefaultVendor(int vendorId) { new Vendor().tap { - it.id = id + it.id = vendorId it.name = PBSUtils.randomString it.purposes = [1, 3, 4, 5] it.legIntPurposes = [2, 7, 10] diff --git a/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy b/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy index 249d6fa3f13..2e0d0765a0f 100644 --- a/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy +++ b/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy @@ -448,6 +448,15 @@ class PrebidServerService implements ObjectMapperWrapper { } } + Boolean isMetricFilled(String metricName) { + try { + PBSUtils.waitUntil({ this.sendCollectedMetricsRequest()[metricName] != 0 }) + true + } catch (IllegalStateException ignored) { + false + } + } + Boolean isContainMetricByValue(String value) { try { PBSUtils.waitUntil({ sendInfluxMetricsRequest()[value] != null }) diff --git a/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy b/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy index 343a118f53a..29b0060af27 100644 --- a/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy +++ b/src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy @@ -4,7 +4,7 @@ import org.mockserver.matchers.TimeToLive import org.mockserver.matchers.Times import org.mockserver.model.Delay import org.mockserver.model.HttpRequest -import org.mockserver.model.HttpResponse +import org.prebid.server.functional.util.privacy.TcfConsent import org.testcontainers.containers.MockServerContainer import static org.mockserver.model.HttpRequest.request @@ -20,10 +20,13 @@ import static org.prebid.server.functional.util.privacy.TcfConsent.TcfPolicyVers class VendorList extends NetworkScaffolding { - private static final String VENDOR_LIST_ENDPOINT = "/v{TCF_POLICY}/vendor-list.json" + private static final String ENDPOINT_TEMPLATE = "/v{TCF_POLICY}/{FILE_NAME}" - VendorList(MockServerContainer mockServerContainer) { - super(mockServerContainer, VENDOR_LIST_ENDPOINT) + private final String fileName + + VendorList(MockServerContainer mockServerContainer, String fileName = "vendor-list.json") { + super(mockServerContainer, ENDPOINT_TEMPLATE.replace("{FILE_NAME}", fileName)) + this.fileName = fileName } @Override @@ -33,29 +36,40 @@ class VendorList extends NetworkScaffolding { @Override protected HttpRequest getRequest() { - request().withPath(VENDOR_LIST_ENDPOINT) + request().withPath(endpoint) } @Override void reset() { - TcfPolicyVersion.values().each { version -> super.reset("/v${version.vendorListVersion}/vendor-list.json") } + TcfPolicyVersion.values().each { + super.reset("/v${it.vendorListVersion}/${fileName}") + } } void setResponse(TcfPolicyVersion tcfPolicyVersion = TCF_POLICY_V2, Delay delay = null, - Map vendors = [(GENERIC_VENDOR_ID): Vendor.getDefaultVendor(GENERIC_VENDOR_ID)]) { + Map vendors = [(GENERIC_VENDOR_ID): Vendor.getDefaultVendor(GENERIC_VENDOR_ID)], + Integer vendorListVersion = TcfConsent.VENDOR_LIST_VERSION, + Times times = Times.unlimited()) { + def prepareEndpoint = endpoint.replace("{TCF_POLICY}", tcfPolicyVersion.vendorListVersion.toString()) def prepareEncodeResponseBody = encode(defaultVendorListResponse.tap { it.tcfPolicyVersion = tcfPolicyVersion.vendorListVersion it.vendors = vendors + it.vendorListVersion = vendorListVersion it.gvlSpecificationVersion = tcfPolicyVersion >= TcfPolicyVersion.TCF_POLICY_V4 ? V3 : V2 }) - mockServerClient.when(request().withPath(prepareEndpoint), Times.unlimited(), TimeToLive.unlimited(), -10) - .respond { request -> - request.withPath(endpoint) - ? response().withStatusCode(OK_200.code()).withDelay(delay).withBody(prepareEncodeResponseBody) - : HttpResponse.notFoundResponse() - } + def mockResponse = response() + .withStatusCode(OK_200.code()) + .withBody(prepareEncodeResponseBody) + + delay?.with { + mockResponse.withDelay(it) + } + + mockServerClient + .when(request().withPath(prepareEndpoint), times, TimeToLive.unlimited(), -10) + .respond(mockResponse) } } diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy index d92bec507cc..a0557a55202 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy @@ -8,10 +8,8 @@ import org.prebid.server.functional.model.config.AccountDsaConfig import org.prebid.server.functional.model.config.AccountGdprConfig import org.prebid.server.functional.model.config.AccountGppConfig import org.prebid.server.functional.model.config.AccountPrivacyConfig -import org.prebid.server.functional.model.config.Purpose import org.prebid.server.functional.model.db.Account import org.prebid.server.functional.model.mock.services.vendorlist.VendorListResponse -import org.prebid.server.functional.model.privacy.EnforcementRequirement import org.prebid.server.functional.model.privacy.gpp.GppDataActivity import org.prebid.server.functional.model.privacy.gpp.UsCaliforniaV1ChildSensitiveData import org.prebid.server.functional.model.privacy.gpp.UsCaliforniaV1SensitiveData @@ -55,12 +53,9 @@ import org.prebid.server.functional.util.privacy.gpp.v1.UsVaV1Consent import static org.prebid.server.functional.model.bidder.BidderName.GENERIC import static org.prebid.server.functional.model.bidder.BidderName.OPENX -import static org.prebid.server.functional.model.config.PurposeEnforcement.BASIC -import static org.prebid.server.functional.model.config.PurposeEnforcement.FULL -import static org.prebid.server.functional.model.config.PurposeEnforcement.NO import static org.prebid.server.functional.model.mock.services.vendorlist.VendorListResponse.getDefaultVendorListResponse -import static org.prebid.server.functional.model.pricefloors.Country.USA import static org.prebid.server.functional.model.pricefloors.Country.BULGARIA +import static org.prebid.server.functional.model.pricefloors.Country.USA import static org.prebid.server.functional.model.request.GppSectionId.US_CA_V1 import static org.prebid.server.functional.model.request.GppSectionId.US_CO_V1 import static org.prebid.server.functional.model.request.GppSectionId.US_CT_V1 @@ -74,10 +69,6 @@ import static org.prebid.server.functional.model.request.auction.TraceLevel.VERB import static org.prebid.server.functional.model.response.cookiesync.UserSyncInfo.Type.REDIRECT import static org.prebid.server.functional.testcontainers.Dependencies.getNetworkServiceContainer import static org.prebid.server.functional.util.privacy.TcfConsent.GENERIC_VENDOR_ID -import static org.prebid.server.functional.util.privacy.TcfConsent.PurposeId.BASIC_ADS -import static org.prebid.server.functional.util.privacy.TcfConsent.RestrictionType.REQUIRE_CONSENT -import static org.prebid.server.functional.util.privacy.TcfConsent.RestrictionType.REQUIRE_LEGITIMATE_INTEREST -import static org.prebid.server.functional.util.privacy.TcfConsent.RestrictionType.UNDEFINED import static org.prebid.server.functional.util.privacy.TcfConsent.TcfPolicyVersion.TCF_POLICY_V2 import static org.prebid.server.functional.util.privacy.model.State.ALABAMA @@ -90,7 +81,7 @@ abstract class PrivacyBaseSpec extends BaseSpec { "adapters.${GENERIC.value}.ortb-version" : "2.6"] private static final Map OPENX_CONFIG = ["adaptrs.${OPENX.value}.enabled" : "true", "adapters.${OPENX.value}.usersync.cookie-family-name": OPENX.value, - "adapters.${OPENX}.ortb-version" : "2.6", + "adapters.${OPENX.value}.ortb-version" : "2.6", "adapters.${OPENX.value}.endpoint" : "$networkServiceContainer.rootUri/auction".toString(), "adapters.${OPENX.value}.enabled" : 'true'] protected static final Map GDPR_VENDOR_LIST_CONFIG = ["gdpr.vendorlist.v2.http-endpoint-template": "$networkServiceContainer.rootUri/v2/vendor-list.json".toString(), diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy index d90bf8da9ac..8b5996089b3 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy @@ -1,5 +1,6 @@ package org.prebid.server.functional.tests.privacy.gdpr +import org.mockserver.matchers.Times import org.mockserver.model.Delay import org.prebid.server.functional.model.ChannelType import org.prebid.server.functional.model.config.AccountGdprConfig @@ -7,24 +8,30 @@ import org.prebid.server.functional.model.config.AccountMetricsConfig import org.prebid.server.functional.model.config.AccountMetricsVerbosityLevel import org.prebid.server.functional.model.config.PurposeConfig import org.prebid.server.functional.model.config.PurposeEnforcement +import org.prebid.server.functional.model.mock.services.vendorlist.VendorListResponse import org.prebid.server.functional.model.pricefloors.Country +import org.prebid.server.functional.model.privacy.EnforcementRequirement import org.prebid.server.functional.model.request.auction.DistributionChannel import org.prebid.server.functional.model.request.auction.Regs import org.prebid.server.functional.model.request.auction.RegsExt import org.prebid.server.functional.model.response.auction.ErrorType +import org.prebid.server.functional.model.response.auction.NoBidResponse +import org.prebid.server.functional.service.PrebidServerService +import org.prebid.server.functional.testcontainers.scaffolding.VendorList import org.prebid.server.functional.tests.privacy.PrivacyBaseSpec import org.prebid.server.functional.util.PBSUtils import org.prebid.server.functional.util.privacy.BogusConsent import org.prebid.server.functional.util.privacy.TcfConsent +import org.prebid.server.functional.util.privacy.TcfUtils import org.prebid.server.functional.util.privacy.VendorListConsent import spock.lang.PendingFeature import java.time.Instant +import java.time.ZonedDateTime import static org.prebid.server.functional.model.ChannelType.PBJS import static org.prebid.server.functional.model.ChannelType.WEB import static org.prebid.server.functional.model.bidder.BidderName.GENERIC - import static org.prebid.server.functional.model.config.AccountMetricsVerbosityLevel.DETAILED import static org.prebid.server.functional.model.config.Purpose.P1 import static org.prebid.server.functional.model.config.Purpose.P2 @@ -46,6 +53,7 @@ import static org.prebid.server.functional.model.request.auction.PublicCountryIp import static org.prebid.server.functional.model.request.auction.TraceLevel.BASIC import static org.prebid.server.functional.model.request.auction.TraceLevel.VERBOSE import static org.prebid.server.functional.model.response.auction.BidRejectionReason.REQUEST_BLOCKED_PRIVACY +import static org.prebid.server.functional.testcontainers.Dependencies.getNetworkServiceContainer import static org.prebid.server.functional.util.privacy.TcfConsent.GENERIC_VENDOR_ID import static org.prebid.server.functional.util.privacy.TcfConsent.PurposeId.BASIC_ADS import static org.prebid.server.functional.util.privacy.TcfConsent.PurposeId.DEVICE_ACCESS @@ -55,6 +63,21 @@ import static org.prebid.server.functional.util.privacy.TcfConsent.TcfPolicyVers class GdprAuctionSpec extends PrivacyBaseSpec { + private static final String LIVE_GVL_FILE_NAME = "live-gvl-vendor-list.json" + private static final VENDOR_LIST_VERSION = PBSUtils.getRandomNumber(2, 4094) + private static final Map REFRESH_LIVE_VENDOR_LIST_CONFIG = GENERAL_PRIVACY_CONFIG + + ["gdpr.vendorlist.live-gvl-url": "$networkServiceContainer.rootUri/v3/$LIVE_GVL_FILE_NAME".toString()] + private static final VendorList liveVendorListResponse = new VendorList(networkServiceContainer, LIVE_GVL_FILE_NAME) + + def setup() { + vendorListResponse.setResponse() + liveVendorListResponse.reset() + } + + def cleanup() { + vendorListResponse.reset() + } + @PendingFeature def "PBS should add debug log for auction request when valid gdpr was passed"() { given: "Default gdpr BidRequest" @@ -1116,4 +1139,261 @@ class GdprAuctionSpec extends PrivacyBaseSpec { assert metrics["adapter.${GENERIC.value}.requests.buyeruid_scrubbed"] == 1 assert metrics["account.${account.uuid}.adapter.${GENERIC.value}.requests.buyeruid_scrubbed"] == 1 } + + def "PBS should treated GVL record as valid when it's don't marked as deleted for GVL file"() { + given: "Vendor list response with not passed delete date for bidder" + def vendorList = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = gvlDeletedDate + } + downloadLiveGvtList(VENDOR_LIST_VERSION, vendorList) + + and: "PBS with proper warmed GVL config" + def pbsWithLiveGvlSetup = pbsServiceFactory.getService(REFRESH_LIVE_VENDOR_LIST_CONFIG) + warmupGvtList(pbsWithLiveGvlSetup, VENDOR_LIST_VERSION) + + and: "Valid tcf full enforcement requirments" + def requirement = EnforcementRequirement.getDefaultFull(GENERIC_VENDOR_ID, VENDOR_LIST_VERSION, P2) + + and: "Tcf consent setup" + def tcfConsent = TcfUtils.getConsentString(requirement) + + and: "Bid request with tcf consent string" + def bidRequest = getGdprBidRequest(tcfConsent) + + and: "Account with tcf config" + def accountGdprConfig = new AccountGdprConfig(purposes: TcfUtils.getPurposeConfigsForPersonalizedAds(requirement)) + def account = getAccountWithGdpr(bidRequest.accountId, accountGdprConfig) + accountDao.save(account) + + and: "Flush metric" + flushMetrics(pbsWithLiveGvlSetup) + + when: "PBS processes auction request" + def response = pbsWithLiveGvlSetup.sendAuctionRequest(bidRequest) + + then: "Response should contain seatBid" + assert response.seatbid.bid.flatten().size() == 1 + + "Response shouldn't contain any nrb, errors and warnings" + assert verifyAll(response) { + !it.noBidResponse + !it.ext.warnings + !it.ext.errors + } + + and: "Bidder request should be valid" + assert bidder.getBidderRequests(bidRequest.id) + + and: "Disallowed metrics shouldn't be updated" + def metrics = pbsWithLiveGvlSetup.sendCollectedMetricsRequest() + assert !metrics[TEMPLATE_ADAPTER_DISALLOWED_COUNT.getValue(bidRequest, FETCH_BIDS)] + + cleanup: "Stop and remove pbs container" + pbsServiceFactory.removeContainer(REFRESH_LIVE_VENDOR_LIST_CONFIG) + + where: + gvlDeletedDate << [null, ZonedDateTime.now().plusYears(1)] + } + + def "PBS should treated GVL record as invalid when it's marked as deleted for GVL file"() { + given: "Vendor list response with passed delete date for bidder" + def vendor = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = liveGvlDeletedDate + } + downloadLiveGvtList(VENDOR_LIST_VERSION, vendor) + + and: "PBS with proper warmed GVL config" + def pbsWithLiveGvlSetup = pbsServiceFactory.getService(REFRESH_LIVE_VENDOR_LIST_CONFIG) + def requestVendor = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = requestGvlDeletedDate + } + warmupGvtList(pbsWithLiveGvlSetup, VENDOR_LIST_VERSION, requestVendor) + + and: "Valid tcf full enforcement requirments" + def requirement = EnforcementRequirement.getDefaultFull(GENERIC_VENDOR_ID, VENDOR_LIST_VERSION, P2) + + and: "Tcf consent setup" + def tcfConsent = TcfUtils.getConsentString(requirement) + + and: "Bid request with tcf consent string" + def bidRequest = getGdprBidRequest(tcfConsent) + + and: "Account with tcf config" + def accountGdprConfig = new AccountGdprConfig(purposes: TcfUtils.getPurposeConfigsForPersonalizedAds(requirement)) + def account = getAccountWithGdpr(bidRequest.accountId, accountGdprConfig) + accountDao.save(account) + + and: "Flush metric" + flushMetrics(pbsWithLiveGvlSetup) + + when: "PBS processes auction request" + def response = pbsWithLiveGvlSetup.sendAuctionRequest(bidRequest) + + then: "Response shouldn't contain any seatBids, errors and warnings" + assert verifyAll(response) { + !it.seatbid.size() + !it.ext.warnings + !it.ext.errors + } + + and: "Response should include nbr" + assert response.noBidResponse == NoBidResponse.UNKNOWN_ERROR + + and: "PBS should cansel request" + assert !bidder.getBidderRequests(bidRequest.id) + + and: "Disallowed metrics should be updated" + def metrics = pbsWithLiveGvlSetup.sendCollectedMetricsRequest() + assert metrics[TEMPLATE_ADAPTER_DISALLOWED_COUNT.getValue(bidRequest, FETCH_BIDS)] == 1 + + cleanup: "Stop and remove pbs container" + pbsServiceFactory.removeContainer(REFRESH_LIVE_VENDOR_LIST_CONFIG) + + where: + liveGvlDeletedDate | requestGvlDeletedDate + ZonedDateTime.now().minusSeconds(1) | ZonedDateTime.now().plusYears(1) + ZonedDateTime.now().plusYears(1) | ZonedDateTime.now().minusSeconds(1) + } + + def "PBS should treated GVL record as invalid when it's marked as deleted for newer GVL file"() { + given: "Newer Vendor list response with passed delete date for bidder" + def vendorList = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = ZonedDateTime.now().minusSeconds(1) + } + downloadLiveGvtList(VENDOR_LIST_VERSION, vendorList) + + and: "PBS with proper warmed older GVL config" + def pbsWithLiveGvlSetup = pbsServiceFactory.getService(REFRESH_LIVE_VENDOR_LIST_CONFIG) + warmupGvtList(pbsWithLiveGvlSetup, VENDOR_LIST_VERSION - 1) + + and: "Valid tcf full enforcement requirments for older gvl list" + def requirement = EnforcementRequirement.getDefaultFull(GENERIC_VENDOR_ID, VENDOR_LIST_VERSION - 1, P2) + + and: "Tcf consent setup" + def tcfConsent = TcfUtils.getConsentString(requirement) + + and: "Bid request with tcf consent string" + def bidRequest = getGdprBidRequest(tcfConsent) + + and: "Account with tcf config" + def accountGdprConfig = new AccountGdprConfig(purposes: TcfUtils.getPurposeConfigsForPersonalizedAds(requirement)) + def account = getAccountWithGdpr(bidRequest.accountId, accountGdprConfig) + accountDao.save(account) + + and: "Flush metric" + flushMetrics(pbsWithLiveGvlSetup) + + when: "PBS processes auction request" + def response = pbsWithLiveGvlSetup.sendAuctionRequest(bidRequest) + + then: "Response shouldn't contain any seatBids, errors and warnings" + assert verifyAll(response) { + !it.seatbid.size() + !it.ext.warnings + !it.ext.errors + } + + and: "Response should include nbr" + assert response.noBidResponse == NoBidResponse.UNKNOWN_ERROR + + and: "PBS should cansel request" + assert !bidder.getBidderRequests(bidRequest.id) + + and: "Disallowed metrics should be updated" + def metrics = pbsWithLiveGvlSetup.sendCollectedMetricsRequest() + assert metrics[TEMPLATE_ADAPTER_DISALLOWED_COUNT.getValue(bidRequest, FETCH_BIDS)] == 1 + + cleanup: "Stop and remove pbs container" + pbsServiceFactory.removeContainer(REFRESH_LIVE_VENDOR_LIST_CONFIG) + } + + def "PBS should treat vendor as valid when deleted only in intermediate GVL versions"() { + given: "Old vendor list response with passed delete date for bidder" + def olderVendorList = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = ZonedDateTime.now().minusSeconds(1) + } + downloadLiveGvtList(VENDOR_LIST_VERSION, olderVendorList, Times.once()) + + and: "PBS with a high-frequency live GVL refresh config" + def config = REFRESH_LIVE_VENDOR_LIST_CONFIG + ['gdpr.vendorlist.live-gvl-refresh-period-ms': '1000'] + def pbsWithLiveGvlSetup = pbsServiceFactory.getService(config) + + and: "Flash metrics" + flushMetrics(pbsWithLiveGvlSetup) + + and: "Newer vendor list where bidder is not deleted yet" + def newerVendorList = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID).tap { + deletedDate = ZonedDateTime.now().plusYears(1) + } + downloadLiveGvtList(VENDOR_LIST_VERSION + 1, newerVendorList, Times.once()) + pbsWithLiveGvlSetup.isContainMetricByValue('privacy.tcf.vendorlist.latest.ok') + + and: "GVL list is warmed up for PBS" + warmupGvtList(pbsWithLiveGvlSetup, VENDOR_LIST_VERSION - 1) + + and: "Valid tcf full enforcement requirments for older GVL list" + def requirement = EnforcementRequirement.getDefaultFull(GENERIC_VENDOR_ID, VENDOR_LIST_VERSION - 1, P2) + + and: "Tcf consent setup" + def tcfConsent = TcfUtils.getConsentString(requirement) + + and: "Bid request with tcf consent string" + def bidRequest = getGdprBidRequest(tcfConsent) + + and: "Account with tcf config" + def accountGdprConfig = new AccountGdprConfig(purposes: TcfUtils.getPurposeConfigsForPersonalizedAds(requirement)) + def account = getAccountWithGdpr(bidRequest.accountId, accountGdprConfig) + accountDao.save(account) + + and: "Flush metric" + flushMetrics(pbsWithLiveGvlSetup) + + when: "PBS processes auction request" + def response = pbsWithLiveGvlSetup.sendAuctionRequest(bidRequest) + + then: "Response should contain seatBid" + assert response.seatbid.bid.flatten().size() == 1 + + "Response shouldn't contain any nrb, errors and warnings" + assert verifyAll(response) { + !it.noBidResponse + !it.ext.warnings + !it.ext.errors + } + + and: "Bidder request should be valid" + assert bidder.getBidderRequests(bidRequest.id) + + and: "Disallowed metrics shouldn't be updated" + def metrics = pbsWithLiveGvlSetup.sendCollectedMetricsRequest() + assert metrics[TEMPLATE_ADAPTER_DISALLOWED_COUNT.getValue(bidRequest, FETCH_BIDS)] == 0 + + cleanup: "Stop and remove pbs container" + pbsServiceFactory.removeContainer(config) + } + + private static void downloadLiveGvtList( + Integer vendorListVersion = VENDOR_LIST_VERSION, + VendorListResponse.Vendor vendor = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID), + Times times = Times.unlimited()) { + + liveVendorListResponse.reset() + liveVendorListResponse.setResponse(TCF_POLICY_V5, null, [(vendor.id): vendor], vendorListVersion, times) + } + + private static void warmupGvtList(PrebidServerService pbsService, + Integer vendorListVersion = VENDOR_LIST_VERSION, + VendorListResponse.Vendor vendor = VendorListResponse.Vendor.getDefaultVendor(GENERIC_VENDOR_ID)) { + + def simpleTcfString = new TcfConsent.Builder() + .setDisclosedVendors([GENERIC_VENDOR_ID]) + .setVendorListVersion(vendorListVersion) + .build() + + vendorList.reset() + vendorList.setResponse(TCF_POLICY_V2, null, [(vendor.id): vendor]) + + def bidRequest = getGdprBidRequest(simpleTcfString) + pbsService.sendAuctionRequest(bidRequest) + } } diff --git a/src/test/groovy/org/prebid/server/functional/util/privacy/TcfConsent.groovy b/src/test/groovy/org/prebid/server/functional/util/privacy/TcfConsent.groovy index 56953549575..7a64c22b3df 100644 --- a/src/test/groovy/org/prebid/server/functional/util/privacy/TcfConsent.groovy +++ b/src/test/groovy/org/prebid/server/functional/util/privacy/TcfConsent.groovy @@ -12,9 +12,9 @@ import static org.prebid.server.functional.util.privacy.TcfConsent.TcfPolicyVers class TcfConsent implements ConsentString { - public static final Integer RUBICON_VENDOR_ID = PBSUtils.getRandomNumber(0, 65534) - public static final Integer GENERIC_VENDOR_ID = PBSUtils.getRandomNumber(0, 65534) - public static final Integer VENDOR_LIST_VERSION = PBSUtils.getRandomNumber(0, 4095) + public static final Integer RUBICON_VENDOR_ID = PBSUtils.getRandomNumber(1, 65534) + public static final Integer GENERIC_VENDOR_ID = PBSUtils.getRandomNumber(1, 65534) + public static final Integer VENDOR_LIST_VERSION = PBSUtils.getRandomNumber(1, 4095) private final TCStringEncoder.Builder tcStringEncoder From f9186e4c55e5133640a5bc5023de5b7a54fc1d0a Mon Sep 17 00:00:00 2001 From: Viktor Kryshtal Date: Wed, 22 Jul 2026 16:31:57 +0300 Subject: [PATCH 07/11] Refactor --- .../server/privacy/gdpr/Tcf2Service.java | 7 ++-- .../vendorlist/LiveVendorListService.java | 7 ++-- .../gdpr/vendorlist/VendorListWrapper.java | 34 +++++++++++++++++++ .../VersionedVendorListService.java | 15 ++------ .../server/privacy/gdpr/Tcf2ServiceTest.java | 4 +-- .../vendorlist/LiveVendorListServiceTest.java | 28 +++++++-------- .../vendorlist/VendorListFileStoreTest.java | 19 +++++++++-- .../VersionedVendorListServiceTest.java | 15 ++++---- 8 files changed, 82 insertions(+), 47 deletions(-) create mode 100644 src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java diff --git a/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java b/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java index 8d47b739426..0d10b45368a 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java @@ -12,6 +12,7 @@ import org.prebid.server.privacy.gdpr.model.VendorPermissionWithGvl; import org.prebid.server.privacy.gdpr.tcfstrategies.purpose.PurposeStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy; +import org.prebid.server.privacy.gdpr.vendorlist.VendorListWrapper; import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService; import org.prebid.server.privacy.gdpr.vendorlist.proto.PurposeCode; import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; @@ -114,7 +115,7 @@ private Future> permissionsForInternal(Collection processDowngradedSupportedPurposeStrategies( tcfConsent, - wrapWithGVL(vendorPermissionsByType, Collections.emptyMap()), + wrapWithGVL(vendorPermissionsByType, VendorListWrapper.empty()), mergedPurposes, mergedPurposeOneTreatmentInterpretation)) .map(ignored -> enforcePurpose4IfRequired(mergedPurposes, vendorPermissionsByType)) @@ -146,7 +147,7 @@ private static VendorPermissionsByType toVendorPermissionsByTy private static VendorPermissionsByType wrapWithGVL( VendorPermissionsByType vendorPermissionsByType, - Map vendorGvlPermissions) { + VendorListWrapper vendorGvlPermissions) { final List weakPermissions = vendorPermissionsByType.getWeakPermissions().stream() .map(vendorPermission -> wrapWithGVL(vendorPermission, vendorGvlPermissions)) @@ -161,7 +162,7 @@ private static VendorPermissionsByType wrapWithGVL( } private static VendorPermissionWithGvl wrapWithGVL(VendorPermission vendorPermission, - Map vendorGvlPermissions) { + VendorListWrapper vendorGvlPermissions) { final Integer vendorId = vendorPermission.getVendorId(); final Vendor vendorGvlByVendorId = Optional.ofNullable(vendorId) diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java index c0d388dfac2..c2bc8507d4a 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java @@ -2,6 +2,7 @@ import io.vertx.core.Promise; import io.vertx.core.Vertx; +import lombok.Getter; import org.prebid.server.exception.PreBidException; import org.prebid.server.json.JacksonMapper; import org.prebid.server.log.Logger; @@ -35,6 +36,7 @@ public class LiveVendorListService implements Initializable { private final JacksonMapper mapper; private final Clock clock; + @Getter private volatile Set deletedVendorIds = Set.of(); public LiveVendorListService(String cacheDir, @@ -60,11 +62,6 @@ public LiveVendorListService(String cacheDir, this.clock = Objects.requireNonNull(clock); } - public boolean isDeleted(Integer id) { - final Set ids = deletedVendorIds; - return !ids.isEmpty() && ids.contains(id); - } - @Override public void initialize(Promise initializePromise) { initializeWithLatestCachedVersion(); diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java new file mode 100644 index 00000000000..dd96058b29c --- /dev/null +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java @@ -0,0 +1,34 @@ +package org.prebid.server.privacy.gdpr.vendorlist; + +import lombok.RequiredArgsConstructor; +import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; + +import java.time.Instant; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +// The purpose of this wrapper class is to avoid filtering the whole vendor list for deleted vendors +// TODO: move filtering logic to vendor list service +@RequiredArgsConstructor(staticName = "of") +public class VendorListWrapper { + + private final Map vendorList; + private final Set deletedVendorIds; + private final Instant timestamp; + + public Vendor get(Integer key) { + final Vendor vendor = vendorList.get(key); + return isRetained(key, vendor) ? vendor : null; + } + + private boolean isRetained(Integer id, Vendor vendor) { + return vendor != null + && !VendorListUtil.vendorIsDeletedAt(vendor, timestamp) + && !deletedVendorIds.contains(id); + } + + public static VendorListWrapper empty() { + return VendorListWrapper.of(Collections.emptyMap(), Collections.emptySet(), null); + } +} diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java index 8f6487ff60e..0f0a63073ca 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListService.java @@ -5,10 +5,8 @@ import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; import java.time.Clock; -import java.time.Instant; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; public class VersionedVendorListService { @@ -28,7 +26,7 @@ public VersionedVendorListService(VendorListService vendorListServiceV2, this.clock = Objects.requireNonNull(clock); } - public Future> forConsent(TCString consent) { + public Future forConsent(TCString consent) { final int tcfPolicyVersion = consent.getTcfPolicyVersion(); final int vendorListVersion = consent.getVendorListVersion(); @@ -36,14 +34,7 @@ public Future> forConsent(TCString consent) { ? vendorListServiceV2.forVersion(vendorListVersion) : vendorListServiceV3.forVersion(vendorListVersion); - return vendorListFuture.map(this::filterDeletedVendors); - } - - private Map filterDeletedVendors(Map vendors) { - final Instant now = clock.instant(); - return vendors.entrySet().stream() - .filter(entry -> !VendorListUtil.vendorIsDeletedAt(entry.getValue(), now)) - .filter(entry -> !liveVendorListService.isDeleted(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return vendorListFuture.map(vendors -> + VendorListWrapper.of(vendors, liveVendorListService.getDeletedVendorIds(), clock.instant())); } } diff --git a/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java index 19c4d034b2d..0e7ac5fd12d 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java @@ -15,6 +15,7 @@ import org.prebid.server.privacy.gdpr.model.VendorPermissionWithGvl; import org.prebid.server.privacy.gdpr.tcfstrategies.purpose.PurposeStrategy; import org.prebid.server.privacy.gdpr.tcfstrategies.specialfeature.SpecialFeaturesStrategy; +import org.prebid.server.privacy.gdpr.vendorlist.VendorListWrapper; import org.prebid.server.privacy.gdpr.vendorlist.VersionedVendorListService; import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; import org.prebid.server.settings.model.AccountGdprConfig; @@ -34,7 +35,6 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static org.apache.commons.collections4.SetUtils.hashSet; @@ -103,7 +103,7 @@ public class Tcf2ServiceTest extends VertxTest { @BeforeEach public void setUp() { - given(vendorListService.forConsent(any())).willReturn(Future.succeededFuture(emptyMap())); + given(vendorListService.forConsent(any())).willReturn(Future.succeededFuture(VendorListWrapper.empty())); given(purposeStrategyOne.getPurpose()).willReturn(ONE); given(purposeStrategyTwo.getPurpose()).willReturn(TWO); diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java index 66676ed51ca..18313ae9049 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java @@ -76,14 +76,13 @@ public void setUp() { } @Test - public void isDeletedShouldReturnFalseWhenFetchNeverSucceeded() { + public void getDeletedVendorIdsShouldReturnEmptySetWhenFetchNeverSucceeded() { // when and then - assertThat(target.isDeleted(1)).isFalse(); - assertThat(target.isDeleted(null)).isFalse(); + assertThat(target.getDeletedVendorIds()).isEmpty(); } @Test - public void isDeletedShouldReturnTrueWhenVendorIsDeletedInLiveVendorList() throws JsonProcessingException { + public void getDeletedVendorIdsShouldReturnVendorThatIsDeletedInLiveVendorList() throws JsonProcessingException { // given final Vendor vendor = givenVendor(42, "2024-01-01T00:00:00Z"); final String responseBody = mapper.writeValueAsString(givenVendorList(vendor)); @@ -94,8 +93,7 @@ public void isDeletedShouldReturnTrueWhenVendorIsDeletedInLiveVendorList() throw target.refresh(); // then - assertThat(target.isDeleted(42)).isTrue(); - assertThat(target.isDeleted(99)).isFalse(); + assertThat(target.getDeletedVendorIds()).containsExactly(42); } @Test @@ -125,7 +123,7 @@ public void refreshShouldUpdateDeletedVendorIdsAndIncrementOkMetric() throws Jso target.refresh(); // then - assertThat(target.isDeleted(1)).isTrue(); + assertThat(target.getDeletedVendorIds()).containsExactly(1); verify(metrics).updatePrivacyTcfVendorListLatestOkMetric(); verify(metrics, never()).updatePrivacyTcfVendorListLatestErrorMetric(); } @@ -147,8 +145,7 @@ public void refreshShouldReplaceDeletedVendorIdsOnSubsequentSuccessfulFetch() th target.refresh(); // then - assertThat(target.isDeleted(1)).isFalse(); - assertThat(target.isDeleted(2)).isTrue(); + assertThat(target.getDeletedVendorIds()).containsExactly(2); } @Test @@ -161,7 +158,7 @@ public void refreshShouldIncrementErrorMetricOnHttpFailure() { target.refresh(); // then - assertThat(target.isDeleted(1)).isFalse(); + assertThat(target.getDeletedVendorIds()).isEmpty(); verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); verify(metrics, never()).updatePrivacyTcfVendorListLatestOkMetric(); } @@ -175,7 +172,7 @@ public void refreshShouldIncrementErrorMetricOnNonOkStatus() { target.refresh(); // then - assertThat(target.isDeleted(1)).isFalse(); + assertThat(target.getDeletedVendorIds()).isEmpty(); verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); } @@ -188,7 +185,7 @@ public void refreshShouldIncrementErrorMetricOnInvalidJson() { target.refresh(); // then - assertThat(target.isDeleted(1)).isFalse(); + assertThat(target.getDeletedVendorIds()).isEmpty(); verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); } @@ -204,7 +201,7 @@ public void refreshShouldIncrementErrorMetricOnInvalidVendorList() throws JsonPr target.refresh(); // then - assertThat(target.isDeleted(1)).isFalse(); + assertThat(target.getDeletedVendorIds()).isEmpty(); verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); } @@ -218,8 +215,7 @@ public void initializeShouldLoadDeletedVendorsFromCachedVendorList() { target.initialize(Promise.promise()); // then - assertThat(target.isDeleted(42)).isTrue(); - assertThat(target.isDeleted(99)).isFalse(); + assertThat(target.getDeletedVendorIds()).containsExactly(42); } @Test @@ -262,7 +258,7 @@ public void refreshShouldKeepLastGoodSetOnFailureAfterSuccessfulFetch() throws J target.refresh(); // then - assertThat(target.isDeleted(1)).isTrue(); + assertThat(target.getDeletedVendorIds()).containsExactly(1); verify(metrics).updatePrivacyTcfVendorListLatestOkMetric(); verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); } diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java index 4a96f8f6dd3..b37471a60e9 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStoreTest.java @@ -36,6 +36,7 @@ import static java.util.Collections.singletonMap; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -57,7 +58,7 @@ public class VendorListFileStoreTest extends VertxTest { private FileSystem fileSystem; @TempDir - Path tempDir; + private Path tempDir; private VendorListFileStore target; @@ -109,7 +110,7 @@ public void createCacheFromDiskShouldFailIfCannotCreateCacheDir() { } @Test - public void createCacheFromDiskShouldFailIfNoWritePermissionsForCacheDir() { + public void createCacheFromDiskShouldFailIfCacheDirIsNotWritable() { // given final String cacheDir = tempDir.toString(); tempDir.toFile().setWritable(false, false); @@ -124,6 +125,20 @@ public void createCacheFromDiskShouldFailIfNoWritePermissionsForCacheDir() { .hasMessage("No write permissions for directory: " + cacheDir); } + @Test + public void createCacheFromDiskShouldNotFailIfCacheDirIsWriable() { + // given + final String cacheDir = tempDir.toString(); + tempDir.toFile().setWritable(true, false); + final FileProps fileProps = mock(FileProps.class); + given(fileSystem.existsBlocking(eq(cacheDir))).willReturn(true); + given(fileSystem.propsBlocking(eq(cacheDir))).willReturn(fileProps); + given(fileProps.isDirectory()).willReturn(true); + + // when and then + assertDoesNotThrow(() -> target.createCacheFromDisk(cacheDir)); + } + @Test public void createCacheFromDiskShouldReturnCacheLoadedFromJsonFiles() throws JsonProcessingException { // given diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java index 2f20041192f..4ca880f61e0 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/VersionedVendorListServiceTest.java @@ -17,6 +17,8 @@ import java.util.concurrent.ThreadLocalRandom; import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static java.util.Collections.singleton; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.BDDMockito.given; @@ -82,11 +84,11 @@ public void versionedVendorListServiceShouldTreatTcfPolicyGreaterOrEqualFourAsVe @Test public void forConsentShouldRemoveVendorsMarkedDeletedInRequestedGvl() { // given - final Vendor deletedVendor = Vendor.empty(1).toBuilder() + final Vendor deletedVendor = Vendor.empty(2).toBuilder() .deletedDate(Instant.parse("2024-01-01T00:00:00Z")) .build(); final Vendor activeVendor = Vendor.empty(52); - final Map vendorList = Map.of(1, deletedVendor, 52, activeVendor); + final Map vendorList = Map.of(2, deletedVendor, 52, activeVendor); final TCString consent = TCStringEncoder.newBuilder() .version(2) .tcfPolicyVersion(3) @@ -94,14 +96,14 @@ public void forConsentShouldRemoveVendorsMarkedDeletedInRequestedGvl() { .toTCString(); given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(vendorList)); - given(liveVendorListService.isDeleted(anyInt())).willReturn(false); + given(liveVendorListService.getDeletedVendorIds()).willReturn(emptySet()); // when and then assertThat(target.forConsent(consent)) .isSucceeded() .unwrap() .satisfies(result -> { - assertThat(result).containsOnlyKeys(52); + assertThat(result.get(2)).isNull(); assertThat(result.get(52)).isSameAs(activeVendor); }); } @@ -119,15 +121,14 @@ public void forConsentShouldRemoveVendorsMarkedDeletedInLiveGvl() { .toTCString(); given(vendorListServiceV2.forVersion(anyInt())).willReturn(Future.succeededFuture(vendorList)); - given(liveVendorListService.isDeleted(1)).willReturn(true); - given(liveVendorListService.isDeleted(52)).willReturn(false); + given(liveVendorListService.getDeletedVendorIds()).willReturn(singleton(1)); // when and then assertThat(target.forConsent(consent)) .isSucceeded() .unwrap() .satisfies(result -> { - assertThat(result).containsOnlyKeys(52); + assertThat(result.get(1)).isNull(); assertThat(result.get(52)).isSameAs(activeVendor); }); } From 67e27abc60a699372aa001cdb4d1c46b0d9d8202 Mon Sep 17 00:00:00 2001 From: Viktor Kryshtal Date: Wed, 22 Jul 2026 19:00:53 +0300 Subject: [PATCH 08/11] Fix comments --- sample/001_banner/configs/config.yaml | 2 ++ sample/configs/localdev-config.yaml | 2 ++ sample/configs/prebid-config-db.yaml | 2 ++ sample/configs/prebid-config-s3.yaml | 2 ++ sample/configs/prebid-config-with-51d-dd.yaml | 2 ++ sample/configs/prebid-config-with-module.yaml | 2 ++ sample/configs/prebid-config-with-optable.yaml | 2 ++ sample/configs/prebid-config-with-wurfl.yaml | 2 ++ sample/configs/prebid-config.yaml | 2 ++ src/main/docker/application.yaml | 2 ++ .../privacy/gdpr/vendorlist/VendorListFileStore.java | 3 ++- .../server/spring/config/PrivacyServiceConfiguration.java | 8 ++++---- src/main/resources/application.yaml | 5 +++-- .../org/prebid/server/it/test-application.properties | 1 + 14 files changed, 30 insertions(+), 7 deletions(-) diff --git a/sample/001_banner/configs/config.yaml b/sample/001_banner/configs/config.yaml index 1dd053ddb22..ca3496226c2 100755 --- a/sample/001_banner/configs/config.yaml +++ b/sample/001_banner/configs/config.yaml @@ -25,6 +25,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/localdev-config.yaml b/sample/configs/localdev-config.yaml index c2991cdf3b6..9c036f3fdb9 100644 --- a/sample/configs/localdev-config.yaml +++ b/sample/configs/localdev-config.yaml @@ -24,3 +24,5 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 diff --git a/sample/configs/prebid-config-db.yaml b/sample/configs/prebid-config-db.yaml index 418b6d365df..666ffb69a55 100644 --- a/sample/configs/prebid-config-db.yaml +++ b/sample/configs/prebid-config-db.yaml @@ -43,6 +43,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/prebid-config-s3.yaml b/sample/configs/prebid-config-s3.yaml index 2a01d4f4d43..09620ceec87 100644 --- a/sample/configs/prebid-config-s3.yaml +++ b/sample/configs/prebid-config-s3.yaml @@ -51,6 +51,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: diff --git a/sample/configs/prebid-config-with-51d-dd.yaml b/sample/configs/prebid-config-with-51d-dd.yaml index 80f73b1a461..1e29ec1d997 100644 --- a/sample/configs/prebid-config-with-51d-dd.yaml +++ b/sample/configs/prebid-config-with-51d-dd.yaml @@ -33,6 +33,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/prebid-config-with-module.yaml b/sample/configs/prebid-config-with-module.yaml index 3619e1cb9fd..959a4cef2af 100644 --- a/sample/configs/prebid-config-with-module.yaml +++ b/sample/configs/prebid-config-with-module.yaml @@ -33,6 +33,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/prebid-config-with-optable.yaml b/sample/configs/prebid-config-with-optable.yaml index 4070cc0aed5..ef2d59037bf 100644 --- a/sample/configs/prebid-config-with-optable.yaml +++ b/sample/configs/prebid-config-with-optable.yaml @@ -39,6 +39,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/prebid-config-with-wurfl.yaml b/sample/configs/prebid-config-with-wurfl.yaml index 55ccd4ffded..9a82d459195 100644 --- a/sample/configs/prebid-config-with-wurfl.yaml +++ b/sample/configs/prebid-config-with-wurfl.yaml @@ -34,6 +34,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/sample/configs/prebid-config.yaml b/sample/configs/prebid-config.yaml index a964c645cc7..f969f25bd60 100644 --- a/sample/configs/prebid-config.yaml +++ b/sample/configs/prebid-config.yaml @@ -34,6 +34,8 @@ gdpr: cache-dir: /var/tmp/vendor2 v3: cache-dir: /var/tmp/vendor3 + live: + startup-cache-dir: /var/tmp/vendor3 admin-endpoints: logging-changelevel: enabled: true diff --git a/src/main/docker/application.yaml b/src/main/docker/application.yaml index 9765fc7ed39..a5e51ed5b4c 100644 --- a/src/main/docker/application.yaml +++ b/src/main/docker/application.yaml @@ -4,3 +4,5 @@ gdpr: cache-dir: /app/prebid-server/data/vendorlist-v2 v3: cache-dir: /app/prebid-server/data/vendorlist-v3 + live: + startup-cache-dir: /app/prebid-server/data/vendorlist-v3 diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java index 6e288f92ca8..8ab45c24b7a 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java @@ -77,7 +77,8 @@ private void createAndCheckWritePermissionsForCacheDir(String cacheDir) { private Map readFileSystemCache(String cacheDir) { return fileSystem.readDirBlocking(cacheDir).stream() .filter(filepath -> filepath.endsWith(JSON_SUFFIX)) - .collect(Collectors.toMap(VendorListFileStore::parseCachedFileVersion, + .collect(Collectors.toMap( + VendorListFileStore::parseCachedFileVersion, filename -> fileSystem.readFileBlocking(filename).toString())); } diff --git a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java index 9ba7da3d839..579dbda2fa6 100644 --- a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java @@ -148,9 +148,9 @@ VendorListServiceConfigurationProperties vendorListServiceV3Properties() { @Bean LiveVendorListService liveVendorListService( - VendorListServiceConfigurationProperties vendorListServiceV3Properties, - @Value("${gdpr.vendorlist.live-gvl-url}") String liveGvlUrl, - @Value("${gdpr.vendorlist.live-gvl-refresh-period-ms}") long refreshPeriodMs, + @Value("${gdpr.vendorlist.live.startup-cache-dir}") String startupCacheDir, + @Value("${gdpr.vendorlist.live.url}") String liveGvlUrl, + @Value("${gdpr.vendorlist.live.refresh-period-ms}") long refreshPeriodMs, @Value("${gdpr.vendorlist.default-timeout-ms}") int defaultTimeoutMs, Vertx vertx, HttpClient httpClient, @@ -160,7 +160,7 @@ LiveVendorListService liveVendorListService( Clock clock) { return new LiveVendorListService( - vendorListServiceV3Properties.getCacheDir(), + startupCacheDir, liveGvlUrl, refreshPeriodMs, defaultTimeoutMs, diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 6743b6fd9d8..ff8880feade 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -208,8 +208,6 @@ gdpr: eea-countries: at,bg,be,cy,cz,dk,ee,fi,fr,de,gr,hu,ie,it,lv,lt,lu,mt,nl,pl,pt,ro,sk,si,es,se,gb,is,no,li,ai,aw,pt,bm,aq,io,vg,ic,ky,fk,re,mw,gp,gf,yt,pf,tf,gl,pt,ms,an,bq,cw,sx,nc,pn,sh,pm,gs,tc,uk,wf vendorlist: default-timeout-ms: 2000 - live-gvl-refresh-period-ms: 86400000 - live-gvl-url: https://vendor-list.consensu.org/v3/vendor-list.json v2: http-endpoint-template: https://vendor-list.consensu.org/v2/archives/vendor-list-v{VERSION}.json refresh-missing-list-period-ms: 3600000 @@ -230,6 +228,9 @@ gdpr: max-delay-millis: 120000 factor: 1.1 jitter: 0.2 + live: + url: https://vendor-list.consensu.org/v3/vendor-list.json + refresh-period-ms: 86400000 purposes: p1: enforce-purpose: full diff --git a/src/test/resources/org/prebid/server/it/test-application.properties b/src/test/resources/org/prebid/server/it/test-application.properties index efabbcfeb1b..10d0c51a2c8 100644 --- a/src/test/resources/org/prebid/server/it/test-application.properties +++ b/src/test/resources/org/prebid/server/it/test-application.properties @@ -769,4 +769,5 @@ gdpr.host-vendor-id=1 gdpr.default-value=1 gdpr.vendorlist.v2.cache-dir=src/test/resources/org/prebid/server/it/gdpr-vendorlist2 gdpr.vendorlist.v3.cache-dir=src/test/resources/org/prebid/server/it/gdpr-vendorlist3 +gdpr.vendorlist.live.startup-cache-dir=src/test/resources/org/prebid/server/it/gdpr-vendorlist3 ccpa.enforce=false From 0a1e6017061aba89bef16d86de1ed3410398c1e7 Mon Sep 17 00:00:00 2001 From: osulzhenko Date: Thu, 23 Jul 2026 11:44:12 +0300 Subject: [PATCH 09/11] update functional tests --- .../functional/tests/privacy/gdpr/GdprAuctionSpec.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy index 8b5996089b3..b3ee78dab78 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy @@ -66,7 +66,7 @@ class GdprAuctionSpec extends PrivacyBaseSpec { private static final String LIVE_GVL_FILE_NAME = "live-gvl-vendor-list.json" private static final VENDOR_LIST_VERSION = PBSUtils.getRandomNumber(2, 4094) private static final Map REFRESH_LIVE_VENDOR_LIST_CONFIG = GENERAL_PRIVACY_CONFIG + - ["gdpr.vendorlist.live-gvl-url": "$networkServiceContainer.rootUri/v3/$LIVE_GVL_FILE_NAME".toString()] + ["gdpr.vendorlist.live.url": "$networkServiceContainer.rootUri/v3/$LIVE_GVL_FILE_NAME".toString()] private static final VendorList liveVendorListResponse = new VendorList(networkServiceContainer, LIVE_GVL_FILE_NAME) def setup() { @@ -1315,7 +1315,7 @@ class GdprAuctionSpec extends PrivacyBaseSpec { downloadLiveGvtList(VENDOR_LIST_VERSION, olderVendorList, Times.once()) and: "PBS with a high-frequency live GVL refresh config" - def config = REFRESH_LIVE_VENDOR_LIST_CONFIG + ['gdpr.vendorlist.live-gvl-refresh-period-ms': '1000'] + def config = REFRESH_LIVE_VENDOR_LIST_CONFIG + ['gdpr.vendorlist.live.refresh-period-ms': '1000'] def pbsWithLiveGvlSetup = pbsServiceFactory.getService(config) and: "Flash metrics" From 1e14ac0e833800e4936bc4078bbd3cebc6a50f63 Mon Sep 17 00:00:00 2001 From: Viktor Kryshtal Date: Thu, 23 Jul 2026 14:46:15 +0300 Subject: [PATCH 10/11] Fix comments --- docs/config-app.md | 5 +++-- docs/metrics.md | 2 +- .../java/org/prebid/server/metric/Metrics.java | 8 ++++---- .../org/prebid/server/metric/TcfMetrics.java | 14 +++++++------- .../server/privacy/gdpr/Tcf2Service.java | 2 +- .../gdpr/vendorlist/LiveVendorListService.java | 4 ++-- .../gdpr/vendorlist/VendorListFileStore.java | 6 ++---- .../gdpr/vendorlist/VendorListWrapper.java | 11 +++++------ .../metrics-config/prometheus-labels.yaml | 4 ++-- .../tests/privacy/gdpr/GdprAuctionSpec.groovy | 2 +- .../org/prebid/server/metric/MetricsTest.java | 12 ++++++------ .../server/privacy/gdpr/Tcf2ServiceTest.java | 2 +- .../vendorlist/LiveVendorListServiceTest.java | 18 +++++++++--------- 13 files changed, 44 insertions(+), 46 deletions(-) diff --git a/docs/config-app.md b/docs/config-app.md index 237e514cc52..f60718695df 100644 --- a/docs/config-app.md +++ b/docs/config-app.md @@ -458,8 +458,9 @@ If not defined in config all other Health Checkers would be disabled and endpoin - `gdpr.special-features.sfN.vendor-exceptions[]` - bidder names that will be treated opposite to `sfN.enforce` value. - `gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow. - `gdpr.vendorlist.default-timeout-ms` - default operation timeout for obtaining new vendor list. -- `gdpr.vendorlist.live-gvl-url` - URL of the latest TCF GVL used to detect vendors with a past `deletedDate`. Default `https://vendor-list.consensu.org/v3/vendor-list.json`. -- `gdpr.vendorlist.live-gvl-refresh-period-ms` - how often to refresh the live GVL deleted-vendor set, in milliseconds. Default `86400000` (24 hours). +- `gdpr.vendorlist.live.url` - URL of the latest TCF GVL used to detect vendors with a past `deletedDate`. Default `https://vendor-list.consensu.org/v3/vendor-list.json`. +- `gdpr.vendorlist.live.refresh-period-ms` - how often to refresh the live GVL deleted-vendor set, in milliseconds. Default `86400000` (24 hours). +- `gdpr.vendorlist.live.startup-cache-dir` - directory for local storage vendor list cache. At startup server will attempt to read latest GVL from this location. - `gdpr.vendorlist.v2.http-endpoint-template` - template string for vendor list url version 2. - `gdpr.vendorlist.v2.refresh-missing-list-period-ms` - time to wait between attempts to fetch vendor list version that previously was reported to be missing by origin. Default `3600000` (one hour). - `gdpr.vendorlist.v2.fallback-vendor-list-path` - location on the file system of the fallback vendor list that will be used in place of missing vendor list versions. Optional. diff --git a/docs/metrics.md b/docs/metrics.md index e5844413032..4dad9b6effb 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -131,7 +131,7 @@ Following metrics are collected and submitted if account is configured with `det - `privacy.tcf.(v1,v2).in-geo` - number of requests received from TCF-concerned geo region with consent string of particular version - `privacy.tcf.(v1,v2).out-geo` - number of requests received outside of TCF-concerned geo region with consent string of particular version - `privacy.tcf.(v1,v2).vendorlist.(missing|ok|err|fallback)` - number of processed vendor lists of particular version -- `privacy.tcf.vendorlist.latest.(ok|err)` - number of successful or failed refreshes of the live GVL used for deleted-vendor detection +- `privacy.tcf.vendorlist.live.(ok|err)` - number of successful or failed refreshes of the live GVL used for deleted-vendor detection - `privacy.usp.specified` - number of requests with a valid US Privacy string (CCPA) - `privacy.usp.opt-out` - number of requests that required privacy enforcement according to CCPA rules - `privacy.lmt` - number of requests that required privacy enforcement according to LMT flag diff --git a/src/main/java/org/prebid/server/metric/Metrics.java b/src/main/java/org/prebid/server/metric/Metrics.java index 3fa56cbd856..c6e3394c807 100644 --- a/src/main/java/org/prebid/server/metric/Metrics.java +++ b/src/main/java/org/prebid/server/metric/Metrics.java @@ -554,12 +554,12 @@ public void updatePrivacyTcfVendorListFallbackMetric(int version) { updatePrivacyTcfVendorListMetric(version, MetricName.fallback); } - public void updatePrivacyTcfVendorListLatestOkMetric() { - privacy().tcf().vendorListLatest().incCounter(MetricName.ok); + public void updatePrivacyTcfLiveVendorListOkMetric() { + privacy().tcf().liveVendorList().incCounter(MetricName.ok); } - public void updatePrivacyTcfVendorListLatestErrorMetric() { - privacy().tcf().vendorListLatest().incCounter(MetricName.err); + public void updatePrivacyTcfLiveVendorListErrorMetric() { + privacy().tcf().liveVendorList().incCounter(MetricName.err); } private void updatePrivacyTcfVendorListMetric(int version, MetricName metricName) { diff --git a/src/main/java/org/prebid/server/metric/TcfMetrics.java b/src/main/java/org/prebid/server/metric/TcfMetrics.java index ba5d8ed86f3..72c9a5cd4f6 100644 --- a/src/main/java/org/prebid/server/metric/TcfMetrics.java +++ b/src/main/java/org/prebid/server/metric/TcfMetrics.java @@ -16,7 +16,7 @@ class TcfMetrics extends UpdatableMetrics { private final TcfVersionMetrics tcfVersion1Metrics; private final TcfVersionMetrics tcfVersion2Metrics; - private final VendorListLatestMetrics vendorListLatestMetrics; + private final LiveVendorListMetrics liveVendorListMetrics; TcfMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { super( @@ -26,7 +26,7 @@ class TcfMetrics extends UpdatableMetrics { tcfVersion1Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v1"); tcfVersion2Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v2"); - vendorListLatestMetrics = new VendorListLatestMetrics(metricRegistry, counterType, createTcfPrefix(prefix)); + liveVendorListMetrics = new LiveVendorListMetrics(metricRegistry, counterType, createTcfPrefix(prefix)); } TcfVersionMetrics fromVersion(int version) { @@ -37,8 +37,8 @@ TcfVersionMetrics fromVersion(int version) { }; } - VendorListLatestMetrics vendorListLatest() { - return vendorListLatestMetrics; + LiveVendorListMetrics liveVendorList() { + return liveVendorListMetrics; } private static String createTcfPrefix(String prefix) { @@ -94,9 +94,9 @@ private static Function nameCreator(String prefix) { } } - static class VendorListLatestMetrics extends UpdatableMetrics { + static class LiveVendorListMetrics extends UpdatableMetrics { - VendorListLatestMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { + LiveVendorListMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) { super( metricRegistry, counterType, @@ -104,7 +104,7 @@ static class VendorListLatestMetrics extends UpdatableMetrics { } private static String createLatestPrefix(String prefix) { - return prefix + ".vendorlist.latest"; + return prefix + ".vendorlist.live"; } private static Function nameCreator(String prefix) { diff --git a/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java b/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java index 0d10b45368a..6010ad950e2 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/Tcf2Service.java @@ -115,7 +115,7 @@ private Future> permissionsForInternal(Collection processDowngradedSupportedPurposeStrategies( tcfConsent, - wrapWithGVL(vendorPermissionsByType, VendorListWrapper.empty()), + wrapWithGVL(vendorPermissionsByType, VendorListWrapper.EMPTY), mergedPurposes, mergedPurposeOneTreatmentInterpretation)) .map(ignored -> enforcePurpose4IfRequired(mergedPurposes, vendorPermissionsByType)) diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java index 49a7e18d115..49fb8e4e6cb 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java @@ -115,13 +115,13 @@ Set extractDeletedVendorIds(VendorList vendorList) { private Void updateDeletedVendorIds(Set ids) { deletedVendorIds = ids; - metrics.updatePrivacyTcfVendorListLatestOkMetric(); + metrics.updatePrivacyTcfLiveVendorListOkMetric(); return null; } private Void handleError(Throwable exception) { logger.warn("Error occurred while fetching live GVL", exception); - metrics.updatePrivacyTcfVendorListLatestErrorMetric(); + metrics.updatePrivacyTcfLiveVendorListErrorMetric(); return null; } } diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java index 8ab45c24b7a..e02669c8e26 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java @@ -1,6 +1,5 @@ package org.prebid.server.privacy.gdpr.vendorlist; -import com.github.benmanes.caffeine.cache.Caffeine; import io.vertx.core.Future; import io.vertx.core.buffer.Buffer; import io.vertx.core.file.FileProps; @@ -23,6 +22,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; public class VendorListFileStore { @@ -49,9 +49,7 @@ Map> createCacheFromDisk(String cacheDir) { createAndCheckWritePermissionsForCacheDir(cacheDir); final Map versionToFileContent = readFileSystemCache(cacheDir); - final Map> cache = Caffeine.newBuilder() - .>build() - .asMap(); + final Map> cache = new ConcurrentHashMap<>(); for (Map.Entry versionAndFileContent : versionToFileContent.entrySet()) { final VendorList vendorList = VendorListUtil.parseVendorList(versionAndFileContent.getValue(), mapper); diff --git a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java index dd96058b29c..64ce79352b5 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListWrapper.java @@ -1,6 +1,6 @@ package org.prebid.server.privacy.gdpr.vendorlist; -import lombok.RequiredArgsConstructor; +import lombok.AllArgsConstructor; import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor; import java.time.Instant; @@ -10,9 +10,12 @@ // The purpose of this wrapper class is to avoid filtering the whole vendor list for deleted vendors // TODO: move filtering logic to vendor list service -@RequiredArgsConstructor(staticName = "of") +@AllArgsConstructor(staticName = "of") public class VendorListWrapper { + public static final VendorListWrapper EMPTY = VendorListWrapper.of( + Collections.emptyMap(), Collections.emptySet(), Instant.EPOCH); + private final Map vendorList; private final Set deletedVendorIds; private final Instant timestamp; @@ -27,8 +30,4 @@ private boolean isRetained(Integer id, Vendor vendor) { && !VendorListUtil.vendorIsDeletedAt(vendor, timestamp) && !deletedVendorIds.contains(id); } - - public static VendorListWrapper empty() { - return VendorListWrapper.of(Collections.emptyMap(), Collections.emptySet(), null); - } } diff --git a/src/main/resources/metrics-config/prometheus-labels.yaml b/src/main/resources/metrics-config/prometheus-labels.yaml index 6941b1d7cb0..162be432f9d 100644 --- a/src/main/resources/metrics-config/prometheus-labels.yaml +++ b/src/main/resources/metrics-config/prometheus-labels.yaml @@ -85,8 +85,8 @@ mappers: labels: tcf: ${0} status: ${1} - - match: privacy.tcf.vendorlist.latest.* - name: privacy.tcf.vendorlist.latest + - match: privacy.tcf.vendorlist.live.* + name: privacy.tcf.vendorlist.live labels: status: ${0} - match: privacy.tcf.*.* diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy index b3ee78dab78..03c249278a4 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/gdpr/GdprAuctionSpec.groovy @@ -1326,7 +1326,7 @@ class GdprAuctionSpec extends PrivacyBaseSpec { deletedDate = ZonedDateTime.now().plusYears(1) } downloadLiveGvtList(VENDOR_LIST_VERSION + 1, newerVendorList, Times.once()) - pbsWithLiveGvlSetup.isContainMetricByValue('privacy.tcf.vendorlist.latest.ok') + pbsWithLiveGvlSetup.isContainMetricByValue('privacy.tcf.vendorlist.live.ok') and: "GVL list is warmed up for PBS" warmupGvtList(pbsWithLiveGvlSetup, VENDOR_LIST_VERSION - 1) diff --git a/src/test/java/org/prebid/server/metric/MetricsTest.java b/src/test/java/org/prebid/server/metric/MetricsTest.java index 1df98cedd68..eb0af781a7d 100644 --- a/src/test/java/org/prebid/server/metric/MetricsTest.java +++ b/src/test/java/org/prebid/server/metric/MetricsTest.java @@ -1005,21 +1005,21 @@ public void updatePrivacyTcfVendorListFallbackMetricShouldIncrementMetric() { } @Test - public void updatePrivacyTcfVendorListLatestOkMetricShouldIncrementMetric() { + public void updatePrivacyTcfLiveVendorListOkMetricShouldIncrementMetric() { // when - metrics.updatePrivacyTcfVendorListLatestOkMetric(); + metrics.updatePrivacyTcfLiveVendorListOkMetric(); // then - assertThat(metricRegistry.counter("privacy.tcf.vendorlist.latest.ok").getCount()).isOne(); + assertThat(metricRegistry.counter("privacy.tcf.vendorlist.live.ok").getCount()).isOne(); } @Test - public void updatePrivacyTcfVendorListLatestErrorMetricShouldIncrementMetric() { + public void updatePrivacyTcfLiveVendorListErrorMetricShouldIncrementMetric() { // when - metrics.updatePrivacyTcfVendorListLatestErrorMetric(); + metrics.updatePrivacyTcfLiveVendorListErrorMetric(); // then - assertThat(metricRegistry.counter("privacy.tcf.vendorlist.latest.err").getCount()).isOne(); + assertThat(metricRegistry.counter("privacy.tcf.vendorlist.live.err").getCount()).isOne(); } @Test diff --git a/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java index 0e7ac5fd12d..bc99140d6db 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/Tcf2ServiceTest.java @@ -103,7 +103,7 @@ public class Tcf2ServiceTest extends VertxTest { @BeforeEach public void setUp() { - given(vendorListService.forConsent(any())).willReturn(Future.succeededFuture(VendorListWrapper.empty())); + given(vendorListService.forConsent(any())).willReturn(Future.succeededFuture(VendorListWrapper.EMPTY)); given(purposeStrategyOne.getPurpose()).willReturn(ONE); given(purposeStrategyTwo.getPurpose()).willReturn(TWO); diff --git a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java index bcb6f4cccd3..863ac4a978b 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListServiceTest.java @@ -123,8 +123,8 @@ public void refreshShouldUpdateDeletedVendorIdsAndIncrementOkMetric() throws Jso // then assertThat(target.getDeletedVendorIds()).containsExactly(1); - verify(metrics).updatePrivacyTcfVendorListLatestOkMetric(); - verify(metrics, never()).updatePrivacyTcfVendorListLatestErrorMetric(); + verify(metrics).updatePrivacyTcfLiveVendorListOkMetric(); + verify(metrics, never()).updatePrivacyTcfLiveVendorListErrorMetric(); } @Test @@ -158,8 +158,8 @@ public void refreshShouldIncrementErrorMetricOnHttpFailure() { // then assertThat(target.getDeletedVendorIds()).isEmpty(); - verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); - verify(metrics, never()).updatePrivacyTcfVendorListLatestOkMetric(); + verify(metrics).updatePrivacyTcfLiveVendorListErrorMetric(); + verify(metrics, never()).updatePrivacyTcfLiveVendorListOkMetric(); } @Test @@ -172,7 +172,7 @@ public void refreshShouldIncrementErrorMetricOnNonOkStatus() { // then assertThat(target.getDeletedVendorIds()).isEmpty(); - verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + verify(metrics).updatePrivacyTcfLiveVendorListErrorMetric(); } @Test @@ -185,7 +185,7 @@ public void refreshShouldIncrementErrorMetricOnInvalidJson() { // then assertThat(target.getDeletedVendorIds()).isEmpty(); - verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + verify(metrics).updatePrivacyTcfLiveVendorListErrorMetric(); } @Test @@ -201,7 +201,7 @@ public void refreshShouldIncrementErrorMetricOnInvalidVendorList() throws JsonPr // then assertThat(target.getDeletedVendorIds()).isEmpty(); - verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + verify(metrics).updatePrivacyTcfLiveVendorListErrorMetric(); } @Test @@ -257,8 +257,8 @@ public void refreshShouldKeepLastGoodSetOnFailureAfterSuccessfulFetch() throws J // then assertThat(target.getDeletedVendorIds()).containsExactly(1); - verify(metrics).updatePrivacyTcfVendorListLatestOkMetric(); - verify(metrics).updatePrivacyTcfVendorListLatestErrorMetric(); + verify(metrics).updatePrivacyTcfLiveVendorListOkMetric(); + verify(metrics).updatePrivacyTcfLiveVendorListErrorMetric(); } private void givenHttpClientReturnsResponse(int statusCode, String response) { From 5c5ec8fc5252c48bb0b71dcaf31d4996bc8bcf56 Mon Sep 17 00:00:00 2001 From: osulzhenko Date: Thu, 23 Jul 2026 17:20:18 +0300 Subject: [PATCH 11/11] fix ft --- .../BidderFieldDisplayBehaviorSpec.groovy | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/test/groovy/org/prebid/server/functional/tests/BidderFieldDisplayBehaviorSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/BidderFieldDisplayBehaviorSpec.groovy index bdf0f633626..6acd01d09cd 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/BidderFieldDisplayBehaviorSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/BidderFieldDisplayBehaviorSpec.groovy @@ -21,6 +21,7 @@ import org.prebid.server.functional.model.request.auction.BidderConfigOrtb import org.prebid.server.functional.model.request.auction.ConsentedProvidersSettings import org.prebid.server.functional.model.request.auction.Device import org.prebid.server.functional.model.request.auction.DeviceExt +import org.prebid.server.functional.model.request.auction.DevicePrebid import org.prebid.server.functional.model.request.auction.Dooh import org.prebid.server.functional.model.request.auction.DoohExt import org.prebid.server.functional.model.request.auction.EidPermission @@ -28,9 +29,7 @@ import org.prebid.server.functional.model.request.auction.ExtPrebidBidderConfig import org.prebid.server.functional.model.request.auction.ExtRequestPrebidData import org.prebid.server.functional.model.request.auction.Interstitial import org.prebid.server.functional.model.request.auction.MultiBid -import org.prebid.server.functional.model.request.auction.PaaFormat import org.prebid.server.functional.model.request.auction.PrebidCache -import org.prebid.server.functional.model.request.auction.DevicePrebid import org.prebid.server.functional.model.request.auction.PrebidCurrency import org.prebid.server.functional.model.request.auction.Renderer import org.prebid.server.functional.model.request.auction.RendererData @@ -47,9 +46,9 @@ import org.prebid.server.functional.util.PBSUtils import static org.prebid.server.functional.model.Currency.USD import static org.prebid.server.functional.model.bidder.BidderName.ALIAS +import static org.prebid.server.functional.model.bidder.BidderName.GENERIC import static org.prebid.server.functional.model.bidder.BidderName.OPENX import static org.prebid.server.functional.model.bidder.BidderName.RUBICON -import static org.prebid.server.functional.model.bidder.BidderName.GENERIC import static org.prebid.server.functional.model.mock.services.currencyconversion.CurrencyConversionRatesResponse.defaultConversionRates import static org.prebid.server.functional.model.request.auction.AdjustmentType.MULTIPLIER import static org.prebid.server.functional.model.request.auction.BidAdjustmentMediaType.BANNER @@ -323,20 +322,6 @@ class BidderFieldDisplayBehaviorSpec extends BaseSpec { } } - def "PBS auction should pass ext.prebid.paaformat to bidder request when paaformat specified"() { - given: "Default bid request" - def bidRequest = BidRequest.defaultBidRequest.tap { - ext.prebid.paaFormat = PaaFormat.ORIGINAL - } - - when: "PBS processes auction request" - defaultPbsService.sendAuctionRequest(bidRequest) - - then: "Bidder request should contain paaFormat value same in request" - def bidderRequest = bidder.getBidderRequest(bidRequest.id) - assert bidderRequest.ext.prebid.paaFormat == bidRequest.ext.prebid.paaFormat - } - def "PBS auction shouldn't pass ext.prebid.kvps to bidder request when kvps specified"() { given: "Default bid request with kvps" def bidRequest = BidRequest.defaultBidRequest.tap {