diff --git a/docs/metrics.md b/docs/metrics.md index 504cd50f5db..85df92bc269 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -37,6 +37,7 @@ where `[DATASOURCE]` is a data source name, `DEFAULT_DS` by defaul. ## General auction metrics - `app_requests` - number of requests received from applications +- `debug_requests` - number of requests received (when debug mode is enabled) - `no_cookie_requests` - number of requests without `uids` cookie or with one that didn't contain at least one live UID - `request_time` - timer tracking how long did it take for Prebid Server to serve a request - `imps_requested` - number if impressions requested @@ -89,6 +90,7 @@ Following metrics are collected and submitted if account is configured with `bas Following metrics are collected and submitted if account is configured with `detailed` verbosity: - `account..requests.type.(openrtb2-web,openrtb-app,amp,legacy)` - number of requests received from account with `` broken down by type of incoming request +- `account..debug_requests` - number of requests received from account with `` broken down by type of incoming request (when debug mode is enabled) - `account..requests.rejected` - number of rejected requests caused by incorrect `accountId` - `account..adapter..request_time` - timer tracking how long did it take to make a request to `` when incoming request was from `` - `account..adapter..bids_received` - number of bids received from `` when incoming request was from `` diff --git a/src/main/java/org/prebid/server/auction/ExchangeService.java b/src/main/java/org/prebid/server/auction/ExchangeService.java index 7e37c48f219..6fff31c6459 100644 --- a/src/main/java/org/prebid/server/auction/ExchangeService.java +++ b/src/main/java/org/prebid/server/auction/ExchangeService.java @@ -249,6 +249,10 @@ private Future runAuction(AuctionContext receivedContext) { final Map bidderToMultiBid = bidderToMultiBids(bidRequest, debugWarnings); receivedContext.getBidRejectionTrackers().putAll(makeBidRejectionTrackers(bidRequest, aliases)); + final boolean debugEnabled = receivedContext.getDebugContext().isDebugEnabled(); + metrics.updateDebugRequestMetrics(debugEnabled); + metrics.updateAccountDebugRequestMetrics(account, debugEnabled); + return storedResponseProcessor.getStoredResponseResult(bidRequest.getImp(), timeout) .map(storedResponseResult -> populateStoredResponse(storedResponseResult, storedAuctionResponses)) .compose(storedResponseResult -> @@ -274,7 +278,7 @@ private Future runAuction(AuctionContext receivedContext) { bidRequest.getImp(), context.getBidRejectionTrackers())) .map(auctionParticipations -> dropZeroNonDealBids( - auctionParticipations, debugWarnings, context.getDebugContext().isDebugEnabled())) + auctionParticipations, debugWarnings, debugEnabled)) .map(auctionParticipations -> bidsAdjuster.validateAndAdjustBids(auctionParticipations, context, aliases)) .map(auctionParticipations -> updateResponsesMetrics(auctionParticipations, account, aliases)) @@ -285,7 +289,7 @@ private Future runAuction(AuctionContext receivedContext) { logger, bidResponse, context.getBidRequest(), - context.getDebugContext().isDebugEnabled())) + debugEnabled)) .compose(bidResponse -> bidResponsePostProcessor.postProcess( context.getHttpRequest(), uidsCookie, bidRequest, bidResponse, account)) .map(context::with)); diff --git a/src/main/java/org/prebid/server/metric/MetricName.java b/src/main/java/org/prebid/server/metric/MetricName.java index 7f27ea2212e..9ea3d15f0fc 100644 --- a/src/main/java/org/prebid/server/metric/MetricName.java +++ b/src/main/java/org/prebid/server/metric/MetricName.java @@ -25,6 +25,7 @@ public enum MetricName { // auction requests, + debug_requests, app_requests, no_cookie_requests, request_time, diff --git a/src/main/java/org/prebid/server/metric/Metrics.java b/src/main/java/org/prebid/server/metric/Metrics.java index c435e361fec..52964a9f8b0 100644 --- a/src/main/java/org/prebid/server/metric/Metrics.java +++ b/src/main/java/org/prebid/server/metric/Metrics.java @@ -167,6 +167,12 @@ HooksMetrics hooks() { return hooksMetrics; } + public void updateDebugRequestMetrics(boolean debugEnabled) { + if (debugEnabled) { + incCounter(MetricName.debug_requests); + } + } + public void updateAppAndNoCookieAndImpsRequestedMetrics(boolean isApp, boolean liveUidsPresent, int numImps) { if (isApp) { incCounter(MetricName.app_requests); @@ -235,12 +241,20 @@ public void updateAccountRequestMetrics(Account account, MetricName requestType) final AccountMetrics accountMetrics = forAccount(account.getId()); accountMetrics.incCounter(MetricName.requests); + if (verbosityLevel.isAtLeast(AccountMetricsVerbosityLevel.detailed)) { accountMetrics.requestType(requestType).incCounter(MetricName.requests); } } } + public void updateAccountDebugRequestMetrics(Account account, boolean debugEnabled) { + final AccountMetricsVerbosityLevel verbosityLevel = accountMetricsVerbosityResolver.forAccount(account); + if (verbosityLevel.isAtLeast(AccountMetricsVerbosityLevel.detailed) && debugEnabled) { + forAccount(account.getId()).incCounter(MetricName.debug_requests); + } + } + public void updateAccountRequestRejectedByInvalidAccountMetrics(String accountId) { updateAccountRequestsMetrics(accountId, MetricName.rejected_by_invalid_account); } diff --git a/src/test/groovy/org/prebid/server/functional/tests/DebugSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/DebugSpec.groovy index 715c20c9dcb..ea169ad00ee 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/DebugSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/DebugSpec.groovy @@ -3,18 +3,24 @@ package org.prebid.server.functional.tests import org.apache.commons.lang3.StringUtils import org.prebid.server.functional.model.config.AccountAuctionConfig import org.prebid.server.functional.model.config.AccountConfig +import org.prebid.server.functional.model.config.AccountMetricsConfig import org.prebid.server.functional.model.db.Account import org.prebid.server.functional.model.db.StoredRequest import org.prebid.server.functional.model.db.StoredResponse import org.prebid.server.functional.model.request.amp.AmpRequest import org.prebid.server.functional.model.request.auction.BidRequest +import org.prebid.server.functional.model.request.auction.Site import org.prebid.server.functional.model.request.auction.StoredBidResponse import org.prebid.server.functional.model.response.auction.BidResponse import org.prebid.server.functional.model.response.auction.ErrorType +import org.prebid.server.functional.service.PrebidServerException import org.prebid.server.functional.util.PBSUtils import spock.lang.PendingFeature import static org.prebid.server.functional.model.bidder.BidderName.GENERIC +import static org.prebid.server.functional.model.config.AccountMetricsVerbosityLevel.BASIC +import static org.prebid.server.functional.model.config.AccountMetricsVerbosityLevel.DETAILED +import static org.prebid.server.functional.model.config.AccountMetricsVerbosityLevel.NONE import static org.prebid.server.functional.model.request.auction.DebugCondition.DISABLED import static org.prebid.server.functional.model.request.auction.DebugCondition.ENABLED import static org.prebid.server.functional.model.response.auction.BidderCallType.STORED_BID_RESPONSE @@ -22,19 +28,33 @@ import static org.prebid.server.functional.model.response.auction.BidderCallType class DebugSpec extends BaseSpec { private static final String overrideToken = PBSUtils.randomString + private static final String ACCOUNT_METRICS_PREFIX_NAME = "account" + private static final String DEBUG_REQUESTS_METRIC = "debug_requests" + private static final String ACCOUNT_DEBUG_REQUESTS_METRIC = "account.%s.debug_requests" + private static final String REQUEST_OK_WEB_METRICS = "requests.ok.openrtb2-web" - def "PBS should return debug information when debug flag is #debug and test flag is #test"() { + def "PBS should return debug information and emit metrics when debug flag is #debug and test flag is #test"() { given: "Default BidRequest with test flag" def bidRequest = BidRequest.defaultBidRequest bidRequest.ext.prebid.debug = debug bidRequest.test = test + and: "Flash metrics" + flushMetrics(defaultPbsService) + when: "PBS processes auction request" def response = defaultPbsService.sendAuctionRequest(bidRequest) then: "Response should contain ext.debug" assert response.ext?.debug + and: "Debug metrics should be incremented" + def metricsRequest = defaultPbsService.sendCollectedMetricsRequest() + assert metricsRequest[DEBUG_REQUESTS_METRIC] == 1 + + and: "Account debug metrics shouldn't be incremented" + assert !metricsRequest.keySet().contains(ACCOUNT_METRICS_PREFIX_NAME) + where: debug | test ENABLED | null @@ -48,12 +68,23 @@ class DebugSpec extends BaseSpec { bidRequest.ext.prebid.debug = test bidRequest.test = test + and: "Flash metrics" + flushMetrics(defaultPbsService) + when: "PBS processes auction request" def response = defaultPbsService.sendAuctionRequest(bidRequest) then: "Response shouldn't contain ext.debug" assert !response.ext?.debug + and: "Debug metrics shouldn't be populated" + def metricsRequest = defaultPbsService.sendCollectedMetricsRequest() + assert !metricsRequest[DEBUG_REQUESTS_METRIC] + assert !metricsRequest.keySet().contains(ACCOUNT_METRICS_PREFIX_NAME) + + and: "General metrics should be present" + assert metricsRequest[REQUEST_OK_WEB_METRICS] == 1 + where: debug | test DISABLED | null @@ -351,4 +382,157 @@ class DebugSpec extends BaseSpec { and: "Response should not contain ext.warnings" assert !response.ext?.warnings } + + def "PBS should return debug information and emit metrics when account debug enabled and verbosity detailed"() { + given: "Default basic generic bid request" + def bidRequest = BidRequest.defaultBidRequest + + and: "Account in the DB" + def accountConfig = new AccountConfig( + metrics: new AccountMetricsConfig(verbosityLevel: DETAILED), + auction: new AccountAuctionConfig(debugAllow: true)) + def account = new Account(uuid: bidRequest.site.publisher.id, config: accountConfig) + accountDao.save(account) + + and: "Flash metrics" + flushMetrics(defaultPbsService) + + when: "PBS processes auction request" + def response = defaultPbsService.sendAuctionRequest(bidRequest) + + then: "Response should contain ext.debug" + assert response.ext?.debug + + and: "Debug metrics should be incremented" + def metricsRequest = defaultPbsService.sendCollectedMetricsRequest() + assert metricsRequest[ACCOUNT_DEBUG_REQUESTS_METRIC.formatted(bidRequest.accountId)] == 1 + assert metricsRequest[DEBUG_REQUESTS_METRIC] == 1 + } + + def "PBS shouldn't return debug information and emit metrics when account debug enabled and verbosity #verbosityLevel"() { + given: "Default basic generic bid request" + def bidRequest = BidRequest.defaultBidRequest + + and: "Account in the DB" + def accountConfig = new AccountConfig( + metrics: new AccountMetricsConfig(verbosityLevel: verbosityLevel), + auction: new AccountAuctionConfig(debugAllow: true)) + def account = new Account(uuid: bidRequest.site.publisher.id, config: accountConfig) + accountDao.save(account) + + and: "Flash metrics" + flushMetrics(defaultPbsService) + + when: "PBS processes auction request" + def response = defaultPbsService.sendAuctionRequest(bidRequest) + + then: "Response should contain ext.debug" + assert response.ext?.debug + + and: "Account debug metrics shouldn't be incremented" + def metricsRequest = defaultPbsService.sendCollectedMetricsRequest() + assert !metricsRequest[ACCOUNT_DEBUG_REQUESTS_METRIC.formatted(bidRequest.accountId)] + + and: "Request debug metrics should be incremented" + assert metricsRequest[DEBUG_REQUESTS_METRIC] == 1 + + where: + verbosityLevel << [NONE, BASIC] + } + + def "PBS amp should return debug information and emit metrics when account debug enabled and verbosity detailed"() { + given: "Default AMP request" + def ampRequest = AmpRequest.defaultAmpRequest + + and: "Default stored request" + def ampStoredRequest = BidRequest.defaultStoredRequest + + and: "Account in the DB" + def accountConfig = new AccountConfig( + metrics: new AccountMetricsConfig(verbosityLevel: DETAILED), + auction: new AccountAuctionConfig(debugAllow: true)) + def account = new Account(uuid: ampRequest.account, config: accountConfig) + accountDao.save(account) + + and: "Flash metrics" + flushMetrics(defaultPbsService) + + and: "Save storedRequest into DB" + def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest) + storedRequestDao.save(storedRequest) + + when: "PBS processes amp request" + def response = defaultPbsService.sendAmpRequest(ampRequest) + + then: "Response should contain ext.debug" + assert response.ext?.debug + + and: "Debug metrics should be incremented" + def metricsRequest = defaultPbsService.sendCollectedMetricsRequest() + assert metricsRequest[ACCOUNT_DEBUG_REQUESTS_METRIC.formatted(ampRequest.account)] == 1 + assert metricsRequest[DEBUG_REQUESTS_METRIC] == 1 + } + + def "PBS amp should return debug information and emit metrics when account debug enabled and verbosity #verbosityLevel"() { + given: "Default AMP request" + def ampRequest = AmpRequest.defaultAmpRequest + + and: "Default stored request" + def ampStoredRequest = BidRequest.defaultStoredRequest + + and: "Account in the DB" + def accountConfig = new AccountConfig( + metrics: new AccountMetricsConfig(verbosityLevel: verbosityLevel), + auction: new AccountAuctionConfig(debugAllow: true)) + def account = new Account(uuid: ampRequest.account, config: accountConfig) + accountDao.save(account) + + and: "Flash metrics" + flushMetrics(defaultPbsService) + + and: "Save storedRequest into DB" + def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest) + storedRequestDao.save(storedRequest) + + when: "PBS processes amp request" + def response = defaultPbsService.sendAmpRequest(ampRequest) + + then: "Response should contain ext.debug" + assert response.ext?.debug + + and: "Account debug metrics shouldn't be incremented" + def metricsRequest = defaultPbsService.sendCollectedMetricsRequest() + assert !metricsRequest[ACCOUNT_DEBUG_REQUESTS_METRIC.formatted(ampRequest.account)] + + and: "Debug metrics should be incremented" + assert metricsRequest[DEBUG_REQUESTS_METRIC] == 1 + + where: + verbosityLevel << [NONE, BASIC] + } + + def "PBS shouldn't emit auction request metric when incoming request invalid"() { + given: "Default basic BidRequest" + def bidRequest = BidRequest.defaultBidRequest + bidRequest.site = new Site(id: null, name: PBSUtils.randomString, page: null) + bidRequest.ext.prebid.debug = ENABLED + + and: "Flash metrics" + flushMetrics(defaultPbsService) + + when: "PBS processes auction request" + defaultPbsService.sendAuctionRequest(bidRequest) + + then: "Request should fail with error" + def exception = thrown(PrebidServerException) + assert exception.responseBody.contains("request.site should include at least one of request.site.id or request.site.page") + + and: "Debug metrics shouldn't be populated" + def metricsRequest = defaultPbsService.sendCollectedMetricsRequest() + assert !metricsRequest[DEBUG_REQUESTS_METRIC] + assert !metricsRequest.keySet().contains(ACCOUNT_METRICS_PREFIX_NAME) + + and: "General metrics shouldn't be present" + assert !metricsRequest[REQUEST_OK_WEB_METRICS] + } } diff --git a/src/test/java/org/prebid/server/auction/ExchangeServiceTest.java b/src/test/java/org/prebid/server/auction/ExchangeServiceTest.java index e8d78c33b1a..c50377f0667 100644 --- a/src/test/java/org/prebid/server/auction/ExchangeServiceTest.java +++ b/src/test/java/org/prebid/server/auction/ExchangeServiceTest.java @@ -2979,6 +2979,8 @@ public void shouldIncrementCommonMetrics() { target.holdAuction(givenRequestContext(bidRequest)); // then + verify(metrics).updateDebugRequestMetrics(false); + verify(metrics).updateAccountDebugRequestMetrics(any(), eq(false)); verify(metrics).updateRequestBidderCardinalityMetric(1); verify(metrics).updateAccountRequestMetrics(any(), eq(MetricName.openrtb2web)); verify(metrics).updateAdapterRequestTypeAndNoCookieMetrics( diff --git a/src/test/java/org/prebid/server/metric/MetricsTest.java b/src/test/java/org/prebid/server/metric/MetricsTest.java index 104df273c3e..f4943895116 100644 --- a/src/test/java/org/prebid/server/metric/MetricsTest.java +++ b/src/test/java/org/prebid/server/metric/MetricsTest.java @@ -331,6 +331,16 @@ public void updateAppAndNoCookieAndImpsRequestedMetricsShouldIncrementMetrics() assertThat(metricRegistry.counter("imps_requested").getCount()).isEqualTo(4); } + @Test + public void updateDebugRequestsMetricsShouldIncrementMetrics() { + // when + metrics.updateDebugRequestMetrics(false); + metrics.updateDebugRequestMetrics(true); + + // then + assertThat(metricRegistry.counter("debug_requests").getCount()).isOne(); + } + @Test public void updateImpTypesMetricsByCountPerMediaTypeShouldIncrementMetrics() { // given @@ -427,6 +437,16 @@ public void updateAccountRequestMetricsShouldIncrementMetrics() { assertThat(metricRegistry.counter("account.accountId.requests.type.openrtb2-web").getCount()).isOne(); } + @Test + public void updateAccountDebugRequestMetricsShouldIncrementMetrics() { + // when + metrics.updateAccountDebugRequestMetrics(Account.empty(ACCOUNT_ID), false); + metrics.updateAccountDebugRequestMetrics(Account.empty(ACCOUNT_ID), true); + + // then + assertThat(metricRegistry.counter("account.accountId.debug_requests").getCount()).isOne(); + } + @Test public void updateAdapterRequestTypeAndNoCookieMetricsShouldUpdateMetricsAsExpected() { @@ -916,6 +936,8 @@ public void shouldNotUpdateAccountMetricsIfVerbosityIsNone() { given(accountMetricsVerbosityResolver.forAccount(any())).willReturn(AccountMetricsVerbosityLevel.none); // when + metrics.updateAccountDebugRequestMetrics(Account.empty(ACCOUNT_ID), false); + metrics.updateAccountDebugRequestMetrics(Account.empty(ACCOUNT_ID), true); metrics.updateAccountRequestMetrics(Account.empty(ACCOUNT_ID), MetricName.openrtb2web); metrics.updateAdapterResponseTime(RUBICON, Account.empty(ACCOUNT_ID), 500); metrics.updateAdapterRequestNobidMetrics(RUBICON, Account.empty(ACCOUNT_ID)); @@ -924,6 +946,7 @@ public void shouldNotUpdateAccountMetricsIfVerbosityIsNone() { // then assertThat(metricRegistry.counter("account.accountId.requests").getCount()).isZero(); + assertThat(metricRegistry.counter("account.accountId.debug_requests").getCount()).isZero(); assertThat(metricRegistry.counter("account.accountId.requests.type.openrtb2-web").getCount()).isZero(); assertThat(metricRegistry.timer("account.accountId.rubicon.request_time").getCount()).isZero(); assertThat(metricRegistry.counter("account.accountId.rubicon.requests.nobid").getCount()).isZero(); @@ -939,6 +962,8 @@ public void shouldUpdateAccountRequestsMetricOnlyIfVerbosityIsBasic() { // when metrics.updateAccountRequestMetrics(Account.empty(ACCOUNT_ID), MetricName.openrtb2web); + metrics.updateAccountDebugRequestMetrics(Account.empty(ACCOUNT_ID), false); + metrics.updateAccountDebugRequestMetrics(Account.empty(ACCOUNT_ID), true); metrics.updateAdapterResponseTime(RUBICON, Account.empty(ACCOUNT_ID), 500); metrics.updateAdapterRequestNobidMetrics(RUBICON, Account.empty(ACCOUNT_ID)); metrics.updateAdapterRequestGotbidsMetrics(RUBICON, Account.empty(ACCOUNT_ID)); @@ -946,6 +971,7 @@ public void shouldUpdateAccountRequestsMetricOnlyIfVerbosityIsBasic() { // then assertThat(metricRegistry.counter("account.accountId.requests").getCount()).isOne(); + assertThat(metricRegistry.counter("account.accountId.debug_requests").getCount()).isZero(); assertThat(metricRegistry.counter("account.accountId.requests.type.openrtb2-web").getCount()).isZero(); assertThat(metricRegistry.timer("account.accountId.rubicon.request_time").getCount()).isZero(); assertThat(metricRegistry.counter("account.accountId.rubicon.requests.nobid").getCount()).isZero();