Add Debug Metrics (#3548)

docs/metrics.md

@@ -37,6 +37,7 @@ where `[DATASOURCE]` is a data source name, `DEFAULT_DS` by defaul.
 
 ## General auction metrics
 - `app_requests` - number of requests received from applications
+- `debug_requests` - number of requests received (when debug mode is enabled)
 - `no_cookie_requests` - number of requests without `uids` cookie or with one that didn't contain at least one live UID
 - `request_time` - timer tracking how long did it take for Prebid Server to serve a request
 - `imps_requested` - number if impressions requested
@@ -89,6 +90,7 @@ Following metrics are collected and submitted if account is configured with `bas
 
 Following metrics are collected and submitted if account is configured with `detailed` verbosity:
 - `account.<account-id>.requests.type.(openrtb2-web,openrtb-app,amp,legacy)` - number of requests received from account with `<account-id>` broken down by type of incoming request
+- `account.<account-id>.debug_requests` - number of requests received from account with `<account-id>` broken down by type of incoming request (when debug mode is enabled)
 - `account.<account-id>.requests.rejected` - number of rejected requests caused by incorrect `accountId`
 - `account.<account-id>.adapter.<bidder-name>.request_time` - timer tracking how long did it take to make a request to `<bidder-name>` when incoming request was from `<account-id>` 
 - `account.<account-id>.adapter.<bidder-name>.bids_received` - number of bids received from `<bidder-name>` when incoming request was from `<account-id>`

src/main/java/org/prebid/server/auction/ExchangeService.java

@@ -249,6 +249,10 @@ private Future<AuctionContext> runAuction(AuctionContext receivedContext) {
         final Map<String, MultiBidConfig> bidderToMultiBid = bidderToMultiBids(bidRequest, debugWarnings);
         receivedContext.getBidRejectionTrackers().putAll(makeBidRejectionTrackers(bidRequest, aliases));
 
+        final boolean debugEnabled = receivedContext.getDebugContext().isDebugEnabled();
+        metrics.updateDebugRequestMetrics(debugEnabled);
+        metrics.updateAccountDebugRequestMetrics(account, debugEnabled);
+
         return storedResponseProcessor.getStoredResponseResult(bidRequest.getImp(), timeout)
                 .map(storedResponseResult -> populateStoredResponse(storedResponseResult, storedAuctionResponses))
                 .compose(storedResponseResult ->
@@ -274,7 +278,7 @@ private Future<AuctionContext> runAuction(AuctionContext receivedContext) {
                                 bidRequest.getImp(),
                                 context.getBidRejectionTrackers()))
                         .map(auctionParticipations -> dropZeroNonDealBids(
-                                auctionParticipations, debugWarnings, context.getDebugContext().isDebugEnabled()))
+                                auctionParticipations, debugWarnings, debugEnabled))
                         .map(auctionParticipations ->
                                 bidsAdjuster.validateAndAdjustBids(auctionParticipations, context, aliases))
                         .map(auctionParticipations -> updateResponsesMetrics(auctionParticipations, account, aliases))
@@ -285,7 +289,7 @@ private Future<AuctionContext> runAuction(AuctionContext receivedContext) {
                                 logger,
                                 bidResponse,
                                 context.getBidRequest(),
-                                context.getDebugContext().isDebugEnabled()))
+                                debugEnabled))
                         .compose(bidResponse -> bidResponsePostProcessor.postProcess(
                                 context.getHttpRequest(), uidsCookie, bidRequest, bidResponse, account))
                         .map(context::with));

src/main/java/org/prebid/server/metric/MetricName.java

@@ -25,6 +25,7 @@ public enum MetricName {
 
     // auction
     requests,
+    debug_requests,
     app_requests,
     no_cookie_requests,
     request_time,

src/main/java/org/prebid/server/metric/Metrics.java

@@ -167,6 +167,12 @@ HooksMetrics hooks() {
         return hooksMetrics;
     }
 
+    public void updateDebugRequestMetrics(boolean debugEnabled) {
+        if (debugEnabled) {
+            incCounter(MetricName.debug_requests);
+        }
+    }
+
     public void updateAppAndNoCookieAndImpsRequestedMetrics(boolean isApp, boolean liveUidsPresent, int numImps) {
         if (isApp) {
             incCounter(MetricName.app_requests);
@@ -235,12 +241,20 @@ public void updateAccountRequestMetrics(Account account, MetricName requestType)
             final AccountMetrics accountMetrics = forAccount(account.getId());
 
             accountMetrics.incCounter(MetricName.requests);
+
             if (verbosityLevel.isAtLeast(AccountMetricsVerbosityLevel.detailed)) {
                 accountMetrics.requestType(requestType).incCounter(MetricName.requests);
             }
         }
     }
 
+    public void updateAccountDebugRequestMetrics(Account account, boolean debugEnabled) {
+        final AccountMetricsVerbosityLevel verbosityLevel = accountMetricsVerbosityResolver.forAccount(account);
+        if (verbosityLevel.isAtLeast(AccountMetricsVerbosityLevel.detailed) && debugEnabled) {
+            forAccount(account.getId()).incCounter(MetricName.debug_requests);
+        }
+    }
+
     public void updateAccountRequestRejectedByInvalidAccountMetrics(String accountId) {
         updateAccountRequestsMetrics(accountId, MetricName.rejected_by_invalid_account);
     }

src/test/groovy/org/prebid/server/functional/tests/DebugSpec.groovy

@@ -3,38 +3,58 @@ package org.prebid.server.functional.tests
 import org.apache.commons.lang3.StringUtils
 import org.prebid.server.functional.model.config.AccountAuctionConfig
 import org.prebid.server.functional.model.config.AccountConfig
+import org.prebid.server.functional.model.config.AccountMetricsConfig
 import org.prebid.server.functional.model.db.Account
 import org.prebid.server.functional.model.db.StoredRequest
 import org.prebid.server.functional.model.db.StoredResponse
 import org.prebid.server.functional.model.request.amp.AmpRequest
 import org.prebid.server.functional.model.request.auction.BidRequest
+import org.prebid.server.functional.model.request.auction.Site
 import org.prebid.server.functional.model.request.auction.StoredBidResponse
 import org.prebid.server.functional.model.response.auction.BidResponse
 import org.prebid.server.functional.model.response.auction.ErrorType
+import org.prebid.server.functional.service.PrebidServerException
 import org.prebid.server.functional.util.PBSUtils
 import spock.lang.PendingFeature
 
 import static org.prebid.server.functional.model.bidder.BidderName.GENERIC
+import static org.prebid.server.functional.model.config.AccountMetricsVerbosityLevel.BASIC
+import static org.prebid.server.functional.model.config.AccountMetricsVerbosityLevel.DETAILED
+import static org.prebid.server.functional.model.config.AccountMetricsVerbosityLevel.NONE
 import static org.prebid.server.functional.model.request.auction.DebugCondition.DISABLED
 import static org.prebid.server.functional.model.request.auction.DebugCondition.ENABLED
 import static org.prebid.server.functional.model.response.auction.BidderCallType.STORED_BID_RESPONSE
 
 class DebugSpec extends BaseSpec {
 
     private static final String overrideToken = PBSUtils.randomString
+    private static final String ACCOUNT_METRICS_PREFIX_NAME = "account"
+    private static final String DEBUG_REQUESTS_METRIC = "debug_requests"
+    private static final String ACCOUNT_DEBUG_REQUESTS_METRIC = "account.%s.debug_requests"
+    private static final String REQUEST_OK_WEB_METRICS = "requests.ok.openrtb2-web"
 
-    def "PBS should return debug information when debug flag is #debug and test flag is #test"() {
+    def "PBS should return debug information and emit metrics when debug flag is #debug and test flag is #test"() {
         given: "Default BidRequest with test flag"
         def bidRequest = BidRequest.defaultBidRequest
         bidRequest.ext.prebid.debug = debug
         bidRequest.test = test
 
+        and: "Flash metrics"
+        flushMetrics(defaultPbsService)
+
         when: "PBS processes auction request"
         def response = defaultPbsService.sendAuctionRequest(bidRequest)
 
         then: "Response should contain ext.debug"
         assert response.ext?.debug
 
+        and: "Debug metrics should be incremented"
+        def metricsRequest = defaultPbsService.sendCollectedMetricsRequest()
+        assert metricsRequest[DEBUG_REQUESTS_METRIC] == 1
+
+        and: "Account debug metrics shouldn't be incremented"
+        assert !metricsRequest.keySet().contains(ACCOUNT_METRICS_PREFIX_NAME)
+
         where:
         debug   | test
         ENABLED | null
@@ -48,12 +68,23 @@ class DebugSpec extends BaseSpec {
         bidRequest.ext.prebid.debug = test
         bidRequest.test = test
 
+        and: "Flash metrics"
+        flushMetrics(defaultPbsService)
+
         when: "PBS processes auction request"
         def response = defaultPbsService.sendAuctionRequest(bidRequest)
 
         then: "Response shouldn't contain ext.debug"
         assert !response.ext?.debug
 
+        and: "Debug metrics shouldn't be populated"
+        def metricsRequest = defaultPbsService.sendCollectedMetricsRequest()
+        assert !metricsRequest[DEBUG_REQUESTS_METRIC]
+        assert !metricsRequest.keySet().contains(ACCOUNT_METRICS_PREFIX_NAME)
+
+        and: "General metrics should be present"
+        assert metricsRequest[REQUEST_OK_WEB_METRICS] == 1
+
         where:
         debug    | test
         DISABLED | null
@@ -351,4 +382,157 @@ class DebugSpec extends BaseSpec {
         and: "Response should not contain ext.warnings"
         assert !response.ext?.warnings
     }
+
+    def "PBS should return debug information and emit metrics when account debug enabled and verbosity detailed"() {
+        given: "Default basic generic bid request"
+        def bidRequest = BidRequest.defaultBidRequest
+
+        and: "Account in the DB"
+        def accountConfig = new AccountConfig(
+                metrics: new AccountMetricsConfig(verbosityLevel: DETAILED),
+                auction: new AccountAuctionConfig(debugAllow: true))
+        def account = new Account(uuid: bidRequest.site.publisher.id, config: accountConfig)
+        accountDao.save(account)
+
+        and: "Flash metrics"
+        flushMetrics(defaultPbsService)
+
+        when: "PBS processes auction request"
+        def response = defaultPbsService.sendAuctionRequest(bidRequest)
+
+        then: "Response should contain ext.debug"
+        assert response.ext?.debug
+
+        and: "Debug metrics should be incremented"
+        def metricsRequest = defaultPbsService.sendCollectedMetricsRequest()
+        assert metricsRequest[ACCOUNT_DEBUG_REQUESTS_METRIC.formatted(bidRequest.accountId)] == 1
+        assert metricsRequest[DEBUG_REQUESTS_METRIC] == 1
+    }
+
+    def "PBS shouldn't return debug information and emit metrics when account debug enabled and verbosity #verbosityLevel"() {
+        given: "Default basic generic bid request"
+        def bidRequest = BidRequest.defaultBidRequest
+
+        and: "Account in the DB"
+        def accountConfig = new AccountConfig(
+                metrics: new AccountMetricsConfig(verbosityLevel: verbosityLevel),
+                auction: new AccountAuctionConfig(debugAllow: true))
+        def account = new Account(uuid: bidRequest.site.publisher.id, config: accountConfig)
+        accountDao.save(account)
+
+        and: "Flash metrics"
+        flushMetrics(defaultPbsService)
+
+        when: "PBS processes auction request"
+        def response = defaultPbsService.sendAuctionRequest(bidRequest)
+
+        then: "Response should contain ext.debug"
+        assert response.ext?.debug
+
+        and: "Account debug metrics shouldn't be incremented"
+        def metricsRequest = defaultPbsService.sendCollectedMetricsRequest()
+        assert !metricsRequest[ACCOUNT_DEBUG_REQUESTS_METRIC.formatted(bidRequest.accountId)]
+
+        and: "Request debug metrics should be incremented"
+        assert metricsRequest[DEBUG_REQUESTS_METRIC] == 1
+
+        where:
+        verbosityLevel << [NONE, BASIC]
+    }
+
+    def "PBS amp should return debug information and emit metrics when account debug enabled and verbosity detailed"() {
+        given: "Default AMP request"
+        def ampRequest = AmpRequest.defaultAmpRequest
+
+        and: "Default stored request"
+        def ampStoredRequest = BidRequest.defaultStoredRequest
+
+        and: "Account in the DB"
+        def accountConfig = new AccountConfig(
+                metrics: new AccountMetricsConfig(verbosityLevel: DETAILED),
+                auction: new AccountAuctionConfig(debugAllow: true))
+        def account = new Account(uuid: ampRequest.account, config: accountConfig)
+        accountDao.save(account)
+
+        and: "Flash metrics"
+        flushMetrics(defaultPbsService)
+
+        and: "Save storedRequest into DB"
+        def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest)
+        storedRequestDao.save(storedRequest)
+
+        when: "PBS processes amp request"
+        def response = defaultPbsService.sendAmpRequest(ampRequest)
+
+        then: "Response should contain ext.debug"
+        assert response.ext?.debug
+
+        and: "Debug metrics should be incremented"
+        def metricsRequest = defaultPbsService.sendCollectedMetricsRequest()
+        assert metricsRequest[ACCOUNT_DEBUG_REQUESTS_METRIC.formatted(ampRequest.account)] == 1
+        assert metricsRequest[DEBUG_REQUESTS_METRIC] == 1
+    }
+
+    def "PBS amp should return debug information and emit metrics when account debug enabled and verbosity #verbosityLevel"() {
+        given: "Default AMP request"
+        def ampRequest = AmpRequest.defaultAmpRequest
+
+        and: "Default stored request"
+        def ampStoredRequest = BidRequest.defaultStoredRequest
+
+        and: "Account in the DB"
+        def accountConfig = new AccountConfig(
+                metrics: new AccountMetricsConfig(verbosityLevel: verbosityLevel),
+                auction: new AccountAuctionConfig(debugAllow: true))
+        def account = new Account(uuid: ampRequest.account, config: accountConfig)
+        accountDao.save(account)
+
+        and: "Flash metrics"
+        flushMetrics(defaultPbsService)
+
+        and: "Save storedRequest into DB"
+        def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest)
+        storedRequestDao.save(storedRequest)
+
+        when: "PBS processes amp request"
+        def response = defaultPbsService.sendAmpRequest(ampRequest)
+
+        then: "Response should contain ext.debug"
+        assert response.ext?.debug
+
+        and: "Account debug metrics shouldn't be incremented"
+        def metricsRequest = defaultPbsService.sendCollectedMetricsRequest()
+        assert !metricsRequest[ACCOUNT_DEBUG_REQUESTS_METRIC.formatted(ampRequest.account)]
+
+        and: "Debug metrics should be incremented"
+        assert metricsRequest[DEBUG_REQUESTS_METRIC] == 1
+
+        where:
+        verbosityLevel << [NONE, BASIC]
+    }
+
+    def "PBS shouldn't emit auction request metric when incoming request invalid"() {
+        given: "Default basic BidRequest"
+        def bidRequest = BidRequest.defaultBidRequest
+        bidRequest.site = new Site(id: null, name: PBSUtils.randomString, page: null)
+        bidRequest.ext.prebid.debug = ENABLED
+
+        and: "Flash metrics"
+        flushMetrics(defaultPbsService)
+
+        when: "PBS processes auction request"
+        defaultPbsService.sendAuctionRequest(bidRequest)
+
+        then: "Request should fail with error"
+        def exception = thrown(PrebidServerException)
+        assert exception.responseBody.contains("request.site should include at least one of request.site.id or request.site.page")
+
+        and: "Debug metrics shouldn't be populated"
+        def metricsRequest = defaultPbsService.sendCollectedMetricsRequest()
+        assert !metricsRequest[DEBUG_REQUESTS_METRIC]
+        assert !metricsRequest.keySet().contains(ACCOUNT_METRICS_PREFIX_NAME)
+
+        and: "General metrics shouldn't be present"
+        assert !metricsRequest[REQUEST_OK_WEB_METRICS]
+    }
 }

src/test/java/org/prebid/server/auction/ExchangeServiceTest.java

@@ -2979,6 +2979,8 @@ public void shouldIncrementCommonMetrics() {
         target.holdAuction(givenRequestContext(bidRequest));
 
         // then
+        verify(metrics).updateDebugRequestMetrics(false);
+        verify(metrics).updateAccountDebugRequestMetrics(any(), eq(false));
         verify(metrics).updateRequestBidderCardinalityMetric(1);
         verify(metrics).updateAccountRequestMetrics(any(), eq(MetricName.openrtb2web));
         verify(metrics).updateAdapterRequestTypeAndNoCookieMetrics(