From 5fa5edc4b6aff1105d47567252af9cb2fc1f1ae8 Mon Sep 17 00:00:00 2001 From: Vitor Anjos Date: Tue, 7 Jul 2020 19:34:45 -0300 Subject: [PATCH 1/3] Log scroll response failures --- elasticsearch/helpers/actions.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/elasticsearch/helpers/actions.py b/elasticsearch/helpers/actions.py index 87fcc305d..652ad1eb5 100644 --- a/elasticsearch/helpers/actions.py +++ b/elasticsearch/helpers/actions.py @@ -524,14 +524,25 @@ def scan( resp["_shards"]["skipped"], resp["_shards"]["total"], ) + + resp_failures = resp['_shards'].get('failures') + failures = [] + if resp_failures is not None: + for resp_failure in resp_failures: + failure = "Failure type[%s] received with reason: %s" % \ + (resp_failure["reason"]['type'], resp_failure["reason"]['reason']) + failures.append(failure) + logger.warning(failure) if raise_on_error: raise ScanError( scroll_id, - "Scroll request has only succeeded on %d (+%d skiped) shards out of %d." + "Scroll request has only succeeded on %d (+%d skiped) shards out of %d.\n" + "%s" % ( resp["_shards"]["successful"], resp["_shards"]["skipped"], resp["_shards"]["total"], + ''.join(failures) ), ) resp = client.scroll( From 349efce8bd7686627333604fe9ba34af7d42e914 Mon Sep 17 00:00:00 2001 From: Vitor Anjos Date: Tue, 7 Jul 2020 19:56:46 -0300 Subject: [PATCH 2/3] Fix lint error --- elasticsearch/helpers/actions.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/elasticsearch/helpers/actions.py b/elasticsearch/helpers/actions.py index 652ad1eb5..93645d417 100644 --- a/elasticsearch/helpers/actions.py +++ b/elasticsearch/helpers/actions.py @@ -525,12 +525,14 @@ def scan( resp["_shards"]["total"], ) - resp_failures = resp['_shards'].get('failures') + resp_failures = resp["_shards"].get("failures") failures = [] if resp_failures is not None: for resp_failure in resp_failures: - failure = "Failure type[%s] received with reason: %s" % \ - (resp_failure["reason"]['type'], resp_failure["reason"]['reason']) + failure = "Failure type[%s] received with reason: %s" % ( + resp_failure["reason"]["type"], + resp_failure["reason"]["reason"], + ) failures.append(failure) logger.warning(failure) if raise_on_error: @@ -542,7 +544,7 @@ def scan( resp["_shards"]["successful"], resp["_shards"]["skipped"], resp["_shards"]["total"], - ''.join(failures) + "".join(failures), ), ) resp = client.scroll( From 327022ebb7b0eb5e2aac7d8414fd6615add98eb0 Mon Sep 17 00:00:00 2001 From: Vitor Anjos Date: Thu, 9 Jul 2020 19:23:42 -0300 Subject: [PATCH 3/3] Add reasonable limit to show failures --- elasticsearch/helpers/actions.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/elasticsearch/helpers/actions.py b/elasticsearch/helpers/actions.py index 93645d417..3042de044 100644 --- a/elasticsearch/helpers/actions.py +++ b/elasticsearch/helpers/actions.py @@ -528,23 +528,30 @@ def scan( resp_failures = resp["_shards"].get("failures") failures = [] if resp_failures is not None: - for resp_failure in resp_failures: - failure = "Failure type[%s] received with reason: %s" % ( - resp_failure["reason"]["type"], - resp_failure["reason"]["reason"], + for i in range(5): + if i < len(resp_failures): + failure = "Failure type[%s] received with reason: %s" % ( + resp_failures[i]["reason"]["type"], + resp_failures[i]["reason"]["reason"], + ) + failures.append(failure) + logger.warning(failure) + + if len(resp_failures) > 5: + omitted_failures = len(resp_failures) - 5 + failures.append( + "...and %d more omitted failures" % omitted_failures ) - failures.append(failure) - logger.warning(failure) if raise_on_error: raise ScanError( scroll_id, - "Scroll request has only succeeded on %d (+%d skiped) shards out of %d.\n" + "Scroll request has only succeeded on %d (+%d skipped) shards out of %d.\n" "%s" % ( resp["_shards"]["successful"], resp["_shards"]["skipped"], resp["_shards"]["total"], - "".join(failures), + "\n".join(failures), ), ) resp = client.scroll(