From d3fabb1bb4c6341527622dfa97b571118e7f2236 Mon Sep 17 00:00:00 2001 From: lordsarcastic Date: Wed, 15 Jan 2025 21:23:01 +0800 Subject: [PATCH] [Integration][Jira] Ocean Rate limiting handling from Jira (#1320) --- integrations/jira/CHANGELOG.md | 9 +++++++++ integrations/jira/jira/client.py | 14 ++++++++++---- integrations/jira/pyproject.toml | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/integrations/jira/CHANGELOG.md b/integrations/jira/CHANGELOG.md index 11bc1da260..859c125cda 100644 --- a/integrations/jira/CHANGELOG.md +++ b/integrations/jira/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +## 0.2.22 (2025-01-15) + + +### Improvements + +- Added rate limit support to avoid failures due to 429 errors + + + ## 0.2.21 (2025-01-15) diff --git a/integrations/jira/jira/client.py b/integrations/jira/jira/client.py index d866ac4afa..3402814eb5 100644 --- a/integrations/jira/jira/client.py +++ b/integrations/jira/jira/client.py @@ -7,7 +7,6 @@ from port_ocean.context.ocean import ocean from port_ocean.utils import http_async_client - PAGE_SIZE = 50 WEBHOOK_NAME = "Port-Ocean-Events-Webhook" MAX_CONCURRENT_REQUESTS = 10 @@ -60,7 +59,14 @@ def __init__(self, jira_url: str, jira_email: str, jira_token: str) -> None: self.client = http_async_client self.client.auth = self.jira_api_auth self.client.timeout = Timeout(30) - self.semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS) + self._semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS) + + async def _handle_rate_limit(self, response: Response) -> None: + if response.status_code == 429: + logger.warning( + f"Jira API rate limit reached. Waiting for {response.headers['Retry-After']} seconds." + ) + await asyncio.sleep(int(response.headers["Retry-After"])) async def _send_api_request( self, @@ -71,13 +77,14 @@ async def _send_api_request( headers: dict[str, str] | None = None, ) -> Any: try: - async with self.semaphore: + async with self._semaphore: response = await self.client.request( method=method, url=url, params=params, json=json, headers=headers ) response.raise_for_status() return response.json() except httpx.HTTPStatusError as e: + await self._handle_rate_limit(e.response) logger.error( f"Jira API request failed with status {e.response.status_code}: {method} {url}" ) @@ -117,7 +124,6 @@ async def _get_cursor_paginated_data( method: str, extract_key: str, initial_params: dict[str, Any] | None = None, - page_size: int = PAGE_SIZE, cursor_param: str = "cursor", ) -> AsyncGenerator[list[dict[str, Any]], None]: params = initial_params or {} diff --git a/integrations/jira/pyproject.toml b/integrations/jira/pyproject.toml index f09c6c4b19..f7b1930d27 100644 --- a/integrations/jira/pyproject.toml +++ b/integrations/jira/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "jira" -version = "0.2.21" +version = "0.2.22" description = "Integration to bring information from Jira into Port" authors = ["Mor Paz "]