From fac709fd8e7c42c338b612ab618d88ba8559f329 Mon Sep 17 00:00:00 2001 From: talsabagport Date: Sun, 1 Sep 2024 11:34:29 +0300 Subject: [PATCH] Add support for search identifier (#909) # Description What - Add support for search identifier Why - New feature to allow updating an entity without knowing its identifier How - Call upsert entity API with query, and allow it in mapping ## Type of change Please leave one option from the following and delete the rest: - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] New Integration (non-breaking change which adds a new integration) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Non-breaking change (fix of existing functionality that will not change current behavior) - [ ] Documentation (added/updated documentation)

All tests should be run against the port production environment(using a testing org).

### Core testing checklist - [X] Integration able to create all default resources from scratch - [X] Resync finishes successfully - [X] Resync able to create entities - [X] Resync able to update entities - [X] Resync able to detect and delete entities - [x] Scheduled resync able to abort existing resync and start a new one - [x] Tested with at least 2 integrations from scratch - [x] Tested with Kafka and Polling event listeners ### Integration testing checklist - [ ] Integration able to create all default resources from scratch - [ ] Resync able to create entities - [ ] Resync able to update entities - [ ] Resync able to detect and delete entities - [ ] Resync finishes successfully - [ ] If new resource kind is added or updated in the integration, add example raw data, mapping and expected result to the `examples` folder in the integration directory. - [ ] If resource kind is updated, run the integration with the example data and check if the expected result is achieved - [ ] If new resource kind is added or updated, validate that live-events for that resource are working as expected - [ ] Docs PR link [here](#) ### Preflight checklist - [ ] Handled rate limiting - [ ] Handled pagination - [ ] Implemented the code in async - [ ] Support Multi account ## Screenshots Include screenshots from your environment showing how the resources of the integration will look. ## API Documentation Provide links to the API documentation used for this integration. --- CHANGELOG.md | 7 ++++++ port_ocean/clients/port/mixins/entities.py | 9 ++++++- .../entities_state_applier/port/applier.py | 25 +++++++++++++------ .../core/handlers/port_app_config/models.py | 12 ++++++--- .../core/integrations/mixins/sync_raw.py | 9 ++++++- port_ocean/core/models.py | 4 +++ pyproject.toml | 2 +- 7 files changed, 53 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ae2462c0d..f960f840e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +## 0.10.7 (2024-08-28) + +### Improvements + +- Add search identifier support (Allow to run a search query to find the identifier of the entity as part of the mapping) + + ## 0.10.6 (2024-08-31) ### Bug Fixes diff --git a/port_ocean/clients/port/mixins/entities.py b/port_ocean/clients/port/mixins/entities.py index a18f4f5e8d..d34d5ce317 100644 --- a/port_ocean/clients/port/mixins/entities.py +++ b/port_ocean/clients/port/mixins/entities.py @@ -29,7 +29,7 @@ async def upsert_entity( request_options: RequestOptions, user_agent_type: UserAgentType | None = None, should_raise: bool = True, - ) -> Entity: + ) -> Entity | None: validation_only = request_options["validation_only"] async with self.semaphore: logger.debug( @@ -58,9 +58,16 @@ async def upsert_entity( ) handle_status_code(response, should_raise) result = response.json() + result_entity = ( Entity.parse_obj(result["entity"]) if result.get("entity") else entity ) + + # Happens when upsert fails and search identifier is defined. + # We return None to ignore the entity later in the delete process + if result_entity.is_using_search_identifier: + return None + # In order to save memory we'll keep only the identifier, blueprint and relations of the # upserted entity result for later calculations reduced_entity = Entity( diff --git a/port_ocean/core/handlers/entities_state_applier/port/applier.py b/port_ocean/core/handlers/entities_state_applier/port/applier.py index dd36a243c1..c1ab47dd75 100644 --- a/port_ocean/core/handlers/entities_state_applier/port/applier.py +++ b/port_ocean/core/handlers/entities_state_applier/port/applier.py @@ -106,18 +106,27 @@ async def upsert( should_raise=False, ) else: + entities_with_search_identifier: list[Entity] = [] + entities_without_search_identifier: list[Entity] = [] + for entity in entities: + if entity.is_using_search_identifier: + entities_with_search_identifier.append(entity) + else: + entities_without_search_identifier.append(entity) + ordered_created_entities = reversed( - order_by_entities_dependencies(entities) + entities_with_search_identifier + + order_by_entities_dependencies(entities_without_search_identifier) ) for entity in ordered_created_entities: - modified_entities.append( - await self.context.port_client.upsert_entity( - entity, - event.port_app_config.get_port_request_options(), - user_agent_type, - should_raise=False, - ) + upsertedEntity = await self.context.port_client.upsert_entity( + entity, + event.port_app_config.get_port_request_options(), + user_agent_type, + should_raise=False, ) + if upsertedEntity: + modified_entities.append(upsertedEntity) return modified_entities async def delete( diff --git a/port_ocean/core/handlers/port_app_config/models.py b/port_ocean/core/handlers/port_app_config/models.py index 7130241181..bb37fd0598 100644 --- a/port_ocean/core/handlers/port_app_config/models.py +++ b/port_ocean/core/handlers/port_app_config/models.py @@ -13,18 +13,22 @@ class Rule(BaseModel): value: str -class SearchRelation(BaseModel): +class IngestSearchQuery(BaseModel): combinator: str - rules: list[Rule | SearchRelation] + rules: list[Rule | IngestSearchQuery] class EntityMapping(BaseModel): - identifier: str + identifier: str | IngestSearchQuery title: str | None blueprint: str team: str | None properties: dict[str, str] = Field(default_factory=dict) - relations: dict[str, str | SearchRelation] = Field(default_factory=dict) + relations: dict[str, str | IngestSearchQuery] = Field(default_factory=dict) + + @property + def is_using_search_identifier(self) -> bool: + return isinstance(self.identifier, dict) class MappingsConfig(BaseModel): diff --git a/port_ocean/core/integrations/mixins/sync_raw.py b/port_ocean/core/integrations/mixins/sync_raw.py index fbcaffd956..0b306ff6ab 100644 --- a/port_ocean/core/integrations/mixins/sync_raw.py +++ b/port_ocean/core/integrations/mixins/sync_raw.py @@ -154,6 +154,12 @@ async def _unregister_resource_raw( results: list[RAW_ITEM], user_agent_type: UserAgentType, ) -> tuple[list[Entity], list[Exception]]: + if resource.port.entity.mappings.is_using_search_identifier: + logger.info( + f"Skip unregistering resource of kind {resource.kind}, as mapping defined with search identifier" + ) + return [], [] + objects_diff = await self._calculate_raw([(resource, results)]) entities_selector_diff, errors = objects_diff[0] @@ -272,7 +278,8 @@ async def register_raw( [ entity for entity in entities_to_delete - if (entity.identifier, entity.blueprint) + if not entity.is_using_search_identifier + and (entity.identifier, entity.blueprint) not in registered_entities_attributes ], ) diff --git a/port_ocean/core/models.py b/port_ocean/core/models.py index 8da040c912..fb588becda 100644 --- a/port_ocean/core/models.py +++ b/port_ocean/core/models.py @@ -19,6 +19,10 @@ class Entity(BaseModel): properties: dict[str, Any] = {} relations: dict[str, Any] = {} + @property + def is_using_search_identifier(self) -> bool: + return isinstance(self.identifier, dict) + class BlueprintRelation(BaseModel): many: bool diff --git a/pyproject.toml b/pyproject.toml index ac4bad4070..d7060f9561 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "port-ocean" -version = "0.10.6" +version = "0.10.7" description = "Port Ocean is a CLI tool for managing your Port projects." readme = "README.md" homepage = "https://app.getport.io"