From 55ac53bc41ee013b4eed7ededfc4a2465dcb8673 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 28 Jan 2025 13:37:04 +0100 Subject: [PATCH 1/2] docs: Move code samples to files and other updates Closes: #337 --- Makefile | 12 +- docs/01_overview/01_introduction.mdx | 26 ++ docs/01_overview/02_setting_up.mdx | 71 ++++ docs/01_overview/03_getting_started.mdx | 74 ++++ docs/01_overview/code/01_usage_async.py | 21 ++ docs/01_overview/code/01_usage_sync.py | 21 ++ docs/01_overview/code/02_auth_async.py | 8 + docs/01_overview/code/02_auth_sync.py | 8 + docs/01_overview/code/03_dataset_async.py | 11 + docs/01_overview/code/03_dataset_sync.py | 11 + docs/01_overview/code/03_input_async.py | 16 + docs/01_overview/code/03_input_sync.py | 16 + docs/02_concepts/01_async_support.mdx | 18 + .../02_single_collection_clients.mdx | 48 +++ docs/02_concepts/03_nested_clients.mdx | 28 ++ docs/02_concepts/04_error_handling.mdx | 26 ++ docs/02_concepts/05_retries.mdx | 42 +++ docs/02_concepts/06_logging.mdx | 33 ++ docs/02_concepts/07_convenience_methods.mdx | 34 ++ docs/02_concepts/08_pagination.mdx | 38 ++ docs/02_concepts/09_streaming.mdx | 38 ++ docs/02_concepts/code/01_async_support.py | 25 ++ docs/02_concepts/code/02_collection_async.py | 16 + docs/02_concepts/code/02_collection_sync.py | 16 + docs/02_concepts/code/02_single_async.py | 16 + docs/02_concepts/code/02_single_sync.py | 16 + docs/02_concepts/code/03_nested_async.py | 22 ++ docs/02_concepts/code/03_nested_sync.py | 22 ++ docs/02_concepts/code/04_error_async.py | 15 + docs/02_concepts/code/04_error_sync.py | 15 + docs/02_concepts/code/05_retries_async.py | 12 + docs/02_concepts/code/05_retries_sync.py | 12 + docs/02_concepts/code/06_logging_config.py | 6 + docs/02_concepts/code/06_logging_formatter.py | 14 + docs/02_concepts/code/07_call_async.py | 14 + docs/02_concepts/code/07_call_sync.py | 14 + docs/02_concepts/code/08_pagination_async.py | 35 ++ docs/02_concepts/code/08_pagination_sync.py | 35 ++ docs/02_concepts/code/09_streaming_async.py | 14 + docs/02_concepts/code/09_streaming_sync.py | 14 + .../03_examples/01_passing_input_to_actor.mdx | 28 ++ .../02_manage_tasks_for_reusable_input.mdx | 28 ++ docs/03_examples/03_retrieve_actor_data.mdx | 28 ++ .../04_integration_with_data_libraries.mdx | 28 ++ docs/03_examples/code/01_input_async.py | 23 ++ docs/03_examples/code/01_input_sync.py | 21 ++ docs/03_examples/code/02_tasks_async.py | 51 +++ docs/03_examples/code/02_tasks_sync.py | 52 +++ docs/03_examples/code/03_retrieve_async.py | 33 ++ docs/03_examples/code/03_retrieve_sync.py | 31 ++ docs/03_examples/code/04_pandas_async.py | 27 ++ docs/03_examples/code/04_pandas_sync.py | 25 ++ docs/examples.md | 140 ------- docs/index.md | 345 ------------------ pyproject.toml | 19 +- website/docusaurus.config.js | 2 +- website/generate_module_shortcuts.py | 23 +- website/sidebars.js | 61 +++- website/src/pages/home_page_example.py | 18 + website/src/pages/index.js | 12 +- 60 files changed, 1413 insertions(+), 515 deletions(-) create mode 100644 docs/01_overview/01_introduction.mdx create mode 100644 docs/01_overview/02_setting_up.mdx create mode 100644 docs/01_overview/03_getting_started.mdx create mode 100644 docs/01_overview/code/01_usage_async.py create mode 100644 docs/01_overview/code/01_usage_sync.py create mode 100644 docs/01_overview/code/02_auth_async.py create mode 100644 docs/01_overview/code/02_auth_sync.py create mode 100644 docs/01_overview/code/03_dataset_async.py create mode 100644 docs/01_overview/code/03_dataset_sync.py create mode 100644 docs/01_overview/code/03_input_async.py create mode 100644 docs/01_overview/code/03_input_sync.py create mode 100644 docs/02_concepts/01_async_support.mdx create mode 100644 docs/02_concepts/02_single_collection_clients.mdx create mode 100644 docs/02_concepts/03_nested_clients.mdx create mode 100644 docs/02_concepts/04_error_handling.mdx create mode 100644 docs/02_concepts/05_retries.mdx create mode 100644 docs/02_concepts/06_logging.mdx create mode 100644 docs/02_concepts/07_convenience_methods.mdx create mode 100644 docs/02_concepts/08_pagination.mdx create mode 100644 docs/02_concepts/09_streaming.mdx create mode 100644 docs/02_concepts/code/01_async_support.py create mode 100644 docs/02_concepts/code/02_collection_async.py create mode 100644 docs/02_concepts/code/02_collection_sync.py create mode 100644 docs/02_concepts/code/02_single_async.py create mode 100644 docs/02_concepts/code/02_single_sync.py create mode 100644 docs/02_concepts/code/03_nested_async.py create mode 100644 docs/02_concepts/code/03_nested_sync.py create mode 100644 docs/02_concepts/code/04_error_async.py create mode 100644 docs/02_concepts/code/04_error_sync.py create mode 100644 docs/02_concepts/code/05_retries_async.py create mode 100644 docs/02_concepts/code/05_retries_sync.py create mode 100644 docs/02_concepts/code/06_logging_config.py create mode 100644 docs/02_concepts/code/06_logging_formatter.py create mode 100644 docs/02_concepts/code/07_call_async.py create mode 100644 docs/02_concepts/code/07_call_sync.py create mode 100644 docs/02_concepts/code/08_pagination_async.py create mode 100644 docs/02_concepts/code/08_pagination_sync.py create mode 100644 docs/02_concepts/code/09_streaming_async.py create mode 100644 docs/02_concepts/code/09_streaming_sync.py create mode 100644 docs/03_examples/01_passing_input_to_actor.mdx create mode 100644 docs/03_examples/02_manage_tasks_for_reusable_input.mdx create mode 100644 docs/03_examples/03_retrieve_actor_data.mdx create mode 100644 docs/03_examples/04_integration_with_data_libraries.mdx create mode 100644 docs/03_examples/code/01_input_async.py create mode 100644 docs/03_examples/code/01_input_sync.py create mode 100644 docs/03_examples/code/02_tasks_async.py create mode 100644 docs/03_examples/code/02_tasks_sync.py create mode 100644 docs/03_examples/code/03_retrieve_async.py create mode 100644 docs/03_examples/code/03_retrieve_sync.py create mode 100644 docs/03_examples/code/04_pandas_async.py create mode 100644 docs/03_examples/code/04_pandas_sync.py delete mode 100644 docs/examples.md delete mode 100644 docs/index.md create mode 100644 website/src/pages/home_page_example.py diff --git a/Makefile b/Makefile index 317e1214..02231a37 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,6 @@ integration-tests format check-async-docstrings check-code fix-async-docstrings \ build-api-reference build-docs run-docs -DIRS_WITH_CODE = src tests scripts - # This is default for local testing, but GitHub workflows override it to a higher value in CI INTEGRATION_TESTS_CONCURRENCY = 1 @@ -23,11 +21,11 @@ publish-to-pypi: poetry publish --no-interaction -vv lint: - poetry run ruff format --check $(DIRS_WITH_CODE) - poetry run ruff check $(DIRS_WITH_CODE) + poetry run ruff format --check + poetry run ruff check type-check: - poetry run mypy $(DIRS_WITH_CODE) + poetry run mypy unit-tests: poetry run pytest --numprocesses=auto --verbose --cov=src/apify_client tests/unit @@ -39,8 +37,8 @@ integration-tests: poetry run pytest --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) tests/integration format: - poetry run ruff check --fix $(DIRS_WITH_CODE) - poetry run ruff format $(DIRS_WITH_CODE) + poetry run ruff check --fix + poetry run ruff format check-async-docstrings: poetry run python scripts/check_async_docstrings.py diff --git a/docs/01_overview/01_introduction.mdx b/docs/01_overview/01_introduction.mdx new file mode 100644 index 00000000..0b49f85d --- /dev/null +++ b/docs/01_overview/01_introduction.mdx @@ -0,0 +1,26 @@ +--- +id: introduction +title: Introduction +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import UsageAsyncExample from '!!raw-loader!./code/01_usage_async.py'; +import UsageSyncExample from '!!raw-loader!./code/01_usage_sync.py'; + +The [Apify client for Python](https://github.com/apify/apify-client-python) is the official library to access the [Apify REST API](https://docs.apify.com/api/v2) from your Python applications. It provides useful features like automatic retries and convenience functions that improve the experience of using the Apify API. All requests and responses (including errors) are encoded in JSON format with UTF-8 encoding. The client provides both synchronous and asynchronous interfaces. + + + + + {UsageAsyncExample} + + + + + {UsageSyncExample} + + + diff --git a/docs/01_overview/02_setting_up.mdx b/docs/01_overview/02_setting_up.mdx new file mode 100644 index 00000000..63e45757 --- /dev/null +++ b/docs/01_overview/02_setting_up.mdx @@ -0,0 +1,71 @@ +--- +id: setting-up +title: Setting up +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import AsyncExample from '!!raw-loader!./code/02_auth_async.py'; +import SyncExample from '!!raw-loader!./code/02_auth_sync.py'; + +This guide will help you get started with [Apify client for Python](https://github.com/apify/apify-client-python) by setting it up on your computer. Follow the steps below to ensure a smooth installation process. + +## Prerequisites + +Before installing `apify-client` itself, make sure that your system meets the following requirements: + +- **Python 3.9 or higher**: `apify-client` requires Python 3.9 or a newer version. You can download Python from the [official website](https://www.python.org/downloads/). +- **Python package manager**: While this guide uses Pip (the most common package manager), you can also use any package manager you want. You can download Pip from the [official website](https://pip.pypa.io/en/stable/installation/). + +### Verifying prerequisites + +To check if Python and the Pip package manager are installed, run the following commands: + +```sh +python --version +``` + +```sh +pip --version +``` + +If these commands return the respective versions, you're ready to continue. + +## Installation + +Apify client for Python is available as the [`apify-client`](https://pypi.org/project/crawlee/) package on PyPI. To install it, run: + +```sh +pip install apify-client +``` + +After installation, verify that `apify-client` is installed correctly by checking its version: + +```sh +python -c 'import apify_client; print(apify_client.__version__)' +``` + +## Authentication and initialization + +To use the client, you need an [API token](https://docs.apify.com/platform/integrations/api#api-token). You can find your token under [Integrations](https://console.apify.com/account/integrations) tab in Apify Console. Copy the token and initialize the client by providing the token (`MY-APIFY-TOKEN`) as a parameter to the `ApifyClient` constructor. + + + + + {AsyncExample} + + + + + {SyncExample} + + + + +:::warning Secure access + +The API token is used to authorize your requests to the Apify API. You can be charged for the usage of the underlying services, so do not share your API token with untrusted parties or expose it on the client side of your applications. + +::: diff --git a/docs/01_overview/03_getting_started.mdx b/docs/01_overview/03_getting_started.mdx new file mode 100644 index 00000000..43b128d0 --- /dev/null +++ b/docs/01_overview/03_getting_started.mdx @@ -0,0 +1,74 @@ +--- +id: getting-started +title: Getting started +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import UsageAsyncExample from '!!raw-loader!./code/01_usage_async.py'; +import UsageSyncExample from '!!raw-loader!./code/01_usage_sync.py'; +import InputAsyncExample from '!!raw-loader!./code/03_input_async.py'; +import InputSyncExample from '!!raw-loader!./code/03_input_sync.py'; +import DatasetAsyncExample from '!!raw-loader!./code/03_dataset_async.py'; +import DatasetSyncExample from '!!raw-loader!./code/03_dataset_sync.py'; + +This guide will walk you through how to use the [Apify Client for Python](https://github.com/apify/apify-client-python) to run [Actors](https://apify.com/actors) on the [Apify platform](https://docs.apify.com/platform), provide input to them, and retrieve results from their datasets. You'll learn the basics of running serverless programs (we're calling them Actors) and managing their output efficiently. + +## Running your first Actor + +To start an Actor, you need its ID (e.g., `john-doe/my-cool-actor`) and an API token. The Actor's ID is a combination of the username and the Actor owner's username. Use the [`ActorClient`](/reference/class/ActorClient) to run the Actor and wait for it to complete. You can run both your own Actors and [Actors from Apify store](https://docs.apify.com/platform/actors/running/actors-in-store). + + + + + {UsageAsyncExample} + + + + + {UsageSyncExample} + + + + +## Providing input to Actor + +Actors often require input, such as URLs to scrape, search terms, or other configuration data. You can pass input as a JSON object when starting the Actor using the [`ActorClient.call`](/reference/class/ActorClient#call) method. Actors respect the input schema defined in the Actor's [input schema](https://docs.apify.com/platform/actors/development/actor-definition/input-schema). + + + + + {InputAsyncExample} + + + + + {InputSyncExample} + + + + +## Getting results from the dataset + +To get the results from the dataset, you can use the [`DatasetClient`](/reference/class/DatasetClient) ([`ApifyClient.dataset`](/reference/class/ApifyClient#dataset) ) and [`DatasetClient.list_items`](/reference/class/DatasetClient#list_items) method. You need to pass the dataset ID to define which dataset you want to access. You can get the dataset ID from the Actor's run dictionary (represented by `defaultDatasetId`). + + + + + {InputAsyncExample} + + + + + {InputSyncExample} + + + + +:::note Dataset access + +Running an Actor might take time, depending on the Actor's complexity and the amount of data it processes. If you want only to get data and have an immediate response you should access the existing dataset of the finished [Actor run](https://docs.apify.com/platform/actors/running/runs-and-builds#runs). + +::: diff --git a/docs/01_overview/code/01_usage_async.py b/docs/01_overview/code/01_usage_async.py new file mode 100644 index 00000000..3ad4e883 --- /dev/null +++ b/docs/01_overview/code/01_usage_async.py @@ -0,0 +1,21 @@ +from apify_client import ApifyClientAsync + +# You can find your API token at https://console.apify.com/settings/integrations. +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + + # Start an Actor and wait for it to finish. + actor_client = apify_client.actor('john-doe/my-cool-actor') + call_result = await actor_client.call() + + if call_result is None: + print('Actor run failed.') + return + + # Fetch results from the Actor run's default dataset. + dataset_client = apify_client.dataset(call_result['defaultDatasetId']) + list_items_result = await dataset_client.list_items() + print(f'Dataset: {list_items_result}') diff --git a/docs/01_overview/code/01_usage_sync.py b/docs/01_overview/code/01_usage_sync.py new file mode 100644 index 00000000..afa15ffb --- /dev/null +++ b/docs/01_overview/code/01_usage_sync.py @@ -0,0 +1,21 @@ +from apify_client import ApifyClient + +# You can find your API token at https://console.apify.com/settings/integrations. +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + + # Start an Actor and wait for it to finish. + actor_client = apify_client.actor('john-doe/my-cool-actor') + call_result = actor_client.call() + + if call_result is None: + print('Actor run failed.') + return + + # Fetch results from the Actor run's default dataset. + dataset_client = apify_client.dataset(call_result['defaultDatasetId']) + list_items_result = dataset_client.list_items() + print(f'Dataset: {list_items_result}') diff --git a/docs/01_overview/code/02_auth_async.py b/docs/01_overview/code/02_auth_async.py new file mode 100644 index 00000000..a584ccba --- /dev/null +++ b/docs/01_overview/code/02_auth_async.py @@ -0,0 +1,8 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + # Client initialization with the API token. + apify_client = ApifyClientAsync(TOKEN) diff --git a/docs/01_overview/code/02_auth_sync.py b/docs/01_overview/code/02_auth_sync.py new file mode 100644 index 00000000..a5f4de39 --- /dev/null +++ b/docs/01_overview/code/02_auth_sync.py @@ -0,0 +1,8 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + # Client initialization with the API token. + apify_client = ApifyClient(TOKEN) diff --git a/docs/01_overview/code/03_dataset_async.py b/docs/01_overview/code/03_dataset_async.py new file mode 100644 index 00000000..99541d2a --- /dev/null +++ b/docs/01_overview/code/03_dataset_async.py @@ -0,0 +1,11 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + dataset_client = apify_client.dataset('dataset-id') + + # Lists items from the Actor's dataset. + dataset_items = (await dataset_client.list_items()).items diff --git a/docs/01_overview/code/03_dataset_sync.py b/docs/01_overview/code/03_dataset_sync.py new file mode 100644 index 00000000..3cc7554d --- /dev/null +++ b/docs/01_overview/code/03_dataset_sync.py @@ -0,0 +1,11 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + dataset_client = apify_client.dataset('dataset-id') + + # Lists items from the Actor's dataset. + dataset_items = dataset_client.list_items().items diff --git a/docs/01_overview/code/03_input_async.py b/docs/01_overview/code/03_input_async.py new file mode 100644 index 00000000..bef7ac72 --- /dev/null +++ b/docs/01_overview/code/03_input_async.py @@ -0,0 +1,16 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + actor_client = apify_client.actor('username/actor-name') + + # Define the input for the Actor. + run_input = { + 'some': 'input', + } + + # Start an Actor and waits for it to finish. + call_result = await actor_client.call(run_input=run_input) diff --git a/docs/01_overview/code/03_input_sync.py b/docs/01_overview/code/03_input_sync.py new file mode 100644 index 00000000..7fab1635 --- /dev/null +++ b/docs/01_overview/code/03_input_sync.py @@ -0,0 +1,16 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + actor_client = apify_client.actor('username/actor-name') + + # Define the input for the Actor. + run_input = { + 'some': 'input', + } + + # Start an Actor and waits for it to finish. + call_result = actor_client.call(run_input=run_input) diff --git a/docs/02_concepts/01_async_support.mdx b/docs/02_concepts/01_async_support.mdx new file mode 100644 index 00000000..bdb6ab3a --- /dev/null +++ b/docs/02_concepts/01_async_support.mdx @@ -0,0 +1,18 @@ +--- +id: asyncio-support +title: Asyncio support +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import AsyncSupportExample from '!!raw-loader!./code/01_async_support.py'; + +The package provides an asynchronous version of the client, [`ApifyClientAsync`](/reference/class/ApifyClientAsync), which allows you to interact with the Apify API using Python's standard async/await syntax. This enables you to perform non-blocking operations, see the Python [asyncio documentation](https://docs.python.org/3/library/asyncio-task.html) for more information. + +The following example demonstrates how to run an Actor asynchronously and stream its logs while it is running: + + + {AsyncSupportExample} + diff --git a/docs/02_concepts/02_single_collection_clients.mdx b/docs/02_concepts/02_single_collection_clients.mdx new file mode 100644 index 00000000..1a5ac0a2 --- /dev/null +++ b/docs/02_concepts/02_single_collection_clients.mdx @@ -0,0 +1,48 @@ +--- +id: single-and-collection-clients +title: Single and collection clients +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import CollectionAsyncExample from '!!raw-loader!./code/02_collection_async.py'; +import CollectionSyncExample from '!!raw-loader!./code/02_collection_sync.py'; +import SingleAsyncExample from '!!raw-loader!./code/02_single_async.py'; +import SingleSyncExample from '!!raw-loader!./code/02_single_sync.py'; + +The Apify client interface is designed to be consistent and intuitive across all of its components. When you call specific methods on the main client, you create specialized clients to manage individual API resources. There are two main types of clients: + +- [`ActorClient`](/reference/class/ActorClient) - Manages a single resource. +- [`ActorCollectionClient`](/reference/class/ActorCollectionClient) - Manages a collection of resources. + + + + + {CollectionAsyncExample} + + + + + {CollectionSyncExample} + + + + +The resource ID can be the resource's `id` or a combination of `username/resource-name`. + + + + + {SingleAsyncExample} + + + + + {SingleSyncExample} + + + + +By utilizing the appropriate collection or resource client, you can simplify how you interact with the Apify API. diff --git a/docs/02_concepts/03_nested_clients.mdx b/docs/02_concepts/03_nested_clients.mdx new file mode 100644 index 00000000..386aefd1 --- /dev/null +++ b/docs/02_concepts/03_nested_clients.mdx @@ -0,0 +1,28 @@ +--- +id: nested-clients +title: Nested clients +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import NestedAsyncExample from '!!raw-loader!./code/03_nested_async.py'; +import NestedSyncExample from '!!raw-loader!./code/03_nested_sync.py'; + +In some cases, the Apify client provides nested clients to simplify working with related collections. For example, you can easily manage the runs of a specific Actor without having to construct multiple endpoints or client instances manually. + + + + + {NestedAsyncExample} + + + + + {NestedSyncExample} + + + + +This direct access to [Dataset](https://docs.apify.com/platform/storage/dataset) (and other storage resources) from the [`RunClient`](/reference/class/RunClient) is especially convenient when used alongside the [`ActorClient.last_run`](/reference/class/ActorClient#last_run) method. diff --git a/docs/02_concepts/04_error_handling.mdx b/docs/02_concepts/04_error_handling.mdx new file mode 100644 index 00000000..d8859eb1 --- /dev/null +++ b/docs/02_concepts/04_error_handling.mdx @@ -0,0 +1,26 @@ +--- +id: error-handling +title: Error handling +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import ErrorAsyncExample from '!!raw-loader!./code/04_error_async.py'; +import ErrorSyncExample from '!!raw-loader!./code/04_error_sync.py'; + +When you use the Apify client, it automatically extracts all relevant data from the endpoint and returns it in the expected format. Date strings, for instance, are seamlessly converted to Python `datetime.datetime` objects. If an error occurs, the client raises an [`ApifyApiError`](/reference/class/ApifyApiError). This exception wraps the raw JSON errors returned by the API and provides additional context, making it easier to debug any issues that arise. + + + + + {ErrorAsyncExample} + + + + + {ErrorSyncExample} + + + diff --git a/docs/02_concepts/05_retries.mdx b/docs/02_concepts/05_retries.mdx new file mode 100644 index 00000000..9cf0ed11 --- /dev/null +++ b/docs/02_concepts/05_retries.mdx @@ -0,0 +1,42 @@ +--- +id: retries +title: Retries +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import RetriesAsyncExample from '!!raw-loader!./code/05_retries_async.py'; +import RetriesSyncExample from '!!raw-loader!./code/05_retries_sync.py'; + +When dealing with network communication, failures can occasionally occur. The Apify client automatically retries requests that fail due to: + +- Network errors +- Internal errors in the Apify API (HTTP status codes 500 and above) +- Rate limit errors (HTTP status code 429) + +By default, the client will retry a failed request up to 8 times. The retry intervals use an exponential backoff strategy: + +- The first retry occurs after approximately 500 milliseconds. +- The second retry occurs after approximately 1,000 milliseconds, and so on. + +You can customize this behavior using the following options in the [`ApifyClient`](/reference/class/ApifyClient) constructor: + +- `max_retries`: Defines the maximum number of retry attempts. +- `min_delay_between_retries_millis`: Sets the minimum delay between retries (in milliseconds). + +Retries with exponential backoff are a common strategy for handling network errors. They help to reduce the load on the server and increase the chances of a successful request. + + + + + {RetriesAsyncExample} + + + + + {RetriesSyncExample} + + + diff --git a/docs/02_concepts/06_logging.mdx b/docs/02_concepts/06_logging.mdx new file mode 100644 index 00000000..2e495878 --- /dev/null +++ b/docs/02_concepts/06_logging.mdx @@ -0,0 +1,33 @@ +--- +id: logging +title: Logging +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import LoggingConfigExample from '!!raw-loader!./code/06_logging_config.py'; +import LoggingFormatterExample from '!!raw-loader!./code/06_logging_formatter.py'; + +The library logs useful debug information to the `apify_client` logger whenever it sends requests to the Apify API. You can configure this logger to print debug information to the standard output by adding a handler: + + + {LoggingConfigExample} + + +The log records include additional properties, provided via the extra argument, which can be helpful for debugging. Some of these properties are: + +- `attempt` - Number of retry attempts for the request. +- `status_code` - HTTP status code of the response. +- `url` - URL of the API endpoint being called. +- `client_method` - Method name of the client that initiated the request. +- `resource_id` - Identifier of the resource being accessed. + +To display these additional properties in the log output, you need to use a custom log formatter. Here's a basic example: + + + {LoggingFormatterExample} + + +For more information on creating and using custom log formatters, refer to the official Python [logging documentation](https://docs.python.org/3/howto/logging.html#formatters). diff --git a/docs/02_concepts/07_convenience_methods.mdx b/docs/02_concepts/07_convenience_methods.mdx new file mode 100644 index 00000000..b3014471 --- /dev/null +++ b/docs/02_concepts/07_convenience_methods.mdx @@ -0,0 +1,34 @@ +--- +id: convenience-methods +title: Convenience methods +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import CallAsyncExample from '!!raw-loader!./code/07_call_async.py'; +import CallSyncExample from '!!raw-loader!./code/07_call_sync.py'; + +The Apify client provides several convenience methods to handle actions that the API alone cannot perform efficiently, such as waiting for an Actor run to finish without running into network timeouts. These methods simplify common tasks and enhance the usability of the client. + +- [`ActorClient.call`](/reference/class/ActorClient#call) - Starts an Actor and waits for it to finish, handling network timeouts internally. +- [`ActorClient.start`](/reference/class/ActorClient#start) - Explicitly waits for an Actor run to finish with customizable timeouts. + +Additionally, storage-related resources offer flexible options for data retrieval: + +- [Key-value store](https://docs.apify.com/platform/storage/key-value-store) records can be retrieved as objects, buffers, or streams. +- [Dataset](https://docs.apify.com/platform/storage/dataset) items can be fetched as individual objects, serialized data, or iterated asynchronously. + + + + + {CallAsyncExample} + + + + + {CallSyncExample} + + + diff --git a/docs/02_concepts/08_pagination.mdx b/docs/02_concepts/08_pagination.mdx new file mode 100644 index 00000000..712aee81 --- /dev/null +++ b/docs/02_concepts/08_pagination.mdx @@ -0,0 +1,38 @@ +--- +id: pagination +title: Pagination +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import PaginationAsyncExample from '!!raw-loader!./code/08_pagination_async.py'; +import PaginationSyncExample from '!!raw-loader!./code/08_pagination_sync.py'; + +Most methods named `list` or `list_something` in the Apify client return a [`ListPage`](/reference/class/ListPage) object. This object provides a consistent interface for working with paginated data and includes the following properties: + +- `items` - The main results you're looking for. +- `total` - The total number of items available. +- `offset` - The starting point of the current page. +- `count` - The number of items in the current page. +- `limit` - The maximum number of items per page. + +Some methods, such as `list_keys` or `list_head`, paginate differently. Regardless, the primary results are always stored under the items property, and the limit property can be used to control the number of results returned. + +The following example demonstrates how to fetch all items from a dataset using pagination: + + + + + {PaginationAsyncExample} + + + + + {PaginationSyncExample} + + + + +The [`ListPage`](/reference/class/ListPage) interface offers several key benefits. Its consistent structure ensures predictable results for most `list` methods, providing a uniform way to work with paginated data. It also offers flexibility, allowing you to customize the `limit` and `offset` parameters to control data fetching according to your needs. Additionally, it provides scalability, enabling you to efficiently handle large datasets through pagination. This approach ensures efficient data retrieval while keeping memory usage under control, making it ideal for managing and processing large collections. diff --git a/docs/02_concepts/09_streaming.mdx b/docs/02_concepts/09_streaming.mdx new file mode 100644 index 00000000..b365e34a --- /dev/null +++ b/docs/02_concepts/09_streaming.mdx @@ -0,0 +1,38 @@ +--- +id: streaming-resources +title: Streaming resources +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import StreamingAsyncExample from '!!raw-loader!./code/09_streaming_async.py'; +import StreamingSyncExample from '!!raw-loader!./code/09_streaming_sync.py'; + +Certain resources, such as dataset items, key-value store records, and logs, support streaming directly from the Apify API. This allows you to process large resources incrementally without downloading them entirely into memory, making it ideal for handling large or continuously updated data. + +Supported streaming methods: + +- [`DatasetClient.stream_items`](/reference/class/DatasetClient#stream_items) - Stream dataset items incrementally. +- [`KeyValueStoreClient.stream_record`](/reference/class/KeyValueStoreClient#stream_record) - Stream key-value store records as raw data. +- [`LogClient.stream`](/reference/class/LogClient#stream) - Stream logs in real time. + +These methods return a raw, context-managed `httpx.Response` object. The response must be consumed within a with block to ensure that the connection is closed automatically, preventing memory leaks or unclosed connections. + +The following example demonstrates how to stream the logs of an Actor run incrementally: + + + + + {StreamingAsyncExample} + + + + + {StreamingSyncExample} + + + + +Streaming offers several key benefits. It ensures memory efficiency by loading only a small portion of the resource into memory at any given time, making it ideal for handling large data. It enables real-time processing, allowing you to start working with data immediately as it is received. With automatic resource management, using the `with` statement ensures that connections are properly closed, preventing memory leaks or unclosed connections. This approach is valuable for processing large logs, datasets, or files on the fly without the need to download them entirely. diff --git a/docs/02_concepts/code/01_async_support.py b/docs/02_concepts/code/01_async_support.py new file mode 100644 index 00000000..28186519 --- /dev/null +++ b/docs/02_concepts/code/01_async_support.py @@ -0,0 +1,25 @@ +import asyncio + +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + actor_client = apify_client.actor('my-actor-id') + + # Start the Actor and get the run ID + run_result = await actor_client.start() + run_client = apify_client.run(run_result['id']) + log_client = run_client.log() + + # Stream the logs + async with log_client.stream() as async_log_stream: + if async_log_stream: + async for line in async_log_stream.aiter_lines(): + print(line) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/02_collection_async.py b/docs/02_concepts/code/02_collection_async.py new file mode 100644 index 00000000..85ed3290 --- /dev/null +++ b/docs/02_concepts/code/02_collection_async.py @@ -0,0 +1,16 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + + # Collection clients do not require a parameter + actor_collection_client = apify_client.actors() + + # Create an Actor with the name: my-actor + my_actor = await actor_collection_client.create(name='my-actor') + + # List all of your Actors + actor_list = (await actor_collection_client.list()).items diff --git a/docs/02_concepts/code/02_collection_sync.py b/docs/02_concepts/code/02_collection_sync.py new file mode 100644 index 00000000..988e41e7 --- /dev/null +++ b/docs/02_concepts/code/02_collection_sync.py @@ -0,0 +1,16 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + + # Collection clients do not require a parameter + actor_collection_client = apify_client.actors() + + # Create an Actor with the name: my-actor + my_actor = actor_collection_client.create(name='my-actor') + + # List all of your Actors + actor_list = actor_collection_client.list().items diff --git a/docs/02_concepts/code/02_single_async.py b/docs/02_concepts/code/02_single_async.py new file mode 100644 index 00000000..c6b1a0ac --- /dev/null +++ b/docs/02_concepts/code/02_single_async.py @@ -0,0 +1,16 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + + # Resource clients accept an ID of the resource + actor_client = apify_client.actor('username/actor-name') + + # Fetch the 'username/actor-name' object from the API + my_actor = await actor_client.get() + + # Start the run of 'username/actor-name' and return the Run object + my_actor_run = await actor_client.start() diff --git a/docs/02_concepts/code/02_single_sync.py b/docs/02_concepts/code/02_single_sync.py new file mode 100644 index 00000000..033e54de --- /dev/null +++ b/docs/02_concepts/code/02_single_sync.py @@ -0,0 +1,16 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + + # Resource clients accept an ID of the resource + actor_client = apify_client.actor('username/actor-name') + + # Fetch the 'username/actor-name' object from the API + my_actor = actor_client.get() + + # Start the run of 'username/actor-name' and return the Run object + my_actor_run = actor_client.start() diff --git a/docs/02_concepts/code/03_nested_async.py b/docs/02_concepts/code/03_nested_async.py new file mode 100644 index 00000000..15e90772 --- /dev/null +++ b/docs/02_concepts/code/03_nested_async.py @@ -0,0 +1,22 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + + actor_client = apify_client.actor('username/actor-name') + runs_client = actor_client.runs() + + # List the last 10 runs of the Actor + actor_runs = (await runs_client.list(limit=10, desc=True)).items + + # Select the last run of the Actor that finished with a SUCCEEDED status + last_succeeded_run_client = actor_client.last_run(status='SUCCEEDED') # type: ignore[arg-type] + + # Get dataset + actor_run_dataset_client = last_succeeded_run_client.dataset() + + # Fetch items from the run's dataset + dataset_items = (await actor_run_dataset_client.list_items()).items diff --git a/docs/02_concepts/code/03_nested_sync.py b/docs/02_concepts/code/03_nested_sync.py new file mode 100644 index 00000000..c02d7600 --- /dev/null +++ b/docs/02_concepts/code/03_nested_sync.py @@ -0,0 +1,22 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + + actor_client = apify_client.actor('username/actor-name') + runs_client = actor_client.runs() + + # List the last 10 runs of the Actor + actor_runs = runs_client.list(limit=10, desc=True).items + + # Select the last run of the Actor that finished with a SUCCEEDED status + last_succeeded_run_client = actor_client.last_run(status='SUCCEEDED') # type: ignore[arg-type] + + # Get dataset + actor_run_dataset_client = last_succeeded_run_client.dataset() + + # Fetch items from the run's dataset + dataset_items = actor_run_dataset_client.list_items().items diff --git a/docs/02_concepts/code/04_error_async.py b/docs/02_concepts/code/04_error_async.py new file mode 100644 index 00000000..0057ea44 --- /dev/null +++ b/docs/02_concepts/code/04_error_async.py @@ -0,0 +1,15 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + + try: + # Try to list items from non-existing dataset + dataset_client = apify_client.dataset('not-existing-dataset-id') + dataset_items = (await dataset_client.list_items()).items + except Exception as ApifyApiError: + # The exception is an instance of ApifyApiError + print(ApifyApiError) diff --git a/docs/02_concepts/code/04_error_sync.py b/docs/02_concepts/code/04_error_sync.py new file mode 100644 index 00000000..83fc3080 --- /dev/null +++ b/docs/02_concepts/code/04_error_sync.py @@ -0,0 +1,15 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + + try: + # Try to list items from non-existing dataset + dataset_client = apify_client.dataset('not-existing-dataset-id') + dataset_items = dataset_client.list_items().items + except Exception as ApifyApiError: + # The exception is an instance of ApifyApiError + print(ApifyApiError) diff --git a/docs/02_concepts/code/05_retries_async.py b/docs/02_concepts/code/05_retries_async.py new file mode 100644 index 00000000..2fd94fec --- /dev/null +++ b/docs/02_concepts/code/05_retries_async.py @@ -0,0 +1,12 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync( + token=TOKEN, + max_retries=8, + min_delay_between_retries_millis=500, # 0.5s + timeout_secs=360, # 6 mins + ) diff --git a/docs/02_concepts/code/05_retries_sync.py b/docs/02_concepts/code/05_retries_sync.py new file mode 100644 index 00000000..25934869 --- /dev/null +++ b/docs/02_concepts/code/05_retries_sync.py @@ -0,0 +1,12 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClient( + token=TOKEN, + max_retries=8, + min_delay_between_retries_millis=500, # 0.5s + timeout_secs=360, # 6 mins + ) diff --git a/docs/02_concepts/code/06_logging_config.py b/docs/02_concepts/code/06_logging_config.py new file mode 100644 index 00000000..e31a7fda --- /dev/null +++ b/docs/02_concepts/code/06_logging_config.py @@ -0,0 +1,6 @@ +import logging + +# Configure the Apify client logger +apify_client_logger = logging.getLogger('apify_client') +apify_client_logger.setLevel(logging.DEBUG) +apify_client_logger.addHandler(logging.StreamHandler()) diff --git a/docs/02_concepts/code/06_logging_formatter.py b/docs/02_concepts/code/06_logging_formatter.py new file mode 100644 index 00000000..8bd6b44d --- /dev/null +++ b/docs/02_concepts/code/06_logging_formatter.py @@ -0,0 +1,14 @@ +import logging + +# Configure the Apify client logger +apify_client_logger = logging.getLogger('apify_client') +apify_client_logger.setLevel(logging.DEBUG) +apify_client_logger.addHandler(logging.StreamHandler()) + +# Create a custom logging formatter +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s - %(attempt)s - %(status_code)s - %(url)s' +) +handler = logging.StreamHandler() +handler.setFormatter(formatter) +apify_client_logger.addHandler(handler) diff --git a/docs/02_concepts/code/07_call_async.py b/docs/02_concepts/code/07_call_async.py new file mode 100644 index 00000000..955f7532 --- /dev/null +++ b/docs/02_concepts/code/07_call_async.py @@ -0,0 +1,14 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + actor_client = apify_client.actor('username/actor-name') + + # Start an Actor and waits for it to finish + finished_actor_run = await actor_client.call() + + # Starts an Actor and waits maximum 60s (1 minute) for the finish + actor_run = await actor_client.start(wait_for_finish=60) diff --git a/docs/02_concepts/code/07_call_sync.py b/docs/02_concepts/code/07_call_sync.py new file mode 100644 index 00000000..af790eaa --- /dev/null +++ b/docs/02_concepts/code/07_call_sync.py @@ -0,0 +1,14 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + actor_client = apify_client.actor('username/actor-name') + + # Start an Actor and waits for it to finish + finished_actor_run = actor_client.call() + + # Starts an Actor and waits maximum 60s (1 minute) for the finish + actor_run = actor_client.start(wait_for_finish=60) diff --git a/docs/02_concepts/code/08_pagination_async.py b/docs/02_concepts/code/08_pagination_async.py new file mode 100644 index 00000000..50e9d047 --- /dev/null +++ b/docs/02_concepts/code/08_pagination_async.py @@ -0,0 +1,35 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + + # Initialize the dataset client + dataset_client = apify_client.dataset('dataset-id') + + # Define the pagination parameters + limit = 1000 # Number of items per page + offset = 0 # Starting offset + all_items = [] # List to store all fetched items + + while True: + # Fetch a page of items + response = await dataset_client.list_items(limit=limit, offset=offset) + items = response.items + total = response.total + + print(f'Fetched {len(items)} items') + + # Add the fetched items to the complete list + all_items.extend(items) + + # Exit the loop if there are no more items to fetch + if offset + limit >= total: + break + + # Increment the offset for the next page + offset += limit + + print(f'Overall fetched {len(all_items)} items') diff --git a/docs/02_concepts/code/08_pagination_sync.py b/docs/02_concepts/code/08_pagination_sync.py new file mode 100644 index 00000000..3beb4fbe --- /dev/null +++ b/docs/02_concepts/code/08_pagination_sync.py @@ -0,0 +1,35 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + + # Initialize the dataset client + dataset_client = apify_client.dataset('dataset-id') + + # Define the pagination parameters + limit = 1000 # Number of items per page + offset = 0 # Starting offset + all_items = [] # List to store all fetched items + + while True: + # Fetch a page of items + response = dataset_client.list_items(limit=limit, offset=offset) + items = response.items + total = response.total + + print(f'Fetched {len(items)} items') + + # Add the fetched items to the complete list + all_items.extend(items) + + # Exit the loop if there are no more items to fetch + if offset + limit >= total: + break + + # Increment the offset for the next page + offset += limit + + print(f'Overall fetched {len(all_items)} items') diff --git a/docs/02_concepts/code/09_streaming_async.py b/docs/02_concepts/code/09_streaming_async.py new file mode 100644 index 00000000..6ff097a8 --- /dev/null +++ b/docs/02_concepts/code/09_streaming_async.py @@ -0,0 +1,14 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + run_client = apify_client.run('MY-RUN-ID') + log_client = run_client.log() + + async with log_client.stream() as log_stream: + if log_stream: + for line in log_stream.iter_lines(): + print(line) diff --git a/docs/02_concepts/code/09_streaming_sync.py b/docs/02_concepts/code/09_streaming_sync.py new file mode 100644 index 00000000..4eb0093d --- /dev/null +++ b/docs/02_concepts/code/09_streaming_sync.py @@ -0,0 +1,14 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + run_client = apify_client.run('MY-RUN-ID') + log_client = run_client.log() + + with log_client.stream() as log_stream: + if log_stream: + for line in log_stream.iter_lines(): + print(line) diff --git a/docs/03_examples/01_passing_input_to_actor.mdx b/docs/03_examples/01_passing_input_to_actor.mdx new file mode 100644 index 00000000..61769d73 --- /dev/null +++ b/docs/03_examples/01_passing_input_to_actor.mdx @@ -0,0 +1,28 @@ +--- +id: passing-input-to-actor +title: Passing input to Actor +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import StreamingAsyncExample from '!!raw-loader!./code/01_input_async.py'; +import StreamingSyncExample from '!!raw-loader!./code/01_input_sync.py'; + +The efficient way to run an Actor and retrieve results is by passing input data directly to the `call` method. This method allows you to configure the Actor's input, execute it, and either get a reference to the running Actor or wait for its completion. + +The following example demonstrates how to pass input to the `apify/instagram-hashtag-scraper` Actor and wait for it to finish. + + + + + {StreamingAsyncExample} + + + + + {StreamingSyncExample} + + + diff --git a/docs/03_examples/02_manage_tasks_for_reusable_input.mdx b/docs/03_examples/02_manage_tasks_for_reusable_input.mdx new file mode 100644 index 00000000..9865e232 --- /dev/null +++ b/docs/03_examples/02_manage_tasks_for_reusable_input.mdx @@ -0,0 +1,28 @@ +--- +id: manage-tasks-for-reusable-input +title: Manage tasks for reusable input +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import TasksAsyncExample from '!!raw-loader!./code/02_tasks_async.py'; +import TasksSyncExample from '!!raw-loader!./code/02_tasks_sync.py'; + +When you need to run multiple inputs with the same Actor, the most convenient approach is to create multiple [tasks](https://docs.apify.com/platform/actors/running/tasks), each with different input configurations. Task inputs are stored on the Apify platform when the task is created, allowing you to reuse them easily. + +The following example demonstrates how to create tasks for the `apify/instagram-hashtag-scraper` Actor with different inputs, manage task clients, and execute them asynchronously: + + + + + {TasksAsyncExample} + + + + + {TasksSyncExample} + + + diff --git a/docs/03_examples/03_retrieve_actor_data.mdx b/docs/03_examples/03_retrieve_actor_data.mdx new file mode 100644 index 00000000..f139b2a9 --- /dev/null +++ b/docs/03_examples/03_retrieve_actor_data.mdx @@ -0,0 +1,28 @@ +--- +id: retrieve-actor-data +title: Retrieve Actor data +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import RetrieveAsyncExample from '!!raw-loader!./code/03_retrieve_async.py'; +import RetrieveSyncExample from '!!raw-loader!./code/03_retrieve_sync.py'; + +Actor output data is stored in [datasets](https://docs.apify.com/platform/storage/dataset), which can be retrieved from individual Actor runs. Dataset items support pagination for efficient retrieval, and multiple datasets can be merged into a single dataset for further analysis. This merged dataset can then be exported into various formats such as CSV, JSON, XLSX, or XML. Additionally, [integrations](https://docs.apify.com/platform/integrations) provide powerful tools to automate data workflows. + +The following example demonstrates how to fetch datasets from an Actor's runs, paginate through their items, and merge them into a single dataset for unified analysis: + + + + + {RetrieveAsyncExample} + + + + + {RetrieveSyncExample} + + + diff --git a/docs/03_examples/04_integration_with_data_libraries.mdx b/docs/03_examples/04_integration_with_data_libraries.mdx new file mode 100644 index 00000000..9c4cad3e --- /dev/null +++ b/docs/03_examples/04_integration_with_data_libraries.mdx @@ -0,0 +1,28 @@ +--- +id: integration-with-data-libraries +title: Integration with data libraries +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import PandasAsyncExample from '!!raw-loader!./code/04_pandas_async.py'; +import PandasSyncExample from '!!raw-loader!./code/04_pandas_sync.py'; + +The Apify client for Python seamlessly integrates with data analysis libraries like [Pandas](https://pandas.pydata.org/). This allows you to load dataset items directly into a Pandas [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) for efficient manipulation and analysis. Pandas provides robust data structures and tools for handling large datasets, making it a powerful addition to your Apify workflows. + +The following example demonstrates how to retrieve items from the most recent dataset of an Actor run and load them into a Pandas DataFrame for further analysis: + + + + + {PandasAsyncExample} + + + + + {PandasSyncExample} + + + diff --git a/docs/03_examples/code/01_input_async.py b/docs/03_examples/code/01_input_async.py new file mode 100644 index 00000000..0310b5da --- /dev/null +++ b/docs/03_examples/code/01_input_async.py @@ -0,0 +1,23 @@ +import asyncio + +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + # Client initialization with the API token + apify_client = ApifyClientAsync(token=TOKEN) + + # Get the Actor client + actor_client = apify_client.actor('apify/instagram-hashtag-scraper') + + input_data = {'hashtags': ['rainbow'], 'resultsLimit': 20} + + # Run the Actor and wait for it to finish up to 60 seconds. + # Input is not persisted for next runs. + run_result = await actor_client.call(run_input=input_data, timeout_secs=60) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/03_examples/code/01_input_sync.py b/docs/03_examples/code/01_input_sync.py new file mode 100644 index 00000000..7f31dc1d --- /dev/null +++ b/docs/03_examples/code/01_input_sync.py @@ -0,0 +1,21 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + # Client initialization with the API token + apify_client = ApifyClient(token=TOKEN) + + # Get the Actor client + actor_client = apify_client.actor('apify/instagram-hashtag-scraper') + + input_data = {'hashtags': ['rainbow'], 'resultsLimit': 20} + + # Run the Actor and wait for it to finish up to 60 seconds. + # Input is not persisted for next runs. + run_result = actor_client.call(run_input=input_data, timeout_secs=60) + + +if __name__ == '__main__': + main() diff --git a/docs/03_examples/code/02_tasks_async.py b/docs/03_examples/code/02_tasks_async.py new file mode 100644 index 00000000..d3e962fa --- /dev/null +++ b/docs/03_examples/code/02_tasks_async.py @@ -0,0 +1,51 @@ +import asyncio + +from apify_client import ApifyClientAsync +from apify_client.clients.resource_clients import TaskClientAsync + +TOKEN = 'MY-APIFY-TOKEN' +HASHTAGS = ['zebra', 'lion', 'hippo'] + + +async def run_apify_task(client: TaskClientAsync) -> dict: + result = await client.call() + return result or {} + + +async def main() -> None: + apify_client = ApifyClientAsync(token=TOKEN) + + # Create Apify tasks + apify_tasks = list[dict]() + apify_tasks_client = apify_client.tasks() + + for hashtag in HASHTAGS: + apify_task = await apify_tasks_client.create( + name=f'hashtags-{hashtag}', + actor_id='apify/instagram-hashtag-scraper', + task_input={'hashtags': [hashtag], 'resultsLimit': 20}, + memory_mbytes=1024, + ) + apify_tasks.append(apify_task) + + print('Tasks created:', apify_tasks) + + # Create Apify task clients + apify_task_clients = list[TaskClientAsync]() + + for apify_task in apify_tasks: + task_id = apify_task['id'] + apify_task_client = apify_client.task(task_id) + apify_task_clients.append(apify_task_client) + + print('Task clients created:', apify_task_clients) + + # Execute Apify tasks + run_apify_tasks = [run_apify_task(client) for client in apify_task_clients] + task_run_results = await asyncio.gather(*run_apify_tasks) + + print('Task results:', task_run_results) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/03_examples/code/02_tasks_sync.py b/docs/03_examples/code/02_tasks_sync.py new file mode 100644 index 00000000..72437742 --- /dev/null +++ b/docs/03_examples/code/02_tasks_sync.py @@ -0,0 +1,52 @@ +from apify_client import ApifyClient +from apify_client.clients.resource_clients import TaskClient + +TOKEN = 'MY-APIFY-TOKEN' +HASHTAGS = ['zebra', 'lion', 'hippo'] + + +def run_apify_task(client: TaskClient) -> dict: + result = client.call() + return result or {} + + +def main() -> None: + apify_client = ApifyClient(token=TOKEN) + + # Create Apify tasks + apify_tasks = list[dict]() + apify_tasks_client = apify_client.tasks() + + for hashtag in HASHTAGS: + apify_task = apify_tasks_client.create( + name=f'hashtags-{hashtag}', + actor_id='apify/instagram-hashtag-scraper', + task_input={'hashtags': [hashtag], 'resultsLimit': 20}, + memory_mbytes=1024, + ) + apify_tasks.append(apify_task) + + print('Tasks created:', apify_tasks) + + # Create Apify task clients + apify_task_clients = list[TaskClient]() + + for apify_task in apify_tasks: + task_id = apify_task['id'] + apify_task_client = apify_client.task(task_id) + apify_task_clients.append(apify_task_client) + + print('Task clients created:', apify_task_clients) + + # Execute Apify tasks + task_run_results = list[dict]() + + for client in apify_task_clients: + result = run_apify_task(client) + task_run_results.append(result) + + print('Task results:', task_run_results) + + +if __name__ == '__main__': + main() diff --git a/docs/03_examples/code/03_retrieve_async.py b/docs/03_examples/code/03_retrieve_async.py new file mode 100644 index 00000000..c6e35095 --- /dev/null +++ b/docs/03_examples/code/03_retrieve_async.py @@ -0,0 +1,33 @@ +import asyncio + +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + # Client initialization with the API token + apify_client = ApifyClientAsync(token=TOKEN) + actor_client = apify_client.actor('apify/instagram-hashtag-scraper') + runs_client = actor_client.runs() + + # See pagination to understand how to get more datasets + actor_datasets = await runs_client.list(limit=20) + + datasets_client = apify_client.datasets() + merging_dataset = await datasets_client.get_or_create(name='merge-dataset') + + for dataset_item in actor_datasets.items: + # Dataset items can be handled here. Dataset items can be paginated + dataset_client = apify_client.dataset(dataset_item['id']) + dataset_items = await dataset_client.list_items(limit=1000) + + # Items can be pushed to single dataset + merging_dataset_client = apify_client.dataset(merging_dataset['id']) + await merging_dataset_client.push_items(dataset_items.items) + + # ... + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/03_examples/code/03_retrieve_sync.py b/docs/03_examples/code/03_retrieve_sync.py new file mode 100644 index 00000000..7d92dd53 --- /dev/null +++ b/docs/03_examples/code/03_retrieve_sync.py @@ -0,0 +1,31 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + # Client initialization with the API token + apify_client = ApifyClient(token=TOKEN) + actor_client = apify_client.actor('apify/instagram-hashtag-scraper') + runs_client = actor_client.runs() + + # See pagination to understand how to get more datasets + actor_datasets = runs_client.list(limit=20) + + datasets_client = apify_client.datasets() + merging_dataset = datasets_client.get_or_create(name='merge-dataset') + + for dataset_item in actor_datasets.items: + # Dataset items can be handled here. Dataset items can be paginated + dataset_client = apify_client.dataset(dataset_item['id']) + dataset_items = dataset_client.list_items(limit=1000) + + # Items can be pushed to single dataset + merging_dataset_client = apify_client.dataset(merging_dataset['id']) + merging_dataset_client.push_items(dataset_items.items) + + # ... + + +if __name__ == '__main__': + main() diff --git a/docs/03_examples/code/04_pandas_async.py b/docs/03_examples/code/04_pandas_async.py new file mode 100644 index 00000000..aecc0655 --- /dev/null +++ b/docs/03_examples/code/04_pandas_async.py @@ -0,0 +1,27 @@ +import asyncio + +import pandas as pd + +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + # Initialize the Apify client + apify_client = ApifyClientAsync(token=TOKEN) + actor_client = apify_client.actor('apify/web-scraper') + run_client = actor_client.last_run() + dataset_client = run_client.dataset() + + # Load items from last dataset run + dataset_data = await dataset_client.list_items() + + # Pass dataset items to Pandas DataFrame + data_frame = pd.DataFrame(dataset_data.items) + + print(data_frame.info) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/03_examples/code/04_pandas_sync.py b/docs/03_examples/code/04_pandas_sync.py new file mode 100644 index 00000000..a42e074f --- /dev/null +++ b/docs/03_examples/code/04_pandas_sync.py @@ -0,0 +1,25 @@ +import pandas as pd + +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + # Initialize the Apify client + apify_client = ApifyClient(token=TOKEN) + actor_client = apify_client.actor('apify/web-scraper') + run_client = actor_client.last_run() + dataset_client = run_client.dataset() + + # Load items from last dataset run + dataset_data = dataset_client.list_items() + + # Pass dataset items to Pandas DataFrame + data_frame = pd.DataFrame(dataset_data.items) + + print(data_frame.info) + + +if __name__ == '__main__': + main() diff --git a/docs/examples.md b/docs/examples.md deleted file mode 100644 index 3a517a4b..00000000 --- a/docs/examples.md +++ /dev/null @@ -1,140 +0,0 @@ ---- -sidebar_label: Examples -title: 'Code examples' ---- - -## Passing an input to the Actor - -The fastest way to get results from an Actor is to pass input directly to the `call` function. -We can set up the input, pass it to `call` function and get the reference of running Actor (or wait for finish). - -```python -from apify_client import ApifyClient - -# Client initialization with the API token -apify_client = ApifyClient(token='MY_APIFY_TOKEN') - -actor_client = apify_client.actor('apify/instagram-hashtag-scraper') - -input_data = { 'hashtags': ['rainbow'], 'resultsLimit': 20 } - -# Run the Actor and wait for it to finish up to 60 seconds. -# Input is not persisted for next runs. -run_data = actor_client.call(run_input=input_data, timeout_secs=60) -``` - -## Manipulating with tasks - -To run multiple inputs with the same Actor, most convenient way is to create multiple [tasks](https://docs.apify.com/platform/actors/running/tasks) with different inputs. -Task input is persisted on Apify platform when task is created. - -```python - -import asyncio - -from apify_client import ApifyClientAsync -from apify_client.clients.resource_clients import TaskClientAsync - -animal_hashtags = ['zebra', 'lion', 'hippo'] - - -async def run_apify_task(client: TaskClientAsync) -> dict: - result = await client.call() - return result or {} - - -async def main() -> None: - apify_client = ApifyClientAsync(token='MY_APIFY_TOKEN') - - # Create Apify tasks - - apify_tasks: list[dict] = [] - apify_tasks_client = apify_client.tasks() - - for hashtag in animal_hashtags: - apify_task = await apify_tasks_client.create( - name=f'hashtags-{hashtag}', - actor_id='apify/instagram-hashtag-scraper', - task_input={'hashtags': [hashtag], 'resultsLimit': 20}, - memory_mbytes=1024, - ) - apify_tasks.append(apify_task) - - print('Tasks created:', apify_tasks) - - # Create Apify task clients - - apify_task_clients: list[TaskClientAsync] = [] - - for apify_task in apify_tasks: - task_id = apify_task['id'] - apify_task_client = apify_client.task(task_id) - apify_task_clients.append(apify_task_client) - - print('Task clients created:', apify_task_clients) - - # Execute Apify tasks - - run_apify_tasks = [run_apify_task(client) for client in apify_task_clients] - task_run_results = await asyncio.gather(*run_apify_tasks) - - print('Task results:', task_run_results) - - -if __name__ == '__main__': - asyncio.run(main()) -``` - -## Getting latest data from an Actor, joining datasets - -Actor data are stored to [datasets](https://docs.apify.com/platform/storage/dataset). Datasets can be retrieved from Actor runs. -Dataset items can be listed with pagination. -Also, datasets can be merged together to make analysis further on with single file as dataset can be exported to various data format (CSV, JSON, XSLX, XML). -[Integrations](https://docs.apify.com/platform/integrations) can do the trick as well. - -```python -from apify_client import ApifyClient - -# Client initialization with the API token -apify_client = ApifyClient(token='MY_APIFY_TOKEN') - -actor_client = apify_client.actor('apify/instagram-hashtag-scraper') - -actor_runs = actor_client.runs() - -# See pagination to understand how to get more datasets -actor_datasets = actor_runs.list(limit=20) - -merging_dataset = apify_client.datasets().get_or_create(name='merge-dataset') - -for dataset_item in actor_datasets.items: - # Dataset items can be handled here. Dataset items can be paginated - dataset_items = apify_client.dataset(dataset_id=dataset_item['id']).list_items(limit=1000) - - # Items can be pushed to single dataset - apify_client.dataset(merging_dataset['id']).push_items(dataset_items.items) - - # ... -``` - -## Integration with data analysis libraries (Pandas) - -The Apify API client for Python can be easily integrated with data analysis libraries. -Following example demonstrates how to load items from the last dataset run and pass them to a Pandas DataFrame for further analysis. -Pandas is a data analysis library that provides data structures and functions to efficiently manipulate large datasets. - -```python -from apify_client import ApifyClient -import pandas - -# Initialize the Apify client -client = ApifyClient(token="MY_APIFY_TOKEN") - -# Load items from last dataset run -dataset_data = client.actor('apify/web-scraper').last_run().dataset().list_items() - -# Pass dataset items to Pandas DataFrame -data_frame = pandas.DataFrame(dataset_data.items) - -print(data_frame.info) -``` diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 2afcf457..00000000 --- a/docs/index.md +++ /dev/null @@ -1,345 +0,0 @@ ---- -sidebar_label: 'Getting started' -title: 'Getting started' ---- - -# Apify API client for Python - -`apify-client` is the official library to access the [Apify REST API](https://docs.apify.com/api/v2) from your Python applications. It provides useful features like automatic retries and convenience functions that improve the experience of using the Apify API. All requests and responses (including errors) are encoded in JSON format with UTF-8 encoding. - -## Pre-requisites - -`apify-client` requires Python version 3.8 or higher. Python is available for download on the [official website](https://www.python.org/). Check for your current Python version by running: - -```bash -python -V -``` - -## Installation - -You can install the client from its [PyPI listing](https://pypi.org/project/apify-client/). -To do that, run: - -```bash -pip install apify-client -``` - -## Authentication and initialization - -To use the client, you need an [API token](https://docs.apify.com/platform/integrations/api#api-token). You can find your token under [Integrations](https://console.apify.com/account/integrations) tab in Apify Console. Copy the token and initialize the client by providing the token (`MY-APIFY-TOKEN`) as a parameter to the `ApifyClient` constructor. - -```python -# import Apify client -from apify_client import ApifyClient - -# Client initialization with the API token -apify_client = ApifyClient('MY-APIFY-TOKEN') -``` - -:::warning Secure access - -The API token is used to authorize your requests to the Apify API. You can be charged for the usage of the underlying services, so do not share your API token with untrusted parties or expose it on the client side of your applications. - -::: - -## Quick start - -One of the most common use cases is starting [Actors](https://docs.apify.com/platform/actors) (serverless programs running in the [Apify cloud](https://docs.apify.com/platform)) and getting results from their [datasets](https://docs.apify.com/platform/storage/dataset) (storage) after they finish the job (usually scraping, automation processes or data processing). - -```python -from apify_client import ApifyClient - -apify_client = ApifyClient('MY-APIFY-TOKEN') - -# Start an Actor and waits for it to finish -actor_call = apify_client.actor('username/actor-name').call() - -# Get a Actor's dataset -dataset_client = apify_client.dataset(actor_call['defaultDatasetId']) - -# Lists items from the Actor's dataset -dataset_items = dataset_client.list_items().items -``` - -### Running Actors - -To start an Actor, you can use the [ActorClient](/reference/class/ActorClient) (`client.actor()`) and pass the Actor's ID (e.g. `john-doe/my-cool-actor`) to define which Actor you want to run. The Actor's ID is a combination of the username and the Actor owner’s username. You can run both your own Actors and [Actors from Apify Store](https://docs.apify.com/platform/actors/running/actors-in-store). - -#### Passing input to the Actor - -To define the Actor's input, you can pass a run input to the [`call()`](/reference/class/ActorClient#call) method. The input can be any JSON object that the Actor expects (respects the Actor's [input schema](https://docs.apify.com/platform/actors/development/actor-definition/input-schema)).The input is used to pass configuration to the Actor, such as URLs to scrape, search terms, or any other data. - -```python -from apify_client import ApifyClient - -apify_client = ApifyClient('MY-APIFY-TOKEN') - -# Define the input for the Actor -actor_input = { - 'some': 'input', -} - -# Start an Actor and waits for it to finish -actor_call = apify_client.actor('username/actor-name').call(run_input=actor_input) -``` - -### Getting results from the dataset - -To get the results from the dataset, you can use the [DatasetClient](/reference/class/DatasetClient) (`client.dataset()`) and [`list_items()`](/reference/class/DatasetClient#list_items) method. You need to pass the dataset ID to define which dataset you want to access. You can get the dataset ID from the Actor's run dictionary (represented by `defaultDatasetId`). - -```python -from apify_client import ApifyClient - -apify_client = ApifyClient('MY-APIFY-TOKEN') - -# Get dataset -dataset_client = apify_client.dataset('dataset-id') - -# Lists items from the Actor's dataset -dataset_items = dataset_client.list_items().items -``` - -:::note Dataset access - -Running an Actor might take time, depending on the Actor's complexity and the amount of data it processes. If you want only to get data and have an immediate response you should access the existing dataset of the finished [Actor run](https://docs.apify.com/platform/actors/running/runs-and-builds#runs). - -::: - -## Usage concepts - -The `ApifyClient` interface follows a generic pattern that applies to all of its components. By calling individual methods of `ApifyClient`, specific clients that target individual API resources are created. There are two types of those clients: - -- [`actorClient`](/reference/class/ActorClient): a client for the management of a single resource -- [`actorCollectionClient`](/reference/class/ActorCollectionClient): a client for the collection of resources - -```python -from apify_client import ApifyClient - -apify_client = ApifyClient('MY-APIFY-TOKEN') - -# Collection clients do not require a parameter -actor_collection_client = apify_client.actors() - -# Create an Actor with the name: my-actor -my_actor = actor_collection_client.create(name='my-actor') - -# List all of your Actors -actor_list = actor_collection_client.list().items -``` - -:::note Resource identification - -The resource ID can be either the `id` of the said resource, or a combination of your `username/resource-name`. - -::: - -```python -# Resource clients accept an ID of the resource -actor_client = apify_client.actor('username/actor-name') - -# Fetch the 'username/actor-name' object from the API -my_actor = actor_client.get() - -# Start the run of 'username/actor-name' and return the Run object -my_actor_run = actor_client.start() -``` - -### Nested clients - -Sometimes clients return other clients. That's to simplify working with nested collections, such as runs of a given Actor. - -```python -from apify_client import ApifyClient - -apify_client = ApifyClient('MY-APIFY-TOKEN') - -actor_client = apify_client.actor('username/actor-name') -runs_client = actor_client.runs() - -# List the last 10 runs of the Actor -actor_runs = runs_client.list(limit=10, desc=True).items - -# Select the last run of the Actor that finished with a SUCCEEDED status -last_succeeded_run_client = actor_client.last_run(status='SUCCEEDED') - -# Get dataset -actor_run_dataset_client = last_succeeded_run_client.dataset() - -# Fetch items from the run's dataset -dataset_items = actor_run_dataset_client.list_items().items -``` - -The quick access to `dataset` and other storage directly from the run client can be used with the [`last_run()`](/reference/class/ActorClient#last_run) method. - -## Features - -Based on the endpoint, the client automatically extracts the relevant data -and returns it in the expected format. -Date strings are automatically converted to `datetime.datetime` objects. -For exceptions, we throw an [`ApifyApiError`](/reference/class/ApifyApiError), -which wraps the plain JSON errors returned by API and enriches them with other context for easier debugging. - -```python -from apify_client import ApifyClient - -apify_client = ApifyClient('MY-APIFY-TOKEN') - -try: - # Try to list items from non-existing dataset - dataset_client = apify_client.dataset('not-existing-dataset-id') - dataset_items = dataset_client.list_items().items -except Exception as ApifyApiError: - # The exception is an instance of ApifyApiError - print(ApifyApiError) -``` - -### Retries with exponential backoff - -Network communication sometimes fails. -The client will automatically retry requests that failed due to a network error, -an internal error of the Apify API (HTTP 500+) or rate limit error (HTTP 429). -By default, it will retry up to 8 times. -First retry will be attempted after ~500ms, second after ~1000ms and so on. -You can configure those parameters using the `max_retries` and `min_delay_between_retries_millis` options -of the [`ApifyClient`](/reference/class/ApifyClient) constructor. - -```python -from apify_client import ApifyClient - -apify_client = ApifyClient( - token='MY-APIFY-TOKEN', - max_retries=8, - min_delay_between_retries_millis=500, # 0.5s - timeout_secs=360, # 6 mins -) -``` - -### Support for asynchronous usage - -The package offers an asynchronous version of the client, -[`ApifyClientAsync`](/reference/class/ApifyClientAsync), -which allows you to work with the Apify API in an asynchronous way, using the standard `async`/`await` syntax [offered by Python](https://docs.python.org/3/library/asyncio-task.html). - -For example, to run an Actor and asynchronously stream its log while it's running, you can use this snippet: - -```python -from apify_client import ApifyClientAsync -apify_client_async = ApifyClientAsync('MY-APIFY-TOKEN') - -async def main(): - run = await apify_client_async.actor('my-actor').start() - - async with apify_client_async.run(run['id']).log().stream() as async_log_stream: - if async_log_stream: - async for line in async_log_stream.aiter_lines(): - print(line) - -asyncio.run(main()) -``` - -### Logging - -The library logs some useful debug information to the `apify_client` logger -when sending requests to the Apify API. -To have them printed out to the standard output, you need to add a handler to the logger: - -```python -import logging -apify_client_logger = logging.getLogger('apify_client') -apify_client_logger.setLevel(logging.DEBUG) -apify_client_logger.addHandler(logging.StreamHandler()) -``` - -The log records have useful properties added with the `extra` argument, -like `attempt`, `status_code`, `url`, `client_method` and `resource_id`. -To print those out, you'll need to use a custom log formatter. -To learn more about log formatters and how to use them, -please refer to the official Python [documentation on logging](https://docs.python.org/3/howto/logging.html#formatters). - -### Convenience functions and options - -Some actions can't be performed by the API itself, such as indefinite waiting for an Actor run to finish (because of network timeouts). -The client provides convenient [`call()`](/reference/class/ActorClient#call) -and [`wait_for_finish()`](/reference/class/ActorClient#wait_for_finish) methods that do that. - -[Key-value store](https://docs.apify.com/platform/storage/key-value-store) records can be retrieved as objects, buffers or streams via the respective options, -dataset items can be fetched as individual objects or serialized data, or iterated asynchronously. - -```python -from apify_client import ApifyClient - -apify_client = ApifyClient('MY-APIFY-TOKEN') - -# Start an Actor and waits for it to finish -finished_actor_run = apify_client.actor('username/actor-name').call() - -# Starts an Actor and waits maximum 60s (1 minute) for the finish -actor_run = apify_client.actor('username/actor-name').start(wait_for_finish=60) -``` - -### Pagination - -Most methods named `list` or `list_something` return a [`ListPage`](/reference/class/ListPage) object, -containing properties `items`, `total`, `offset`, `count` and `limit`. -There are some exceptions though, like `list_keys` or `list_head` which paginate differently. -The results you're looking for are always stored under `items` and you can use the `limit` -property to get only a subset of results. Other properties can be available depending on the method. - -```python -from apify_client import ApifyClient - -apify_client = ApifyClient('MY-APIFY-TOKEN') - -# Resource clients accept an ID of the resource -dataset_client = apify_client.dataset('dataset-id') - -# Number of items per page -limit = 1000 -# Initial offset -offset = 0 -# List to store all items -all_items = [] - -while True: - response = dataset_client.list_items(limit=limit, offset=offset) - items = response.items - total = response.total - - print(f'Fetched {len(items)} items') - - # Merge new items with other already loaded items - all_items.extend(items) - - # If there are no more items to fetch, exit the loading - if offset + limit >= total: - break - - offset += limit - -print(f'Overall fetched {len(all_items)} items') -``` - -### Streaming resources - -Some resources (dataset items, key-value store records and logs) -support streaming the resource from the Apify API in parts, -without having to download the whole (potentially huge) resource to memory before processing it. - -The methods to stream these resources are -[`DatasetClient.stream_items()`](/reference/class/DatasetClient#stream_items), -[`KeyValueStoreClient.stream_record()`](/reference/class/KeyValueStoreClient#stream_record), -and [`LogClient.stream()`](/reference/class/LogClient#stream). - -Instead of the parsed resource, they return a raw, context-managed -[`httpx.Response`](https://www.python-httpx.org/quickstart/#streaming-responses) object, -which has to be consumed using the `with` keyword, -and automatically gets closed once you exit the `with` block, preventing memory leaks and unclosed connections. - -For example, to consume an Actor run log in a streaming fashion, you can use this snippet: - -```python -with apify_client.run('MY-RUN-ID').log().stream() as log_stream: - if log_stream: - for line in log_stream.iter_lines(): - print(line) -``` diff --git a/pyproject.toml b/pyproject.toml index 39a077b7..9926c00b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ setuptools = "~75.8.0" # setuptools are used by pytest but not explicitly re [tool.ruff] line-length = 120 +include = ["src/**/*.py", "tests/**/*.py", "scripts/**/*.py", "docs/**/*.py", "website/**/*.py"] [tool.ruff.lint] select = ["ALL"] @@ -120,6 +121,13 @@ indent-style = "space" "TRY301", # Abstract `raise` to an inner function "TID252", # Prefer absolute imports over relative imports from parent modules ] +"**/{docs,website}/**" = [ + "D", # Everything from the pydocstyle + "INP001", # File {filename} is part of an implicit namespace package, add an __init__.py + "F841", # Local variable {variable} is assigned to but never used + "N999", # Invalid module name + "T201", # print found +] [tool.ruff.lint.flake8-quotes] docstring-quotes = "double" @@ -145,7 +153,7 @@ timeout = 1200 [tool.mypy] python_version = "3.9" -files = ["scripts", "src", "tests"] +files = ["src", "tests", "scripts", "docs", "website"] check_untyped_defs = true disallow_incomplete_defs = true disallow_untyped_calls = true @@ -158,10 +166,17 @@ warn_unreachable = true warn_unused_ignores = true exclude = [] +[[tool.mypy.overrides]] +module = [ + "pandas", + "respx", +] +ignore_missing_imports = true + [tool.basedpyright] pythonVersion = "3.9" typeCheckingMode = "standard" -include = ["scripts", "src", "tests"] +include = ["src", "tests", "scripts", "docs", "website"] [tool.coverage.report] exclude_lines = [ diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index d93b2d44..3c056a51 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -27,7 +27,7 @@ module.exports = { title: 'API Client for Python', items: [ { - to: 'docs', + to: 'docs/overview/introduction', label: 'Docs', position: 'left', activeBaseRegex: '/docs(?!/changelog)', diff --git a/website/generate_module_shortcuts.py b/website/generate_module_shortcuts.py index 1e245cc1..18516ef5 100755 --- a/website/generate_module_shortcuts.py +++ b/website/generate_module_shortcuts.py @@ -1,40 +1,47 @@ #!/usr/bin/env python3 +from __future__ import annotations + import importlib import inspect import json +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from types import ModuleType -def get_module_shortcuts(module, parent_classes=None): - """Traverse a module and its submodules, and if some class is present in both a module and its submodule, register a shortcut.""" +def get_module_shortcuts(module: ModuleType, parent_classes: list | None = None) -> dict: + """Traverse a module and its submodules to identify and register shortcuts for classes.""" shortcuts = {} if parent_classes is None: parent_classes = [] + parent_module_name = '.'.join(module.__name__.split('.')[:-1]) module_classes = [] + for classname, cls in inspect.getmembers(module, inspect.isclass): module_classes.append(cls) if cls in parent_classes: shortcuts[f'{module.__name__}.{classname}'] = f'{parent_module_name}.{classname}' for _, submodule in inspect.getmembers(module, inspect.ismodule): - if (submodule.__name__.startswith('apify')): + if submodule.__name__.startswith('apify'): shortcuts.update(get_module_shortcuts(submodule, module_classes)) return shortcuts -def resolve_shortcuts(shortcuts): +def resolve_shortcuts(shortcuts: dict) -> None: """Resolve linked shortcuts. - For example, if there are shortcuts A -> B and B -> C, - resolve them to A -> C. + For example, if there are shortcuts A -> B and B -> C, resolve them to A -> C. """ for source, target in shortcuts.items(): while target in shortcuts: shortcuts[source] = shortcuts[target] - target = shortcuts[target] + target = shortcuts[target] # noqa: PLW2901 shortcuts = {} @@ -43,7 +50,7 @@ def resolve_shortcuts(shortcuts): module = importlib.import_module(module_name) module_shortcuts = get_module_shortcuts(module) shortcuts.update(module_shortcuts) - except ModuleNotFoundError: + except ModuleNotFoundError: # noqa: PERF203 pass resolve_shortcuts(shortcuts) diff --git a/website/sidebars.js b/website/sidebars.js index 969ccba9..1443dd48 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -1,12 +1,67 @@ module.exports = { sidebar: [ { - type: 'doc', - id: 'index', + type: 'category', + label: 'Overview', + collapsed: false, + items: [ + { + type: 'autogenerated', + dirName: '01_overview', + }, + ], + }, + { + type: 'category', + label: 'Concepts', + collapsed: false, + items: [ + { + type: 'autogenerated', + dirName: '02_concepts', + }, + ], + }, + { + type: 'category', + label: 'Examples', + collapsed: false, + items: [ + { + type: 'autogenerated', + dirName: '03_examples', + }, + ], }, + // { + // type: 'category', + // label: 'Upgrading', + // collapsed: false, + // items: [ + // { + // type: 'autogenerated', + // dirName: '04_upgrading', + // }, + // ], + // }, { type: 'doc', - id: 'examples', + id: 'changelog', }, ], }; + + + +// module.exports = { +// sidebar: [ +// { +// type: 'doc', +// id: 'index', +// }, +// { +// type: 'doc', +// id: 'examples', +// }, +// ], +// }; diff --git a/website/src/pages/home_page_example.py b/website/src/pages/home_page_example.py new file mode 100644 index 00000000..13236b83 --- /dev/null +++ b/website/src/pages/home_page_example.py @@ -0,0 +1,18 @@ +from apify_client import ApifyClientAsync + + +async def main() -> None: + apify_client = ApifyClientAsync('MY-APIFY-TOKEN') + + # Start an Actor and wait for it to finish. + actor_client = apify_client.actor('john-doe/my-cool-actor') + call_result = await actor_client.call() + + if call_result is None: + print('Actor run failed.') + return + + # Fetch results from the Actor run's default dataset. + dataset_client = apify_client.dataset(call_result['defaultDatasetId']) + list_items_result = await dataset_client.list_items() + print(f'Dataset: {list_items_result}') diff --git a/website/src/pages/index.js b/website/src/pages/index.js index cf975834..ee148ea3 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -7,6 +7,8 @@ import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; import useBaseUrl from '@docusaurus/useBaseUrl'; import styles from './index.module.css'; +import HomePageExample from '!!raw-loader!./home_page_example.py'; + function Hero() { return (
@@ -72,15 +74,7 @@ export default function Home() {

- {`from apify_client import ApifyClient - -apify_client = ApifyClient('MY-APIFY-TOKEN') - -# Start an Actor and wait for it to finish -actor_call = apify_client.actor('john-doe/my-cool-actor').call() - -# Fetch results from the Actor run's default dataset -dataset_items = apify_client.dataset(actor_call['defaultDatasetId']).list_items().items`} + {HomePageExample}
From 100acdad48a53a1c0d1510afeffa2b37b7ccc816 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 28 Jan 2025 13:38:49 +0100 Subject: [PATCH 2/2] update sidebars --- website/sidebars.js | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/website/sidebars.js b/website/sidebars.js index 1443dd48..e45f2828 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -50,18 +50,3 @@ module.exports = { }, ], }; - - - -// module.exports = { -// sidebar: [ -// { -// type: 'doc', -// id: 'index', -// }, -// { -// type: 'doc', -// id: 'examples', -// }, -// ], -// };