diff --git a/.gitignore b/.gitignore index 1b4d081..804401a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,13 @@ archive/ tests +# Track the Business Search contract tests; keep local test artifacts ignored. +!tests/ +tests/* +!tests/unit/ +tests/unit/* +!tests/unit/test_bright_search.py + # Byte-compiled / optimized / DLL files __pycache__/ *.py[codz] diff --git a/CHANGELOG.md b/CHANGELOG.md index a8bffcc..16edcb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Bright Data Python SDK Changelog +## Version 2.6.0 - Business Search and structured errors + +- **Business Search**: `client.search.business.companies()` and `.people()` search indexed company and professional records using structured Ludicrous queries or natural-language Instant/Smart queries. Both async and sync clients return one `BrightSearchPage` with records, request ID, paging metadata, and index coverage. +- **E-commerce search**: `client.search.ecommerce.products()` supports structured Ludicrous queries. Search calls use the shared engine without requiring zones; pagination and retries are explicit caller decisions. +- **Search request and response handling**: validate JSON payloads before dispatch, preserve available HTTP diagnostics, and retain timeout context through `RequestTimeoutError`, also catchable as `TimeoutError`. +- **Examples**: README usage, an executable example, and a notebook with simple and paginated company searches and plain-text results. +- **HTTP errors now raise typed exceptions.** `AsyncEngine` previously converted only 401/403 and handed every other failure back as a normal response, leaving each subsystem to improvise. Classification now happens once, at the single point every request passes through: `429` → the new **`RateLimitError`** (with `retry_after` parsed from the header), other 4xx/5xx → `APIError`. +- **Exceptions carry data, not just prose.** `BrightDataError` gained `status_code`, `url`, `method`, `retry_after`, `retryable` and `raw` (the response body, bounded to 4 KB). Callers can branch on the failure instead of parsing its message. +- **Fixed**: a rate-limited dataset request surfaced as an aiohttp *content-type* error, because the 429 body is a bare string and the response was parsed as JSON before its status was checked. It now raises `RateLimitError`. +- **Fixed**: a token expiring mid-poll was reported as `"Job failed with status: error"` — blaming the user's scrape for an authentication problem. `DatasetAPIClient.get_status` no longer collapses every non-200 into the status string `"error"`. +- **Results carry the failure too.** `ScrapeResult` / `CrawlResult` gained `cause`, the originating exception, populated wherever one is converted into a result. `error` remains the human-readable message: + ```python + result = await client.scrape.x.posts(url) + if not result.success and isinstance(result.cause, RateLimitError): + await asyncio.sleep(result.cause.retry_after or 60) + ``` +- **Retry decisions respect service policy.** `retry_with_backoff` honors SDK errors' `retryable` flag. Outside Bright Search, HTTP 5xx, network errors, and plain timeouts retain their default retries; 429 is excluded from the default policy. Bright Search sets `retryable=False` because an unsuccessful response can follow an executed search. An explicit `retryable_exceptions` list overrides the default policy. +- `DatasetError` now subclasses `BrightDataError` (still catchable as before). `RateLimitError` and `DataNotReadyError` are exported from `brightdata` top level. +- **Note on messages**: error messages are shorter and more uniform, with detail moved to attributes. Code matching on message *text* may need updating; use `status_code` instead. +- **Known limit**: this is status-based, so it cannot see HTTP 200 responses carrying an error in the body (SERP's inner envelope, Web Unlocker error content). Those remain string-shaped. + +--- + ## Version 2.5.2 - Safer CSV exports - **Security fix**: `export_csv()` (and `export()` for `.csv` files) now automatically escapes cell values that spreadsheet apps like Excel, Google Sheets, or LibreOffice could otherwise misinterpret as formulas, so scraped data can't unexpectedly trigger links or formulas when the exported file is opened. diff --git a/README.md b/README.md index cbee752..9e3d63a 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,34 @@ export BRIGHTDATA_API_TOKEN="your_api_token_here" **Already logged in with the CLI?** The SDK works with no configuration — it automatically falls back to the credentials stored by `brightdata login`. +## Handling errors + +Failures carry structured data, not just a message: + +```python +from brightdata import BrightDataClient, RateLimitError, APIError + +async with BrightDataClient() as client: + try: + data = await client.datasets.instagram_profiles.download(snapshot_id) + except RateLimitError as e: + await asyncio.sleep(e.retry_after or 60) # the API told us how long + except APIError as e: + print(e.status_code, e.raw) # not a message to parse +``` + +Methods that return a result instead of raising expose the same information on `cause`: + +```python +result = await client.scrape.x.posts(url) +if not result.success and isinstance(result.cause, RateLimitError): + await asyncio.sleep(result.cause.retry_after or 60) +``` + +Use `e.retryable` to decide whether repeating the call is safe. Rate limits are never +retryable — a 429 response itself consumes quota, so retrying extends the lockout; wait +`retry_after` instead. + ## Quick Start This SDK is **async-native**. A sync client is also available (see [Sync Client](#sync-client)). @@ -184,6 +212,101 @@ async with async_playwright() as pw: **When to use:** sites that require full browser automation — JS rendering, login flows, interactive clicks. For plain HTML fetches, prefer `client.scrape_url()`. +### Indexed Search: Business and E-commerce + +Business Search groups company and professional records under +`client.search.business.companies()` and `client.search.business.people()`. Use +structured queries with `mode="ludicrous"`, or natural language with +`mode="instant"` / `mode="smart"`. Natural-language queries allow up to 200 characters. + +Product search is a separate category: `client.search.ecommerce.products()`. +It accepts structured queries with `mode="ludicrous"`. Each method returns one page. + +```python +from brightdata import BrightDataClient + +# Direct Bright Search requests do not need Web Unlocker or SERP zones. +async with BrightDataClient(auto_create_zones=False) as client: + page = await client.search.business.companies( + query="US software companies with 50 to 500 employees", + mode="instant", + view="summary", + limit=10, + ) + for document in page.documents: + print(document["bright_id"], document["data"].get("name")) + print(page.req_id, page.meta["matched"], page.meta["coverage_percent"]) + + products = await client.search.ecommerce.products( + query={"and": [ + {"equals": {"retailer": "Argos"}}, + {"equals": {"sku": "4326522"}}, + ]}, + mode="ludicrous", + view="full", + ) +``` + +The same methods return `BrightSearchPage` from the synchronous client: + +```python +from brightdata import SyncBrightDataClient + +with SyncBrightDataClient(auto_create_zones=False) as client: + page = client.search.business.people( + "Product managers in the United States", + mode="instant", + view={"fields": ["name", "current_title", "url"]}, + ) +``` + +`source=None` uses the category's server default; `view=None` uses `id_only`. +E-commerce supports only `ludicrous`, and its default `retail_products_live` source +supports `id_only`, `full`, or a custom field list, but no `summary` view. Every +document includes `bright_id`; fields inside `data` are optional. `page.raw`, +`page.to_dict()`, and `page.to_json()` preserve the complete response body, +including additional fields. Response headers are separate in `page.response_headers`. + +Use explicit `offset` and `limit` for paging. Ludicrous allows limits up to 100 +and offsets up to 1000. Instant and Smart expose a 100-record window: +`offset + limit` must not exceed 100, with a maximum page size of 100 for Instant +and 10 for Smart. `matched` is generally a reached-count lower bound, not a total, and partial +coverage is reported in `coverage_percent`. Pages are separate searches against +the current index and are billed separately, including served empty pages. + +The SDK does not automatically paginate these calls. Bright Search failures set +`retryable=False`, so the generic `retry_with_backoff` helper does not replay them +under its default policy, including HTTP 5xx and response-body failures. A failed +response can follow an executed search. Callers can make another explicit request; +supplying `retryable_exceptions` to the helper is an explicit policy override. + +Redirects raise `APIError` without following another URL. HTTP errors retain +available `req_id`, `response_json`, and selected response headers. A +`RateLimitError` exposes `retry_after` for deciding when to make another request. +Interrupted successful response bodies raise `NetworkError`; malformed text or +JSON raises `APIError`. If an error body has invalid text encoding, `raw` contains +readable diagnostic text, while `response_json` and `req_id` remain unset. + +Request and body timeouts raise `RequestTimeoutError`, which is also catchable +as the built-in `TimeoutError`. It retains the URL, method, and any known HTTP +status, selected headers, and `retry_after`. Unread bodies do not supply a request +ID or parsed JSON. Legacy engine calls keep their existing timeout behavior. + +The complete request payload must be JSON-compatible. Non-finite numbers, +non-string object keys, unsupported objects, circular references, and structures +the JSON encoder cannot serialize raise `ValidationError` before dispatch. Valid +nested values and unknown field names are passed through without changing them. + +For a debug-enabled account, the optional `debug` mapping accepts the wire keys +`explain`, `benchmark`, `trace`, and `__overrides__`. These keys are omitted unless +explicitly supplied; nested null, false, and zero values are preserved. Diagnostic +output is available in `page.explain`, and its internal structure is unstable. +Flights is experimental and is not exposed by these methods. + +See [the Bright Search example](examples/bright_search.py) for async and sync usage. +The [Business Search notebook](notebooks/10_bright_search.ipynb) walks through a +simple company search and a structured search with selected fields and pagination. + ### Discover API AI-ranked web search. Unlike SERP (which returns engine-ordered results), Discover takes a `query` plus an `intent` phrase and re-ranks by relevance. Optionally extracts full page content as markdown. diff --git a/examples/bright_search.py b/examples/bright_search.py new file mode 100644 index 0000000..39f14c0 --- /dev/null +++ b/examples/bright_search.py @@ -0,0 +1,54 @@ +"""Bright Search examples using the configured Bright Data API key. + +Run this file to make one Company search. The other functions demonstrate +People and structured E-commerce search when called explicitly. +""" + +import asyncio + +from brightdata import BrightDataClient, BrightSearchPage, SyncBrightDataClient + + +async def company_search() -> BrightSearchPage: + """Make one request, leaving paging and recovery under caller control.""" + async with BrightDataClient(auto_create_zones=False) as client: + page = await client.search.business.companies( + "US software companies with 50 to 500 employees", + mode="instant", + view="summary", + limit=10, + ) + print(f"Request: {page.req_id}") + print(f"Reached: {page.meta['matched']}; coverage: {page.meta['coverage_percent']}%") + for document in page.documents: + print(document["bright_id"], document["data"].get("name")) + return page + + +def people_search_sync() -> BrightSearchPage: + """Use the same page contract from an ordinary synchronous script.""" + with SyncBrightDataClient(auto_create_zones=False) as client: + return client.search.business.people( + "Product managers in the United States", + mode="instant", + view={"fields": ["name", "current_title", "url"]}, + ) + + +async def product_search() -> BrightSearchPage: + """Send the native structured query without a separate query-builder API.""" + async with BrightDataClient(auto_create_zones=False) as client: + return await client.search.ecommerce.products( + query={ + "and": [ + {"equals": {"retailer": "Argos"}}, + {"equals": {"sku": "4326522"}}, + ] + }, + mode="ludicrous", + view="full", + ) + + +if __name__ == "__main__": + asyncio.run(company_search()) diff --git a/notebooks/10_bright_search.ipynb b/notebooks/10_bright_search.ipynb new file mode 100644 index 0000000..598a988 --- /dev/null +++ b/notebooks/10_bright_search.ipynb @@ -0,0 +1,301 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "overview", + "metadata": {}, + "source": [ + "# Business Search with the Bright Data Python SDK\n", + "\n", + "Find company records through `client.search.business.companies()` and work with the returned data directly. The Business category also provides `people()` for professional profiles.\n", + "\n", + "This notebook contains two examples:\n", + "\n", + "1. **Simple:** describe the companies you want in plain language and return three records.\n", + "2. **More detailed:** write structured filters, select output fields, fetch two pages, and combine the records without duplicate IDs.\n", + "\n", + "Running both examples makes **at most three search requests**: one Instant request and up to two Ludicrous requests. Running a request cell again makes fresh requests. Search results and timings can differ between runs.\n", + "\n", + "[Business Search documentation](https://docs.brightdata.com/products/business-search/introduction)" + ] + }, + { + "cell_type": "markdown", + "id": "setup-notes", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Use Python 3.10 or later and a checkout of this repository that includes Business Search support. Install the checkout and a notebook kernel from the repository root:\n", + "\n", + "```bash\n", + "python -m pip install -e . ipykernel\n", + "```\n", + "\n", + "Open this notebook using that Python environment. Set `BRIGHTDATA_API_TOKEN` or `BRIGHTDATA_API_KEY` in your environment or the repository's `.env` file. The account behind the key needs Business Search access. The SDK reads the credentials; they are not printed or embedded in this notebook.\n", + "\n", + "The examples use `await` directly in notebook cells. No Web Unlocker or SERP zone is needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "setup", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from time import perf_counter\n", + "\n", + "from dotenv import load_dotenv\n", + "\n", + "from brightdata import BrightDataClient\n", + "\n", + "load_dotenv()\n", + "\n", + "CLIENT_OPTIONS = {\n", + " \"auto_create_zones\": False,\n", + " \"validate_token\": False,\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "simple-search-notes", + "metadata": {}, + "source": [ + "## Example 1: Find companies with a plain-language query\n", + "\n", + "Find **US software companies with 50 to 500 employees**.\n", + "\n", + "- `mode=\"instant\"` asks Bright Data to turn the sentence into a structured search.\n", + "- `view=\"summary\"` requests a compact company record. Omitting `view` would return the identifier-only view.\n", + "- `limit=3` requests up to three records.\n", + "\n", + "This cell makes one search request." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "simple-request", + "metadata": {}, + "outputs": [], + "source": [ + "async with BrightDataClient(**CLIENT_OPTIONS) as client:\n", + " started = perf_counter()\n", + " simple_page = await client.search.business.companies(\n", + " query=\"US software companies with 50 to 500 employees\",\n", + " mode=\"instant\",\n", + " view=\"summary\",\n", + " limit=3,\n", + " )\n", + " simple_elapsed_ms = (perf_counter() - started) * 1000" + ] + }, + { + "cell_type": "markdown", + "id": "simple-results-notes", + "metadata": {}, + "source": [ + "### Read the records and request metadata\n", + "\n", + "Each item in `page.documents` has a `bright_id` and a `data` dictionary. The next cell prints every returned record as plain, indented JSON, followed by request metadata. No rich JSON viewer is required. Individual fields can be missing.\n", + "\n", + "This prints all fields returned by `view=\"summary\"`. Change the request to `view=\"full\"` if you want every stored field available for each record.\n", + "\n", + "`matched` is a lower bound on matches reached by this search, not an exhaustive total. `coverage_percent` below 100 means the response is partial. `req_id` identifies the request for troubleshooting. An empty list is a valid result.\n", + "\n", + "The elapsed time below includes the network, Bright Data processing, and SDK handling. It is one observation, not a benchmark." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "simple-results", + "metadata": { + "collapsed": false, + "scrolled": false + }, + "outputs": [], + "source": [ + "simple_companies = simple_page.documents\n", + "simple_metadata = {\n", + " \"req_id\": simple_page.req_id,\n", + " \"source\": simple_page.source,\n", + " \"returned_records\": len(simple_page.documents),\n", + " **simple_page.meta,\n", + " \"elapsed_ms\": round(simple_elapsed_ms, 1),\n", + "}\n", + "\n", + "print(f\"Returned {len(simple_companies)} company records.\")\n", + "if not simple_companies:\n", + " print(\"No companies matched this search.\")\n", + "\n", + "for index, document in enumerate(simple_companies, start=1):\n", + " name = document[\"data\"].get(\"name\") or \"Name not returned\"\n", + " print(f\"\\nCompany {index}: {name}\")\n", + " print(json.dumps(document, indent=2, ensure_ascii=False))\n", + "\n", + "print(\"\\nRequest metadata\")\n", + "print(json.dumps(simple_metadata, indent=2, ensure_ascii=False))" + ] + }, + { + "cell_type": "markdown", + "id": "advanced-search-notes", + "metadata": {}, + "source": [ + "## Example 2: Build a company shortlist with filters and pagination\n", + "\n", + "Write the conditions explicitly using `mode=\"ludicrous\"`:\n", + "\n", + "- The industry contains the word **software**.\n", + "- The headquarters country is **US**.\n", + "- The employee count recorded on LinkedIn is between **50 and 500**, inclusive.\n", + "\n", + "`query` selects matching records; `view` selects the fields returned for each record. Here we request just five useful company fields.\n", + "\n", + "Fetch at most two pages, at offsets 0 and 3, with up to three records each. Each page is a separate search against the index, so results can overlap or change. Keep the first record for each `bright_id` and record metadata for each page. Stop if a page is empty; do not use `matched` as a target for an unbounded paging loop." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "advanced-inputs", + "metadata": {}, + "outputs": [], + "source": [ + "company_query = {\n", + " \"and\": [\n", + " {\"text\": {\"industry\": \"software\"}},\n", + " {\"equals\": {\"headquarters_country_code\": \"US\"}},\n", + " {\"range\": {\"employees_in_linkedin\": {\">=\": 50, \"<=\": 500}}},\n", + " ]\n", + "}\n", + "company_fields = [\n", + " \"name\",\n", + " \"website\",\n", + " \"industry\",\n", + " \"headquarters_country_code\",\n", + " \"employees_in_linkedin\",\n", + "]\n", + "page_size = 3\n", + "max_pages = 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "advanced-request", + "metadata": {}, + "outputs": [], + "source": [ + "companies_by_id = {}\n", + "page_metadata = []\n", + "\n", + "async with BrightDataClient(**CLIENT_OPTIONS) as client:\n", + " for page_number in range(max_pages):\n", + " started = perf_counter()\n", + " page = await client.search.business.companies(\n", + " query=company_query,\n", + " mode=\"ludicrous\",\n", + " view={\"fields\": company_fields},\n", + " offset=page_number * page_size,\n", + " limit=page_size,\n", + " )\n", + " elapsed_ms = (perf_counter() - started) * 1000\n", + " page_metadata.append(\n", + " {\n", + " \"req_id\": page.req_id,\n", + " \"returned_records\": len(page.documents),\n", + " **page.meta,\n", + " \"elapsed_ms\": round(elapsed_ms, 1),\n", + " }\n", + " )\n", + "\n", + " for document in page.documents:\n", + " companies_by_id.setdefault(\n", + " document[\"bright_id\"],\n", + " {\n", + " \"bright_id\": document[\"bright_id\"],\n", + " **{field: document[\"data\"].get(field) for field in company_fields},\n", + " },\n", + " )\n", + "\n", + " if not page.documents:\n", + " break\n", + "\n", + "company_shortlist = list(companies_by_id.values())" + ] + }, + { + "cell_type": "markdown", + "id": "advanced-results-notes", + "metadata": {}, + "source": [ + "### Inspect the shortlist and the pages it came from\n", + "\n", + "The next cell prints every company in the shortlist, followed by metadata for each page. Each record includes its ID and the five fields requested by the custom view. A repeated company ID appears only once in the shortlist.\n", + "\n", + "The output is plain text, with no row, column, or string truncation applied by this notebook.\n", + "\n", + "The page cap keeps this example small; it does not establish that every matching company was retrieved. The employee count is what the source records, not a guaranteed current headcount." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "advanced-results", + "metadata": { + "collapsed": false, + "scrolled": false + }, + "outputs": [], + "source": [ + "print(f\"Fetched {len(page_metadata)} pages; kept {len(company_shortlist)} unique companies.\")\n", + "if not company_shortlist:\n", + " print(\"No companies matched the structured search.\")\n", + "\n", + "for index, company in enumerate(company_shortlist, start=1):\n", + " name = company.get(\"name\") or \"Name not returned\"\n", + " print(f\"\\nCompany {index}: {name}\")\n", + " print(json.dumps(company, indent=2, ensure_ascii=False))\n", + "\n", + "print(\"\\nPage metadata\")\n", + "for index, metadata in enumerate(page_metadata, start=1):\n", + " print(f\"\\nPage {index}\")\n", + " print(json.dumps(metadata, indent=2, ensure_ascii=False))\n", + "\n", + "if any(item[\"coverage_percent\"] < 100 for item in page_metadata):\n", + " print(\"At least one page has partial index coverage; the shortlist is incomplete.\")" + ] + }, + { + "cell_type": "markdown", + "id": "next-steps", + "metadata": {}, + "source": [ + "## Adapt the examples\n", + "\n", + "Change the country or employee range in `company_query` to search a different group of companies. Change `company_fields` to request a different set of supported output fields.\n", + "\n", + "Keep pagination deliberate: increasing `max_pages` increases the number of API requests. These examples make no automatic retries.\n", + "\n", + "The Business category also exposes `client.search.business.people()` for structured and plain-language queries. E-commerce is a separate category at `client.search.ecommerce.products()`, using structured queries with `mode=\"ludicrous\"`." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index a963515..69dd705 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ where = ["src"] [project] name = "brightdata-sdk" -version = "2.5.2" +version = "2.6.0" description = "Modern async-first Python SDK for Bright Data APIs" authors = [{name = "Bright Data", email = "support@brightdata.com"}] license = {text = "MIT"} diff --git a/src/brightdata/__init__.py b/src/brightdata/__init__.py index 0f5f7c7..9ebbd53 100644 --- a/src/brightdata/__init__.py +++ b/src/brightdata/__init__.py @@ -9,6 +9,9 @@ __version__ = "0.0.0.dev" # Export main client (async) +# Export direct Bright Search responses +from .bright_search.models import BrightSearchPage +from .bright_search.service import BrightSearchService from .browser.service import BrowserService from .client import BrightDataClient from .core.zone_manager import ZoneManager @@ -21,7 +24,10 @@ APIError, AuthenticationError, BrightDataError, + DataNotReadyError, NetworkError, + RateLimitError, + RequestTimeoutError, SSLError, ValidationError, ZoneError, @@ -127,8 +133,11 @@ "ValidationError", "AuthenticationError", "APIError", + "RateLimitError", + "DataNotReadyError", "ZoneError", "NetworkError", + "RequestTimeoutError", "SSLError", # Scraper Studio "ScraperStudioJob", @@ -138,6 +147,9 @@ "DiscoverResult", "DiscoverJob", "DiscoverSnapshot", + # Bright Search + "BrightSearchPage", + "BrightSearchService", # Services "WebUnlockerService", "BrowserService", diff --git a/src/brightdata/bright_search/__init__.py b/src/brightdata/bright_search/__init__.py new file mode 100644 index 0000000..80e996b --- /dev/null +++ b/src/brightdata/bright_search/__init__.py @@ -0,0 +1,6 @@ +"""Business and E-commerce record search through Bright Search.""" + +from .models import BrightSearchPage +from .service import BrightSearchService + +__all__ = ["BrightSearchPage", "BrightSearchService"] diff --git a/src/brightdata/bright_search/models.py b/src/brightdata/bright_search/models.py new file mode 100644 index 0000000..5cc527c --- /dev/null +++ b/src/brightdata/bright_search/models.py @@ -0,0 +1,128 @@ +"""Bright Search response models and request types.""" + +from __future__ import annotations + +import json +import math +from copy import deepcopy +from dataclasses import dataclass, field +from typing import Any, Dict, List, Literal, Optional, TypedDict, Union, cast + +BrightSearchMode = Literal["ludicrous", "instant", "smart"] +BrightSearchQuery = Union[str, Dict[str, Any]] +BrightSearchView = Union[str, Dict[str, List[str]]] + + +class BrightSearchDebug(TypedDict, total=False): + """Optional, unstable controls requiring a debug-enabled account.""" + + explain: Union[bool, int, Dict[str, Any]] + benchmark: Union[bool, int] + trace: Union[bool, int] + __overrides__: Optional[Dict[str, Any]] + + +class BrightSearchMeta(TypedDict): + """Paging metadata. matched is a reached count, not generally a total.""" + + matched: Union[int, float] + coverage_percent: Union[int, float] + offset: int + limit: int + + +class _DocumentDiagnostics(TypedDict, total=False): + score: float + + +class BrightSearchDocument(_DocumentDiagnostics): + """A record's identity and projected data; all fields inside data are optional.""" + + bright_id: str + data: Dict[str, Any] + + +@dataclass +class BrightSearchPage: + """One search response, preserving its original body and document order. + + Attributes: + raw: Original JSON body, including any additional or diagnostic fields. + Accessors refer to this mapping rather than duplicating its values. + response_headers: Selected response headers, separate from the JSON body. + + Raises: + ValueError: If the response does not contain a valid search envelope. + """ + + raw: Dict[str, Any] = field(repr=False) + response_headers: Dict[str, str] = field(default_factory=dict, repr=False) + + def __post_init__(self) -> None: + if not isinstance(self.raw, dict): + raise ValueError("response must be an object") + for name in ("req_id", "source"): + if not isinstance(self.raw.get(name), str): + raise ValueError(f"{name} must be a string") + + meta = self.raw.get("meta") + if not isinstance(meta, dict): + raise ValueError("meta must be an object") + for name in ("matched", "coverage_percent"): + value = meta.get(name) + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or (isinstance(value, float) and not math.isfinite(value)) + ): + raise ValueError(f"meta.{name} must be a finite number") + for name in ("offset", "limit"): + value = meta.get(name) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"meta.{name} must be an integer") + + documents = self.raw.get("documents") + if not isinstance(documents, list): + raise ValueError("documents must be a list") + for document in documents: + if not isinstance(document, dict): + raise ValueError("each document must be an object") + if not isinstance(document.get("bright_id"), str): + raise ValueError("document.bright_id must be a string") + if not isinstance(document.get("data"), dict): + raise ValueError("document.data must be an object") + + @property + def req_id(self) -> str: + return self.raw["req_id"] + + @property + def source(self) -> str: + return self.raw["source"] + + @property + def meta(self) -> BrightSearchMeta: + return cast(BrightSearchMeta, self.raw["meta"]) + + @property + def documents(self) -> List[BrightSearchDocument]: + return cast(List[BrightSearchDocument], self.raw["documents"]) + + @property + def explain(self) -> Any: + """Diagnostic output, if supplied; its internal shape is not stable.""" + return self.raw.get("__explain__") + + def to_dict(self) -> Dict[str, Any]: + """Copy the wire body, preserving missing keys, nulls, and extra fields.""" + return deepcopy(self.raw) + + def to_json(self, indent: Optional[int] = None) -> str: + """Serialize only the response body, without transport headers.""" + return json.dumps(self.raw, indent=indent, ensure_ascii=False) + + def __repr__(self) -> str: + return ( + f"" + ) diff --git a/src/brightdata/bright_search/namespaces.py b/src/brightdata/bright_search/namespaces.py new file mode 100644 index 0000000..43d5e39 --- /dev/null +++ b/src/brightdata/bright_search/namespaces.py @@ -0,0 +1,119 @@ +"""Public Business and E-commerce categories within client.search.""" + +from typing import TYPE_CHECKING, Any, Dict, Literal, Optional + +from .models import ( + BrightSearchDebug, + BrightSearchMode, + BrightSearchPage, + BrightSearchQuery, + BrightSearchView, +) + +if TYPE_CHECKING: + from ..serp.service import SearchService + + +class BusinessSearchService: + """Search company and professional records through client.search.business.""" + + def __init__(self, search_service: "SearchService"): + self._search_service = search_service + + async def companies( + self, + query: BrightSearchQuery, + *, + mode: BrightSearchMode, + source: Optional[str] = None, + view: Optional[BrightSearchView] = None, + offset: int = 0, + limit: int = 10, + debug: Optional[BrightSearchDebug] = None, + ) -> BrightSearchPage: + """Search company records and return one page. + + Use a structured query with mode="ludicrous", or natural language + with mode="instant" or "smart". Source and view default on the server + to linkedin_company and id_only. Pagination and retries are explicit + caller decisions. + + Example: + >>> async with BrightDataClient(auto_create_zones=False) as client: + ... page = await client.search.business.companies( + ... "US software companies", mode="instant", view="summary" + ... ) + ... for document in page.documents: + ... print(document["bright_id"], document["data"].get("name")) + """ + return await self._search_service._bright_search().search( + "company", + query, + mode=mode, + source=source, + view=view, + offset=offset, + limit=limit, + debug=debug, + ) + + async def people( + self, + query: BrightSearchQuery, + *, + mode: BrightSearchMode, + source: Optional[str] = None, + view: Optional[BrightSearchView] = None, + offset: int = 0, + limit: int = 10, + debug: Optional[BrightSearchDebug] = None, + ) -> BrightSearchPage: + """Search professional records using the same options as companies(). + + The default source is linkedin_people. All projected entity fields + are optional; matched is a reached count, not generally a total. + """ + return await self._search_service._bright_search().search( + "people", + query, + mode=mode, + source=source, + view=view, + offset=offset, + limit=limit, + debug=debug, + ) + + +class EcommerceSearchService: + """Search product records through client.search.ecommerce.""" + + def __init__(self, search_service: "SearchService"): + self._search_service = search_service + + async def products( + self, + query: Dict[str, Any], + *, + mode: Literal["ludicrous"], + source: Optional[str] = None, + view: Optional[BrightSearchView] = None, + offset: int = 0, + limit: int = 10, + debug: Optional[BrightSearchDebug] = None, + ) -> BrightSearchPage: + """Search products with a structured query and mode="ludicrous". + + The default source retail_products_live supports id_only, full, and custom + field lists; it does not publish a summary view. + """ + return await self._search_service._bright_search().search( + "ecommerce", + query, + mode=mode, + source=source, + view=view, + offset=offset, + limit=limit, + debug=debug, + ) diff --git a/src/brightdata/bright_search/service.py b/src/brightdata/bright_search/service.py new file mode 100644 index 0000000..a5d9b87 --- /dev/null +++ b/src/brightdata/bright_search/service.py @@ -0,0 +1,228 @@ +"""Direct Bright Search requests using the client's shared async engine.""" + +import asyncio +import json +from http import HTTPStatus +from typing import Any, Dict, Optional + +import aiohttp + +from ..core.engine import AsyncEngine +from ..exceptions import ( + APIError, + BrightDataError, + NetworkError, + RequestTimeoutError, + ValidationError, +) +from ..utils.http import diagnostic_response_headers, parse_retry_after +from .models import ( + BrightSearchDebug, + BrightSearchMode, + BrightSearchPage, + BrightSearchQuery, + BrightSearchView, +) + + +class BrightSearchService: + """Search a supported category and return one page directly. + + Requests are not redirected, retried, or paginated automatically. Failures + opt out of the generic retry helper because the search may have executed. + Field/operator validation, natural-language planning, and ranking remain with the API. + The owning client manages the engine's session. + """ + + _CATEGORY_MODES = { + "company": ("ludicrous", "instant", "smart"), + "people": ("ludicrous", "instant", "smart"), + "ecommerce": ("ludicrous",), + } + _MODE_CAPS = {"ludicrous": (100, 1000), "instant": (100, 100), "smart": (10, 100)} + _DEBUG_KEYS = frozenset({"explain", "benchmark", "trace", "__overrides__"}) + + def __init__(self, engine: AsyncEngine): + self.engine = engine + + async def search( + self, + category: str, + query: BrightSearchQuery, + *, + mode: BrightSearchMode, + source: Optional[str] = None, + view: Optional[BrightSearchView] = None, + offset: int = 0, + limit: int = 10, + debug: Optional[BrightSearchDebug] = None, + ) -> BrightSearchPage: + """POST one query to Company, People, or E-commerce search. + + Args: + category: company, people, or ecommerce. Flights is experimental and + is not exposed by this service. + query: A native structured object for ludicrous mode, or a string + of at most 200 characters for instant/smart. + mode: Explicit search mode. E-commerce supports ludicrous only. + source: Source name; None uses the category's server default. + view: View profile or {"fields": [...]}; None uses id_only. + offset: Page offset, up to 1000 for ludicrous. Instant and smart + require offset + limit <= 100. + limit: Page size, up to 100 (10 for smart). + debug: Explicit diagnostic wire keys for a debug-enabled account. + Nested null, false, and zero values are preserved. + + Returns: + BrightSearchPage preserving request identity and coverage metadata. + + Raises: + ValidationError: Invalid request shape, capability, or JSON payload. + APIError: Upstream failure or malformed search response. Rate limits + raise RateLimitError with retry_after; callers control recovery. + NetworkError: The response body could not be fully received. + RequestTimeoutError: Request/body timeout with available context; + also catchable as the built-in TimeoutError. + """ + payload = self._build_payload(category, query, mode, source, view, offset, limit, debug) + endpoint = f"/search/{category}" + try: + return await self._request_page(endpoint, payload) + except BrightDataError as exc: + # This includes errors raised while entering the engine context. + # An HTTP failure or an unreadable response can follow execution. + exc.retryable = False + raise + + async def _request_page(self, endpoint: str, payload: Dict[str, Any]) -> BrightSearchPage: + async with self.engine.post( + endpoint, json_data=payload, preserve_error_response=True, allow_redirects=False + ) as response: + response_headers = diagnostic_response_headers(response.headers) + context: Dict[str, Any] = { + "status_code": response.status, + "url": f"{self.engine.BASE_URL}{endpoint}", + "method": "POST", + "retryable": False, + "response_headers": response_headers, + "retry_after": parse_retry_after(response.headers), + } + try: + text = await response.text() + context["raw"] = text + data = json.loads(text) + except asyncio.TimeoutError as exc: + raise RequestTimeoutError( + f"Request timeout after {self.engine.timeout.total} seconds", **context + ) from exc + except (aiohttp.ClientError, OSError) as exc: + raise NetworkError( + f"Network error reading Bright Search response: {exc}", **context + ) from exc + except (ValueError, TypeError, RecursionError) as exc: + if isinstance(exc, UnicodeDecodeError): + context["raw"] = exc.object.decode(exc.encoding, errors="replace") + message = ( + f"Unexpected Bright Search status: {response.status}" + if response.status != HTTPStatus.OK + else "Bright Search returned an invalid JSON response" + ) + raise APIError(message, **context) from exc + + context.update( + req_id=data.get("req_id") if isinstance(data, dict) else None, + response_json=data, + ) + if response.status != HTTPStatus.OK: + raise APIError(f"Unexpected Bright Search status: {response.status}", **context) + try: + return BrightSearchPage(raw=data, response_headers=response_headers) + except ValueError as exc: + raise APIError(f"Invalid Bright Search response: {exc}", **context) from exc + + @classmethod + def _build_payload( + cls, + category: str, + query: BrightSearchQuery, + mode: BrightSearchMode, + source: Optional[str], + view: Optional[BrightSearchView], + offset: int, + limit: int, + debug: Optional[BrightSearchDebug], + ) -> Dict[str, Any]: + if not isinstance(category, str) or category not in cls._CATEGORY_MODES: + raise ValidationError("category must be company, people, or ecommerce") + if not isinstance(mode, str) or mode not in cls._CATEGORY_MODES[category]: + raise ValidationError( + f"Unsupported mode {mode!r} for {category}; " + f"expected one of {cls._CATEGORY_MODES[category]}" + ) + if mode == "ludicrous": + if not isinstance(query, dict): + raise ValidationError("ludicrous query must be a structured object") + elif not isinstance(query, str) or len(query) > 200: + raise ValidationError("instant/smart query must be a string of at most 200 characters") + + max_limit, max_offset = cls._MODE_CAPS[mode] + for name, value, maximum in (("offset", offset, max_offset), ("limit", limit, max_limit)): + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 0 + or value > maximum + ): + raise ValidationError(f"{name} must be an integer between 0 and {maximum}") + if mode in ("instant", "smart") and offset + limit > 100: + raise ValidationError("instant and smart require offset + limit <= 100") + if source is not None and not isinstance(source, str): + raise ValidationError("source must be a string") + if view is not None and not isinstance(view, str): + if ( + not isinstance(view, dict) + or set(view) != {"fields"} + or not isinstance(view["fields"], list) + or not all(isinstance(name, str) for name in view["fields"]) + ): + raise ValidationError('view must be a profile name or {"fields": [...]}') + if ( + category == "ecommerce" + and source in (None, "retail_products_live") + and view == "summary" + ): + raise ValidationError("retail_products_live does not support the summary view") + if debug is not None: + if not isinstance(debug, dict) or not cls._DEBUG_KEYS.issuperset(debug): + raise ValidationError( + "debug may contain only explain, benchmark, trace, and __overrides__" + ) + + payload: Dict[str, Any] = {"mode": mode, "query": query, "offset": offset, "limit": limit} + if source is not None: + payload["source"] = source + if view is not None: + payload["view"] = view + if debug is not None: + payload.update(debug) + cls._validate_json_payload(payload) + return payload + + @staticmethod + def _validate_json_payload(payload: Dict[str, Any]) -> None: + """Reject invalid JSON and object-key coercion without changing the input.""" + try: + # The encoder checks unsupported values, non-finite numbers, cycles, + # and nesting it cannot serialize before the request is dispatched. + json.dumps(payload, allow_nan=False) + pending = [payload] + while pending: + value = pending.pop() + if isinstance(value, dict): + if any(not isinstance(key, str) for key in value): + raise ValueError("JSON object keys must be strings") + pending.extend(value.values()) + elif isinstance(value, (list, tuple)): + pending.extend(value) + except (TypeError, ValueError, OverflowError, RecursionError) as exc: + raise ValidationError(f"Bright Search payload must be valid JSON: {exc}") from exc diff --git a/src/brightdata/client.py b/src/brightdata/client.py index b559ba7..3014d73 100644 --- a/src/brightdata/client.py +++ b/src/brightdata/client.py @@ -280,9 +280,12 @@ def scrape(self) -> ScrapeService: @property def search(self) -> SearchService: """ - Access search services (SERP API). + Access indexed search, search engines, and platform discovery. - Provides access to search engine result scrapers: + Search through the appropriate category or provider: + - client.search.business.companies(query="...", mode="instant") + - client.search.business.people(query="...", mode="instant") + - client.search.ecommerce.products(query={...}, mode="ludicrous") - client.search.google(query="...") - client.search.bing(query="...") - client.search.linkedin.jobs(...) diff --git a/src/brightdata/core/engine.py b/src/brightdata/core/engine.py index 473423c..e7b6843 100644 --- a/src/brightdata/core/engine.py +++ b/src/brightdata/core/engine.py @@ -1,6 +1,7 @@ """Async HTTP engine for Bright Data API operations.""" import asyncio +import json import ssl import warnings from http import HTTPStatus @@ -9,7 +10,15 @@ import aiohttp from .. import __version__ -from ..exceptions import AuthenticationError, NetworkError, SSLError +from ..exceptions import ( + APIError, + AuthenticationError, + NetworkError, + RateLimitError, + RequestTimeoutError, + SSLError, +) +from ..utils.http import diagnostic_response_headers, parse_retry_after, status_phrase from ..utils.ssl_helpers import get_ssl_error_message, is_ssl_certificate_error # Rate limiting support @@ -184,6 +193,9 @@ def request( json_data: Optional[Dict[str, Any]] = None, params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, + *, + preserve_error_response: bool = False, + allow_redirects: bool = True, ): """ Make an async HTTP request. @@ -196,6 +208,10 @@ def request( json_data: Optional JSON payload. params: Optional query parameters. headers: Optional additional headers. + preserve_error_response: Retain parsed error JSON, request ID, and + selected response headers in addition to the bounded raw body. + allow_redirects: Follow HTTP redirects. Disable for operations that + must make at most one upstream request. Returns: Context manager for aiohttp ClientResponse (use with async with). @@ -223,6 +239,8 @@ def request( params=params, headers=request_headers, rate_limiter=self._rate_limiter, + preserve_error_response=preserve_error_response, + allow_redirects=allow_redirects, ) def post( @@ -231,9 +249,20 @@ def post( json_data: Optional[Dict[str, Any]] = None, params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, + *, + preserve_error_response: bool = False, + allow_redirects: bool = True, ): """Make POST request. Returns context manager.""" - return self.request("POST", endpoint, json_data=json_data, params=params, headers=headers) + return self.request( + "POST", + endpoint, + json_data=json_data, + params=params, + headers=headers, + preserve_error_response=preserve_error_response, + allow_redirects=allow_redirects, + ) def get( self, @@ -358,6 +387,9 @@ def _make_request( headers: Optional[Dict[str, str]] = None, timeout: Optional[aiohttp.ClientTimeout] = None, rate_limiter: Optional[Any] = None, + preserve_error_response: bool = False, + *, + allow_redirects: bool = True, ): """ Internal method to make HTTP request with error handling. @@ -370,6 +402,8 @@ def _make_request( headers: Request headers timeout: Optional timeout override rate_limiter: Optional rate limiter to apply + preserve_error_response: Retain structured error context when opted in + allow_redirects: Whether the HTTP client may follow redirects Returns: Context manager for aiohttp ClientResponse @@ -410,26 +444,100 @@ async def __aenter__(self): params=self._params, headers=self._headers, timeout=self._timeout, + allow_redirects=allow_redirects, ) - # Check status codes that should raise exceptions - if self._response.status == HTTPStatus.UNAUTHORIZED: + status = self._response.status + + # 202 MUST pass through. It is a success for + # scraper_studio.trigger_immediate, and elsewhere it means + # "ready but still building", which fetch_result turns into + # DataNotReadyError -- the SDK's only recovery path. + if status < 400 or status == HTTPStatus.ACCEPTED: + return self._response + + body_error = None + if preserve_error_response: + try: + text = await self._response.text() + except UnicodeDecodeError as e: + # Keep the known HTTP classification even when an + # error message contains invalid text bytes. + text = e.object.decode(e.encoding, errors="replace") + body_error = e + except asyncio.TimeoutError: + raise + except (aiohttp.ClientError, OSError) as e: + # A broken error stream must not turn a known 429 + # into a generic network error and lose Retry-After. + text = "" + body_error = e + finally: + # __aexit__ is not called when entry raises. + self._response.close() + else: text = await self._response.text() await self._response.release() - raise AuthenticationError( - f"Unauthorized ({HTTPStatus.UNAUTHORIZED}): {text}" + + context = { + "status_code": status, + "url": self._url, + "method": self._method, + "raw": text, + } + if preserve_error_response: + # Extract identity while the complete body is still + # available; replacement-decoded text is diagnostic only. + error_data = None + if body_error is None: + try: + error_data = json.loads(text) + except (ValueError, RecursionError): + pass + req_id = error_data.get("req_id") if isinstance(error_data, dict) else None + context.update( + req_id=req_id, + response_json=error_data, + response_headers=diagnostic_response_headers(self._response.headers), ) - elif self._response.status == HTTPStatus.FORBIDDEN: - text = await self._response.text() - await self._response.release() - raise AuthenticationError(f"Forbidden ({HTTPStatus.FORBIDDEN}): {text}") + # A short body excerpt keeps a bare print(exc) useful; the + # full (bounded) body stays on .raw. + detail = " ".join(text.split())[:200] + suffix = f": {detail}" if detail else "" - return self._response + if status in (HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN): + raise AuthenticationError( + f"{status_phrase(status)} ({status})", **context + ) from body_error + + if status == HTTPStatus.TOO_MANY_REQUESTS: + # Keep generic retries disabled; callers can use + # Retry-After with their service's recovery policy. + raise RateLimitError( + f"Rate limited ({status}){suffix}", + retry_after=parse_retry_after(self._response.headers), + **context, + ) from body_error + + # retryable resolves from status_code: 5xx yes, 4xx no. + raise APIError( + f"Request failed (HTTP {status}){suffix}", **context + ) from body_error except asyncio.TimeoutError as e: # Must be caught before OSError — on Python 3.11+, # TimeoutError is a subclass of OSError - raise TimeoutError( - f"Request timeout after {self._timeout.total} seconds" - ) from e + message = f"Request timeout after {self._timeout.total} seconds" + if preserve_error_response: + response = self._response + headers = response.headers if response is not None else None + raise RequestTimeoutError( + message, + status_code=response.status if response is not None else None, + url=self._url, + method=self._method, + response_headers=diagnostic_response_headers(headers), + retry_after=parse_retry_after(headers), + ) from e + raise TimeoutError(message) from e except (aiohttp.ClientError, ssl.SSLError, OSError) as e: # Check for SSL certificate errors first # aiohttp wraps SSL errors in ClientConnectorError or ClientSSLError diff --git a/src/brightdata/crawler/models.py b/src/brightdata/crawler/models.py index 54d72e2..e01f53f 100644 --- a/src/brightdata/crawler/models.py +++ b/src/brightdata/crawler/models.py @@ -4,6 +4,8 @@ from datetime import datetime from typing import Any, Dict, List, Optional +from ..exceptions import BrightDataError + @dataclass class CrawlResult: @@ -22,6 +24,9 @@ class CrawlResult: trigger_sent_at: Optional[datetime] = None data_fetched_at: Optional[datetime] = None error: Optional[str] = None + # Underlying exception when one was converted into this result; `error` + # stays the message, `cause` is what code branches on. + cause: Optional[BrightDataError] = field(default=None, repr=False, compare=False) def __repr__(self) -> str: sid = f" snapshot_id={self.snapshot_id}" if self.snapshot_id else "" diff --git a/src/brightdata/crawler/service.py b/src/brightdata/crawler/service.py index 2d3821b..1302552 100644 --- a/src/brightdata/crawler/service.py +++ b/src/brightdata/crawler/service.py @@ -21,7 +21,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from ..exceptions import APIError, ValidationError +from ..exceptions import APIError, BrightDataError, ValidationError from ..utils.function_detection import get_caller_function_name from ..utils.validation import validate_url, validate_url_list from .models import CrawlJob, CrawlResult @@ -222,6 +222,7 @@ async def download( trigger_sent_at=trigger_sent_at, data_fetched_at=datetime.now(timezone.utc), error=f"Status check failed: {exc}", + cause=exc, ) if current == "ready": @@ -298,8 +299,19 @@ async def _scrape_sync( trigger_sent_at=trigger_sent_at, data_fetched_at=data_fetched_at, ) - except (ValidationError, APIError): + except ValidationError: raise + except APIError as exc: + # The engine now raises for non-2xx, so this is the same condition + # the status check above used to handle inline. Keep returning a + # CrawlResult rather than raising, which is crawl()'s contract. + return CrawlResult( + success=False, + trigger_sent_at=trigger_sent_at, + data_fetched_at=datetime.now(timezone.utc), + error=f"HTTP {exc.status_code}: {exc.raw or exc.message}", + cause=exc, + ) except Exception as exc: return CrawlResult( success=False, @@ -347,6 +359,7 @@ async def _fetch_snapshot( trigger_sent_at=trigger_sent_at, data_fetched_at=datetime.now(timezone.utc), error=f"Snapshot fetch error: {exc}", + cause=exc if isinstance(exc, BrightDataError) else None, ) @staticmethod diff --git a/src/brightdata/datasets/base.py b/src/brightdata/datasets/base.py index 2937102..2e180cf 100644 --- a/src/brightdata/datasets/base.py +++ b/src/brightdata/datasets/base.py @@ -3,21 +3,44 @@ """ import asyncio +import json import time from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional +from ..exceptions import APIError, BrightDataError, RateLimitError from .models import DatasetMetadata, SnapshotStatus if TYPE_CHECKING: from ..core.engine import AsyncEngine -class DatasetError(Exception): +class DatasetError(BrightDataError): """Error related to dataset operations.""" pass +def _as_dataset_error(exc: APIError, what: str) -> DatasetError: + """ + Re-type an engine-raised APIError as a DatasetError, preserving context. + + The engine classifies every non-2xx centrally, so by the time a failure + reaches this layer it is already structured. Callers of the datasets API + catch DatasetError, though, so convert rather than let a sibling type + escape. RateLimitError is deliberately NOT converted: it is the more + specific, actionable type and users are told to catch it directly. + """ + return DatasetError( + f"{what} failed (HTTP {exc.status_code})", + status_code=exc.status_code, + url=exc.url, + method=exc.method, + retry_after=exc.retry_after, + retryable=exc.retryable, + raw=exc.raw, + ) + + class BaseDataset: """ Base class for all dataset types. @@ -95,11 +118,17 @@ async def __call__( if records_limit is not None: payload["records_limit"] = records_limit - async with self._engine.post_to_url( - f"{self.BASE_URL}/datasets/filter", - json_data=payload, - ) as response: - data = await response.json() + try: + async with self._engine.post_to_url( + f"{self.BASE_URL}/datasets/filter", + json_data=payload, + ) as response: + body = await response.text() + data = json.loads(body) if body.strip() else {} + except RateLimitError: + raise + except APIError as exc: + raise _as_dataset_error(exc, "Filter request") from exc if "snapshot_id" not in data: error_msg = ( @@ -142,10 +171,16 @@ async def get_status(self, snapshot_id: str) -> SnapshotStatus: Returns: SnapshotStatus with status field: "scheduled", "building", "ready", or "failed" """ - async with self._engine.get_from_url( - f"{self.BASE_URL}/datasets/snapshots/{snapshot_id}" - ) as response: - data = await response.json() + try: + async with self._engine.get_from_url( + f"{self.BASE_URL}/datasets/snapshots/{snapshot_id}" + ) as response: + body = await response.text() + data = json.loads(body) if body.strip() else {} + except RateLimitError: + raise + except APIError as exc: + raise _as_dataset_error(exc, "Snapshot status check") from exc return SnapshotStatus.from_dict(data) async def download( @@ -197,8 +232,6 @@ async def download( f"{self.BASE_URL}/datasets/snapshots/{snapshot_id}/download", params={"format": format}, ) as response: - import json - # Check for HTTP errors if response.status >= 400: error_text = await response.text() diff --git a/src/brightdata/exceptions/__init__.py b/src/brightdata/exceptions/__init__.py index a0960c0..44e6472 100644 --- a/src/brightdata/exceptions/__init__.py +++ b/src/brightdata/exceptions/__init__.py @@ -6,6 +6,8 @@ BrightDataError, DataNotReadyError, NetworkError, + RateLimitError, + RequestTimeoutError, SSLError, ValidationError, ZoneError, @@ -16,8 +18,10 @@ "ValidationError", "AuthenticationError", "APIError", + "RateLimitError", "DataNotReadyError", "ZoneError", "NetworkError", + "RequestTimeoutError", "SSLError", ] diff --git a/src/brightdata/exceptions/errors.py b/src/brightdata/exceptions/errors.py index fc476d3..754b4d3 100644 --- a/src/brightdata/exceptions/errors.py +++ b/src/brightdata/exceptions/errors.py @@ -2,13 +2,71 @@ from __future__ import annotations +from typing import Any + +# Bound how much of a response body an exception retains. Error pages from +# proxies (trust_env=True is on) or CDNs can be hundreds of KB, and exceptions +# are routinely captured by loggers and error trackers. +RAW_LIMIT = 4096 + + +def _truncate(raw: Any) -> Any: + """Bound a retained response body, marking it when shortened.""" + if isinstance(raw, str) and len(raw) > RAW_LIMIT: + return f"{raw[:RAW_LIMIT]}…[truncated {len(raw) - RAW_LIMIT} bytes]" + return raw + class BrightDataError(Exception): - """Base exception for all Bright Data errors.""" + """ + Base exception for all Bright Data errors. + + Carries structured context alongside the human message so callers can react + programmatically instead of parsing the message string. + + Attributes: + message: Short human-readable description. + status_code: HTTP status, when the failure came from a response. + url: URL of the failing request, when known. + method: HTTP method of the failing request, when known. + retry_after: Seconds parsed from a Retry-After header, when present. + retryable: Whether the generic helper should repeat the operation. + Defaults from the HTTP status (5xx). NetworkError defaults + to True to match legacy retry behavior. Services can opt out. + raw: Response body, truncated to RAW_LIMIT. + req_id: Request identifier extracted before truncation, when opted in. + response_json: Parsed error response, when explicitly retained. + response_headers: Selected response headers, when explicitly retained. + """ - def __init__(self, message: str, *args, **kwargs): + def __init__( + self, + message: str, + *args, + status_code: int | None = None, + url: str | None = None, + method: str | None = None, + retry_after: float | None = None, + retryable: bool | None = None, + raw: Any = None, + req_id: str | None = None, + response_json: Any = None, + response_headers: dict[str, str] | None = None, + **kwargs, + ): super().__init__(message, *args) self.message = message + self.status_code = status_code + self.url = url + self.method = method + self.retry_after = retry_after + if retryable is None: + retryable = status_code is not None and status_code >= 500 + self.retryable = retryable + self.raw = _truncate(raw) + self.req_id = req_id if isinstance(req_id, str) else None + self.response_json = response_json + self.response_headers = response_headers if response_headers is not None else {} class ValidationError(BrightDataError): @@ -34,11 +92,27 @@ def __init__( *args, **kwargs, ): - super().__init__(message, *args, **kwargs) - self.status_code = status_code + # status_code stays the second positional parameter for backwards + # compatibility (APIError("msg", 429) is legal); drop any duplicate + # keyword before forwarding it to the base. + kwargs.pop("status_code", None) + super().__init__(message, *args, status_code=status_code, **kwargs) self.response_text = response_text +class RateLimitError(APIError): + """ + HTTP 429 — request rate or quota exceeded. + + Excluded from the generic retry helper. Use `retry_after` to decide how + long to pause before an explicit retry according to the service's contract. + """ + + def __init__(self, message: str, *args, **kwargs): + kwargs["retryable"] = False # not overridable — see the class docstring + super().__init__(message, *args, **kwargs) + + class DataNotReadyError(BrightDataError): """Data is not ready yet (HTTP 202). Should retry.""" @@ -52,7 +126,16 @@ class ZoneError(BrightDataError): class NetworkError(BrightDataError): - """Network connectivity issue.""" + """Network failure; retain legacy retries unless explicitly disabled.""" + + def __init__(self, message: str, *args, retryable: bool | None = None, **kwargs): + super().__init__( + message, *args, retryable=True if retryable is None else retryable, **kwargs + ) + + +class RequestTimeoutError(BrightDataError, TimeoutError): + """Request/body timeout with context, also catchable as built-in TimeoutError.""" pass diff --git a/src/brightdata/models.py b/src/brightdata/models.py index 86773aa..83f41b0 100644 --- a/src/brightdata/models.py +++ b/src/brightdata/models.py @@ -8,6 +8,8 @@ from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Union +from .exceptions import BrightDataError + StatusType = Literal["ready", "error", "timeout", "in_progress"] PlatformType = Optional[Literal["linkedin", "amazon", "chatgpt", "instagram", "facebook"]] SearchEngineType = Optional[Literal["google", "bing", "yandex"]] @@ -27,6 +29,10 @@ class BaseResult: error: Error message if operation failed, None otherwise. trigger_sent_at: Timestamp when the trigger request was sent to Bright Data (UTC-aware). data_fetched_at: Timestamp when data was fetched after polling completed (UTC-aware). + cause: The underlying exception when one was converted into this result. + `error` stays the human-readable message; `cause` is what code + branches on -- e.g. `isinstance(result.cause, RateLimitError)` + then `result.cause.retry_after`. """ success: bool @@ -34,6 +40,7 @@ class BaseResult: error: Optional[str] = None trigger_sent_at: Optional[datetime] = None data_fetched_at: Optional[datetime] = None + cause: Optional[BrightDataError] = field(default=None, repr=False, compare=False) def __post_init__(self) -> None: """Validate data after initialization.""" @@ -83,6 +90,17 @@ def to_dict(self) -> Dict[str, Any]: result[key] = value.isoformat() elif isinstance(value, list) and value and isinstance(value[0], datetime): result[key] = [v.isoformat() if isinstance(v, datetime) else v for v in value] + + # An exception is not JSON-serializable; keep a summary so to_json() + # and save_to_file() still work. + if self.cause is not None: + result["cause"] = { + "type": type(self.cause).__name__, + "message": self.cause.message, + "status_code": self.cause.status_code, + "retry_after": self.cause.retry_after, + "retryable": self.cause.retryable, + } return result def to_json(self, indent: Optional[int] = None) -> str: diff --git a/src/brightdata/scrapers/api_client.py b/src/brightdata/scrapers/api_client.py index 2b2bc02..88a6aa9 100644 --- a/src/brightdata/scrapers/api_client.py +++ b/src/brightdata/scrapers/api_client.py @@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional from ..core.engine import AsyncEngine -from ..exceptions import APIError, DataNotReadyError +from ..exceptions import APIError, DataNotReadyError, RateLimitError class DatasetAPIClient: @@ -108,12 +108,22 @@ async def get_status(self, snapshot_id: str) -> str: """ url = f"{self.STATUS_URL}/{snapshot_id}" - async with self.engine.get_from_url(url) as response: - if response.status == HTTPStatus.OK: + # Do NOT collapse transport failures into a job status. Returning + # "error" here made an expired token look like a failed scrape; the + # engine raises for non-2xx, and re-raising keeps the endpoint context. + try: + async with self.engine.get_from_url(url) as response: data = await response.json() return data.get("status", "unknown") - else: - return "error" + except RateLimitError: + raise + except APIError as exc: + raise APIError( + f"Status check failed (HTTP {exc.status_code})", + status_code=exc.status_code, + raw=exc.raw, + retryable=exc.retryable, + ) from exc async def fetch_result(self, snapshot_id: str, format: str = "json") -> Any: """ diff --git a/src/brightdata/scrapers/workflow.py b/src/brightdata/scrapers/workflow.py index e228e5b..8054f9c 100644 --- a/src/brightdata/scrapers/workflow.py +++ b/src/brightdata/scrapers/workflow.py @@ -89,6 +89,7 @@ async def execute( url="", status="error", error=f"Trigger failed: {str(e)}", + cause=e, platform=self.platform_name, method="web_scraper", trigger_sent_at=trigger_sent_at, diff --git a/src/brightdata/serp/service.py b/src/brightdata/serp/service.py index c1ea7c3..d699e79 100644 --- a/src/brightdata/serp/service.py +++ b/src/brightdata/serp/service.py @@ -1,8 +1,7 @@ """ -Search service namespace (SERP API). +Search namespace for Business, E-commerce, search engines, and platform discovery. -Provides access to search engine result scrapers with normalized -data across different search engines. +Groups indexed record search, search engine results, and platform discovery. All methods are async-only. For sync usage, use SyncBrightDataClient. """ @@ -11,6 +10,8 @@ from ..models import SearchResult if TYPE_CHECKING: + from ..bright_search.namespaces import BusinessSearchService, EcommerceSearchService + from ..bright_search.service import BrightSearchService from ..client import BrightDataClient from ..scrapers.amazon.search import AmazonSearchScraper from ..scrapers.chatgpt.search import ChatGPTSearchService @@ -25,10 +26,10 @@ class SearchService: """ - Search service namespace (SERP API). + Search namespace with Business and E-commerce categories. - Provides access to search engine result scrapers with normalized - data across different search engines. + Use business.companies() and business.people() for business records, + ecommerce.products() for products, or the existing provider search methods. Example: >>> # Google search @@ -55,6 +56,35 @@ def __init__(self, client: "BrightDataClient"): self._tiktok_search: Optional["TikTokSearchScraper"] = None self._youtube_search: Optional["YouTubeSearchScraper"] = None self._pinterest_search = None + self._bright_search_service: Optional["BrightSearchService"] = None + self._business: Optional["BusinessSearchService"] = None + self._ecommerce: Optional["EcommerceSearchService"] = None + + def _bright_search(self) -> "BrightSearchService": + self._client._ensure_initialized() + if self._bright_search_service is None: + from ..bright_search.service import BrightSearchService + + self._bright_search_service = BrightSearchService(self._client.engine) + return self._bright_search_service + + @property + def business(self) -> "BusinessSearchService": + """Search company and professional records with companies() and people().""" + if self._business is None: + from ..bright_search.namespaces import BusinessSearchService + + self._business = BusinessSearchService(self) + return self._business + + @property + def ecommerce(self) -> "EcommerceSearchService": + """Search product records with products().""" + if self._ecommerce is None: + from ..bright_search.namespaces import EcommerceSearchService + + self._ecommerce = EcommerceSearchService(self) + return self._ecommerce async def google( self, diff --git a/src/brightdata/sync_client.py b/src/brightdata/sync_client.py index 447ee1b..ff79b5c 100644 --- a/src/brightdata/sync_client.py +++ b/src/brightdata/sync_client.py @@ -6,10 +6,11 @@ import asyncio import logging -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional logger = logging.getLogger(__name__) +from .bright_search.models import BrightSearchMode, BrightSearchPage, BrightSearchQuery from .browser.service import BrowserService from .client import BrightDataClient from .discover.models import DiscoverResult, DiscoverSnapshot @@ -263,7 +264,7 @@ def datasets(self) -> "SyncDatasetsClient": @property def search(self) -> "SyncSearchService": """Access search services (sync).""" - if self._search is None: + if self._search is None or self._search._loop is not self._loop: self._search = SyncSearchService(self._async_client.search, self._loop) return self._search @@ -965,6 +966,47 @@ def profiles_by_username_fetch(self, snapshot_id): # ============================================================================ +class SyncBusinessSearchService: + """Synchronous company and professional search through search.business.""" + + def __init__(self, search_service: "SyncSearchService"): + self._search_service = search_service + + def companies( + self, query: BrightSearchQuery, *, mode: BrightSearchMode, **kwargs + ) -> BrightSearchPage: + """Return one company page; options match search.business.companies().""" + self._search_service._ensure_initialized() + return self._search_service._loop.run_until_complete( + self._search_service._async.business.companies(query, mode=mode, **kwargs) + ) + + def people( + self, query: BrightSearchQuery, *, mode: BrightSearchMode, **kwargs + ) -> BrightSearchPage: + """Return one professional page; options match search.business.people().""" + self._search_service._ensure_initialized() + return self._search_service._loop.run_until_complete( + self._search_service._async.business.people(query, mode=mode, **kwargs) + ) + + +class SyncEcommerceSearchService: + """Synchronous product search through search.ecommerce.""" + + def __init__(self, search_service: "SyncSearchService"): + self._search_service = search_service + + def products( + self, query: Dict[str, Any], *, mode: Literal["ludicrous"], **kwargs + ) -> BrightSearchPage: + """Return one product page; options match search.ecommerce.products().""" + self._search_service._ensure_initialized() + return self._search_service._loop.run_until_complete( + self._search_service._async.ecommerce.products(query, mode=mode, **kwargs) + ) + + class SyncSearchService: """Sync wrapper for SearchService - COMPLETE.""" @@ -978,6 +1020,38 @@ def __init__(self, async_service, loop): self._tiktok = None self._youtube = None + self._business: Optional["SyncBusinessSearchService"] = None + self._ecommerce: Optional["SyncEcommerceSearchService"] = None + + def _ensure_initialized(self) -> None: + if self._loop is None or self._loop.is_closed(): + raise RuntimeError( + "SyncBrightDataClient not initialized. " + "Use: with SyncBrightDataClient() as client: ..." + ) + try: + asyncio.get_running_loop() + except RuntimeError: + return + raise RuntimeError( + "SyncBrightDataClient cannot be used inside an async context. " + "Use BrightDataClient with async/await instead." + ) + + @property + def business(self) -> "SyncBusinessSearchService": + """Business Search: companies() and people().""" + if self._business is None: + self._business = SyncBusinessSearchService(self) + return self._business + + @property + def ecommerce(self) -> "SyncEcommerceSearchService": + """E-commerce search: products().""" + if self._ecommerce is None: + self._ecommerce = SyncEcommerceSearchService(self) + return self._ecommerce + def google(self, query, **kwargs) -> SearchResult: """Search Google.""" return self._loop.run_until_complete(self._async.google(query, **kwargs)) diff --git a/src/brightdata/utils/http.py b/src/brightdata/utils/http.py new file mode 100644 index 0000000..b07e1b4 --- /dev/null +++ b/src/brightdata/utils/http.py @@ -0,0 +1,67 @@ +"""HTTP header helpers shared by the engine and the datasets layer.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from http import HTTPStatus +from typing import Any, Dict, Optional + + +def diagnostic_response_headers(headers: Any) -> Dict[str, str]: + """Keep documented diagnostics without retaining unrelated response headers.""" + if not headers: + return {} + allowed = {"retry-after", "x-debug-trace", "x-debug-trace-page"} + return { + name.lower(): value + for name, value in headers.items() + if isinstance(name, str) and name.lower() in allowed and isinstance(value, str) + } + + +def parse_retry_after(headers: Any) -> Optional[float]: + """ + Parse a Retry-After header into seconds. + + Supports both documented forms: delta-seconds ("30") and an HTTP-date + ("Wed, 21 Oct 2015 07:28:00 GMT"). Returns None when the header is absent + or unparseable — callers treat that as "the server did not say". + """ + if not headers: + return None + try: + value = headers.get("Retry-After") + except AttributeError: + return None + if not value: + return None + + value = str(value).strip() + try: + return max(0.0, float(value)) + except ValueError: + pass + + try: + when = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + if when is None: + return None + if when.tzinfo is None: + when = when.replace(tzinfo=timezone.utc) + return max(0.0, (when - datetime.now(timezone.utc)).total_seconds()) + + +def status_phrase(status: int) -> str: + """ + Human-readable phrase for an HTTP status. + + HTTPStatus(...) raises ValueError on non-standard codes, and those are real + in the wild (Cloudflare uses 520-526), so fall back rather than raise. + """ + try: + return HTTPStatus(status).phrase + except ValueError: + return "HTTP error" diff --git a/src/brightdata/utils/polling.py b/src/brightdata/utils/polling.py index e916ea6..1185d73 100644 --- a/src/brightdata/utils/polling.py +++ b/src/brightdata/utils/polling.py @@ -15,7 +15,7 @@ from typing import Any, Awaitable, Callable, List from ..constants import DEFAULT_POLL_INTERVAL, DEFAULT_POLL_TIMEOUT -from ..exceptions import DataNotReadyError +from ..exceptions import BrightDataError, DataNotReadyError from ..models import ScrapeResult @@ -108,6 +108,7 @@ async def poll_until_ready( url="", status="error", error=f"Failed to get status: {str(e)}", + cause=e if isinstance(e, BrightDataError) else None, snapshot_id=snapshot_id, platform=platform, method=method or "web_scraper", @@ -135,6 +136,7 @@ async def poll_until_ready( url="", status="error", error=f"Failed to fetch results: {str(e)}", + cause=e if isinstance(e, BrightDataError) else None, snapshot_id=snapshot_id, platform=platform, method=method or "web_scraper", diff --git a/src/brightdata/utils/retry.py b/src/brightdata/utils/retry.py index 47a69c5..2b1becb 100644 --- a/src/brightdata/utils/retry.py +++ b/src/brightdata/utils/retry.py @@ -3,11 +3,28 @@ import asyncio from typing import Awaitable, Callable, List, Optional, Type, TypeVar -from ..exceptions import APIError, NetworkError +from ..exceptions import BrightDataError T = TypeVar("T") +def is_retryable(exc: Exception) -> bool: + """ + Decide whether repeating an operation is safe and worthwhile. + + Honor SDK errors' retryable flag, including explicit service policies. + NetworkError defaults to retryable and plain built-in timeouts keep their + legacy retry behavior. Other errors need an explicit retryable decision. + A failed response does not prove that the server did not execute the work. + RateLimitError stays non-retryable; callers can inspect retry_after. + """ + if isinstance(exc, BrightDataError): + return exc.retryable + if isinstance(exc, TimeoutError): + return True + return False + + async def retry_with_backoff( func: Callable[[], Awaitable[T]], max_retries: int = 3, @@ -25,7 +42,8 @@ async def retry_with_backoff( initial_delay: Initial delay in seconds max_delay: Maximum delay in seconds backoff_factor: Multiplier for exponential backoff - retryable_exceptions: List of exception types to retry on + retryable_exceptions: Explicit exception-type override of the default + retry policy. Supplying this list opts into retries for those types. Returns: Result from successful function call @@ -33,9 +51,6 @@ async def retry_with_backoff( Raises: Last exception if all retries fail """ - if retryable_exceptions is None: - retryable_exceptions = [NetworkError, TimeoutError, APIError] - last_exception = None delay = initial_delay @@ -46,7 +61,10 @@ async def retry_with_backoff( last_exception = e # Check if exception is retryable - if not any(isinstance(e, exc_type) for exc_type in retryable_exceptions): + if retryable_exceptions is None: + if not is_retryable(e): + raise + elif not any(isinstance(e, exc_type) for exc_type in retryable_exceptions): raise # Don't retry on last attempt diff --git a/src/brightdata/web_unlocker/async_client.py b/src/brightdata/web_unlocker/async_client.py index 8ba837e..fd7655e 100644 --- a/src/brightdata/web_unlocker/async_client.py +++ b/src/brightdata/web_unlocker/async_client.py @@ -142,16 +142,21 @@ async def get_status(self, zone: str, response_id: str, customer: Optional[str] if customer: params["customer"] = customer - async with self.engine.get_from_url( - f"{self.engine.BASE_URL}{self.FETCH_ENDPOINT}", params=params - ) as response: - if response.status == 200: - return "ready" - elif response.status == 202: - return "pending" - else: - # Any other status (4xx, 5xx) is treated as error + try: + async with self.engine.get_from_url( + f"{self.engine.BASE_URL}{self.FETCH_ENDPOINT}", params=params + ) as response: + if response.status == 200: + return "ready" + elif response.status == 202: + return "pending" + # Unreachable in practice: the engine raises for other statuses. return "error" + except APIError: + # This method's contract is a status string, not an exception -- its + # caller is a poll loop. The engine now raises for 4xx/5xx, so map + # that back to the documented "error" value. + return "error" async def fetch_result( self, diff --git a/tests/unit/test_bright_search.py b/tests/unit/test_bright_search.py new file mode 100644 index 0000000..7b0f52d --- /dev/null +++ b/tests/unit/test_bright_search.py @@ -0,0 +1,951 @@ +"""Business Search request, response, and transport contract tests. + +Cover page envelopes, projections, paging, diagnostics, and failure identity +using mocks or a loopback HTTP peer, without live API requests. +""" + +import asyncio +import json +from contextlib import asynccontextmanager +from copy import deepcopy +from types import SimpleNamespace +from typing import Any, Dict, Literal, get_type_hints +from unittest.mock import AsyncMock, MagicMock, patch + +import aiohttp +import pytest +from multidict import CIMultiDict + +from brightdata import BrightDataClient, BrightSearchPage, BrightSearchService, SyncBrightDataClient +from brightdata.core.engine import AsyncEngine +from brightdata.exceptions import ( + APIError, + AuthenticationError, + BrightDataError, + NetworkError, + RateLimitError, + ValidationError, +) +from brightdata.sync_client import SyncSearchService +from brightdata.utils.retry import is_retryable, retry_with_backoff +from tests.conftest import MockContextManager, MockResponse + +PUBLIC_SEARCHES = [ + ("business", "companies", "company"), + ("business", "people", "people"), + ("ecommerce", "products", "ecommerce"), +] + + +@pytest.fixture +def body(): + return { + "req_id": "request_123", + "source": "linkedin_company", + "meta": {"matched": 42, "coverage_percent": 100, "offset": 0, "limit": 10}, + "documents": [ + {"bright_id": "northstar", "data": {"name": "Northstar Systems"}}, + {"bright_id": "second", "data": {}}, + ], + } + + +def response(body, status=200, headers=None): + return MockResponse(status, json_data=body, text_data=json.dumps(body), headers=headers) + + +@pytest.fixture +def service(mock_engine, body): + mock_engine.BASE_URL = AsyncEngine.BASE_URL + mock_engine.post.return_value = MockContextManager(response(body)) + return BrightSearchService(mock_engine) + + +@pytest.mark.parametrize( + "category,mode", + [ + ("company", "ludicrous"), + ("company", "instant"), + ("company", "smart"), + ("people", "ludicrous"), + ("people", "instant"), + ("people", "smart"), + ("ecommerce", "ludicrous"), + ], +) +async def test_supported_requests_are_one_native_post(service, mock_engine, body, category, mode): + query = {"equals": {"country_code": "US"}} if mode == "ludicrous" else "US companies" + page = await service.search(category, query, mode=mode) + + mock_engine.post.assert_called_once_with( + f"/search/{category}", + json_data={"mode": mode, "query": query, "offset": 0, "limit": 10}, + preserve_error_response=True, + allow_redirects=False, + ) + mock_engine.get.assert_not_called() + assert page.to_dict() == body + assert page.req_id == "request_123" + + +async def test_structured_queries_custom_fields_and_debug_values_are_not_rewritten( + service, mock_engine +): + query = { + "and": [ + {"text": {"role": {"value": "engineer", "mode": "all-or-top-bm25", "top-k": 100}}}, + {"or": [{"equals": {"city": "Berlin"}}, {"in": {"country_code": ["DE", "GB"]}}]}, + {"not": {"equals": {"future_field": False}}}, + {"range": {"experience_years": {">=": 0, "<": 10}}}, + ] + } + debug = { + "explain": {"trace": 1}, + "trace": False, + "benchmark": 0, + "__overrides__": { + "hornet": {"score": {"parameters": {"name_weight": None, "about_weight": 0}}}, + "reranker": {"enabled": False}, + }, + } + before = deepcopy((query, debug)) + view = {"fields": ["name", "future_released_field"]} + await service.search( + "people", + query, + mode="ludicrous", + source="future_people", + view=view, + offset=20, + limit=20, + debug=debug, + ) + sent = mock_engine.post.call_args.kwargs["json_data"] + assert sent == { + "query": query, + "mode": "ludicrous", + "source": "future_people", + "view": view, + "offset": 20, + "limit": 20, + **debug, + } + assert (query, debug) == before + + +async def test_explicit_false_and_null_debug_are_retained(service, mock_engine): + await service.search( + "company", "query", mode="instant", debug={"explain": False, "__overrides__": None} + ) + sent = mock_engine.post.call_args.kwargs["json_data"] + assert sent["explain"] is False + assert "__overrides__" in sent and sent["__overrides__"] is None + assert "trace" not in sent and "benchmark" not in sent + + +@pytest.mark.parametrize( + "changes", + [ + {"category": "flights"}, + {"category": "ecom"}, # The API accepts ecommerce, not the old spelling. + {"category": ["company"]}, + {"mode": "fast"}, + {"mode": None}, + {"category": "ecommerce", "mode": "instant"}, + {"category": "ecommerce", "mode": "smart"}, + {"mode": "ludicrous", "query": "text"}, + {"mode": "ludicrous", "query": []}, + {"query": {}}, + {"query": None}, + {"query": "x" * 201}, + {"offset": -1}, + {"offset": True}, + {"offset": 1.5}, + {"offset": "1"}, + {"offset": 1001}, + {"mode": "smart", "offset": 101}, + {"limit": -1}, + {"limit": False}, + {"limit": 1.5}, + {"limit": None}, + {"limit": 101}, + {"mode": "smart", "limit": 11}, + {"source": ["linkedin_company"]}, + {"view": ["name"]}, + {"view": {"fields": "name"}}, + {"view": {"fields": [1]}}, + {"view": {"other": []}}, + {"view": {"fields": [], "other": 1}}, + {"category": "ecommerce", "mode": "ludicrous", "query": {}, "view": "summary"}, + { + "category": "ecommerce", + "mode": "ludicrous", + "query": {}, + "source": "retail_products_live", + "view": "summary", + }, + {"debug": False}, + {"debug": {"query": {}}}, + {"debug": {"mode": "smart"}}, + {"debug": {"limit": 100}}, + {"debug": {"source": "other"}}, + ], +) +async def test_invalid_outer_requests_fail_before_network(service, mock_engine, changes): + arguments = {"category": "company", "query": "query", "mode": "instant", **changes} + with pytest.raises(ValidationError): + await service.search(**arguments) + mock_engine.post.assert_not_called() + + +@pytest.mark.parametrize("target", ["query", "debug"]) +@pytest.mark.parametrize( + "value", + [ + pytest.param(float("nan"), id="nan"), + pytest.param(float("inf"), id="infinity"), + pytest.param(float("-inf"), id="negative-infinity"), + pytest.param({1: "value"}, id="integer-key"), + pytest.param({False: "value"}, id="boolean-key"), + pytest.param({None: "value"}, id="null-key"), + pytest.param({1.5: "value"}, id="float-key"), + pytest.param({("key",): "value"}, id="tuple-key"), + pytest.param(b"bytes", id="bytes"), + pytest.param({1, 2}, id="set"), + pytest.param(object(), id="object"), + ], +) +async def test_nested_non_json_values_fail_before_dispatch(service, mock_engine, target, value): + query = {"future_field": [value]} if target == "query" else {} + debug = {"__overrides__": {"future_option": [value]}} if target == "debug" else None + with pytest.raises(ValidationError): + await service.search("company", query, mode="ludicrous", debug=debug) + mock_engine.post.assert_not_called() + + +@pytest.mark.parametrize("target", ["query", "debug"]) +@pytest.mark.parametrize("container", ["list", "dict"]) +async def test_circular_payload_fails_before_dispatch(service, mock_engine, target, container): + if container == "list": + value = [] + value.append(value) + else: + value = {} + value["self"] = value + query = {"future_field": value} if target == "query" else {} + debug = {"__overrides__": value} if target == "debug" else None + with pytest.raises(ValidationError): + await service.search("company", query, mode="ludicrous", debug=debug) + mock_engine.post.assert_not_called() + assert (value[0] if container == "list" else value["self"]) is value + + +async def test_deep_payload_fails_before_dispatch(service, mock_engine): + query = {} + # Python 3.12's JSON encoder accepts deeper nesting than earlier versions. + for _ in range(10_000): + query = {"nested": query} + with pytest.raises(ValidationError): + await service.search("company", query, mode="ludicrous") + mock_engine.post.assert_not_called() + + +async def test_json_validation_preserves_nested_values_and_shared_containers(service, mock_engine): + shared = {"future_field": [None, False, 0, [], {}, "text", 1.5]} + query = {"future_query": [shared, shared]} + debug = {"__overrides__": {"future_debug": shared}, "trace": False, "benchmark": 0} + before = deepcopy((query, debug)) + + await service.search("company", query, mode="ludicrous", debug=debug) + + sent = mock_engine.post.call_args.kwargs["json_data"] + assert (query, debug) == before + assert sent["query"] == before[0] + assert sent["query"]["future_query"][0] is shared + assert sent["__overrides__"] == before[1]["__overrides__"] + assert sent["trace"] is False and sent["benchmark"] == 0 + assert json.loads(json.dumps(sent, allow_nan=False)) == sent + + +@pytest.mark.parametrize("category", ["company", "people"]) +@pytest.mark.parametrize("mode", ["instant", "smart"]) +@pytest.mark.parametrize("offset,limit", [(91, 10), (100, 1)]) +async def test_natural_language_window_fails_before_network( + service, mock_engine, category, mode, offset, limit +): + with pytest.raises(ValidationError, match=r"offset.*limit.*100"): + await service.search(category, "US companies", mode=mode, offset=offset, limit=limit) + mock_engine.post.assert_not_called() + + +@pytest.mark.parametrize( + "mode,limit,offset", + [ + ("ludicrous", 100, 1000), + ("instant", 100, 0), + ("instant", 1, 99), + ("smart", 10, 90), + ("smart", 1, 99), + ], +) +async def test_documented_boundaries_are_inclusive(service, mock_engine, mode, limit, offset): + query = {"equals": {"headquarters_country_code": "US"}} if mode == "ludicrous" else "x" * 200 + await service.search("company", query, mode=mode, limit=limit, offset=offset) + sent = mock_engine.post.call_args.kwargs["json_data"] + assert sent["limit"] == limit and sent["offset"] == offset + assert sent["query"] == query + + +@pytest.mark.parametrize("mode,query", [("instant", ""), ("ludicrous", {})]) +async def test_unspecified_zero_and_empty_semantics_are_left_to_server( + service, mock_engine, mode, query +): + await service.search("company", query, mode=mode, limit=0) + assert mock_engine.post.call_args.kwargs["json_data"]["limit"] == 0 + assert mock_engine.post.call_args.kwargs["json_data"]["query"] == query + + +async def test_future_source_view_rules_are_server_owned(service, mock_engine): + await service.search("ecommerce", {}, mode="ludicrous", source="future_retail", view="summary") + assert mock_engine.post.call_args.kwargs["json_data"]["view"] == "summary" + + +def test_page_preserves_sparse_fields_order_diagnostics_and_unknown_fields(body): + body["meta"].update(matched=100, coverage_percent=72.5, future_meta="value") + body["documents"][0].update(score=0.8, future_document="value") + body["documents"][0]["data"]["future_field"] = None + body["documents"][1]["data"] = {"name": "Second"} + body["__explain__"] = {"future_diagnostic": {"value": None}} + body["response_headers"] = {"future_wire_field": True} + page = BrightSearchPage(body, {"x-debug-trace-page": "https://trace.example/page"}) + + assert page.to_dict() == body + assert json.loads(page.to_json()) == body + assert page.source == body["source"] + assert page.meta["matched"] == 100 and page.meta["coverage_percent"] == 72.5 + assert [doc["bright_id"] for doc in page.documents] == ["northstar", "second"] + assert "score" not in page.documents[1] + assert "future_field" not in page.documents[1]["data"] + assert page.explain == body["__explain__"] + assert page.response_headers != page.raw["response_headers"] + + copied = page.to_dict() + copied["documents"][0]["data"]["name"] = "Changed" + assert page.documents[0]["data"]["name"] == "Northstar Systems" + page.raw["meta"]["matched"] = 43 + assert page.meta["matched"] == 43 # no duplicate cached metadata + + +@pytest.mark.parametrize("documents", [[], [{"bright_id": "id_only", "data": {}}]]) +def test_empty_pages_and_id_only_documents_are_valid(body, documents): + body["documents"] = documents + page = BrightSearchPage(body) + assert page.documents == documents + assert page.explain is None + assert "__explain__" not in page.to_dict() + + +@pytest.mark.parametrize( + "field,value", + [ + ("req_id", None), + ("source", 1), + ("meta", None), + ("documents", {}), + ("documents", [None]), + ("documents", [{"bright_id": 1, "data": {}}]), + ("documents", [{"bright_id": "id", "data": None}]), + ], +) +async def test_malformed_envelope_raises_api_error(service, mock_engine, body, field, value): + body[field] = value + mock_engine.post.return_value = MockContextManager(response(body)) + with pytest.raises(APIError, match="Invalid Bright Search response") as error: + await service.search("company", "query", mode="instant") + assert error.value.response_json == body + assert error.value.status_code == 200 and error.value.retryable is False + mock_engine.post.assert_called_once() + + +@pytest.mark.parametrize( + "field,value", + [ + ("matched", True), + ("matched", "42"), + ("matched", float("nan")), + ("coverage_percent", None), + ("coverage_percent", float("inf")), + ("offset", False), + ("limit", 10.5), + ], +) +def test_malformed_meta_does_not_become_successful_empty_data(body, field, value): + body["meta"][field] = value + with pytest.raises(ValueError, match=f"meta.{field}"): + BrightSearchPage(body) + + +@pytest.mark.parametrize("data", [[], None, "text", {}]) +async def test_invalid_or_missing_response_envelope(service, mock_engine, data): + mock_engine.post.return_value = MockContextManager(response(data)) + with pytest.raises(APIError, match="Invalid Bright Search response"): + await service.search("company", "query", mode="instant") + mock_engine.post.assert_called_once() + + +async def test_non_json_success_remains_an_api_failure(service, mock_engine): + mock_engine.post.return_value = MockContextManager( + MockResponse(200, text_data="error") + ) + with pytest.raises(APIError, match="invalid JSON") as error: + await service.search("company", "query", mode="instant") + assert error.value.raw == "error" + mock_engine.post.assert_called_once() + + +async def test_unexpected_accepted_response_does_not_start_polling(service, mock_engine, body): + mock_engine.post.return_value = MockContextManager(response(body, status=202)) + with pytest.raises(APIError, match="Unexpected Bright Search status") as error: + await service.search("company", "query", mode="instant") + assert error.value.req_id == "request_123" + mock_engine.post.assert_called_once() + mock_engine.get.assert_not_called() + + +@pytest.mark.parametrize( + "status,error_type", + [ + (400, APIError), + (401, AuthenticationError), + (403, AuthenticationError), + (429, RateLimitError), + (500, APIError), + (502, APIError), + ], +) +async def test_engine_preserves_error_identity_before_truncation_without_retry(status, error_type): + data = {"__explain__": {"detail": "x" * 6000}, "req_id": "tail_request_id", "error": "failed"} + headers = CIMultiDict( + { + "Retry-After": "2", + "X-Debug-Trace": "https://trace.example/raw", + "X-Debug-Trace-Page": "https://trace.example/page", + "Set-Cookie": "omit-me", + } + ) + engine = AsyncEngine("test_token") + async with engine: + engine._session.request = AsyncMock(return_value=response(data, status, headers)) + with pytest.raises(error_type) as error: + await BrightSearchService(engine).search("company", "query", mode="instant") + engine._session.request.assert_awaited_once() + assert "preserve_error_response" not in engine._session.request.call_args.kwargs + + exc = error.value + assert exc.req_id == "tail_request_id" and exc.response_json == data + assert "tail_request_id" not in exc.raw and "truncated" in exc.raw + assert exc.response_headers == { + "retry-after": "2", + "x-debug-trace": "https://trace.example/raw", + "x-debug-trace-page": "https://trace.example/page", + } + assert exc.status_code == status and exc.method == "POST" + assert exc.url == "https://api.brightdata.com/search/company" + assert exc.retryable is False + if status == 429: + assert exc.retry_after == 2 + + +async def test_error_capture_is_opt_in_and_non_json_errors_keep_their_type(): + engine = AsyncEngine("test_token") + async with engine: + engine._session.request = AsyncMock(return_value=response({"req_id": "legacy"}, 400)) + with pytest.raises(APIError) as legacy: + async with engine.post("/legacy"): + pass + assert legacy.value.req_id is None + assert legacy.value.response_json is None + assert legacy.value.response_headers == {} + + engine._session.request = AsyncMock( + return_value=MockResponse(429, text_data="rate limited") + ) + with pytest.raises(RateLimitError) as non_json: + await BrightSearchService(engine).search("people", "query", mode="instant") + assert non_json.value.raw == "rate limited" + assert non_json.value.response_json is None + assert non_json.value.req_id is None + engine._session.request.assert_awaited_once() + + +@pytest.mark.parametrize( + "failure,expected", + [(asyncio.TimeoutError(), TimeoutError), (aiohttp.ClientError("offline"), NetworkError)], +) +async def test_transport_failures_are_not_replayed(failure, expected): + engine = AsyncEngine("test_token") + async with engine: + engine._session.request = AsyncMock(side_effect=failure) + with pytest.raises(expected) as error: + await retry_with_backoff( + lambda: BrightSearchService(engine).search("company", "query", mode="instant"), + max_retries=1, + initial_delay=0, + ) + engine._session.request.assert_awaited_once() + assert not is_retryable(error.value) + + +@pytest.mark.parametrize("status", [200, 429]) +async def test_body_timeout_keeps_known_context(status): + failure = asyncio.TimeoutError("body timed out") + resp = response({}, status, CIMultiDict({"Retry-After": "2", "X-Debug-Trace": "trace"})) + resp.text = AsyncMock(side_effect=failure) + async with AsyncEngine("test_token") as engine: + engine._session.request = AsyncMock(return_value=resp) + with pytest.raises(TimeoutError) as error: + await retry_with_backoff( + lambda: BrightSearchService(engine).search("company", "query", mode="instant"), + max_retries=1, + initial_delay=0, + ) + engine._session.request.assert_awaited_once() + assert resp.closed + exc = error.value + assert isinstance(exc, BrightDataError) + assert exc.status_code == status + assert exc.response_headers == {"retry-after": "2", "x-debug-trace": "trace"} + assert exc.retry_after == 2 + assert exc.url == "https://api.brightdata.com/search/company" and exc.method == "POST" + assert exc.req_id is None and exc.response_json is None + assert exc.__cause__ is failure + assert not is_retryable(exc) + + +async def test_legacy_request_timeout_remains_plain_and_retryable(): + failure = asyncio.TimeoutError("connection timed out") + async with AsyncEngine("test_token") as engine: + engine._session.request = AsyncMock(side_effect=failure) + + async def request(): + async with engine.post("/legacy"): + pytest.fail("The mocked request should time out") + + with pytest.raises(TimeoutError) as error: + await retry_with_backoff(request, max_retries=1, initial_delay=0) + assert engine._session.request.await_count == 2 + assert type(error.value) is TimeoutError + assert error.value.__cause__ is failure + assert is_retryable(error.value) + + +@pytest.mark.parametrize("client_type", [BrightDataClient, SyncBrightDataClient]) +def test_search_categories_are_lazy_cached_and_separate(client_type): + client = client_type(token="test_token_12345", auto_create_zones=False) + search = client.search + assert search.business is search.business + assert search.ecommerce is search.ecommerce + assert search.business is not search.ecommerce + assert callable(search.business.companies) + assert callable(search.business.people) + assert callable(search.ecommerce.products) + for name in ("company", "people", "ecom"): + assert not hasattr(search, name) + assert not hasattr(search.business, "company") + assert not hasattr(search.business, "products") + assert not hasattr(search.ecommerce, "companies") + assert not hasattr(search.ecommerce, "people") + async_search = search if client_type is BrightDataClient else search._async + assert async_search._bright_search_service is None + + +@pytest.mark.parametrize("client_type", [BrightDataClient, SyncBrightDataClient]) +def test_ecommerce_product_annotations_match_supported_inputs(client_type): + client = client_type(token="test_token_12345", auto_create_zones=False) + hints = get_type_hints(client.search.ecommerce.products) + assert hints["query"] == Dict[str, Any] + assert hints["mode"] == Literal["ludicrous"] + assert hints["return"] is BrightSearchPage + + +@pytest.mark.parametrize("namespace,name,category", PUBLIC_SEARCHES) +async def test_async_facade_uses_shared_engine_and_no_zones(body, namespace, name, category): + client = BrightDataClient(token="test_token_12345", auto_create_zones=False) + search = getattr(client.search, namespace) + with patch( + "brightdata.client.ZoneManager.ensure_required_zones", new_callable=AsyncMock + ) as zones: + async with client: + assert search is getattr(client.search, namespace) + session = client.engine._session + session.request = AsyncMock( + return_value=response(body, headers={"X-Debug-Trace-Page": "trace"}) + ) + page = await getattr(search, name)({}, mode="ludicrous", view="full") + assert page.to_dict() == body + assert page.response_headers == {"x-debug-trace-page": "trace"} + assert client.search._bright_search_service.engine is client.engine + cached = client.search._bright_search_service + await getattr(search, name)({}, mode="ludicrous") + assert client.search._bright_search_service is cached + assert not session.closed + assert session.request.await_count == 2 + assert session.request.call_args.kwargs["url"].endswith(f"/search/{category}") + assert session.closed + zones.assert_not_awaited() + + +@pytest.mark.parametrize("namespace,name,category", PUBLIC_SEARCHES) +async def test_async_facade_requires_initialization(namespace, name, category): + client = BrightDataClient(token="test_token_12345", auto_create_zones=False) + with pytest.raises(RuntimeError, match="not initialized"): + await getattr(getattr(client.search, namespace), name)({}, mode="ludicrous") + + +@pytest.mark.parametrize("namespace,name,category", PUBLIC_SEARCHES) +def test_sync_facade_has_the_same_wire_contract_and_page(body, namespace, name, category): + client = SyncBrightDataClient(token="test_token_12345", auto_create_zones=False) + before_enter = getattr(client.search, namespace) + with pytest.raises(RuntimeError, match="not initialized"): + getattr(before_enter, name)({}, mode="ludicrous") + retired = None + with patch( + "brightdata.client.ZoneManager.ensure_required_zones", new_callable=AsyncMock + ) as zones: + for _ in range(2): + with client: + search = getattr(client.search, namespace) + assert search is getattr(client.search, namespace) + assert search is not before_enter + if retired is not None: + assert search is not retired + with pytest.raises(RuntimeError, match="not initialized"): + getattr(retired, name)({}, mode="ludicrous") + session = client._async_client.engine._session + session.request = AsyncMock(return_value=response(body)) + page = getattr(search, name)( + {}, + mode="ludicrous", + source="future_source", + view={"fields": ["name"]}, + limit=20, + ) + assert isinstance(page, BrightSearchPage) and page.to_dict() == body + session.request.assert_awaited_once() + sent = session.request.call_args.kwargs + assert sent["url"].endswith(f"/search/{category}") + assert sent["json"] == { + "mode": "ludicrous", + "query": {}, + "source": "future_source", + "view": {"fields": ["name"]}, + "offset": 0, + "limit": 20, + } + retired = search + assert session.closed + zones.assert_not_awaited() + with pytest.raises(RuntimeError, match="not initialized"): + getattr(getattr(client.search, namespace), name)({}, mode="ludicrous") + + +@pytest.mark.parametrize("namespace,name,category", PUBLIC_SEARCHES) +async def test_sync_call_in_async_context_fails_before_creating_a_coroutine( + namespace, name, category +): + loop = asyncio.new_event_loop() + async_service = MagicMock() + try: + service = SyncSearchService(async_service, loop) + with pytest.raises(RuntimeError, match="async context"): + getattr(getattr(service, namespace), name)({}, mode="ludicrous") + getattr(getattr(async_service, namespace), name).assert_not_called() + finally: + loop.close() + + +@pytest.fixture +def wire_server(monkeypatch, body): + """Real HTTP responses, including bodies aiohttp must reject while reading.""" + monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost") + monkeypatch.setenv("no_proxy", "127.0.0.1,localhost") + + @asynccontextmanager + async def serve(status=200, payload=b"", *, headers=None, length=None, hold_open=False): + requests = [] + handlers = set() + disconnected = asyncio.Event() + + async def handle(reader, writer): + task = asyncio.current_task() + handlers.add(task) + try: + request = (await reader.readuntil(b"\r\n\r\n")).decode("ascii").split("\r\n") + method, path, _ = request[0].split(" ") + request_headers = { + name.lower(): value.strip() + for line in request[1:] + if ":" in line + for name, value in [line.split(":", 1)] + } + data = await reader.readexactly(int(request_headers.get("content-length", 0))) + requests.append((method, path, data)) + response_status, response_body = status, payload + response_headers = { + "Content-Type": "application/json", + "Content-Length": str(len(payload) if length is None else length), + "Connection": "close", + "X-Debug-Trace-Page": "local-trace", + "Retry-After": "2", + **(headers or {}), + } + if path == "/redirected": + response_status, response_body = 200, json.dumps(body).encode() + response_headers["Content-Length"] = str(len(response_body)) + head = f"HTTP/1.1 {response_status} Test\r\n" + "".join( + f"{key}: {value}\r\n" for key, value in response_headers.items() + ) + writer.write(head.encode("ascii") + b"\r\n" + response_body) + await writer.drain() + if hold_open: + await reader.read() + finally: + writer.close() + await writer.wait_closed() + disconnected.set() + handlers.discard(task) + + server = await asyncio.start_server(handle, "127.0.0.1", 0) + try: + yield SimpleNamespace( + url=f"http://127.0.0.1:{server.sockets[0].getsockname()[1]}", + requests=requests, + disconnected=disconnected, + ) + finally: + server.close() + await server.wait_closed() + pending = list(handlers) + for task in pending: + task.cancel() + await asyncio.gather(*pending, return_exceptions=True) + + return serve + + +def assert_wire_error_context(exc, peer, status): + assert exc.status_code == status + assert exc.url == f"{peer.url}/search/company" + assert exc.method == "POST" + assert exc.response_headers == {"x-debug-trace-page": "local-trace", "retry-after": "2"} + assert len(peer.requests) == 1 + assert peer.requests[0][:2] == ("POST", "/search/company") + + +@pytest.mark.parametrize("status", [500, 502]) +async def test_wire_http_failure_is_not_replayed_through_generic_backoff(wire_server, status): + payload = b'{"req_id": "failed-search", "error": "upstream failed"}' + async with wire_server(status, payload) as peer: + async with BrightDataClient(token="local_test_token", auto_create_zones=False) as client: + client.engine.BASE_URL = peer.url + with pytest.raises(APIError) as error: + await retry_with_backoff( + lambda: client.search.business.companies("query", mode="instant"), + max_retries=1, + initial_delay=0, + ) + assert_wire_error_context(error.value, peer, status) + assert error.value.req_id == "failed-search" + assert error.value.retryable is False + + +@pytest.mark.parametrize("status", [500, 502]) +async def test_wire_legacy_http_failure_still_retries(wire_server, status): + async with wire_server(status, b'{"error": "upstream failed"}') as peer: + async with AsyncEngine("local_test_token") as engine: + engine.BASE_URL = peer.url + + async def request(): + async with engine.post("/legacy"): + pytest.fail("The HTTP error should be raised during entry") + + with pytest.raises(APIError) as error: + await retry_with_backoff(request, max_retries=1, initial_delay=0) + assert len(peer.requests) == 2 + assert error.value.retryable is True + + +@pytest.mark.parametrize("status", [301, 302, 303, 307, 308]) +async def test_wire_redirect_is_an_error_without_another_request(wire_server, status): + async with wire_server(status, headers={"Location": "/redirected"}) as peer: + async with BrightDataClient(token="local_test_token", auto_create_zones=False) as client: + client.engine.BASE_URL = peer.url + with pytest.raises(APIError, match="Unexpected Bright Search status") as error: + await client.search.business.companies("query", mode="instant") + assert_wire_error_context(error.value, peer, status) + assert error.value.retryable is False + + +@pytest.mark.parametrize("status,next_method", [(302, "GET"), (307, "POST")]) +async def test_wire_legacy_post_still_follows_redirects(wire_server, body, status, next_method): + async with wire_server(status, headers={"Location": "/redirected"}) as peer: + async with AsyncEngine("local_test_token") as engine: + engine.BASE_URL = peer.url + async with engine.post("/legacy", json_data={"query": "original"}) as resp: + assert await resp.json() == body + assert [request[:2] for request in peer.requests] == [ + ("POST", "/legacy"), + (next_method, "/redirected"), + ] + + +@pytest.mark.parametrize( + "payload,length,error_type,cause_type", + [ + (b'{"req_id": "partial"', 100, NetworkError, aiohttp.ClientPayloadError), + (b'{"req_id": "ok", "data": "\xff"}', None, APIError, UnicodeDecodeError), + # Exceed the decoder's recursion guard on every supported Python version. + (b"[" * 10_000 + b"0" + b"]" * 10_000, None, APIError, RecursionError), + ], + ids=["truncated", "invalid-utf8", "deep-json"], +) +async def test_wire_body_failures_use_sdk_errors( + wire_server, payload, length, error_type, cause_type +): + async with wire_server(payload=payload, length=length) as peer: + async with BrightDataClient(token="local_test_token", auto_create_zones=False) as client: + client.engine.BASE_URL = peer.url + with pytest.raises(error_type) as error: + await retry_with_backoff( + lambda: client.search.business.companies("query", mode="instant"), + max_retries=1, + initial_delay=0, + ) + assert_wire_error_context(error.value, peer, 200) + assert isinstance(error.value.__cause__, cause_type) + assert error.value.retryable is False + + +@pytest.mark.parametrize( + "status,error_type", + [ + (400, APIError), + (401, AuthenticationError), + (403, AuthenticationError), + (429, RateLimitError), + (500, APIError), + (502, APIError), + ], +) +@pytest.mark.parametrize( + "payload", + [ + b'{"req_id": "recoverable", "error": "\xff"}', + b'{"req_id": "corrupt\xff", "error": "failed"}', + ], + ids=["outside-request-id", "inside-request-id"], +) +async def test_wire_invalid_error_encoding_keeps_http_classification( + wire_server, status, error_type, payload +): + async with wire_server(status, payload) as peer: + async with BrightDataClient(token="local_test_token", auto_create_zones=False) as client: + client.engine.BASE_URL = peer.url + with pytest.raises(error_type) as error: + await client.search.business.companies("query", mode="instant") + assert_wire_error_context(error.value, peer, status) + assert error.value.req_id is None + assert error.value.response_json is None + assert "\ufffd" in error.value.raw + assert isinstance(error.value.__cause__, UnicodeDecodeError) + if status == 429: + assert error.value.retry_after == 2 + assert error.value.retryable is False + + +async def test_wire_valid_unicode_error_json_remains_authoritative(wire_server): + data = {"req_id": "valid-\ufffd", "error": "failed"} + async with wire_server(500, json.dumps(data).encode()) as peer: + async with BrightDataClient(token="local_test_token", auto_create_zones=False) as client: + client.engine.BASE_URL = peer.url + with pytest.raises(APIError) as error: + await client.search.business.companies("query", mode="instant") + assert_wire_error_context(error.value, peer, 500) + assert error.value.req_id == data["req_id"] + assert error.value.response_json == data + assert error.value.__cause__ is None + + +@pytest.mark.parametrize("status,error_type", [(429, RateLimitError), (500, APIError)]) +async def test_wire_truncated_error_keeps_known_status_and_headers(wire_server, status, error_type): + async with wire_server(status, b'{"req_id": "incomplete"', length=100) as peer: + async with BrightDataClient(token="local_test_token", auto_create_zones=False) as client: + client.engine.BASE_URL = peer.url + with pytest.raises(error_type) as error: + await client.search.business.companies("query", mode="instant") + assert_wire_error_context(error.value, peer, status) + assert isinstance(error.value.__cause__, aiohttp.ClientPayloadError) + assert error.value.req_id is None and error.value.response_json is None + if status == 429: + assert error.value.retry_after == 2 and error.value.retryable is False + + +async def test_wire_legacy_error_decoding_remains_strict(wire_server): + async with wire_server(400, b"\xff") as peer: + async with AsyncEngine("local_test_token") as engine: + engine.BASE_URL = peer.url + with pytest.raises(UnicodeDecodeError): + async with engine.post("/legacy"): + pytest.fail("Invalid encoding should fail before entry") + assert len(peer.requests) == 1 + + +@pytest.mark.parametrize("status", [200, 429]) +async def test_wire_body_timeout_stays_timeout_and_closes_response(wire_server, status): + async with wire_server(status, length=100, hold_open=True) as peer: + async with BrightDataClient(token="local_test_token", auto_create_zones=False) as client: + client.engine.BASE_URL = peer.url + client.engine.timeout = aiohttp.ClientTimeout(total=0.2) + with pytest.raises(TimeoutError) as error: + await retry_with_backoff( + lambda: client.search.business.companies("query", mode="instant"), + max_retries=1, + initial_delay=0, + ) + assert_wire_error_context(error.value, peer, status) + assert error.value.retry_after == 2 + assert error.value.req_id is None and error.value.response_json is None + assert isinstance(error.value.__cause__, asyncio.TimeoutError) + assert not is_retryable(error.value) + await asyncio.wait_for(peer.disconnected.wait(), timeout=2) + assert len(peer.requests) == 1 + assert not client.engine._session.closed + + +@pytest.mark.parametrize("status", [200, 429]) +async def test_wire_body_cancellation_propagates_and_closes_response( + wire_server, monkeypatch, status +): + reading = asyncio.Event() + original_text = aiohttp.ClientResponse.text + + async def observe_read(response, *args, **kwargs): + reading.set() + return await original_text(response, *args, **kwargs) + + monkeypatch.setattr(aiohttp.ClientResponse, "text", observe_read) + async with wire_server(status, length=100, hold_open=True) as peer: + async with BrightDataClient(token="local_test_token", auto_create_zones=False) as client: + client.engine.BASE_URL = peer.url + task = asyncio.create_task(client.search.business.companies("query", mode="instant")) + try: + await asyncio.wait_for(reading.wait(), timeout=2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.wait_for(peer.disconnected.wait(), timeout=2) + assert len(peer.requests) == 1 + assert not client.engine._session.closed + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) diff --git a/tests/unit/test_engine.py b/tests/unit/test_engine.py index 79922a6..cd48250 100644 --- a/tests/unit/test_engine.py +++ b/tests/unit/test_engine.py @@ -114,7 +114,13 @@ async def test_post_delegates_to_request(self): with patch.object(engine, "request", return_value="cm") as mock_req: result = engine.post("/endpoint", json_data={"k": "v"}) mock_req.assert_called_once_with( - "POST", "/endpoint", json_data={"k": "v"}, params=None, headers=None + "POST", + "/endpoint", + json_data={"k": "v"}, + params=None, + headers=None, + preserve_error_response=False, + allow_redirects=True, ) @pytest.mark.asyncio diff --git a/tests/unit/test_error_translation.py b/tests/unit/test_error_translation.py new file mode 100644 index 0000000..dacd2ac --- /dev/null +++ b/tests/unit/test_error_translation.py @@ -0,0 +1,371 @@ +""" +Tests for engine-level HTTP error translation and structured exceptions. + +Covers: per-status translation at the choke point, the 202 pass-through that +protects the SDK's only recovery path, `retryable` correctness (the +duplicate-job guard), export surface, and back-compat of the exception +constructors. +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import brightdata +import brightdata.exceptions as ex +from brightdata.core.engine import AsyncEngine +from brightdata.exceptions import ( + APIError, + AuthenticationError, + BrightDataError, + DataNotReadyError, + RateLimitError, +) +from brightdata.utils.http import parse_retry_after, status_phrase +from brightdata.utils.retry import is_retryable + + +def _resp(status, text="body", headers=None): + r = AsyncMock() + r.status = status + r.text = AsyncMock(return_value=text) + r.release = AsyncMock() + r.close = MagicMock() # aiohttp's close() is sync + r.headers = headers or {} + return r + + +async def _enter(engine, response): + """Send one request through the engine with a canned response.""" + engine._session.request = AsyncMock(return_value=response) + async with engine.get("/test") as r: + return r + + +# --------------------------------------------------------------------------- +# Per-status translation +# --------------------------------------------------------------------------- + + +class TestStatusTranslation: + @pytest.mark.parametrize("status", [401, 403]) + async def test_auth_statuses(self, status): + engine = AsyncEngine(bearer_token="tok") + async with engine: + with pytest.raises(AuthenticationError) as ei: + await _enter(engine, _resp(status, "nope")) + assert ei.value.status_code == status + assert ei.value.raw == "nope" + assert ei.value.retryable is False + + @pytest.mark.parametrize("status", [400, 404, 422]) + async def test_client_errors_are_api_error_not_retryable(self, status): + engine = AsyncEngine(bearer_token="tok") + async with engine: + with pytest.raises(APIError) as ei: + await _enter(engine, _resp(status)) + assert not isinstance(ei.value, RateLimitError) + assert ei.value.status_code == status + assert ei.value.retryable is False + + @pytest.mark.parametrize("status", [500, 502, 503]) + async def test_server_errors_are_retryable(self, status): + engine = AsyncEngine(bearer_token="tok") + async with engine: + with pytest.raises(APIError) as ei: + await _enter(engine, _resp(status)) + assert ei.value.status_code == status + assert ei.value.retryable is True + + async def test_429_becomes_rate_limit_error_with_retry_after(self): + engine = AsyncEngine(bearer_token="tok") + async with engine: + with pytest.raises(RateLimitError) as ei: + await _enter(engine, _resp(429, "slow down", {"Retry-After": "30"})) + assert ei.value.status_code == 429 + assert ei.value.retry_after == 30.0 + assert ei.value.retryable is False + + async def test_429_with_non_json_body_is_not_a_parse_error(self): + """The field case: a bare string under a non-JSON content type.""" + engine = AsyncEngine(bearer_token="tok") + async with engine: + with pytest.raises(RateLimitError) as ei: + await _enter(engine, _resp(429, "too_many_parallel_jobs")) + assert "too_many_parallel_jobs" in ei.value.raw + assert ei.value.retry_after is None + + async def test_non_standard_status_does_not_crash_phrase_lookup(self): + engine = AsyncEngine(bearer_token="tok") + async with engine: + with pytest.raises(APIError) as ei: + await _enter(engine, _resp(520)) # Cloudflare + assert ei.value.status_code == 520 + + async def test_error_carries_url_and_method(self): + engine = AsyncEngine(bearer_token="tok") + async with engine: + with pytest.raises(APIError) as ei: + await _enter(engine, _resp(500)) + assert ei.value.method == "GET" + assert ei.value.url.endswith("/test") + + async def test_raw_is_bounded(self): + engine = AsyncEngine(bearer_token="tok") + async with engine: + with pytest.raises(APIError) as ei: + await _enter(engine, _resp(500, "x" * 20000)) + assert len(ei.value.raw) < 20000 + assert "truncated" in ei.value.raw + + +# --------------------------------------------------------------------------- +# 202 must pass through — the SDK's only recovery path depends on it +# --------------------------------------------------------------------------- + + +class TestAcceptedPassesThrough: + @pytest.mark.parametrize("status", [200, 201, 202, 204]) + async def test_success_and_accepted_are_returned_not_raised(self, status): + engine = AsyncEngine(bearer_token="tok") + async with engine: + r = await _enter(engine, _resp(status)) + assert r.status == status + + async def test_fetch_result_still_raises_data_not_ready_on_202(self): + from brightdata.scrapers.api_client import DatasetAPIClient + + resp = _resp(202, "still building") + engine = MagicMock() + engine.get_from_url = MagicMock(return_value=_ctx(resp)) + with pytest.raises(DataNotReadyError): + await DatasetAPIClient(engine).fetch_result("snap_1") + + +def _ctx(response): + cm = MagicMock() + cm.__aenter__ = AsyncMock(return_value=response) + cm.__aexit__ = AsyncMock(return_value=False) + return cm + + +# --------------------------------------------------------------------------- +# retryable — the duplicate-job guard +# --------------------------------------------------------------------------- + + +class TestRetryable: + def test_missing_status_is_never_retryable(self): + """Raised locally, possibly after the server accepted the work.""" + assert is_retryable(APIError("Failed to trigger scrape - no snapshot_id returned")) is False + + def test_explicit_5xx_is_retryable(self): + assert is_retryable(APIError("x", status_code=500)) is True + + def test_4xx_is_not_retryable(self): + assert is_retryable(APIError("x", status_code=400)) is False + + def test_rate_limit_is_never_retryable_even_if_flagged(self): + assert is_retryable(RateLimitError("x", status_code=429, retryable=True)) is False + + def test_network_and_timeout_are_retryable(self): + from brightdata.exceptions import NetworkError + + assert is_retryable(NetworkError("down")) is True + assert is_retryable(TimeoutError("slow")) is True + + +# --------------------------------------------------------------------------- +# Behaviour preserved at repaired call sites +# --------------------------------------------------------------------------- + + +class TestRepairedCallSites: + async def test_crawler_scrape_returns_result_on_http_error(self): + from brightdata.crawler.service import CrawlerService + + client = MagicMock() + cm = MagicMock() + cm.__aenter__ = AsyncMock(side_effect=APIError("boom", status_code=500, raw="oops")) + cm.__aexit__ = AsyncMock(return_value=False) + client.engine.post_to_url = MagicMock(return_value=cm) + + result = await CrawlerService(client).crawl(urls="https://example.com") + assert result.success is False + assert "500" in result.error + + @pytest.mark.parametrize( + "raised,expected", + [ + (APIError("x", status_code=500), "error"), + (RateLimitError("x", status_code=429), "error"), + ], + ) + async def test_unlocker_status_returns_string_never_raises(self, raised, expected): + from brightdata.web_unlocker.async_client import AsyncUnblockerClient + + engine = MagicMock() + engine.BASE_URL = "https://api.brightdata.com" + cm = MagicMock() + cm.__aenter__ = AsyncMock(side_effect=raised) + cm.__aexit__ = AsyncMock(return_value=False) + engine.get_from_url = MagicMock(return_value=cm) + + got = await AsyncUnblockerClient(engine).get_status(zone="z", response_id="r") + assert got == expected + + async def test_poll_reports_transport_failure_not_job_failure(self): + from brightdata.utils.polling import poll_until_ready + + async def bad_status(_): + raise AuthenticationError("Unauthorized (401)", status_code=401) + + async def never(_): + raise AssertionError("should not fetch") + + r = await poll_until_ready(bad_status, never, "snap_1", poll_interval=0, poll_timeout=5) + assert r.success is False + assert "Failed to get status" in r.error + assert "Job failed" not in r.error + + +# --------------------------------------------------------------------------- +# Exports and back-compat +# --------------------------------------------------------------------------- + + +class TestExportsAndBackCompat: + def test_rate_limit_error_importable_at_both_levels(self): + assert brightdata.RateLimitError is ex.RateLimitError + + def test_every_exception_name_is_exported_top_level(self): + missing = [n for n in ex.__all__ if n not in brightdata.__all__] + assert not missing, f"not re-exported from brightdata: {missing}" + + def test_legacy_constructors_still_work(self): + assert BrightDataError("m").message == "m" + assert APIError("m").status_code is None + assert APIError("m", 429).status_code == 429 # positional, as before + assert APIError("m", 429, "body").response_text == "body" + + def test_rate_limit_is_catchable_as_api_error(self): + with pytest.raises(APIError): + raise RateLimitError("x", status_code=429) + + def test_dataset_error_is_catchable_both_ways(self): + from brightdata.datasets import DatasetError + + assert issubclass(DatasetError, BrightDataError) + assert issubclass(DatasetError, Exception) + + +class TestHttpHelpers: + @pytest.mark.parametrize( + "value,expected", [("30", 30.0), ("0", 0.0), (None, None), ("soon", None)] + ) + def test_parse_retry_after(self, value, expected): + headers = {"Retry-After": value} if value is not None else {} + assert parse_retry_after(headers) == expected + + def test_status_phrase_handles_non_standard(self): + assert status_phrase(429) == "Too Many Requests" + assert status_phrase(520) # must not raise + + +# --------------------------------------------------------------------------- +# Structure must survive to the caller (the point of the whole change) +# --------------------------------------------------------------------------- + + +class TestCausePropagation: + async def test_rate_limit_during_polling_reaches_the_caller(self): + from brightdata.utils.polling import poll_until_ready + + async def rate_limited(_): + raise RateLimitError("Rate limited (429)", status_code=429, retry_after=45.0) + + async def never(_): + raise AssertionError("should not fetch") + + r = await poll_until_ready(rate_limited, never, "snap_1", poll_interval=0, poll_timeout=5) + + assert r.success is False + assert isinstance(r.cause, RateLimitError) + assert r.cause.status_code == 429 + assert r.cause.retry_after == 45.0 + assert r.cause.retryable is False + + async def test_trigger_failure_carries_cause(self): + from brightdata.scrapers.workflow import WorkflowExecutor + + api = MagicMock() + api.trigger = AsyncMock(side_effect=APIError("boom", status_code=503)) + r = await WorkflowExecutor(api).execute(payload=[{"url": "u"}], dataset_id="gd_x") + + assert r.success is False + assert isinstance(r.cause, APIError) + assert r.cause.status_code == 503 + assert r.cause.retryable is True + + async def test_crawler_result_carries_cause(self): + from brightdata.crawler.service import CrawlerService + + client = MagicMock() + cm = MagicMock() + cm.__aenter__ = AsyncMock(side_effect=RateLimitError("rl", status_code=429, retry_after=10)) + cm.__aexit__ = AsyncMock(return_value=False) + client.engine.post_to_url = MagicMock(return_value=cm) + + r = await CrawlerService(client).crawl(urls="https://example.com") + assert isinstance(r.cause, RateLimitError) + assert r.cause.retry_after == 10 + + def test_result_with_cause_is_still_serializable(self): + from brightdata.models import ScrapeResult + + r = ScrapeResult( + success=False, + error="rate limited", + cause=RateLimitError("rl", status_code=429, retry_after=30.0), + ) + payload = r.to_dict() + assert payload["cause"]["type"] == "RateLimitError" + assert payload["cause"]["status_code"] == 429 + r.to_json() # must not raise + + +class TestDatasetErrorContract: + """The datasets layer must keep raising DatasetError after central translation.""" + + async def _call(self, status, body, headers=None): + from brightdata.datasets.base import BaseDataset + + class DS(BaseDataset): + DATASET_ID = "gd_x" + NAME = "x" + + engine = AsyncEngine(bearer_token="tok") + async with engine: + engine._session.request = AsyncMock(return_value=_resp(status, body, headers)) + return await DS(engine)(filter={"name": "f", "operator": "is_not_null"}) + + async def test_client_error_is_still_a_dataset_error(self): + from brightdata.datasets import DatasetError + + with pytest.raises(DatasetError) as ei: + await self._call(400, '{"error":"bad filter"}') + assert ei.value.status_code == 400 + assert "bad filter" in ei.value.raw + + async def test_rate_limit_stays_rate_limit_error(self): + with pytest.raises(RateLimitError) as ei: + await self._call(429, "too_many_parallel_jobs", {"Retry-After": "60"}) + assert ei.value.retry_after == 60.0 + + async def test_message_keeps_a_body_excerpt(self): + with pytest.raises(APIError) as ei: + engine = AsyncEngine(bearer_token="tok") + async with engine: + await _enter(engine, _resp(400, "helpful detail")) + assert "helpful detail" in str(ei.value) + assert len(str(ei.value)) < 300 # bounded diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 311ec7f..14cf42f 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -5,7 +5,7 @@ import pytest from brightdata.exceptions import APIError, AuthenticationError, NetworkError, ValidationError -from brightdata.utils.retry import retry_with_backoff +from brightdata.utils.retry import is_retryable, retry_with_backoff # --------------------------------------------------------------------------- # Happy path @@ -37,6 +37,26 @@ async def test_retries_then_succeeds(self): class TestRetryableExceptions: + @pytest.mark.parametrize("retryable,attempts", [(False, 1), (True, 2), (None, 2)]) + async def test_explicit_network_retry_policy(self, retryable, attempts): + error = NetworkError("response failed", retryable=retryable) + func = AsyncMock(side_effect=error) + with pytest.raises(NetworkError): + await retry_with_backoff(func, max_retries=1, initial_delay=0) + assert func.await_count == attempts + assert is_retryable(error) is (attempts == 2) + assert error.retryable is is_retryable(error) + + async def test_custom_exception_list_remains_an_explicit_policy_override(self): + func = AsyncMock(side_effect=[NetworkError("read failed", retryable=False), "ok"]) + assert ( + await retry_with_backoff( + func, max_retries=1, initial_delay=0, retryable_exceptions=[NetworkError] + ) + == "ok" + ) + assert func.await_count == 2 + @pytest.mark.asyncio async def test_retries_on_network_error(self): func = AsyncMock(side_effect=[NetworkError("net"), "ok"])