From f15add299d50c81d179afdfdcda99eb9687aaa31 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:49:08 +0000 Subject: [PATCH 01/10] feat(api): api update --- .stats.yml | 4 ++-- web.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.stats.yml b/.stats.yml index 74e73bc..9ea03e0 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 38 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-155e4761d62255841349c0f8a01b0a9c463ea1d1f2d6c4fd8d1a75c8bef6f226.yml -openapi_spec_hash: ab91f77e7c9d992400cbc7fc8a9e76c1 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-8a92cb064637006331b2910caedca8512290c1d8cc75eb91218393cc1581e59c.yml +openapi_spec_hash: c811ec47143ad01cdbcbd8324caa4eda config_hash: bff282047fafdad771fb7ec685f56944 diff --git a/web.go b/web.go index b4c3e51..fd0c3e2 100644 --- a/web.go +++ b/web.go @@ -132,6 +132,7 @@ func (r *WebService) WebScrapeImages(ctx context.Context, query WebWebScrapeImag // | 401 / 403 | No | Invalid/disabled key, insufficient permissions, or credits exhausted; inspect error_code | // | 404 | No | Target page returned or fingerprinted as not found | // | 408 | No | Request timed out | +// | 413 | No | Target content exceeds the maximum supported size (20 MB) | // | 415 | No | Unsupported content type | // | 429 | No | Per-minute rate limit exceeded; honor Retry-After | // | 500 | No | Internal error | From e71dbb9625e935c311e8ab0f57bb112791b3b1cf Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:08:55 +0000 Subject: [PATCH 02/10] feat(api): api update --- .stats.yml | 4 ++-- web.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.stats.yml b/.stats.yml index 9ea03e0..9a1a53d 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 38 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-8a92cb064637006331b2910caedca8512290c1d8cc75eb91218393cc1581e59c.yml -openapi_spec_hash: c811ec47143ad01cdbcbd8324caa4eda +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-e6a3365124af52952a85c8af5a09d6fdc02a686db7949985d0a6313dfad8dee5.yml +openapi_spec_hash: d7d0e17252f6f5665fb1698e37e09fe2 config_hash: bff282047fafdad771fb7ec685f56944 diff --git a/web.go b/web.go index fd0c3e2..aa425d9 100644 --- a/web.go +++ b/web.go @@ -4117,7 +4117,7 @@ type WebWebScrapeHTMLParamsPdf struct { // the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. Ocr WebWebScrapeHTMLParamsPdfOcrUnion `query:"ocr,omitzero" json:"-"` // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and - // a 400 WEBSITE_ACCESS_ERROR is returned. + // a 400 PDF_SKIPPED is returned. ShouldParse WebWebScrapeHTMLParamsPdfShouldParseUnion `query:"shouldParse,omitzero" json:"-"` paramObj } @@ -4834,7 +4834,7 @@ type WebWebScrapeMdParamsPdf struct { // the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. Ocr WebWebScrapeMdParamsPdfOcrUnion `query:"ocr,omitzero" json:"-"` // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and - // a 400 WEBSITE_ACCESS_ERROR is returned. + // a 400 PDF_SKIPPED is returned. ShouldParse WebWebScrapeMdParamsPdfShouldParseUnion `query:"shouldParse,omitzero" json:"-"` paramObj } From bd340b8ad89d0530f3a8396d4ec3f380fc4137d5 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 05:45:54 +0000 Subject: [PATCH 03/10] feat(api): api update --- .stats.yml | 4 ++-- parse.go | 2 +- web.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.stats.yml b/.stats.yml index 9a1a53d..f57748b 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 38 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-e6a3365124af52952a85c8af5a09d6fdc02a686db7949985d0a6313dfad8dee5.yml -openapi_spec_hash: d7d0e17252f6f5665fb1698e37e09fe2 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-b73de7a5f87b3bed7d9e4259bd109cd62f4eb5f61fa082f31fc22d5bf8e69bcc.yml +openapi_spec_hash: 5dcb5ba9b8487329adc2f89eeb4df7c4 config_hash: bff282047fafdad771fb7ec685f56944 diff --git a/parse.go b/parse.go index 8fc2cbc..8b34fbe 100644 --- a/parse.go +++ b/parse.go @@ -174,7 +174,7 @@ type ParseHandleParams struct { // When true for PDF inputs, detect and OCR images embedded in the selected pages, // inserting recognized text at each image's position in page reading order while // preserving the PDF text layer. pdf.start/pdf.end limit the inclusive page range. - // When false, all OCR is disabled, including the automatic scanned-PDF fallback. + // When false, no OCR runs. Ocr ParseHandleParamsOcrUnion `query:"ocr,omitzero" json:"-"` // PDF page-range options as a JSON object, e.g. {"start": 2, "end": 5}. Pdf ParseHandleParamsPdf `query:"pdf,omitzero" json:"-"` diff --git a/web.go b/web.go index aa425d9..ab84e52 100644 --- a/web.go +++ b/web.go @@ -4114,7 +4114,7 @@ type WebWebScrapeHTMLParamsPdf struct { Start param.Opt[int64] `query:"start,omitzero" json:"-"` // When true, detect and OCR images embedded in the selected PDF pages, inserting // recognized text at each image's position in page reading order while preserving - // the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. + // the PDF text layer. When false, no OCR runs. Ocr WebWebScrapeHTMLParamsPdfOcrUnion `query:"ocr,omitzero" json:"-"` // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and // a 400 PDF_SKIPPED is returned. @@ -4831,7 +4831,7 @@ type WebWebScrapeMdParamsPdf struct { Start param.Opt[int64] `query:"start,omitzero" json:"-"` // When true, detect and OCR images embedded in the selected PDF pages, inserting // recognized text at each image's position in page reading order while preserving - // the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. + // the PDF text layer. When false, no OCR runs. Ocr WebWebScrapeMdParamsPdfOcrUnion `query:"ocr,omitzero" json:"-"` // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and // a 400 PDF_SKIPPED is returned. From 46f18c27fb20790758beb40150fba47971a91a5a Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 07:45:39 +0000 Subject: [PATCH 04/10] feat(api): api update --- .stats.yml | 4 ++-- web.go | 13 +++++++++++-- web_test.go | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.stats.yml b/.stats.yml index f57748b..15a2673 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 38 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-b73de7a5f87b3bed7d9e4259bd109cd62f4eb5f61fa082f31fc22d5bf8e69bcc.yml -openapi_spec_hash: 5dcb5ba9b8487329adc2f89eeb4df7c4 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-b5a46cf886759797353b250da971e46cf66804b825ded9c51609f34d72d1169c.yml +openapi_spec_hash: 1947de2a731ddab05bfb781f37cd42b6 config_hash: bff282047fafdad771fb7ec685f56944 diff --git a/web.go b/web.go index ab84e52..1fc1237 100644 --- a/web.go +++ b/web.go @@ -143,7 +143,11 @@ func (r *WebService) WebScrapeMd(ctx context.Context, query WebWebScrapeMdParams return res, err } -// Crawl an entire website's sitemap and return all discovered page URLs. +// Crawl an entire website's sitemap and return all discovered page URLs. Pass +// `search` to have the crawled sitemap filtered down to the pages about a phrase +// (for example `pricing and plans` or `api authentication docs`), most relevant +// first — a searched crawl scans the whole sitemap and costs 2 credits instead +// of 1. func (r *WebService) WebScrapeSitemap(ctx context.Context, query WebWebScrapeSitemapParams, opts ...option.RequestOption) (res *WebWebScrapeSitemapResponse, err error) { opts = slices.Concat(r.options, opts) path := "web/scrape/sitemap" @@ -2370,7 +2374,8 @@ type WebWebScrapeSitemapResponse struct { // // Any of true. Success bool `json:"success" api:"required"` - // Array of discovered page URLs from the sitemap (max 500) + // Discovered page URLs from the sitemap, up to `maxLinks`. When `search` is set + // these are only the matching pages, most relevant first. URLs []string `json:"urls" api:"required"` // Metadata about the API key used for the request. Included in every response // whenever a valid API key is provided, even when the response status is not 200. @@ -4955,6 +4960,10 @@ type WebWebScrapeSitemapParams struct { // Maximum number of links to return from the sitemap crawl. Defaults to 10,000. // Minimum is 1, maximum is 100,000. MaxLinks param.Opt[int64] `query:"maxLinks,omitzero" json:"-"` + // Optional search phrase. When provided, the crawled sitemap is filtered to the + // pages whose URLs are about that phrase, most relevant first, and the request + // costs 2 credits instead of 1. + Search param.Opt[string] `query:"search,omitzero" json:"-"` // Optional explicit sitemap URL. When provided, exactly this sitemap is crawled // instead of discovering the domain's sitemaps. SitemapURL param.Opt[string] `query:"sitemapUrl,omitzero" format:"uri" json:"-"` diff --git a/web_test.go b/web_test.go index 11779f3..c5ba889 100644 --- a/web_test.go +++ b/web_test.go @@ -490,6 +490,7 @@ func TestWebWebScrapeSitemapWithOptionalParams(t *testing.T) { "foo": "J!", }, MaxLinks: contextdev.Int(1), + Search: contextdev.String("help center and troubleshooting articles"), SitemapURL: contextdev.String("https://example.com"), Tags: []string{"production", "team-alpha"}, TimeoutMs: contextdev.Int(1), From e4803c4cf0aebf68aeb1653630e60e39caae68c1 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:23:15 +0000 Subject: [PATCH 05/10] feat(api): api update --- .stats.yml | 4 ++-- batch.go | 23 +++++++++++++++++++---- brand.go | 33 +++++++++++++++++++++++++++++++-- parse.go | 9 +++++---- web.go | 21 ++++++++++++--------- 5 files changed, 69 insertions(+), 21 deletions(-) diff --git a/.stats.yml b/.stats.yml index 15a2673..3eddd36 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 38 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-b5a46cf886759797353b250da971e46cf66804b825ded9c51609f34d72d1169c.yml -openapi_spec_hash: 1947de2a731ddab05bfb781f37cd42b6 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-3ecc6285243f60f7619d543cac59991015fbb0a156648741e7e42e619e0cbc5c.yml +openapi_spec_hash: 0130f29a501366c63cbd3a446c5eb109 config_hash: bff282047fafdad771fb7ec685f56944 diff --git a/batch.go b/batch.go index 9415b9b..6034d93 100644 --- a/batch.go +++ b/batch.go @@ -355,9 +355,12 @@ func (r *BatchGetResponse) UnmarshalJSON(data []byte) error { // What this batch has done to your credit balance. type BatchGetResponseCredits struct { - // `reserved` minus `refunded` — what the batch has cost so far. Equal to - // `reserved` until the batch settles. + // `reserved` minus `refunded` plus `ocr_charged` — what the batch has cost so far. + // Equal to `reserved` until the batch settles. Net int64 `json:"net" api:"required"` + // Credits charged for PDF pages recovered by OCR (pdf.ocr=true), 1 per recovered + // page, on top of `reserved`. Stays 0 until the batch settles. + OcrCharged int64 `json:"ocr_charged" api:"required"` // Credits returned for pages that did not succeed. Stays 0 until the batch reaches // a final status, then settles in one movement. Refunded int64 `json:"refunded" api:"required"` @@ -367,6 +370,7 @@ type BatchGetResponseCredits struct { // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { Net respjson.Field + OcrCharged respjson.Field Refunded respjson.Field Reserved respjson.Field ExtraFields map[string]respjson.Field @@ -637,9 +641,12 @@ func (r *BatchListResponseData) UnmarshalJSON(data []byte) error { // What this batch has done to your credit balance. type BatchListResponseDataCredits struct { - // `reserved` minus `refunded` — what the batch has cost so far. Equal to - // `reserved` until the batch settles. + // `reserved` minus `refunded` plus `ocr_charged` — what the batch has cost so far. + // Equal to `reserved` until the batch settles. Net int64 `json:"net" api:"required"` + // Credits charged for PDF pages recovered by OCR (pdf.ocr=true), 1 per recovered + // page, on top of `reserved`. Stays 0 until the batch settles. + OcrCharged int64 `json:"ocr_charged" api:"required"` // Credits returned for pages that did not succeed. Stays 0 until the batch reaches // a final status, then settles in one movement. Refunded int64 `json:"refunded" api:"required"` @@ -649,6 +656,7 @@ type BatchListResponseDataCredits struct { // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { Net respjson.Field + OcrCharged respjson.Field Refunded respjson.Field Reserved respjson.Field ExtraFields map[string]respjson.Field @@ -995,6 +1003,8 @@ type BatchGetResultsResponseDataUnion struct { // This field is from variant [BatchGetResultsResponseDataOk]. Markdown string `json:"markdown"` Meta any `json:"meta"` + // This field is from variant [BatchGetResultsResponseDataOk]. + OcrPages int64 `json:"ocr_pages"` // This field is from variant [BatchGetResultsResponseDataError]. ErrorCode string `json:"error_code"` // This field is from variant [BatchGetResultsResponseDataError]. @@ -1009,6 +1019,7 @@ type BatchGetResultsResponseDataUnion struct { ItemID respjson.Field Markdown respjson.Field Meta respjson.Field + OcrPages respjson.Field ErrorCode respjson.Field Message respjson.Field raw string @@ -1080,6 +1091,9 @@ type BatchGetResultsResponseDataOk struct { Markdown string `json:"markdown"` // Caller-supplied metadata echoed from submission. Meta map[string]any `json:"meta"` + // PDF pages of this document recovered by OCR (pdf.ocr=true). Each recovered page + // bills 1 credit on top of the page base credit; absent when no OCR ran. + OcrPages int64 `json:"ocr_pages"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { FinalURL respjson.Field @@ -1091,6 +1105,7 @@ type BatchGetResultsResponseDataOk struct { ItemID respjson.Field Markdown respjson.Field Meta respjson.Field + OcrPages respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` diff --git a/brand.go b/brand.go index f979a30..54577a8 100644 --- a/brand.go +++ b/brand.go @@ -59,7 +59,8 @@ func (r *BrandService) GetSimplified(ctx context.Context, query BrandGetSimplifi } // Search brands by name or domain and get back up to 10 lightweight matches -// (domain, name, logo), most popular first: by Tranco rank, then market cap for +// (domain, name, logo). Name matches rank ahead of domain matches; within each +// group the most popular brands come first: by Tranco rank, then market cap for // brands outside the Tranco list, with text relevance breaking ties. Matching is // prefix-based with no typo tolerance, so it is suited to autocomplete. Only // brands already in the Context.dev index are returned — use /brand/retrieve to @@ -113,6 +114,8 @@ type BrandGetResponseBrand struct { Domain string `json:"domain"` // Company email address Email string `json:"email"` + // Employee headcount information for the brand (will be null if unknown) + Employees BrandGetResponseBrandEmployees `json:"employees"` // Industry classification information for the brand Industries BrandGetResponseBrandIndustries `json:"industries"` // Indicates whether the brand content is not safe for work (NSFW) @@ -161,6 +164,7 @@ type BrandGetResponseBrand struct { Description respjson.Field Domain respjson.Field Email respjson.Field + Employees respjson.Field Industries respjson.Field IsNsfw respjson.Field Links respjson.Field @@ -305,6 +309,30 @@ func (r *BrandGetResponseBrandColor) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } +// Employee headcount information for the brand (will be null if unknown) +type BrandGetResponseBrandEmployees struct { + // Exact employee count when a precise headcount is known + Exact int64 `json:"exact"` + // Employee count range for the brand (e.g. '11 to 50') + // + // Any of "1 to 10", "11 to 50", "51 to 200", "201 to 500", "501 to 1000", "1001 to + // 5000", "5001 to 10000", "10001+". + Range string `json:"range"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Exact respjson.Field + Range respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r BrandGetResponseBrandEmployees) RawJSON() string { return r.JSON.raw } +func (r *BrandGetResponseBrandEmployees) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + // Industry classification information for the brand type BrandGetResponseBrandIndustries struct { // Easy Industry Classification - array of industry and subindustry pairs @@ -876,7 +904,8 @@ func (r *BrandGetSimplifiedResponseKeyMetadata) UnmarshalJSON(data []byte) error } type BrandSearchResponse struct { - // Up to 10 matching brands, most popular first. Empty when nothing matches. + // Up to 10 matching brands, name matches first, then domain matches, most popular + // first within each group. Empty when nothing matches. Results []BrandSearchResponseResult `json:"results" api:"required"` // Metadata about the API key used for the request. Included in every response // whenever a valid API key is provided, even when the response status is not 200. diff --git a/parse.go b/parse.go index 8b34fbe..f6198d0 100644 --- a/parse.go +++ b/parse.go @@ -171,10 +171,11 @@ type ParseHandleParams struct { IncludeImages ParseHandleParamsIncludeImagesUnion `query:"includeImages,omitzero" json:"-"` // Preserve hyperlinks in Markdown output IncludeLinks ParseHandleParamsIncludeLinksUnion `query:"includeLinks,omitzero" json:"-"` - // When true for PDF inputs, detect and OCR images embedded in the selected pages, - // inserting recognized text at each image's position in page reading order while - // preserving the PDF text layer. pdf.start/pdf.end limit the inclusive page range. - // When false, no OCR runs. + // When true for PDF inputs, OCR the selected pages that have no usable text layer + // (scans), replacing each recovered page's text with the OCR result while pages + // with a real text layer keep it. pdf.start/pdf.end limit the inclusive page + // range. Billed at 1 credit per page OCR actually recovered, on top of the base + // request cost. When false, no OCR runs. Ocr ParseHandleParamsOcrUnion `query:"ocr,omitzero" json:"-"` // PDF page-range options as a JSON object, e.g. {"start": 2, "end": 5}. Pdf ParseHandleParamsPdf `query:"pdf,omitzero" json:"-"` diff --git a/web.go b/web.go index 1fc1237..ab3b01c 100644 --- a/web.go +++ b/web.go @@ -3710,9 +3710,10 @@ type WebWebCrawlMdParamsPdf struct { // Last 1-based PDF page to parse. When omitted, parsing ends at the last page. // Must be greater than or equal to start when both are provided. End param.Opt[int64] `json:"end,omitzero"` - // When true, detect and OCR images embedded in the selected PDF pages, inserting - // recognized text at each image's position in page reading order while preserving - // the PDF text layer. This is separate from automatic scanned-PDF OCR fallback. + // When true, OCR the selected PDF pages that have no usable text layer (scans), + // replacing each recovered page's text with the OCR result while pages with a real + // text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + // of the base request cost. Ocr param.Opt[bool] `json:"ocr,omitzero"` // When true, PDF pages are fetched and parsed. When false, PDF pages are skipped // entirely (not included in results and not counted as failures). @@ -4117,9 +4118,10 @@ type WebWebScrapeHTMLParamsPdf struct { End param.Opt[int64] `query:"end,omitzero" json:"-"` // First 1-based PDF page to parse. When omitted, parsing starts at the first page. Start param.Opt[int64] `query:"start,omitzero" json:"-"` - // When true, detect and OCR images embedded in the selected PDF pages, inserting - // recognized text at each image's position in page reading order while preserving - // the PDF text layer. When false, no OCR runs. + // When true, OCR the selected PDF pages that have no usable text layer (scans), + // replacing each recovered page's text with the OCR result while pages with a real + // text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + // of the base request cost. When false, no OCR runs. Ocr WebWebScrapeHTMLParamsPdfOcrUnion `query:"ocr,omitzero" json:"-"` // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and // a 400 PDF_SKIPPED is returned. @@ -4834,9 +4836,10 @@ type WebWebScrapeMdParamsPdf struct { End param.Opt[int64] `query:"end,omitzero" json:"-"` // First 1-based PDF page to parse. When omitted, parsing starts at the first page. Start param.Opt[int64] `query:"start,omitzero" json:"-"` - // When true, detect and OCR images embedded in the selected PDF pages, inserting - // recognized text at each image's position in page reading order while preserving - // the PDF text layer. When false, no OCR runs. + // When true, OCR the selected PDF pages that have no usable text layer (scans), + // replacing each recovered page's text with the OCR result while pages with a real + // text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + // of the base request cost. When false, no OCR runs. Ocr WebWebScrapeMdParamsPdfOcrUnion `query:"ocr,omitzero" json:"-"` // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and // a 400 PDF_SKIPPED is returned. From 14015cba3fde78c7a408c6c45b1c2a740fd603bb Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:35:39 +0000 Subject: [PATCH 06/10] codegen metadata --- .stats.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.stats.yml b/.stats.yml index 3eddd36..2c6e57f 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 38 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-3ecc6285243f60f7619d543cac59991015fbb0a156648741e7e42e619e0cbc5c.yml -openapi_spec_hash: 0130f29a501366c63cbd3a446c5eb109 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-2f28cd1bf5b88305989d68dc55037fe25bdba2800d61c071a848cc1eb1ccb225.yml +openapi_spec_hash: 6dd2807c7804e08c1d49479c0ff82e47 config_hash: bff282047fafdad771fb7ec685f56944 From e1a197fe3667d68af27bd368470f743eee85ec01 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:45:33 +0000 Subject: [PATCH 07/10] feat(api): api update --- .stats.yml | 4 ++-- web.go | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.stats.yml b/.stats.yml index 2c6e57f..b90c8da 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 38 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-2f28cd1bf5b88305989d68dc55037fe25bdba2800d61c071a848cc1eb1ccb225.yml -openapi_spec_hash: 6dd2807c7804e08c1d49479c0ff82e47 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-c1b6ca84f9bec1b1936cebb61f058924c3d7a5f6977e4cd76cf0544c4c7d8737.yml +openapi_spec_hash: 141bfa8430ddccfb5a37c4bd703d163b config_hash: bff282047fafdad771fb7ec685f56944 diff --git a/web.go b/web.go index ab3b01c..1d26a24 100644 --- a/web.go +++ b/web.go @@ -123,6 +123,18 @@ func (r *WebService) WebScrapeImages(ctx context.Context, query WebWebScrapeImag // responses from a recognized API key; use error_code to distinguish stable // failure categories. // +// ### YouTube +// +// YouTube URLs return the video or channel itself rather than the surrounding +// player and navigation chrome. A URL addressing a single video (`/watch`, +// `youtu.be`, `/shorts`, `/embed`, `/live`) returns its title, channel, duration, +// view count, keywords, full description, and the transcript when the video has +// captions that can be retrieved; videos without captions return everything except +// the transcript. A channel URL (`/channel/UC…`, `/@handle`, `/c/…`, `/user/…`) +// returns its name, handle, subscriber count, video count, and full description. +// When `includeImages=true`, video responses also include the thumbnail and +// channel responses include the avatar. Costs the same as any other scrape. +// // ### Billing & errors // // | HTTP status | Billed? | Meaning | @@ -1253,7 +1265,8 @@ func (r *WebSearchResponseResult) UnmarshalJSON(data []byte) error { type WebSearchResponseResultMarkdown struct { // Per-result scrape outcome. Inspect this before reading `markdown`. // - // Any of "SUCCESS", "NOT_REQUESTED", "TIMEOUT", "WEBSITE_ACCESS_ERROR", "ERROR". + // Any of "SUCCESS", "NOT_REQUESTED", "TIMEOUT", "CONTENT_TOO_LARGE", + // "WEBSITE_ACCESS_ERROR", "ERROR". Code string `json:"code" api:"required"` // GFM Markdown of the page. Null unless markdownOptions.enabled is true and // scraping succeeded. From 3362570a65bffd75adee223b860704feb4b90d6a Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:28:14 +0000 Subject: [PATCH 08/10] feat(api): manual updates --- .stats.yml | 6 +- api.md | 14 +- batch.go | 1559 +++++++++++++++++++++++++--------- batch_test.go | 72 +- client.go | 5 +- person.go | 727 ++++++++++++++++ person_test.go | 64 ++ shared/constant/constants.go | 21 + 8 files changed, 2072 insertions(+), 396 deletions(-) create mode 100644 person.go create mode 100644 person_test.go diff --git a/.stats.yml b/.stats.yml index b90c8da..ca57822 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ -configured_endpoints: 38 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-c1b6ca84f9bec1b1936cebb61f058924c3d7a5f6977e4cd76cf0544c4c7d8737.yml +configured_endpoints: 40 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-e2ba5381de3010ee53f27388aa6727ad567eb18e4b0b4eb1c4c66402bfea64cc.yml openapi_spec_hash: 141bfa8430ddccfb5a37c4bd703d163b -config_hash: bff282047fafdad771fb7ec685f56944 +config_hash: 920678668dd2da6f8966fbf1b8fde4e2 diff --git a/api.md b/api.md index 69d7939..4891172 100644 --- a/api.md +++ b/api.md @@ -131,6 +131,7 @@ Response Types: - contextdev.Intake - contextdev.BatchGetResponse - contextdev.BatchListResponse +- contextdev.BatchDeleteResponse - contextdev.BatchCancelResponse - contextdev.BatchGetResultsResponse - contextdev.BatchSubmitResponse @@ -139,6 +140,17 @@ Methods: - client.Batch.Get(ctx context.Context, batchID string) (\*contextdev.BatchGetResponse, error) - client.Batch.List(ctx context.Context, query contextdev.BatchListParams) (\*contextdev.BatchListResponse, error) +- client.Batch.Delete(ctx context.Context, batchID string) (\*contextdev.BatchDeleteResponse, error) - client.Batch.Cancel(ctx context.Context, batchID string) (\*contextdev.BatchCancelResponse, error) - client.Batch.GetResults(ctx context.Context, batchID string, query contextdev.BatchGetResultsParams) (\*contextdev.BatchGetResultsResponse, error) -- client.Batch.Submit(ctx context.Context, body contextdev.BatchSubmitParams) (\*contextdev.BatchSubmitResponse, error) +- client.Batch.Submit(ctx context.Context, params contextdev.BatchSubmitParams) (\*contextdev.BatchSubmitResponse, error) + +# People + +Response Types: + +- contextdev.PersonEnrichResponse + +Methods: + +- client.People.Enrich(ctx context.Context, body contextdev.PersonEnrichParams) (\*contextdev.PersonEnrichResponse, error) diff --git a/batch.go b/batch.go index 6034d93..5bda26e 100644 --- a/batch.go +++ b/batch.go @@ -20,6 +20,8 @@ import ( "github.com/context-dot-dev/context-go-sdk/v2/shared/constant" ) +// Scrape many pages or crawl a site asynchronously. +// // BatchService contains methods and other services that help with interacting with // the context.dev API. // @@ -60,6 +62,19 @@ func (r *BatchService) List(ctx context.Context, query BatchListParams, opts ... return res, err } +// Permanently delete a finished batch and its stored results. Active batches must +// settle first. +func (r *BatchService) Delete(ctx context.Context, batchID string, opts ...option.RequestOption) (res *BatchDeleteResponse, err error) { + opts = slices.Concat(r.options, opts) + if batchID == "" { + err = errors.New("missing required batch_id parameter") + return nil, err + } + path := fmt.Sprintf("batch/%s", url.PathEscape(batchID)) + err = requestconfig.ExecuteNewRequest(ctx, http.MethodDelete, path, nil, &res, opts...) + return res, err +} + // Stop a batch from starting new pages. In-progress pages finish, and unused // credits are refunded. func (r *BatchService) Cancel(ctx context.Context, batchID string, opts ...option.RequestOption) (res *BatchCancelResponse, err error) { @@ -86,11 +101,14 @@ func (r *BatchService) GetResults(ctx context.Context, batchID string, query Bat return res, err } -// Retrieve and normalize a person profile from identifiers. -func (r *BatchService) Submit(ctx context.Context, body BatchSubmitParams, opts ...option.RequestOption) (res *BatchSubmitResponse, err error) { +// Scrape 25K URLs or crawl large websites asynchronously. +func (r *BatchService) Submit(ctx context.Context, params BatchSubmitParams, opts ...option.RequestOption) (res *BatchSubmitResponse, err error) { + if !param.IsOmitted(params.IdempotencyKey) { + opts = append(opts, option.WithHeader("Idempotency-Key", fmt.Sprintf("%v", params.IdempotencyKey.Value))) + } opts = slices.Concat(r.options, opts) - path := "people/retrieve" - err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, body, &res, opts...) + path := "batch/submit" + err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, params, &res, opts...) return res, err } @@ -786,6 +804,52 @@ func (r *BatchListResponseKeyMetadata) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } +type BatchDeleteResponse struct { + // ID of the deleted batch. + ID string `json:"id"` + // Always true on success. + Deleted bool `json:"deleted"` + // Metadata about the API key used for the request. Included in every response + // whenever a valid API key is provided, even when the response status is not 200. + KeyMetadata BatchDeleteResponseKeyMetadata `json:"key_metadata"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + ID respjson.Field + Deleted respjson.Field + KeyMetadata respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r BatchDeleteResponse) RawJSON() string { return r.JSON.raw } +func (r *BatchDeleteResponse) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Metadata about the API key used for the request. Included in every response +// whenever a valid API key is provided, even when the response status is not 200. +type BatchDeleteResponseKeyMetadata struct { + // The number of credits consumed by this request. + CreditsConsumed int64 `json:"credits_consumed" api:"required"` + // The number of credits remaining for your organization after this request. + CreditsRemaining int64 `json:"credits_remaining" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + CreditsConsumed respjson.Field + CreditsRemaining respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r BatchDeleteResponseKeyMetadata) RawJSON() string { return r.JSON.raw } +func (r *BatchDeleteResponseKeyMetadata) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + type BatchCancelResponse struct { // Batch ID. ID string `json:"id" api:"required"` @@ -1380,421 +1444,126 @@ func (r *BatchGetResultsResponseKeyMetadata) UnmarshalJSON(data []byte) error { } type BatchSubmitResponse struct { - // HTTP status code. + // Batch ID. Poll GET /batch/{batch_id} with it. + ID string `json:"id" api:"required"` + // The crawl controls as submitted, so the limits requested can be compared against + // what the crawl reached. + Crawl CrawlControls `json:"crawl" api:"required"` + // When the batch was created. + CreatedAt string `json:"created_at" api:"required"` + // What accepting this batch cost. + Credits BatchSubmitResponseCredits `json:"credits" api:"required"` + // What each page will be returned as. + // + // Any of "markdown", "html". + Format BatchSubmitResponseFormat `json:"format" api:"required"` + // What submission took in, and what it charged for. + Input Intake `json:"input" api:"required"` + // Rejected URLs, up to 100. These are not charged. + InvalidURLs []BatchSubmitResponseInvalidURL `json:"invalid_urls" api:"required"` + // How pages will be selected. // - // Any of 200. - Code int64 `json:"code" api:"required"` - // Additional response details. - Metadata BatchSubmitResponseMetadata `json:"metadata" api:"required"` - // Retrieved person profile. - Person BatchSubmitResponsePerson `json:"person" api:"required"` - // Response status. + // Any of "scrape", "crawl". + Mode BatchSubmitResponseMode `json:"mode" api:"required"` + // Always `queued`. An accepted batch has not started yet. // - // Any of "ok". + // Any of "queued". Status BatchSubmitResponseStatus `json:"status" api:"required"` - // Metadata about the API key used for the request. Included in every response - // whenever a valid API key is provided, even when the response status is not 200. + // Tags stored on the batch. + Tags []string `json:"tags" api:"required"` + // API key usage for this request. KeyMetadata BatchSubmitResponseKeyMetadata `json:"key_metadata"` + // Signing secret for the completion webhook, returned only here and never again. + // Store it now; it is not repeated by GET /batch/{batch_id}. + WebhookSecret string `json:"webhook_secret"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { - Code respjson.Field - Metadata respjson.Field - Person respjson.Field - Status respjson.Field - KeyMetadata respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponse) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponse) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// Additional response details. -type BatchSubmitResponseMetadata struct { - // Identifiers returned for the person. - Identifiers BatchSubmitResponseMetadataIdentifiers `json:"identifiers" api:"required"` - // Source categories checked. - // - // Any of "linkedin", "cv", "manual", "github", "other". - SourcesAttempted []string `json:"sourcesAttempted" api:"required"` - // Source categories with data. - // - // Any of "linkedin", "cv", "manual", "github", "other". - SourcesSucceeded []string `json:"sourcesSucceeded" api:"required"` - // URLs reviewed for this profile. - URLsAnalyzed []string `json:"urlsAnalyzed" api:"required" format:"uri"` - // Personal website URL, when found. - PersonalWebsiteURL string `json:"personalWebsiteUrl" format:"uri"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Identifiers respjson.Field - SourcesAttempted respjson.Field - SourcesSucceeded respjson.Field - URLsAnalyzed respjson.Field - PersonalWebsiteURL respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponseMetadata) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponseMetadata) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// Identifiers returned for the person. -type BatchSubmitResponseMetadataIdentifiers struct { - // LinkedIn profile URL. - LinkedinURL string `json:"linkedinUrl" format:"uri"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - LinkedinURL respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponseMetadataIdentifiers) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponseMetadataIdentifiers) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// Retrieved person profile. -type BatchSubmitResponsePerson struct { - // Education history. - Education []BatchSubmitResponsePersonEducation `json:"education" api:"required"` - // Work history. - Experience []BatchSubmitResponsePersonExperience `json:"experience" api:"required"` - // Core profile details. - Profile BatchSubmitResponsePersonProfile `json:"profile" api:"required"` - // Listed skills. - Skills []BatchSubmitResponsePersonSkill `json:"skills" api:"required"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Education respjson.Field - Experience respjson.Field - Profile respjson.Field - Skills respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePerson) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePerson) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -type BatchSubmitResponsePersonEducation struct { - // School or institution name. - Institution BatchSubmitResponsePersonEducationInstitution `json:"institution" api:"required"` - // Education dates. - Dates BatchSubmitResponsePersonEducationDates `json:"dates"` - // Additional education details. - Description string `json:"description"` - // Area of study. - FieldOfStudy string `json:"fieldOfStudy"` - // Degree, certificate, or credential. - Qualification string `json:"qualification"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Institution respjson.Field - Dates respjson.Field - Description respjson.Field - FieldOfStudy respjson.Field - Qualification respjson.Field + ID respjson.Field + Crawl respjson.Field + CreatedAt respjson.Field + Credits respjson.Field + Format respjson.Field + Input respjson.Field + InvalidURLs respjson.Field + Mode respjson.Field + Status respjson.Field + Tags respjson.Field + KeyMetadata respjson.Field + WebhookSecret respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` } // Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonEducation) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonEducation) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// School or institution name. -type BatchSubmitResponsePersonEducationInstitution struct { - // Display name. - Display string `json:"display" api:"required"` - // Standardized name, when available. - Normalized string `json:"normalized"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Display respjson.Field - Normalized respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonEducationInstitution) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonEducationInstitution) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// Education dates. -type BatchSubmitResponsePersonEducationDates struct { - // End date, when known. - EndDate BatchSubmitResponsePersonEducationDatesEndDate `json:"endDate"` - // Whether the entry is current. - IsCurrent bool `json:"isCurrent"` - // Start date, when known. - StartDate BatchSubmitResponsePersonEducationDatesStartDate `json:"startDate"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - EndDate respjson.Field - IsCurrent respjson.Field - StartDate respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonEducationDates) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonEducationDates) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// End date, when known. -type BatchSubmitResponsePersonEducationDatesEndDate struct { - // Year value. - Year int64 `json:"year" api:"required"` - // Day value, when known. - Day int64 `json:"day"` - // Month value, when known. - Month int64 `json:"month"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Year respjson.Field - Day respjson.Field - Month respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonEducationDatesEndDate) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonEducationDatesEndDate) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// Start date, when known. -type BatchSubmitResponsePersonEducationDatesStartDate struct { - // Year value. - Year int64 `json:"year" api:"required"` - // Day value, when known. - Day int64 `json:"day"` - // Month value, when known. - Month int64 `json:"month"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Year respjson.Field - Day respjson.Field - Month respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonEducationDatesStartDate) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonEducationDatesStartDate) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -type BatchSubmitResponsePersonExperience struct { - // Company or organization name. - Company BatchSubmitResponsePersonExperienceCompany `json:"company" api:"required"` - // Role or job title. - Title string `json:"title" api:"required"` - // Role dates. - Dates BatchSubmitResponsePersonExperienceDates `json:"dates"` - // Role description. - Description string `json:"description"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Company respjson.Field - Title respjson.Field - Dates respjson.Field - Description respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonExperience) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonExperience) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// Company or organization name. -type BatchSubmitResponsePersonExperienceCompany struct { - // Display name. - Display string `json:"display" api:"required"` - // Standardized name, when available. - Normalized string `json:"normalized"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Display respjson.Field - Normalized respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonExperienceCompany) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonExperienceCompany) UnmarshalJSON(data []byte) error { +func (r BatchSubmitResponse) RawJSON() string { return r.JSON.raw } +func (r *BatchSubmitResponse) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Role dates. -type BatchSubmitResponsePersonExperienceDates struct { - // End date, when known. - EndDate BatchSubmitResponsePersonExperienceDatesEndDate `json:"endDate"` - // Whether the entry is current. - IsCurrent bool `json:"isCurrent"` - // Start date, when known. - StartDate BatchSubmitResponsePersonExperienceDatesStartDate `json:"startDate"` +// What accepting this batch cost. +type BatchSubmitResponseCredits struct { + // Credits just debited from your balance. Whatever the batch does not spend is + // refunded when it settles. + Reserved int64 `json:"reserved" api:"required"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { - EndDate respjson.Field - IsCurrent respjson.Field - StartDate respjson.Field + Reserved respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` } // Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonExperienceDates) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonExperienceDates) UnmarshalJSON(data []byte) error { +func (r BatchSubmitResponseCredits) RawJSON() string { return r.JSON.raw } +func (r *BatchSubmitResponseCredits) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// End date, when known. -type BatchSubmitResponsePersonExperienceDatesEndDate struct { - // Year value. - Year int64 `json:"year" api:"required"` - // Day value, when known. - Day int64 `json:"day"` - // Month value, when known. - Month int64 `json:"month"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Year respjson.Field - Day respjson.Field - Month respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} +// What each page will be returned as. +type BatchSubmitResponseFormat string -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonExperienceDatesEndDate) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonExperienceDatesEndDate) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} +const ( + BatchSubmitResponseFormatMarkdown BatchSubmitResponseFormat = "markdown" + BatchSubmitResponseFormatHTML BatchSubmitResponseFormat = "html" +) -// Start date, when known. -type BatchSubmitResponsePersonExperienceDatesStartDate struct { - // Year value. - Year int64 `json:"year" api:"required"` - // Day value, when known. - Day int64 `json:"day"` - // Month value, when known. - Month int64 `json:"month"` +type BatchSubmitResponseInvalidURL struct { + // Why it was rejected. + Reason string `json:"reason" api:"required"` + // Rejected URL. + URL string `json:"url" api:"required"` // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. JSON struct { - Year respjson.Field - Day respjson.Field - Month respjson.Field + Reason respjson.Field + URL respjson.Field ExtraFields map[string]respjson.Field raw string } `json:"-"` } // Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonExperienceDatesStartDate) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonExperienceDatesStartDate) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} - -// Core profile details. -type BatchSubmitResponsePersonProfile struct { - // Person's full name. - FullName string `json:"fullName"` - // Short professional headline. - Headline string `json:"headline"` - // Person's listed location. - Location string `json:"location"` - // Profile image URL. - ProfilePictureURL string `json:"profilePictureUrl" format:"uri"` - // Brief profile summary. - Summary string `json:"summary"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - FullName respjson.Field - Headline respjson.Field - Location respjson.Field - ProfilePictureURL respjson.Field - Summary respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} - -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonProfile) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonProfile) UnmarshalJSON(data []byte) error { +func (r BatchSubmitResponseInvalidURL) RawJSON() string { return r.JSON.raw } +func (r *BatchSubmitResponseInvalidURL) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -type BatchSubmitResponsePersonSkill struct { - // Skill name. - Name string `json:"name" api:"required"` - // Standardized skill name, when available. - Normalized string `json:"normalized"` - // Skill proficiency, when available. - Proficiency string `json:"proficiency"` - // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. - JSON struct { - Name respjson.Field - Normalized respjson.Field - Proficiency respjson.Field - ExtraFields map[string]respjson.Field - raw string - } `json:"-"` -} +// How pages will be selected. +type BatchSubmitResponseMode string -// Returns the unmodified JSON received from the API -func (r BatchSubmitResponsePersonSkill) RawJSON() string { return r.JSON.raw } -func (r *BatchSubmitResponsePersonSkill) UnmarshalJSON(data []byte) error { - return apijson.UnmarshalRoot(data, r) -} +const ( + BatchSubmitResponseModeScrape BatchSubmitResponseMode = "scrape" + BatchSubmitResponseModeCrawl BatchSubmitResponseMode = "crawl" +) -// Response status. +// Always `queued`. An accepted batch has not started yet. type BatchSubmitResponseStatus string const ( - BatchSubmitResponseStatusOk BatchSubmitResponseStatus = "ok" + BatchSubmitResponseStatusQueued BatchSubmitResponseStatus = "queued" ) -// Metadata about the API key used for the request. Included in every response -// whenever a valid API key is provided, even when the response status is not 200. +// API key usage for this request. type BatchSubmitResponseKeyMetadata struct { // The number of credits consumed by this request. CreditsConsumed int64 `json:"credits_consumed" api:"required"` @@ -1884,13 +1653,14 @@ func (r BatchGetResultsParams) URLQuery() (v url.Values, err error) { } type BatchSubmitParams struct { - // Known identifiers for the person. At least one identifier is required. - Identifiers BatchSubmitParamsIdentifiers `json:"identifiers,omitzero" api:"required"` - // Optional timeout in milliseconds for the request. If the request takes longer - // than this value, it will be aborted with a 408 status code. Maximum allowed - // value is 300000ms (5 minutes). - TimeoutMs param.Opt[int64] `json:"timeoutMS,omitzero"` - // Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters. + // Choose a URL list or a site crawl. + Input BatchSubmitParamsInputUnion `json:"input,omitzero" api:"required"` + // URL notified when the batch finishes. + WebhookURL param.Opt[string] `json:"webhookUrl,omitzero"` + // Any string unique to this submission. Retries with the same key return the + // original batch. + IdempotencyKey param.Opt[string] `header:"Idempotency-Key,omitzero" json:"-"` + // Tags stored on the batch. Filter the batch list by them later. Tags []string `json:"tags,omitzero"` paramObj } @@ -1903,17 +1673,1032 @@ func (r *BatchSubmitParams) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -// Known identifiers for the person. At least one identifier is required. -type BatchSubmitParamsIdentifiers struct { - // LinkedIn profile URL, e.g. https://www.linkedin.com/in/yahia-bakour/. - LinkedinURL param.Opt[string] `json:"linkedinUrl,omitzero" format:"uri"` +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputUnion struct { + OfScrape *BatchSubmitParamsInputScrape `json:",omitzero,inline"` + OfCrawl *BatchSubmitParamsInputCrawl `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfScrape, u.OfCrawl) +} +func (u *BatchSubmitParamsInputUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +func init() { + apijson.RegisterUnion[BatchSubmitParamsInputUnion]( + "mode", + apijson.Discriminator[BatchSubmitParamsInputScrape]("scrape"), + apijson.Discriminator[BatchSubmitParamsInputCrawl]("crawl"), + ) +} + +// Scrape up to 25K URLs in one batch. +// +// The properties Data, Mode are required. +type BatchSubmitParamsInputScrape struct { + // Pages to scrape and their output format. + Data BatchSubmitParamsInputScrapeDataUnion `json:"data,omitzero" api:"required"` + // Scrape the pages in `data.urls`. + // + // This field can be elided, and will marshal its zero value as "scrape". + Mode constant.Scrape `json:"mode" default:"scrape"` + paramObj +} + +func (r BatchSubmitParamsInputScrape) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputScrape + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputScrape) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputScrapeDataUnion struct { + OfMarkdown *BatchSubmitParamsInputScrapeDataMarkdown `json:",omitzero,inline"` + OfHTML *BatchSubmitParamsInputScrapeDataHTML `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputScrapeDataUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfMarkdown, u.OfHTML) +} +func (u *BatchSubmitParamsInputScrapeDataUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +func init() { + apijson.RegisterUnion[BatchSubmitParamsInputScrapeDataUnion]( + "format", + apijson.Discriminator[BatchSubmitParamsInputScrapeDataMarkdown]("markdown"), + apijson.Discriminator[BatchSubmitParamsInputScrapeDataHTML]("html"), + ) +} + +// Scrape the listed pages as Markdown. +// +// The properties Format, URLs are required. +type BatchSubmitParamsInputScrapeDataMarkdown struct { + // Pages to scrape. Maximum 25000. + URLs []BatchSubmitParamsInputScrapeDataMarkdownURL `json:"urls,omitzero" api:"required"` + // Options for Markdown output. + Options BatchSubmitParamsInputScrapeDataMarkdownOptions `json:"options,omitzero"` + // Return page content as Markdown. + // + // This field can be elided, and will marshal its zero value as "markdown". + Format constant.Markdown `json:"format" default:"markdown"` paramObj } -func (r BatchSubmitParamsIdentifiers) MarshalJSON() (data []byte, err error) { - type shadow BatchSubmitParamsIdentifiers +func (r BatchSubmitParamsInputScrapeDataMarkdown) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputScrapeDataMarkdown return param.MarshalObject(r, (*shadow)(&r)) } -func (r *BatchSubmitParamsIdentifiers) UnmarshalJSON(data []byte) error { +func (r *BatchSubmitParamsInputScrapeDataMarkdown) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } + +// A page to scrape, with optional data for matching results. +// +// The property URL is required. +type BatchSubmitParamsInputScrapeDataMarkdownURL struct { + // Page URL to scrape. + URL string `json:"url" api:"required"` + // Your ID for this page, returned with its result. The same URL can use different + // IDs. + ItemID param.Opt[string] `json:"itemId,omitzero"` + // Custom JSON returned unchanged with this page result. + Meta map[string]any `json:"meta,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputScrapeDataMarkdownURL) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputScrapeDataMarkdownURL + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputScrapeDataMarkdownURL) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Options for Markdown output. +type BatchSubmitParamsInputScrapeDataMarkdownOptions struct { + // Return a cached result if a prior scrape for the same parameters exists and is + // younger than this many milliseconds. Defaults to 1 day (86400000 ms) when + // omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. + MaxAgeMs param.Opt[int64] `json:"maxAgeMs,omitzero"` + // Include image references in the Markdown. + IncludeImages param.Opt[bool] `json:"includeImages,omitzero"` + // Include links in the Markdown. + IncludeLinks param.Opt[bool] `json:"includeLinks,omitzero"` + // Wait briefly for CSS and transition animations to settle before extraction, on + // pages that render in a browser. + SettleAnimations param.Opt[bool] `json:"settleAnimations,omitzero"` + // Shorten inline base64 image data. + ShortenBase64Images param.Opt[bool] `json:"shortenBase64Images,omitzero"` + // Return the main content without navigation or footers. + UseMainContentOnly param.Opt[bool] `json:"useMainContentOnly,omitzero"` + // How long to wait after initial page load, in milliseconds. `0` waits 500 ms. + WaitForMs param.Opt[int64] `json:"waitForMs,omitzero"` + // Remove elements matching these CSS selectors. Applied after `includeSelectors`, + // so an element matching both is removed. + ExcludeSelectors []string `json:"excludeSelectors,omitzero"` + // Keep only the subtrees matching these CSS selectors. Filtered pages are always + // fetched fresh, ignoring `maxAgeMs`. + IncludeSelectors []string `json:"includeSelectors,omitzero"` + // Fetch the target page through a residential proxy in this country (ISO 3166-1 + // alpha-2). + // + // Any of "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", + // "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", + // "bq", "br", "bs", "bw", "by", "bz", "ca", "cd", "cf", "cg", "ch", "ci", "cl", + // "cm", "cn", "co", "cr", "cv", "cw", "cy", "cz", "de", "dj", "dk", "dm", "do", + // "dz", "ec", "ee", "eg", "es", "et", "fi", "fj", "fr", "ga", "gb", "gd", "ge", + // "gf", "gg", "gh", "gm", "gn", "gp", "gq", "gr", "gt", "gu", "gw", "gy", "hk", + // "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "iq", "ir", "is", "it", + // "je", "jm", "jo", "jp", "ke", "kg", "kh", "kn", "kr", "kw", "ky", "kz", "la", + // "lb", "lc", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", + // "mf", "mg", "mk", "ml", "mm", "mn", "mo", "mq", "mr", "mt", "mu", "mv", "mw", + // "mx", "my", "mz", "na", "nc", "ne", "ng", "ni", "nl", "no", "np", "nz", "om", + // "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pr", "ps", "pt", "py", "qa", "re", + // "ro", "rs", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", "sl", "sm", + // "sn", "so", "sr", "ss", "st", "sv", "sx", "sy", "sz", "tc", "td", "tg", "th", + // "tj", "tl", "tm", "tn", "tr", "tt", "tw", "tz", "ua", "ug", "us", "uy", "uz", + // "vc", "ve", "vg", "vi", "vn", "ye", "yt", "za", "zm", "zw". + Country string `json:"country,omitzero"` + // PDF parsing controls. Use start/end to limit text extraction and embedded-image + // detection/OCR to an inclusive 1-based page range. + Pdf BatchSubmitParamsInputScrapeDataMarkdownOptionsPdf `json:"pdf,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputScrapeDataMarkdownOptions) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputScrapeDataMarkdownOptions + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputScrapeDataMarkdownOptions) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +func init() { + apijson.RegisterFieldValidator[BatchSubmitParamsInputScrapeDataMarkdownOptions]( + "country", "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", "bq", "br", "bs", "bw", "by", "bz", "ca", "cd", "cf", "cg", "ch", "ci", "cl", "cm", "cn", "co", "cr", "cv", "cw", "cy", "cz", "de", "dj", "dk", "dm", "do", "dz", "ec", "ee", "eg", "es", "et", "fi", "fj", "fr", "ga", "gb", "gd", "ge", "gf", "gg", "gh", "gm", "gn", "gp", "gq", "gr", "gt", "gu", "gw", "gy", "hk", "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "iq", "ir", "is", "it", "je", "jm", "jo", "jp", "ke", "kg", "kh", "kn", "kr", "kw", "ky", "kz", "la", "lb", "lc", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", "mf", "mg", "mk", "ml", "mm", "mn", "mo", "mq", "mr", "mt", "mu", "mv", "mw", "mx", "my", "mz", "na", "nc", "ne", "ng", "ni", "nl", "no", "np", "nz", "om", "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pr", "ps", "pt", "py", "qa", "re", "ro", "rs", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", "sl", "sm", "sn", "so", "sr", "ss", "st", "sv", "sx", "sy", "sz", "tc", "td", "tg", "th", "tj", "tl", "tm", "tn", "tr", "tt", "tw", "tz", "ua", "ug", "us", "uy", "uz", "vc", "ve", "vg", "vi", "vn", "ye", "yt", "za", "zm", "zw", + ) +} + +// PDF parsing controls. Use start/end to limit text extraction and embedded-image +// detection/OCR to an inclusive 1-based page range. +type BatchSubmitParamsInputScrapeDataMarkdownOptionsPdf struct { + // Last 1-based PDF page to parse. When omitted, parsing ends at the last page. + // Must be greater than or equal to start when both are provided. + End param.Opt[int64] `json:"end,omitzero"` + // First 1-based PDF page to parse. When omitted, parsing starts at the first page. + Start param.Opt[int64] `json:"start,omitzero"` + // When true, OCR the selected PDF pages that have no usable text layer (scans), + // replacing each recovered page's text with the OCR result while pages with a real + // text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + // of the base request cost. When false, no OCR runs. + Ocr BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfOcrUnion `json:"ocr,omitzero"` + // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and + // a 400 PDF_SKIPPED is returned. + ShouldParse BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfShouldParseUnion `json:"shouldParse,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputScrapeDataMarkdownOptionsPdf) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputScrapeDataMarkdownOptionsPdf + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputScrapeDataMarkdownOptionsPdf) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfOcrUnion struct { + OfBool param.Opt[bool] `json:",omitzero,inline"` + // Check if union is this variant with + // !param.IsOmitted(union.OfBatchSubmitsInputScrapeDataMarkdownOptionsPdfOcrString) + OfBatchSubmitsInputScrapeDataMarkdownOptionsPdfOcrString param.Opt[string] `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfOcrUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfBool, u.OfBatchSubmitsInputScrapeDataMarkdownOptionsPdfOcrString) +} +func (u *BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfOcrUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +type BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfOcrString string + +const ( + BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfOcrStringTrue BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfOcrString = "true" + BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfOcrStringFalse BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfOcrString = "false" +) + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfShouldParseUnion struct { + OfBool param.Opt[bool] `json:",omitzero,inline"` + // Check if union is this variant with + // !param.IsOmitted(union.OfBatchSubmitsInputScrapeDataMarkdownOptionsPdfShouldParseString) + OfBatchSubmitsInputScrapeDataMarkdownOptionsPdfShouldParseString param.Opt[string] `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfShouldParseUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfBool, u.OfBatchSubmitsInputScrapeDataMarkdownOptionsPdfShouldParseString) +} +func (u *BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfShouldParseUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +type BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfShouldParseString string + +const ( + BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfShouldParseStringTrue BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfShouldParseString = "true" + BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfShouldParseStringFalse BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfShouldParseString = "false" +) + +// Scrape the listed pages as HTML. +// +// The properties Format, URLs are required. +type BatchSubmitParamsInputScrapeDataHTML struct { + // Pages to scrape. Maximum 25000. + URLs []BatchSubmitParamsInputScrapeDataHTMLURL `json:"urls,omitzero" api:"required"` + // Options for HTML output. + Options BatchSubmitParamsInputScrapeDataHTMLOptions `json:"options,omitzero"` + // Return page content as HTML. + // + // This field can be elided, and will marshal its zero value as "html". + Format constant.HTML `json:"format" default:"html"` + paramObj +} + +func (r BatchSubmitParamsInputScrapeDataHTML) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputScrapeDataHTML + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputScrapeDataHTML) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// A page to scrape, with optional data for matching results. +// +// The property URL is required. +type BatchSubmitParamsInputScrapeDataHTMLURL struct { + // Page URL to scrape. + URL string `json:"url" api:"required"` + // Your ID for this page, returned with its result. The same URL can use different + // IDs. + ItemID param.Opt[string] `json:"itemId,omitzero"` + // Custom JSON returned unchanged with this page result. + Meta map[string]any `json:"meta,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputScrapeDataHTMLURL) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputScrapeDataHTMLURL + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputScrapeDataHTMLURL) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Options for HTML output. +type BatchSubmitParamsInputScrapeDataHTMLOptions struct { + // Return a cached result if a prior scrape for the same parameters exists and is + // younger than this many milliseconds. Defaults to 1 day (86400000 ms) when + // omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. + MaxAgeMs param.Opt[int64] `json:"maxAgeMs,omitzero"` + // Wait briefly for CSS and transition animations to settle before extraction, on + // pages that render in a browser. + SettleAnimations param.Opt[bool] `json:"settleAnimations,omitzero"` + // Return the main content without navigation or footers. + UseMainContentOnly param.Opt[bool] `json:"useMainContentOnly,omitzero"` + // How long to wait after initial page load, in milliseconds. `0` waits 500 ms. + WaitForMs param.Opt[int64] `json:"waitForMs,omitzero"` + // Remove elements matching these CSS selectors. Applied after `includeSelectors`, + // so an element matching both is removed. + ExcludeSelectors []string `json:"excludeSelectors,omitzero"` + // Keep only the subtrees matching these CSS selectors. Filtered pages are always + // fetched fresh, ignoring `maxAgeMs`. + IncludeSelectors []string `json:"includeSelectors,omitzero"` + // Fetch the target page through a residential proxy in this country (ISO 3166-1 + // alpha-2). + // + // Any of "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", + // "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", + // "bq", "br", "bs", "bw", "by", "bz", "ca", "cd", "cf", "cg", "ch", "ci", "cl", + // "cm", "cn", "co", "cr", "cv", "cw", "cy", "cz", "de", "dj", "dk", "dm", "do", + // "dz", "ec", "ee", "eg", "es", "et", "fi", "fj", "fr", "ga", "gb", "gd", "ge", + // "gf", "gg", "gh", "gm", "gn", "gp", "gq", "gr", "gt", "gu", "gw", "gy", "hk", + // "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "iq", "ir", "is", "it", + // "je", "jm", "jo", "jp", "ke", "kg", "kh", "kn", "kr", "kw", "ky", "kz", "la", + // "lb", "lc", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", + // "mf", "mg", "mk", "ml", "mm", "mn", "mo", "mq", "mr", "mt", "mu", "mv", "mw", + // "mx", "my", "mz", "na", "nc", "ne", "ng", "ni", "nl", "no", "np", "nz", "om", + // "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pr", "ps", "pt", "py", "qa", "re", + // "ro", "rs", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", "sl", "sm", + // "sn", "so", "sr", "ss", "st", "sv", "sx", "sy", "sz", "tc", "td", "tg", "th", + // "tj", "tl", "tm", "tn", "tr", "tt", "tw", "tz", "ua", "ug", "us", "uy", "uz", + // "vc", "ve", "vg", "vi", "vn", "ye", "yt", "za", "zm", "zw". + Country string `json:"country,omitzero"` + // PDF parsing controls. Use start/end to limit text extraction and embedded-image + // detection/OCR to an inclusive 1-based page range. + Pdf BatchSubmitParamsInputScrapeDataHTMLOptionsPdf `json:"pdf,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputScrapeDataHTMLOptions) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputScrapeDataHTMLOptions + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputScrapeDataHTMLOptions) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +func init() { + apijson.RegisterFieldValidator[BatchSubmitParamsInputScrapeDataHTMLOptions]( + "country", "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", "bq", "br", "bs", "bw", "by", "bz", "ca", "cd", "cf", "cg", "ch", "ci", "cl", "cm", "cn", "co", "cr", "cv", "cw", "cy", "cz", "de", "dj", "dk", "dm", "do", "dz", "ec", "ee", "eg", "es", "et", "fi", "fj", "fr", "ga", "gb", "gd", "ge", "gf", "gg", "gh", "gm", "gn", "gp", "gq", "gr", "gt", "gu", "gw", "gy", "hk", "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "iq", "ir", "is", "it", "je", "jm", "jo", "jp", "ke", "kg", "kh", "kn", "kr", "kw", "ky", "kz", "la", "lb", "lc", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", "mf", "mg", "mk", "ml", "mm", "mn", "mo", "mq", "mr", "mt", "mu", "mv", "mw", "mx", "my", "mz", "na", "nc", "ne", "ng", "ni", "nl", "no", "np", "nz", "om", "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pr", "ps", "pt", "py", "qa", "re", "ro", "rs", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", "sl", "sm", "sn", "so", "sr", "ss", "st", "sv", "sx", "sy", "sz", "tc", "td", "tg", "th", "tj", "tl", "tm", "tn", "tr", "tt", "tw", "tz", "ua", "ug", "us", "uy", "uz", "vc", "ve", "vg", "vi", "vn", "ye", "yt", "za", "zm", "zw", + ) +} + +// PDF parsing controls. Use start/end to limit text extraction and embedded-image +// detection/OCR to an inclusive 1-based page range. +type BatchSubmitParamsInputScrapeDataHTMLOptionsPdf struct { + // Last 1-based PDF page to parse. When omitted, parsing ends at the last page. + // Must be greater than or equal to start when both are provided. + End param.Opt[int64] `json:"end,omitzero"` + // First 1-based PDF page to parse. When omitted, parsing starts at the first page. + Start param.Opt[int64] `json:"start,omitzero"` + // When true, OCR the selected PDF pages that have no usable text layer (scans), + // replacing each recovered page's text with the OCR result while pages with a real + // text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + // of the base request cost. When false, no OCR runs. + Ocr BatchSubmitParamsInputScrapeDataHTMLOptionsPdfOcrUnion `json:"ocr,omitzero"` + // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and + // a 400 PDF_SKIPPED is returned. + ShouldParse BatchSubmitParamsInputScrapeDataHTMLOptionsPdfShouldParseUnion `json:"shouldParse,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputScrapeDataHTMLOptionsPdf) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputScrapeDataHTMLOptionsPdf + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputScrapeDataHTMLOptionsPdf) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputScrapeDataHTMLOptionsPdfOcrUnion struct { + OfBool param.Opt[bool] `json:",omitzero,inline"` + // Check if union is this variant with + // !param.IsOmitted(union.OfBatchSubmitsInputScrapeDataHTMLOptionsPdfOcrString) + OfBatchSubmitsInputScrapeDataHTMLOptionsPdfOcrString param.Opt[string] `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputScrapeDataHTMLOptionsPdfOcrUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfBool, u.OfBatchSubmitsInputScrapeDataHTMLOptionsPdfOcrString) +} +func (u *BatchSubmitParamsInputScrapeDataHTMLOptionsPdfOcrUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +type BatchSubmitParamsInputScrapeDataHTMLOptionsPdfOcrString string + +const ( + BatchSubmitParamsInputScrapeDataHTMLOptionsPdfOcrStringTrue BatchSubmitParamsInputScrapeDataHTMLOptionsPdfOcrString = "true" + BatchSubmitParamsInputScrapeDataHTMLOptionsPdfOcrStringFalse BatchSubmitParamsInputScrapeDataHTMLOptionsPdfOcrString = "false" +) + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputScrapeDataHTMLOptionsPdfShouldParseUnion struct { + OfBool param.Opt[bool] `json:",omitzero,inline"` + // Check if union is this variant with + // !param.IsOmitted(union.OfBatchSubmitsInputScrapeDataHTMLOptionsPdfShouldParseString) + OfBatchSubmitsInputScrapeDataHTMLOptionsPdfShouldParseString param.Opt[string] `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputScrapeDataHTMLOptionsPdfShouldParseUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfBool, u.OfBatchSubmitsInputScrapeDataHTMLOptionsPdfShouldParseString) +} +func (u *BatchSubmitParamsInputScrapeDataHTMLOptionsPdfShouldParseUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +type BatchSubmitParamsInputScrapeDataHTMLOptionsPdfShouldParseString string + +const ( + BatchSubmitParamsInputScrapeDataHTMLOptionsPdfShouldParseStringTrue BatchSubmitParamsInputScrapeDataHTMLOptionsPdfShouldParseString = "true" + BatchSubmitParamsInputScrapeDataHTMLOptionsPdfShouldParseStringFalse BatchSubmitParamsInputScrapeDataHTMLOptionsPdfShouldParseString = "false" +) + +// Crawl pages starting from a URL or from a domain's sitemap. +// +// The properties Data, Mode are required. +type BatchSubmitParamsInputCrawl struct { + // Crawl source and output format. + Data BatchSubmitParamsInputCrawlDataUnion `json:"data,omitzero" api:"required"` + // Discover and scrape pages from `data.source`. + // + // This field can be elided, and will marshal its zero value as "crawl". + Mode constant.Crawl `json:"mode" default:"crawl"` + paramObj +} + +func (r BatchSubmitParamsInputCrawl) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawl + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawl) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputCrawlDataUnion struct { + OfMarkdown *BatchSubmitParamsInputCrawlDataMarkdown `json:",omitzero,inline"` + OfHTML *BatchSubmitParamsInputCrawlDataHTML `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputCrawlDataUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfMarkdown, u.OfHTML) +} +func (u *BatchSubmitParamsInputCrawlDataUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +func init() { + apijson.RegisterUnion[BatchSubmitParamsInputCrawlDataUnion]( + "format", + apijson.Discriminator[BatchSubmitParamsInputCrawlDataMarkdown]("markdown"), + apijson.Discriminator[BatchSubmitParamsInputCrawlDataHTML]("html"), + ) +} + +// Crawl pages and return Markdown. +// +// The properties Format, Source are required. +type BatchSubmitParamsInputCrawlDataMarkdown struct { + // How to find pages to crawl. + Source BatchSubmitParamsInputCrawlDataMarkdownSourceUnion `json:"source,omitzero" api:"required"` + // Options for Markdown output. + Options BatchSubmitParamsInputCrawlDataMarkdownOptions `json:"options,omitzero"` + // Return page content as Markdown. + // + // This field can be elided, and will marshal its zero value as "markdown". + Format constant.Markdown `json:"format" default:"markdown"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataMarkdown) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataMarkdown + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataMarkdown) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputCrawlDataMarkdownSourceUnion struct { + OfStartURL *BatchSubmitParamsInputCrawlDataMarkdownSourceStartURL `json:",omitzero,inline"` + OfSitemap *BatchSubmitParamsInputCrawlDataMarkdownSourceSitemap `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputCrawlDataMarkdownSourceUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfStartURL, u.OfSitemap) +} +func (u *BatchSubmitParamsInputCrawlDataMarkdownSourceUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +func init() { + apijson.RegisterUnion[BatchSubmitParamsInputCrawlDataMarkdownSourceUnion]( + "type", + apijson.Discriminator[BatchSubmitParamsInputCrawlDataMarkdownSourceStartURL]("start_url"), + apijson.Discriminator[BatchSubmitParamsInputCrawlDataMarkdownSourceSitemap]("sitemap"), + ) +} + +// Discover pages by following links from one URL. +// +// The properties Type, URL are required. +type BatchSubmitParamsInputCrawlDataMarkdownSourceStartURL struct { + // Page where crawling begins. A URL without a scheme is read as https://. + URL string `json:"url" api:"required"` + // Limits and filters for page discovery. + Controls BatchSubmitParamsInputCrawlDataMarkdownSourceStartURLControls `json:"controls,omitzero"` + // Start from one page. + // + // This field can be elided, and will marshal its zero value as "start_url". + Type constant.StartURL `json:"type" default:"start_url"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataMarkdownSourceStartURL) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataMarkdownSourceStartURL + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataMarkdownSourceStartURL) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Limits and filters for page discovery. +type BatchSubmitParamsInputCrawlDataMarkdownSourceStartURLControls struct { + // Follow links to subdomains. + FollowSubdomains param.Opt[bool] `json:"followSubdomains,omitzero"` + // Maximum link depth. Source pages are depth 0. No limit when omitted. + MaxDepth param.Opt[int64] `json:"maxDepth,omitzero"` + // Maximum pages to fetch. Unused reserved credits are refunded. Maximum 25000. + MaxURLs param.Opt[int64] `json:"maxUrls,omitzero"` + // RE2 pattern for URLs to include. The `start_url` itself is always included. + Regex param.Opt[string] `json:"regex,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataMarkdownSourceStartURLControls) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataMarkdownSourceStartURLControls + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataMarkdownSourceStartURLControls) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Scrape the pages listed in a domain's sitemap. Links on those pages are not +// followed. +// +// The properties Domain, Type are required. +type BatchSubmitParamsInputCrawlDataMarkdownSourceSitemap struct { + // Domain whose sitemap lists the pages to scrape. A full URL is reduced to its + // domain. + Domain string `json:"domain" api:"required"` + // Limits and filters for the sitemap URLs. A sitemap batch scrapes exactly those + // URLs and never follows links off them, so there is no crawl depth here. + Controls BatchSubmitParamsInputCrawlDataMarkdownSourceSitemapControls `json:"controls,omitzero"` + // Scrape the URLs in the domain's sitemap. + // + // This field can be elided, and will marshal its zero value as "sitemap". + Type constant.Sitemap `json:"type" default:"sitemap"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataMarkdownSourceSitemap) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataMarkdownSourceSitemap + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataMarkdownSourceSitemap) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Limits and filters for the sitemap URLs. A sitemap batch scrapes exactly those +// URLs and never follows links off them, so there is no crawl depth here. +type BatchSubmitParamsInputCrawlDataMarkdownSourceSitemapControls struct { + // Maximum pages to fetch. Unused reserved credits are refunded. Maximum 25000. + MaxURLs param.Opt[int64] `json:"maxUrls,omitzero"` + // RE2 pattern; only sitemap URLs matching it are scraped. + Regex param.Opt[string] `json:"regex,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataMarkdownSourceSitemapControls) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataMarkdownSourceSitemapControls + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataMarkdownSourceSitemapControls) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Options for Markdown output. +type BatchSubmitParamsInputCrawlDataMarkdownOptions struct { + // Return a cached result if a prior scrape for the same parameters exists and is + // younger than this many milliseconds. Defaults to 1 day (86400000 ms) when + // omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. + MaxAgeMs param.Opt[int64] `json:"maxAgeMs,omitzero"` + // Include image references in the Markdown. + IncludeImages param.Opt[bool] `json:"includeImages,omitzero"` + // Include links in the Markdown. + IncludeLinks param.Opt[bool] `json:"includeLinks,omitzero"` + // Wait briefly for CSS and transition animations to settle before extraction, on + // pages that render in a browser. + SettleAnimations param.Opt[bool] `json:"settleAnimations,omitzero"` + // Shorten inline base64 image data. + ShortenBase64Images param.Opt[bool] `json:"shortenBase64Images,omitzero"` + // Return the main content without navigation or footers. + UseMainContentOnly param.Opt[bool] `json:"useMainContentOnly,omitzero"` + // How long to wait after initial page load, in milliseconds. `0` waits 500 ms. + WaitForMs param.Opt[int64] `json:"waitForMs,omitzero"` + // Remove elements matching these CSS selectors. Applied after `includeSelectors`, + // so an element matching both is removed. + ExcludeSelectors []string `json:"excludeSelectors,omitzero"` + // Keep only the subtrees matching these CSS selectors. Filtered pages are always + // fetched fresh, ignoring `maxAgeMs`. + IncludeSelectors []string `json:"includeSelectors,omitzero"` + // Fetch the target page through a residential proxy in this country (ISO 3166-1 + // alpha-2). + // + // Any of "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", + // "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", + // "bq", "br", "bs", "bw", "by", "bz", "ca", "cd", "cf", "cg", "ch", "ci", "cl", + // "cm", "cn", "co", "cr", "cv", "cw", "cy", "cz", "de", "dj", "dk", "dm", "do", + // "dz", "ec", "ee", "eg", "es", "et", "fi", "fj", "fr", "ga", "gb", "gd", "ge", + // "gf", "gg", "gh", "gm", "gn", "gp", "gq", "gr", "gt", "gu", "gw", "gy", "hk", + // "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "iq", "ir", "is", "it", + // "je", "jm", "jo", "jp", "ke", "kg", "kh", "kn", "kr", "kw", "ky", "kz", "la", + // "lb", "lc", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", + // "mf", "mg", "mk", "ml", "mm", "mn", "mo", "mq", "mr", "mt", "mu", "mv", "mw", + // "mx", "my", "mz", "na", "nc", "ne", "ng", "ni", "nl", "no", "np", "nz", "om", + // "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pr", "ps", "pt", "py", "qa", "re", + // "ro", "rs", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", "sl", "sm", + // "sn", "so", "sr", "ss", "st", "sv", "sx", "sy", "sz", "tc", "td", "tg", "th", + // "tj", "tl", "tm", "tn", "tr", "tt", "tw", "tz", "ua", "ug", "us", "uy", "uz", + // "vc", "ve", "vg", "vi", "vn", "ye", "yt", "za", "zm", "zw". + Country string `json:"country,omitzero"` + // PDF parsing controls. Use start/end to limit text extraction and embedded-image + // detection/OCR to an inclusive 1-based page range. + Pdf BatchSubmitParamsInputCrawlDataMarkdownOptionsPdf `json:"pdf,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataMarkdownOptions) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataMarkdownOptions + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataMarkdownOptions) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +func init() { + apijson.RegisterFieldValidator[BatchSubmitParamsInputCrawlDataMarkdownOptions]( + "country", "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", "bq", "br", "bs", "bw", "by", "bz", "ca", "cd", "cf", "cg", "ch", "ci", "cl", "cm", "cn", "co", "cr", "cv", "cw", "cy", "cz", "de", "dj", "dk", "dm", "do", "dz", "ec", "ee", "eg", "es", "et", "fi", "fj", "fr", "ga", "gb", "gd", "ge", "gf", "gg", "gh", "gm", "gn", "gp", "gq", "gr", "gt", "gu", "gw", "gy", "hk", "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "iq", "ir", "is", "it", "je", "jm", "jo", "jp", "ke", "kg", "kh", "kn", "kr", "kw", "ky", "kz", "la", "lb", "lc", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", "mf", "mg", "mk", "ml", "mm", "mn", "mo", "mq", "mr", "mt", "mu", "mv", "mw", "mx", "my", "mz", "na", "nc", "ne", "ng", "ni", "nl", "no", "np", "nz", "om", "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pr", "ps", "pt", "py", "qa", "re", "ro", "rs", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", "sl", "sm", "sn", "so", "sr", "ss", "st", "sv", "sx", "sy", "sz", "tc", "td", "tg", "th", "tj", "tl", "tm", "tn", "tr", "tt", "tw", "tz", "ua", "ug", "us", "uy", "uz", "vc", "ve", "vg", "vi", "vn", "ye", "yt", "za", "zm", "zw", + ) +} + +// PDF parsing controls. Use start/end to limit text extraction and embedded-image +// detection/OCR to an inclusive 1-based page range. +type BatchSubmitParamsInputCrawlDataMarkdownOptionsPdf struct { + // Last 1-based PDF page to parse. When omitted, parsing ends at the last page. + // Must be greater than or equal to start when both are provided. + End param.Opt[int64] `json:"end,omitzero"` + // First 1-based PDF page to parse. When omitted, parsing starts at the first page. + Start param.Opt[int64] `json:"start,omitzero"` + // When true, OCR the selected PDF pages that have no usable text layer (scans), + // replacing each recovered page's text with the OCR result while pages with a real + // text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + // of the base request cost. When false, no OCR runs. + Ocr BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfOcrUnion `json:"ocr,omitzero"` + // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and + // a 400 PDF_SKIPPED is returned. + ShouldParse BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfShouldParseUnion `json:"shouldParse,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataMarkdownOptionsPdf) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataMarkdownOptionsPdf + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataMarkdownOptionsPdf) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfOcrUnion struct { + OfBool param.Opt[bool] `json:",omitzero,inline"` + // Check if union is this variant with + // !param.IsOmitted(union.OfBatchSubmitsInputCrawlDataMarkdownOptionsPdfOcrString) + OfBatchSubmitsInputCrawlDataMarkdownOptionsPdfOcrString param.Opt[string] `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfOcrUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfBool, u.OfBatchSubmitsInputCrawlDataMarkdownOptionsPdfOcrString) +} +func (u *BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfOcrUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +type BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfOcrString string + +const ( + BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfOcrStringTrue BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfOcrString = "true" + BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfOcrStringFalse BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfOcrString = "false" +) + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfShouldParseUnion struct { + OfBool param.Opt[bool] `json:",omitzero,inline"` + // Check if union is this variant with + // !param.IsOmitted(union.OfBatchSubmitsInputCrawlDataMarkdownOptionsPdfShouldParseString) + OfBatchSubmitsInputCrawlDataMarkdownOptionsPdfShouldParseString param.Opt[string] `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfShouldParseUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfBool, u.OfBatchSubmitsInputCrawlDataMarkdownOptionsPdfShouldParseString) +} +func (u *BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfShouldParseUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +type BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfShouldParseString string + +const ( + BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfShouldParseStringTrue BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfShouldParseString = "true" + BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfShouldParseStringFalse BatchSubmitParamsInputCrawlDataMarkdownOptionsPdfShouldParseString = "false" +) + +// Crawl pages and return HTML. +// +// The properties Format, Source are required. +type BatchSubmitParamsInputCrawlDataHTML struct { + // How to find pages to crawl. + Source BatchSubmitParamsInputCrawlDataHTMLSourceUnion `json:"source,omitzero" api:"required"` + // Options for HTML output. + Options BatchSubmitParamsInputCrawlDataHTMLOptions `json:"options,omitzero"` + // Return page content as HTML. + // + // This field can be elided, and will marshal its zero value as "html". + Format constant.HTML `json:"format" default:"html"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataHTML) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataHTML + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataHTML) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputCrawlDataHTMLSourceUnion struct { + OfStartURL *BatchSubmitParamsInputCrawlDataHTMLSourceStartURL `json:",omitzero,inline"` + OfSitemap *BatchSubmitParamsInputCrawlDataHTMLSourceSitemap `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputCrawlDataHTMLSourceUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfStartURL, u.OfSitemap) +} +func (u *BatchSubmitParamsInputCrawlDataHTMLSourceUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +func init() { + apijson.RegisterUnion[BatchSubmitParamsInputCrawlDataHTMLSourceUnion]( + "type", + apijson.Discriminator[BatchSubmitParamsInputCrawlDataHTMLSourceStartURL]("start_url"), + apijson.Discriminator[BatchSubmitParamsInputCrawlDataHTMLSourceSitemap]("sitemap"), + ) +} + +// Discover pages by following links from one URL. +// +// The properties Type, URL are required. +type BatchSubmitParamsInputCrawlDataHTMLSourceStartURL struct { + // Page where crawling begins. A URL without a scheme is read as https://. + URL string `json:"url" api:"required"` + // Limits and filters for page discovery. + Controls BatchSubmitParamsInputCrawlDataHTMLSourceStartURLControls `json:"controls,omitzero"` + // Start from one page. + // + // This field can be elided, and will marshal its zero value as "start_url". + Type constant.StartURL `json:"type" default:"start_url"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataHTMLSourceStartURL) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataHTMLSourceStartURL + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataHTMLSourceStartURL) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Limits and filters for page discovery. +type BatchSubmitParamsInputCrawlDataHTMLSourceStartURLControls struct { + // Follow links to subdomains. + FollowSubdomains param.Opt[bool] `json:"followSubdomains,omitzero"` + // Maximum link depth. Source pages are depth 0. No limit when omitted. + MaxDepth param.Opt[int64] `json:"maxDepth,omitzero"` + // Maximum pages to fetch. Unused reserved credits are refunded. Maximum 25000. + MaxURLs param.Opt[int64] `json:"maxUrls,omitzero"` + // RE2 pattern for URLs to include. The `start_url` itself is always included. + Regex param.Opt[string] `json:"regex,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataHTMLSourceStartURLControls) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataHTMLSourceStartURLControls + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataHTMLSourceStartURLControls) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Scrape the pages listed in a domain's sitemap. Links on those pages are not +// followed. +// +// The properties Domain, Type are required. +type BatchSubmitParamsInputCrawlDataHTMLSourceSitemap struct { + // Domain whose sitemap lists the pages to scrape. A full URL is reduced to its + // domain. + Domain string `json:"domain" api:"required"` + // Limits and filters for the sitemap URLs. A sitemap batch scrapes exactly those + // URLs and never follows links off them, so there is no crawl depth here. + Controls BatchSubmitParamsInputCrawlDataHTMLSourceSitemapControls `json:"controls,omitzero"` + // Scrape the URLs in the domain's sitemap. + // + // This field can be elided, and will marshal its zero value as "sitemap". + Type constant.Sitemap `json:"type" default:"sitemap"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataHTMLSourceSitemap) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataHTMLSourceSitemap + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataHTMLSourceSitemap) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Limits and filters for the sitemap URLs. A sitemap batch scrapes exactly those +// URLs and never follows links off them, so there is no crawl depth here. +type BatchSubmitParamsInputCrawlDataHTMLSourceSitemapControls struct { + // Maximum pages to fetch. Unused reserved credits are refunded. Maximum 25000. + MaxURLs param.Opt[int64] `json:"maxUrls,omitzero"` + // RE2 pattern; only sitemap URLs matching it are scraped. + Regex param.Opt[string] `json:"regex,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataHTMLSourceSitemapControls) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataHTMLSourceSitemapControls + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataHTMLSourceSitemapControls) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Options for HTML output. +type BatchSubmitParamsInputCrawlDataHTMLOptions struct { + // Return a cached result if a prior scrape for the same parameters exists and is + // younger than this many milliseconds. Defaults to 1 day (86400000 ms) when + // omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. + MaxAgeMs param.Opt[int64] `json:"maxAgeMs,omitzero"` + // Wait briefly for CSS and transition animations to settle before extraction, on + // pages that render in a browser. + SettleAnimations param.Opt[bool] `json:"settleAnimations,omitzero"` + // Return the main content without navigation or footers. + UseMainContentOnly param.Opt[bool] `json:"useMainContentOnly,omitzero"` + // How long to wait after initial page load, in milliseconds. `0` waits 500 ms. + WaitForMs param.Opt[int64] `json:"waitForMs,omitzero"` + // Remove elements matching these CSS selectors. Applied after `includeSelectors`, + // so an element matching both is removed. + ExcludeSelectors []string `json:"excludeSelectors,omitzero"` + // Keep only the subtrees matching these CSS selectors. Filtered pages are always + // fetched fresh, ignoring `maxAgeMs`. + IncludeSelectors []string `json:"includeSelectors,omitzero"` + // Fetch the target page through a residential proxy in this country (ISO 3166-1 + // alpha-2). + // + // Any of "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", + // "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", + // "bq", "br", "bs", "bw", "by", "bz", "ca", "cd", "cf", "cg", "ch", "ci", "cl", + // "cm", "cn", "co", "cr", "cv", "cw", "cy", "cz", "de", "dj", "dk", "dm", "do", + // "dz", "ec", "ee", "eg", "es", "et", "fi", "fj", "fr", "ga", "gb", "gd", "ge", + // "gf", "gg", "gh", "gm", "gn", "gp", "gq", "gr", "gt", "gu", "gw", "gy", "hk", + // "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "iq", "ir", "is", "it", + // "je", "jm", "jo", "jp", "ke", "kg", "kh", "kn", "kr", "kw", "ky", "kz", "la", + // "lb", "lc", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", + // "mf", "mg", "mk", "ml", "mm", "mn", "mo", "mq", "mr", "mt", "mu", "mv", "mw", + // "mx", "my", "mz", "na", "nc", "ne", "ng", "ni", "nl", "no", "np", "nz", "om", + // "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pr", "ps", "pt", "py", "qa", "re", + // "ro", "rs", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", "sl", "sm", + // "sn", "so", "sr", "ss", "st", "sv", "sx", "sy", "sz", "tc", "td", "tg", "th", + // "tj", "tl", "tm", "tn", "tr", "tt", "tw", "tz", "ua", "ug", "us", "uy", "uz", + // "vc", "ve", "vg", "vi", "vn", "ye", "yt", "za", "zm", "zw". + Country string `json:"country,omitzero"` + // PDF parsing controls. Use start/end to limit text extraction and embedded-image + // detection/OCR to an inclusive 1-based page range. + Pdf BatchSubmitParamsInputCrawlDataHTMLOptionsPdf `json:"pdf,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataHTMLOptions) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataHTMLOptions + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataHTMLOptions) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +func init() { + apijson.RegisterFieldValidator[BatchSubmitParamsInputCrawlDataHTMLOptions]( + "country", "ad", "ae", "af", "ag", "ai", "al", "am", "ao", "ar", "at", "au", "aw", "az", "ba", "bb", "bd", "be", "bf", "bg", "bh", "bi", "bj", "bm", "bn", "bo", "bq", "br", "bs", "bw", "by", "bz", "ca", "cd", "cf", "cg", "ch", "ci", "cl", "cm", "cn", "co", "cr", "cv", "cw", "cy", "cz", "de", "dj", "dk", "dm", "do", "dz", "ec", "ee", "eg", "es", "et", "fi", "fj", "fr", "ga", "gb", "gd", "ge", "gf", "gg", "gh", "gm", "gn", "gp", "gq", "gr", "gt", "gu", "gw", "gy", "hk", "hn", "hr", "ht", "hu", "id", "ie", "il", "im", "in", "iq", "ir", "is", "it", "je", "jm", "jo", "jp", "ke", "kg", "kh", "kn", "kr", "kw", "ky", "kz", "la", "lb", "lc", "lk", "lr", "ls", "lt", "lu", "lv", "ly", "ma", "mc", "md", "me", "mf", "mg", "mk", "ml", "mm", "mn", "mo", "mq", "mr", "mt", "mu", "mv", "mw", "mx", "my", "mz", "na", "nc", "ne", "ng", "ni", "nl", "no", "np", "nz", "om", "pa", "pe", "pf", "pg", "ph", "pk", "pl", "pr", "ps", "pt", "py", "qa", "re", "ro", "rs", "ru", "rw", "sa", "sc", "sd", "se", "sg", "si", "sk", "sl", "sm", "sn", "so", "sr", "ss", "st", "sv", "sx", "sy", "sz", "tc", "td", "tg", "th", "tj", "tl", "tm", "tn", "tr", "tt", "tw", "tz", "ua", "ug", "us", "uy", "uz", "vc", "ve", "vg", "vi", "vn", "ye", "yt", "za", "zm", "zw", + ) +} + +// PDF parsing controls. Use start/end to limit text extraction and embedded-image +// detection/OCR to an inclusive 1-based page range. +type BatchSubmitParamsInputCrawlDataHTMLOptionsPdf struct { + // Last 1-based PDF page to parse. When omitted, parsing ends at the last page. + // Must be greater than or equal to start when both are provided. + End param.Opt[int64] `json:"end,omitzero"` + // First 1-based PDF page to parse. When omitted, parsing starts at the first page. + Start param.Opt[int64] `json:"start,omitzero"` + // When true, OCR the selected PDF pages that have no usable text layer (scans), + // replacing each recovered page's text with the OCR result while pages with a real + // text layer keep it. Billed at 1 credit per page OCR actually recovered, on top + // of the base request cost. When false, no OCR runs. + Ocr BatchSubmitParamsInputCrawlDataHTMLOptionsPdfOcrUnion `json:"ocr,omitzero"` + // When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and + // a 400 PDF_SKIPPED is returned. + ShouldParse BatchSubmitParamsInputCrawlDataHTMLOptionsPdfShouldParseUnion `json:"shouldParse,omitzero"` + paramObj +} + +func (r BatchSubmitParamsInputCrawlDataHTMLOptionsPdf) MarshalJSON() (data []byte, err error) { + type shadow BatchSubmitParamsInputCrawlDataHTMLOptionsPdf + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *BatchSubmitParamsInputCrawlDataHTMLOptionsPdf) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputCrawlDataHTMLOptionsPdfOcrUnion struct { + OfBool param.Opt[bool] `json:",omitzero,inline"` + // Check if union is this variant with + // !param.IsOmitted(union.OfBatchSubmitsInputCrawlDataHTMLOptionsPdfOcrString) + OfBatchSubmitsInputCrawlDataHTMLOptionsPdfOcrString param.Opt[string] `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputCrawlDataHTMLOptionsPdfOcrUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfBool, u.OfBatchSubmitsInputCrawlDataHTMLOptionsPdfOcrString) +} +func (u *BatchSubmitParamsInputCrawlDataHTMLOptionsPdfOcrUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +type BatchSubmitParamsInputCrawlDataHTMLOptionsPdfOcrString string + +const ( + BatchSubmitParamsInputCrawlDataHTMLOptionsPdfOcrStringTrue BatchSubmitParamsInputCrawlDataHTMLOptionsPdfOcrString = "true" + BatchSubmitParamsInputCrawlDataHTMLOptionsPdfOcrStringFalse BatchSubmitParamsInputCrawlDataHTMLOptionsPdfOcrString = "false" +) + +// Only one field can be non-zero. +// +// Use [param.IsOmitted] to confirm if a field is set. +type BatchSubmitParamsInputCrawlDataHTMLOptionsPdfShouldParseUnion struct { + OfBool param.Opt[bool] `json:",omitzero,inline"` + // Check if union is this variant with + // !param.IsOmitted(union.OfBatchSubmitsInputCrawlDataHTMLOptionsPdfShouldParseString) + OfBatchSubmitsInputCrawlDataHTMLOptionsPdfShouldParseString param.Opt[string] `json:",omitzero,inline"` + paramUnion +} + +func (u BatchSubmitParamsInputCrawlDataHTMLOptionsPdfShouldParseUnion) MarshalJSON() ([]byte, error) { + return param.MarshalUnion(u, u.OfBool, u.OfBatchSubmitsInputCrawlDataHTMLOptionsPdfShouldParseString) +} +func (u *BatchSubmitParamsInputCrawlDataHTMLOptionsPdfShouldParseUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, u) +} + +type BatchSubmitParamsInputCrawlDataHTMLOptionsPdfShouldParseString string + +const ( + BatchSubmitParamsInputCrawlDataHTMLOptionsPdfShouldParseStringTrue BatchSubmitParamsInputCrawlDataHTMLOptionsPdfShouldParseString = "true" + BatchSubmitParamsInputCrawlDataHTMLOptionsPdfShouldParseStringFalse BatchSubmitParamsInputCrawlDataHTMLOptionsPdfShouldParseString = "false" +) diff --git a/batch_test.go b/batch_test.go index cdb86d8..2011fab 100644 --- a/batch_test.go +++ b/batch_test.go @@ -66,6 +66,29 @@ func TestBatchListWithOptionalParams(t *testing.T) { } } +func TestBatchDelete(t *testing.T) { + t.Skip("Mock server tests are disabled") + baseURL := "http://localhost:4010" + if envURL, ok := os.LookupEnv("TEST_API_BASE_URL"); ok { + baseURL = envURL + } + if !testutil.CheckTestServer(t, baseURL) { + return + } + client := contextdev.NewClient( + option.WithBaseURL(baseURL), + option.WithAPIKey("My API Key"), + ) + _, err := client.Batch.Delete(context.TODO(), "batch_9f2c8a") + if err != nil { + var apierr *contextdev.Error + if errors.As(err, &apierr) { + t.Log(string(apierr.DumpRequest(true))) + } + t.Fatalf("err should be nil: %s", err.Error()) + } +} + func TestBatchCancel(t *testing.T) { t.Skip("Mock server tests are disabled") baseURL := "http://localhost:4010" @@ -133,11 +156,52 @@ func TestBatchSubmitWithOptionalParams(t *testing.T) { option.WithAPIKey("My API Key"), ) _, err := client.Batch.Submit(context.TODO(), contextdev.BatchSubmitParams{ - Identifiers: contextdev.BatchSubmitParamsIdentifiers{ - LinkedinURL: contextdev.String("https://www.linkedin.com/in/yahia-bakour/"), + Input: contextdev.BatchSubmitParamsInputUnion{ + OfScrape: &contextdev.BatchSubmitParamsInputScrape{ + Data: contextdev.BatchSubmitParamsInputScrapeDataUnion{ + OfMarkdown: &contextdev.BatchSubmitParamsInputScrapeDataMarkdown{ + URLs: []contextdev.BatchSubmitParamsInputScrapeDataMarkdownURL{{ + URL: "https://example.com/products/anvil", + ItemID: contextdev.String("sku-1"), + Meta: map[string]any{ + "category": "bar", + }, + }, { + URL: "https://example.com/products/hammer", + ItemID: contextdev.String("sku-2"), + Meta: map[string]any{ + "foo": "bar", + }, + }}, + Options: contextdev.BatchSubmitParamsInputScrapeDataMarkdownOptions{ + Country: "de", + ExcludeSelectors: []string{"x"}, + IncludeImages: contextdev.Bool(true), + IncludeLinks: contextdev.Bool(true), + IncludeSelectors: []string{"x"}, + MaxAgeMs: contextdev.Int(0), + Pdf: contextdev.BatchSubmitParamsInputScrapeDataMarkdownOptionsPdf{ + End: contextdev.Int(1), + Ocr: contextdev.BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfOcrUnion{ + OfBatchSubmitsInputScrapeDataMarkdownOptionsPdfOcrString: contextdev.String("true"), + }, + ShouldParse: contextdev.BatchSubmitParamsInputScrapeDataMarkdownOptionsPdfShouldParseUnion{ + OfBatchSubmitsInputScrapeDataMarkdownOptionsPdfShouldParseString: contextdev.String("true"), + }, + Start: contextdev.Int(1), + }, + SettleAnimations: contextdev.Bool(true), + ShortenBase64Images: contextdev.Bool(true), + UseMainContentOnly: contextdev.Bool(true), + WaitForMs: contextdev.Int(0), + }, + }, + }, + }, }, - Tags: []string{"production", "team-alpha"}, - TimeoutMs: contextdev.Int(1000), + Tags: []string{"docs", "competitor"}, + WebhookURL: contextdev.String("webhookUrl"), + IdempotencyKey: contextdev.String("Idempotency-Key"), }) if err != nil { var apierr *contextdev.Error diff --git a/client.go b/client.go index b445caf..4561ee9 100644 --- a/client.go +++ b/client.go @@ -29,7 +29,9 @@ type Client struct { // MonitorsChangeDetectedWebhookPayload and MonitorsRunCompletedWebhookPayload // schemas. Monitors MonitorService - Batch BatchService + // Scrape many pages or crawl a site asynchronously. + Batch BatchService + People PersonService } // DefaultClientOptions read from the environment (CONTEXT_DEV_API_KEY, @@ -70,6 +72,7 @@ func NewClient(opts ...option.RequestOption) (r Client) { r.Utility = NewUtilityService(opts...) r.Monitors = NewMonitorService(opts...) r.Batch = NewBatchService(opts...) + r.People = NewPersonService(opts...) return } diff --git a/person.go b/person.go new file mode 100644 index 0000000..c750f6a --- /dev/null +++ b/person.go @@ -0,0 +1,727 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +package contextdev + +import ( + "context" + "encoding/json" + "net/http" + "slices" + + "github.com/context-dot-dev/context-go-sdk/v2/internal/apijson" + "github.com/context-dot-dev/context-go-sdk/v2/internal/requestconfig" + "github.com/context-dot-dev/context-go-sdk/v2/option" + "github.com/context-dot-dev/context-go-sdk/v2/packages/param" + "github.com/context-dot-dev/context-go-sdk/v2/packages/respjson" + "github.com/context-dot-dev/context-go-sdk/v2/shared/constant" +) + +// PersonService contains methods and other services that help with interacting +// with the context.dev API. +// +// Note, unlike clients, this service does not read variables from the environment +// automatically. You should not instantiate this service directly, and instead use +// the [NewPersonService] method instead. +type PersonService struct { + options []option.RequestOption +} + +// NewPersonService generates a new service that applies the given options to each +// request. These options are applied after the parent client's options (if there +// is one), and before any request-specific options. +func NewPersonService(opts ...option.RequestOption) (r PersonService) { + r = PersonService{} + r.options = opts + return +} + +// Finds and normalizes the best available person candidate from additive identity +// clues, then assigns an identity match score from 0 to 100. Available on Pro and +// Scale plans. Successful requests cost 20 credits. Disposable and free email +// addresses (like gmail.com, yahoo.com) will throw a 422 error. +func (r *PersonService) Enrich(ctx context.Context, body PersonEnrichParams, opts ...option.RequestOption) (res *PersonEnrichResponse, err error) { + opts = slices.Concat(r.options, opts) + path := "people/enrich" + err = requestconfig.ExecuteNewRequest(ctx, http.MethodPost, path, body, &res, opts...) + return res, err +} + +type PersonEnrichResponse struct { + // The highest-scoring person candidate. + Match PersonEnrichResponseMatchUnion `json:"match" api:"required"` + // Metadata about the API key used for the request. Included in every response + // whenever a valid API key is provided, even when the response status is not 200. + KeyMetadata PersonEnrichResponseKeyMetadata `json:"key_metadata"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Match respjson.Field + KeyMetadata respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponse) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponse) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// PersonEnrichResponseMatchUnion contains all possible properties and values from +// [PersonEnrichResponseMatchCandidate], [PersonEnrichResponseMatchNotFound]. +// +// Use the [PersonEnrichResponseMatchUnion.AsAny] method to switch on the variant. +// +// Use the methods beginning with 'As' to cast the union to one of its variants. +type PersonEnrichResponseMatchUnion struct { + // This field is a union of [PersonEnrichResponseMatchCandidatePerson], [any] + Person PersonEnrichResponseMatchUnionPerson `json:"person"` + // This field is a union of [int64], [any] + Score PersonEnrichResponseMatchUnionScore `json:"score"` + // Any of "candidate", "not_found". + Status string `json:"status"` + JSON struct { + Person respjson.Field + Score respjson.Field + Status respjson.Field + raw string + } `json:"-"` +} + +// anyPersonEnrichResponseMatch is implemented by each variant of +// [PersonEnrichResponseMatchUnion] to add type safety for the return type of +// [PersonEnrichResponseMatchUnion.AsAny] +type anyPersonEnrichResponseMatch interface { + implPersonEnrichResponseMatchUnion() +} + +func (PersonEnrichResponseMatchCandidate) implPersonEnrichResponseMatchUnion() {} +func (PersonEnrichResponseMatchNotFound) implPersonEnrichResponseMatchUnion() {} + +// Use the following switch statement to find the correct variant +// +// switch variant := PersonEnrichResponseMatchUnion.AsAny().(type) { +// case contextdev.PersonEnrichResponseMatchCandidate: +// case contextdev.PersonEnrichResponseMatchNotFound: +// default: +// fmt.Errorf("no variant present") +// } +func (u PersonEnrichResponseMatchUnion) AsAny() anyPersonEnrichResponseMatch { + switch u.Status { + case "candidate": + return u.AsCandidate() + case "not_found": + return u.AsNotFound() + } + return nil +} + +func (u PersonEnrichResponseMatchUnion) AsCandidate() (v PersonEnrichResponseMatchCandidate) { + apijson.UnmarshalRoot(json.RawMessage(u.JSON.raw), &v) + return +} + +func (u PersonEnrichResponseMatchUnion) AsNotFound() (v PersonEnrichResponseMatchNotFound) { + apijson.UnmarshalRoot(json.RawMessage(u.JSON.raw), &v) + return +} + +// Returns the unmodified JSON received from the API +func (u PersonEnrichResponseMatchUnion) RawJSON() string { return u.JSON.raw } + +func (r *PersonEnrichResponseMatchUnion) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// PersonEnrichResponseMatchUnionPerson is an implicit subunion of +// [PersonEnrichResponseMatchUnion]. PersonEnrichResponseMatchUnionPerson provides +// convenient access to the sub-properties of the union. +// +// For type safety it is recommended to directly use a variant of the +// [PersonEnrichResponseMatchUnion]. +// +// If the underlying value is not a json object, one of the following properties +// will be valid: OfPersonEnrichResponseMatchNotFoundPerson] +type PersonEnrichResponseMatchUnionPerson struct { + // This field will be present if the value is a [any] instead of an object. + OfPersonEnrichResponseMatchNotFoundPerson any `json:",inline"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + Education []PersonEnrichResponseMatchCandidatePersonEducation `json:"education"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + Experience []PersonEnrichResponseMatchCandidatePersonExperience `json:"experience"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + Skills []string `json:"skills"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + SocialURLs []string `json:"social_urls"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + WebsiteURLs []string `json:"website_urls"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + AvatarURL string `json:"avatar_url"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + Bio string `json:"bio"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + CurrentRole PersonEnrichResponseMatchCandidatePersonCurrentRole `json:"current_role"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + Email string `json:"email"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + Location PersonEnrichResponseMatchCandidatePersonLocation `json:"location"` + // This field is from variant [PersonEnrichResponseMatchCandidatePerson]. + Name PersonEnrichResponseMatchCandidatePersonName `json:"name"` + JSON struct { + OfPersonEnrichResponseMatchNotFoundPerson respjson.Field + Education respjson.Field + Experience respjson.Field + Skills respjson.Field + SocialURLs respjson.Field + WebsiteURLs respjson.Field + AvatarURL respjson.Field + Bio respjson.Field + CurrentRole respjson.Field + Email respjson.Field + Location respjson.Field + Name respjson.Field + raw string + } `json:"-"` +} + +func (r *PersonEnrichResponseMatchUnionPerson) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// PersonEnrichResponseMatchUnionScore is an implicit subunion of +// [PersonEnrichResponseMatchUnion]. PersonEnrichResponseMatchUnionScore provides +// convenient access to the sub-properties of the union. +// +// For type safety it is recommended to directly use a variant of the +// [PersonEnrichResponseMatchUnion]. +// +// If the underlying value is not a json object, one of the following properties +// will be valid: OfInt OfPersonEnrichResponseMatchNotFoundScore] +type PersonEnrichResponseMatchUnionScore struct { + // This field will be present if the value is a [int64] instead of an object. + OfInt int64 `json:",inline"` + // This field will be present if the value is a [any] instead of an object. + OfPersonEnrichResponseMatchNotFoundScore any `json:",inline"` + JSON struct { + OfInt respjson.Field + OfPersonEnrichResponseMatchNotFoundScore respjson.Field + raw string + } `json:"-"` +} + +func (r *PersonEnrichResponseMatchUnionScore) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// The highest-scoring person candidate. +type PersonEnrichResponseMatchCandidate struct { + Person PersonEnrichResponseMatchCandidatePerson `json:"person" api:"required"` + Score int64 `json:"score" api:"required"` + Status constant.Candidate `json:"status" default:"candidate"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Person respjson.Field + Score respjson.Field + Status respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidate) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponseMatchCandidate) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePerson struct { + Education []PersonEnrichResponseMatchCandidatePersonEducation `json:"education" api:"required"` + Experience []PersonEnrichResponseMatchCandidatePersonExperience `json:"experience" api:"required"` + Skills []string `json:"skills" api:"required"` + SocialURLs []string `json:"social_urls" api:"required" format:"uri"` + WebsiteURLs []string `json:"website_urls" api:"required" format:"uri"` + AvatarURL string `json:"avatar_url"` + Bio string `json:"bio"` + CurrentRole PersonEnrichResponseMatchCandidatePersonCurrentRole `json:"current_role"` + Email string `json:"email" format:"email"` + Location PersonEnrichResponseMatchCandidatePersonLocation `json:"location"` + Name PersonEnrichResponseMatchCandidatePersonName `json:"name"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Education respjson.Field + Experience respjson.Field + Skills respjson.Field + SocialURLs respjson.Field + WebsiteURLs respjson.Field + AvatarURL respjson.Field + Bio respjson.Field + CurrentRole respjson.Field + Email respjson.Field + Location respjson.Field + Name respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePerson) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponseMatchCandidatePerson) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonEducation struct { + Institution PersonEnrichResponseMatchCandidatePersonEducationInstitution `json:"institution" api:"required"` + Degree string `json:"degree"` + Description string `json:"description"` + EndDate PersonEnrichResponseMatchCandidatePersonEducationEndDate `json:"end_date"` + FieldOfStudy string `json:"field_of_study"` + StartDate PersonEnrichResponseMatchCandidatePersonEducationStartDate `json:"start_date"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Institution respjson.Field + Degree respjson.Field + Description respjson.Field + EndDate respjson.Field + FieldOfStudy respjson.Field + StartDate respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonEducation) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponseMatchCandidatePersonEducation) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonEducationInstitution struct { + Name string `json:"name" api:"required"` + Domain string `json:"domain"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Name respjson.Field + Domain respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonEducationInstitution) RawJSON() string { + return r.JSON.raw +} +func (r *PersonEnrichResponseMatchCandidatePersonEducationInstitution) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonEducationEndDate struct { + Year int64 `json:"year" api:"required"` + Day int64 `json:"day"` + Month int64 `json:"month"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Year respjson.Field + Day respjson.Field + Month respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonEducationEndDate) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponseMatchCandidatePersonEducationEndDate) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonEducationStartDate struct { + Year int64 `json:"year" api:"required"` + Day int64 `json:"day"` + Month int64 `json:"month"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Year respjson.Field + Day respjson.Field + Month respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonEducationStartDate) RawJSON() string { + return r.JSON.raw +} +func (r *PersonEnrichResponseMatchCandidatePersonEducationStartDate) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonExperience struct { + Organization PersonEnrichResponseMatchCandidatePersonExperienceOrganization `json:"organization" api:"required"` + Title string `json:"title" api:"required"` + Description string `json:"description"` + EndDate PersonEnrichResponseMatchCandidatePersonExperienceEndDate `json:"end_date"` + IsCurrent bool `json:"is_current"` + Location string `json:"location"` + StartDate PersonEnrichResponseMatchCandidatePersonExperienceStartDate `json:"start_date"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Organization respjson.Field + Title respjson.Field + Description respjson.Field + EndDate respjson.Field + IsCurrent respjson.Field + Location respjson.Field + StartDate respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonExperience) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponseMatchCandidatePersonExperience) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonExperienceOrganization struct { + Name string `json:"name" api:"required"` + Domain string `json:"domain"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Name respjson.Field + Domain respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonExperienceOrganization) RawJSON() string { + return r.JSON.raw +} +func (r *PersonEnrichResponseMatchCandidatePersonExperienceOrganization) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonExperienceEndDate struct { + Year int64 `json:"year" api:"required"` + Day int64 `json:"day"` + Month int64 `json:"month"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Year respjson.Field + Day respjson.Field + Month respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonExperienceEndDate) RawJSON() string { + return r.JSON.raw +} +func (r *PersonEnrichResponseMatchCandidatePersonExperienceEndDate) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonExperienceStartDate struct { + Year int64 `json:"year" api:"required"` + Day int64 `json:"day"` + Month int64 `json:"month"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Year respjson.Field + Day respjson.Field + Month respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonExperienceStartDate) RawJSON() string { + return r.JSON.raw +} +func (r *PersonEnrichResponseMatchCandidatePersonExperienceStartDate) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonCurrentRole struct { + Organization PersonEnrichResponseMatchCandidatePersonCurrentRoleOrganization `json:"organization" api:"required"` + Title string `json:"title" api:"required"` + Description string `json:"description"` + EndDate PersonEnrichResponseMatchCandidatePersonCurrentRoleEndDate `json:"end_date"` + IsCurrent bool `json:"is_current"` + Location string `json:"location"` + StartDate PersonEnrichResponseMatchCandidatePersonCurrentRoleStartDate `json:"start_date"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Organization respjson.Field + Title respjson.Field + Description respjson.Field + EndDate respjson.Field + IsCurrent respjson.Field + Location respjson.Field + StartDate respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonCurrentRole) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponseMatchCandidatePersonCurrentRole) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonCurrentRoleOrganization struct { + Name string `json:"name" api:"required"` + Domain string `json:"domain"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Name respjson.Field + Domain respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonCurrentRoleOrganization) RawJSON() string { + return r.JSON.raw +} +func (r *PersonEnrichResponseMatchCandidatePersonCurrentRoleOrganization) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonCurrentRoleEndDate struct { + Year int64 `json:"year" api:"required"` + Day int64 `json:"day"` + Month int64 `json:"month"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Year respjson.Field + Day respjson.Field + Month respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonCurrentRoleEndDate) RawJSON() string { + return r.JSON.raw +} +func (r *PersonEnrichResponseMatchCandidatePersonCurrentRoleEndDate) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonCurrentRoleStartDate struct { + Year int64 `json:"year" api:"required"` + Day int64 `json:"day"` + Month int64 `json:"month"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Year respjson.Field + Day respjson.Field + Month respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonCurrentRoleStartDate) RawJSON() string { + return r.JSON.raw +} +func (r *PersonEnrichResponseMatchCandidatePersonCurrentRoleStartDate) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonLocation struct { + City string `json:"city"` + Country string `json:"country"` + CountryCode string `json:"country_code"` + Display string `json:"display"` + Region string `json:"region"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + City respjson.Field + Country respjson.Field + CountryCode respjson.Field + Display respjson.Field + Region respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonLocation) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponseMatchCandidatePersonLocation) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichResponseMatchCandidatePersonName struct { + First string `json:"first"` + Full string `json:"full"` + Last string `json:"last"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + First respjson.Field + Full respjson.Field + Last respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchCandidatePersonName) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponseMatchCandidatePersonName) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// No usable person candidate was found. +type PersonEnrichResponseMatchNotFound struct { + Person any `json:"person" api:"required"` + Score any `json:"score" api:"required"` + Status constant.NotFound `json:"status" default:"not_found"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + Person respjson.Field + Score respjson.Field + Status respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseMatchNotFound) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponseMatchNotFound) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +// Metadata about the API key used for the request. Included in every response +// whenever a valid API key is provided, even when the response status is not 200. +type PersonEnrichResponseKeyMetadata struct { + // The number of credits consumed by this request. + CreditsConsumed int64 `json:"credits_consumed" api:"required"` + // The number of credits remaining for your organization after this request. + CreditsRemaining int64 `json:"credits_remaining" api:"required"` + // JSON contains metadata for fields, check presence with [respjson.Field.Valid]. + JSON struct { + CreditsConsumed respjson.Field + CreditsRemaining respjson.Field + ExtraFields map[string]respjson.Field + raw string + } `json:"-"` +} + +// Returns the unmodified JSON received from the API +func (r PersonEnrichResponseKeyMetadata) RawJSON() string { return r.JSON.raw } +func (r *PersonEnrichResponseKeyMetadata) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichParams struct { + Email param.Opt[string] `json:"email,omitzero" format:"email"` + // Optional timeout in milliseconds for the request. If the request takes longer + // than this value, it will be aborted with a 408 status code. Maximum allowed + // value is 300000ms (5 minutes). + TimeoutMs param.Opt[int64] `json:"timeoutMS,omitzero"` + Company PersonEnrichParamsCompany `json:"company,omitzero"` + Education []PersonEnrichParamsEducation `json:"education,omitzero"` + Location PersonEnrichParamsLocation `json:"location,omitzero"` + Name PersonEnrichParamsName `json:"name,omitzero"` + SocialURLs []string `json:"social_urls,omitzero" format:"uri"` + // Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters. + Tags []string `json:"tags,omitzero"` + paramObj +} + +func (r PersonEnrichParams) MarshalJSON() (data []byte, err error) { + type shadow PersonEnrichParams + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *PersonEnrichParams) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichParamsCompany struct { + Domain param.Opt[string] `json:"domain,omitzero"` + Name param.Opt[string] `json:"name,omitzero"` + paramObj +} + +func (r PersonEnrichParamsCompany) MarshalJSON() (data []byte, err error) { + type shadow PersonEnrichParamsCompany + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *PersonEnrichParamsCompany) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichParamsEducation struct { + Degree param.Opt[string] `json:"degree,omitzero"` + FieldOfStudy param.Opt[string] `json:"field_of_study,omitzero"` + GraduationYear param.Opt[int64] `json:"graduation_year,omitzero"` + Institution PersonEnrichParamsEducationInstitution `json:"institution,omitzero"` + paramObj +} + +func (r PersonEnrichParamsEducation) MarshalJSON() (data []byte, err error) { + type shadow PersonEnrichParamsEducation + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *PersonEnrichParamsEducation) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichParamsEducationInstitution struct { + Domain param.Opt[string] `json:"domain,omitzero"` + Name param.Opt[string] `json:"name,omitzero"` + paramObj +} + +func (r PersonEnrichParamsEducationInstitution) MarshalJSON() (data []byte, err error) { + type shadow PersonEnrichParamsEducationInstitution + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *PersonEnrichParamsEducationInstitution) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichParamsLocation struct { + City param.Opt[string] `json:"city,omitzero"` + Country param.Opt[string] `json:"country,omitzero"` + Region param.Opt[string] `json:"region,omitzero"` + paramObj +} + +func (r PersonEnrichParamsLocation) MarshalJSON() (data []byte, err error) { + type shadow PersonEnrichParamsLocation + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *PersonEnrichParamsLocation) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} + +type PersonEnrichParamsName struct { + First param.Opt[string] `json:"first,omitzero"` + Last param.Opt[string] `json:"last,omitzero"` + paramObj +} + +func (r PersonEnrichParamsName) MarshalJSON() (data []byte, err error) { + type shadow PersonEnrichParamsName + return param.MarshalObject(r, (*shadow)(&r)) +} +func (r *PersonEnrichParamsName) UnmarshalJSON(data []byte) error { + return apijson.UnmarshalRoot(data, r) +} diff --git a/person_test.go b/person_test.go new file mode 100644 index 0000000..1589209 --- /dev/null +++ b/person_test.go @@ -0,0 +1,64 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +package contextdev_test + +import ( + "context" + "errors" + "os" + "testing" + + "github.com/context-dot-dev/context-go-sdk/v2" + "github.com/context-dot-dev/context-go-sdk/v2/internal/testutil" + "github.com/context-dot-dev/context-go-sdk/v2/option" +) + +func TestPersonEnrichWithOptionalParams(t *testing.T) { + t.Skip("Mock server tests are disabled") + baseURL := "http://localhost:4010" + if envURL, ok := os.LookupEnv("TEST_API_BASE_URL"); ok { + baseURL = envURL + } + if !testutil.CheckTestServer(t, baseURL) { + return + } + client := contextdev.NewClient( + option.WithBaseURL(baseURL), + option.WithAPIKey("My API Key"), + ) + _, err := client.People.Enrich(context.TODO(), contextdev.PersonEnrichParams{ + Company: contextdev.PersonEnrichParamsCompany{ + Domain: contextdev.String("analyticalengines.example"), + Name: contextdev.String("Analytical Engines"), + }, + Education: []contextdev.PersonEnrichParamsEducation{{ + Degree: contextdev.String("x"), + FieldOfStudy: contextdev.String("x"), + GraduationYear: contextdev.Int(1900), + Institution: contextdev.PersonEnrichParamsEducationInstitution{ + Domain: contextdev.String("x"), + Name: contextdev.String("x"), + }, + }}, + Email: contextdev.String("dev@stainless.com"), + Location: contextdev.PersonEnrichParamsLocation{ + City: contextdev.String("x"), + Country: contextdev.String("x"), + Region: contextdev.String("x"), + }, + Name: contextdev.PersonEnrichParamsName{ + First: contextdev.String("Ada"), + Last: contextdev.String("Lovelace"), + }, + SocialURLs: []string{"https://www.linkedin.com/in/ada-lovelace/"}, + Tags: []string{"production", "team-alpha"}, + TimeoutMs: contextdev.Int(1000), + }) + if err != nil { + var apierr *contextdev.Error + if errors.As(err, &apierr) { + t.Log(string(apierr.DumpRequest(true))) + } + t.Fatalf("err should be nil: %s", err.Error()) + } +} diff --git a/shared/constant/constants.go b/shared/constant/constants.go index 816f4a0..925106e 100644 --- a/shared/constant/constants.go +++ b/shared/constant/constants.go @@ -24,14 +24,21 @@ type ByEmail string // Always "by_email" type ByName string // Always "by_name" type ByTicker string // Always "by_ticker" type ByTransaction string // Always "by_transaction" +type Candidate string // Always "candidate" +type Crawl string // Always "crawl" type Error string // Always "error" type Exact string // Always "exact" type Extract string // Always "extract" +type HTML string // Always "html" +type Markdown string // Always "markdown" +type NotFound string // Always "not_found" type Ok string // Always "ok" type Page string // Always "page" type Perform string // Always "perform" +type Scrape string // Always "scrape" type Semantic string // Always "semantic" type Sitemap string // Always "sitemap" +type StartURL string // Always "start_url" type Wait string // Always "wait" func (c ByDirectURL) Default() ByDirectURL { return "by_direct_url" } @@ -40,14 +47,21 @@ func (c ByEmail) Default() ByEmail { return "by_email" } func (c ByName) Default() ByName { return "by_name" } func (c ByTicker) Default() ByTicker { return "by_ticker" } func (c ByTransaction) Default() ByTransaction { return "by_transaction" } +func (c Candidate) Default() Candidate { return "candidate" } +func (c Crawl) Default() Crawl { return "crawl" } func (c Error) Default() Error { return "error" } func (c Exact) Default() Exact { return "exact" } func (c Extract) Default() Extract { return "extract" } +func (c HTML) Default() HTML { return "html" } +func (c Markdown) Default() Markdown { return "markdown" } +func (c NotFound) Default() NotFound { return "not_found" } func (c Ok) Default() Ok { return "ok" } func (c Page) Default() Page { return "page" } func (c Perform) Default() Perform { return "perform" } +func (c Scrape) Default() Scrape { return "scrape" } func (c Semantic) Default() Semantic { return "semantic" } func (c Sitemap) Default() Sitemap { return "sitemap" } +func (c StartURL) Default() StartURL { return "start_url" } func (c Wait) Default() Wait { return "wait" } func (c ByDirectURL) MarshalJSON() ([]byte, error) { return marshalString(c) } @@ -56,14 +70,21 @@ func (c ByEmail) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c ByName) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c ByTicker) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c ByTransaction) MarshalJSON() ([]byte, error) { return marshalString(c) } +func (c Candidate) MarshalJSON() ([]byte, error) { return marshalString(c) } +func (c Crawl) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c Error) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c Exact) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c Extract) MarshalJSON() ([]byte, error) { return marshalString(c) } +func (c HTML) MarshalJSON() ([]byte, error) { return marshalString(c) } +func (c Markdown) MarshalJSON() ([]byte, error) { return marshalString(c) } +func (c NotFound) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c Ok) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c Page) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c Perform) MarshalJSON() ([]byte, error) { return marshalString(c) } +func (c Scrape) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c Semantic) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c Sitemap) MarshalJSON() ([]byte, error) { return marshalString(c) } +func (c StartURL) MarshalJSON() ([]byte, error) { return marshalString(c) } func (c Wait) MarshalJSON() ([]byte, error) { return marshalString(c) } type constant[T any] interface { From 4bcf678f82a1f49dc7061e6f9f5984013fc1ed99 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 01:02:58 +0000 Subject: [PATCH 09/10] feat(api): api update --- .stats.yml | 4 ++-- batch.go | 24 +++++++++++++----------- person.go | 6 +++--- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.stats.yml b/.stats.yml index ca57822..97db86d 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 40 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-e2ba5381de3010ee53f27388aa6727ad567eb18e4b0b4eb1c4c66402bfea64cc.yml -openapi_spec_hash: 141bfa8430ddccfb5a37c4bd703d163b +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-a0dda03bbb600917cfb9add468cc4c8c84351a8dbbf61644dbc353263ca1748f.yml +openapi_spec_hash: c24264f32a46d9317aac5af9d6a396f7 config_hash: 920678668dd2da6f8966fbf1b8fde4e2 diff --git a/batch.go b/batch.go index 5bda26e..c58dac2 100644 --- a/batch.go +++ b/batch.go @@ -189,14 +189,14 @@ func (r *CrawlControls) UnmarshalJSON(data []byte) error { } // CrawlControlsSourceUnion contains all possible properties and values from -// [CrawlControlsSourceObject], [CrawlControlsSourceObject2]. +// [CrawlControlsSourceStartURL], [CrawlControlsSourceSitemap]. // // Use the methods beginning with 'As' to cast the union to one of its variants. type CrawlControlsSourceUnion struct { Type string `json:"type"` - // This field is from variant [CrawlControlsSourceObject]. + // This field is from variant [CrawlControlsSourceStartURL]. URL string `json:"url"` - // This field is from variant [CrawlControlsSourceObject2]. + // This field is from variant [CrawlControlsSourceSitemap]. Domain string `json:"domain"` JSON struct { Type respjson.Field @@ -206,12 +206,12 @@ type CrawlControlsSourceUnion struct { } `json:"-"` } -func (u CrawlControlsSourceUnion) AsCrawlControlsSourceObject() (v CrawlControlsSourceObject) { +func (u CrawlControlsSourceUnion) AsStartURL() (v CrawlControlsSourceStartURL) { apijson.UnmarshalRoot(json.RawMessage(u.JSON.raw), &v) return } -func (u CrawlControlsSourceUnion) AsCrawlControlsSourceObject2() (v CrawlControlsSourceObject2) { +func (u CrawlControlsSourceUnion) AsSitemap() (v CrawlControlsSourceSitemap) { apijson.UnmarshalRoot(json.RawMessage(u.JSON.raw), &v) return } @@ -223,7 +223,8 @@ func (r *CrawlControlsSourceUnion) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -type CrawlControlsSourceObject struct { +// The crawl discovered pages by following links from one URL. +type CrawlControlsSourceStartURL struct { // Any of "start_url". Type string `json:"type" api:"required"` // Page the crawl started from. @@ -238,12 +239,13 @@ type CrawlControlsSourceObject struct { } // Returns the unmodified JSON received from the API -func (r CrawlControlsSourceObject) RawJSON() string { return r.JSON.raw } -func (r *CrawlControlsSourceObject) UnmarshalJSON(data []byte) error { +func (r CrawlControlsSourceStartURL) RawJSON() string { return r.JSON.raw } +func (r *CrawlControlsSourceStartURL) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } -type CrawlControlsSourceObject2 struct { +// The crawl scraped the pages listed in the domain's sitemap. +type CrawlControlsSourceSitemap struct { // Domain whose sitemap supplied the pages. Domain string `json:"domain" api:"required"` // Any of "sitemap". @@ -258,8 +260,8 @@ type CrawlControlsSourceObject2 struct { } // Returns the unmodified JSON received from the API -func (r CrawlControlsSourceObject2) RawJSON() string { return r.JSON.raw } -func (r *CrawlControlsSourceObject2) UnmarshalJSON(data []byte) error { +func (r CrawlControlsSourceSitemap) RawJSON() string { return r.JSON.raw } +func (r *CrawlControlsSourceSitemap) UnmarshalJSON(data []byte) error { return apijson.UnmarshalRoot(data, r) } diff --git a/person.go b/person.go index c750f6a..c9f4863 100644 --- a/person.go +++ b/person.go @@ -36,9 +36,9 @@ func NewPersonService(opts ...option.RequestOption) (r PersonService) { } // Finds and normalizes the best available person candidate from additive identity -// clues, then assigns an identity match score from 0 to 100. Available on Pro and -// Scale plans. Successful requests cost 20 credits. Disposable and free email -// addresses (like gmail.com, yahoo.com) will throw a 422 error. +// clues, then assigns an identity match score from 0 to 100. Available on all paid +// plans. Successful requests cost 20 credits. Disposable and free email addresses +// (like gmail.com, yahoo.com) will throw a 422 error. func (r *PersonService) Enrich(ctx context.Context, body PersonEnrichParams, opts ...option.RequestOption) (res *PersonEnrichResponse, err error) { opts = slices.Concat(r.options, opts) path := "people/enrich" From bb391ccd0721c566796867b604241cdc9dafe503 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 01:03:20 +0000 Subject: [PATCH 10/10] release: 2.9.0 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 15 +++++++++++++++ README.md | 2 +- internal/version.go | 2 +- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 64f9ff4..427b8ec 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.8.0" + ".": "2.9.0" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 643ebcb..3ac7ed8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 2.9.0 (2026-08-07) + +Full Changelog: [v2.8.0...v2.9.0](https://github.com/context-dot-dev/context-go-sdk/compare/v2.8.0...v2.9.0) + +### Features + +* **api:** api update ([4bcf678](https://github.com/context-dot-dev/context-go-sdk/commit/4bcf678f82a1f49dc7061e6f9f5984013fc1ed99)) +* **api:** api update ([e1a197f](https://github.com/context-dot-dev/context-go-sdk/commit/e1a197fe3667d68af27bd368470f743eee85ec01)) +* **api:** api update ([e4803c4](https://github.com/context-dot-dev/context-go-sdk/commit/e4803c4cf0aebf68aeb1653630e60e39caae68c1)) +* **api:** api update ([46f18c2](https://github.com/context-dot-dev/context-go-sdk/commit/46f18c27fb20790758beb40150fba47971a91a5a)) +* **api:** api update ([bd340b8](https://github.com/context-dot-dev/context-go-sdk/commit/bd340b8ad89d0530f3a8396d4ec3f380fc4137d5)) +* **api:** api update ([e71dbb9](https://github.com/context-dot-dev/context-go-sdk/commit/e71dbb9625e935c311e8ab0f57bb112791b3b1cf)) +* **api:** api update ([f15add2](https://github.com/context-dot-dev/context-go-sdk/commit/f15add299d50c81d179afdfdcda99eb9687aaa31)) +* **api:** manual updates ([3362570](https://github.com/context-dot-dev/context-go-sdk/commit/3362570a65bffd75adee223b860704feb4b90d6a)) + ## 2.8.0 (2026-08-05) Full Changelog: [v2.7.0...v2.8.0](https://github.com/context-dot-dev/context-go-sdk/compare/v2.7.0...v2.8.0) diff --git a/README.md b/README.md index 9532e29..ae38305 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Or to pin the version: ```sh -go get -u 'github.com/context-dot-dev/context-go-sdk@v2.8.0' +go get -u 'github.com/context-dot-dev/context-go-sdk@v2.9.0' ``` diff --git a/internal/version.go b/internal/version.go index ac38466..24ea833 100644 --- a/internal/version.go +++ b/internal/version.go @@ -2,4 +2,4 @@ package internal -const PackageVersion = "2.8.0" // x-release-please-version +const PackageVersion = "2.9.0" // x-release-please-version