anycrawler.js

class AnyCrawlerError extends Error {
  constructor({ status, errorCode, errorMessage, data, meta }) {
    super(errorMessage);
    this.name = "AnyCrawlerError";
    this.status = status;
    this.errorCode = errorCode;
    this.errorMessage = errorMessage;
    this.data = data;
    this.meta = meta;
  }
}

function normalizeBaseUrl(baseUrl) {
  return baseUrl.replace(/\/+$/, "");
}

function parseOptionalNumber(value) {
  if (value === null || value === undefined || value === "") {
    return null;
  }

  const parsed = Number(value);
  return Number.isFinite(parsed) ? parsed : null;
}

function buildMeta(response) {
  return {
    status: response.status,
    requestId: response.headers.get("x-request-id"),
    creditsReserved: parseOptionalNumber(response.headers.get("x-credits-reserved")),
    creditsUsed: parseOptionalNumber(response.headers.get("x-credits-used")),
    browserMsUsed: parseOptionalNumber(response.headers.get("x-browser-ms-used")),
  };
}

function parseJson(text) {
  if (!text) {
    return null;
  }

  return JSON.parse(text);
}

class AnyCrawlerClient {
  constructor(options) {
    if (!options || !options.apiKey) {
      throw new Error("AnyCrawlerClient requires an apiKey.");
    }

    this.apiKey = options.apiKey;
    this.baseUrl = normalizeBaseUrl(options.baseUrl || "https://api.anycrawler.com");
    this.defaultHeaders = options.headers || {};
    this.fetchImpl = options.fetch || fetch;
  }

  async getAccountUsage() {
    return this.request("/v1/account/usage", {
      method: "GET",
    });
  }

  async crawlPage(payload) {
    return this.request("/v1/crawl/page", {
      method: "POST",
      body: JSON.stringify(payload),
    });
  }

  async crawlScreenshot(payload) {
    return this.request("/v1/crawl/screenshot", {
      method: "POST",
      body: JSON.stringify(payload),
    });
  }

  async search(payload) {
    return this.request("/v1/search", {
      method: "POST",
      body: JSON.stringify(payload),
    });
  }

  async searchPage(payload) {
    return this.request("/v1/search/page", {
      method: "POST",
      body: JSON.stringify(payload),
    });
  }

  async searchNews(payload) {
    return this.request("/v1/search/news", {
      method: "POST",
      body: JSON.stringify(payload),
    });
  }

  async request(pathname, init) {
    const headers = new Headers(this.defaultHeaders);
    headers.set("accept", "application/json");
    headers.set("authorization", `Bearer ${this.apiKey}`);

    if (init.body !== undefined) {
      headers.set("content-type", "application/json");
    }

    const response = await this.fetchImpl(`${this.baseUrl}${pathname}`, {
      ...init,
      headers,
    });

    const meta = buildMeta(response);
    const text = await response.text();
    const data = parseJson(text);

    if (!response.ok) {
      const body = data && typeof data === "object" && !Array.isArray(data) ? data : null;
      const errorCode =
        body && typeof body.error_code === "string"
          ? body.error_code
          : body && typeof body.errorCode === "string"
            ? body.errorCode
            : null;
      const errorMessage =
        body && typeof body.error_message === "string"
          ? body.error_message
          : body && typeof body.message === "string"
            ? body.message
            : `AnyCrawler request failed with status ${response.status}.`;

      throw new AnyCrawlerError({
        status: response.status,
        errorCode,
        errorMessage,
        data,
        meta,
      });
    }

    return {
      data,
      meta,
    };
  }
}

module.exports = {
  AnyCrawlerClient,
  AnyCrawlerError,
};