import { Request, Response } from "express";
import { config } from "../../config";
import { z } from "zod";
import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
import { countries } from "../../lib/validate-country";
import { includesFormat } from "../../lib/format-utils";
import {
  ExtractorOptions,
  PageOptions,
  ScrapeActionContent,
  Document as V0Document,
  WebSearchResult,
} from "../../lib/entities";
import {
  agentOptionsExtract,
  AuthCreditUsageChunk,
  ScrapeOptions as V1ScrapeOptions,
} from "../v1/types";
import type { InternalOptions } from "../../scraper/scrapeURL";
import { ErrorCodes } from "../../lib/error";
import Ajv from "ajv";
import addFormats from "ajv-formats";
import { integrationSchema } from "../../utils/integration";
import {
  webhookSchema,
  createWebhookSchema,
} from "../../services/webhook/schema";
import { BrandingProfile } from "../../types/branding";

// Base URL schema with common validation logic
export const URL = z.preprocess(
  x => {
    if (!protocolIncluded(x as string)) {
      x = `http://${x}`;
    }

    // transforming the query parameters is breaking certain sites, so we're not doing it - mogery
    // try {
    //   const urlObj = new URL(x as string);
    //   if (urlObj.search) {
    //     const searchParams = new URLSearchParams(urlObj.search.substring(1));
    //     return `${urlObj.origin}${urlObj.pathname}?${searchParams.toString()}`;
    //   }
    // } catch (e) {
    // }

    return x;
  },
  z
    .url()
    .regex(/^https?:\/\//i, "URL uses unsupported protocol")
    .refine(x => {
      if (config.TEST_SUITE_SELF_HOSTED && config.ALLOW_LOCAL_WEBHOOKS) {
        if (
          /^https?:\/\/(localhost|127\.0\.0\.1|10\.\d{1,3}\.\d{1,3}\.\d{1,3}|192\.168\.\d{1,3}\.\d{1,3}|172\.(1[6-9]|2\d|3[0-1])\.\d{1,3}\.\d{1,3})(:\d+)?([\/?#]|$)/i.test(
            x as string,
          )
        ) {
          return true;
        }
      }
      return /(\.[a-zA-Z0-9-\u0400-\u04FF\u0500-\u052F\u2DE0-\u2DFF\uA640-\uA69F]{2,}|\.xn--[a-zA-Z0-9-]{1,})(:\d+)?([\/?#]|$)/i.test(
        x,
      );
    }, "URL must have a valid top-level domain or be a valid path")
    .refine(x => {
      try {
        checkUrl(x as string);
        return true;
      } catch (_) {
        return false;
      }
    }, "Invalid URL"),
  // .refine((x) => !isUrlBlocked(x as string), UNSUPPORTED_SITE_MESSAGE),
);

const strictMessage =
  "Unrecognized key in body -- please review the v2 API documentation for request body changes";

// Helper function to add strict validation
// In zod v4, .strict() doesn't accept arguments
// The custom error message is handled in the error handler (see src/index.ts)
// We use a type assertion to preserve the input type so optional fields with defaults remain optional
// The 'as any' is necessary because zod v4's .strict() changes type inference in a way that makes
// optional fields with defaults appear required, even though they're not at runtime
function strictWithMessage<T extends z.ZodObject<any>>(schema: T): T {
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  return schema.strict() as any as T;
}

function normalizeSchemaForOpenAI(schema: any): any {
  if (!schema || typeof schema !== "object") {
    return schema;
  }

  const visited = new WeakSet();

  function normalizeObject(obj: any): any {
    if (typeof obj !== "object" || obj === null) return obj;
    if (Array.isArray(obj)) {
      return obj.map(item => normalizeObject(item));
    }

    if (visited.has(obj)) return obj;
    visited.add(obj);

    const normalized = { ...obj };

    // Handle $ref recursion - preserve as-is for OpenAI compatibility
    if (normalized.hasOwnProperty("$ref")) {
      return normalized;
    }

    if (normalized.hasOwnProperty("$defs")) {
      const { $defs, ...rest } = normalized;
      const processedRest = {};

      for (const [key, value] of Object.entries(rest)) {
        if (
          typeof value === "object" &&
          value !== null &&
          !value.hasOwnProperty("$ref")
        ) {
          processedRest[key] = normalizeObject(value);
        } else {
          processedRest[key] = value;
        }
      }

      const normalizedDefs = Object.fromEntries(
        Object.entries($defs ?? {}).map(([key, value]) => [
          key,
          normalizeObject(value),
        ]),
      );

      return { ...processedRest, $defs: normalizedDefs };
    }

    if (
      normalized.type === "object" &&
      normalized.hasOwnProperty("properties") &&
      normalized.hasOwnProperty("additionalProperties")
    ) {
      delete normalized.additionalProperties;
    }

    if (
      normalized.type === "object" &&
      normalized.hasOwnProperty("required") &&
      normalized.hasOwnProperty("properties")
    ) {
      if (
        Array.isArray(normalized.required) &&
        typeof normalized.properties === "object" &&
        normalized.properties !== null
      ) {
        const validRequired = normalized.required.filter((field: string) =>
          normalized.properties.hasOwnProperty(field),
        );
        if (validRequired.length > 0) {
          normalized.required = validRequired;
        } else {
          delete normalized.required;
        }
      } else {
        delete normalized.required;
      }
    }

    for (const [key, value] of Object.entries(normalized)) {
      if (
        typeof value === "object" &&
        value !== null &&
        !value.hasOwnProperty("$ref")
      ) {
        normalized[key] = normalizeObject(value);
      }
    }

    return normalized;
  }

  return normalizeObject(schema);
}

function validateSchemaForOpenAI(schema: any): boolean {
  if (!schema || typeof schema !== "object") {
    return true;
  }

  const visited = new WeakSet();

  function hasInvalidStructure(obj: any): boolean {
    if (typeof obj !== "object" || obj === null) return false;

    if (visited.has(obj)) return false;
    visited.add(obj);

    if (obj.hasOwnProperty("$ref")) {
      return false;
    }

    if (
      obj.type === "object" &&
      !obj.hasOwnProperty("properties") &&
      !obj.hasOwnProperty("patternProperties") &&
      obj.additionalProperties === true
    ) {
      return true;
    }

    for (const value of Object.values(obj)) {
      if (
        typeof value === "object" &&
        value !== null &&
        !value.hasOwnProperty("$ref")
      ) {
        if (hasInvalidStructure(value)) return true;
      }
    }
    return false;
  }

  return !hasInvalidStructure(schema);
}

const OPENAI_SCHEMA_ERROR_MESSAGE =
  "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object. Note: Recursive schemas using '$ref' are supported.";

const ACTIONS_MAX_WAIT_TIME = 60;
const MAX_ACTIONS = 50;
function calculateTotalWaitTime(
  actions: any[] = [],
  waitFor: number = 0,
): number {
  const actionWaitTime = actions.reduce((acc, action) => {
    if (action.type === "wait") {
      if (action.milliseconds) {
        return acc + action.milliseconds;
      }
      // Consider selector actions as 1 second
      if (action.selector) {
        return acc + 1000;
      }
    }
    return acc;
  }, 0);

  return waitFor + actionWaitTime;
}

const actionSchema = z.union([
  z
    .object({
      type: z.literal("wait"),
      milliseconds: z.int().positive().finite().optional(),
      selector: z.string().optional(),
    })
    .refine(
      data =>
        (data.milliseconds !== undefined || data.selector !== undefined) &&
        !(data.milliseconds !== undefined && data.selector !== undefined),
      {
        error:
          "Either 'milliseconds' or 'selector' must be provided, but not both.",
      },
    ),
  z.object({
    type: z.literal("click"),
    selector: z.string(),
    all: z.boolean().prefault(false),
  }),
  z.object({
    type: z.literal("screenshot"),
    fullPage: z.boolean().prefault(false),
    quality: z.number().min(1).max(100).optional(),
    viewport: z
      .object({
        width: z.int().positive().finite().max(7680), // 8K resolution width
        height: z.int().positive().finite().max(4320), // 8K resolution height
      })
      .optional(),
  }),
  z.object({
    type: z.literal("write"),
    text: z.string(),
  }),
  z.object({
    type: z.literal("press"),
    key: z.string(),
  }),
  z.object({
    type: z.literal("scroll"),
    direction: z.enum(["up", "down"]).optional().prefault("down"),
    selector: z.string().optional(),
  }),
  z.object({
    type: z.literal("scrape"),
  }),
  z.object({
    type: z.literal("executeJavascript"),
    script: z.string(),
  }),
  z.object({
    type: z.literal("pdf"),
    landscape: z.boolean().prefault(false),
    scale: z.number().prefault(1),
    format: z
      .enum([
        "A0",
        "A1",
        "A2",
        "A3",
        "A4",
        "A5",
        "A6",
        "Letter",
        "Legal",
        "Tabloid",
        "Ledger",
      ])
      .prefault("Letter"),
  }),
]);

const actionsSchema = z
  .array(actionSchema)
  .refine(actions => actions.length <= MAX_ACTIONS, {
    message: `Number of actions cannot exceed ${MAX_ACTIONS}`,
  })
  .refine(
    actions => calculateTotalWaitTime(actions) <= ACTIONS_MAX_WAIT_TIME * 1000,
    {
      message: `Total wait time (waitFor + wait actions) cannot exceed ${ACTIONS_MAX_WAIT_TIME} seconds`,
    },
  );

const jsonFormatWithOptions = z.strictObject({
  type: z.literal("json"),
  schema: z
    .any()
    .optional()
    .transform(val => normalizeSchemaForOpenAI(val))
    .refine(val => validateSchemaForOpenAI(val), {
      message: OPENAI_SCHEMA_ERROR_MESSAGE,
    }),
  prompt: z.string().max(10000).optional(),
});

export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>;

const changeTrackingFormatWithOptions = z.strictObject({
  type: z.literal("changeTracking"),
  prompt: z.string().optional(),
  schema: z
    .any()
    .optional()
    .transform(val => normalizeSchemaForOpenAI(val))
    .refine(val => validateSchemaForOpenAI(val), {
      message: OPENAI_SCHEMA_ERROR_MESSAGE,
    }),
  modes: z.enum(["json", "git-diff"]).array().optional().prefault([]),
  tag: z.string().or(z.null()).prefault(null),
});

type ChangeTrackingFormatWithOptions = z.output<
  typeof changeTrackingFormatWithOptions
>;

const screenshotFormatWithOptions = z.object({
  type: z.literal("screenshot"),
  fullPage: z.boolean().prefault(false),
  quality: z.number().min(1).max(100).optional(),
  viewport: z
    .object({
      width: z.int().positive().finite().max(7680), // 8K resolution width
      height: z.int().positive().finite().max(4320), // 8K resolution height
    })
    .optional(),
});

type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;

const attributesFormatWithOptions = z.strictObject({
  type: z.literal("attributes"),
  selectors: z
    .array(
      z.strictObject({
        selector: z.string().describe("CSS selector to find elements"),
        attribute: z
          .string()
          .describe(
            "Attribute name to extract (e.g., 'data-vehicle-name' or 'id')",
          ),
      }),
    )
    .describe("Extract specific attributes from elements"),
});

type AttributesFormatWithOptions = z.output<typeof attributesFormatWithOptions>;

const questionFormatWithOptions = z.strictObject({
  type: z.literal("question"),
  question: z.string().min(1).max(10000),
});

type QuestionFormatWithOptions = z.output<typeof questionFormatWithOptions>;

const highlightsFormatWithOptions = z.strictObject({
  type: z.literal("highlights"),
  query: z.string().min(1).max(10000),
});

type HighlightsFormatWithOptions = z.output<typeof highlightsFormatWithOptions>;

/** @deprecated Use `question` or `highlights` format instead. */
const queryFormatWithOptions = z.strictObject({
  type: z.literal("query"),
  prompt: z.string().max(10000),
  mode: z.enum(["freeform", "directQuote"]).optional().default("freeform"),
});

type QueryFormatWithOptions = z.output<typeof queryFormatWithOptions>;

export type FormatObject =
  | { type: "markdown" }
  | { type: "html" }
  | { type: "rawHtml" }
  | { type: "links" }
  | { type: "images" }
  | { type: "summary" }
  | JsonFormatWithOptions
  | ChangeTrackingFormatWithOptions
  | ScreenshotFormatWithOptions
  | AttributesFormatWithOptions
  | QuestionFormatWithOptions
  | HighlightsFormatWithOptions
  | QueryFormatWithOptions
  | { type: "branding" }
  | { type: "audio" }
  | { type: "video" };

const pdfModeSchema = z.enum(["fast", "auto", "ocr"]);

export type PDFMode = z.infer<typeof pdfModeSchema>;

const pdfParserWithOptions = z.strictObject({
  type: z.literal("pdf"),
  mode: pdfModeSchema.optional(),
  maxPages: z.int().positive().finite().max(10000).optional(),
  // Experimental: route this request through the fire-pdf async pipeline
  // (POST /jobs + poll) instead of the sync POST /ocr endpoint. Falls back
  // to sync on any async-path failure, so user-visible behavior is unchanged
  // beyond latency variance. Underscored to mark as internal/experimental.
  __firePdfAsync: z.boolean().optional(),
});

const parsersSchema = z
  .array(z.union([z.literal("pdf"), pdfParserWithOptions]))
  .prefault(["pdf"]);

type Parsers = z.infer<typeof parsersSchema>;

export function shouldParsePDF(parsers?: Parsers): boolean {
  if (!parsers) return true;
  return parsers.some(parser => {
    if (parser === "pdf") return true;
    if (typeof parser === "object" && parser !== null && "type" in parser) {
      return (parser as any).type === "pdf";
    }
    return false;
  });
}

export function getPDFMaxPages(parsers?: Parsers): number | undefined {
  if (!parsers) return undefined;
  const pdfParser = parsers.find(parser => {
    if (typeof parser === "object" && parser !== null && "type" in parser) {
      return (parser as any).type === "pdf";
    }
    return false;
  });
  if (pdfParser && typeof pdfParser === "object" && "maxPages" in pdfParser) {
    return (pdfParser as any).maxPages;
  }
  return undefined;
}

export function getPDFMode(parsers?: Parsers): PDFMode {
  if (!parsers) return "auto";
  for (const parser of parsers) {
    if (parser === "pdf") return "auto";
    if (typeof parser === "object" && parser.type === "pdf") {
      return parser.mode ?? "auto";
    }
  }
  return "auto";
}

export function getFirePdfAsync(parsers?: Parsers): boolean {
  if (!parsers) return false;
  for (const parser of parsers) {
    if (parser === "pdf") return false;
    if (typeof parser === "object" && parser.type === "pdf") {
      return parser.__firePdfAsync === true;
    }
  }
  return false;
}

function transformIframeSelector(selector: string): string {
  return selector.replace(/(?:^|[\s,])iframe(?=\s|$|[.#\[:,])/g, match => {
    const prefix = match.match(/^[\s,]/)?.[0] || "";
    return prefix + 'div[data-original-tag="iframe"]';
  });
}

const SPECIAL_COUNTRIES = ["us-generic", "us-whitelist"];

const locationSchema = z
  .object({
    country: z
      .string()
      .optional()
      .refine(
        val =>
          !val ||
          Object.keys(countries).includes(val.toUpperCase()) ||
          SPECIAL_COUNTRIES.includes(val.toLowerCase()),
        "Invalid country code. Use a valid ISO 3166-1 alpha-2 country code.",
      )
      .transform(val => {
        if (!val) return "us-generic";
        return val.toLowerCase();
      }),
    languages: z.array(z.string()).optional(),
  })
  .optional();

const baseScrapeOptions = z.strictObject({
  formats: z
    .preprocess(
      val => {
        if (!Array.isArray(val)) return val;
        return val.map(format => {
          if (typeof format === "string") {
            return { type: format };
          }
          return format;
        });
      },
      z
        .union([
          z.strictObject({ type: z.literal("markdown") }),
          z.strictObject({ type: z.literal("html") }),
          z.strictObject({ type: z.literal("rawHtml") }),
          z.strictObject({ type: z.literal("links") }),
          z.strictObject({ type: z.literal("images") }),
          z.strictObject({ type: z.literal("summary") }),
          jsonFormatWithOptions,
          changeTrackingFormatWithOptions,
          screenshotFormatWithOptions,
          attributesFormatWithOptions,
          z.strictObject({ type: z.literal("branding") }),
          questionFormatWithOptions,
          highlightsFormatWithOptions,
          queryFormatWithOptions,
          z.strictObject({ type: z.literal("audio") }),
          z.strictObject({ type: z.literal("video") }),
        ])
        .array()
        .optional()
        .prefault([{ type: "markdown" }]),
    )
    .refine(x => {
      return x.filter(f => f.type === "screenshot").length <= 1;
    }, "You may only specify one screenshot format")
    .refine(x => {
      const hasChangeTracking = x.find(f => f.type === "changeTracking");
      const hasMarkdown = x.find(f => f.type === "markdown");
      return !hasChangeTracking || hasMarkdown;
    }, "The changeTracking format requires the markdown format to be specified as well"),
  headers: z.record(z.string(), z.string()).optional(),
  includeTags: z
    .string()
    .array()
    .transform(tags => tags.map(transformIframeSelector))
    .optional(),
  excludeTags: z
    .string()
    .array()
    .transform(tags => tags.map(transformIframeSelector))
    .optional(),
  onlyMainContent: z.boolean().prefault(true),
  onlyCleanContent: z.boolean().prefault(false),
  timeout: z.int().positive().min(1000).optional(),
  waitFor: z.int().nonnegative().max(60000).prefault(0),
  mobile: z.boolean().prefault(false),
  parsers: parsersSchema.optional(),
  actions: actionsSchema.optional(),

  location: locationSchema,

  skipTlsVerification: z.boolean().optional(),
  removeBase64Images: z.boolean().prefault(true),
  fastMode: z.boolean().prefault(false),
  useMock: z.string().optional(),
  blockAds: z.boolean().prefault(true),
  proxy: z.enum(["basic", "stealth", "enhanced", "auto"]).prefault("auto"),
  maxAge: z.int().gte(0).optional(),
  minAge: z.int().gte(0).optional(),
  storeInCache: z.boolean().prefault(true),
  lockdown: z.boolean().prefault(false),

  profile: z
    .object({
      name: z.string().min(1).max(128),
      saveChanges: z.boolean().default(true),
    })
    .optional(),

  // @deprecated
  __searchPreviewToken: z.string().optional(),
  __experimental_omce: z.boolean().prefault(false).optional(),
  __experimental_omceDomain: z.string().optional(),
  __experimental_engpicker: z.boolean().prefault(false).optional(),
  __forceFirePDF: z.boolean().prefault(false).optional(),
});

type ScrapeOptionsBase = z.infer<typeof baseScrapeOptions>;

const waitForRefine = (obj?: ScrapeOptionsBase): boolean => {
  if (obj && obj.waitFor && obj.timeout) {
    if (typeof obj.timeout !== "number" || obj.timeout <= 0) {
      return false;
    }
    return obj.waitFor <= obj.timeout / 2;
  }
  return true;
};
const waitForRefineOpts = {
  message: "waitFor must not exceed half of timeout",
  path: ["waitFor"],
};

// Base transform function that handles both nullable and non-nullable cases
// Uses generic type to preserve all fields from extended schemas
const extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(
  obj: T,
): T extends undefined ? undefined : T => {
  if (!obj) return obj as T extends undefined ? undefined : T;
  // Handle timeout
  let result = { ...obj };
  if (
    obj.formats.find(x => typeof x === "object" && x.type === "json") &&
    obj.timeout === 30000
  ) {
    result = { ...result, timeout: 60000 };
  }

  const changeTracking = obj.formats?.find(
    x => typeof x === "object" && x.type === "changeTracking",
  );

  if (changeTracking && (obj.waitFor === undefined || obj.waitFor < 5000)) {
    result = { ...result, waitFor: 5000 };
  }

  if (changeTracking && obj.timeout === 30000) {
    result = { ...result, timeout: 60000 };
  }

  if (
    (obj.proxy === "stealth" ||
      obj.proxy === "enhanced" ||
      obj.proxy === "auto") &&
    obj.timeout === 30000
  ) {
    result = { ...result, timeout: 120000 };
  }

  if (obj.lockdown && obj.maxAge === undefined) {
    // 2 years in ms. Number.MAX_SAFE_INTEGER lands ~285,000 years which
    // overflows Postgres TIMESTAMP arithmetic in the index lookup and silently
    // returns no rows. 2 years covers any practical cache retention window.
    result = { ...result, maxAge: 2 * 365 * 24 * 60 * 60 * 1000 };
  }

  return result as T extends undefined ? undefined : T;
};

// Type-safe wrapper for nullable cases (used in optional scrapeOptions)
const extractTransform = (
  obj?: ScrapeOptionsBase,
): ScrapeOptionsBase | undefined => {
  return extractTransformImpl(obj);
};

// Type-safe wrapper for non-nullable cases (used in required scrapeOptions schema)
// This ensures TypeScript knows the output is always ScrapeOptionsBase, not undefined
const extractTransformRequired = <T extends ScrapeOptionsBase>(obj: T): T => {
  return extractTransformImpl(obj)! as T;
};

export const scrapeOptions = strictWithMessage(baseScrapeOptions)
  .refine(
    obj => {
      if (!obj.actions) return true;
      return (
        calculateTotalWaitTime(obj.actions, obj.waitFor) <=
        ACTIONS_MAX_WAIT_TIME * 1000
      );
    },
    {
      message: `Total wait time (waitFor + wait actions) cannot exceed ${ACTIONS_MAX_WAIT_TIME} seconds`,
    },
  )
  .refine(waitForRefine, waitForRefineOpts)
  .transform(extractTransformRequired);

export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;

export type ScrapeOptions = BaseScrapeOptions;

export type UploadedParseFileKind = "html" | "pdf" | "document";

export type UploadedParseFile = {
  buffer: Buffer;
  filename: string;
  contentType?: string;
  kind?: UploadedParseFileKind;
};

const ajv = new Ajv();
const agentAjv = new Ajv();
addFormats(agentAjv);

const extractOptions = z
  .strictObject({
    urls: URL.array()
      .max(10, "Maximum of 10 URLs allowed per request while in beta.")
      .optional(),
    prompt: z.string().max(10000).optional(),
    systemPrompt: z.string().max(10000).optional(),
    schema: z
      .any()
      .optional()
      .refine(
        val => {
          if (!val) return true; // Allow undefined schema
          try {
            const validate = ajv.compile(val);
            return typeof validate === "function";
          } catch (e) {
            return false;
          }
        },
        {
          error: "Invalid JSON schema.",
        },
      )
      .transform(val => normalizeSchemaForOpenAI(val))
      .refine(val => validateSchemaForOpenAI(val), {
        message: OPENAI_SCHEMA_ERROR_MESSAGE,
      }),
    limit: z.int().positive().finite().optional(),
    ignoreSitemap: z.boolean().prefault(false),
    includeSubdomains: z.boolean().prefault(true),
    allowExternalLinks: z.boolean().prefault(false),
    enableWebSearch: z.boolean().prefault(false),
    scrapeOptions: baseScrapeOptions.optional(),
    origin: z.string().optional().prefault("api"),
    integration: integrationSchema.optional().transform(val => val || null),
    urlTrace: z.boolean().prefault(false),
    timeout: z.int().positive().min(1000).optional(),
    agent: agentOptionsExtract.optional(),
    __experimental_streamSteps: z.boolean().prefault(false),
    __experimental_llmUsage: z.boolean().prefault(false),
    __experimental_showSources: z.boolean().prefault(false),
    showSources: z.boolean().prefault(false),
    // These two below don't do anything anymore
    __experimental_cacheKey: z.string().optional(),
    __experimental_cacheMode: z
      .enum(["direct", "save", "load"])
      .prefault("direct")
      .optional(),
    __experimental_showCostTracking: z.boolean().prefault(false),
    ignoreInvalidURLs: z.boolean().prefault(true),
    webhook: webhookSchema.optional(),
  })
  .refine(obj => obj.urls || obj.prompt, {
    error: "Either 'urls' or 'prompt' must be provided.",
  })
  .transform(obj => ({
    ...obj,
    allowExternalLinks: obj.allowExternalLinks || obj.enableWebSearch,
  }))
  .refine(
    x => (x.scrapeOptions ? waitForRefine(x.scrapeOptions) : true),
    waitForRefineOpts,
  )
  .transform(x => ({
    ...x,
    scrapeOptions: extractTransform(x.scrapeOptions),
  }));

export const extractRequestSchema = extractOptions;
export type ExtractRequest = z.infer<typeof extractRequestSchema>;
export type ExtractRequestInput = z.input<typeof extractRequestSchema>;

const agentWebhookSchema = createWebhookSchema([
  "started",
  "action",
  "completed",
  "failed",
  "cancelled",
]);

export const agentRequestSchema = z.strictObject({
  urls: URL.array().optional(),
  prompt: z.string().max(10000),
  schema: z
    .any()
    .optional()
    .superRefine((val, ctx) => {
      if (!val) return; // Allow undefined schema
      try {
        agentAjv.compile(val);
      } catch (e) {
        const message =
          e instanceof Error
            ? e.message
            : typeof e === "string"
              ? e
              : "Unknown error";
        ctx.addIssue({
          code: "custom",
          message: `Invalid JSON schema: ${message}`,
        });
      }
    }),
  origin: z.string().optional().prefault("api"),
  integration: integrationSchema.optional().transform(val => val || null),
  maxCredits: z.number().optional(),
  strictConstrainToURLs: z.boolean().optional(),
  webhook: agentWebhookSchema.optional(),

  overrideWhitelist: z.string().optional(),
  model: z.enum(["spark-1-pro", "spark-1-mini"]).default("spark-1-pro"),
});

export type AgentRequest = z.infer<typeof agentRequestSchema>;
// export type AgentRequestInput = z.input<typeof agentRequestSchema>;

const scrapeRequestSchemaBase = baseScrapeOptions.extend({
  url: URL,
  origin: z.string().optional().prefault("api"),
  integration: integrationSchema.optional().transform(val => val || null),
  zeroDataRetention: z.boolean().optional(),
  __agentInterop: z
    .object({
      auth: z.string(),
      requestId: z.string(),
      shouldBill: z.boolean(),
      boostConcurrency: z.boolean().optional(),
    })
    .optional(),
});

export const scrapeRequestSchema = strictWithMessage(scrapeRequestSchemaBase)
  .refine(waitForRefine, waitForRefineOpts)
  .transform(extractTransformRequired);

export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;
// Use z.input on the base schema before strict() to preserve optional fields with defaults
// This is needed because zod v4's .strict() changes type inference for optional fields
// We explicitly make formats optional since it has .prefault() which should make it optional
export type ScrapeRequestInput = Omit<
  z.input<typeof baseScrapeOptions>,
  "formats"
> & {
  formats?: z.input<typeof baseScrapeOptions>["formats"];
} & {
  url: z.input<typeof URL>;
  origin?: string;
  integration?: z.input<typeof integrationSchema> | null;
  zeroDataRetention?: boolean;
};

const uploadedParseFileSchema = z.custom<UploadedParseFile>(
  value =>
    typeof value === "object" &&
    value !== null &&
    "buffer" in value &&
    Buffer.isBuffer((value as any).buffer) &&
    "filename" in value &&
    typeof (value as any).filename === "string" &&
    (value as any).filename.trim().length > 0 &&
    (!("contentType" in value) ||
      (value as any).contentType === undefined ||
      typeof (value as any).contentType === "string") &&
    (!("kind" in value) ||
      (value as any).kind === undefined ||
      (value as any).kind === "html" ||
      (value as any).kind === "pdf" ||
      (value as any).kind === "document"),
  {
    error: "A file upload is required.",
  },
);

const parseRequestSchemaBase = baseScrapeOptions.extend({
  origin: z.string().optional().prefault("api"),
  integration: integrationSchema.optional().transform(val => val || null),
  zeroDataRetention: z.boolean().optional(),
  __agentInterop: z
    .object({
      auth: z.string(),
      requestId: z.string(),
      shouldBill: z.boolean(),
      boostConcurrency: z.boolean().optional(),
    })
    .optional(),
  file: uploadedParseFileSchema,
});

export const parseRequestSchema = strictWithMessage(parseRequestSchemaBase)
  .refine(waitForRefine, waitForRefineOpts)
  .transform(x => {
    const { file, ...scrapeLike } = x;
    return {
      ...extractTransformRequired(scrapeLike),
      file,
    };
  });

export type ParseRequest = z.infer<typeof parseRequestSchema>;
export type ParseRequestInput = z.input<typeof parseRequestSchemaBase>;

const batchScrapeRequestSchemaBase = baseScrapeOptions.extend({
  urls: URL.array().min(1),
  origin: z.string().optional().prefault("api"),
  integration: integrationSchema.optional().transform(val => val || null),
  webhook: webhookSchema.optional(),
  appendToId: z.uuid().optional(),
  ignoreInvalidURLs: z.boolean().prefault(true),
  maxConcurrency: z.int().positive().optional(),
  zeroDataRetention: z.boolean().optional(),
  __agentInterop: z
    .object({
      auth: z.string(),
      requestId: z.string(),
      shouldBill: z.boolean(),
    })
    .optional(),
});

export const batchScrapeRequestSchema = strictWithMessage(
  batchScrapeRequestSchemaBase,
)
  .refine(waitForRefine, waitForRefineOpts)
  .transform(extractTransformRequired);

const batchScrapeRequestSchemaNoURLValidationBase = baseScrapeOptions.extend({
  urls: z.string().array().min(1),
  origin: z.string().optional().prefault("api"),
  integration: integrationSchema.optional().transform(val => val || null),
  webhook: webhookSchema.optional(),
  appendToId: z.uuid().optional(),
  ignoreInvalidURLs: z.boolean().prefault(true),
  maxConcurrency: z.int().positive().optional(),
  zeroDataRetention: z.boolean().optional(),
  __agentInterop: z
    .object({
      auth: z.string(),
      requestId: z.string(),
      shouldBill: z.boolean(),
    })
    .optional(),
});

export const batchScrapeRequestSchemaNoURLValidation = strictWithMessage(
  batchScrapeRequestSchemaNoURLValidationBase,
)
  .refine(waitForRefine, waitForRefineOpts)
  .transform(extractTransformRequired);

export type BatchScrapeRequest = z.infer<typeof batchScrapeRequestSchema>;
// Use z.input on the base schema before strict() to preserve optional fields with defaults
// We explicitly make formats optional since it has .prefault() which should make it optional
export type BatchScrapeRequestInput = Omit<
  z.input<typeof baseScrapeOptions>,
  "formats"
> & {
  formats?: z.input<typeof baseScrapeOptions>["formats"];
} & {
  urls: z.input<typeof URL>[];
  origin?: string;
  integration?: z.input<typeof integrationSchema> | null;
  webhook?: z.input<typeof webhookSchema>;
  appendToId?: string;
  ignoreInvalidURLs?: boolean;
  maxConcurrency?: number;
  zeroDataRetention?: boolean;
};

export const crawlerOptions = z.strictObject({
  includePaths: z.string().array().prefault([]),
  excludePaths: z.string().array().prefault([]),
  maxDiscoveryDepth: z.number().optional(),
  limit: z.number().prefault(10000), // default?
  crawlEntireDomain: z.boolean().optional(),
  allowExternalLinks: z.boolean().prefault(false),
  allowSubdomains: z.boolean().prefault(false),
  ignoreRobotsTxt: z.boolean().prefault(false),
  robotsUserAgent: z.string().optional(),
  sitemap: z.enum(["skip", "include", "only"]).prefault("include"),
  deduplicateSimilarURLs: z.boolean().prefault(true),
  ignoreQueryParameters: z.boolean().prefault(false),
  regexOnFullURL: z.boolean().prefault(false),
  delay: z.number().positive().optional(),
});

// export type CrawlerOptions = {
//   includePaths?: string[];
//   excludePaths?: string[];
//   maxDepth?: number;
//   limit?: number;
//   crawlEntireDomain?: boolean;
//   allowExternalLinks?: boolean;
//   ignoreSitemap?: boolean;
// };

type CrawlerOptions = z.infer<typeof crawlerOptions>;

const crawlRequestSchemaBase = crawlerOptions.extend({
  url: URL,
  origin: z.string().optional().prefault("api"),
  integration: integrationSchema.optional().transform(val => val || null),
  scrapeOptions: baseScrapeOptions.prefault(() => baseScrapeOptions.parse({})),
  webhook: webhookSchema.optional(),
  limit: z.number().prefault(10000),
  maxConcurrency: z.int().positive().optional(),
  zeroDataRetention: z.boolean().optional(),
  prompt: z.string().max(10000).optional(),
});

export const crawlRequestSchema = strictWithMessage(crawlRequestSchemaBase)
  .refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts)
  .transform(x => {
    const scrapeOptionsValue = x.scrapeOptions ?? baseScrapeOptions.parse({});
    return {
      ...x,
      url: x.url,
      scrapeOptions: extractTransformRequired(scrapeOptionsValue),
    };
  });

// export type CrawlRequest = {
//   url: string;
//   crawlerOptions?: CrawlerOptions;
//   scrapeOptions?: Exclude<ScrapeRequest, "url">;
// };

// export type ExtractorOptions = {
//   mode: "markdown" | "llm-extraction" | "llm-extraction-from-markdown" | "llm-extraction-from-raw-html";
//   extractionPrompt?: string;
//   extractionSchema?: Record<string, any>;
// }

export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;

export const MAX_MAP_LIMIT = 100000;

const mapRequestSchemaBase = crawlerOptions
  .omit({ sitemap: true, ignoreQueryParameters: true })
  .extend({
    url: URL,
    origin: z.string().optional().prefault("api"),
    integration: integrationSchema.optional().transform(val => val || null),
    includeSubdomains: z.boolean().prefault(true),
    ignoreQueryParameters: z.boolean().prefault(true),
    search: z.string().optional(),
    sitemap: z.enum(["only", "include", "skip"]).prefault("include"),
    limit: z.number().min(1).max(MAX_MAP_LIMIT).prefault(5000),
    timeout: z.number().positive().finite().optional(),
    useMock: z.string().optional(),
    filterByPath: z.boolean().prefault(true),
    useIndex: z.boolean().prefault(true),
    ignoreCache: z.boolean().prefault(false),
    location: locationSchema,
    headers: z.record(z.string(), z.string()).optional(),
  });

export const mapRequestSchema = strictWithMessage(mapRequestSchemaBase);

// export type MapRequest = {
//   url: string;
//   crawlerOptions?: CrawlerOptions;
// };

export type MapRequest = z.infer<typeof mapRequestSchema>;
export type MapRequestInput = z.input<typeof mapRequestSchema>;

export type Document = {
  title?: string;
  description?: string;
  url?: string;
  markdown?: string;
  html?: string;
  rawHtml?: string;
  links?: string[];
  images?: string[];
  screenshot?: string;
  audio?: string;
  video?: string;
  extract?: any;
  json?: any;
  summary?: string;
  answer?: string;
  highlights?: string;
  branding?: BrandingProfile;
  warning?: string;
  attributes?: {
    selector: string;
    attribute: string;
    values: string[];
  }[];
  actions?: {
    screenshots?: string[];
    scrapes?: ScrapeActionContent[];
    javascriptReturns?: {
      type: string;
      value: unknown;
    }[];
    pdfs?: string[];
  };
  changeTracking?: {
    previousScrapeAt: string | null;
    changeStatus: "new" | "same" | "changed" | "removed";
    visibility: "visible" | "hidden";
    diff?: {
      text: string;
      json: {
        files: Array<{
          from: string | null;
          to: string | null;
          chunks: Array<{
            content: string;
            changes: Array<{
              type: string;
              normal?: boolean;
              ln?: number;
              ln1?: number;
              ln2?: number;
              content: string;
            }>;
          }>;
        }>;
      };
    };
    json?: any;
  };
  metadata: {
    title?: string;
    description?: string;
    language?: string;
    keywords?: string;
    robots?: string;
    ogTitle?: string;
    ogDescription?: string;
    ogUrl?: string;
    ogImage?: string;
    ogAudio?: string;
    ogDeterminer?: string;
    ogLocale?: string;
    ogLocaleAlternate?: string[];
    ogSiteName?: string;
    ogVideo?: string;
    favicon?: string;
    dcTermsCreated?: string;
    dcDateCreated?: string;
    dcDate?: string;
    dcTermsType?: string;
    dcType?: string;
    dcTermsAudience?: string;
    dcTermsSubject?: string;
    dcSubject?: string;
    dcDescription?: string;
    dcTermsKeywords?: string;
    modifiedTime?: string;
    publishedTime?: string;
    articleTag?: string;
    articleSection?: string;
    url?: string;
    sourceURL?: string;
    statusCode: number;
    scrapeId?: string;
    error?: string;
    numPages?: number;
    contentType?: string;
    timezone?: string;
    proxyUsed: "basic" | "stealth";
    cacheState?: "hit" | "miss";
    cachedAt?: string;
    creditsUsed?: number;
    postprocessorsUsed?: string[];
    indexId?: string; // ID used to store the document in the index (GCS)
    concurrencyLimited?: boolean;
    concurrencyQueueDurationMs?: number;
    // [key: string]: string | string[] | number | { smartScrape: number; other: number; total: number } | undefined;
  };
  serpResults?: {
    title: string;
    description: string;
    url: string;
  };
};

export type ErrorResponse = {
  success: false;
  code?: ErrorCodes;
  error: string;
  details?: any;
  sponsor_status?: string;
  login_url?: string;
};

export type ScrapeResponse =
  | ErrorResponse
  | {
      success: true;
      warning?: string;
      data: Document;
      scrape_id?: string;
    };

export interface URLTrace {
  url: string;
  status: "mapped" | "scraped" | "error";
  timing: {
    discoveredAt: string;
    scrapedAt?: string;
    completedAt?: string;
  };
  error?: string;
  warning?: string;
  contentStats?: {
    rawContentLength: number;
    processedContentLength: number;
    tokensUsed: number;
  };
  relevanceScore?: number;
  usedInCompletion?: boolean;
  extractedFields?: string[];
}

export interface ExtractResponse {
  success: boolean;
  error?: string;
  data?: any;
  scrape_id?: string;
  id?: string;
  warning?: string;
  urlTrace?: URLTrace[];
  sources?: {
    [key: string]: string[];
  };
  tokensUsed?: number;
  creditsUsed?: number;
}

export type AgentResponse =
  | ErrorResponse
  | {
      success: boolean;
      id: string;
    };

export type AgentStatusResponse =
  | ErrorResponse
  | {
      success: boolean;
      status: "processing" | "completed" | "failed";
      error?: string;
      data?: any;
      model?: "spark-1-pro" | "spark-1-mini";
      expiresAt: string;
      creditsUsed?: number;
    };

export type AgentCancelResponse =
  | ErrorResponse
  | {
      success: boolean;
    };

export type CrawlResponse =
  | ErrorResponse
  | {
      success: true;
      id: string;
      url: string;
    };

export type BatchScrapeResponse =
  | ErrorResponse
  | {
      success: true;
      id: string;
      url: string;
      invalidURLs?: string[];
    };

// Map document interface (transitioned from v1)
export interface MapDocument {
  url: string;
  title?: string;
  description?: string;
}

// V2 Map Response with dictionary format
export type MapResponse =
  | ErrorResponse
  | {
      success: true;
      links?: MapDocument[];
      warning?: string;
    };

export type CrawlStatusParams = {
  jobId: string;
};

export type ConcurrencyCheckParams = {
  teamId: string;
};

export type ConcurrencyCheckResponse =
  | ErrorResponse
  | {
      success: true;
      concurrency: number;
      maxConcurrency: number;
    };

export type CrawlStatusResponse =
  | ErrorResponse
  | {
      success: true;
      status: "scraping" | "completed" | "failed" | "cancelled";
      completed: number;
      total: number;
      creditsUsed: number;
      expiresAt: string;
      next?: string;
      data: Document[];
      warning?: string;
    }
  | {
      success: false;
      status: "failed";
      error: string;
      completed: number;
      total: number;
      creditsUsed: number;
      expiresAt: string;
      data: Document[];
    };

export type OngoingCrawlsResponse =
  | ErrorResponse
  | {
      success: true;
      crawls: {
        id: string;
        teamId: string;
        url: string;
        created_at: string;
        options: CrawlerOptions;
      }[];
    };

export type CrawlErrorsResponse =
  | ErrorResponse
  | {
      errors: {
        id: string;
        timestamp?: string;
        url: string;
        code?: ErrorCodes;
        error: string;
      }[];
      robotsBlocked: string[];
    };

type AuthObject = {
  team_id: string;
  org_id?: string | null;
};

type Account = {
  remainingCredits: number;
};

export type TeamFlags = {
  ignoreRobots?: "disabled" | "allowed" | "forced";
  customRobotsAgent?: "disabled" | "allowed";
  unblockedDomains?: string[];
  forceZDR?: boolean;
  allowZDR?: boolean;
  scrapeZDR?: "disabled" | "allowed" | "forced";
  searchZDR?: "disabled" | "allowed" | "forced";
  zdrCost?: number;
  checkRobotsOnScrape?: boolean;
  crawlTtlHours?: number;
  ipWhitelist?: boolean;
  bypassCreditChecks?: boolean;
  debugBranding?: boolean;
  maxBrowserSessions?: number;
  researchBeta?: boolean;
} | null;

interface RequestWithMaybeACUC<
  ReqParams = {},
  ReqBody = undefined,
  ResBody = undefined,
> extends Request<ReqParams, ReqBody, ResBody> {
  acuc?: AuthCreditUsageChunk;
}

export interface RequestWithAuth<
  ReqParams = {},
  ReqBody = undefined,
  ResBody = undefined,
> extends RequestWithMaybeACUC<ReqParams, ReqBody, ResBody> {
  auth: AuthObject;
  account?: Account;
}

export function toV0CrawlerOptions(x: CrawlerOptions) {
  return {
    includes: x.includePaths,
    excludes: x.excludePaths,
    maxCrawledLinks: x.limit,
    maxDepth: 9999,
    limit: x.limit,
    generateImgAltText: false,
    allowBackwardCrawling: x.crawlEntireDomain,
    allowExternalContentLinks: x.allowExternalLinks,
    allowSubdomains: x.allowSubdomains,
    ignoreRobotsTxt: x.ignoreRobotsTxt,
    robotsUserAgent: x.robotsUserAgent,
    ignoreSitemap: x.sitemap === "skip",
    sitemapOnly: x.sitemap === "only",
    deduplicateSimilarURLs: x.deduplicateSimilarURLs,
    ignoreQueryParameters: x.ignoreQueryParameters,
    regexOnFullURL: x.regexOnFullURL,
    maxDiscoveryDepth: x.maxDiscoveryDepth,
    currentDiscoveryDepth: 0,
    delay: x.delay,
  };
}

export function toV2CrawlerOptions(x: any): CrawlerOptions {
  return {
    includePaths: x.includes,
    excludePaths: x.excludes,
    limit: x.limit,
    crawlEntireDomain: x.allowBackwardCrawling,
    allowExternalLinks: x.allowExternalContentLinks,
    allowSubdomains: x.allowSubdomains,
    ignoreRobotsTxt: x.ignoreRobotsTxt,
    robotsUserAgent: x.robotsUserAgent,
    sitemap: x.sitemapOnly ? "only" : x.ignoreSitemap ? "skip" : "include",
    deduplicateSimilarURLs: x.deduplicateSimilarURLs,
    ignoreQueryParameters: x.ignoreQueryParameters,
    regexOnFullURL: x.regexOnFullURL,
    maxDiscoveryDepth: x.maxDiscoveryDepth,
    delay: x.delay,
  };
}

function fromV0CrawlerOptions(
  x: any,
  teamId: string,
): {
  crawlOptions: CrawlerOptions;
  internalOptions: InternalOptions;
} {
  return {
    crawlOptions: crawlerOptions.parse({
      includePaths: x.includes,
      excludePaths: x.excludes,
      limit: x.maxCrawledLinks ?? x.limit,
      crawlEntireDomain: x.allowBackwardCrawling,
      allowExternalLinks: x.allowExternalContentLinks,
      allowSubdomains: x.allowSubdomains,
      ignoreRobotsTxt: x.ignoreRobotsTxt,
      sitemap: x.sitemapOnly ? "only" : x.ignoreSitemap ? "skip" : "include",
      deduplicateSimilarURLs: x.deduplicateSimilarURLs,
      ignoreQueryParameters: x.ignoreQueryParameters,
      regexOnFullURL: x.regexOnFullURL,
      maxDiscoveryDepth: x.maxDiscoveryDepth,
      delay: x.delay,
    }),
    internalOptions: {
      v0CrawlOnlyUrls: x.returnOnlyUrls,
      teamId,
    },
  };
}

export interface MapDocument {
  url: string;
  title?: string;
  description?: string;
}
export function fromV0ScrapeOptions(
  pageOptions: PageOptions,
  extractorOptions: ExtractorOptions | undefined,
  timeout: number | undefined,
  teamId: string,
): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {
  return {
    scrapeOptions: scrapeOptions.parse({
      formats: [
        (pageOptions.includeMarkdown ?? true) ? ("markdown" as const) : null,
        (pageOptions.includeHtml ?? false) ? ("html" as const) : null,
        (pageOptions.includeRawHtml ?? false) ? ("rawHtml" as const) : null,
        (pageOptions.screenshot ?? false) ? ("screenshot" as const) : null,
        (pageOptions.fullPageScreenshot ?? false)
          ? { type: "screenshot" as const, fullPage: true }
          : null,
        extractorOptions !== undefined &&
        extractorOptions.mode.includes("llm-extraction")
          ? {
              type: "json" as const,
              prompt: extractorOptions.userPrompt,
              schema: extractorOptions.extractionSchema,
            }
          : null,
        "links",
      ].filter(x => x !== null),
      waitFor: pageOptions.waitFor,
      headers: pageOptions.headers,
      includeTags:
        typeof pageOptions.onlyIncludeTags === "string"
          ? [pageOptions.onlyIncludeTags]
          : pageOptions.onlyIncludeTags,
      excludeTags:
        typeof pageOptions.removeTags === "string"
          ? [pageOptions.removeTags]
          : pageOptions.removeTags,
      onlyMainContent: pageOptions.onlyMainContent ?? false,
      timeout: timeout,
      parsers:
        pageOptions.parsePDF !== undefined
          ? pageOptions.parsePDF
            ? ["pdf"]
            : []
          : undefined,
      actions: pageOptions.actions,
      location: pageOptions.geolocation,
      skipTlsVerification: pageOptions.skipTlsVerification,
      removeBase64Images: pageOptions.removeBase64Images,
      mobile: pageOptions.mobile,
      fastMode: pageOptions.useFastMode,
    }),
    internalOptions: {
      atsv: pageOptions.atsv,
      v0DisableJsDom: pageOptions.disableJsDom,
      teamId,
      ...(extractorOptions !== undefined &&
      extractorOptions.mode.includes("llm-extraction")
        ? {
            v1JSONSystemPrompt: extractorOptions.extractionPrompt,
          }
        : {}),
    },
    // TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks
  };
}

export function fromV1ScrapeOptions(
  v1ScrapeOptions: V1ScrapeOptions,
  timeout: number | undefined,
  teamId: string,
): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {
  const spreadScrapeOptions = { ...v1ScrapeOptions };
  delete (spreadScrapeOptions as any).urls;
  delete (spreadScrapeOptions as any).ignoreInvalidURLs;
  delete (spreadScrapeOptions as any).url;
  delete (spreadScrapeOptions as any).origin;
  delete (spreadScrapeOptions as any).integration;
  delete (spreadScrapeOptions as any).webhook;
  delete (spreadScrapeOptions as any).zeroDataRetention;
  delete (spreadScrapeOptions as any).maxConcurrency;
  // v2 scrapeOptions schema is strict and does not include `agent`. We carry it via internalOptions below.
  delete (spreadScrapeOptions as any).agent;

  delete spreadScrapeOptions.__experimental_cache;
  delete spreadScrapeOptions.jsonOptions;
  delete spreadScrapeOptions.changeTrackingOptions;
  delete spreadScrapeOptions.extract;
  delete spreadScrapeOptions.geolocation;
  delete (spreadScrapeOptions as any).parsePDF;

  // Track the original format for v1 backward compatibility
  // Check json first since when user specifies "json", both "json" and "extract" are present
  // When user specifies "extract", only "extract" is present
  let v1OriginalFormat: "extract" | "json" | undefined;
  if (includesFormat(v1ScrapeOptions.formats as any, "json")) {
    v1OriginalFormat = "json";
  } else if (includesFormat(v1ScrapeOptions.formats as any, "extract")) {
    v1OriginalFormat = "extract";
  }

  return {
    scrapeOptions: scrapeOptions.parse({
      ...spreadScrapeOptions,

      ...(v1ScrapeOptions.__experimental_cache
        ? {
            maxAge: v1ScrapeOptions.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours
          }
        : {}),
      location: v1ScrapeOptions.location ?? v1ScrapeOptions.geolocation,
      formats: v1ScrapeOptions.formats
        .map(x => {
          // json and extract is standardized down to extract fmt in v1 -- fine to take one and dismiss the other
          if (x === "extract") {
            const opts = v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract;
            const fmt: JsonFormatWithOptions = {
              type: "json",
              schema: opts?.schema,
              prompt: opts?.prompt,
            };
            return fmt;
          } else if (x === "json") {
            // If jsonOptions are provided with json format, create JsonFormatWithOptions
            const opts = v1ScrapeOptions.jsonOptions;
            if (opts) {
              const fmt: JsonFormatWithOptions = {
                type: "json",
                schema: opts.schema,
                prompt: opts.prompt,
              };
              return includesFormat(v1ScrapeOptions.formats as any, "extract")
                ? null
                : fmt;
            }
            return null;
          } else if (x === "changeTracking") {
            const opts = v1ScrapeOptions.changeTrackingOptions;
            const fmt: ChangeTrackingFormatWithOptions = {
              type: "changeTracking",
              modes: opts?.modes ?? [],
              tag: opts?.tag ?? null,
              schema: opts?.schema,
              prompt: opts?.prompt,
            };
            return fmt;
          } else if (x === "screenshot@fullPage") {
            return { type: "screenshot" as const, fullPage: true };
          } else if (x === "branding") {
            return { type: "branding" as const };
          } else {
            return x;
          }
        })
        .filter(x => x !== null),
      parsers:
        v1ScrapeOptions.parsePDF !== undefined
          ? v1ScrapeOptions.parsePDF
            ? ["pdf"]
            : []
          : undefined,
    }),
    internalOptions: {
      teamId,
      v1Agent: v1ScrapeOptions.agent,
      v1JSONSystemPrompt: (
        v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract
      )?.systemPrompt,
      v1JSONAgent: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract)
        ?.agent,
      v1OriginalFormat,
    },
  };
}

export function fromV0Combo(
  pageOptions: PageOptions,
  extractorOptions: ExtractorOptions | undefined,
  timeout: number | undefined,
  crawlerOptions: any,
  teamId: string,
): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {
  const { scrapeOptions, internalOptions: i1 } = fromV0ScrapeOptions(
    pageOptions,
    extractorOptions,
    timeout,
    teamId,
  );
  const { internalOptions: i2 } = fromV0CrawlerOptions(crawlerOptions, teamId);
  return { scrapeOptions, internalOptions: Object.assign(i1, i2) };
}

// Search source type definitions
// These allow fine-grained control over each search source type
// Similar to how scrape formats work with jsonFormatWithOptions, etc.

const webSearchSourceOptions = z.strictObject({
  type: z.literal("web"),
  tbs: z.string().optional(), // Time-based search (e.g., "qdr:d" for past day)
  filter: z.string().optional(), // Search filter
  lang: z.string().optional(), // Language override for this source
  country: z.string().optional(), // Country override for this source
  location: z.string().optional(), // Location override for this source
});

const imagesSearchSourceOptions = z.strictObject({
  type: z.literal("images"),
});

const newsSearchSourceOptions = z.strictObject({
  type: z.literal("news"),
});

// Category source type definitions
const githubCategoryOptions = z.strictObject({
  type: z.literal("github"),
});

const researchCategoryOptions = z.strictObject({
  type: z.literal("research"),
});

const pdfCategoryOptions = z.strictObject({
  type: z.literal("pdf"),
});

const searchDomainSchema = z
  .string()
  .trim()
  .toLowerCase()
  .refine(
    value =>
      /^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/.test(
        value,
      ),
    "Domain must be a valid hostname without protocol or path",
  );

export const searchRequestSchema = z
  .strictObject({
    query: z.string(),
    limit: z.int().positive().finite().max(100).optional().prefault(10),
    tbs: z.string().optional(),
    filter: z.string().optional(),
    sources: z
      .union([
        // Array of strings (simple format)
        z.array(z.enum(["web", "images", "news"])),
        // Array of objects (advanced format)
        z.array(
          z.union([
            webSearchSourceOptions,
            imagesSearchSourceOptions,
            newsSearchSourceOptions,
          ]),
        ),
      ])
      .optional()
      .prefault(["web"]),
    categories: z
      .union([
        // Array of strings (simple format)
        z.array(z.enum(["github", "research", "pdf"])),
        // Array of objects (advanced format)
        z.array(
          z.union([
            githubCategoryOptions,
            researchCategoryOptions,
            pdfCategoryOptions,
          ]),
        ),
      ])
      .optional(),
    includeDomains: z.array(searchDomainSchema).optional(),
    excludeDomains: z.array(searchDomainSchema).optional(),
    lang: z.string().optional().prefault("en"),
    enterprise: z.array(z.enum(["default", "anon", "zdr"])).optional(),
    country: z.string().optional(),
    location: z.string().optional(),
    origin: z.string().optional().prefault("api"),
    integration: integrationSchema.optional().transform(val => val || null),
    timeout: z.int().positive().finite().prefault(60000),
    ignoreInvalidURLs: z.boolean().optional().prefault(false),
    asyncScraping: z.boolean().optional().prefault(false),
    __searchPreviewToken: z.string().optional(),
    scrapeOptions: baseScrapeOptions
      .extend({
        formats: z
          .preprocess(
            val => {
              if (!Array.isArray(val)) return val;
              return val.map(format => {
                if (typeof format === "string") {
                  return { type: format };
                }
                return format;
              });
            },
            z
              .union([
                z.strictObject({ type: z.literal("markdown") }),
                z.strictObject({ type: z.literal("html") }),
                z.strictObject({ type: z.literal("rawHtml") }),
                z.strictObject({ type: z.literal("links") }),
                z.strictObject({ type: z.literal("images") }),
                z.strictObject({ type: z.literal("summary") }),
                jsonFormatWithOptions,
                questionFormatWithOptions,
                highlightsFormatWithOptions,
                queryFormatWithOptions,
                screenshotFormatWithOptions,
              ])
              .array()
              .optional()
              .prefault([]),
          )
          .refine(x => {
            return x.filter(f => f.type === "screenshot").length <= 1;
          }, "You may only specify one screenshot format"),
      })
      .optional(),
    __agentInterop: z
      .object({
        auth: z.string(),
        requestId: z.string(),
        shouldBill: z.boolean(),
      })
      .optional(),
  })
  .refine(
    x => !(x.includeDomains?.length && x.excludeDomains?.length),
    "includeDomains and excludeDomains cannot both be specified",
  )
  .refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts)
  .transform(x => {
    const country =
      x.country !== undefined ? x.country : x.location ? undefined : "us";

    // Transform string array sources to object format
    let sources = x.sources;
    if (sources && Array.isArray(sources) && sources.length > 0) {
      // Check if it's a string array by checking the first element
      if (typeof sources[0] === "string") {
        // It's a string array, transform to object array
        sources = (sources as string[]).map(s => {
          switch (s) {
            case "web":
              return {
                type: "web" as const,
                tbs: x.tbs,
                filter: x.filter,
                lang: x.lang,
                country,
                location: x.location,
              };
            case "images":
              return {
                type: "images" as const,
                // Images don't inherit global params in the simple format
              };
            case "news":
              return {
                type: "news" as const,
                tbs: x.tbs,
                lang: x.lang,
                country,
                location: x.location,
              };
            default:
              return { type: s as any };
          }
        });
      }
      // Otherwise it's already an object array, keep as is
    }

    // Transform string array categories to object format
    let categories = x.categories;
    if (categories && Array.isArray(categories) && categories.length > 0) {
      // Check if it's a string array by checking the first element
      if (typeof categories[0] === "string") {
        // It's a string array, transform to object array
        categories = (categories as string[]).map(c => {
          switch (c) {
            case "github":
              return {
                type: "github" as const,
              };
            case "research":
              return {
                type: "research" as const,
              };
            case "pdf":
              return {
                type: "pdf" as const,
              };
            default:
              return { type: c as any };
          }
        });
      }
      // Otherwise it's already an object array, keep as is
    }

    return {
      ...x,
      country,
      sources,
      categories,
      scrapeOptions: extractTransform(x.scrapeOptions),
    };
  });

export type SearchRequest = z.infer<typeof searchRequestSchema>;
export type SearchRequestInput = z.input<typeof searchRequestSchema>;

export type SearchResponse =
  | ErrorResponse
  | {
      success: true;
      warning?: string;
      data: Document[];
      creditsUsed: number;
      id: string;
    }
  | {
      success: true;
      warning?: string;
      data: import("../../lib/entities").SearchV2Response;
      creditsUsed: number;
      id: string;
    }
  | {
      success: true;
      warning?: string;
      data: import("../../lib/entities").SearchV2Response;
      scrapeIds: {
        web?: string[];
        news?: string[];
        images?: string[];
      };
      creditsUsed: number;
      id: string;
    };

// =============================================
// Search Feedback
// =============================================

const searchFeedbackUrlSchema = z
  .string()
  .trim()
  .min(1)
  .max(2048)
  .refine(value => {
    try {
      const u = new globalThis.URL(value);
      return u.protocol === "http:" || u.protocol === "https:";
    } catch {
      return false;
    }
  }, "Must be a valid http(s) URL");

const missingContentEntrySchema = z.strictObject({
  topic: z
    .string()
    .trim()
    .min(1, "topic must not be empty")
    .max(200, "topic must be 200 characters or fewer"),
  description: z.string().trim().max(2000).optional(),
});

export const searchFeedbackSchema = z
  .strictObject({
    rating: z.enum(["good", "bad", "partial"]),
    valuableSources: z
      .array(
        z.strictObject({
          url: searchFeedbackUrlSchema,
          reason: z.string().trim().max(1000).optional(),
        }),
      )
      .max(50)
      .optional(),
    missingContent: z.array(missingContentEntrySchema).max(20).optional(),
    querySuggestions: z.string().trim().max(2000).optional(),
    origin: z.string().optional().prefault("api"),
    integration: integrationSchema.optional().transform(val => val || null),
  })
  .refine(
    data => {
      const hasSources = (data.valuableSources?.length ?? 0) > 0;
      const hasMissing = (data.missingContent?.length ?? 0) > 0;
      const hasSuggestions =
        !!data.querySuggestions && data.querySuggestions.length > 0;

      switch (data.rating) {
        case "good":
          return hasSources;
        case "partial":
          return hasSources || hasMissing;
        case "bad":
          return hasMissing || hasSuggestions;
      }
    },
    {
      message:
        "Feedback must be substantive. 'good' requires at least one valuableSources entry; 'partial' requires valuableSources or at least one missingContent entry; 'bad' requires at least one missingContent entry or querySuggestions.",
    },
  );

export type SearchFeedbackRequest = z.infer<typeof searchFeedbackSchema>;
export type SearchFeedbackRequestInput = z.input<typeof searchFeedbackSchema>;

export type SearchFeedbackErrorCode =
  | "SEARCH_NOT_FOUND"
  | "FEEDBACK_WINDOW_EXPIRED"
  | "SEARCH_FAILED"
  | "PREVIEW_TEAM_NOT_ALLOWED"
  | "TEAM_OPTED_OUT"
  | "INVALID_BODY"
  | "DB_DISABLED"
  | "INTERNAL";

export type SearchFeedbackResponse =
  | (ErrorResponse & { feedbackErrorCode?: SearchFeedbackErrorCode })
  | {
      success: true;
      feedbackId: string;
      creditsRefunded: number;
      alreadySubmitted?: boolean;
      dailyCapReached?: boolean;
      creditsRefundedToday?: number;
      dailyRefundCap?: number;
      warning?: string;
    };

export type TokenUsage = {
  promptTokens: number;
  completionTokens: number;
  totalTokens: number;
  step?: string;
  model?: string;
};

const generateLLMsTextRequestSchema = z.object({
  url: URL.describe("The URL to generate text from"),
  maxUrls: z
    .number()
    .min(1)
    .max(5000)
    .prefault(10)
    .describe("Maximum number of URLs to process"),
  showFullText: z
    .boolean()
    .prefault(false)
    .describe("Whether to show the full LLMs-full.txt in the response"),
  cache: z
    .boolean()
    .prefault(true)
    .describe("Whether to use cached content if available"),
  __experimental_stream: z.boolean().optional(),
});

type GenerateLLMsTextRequest = z.infer<typeof generateLLMsTextRequestSchema>;