/**
 * Tests for LangSmith mode runner.
 *
 * These tests mock the LangSmith evaluate() function to verify:
 * - Target function does all work (generation + evaluation)
 * - Evaluator just extracts pre-computed feedback
 * - Dataset context extraction is respected
 * - Filters trigger dataset example preloading
 */

import { mock } from 'jest-mock-extended';
import type { Client } from 'langsmith/client';
import { evaluate as langsmithEvaluate } from 'langsmith/evaluation';
import type { Dataset, Example } from 'langsmith/schemas';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';

import type { SimpleWorkflow } from '@/types/workflow';

import type { Evaluator, Feedback, RunConfig } from '../harness/harness-types';
import { createLogger } from '../harness/logger';

const silentLogger = createLogger(false);

jest.mock('langsmith/evaluation', () => ({
	evaluate: jest.fn().mockResolvedValue({ experimentName: 'test-experiment' }),
}));

jest.mock('langsmith/traceable', () => ({
	traceable: jest.fn(
		<T extends (...args: unknown[]) => unknown>(fn: T, _options: unknown): T => fn,
	),
}));

function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
	return { name, nodes: [], connections: {} };
}

function createMockEvaluator(
	name: string,
	feedback: Feedback[] = [{ evaluator: name, metric: 'score', score: 1, kind: 'score' }],
): Evaluator {
	return {
		name,
		evaluate: jest.fn().mockResolvedValue(feedback),
	};
}

function isRecord(value: unknown): value is Record<string, unknown> {
	return typeof value === 'object' && value !== null && !Array.isArray(value);
}

function isCallable(value: unknown): value is (...args: unknown[]) => unknown {
	return typeof value === 'function';
}

type LangsmithTargetOutput = {
	workflow: SimpleWorkflow;
	prompt: string;
	feedback: Feedback[];
};

function isSimpleWorkflow(value: unknown): value is SimpleWorkflow {
	return isRecord(value) && Array.isArray(value.nodes) && isRecord(value.connections);
}

function isFeedback(value: unknown): value is Feedback {
	return (
		isRecord(value) &&
		typeof value.evaluator === 'string' &&
		typeof value.metric === 'string' &&
		typeof value.score === 'number' &&
		(value.kind === 'score' || value.kind === 'metric' || value.kind === 'detail')
	);
}

function isLangsmithTargetOutput(value: unknown): value is LangsmithTargetOutput {
	return (
		isRecord(value) &&
		isSimpleWorkflow(value.workflow) &&
		typeof value.prompt === 'string' &&
		Array.isArray(value.feedback) &&
		value.feedback.every(isFeedback)
	);
}

async function callLangsmithTarget(target: unknown, inputs: unknown): Promise<unknown> {
	if (isCallable(target)) return await target(inputs);
	if (isRecord(target) && isCallable(target.invoke)) return await target.invoke(inputs);
	throw new Error('Expected LangSmith target to be callable');
}

function createMockLangsmithClient() {
	const lsClient = mock<Client>();
	lsClient.readDataset.mockResolvedValue(mock<Dataset>({ id: 'test-dataset-id' }));
	lsClient.listExamples.mockReturnValue((async function* () {})());
	lsClient.awaitPendingTraceBatches.mockResolvedValue(undefined);
	return lsClient;
}

describe('Runner - LangSmith Mode', () => {
	beforeEach(() => {
		jest.clearAllMocks();
	});

	describe('runEvaluation() with LangSmith', () => {
		it('should call langsmith evaluate() with correct options', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);
			const lsClient = createMockLangsmithClient();

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'my-dataset',
				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
				evaluators: [createMockEvaluator('test')],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test-experiment',
					repetitions: 2,
					concurrency: 4,
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await runEvaluation(config);

			expect(mockEvaluate).toHaveBeenCalledTimes(1);
			const [_target, options] = mockEvaluate.mock.calls[0];
			expect(options).toEqual(
				expect.objectContaining({
					data: 'my-dataset',
					experimentPrefix: 'test-experiment',
					numRepetitions: 2,
					maxConcurrency: 4,
					client: lsClient,
				}),
			);
		});

		it('should create target function that generates workflow and runs evaluators', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);
			const lsClient = createMockLangsmithClient();

			const workflow = createMockWorkflow('Generated');
			const generateWorkflow = jest.fn().mockResolvedValue(workflow);
			const evaluator = createMockEvaluator('test', [
				{ evaluator: 'test', metric: 'score', score: 0.9, kind: 'score' },
			]);

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'test-dataset',
				generateWorkflow,
				evaluators: [evaluator],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test',
					repetitions: 1,
					concurrency: 1,
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await runEvaluation(config);

			expect(mockEvaluate).toHaveBeenCalledTimes(1);
			const [target] = mockEvaluate.mock.calls[0];

			const result = await callLangsmithTarget(target, { prompt: 'Create a workflow' });
			expect(isLangsmithTargetOutput(result)).toBe(true);
			if (!isLangsmithTargetOutput(result)) throw new Error('Expected LangSmith target output');

			// Collectors are passed explicitly from the traceable wrapper to capture token usage and subgraph metrics
			expect(generateWorkflow).toHaveBeenCalledWith(
				'Create a workflow',
				undefined,
				expect.objectContaining({
					tokenUsage: expect.any(Function),
					subgraphMetrics: expect.any(Function),
				}),
			);
			expect(evaluator.evaluate).toHaveBeenCalledWith(
				workflow,
				expect.objectContaining({ prompt: 'Create a workflow' }),
			);
			expect(result).toEqual({
				workflow,
				prompt: 'Create a workflow',
				feedback: [{ evaluator: 'test', metric: 'score', score: 0.9, kind: 'score' }],
			});
		});

		it('should write artifacts when outputDir is provided', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);
			const lsClient = createMockLangsmithClient();

			const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'v2-evals-langsmith-out-'));
			try {
				const config: RunConfig = {
					mode: 'langsmith',
					dataset: 'test-dataset',
					outputDir: tempDir,
					generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow('Generated')),
					evaluators: [createMockEvaluator('test')],
					langsmithClient: lsClient,
					langsmithOptions: {
						experimentName: 'test',
						repetitions: 1,
						concurrency: 1,
					},
					logger: silentLogger,
				};

				const { runEvaluation } = await import('../harness/runner');
				await runEvaluation(config);

				expect(mockEvaluate).toHaveBeenCalledTimes(1);
				const [target] = mockEvaluate.mock.calls[0];

				await callLangsmithTarget(target, { prompt: 'Create a workflow' });

				const entries = fs.readdirSync(tempDir, { withFileTypes: true });
				const exampleDir = entries.find(
					(e) => e.isDirectory() && e.name.startsWith('example-001-'),
				)?.name;
				expect(exampleDir).toBeDefined();

				expect(fs.existsSync(path.join(tempDir, exampleDir!, 'prompt.txt'))).toBe(true);
				expect(fs.existsSync(path.join(tempDir, exampleDir!, 'workflow.json'))).toBe(true);
				expect(fs.existsSync(path.join(tempDir, exampleDir!, 'feedback.json'))).toBe(true);
			} finally {
				fs.rmSync(tempDir, { recursive: true, force: true });
			}
		});

		it('should aggregate feedback from multiple evaluators in target', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);
			const lsClient = createMockLangsmithClient();

			const evaluator1 = createMockEvaluator('e1', [
				{ evaluator: 'e1', metric: 'score', score: 0.8, kind: 'score' },
			]);
			const evaluator2 = createMockEvaluator('e2', [
				{ evaluator: 'e2', metric: 'a', score: 0.9, kind: 'metric' },
				{ evaluator: 'e2', metric: 'b', score: 1.0, kind: 'metric' },
			]);

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'test-dataset',
				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
				evaluators: [evaluator1, evaluator2],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test',
					repetitions: 1,
					concurrency: 1,
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await runEvaluation(config);

			expect(mockEvaluate).toHaveBeenCalledTimes(1);
			const [target] = mockEvaluate.mock.calls[0];
			const result = await callLangsmithTarget(target, { prompt: 'Test' });
			expect(isLangsmithTargetOutput(result)).toBe(true);
			if (!isLangsmithTargetOutput(result)) throw new Error('Expected LangSmith target output');

			expect(result.feedback).toHaveLength(3);
			expect(result.feedback).toContainEqual({
				evaluator: 'e1',
				metric: 'score',
				score: 0.8,
				kind: 'score',
			});
			expect(result.feedback).toContainEqual({
				evaluator: 'e2',
				metric: 'a',
				score: 0.9,
				kind: 'metric',
			});
			expect(result.feedback).toContainEqual({
				evaluator: 'e2',
				metric: 'b',
				score: 1.0,
				kind: 'metric',
			});
		});

		it('should handle evaluator errors gracefully in target', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);
			const lsClient = createMockLangsmithClient();

			const goodEvaluator = createMockEvaluator('good', [
				{ evaluator: 'good', metric: 'score', score: 1, kind: 'score' },
			]);
			const badEvaluator: Evaluator = {
				name: 'bad',
				evaluate: jest.fn().mockRejectedValue(new Error('Evaluator crashed')),
			};

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'test-dataset',
				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
				evaluators: [goodEvaluator, badEvaluator],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test',
					repetitions: 1,
					concurrency: 1,
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await runEvaluation(config);

			expect(mockEvaluate).toHaveBeenCalledTimes(1);
			const [target] = mockEvaluate.mock.calls[0];

			const result = await callLangsmithTarget(target, { prompt: 'Test' });
			expect(isLangsmithTargetOutput(result)).toBe(true);
			if (!isLangsmithTargetOutput(result)) throw new Error('Expected LangSmith target output');

			expect(result.feedback).toContainEqual({
				evaluator: 'good',
				metric: 'score',
				score: 1,
				kind: 'score',
			});
			expect(result.feedback).toContainEqual({
				evaluator: 'bad',
				metric: 'error',
				score: 0,
				kind: 'score',
				comment: 'Evaluator crashed',
			});
		});

		it('should create evaluator that extracts pre-computed feedback', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);
			const lsClient = createMockLangsmithClient();

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'test-dataset',
				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
				evaluators: [createMockEvaluator('test')],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test',
					repetitions: 1,
					concurrency: 1,
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await runEvaluation(config);

			expect(mockEvaluate).toHaveBeenCalledTimes(1);
			const [_target, options] = mockEvaluate.mock.calls[0];

			expect(Array.isArray(options.evaluators)).toBe(true);
			if (!Array.isArray(options.evaluators))
				throw new Error('Expected LangSmith evaluators array');
			expect(options.evaluators).toHaveLength(1);

			const evaluatorFn = options.evaluators[0];
			expect(isCallable(evaluatorFn)).toBe(true);
			if (!isCallable(evaluatorFn)) throw new Error('Expected evaluator function');

			const extracted = await evaluatorFn({
				outputs: {
					feedback: [
						{ evaluator: 'test', metric: 'score', score: 0.9, kind: 'score' },
						{ evaluator: 'other', metric: 'trigger', score: 0.8, kind: 'metric' },
					],
				},
			});

			expect(extracted).toEqual([
				{ key: 'test.score', score: 0.9 },
				{ key: 'other.trigger', score: 0.8 },
			]);
		});

		it('should keep programmatic prefixes but not llm-judge metric prefixes', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);
			const lsClient = createMockLangsmithClient();

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'test-dataset',
				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
				evaluators: [createMockEvaluator('test')],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test',
					repetitions: 1,
					concurrency: 1,
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await runEvaluation(config);

			expect(mockEvaluate).toHaveBeenCalledTimes(1);
			const [_target, options] = mockEvaluate.mock.calls[0];
			expect(Array.isArray(options.evaluators)).toBe(true);
			if (!Array.isArray(options.evaluators))
				throw new Error('Expected LangSmith evaluators array');

			const evaluatorFn = options.evaluators[0];
			expect(isCallable(evaluatorFn)).toBe(true);
			if (!isCallable(evaluatorFn)) throw new Error('Expected evaluator function');

			const extracted = await evaluatorFn({
				outputs: {
					feedback: [
						{ evaluator: 'llm-judge', metric: 'functionality', score: 0.9, kind: 'metric' },
						{ evaluator: 'programmatic', metric: 'trigger', score: 0.8, kind: 'metric' },
						{
							evaluator: 'llm-judge',
							metric: 'maintainability.nodeNamingQuality',
							score: 0.7,
							kind: 'detail',
						},
					],
				},
			});

			expect(extracted).toEqual([
				{ key: 'functionality', score: 0.9 },
				{ key: 'programmatic.trigger', score: 0.8 },
				{ key: 'maintainability.nodeNamingQuality', score: 0.7 },
			]);
		});

		it('should handle missing feedback in outputs', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);
			const lsClient = createMockLangsmithClient();

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'test-dataset',
				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
				evaluators: [],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test',
					repetitions: 1,
					concurrency: 1,
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await runEvaluation(config);

			expect(mockEvaluate).toHaveBeenCalledTimes(1);
			const [_target, options] = mockEvaluate.mock.calls[0];

			expect(Array.isArray(options.evaluators)).toBe(true);
			if (!Array.isArray(options.evaluators))
				throw new Error('Expected LangSmith evaluators array');
			const evaluatorFn = options.evaluators[0];
			expect(isCallable(evaluatorFn)).toBe(true);
			if (!isCallable(evaluatorFn)) throw new Error('Expected evaluator function');

			const extracted = await evaluatorFn({ outputs: {} });
			expect(extracted).toEqual([
				{
					key: 'evaluationError',
					score: 0,
					comment: 'No feedback found in target output',
				},
			]);
		});

		it('should pass dataset-level context to evaluators', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);
			const lsClient = createMockLangsmithClient();

			const evaluateContextual: Evaluator['evaluate'] = async (_workflow, ctx) => [
				{ evaluator: 'contextual', metric: 'score', score: ctx.dos ? 1 : 0, kind: 'score' },
			];

			const evaluator: Evaluator = {
				name: 'contextual',
				evaluate: jest.fn(evaluateContextual),
			};

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'test-dataset',
				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
				evaluators: [evaluator],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test',
					repetitions: 1,
					concurrency: 1,
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await runEvaluation(config);

			expect(mockEvaluate).toHaveBeenCalledTimes(1);
			const [target] = mockEvaluate.mock.calls[0];

			const result = await callLangsmithTarget(target, {
				prompt: 'Test',
				evals: { dos: 'Use Slack', donts: 'No HTTP' },
			});
			expect(isLangsmithTargetOutput(result)).toBe(true);
			if (!isLangsmithTargetOutput(result)) throw new Error('Expected LangSmith target output');

			expect(evaluator.evaluate).toHaveBeenCalledWith(
				expect.anything(),
				expect.objectContaining({ dos: 'Use Slack', donts: 'No HTTP' }),
			);
			expect(result.feedback).toContainEqual({
				evaluator: 'contextual',
				metric: 'score',
				score: 1,
				kind: 'score',
			});
		});

		it('should ignore invalid referenceWorkflow in dataset context', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);
			const lsClient = createMockLangsmithClient();

			const evaluate = jest.fn<
				ReturnType<Evaluator['evaluate']>,
				Parameters<Evaluator['evaluate']>
			>(async (_workflow, ctx) => [
				{
					evaluator: 'ref-check',
					metric: 'hasRef',
					score: ctx.referenceWorkflows && ctx.referenceWorkflows.length > 0 ? 1 : 0,
					kind: 'score',
				},
			]);

			const evaluator: Evaluator = {
				name: 'ref-check',
				evaluate,
			};

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'test-dataset',
				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
				evaluators: [evaluator],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test',
					repetitions: 1,
					concurrency: 1,
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await runEvaluation(config);

			expect(mockEvaluate).toHaveBeenCalledTimes(1);
			const [target] = mockEvaluate.mock.calls[0];

			const result = await callLangsmithTarget(target, {
				prompt: 'Test',
				evals: {
					referenceWorkflow: { nodes: [{}], connections: {} },
				},
			});
			expect(isLangsmithTargetOutput(result)).toBe(true);
			if (!isLangsmithTargetOutput(result)) throw new Error('Expected LangSmith target output');

			const ctx = evaluate.mock.calls[0]?.[1];
			expect(ctx?.referenceWorkflows).toBeUndefined();

			expect(result.feedback).toContainEqual({
				evaluator: 'ref-check',
				metric: 'hasRef',
				score: 0,
				kind: 'score',
			});
		});

		it('should pre-load and filter examples when filters are provided', async () => {
			const mockEvaluate = jest.mocked(langsmithEvaluate);

			const examples: Example[] = [
				mock<Example>({
					id: 'e1',
					inputs: { prompt: 'One', evals: { dos: 'Use Slack', donts: 'No HTTP' } },
					metadata: { notion_id: 'n1', categories: ['data_transformation'] },
				}),
				mock<Example>({
					id: 'e2',
					inputs: { prompt: 'Two', evals: { dos: 'Use Gmail', donts: 'No Slack' } },
					metadata: { notion_id: 'n2', categories: ['other'] },
				}),
			];

			const lsClient = createMockLangsmithClient();
			lsClient.listExamples.mockReturnValue(
				(async function* () {
					for (const ex of examples) yield ex;
				})(),
			);

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'test-dataset',
				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
				evaluators: [createMockEvaluator('test')],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test',
					repetitions: 1,
					concurrency: 1,
					filters: { notionId: 'n1', technique: 'data_transformation', doSearch: 'slack' },
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await runEvaluation(config);

			expect(mockEvaluate).toHaveBeenCalledTimes(1);
			const [_target, options] = mockEvaluate.mock.calls[0];
			const data: unknown = options.data;
			expect(Array.isArray(data)).toBe(true);
			if (!Array.isArray(data)) throw new Error('Expected `evaluate()` to receive example array');

			const ids = data
				.filter((e): e is { id: string } => isRecord(e) && typeof e.id === 'string')
				.map((e) => e.id);
			expect(ids).toEqual(['e1']);
		});

		it('should throw when filters match no examples', async () => {
			const lsClient = createMockLangsmithClient();
			lsClient.listExamples.mockReturnValue(
				(async function* () {
					yield mock<Example>({
						id: 'e1',
						inputs: { prompt: 'One', evals: { dos: 'Use Slack', donts: 'No HTTP' } },
						metadata: { notion_id: 'n1', categories: ['data_transformation'] },
					});
				})(),
			);

			const config: RunConfig = {
				mode: 'langsmith',
				dataset: 'test-dataset',
				generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
				evaluators: [createMockEvaluator('test')],
				langsmithClient: lsClient,
				langsmithOptions: {
					experimentName: 'test',
					repetitions: 1,
					concurrency: 1,
					filters: { notionId: 'does-not-exist' },
				},
				logger: silentLogger,
			};

			const { runEvaluation } = await import('../harness/runner');
			await expect(runEvaluation(config)).rejects.toThrow('No examples matched filters');
		});

		it('should include evaluatorAverages in summary', async () => {
			const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-test-'));
			try {
				const mockEvaluate = jest.mocked(langsmithEvaluate);
				const lsClient = createMockLangsmithClient();

				const evaluator1 = createMockEvaluator('pairwise', [
					{ evaluator: 'pairwise', metric: 'score', score: 0.8, kind: 'score' },
				]);
				const evaluator2 = createMockEvaluator('programmatic', [
					{ evaluator: 'programmatic', metric: 'overall', score: 0.9, kind: 'score' },
				]);

				// Mock evaluate to call the target function with test inputs
				mockEvaluate.mockImplementationOnce(async (target, _options) => {
					// Call the target to populate capturedResults
					await callLangsmithTarget(target, { prompt: 'Test prompt 1' });
					await callLangsmithTarget(target, { prompt: 'Test prompt 2' });
					return { experimentName: 'test-experiment' } as Awaited<
						ReturnType<typeof langsmithEvaluate>
					>;
				});

				const config: RunConfig = {
					mode: 'langsmith',
					dataset: 'test-dataset',
					generateWorkflow: jest.fn().mockResolvedValue(createMockWorkflow()),
					evaluators: [evaluator1, evaluator2],
					langsmithClient: lsClient,
					langsmithOptions: {
						experimentName: 'test',
						repetitions: 1,
						concurrency: 1,
					},
					outputDir: tempDir, // Enable artifact saving to capture results
					logger: silentLogger,
				};

				const { runEvaluation } = await import('../harness/runner');
				const summary = await runEvaluation(config);

				// The summary should include evaluatorAverages computed from captured results
				expect(summary.evaluatorAverages).toBeDefined();
				expect(summary.evaluatorAverages).toEqual({
					pairwise: 0.8,
					programmatic: 0.9,
				});
			} finally {
				fs.rmSync(tempDir, { recursive: true, force: true });
			}
		});
	});
});