test-harness/dist¶

Classes¶

CognideskTestHarness¶

Constructors¶

Constructor¶

new CognideskTestHarness(options): CognideskTestHarness;

Parameters¶

Parameter	Type
`options`	`TestHarnessOptions`

Returns¶

CognideskTestHarness

Methods¶

runScenario()¶

runScenario(scenario): Promise<ScenarioResult>;

Parameters¶

Parameter	Type
`scenario`	`HarnessScenario`

Returns¶

Promise\<ScenarioResult>

Interfaces¶

AssertionResult¶

Properties¶

assertionId¶

assertionId: string;

passed¶

passed: boolean;

reasoning¶

reasoning: string;

CriterionJudgement¶

Properties¶

criterionId¶

criterionId: string;

passed¶

passed: boolean;

reasoning¶

reasoning: string;

score¶

score: number;

EvaluationCriterion¶

Properties¶

description¶

description: string;

id¶

id: string;

threshold?¶

optional threshold?: number;

weight?¶

optional weight?: number;

HarnessAgentClient¶

Methods¶

createConversation()¶

createConversation(input): Promise<{
  id: string;
}>;

Parameters¶

Parameter	Type
`input`	`CreateRuntimeConversationInput`

Returns¶

Promise\<{ id: string; }>

sendMessage()¶

sendMessage(input): Promise<Pick<HandleUserMessageResult, "text" | "events" | "activeJourneyId">>;

Parameters¶

Parameter	Type
`input`	{ `conversationId`: `string`; `signal?`: `AbortSignal`; `text`: `string`; `turn?`: `unknown`; }
`input.conversationId`	`string`
`input.signal?`	`AbortSignal`
`input.text`	`string`
`input.turn?`	`unknown`

Returns¶

Promise\<Pick\<HandleUserMessageResult, "text" | "events" | "activeJourneyId">>

submitWidget()?¶

optional submitWidget(input): Promise<{
  event: RuntimeEvent;
  events?: RuntimeEvent[];
}>;

Parameters¶

Parameter	Type
`input`	{ `conversationId`: `string`; `output`: `unknown`; `promptId`: `string`; `signal?`: `AbortSignal`; `widgetKind`: `string`; }
`input.conversationId`	`string`
`input.output`	`unknown`
`input.promptId`	`string`
`input.signal?`	`AbortSignal`
`input.widgetKind`	`string`

Returns¶

Promise\<{ event: RuntimeEvent; events?: RuntimeEvent[]; }>

HarnessScenario¶

Properties¶

agentId¶

agentId: string;

assertions?¶

optional assertions?: ScenarioAssertion[];

channel?¶

optional channel?: ConversationChannelInput;

context?¶

optional context?: unknown;

criteria?¶

optional criteria?: EvaluationCriterion[];

description?¶

optional description?: string;

id¶

id: string;

judge?¶

optional judge?: {
  instructions?: string;
  threshold?: number;
};

instructions?¶

optional instructions?: string;

threshold?¶

optional threshold?: number;

maxTurns?¶

optional maxTurns?: number;

name?¶

optional name?: string;

timeoutMs?¶

optional timeoutMs?: number;

user¶

user: SimulatedUserDefinition;

ScenarioAssertionInput¶

Properties¶

activeJourneyIds¶

activeJourneyIds: string[];

conversationId¶

conversationId: string;

events¶

events: RuntimeEvent[];

scenario¶

scenario: HarnessScenario;

transcript¶

transcript: TranscriptTurn[];

ScenarioResult¶

Properties¶

assertions¶

assertions: AssertionResult[];

conversationId¶

conversationId: string;

error?¶

optional error?: string;

events¶

events: RuntimeEvent[];

judgements¶

judgements: CriterionJudgement[];

passed¶

passed: boolean;

scenarioId¶

scenarioId: string;

score¶

score: number;

status¶

status: "error" | "failed" | "passed";

transcript¶

transcript: TranscriptTurn[];

SimulatedUserDefinition¶

Properties¶

background?¶

optional background?: string;

goal¶

goal: string;

identity¶

identity: string;

scriptedTurns?¶

optional scriptedTurns?: ScriptedUserTurn[];

stopWhen?¶

optional stopWhen?: string;

TestHarnessOptions¶

Properties¶

client¶

client: HarnessAgentClient;

judgeModel?¶

optional judgeModel?: ModelAdapter;

privacy?¶

optional privacy?: {
  redactEvents?:   | RuntimeEvent[]
     | Promise<RuntimeEvent[]>;
  redactTranscript?:   | TranscriptTurn[]
     | Promise<TranscriptTurn[]>;
};

redactEvents()?¶

optional redactEvents(input):
  | RuntimeEvent[]
| Promise<RuntimeEvent[]>;

Parameters¶

Parameter	Type
`input`	{ `events`: `RuntimeEvent`[]; `scenario`: `HarnessScenario`; }
`input.events`	`RuntimeEvent`[]
`input.scenario`	`HarnessScenario`

Returns¶

| RuntimeEvent[] | Promise\<RuntimeEvent[]>

redactTranscript()?¶

optional redactTranscript(input):
  | TranscriptTurn[]
| Promise<TranscriptTurn[]>;

Parameters¶

Parameter	Type
`input`	{ `scenario`: `HarnessScenario`; `transcript`: `TranscriptTurn`[]; }
`input.scenario`	`HarnessScenario`
`input.transcript`	`TranscriptTurn`[]

Returns¶

| TranscriptTurn[] | Promise\<TranscriptTurn[]>

simulatedUserModel?¶

optional simulatedUserModel?: ModelAdapter;

timeoutMs?¶

optional timeoutMs?: number;

TranscriptTurn¶

Properties¶

content¶

content: string;

role¶

role: "user" | "assistant";

Type Aliases¶

ScenarioAssertion¶

type ScenarioAssertion =
  | {
  caseSensitive?: boolean;
  id: string;
  text: string;
  type: "assistantContains";
}
  | {
  id: string;
  journeyId: string;
  type: "journeyActivated";
}
  | {
  eventType: RuntimeEvent["type"];
  id: string;
  type: "eventEmitted";
}
  | {
  eventType: RuntimeEvent["type"];
  expected: Record<string, unknown>;
  id: string;
  type: "eventPayloadMatches";
}
  | {
  code?: string;
  id: string;
  toolName?: string;
  type: "policyBlocked";
}
  | {
  id: string;
  toolName: string;
  type: "toolCalled";
}
  | {
  id: string;
  toolName: string;
  type: "toolNotCalled";
}
  | {
  id: string;
  maxWords: number;
  type: "assistantMaxWords";
}
  | {
  id: string;
  maxCharacters: number;
  type: "assistantMaxCharacters";
}
  | {
  description?: string;
  evaluate: (input) => MaybePromise<
     | boolean
     | {
     passed: boolean;
     reasoning?: string;
  }>;
  id: string;
  type: "custom";
};

ScriptedUserTurn¶

type ScriptedUserTurn =
  | string
  | {
  text: string;
  type: "message";
}
  | {
  output: unknown;
  promptId?: string;
  type: "widget";
  widgetKind?: string;
};

Functions¶

createTestHarness()¶

function createTestHarness(options): CognideskTestHarness;

Parameters¶

Parameter	Type
`options`	`TestHarnessOptions`

Returns¶

CognideskTestHarness

runtimeHarnessClient()¶

function runtimeHarnessClient(runtime): HarnessAgentClient;

Parameters¶

Parameter	Type
`runtime`	`CognideskRuntime`

Returns¶

HarnessAgentClient

References¶