Enrichment Engine, Phase 2a: Provider + Deterministic Core + Cache-First Service, Implementation Plan

  OPENROUTER_API_KEY: nonEmptyOptional(20),

import { afterEach, describe, expect, it, vi } from "vitest";
 
import { researchCompany, extractCompany } from "./provider";
 
afterEach(() => vi.restoreAllMocks());
 
function mockFetchOnce(content: string) {
  return vi.spyOn(globalThis, "fetch").mockResolvedValue(
    new Response(JSON.stringify({ choices: [{ message: { content } }] }), {
      status: 200,
      headers: { "content-type": "application/json" },
    }),
  );
}
 
describe("researchCompany", () => {
  it("calls OpenRouter with the sonar model and returns text + the citations sent", async () => {
    const spy = mockFetchOnce("Acme BV is a 40-person SaaS in NL. Source: https://acme.nl/about");
    const res = await researchCompany("acme.nl", { apiKey: "test-key" });
    expect(res.text).toContain("Acme BV");
    const body = JSON.parse((spy.mock.calls[0]![1] as RequestInit).body as string);
    expect(body.model).toBe("perplexity/sonar");
    expect(JSON.stringify(body.messages)).toContain("acme.nl");
  });
 
  it("throws a clear error when the api key is missing", async () => {
    await expect(researchCompany("acme.nl", { apiKey: undefined })).rejects.toThrow(
      /OPENROUTER_API_KEY/,
    );
  });
});
 
describe("extractCompany", () => {
  it("parses the model's JSON block into an EnrichedCompany-shaped object", async () => {
    mockFetchOnce(
      'Here is the data:\n```json\n{"domain":"acme.nl","company_name":"Acme BV","country":"NL","industry":"saas","headcount_estimate":40,"roles":[{"role":"sales","headcount":5,"confidence":"llm_inferred"}],"confidence":{"overall":0.6},"sources":["https://acme.nl/about"]}\n```',
    );
    const out = await extractCompany("acme.nl", "Acme BV is a 40-person SaaS in NL.", {
      apiKey: "k",
    });
    expect(out.domain).toBe("acme.nl");
    expect(out.roles[0]!.role).toBe("sales");
  });
});

import "server-only";
 
import { EnrichedCompanySchema, type EnrichedCompany } from "@agent-metrics/types";
 
const OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions";
 
// Model ids carried over from the prior n8n engine. Sonar does grounded web
// research with citations; Claude does structured extraction and the narrative.
export const RESEARCH_MODEL = "perplexity/sonar";
export const REASONING_MODEL = "anthropic/claude-sonnet-4.6";
 
export interface ProviderOpts {
  apiKey: string | undefined;
}
 
interface ChatResponse {
  choices?: { message?: { content?: string } }[];
}
 
async function chat(
  model: string,
  messages: unknown[],
  apiKey: string | undefined,
): Promise<string> {
  if (!apiKey)
    throw new Error("OPENROUTER_API_KEY is not set; cannot call the enrichment provider.");
  const res = await fetch(OPENROUTER_URL, {
    method: "POST",
    headers: {
      Authorization: `Bearer ${apiKey}`,
      "Content-Type": "application/json",
      "HTTP-Referer": "https://humanhours.dev",
      "X-Title": "HumanHours Enrichment",
    },
    body: JSON.stringify({ model, messages }),
  });
  if (!res.ok) {
    const detail = await res.text().catch(() => "");
    throw new Error(`OpenRouter ${model} failed: ${res.status} ${detail.slice(0, 300)}`);
  }
  const json = (await res.json()) as ChatResponse;
  const content = json.choices?.[0]?.message?.content;
  if (!content) throw new Error(`OpenRouter ${model} returned no content.`);
  return content;
}
 
export interface ResearchResult {
  text: string;
}
 
// Grounded research: ask Sonar to find verifiable facts with source URLs.
export async function researchCompany(domain: string, opts: ProviderOpts): Promise<ResearchResult> {
  const text = await chat(
    RESEARCH_MODEL,
    [
      {
        role: "system",
        content:
          "You are a company research analyst. Use web search. Report only facts you can attribute to a source URL. If unsure, say so. Always include the source URL next to each material fact.",
      },
      {
        role: "user",
        content: `Research the company at domain "${domain}". Find: legal name, primary country, industry, total headcount, the main departments/role categories with rough sizes, customer channels (chat/phone/email), and languages. Give a short factual brief with a source URL for each material fact.`,
      },
    ],
    opts.apiKey,
  );
  return { text };
}
 
// Extraction: Claude turns the research brief into the fixed JSON shape.
export async function extractCompany(
  domain: string,
  researchText: string,
  opts: ProviderOpts,
): Promise<EnrichedCompany> {
  const content = await chat(
    REASONING_MODEL,
    [
      {
        role: "system",
        content:
          'Convert the research brief into a single JSON object. Output ONLY a json code block. Shape: {"domain","company_name","country"(ISO-2),"industry","headcount_estimate"(number),"roles":[{"role","headcount"(number),"confidence"}],"confidence":{"overall"(0..1)},"sources":[url]}. For each role pick a short snake_case key. Set role/overall confidence to "llm_inferred" / a number you can defend from the brief; never invent precise figures the brief does not support.',
      },
      { role: "user", content: `Domain: ${domain}\n\nResearch brief:\n${researchText}` },
    ],
    opts.apiKey,
  );
  const jsonText = extractJsonBlock(content);
  const parsed = JSON.parse(jsonText) as unknown;
  // Loose-parse: the engine tolerates extra keys but guarantees the core shape.
  return EnrichedCompanySchema.parse({ domain, ...(parsed as Record<string, unknown>) });
}
 
// Narrative: Claude writes the business case from deterministic numbers only.
export async function writeBusinessCase(
  domain: string,
  totals: { annual_labour_cost_eur: number; potential_saving_eur: number; net_saving_eur: number },
  opts: ProviderOpts,
): Promise<string> {
  return chat(
    REASONING_MODEL,
    [
      {
        role: "system",
        content:
          "Write a concise 3-sentence business case for automating repetitive work at this company. Use ONLY the numbers given; do not invent figures. No em-dashes. Plain, factual tone.",
      },
      {
        role: "user",
        content: `Company ${domain}. Annual labour cost EUR ${totals.annual_labour_cost_eur}. Potential automation saving EUR ${totals.potential_saving_eur}. Net saving EUR ${totals.net_saving_eur}.`,
      },
    ],
    opts.apiKey,
  );
}
 
function extractJsonBlock(s: string): string {
  const fenced = s.match(/```(?:json)?\s*([\s\S]*?)```/i);
  if (fenced?.[1]) return fenced[1].trim();
  const brace = s.indexOf("{");
  const end = s.lastIndexOf("}");
  if (brace !== -1 && end > brace) return s.slice(brace, end + 1);
  throw new Error("No JSON object found in model output.");
}

import { describe, expect, it } from "vitest";
 
import { CANONICAL_ROLE_KEYS, normalizeRole } from "./normalize-roles";
 
describe("normalizeRole", () => {
  it("maps common synonyms to canonical keys", () => {
    expect(normalizeRole("Customer Service Representative")).toBe("customer_service");
    expect(normalizeRole("software engineer")).toBe("engineering_it");
    expect(normalizeRole("Sales Associate")).toBe("sales");
    expect(normalizeRole("Ops Manager")).toBe("operations");
  });
 
  it("trims and lowercases noisy input", () => {
    expect(normalizeRole("  Consultant ")).toBe("operations");
  });
 
  it("falls back to 'other' for unknown roles", () => {
    expect(normalizeRole("Chief Vibes Officer")).toBe("other");
  });
 
  it("only ever returns a canonical key", () => {
    for (const raw of ["marketing", "legal counsel", "warehouse", "data scientist"]) {
      expect(CANONICAL_ROLE_KEYS).toContain(normalizeRole(raw));
    }
  });
});

// Maps a raw department/role label onto one canonical role key. Keep this in
// sync with the canonical_roles table (migration 0041). Pure + deterministic.
 
export const CANONICAL_ROLE_KEYS = [
  "customer_service",
  "sales",
  "marketing",
  "operations",
  "finance_admin",
  "human_resources",
  "engineering_it",
  "product",
  "data_analytics",
  "logistics",
  "legal",
  "support_helpdesk",
  "management",
  "other",
] as const;
 
export type CanonicalRole = (typeof CANONICAL_ROLE_KEYS)[number];
 
// Ordered: more specific patterns first so e.g. "data" beats generic matches.
const PATTERNS: [RegExp, CanonicalRole][] = [
  [/help ?desk|it support|service desk|first line/, "support_helpdesk"],
  [/data|analytic|scientist|bi\b|business intelligence/, "data_analytics"],
  [/engineer|developer|software|devops|\bit\b|technical|programmer/, "engineering_it"],
  [/product manager|product owner|\bproduct\b/, "product"],
  [/customer (service|support|success)|support (rep|agent)|helpdesk agent/, "customer_service"],
  [/sales|account (exec|manager)|business development|\bbdr\b|\bsdr\b/, "sales"],
  [/market|growth|seo|content|brand|communications/, "marketing"],
  [/finance|account(ing|ant)|controller|bookkeep|admin|payroll/, "finance_admin"],
  [/\bhr\b|human resources|recruit|people|talent/, "human_resources"],
  [/legal|counsel|compliance|lawyer/, "legal"],
  [/logistic|warehouse|supply chain|fulfil|driver|courier/, "logistics"],
  [/operation|\bops\b|consultant|project manager|process/, "operations"],
  [/manager|director|head of|chief|lead\b|executive|\bceo\b|\bcto\b/, "management"],
];
 
export function normalizeRole(raw: string): CanonicalRole {
  const s = raw.trim().toLowerCase();
  if (!s) return "other";
  if ((CANONICAL_ROLE_KEYS as readonly string[]).includes(s)) return s as CanonicalRole;
  for (const [re, role] of PATTERNS) {
    if (re.test(s)) return role;
  }
  return "other";
}

export { normalizeRole, CANONICAL_ROLE_KEYS } from "./enrichment/normalize-roles";
export type { CanonicalRole } from "./enrichment/normalize-roles";

import { describe, expect, it } from "vitest";
 
import { loadedHourly, roleAnnualCost, aggregateAnnualCost } from "./cost";
 
describe("loadedHourly", () => {
  it("applies the employer factor to the gross hourly wage", () => {
    expect(loadedHourly(25, 1.4)).toBeCloseTo(35, 5);
  });
});
 
describe("roleAnnualCost", () => {
  it("multiplies loaded hourly by hours/year by headcount", () => {
    // 35 EUR/h * 1720 h * 10 people = 602000
    expect(roleAnnualCost({ loadedHourlyEur: 35, hoursPerYear: 1720, headcount: 10 })).toBeCloseTo(
      602000,
      2,
    );
  });
 
  it("returns 0 for zero headcount", () => {
    expect(roleAnnualCost({ loadedHourlyEur: 35, hoursPerYear: 1720, headcount: 0 })).toBe(0);
  });
});
 
describe("aggregateAnnualCost", () => {
  it("sums per-role annual costs", () => {
    const total = aggregateAnnualCost([
      { loadedHourlyEur: 35, hoursPerYear: 1720, headcount: 10 },
      { loadedHourlyEur: 50, hoursPerYear: 1720, headcount: 2 },
    ]);
    expect(total).toBeCloseTo(602000 + 172000, 2);
  });
});

// Pure labour-cost math. No I/O. EUR throughout.
 
export function loadedHourly(grossHourlyEur: number, employerFactor: number): number {
  return grossHourlyEur * employerFactor;
}
 
export interface RoleCostInput {
  loadedHourlyEur: number;
  hoursPerYear: number;
  headcount: number;
}
 
export function roleAnnualCost(input: RoleCostInput): number {
  if (input.headcount <= 0) return 0;
  return input.loadedHourlyEur * input.hoursPerYear * input.headcount;
}
 
export function aggregateAnnualCost(roles: RoleCostInput[]): number {
  return roles.reduce((sum, r) => sum + roleAnnualCost(r), 0);
}

export { loadedHourly, roleAnnualCost, aggregateAnnualCost } from "./enrichment/cost";
export type { RoleCostInput } from "./enrichment/cost";

import { describe, expect, it } from "vitest";
 
import { roleSaving, netSaving } from "./savings";
 
describe("roleSaving", () => {
  it("applies the automation rate to the role's annual cost", () => {
    expect(roleSaving(100000, 0.32)).toBeCloseTo(32000, 2);
  });
  it("clamps the rate into 0..1", () => {
    expect(roleSaving(100000, 1.5)).toBeCloseTo(100000, 2);
    expect(roleSaving(100000, -0.2)).toBe(0);
  });
});
 
describe("netSaving", () => {
  it("subtracts agent cost from gross saving", () => {
    expect(netSaving(32000, 5000)).toBe(27000);
  });
  it("treats missing agent cost as zero", () => {
    expect(netSaving(32000, undefined)).toBe(32000);
  });
});

import { describe, expect, it } from "vitest";
 
import { tierScore, rollupConfidence } from "./confidence";
 
describe("tierScore", () => {
  it("ranks tiers from fetched_cited (high) to hard_fallback (low)", () => {
    expect(tierScore("fetched_cited")).toBeGreaterThan(tierScore("official_statistic"));
    expect(tierScore("official_statistic")).toBeGreaterThan(tierScore("seeded_reference"));
    expect(tierScore("seeded_reference")).toBeGreaterThan(tierScore("llm_inferred"));
    expect(tierScore("llm_inferred")).toBeGreaterThan(tierScore("hard_fallback"));
  });
});
 
describe("rollupConfidence", () => {
  it("is the cost-weighted average of per-item tier scores", () => {
    const overall = rollupConfidence([
      { weight: 90000, tier: "official_statistic" },
      { weight: 10000, tier: "hard_fallback" },
    ]);
    // dominated by the high-weight official_statistic item
    expect(overall).toBeGreaterThan(tierScore("seeded_reference"));
    expect(overall).toBeLessThanOrEqual(tierScore("official_statistic"));
  });
  it("returns 0 for no items", () => {
    expect(rollupConfidence([])).toBe(0);
  });
});

// Automation-savings math. Rates are fractions of a role's annual cost.
 
export function roleSaving(roleAnnualCostEur: number, automationRate: number): number {
  const rate = Math.min(1, Math.max(0, automationRate));
  return roleAnnualCostEur * rate;
}
 
export function netSaving(grossSavingEur: number, agentCostEur: number | undefined): number {
  return grossSavingEur - (agentCostEur ?? 0);
}

import type { ConfidenceTier } from "@agent-metrics/types";
 
// Source tier -> a 0..1 trust score. The single place that ranks evidence.
const TIER_SCORE: Record<ConfidenceTier, number> = {
  fetched_cited: 0.95,
  official_statistic: 0.8,
  seeded_reference: 0.6,
  llm_inferred: 0.4,
  hard_fallback: 0.2,
};
 
export function tierScore(tier: ConfidenceTier): number {
  return TIER_SCORE[tier];
}
 
export interface WeightedTier {
  weight: number; // cost weight (EUR) of this datapoint
  tier: ConfidenceTier;
}
 
// Cost-weighted average of tier scores. Bigger spend -> bigger say.
export function rollupConfidence(items: WeightedTier[]): number {
  const totalWeight = items.reduce((s, i) => s + Math.max(0, i.weight), 0);
  if (totalWeight <= 0) return 0;
  const weighted = items.reduce((s, i) => s + Math.max(0, i.weight) * tierScore(i.tier), 0);
  return weighted / totalWeight;
}

export { roleSaving, netSaving } from "./enrichment/savings";
export { tierScore, rollupConfidence } from "./enrichment/confidence";
export type { WeightedTier } from "./enrichment/confidence";

-- 0042_enrichment_reference_seed.sql
-- PROVISIONAL methodology seed: a generic industry-agnostic role distribution
-- and role automation rates. version 'v0-provisional'. Phase 2b supersedes
-- wage_reference and may refine these with cited sources at a higher version.
-- Idempotent via on conflict do nothing on the table's unique keys.
 
-- Generic role distribution under the catch-all industry 'default'. Shares sum
-- to ~1.0; the engine uses this when no industry-specific distribution exists.
insert into public.role_distributions (industry, role, share, source, version) values
  ('default', 'customer_service', 0.16, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'sales',            0.14, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'operations',       0.16, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'engineering_it',   0.14, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'marketing',        0.07, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'finance_admin',    0.08, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'human_resources',  0.05, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'logistics',        0.08, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'support_helpdesk', 0.04, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'data_analytics',   0.03, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'product',          0.02, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'legal',            0.01, 'provisional generic distribution', 'v0-provisional'),
  ('default', 'management',       0.02, 'provisional generic distribution', 'v0-provisional')
on conflict (industry, role, version) do nothing;
 
-- Role automation rates: fraction of a role's hours automatable. Anchored on
-- the two values carried from the n8n engine (customer_service 0.32, logistics
-- 0.08); the rest are provisional mid-points pending a cited source in 2b.
insert into public.automation_rates (role, rate, source, version) values
  ('customer_service', 0.32, 'provisional; n8n carry-over anchor', 'v0-provisional'),
  ('sales',            0.20, 'provisional mid-point', 'v0-provisional'),
  ('operations',       0.22, 'provisional mid-point', 'v0-provisional'),
  ('engineering_it',   0.18, 'provisional mid-point', 'v0-provisional'),
  ('marketing',        0.25, 'provisional mid-point', 'v0-provisional'),
  ('finance_admin',    0.30, 'provisional mid-point', 'v0-provisional'),
  ('human_resources',  0.20, 'provisional mid-point', 'v0-provisional'),
  ('logistics',        0.08, 'provisional; n8n carry-over anchor', 'v0-provisional'),
  ('support_helpdesk', 0.35, 'provisional mid-point', 'v0-provisional'),
  ('data_analytics',   0.15, 'provisional mid-point', 'v0-provisional'),
  ('product',          0.12, 'provisional mid-point', 'v0-provisional'),
  ('legal',            0.18, 'provisional mid-point', 'v0-provisional'),
  ('management',       0.10, 'provisional mid-point', 'v0-provisional'),
  ('other',            0.15, 'provisional mid-point', 'v0-provisional')
on conflict (role, version) do nothing;

import "server-only";
 
import {
  aggregateAnnualCost,
  loadedHourly,
  netSaving,
  normalizeRole,
  rollupConfidence,
  roleAnnualCost,
  roleSaving,
  type WeightedTier,
} from "@agent-metrics/core";
import { type EnrichedCompany } from "@agent-metrics/types";
 
import { createSupabaseAdminClient } from "@/lib/supabase/admin";
import { env } from "@/lib/env";
 
import { extractCompany, researchCompany, writeBusinessCase } from "./provider";
 
const DEFAULT_HOURS_PER_YEAR = 1720;
const DEFAULT_YEAR = 2025;
 
export interface EnrichOptions {
  forceRefresh?: boolean;
}
 
// Returns a cached research row if present and fresh; null otherwise.
async function readResearchCache(domain: string): Promise<EnrichedCompany | null> {
  const db = createSupabaseAdminClient();
  const { data } = await db
    .from("company_research")
    .select("company_data, researched_at, ttl_days")
    .eq("domain", domain)
    .maybeSingle();
  if (!data) return null;
  const ageDays = (Date.now() - new Date(data.researched_at as string).getTime()) / 86_400_000;
  if (ageDays > (data.ttl_days as number)) return null;
  return data.company_data as EnrichedCompany;
}
 
// Loaded hourly cost for a country/role. Cache-first on wages, then
// wage_reference (Phase 2b), then a hard fallback. Returns the tier used.
async function resolveLoadedHourly(
  country: string,
  role: string,
): Promise<{
  loadedHourlyEur: number;
  tier: "official_statistic" | "seeded_reference" | "hard_fallback";
}> {
  const db = createSupabaseAdminClient();
 
  const { data: cached } = await db
    .from("wages")
    .select("wage_data")
    .eq("country", country)
    .eq("role", role)
    .eq("year", DEFAULT_YEAR)
    .maybeSingle();
  const cachedLoaded = (cached?.wage_data as { blended_hourly_eur?: number } | undefined)
    ?.blended_hourly_eur;
  if (typeof cachedLoaded === "number")
    return { loadedHourlyEur: cachedLoaded, tier: "official_statistic" };
 
  const { data: factorRow } = await db
    .from("employer_factors")
    .select("factor, hours_per_year")
    .eq("country", country)
    .maybeSingle();
  const factor = (factorRow?.factor as number | undefined) ?? 1.3;
 
  const { data: refRow } = await db
    .from("wage_reference")
    .select("gross_hourly_eur")
    .eq("country", country)
    .eq("role", role)
    .order("year", { ascending: false })
    .limit(1)
    .maybeSingle();
  if (refRow) {
    return {
      loadedHourlyEur: loadedHourly(refRow.gross_hourly_eur as number, factor),
      tier: "seeded_reference",
    };
  }
 
  // No data anywhere: a conservative fallback, explicitly low-confidence.
  return { loadedHourlyEur: loadedHourly(25, factor), tier: "hard_fallback" };
}
 
export interface EnrichResult {
  record: EnrichedCompany;
  fromCache: boolean;
}
 
export async function enrichCompany(
  domain: string,
  opts: EnrichOptions = {},
): Promise<EnrichResult> {
  if (!opts.forceRefresh) {
    const cached = await readResearchCache(domain);
    if (cached) return { record: cached, fromCache: true };
  }
 
  const provider = { apiKey: env.OPENROUTER_API_KEY };
  const research = await researchCompany(domain, provider);
  const extracted = await extractCompany(domain, research.text, provider);
 
  const country = (extracted.country ?? "NL").toUpperCase();
  const factorRow = await createSupabaseAdminClient()
    .from("employer_factors")
    .select("hours_per_year")
    .eq("country", country)
    .maybeSingle();
  const hoursPerYear =
    (factorRow.data?.hours_per_year as number | undefined) ?? DEFAULT_HOURS_PER_YEAR;
 
  const weighted: WeightedTier[] = [];
  let annualTotal = 0;
  let savingTotal = 0;
  const wages: EnrichedCompany["wages"] = [];
 
  for (const r of extracted.roles) {
    const role = normalizeRole(r.role);
    const { loadedHourlyEur, tier } = await resolveLoadedHourly(country, role);
    const annual = roleAnnualCost({ loadedHourlyEur, hoursPerYear, headcount: r.headcount });
    annualTotal += annual;
 
    const db = createSupabaseAdminClient();
    const { data: rateRow } = await db
      .from("automation_rates")
      .select("rate")
      .eq("role", role)
      .order("version", { ascending: false })
      .limit(1)
      .maybeSingle();
    savingTotal += roleSaving(annual, (rateRow?.rate as number | undefined) ?? 0.15);
 
    weighted.push({ weight: annual, tier });
    wages.push({
      country,
      role,
      wage_data: {
        blended_hourly_eur: loadedHourlyEur,
        gross_hourly_eur: loadedHourlyEur,
        employer_factor: 1,
        hours_per_year: hoursPerYear,
      },
      confidence: tier,
    });
  }
 
  const net = netSaving(savingTotal, undefined);
  const narrative = await writeBusinessCase(
    domain,
    { annual_labour_cost_eur: annualTotal, potential_saving_eur: savingTotal, net_saving_eur: net },
    provider,
  );
 
  const record: EnrichedCompany = {
    ...extracted,
    wages,
    business_case: {
      annual_labour_cost_eur: annualTotal,
      potential_saving_eur: savingTotal,
      net_saving_eur: net,
      summary: narrative,
    },
    confidence: { overall: rollupConfidence(weighted) },
  };
 
  // Cache-first write: upsert the global research row.
  await createSupabaseAdminClient()
    .from("company_research")
    .upsert(
      {
        domain,
        company_data: record,
        roles: record.roles,
        wages: record.wages,
        totals: { annual_cost_eur: annualTotal, potential_saving_eur: savingTotal },
        business_case: record.business_case,
        confidence: record.confidence,
        sources: record.sources,
        researched_at: new Date().toISOString(),
      },
      { onConflict: "domain" },
    );
 
  return { record, fromCache: false };
}