Enrichment Engine, Phase 2b: Global Grounded Wages + Accuracy Eval, Implementation Plan

import { describe, expect, it } from "vitest";
 
import { parseHeadcount, parseWageEur, parseCountryToIso, parseGroundTruth } from "./ground-truth";
 
describe("parseHeadcount", () => {
  it("reads European thousands-separated integers", () => {
    expect(parseHeadcount("65.340")).toBe(65340);
    expect(parseHeadcount("1.156")).toBe(1156);
    expect(parseHeadcount("414.000")).toBe(414000);
    expect(parseHeadcount("464")).toBe(464);
  });
});
 
describe("parseWageEur", () => {
  it("extracts a euro hourly number from dirty strings", () => {
    expect(parseWageEur("€40.70/hr")).toBeCloseTo(40.7, 2);
    expect(parseWageEur("€40.39hr")).toBeCloseTo(40.39, 2);
    expect(parseWageEur("€62.67hr")).toBeCloseTo(62.67, 2);
    expect(parseWageEur("€15.4/hr")).toBeCloseTo(15.4, 2);
  });
  it("returns null when there is no number", () => {
    expect(parseWageEur("n/a")).toBeNull();
  });
});
 
describe("parseCountryToIso", () => {
  it("maps full country names to ISO-2", () => {
    expect(parseCountryToIso("Netherlands")).toBe("NL");
    expect(parseCountryToIso("Germany")).toBe("DE");
    expect(parseCountryToIso("United States")).toBe("US");
  });
  it("passes through an already-ISO code", () => {
    expect(parseCountryToIso("FR")).toBe("FR");
  });
});
 
describe("parseGroundTruth", () => {
  it("parses CSV rows, trimming role whitespace", () => {
    const csv =
      "domain,legal_name,true_primary_country,true_headcount,role_1,true_avg_wage_1\n" +
      "afas.nl,AFAS Software B.V.,Netherlands,739,Consultant ,€32.31/hr\n";
    const rows = parseGroundTruth(csv);
    expect(rows).toHaveLength(1);
    expect(rows[0]!.domain).toBe("afas.nl");
    expect(rows[0]!.trueHeadcount).toBe(739);
    expect(rows[0]!.role).toBe("Consultant");
    expect(rows[0]!.trueCountry).toBe("NL");
    expect(rows[0]!.trueWageEur).toBeCloseTo(32.31, 2);
  });
});

// Pure parsing of the ground-truth dataset. Defensive about European decimals
// (a dot used as a thousands separator) and dirty wage strings.
 
export interface GroundTruthRow {
  domain: string;
  legalName: string;
  trueCountry: string; // ISO-2
  trueHeadcount: number | null;
  role: string;
  trueWageEur: number | null;
}
 
// "65.340" / "1.156" / "414.000" are thousands-separated integers, not decimals.
export function parseHeadcount(raw: string): number | null {
  const s = raw.trim();
  if (!s) return null;
  const digits = s.replace(/[.\s]/g, "");
  if (!/^\d+$/.test(digits)) return null;
  return Number(digits);
}
 
export function parseWageEur(raw: string): number | null {
  const m = raw.replace(",", ".").match(/(\d+(?:\.\d+)?)/);
  return m ? Number(m[1]) : null;
}
 
const COUNTRY_TO_ISO: Record<string, string> = {
  netherlands: "NL",
  germany: "DE",
  "united kingdom": "GB",
  uk: "GB",
  "united states": "US",
  usa: "US",
  france: "FR",
  belgium: "BE",
};
 
export function parseCountryToIso(raw: string): string {
  const s = raw.trim();
  if (/^[A-Za-z]{2}$/.test(s)) return s.toUpperCase();
  return COUNTRY_TO_ISO[s.toLowerCase()] ?? s.toUpperCase();
}
 
// Minimal CSV split (the dataset has no quoted commas in the fields we use).
export function parseGroundTruth(csv: string): GroundTruthRow[] {
  const lines = csv.split(/\r?\n/).filter((l) => l.trim().length > 0);
  const rows: GroundTruthRow[] = [];
  for (let i = 1; i < lines.length; i++) {
    const cols = lines[i]!.split(",");
    if (cols.length < 6) continue;
    rows.push({
      domain: cols[0]!.trim().toLowerCase(),
      legalName: cols[1]!.trim(),
      trueCountry: parseCountryToIso(cols[2]!),
      trueHeadcount: parseHeadcount(cols[3]!),
      role: cols[4]!.trim(),
      trueWageEur: parseWageEur(cols[5]!),
    });
  }
  return rows;
}

import { afterEach, describe, expect, it, vi } from "vitest";
 
import { researchGrossHourly } from "./wage-provider";
 
afterEach(() => vi.restoreAllMocks());
 
function mockTwoCalls(researchText: string, extractJson: string) {
  const fetchMock = vi.spyOn(globalThis, "fetch");
  for (const content of [researchText, extractJson]) {
    fetchMock.mockResolvedValueOnce(
      new Response(JSON.stringify({ choices: [{ message: { content } }] }), { status: 200 }),
    );
  }
  return fetchMock;
}
 
describe("researchGrossHourly", () => {
  it("returns a euro hourly number and a fetched_cited tier when a source is found", async () => {
    mockTwoCalls(
      "A software engineer in Brazil earns about R$ ... ~ EUR 12/hour. Source: https://example.org/br",
      '```json\n{"gross_hourly_eur": 12.0, "source_url": "https://example.org/br"}\n```',
    );
    const out = await researchGrossHourly("BR", "engineering_it", 2025, { apiKey: "k" });
    expect(out.grossHourlyEur).toBeCloseTo(12, 2);
    expect(out.tier).toBe("fetched_cited");
    expect(out.sourceUrl).toContain("example.org");
  });
 
  it("falls back to llm_inferred tier when no source url is present", async () => {
    mockTwoCalls(
      "Roughly EUR 30/hour for this role.",
      '{"gross_hourly_eur": 30, "source_url": null}',
    );
    const out = await researchGrossHourly("JP", "sales", 2025, { apiKey: "k" });
    expect(out.grossHourlyEur).toBeCloseTo(30, 2);
    expect(out.tier).toBe("llm_inferred");
  });
 
  it("throws when no number can be extracted", async () => {
    mockTwoCalls("No reliable data.", '{"gross_hourly_eur": null, "source_url": null}');
    await expect(researchGrossHourly("ZZ", "legal", 2025, { apiKey: "k" })).rejects.toThrow();
  });
});

import "server-only";
 
import { type ConfidenceTier } from "@agent-metrics/types";
 
import { chat, REASONING_MODEL, RESEARCH_MODEL } from "./provider";
import type { ProviderOpts } from "./provider";
 
export interface WageResearch {
  grossHourlyEur: number;
  sourceUrl: string | null;
  tier: ConfidenceTier;
}
 
// Global wage lookup for ANY country/role. Sonar researches the prevailing
// gross wage with a source; Claude extracts a single EUR-per-hour number.
export async function researchGrossHourly(
  country: string,
  role: string,
  year: number,
  opts: ProviderOpts,
): Promise<WageResearch> {
  const research = await chat(
    RESEARCH_MODEL,
    [
      {
        role: "system",
        content:
          "You research labour-market wages. Use web search. Give the typical GROSS wage (before employer costs) and include a source URL. If the local currency is not EUR, convert to EUR.",
      },
      {
        role: "user",
        content: `What is the typical gross hourly wage in EUR for the role "${role}" in country "${country}" around ${year}? If only an annual or monthly salary is available, convert to an hourly figure assuming ~1720 working hours per year. Give the number and a source URL.`,
      },
    ],
    opts.apiKey,
  );
 
  const extract = await chat(
    REASONING_MODEL,
    [
      {
        role: "system",
        content:
          'Extract a single JSON object {"gross_hourly_eur": number|null, "source_url": string|null} from the text. gross_hourly_eur is EUR per hour. Output only the JSON.',
      },
      { role: "user", content: research },
    ],
    opts.apiKey,
  );
 
  const parsed = JSON.parse(extractJsonBlock(extract)) as {
    gross_hourly_eur: number | null;
    source_url: string | null;
  };
  if (typeof parsed.gross_hourly_eur !== "number" || !(parsed.gross_hourly_eur > 0)) {
    throw new Error(`No usable wage for ${role} in ${country}.`);
  }
  const sourceUrl = parsed.source_url ?? null;
  return {
    grossHourlyEur: parsed.gross_hourly_eur,
    sourceUrl,
    tier: sourceUrl ? "fetched_cited" : "llm_inferred",
  };
}
 
function extractJsonBlock(s: string): string {
  const fenced = s.match(/```(?:json)?\s*([\s\S]*?)```/i);
  if (fenced?.[1]) return fenced[1].trim();
  const a = s.indexOf("{");
  const b = s.lastIndexOf("}");
  if (a !== -1 && b > a) return s.slice(a, b + 1);
  throw new Error("No JSON in wage extraction output.");
}

import { researchGrossHourly } from "./wage-provider";

// Loaded hourly cost for a country/role. Cache-first on wages, then grounded
// research (works for ANY country, cached into wages), then a hard fallback.
async function resolveLoadedHourly(
  country: string,
  role: string,
): Promise<{ loadedHourlyEur: number; tier: ConfidenceTier }> {
  const db = createSupabaseAdminClient();
 
  const { data: cached } = await db
    .from("wages")
    .select("wage_data, confidence")
    .eq("country", country)
    .eq("role", role)
    .eq("year", DEFAULT_YEAR)
    .maybeSingle();
  const cachedLoaded = (cached?.wage_data as { blended_hourly_eur?: number } | undefined)
    ?.blended_hourly_eur;
  if (typeof cachedLoaded === "number") {
    return { loadedHourlyEur: cachedLoaded, tier: "seeded_reference" };
  }
 
  const { data: factorRow } = await db
    .from("employer_factors")
    .select("factor")
    .eq("country", country)
    .maybeSingle();
  const factor = (factorRow?.factor as number | undefined) ?? 1.3;
 
  try {
    const research = await researchGrossHourly(country, role, DEFAULT_YEAR, {
      apiKey: env.OPENROUTER_API_KEY,
    });
    const loaded = loadedHourly(research.grossHourlyEur, factor);
    // Cache once: pay per country/role, reuse globally.
    await db.from("wages").upsert(
      {
        country,
        role,
        year: DEFAULT_YEAR,
        wage_data: {
          blended_hourly_eur: loaded,
          gross_hourly_eur: research.grossHourlyEur,
          employer_factor: factor,
          hours_per_year: 1720,
        },
        source: "grounded research",
        source_url: research.sourceUrl,
        last_researched: new Date().toISOString(),
      },
      { onConflict: "country,role,year" },
    );
    return { loadedHourlyEur: loaded, tier: research.tier };
  } catch {
    // Research failed: conservative, explicitly low-confidence, never a silent 0.
    return { loadedHourlyEur: loadedHourly(25, factor), tier: "hard_fallback" };
  }
}

import { type ConfidenceTier, type EnrichedCompany } from "@agent-metrics/types";

import { enrichCompany } from "../lib/enrichment/service";
const r = await enrichCompany("catawiki.com", { forceRefresh: true });
console.log(
  "country",
  r.record.country,
  "annual",
  r.record.business_case?.annual_labour_cost_eur,
  "conf",
  r.record.confidence.overall,
);
process.exit(0);

import { readFileSync } from "node:fs";
import { join } from "node:path";
 
import { describe, expect, it } from "vitest";
 
import { parseGroundTruth } from "./ground-truth";
 
const RUN = process.env.RUN_EVAL === "1";
const SAMPLE = Number(process.env.EVAL_N ?? "8");
 
// Headcount is judged on order-of-magnitude (outside-in estimates are coarse);
// wage within +/-40%; country exact.
function headcountClose(pred: number, truth: number): boolean {
  if (truth <= 0 || pred <= 0) return false;
  const ratio = pred / truth;
  return ratio >= 1 / 3 && ratio <= 3;
}
function wageClose(pred: number, truth: number): boolean {
  if (truth <= 0 || pred <= 0) return false;
  return Math.abs(pred - truth) / truth <= 0.4;
}
 
describe.runIf(RUN)("enrichment accuracy eval", () => {
  it(
    `scores a sample of ${SAMPLE} companies`,
    async () => {
      const { config } = await import("dotenv");
      config({ path: join(process.cwd(), "apps/web/.env.local") });
      const { enrichCompany } = await import("./service");
 
      const csv = readFileSync(
        join(process.cwd(), "apps/web/lib/enrichment/__fixtures__/ground-truth.csv"),
        "utf8",
      );
      const rows = parseGroundTruth(csv).slice(0, SAMPLE);
 
      let countryHits = 0;
      let headcountHits = 0;
      let wageHits = 0;
      let wageJudged = 0;
 
      for (const row of rows) {
        const { record } = await enrichCompany(row.domain, { forceRefresh: true });
        const country = (record.country ?? "").toUpperCase();
        if (country === row.trueCountry) countryHits++;
        if (
          row.trueHeadcount &&
          record.headcount_estimate &&
          headcountClose(record.headcount_estimate, row.trueHeadcount)
        ) {
          headcountHits++;
        }
        if (row.trueWageEur) {
          wageJudged++;
          const predWage = record.wages[0]?.wage_data.gross_hourly_eur;
          if (predWage && wageClose(predWage, row.trueWageEur)) wageHits++;
        }
        // eslint-disable-next-line no-console
        console.log(
          `${row.domain}: country ${country}/${row.trueCountry}, headcount ${record.headcount_estimate}/${row.trueHeadcount}`,
        );
      }
 
      const n = rows.length;
      const countryAcc = countryHits / n;
      const headcountAcc = headcountHits / n;
      const wageAcc = wageJudged ? wageHits / wageJudged : 0;
      const overall = (countryAcc + headcountAcc + wageAcc) / 3;
      // eslint-disable-next-line no-console
      console.log(
        `\nACCURACY country=${countryAcc.toFixed(2)} headcount=${headcountAcc.toFixed(2)} wage=${wageAcc.toFixed(2)} overall=${overall.toFixed(2)}`,
      );
 
      expect(overall).toBeGreaterThan(0); // informational gate; see report below
    },
    1000 * 60 * 10,
  );
});