{
  "schemaVersion": 1,
  "title": "Traceroster keyword, semantic, and hybrid retrieval benchmark",
  "status": "preregistered-awaiting-human-review-and-hybrid-run",
  "created": "2026-09-01",
  "method": {
    "corpusSize": 20,
    "queryCount": 10,
    "cutoff": 5,
    "metrics": ["Precision@5", "NDCG@5", "zero-result rate", "hard-filter pass rate"],
    "protectedAttributes": "Excluded from profiles, queries, labels, and ranking inputs.",
    "judgmentScale": {
      "0": "not relevant",
      "1": "adjacent experience only",
      "2": "plausible match with a material gap",
      "3": "strong match to the declared information"
    },
    "labelRule": "Profiles omitted from a query's draftGrades map have grade 0. Draft grades were created with the synthetic corpus and must be approved by the named human reviewer before any result is published as reviewed evidence."
  },
  "targetConfiguration": {
    "embeddingModel": "text-embedding-3-small",
    "lexicalMethod": "BM25Okapi using the application token pattern",
    "fusion": "reciprocal rank fusion with k=60",
    "reranker": "cross-encoder/ms-marco-MiniLM-L-6-v2",
    "minimumDenseSimilarity": 0.45,
    "minimumSparseRelevance": 0.3,
    "limitations": [
      "The harness uses one bounded document per synthetic profile rather than production resume chunks in Qdrant.",
      "It does not measure recruiter productivity, candidate quality, fairness, interviews, or hiring outcomes.",
      "Production flags and model configuration must be verified against the deployed environment before results are described as representative of the live path."
    ]
  },
  "profiles": [
    {"id":"p01","headline":"Senior backend engineer","location":"Toronto, Canada","remote":true,"years":8,"searchText":"Senior backend engineer. Built TypeScript and Python payments APIs, ledger reconciliation, idempotent checkout services, Kafka events, and PCI-aware systems for remote product teams."},
    {"id":"p02","headline":"Java backend engineer","location":"Berlin, Germany","remote":false,"years":7,"searchText":"Java and Spring backend engineer in healthcare. Built patient scheduling services, PostgreSQL APIs, and audit logging for an on-site hospital software team."},
    {"id":"p03","headline":"Frontend engineer","location":"Lahore, Pakistan","remote":true,"years":5,"searchText":"React and TypeScript frontend engineer. Built accessible component libraries, design-system documentation, dashboards, and responsive SaaS interfaces."},
    {"id":"p04","headline":"iOS engineer","location":"London, United Kingdom","remote":false,"years":6,"searchText":"Swift and SwiftUI mobile engineer. Shipped consumer subscription apps, offline synchronization, StoreKit purchases, and mobile analytics."},
    {"id":"p05","headline":"Data engineer","location":"Karachi, Pakistan","remote":true,"years":6,"searchText":"Data engineer using Python, Airflow, dbt, Snowflake, and Kafka. Built warehouse pipelines, data quality checks, and reliable batch orchestration."},
    {"id":"p06","headline":"Machine learning engineer","location":"Vancouver, Canada","remote":true,"years":5,"searchText":"Machine learning engineer focused on NLP, embeddings, semantic retrieval, evaluation datasets, reranking, and production search observability."},
    {"id":"p07","headline":"Platform engineer","location":"Warsaw, Poland","remote":true,"years":7,"searchText":"Platform engineer operating Kubernetes on AWS. Built Terraform modules, delivery pipelines, service observability, incident response, and multi-region infrastructure."},
    {"id":"p08","headline":"Security engineer","location":"New York, United States","remote":true,"years":7,"searchText":"Application security engineer. Led threat modeling, incident response, SOC 2 controls, vulnerability management, and secure software reviews."},
    {"id":"p09","headline":"Senior product manager","location":"Toronto, Canada","remote":true,"years":8,"searchText":"Senior B2B fintech product manager. Took payments and reconciliation products from early customer discovery through launch and enterprise adoption."},
    {"id":"p10","headline":"Consumer mobile product manager","location":"Austin, United States","remote":false,"years":6,"searchText":"Product manager for consumer mobile social products. Owned engagement experiments, push notifications, subscription conversion, and app-store launches."},
    {"id":"p11","headline":"Product designer","location":"Lisbon, Portugal","remote":true,"years":6,"searchText":"B2B SaaS product designer. Ran customer research, mapped complex workflows, maintained a design system, and partnered with engineers on accessible interfaces."},
    {"id":"p12","headline":"Growth marketing lead","location":"Manchester, United Kingdom","remote":true,"years":7,"searchText":"Early-stage growth and brand lead. Built a marketing function from the first year, clarified positioning, launched content programs, and measured acquisition funnels."},
    {"id":"p13","headline":"Enterprise account executive","location":"Chicago, United States","remote":true,"years":9,"searchText":"Enterprise SaaS account executive. Managed security-heavy sales cycles, procurement, forecasting, and multi-stakeholder deals above six figures."},
    {"id":"p14","headline":"Technical recruiter","location":"Islamabad, Pakistan","remote":true,"years":5,"searchText":"Technical recruiter sourcing passive software candidates. Uses intake calibration, talent pools, structured outreach, shortlist reviews, and hiring-manager feedback."},
    {"id":"p15","headline":"Customer success manager","location":"Dublin, Ireland","remote":true,"years":6,"searchText":"B2B customer success manager for SaaS accounts. Led onboarding, adoption reviews, renewals, escalation management, and customer education."},
    {"id":"p16","headline":"QA automation engineer","location":"Prague, Czechia","remote":true,"years":5,"searchText":"QA automation engineer using Playwright, Cypress, API contract tests, and CI quality gates for web applications."},
    {"id":"p17","headline":"Financial analyst","location":"Boston, United States","remote":false,"years":4,"searchText":"Financial analyst building forecasting models, board reporting, budget variance analysis, and spreadsheet automation for a logistics company."},
    {"id":"p18","headline":"Operations manager","location":"Dubai, United Arab Emirates","remote":false,"years":8,"searchText":"Logistics operations manager. Improved warehouse planning, carrier coordination, delivery performance, and frontline team processes."},
    {"id":"p19","headline":"Distributed systems engineer","location":"Amsterdam, Netherlands","remote":true,"years":8,"searchText":"Backend engineer using Go for distributed systems. Built high-throughput services, event streaming, service reliability controls, and cloud-native APIs."},
    {"id":"p20","headline":"Engineering manager","location":"Cape Town, South Africa","remote":true,"years":10,"searchText":"Engineering manager who managed five engineers, coached senior developers, planned delivery, and stayed hands-on with Node.js services during a startup scale-up."}
  ],
  "queries": [
    {"id":"q01","query":"senior backend engineer who has built payments infrastructure for a remote product team","filters":{"remote":true,"minimumYears":6},"draftGrades":{"p01":3,"p19":1}},
    {"id":"q02","query":"engineering leader who managed a small team and remained hands-on with Node.js","filters":{"remote":true,"minimumYears":8},"draftGrades":{"p20":3}},
    {"id":"q03","query":"product manager with early-stage B2B fintech and payments experience","filters":{"remote":true,"minimumYears":6},"draftGrades":{"p09":3,"p12":1}},
    {"id":"q04","query":"B2B product designer experienced in customer research and design systems","filters":{"remote":true,"minimumYears":5},"draftGrades":{"p11":3,"p03":1}},
    {"id":"q05","query":"machine learning engineer for NLP semantic search embeddings and reranking","filters":{"remote":true,"minimumYears":4},"draftGrades":{"p06":3,"p05":1}},
    {"id":"q06","query":"platform engineer with Kubernetes AWS Terraform and incident response","filters":{"remote":true,"minimumYears":6},"draftGrades":{"p07":3,"p19":1}},
    {"id":"q07","query":"security engineer with SOC 2 controls threat modeling and incident response","filters":{"remote":true,"minimumYears":6},"draftGrades":{"p08":3,"p07":1}},
    {"id":"q08","query":"technical recruiter who sources passive software candidates and calibrates with hiring managers","filters":{"remote":true,"minimumYears":4},"draftGrades":{"p14":3}},
    {"id":"q09","query":"senior Go backend engineer for high-throughput distributed services","filters":{"remote":true,"minimumYears":7},"draftGrades":{"p19":3,"p01":1}},
    {"id":"q10","query":"data engineer with Airflow dbt Snowflake and warehouse quality checks","filters":{"remote":true,"minimumYears":5},"draftGrades":{"p05":3,"p17":1}}
  ],
  "review": {
    "assignedReviewer": "Syed Ayaan Ali",
    "reviewer": null,
    "reviewedAt": null,
    "approved": false,
    "instructions": "Syed Ayaan Ali independently grades every query against all 20 profiles without viewing draft labels or ranking output. After the completed blind review sheet is reconciled, enter the reviewer name/date and set approved true."
  }
}