Files
redsen-lean-harness/src/lib/repomap.mjs
T
mozempkandCopilot 383129f571 feat: scaffold redsen-lean-harness v0.1.0
Recovered from crashed session (Node OOM). Repo contains full P0-P6
scaffold: plugin.json/marketplace.json, AGENTS.md, ADRs 0001-0006,
lh CLI (init/index/graph/lane/run/memory/host/report/doctor), 10
.github/agents, 12 CLI skills, instructions, context7 mcp.json, and
unit/e2e test suite.

Fixed: run.mjs read --in-tokens/--out-tokens but tests and CLI docs
use --input-tokens/--output-tokens, so telemetry totals were always 0.
Now accepts both forms.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-09-09 22:44:15 +02:00

583 lines
18 KiB
JavaScript

import fs from 'node:fs';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { pathToFileURL } from 'node:url';
import { detect, fallbackFor, queriesFor } from './langs.mjs';
import { ensureDir, estimateTokens, readIfExists, writeFile } from './paths.mjs';
const INDEX_VERSION = 1;
const MAX_FILE_SIZE = 1024 * 1024;
const DEFAULT_WASM_BASE = 'https://cdn.jsdelivr.net/npm/tree-sitter-wasms@latest/out';
let ParserClass = null;
let parserInit = null;
const languageCache = new Map();
export async function buildIndex(ctx, options = {}) {
const started = Date.now();
const config = ctx.config ?? {};
const p = ctx.paths;
const budget = numberOption(options.budget, config.index?.budget ?? 1200);
const files = listFiles(p.root, config.index ?? {});
const cache = loadCache(p.symbols);
const gitHead = git(['rev-parse', 'HEAD'], p.root);
const changedByGit = changedFiles(p.root, cache?.gitHead, gitHead);
const nextFiles = {};
const degraded = new Map();
const stats = {
files: 0,
parsed: 0,
cacheHits: 0,
skipped: 0,
symbols: 0,
refs: 0,
languages: {},
degradedLanguages: {},
elapsedMs: 0,
tokens: 0,
};
for (const rel of files) {
const abs = path.join(p.root, rel);
const stat = safeStat(abs);
if (!stat?.isFile()) continue;
if (stat.size > MAX_FILE_SIZE || isBinary(abs)) {
stats.skipped += 1;
continue;
}
const lang = detect(rel);
if (!lang) {
stats.skipped += 1;
continue;
}
const cached = cache?.files?.[rel];
if (
cached &&
!options.force &&
cached.mtimeMs === stat.mtimeMs &&
cached.size === stat.size &&
!changedByGit.has(rel)
) {
nextFiles[rel] = cached;
stats.cacheHits += 1;
if (cached.degraded) degraded.set(cached.lang, cached.parser === 'regex' ? 'wasm unavailable' : 'degraded');
countFile(stats, cached);
continue;
}
let source;
try {
source = fs.readFileSync(abs, 'utf8');
} catch {
stats.skipped += 1;
continue;
}
const extracted = await extractFile(ctx, rel, source, lang, options.fetch, degraded);
const entry = {
path: rel,
lang: lang.id,
parser: extracted.parser,
degraded: extracted.degraded,
mtimeMs: stat.mtimeMs,
size: stat.size,
hash: hash(source),
symbols: normalizeSymbols(extracted.symbols, source),
refs: normalizeRefs(extracted.refs),
imports: normalizeRefs(extracted.imports ?? []).map((ref) => ref.name),
};
nextFiles[rel] = entry;
stats.parsed += 1;
countFile(stats, entry);
}
const index = {
version: INDEX_VERSION,
root: p.root,
generatedAt: new Date().toISOString(),
gitHead,
files: nextFiles,
graph: buildReferenceGraph(nextFiles),
ranks: {},
stats,
};
index.ranks = rankIndex(index, { focus: options.focus });
const rendered = renderMap(index, { budget, focus: options.focus });
index.stats.elapsedMs = Date.now() - started;
index.stats.tokens = estimateTokens(rendered);
index.stats.degradedLanguages = Object.fromEntries(degraded);
writeFile(p.symbols, `${JSON.stringify(index, null, 2)}\n`);
writeFile(
p.repomap,
`${JSON.stringify(
{
version: INDEX_VERSION,
generatedAt: index.generatedAt,
budget,
focus: options.focus ?? null,
tokens: index.stats.tokens,
map: rendered,
},
null,
2,
)}\n`,
);
return index;
}
export function renderMap(index, options = {}) {
const budget = numberOption(options.budget, index?.stats?.budget ?? 1200);
const ranks = index.ranks?.files ?? rankIndex(index, { focus: options.focus }).files;
const files = Object.values(index.files ?? {}).sort((a, b) => (ranks[b.path] ?? 0) - (ranks[a.path] ?? 0));
const lines = ['# Repo map'];
if (options.focus) lines.push(`focus: ${options.focus}`);
lines.push(`files: ${files.length} symbols: ${index.stats?.symbols ?? countSymbols(index)}`);
for (const file of files) {
const full = renderFile(file, ranks[file.path] ?? 0, 'full');
if (fits(lines, full, budget)) {
lines.push(...full);
continue;
}
const names = renderFile(file, ranks[file.path] ?? 0, 'names');
if (fits(lines, names, budget)) {
lines.push(...names);
continue;
}
const counts = renderFile(file, ranks[file.path] ?? 0, 'counts');
if (fits(lines, counts, budget)) lines.push(...counts);
}
return `${lines.join('\n')}\n`;
}
export function globMatch(pattern, candidate) {
const normalized = normalizePath(candidate);
return globRegex(pattern).test(normalized);
}
async function extractFile(ctx, rel, source, lang, shouldFetch, degraded) {
const wasmPath = await ensureGrammar(ctx, lang, shouldFetch);
if (wasmPath) {
try {
return { ...(await extractTreeSitter(source, lang, wasmPath)), parser: 'tree-sitter', degraded: false };
} catch (error) {
degraded.set(lang.id, `tree-sitter failed: ${error.message}`);
}
} else {
degraded.set(lang.id, shouldFetch ? 'wasm fetch failed' : 'wasm unavailable');
}
const fallback = fallbackFor(lang.id);
const result = fallback(source, rel) ?? {};
return {
symbols: result.symbols ?? [],
refs: result.refs ?? [],
imports: result.imports ?? [],
parser: 'regex',
degraded: true,
};
}
async function ensureGrammar(ctx, lang, shouldFetch) {
const target = path.join(ctx.paths.wasm, lang.wasm);
if (fs.existsSync(target)) return target;
if (!shouldFetch) return null;
try {
ensureDir(ctx.paths.wasm);
const base = process.env.LH_WASM_BASE_URL || ctx.config?.index?.wasmBaseUrl || DEFAULT_WASM_BASE;
const url = base.includes('{file}') ? base.replace('{file}', lang.wasm) : `${base.replace(/\/$/, '')}/${lang.wasm}`;
const response = await fetch(url);
if (!response.ok) return null;
const bytes = Buffer.from(await response.arrayBuffer());
fs.writeFileSync(target, bytes);
return target;
} catch {
return null;
}
}
async function extractTreeSitter(source, lang, wasmPath) {
await initParser();
const parser = new ParserClass();
let language = languageCache.get(wasmPath);
if (!language) {
language = await ParserClass.Language.load(pathToFileURL(wasmPath).href);
languageCache.set(wasmPath, language);
}
parser.setLanguage(language);
const tree = parser.parse(source);
const queries = queriesFor(lang.id);
const symbols = [];
const imports = [];
const refs = [];
collectDefinitions(language, tree, queries.definitions, source, symbols);
collectImports(language, tree, queries.imports, source, imports, refs);
collectCalls(language, tree, queries.calls, source, refs);
markExports(symbols, source);
return { symbols, imports, refs };
}
async function initParser() {
if (!parserInit) {
parserInit = import('web-tree-sitter').then(async (mod) => {
ParserClass = mod.default ?? mod;
await ParserClass.init();
return ParserClass;
});
}
return parserInit;
}
function collectDefinitions(language, tree, querySource, source, out) {
for (const match of runQuery(language, tree, querySource)) {
const def = match.captures.find((cap) => cap.name.startsWith('def.'));
const nameCap = match.captures.find((cap) => cap.name === 'name');
if (!def || !nameCap) continue;
const name = textFor(source, nameCap.node).replace(/^["']|["']$/g, '');
if (!isIdentifierish(name)) continue;
out.push({
name,
kind: def.name.slice(4) || inferKind(def.node.type),
line: def.node.startPosition.row + 1,
endLine: def.node.endPosition.row + 1,
signature: firstLine(source, def.node),
exported: hasExportAncestor(def.node) || /\b(export|pub|public)\b/.test(firstLine(source, def.node)),
});
}
}
function collectImports(language, tree, querySource, source, imports, refs) {
for (const match of runQuery(language, tree, querySource)) {
for (const cap of match.captures) {
if (!cap.name.startsWith('import.')) continue;
const raw = textFor(source, cap.node).replace(/^["'`<]|["'`>]$/g, '');
const name = cap.name === 'import.source' ? lastPathPart(raw) : raw;
if (!isIdentifierish(name)) continue;
imports.push({ name, kind: 'import', line: cap.node.startPosition.row + 1 });
refs.push({ name, kind: 'import', line: cap.node.startPosition.row + 1 });
}
}
}
function collectCalls(language, tree, querySource, source, refs) {
for (const match of runQuery(language, tree, querySource)) {
for (const cap of match.captures) {
if (cap.name !== 'call.name') continue;
const name = textFor(source, cap.node);
if (!isIdentifierish(name)) continue;
refs.push({ name, kind: 'call', line: cap.node.startPosition.row + 1 });
}
}
}
function runQuery(language, tree, querySource) {
if (!querySource?.trim()) return [];
try {
const query = language.query(querySource);
return query.matches(tree.rootNode);
} catch {
return [];
}
}
function markExports(symbols, source) {
const lines = source.split(/\r?\n/);
for (const symbol of symbols) {
const line = lines[symbol.line - 1] ?? '';
if (/\b(export|pub|public)\b/.test(line)) symbol.exported = true;
}
}
function hasExportAncestor(node) {
let current = node;
while (current) {
if (/export/.test(current.type)) return true;
current = current.parent;
}
return false;
}
function normalizeSymbols(symbols, source) {
const seen = new Set();
const lines = source.split(/\r?\n/);
return (symbols ?? [])
.filter((symbol) => symbol?.name && isIdentifierish(symbol.name))
.map((symbol) => ({
name: String(symbol.name),
kind: symbol.kind ?? 'symbol',
line: Math.max(1, Number(symbol.line) || 1),
endLine: Math.max(Number(symbol.endLine) || Number(symbol.line) || 1, Number(symbol.line) || 1),
signature: String(symbol.signature || lines[(Number(symbol.line) || 1) - 1] || symbol.name).trim().slice(0, 180),
exported: Boolean(symbol.exported),
arity: arity(symbol.signature || ''),
}))
.filter((symbol) => {
const key = `${symbol.name}:${symbol.kind}:${symbol.line}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
});
}
function normalizeRefs(refs) {
const seen = new Set();
return (refs ?? [])
.filter((ref) => ref?.name && isIdentifierish(ref.name))
.map((ref) => ({ name: String(ref.name), kind: ref.kind ?? 'call', line: Math.max(1, Number(ref.line) || 1) }))
.filter((ref) => {
const key = `${ref.kind}:${ref.name}:${ref.line}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
});
}
function buildReferenceGraph(files) {
const definitions = new Map();
for (const file of Object.values(files)) {
for (const symbol of file.symbols ?? []) {
if (!definitions.has(symbol.name)) definitions.set(symbol.name, new Set());
definitions.get(symbol.name).add(file.path);
}
}
const edges = {};
for (const file of Object.values(files)) {
const out = new Set();
for (const ref of file.refs ?? []) {
for (const target of definitions.get(ref.name) ?? []) {
if (target !== file.path) out.add(target);
}
}
edges[file.path] = [...out].sort();
}
return { edges };
}
function rankIndex(index, options = {}) {
const paths = Object.keys(index.files ?? {}).sort();
const n = paths.length || 1;
const edges = index.graph?.edges ?? buildReferenceGraph(index.files ?? {}).edges;
let scores = Object.fromEntries(paths.map((file) => [file, 1 / n]));
const focusMatchers = [options.focus].filter(Boolean);
const seeds = Object.fromEntries(
paths.map((file) => {
const focused = focusMatchers.some((pattern) => globMatch(pattern, file));
return [file, focused ? 4 : 1];
}),
);
const seedTotal = Object.values(seeds).reduce((sum, value) => sum + value, 0) || 1;
for (const file of paths) seeds[file] /= seedTotal;
for (let i = 0; i < 20; i += 1) {
const next = Object.fromEntries(paths.map((file) => [file, (1 - 0.85) * seeds[file]]));
for (const from of paths) {
const outs = edges[from] ?? [];
if (!outs.length) {
for (const file of paths) next[file] += (0.85 * scores[from]) / n;
} else {
for (const to of outs) next[to] += (0.85 * scores[from]) / outs.length;
}
}
scores = next;
}
const symbols = {};
for (const file of Object.values(index.files ?? {})) {
for (const symbol of file.symbols ?? []) symbols[`${file.path}:${symbol.name}:${symbol.line}`] = scores[file.path] ?? 0;
}
return { files: scores, symbols };
}
function listFiles(root, indexConfig) {
const include = indexConfig.include?.length ? indexConfig.include : ['**/*'];
const exclude = indexConfig.exclude ?? [];
const depth = Number(indexConfig.depth ?? 0);
const gitFiles = git(['ls-files', '--cached', '--others', '--exclude-standard'], root);
const files = gitFiles ? gitFiles.split(/\r?\n/).filter(Boolean) : manualFiles(root);
return [...new Set(files.map(normalizePath))]
.filter((file) => withinDepth(file, depth))
.filter((file) => include.some((pattern) => globMatch(pattern, file)))
.filter((file) => !exclude.some((pattern) => globMatch(pattern, file)))
.sort();
}
function manualFiles(root) {
const out = [];
const walk = (dir) => {
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
if (['.git', 'node_modules', '.agents'].includes(entry.name)) continue;
const abs = path.join(dir, entry.name);
const rel = normalizePath(path.relative(root, abs));
if (entry.isDirectory()) walk(abs);
else if (entry.isFile()) out.push(rel);
}
};
walk(root);
return out;
}
function changedFiles(root, cachedHead, currentHead) {
const changed = new Set();
if (cachedHead && currentHead && cachedHead !== currentHead) {
for (const file of lines(git(['diff', '--name-only', `${cachedHead}..${currentHead}`], root))) changed.add(normalizePath(file));
}
for (const file of lines(git(['diff', '--name-only'], root))) changed.add(normalizePath(file));
for (const file of lines(git(['diff', '--name-only', '--cached'], root))) changed.add(normalizePath(file));
return changed;
}
function renderFile(file, score, mode) {
const suffix = file.degraded ? ' regex' : '';
if (mode === 'counts') return [`${file.path} [${file.lang}${suffix}] ${file.symbols.length} symbols ${file.refs.length} refs`];
const symbols = [...(file.symbols ?? [])].sort((a, b) => a.line - b.line).slice(0, mode === 'full' ? 12 : 18);
if (!symbols.length) return [`${file.path} [${file.lang}${suffix}] score ${score.toFixed(3)}`];
if (mode === 'names') {
return [`${file.path} [${file.lang}${suffix}] ${symbols.map((s) => `${s.exported ? '+' : ''}${s.name}`).join(', ')}`];
}
return [
`${file.path} [${file.lang}${suffix}] score ${score.toFixed(3)}`,
...symbols.map((s) => ` L${s.line} ${s.exported ? 'export ' : ''}${s.kind} ${s.name}: ${s.signature}`),
];
}
function fits(lines, addition, budget) {
return estimateTokens(`${lines.concat(addition).join('\n')}\n`) <= budget;
}
function globRegex(pattern) {
const normalized = normalizePath(pattern);
let rx = '^';
const parts = normalized.split('/');
for (let i = 0; i < parts.length; i += 1) {
const part = parts[i];
const last = i === parts.length - 1;
if (part === '**') {
rx += last ? '.*' : '(?:[^/]+/)*';
continue;
}
rx += [...part]
.map((ch) => {
if (ch === '*') return '[^/]*';
if (ch === '?') return '[^/]';
return ch.replace(/[|\\{}()[\]^$+?.]/g, '\\$&');
})
.join('');
if (!last) rx += '/';
}
rx += '$';
return new RegExp(rx);
}
function withinDepth(file, depth) {
if (!depth || depth < 1) return true;
return file.split('/').length - 1 <= depth;
}
function isBinary(file) {
try {
const fd = fs.openSync(file, 'r');
const buffer = Buffer.alloc(8192);
const read = fs.readSync(fd, buffer, 0, buffer.length, 0);
fs.closeSync(fd);
return buffer.subarray(0, read).includes(0);
} catch {
return true;
}
}
function safeStat(file) {
try {
return fs.statSync(file);
} catch {
return null;
}
}
function git(args, cwd) {
try {
return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
} catch {
return null;
}
}
function lines(text) {
return text ? text.split(/\r?\n/).filter(Boolean) : [];
}
function loadCache(file) {
try {
const cache = JSON.parse(readIfExists(file) ?? 'null');
return cache?.version === INDEX_VERSION ? cache : null;
} catch {
return null;
}
}
function countFile(stats, file) {
stats.files += 1;
stats.symbols += file.symbols?.length ?? 0;
stats.refs += file.refs?.length ?? 0;
stats.languages[file.lang] = (stats.languages[file.lang] ?? 0) + 1;
}
function countSymbols(index) {
return Object.values(index.files ?? {}).reduce((sum, file) => sum + (file.symbols?.length ?? 0), 0);
}
function hash(source) {
return createHash('sha1').update(source).digest('hex');
}
function arity(signature) {
const args = String(signature).match(/\(([^)]*)\)/)?.[1];
if (args === undefined) return -1;
const trimmed = args.trim();
if (!trimmed) return 0;
return trimmed.split(',').filter(Boolean).length;
}
function inferKind(type) {
if (/class/.test(type)) return 'class';
if (/interface/.test(type)) return 'interface';
if (/struct/.test(type)) return 'struct';
if (/enum/.test(type)) return 'enum';
if (/type/.test(type)) return 'type';
if (/method/.test(type)) return 'method';
if (/function/.test(type)) return 'function';
return 'symbol';
}
function textFor(source, node) {
return source.slice(node.startIndex, node.endIndex);
}
function firstLine(source, node) {
return textFor(source, node).split(/\r?\n/, 1)[0].trim().slice(0, 180);
}
function lastPathPart(value) {
return String(value ?? '').split(/[\/.]/).filter(Boolean).pop() ?? '';
}
function isIdentifierish(name) {
return /^[A-Za-z_$#][\w$#-]*$/.test(name) && name.length <= 120;
}
function normalizePath(value) {
return String(value).replace(/\\/g, '/').replace(/^\.\//, '');
}
function numberOption(value, fallback) {
const n = Number(value);
return Number.isFinite(n) && n > 0 ? n : fallback;
}