mirror of
https://github.com/volcengine/OpenViking.git
synced 2026-09-28 11:43:00 +08:00
fix(plugins): rank skill hits like memory leaves in the recall fallbacks
Naming a skill hit by its directory instead of its .abstract.md cost it the 0.12 leaf boost, since the boost keyed on a ".md" URI, so a skill that main would recall lost to ten slightly weaker memory leaves. In Codex, skills were also never picked while enough memory leaves passed the threshold (leaves are picked first), and a hit the server labeled with another category lost its "skills" label. Skill hits now count as leaves in ranking and picking in both recall-core and Codex, and Codex always labels them "skills". Claude-Session: https://claude.ai/code/session_01ECVkFufAU2LKe4g83cxt28
This commit is contained in:
@@ -257,7 +257,8 @@ function rankItem(item, profile) {
|
||||
const abstract = (item.abstract || item.overview || "").trim();
|
||||
const cat = (item.category || "").toLowerCase();
|
||||
const uri = (item.uri || "").toLowerCase();
|
||||
const leafBoost = (item.level === 2 || uri.endsWith(".md")) ? 0.12 : 0;
|
||||
// A skill hit names its directory, but it is as complete a unit as a memory leaf.
|
||||
const leafBoost = (item.level === 2 || item._sourceType === "skill" || uri.endsWith(".md")) ? 0.12 : 0;
|
||||
const eventBoost = profile.wantsTemporal && (cat === "events" || uri.includes("/events/")) ? 0.1 : 0;
|
||||
const prefBoost = profile.wantsPreference && (cat === "preferences" || uri.includes("/preferences/")) ? 0.08 : 0;
|
||||
const overlapBoost = lexicalOverlapBoost(profile.tokens, `${item.uri} ${abstract}`);
|
||||
|
||||
@@ -150,12 +150,17 @@ function lexicalOverlapBoost(tokens, text) {
|
||||
return Math.min(0.2, (matched / Math.min(tokens.length, 4)) * 0.2);
|
||||
}
|
||||
|
||||
// A skill hit names its directory, but it is as complete a unit as a memory leaf.
|
||||
function isLeafHit(item) {
|
||||
return item.level === 2 || item.category === "skills" || String(item.uri || "").endsWith(".md");
|
||||
}
|
||||
|
||||
function getRankingBreakdown(item, profile) {
|
||||
const base = clampScore(item.score);
|
||||
const abstract = (item.abstract || item.overview || "").trim();
|
||||
const cat = (item.category || "").toLowerCase();
|
||||
const uri = item.uri.toLowerCase();
|
||||
const leafBoost = (item.level === 2 || uri.endsWith(".md")) ? 0.12 : 0;
|
||||
const leafBoost = isLeafHit(item) ? 0.12 : 0;
|
||||
const eventBoost = profile.wantsTemporal && (cat === "events" || uri.includes("/events/")) ? 0.1 : 0;
|
||||
const prefBoost = profile.wantsPreference && (cat === "preferences" || uri.includes("/preferences/")) ? 0.08 : 0;
|
||||
const overlapBoost = lexicalOverlapBoost(profile.tokens, `${item.uri} ${abstract}`);
|
||||
@@ -188,7 +193,7 @@ function pickMemories(items, limit, queryText) {
|
||||
const profile = buildQueryProfile(queryText);
|
||||
const sorted = [...items].sort((a, b) => rankForInjection(b, profile) - rankForInjection(a, profile));
|
||||
const deduped = dedupeByAbstract(sorted);
|
||||
const leaves = deduped.filter((m) => m.level === 2 || m.uri.endsWith(".md"));
|
||||
const leaves = deduped.filter(isLeafHit);
|
||||
if (leaves.length >= limit) return leaves.slice(0, limit);
|
||||
const picked = [...leaves];
|
||||
const used = new Set(picked.map((m) => m.uri));
|
||||
@@ -277,7 +282,7 @@ async function searchAll(query, limit, sessionId = null) {
|
||||
const skills = [...userSkills, ...sharedSkills].map((m) => ({
|
||||
...m,
|
||||
uri: skillHitUri(m.uri),
|
||||
category: m.category || "skills",
|
||||
category: "skills",
|
||||
}));
|
||||
const all = [...userMems, ...skills];
|
||||
const seen = new Set();
|
||||
|
||||
@@ -749,6 +749,84 @@ test("the raw-search fallback also searches shared skills and flags them", async
|
||||
}
|
||||
});
|
||||
|
||||
test("the raw-search fallback keeps a strong skill hit among many memory leaves", async () => {
|
||||
const stateDir = await mkdtemp(join(tmpdir(), "ov-auto-recall-skill-vs-leaves-"));
|
||||
|
||||
try {
|
||||
await withMockOpenViking(async (req, res) => {
|
||||
const url = new URL(req.url, "http://127.0.0.1");
|
||||
if (req.method === "GET" && url.pathname === "/health") {
|
||||
writeJson(res, { status: "ok", result: { ok: true } });
|
||||
return;
|
||||
}
|
||||
if (req.method === "POST" && url.pathname === "/api/v1/search/search") {
|
||||
const body = await readRequestBody(req);
|
||||
if (body.mode === "context") {
|
||||
writeStatusJson(res, 400, { status: "error", error: "Extra inputs are not permitted: mode" });
|
||||
return;
|
||||
}
|
||||
if (body.target_uri === "viking://~/memories") {
|
||||
writeJson(res, {
|
||||
status: "ok",
|
||||
result: {
|
||||
memories: Array.from({ length: 12 }, (_, i) => ({
|
||||
uri: `viking://user/zeus/memories/events/deploy-${i}.md`,
|
||||
level: 2,
|
||||
score: 0.6,
|
||||
category: "events",
|
||||
abstract: `deploy note ${i}`,
|
||||
})),
|
||||
skills: [],
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
const skills = body.target_uri === "viking://agent/skills"
|
||||
? [{
|
||||
uri: "viking://agent/skills/deploy-runbook/.abstract.md",
|
||||
level: 0,
|
||||
score: 0.95,
|
||||
// The server may label a skill hit with any category; it is still a skill.
|
||||
category: "events",
|
||||
abstract: "name: deploy-runbook description: Shared deployment runbook",
|
||||
}]
|
||||
: [];
|
||||
writeJson(res, { status: "ok", result: { memories: [], skills } });
|
||||
return;
|
||||
}
|
||||
if (req.method === "GET" && url.pathname === "/api/v1/content/read") {
|
||||
writeJson(res, { status: "ok", result: "deploy note detail" });
|
||||
return;
|
||||
}
|
||||
res.writeHead(404, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ status: "error", error: "not found" }));
|
||||
}, async (baseUrl) => {
|
||||
const result = await runAutoRecall(
|
||||
{ prompt: "how do we roll back the payments deploy", session_id: "codex:790" },
|
||||
{
|
||||
OPENVIKING_AUTO_RECALL: "1",
|
||||
OPENVIKING_CODEX_STATE_DIR: stateDir,
|
||||
OPENVIKING_STATE_DIR: stateDir,
|
||||
OPENVIKING_CONFIG_FILE: join(stateDir, "missing-ov.conf"),
|
||||
OPENVIKING_CLI_CONFIG_FILE: join(stateDir, "missing-ovcli.conf"),
|
||||
OPENVIKING_CREDENTIAL_SOURCE: "env",
|
||||
OPENVIKING_RECALL_COMPRESS: "0",
|
||||
OPENVIKING_RECALL_TIMEOUT_MS: "10000",
|
||||
OPENVIKING_MIN_QUERY_LENGTH: "1",
|
||||
OPENVIKING_SCORE_THRESHOLD: "0",
|
||||
OPENVIKING_TIMEOUT_MS: "5000",
|
||||
OPENVIKING_URL: baseUrl,
|
||||
},
|
||||
);
|
||||
|
||||
const context = JSON.parse(result.stdout.trim()).hookSpecificOutput.additionalContext;
|
||||
assert.match(context, /\[skills\] name: deploy-runbook .*\(viking:\/\/agent\/skills\/deploy-runbook\)/);
|
||||
});
|
||||
} finally {
|
||||
await rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("auto-recall preserves explicit default user memory target", async () => {
|
||||
const stateDir = await mkdtemp(join(tmpdir(), "ov-auto-recall-default-user-"));
|
||||
const requests = [];
|
||||
|
||||
+2
-1
@@ -257,7 +257,8 @@ function rankItem(item, profile) {
|
||||
const abstract = (item.abstract || item.overview || "").trim();
|
||||
const cat = (item.category || "").toLowerCase();
|
||||
const uri = (item.uri || "").toLowerCase();
|
||||
const leafBoost = (item.level === 2 || uri.endsWith(".md")) ? 0.12 : 0;
|
||||
// A skill hit names its directory, but it is as complete a unit as a memory leaf.
|
||||
const leafBoost = (item.level === 2 || item._sourceType === "skill" || uri.endsWith(".md")) ? 0.12 : 0;
|
||||
const eventBoost = profile.wantsTemporal && (cat === "events" || uri.includes("/events/")) ? 0.1 : 0;
|
||||
const prefBoost = profile.wantsPreference && (cat === "preferences" || uri.includes("/preferences/")) ? 0.08 : 0;
|
||||
const overlapBoost = lexicalOverlapBoost(profile.tokens, `${item.uri} ${abstract}`);
|
||||
|
||||
@@ -256,7 +256,8 @@ function rankItem(item, profile) {
|
||||
const abstract = (item.abstract || item.overview || "").trim();
|
||||
const cat = (item.category || "").toLowerCase();
|
||||
const uri = (item.uri || "").toLowerCase();
|
||||
const leafBoost = (item.level === 2 || uri.endsWith(".md")) ? 0.12 : 0;
|
||||
// A skill hit names its directory, but it is as complete a unit as a memory leaf.
|
||||
const leafBoost = (item.level === 2 || item._sourceType === "skill" || uri.endsWith(".md")) ? 0.12 : 0;
|
||||
const eventBoost = profile.wantsTemporal && (cat === "events" || uri.includes("/events/")) ? 0.1 : 0;
|
||||
const prefBoost = profile.wantsPreference && (cat === "preferences" || uri.includes("/preferences/")) ? 0.08 : 0;
|
||||
const overlapBoost = lexicalOverlapBoost(profile.tokens, `${item.uri} ${abstract}`);
|
||||
|
||||
@@ -519,3 +519,37 @@ test("the skill hint appears only when the assembled context carries a skill", a
|
||||
assert.equal(/Skill entries are OpenViking skills/.test(block), expected, rendered);
|
||||
}
|
||||
});
|
||||
|
||||
test("a skill hit competes with memory leaves on equal footing in the fallback", async () => {
|
||||
const legacyCachePath = await tempPath("context-face.json");
|
||||
const fetchJSON = async (path, init) => {
|
||||
if (path === "/api/v1/search/search") return { ok: false, status: 503 };
|
||||
if (path === "/api/v1/search/recall") return { ok: false, status: 404 };
|
||||
if (path === "/api/v1/search/find") {
|
||||
const body = JSON.parse(init.body);
|
||||
if (body.target_uri === "viking://~/memories") {
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
memories: Array.from({ length: 10 }, (_, i) => ({
|
||||
uri: `viking://user/alice/memories/events/e${i}.md`, score: 0.62, abstract: `event ${i}`, level: 2,
|
||||
})),
|
||||
skills: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
if (body.target_uri === "viking://~/skills") {
|
||||
return { ok: true, result: { memories: [], skills: [{ uri: "viking://user/alice/skills/pr-review/.abstract.md", score: 0.7, abstract: "name: pr-review", level: 0 }] } };
|
||||
}
|
||||
return { ok: true, result: { memories: [], skills: [] } };
|
||||
}
|
||||
return { ok: false, status: 404 };
|
||||
};
|
||||
const events = [];
|
||||
await buildRecallBlock(fetchJSON, { recallLimit: 10, recallPreferAbstract: true, scoreThreshold: 0.35 }, "review this pull request", {
|
||||
legacyCachePath,
|
||||
log: (event, data) => events.push({ event, data }),
|
||||
});
|
||||
const picked = events.find((e) => e.event === "recall_picked").data.items.map((item) => item.uri);
|
||||
assert.ok(picked.includes("viking://user/alice/skills/pr-review"), picked.join("\n"));
|
||||
});
|
||||
|
||||
@@ -257,7 +257,8 @@ function rankItem(item, profile) {
|
||||
const abstract = (item.abstract || item.overview || "").trim();
|
||||
const cat = (item.category || "").toLowerCase();
|
||||
const uri = (item.uri || "").toLowerCase();
|
||||
const leafBoost = (item.level === 2 || uri.endsWith(".md")) ? 0.12 : 0;
|
||||
// A skill hit names its directory, but it is as complete a unit as a memory leaf.
|
||||
const leafBoost = (item.level === 2 || item._sourceType === "skill" || uri.endsWith(".md")) ? 0.12 : 0;
|
||||
const eventBoost = profile.wantsTemporal && (cat === "events" || uri.includes("/events/")) ? 0.1 : 0;
|
||||
const prefBoost = profile.wantsPreference && (cat === "preferences" || uri.includes("/preferences/")) ? 0.08 : 0;
|
||||
const overlapBoost = lexicalOverlapBoost(profile.tokens, `${item.uri} ${abstract}`);
|
||||
|
||||
Reference in New Issue
Block a user