zzos/Octop
1
0
Fork 0
mirror of https://github.com/TencentCloud/Octop.git synced 2026-09-28 11:23:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(knowledge): 支持按知识库配置文档数量上限 (#427)

Browse Source 
- schema v10: knowledge_bases 新增 max_documents 列（默认 100，0=不限制，上限 10000）
- 上传时按 KB 自身上限在 SQL 事务内原子拦截，第 N+1 个文档被拒绝
- 前端编辑模态框在"描述"与"图标"之间新增"文档数量上限"输入项，保存即同步
- 详情面板"X/Y 文档"文本、上限 Alert、上传按钮均改用该 KB 自身上限
- SQLite 通过启动兜底加列，PostgreSQL 使用 ADD COLUMN IF NOT EXISTS 幂等迁移
This commit is contained in:
chujieHong
2026-08-27 15:38:05 +08:00
committed by GitHub
parent a64c6403bd
commit 90b8ca2560
13 changed files with 249 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dashboard/src/api/modules/knowledgeBases.ts
+3
View File
@@ -37,6 +37,7 @@ export interface KnowledgeBase {
embedding_model: string;
embedding_dim: number;
doc_count: number;
max_documents: number;
created_at: number;
updated_at: number;
}
@@ -151,6 +152,7 @@ export const knowledgeBasesApi = {
default_open?: boolean;
shared?: boolean;
icon_name?: string;
max_documents?: number;
}) =>
request<KnowledgeBase>("/knowledge-bases", {
method: "POST",
@@ -165,6 +167,7 @@ export const knowledgeBasesApi = {
default_open?: boolean;
shared?: boolean;
icon_name?: string;
max_documents?: number;
},
) =>
request<KnowledgeBase>(`/knowledge-bases/${id}`, {
dashboard/src/locales/en.json
+2
View File
@@ -322,6 +322,8 @@
"documents": "Documents",
"documentLimit": "{{count}} / {{max}} documents",
"documentLimitReached": "This knowledge base already contains the maximum of {{count}} documents.",
"maxDocuments": "Document limit",
"maxDocumentsHint": "Maximum number of files in this knowledge base. 0 = unlimited, default 100.",
"documentTooLarge": "Each document must be at most {{sizeMb}} MB.",
"baseLimitReached": "You can create at most {{count}} knowledge bases.",
"uploadHint": "Supports md / txt / pdf / docx / pptx. Max {{sizeMb}} MB per file.",
dashboard/src/locales/zh.json
+2
View File
@@ -322,6 +322,8 @@
"documents": "文档",
"documentLimit": "{{count}} / {{max}} 个文档",
"documentLimitReached": "此知识库已达到 {{count}} 个文档的上限。",
"maxDocuments": "文档数量上限",
"maxDocumentsHint": "本知识库可容纳的最大文档数，0 表示不限制，默认为 100。",
"documentTooLarge": "单个文档不能超过 {{sizeMb}} MB。",
"baseLimitReached": "每个用户最多可创建 {{count}} 个知识库。",
"uploadHint": "支持 md / txt / pdf / docx / pptx，单文件不超过 {{sizeMb}} MB。",
dashboard/src/pages/KnowledgeBases/index.tsx
+25 -4
View File
@@ -13,6 +13,7 @@ import {
Empty,
Form,
Input,
InputNumber,
List,
Modal,
Popconfirm,
@@ -94,6 +95,7 @@ type BaseFormValues = {
name: string;
description?: string;
icon_name?: string;
max_documents?: number;
};
type DocsViewMode = "card" | "table";
@@ -328,7 +330,8 @@ export default function KnowledgeBasesPage() {
: 0;
const atBaseLimit = ownedBaseCount >= limits.max_bases_per_owner;
const fileCount = documents.filter((document) => !document.is_dir).length;
const isAtDocumentLimit = fileCount >= limits.max_docs_per_kb;
const isAtDocumentLimit =
fileCount >= (selected?.max_documents ?? limits.max_docs_per_kb);
const folderEntries = documents
.filter((document) =>
isDirectKnowledgeChild(document.path || document.filename, currentFolder),
@@ -599,6 +602,7 @@ export default function KnowledgeBasesPage() {
name: "",
description: "",
icon_name: "book-open",
max_documents: 100,
});
setDefaultOpenChecked(false);
setSharedChecked(false);
@@ -612,6 +616,7 @@ export default function KnowledgeBasesPage() {
name: selected.name,
description: selected.description,
icon_name: selected.icon_name || undefined,
max_documents: selected.max_documents,
});
setDefaultOpenChecked(selected.default_open);
setSharedChecked(selected.shared);
@@ -814,7 +819,10 @@ export default function KnowledgeBasesPage() {
const uploadDocuments = async (files: FileList | null) => {
if (!selected || !files || !usable || isAtDocumentLimit) return;
const remaining = Math.max(0, limits.max_docs_per_kb - fileCount);
const remaining = Math.max(
0,
(selected?.max_documents ?? limits.max_docs_per_kb) - fileCount,
);
const chosen = Array.from(files).slice(0, remaining);
const oversized = chosen.filter(
(file) => file.size > limits.max_document_bytes,
@@ -1477,7 +1485,7 @@ export default function KnowledgeBasesPage() {
<span className={skillStyles.gridCount}>
{t("knowledgeBases.documentLimit", {
count: fileCount,
max: limits.max_docs_per_kb,
max: selected?.max_documents ?? limits.max_docs_per_kb,
})}
</span>
<div className={skillStyles.gridToolbarRight}>
@@ -1608,7 +1616,7 @@ export default function KnowledgeBasesPage() {
type="info"
showIcon
message={t("knowledgeBases.documentLimitReached", {
count: limits.max_docs_per_kb,
count: selected?.max_documents ?? limits.max_docs_per_kb,
})}
/>
) : null}
@@ -1938,6 +1946,19 @@ export default function KnowledgeBasesPage() {
showCount
/>
</Form.Item>
<Form.Item
name="max_documents"
label={t("knowledgeBases.maxDocuments")}
extra={t("knowledgeBases.maxDocumentsHint")}
>
<InputNumber
min={0}
max={10000}
step={1}
precision={0}
style={{ width: "100%" }}
/>
</Form.Item>
<Form.Item name="icon_name" label={t("knowledgeBases.icon")}>
<KnowledgeIconPicker />
</Form.Item>
src/octop/api/routers/knowledge_bases.py
+14
View File
@@ -69,6 +69,12 @@ class CreateBaseBody(BaseModel):
default_open: bool = False
shared: bool = False
icon_name: str = Field(default="", max_length=64)
max_documents: int | None = Field(
default=None,
ge=0,
le=10_000,
description="Per-base document limit. 0 = unlimited, default 100.",
)
class CreateFolderBody(BaseModel):
@@ -98,6 +104,12 @@ class UpdateBaseBody(BaseModel):
default_open: bool | None = None
shared: bool | None = None
icon_name: str | None = Field(default=None, max_length=64)
max_documents: int | None = Field(
default=None,
ge=0,
le=10_000,
description="Per-base document limit. 0 = unlimited.",
)
class RenameDocumentBody(BaseModel):
@@ -418,6 +430,7 @@ async def create_base(
default_open=body.default_open,
shared=body.shared,
icon_name=body.icon_name.strip(),
max_documents=body.max_documents if body.max_documents is not None else MAX_DOCS_PER_KB,
)
return _base_payload(server, base)
except Exception as exc:
@@ -480,6 +493,7 @@ async def update_base(
default_open=body.default_open,
shared=body.shared,
icon_name=body.icon_name.strip() if body.icon_name is not None else None,
max_documents=body.max_documents,
is_admin=_is_admin(user),
),
)
src/octop/infra/db/migrate.py
+2
View File
@@ -637,6 +637,8 @@ def _ensure_knowledge_bases_schema(db: DatabasePool) -> None:
"""Create or rebuild knowledge tables to the integer-PK identity schema."""
_rebuild_knowledge_identity_schema(db)
_drop_knowledge_base_members(db)
# Schema v10: per-knowledge-base configurable document limit.
_ensure_column(db, "knowledge_bases", "max_documents", "INTEGER NOT NULL DEFAULT 100")
def _ensure_sso_oidc_schema(db: DatabasePool) -> None:
src/octop/infra/db/migrations/010_kb_max_documents.pg.sql
+7
View File
@@ -0,0 +1,7 @@
-- Schema v10: per-knowledge-base configurable document limit.
-- PostgreSQL: ADD COLUMN IF NOT EXISTS is safe (skips if already present).
ALTER TABLE knowledge_bases
ADD COLUMN IF NOT EXISTS max_documents INTEGER NOT NULL DEFAULT 100;
UPDATE _schema_version SET version = 10;
src/octop/infra/db/migrations/010_kb_max_documents.sql
+6
View File
@@ -0,0 +1,6 @@
-- Schema v10: per-knowledge-base configurable document limit.
-- The column is added by migrate.py::_ensure_knowledge_bases_schema so that
-- boot-time repair covers pre-v10 databases. This file only bumps _schema_version.
-- 100 is the previous system-wide default; 0 means unlimited.
UPDATE _schema_version SET version = 10;
src/octop/infra/db/repos/knowledge.py
+58 -21
View File
@@ -31,11 +31,16 @@ class KnowledgeBaseRow:
embedding_model: str
embedding_dim: int
doc_count: int
max_documents: int
created_at: int
updated_at: int
@classmethod
def from_row(cls, r: DbRow) -> KnowledgeBaseRow:
# Schema v10 adds max_documents. Fall back to 100 for pre-v10 DBs.
# sqlite3.Row has no __contains__; use keys() (like users.py).
keys = frozenset(r.keys()) if hasattr(r, "keys") else frozenset()
max_doc = int(r["max_documents"]) if "max_documents" in keys else 100
return cls(
id=str(r["knowledge_base_id"]),
pk=int(r["id"]),
@@ -48,6 +53,7 @@ class KnowledgeBaseRow:
embedding_model=r["embedding_model"],
embedding_dim=r["embedding_dim"],
doc_count=r["doc_count"],
max_documents=max_doc,
created_at=r["created_at"],
updated_at=r["updated_at"],
)
@@ -112,29 +118,54 @@ class KnowledgeRepo:
icon_name: str = "",
embedding_model: str = "",
embedding_dim: int = 0,
max_documents: int | None = None,
) -> KnowledgeBaseRow:
kb_id = self._allocate_base_id()
ts = now_ts()
with self._db.transaction() as conn:
conn.execute(
"INSERT INTO knowledge_bases("
"knowledge_base_id, owner_user_id, name, description, default_open, shared, "
"icon_name, embedding_model, embedding_dim, doc_count, created_at, updated_at"
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?)",
(
kb_id,
owner_user_id,
name,
description,
bool_int(default_open),
bool_int(shared),
icon_name,
embedding_model,
embedding_dim,
ts,
ts,
),
)
if max_documents is None:
# Rely on the column DEFAULT (100) for max_documents.
conn.execute(
"INSERT INTO knowledge_bases("
"knowledge_base_id, owner_user_id, name, description, default_open, shared, "
"icon_name, embedding_model, embedding_dim, doc_count, created_at, updated_at"
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?)",
(
kb_id,
owner_user_id,
name,
description,
bool_int(default_open),
bool_int(shared),
icon_name,
embedding_model,
embedding_dim,
ts,
ts,
),
)
else:
conn.execute(
"INSERT INTO knowledge_bases("
"knowledge_base_id, owner_user_id, name, description, default_open, shared, "
"icon_name, embedding_model, embedding_dim, doc_count, max_documents, "
"created_at, updated_at"
") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?)",
(
kb_id,
owner_user_id,
name,
description,
bool_int(default_open),
bool_int(shared),
icon_name,
embedding_model,
embedding_dim,
max_documents,
ts,
ts,
),
)
row = self.get_base(kb_id)
if row is None:
raise RuntimeError(f"knowledge base insert failed: {kb_id}")
@@ -183,6 +214,7 @@ class KnowledgeRepo:
embedding_model: str | None = None,
embedding_dim: int | None = None,
doc_count: int | None = None,
max_documents: int | None = None,
) -> None:
fields, params = partial_updates(
[
@@ -194,6 +226,7 @@ class KnowledgeRepo:
("embedding_model", embedding_model),
("embedding_dim", embedding_dim),
("doc_count", doc_count),
("max_documents", max_documents),
]
)
if not fields:
@@ -264,8 +297,12 @@ class KnowledgeRepo:
self.ensure_folder(kb_id, folder)
doc_id = new_ulid()
ts = now_ts()
# Treat both None (caller did not specify) and 0 (per-base "unlimited"
# sentinel) as unbounded. Otherwise 0 would be enforced literally as
# "at most 0 documents" and reject every create.
enforce_limit = max_documents is not None and max_documents > 0
with self._db.transaction() as conn:
if max_documents is not None:
if enforce_limit:
cursor = conn.execute(
"UPDATE knowledge_bases SET doc_count = doc_count + 1, updated_at = ? "
"WHERE knowledge_base_id = ? AND doc_count < ?",
@@ -280,7 +317,7 @@ class KnowledgeRepo:
") VALUES (?, ?, ?, ?, 0, ?, ?, ?, ?, '', 0, ?, ?)",
(doc_id, kb_id, rel, name, content_type, byte_size, content_hash, status, ts, ts),
)
if max_documents is None:
if not enforce_limit:
conn.execute(
"UPDATE knowledge_bases SET doc_count = doc_count + 1, updated_at = ? "
"WHERE knowledge_base_id = ?",
src/octop/infra/knowledge/service.py
+15 -2
View File
@@ -24,6 +24,8 @@ from octop.infra.knowledge.relpath import normalize_kb_path, path_basename, path
MAX_DOCS_PER_KB = 100
MAX_BASES_PER_OWNER = 20
MAX_DOCUMENT_BYTES = upload_mb_to_bytes(DEFAULT_MAX_UPLOAD_MB)
# Upper bound for the per-base max_documents field. Mirrors Field(le=10000).
MAX_KB_MAX_DOCUMENTS = 10_000
_MAX_PREVIEW_CHARS = 200_000
_EXT_TO_CONTENT_TYPE = {
".txt": "text/plain",
@@ -73,10 +75,13 @@ class KnowledgeService:
default_open: bool = False,
shared: bool = False,
icon_name: str = "",
max_documents: int = MAX_DOCS_PER_KB,
) -> KnowledgeBaseRow:
assert_knowledge_usable(
self._services.settings_repo.get, getattr(self._services, "provider_repo", None)
)
if max_documents < 0 or max_documents > MAX_KB_MAX_DOCUMENTS:
raise ValueError(f"max_documents must be between 0 and {MAX_KB_MAX_DOCUMENTS}")
owned = self._repo.count_bases_for_owner(owner_user_id)
if owned >= MAX_BASES_PER_OWNER:
raise ValueError(f"a user can own at most {MAX_BASES_PER_OWNER} knowledge bases")
@@ -91,6 +96,7 @@ class KnowledgeService:
shared=shared,
icon_name=icon_name,
embedding_model=model,
max_documents=max_documents,
),
)
@@ -111,9 +117,14 @@ class KnowledgeService:
default_open: bool | None = None,
shared: bool | None = None,
icon_name: str | None = None,
max_documents: int | None = None,
is_admin: bool = False,
) -> KnowledgeBaseRow:
self.require_owner(kb_id, actor_user_id=actor_user_id, is_admin=is_admin)
if max_documents is not None and (
max_documents < 0 or max_documents > MAX_KB_MAX_DOCUMENTS
):
raise ValueError(f"max_documents must be between 0 and {MAX_KB_MAX_DOCUMENTS}")
self._repo.update_base(
kb_id,
name=name,
@@ -121,6 +132,7 @@ class KnowledgeService:
default_open=default_open,
shared=shared,
icon_name=icon_name,
max_documents=max_documents,
)
return self._require_base(kb_id)
@@ -294,7 +306,7 @@ class KnowledgeService:
assert_knowledge_usable(
self._services.settings_repo.get, getattr(self._services, "provider_repo", None)
)
self.get_writable_base(kb_id, actor_user_id=actor_user_id, is_admin=is_admin)
base = self.get_writable_base(kb_id, actor_user_id=actor_user_id, is_admin=is_admin)
limit = self._max_document_bytes()
if len(content) > limit:
raise ValueError(f"knowledge document size exceeds maximum of {limit} bytes")
@@ -305,13 +317,14 @@ class KnowledgeService:
resolved_type = _resolve_content_type(name, content_type)
if resolved_type not in _ALLOWED_CONTENT_TYPES:
raise ValueError(f"unsupported knowledge document content type: {content_type}")
# The per-base limit lives on the KB row (schema v10). 0 = unlimited.
document = self._repo.create_document(
kb_id=kb_id,
filename=name,
path=rel,
content_type=resolved_type,
byte_size=len(content),
max_documents=MAX_DOCS_PER_KB,
max_documents=base.max_documents,
)
try:
write_document(kb_id, document.id, name, content)
tests/unit/api/test_knowledge_bases.py
+41
View File
@@ -39,6 +39,7 @@ class _Base:
embedding_model: str = "model"
embedding_dim: int = 0
doc_count: int = 0
max_documents: int = 100
created_at: int = 1
updated_at: int = 1
@@ -199,6 +200,7 @@ async def test_create_base_uses_selected_model_when_usable(
"default_open": False,
"shared": False,
"icon_name": "",
"max_documents": 100,
}
@@ -550,3 +552,42 @@ async def test_rename_document_maps_invalid_name(
)
assert raised.value.code == ErrorCode.KNOWLEDGE_NAME_INVALID
@pytest.mark.asyncio
async def test_update_base_accepts_max_documents(monkeypatch: pytest.MonkeyPatch) -> None:
from octop.api.routers import knowledge_bases
received: dict[str, object] = {}
class _StubSvc:
def update_base(self, *_: object, **kw: object) -> object:
received.update(kw)
return _Base(max_documents=42)
monkeypatch.setattr(knowledge_bases, "_knowledge_service", lambda _s: _StubSvc())
response = await knowledge_bases.update_base(
kb_id="kb-1",
body=knowledge_bases.UpdateBaseBody(max_documents=42),
request=_request(),
server=SimpleNamespace(services=_services()),
user=SimpleNamespace(id=1, is_admin=False),
)
assert received["max_documents"] == 42
assert response["max_documents"] == 42
@pytest.mark.asyncio
async def test_update_base_rejects_max_documents_out_of_range() -> None:
from pydantic import ValidationError
from octop.api.routers import knowledge_bases
with pytest.raises(ValidationError):
knowledge_bases.UpdateBaseBody(max_documents=-1)
with pytest.raises(ValidationError):
knowledge_bases.UpdateBaseBody(max_documents=10_001)
assert knowledge_bases.UpdateBaseBody(max_documents=0).max_documents == 0
assert knowledge_bases.UpdateBaseBody(max_documents=10_000).max_documents == 10_000
assert knowledge_bases.UpdateBaseBody().max_documents is None
tests/unit/db/test_repo_knowledge.py
+53 -3
View File
@@ -52,9 +52,9 @@ def test_knowledge_tables_migrated(db: SqlitePool) -> None:
}.issubset(names)
assert v == 10
assert "knowledge_base_members" not in names
assert "knowledge_base_id" in {
r["name"] for r in conn.execute("PRAGMA table_info(knowledge_bases)").fetchall()
}
cols = {r["name"] for r in conn.execute("PRAGMA table_info(knowledge_bases)").fetchall()}
assert "knowledge_base_id" in cols
assert "max_documents" in cols
def test_path_layout_knowledge_dir(tmp_path: Path) -> None:
@@ -296,3 +296,53 @@ def test_rename_document_missing_or_wrong_kb_returns_none(
assert repo.rename_document("missing-kb", folder.id, "x") is None
assert repo.rename_document(other.id, folder.id, "x") is None
assert repo.rename_document(kb.id, "missing-doc", "x") is None
def test_update_base_persists_max_documents(repo: KnowledgeRepo, owner_id: int) -> None:
kb = repo.create_base(owner_user_id=owner_id, name="Docs")
assert kb.max_documents == 100
repo.update_base(kb.id, max_documents=42)
refreshed = repo.get_base(kb.id)
assert refreshed is not None
assert refreshed.max_documents == 42
repo.update_base(kb.id, description="unchanged")
assert repo.get_base(kb.id).max_documents == 42 # type: ignore[union-attr]
def test_create_document_uses_kb_max_documents_for_limit(
repo: KnowledgeRepo, owner_id: int
) -> None:
# The repo enforces whatever max_documents the caller passes (the service
# layer forwards kb.max_documents). A cap of 2 must reject the 3rd.
kb = repo.create_base(owner_user_id=owner_id, name="Tiny", max_documents=2)
assert kb.max_documents == 2
for i in range(2):
repo.create_document(
kb_id=kb.id,
filename=f"d{i}.md",
content_type="text/markdown",
byte_size=2,
max_documents=kb.max_documents,
)
with pytest.raises(ValueError, match="at most 2 documents"):
repo.create_document(
kb_id=kb.id,
filename="d2.md",
content_type="text/markdown",
byte_size=2,
max_documents=kb.max_documents,
)
def test_create_document_zero_max_means_unlimited(repo: KnowledgeRepo, owner_id: int) -> None:
kb = repo.create_base(owner_user_id=owner_id, name="Unbounded", max_documents=0)
assert kb.max_documents == 0
for i in range(150):
repo.create_document(
kb_id=kb.id,
filename=f"d{i:03}.md",
content_type="text/markdown",
byte_size=2,
max_documents=kb.max_documents,
)
assert repo.get_base(kb.id).doc_count == 150 # type: ignore[union-attr]
tests/unit/knowledge/test_service_acl.py
+21
View File
@@ -256,3 +256,24 @@ def test_shared_reader_cannot_rename(service: KnowledgeService) -> None:
with pytest.raises(PermissionError, match="write"):
service.rename_document(kb.id, folder.id, actor_user_id=viewer, new_name="b")
def test_update_base_validates_max_documents_range(service: KnowledgeService) -> None:
users = service._services.user_repo
owner = users.create(username="ow", password_hash="h", role="user")
kb = service.create_base(owner_user_id=owner, name="Docs")
# In range
service.update_base(kb.id, actor_user_id=owner, max_documents=0)
assert service.get_readable_base(kb.id, actor_user_id=owner).max_documents == 0
service.update_base(kb.id, actor_user_id=owner, max_documents=500)
assert service.get_readable_base(kb.id, actor_user_id=owner).max_documents == 500
service.update_base(kb.id, actor_user_id=owner, max_documents=10_000)
assert service.get_readable_base(kb.id, actor_user_id=owner).max_documents == 10_000
# Out of range
with pytest.raises(ValueError, match="max_documents must be between"):
service.update_base(kb.id, actor_user_id=owner, max_documents=-1)
with pytest.raises(ValueError, match="max_documents must be between"):
service.update_base(kb.id, actor_user_id=owner, max_documents=10_001)
# Omit keeps value
service.update_base(kb.id, actor_user_id=owner, description="keep")
assert service.get_readable_base(kb.id, actor_user_id=owner).max_documents == 10_000