114 lines
3.2 KiB
Python
114 lines
3.2 KiB
Python
"""导入资料结构解析（纯函数，无 IO）。
|
|
|
|
按 ``material_type`` / ``format`` 把原始资料切片为条目草稿。只做结构解析
|
|
（消息数组、段落、字段），语义抽取（档案 / 记忆 / 人格候选）留给后续
|
|
模型驱动阶段。领域层禁止 IO，本模块仅产出数据结构。
|
|
"""
|
|
|
|
import json
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
# 5a 不切片的媒体资料（仅存元数据与授权记录，spec §18 仅预留）。
|
|
_NON_TEXT_MATERIAL_TYPES = {"voice_sample"}
|
|
|
|
|
|
@dataclass
|
|
class ParsedItemDraft:
|
|
"""解析产出的条目草稿；落库前由 service 补 sequence / id。"""
|
|
|
|
item_type: str
|
|
role: str
|
|
content: str
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
def parse_import(
|
|
material_type: str, fmt: str, *, content: str | None = None
|
|
) -> list[ParsedItemDraft]:
|
|
"""按格式解析文本资料；媒体资料返回空。"""
|
|
if material_type in _NON_TEXT_MATERIAL_TYPES:
|
|
return []
|
|
|
|
if fmt == "json" and content:
|
|
return _parse_json(content)
|
|
|
|
if content:
|
|
return _parse_paragraphs(content)
|
|
|
|
return []
|
|
|
|
|
|
def _parse_json(content: str) -> list[ParsedItemDraft]:
|
|
try:
|
|
data = json.loads(content)
|
|
except json.JSONDecodeError:
|
|
return _parse_paragraphs(content)
|
|
|
|
if (
|
|
isinstance(data, list)
|
|
and data
|
|
and isinstance(data[0], dict)
|
|
and ("role" in data[0] or "content" in data[0])
|
|
):
|
|
return _parse_messages(data)
|
|
if isinstance(data, dict):
|
|
return _parse_fields(data)
|
|
return _parse_paragraphs(content)
|
|
|
|
|
|
def _parse_messages(messages: list[dict[str, Any]]) -> list[ParsedItemDraft]:
|
|
drafts: list[ParsedItemDraft] = []
|
|
for msg in messages:
|
|
if not isinstance(msg, dict):
|
|
continue
|
|
text = str(msg.get("content", "")).strip()
|
|
if not text:
|
|
continue
|
|
meta = {k: v for k, v in msg.items() if k not in ("role", "content")}
|
|
drafts.append(
|
|
ParsedItemDraft(
|
|
item_type="message",
|
|
role=_normalize_role(str(msg.get("role", "other"))),
|
|
content=text,
|
|
metadata=meta,
|
|
)
|
|
)
|
|
return drafts
|
|
|
|
|
|
def _parse_fields(data: dict[str, Any]) -> list[ParsedItemDraft]:
|
|
drafts: list[ParsedItemDraft] = []
|
|
for key, value in data.items():
|
|
text = str(value).strip()
|
|
if text:
|
|
drafts.append(
|
|
ParsedItemDraft(
|
|
item_type="profile_field",
|
|
role="other",
|
|
content=text,
|
|
metadata={"field": key},
|
|
)
|
|
)
|
|
return drafts
|
|
|
|
|
|
def _parse_paragraphs(text: str) -> list[ParsedItemDraft]:
|
|
drafts: list[ParsedItemDraft] = []
|
|
for chunk in text.split("\n\n"):
|
|
clean = chunk.strip()
|
|
if clean:
|
|
drafts.append(ParsedItemDraft(item_type="paragraph", role="other", content=clean))
|
|
return drafts
|
|
|
|
|
|
def _normalize_role(role: str) -> str:
|
|
lowered = role.lower()
|
|
if lowered in ("user", "human"):
|
|
return "user"
|
|
if lowered in ("assistant", "ai", "bot"):
|
|
return "assistant"
|
|
if lowered == "system":
|
|
return "system"
|
|
return "other"
|