chenxin
c63b75152a
- 天工阁域 domains/tiangong：backstage_tasks（委托包+验收标准+结果证据）；有限分解（parent 链）与受控并行（各自后台 Run）；子任务全终态级联父任务进入待验收；按证据 accept/reject - kernel.start_backstage_run（kind=background_task）：loop 后台分支不占前台席位、与前台对话并行；断言放宽容纳无人物功能任务；结果只经回调回呈任务——无身份写入路径 - 叩心域 domains/kouxin：knocking_candidates（event/time_anchor/manual 触发 + 草稿 + 裁决快照）；四约束裁决（总开关→静默窗口【本地时区判定】→24h 频率→预算），dispatch 未过 429 且快照留痕；候选默认不自动发送；expire_stale 过期推进不触碰缘契 - 辰正 domains/zhengchen：temporal_context() 复用 time_sense 计算核（快照/农历/季节），产出【时态感知】投影段接入镜树生成 prompt（客观时间语境，不附情绪臆测——守一纪律） - 退役：modules/goal_orchestrator|planner|perception|big_cycle、application/planning、services/governance|goal_persist、candidates 表链（imports 候选抽取 API 面）、goals/plans/agent_roster/perception_channels/big_cycles/旧候选表（迁移 009）；前端 admin goals/plans/agents/perception/world 页与六组件；governance 只留 GenerationPreferences；admin 治理面收敛为司命+百工两枢；pivots 健康以域表计数重建 - 验收（test_m8_tiangong 五用例）：后台任务无身份写入（记忆/魂籍/缘契零新增）；后台与前台并行；分解级联；叩心四约束逐项；不回应不降低关系（缘契维度/状态/版本零变化） - 四套件：后端 345、前端 203、BFF 4、神识 5 包全绿（2 个预存环境失败不变）；迁移 009 净库干净
297 lines
9.7 KiB
Python
297 lines
9.7 KiB
Python
"""资料导入服务（api-spec §8）。
|
||
|
||
5a：结构解析（消息数组 / 段落 / 字段）落 ``import_items``，不调模型；媒体资料
|
||
（``voice_sample``）仅存元数据与授权记录（spec §18 仅预留，不落盘不解析）。
|
||
候选抽取（档案 / 记忆 / 人格）与应用留 5b / 5c。
|
||
"""
|
||
|
||
import hashlib
|
||
|
||
from fastapi import UploadFile
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.core.config import settings
|
||
from app.core.errors import AppError
|
||
from app.core.time import utc_now
|
||
from app.db.session import AsyncSessionLocal
|
||
from app.models.imports import CharacterImportJob, ImportItem
|
||
from app.modules.imports import parse_import
|
||
from app.repositories import imports as imports_repo
|
||
from app.repositories.json_utils import dumps, loads_dict
|
||
from app.schemas.common import Page
|
||
from app.schemas.imports import (
|
||
CreateImportRequest,
|
||
ImportItemResponse,
|
||
ImportJobResponse,
|
||
)
|
||
from app.services.audit import record_audit_event
|
||
from app.services.characters import get_owned_character
|
||
from app.services.serializers import (
|
||
import_item_to_response,
|
||
import_job_to_response,
|
||
)
|
||
|
||
_TEXT_MIME_FORMAT: dict[str, str] = {
|
||
"text/plain": "text",
|
||
"text/markdown": "markdown",
|
||
"application/json": "json",
|
||
"text/csv": "csv",
|
||
"application/yaml": "yaml",
|
||
"text/yaml": "yaml",
|
||
}
|
||
_AUDIO_FORMAT: dict[str, str] = {
|
||
"audio/wav": "wav",
|
||
"audio/mpeg": "mp3",
|
||
"audio/mp4": "m4a",
|
||
"audio/x-m4a": "m4a",
|
||
"audio/flac": "flac",
|
||
}
|
||
|
||
|
||
async def create_import_from_json(
|
||
character_id: str,
|
||
payload: CreateImportRequest,
|
||
owner_scope: str = settings.default_owner_scope,
|
||
) -> ImportJobResponse:
|
||
"""JSON 提交导入（粘贴文本 / 结构化消息 / 手动设定）。"""
|
||
async with AsyncSessionLocal() as session:
|
||
character = await get_owned_character(session, character_id, owner_scope=owner_scope)
|
||
job = CharacterImportJob(
|
||
character_id=character.id,
|
||
material_type=payload.material_type,
|
||
source_type="pasted_text",
|
||
title=payload.title,
|
||
status="pending",
|
||
format=payload.format,
|
||
source_description=payload.source_note,
|
||
raw_content=payload.content,
|
||
checksum=_checksum(payload.content),
|
||
)
|
||
await imports_repo.add_job(session, job)
|
||
await _parse_into_items(session, job, payload.content)
|
||
await _audit(session, character.id, "import.created", job.id, payload.material_type)
|
||
await session.commit()
|
||
await session.refresh(job)
|
||
return import_job_to_response(job)
|
||
|
||
|
||
async def create_import_from_upload(
|
||
character_id: str,
|
||
file: UploadFile,
|
||
*,
|
||
title: str | None = None,
|
||
source_description: str | None = None,
|
||
speaker_label: str | None = None,
|
||
consent_status: str = "unknown",
|
||
usage_scope: str = "reference_only",
|
||
owner_scope: str = settings.default_owner_scope,
|
||
) -> ImportJobResponse:
|
||
"""文件上传导入（文本类解析为 items；音频类仅存元数据，不落盘）。"""
|
||
content_bytes = await file.read()
|
||
mime = file.content_type or "application/octet-stream"
|
||
filename = file.filename or "upload"
|
||
|
||
async with AsyncSessionLocal() as session:
|
||
character = await get_owned_character(session, character_id, owner_scope=owner_scope)
|
||
if mime in _AUDIO_FORMAT:
|
||
job = CharacterImportJob(
|
||
character_id=character.id,
|
||
material_type="voice_sample",
|
||
source_type="media_upload",
|
||
title=title,
|
||
status="uploaded",
|
||
format=_AUDIO_FORMAT[mime],
|
||
media_type="audio",
|
||
mime_type=mime,
|
||
source_description=source_description,
|
||
speaker_label=speaker_label,
|
||
consent_status=consent_status,
|
||
usage_scope=usage_scope,
|
||
checksum=_checksum_bytes(content_bytes),
|
||
)
|
||
await imports_repo.add_job(session, job)
|
||
else:
|
||
try:
|
||
content = content_bytes.decode("utf-8")
|
||
except UnicodeDecodeError as exc:
|
||
raise AppError("unsupported_media_type", "仅支持 UTF-8 文本文件。") from exc
|
||
job = CharacterImportJob(
|
||
character_id=character.id,
|
||
material_type=_infer_material_type(filename, mime),
|
||
source_type="uploaded_file",
|
||
title=title or filename,
|
||
status="pending",
|
||
format=_TEXT_MIME_FORMAT.get(mime, "text"),
|
||
mime_type=mime,
|
||
source_description=source_description,
|
||
raw_content=content,
|
||
checksum=_checksum_bytes(content_bytes),
|
||
)
|
||
await imports_repo.add_job(session, job)
|
||
await _parse_into_items(session, job, content)
|
||
await _audit(session, character.id, "import.created", job.id, job.material_type)
|
||
await session.commit()
|
||
await session.refresh(job)
|
||
return import_job_to_response(job)
|
||
|
||
|
||
async def get_import(
|
||
character_id: str,
|
||
import_id: str,
|
||
owner_scope: str = settings.default_owner_scope,
|
||
) -> ImportJobResponse:
|
||
async with AsyncSessionLocal() as session:
|
||
character = await get_owned_character(session, character_id, owner_scope=owner_scope)
|
||
job = await imports_repo.get_job(session, import_id)
|
||
if job is None or job.character_id != character.id:
|
||
raise AppError("not_found", "导入任务不存在。")
|
||
return import_job_to_response(job)
|
||
|
||
|
||
async def list_jobs_page(
|
||
character_id: str,
|
||
*,
|
||
page: int = 1,
|
||
page_size: int = 20,
|
||
owner_scope: str = settings.default_owner_scope,
|
||
) -> Page[ImportJobResponse]:
|
||
async with AsyncSessionLocal() as session:
|
||
character = await get_owned_character(session, character_id, owner_scope=owner_scope)
|
||
safe_page = max(page, 1)
|
||
safe_size = min(max(page_size, 1), 100)
|
||
items, total = await imports_repo.list_jobs(
|
||
session, character.id, page=safe_page, page_size=safe_size
|
||
)
|
||
return Page(
|
||
items=[import_job_to_response(item) for item in items],
|
||
page=safe_page,
|
||
page_size=safe_size,
|
||
total=total,
|
||
)
|
||
|
||
|
||
async def list_import_items(
|
||
character_id: str,
|
||
import_id: str,
|
||
*,
|
||
page: int = 1,
|
||
page_size: int = 20,
|
||
owner_scope: str = settings.default_owner_scope,
|
||
) -> Page[ImportItemResponse]:
|
||
async with AsyncSessionLocal() as session:
|
||
character = await get_owned_character(session, character_id, owner_scope=owner_scope)
|
||
job = await imports_repo.get_job(session, import_id)
|
||
if job is None or job.character_id != character.id:
|
||
raise AppError("not_found", "导入任务不存在。")
|
||
safe_page = max(page, 1)
|
||
safe_size = min(max(page_size, 1), 100)
|
||
items, total = await imports_repo.list_items(
|
||
session, import_id, page=safe_page, page_size=safe_size
|
||
)
|
||
return Page(
|
||
items=[import_item_to_response(item) for item in items],
|
||
page=safe_page,
|
||
page_size=safe_size,
|
||
total=total,
|
||
)
|
||
|
||
|
||
async def _parse_into_items(session: AsyncSession, job: CharacterImportJob, content: str) -> None:
|
||
"""结构解析为 import_items 并回填 job 状态；纯结构，不调模型。"""
|
||
job.status = "parsing"
|
||
await session.flush()
|
||
drafts = parse_import(job.material_type, job.format, content=content)
|
||
items = [
|
||
ImportItem(
|
||
import_job_id=job.id,
|
||
character_id=job.character_id,
|
||
item_type=draft.item_type,
|
||
role=draft.role,
|
||
sequence=idx,
|
||
content=draft.content,
|
||
metadata_json=dumps(draft.metadata),
|
||
)
|
||
for idx, draft in enumerate(drafts)
|
||
]
|
||
await imports_repo.add_items(session, items)
|
||
job.item_count = len(items)
|
||
job.status = "parsed" if items else "uploaded"
|
||
|
||
|
||
async def _audit(
|
||
session: AsyncSession,
|
||
character_id: str,
|
||
action: str,
|
||
resource_id: str,
|
||
summary: str,
|
||
) -> None:
|
||
await record_audit_event(
|
||
session,
|
||
action=action,
|
||
character_id=character_id,
|
||
actor="user",
|
||
decision="auto",
|
||
input_summary=summary,
|
||
resource_type="import_job",
|
||
resource_id=resource_id,
|
||
)
|
||
|
||
|
||
def _checksum(text: str) -> str:
|
||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def _checksum_bytes(data: bytes) -> str:
|
||
return hashlib.sha256(data).hexdigest()
|
||
|
||
|
||
def _infer_material_type(filename: str, mime: str) -> str:
|
||
name = filename.lower()
|
||
if "chat" in name or "history" in name:
|
||
return "chat_history"
|
||
if mime == "application/json":
|
||
return "other"
|
||
return "background_story"
|
||
|
||
|
||
# —— 候选抽取触发与列表（5b，异步后台）——
|
||
|
||
_EXTRACTABLE_STATUSES = frozenset({"parsed", "uploaded", "reviewing"})
|
||
|
||
|
||
async def _get_owned_job(
|
||
session: AsyncSession, character_id: str, import_id: str
|
||
) -> CharacterImportJob:
|
||
job = await imports_repo.get_job(session, import_id)
|
||
if job is None or job.character_id != character_id:
|
||
raise AppError("not_found", "导入任务不存在。")
|
||
return job
|
||
|
||
|
||
async def _reload_job(import_id: str) -> ImportJobResponse:
|
||
async with AsyncSessionLocal() as session:
|
||
job = await imports_repo.get_job(session, import_id)
|
||
return import_job_to_response(job)
|
||
|
||
|
||
|
||
|
||
async def trigger_personality_extraction(
|
||
character_id: str,
|
||
import_id: str,
|
||
owner_scope: str = settings.default_owner_scope,
|
||
) -> ImportJobResponse:
|
||
return await _trigger_extraction(
|
||
character_id,
|
||
import_id,
|
||
"generating_personality_proposals",
|
||
"personality",
|
||
owner_scope=owner_scope,
|
||
)
|
||
|
||
|
||
|
||
|
||
|
||
|