初始化提交

backend/app/services/imports.py

@@ -0,0 +1,481 @@
+"""资料导入服务（api-spec §8）。
+
+5a：结构解析（消息数组 / 段落 / 字段）落 ``import_items``，不调模型；媒体资料
+（``voice_sample``）仅存元数据与授权记录（spec §18 仅预留，不落盘不解析）。
+候选抽取（档案 / 记忆 / 人格）与应用留 5b / 5c。
+"""
+
+import hashlib
+
+from fastapi import UploadFile
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.config import settings
+from app.core.errors import AppError
+from app.core.time import utc_now
+from app.db.session import AsyncSessionLocal
+from app.models.imports import CharacterImportJob, ImportItem
+from app.models.memory import Memory
+from app.modules.imports import parse_import
+from app.repositories import candidates as candidates_repo
+from app.repositories import imports as imports_repo
+from app.repositories.json_utils import dumps, loads_dict
+from app.schemas.candidates import (
+    BatchReviewRequest,
+    BatchReviewResult,
+    MemoryCandidateResponse,
+    ProfileCandidateResponse,
+)
+from app.schemas.common import Page
+from app.schemas.imports import (
+    CreateImportRequest,
+    ImportItemResponse,
+    ImportJobResponse,
+)
+from app.services.audit import record_audit_event
+from app.services.characters import get_owned_character
+from app.services.serializers import (
+    import_item_to_response,
+    import_job_to_response,
+    memory_candidate_to_response,
+    profile_candidate_to_response,
+)
+
+_TEXT_MIME_FORMAT: dict[str, str] = {
+    "text/plain": "text",
+    "text/markdown": "markdown",
+    "application/json": "json",
+    "text/csv": "csv",
+    "application/yaml": "yaml",
+    "text/yaml": "yaml",
+}
+_AUDIO_FORMAT: dict[str, str] = {
+    "audio/wav": "wav",
+    "audio/mpeg": "mp3",
+    "audio/mp4": "m4a",
+    "audio/x-m4a": "m4a",
+    "audio/flac": "flac",
+}
+
+
+async def create_import_from_json(
+    character_id: str,
+    payload: CreateImportRequest,
+    owner_scope: str = settings.default_owner_scope,
+) -> ImportJobResponse:
+    """JSON 提交导入（粘贴文本 / 结构化消息 / 手动设定）。"""
+    async with AsyncSessionLocal() as session:
+        character = await get_owned_character(session, character_id, owner_scope=owner_scope)
+        job = CharacterImportJob(
+            character_id=character.id,
+            material_type=payload.material_type,
+            source_type="pasted_text",
+            title=payload.title,
+            status="pending",
+            format=payload.format,
+            source_description=payload.source_note,
+            raw_content=payload.content,
+            checksum=_checksum(payload.content),
+        )
+        await imports_repo.add_job(session, job)
+        await _parse_into_items(session, job, payload.content)
+        await _audit(session, character.id, "import.created", job.id, payload.material_type)
+        await session.commit()
+        await session.refresh(job)
+        return import_job_to_response(job)
+
+
+async def create_import_from_upload(
+    character_id: str,
+    file: UploadFile,
+    *,
+    title: str | None = None,
+    source_description: str | None = None,
+    speaker_label: str | None = None,
+    consent_status: str = "unknown",
+    usage_scope: str = "reference_only",
+    owner_scope: str = settings.default_owner_scope,
+) -> ImportJobResponse:
+    """文件上传导入（文本类解析为 items；音频类仅存元数据，不落盘）。"""
+    content_bytes = await file.read()
+    mime = file.content_type or "application/octet-stream"
+    filename = file.filename or "upload"
+
+    async with AsyncSessionLocal() as session:
+        character = await get_owned_character(session, character_id, owner_scope=owner_scope)
+        if mime in _AUDIO_FORMAT:
+            job = CharacterImportJob(
+                character_id=character.id,
+                material_type="voice_sample",
+                source_type="media_upload",
+                title=title,
+                status="uploaded",
+                format=_AUDIO_FORMAT[mime],
+                media_type="audio",
+                mime_type=mime,
+                source_description=source_description,
+                speaker_label=speaker_label,
+                consent_status=consent_status,
+                usage_scope=usage_scope,
+                checksum=_checksum_bytes(content_bytes),
+            )
+            await imports_repo.add_job(session, job)
+        else:
+            try:
+                content = content_bytes.decode("utf-8")
+            except UnicodeDecodeError as exc:
+                raise AppError("unsupported_media_type", "仅支持 UTF-8 文本文件。") from exc
+            job = CharacterImportJob(
+                character_id=character.id,
+                material_type=_infer_material_type(filename, mime),
+                source_type="uploaded_file",
+                title=title or filename,
+                status="pending",
+                format=_TEXT_MIME_FORMAT.get(mime, "text"),
+                mime_type=mime,
+                source_description=source_description,
+                raw_content=content,
+                checksum=_checksum_bytes(content_bytes),
+            )
+            await imports_repo.add_job(session, job)
+            await _parse_into_items(session, job, content)
+        await _audit(session, character.id, "import.created", job.id, job.material_type)
+        await session.commit()
+        await session.refresh(job)
+        return import_job_to_response(job)
+
+
+async def get_import(
+    character_id: str,
+    import_id: str,
+    owner_scope: str = settings.default_owner_scope,
+) -> ImportJobResponse:
+    async with AsyncSessionLocal() as session:
+        character = await get_owned_character(session, character_id, owner_scope=owner_scope)
+        job = await imports_repo.get_job(session, import_id)
+        if job is None or job.character_id != character.id:
+            raise AppError("not_found", "导入任务不存在。")
+        return import_job_to_response(job)
+
+
+async def list_jobs_page(
+    character_id: str,
+    *,
+    page: int = 1,
+    page_size: int = 20,
+    owner_scope: str = settings.default_owner_scope,
+) -> Page[ImportJobResponse]:
+    async with AsyncSessionLocal() as session:
+        character = await get_owned_character(session, character_id, owner_scope=owner_scope)
+        safe_page = max(page, 1)
+        safe_size = min(max(page_size, 1), 100)
+        items, total = await imports_repo.list_jobs(
+            session, character.id, page=safe_page, page_size=safe_size
+        )
+        return Page(
+            items=[import_job_to_response(item) for item in items],
+            page=safe_page,
+            page_size=safe_size,
+            total=total,
+        )
+
+
+async def list_import_items(
+    character_id: str,
+    import_id: str,
+    *,
+    page: int = 1,
+    page_size: int = 20,
+    owner_scope: str = settings.default_owner_scope,
+) -> Page[ImportItemResponse]:
+    async with AsyncSessionLocal() as session:
+        character = await get_owned_character(session, character_id, owner_scope=owner_scope)
+        job = await imports_repo.get_job(session, import_id)
+        if job is None or job.character_id != character.id:
+            raise AppError("not_found", "导入任务不存在。")
+        safe_page = max(page, 1)
+        safe_size = min(max(page_size, 1), 100)
+        items, total = await imports_repo.list_items(
+            session, import_id, page=safe_page, page_size=safe_size
+        )
+        return Page(
+            items=[import_item_to_response(item) for item in items],
+            page=safe_page,
+            page_size=safe_size,
+            total=total,
+        )
+
+
+async def _parse_into_items(session: AsyncSession, job: CharacterImportJob, content: str) -> None:
+    """结构解析为 import_items 并回填 job 状态；纯结构，不调模型。"""
+    job.status = "parsing"
+    await session.flush()
+    drafts = parse_import(job.material_type, job.format, content=content)
+    items = [
+        ImportItem(
+            import_job_id=job.id,
+            character_id=job.character_id,
+            item_type=draft.item_type,
+            role=draft.role,
+            sequence=idx,
+            content=draft.content,
+            metadata_json=dumps(draft.metadata),
+        )
+        for idx, draft in enumerate(drafts)
+    ]
+    await imports_repo.add_items(session, items)
+    job.item_count = len(items)
+    job.status = "parsed" if items else "uploaded"
+
+
+async def _audit(
+    session: AsyncSession,
+    character_id: str,
+    action: str,
+    resource_id: str,
+    summary: str,
+) -> None:
+    await record_audit_event(
+        session,
+        action=action,
+        character_id=character_id,
+        actor="user",
+        decision="auto",
+        input_summary=summary,
+        resource_type="import_job",
+        resource_id=resource_id,
+    )
+
+
+def _checksum(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _checksum_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _infer_material_type(filename: str, mime: str) -> str:
+    name = filename.lower()
+    if "chat" in name or "history" in name:
+        return "chat_history"
+    if mime == "application/json":
+        return "other"
+    return "background_story"
+
+
+# —— 候选抽取触发与列表（5b，异步后台）——
+
+_EXTRACTABLE_STATUSES = frozenset({"parsed", "uploaded", "reviewing"})
+
+
+async def _get_owned_job(
+    session: AsyncSession, character_id: str, import_id: str
+) -> CharacterImportJob:
+    job = await imports_repo.get_job(session, import_id)
+    if job is None or job.character_id != character_id:
+        raise AppError("not_found", "导入任务不存在。")
+    return job
+
+
+async def _reload_job(import_id: str) -> ImportJobResponse:
+    async with AsyncSessionLocal() as session:
+        job = await imports_repo.get_job(session, import_id)
+        return import_job_to_response(job)
+
+
+async def trigger_profile_extraction(
+    character_id: str,
+    import_id: str,
+    owner_scope: str = settings.default_owner_scope,
+) -> ImportJobResponse:
+    return await _trigger_extraction(
+        character_id, import_id, "extracting_profile", "profile", owner_scope=owner_scope
+    )
+
+
+async def trigger_memory_extraction(
+    character_id: str,
+    import_id: str,
+    owner_scope: str = settings.default_owner_scope,
+) -> ImportJobResponse:
+    return await _trigger_extraction(
+        character_id, import_id, "extracting_memory", "memory", owner_scope=owner_scope
+    )
+
+
+async def trigger_personality_extraction(
+    character_id: str,
+    import_id: str,
+    owner_scope: str = settings.default_owner_scope,
+) -> ImportJobResponse:
+    return await _trigger_extraction(
+        character_id,
+        import_id,
+        "generating_personality_proposals",
+        "personality",
+        owner_scope=owner_scope,
+    )
+
+
+async def _trigger_extraction(
+    character_id: str,
+    import_id: str,
+    status: str,
+    kind: str,
+    owner_scope: str = settings.default_owner_scope,
+) -> ImportJobResponse:
+    """校验 job → 置 extracting_* → 启动后台抽取 → 返回 job 当前状态。"""
+    async with AsyncSessionLocal() as session:
+        character = await get_owned_character(session, character_id, owner_scope=owner_scope)
+        job = await _get_owned_job(session, character.id, import_id)
+        if job.status not in _EXTRACTABLE_STATUSES:
+            raise AppError("conflict", f"任务状态 {job.status} 不可触发抽取。")
+        job.status = status
+        await session.commit()
+
+    from app.application.imports import (
+        start_memory_extraction,
+        start_personality_extraction,
+        start_profile_extraction,
+    )
+
+    starters = {
+        "profile": start_profile_extraction,
+        "memory": start_memory_extraction,
+        "personality": start_personality_extraction,
+    }
+    starters[kind](import_id)
+    return await _reload_job(import_id)
+
+
+async def list_profile_candidates(
+    character_id: str,
+    import_id: str,
+    *,
+    status: str | None = None,
+    owner_scope: str = settings.default_owner_scope,
+) -> list[ProfileCandidateResponse]:
+    async with AsyncSessionLocal() as session:
+        character = await get_owned_character(session, character_id, owner_scope=owner_scope)
+        await _get_owned_job(session, character.id, import_id)
+        items = await candidates_repo.list_profile_candidates(
+            session, character.id, import_job_id=import_id, status=status
+        )
+        return [profile_candidate_to_response(c) for c in items]
+
+
+async def list_memory_candidates(
+    character_id: str,
+    import_id: str,
+    *,
+    status: str | None = None,
+    owner_scope: str = settings.default_owner_scope,
+) -> list[MemoryCandidateResponse]:
+    async with AsyncSessionLocal() as session:
+        character = await get_owned_character(session, character_id, owner_scope=owner_scope)
+        await _get_owned_job(session, character.id, import_id)
+        items = await candidates_repo.list_memory_candidates(
+            session, character.id, import_job_id=import_id, status=status
+        )
+        return [memory_candidate_to_response(c) for c in items]
+
+
+# —— 候选审核应用（5c）——
+
+
+async def batch_review_profile_candidates(
+    character_id: str,
+    payload: BatchReviewRequest,
+    owner_scope: str = settings.default_owner_scope,
+) -> BatchReviewResult:
+    """批量审核档案候选：approve 写入人物档案，reject 标记驳回，非 pending 跳过。"""
+    async with AsyncSessionLocal() as session:
+        character = await get_owned_character(session, character_id, owner_scope=owner_scope)
+        profile = loads_dict(character.profile_json)
+        applied = rejected = skipped = 0
+        for decision in payload.decisions:
+            candidate = await candidates_repo.get_profile_candidate(session, decision.candidate_id)
+            if (
+                candidate is None
+                or candidate.character_id != character.id
+                or candidate.status != "pending"
+            ):
+                skipped += 1
+                continue
+            if decision.decision == "approve":
+                profile[candidate.field] = candidate.value
+                if candidate.field == "display_name":
+                    character.display_name = candidate.value
+                candidate.status = "applied"
+                applied += 1
+            elif decision.decision == "reject":
+                candidate.status = "rejected"
+                rejected += 1
+            else:
+                skipped += 1
+                continue
+            candidate.reviewed_at = utc_now()
+        character.profile_json = dumps(profile)
+        await _audit(
+            session,
+            character.id,
+            "import.profile_candidate_reviewed",
+            character.id,
+            f"applied={applied} rejected={rejected}",
+        )
+        await session.commit()
+        return BatchReviewResult(applied=applied, rejected=rejected, skipped=skipped)
+
+
+async def batch_review_memory_candidates(
+    character_id: str,
+    payload: BatchReviewRequest,
+    owner_scope: str = settings.default_owner_scope,
+) -> BatchReviewResult:
+    """批量审核记忆候选：approve 创建记忆，reject 标记驳回。"""
+    async with AsyncSessionLocal() as session:
+        character = await get_owned_character(session, character_id, owner_scope=owner_scope)
+        applied = rejected = skipped = 0
+        for decision in payload.decisions:
+            candidate = await candidates_repo.get_memory_candidate(session, decision.candidate_id)
+            if (
+                candidate is None
+                or candidate.character_id != character.id
+                or candidate.status != "pending"
+            ):
+                skipped += 1
+                continue
+            if decision.decision == "approve":
+                session.add(
+                    Memory(
+                        character_id=character.id,
+                        type=candidate.type,
+                        title=candidate.title,
+                        content=candidate.content,
+                        source="user_explicit",
+                        confidence=candidate.confidence,
+                        importance=candidate.importance,
+                        sensitivity=candidate.sensitivity,
+                        evidence_ids_json=candidate.evidence_ids_json,
+                        status="active",
+                    )
+                )
+                candidate.status = "applied"
+                applied += 1
+            elif decision.decision == "reject":
+                candidate.status = "rejected"
+                rejected += 1
+            else:
+                skipped += 1
+                continue
+            candidate.reviewed_at = utc_now()
+        await _audit(
+            session,
+            character.id,
+            "import.memory_candidate_reviewed",
+            character.id,
+            f"applied={applied} rejected={rejected}",
+        )
+        await session.commit()
+        return BatchReviewResult(applied=applied, rejected=rejected, skipped=skipped)