feat: Add batch embedding retrieval and vector update methods for Manticore integration

- Implemented `getBatchEmbeddings` method in AI module for retrieving embeddings for multiple texts.
- Added vector update methods for messages, files, tasks, projects, and users in ManticoreBase.
- Enhanced ManticoreFile, ManticoreMsg, ManticoreProject, ManticoreTask, and ManticoreUser to support vector generation during sync operations.
- Introduced `generateVectorsBatch` methods for batch processing of vector generation in Manticore modules.
- Updated ManticoreSyncTask to handle incremental updates and vector generation asynchronously.
This commit is contained in:
kuaifan
2026-01-03 15:19:23 +00:00
parent 7a21a2d800
commit e020a80020
9 changed files with 1087 additions and 31 deletions

View File

@@ -207,9 +207,10 @@ class ManticoreTask
* 同步单个任务到 Manticore(含 allowed_users)
*
* @param ProjectTask $task 任务模型
* @param bool $withVector 是否同时生成向量(默认 false,向量由后台任务生成)
* @return bool 是否成功
*/
public static function sync(ProjectTask $task): bool
public static function sync(ProjectTask $task, bool $withVector = false): bool
{
if (!Apps::isInstalled("manticore")) {
return false;
@@ -227,9 +228,9 @@ class ManticoreTask
// 构建用于搜索的文本内容
$searchableContent = self::buildSearchableContent($task, $taskContent);
// 获取 embedding(如果 AI 可用)
// 只有明确要求时才生成向量(默认不生成,由后台任务处理)
$embedding = null;
if (!empty($searchableContent) && Apps::isInstalled('ai')) {
if ($withVector && !empty($searchableContent) && Apps::isInstalled('ai')) {
$embeddingResult = self::getEmbedding($searchableContent);
if (!empty($embeddingResult)) {
$embedding = '[' . implode(',', $embeddingResult) . ']';
@@ -353,9 +354,10 @@ class ManticoreTask
* 批量同步任务
*
* @param iterable $tasks 任务列表
* @param bool $withVector 是否同时生成向量
* @return int 成功同步的数量
*/
public static function batchSync(iterable $tasks): int
public static function batchSync(iterable $tasks, bool $withVector = false): int
{
if (!Apps::isInstalled("manticore")) {
return 0;
@@ -363,7 +365,7 @@ class ManticoreTask
$count = 0;
foreach ($tasks as $task) {
if (self::sync($task)) {
if (self::sync($task, $withVector)) {
$count++;
}
}
@@ -520,4 +522,86 @@ class ManticoreTask
Log::error('Manticore cascadeToChildren error: ' . $e->getMessage(), ['task_id' => $taskId]);
}
}
// ==============================
// 批量向量生成方法
// ==============================
/**
* 批量生成任务向量
* 用于后台异步处理,将已索引任务的向量批量生成
*
* @param array $taskIds 任务ID数组
* @param int $batchSize 每批 embedding 数量(默认20)
* @return int 成功处理的数量
*/
public static function generateVectorsBatch(array $taskIds, int $batchSize = 20): int
{
if (!Apps::isInstalled("manticore") || !Apps::isInstalled("ai") || empty($taskIds)) {
return 0;
}
try {
// 1. 查询任务信息
$tasks = ProjectTask::whereIn('id', $taskIds)
->whereNull('deleted_at')
->whereNull('archived_at')
->get();
if ($tasks->isEmpty()) {
return 0;
}
// 2. 提取每个任务的内容
$taskContents = [];
foreach ($tasks as $task) {
$taskContent = self::getTaskContent($task);
$searchableContent = self::buildSearchableContent($task, $taskContent);
if (!empty($searchableContent)) {
// 限制内容长度
$searchableContent = mb_substr($searchableContent, 0, self::MAX_CONTENT_LENGTH);
$taskContents[$task->id] = $searchableContent;
}
}
if (empty($taskContents)) {
return 0;
}
// 3. 分批处理
$successCount = 0;
$chunks = array_chunk($taskContents, $batchSize, true);
foreach ($chunks as $chunk) {
$texts = array_values($chunk);
$ids = array_keys($chunk);
// 4. 批量获取 embedding
$result = AI::getBatchEmbeddings($texts);
if (!Base::isSuccess($result) || empty($result['data'])) {
Log::warning('ManticoreTask: Batch embedding failed', ['task_ids' => $ids]);
continue;
}
$embeddings = $result['data'];
// 5. 逐个更新向量到 Manticore
foreach ($ids as $index => $taskId) {
if (!isset($embeddings[$index]) || empty($embeddings[$index])) {
continue;
}
$vectorStr = '[' . implode(',', $embeddings[$index]) . ']';
if (ManticoreBase::updateTaskVector($taskId, $vectorStr)) {
$successCount++;
}
}
}
return $successCount;
} catch (\Exception $e) {
Log::error('ManticoreTask generateVectorsBatch error: ' . $e->getMessage());
return 0;
}
}
}