feat: Add batch embedding retrieval and vector update methods for Manticore integration

- Implemented `getBatchEmbeddings` method in AI module for retrieving embeddings for multiple texts.
- Added vector update methods for messages, files, tasks, projects, and users in ManticoreBase.
- Enhanced ManticoreFile, ManticoreMsg, ManticoreProject, ManticoreTask, and ManticoreUser to support vector generation during sync operations.
- Introduced `generateVectorsBatch` methods for batch processing of vector generation in Manticore modules.
- Updated ManticoreSyncTask to handle incremental updates and vector generation asynchronously.
This commit is contained in:
kuaifan
2026-01-03 15:19:23 +00:00
parent 7a21a2d800
commit e020a80020
9 changed files with 1087 additions and 31 deletions

View File

@@ -242,9 +242,10 @@ class ManticoreFile
* 同步单个文件到 Manticore含 allowed_users
*
* @param File $file 文件模型
* @param bool $withVector 是否同时生成向量(默认 false向量由后台任务生成
* @return bool 是否成功
*/
public static function sync(File $file): bool
public static function sync(File $file, bool $withVector = false): bool
{
if (!Apps::isInstalled("manticore")) {
return false;
@@ -269,9 +270,9 @@ class ManticoreFile
// 限制提取后的内容长度
$content = mb_substr($content, 0, self::MAX_CONTENT_LENGTH);
// 获取 embedding如果有内容且 AI 可用
// 只有明确要求时才生成向量(默认不生成,由后台任务处理
$embedding = null;
if (!empty($content) && Apps::isInstalled('ai')) {
if ($withVector && !empty($content) && Apps::isInstalled('ai')) {
$embeddingResult = self::getEmbedding($content);
if (!empty($embeddingResult)) {
$embedding = '[' . implode(',', $embeddingResult) . ']';
@@ -339,9 +340,10 @@ class ManticoreFile
* 批量同步文件
*
* @param iterable $files 文件列表
* @param bool $withVector 是否同时生成向量
* @return int 成功同步的数量
*/
public static function batchSync(iterable $files): int
public static function batchSync(iterable $files, bool $withVector = false): int
{
if (!Apps::isInstalled("manticore")) {
return 0;
@@ -349,7 +351,7 @@ class ManticoreFile
$count = 0;
foreach ($files as $file) {
if (self::sync($file)) {
if (self::sync($file, $withVector)) {
$count++;
}
}
@@ -477,4 +479,90 @@ class ManticoreFile
return false;
}
}
// ==============================
// 批量向量生成方法
// ==============================
/**
* 批量生成文件向量
* 用于后台异步处理,将已索引文件的向量批量生成
*
* @param array $fileIds 文件ID数组
* @param int $batchSize 每批 embedding 数量默认20
* @return int 成功处理的数量
*/
public static function generateVectorsBatch(array $fileIds, int $batchSize = 20): int
{
if (!Apps::isInstalled("manticore") || !Apps::isInstalled("ai") || empty($fileIds)) {
return 0;
}
try {
// 1. 查询文件信息
$files = File::whereIn('id', $fileIds)
->where('type', '!=', 'folder')
->get();
if ($files->isEmpty()) {
return 0;
}
// 2. 提取每个文件的内容
$fileContents = [];
foreach ($files as $file) {
// 检查文件大小限制
$maxSize = self::getMaxFileSizeByExt($file->ext);
if ($file->size > $maxSize) {
continue;
}
$content = self::extractFileContent($file);
if (!empty($content)) {
// 限制内容长度
$content = mb_substr($content, 0, self::MAX_CONTENT_LENGTH);
$fileContents[$file->id] = $content;
}
}
if (empty($fileContents)) {
return 0;
}
// 3. 分批处理
$successCount = 0;
$chunks = array_chunk($fileContents, $batchSize, true);
foreach ($chunks as $chunk) {
$texts = array_values($chunk);
$ids = array_keys($chunk);
// 4. 批量获取 embedding
$result = AI::getBatchEmbeddings($texts);
if (!Base::isSuccess($result) || empty($result['data'])) {
Log::warning('ManticoreFile: Batch embedding failed', ['file_ids' => $ids]);
continue;
}
$embeddings = $result['data'];
// 5. 逐个更新向量到 Manticore
foreach ($ids as $index => $fileId) {
if (!isset($embeddings[$index]) || empty($embeddings[$index])) {
continue;
}
$vectorStr = '[' . implode(',', $embeddings[$index]) . ']';
if (ManticoreBase::updateFileVector($fileId, $vectorStr)) {
$successCount++;
}
}
}
return $successCount;
} catch (\Exception $e) {
Log::error('ManticoreFile generateVectorsBatch error: ' . $e->getMessage());
return 0;
}
}
}