feat: Add batch embedding retrieval and vector update methods for Manticore integration
- Implemented `getBatchEmbeddings` method in AI module for retrieving embeddings for multiple texts. - Added vector update methods for messages, files, tasks, projects, and users in ManticoreBase. - Enhanced ManticoreFile, ManticoreMsg, ManticoreProject, ManticoreTask, and ManticoreUser to support vector generation during sync operations. - Introduced `generateVectorsBatch` methods for batch processing of vector generation in Manticore modules. - Updated ManticoreSyncTask to handle incremental updates and vector generation asynchronously.
This commit is contained in:
@@ -242,9 +242,10 @@ class ManticoreFile
|
||||
* 同步单个文件到 Manticore(含 allowed_users)
|
||||
*
|
||||
* @param File $file 文件模型
|
||||
* @param bool $withVector 是否同时生成向量(默认 false,向量由后台任务生成)
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function sync(File $file): bool
|
||||
public static function sync(File $file, bool $withVector = false): bool
|
||||
{
|
||||
if (!Apps::isInstalled("manticore")) {
|
||||
return false;
|
||||
@@ -269,9 +270,9 @@ class ManticoreFile
|
||||
// 限制提取后的内容长度
|
||||
$content = mb_substr($content, 0, self::MAX_CONTENT_LENGTH);
|
||||
|
||||
// 获取 embedding(如果有内容且 AI 可用)
|
||||
// 只有明确要求时才生成向量(默认不生成,由后台任务处理)
|
||||
$embedding = null;
|
||||
if (!empty($content) && Apps::isInstalled('ai')) {
|
||||
if ($withVector && !empty($content) && Apps::isInstalled('ai')) {
|
||||
$embeddingResult = self::getEmbedding($content);
|
||||
if (!empty($embeddingResult)) {
|
||||
$embedding = '[' . implode(',', $embeddingResult) . ']';
|
||||
@@ -339,9 +340,10 @@ class ManticoreFile
|
||||
* 批量同步文件
|
||||
*
|
||||
* @param iterable $files 文件列表
|
||||
* @param bool $withVector 是否同时生成向量
|
||||
* @return int 成功同步的数量
|
||||
*/
|
||||
public static function batchSync(iterable $files): int
|
||||
public static function batchSync(iterable $files, bool $withVector = false): int
|
||||
{
|
||||
if (!Apps::isInstalled("manticore")) {
|
||||
return 0;
|
||||
@@ -349,7 +351,7 @@ class ManticoreFile
|
||||
|
||||
$count = 0;
|
||||
foreach ($files as $file) {
|
||||
if (self::sync($file)) {
|
||||
if (self::sync($file, $withVector)) {
|
||||
$count++;
|
||||
}
|
||||
}
|
||||
@@ -477,4 +479,90 @@ class ManticoreFile
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ==============================
|
||||
// 批量向量生成方法
|
||||
// ==============================
|
||||
|
||||
/**
|
||||
* 批量生成文件向量
|
||||
* 用于后台异步处理,将已索引文件的向量批量生成
|
||||
*
|
||||
* @param array $fileIds 文件ID数组
|
||||
* @param int $batchSize 每批 embedding 数量(默认20)
|
||||
* @return int 成功处理的数量
|
||||
*/
|
||||
public static function generateVectorsBatch(array $fileIds, int $batchSize = 20): int
|
||||
{
|
||||
if (!Apps::isInstalled("manticore") || !Apps::isInstalled("ai") || empty($fileIds)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
// 1. 查询文件信息
|
||||
$files = File::whereIn('id', $fileIds)
|
||||
->where('type', '!=', 'folder')
|
||||
->get();
|
||||
|
||||
if ($files->isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 2. 提取每个文件的内容
|
||||
$fileContents = [];
|
||||
foreach ($files as $file) {
|
||||
// 检查文件大小限制
|
||||
$maxSize = self::getMaxFileSizeByExt($file->ext);
|
||||
if ($file->size > $maxSize) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$content = self::extractFileContent($file);
|
||||
if (!empty($content)) {
|
||||
// 限制内容长度
|
||||
$content = mb_substr($content, 0, self::MAX_CONTENT_LENGTH);
|
||||
$fileContents[$file->id] = $content;
|
||||
}
|
||||
}
|
||||
|
||||
if (empty($fileContents)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 3. 分批处理
|
||||
$successCount = 0;
|
||||
$chunks = array_chunk($fileContents, $batchSize, true);
|
||||
|
||||
foreach ($chunks as $chunk) {
|
||||
$texts = array_values($chunk);
|
||||
$ids = array_keys($chunk);
|
||||
|
||||
// 4. 批量获取 embedding
|
||||
$result = AI::getBatchEmbeddings($texts);
|
||||
if (!Base::isSuccess($result) || empty($result['data'])) {
|
||||
Log::warning('ManticoreFile: Batch embedding failed', ['file_ids' => $ids]);
|
||||
continue;
|
||||
}
|
||||
|
||||
$embeddings = $result['data'];
|
||||
|
||||
// 5. 逐个更新向量到 Manticore
|
||||
foreach ($ids as $index => $fileId) {
|
||||
if (!isset($embeddings[$index]) || empty($embeddings[$index])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$vectorStr = '[' . implode(',', $embeddings[$index]) . ']';
|
||||
if (ManticoreBase::updateFileVector($fileId, $vectorStr)) {
|
||||
$successCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $successCount;
|
||||
} catch (\Exception $e) {
|
||||
Log::error('ManticoreFile generateVectorsBatch error: ' . $e->getMessage());
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user