refactor: 使用 Manticore Search 替换 SeekDB

This commit is contained in:
kuaifan
2026-01-01 03:17:27 +00:00
parent 10c6177a9f
commit 48ef4cfdef
24 changed files with 1017 additions and 1102 deletions
@@ -4,12 +4,12 @@ namespace App\Console\Commands;
use App\Models\File;
use App\Module\Apps;
use App\Module\SeekDB\SeekDBFile;
use App\Module\SeekDB\SeekDBKeyValue;
use App\Module\Manticore\ManticoreFile;
use App\Module\Manticore\ManticoreKeyValue;
use Cache;
use Illuminate\Console\Command;
class SyncFileToSeekDB extends Command
class SyncFileToManticore extends Command
{
/**
* 更新数据
@@ -21,16 +21,16 @@ class SyncFileToSeekDB extends Command
* --c: 清除索引
*/
protected $signature = 'seekdb:sync-files {--f} {--i} {--c} {--u} {--batch=100}';
protected $description = '同步文件内容到 SeekDB';
protected $signature = 'manticore:sync-files {--f} {--i} {--c} {--u} {--batch=100}';
protected $description = '同步文件内容到 Manticore Search';
/**
* @return int
*/
public function handle(): int
{
if (!Apps::isInstalled("seekdb")) {
$this->error("应用「SeekDB」未安装");
if (!Apps::isInstalled("manticore")) {
$this->error("应用「Manticore Search」未安装");
return 1;
}
@@ -54,8 +54,8 @@ class SyncFileToSeekDB extends Command
// 清除索引
if ($this->option('c')) {
$this->info('清除索引...');
SeekDBKeyValue::clear();
SeekDBFile::clear();
ManticoreKeyValue::clear();
ManticoreFile::clear();
$this->info("索引删除成功");
$this->releaseLock();
return 0;
@@ -64,7 +64,7 @@ class SyncFileToSeekDB extends Command
// 仅同步文件用户关系
if ($this->option('u')) {
$this->info('开始同步文件用户关系...');
$count = SeekDBFile::syncAllFileUsers(function ($count) {
$count = ManticoreFile::syncAllFileUsers(function ($count) {
if ($count % 1000 === 0) {
$this->info(" 已同步 {$count} 条关系...");
}
@@ -83,7 +83,7 @@ class SyncFileToSeekDB extends Command
if ($this->option('f') || (!$this->option('i') && !$this->option('u'))) {
// 全量同步:清空后重建
$this->info("\n全量同步文件用户关系...");
$count = SeekDBFile::syncAllFileUsers(function ($count) {
$count = ManticoreFile::syncAllFileUsers(function ($count) {
if ($count % 1000 === 0) {
$this->info(" 已同步 {$count} 条关系...");
}
@@ -92,7 +92,7 @@ class SyncFileToSeekDB extends Command
} elseif ($this->option('i')) {
// 增量同步:只同步新增的
$this->info("\n增量同步文件用户关系...");
$count = SeekDBFile::syncFileUsersIncremental(function ($count) {
$count = ManticoreFile::syncFileUsersIncremental(function ($count) {
if ($count % 1000 === 0) {
$this->info(" 已同步 {$count} 条关系...");
}
@@ -161,8 +161,8 @@ class SyncFileToSeekDB extends Command
private function syncFiles(): void
{
// 获取上次同步的最后ID
$lastKey = "sync:seekdbFileLastId";
$lastId = $this->option('i') ? intval(SeekDBKeyValue::get($lastKey, 0)) : 0;
$lastKey = "sync:manticoreFileLastId";
$lastId = $this->option('i') ? intval(ManticoreKeyValue::get($lastKey, 0)) : 0;
if ($lastId > 0) {
$this->info("\n同步文件数据({$lastId})...");
@@ -171,8 +171,8 @@ class SyncFileToSeekDB extends Command
}
// 查询条件:排除文件夹,使用最大文件限制
// 具体的文件类型大小检查在 SeekDBFile::sync 中进行
$maxFileSize = SeekDBFile::getMaxFileSize();
// 具体的文件类型大小检查在 ManticoreFile::sync 中进行
$maxFileSize = ManticoreFile::getMaxFileSize();
$query = File::where('id', '>', $lastId)
->where('type', '!=', 'folder')
->where('size', '<=', $maxFileSize);
@@ -208,16 +208,16 @@ class SyncFileToSeekDB extends Command
$this->setLock();
// 同步数据
$lastNum = SeekDBFile::batchSync($files);
$lastNum = ManticoreFile::batchSync($files);
$total += $lastNum;
// 更新最后ID
$lastId = $files->last()->id;
SeekDBKeyValue::set($lastKey, $lastId);
ManticoreKeyValue::set($lastKey, $lastId);
} while (count($files) == $batchSize);
$this->info("同步文件结束 - 最后ID {$lastId}");
$this->info("已索引文件数量: " . SeekDBFile::getIndexedCount());
$this->info("已索引文件数量: " . ManticoreFile::getIndexedCount());
}
}
@@ -4,12 +4,12 @@ namespace App\Console\Commands;
use App\Models\Project;
use App\Module\Apps;
use App\Module\SeekDB\SeekDBProject;
use App\Module\SeekDB\SeekDBKeyValue;
use App\Module\Manticore\ManticoreProject;
use App\Module\Manticore\ManticoreKeyValue;
use Cache;
use Illuminate\Console\Command;
class SyncProjectToSeekDB extends Command
class SyncProjectToManticore extends Command
{
/**
* 更新数据
@@ -21,16 +21,16 @@ class SyncProjectToSeekDB extends Command
* --c: 清除索引
*/
protected $signature = 'seekdb:sync-projects {--f} {--i} {--c} {--u} {--batch=100}';
protected $description = '同步项目数据到 SeekDB';
protected $signature = 'manticore:sync-projects {--f} {--i} {--c} {--u} {--batch=100}';
protected $description = '同步项目数据到 Manticore Search';
/**
* @return int
*/
public function handle(): int
{
if (!Apps::isInstalled("seekdb")) {
$this->error("应用「SeekDB」未安装");
if (!Apps::isInstalled("manticore")) {
$this->error("应用「Manticore Search」未安装");
return 1;
}
@@ -53,8 +53,7 @@ class SyncProjectToSeekDB extends Command
// 清除索引
if ($this->option('c')) {
$this->info('清除索引...');
SeekDBProject::clear();
SeekDBKeyValue::set('sync:seekdbProjectLastId', 0);
ManticoreProject::clear();
$this->info("索引删除成功");
$this->releaseLock();
return 0;
@@ -63,7 +62,7 @@ class SyncProjectToSeekDB extends Command
// 仅同步项目成员关系
if ($this->option('u')) {
$this->info('开始同步项目成员关系...');
$count = SeekDBProject::syncAllProjectUsers(function ($count) {
$count = ManticoreProject::syncAllProjectUsers(function ($count) {
if ($count % 1000 === 0) {
$this->info(" 已同步 {$count} 条关系...");
}
@@ -78,18 +77,16 @@ class SyncProjectToSeekDB extends Command
// 同步项目成员关系
if ($this->option('f') || (!$this->option('i') && !$this->option('u'))) {
// 全量同步:清空后重建
$this->info("\n全量同步项目成员关系...");
$count = SeekDBProject::syncAllProjectUsers(function ($count) {
$count = ManticoreProject::syncAllProjectUsers(function ($count) {
if ($count % 1000 === 0) {
$this->info(" 已同步 {$count} 条关系...");
}
});
$this->info("项目成员关系同步完成,共 {$count} 条");
} elseif ($this->option('i')) {
// 增量同步:只同步新增的
$this->info("\n增量同步项目成员关系...");
$count = SeekDBProject::syncProjectUsersIncremental(function ($count) {
$count = ManticoreProject::syncProjectUsersIncremental(function ($count) {
if ($count % 1000 === 0) {
$this->info(" 已同步 {$count} 条关系...");
}
@@ -113,12 +110,13 @@ class SyncProjectToSeekDB extends Command
private function setLock(): void
{
$lockKey = md5($this->signature);
Cache::put($lockKey, ['started_at' => date('Y-m-d H:i:s')], 300);
Cache::put($lockKey, ['started_at' => date('Y-m-d H:i:s')], 600);
}
private function releaseLock(): void
{
Cache::forget(md5($this->signature));
$lockKey = md5($this->signature);
Cache::forget($lockKey);
}
public function handleSignal(int $signal): void
@@ -129,23 +127,25 @@ class SyncProjectToSeekDB extends Command
private function syncProjects(): void
{
$lastKey = "sync:seekdbProjectLastId";
$lastId = $this->option('i') ? intval(SeekDBKeyValue::get($lastKey, 0)) : 0;
$lastKey = "sync:manticoreProjectLastId";
$lastId = $this->option('i') ? intval(ManticoreKeyValue::get($lastKey, 0)) : 0;
if ($lastId > 0) {
$this->info("\n增量同步项目数据(从ID: {$lastId})...");
$this->info("\n同步项目数据({$lastId})...");
} else {
$this->info("\n全量同步项目数据...");
$this->info("\n同步项目数据...");
}
// 只同步未归档的项目
// 排除已归档项目
$query = Project::where('id', '>', $lastId)
->whereNull('archived_at');
$num = 0;
$count = $query->count();
$batchSize = $this->option('batch');
$total = 0;
$lastNum = 0;
do {
$projects = Project::where('id', '>', $lastId)
@@ -160,19 +160,22 @@ class SyncProjectToSeekDB extends Command
$num += count($projects);
$progress = $count > 0 ? round($num / $count * 100, 2) : 100;
$this->info("{$num}/{$count} ({$progress}%) 正在同步项目ID {$projects->first()->id} ~ {$projects->last()->id}");
if ($progress < 100) {
$progress = number_format($progress, 2);
}
$this->info("{$num}/{$count} ({$progress}%) 正在同步项目ID {$projects->first()->id} ~ {$projects->last()->id} ({$total}|{$lastNum})");
$this->setLock();
$synced = SeekDBProject::batchSync($projects);
$total += $synced;
$lastNum = ManticoreProject::batchSync($projects);
$total += $lastNum;
$lastId = $projects->last()->id;
SeekDBKeyValue::set($lastKey, $lastId);
ManticoreKeyValue::set($lastKey, $lastId);
} while (count($projects) == $batchSize);
$this->info("同步项目结束 - 最后ID {$lastId},共同步 {$total} 个项目");
$this->info("已索引项目数量: " . SeekDBProject::getIndexedCount());
$this->info("同步项目结束 - 最后ID {$lastId}");
$this->info("已索引项目数量: " . ManticoreProject::getIndexedCount());
}
}
@@ -4,12 +4,12 @@ namespace App\Console\Commands;
use App\Models\ProjectTask;
use App\Module\Apps;
use App\Module\SeekDB\SeekDBTask;
use App\Module\SeekDB\SeekDBKeyValue;
use App\Module\Manticore\ManticoreTask;
use App\Module\Manticore\ManticoreKeyValue;
use Cache;
use Illuminate\Console\Command;
class SyncTaskToSeekDB extends Command
class SyncTaskToManticore extends Command
{
/**
* 更新数据
@@ -21,16 +21,16 @@ class SyncTaskToSeekDB extends Command
* --c: 清除索引
*/
protected $signature = 'seekdb:sync-tasks {--f} {--i} {--c} {--u} {--batch=100}';
protected $description = '同步任务数据到 SeekDB';
protected $signature = 'manticore:sync-tasks {--f} {--i} {--c} {--u} {--batch=100}';
protected $description = '同步任务数据到 Manticore Search';
/**
* @return int
*/
public function handle(): int
{
if (!Apps::isInstalled("seekdb")) {
$this->error("应用「SeekDB」未安装");
if (!Apps::isInstalled("manticore")) {
$this->error("应用「Manticore Search」未安装");
return 1;
}
@@ -53,8 +53,7 @@ class SyncTaskToSeekDB extends Command
// 清除索引
if ($this->option('c')) {
$this->info('清除索引...');
SeekDBTask::clear();
SeekDBKeyValue::set('sync:seekdbTaskLastId', 0);
ManticoreTask::clear();
$this->info("索引删除成功");
$this->releaseLock();
return 0;
@@ -63,7 +62,7 @@ class SyncTaskToSeekDB extends Command
// 仅同步任务成员关系
if ($this->option('u')) {
$this->info('开始同步任务成员关系...');
$count = SeekDBTask::syncAllTaskUsers(function ($count) {
$count = ManticoreTask::syncAllTaskUsers(function ($count) {
if ($count % 1000 === 0) {
$this->info(" 已同步 {$count} 条关系...");
}
@@ -78,18 +77,16 @@ class SyncTaskToSeekDB extends Command
// 同步任务成员关系
if ($this->option('f') || (!$this->option('i') && !$this->option('u'))) {
// 全量同步:清空后重建
$this->info("\n全量同步任务成员关系...");
$count = SeekDBTask::syncAllTaskUsers(function ($count) {
$count = ManticoreTask::syncAllTaskUsers(function ($count) {
if ($count % 1000 === 0) {
$this->info(" 已同步 {$count} 条关系...");
}
});
$this->info("任务成员关系同步完成,共 {$count} 条");
} elseif ($this->option('i')) {
// 增量同步:只同步新增的
$this->info("\n增量同步任务成员关系...");
$count = SeekDBTask::syncTaskUsersIncremental(function ($count) {
$count = ManticoreTask::syncTaskUsersIncremental(function ($count) {
if ($count % 1000 === 0) {
$this->info(" 已同步 {$count} 条关系...");
}
@@ -113,12 +110,13 @@ class SyncTaskToSeekDB extends Command
private function setLock(): void
{
$lockKey = md5($this->signature);
Cache::put($lockKey, ['started_at' => date('Y-m-d H:i:s')], 600); // 任务可能较多,10分钟
Cache::put($lockKey, ['started_at' => date('Y-m-d H:i:s')], 600);
}
private function releaseLock(): void
{
Cache::forget(md5($this->signature));
$lockKey = md5($this->signature);
Cache::forget($lockKey);
}
public function handleSignal(int $signal): void
@@ -129,27 +127,31 @@ class SyncTaskToSeekDB extends Command
private function syncTasks(): void
{
$lastKey = "sync:seekdbTaskLastId";
$lastId = $this->option('i') ? intval(SeekDBKeyValue::get($lastKey, 0)) : 0;
$lastKey = "sync:manticoreTaskLastId";
$lastId = $this->option('i') ? intval(ManticoreKeyValue::get($lastKey, 0)) : 0;
if ($lastId > 0) {
$this->info("\n增量同步任务数据(从ID: {$lastId})...");
$this->info("\n同步任务数据({$lastId})...");
} else {
$this->info("\n全量同步任务数据...");
$this->info("\n同步任务数据...");
}
// 只同步未归档的任务(包括软删除恢复的情况)
// 排除已归档和已删除的任务
$query = ProjectTask::where('id', '>', $lastId)
->whereNull('archived_at');
->whereNull('archived_at')
->whereNull('deleted_at');
$num = 0;
$count = $query->count();
$batchSize = $this->option('batch');
$total = 0;
$lastNum = 0;
do {
$tasks = ProjectTask::where('id', '>', $lastId)
->whereNull('archived_at')
->whereNull('deleted_at')
->orderBy('id')
->limit($batchSize)
->get();
@@ -160,19 +162,22 @@ class SyncTaskToSeekDB extends Command
$num += count($tasks);
$progress = $count > 0 ? round($num / $count * 100, 2) : 100;
$this->info("{$num}/{$count} ({$progress}%) 正在同步任务ID {$tasks->first()->id} ~ {$tasks->last()->id}");
if ($progress < 100) {
$progress = number_format($progress, 2);
}
$this->info("{$num}/{$count} ({$progress}%) 正在同步任务ID {$tasks->first()->id} ~ {$tasks->last()->id} ({$total}|{$lastNum})");
$this->setLock();
$synced = SeekDBTask::batchSync($tasks);
$total += $synced;
$lastNum = ManticoreTask::batchSync($tasks);
$total += $lastNum;
$lastId = $tasks->last()->id;
SeekDBKeyValue::set($lastKey, $lastId);
ManticoreKeyValue::set($lastKey, $lastId);
} while (count($tasks) == $batchSize);
$this->info("同步任务结束 - 最后ID {$lastId},共同步 {$total} 个任务");
$this->info("已索引任务数量: " . SeekDBTask::getIndexedCount());
$this->info("同步任务结束 - 最后ID {$lastId}");
$this->info("已索引任务数量: " . ManticoreTask::getIndexedCount());
}
}
@@ -4,12 +4,12 @@ namespace App\Console\Commands;
use App\Models\User;
use App\Module\Apps;
use App\Module\SeekDB\SeekDBUser;
use App\Module\SeekDB\SeekDBKeyValue;
use App\Module\Manticore\ManticoreUser;
use App\Module\Manticore\ManticoreKeyValue;
use Cache;
use Illuminate\Console\Command;
class SyncUserToSeekDB extends Command
class SyncUserToManticore extends Command
{
/**
* 更新数据
@@ -20,16 +20,16 @@ class SyncUserToSeekDB extends Command
* --c: 清除索引
*/
protected $signature = 'seekdb:sync-users {--f} {--i} {--c} {--batch=100}';
protected $description = '同步用户数据到 SeekDB(联系人搜索)';
protected $signature = 'manticore:sync-users {--f} {--i} {--c} {--batch=100}';
protected $description = '同步用户数据到 Manticore Search';
/**
* @return int
*/
public function handle(): int
{
if (!Apps::isInstalled("seekdb")) {
$this->error("应用「SeekDB」未安装");
if (!Apps::isInstalled("manticore")) {
$this->error("应用「Manticore Search」未安装");
return 1;
}
@@ -52,8 +52,7 @@ class SyncUserToSeekDB extends Command
// 清除索引
if ($this->option('c')) {
$this->info('清除索引...');
SeekDBUser::clear();
SeekDBKeyValue::set('sync:seekdbUserLastId', 0);
ManticoreUser::clear();
$this->info("索引删除成功");
$this->releaseLock();
return 0;
@@ -76,12 +75,13 @@ class SyncUserToSeekDB extends Command
private function setLock(): void
{
$lockKey = md5($this->signature);
Cache::put($lockKey, ['started_at' => date('Y-m-d H:i:s')], 300);
Cache::put($lockKey, ['started_at' => date('Y-m-d H:i:s')], 600);
}
private function releaseLock(): void
{
Cache::forget(md5($this->signature));
$lockKey = md5($this->signature);
Cache::forget($lockKey);
}
public function handleSignal(int $signal): void
@@ -92,16 +92,16 @@ class SyncUserToSeekDB extends Command
private function syncUsers(): void
{
$lastKey = "sync:seekdbUserLastId";
$lastId = $this->option('i') ? intval(SeekDBKeyValue::get($lastKey, 0)) : 0;
$lastKey = "sync:manticoreUserLastId";
$lastId = $this->option('i') ? intval(ManticoreKeyValue::get($lastKey, 0)) : 0;
if ($lastId > 0) {
$this->info("\n增量同步用户数据(从ID: {$lastId})...");
$this->info("\n同步用户数据({$lastId})...");
} else {
$this->info("\n全量同步用户数据...");
$this->info("\n同步用户数据...");
}
// 只同步非机器人且未禁用的用户
// 排除机器人和已禁用账号
$query = User::where('userid', '>', $lastId)
->where('bot', 0)
->whereNull('disable_at');
@@ -109,7 +109,9 @@ class SyncUserToSeekDB extends Command
$num = 0;
$count = $query->count();
$batchSize = $this->option('batch');
$total = 0;
$lastNum = 0;
do {
$users = User::where('userid', '>', $lastId)
@@ -125,19 +127,22 @@ class SyncUserToSeekDB extends Command
$num += count($users);
$progress = $count > 0 ? round($num / $count * 100, 2) : 100;
$this->info("{$num}/{$count} ({$progress}%) 正在同步用户ID {$users->first()->userid} ~ {$users->last()->userid}");
if ($progress < 100) {
$progress = number_format($progress, 2);
}
$this->info("{$num}/{$count} ({$progress}%) 正在同步用户ID {$users->first()->userid} ~ {$users->last()->userid} ({$total}|{$lastNum})");
$this->setLock();
$synced = SeekDBUser::batchSync($users);
$total += $synced;
$lastNum = ManticoreUser::batchSync($users);
$total += $lastNum;
$lastId = $users->last()->userid;
SeekDBKeyValue::set($lastKey, $lastId);
ManticoreKeyValue::set($lastKey, $lastId);
} while (count($users) == $batchSize);
$this->info("同步用户结束 - 最后ID {$lastId},共同步 {$total} 个用户");
$this->info("已索引用户数量: " . SeekDBUser::getIndexedCount());
$this->info("同步用户结束 - 最后ID {$lastId}");
$this->info("已索引用户数量: " . ManticoreUser::getIndexedCount());
}
}