Scarfski/dootask
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: 优化全文搜索

Browse Source
This commit is contained in:
kuaifan
2025-04-18 12:30:44 +08:00
parent 7a7cd72db9
commit 924f0a9f7c
6 changed files with 175 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

356
app/Module/ZincSearch/ZincSearchUserMsg.php → app/Module/ZincSearch/ZincSearchDialogUserMsg.php
View File

@@ -12,28 +12,24 @@ use Illuminate\Support\Facades\Log;
* 使用方法:
*
* 1. 基础方法
* - 确保索引存在: ensureIndex();
* - 清空所有数据: clear();
*
* 2. 搜索方法
* - 关键词搜索: searchByKeyword('用户ID', '关键词');
* - 关键词搜索: search('用户ID', '关键词');
*
* 3. 基本方法
* - 单个同步: syncMsg($dialogMsg);
* - 批量同步: batchSyncMsgs($dialogMsgs);
* - 删除消息: deleteMsg($dialogMsg);
*
* 4. 用户方法
* - 单个同步: syncUser($dialogUser);
* - 批量同步: batchSyncUsers($dialogUsers);
* - 删除消息: deleteUser($dialogUser);
* - 单个同步: sync(WebSocketDialogMsg $dialogMsg);
* - 批量同步: batchSync(WebSocketDialogMsg[] $dialogMsgs);
* - 用户同步: userSync(WebSocketDialogUser $dialogUser);
* - 删除消息: delete(WebSocketDialogMsg|WebSocketDialogUser $data);
*/
class ZincSearchUserMsg
class ZincSearchDialogUserMsg
{
/**
* 索引名称
*/
protected static string $indexName = 'userMsg';
protected static string $indexNameMsg = 'dialogMsg';
protected static string $indexNameUser = 'dialogUser';
// ==============================
// 基础方法
@@ -42,36 +38,50 @@ class ZincSearchUserMsg
/**
* 确保索引存在
*/
public static function ensureIndex(): bool
private static function ensureIndex(): bool
{
if (!ZincSearchBase::indexExists(self::$indexName)) {
if (!ZincSearchBase::indexExists(self::$indexNameMsg)) {
$mappings = [
'properties' => [
// 关联字段
'_id' => ['type' => 'keyword', 'index' => true],
'_dialog_userid' => ['type' => 'keyword', 'index' => true],
// 拓展数据
'dialog_userid' => ['type' => 'keyword', 'index' => true], // 对话ID+用户ID
// dialog_users 字段
'userid' => ['type' => 'keyword', 'index' => true],
// 消息数据
'id' => ['type' => 'numeric', 'index' => true],
'dialog_id' => ['type' => 'numeric', 'index' => true],
'dialog_type' => ['type' => 'keyword', 'index' => true],
'session_id' => ['type' => 'numeric', 'index' => true],
'userid' => ['type' => 'numeric', 'index' => true],
'type' => ['type' => 'keyword', 'index' => true],
'key' => ['type' => 'text', 'index' => true],
'created_at' => ['type' => 'date', 'index' => true],
'updated_at' => ['type' => 'date', 'index' => true],
]
];
$result = ZincSearchBase::createIndex(self::$indexNameMsg, $mappings);
return $result['success'] ?? false;
}
if (!ZincSearchBase::indexExists(self::$indexNameUser)) {
$mappings = [
'properties' => [
// 拓展数据
'dialog_userid' => ['type' => 'keyword', 'index' => true], // 对话ID+用户ID
// 用户数据
'id' => ['type' => 'numeric', 'index' => true],
'dialog_id' => ['type' => 'numeric', 'index' => true],
'userid' => ['type' => 'numeric', 'index' => true],
'top_at' => ['type' => 'date', 'index' => true],
'last_at' => ['type' => 'date', 'index' => true],
'mark_unread' => ['type' => 'numeric', 'index' => true],
'silence' => ['type' => 'numeric', 'index' => true],
'hide' => ['type' => 'numeric', 'index' => true],
'color' => ['type' => 'keyword', 'index' => true],
// dialog_msgs 字段
'msg_id' => ['type' => 'keyword', 'index' => true],
'dialog_id' => ['type' => 'keyword', 'index' => true],
'sender_userid' => ['type' => 'keyword', 'index' => true],
'msg_type' => ['type' => 'keyword', 'index' => true],
'key' => ['type' => 'text', 'index' => true],
'bot' => ['type' => 'numeric', 'index' => true],
'created_at' => ['type' => 'date', 'index' => true],
'updated_at' => ['type' => 'date', 'index' => true],
]
];
$result = ZincSearchBase::createIndex(self::$indexName, $mappings);
$result = ZincSearchBase::createIndex(self::$indexNameUser, $mappings);
return $result['success'] ?? false;
}
return true;
@@ -84,15 +94,18 @@ class ZincSearchUserMsg
*/
public static function clear(): bool
{
// 检查索引是否存在
if (!ZincSearchBase::indexExists(self::$indexName)) {
return true;
// 检查索引是否存在然后删除
if (ZincSearchBase::indexExists(self::$indexNameMsg)) {
$deleteResult = ZincSearchBase::deleteIndex(self::$indexNameMsg);
if (!($deleteResult['success'] ?? false)) {
return false;
}
}
// 删除再重建索引
$deleteResult = ZincSearchBase::deleteIndex(self::$indexName);
if (!($deleteResult['success'] ?? false)) {
return false;
if (ZincSearchBase::indexExists(self::$indexNameUser)) {
$deleteResult = ZincSearchBase::deleteIndex(self::$indexNameUser);
if (!($deleteResult['success'] ?? false)) {
return false;
}
}
return self::ensureIndex();
@@ -103,7 +116,7 @@ class ZincSearchUserMsg
// ==============================
/**
* 构建对话系统特定的搜索 - 根据用户ID和消息关键词搜索会话
* 根据用户ID和消息关键词搜索会话
*
* @param string $userid 用户ID
* @param string $keyword 消息关键词
@@ -111,7 +124,7 @@ class ZincSearchUserMsg
* @param int $size 返回结果数量
* @return array
*/
public static function searchByKeyword(string $userid, string $keyword, int $from = 0, int $size = 20): array
public static function search(string $userid, string $keyword, int $from = 0, int $size = 20): array
{
$searchParams = [
'query' => [
@@ -131,20 +144,10 @@ class ZincSearchUserMsg
];
try {
$result = ZincSearchBase::elasticSearch(self::$indexName, $searchParams);
$result = ZincSearchBase::elasticSearch(self::$indexNameMsg, $searchParams);
return array_map(function ($hit) {
$source = $hit['_source'];
return [
'id' => $source['dialog_id'],
'top_at' => $source['top_at'],
'last_at' => $source['last_at'],
'mark_unread' => $source['mark_unread'],
'silence' => $source['silence'],
'hide' => $source['hide'],
'color' => $source['color'],
'user_at' => $source['updated_at'],
'search_msg_id' => $source['msg_id'],
];
// todo 格式化消息
return $hit['_source'];
}, $result['data']['hits']['hits'] ?? []);
} catch (\Exception $e) {
Log::error('搜索对话消息失败: ' . $e->getMessage());
@@ -161,23 +164,16 @@ class ZincSearchUserMsg
// ==============================
/**
* 生成文档ID（消息）
* 生成文档ID
*
* @param WebSocketDialogMsg $dialogMsg
* @param WebSocketDialogUser $dialogUser
* @param int $userid
* @return string
*/
private static function generateDocId(WebSocketDialogMsg $dialogMsg, WebSocketDialogUser $dialogUser): string
private static function generateDocId(WebSocketDialogMsg $dialogMsg, int $userid): string
{
return "{$dialogMsg->id}_{$dialogUser->userid}";
return "{$dialogMsg->id}_{$userid}";
}
/**
* 生成文档ID（会话）
*
* @param WebSocketDialogUser $dialogUser
* @return string
*/
private static function generateDialogUserid(WebSocketDialogUser $dialogUser): string
{
return "{$dialogUser->dialog_id}_{$dialogUser->userid}";
@@ -190,28 +186,40 @@ class ZincSearchUserMsg
* @param WebSocketDialogUser $dialogUser
* @return array
*/
private static function generateMsgFormat(WebSocketDialogMsg $dialogMsg, WebSocketDialogUser $dialogUser): array
private static function generateMsgData(WebSocketDialogMsg $dialogMsg, WebSocketDialogUser $dialogUser): array
{
return [
'_id' => self::generateDocId($dialogMsg, $dialogUser),
'_dialog_userid' => self::generateDialogUserid($dialogUser),
'_id' => self::generateDocId($dialogMsg, $dialogUser->userid),
'dialog_userid' => self::generateDialogUserid($dialogUser),
'id' => $dialogMsg->id,
'dialog_id' => $dialogMsg->dialog_id,
'dialog_type' => $dialogMsg->dialog_type,
'session_id' => $dialogMsg->session_id,
'userid' => $dialogMsg->userid,
'type' => $dialogMsg->type,
'key' => $dialogMsg->key,
'created_at' => $dialogMsg->created_at,
'updated_at' => $dialogMsg->updated_at,
];
}
private static function generateUserData(WebSocketDialogUser $dialogUser): array
{
return [
'_id' => $dialogUser->id,
'dialog_userid' => self::generateDialogUserid($dialogUser),
'id' => $dialogUser->id,
'dialog_id' => $dialogUser->dialog_id,
'userid' => $dialogUser->userid,
'top_at' => $dialogUser->top_at,
'last_at' => $dialogUser->last_at,
'mark_unread' => $dialogUser->mark_unread ? 1 : 0,
'silence' => $dialogUser->silence ? 1 : 0,
'hide' => $dialogUser->hide ? 1 : 0,
'mark_unread' => $dialogUser->mark_unread,
'silence' => $dialogUser->silence,
'hide' => $dialogUser->hide,
'color' => $dialogUser->color,
'msg_id' => $dialogMsg->id,
'dialog_id' => $dialogMsg->dialog_id,
'sender_userid' => $dialogMsg->userid,
'msg_type' => $dialogMsg->type,
'key' => $dialogMsg->key,
'bot' => $dialogMsg->bot ? 1 : 0,
'created_at' => $dialogMsg->created_at,
'updated_at' => $dialogMsg->updated_at,
'created_at' => $dialogUser->created_at,
'updated_at' => $dialogUser->updated_at,
];
}
@@ -221,12 +229,17 @@ class ZincSearchUserMsg
* @param WebSocketDialogMsg $dialogMsg
* @return bool
*/
public static function syncMsg(WebSocketDialogMsg $dialogMsg): bool
public static function sync(WebSocketDialogMsg $dialogMsg): bool
{
if (!self::ensureIndex()) {
return false;
}
if ($dialogMsg->bot) {
// 如果是机器人消息，跳过
return true;
}
try {
// 获取此会话的所有用户
$dialogUsers = WebSocketDialogUser::whereDialogId($dialogMsg->dialog_id)->get();
@@ -235,7 +248,8 @@ class ZincSearchUserMsg
return true;
}
$docs = [];
$msgs = [];
$users = [];
foreach ($dialogUsers as $dialogUser) {
if (empty($dialogMsg->key)) {
// 如果消息没有关键词，跳过
@@ -245,11 +259,18 @@ class ZincSearchUserMsg
// 跳过系统用户
continue;
}
$docs[] = self::generateMsgFormat($dialogMsg, $dialogUser);
$msgs[] = self::generateMsgData($dialogMsg, $dialogUser);
$users[$dialogUser->id] = self::generateUserData($dialogUser);
}
if (!empty($docs)) {
ZincSearchBase::addDocs(self::$indexName, $docs);
if ($msgs) {
// 批量写入消息
ZincSearchBase::addDocs(self::$indexNameMsg, $msgs);
}
if ($users) {
// 批量写入用户
ZincSearchBase::addDocs(self::$indexNameUser, array_values($users));
}
return true;
@@ -265,7 +286,7 @@ class ZincSearchUserMsg
* @param WebSocketDialogMsg[] $dialogMsgs
* @return int 成功同步的消息数
*/
public static function batchSyncMsgs($dialogMsgs): int
public static function batchSync($dialogMsgs): int
{
if (!self::ensureIndex()) {
return 0;
@@ -273,7 +294,8 @@ class ZincSearchUserMsg
$count = 0;
try {
$docs = [];
$msgs = [];
$users = [];
$userDialogs = [];
// 预处理：收集所有涉及的对话ID
@@ -299,6 +321,11 @@ class ZincSearchUserMsg
// 如果该会话没有用户，跳过
continue;
}
if ($dialogMsg->bot) {
// 如果是机器人消息，跳过
return true;
}
/** @var WebSocketDialogUser $dialogUser */
foreach ($userDialogs[$dialogMsg->dialog_id] as $dialogUser) {
if (empty($dialogMsg->key)) {
// 如果消息没有关键词，跳过
@@ -308,15 +335,22 @@ class ZincSearchUserMsg
// 跳过系统用户
continue;
}
$docs[] = self::generateMsgFormat($dialogMsg, $dialogUser);
$msgs[] = self::generateMsgData($dialogMsg, $dialogUser);
$users[$dialogUser->id] = self::generateUserData($dialogUser);
$count++;
}
}
// 批量写入
if (!empty($docs)) {
ZincSearchBase::addDocs(self::$indexName, $docs);
if ($msgs) {
// 批量写入消息
ZincSearchBase::addDocs(self::$indexNameMsg, $msgs);
}
if ($users) {
// 批量写入用户
ZincSearchBase::addDocs(self::$indexNameUser, array_values($users));
}
} catch (\Exception $e) {
Log::error('batchSyncMsgs: ' . $e->getMessage());
}
@@ -325,26 +359,53 @@ class ZincSearchUserMsg
}
/**
* 删除消息
* 同步用户
* @param WebSocketDialogUser $dialogUser
* @return bool
*/
public static function userSync(WebSocketDialogUser $dialogUser): bool
{
if (!self::ensureIndex()) {
return false;
}
$data = self::generateUserData($dialogUser);
$result = ZincSearchBase::addDoc(self::$indexNameUser, $data);
return $result['success'] ?? false;
}
/**
* 删除
*
* @param WebSocketDialogMsg $dialogMsg
* @param WebSocketDialogMsg|WebSocketDialogUser $data
* @return int
*/
public static function deleteMsg(WebSocketDialogMsg $dialogMsg): int
public static function delete(mixed $data): int
{
$batchSize = 500; // 每批处理的文档数量
$totalDeleted = 0; // 总共删除的文档数量
$from = 0;
// 根据数据类型生成查询条件
if ($data instanceof WebSocketDialogMsg) {
$query = [
'field' => 'id',
'term' => (string) $data->id
];
} elseif ($data instanceof WebSocketDialogUser) {
$query = [
'field' => 'dialog_userid',
'term' => self::generateDialogUserid($data),
];
} else {
return 0;
}
try {
while (true) {
// 根据消息ID查找相关文档
$result = ZincSearchBase::advancedSearch(self::$indexName, [
$result = ZincSearchBase::advancedSearch(self::$indexNameMsg, [
'search_type' => 'term',
'query' => [
'field' => 'msg_id',
'term' => (string) $dialogMsg->id
],
'query' => $query,
'from' => $from,
'max_results' => $batchSize
]);
@@ -358,7 +419,7 @@ class ZincSearchUserMsg
// 删除本批次找到的所有文档
foreach ($hits as $hit) {
if (isset($hit['_id'])) {
ZincSearchBase::deleteDoc(self::$indexName, $hit['_id']);
ZincSearchBase::deleteDoc(self::$indexNameMsg, $hit['_id']);
$totalDeleted++;
}
}
@@ -377,105 +438,4 @@ class ZincSearchUserMsg
return $totalDeleted;
}
// ==============================
// 用户方法
// ==============================
/**
* 同步用户
*
* @param WebSocketDialogUser $dialogUser
* @return void
*/
public static function syncUser(WebSocketDialogUser $dialogUser): void
{
$batchSize = 500; // 每批处理的文档数量
$lastId = 0; // 上次处理的最后ID
do {
$dialogMsgs = WebSocketDialogMsg::whereDialogId($dialogUser->dialog_id)
->where('id', '>', $lastId)
->orderBy('id')
->limit($batchSize)
->get();
if ($dialogMsgs->isEmpty()) {
break;
}
ZincSearchUserMsg::batchSyncMsgs($dialogMsgs);
// 记录最后处理的ID
$lastId = $dialogMsgs->last()->id;
} while (count($dialogMsgs) == $batchSize);
}
/**
* 批量同步用户
*
* @param WebSocketDialogUser[] $dialogUsers
* @return void
*/
public static function batchSyncUsers($dialogUsers): void
{
foreach ($dialogUsers as $dialogUser) {
self::syncUser($dialogUser);
}
}
/**
* 删除用户
*
* @param WebSocketDialogUser $dialogUser
* @return int
*/
public static function deleteUser(WebSocketDialogUser $dialogUser): int
{
$batchSize = 500; // 每批处理的文档数量
$totalDeleted = 0; // 总共删除的文档数量
$from = 0;
try {
while (true) {
// 根据消息ID查找相关文档
$result = ZincSearchBase::advancedSearch(self::$indexName, [
'search_type' => 'term',
'query' => [
'field' => '_dialog_userid',
'term' => self::generateDialogUserid($dialogUser),
],
'from' => $from,
'max_results' => $batchSize
]);
$hits = $result['data']['hits']['hits'] ?? [];
// 如果没有更多文档，退出循环
if (empty($hits)) {
break;
}
// 删除本批次找到的所有文档
foreach ($hits as $hit) {
if (isset($hit['_id'])) {
ZincSearchBase::deleteDoc(self::$indexName, $hit['_id']);
$totalDeleted++;
}
}
// 如果返回的文档数少于批次大小，说明已经没有更多文档了
if (count($hits) < $batchSize) {
break;
}
// 移动到下一批
$from += $batchSize;
}
} catch (\Exception $e) {
Log::error('deleteUser: ' . $e->getMessage());
}
return $totalDeleted;
}
}