feat: 集成 SeekDB AI 搜索引擎实现文件内容搜索
This commit is contained in:
kuaifan
@@ -738,4 +738,137 @@ class AI
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过 OpenAI 兼容接口获取文本的 Embedding 向量
|
||||
*
|
||||
* @param string $text 需要转换的文本
|
||||
* @param bool $noCache 是否禁用缓存
|
||||
* @return array 返回结果，成功时 data 为向量数组
|
||||
*/
|
||||
public static function getEmbedding($text, $noCache = false)
|
||||
{
|
||||
if (!Apps::isInstalled('ai')) {
|
||||
return Base::retError('应用「AI Assistant」未安装');
|
||||
}
|
||||
|
||||
if (empty($text)) {
|
||||
return Base::retError('文本内容不能为空');
|
||||
}
|
||||
|
||||
// 截断过长的文本（OpenAI 限制 8191 tokens，约 32K 字符）
|
||||
$text = mb_substr($text, 0, 30000);
|
||||
|
||||
$cacheKey = "openAIEmbedding::" . md5($text);
|
||||
if ($noCache) {
|
||||
Cache::forget($cacheKey);
|
||||
}
|
||||
|
||||
$provider = self::resolveEmbeddingProvider();
|
||||
if (!$provider) {
|
||||
return Base::retError("请先在「AI 助手」设置中配置支持 Embedding 的 AI 服务");
|
||||
}
|
||||
|
||||
$result = Cache::remember($cacheKey, Carbon::now()->addDays(7), function () use ($text, $provider) {
|
||||
$payload = [
|
||||
"model" => $provider['model'],
|
||||
"input" => $text,
|
||||
];
|
||||
$post = json_encode($payload);
|
||||
|
||||
$ai = new self($post);
|
||||
$ai->setProvider($provider);
|
||||
$ai->setUrlPath('/embeddings');
|
||||
$ai->setTimeout(30);
|
||||
|
||||
$res = $ai->request(true);
|
||||
if (Base::isError($res)) {
|
||||
return Base::retError("Embedding 请求失败", $res);
|
||||
}
|
||||
|
||||
$resData = Base::json2array($res['data']);
|
||||
if (empty($resData['data'][0]['embedding'])) {
|
||||
return Base::retError("Embedding 接口返回数据格式错误", $resData);
|
||||
}
|
||||
|
||||
$embedding = $resData['data'][0]['embedding'];
|
||||
if (!is_array($embedding) || empty($embedding)) {
|
||||
return Base::retError("Embedding 向量为空");
|
||||
}
|
||||
|
||||
return Base::retSuccess("success", $embedding);
|
||||
});
|
||||
|
||||
if (Base::isError($result)) {
|
||||
Cache::forget($cacheKey);
|
||||
}
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 Embedding 模型配置
|
||||
*
|
||||
* @return array|null
|
||||
*/
|
||||
protected static function resolveEmbeddingProvider()
|
||||
{
|
||||
$setting = Base::setting('aibotSetting');
|
||||
if (!is_array($setting)) {
|
||||
$setting = [];
|
||||
}
|
||||
|
||||
// 优先使用 OpenAI（支持 embedding 接口）
|
||||
$key = trim((string)($setting['openai_key'] ?? ''));
|
||||
if ($key !== '') {
|
||||
$baseUrl = trim((string)($setting['openai_base_url'] ?? ''));
|
||||
$baseUrl = $baseUrl ?: 'https://api.openai.com/v1';
|
||||
$agency = trim((string)($setting['openai_agency'] ?? ''));
|
||||
|
||||
return [
|
||||
'vendor' => 'openai',
|
||||
'model' => 'text-embedding-ada-002',
|
||||
'api_key' => $key,
|
||||
'base_url' => rtrim($baseUrl, '/'),
|
||||
'agency' => $agency,
|
||||
];
|
||||
}
|
||||
|
||||
// 各厂商的默认 baseUrl 和 embedding 模型
|
||||
$vendorDefaults = [
|
||||
'deepseek' => [
|
||||
'base_url' => 'https://api.deepseek.com',
|
||||
'model' => 'deepseek-embedding',
|
||||
],
|
||||
'zhipu' => [
|
||||
'base_url' => 'https://open.bigmodel.cn/api/paas/v4',
|
||||
'model' => 'embedding-2',
|
||||
],
|
||||
'qianwen' => [
|
||||
'base_url' => 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
||||
'model' => 'text-embedding-v3',
|
||||
],
|
||||
];
|
||||
|
||||
// 尝试其他支持 embedding 的服务（如 deepseek、zhipu、qianwen 等）
|
||||
foreach ($vendorDefaults as $vendor => $defaults) {
|
||||
$key = trim((string)($setting[$vendor . '_key'] ?? ''));
|
||||
|
||||
if ($key !== '') {
|
||||
$baseUrl = trim((string)($setting[$vendor . '_base_url'] ?? ''));
|
||||
$baseUrl = $baseUrl ?: $defaults['base_url']; // 使用配置或默认值
|
||||
$agency = trim((string)($setting[$vendor . '_agency'] ?? ''));
|
||||
|
||||
return [
|
||||
'vendor' => $vendor,
|
||||
'model' => $defaults['model'],
|
||||
'api_key' => $key,
|
||||
'base_url' => rtrim($baseUrl, '/'),
|
||||
'agency' => $agency,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,6 +55,7 @@ class Apps
|
||||
'drawio' => 'Drawio',
|
||||
'minder' => 'Minder',
|
||||
'search' => 'ZincSearch',
|
||||
'seekdb' => 'SeekDB',
|
||||
default => $appId,
|
||||
};
|
||||
throw new ApiException("应用「{$name}」未安装", [], 0, false);
|
||||
|
||||
776
app/Module/SeekDB/SeekDBBase.php
Normal file
776
app/Module/SeekDB/SeekDBBase.php
Normal file
@@ -0,0 +1,776 @@
|
||||
<?php
|
||||
|
||||
namespace App\Module\SeekDB;
|
||||
|
||||
use App\Module\Apps;
|
||||
use App\Module\Doo;
|
||||
use PDO;
|
||||
use PDOException;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* SeekDB 基础类
|
||||
*
|
||||
* SeekDB 兼容 MySQL 协议，可以直接使用 PDO 连接
|
||||
*/
|
||||
class SeekDBBase
|
||||
{
|
||||
private static ?PDO $pdo = null;
|
||||
private static bool $initialized = false;
|
||||
|
||||
private string $host;
|
||||
private int $port;
|
||||
private string $user;
|
||||
private string $pass;
|
||||
private string $database;
|
||||
|
||||
/**
|
||||
* 构造函数
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->host = env('SEEKDB_HOST', 'seekdb');
|
||||
$this->port = (int) env('SEEKDB_PORT', 2881);
|
||||
$this->user = env('SEEKDB_USER', 'root');
|
||||
$this->pass = env('SEEKDB_PASSWORD', '');
|
||||
$this->database = env('SEEKDB_DATABASE', 'dootask_search');
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取 PDO 连接
|
||||
*/
|
||||
private function getConnection(): ?PDO
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (self::$pdo === null) {
|
||||
try {
|
||||
// 先连接不指定数据库，用于初始化
|
||||
$dsn = "mysql:host={$this->host};port={$this->port};charset=utf8mb4";
|
||||
$pdo = new PDO($dsn, $this->user, $this->pass, [
|
||||
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
|
||||
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
|
||||
PDO::ATTR_TIMEOUT => 30,
|
||||
]);
|
||||
|
||||
// 初始化数据库和表
|
||||
if (!self::$initialized) {
|
||||
$this->initializeDatabase($pdo);
|
||||
self::$initialized = true;
|
||||
}
|
||||
|
||||
// 切换到目标数据库
|
||||
$pdo->exec("USE `{$this->database}`");
|
||||
self::$pdo = $pdo;
|
||||
} catch (PDOException $e) {
|
||||
Log::error('SeekDB connection failed: ' . $e->getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return self::$pdo;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化数据库和表结构
|
||||
*/
|
||||
private function initializeDatabase(PDO $pdo): void
|
||||
{
|
||||
try {
|
||||
// 创建数据库
|
||||
$pdo->exec("CREATE DATABASE IF NOT EXISTS `{$this->database}`");
|
||||
$pdo->exec("USE `{$this->database}`");
|
||||
|
||||
// 创建文件向量表
|
||||
$pdo->exec("
|
||||
CREATE TABLE IF NOT EXISTS file_vectors (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
file_id BIGINT NOT NULL,
|
||||
userid BIGINT NOT NULL,
|
||||
pshare BIGINT NOT NULL DEFAULT 0,
|
||||
file_name VARCHAR(500),
|
||||
file_type VARCHAR(50),
|
||||
file_ext VARCHAR(20),
|
||||
content LONGTEXT,
|
||||
content_vector VECTOR(1536),
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
UNIQUE KEY uk_file_id (file_id),
|
||||
KEY idx_userid (userid),
|
||||
KEY idx_pshare (pshare),
|
||||
FULLTEXT KEY ft_content (file_name, content)
|
||||
)
|
||||
");
|
||||
|
||||
// 创建键值存储表
|
||||
$pdo->exec("
|
||||
CREATE TABLE IF NOT EXISTS key_values (
|
||||
k VARCHAR(255) PRIMARY KEY,
|
||||
v TEXT,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
)
|
||||
");
|
||||
|
||||
// 创建文件用户关系表（用于权限过滤）
|
||||
$pdo->exec("
|
||||
CREATE TABLE IF NOT EXISTS file_users (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
file_id BIGINT NOT NULL,
|
||||
userid BIGINT NOT NULL,
|
||||
permission TINYINT DEFAULT 0,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
|
||||
UNIQUE KEY uk_file_user (file_id, userid),
|
||||
KEY idx_userid (userid),
|
||||
KEY idx_file_id (file_id)
|
||||
)
|
||||
");
|
||||
|
||||
Log::info('SeekDB database initialized successfully');
|
||||
} catch (PDOException $e) {
|
||||
Log::warning('SeekDB initialization warning: ' . $e->getMessage());
|
||||
// 不抛出异常，表可能已存在
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 重置连接（在长连接环境中使用）
|
||||
*/
|
||||
public static function resetConnection(): void
|
||||
{
|
||||
self::$pdo = null;
|
||||
self::$initialized = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查是否已安装
|
||||
*/
|
||||
public static function isInstalled(): bool
|
||||
{
|
||||
return Apps::isInstalled("seekdb");
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行 SQL（不返回结果）
|
||||
*
|
||||
* @param string $sql SQL语句
|
||||
* @param array $params 参数
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public function execute(string $sql, array $params = []): bool
|
||||
{
|
||||
$pdo = $this->getConnection();
|
||||
if (!$pdo) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
$stmt = $pdo->prepare($sql);
|
||||
return $stmt->execute($params);
|
||||
} catch (PDOException $e) {
|
||||
Log::error('SeekDB execute error: ' . $e->getMessage(), [
|
||||
'sql' => $sql,
|
||||
'params' => $params
|
||||
]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行 SQL 并返回影响行数
|
||||
*
|
||||
* @param string $sql SQL语句
|
||||
* @param array $params 参数
|
||||
* @return int 影响行数，-1 表示失败
|
||||
*/
|
||||
public function executeWithRowCount(string $sql, array $params = []): int
|
||||
{
|
||||
$pdo = $this->getConnection();
|
||||
if (!$pdo) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
try {
|
||||
$stmt = $pdo->prepare($sql);
|
||||
$stmt->execute($params);
|
||||
return $stmt->rowCount();
|
||||
} catch (PDOException $e) {
|
||||
Log::error('SeekDB execute error: ' . $e->getMessage(), [
|
||||
'sql' => $sql,
|
||||
'params' => $params
|
||||
]);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询并返回结果
|
||||
*
|
||||
* @param string $sql SQL语句
|
||||
* @param array $params 参数
|
||||
* @return array 查询结果
|
||||
*/
|
||||
public function query(string $sql, array $params = []): array
|
||||
{
|
||||
$pdo = $this->getConnection();
|
||||
if (!$pdo) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
$stmt = $pdo->prepare($sql);
|
||||
$stmt->execute($params);
|
||||
return $stmt->fetchAll();
|
||||
} catch (PDOException $e) {
|
||||
Log::error('SeekDB query error: ' . $e->getMessage(), [
|
||||
'sql' => $sql,
|
||||
'params' => $params
|
||||
]);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询单行
|
||||
*
|
||||
* @param string $sql SQL语句
|
||||
* @param array $params 参数
|
||||
* @return array|null 单行结果
|
||||
*/
|
||||
public function queryOne(string $sql, array $params = []): ?array
|
||||
{
|
||||
$pdo = $this->getConnection();
|
||||
if (!$pdo) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
$stmt = $pdo->prepare($sql);
|
||||
$stmt->execute($params);
|
||||
$result = $stmt->fetch();
|
||||
return $result ?: null;
|
||||
} catch (PDOException $e) {
|
||||
Log::error('SeekDB queryOne error: ' . $e->getMessage(), [
|
||||
'sql' => $sql,
|
||||
'params' => $params
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取最后插入的 ID
|
||||
*/
|
||||
public function lastInsertId(): ?int
|
||||
{
|
||||
$pdo = $this->getConnection();
|
||||
if (!$pdo) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return (int) $pdo->lastInsertId();
|
||||
} catch (PDOException $e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ==============================
|
||||
// 静态便捷方法
|
||||
// ==============================
|
||||
|
||||
/**
|
||||
* 全文搜索
|
||||
*
|
||||
* @param string $keyword 关键词
|
||||
* @param int $userid 用户ID（0表示不限制权限）
|
||||
* @param int $limit 返回数量
|
||||
* @param int $offset 偏移量
|
||||
* @return array 搜索结果
|
||||
*/
|
||||
public static function fullTextSearch(string $keyword, int $userid = 0, int $limit = 20, int $offset = 0): array
|
||||
{
|
||||
if (empty($keyword)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$instance = new self();
|
||||
$likeKeyword = "%{$keyword}%";
|
||||
|
||||
// 构建 SQL - 同时搜索文件名和内容
|
||||
// 权限过滤通过 JOIN file_users 表实现
|
||||
if ($userid > 0) {
|
||||
// 用户可以看到：1) 自己的文件 2) 共享给自己或公开的文件
|
||||
// 注意：pshare 指向共享根文件夹的 ID，file_users 存储的是共享文件夹的权限关系
|
||||
$sql = "
|
||||
SELECT DISTINCT
|
||||
fv.file_id,
|
||||
fv.userid,
|
||||
fv.file_name,
|
||||
fv.file_type,
|
||||
fv.file_ext,
|
||||
SUBSTRING(fv.content, 1, 500) as content_preview,
|
||||
(
|
||||
CASE WHEN fv.file_name LIKE ? THEN 10 ELSE 0 END +
|
||||
IFNULL(MATCH(fv.content) AGAINST(? IN NATURAL LANGUAGE MODE), 0)
|
||||
) AS relevance
|
||||
FROM file_vectors fv
|
||||
LEFT JOIN file_users fu ON fv.pshare = fu.file_id AND fv.pshare > 0
|
||||
WHERE (fv.file_name LIKE ? OR MATCH(fv.content) AGAINST(? IN NATURAL LANGUAGE MODE))
|
||||
AND (fv.userid = ? OR fu.userid IN (0, ?))
|
||||
ORDER BY relevance DESC
|
||||
LIMIT " . (int)$limit . " OFFSET " . (int)$offset;
|
||||
|
||||
$params = [$likeKeyword, $keyword, $likeKeyword, $keyword, $userid, $userid];
|
||||
} else {
|
||||
// 不限制权限（管理员或后台）
|
||||
$sql = "
|
||||
SELECT
|
||||
file_id,
|
||||
userid,
|
||||
file_name,
|
||||
file_type,
|
||||
file_ext,
|
||||
SUBSTRING(content, 1, 500) as content_preview,
|
||||
(
|
||||
CASE WHEN file_name LIKE ? THEN 10 ELSE 0 END +
|
||||
IFNULL(MATCH(content) AGAINST(? IN NATURAL LANGUAGE MODE), 0)
|
||||
) AS relevance
|
||||
FROM file_vectors
|
||||
WHERE file_name LIKE ? OR MATCH(content) AGAINST(? IN NATURAL LANGUAGE MODE)
|
||||
ORDER BY relevance DESC
|
||||
LIMIT " . (int)$limit . " OFFSET " . (int)$offset;
|
||||
|
||||
$params = [$likeKeyword, $keyword, $likeKeyword, $keyword];
|
||||
}
|
||||
|
||||
return $instance->query($sql, $params);
|
||||
}
|
||||
|
||||
/**
|
||||
* 向量相似度搜索
|
||||
*
|
||||
* @param array $queryVector 查询向量
|
||||
* @param int $userid 用户ID（0表示不限制权限）
|
||||
* @param int $limit 返回数量
|
||||
* @return array 搜索结果
|
||||
*/
|
||||
public static function vectorSearch(array $queryVector, int $userid = 0, int $limit = 20): array
|
||||
{
|
||||
if (empty($queryVector)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$instance = new self();
|
||||
$vectorStr = '[' . implode(',', $queryVector) . ']';
|
||||
|
||||
if ($userid > 0) {
|
||||
// 权限过滤：pshare 指向共享根文件夹的 ID
|
||||
$sql = "
|
||||
SELECT DISTINCT
|
||||
fv.file_id,
|
||||
fv.userid,
|
||||
fv.file_name,
|
||||
fv.file_type,
|
||||
fv.file_ext,
|
||||
SUBSTRING(fv.content, 1, 500) as content_preview,
|
||||
COSINE_SIMILARITY(fv.content_vector, ?) AS similarity
|
||||
FROM file_vectors fv
|
||||
LEFT JOIN file_users fu ON fv.pshare = fu.file_id AND fv.pshare > 0
|
||||
WHERE fv.content_vector IS NOT NULL
|
||||
AND (fv.userid = ? OR fu.userid IN (0, ?))
|
||||
ORDER BY similarity DESC
|
||||
LIMIT " . (int)$limit;
|
||||
|
||||
$params = [$vectorStr, $userid, $userid];
|
||||
} else {
|
||||
// 不限制权限
|
||||
$sql = "
|
||||
SELECT
|
||||
file_id,
|
||||
userid,
|
||||
file_name,
|
||||
file_type,
|
||||
file_ext,
|
||||
SUBSTRING(content, 1, 500) as content_preview,
|
||||
COSINE_SIMILARITY(content_vector, ?) AS similarity
|
||||
FROM file_vectors
|
||||
WHERE content_vector IS NOT NULL
|
||||
ORDER BY similarity DESC
|
||||
LIMIT " . (int)$limit;
|
||||
|
||||
$params = [$vectorStr];
|
||||
}
|
||||
|
||||
return $instance->query($sql, $params);
|
||||
}
|
||||
|
||||
/**
|
||||
* 混合搜索（全文 + 向量，使用 RRF 融合）
|
||||
*
|
||||
* @param string $keyword 关键词
|
||||
* @param array $queryVector 查询向量
|
||||
* @param int $userid 用户ID（0表示不限制权限）
|
||||
* @param int $limit 返回数量
|
||||
* @param float $textWeight 全文搜索权重
|
||||
* @param float $vectorWeight 向量搜索权重
|
||||
* @return array 搜索结果
|
||||
*/
|
||||
public static function hybridSearch(
|
||||
string $keyword,
|
||||
array $queryVector,
|
||||
int $userid = 0,
|
||||
int $limit = 20,
|
||||
float $textWeight = 0.5,
|
||||
float $vectorWeight = 0.5
|
||||
): array {
|
||||
// 分别执行两种搜索（权限过滤在各自方法内通过 JOIN 实现）
|
||||
$textResults = self::fullTextSearch($keyword, $userid, 50, 0);
|
||||
$vectorResults = !empty($queryVector)
|
||||
? self::vectorSearch($queryVector, $userid, 50)
|
||||
: [];
|
||||
|
||||
// 使用 RRF (Reciprocal Rank Fusion) 融合结果
|
||||
$scores = [];
|
||||
$items = [];
|
||||
$k = 60; // RRF 常数
|
||||
|
||||
// 处理全文搜索结果
|
||||
foreach ($textResults as $rank => $item) {
|
||||
$fileId = $item['file_id'];
|
||||
$scores[$fileId] = ($scores[$fileId] ?? 0) + $textWeight / ($k + $rank + 1);
|
||||
$items[$fileId] = $item;
|
||||
}
|
||||
|
||||
// 处理向量搜索结果
|
||||
foreach ($vectorResults as $rank => $item) {
|
||||
$fileId = $item['file_id'];
|
||||
$scores[$fileId] = ($scores[$fileId] ?? 0) + $vectorWeight / ($k + $rank + 1);
|
||||
if (!isset($items[$fileId])) {
|
||||
$items[$fileId] = $item;
|
||||
}
|
||||
}
|
||||
|
||||
// 按融合分数排序
|
||||
arsort($scores);
|
||||
|
||||
// 构建最终结果
|
||||
$results = [];
|
||||
$count = 0;
|
||||
foreach ($scores as $fileId => $score) {
|
||||
if ($count >= $limit) {
|
||||
break;
|
||||
}
|
||||
$item = $items[$fileId];
|
||||
$item['rrf_score'] = $score;
|
||||
$results[] = $item;
|
||||
$count++;
|
||||
}
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
/**
|
||||
* 插入或更新文件向量
|
||||
*
|
||||
* @param array $data 文件数据
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function upsertFileVector(array $data): bool
|
||||
{
|
||||
$instance = new self();
|
||||
|
||||
$fileId = $data['file_id'] ?? 0;
|
||||
if ($fileId <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 检查是否存在
|
||||
$existing = $instance->queryOne(
|
||||
"SELECT id FROM file_vectors WHERE file_id = ?",
|
||||
[$fileId]
|
||||
);
|
||||
|
||||
if ($existing) {
|
||||
// 更新
|
||||
$sql = "UPDATE file_vectors SET
|
||||
userid = ?,
|
||||
pshare = ?,
|
||||
file_name = ?,
|
||||
file_type = ?,
|
||||
file_ext = ?,
|
||||
content = ?,
|
||||
content_vector = ?,
|
||||
updated_at = NOW()
|
||||
WHERE file_id = ?";
|
||||
|
||||
$params = [
|
||||
$data['userid'] ?? 0,
|
||||
$data['pshare'] ?? 0,
|
||||
$data['file_name'] ?? '',
|
||||
$data['file_type'] ?? '',
|
||||
$data['file_ext'] ?? '',
|
||||
$data['content'] ?? '',
|
||||
$data['content_vector'] ?? null,
|
||||
$fileId
|
||||
];
|
||||
} else {
|
||||
// 插入
|
||||
$sql = "INSERT INTO file_vectors
|
||||
(file_id, userid, pshare, file_name, file_type, file_ext, content, content_vector, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, NOW(), NOW())";
|
||||
|
||||
$params = [
|
||||
$fileId,
|
||||
$data['userid'] ?? 0,
|
||||
$data['pshare'] ?? 0,
|
||||
$data['file_name'] ?? '',
|
||||
$data['file_type'] ?? '',
|
||||
$data['file_ext'] ?? '',
|
||||
$data['content'] ?? '',
|
||||
$data['content_vector'] ?? null
|
||||
];
|
||||
}
|
||||
|
||||
return $instance->execute($sql, $params);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除文件向量
|
||||
*
|
||||
* @param int $fileId 文件ID
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function deleteFileVector(int $fileId): bool
|
||||
{
|
||||
if ($fileId <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$instance = new self();
|
||||
return $instance->execute(
|
||||
"DELETE FROM file_vectors WHERE file_id = ?",
|
||||
[$fileId]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量删除文件向量
|
||||
*
|
||||
* @param array $fileIds 文件ID列表
|
||||
* @return int 删除数量
|
||||
*/
|
||||
public static function batchDeleteFileVectors(array $fileIds): int
|
||||
{
|
||||
if (empty($fileIds)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$instance = new self();
|
||||
$placeholders = implode(',', array_fill(0, count($fileIds), '?'));
|
||||
|
||||
return $instance->executeWithRowCount(
|
||||
"DELETE FROM file_vectors WHERE file_id IN ({$placeholders})",
|
||||
$fileIds
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量更新文件的 pshare 值
|
||||
*
|
||||
* @param array $fileIds 文件ID列表
|
||||
* @param int $pshare 新的 pshare 值
|
||||
* @return int 更新数量
|
||||
*/
|
||||
public static function batchUpdatePshare(array $fileIds, int $pshare): int
|
||||
{
|
||||
if (empty($fileIds)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$instance = new self();
|
||||
$placeholders = implode(',', array_fill(0, count($fileIds), '?'));
|
||||
$params = array_merge([$pshare], $fileIds);
|
||||
|
||||
return $instance->executeWithRowCount(
|
||||
"UPDATE file_vectors SET pshare = ?, updated_at = NOW() WHERE file_id IN ({$placeholders})",
|
||||
$params
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空所有文件向量
|
||||
*
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function clearAllFileVectors(): bool
|
||||
{
|
||||
$instance = new self();
|
||||
return $instance->execute("TRUNCATE TABLE file_vectors");
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取已索引的文件数量
|
||||
*
|
||||
* @return int 文件数量
|
||||
*/
|
||||
public static function getIndexedFileCount(): int
|
||||
{
|
||||
$instance = new self();
|
||||
$result = $instance->queryOne("SELECT COUNT(*) as cnt FROM file_vectors");
|
||||
return $result ? (int) $result['cnt'] : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取最后索引的文件ID
|
||||
*
|
||||
* @return int 文件ID
|
||||
*/
|
||||
public static function getLastIndexedFileId(): int
|
||||
{
|
||||
$instance = new self();
|
||||
$result = $instance->queryOne("SELECT MAX(file_id) as max_id FROM file_vectors");
|
||||
return $result ? (int) ($result['max_id'] ?? 0) : 0;
|
||||
}
|
||||
|
||||
// ==============================
|
||||
// 文件用户关系方法
|
||||
// ==============================
|
||||
|
||||
/**
|
||||
* 插入或更新文件用户关系
|
||||
*
|
||||
* @param int $fileId 文件ID
|
||||
* @param int $userid 用户ID（0表示公开）
|
||||
* @param int $permission 权限（0只读，1读写）
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function upsertFileUser(int $fileId, int $userid, int $permission = 0): bool
|
||||
{
|
||||
if ($fileId <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$instance = new self();
|
||||
|
||||
// 检查是否存在
|
||||
$existing = $instance->queryOne(
|
||||
"SELECT id FROM file_users WHERE file_id = ? AND userid = ?",
|
||||
[$fileId, $userid]
|
||||
);
|
||||
|
||||
if ($existing) {
|
||||
// 更新
|
||||
return $instance->execute(
|
||||
"UPDATE file_users SET permission = ?, updated_at = NOW() WHERE file_id = ? AND userid = ?",
|
||||
[$permission, $fileId, $userid]
|
||||
);
|
||||
} else {
|
||||
// 插入
|
||||
return $instance->execute(
|
||||
"INSERT INTO file_users (file_id, userid, permission) VALUES (?, ?, ?)",
|
||||
[$fileId, $userid, $permission]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量同步文件用户关系（替换指定文件的所有关系）
|
||||
*
|
||||
* @param int $fileId 文件ID
|
||||
* @param array $users 用户列表 [['userid' => int, 'permission' => int], ...]
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function syncFileUsers(int $fileId, array $users): bool
|
||||
{
|
||||
if ($fileId <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$instance = new self();
|
||||
|
||||
try {
|
||||
// 删除旧关系
|
||||
$instance->execute("DELETE FROM file_users WHERE file_id = ?", [$fileId]);
|
||||
|
||||
// 插入新关系
|
||||
foreach ($users as $user) {
|
||||
$userid = (int)($user['userid'] ?? 0);
|
||||
$permission = (int)($user['permission'] ?? 0);
|
||||
$instance->execute(
|
||||
"INSERT INTO file_users (file_id, userid, permission) VALUES (?, ?, ?)",
|
||||
[$fileId, $userid, $permission]
|
||||
);
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch (\Exception $e) {
|
||||
Log::error('SeekDB syncFileUsers error: ' . $e->getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除文件的所有用户关系
|
||||
*
|
||||
* @param int $fileId 文件ID
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function deleteFileUsers(int $fileId): bool
|
||||
{
|
||||
if ($fileId <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$instance = new self();
|
||||
return $instance->execute("DELETE FROM file_users WHERE file_id = ?", [$fileId]);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除指定文件和用户的关系
|
||||
*
|
||||
* @param int $fileId 文件ID
|
||||
* @param int $userid 用户ID
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function deleteFileUser(int $fileId, int $userid): bool
|
||||
{
|
||||
if ($fileId <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$instance = new self();
|
||||
return $instance->execute(
|
||||
"DELETE FROM file_users WHERE file_id = ? AND userid = ?",
|
||||
[$fileId, $userid]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取文件用户关系数量
|
||||
*
|
||||
* @return int 关系数量
|
||||
*/
|
||||
public static function getFileUserCount(): int
|
||||
{
|
||||
$instance = new self();
|
||||
$result = $instance->queryOne("SELECT COUNT(*) as cnt FROM file_users");
|
||||
return $result ? (int) $result['cnt'] : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空所有文件用户关系
|
||||
*
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function clearAllFileUsers(): bool
|
||||
{
|
||||
$instance = new self();
|
||||
return $instance->execute("TRUNCATE TABLE file_users");
|
||||
}
|
||||
}
|
||||
|
||||
539
app/Module/SeekDB/SeekDBFile.php
Normal file
539
app/Module/SeekDB/SeekDBFile.php
Normal file
@@ -0,0 +1,539 @@
|
||||
<?php
|
||||
|
||||
namespace App\Module\SeekDB;
|
||||
|
||||
use App\Models\File;
|
||||
use App\Models\FileContent;
|
||||
use App\Models\FileUser;
|
||||
use App\Module\Apps;
|
||||
use App\Module\Base;
|
||||
use App\Module\TextExtractor;
|
||||
use App\Module\AI;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
/**
|
||||
* SeekDB 文件搜索类
|
||||
*
|
||||
* 使用方法:
|
||||
*
|
||||
* 1. 搜索方法
|
||||
* - 搜索文件: search($userid, $keyword, $searchType, $from, $size);
|
||||
*
|
||||
* 2. 同步方法
|
||||
* - 单个同步: sync(File $file);
|
||||
* - 批量同步: batchSync($files);
|
||||
* - 删除索引: delete($fileId);
|
||||
*
|
||||
* 3. 工具方法
|
||||
* - 清空索引: clear();
|
||||
*/
|
||||
class SeekDBFile
|
||||
{
|
||||
/**
|
||||
* 可搜索的文件类型
|
||||
*/
|
||||
public const SEARCHABLE_TYPES = ['document', 'word', 'excel', 'ppt', 'txt', 'md', 'text', 'code'];
|
||||
|
||||
/**
|
||||
* 最大内容长度（字符）- 提取后的文本内容限制
|
||||
*/
|
||||
public const MAX_CONTENT_LENGTH = 100000; // 100K 字符
|
||||
|
||||
/**
|
||||
* 不同文件类型的最大大小限制（字节）
|
||||
*/
|
||||
public const MAX_FILE_SIZE = [
|
||||
'office' => 50 * 1024 * 1024, // 50MB - Office 文件图片占空间大但文本少
|
||||
'text' => 5 * 1024 * 1024, // 5MB - 纯文本文件
|
||||
'other' => 20 * 1024 * 1024, // 20MB - PDF 等其他文件
|
||||
];
|
||||
|
||||
/**
|
||||
* Office 文件扩展名
|
||||
*/
|
||||
public const OFFICE_EXTENSIONS = [
|
||||
'doc', 'docx', 'dot', 'dotx', 'odt', 'ott', 'rtf',
|
||||
'xls', 'xlsx', 'xlsm', 'xlt', 'xltx', 'ods', 'ots', 'csv', 'tsv',
|
||||
'ppt', 'pptx', 'pps', 'ppsx', 'odp', 'otp'
|
||||
];
|
||||
|
||||
/**
|
||||
* 纯文本文件扩展名
|
||||
*/
|
||||
public const TEXT_EXTENSIONS = [
|
||||
'txt', 'md', 'text', 'log', 'json', 'xml', 'html', 'htm', 'css', 'js', 'ts',
|
||||
'php', 'py', 'java', 'c', 'cpp', 'h', 'go', 'rs', 'rb', 'sh', 'bash', 'sql',
|
||||
'yaml', 'yml', 'ini', 'conf', 'vue', 'jsx', 'tsx'
|
||||
];
|
||||
|
||||
/**
|
||||
* 搜索文件（支持全文、向量、混合搜索）
|
||||
*
|
||||
* @param int $userid 用户ID
|
||||
* @param string $keyword 搜索关键词
|
||||
* @param string $searchType 搜索类型: text/vector/hybrid
|
||||
* @param int $from 起始位置
|
||||
* @param int $size 返回数量
|
||||
* @return array 搜索结果
|
||||
*/
|
||||
public static function search(int $userid, string $keyword, string $searchType = 'hybrid', int $from = 0, int $size = 20): array
|
||||
{
|
||||
if (empty($keyword)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (!Apps::isInstalled("seekdb")) {
|
||||
// 未安装 SeekDB，降级到 MySQL LIKE 搜索
|
||||
return self::searchByMysql($userid, $keyword, $from, $size);
|
||||
}
|
||||
|
||||
try {
|
||||
// 权限过滤已在 SeekDBBase 中通过 JOIN file_users 表实现
|
||||
switch ($searchType) {
|
||||
case 'text':
|
||||
// 纯全文搜索
|
||||
return self::formatSearchResults(
|
||||
SeekDBBase::fullTextSearch($keyword, $userid, $size, $from)
|
||||
);
|
||||
|
||||
case 'vector':
|
||||
// 纯向量搜索（需要先获取 embedding）
|
||||
$embedding = self::getEmbedding($keyword);
|
||||
if (empty($embedding)) {
|
||||
// embedding 获取失败，降级到全文搜索
|
||||
return self::formatSearchResults(
|
||||
SeekDBBase::fullTextSearch($keyword, $userid, $size, $from)
|
||||
);
|
||||
}
|
||||
return self::formatSearchResults(
|
||||
SeekDBBase::vectorSearch($embedding, $userid, $size)
|
||||
);
|
||||
|
||||
case 'hybrid':
|
||||
default:
|
||||
// 混合搜索
|
||||
$embedding = self::getEmbedding($keyword);
|
||||
return self::formatSearchResults(
|
||||
SeekDBBase::hybridSearch($keyword, $embedding, $userid, $size)
|
||||
);
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
Log::error('SeekDB search error: ' . $e->getMessage());
|
||||
return self::searchByMysql($userid, $keyword, $from, $size);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取文本的 Embedding 向量
|
||||
*
|
||||
* @param string $text 文本
|
||||
* @return array 向量数组（空数组表示失败）
|
||||
*/
|
||||
private static function getEmbedding(string $text): array
|
||||
{
|
||||
if (empty($text)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
// 调用 AI 模块获取 embedding
|
||||
$result = AI::getEmbedding($text);
|
||||
if (Base::isSuccess($result)) {
|
||||
return $result['data'] ?? [];
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
Log::warning('Get embedding error: ' . $e->getMessage());
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化搜索结果
|
||||
*
|
||||
* @param array $results SeekDB 返回的结果
|
||||
* @return array 格式化后的结果
|
||||
*/
|
||||
private static function formatSearchResults(array $results): array
|
||||
{
|
||||
$formatted = [];
|
||||
foreach ($results as $item) {
|
||||
$formatted[] = [
|
||||
'id' => $item['file_id'],
|
||||
'file_id' => $item['file_id'],
|
||||
'name' => $item['file_name'],
|
||||
'type' => $item['file_type'],
|
||||
'ext' => $item['file_ext'],
|
||||
'userid' => $item['userid'],
|
||||
'content_preview' => $item['content_preview'] ?? null,
|
||||
'relevance' => $item['relevance'] ?? $item['similarity'] ?? $item['rrf_score'] ?? 0,
|
||||
];
|
||||
}
|
||||
return $formatted;
|
||||
}
|
||||
|
||||
/**
|
||||
* MySQL 降级搜索（仅搜索文件名）
|
||||
*
|
||||
* @param int $userid 用户ID
|
||||
* @param string $keyword 关键词
|
||||
* @param int $from 起始位置
|
||||
* @param int $size 返回数量
|
||||
* @return array 搜索结果
|
||||
*/
|
||||
private static function searchByMysql(int $userid, string $keyword, int $from, int $size): array
|
||||
{
|
||||
// 搜索用户自己的文件
|
||||
$builder = File::where('userid', $userid)
|
||||
->where('name', 'like', "%{$keyword}%")
|
||||
->where('type', '!=', 'folder');
|
||||
|
||||
$results = $builder->skip($from)->take($size)->get();
|
||||
|
||||
return $results->map(function ($file) {
|
||||
return [
|
||||
'id' => $file->id,
|
||||
'file_id' => $file->id,
|
||||
'name' => $file->name,
|
||||
'type' => $file->type,
|
||||
'ext' => $file->ext,
|
||||
'userid' => $file->userid,
|
||||
'content_preview' => null,
|
||||
'relevance' => 0,
|
||||
];
|
||||
})->toArray();
|
||||
}
|
||||
|
||||
// ==============================
|
||||
// 同步方法
|
||||
// ==============================
|
||||
|
||||
/**
|
||||
* 同步单个文件到 SeekDB
|
||||
*
|
||||
* @param File $file 文件模型
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function sync(File $file): bool
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb")) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 不处理文件夹
|
||||
if ($file->type === 'folder') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 根据文件类型检查大小限制
|
||||
$maxSize = self::getMaxFileSizeByExt($file->ext);
|
||||
if ($file->size > $maxSize) {
|
||||
Log::info("SeekDB: Skip large file {$file->id} ({$file->size} bytes, max: {$maxSize})");
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
// 提取文件内容
|
||||
$content = self::extractFileContent($file);
|
||||
|
||||
// 限制提取后的内容长度
|
||||
$content = mb_substr($content, 0, self::MAX_CONTENT_LENGTH);
|
||||
|
||||
// 获取 embedding（如果有内容且 AI 可用）
|
||||
$embedding = null;
|
||||
if (!empty($content) && Apps::isInstalled('ai')) {
|
||||
$embeddingResult = self::getEmbedding($content);
|
||||
if (!empty($embeddingResult)) {
|
||||
$embedding = '[' . implode(',', $embeddingResult) . ']';
|
||||
}
|
||||
}
|
||||
|
||||
// 写入 SeekDB
|
||||
// pshare 指向共享根文件夹的 ID，用于权限过滤
|
||||
$result = SeekDBBase::upsertFileVector([
|
||||
'file_id' => $file->id,
|
||||
'userid' => $file->userid,
|
||||
'pshare' => $file->pshare ?? 0,
|
||||
'file_name' => $file->name,
|
||||
'file_type' => $file->type,
|
||||
'file_ext' => $file->ext,
|
||||
'content' => $content,
|
||||
'content_vector' => $embedding,
|
||||
]);
|
||||
|
||||
// 注意：file_users 只需要同步共享文件夹的关系，不需要同步每个文件
|
||||
// 因为搜索时是通过 pshare 关联 file_users 表
|
||||
|
||||
return $result;
|
||||
} catch (\Exception $e) {
|
||||
Log::error('SeekDB sync error: ' . $e->getMessage(), [
|
||||
'file_id' => $file->id,
|
||||
'file_name' => $file->name,
|
||||
]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据文件扩展名获取最大文件大小限制
|
||||
*
|
||||
* @param string|null $ext 文件扩展名
|
||||
* @return int 最大文件大小（字节）
|
||||
*/
|
||||
private static function getMaxFileSizeByExt(?string $ext): int
|
||||
{
|
||||
$ext = strtolower($ext ?? '');
|
||||
|
||||
if (in_array($ext, self::OFFICE_EXTENSIONS)) {
|
||||
return self::MAX_FILE_SIZE['office'];
|
||||
}
|
||||
|
||||
if (in_array($ext, self::TEXT_EXTENSIONS)) {
|
||||
return self::MAX_FILE_SIZE['text'];
|
||||
}
|
||||
|
||||
return self::MAX_FILE_SIZE['other'];
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有文件类型中的最大文件大小限制
|
||||
*
|
||||
* @return int 最大文件大小（字节）
|
||||
*/
|
||||
public static function getMaxFileSize(): int
|
||||
{
|
||||
return max(self::MAX_FILE_SIZE);
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量同步文件
|
||||
*
|
||||
* @param iterable $files 文件列表
|
||||
* @return int 成功同步的数量
|
||||
*/
|
||||
public static function batchSync(iterable $files): int
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb")) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$count = 0;
|
||||
foreach ($files as $file) {
|
||||
if (self::sync($file)) {
|
||||
$count++;
|
||||
}
|
||||
}
|
||||
return $count;
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除文件索引
|
||||
*
|
||||
* @param int $fileId 文件ID
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function delete(int $fileId): bool
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb")) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return SeekDBBase::deleteFileVector($fileId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取文件内容
|
||||
*
|
||||
* @param File $file 文件模型
|
||||
* @return string 文件内容文本
|
||||
*/
|
||||
private static function extractFileContent(File $file): string
|
||||
{
|
||||
// 1. 先尝试从 FileContent 的 text 字段获取（已提取的文本内容）
|
||||
$fileContent = FileContent::where('fid', $file->id)->orderByDesc('id')->first();
|
||||
if ($fileContent && !empty($fileContent->text)) {
|
||||
return $fileContent->text;
|
||||
}
|
||||
|
||||
// 2. 尝试从 FileContent 的 content 字段获取
|
||||
if ($fileContent && !empty($fileContent->content)) {
|
||||
$contentData = Base::json2array($fileContent->content);
|
||||
|
||||
// 2.1 某些文件类型直接存储内容
|
||||
if (!empty($contentData['content'])) {
|
||||
return is_string($contentData['content']) ? $contentData['content'] : '';
|
||||
}
|
||||
|
||||
// 2.2 尝试使用 TextExtractor 提取文件内容
|
||||
$filePath = $contentData['url'] ?? null;
|
||||
if ($filePath && str_starts_with($filePath, 'uploads/')) {
|
||||
$fullPath = public_path($filePath);
|
||||
if (file_exists($fullPath)) {
|
||||
// 根据文件类型设置不同的大小限制
|
||||
$ext = strtolower(pathinfo($fullPath, PATHINFO_EXTENSION));
|
||||
$maxFileSize = self::getMaxFileSizeByExt($ext);
|
||||
$maxContentSize = self::MAX_CONTENT_LENGTH;
|
||||
|
||||
$result = TextExtractor::extractFile(
|
||||
$fullPath,
|
||||
(int) ($maxFileSize / 1024), // 转换为 KB
|
||||
(int) ($maxContentSize / 1024) // 转换为 KB
|
||||
);
|
||||
if (Base::isSuccess($result)) {
|
||||
return $result['data'] ?? '';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空所有索引
|
||||
*
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function clear(): bool
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb")) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return SeekDBBase::clearAllFileVectors();
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取已索引文件数量
|
||||
*
|
||||
* @return int 数量
|
||||
*/
|
||||
public static function getIndexedCount(): int
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb")) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return SeekDBBase::getIndexedFileCount();
|
||||
}
|
||||
|
||||
// ==============================
|
||||
// 文件用户关系同步方法
|
||||
// ==============================
|
||||
|
||||
/**
|
||||
* 同步单个文件的用户关系到 SeekDB
|
||||
*
|
||||
* @param int $fileId 文件ID
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function syncFileUsers(int $fileId): bool
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb") || $fileId <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
// 从 MySQL 获取文件的用户关系
|
||||
$users = FileUser::where('file_id', $fileId)
|
||||
->select(['userid', 'permission'])
|
||||
->get()
|
||||
->map(function ($item) {
|
||||
return [
|
||||
'userid' => $item->userid,
|
||||
'permission' => $item->permission,
|
||||
];
|
||||
})
|
||||
->toArray();
|
||||
|
||||
// 同步到 SeekDB
|
||||
return SeekDBBase::syncFileUsers($fileId, $users);
|
||||
} catch (\Exception $e) {
|
||||
Log::error('SeekDB syncFileUsers error: ' . $e->getMessage(), ['file_id' => $fileId]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 添加文件用户关系到 SeekDB
|
||||
*
|
||||
* @param int $fileId 文件ID
|
||||
* @param int $userid 用户ID
|
||||
* @param int $permission 权限
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function addFileUser(int $fileId, int $userid, int $permission = 0): bool
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb") || $fileId <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return SeekDBBase::upsertFileUser($fileId, $userid, $permission);
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除文件用户关系
|
||||
*
|
||||
* @param int $fileId 文件ID
|
||||
* @param int|null $userid 用户ID，null 表示删除所有
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function removeFileUser(int $fileId, ?int $userid = null): bool
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb") || $fileId <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if ($userid === null) {
|
||||
return SeekDBBase::deleteFileUsers($fileId);
|
||||
}
|
||||
|
||||
return SeekDBBase::deleteFileUser($fileId, $userid);
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量同步所有文件用户关系（全量同步）
|
||||
*
|
||||
* @param callable|null $progressCallback 进度回调
|
||||
* @return int 同步数量
|
||||
*/
|
||||
public static function syncAllFileUsers(?callable $progressCallback = null): int
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb")) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$count = 0;
|
||||
$lastId = 0;
|
||||
$batchSize = 1000;
|
||||
|
||||
// 先清空 SeekDB 中的 file_users 表
|
||||
SeekDBBase::clearAllFileUsers();
|
||||
|
||||
// 分批同步
|
||||
while (true) {
|
||||
$records = FileUser::where('id', '>', $lastId)
|
||||
->orderBy('id')
|
||||
->limit($batchSize)
|
||||
->get();
|
||||
|
||||
if ($records->isEmpty()) {
|
||||
break;
|
||||
}
|
||||
|
||||
foreach ($records as $record) {
|
||||
SeekDBBase::upsertFileUser($record->file_id, $record->userid, $record->permission);
|
||||
$count++;
|
||||
$lastId = $record->id;
|
||||
}
|
||||
|
||||
if ($progressCallback) {
|
||||
$progressCallback($count);
|
||||
}
|
||||
}
|
||||
|
||||
return $count;
|
||||
}
|
||||
}
|
||||
|
||||
181
app/Module/SeekDB/SeekDBKeyValue.php
Normal file
181
app/Module/SeekDB/SeekDBKeyValue.php
Normal file
@@ -0,0 +1,181 @@
|
||||
<?php
|
||||
|
||||
namespace App\Module\SeekDB;
|
||||
|
||||
use App\Module\Apps;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* SeekDB 键值存储类
|
||||
*
|
||||
* 用于存储同步进度、配置等键值数据
|
||||
*
|
||||
* 使用方法:
|
||||
*
|
||||
* 1. 基本操作
|
||||
* - 设置键值: set('sync_last_id', 12345);
|
||||
* - 获取键值: $lastId = get('sync_last_id', 0);
|
||||
* - 删除键值: delete('sync_last_id');
|
||||
*
|
||||
* 2. 批量操作
|
||||
* - 批量设置: batchSet(['key1' => 'value1', 'key2' => 'value2']);
|
||||
* - 批量获取: $values = batchGet(['key1', 'key2']);
|
||||
*/
|
||||
class SeekDBKeyValue
|
||||
{
|
||||
/**
|
||||
* 设置键值
|
||||
*
|
||||
* @param string $key 键名
|
||||
* @param mixed $value 值（会被 JSON 编码）
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function set(string $key, mixed $value): bool
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb") || empty($key)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$instance = new SeekDBBase();
|
||||
$jsonValue = is_string($value) ? $value : json_encode($value, JSON_UNESCAPED_UNICODE);
|
||||
|
||||
// 使用 REPLACE INTO 实现 upsert
|
||||
$sql = "REPLACE INTO key_values (k, v, updated_at) VALUES (?, ?, NOW())";
|
||||
|
||||
return $instance->execute($sql, [$key, $jsonValue]);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取键值
|
||||
*
|
||||
* @param string $key 键名
|
||||
* @param mixed $default 默认值
|
||||
* @return mixed 值或默认值
|
||||
*/
|
||||
public static function get(string $key, mixed $default = null): mixed
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb") || empty($key)) {
|
||||
return $default;
|
||||
}
|
||||
|
||||
$instance = new SeekDBBase();
|
||||
$result = $instance->queryOne(
|
||||
"SELECT v FROM key_values WHERE k = ?",
|
||||
[$key]
|
||||
);
|
||||
|
||||
if (!$result || !isset($result['v'])) {
|
||||
return $default;
|
||||
}
|
||||
|
||||
$value = $result['v'];
|
||||
|
||||
// 尝试 JSON 解码
|
||||
$decoded = json_decode($value, true);
|
||||
if (json_last_error() === JSON_ERROR_NONE) {
|
||||
return $decoded;
|
||||
}
|
||||
|
||||
return $value;
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除键值
|
||||
*
|
||||
* @param string $key 键名
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function delete(string $key): bool
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb") || empty($key)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$instance = new SeekDBBase();
|
||||
return $instance->execute(
|
||||
"DELETE FROM key_values WHERE k = ?",
|
||||
[$key]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量设置键值
|
||||
*
|
||||
* @param array $keyValues 键值对数组
|
||||
* @return bool 是否全部成功
|
||||
*/
|
||||
public static function batchSet(array $keyValues): bool
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb") || empty($keyValues)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$instance = new SeekDBBase();
|
||||
$success = true;
|
||||
|
||||
foreach ($keyValues as $key => $value) {
|
||||
$jsonValue = is_string($value) ? $value : json_encode($value, JSON_UNESCAPED_UNICODE);
|
||||
$result = $instance->execute(
|
||||
"REPLACE INTO key_values (k, v, updated_at) VALUES (?, ?, NOW())",
|
||||
[$key, $jsonValue]
|
||||
);
|
||||
if (!$result) {
|
||||
$success = false;
|
||||
}
|
||||
}
|
||||
|
||||
return $success;
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量获取键值
|
||||
*
|
||||
* @param array $keys 键名数组
|
||||
* @return array 键值对数组
|
||||
*/
|
||||
public static function batchGet(array $keys): array
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb") || empty($keys)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$instance = new SeekDBBase();
|
||||
$placeholders = implode(',', array_fill(0, count($keys), '?'));
|
||||
$results = $instance->query(
|
||||
"SELECT k, v FROM key_values WHERE k IN ({$placeholders})",
|
||||
$keys
|
||||
);
|
||||
|
||||
$values = [];
|
||||
foreach ($results as $row) {
|
||||
$value = $row['v'];
|
||||
$decoded = json_decode($value, true);
|
||||
$values[$row['k']] = (json_last_error() === JSON_ERROR_NONE) ? $decoded : $value;
|
||||
}
|
||||
|
||||
// 填充未找到的键为 null
|
||||
foreach ($keys as $key) {
|
||||
if (!isset($values[$key])) {
|
||||
$values[$key] = null;
|
||||
}
|
||||
}
|
||||
|
||||
return $values;
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空所有键值
|
||||
*
|
||||
* @return bool 是否成功
|
||||
*/
|
||||
public static function clear(): bool
|
||||
{
|
||||
if (!Apps::isInstalled("seekdb")) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$instance = new SeekDBBase();
|
||||
return $instance->execute("TRUNCATE TABLE key_values");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user