feat: 迁移Pocketbase至Matrix

.env.example

@@ -1,17 +1,9 @@
 # Node Environment: dev, production
 NODE_ENV=dev
 
-LLM_MODEL=
-LLM_API_KEY=
-LLM_BASE_URL=
+DATABASE_URL=
 
-LANGFUSE_PK=
-LANGFUSE_SK=
-LANGFUSE_BASE_URL=
-
-# Map of apps such as '{"michat": { "app_id": "123", "app_secret": "456", "app_name": "Mi Chat" }}'
-LARK_APP_MAP=
-
-ATTACH_APP_SECRET=
-
-DATABASE_URL=
+# PocketBase Auth
+PB_USER=
+PB_PASS=
+PB_URL=

constant/appMap.ts

@@ -1,13 +0,0 @@
-import { parseJsonString } from "@egg/hooks"
-
-export interface AppInfo {
-  app_id: string
-  app_secret: string
-  app_name: string
-}
-
-// 获取所有应用信息
-export const appMap = parseJsonString(process.env.LARK_APP_MAP, {}) as Record<
-  string,
-  AppInfo
->

constant/config.ts

@@ -0,0 +1,45 @@
+import logger from "@egg/logger"
+import { RecordModel } from "pocketbase"
+
+import pbClient from "../db/pbClient"
+
+interface Config extends RecordModel {
+  key: string
+  value: string
+  desc: string
+}
+
+export interface AppInfo extends RecordModel {
+  name: string
+  app_id: string
+  app_secret: string
+  app_name: string
+}
+
+export const APP_CONFIG: Record<string, string> = {}
+
+export const APP_MAP: Record<string, Omit<AppInfo, "name">> = {}
+
+/**
+ * 初始化应用配置
+ */
+const initAppConfig = async () => {
+  // 获取所有环境变量
+  const envList = await pbClient.collection<Config>("env").getFullList()
+  for (const env of envList) {
+    APP_CONFIG[env.key] = env.value
+  }
+  logger.info(`Get env list: ${JSON.stringify(APP_CONFIG)}`)
+  // 获取所有应用信息
+  const appList = await pbClient.collection<AppInfo>("app").getFullList()
+  for (const app of appList) {
+    APP_MAP[app.name] = {
+      app_id: app.app_id,
+      app_secret: app.app_secret,
+      app_name: app.app_name,
+    }
+  }
+  logger.info(`Get app list: ${JSON.stringify(APP_MAP)}`)
+}
+
+export default initAppConfig

controller/groupAgent/report.ts

@@ -1,6 +1,6 @@
 import { LarkService } from "@egg/net-tool"
 
-import { appMap } from "../../constant/appMap"
+import { APP_MAP } from "../../constant/config"
 import { RespMessage } from "../../constant/message"
 import prisma from "../../prisma"
 import { Context } from "../../types"
@@ -31,7 +31,7 @@ const genReport = async (
   try {
     const { chat_id: chatId, robot_id: robotId } = subscription
     // 获取接口信息
-    const appInfo = appMap[robotId]
+    const appInfo = APP_MAP[robotId]
     if (!appInfo) {
       logger.error(`Failed to get app info for ${robotId}`)
       return

db/apiKey/index.ts

@@ -1,15 +1,47 @@
-import { DB } from "../../types"
+import { RecordModel } from "pocketbase"
+
+import { AppInfo } from "../../constant/config"
 import { managePbError } from "../../utils/pbTools"
 import pbClient from "../pbClient"
 
+const DB_NAME = "api_key"
+
+export interface ApiKey {
+  name: string
+  owner: string
+  app: string
+}
+
+export type ApiKeyModel = ApiKey & RecordModel
+
+export interface ApiKeyModelWithApp extends ApiKeyModel {
+  expand: {
+    app: AppInfo
+  }
+}
+
+/**
+ * 获取单个 API Key 对应的 App 信息
+ * @param id
+ * @returns
+ */
 const getOne = (id: string) =>
-  managePbError<DB.MessageGroup>(() =>
-    pbClient.collection("api_key").getOne(id, {
+  managePbError<ApiKeyModelWithApp>(() =>
+    pbClient.collection(DB_NAME).getOne(id, {
       expand: "app",
     })
   )
 
+/**
+ * 创建 API Key
+ * @param data
+ * @returns
+ */
+const create = (data: ApiKey) =>
+  managePbError<ApiKey>(() => pbClient.collection(DB_NAME).create(data))
+
 const apiKey = {
+  create,
   getOne,
 }
 

db/index.ts

@@ -1,10 +1,10 @@
 import apiKey from "./apiKey"
 import log from "./log"
-import messageGroup from "./messageGroup"
+import receiveGroup from "./receiveGroup"
 
 const db = {
   apiKey,
-  messageGroup,
+  receiveGroup,
   log,
 }
 

db/log/index.ts

@@ -1,9 +1,31 @@
-import { DB } from "../../types"
+import { RecordModel } from "pocketbase"
+
 import { managePbError } from "../../utils/pbTools"
 import pbClient from "../pbClient"
 
-const create = (collection: DB.LogCollection, log: DB.Log) =>
-  managePbError(() => pbClient.collection(collection).create(log))
+const DB_NAME = "message_log"
+
+export interface Log {
+  api_key: string
+  group_id?: string
+  receive_id?: string
+  receive_id_type?: string
+  msg_type: string
+  content: string
+  final_content?: string
+  send_result?: any
+  error?: string
+}
+
+export type LogModel = Log & RecordModel
+
+/**
+ * 创建一条日志
+ * @param log
+ * @returns
+ */
+const create = (log: Log) =>
+  managePbError(() => pbClient.collection(DB_NAME).create(log))
 
 const log = {
   create,

db/messageGroup/index.ts

@@ -1,14 +0,0 @@
-import { DB } from "../../types"
-import { managePbError } from "../../utils/pbTools"
-import pbClient from "../pbClient"
-
-const getOne = (groupId: string) =>
-  managePbError<DB.MessageGroup>(() =>
-    pbClient.collection("message_group").getOne(groupId)
-  )
-
-const messageGroup = {
-  getOne,
-}
-
-export default messageGroup

db/pbClient.ts

@@ -1,7 +1,11 @@
 import PocketBase from "pocketbase"
 
-const pbClient = new PocketBase("https://egg-pb.xiaomiwh.cn")
+const pbClient = new PocketBase("https://lark-egg-preview.ai.xiaomi.com")
 
 pbClient.autoCancellation(false)
 
+await pbClient
+  .collection("_superusers")
+  .authWithPassword(Bun.env.PB_USER!, Bun.env.PB_PASS!)
+
 export default pbClient

db/receiveGroup/index.ts

@@ -0,0 +1,33 @@
+import { RecordModel } from "pocketbase"
+
+import { managePbError } from "../../utils/pbTools"
+import pbClient from "../pbClient"
+
+const DB_NAME = "message_group"
+
+export interface ReceiveGroup {
+  name: string
+  email?: string[]
+  chat_id?: string[]
+  open_id?: string[]
+  union_id?: string[]
+  user_id?: string[]
+}
+
+export type ReceiveGroupModel = ReceiveGroup & RecordModel
+
+/**
+ * 根据ID获取指定消息组
+ * @param id
+ * @returns
+ */
+const getOne = (id: string) =>
+  managePbError<ReceiveGroupModel>(() =>
+    pbClient.collection(DB_NAME).getOne(id)
+  )
+
+const receiveGroup = {
+  getOne,
+}
+
+export default receiveGroup

index.ts

@@ -1,5 +1,6 @@
 import logger from "@egg/logger"
 
+import initAppConfig from "./constant/config"
 import prisma from "./prisma"
 import { manageBotReq } from "./routes/bot"
 import { manageMessageReq } from "./routes/message"
@@ -10,6 +11,8 @@ import genContext from "./utils/genContext"
 
 initSchedule()
 
+await initAppConfig()
+
 const server = Bun.serve({
   async fetch(req) {
     // 生成上下文

package.json

@@ -17,39 +17,39 @@
     ]
   },
   "devDependencies": {
-    "@commitlint/cli": "^19.6.0",
+    "@commitlint/cli": "^19.6.1",
     "@commitlint/config-conventional": "^19.6.0",
-    "@eslint/js": "^9.16.0",
+    "@eslint/js": "^9.17.0",
     "@types/node-schedule": "^2.1.7",
     "@types/uuid": "^10.0.0",
     "bun-types": "^1.1.38",
-    "eslint": "^9.16.0",
+    "eslint": "^9.17.0",
     "eslint-plugin-simple-import-sort": "^12.1.1",
     "eslint-plugin-unused-imports": "^4.1.4",
     "husky": "^9.1.7",
-    "lint-staged": "^15.2.10",
+    "lint-staged": "^15.2.11",
     "oxlint": "^0.13.2",
     "prettier": "^3.4.2",
     "prisma": "5.22.0",
-    "typescript-eslint": "^8.17.0"
+    "typescript-eslint": "^8.18.1"
   },
   "peerDependencies": {
     "typescript": "^5.5.4"
   },
   "dependencies": {
     "@egg/hooks": "^1.2.0",
-    "@egg/lark-msg-tool": "^1.19.0",
+    "@egg/lark-msg-tool": "^1.21.0",
     "@egg/logger": "^1.6.0",
     "@egg/net-tool": "^1.19.0",
     "@egg/path-tool": "^1.4.1",
-    "@langchain/core": "^0.3.20",
+    "@langchain/core": "^0.3.24",
     "@langchain/openai": "^0.3.14",
     "@prisma/client": "5.22.0",
     "joi": "^17.13.3",
-    "langfuse-langchain": "^3.31.1",
+    "langfuse-langchain": "^3.32.0",
     "node-schedule": "^2.1.1",
     "p-limit": "^6.1.0",
-    "pocketbase": "^0.21.5",
+    "pocketbase": "^0.23.0",
     "uuid": "^10.0.0"
   }
 }

routes/message/index.ts

@@ -1,11 +1,10 @@
 import { stringifyJson } from "@egg/hooks"
 import { LarkService } from "@egg/net-tool"
 
-import { appMap } from "../../constant/appMap"
+import { APP_MAP } from "../../constant/config"
 import db from "../../db"
-import { Context, DB, LarkServer, MsgProxy } from "../../types"
-
-const LOG_COLLECTION = "message_log"
+import { Log } from "../../db/log"
+import { Context, LarkServer, MsgProxy } from "../../types"
 
 /**
  * 校验消息请求的参数
@@ -68,7 +67,7 @@ export const manageMessageReq = async (
   const sendList = [] as Promise<any>[]
 
   // 构造消息记录
-  const baseLog: DB.MessageLogCreate = {
+  const baseLog: Log = {
     ...body,
     final_content: finalContent,
   }
@@ -77,7 +76,7 @@ export const manageMessageReq = async (
   const apiKeyInfo = await db.apiKey.getOne(body.api_key)
   if (!apiKeyInfo) {
     const error = "api key not found"
-    db.log.create(LOG_COLLECTION, { ...baseLog, error })
+    db.log.create({ ...baseLog, error })
     return genResp.notFound(error)
   }
 
@@ -85,15 +84,15 @@ export const manageMessageReq = async (
   const appName = apiKeyInfo.expand?.app?.name
   if (!appName) {
     const error = "app name not found"
-    db.log.create(LOG_COLLECTION, { ...baseLog, error })
+    db.log.create({ ...baseLog, error })
     return genResp.notFound(error)
   }
 
   // 获取 app info
-  const appInfo = appMap[appName]
+  const appInfo = APP_MAP[appName]
   if (!appInfo) {
     const error = "app not found"
-    db.log.create(LOG_COLLECTION, { ...baseLog, error })
+    db.log.create({ ...baseLog, error })
     return genResp.notFound(error)
   }
 
@@ -106,10 +105,10 @@ export const manageMessageReq = async (
   // 如果有 group_id，则发送给所有 group_id 中的人
   if (body.group_id) {
     // 获取所有接收者
-    const group = await db.messageGroup.getOne(body.group_id!)
+    const group = await db.receiveGroup.getOne(body.group_id!)
     if (!group) {
       const error = "message group not found"
-      db.log.create(LOG_COLLECTION, { ...baseLog, error })
+      db.log.create({ ...baseLog, error })
       return genResp.notFound(error)
     }
 
@@ -153,11 +152,11 @@ export const manageMessageReq = async (
     // 发送消息
     await Promise.allSettled(sendList)
     // 保存消息记录
-    db.log.create(LOG_COLLECTION, { ...baseLog, send_result: sendRes })
+    db.log.create({ ...baseLog, send_result: sendRes })
     return genResp.ok(sendRes)
   } catch {
     const error = "send msg failed"
-    db.log.create(LOG_COLLECTION, { ...baseLog, error })
+    db.log.create({ ...baseLog, error })
     return genResp.serverError(error, sendRes)
   }
 }

routes/microApp/index.ts

@@ -1,6 +1,6 @@
 import { LarkService } from "@egg/net-tool"
 
-import { appMap } from "../../constant/appMap"
+import { APP_MAP } from "../../constant/config"
 import { Context } from "../../types"
 
 /**
@@ -21,7 +21,7 @@ const manageLogin = async (ctx: Context.Data) => {
     return genResp.badRequest("app_name not found")
   }
   // 获取 app info
-  const appInfo = appMap[appName]
+  const appInfo = APP_MAP[appName]
   if (!appInfo) {
     return genResp.badRequest("app not found")
   }
@@ -67,7 +67,7 @@ const manageBatchUser = async (ctx: Context.Data) => {
     return genResp.badRequest("app_name not found")
   }
   // 获取 app info
-  const appInfo = appMap[app_name]
+  const appInfo = APP_MAP[app_name]
   if (!appInfo) {
     return genResp.badRequest("app not found")
   }

routes/sheet/index.ts

@@ -1,7 +1,7 @@
 import { LarkService } from "@egg/net-tool"
 import Joi from "joi"
 
-import { appMap } from "../../constant/appMap"
+import { APP_MAP } from "../../constant/config"
 import insertSheet from "../../controller/sheet/insert"
 import db from "../../db"
 import { Context } from "../../types"
@@ -85,7 +85,7 @@ export const manageSheetReq = async (ctx: Context.Data): Promise<Response> => {
   }
 
   // 获取APP信息
-  const appInfo = appMap[appName]
+  const appInfo = APP_MAP[appName]
   if (!appInfo) {
     return genResp.notFound("app not found")
   }

types/db.ts

@@ -1,49 +0,0 @@
-import { RecordModel } from "pocketbase"
-
-export namespace DB {
-  export interface ApiKey extends RecordModel {
-    name: string
-    user: string
-    app: string
-    apply_reason: string
-  }
-
-  export interface MessageGroup extends RecordModel {
-    desc: string
-    id: string
-    name: string
-    email?: string[]
-    chat_id?: string[]
-    open_id?: string[]
-    union_id?: string[]
-    user_id?: string[]
-  }
-
-  export interface MessageLog extends RecordModel {
-    api_key: string
-    group_id?: string
-    receive_id?: string
-    receive_id_type?: string
-    msg_type: string
-    content: string
-    final_content?: string
-    send_result?: any
-    error?: string
-  }
-
-  export type MessageLogCreate = Pick<
-    MessageLog,
-    | "api_key"
-    | "group_id"
-    | "receive_id"
-    | "receive_id_type"
-    | "msg_type"
-    | "content"
-    | "final_content"
-    | "send_result"
-    | "error"
-  >
-
-  export type Log = MessageLogCreate
-  export type LogCollection = "message_log"
-}

types/index.ts

@@ -1,6 +1,5 @@
 import type { Context } from "./context"
-import type { DB } from "./db"
 import type { LarkServer } from "./larkServer"
 import type { MsgProxy } from "./msgProxy"
 
-export { Context, DB, LarkServer, MsgProxy }
+export { Context, LarkServer, MsgProxy }

utils/genContext.ts

@@ -1,15 +1,16 @@
 import { LarkBody, LarkCard } from "@egg/lark-msg-tool"
 import loggerIns from "@egg/logger"
-import { LarkService, NetTool } from "@egg/net-tool"
+import { NetTool } from "@egg/net-tool"
 import { PathCheckTool } from "@egg/path-tool"
 import { v4 as uuid } from "uuid"
 
-import { appMap } from "../constant/appMap"
 import cardMap from "../constant/card"
+import { APP_MAP } from "../constant/config"
 import functionMap from "../constant/function"
 import tempMap from "../constant/template"
 import { AttachService } from "../services"
 import { Context } from "../types"
+import genLarkService from "./genLarkService"
 
 /**
  * 获取之前的requestId。
@@ -42,15 +43,11 @@ const genContext = async (req: Request) => {
   const larkBody = new LarkBody(body)
   const searchParams = new URL(req.url).searchParams
   const app = searchParams.get("app") || "egg"
-  const appInfo = appMap[app]
+  const appInfo = APP_MAP[app]
   const requestId = getPreRequestId(larkBody) || uuid()
   const logger = loggerIns.child({ requestId })
   const genResp = new NetTool({ requestId })
-  const larkService = new LarkService({
-    appId: appInfo.app_id,
-    appSecret: appInfo.app_secret,
-    requestId,
-  })
+  const larkService = genLarkService("egg", requestId)
   const attachService = new AttachService({ requestId })
   const path = new PathCheckTool(req.url)
   const larkCard = new LarkCard(

utils/genLarkService.ts

@@ -0,0 +1,14 @@
+import { LarkService } from "@egg/net-tool"
+
+import { APP_MAP } from "../constant/config"
+
+const genLarkService = (app: string, requestId: string) => {
+  const appInfo = APP_MAP[app]
+  return new LarkService({
+    appId: appInfo.app_id,
+    appSecret: appInfo.app_secret,
+    requestId,
+  })
+}
+
+export default genLarkService

utils/llm/base.ts

@@ -1,6 +1,8 @@
 import { ChatOpenAI } from "@langchain/openai"
 import CallbackHandler, { Langfuse } from "langfuse-langchain"
 
+import { APP_CONFIG } from "../../constant/config"
+
 /**
  * 获取模型
  * @param temperature 温度
@@ -10,11 +12,11 @@ export const getModel = (temperature = 0) => {
   return new ChatOpenAI(
     {
       temperature,
-      model: Bun.env.LLM_MODEL,
-      apiKey: Bun.env.LLM_API_KEY,
+      model: APP_CONFIG.TEXT_LLM_MODEL,
+      apiKey: APP_CONFIG.TEXT_LLM_API_KEY,
     },
     {
-      baseURL: Bun.env.LLM_BASE_URL,
+      baseURL: APP_CONFIG.TEXT_LLM_BASE_URL,
     }
   )
 }
@@ -25,9 +27,9 @@ export const getModel = (temperature = 0) => {
  */
 export const getLangfuse = async (name: string, requestId: string) => {
   const langfuseParams = {
-    publicKey: Bun.env.LANGFUSE_PK,
-    secretKey: Bun.env.LANGFUSE_SK,
-    baseUrl: Bun.env.LANGFUSE_BASE_URL,
+    publicKey: APP_CONFIG.LANGFUSE_PK,
+    secretKey: APP_CONFIG.LANGFUSE_SK,
+    baseUrl: APP_CONFIG.LANGFUSE_BASE_URL,
     sessionId: requestId,
     name,
   }

utils/pbTools.ts

@@ -11,11 +11,12 @@ export const managePb404 = async <T>(
 }
 
 export const managePbError = async <T>(
-  dbFunc: () => Promise<T>
+  dbFunc: () => Promise<T>,
+  defaultVal?: T
 ): Promise<T | null> => {
   try {
     return await dbFunc()
   } catch {
-    return null
+    return defaultVal || null
   }
 }