From 3773e1150501e852804ebd30f81e8be2d75544c9 Mon Sep 17 00:00:00 2001 From: zhaoyingbo Date: Wed, 6 Mar 2024 09:14:28 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=8C=E6=88=90=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devcontainer/devcontainer.json | 2 +- bun.lockb | Bin 5283 -> 6436 bytes controllers/managePipeLine/index.ts | 79 ++++++++++++++++++++++++++++ controllers/manageProject/index.ts | 54 +++++++++++++++++++ controllers/manageUser/index.ts | 38 +++++++++++++ db/index.ts | 11 ++++ db/pb.ts | 8 --- db/pbClient.ts | 7 +++ db/pipeline/index.ts | 48 +++++++++++++++++ db/project/index.ts | 32 +++++++++++ db/user/index.ts | 46 ++++++++++++++++ index.ts | 22 ++++++++ package.json | 3 ++ readme.md | 62 ++++++++++++++++++++++ service/index.ts | 71 +++++++++++++++++++++++++ service/test.ts | 39 -------------- service/typings.d.ts | 52 ++++++++++++++++++ utils/pbTools.ts | 10 ++++ 18 files changed, 536 insertions(+), 48 deletions(-) create mode 100644 controllers/managePipeLine/index.ts create mode 100644 controllers/manageProject/index.ts create mode 100644 controllers/manageUser/index.ts create mode 100644 db/index.ts delete mode 100644 db/pb.ts create mode 100644 db/pbClient.ts create mode 100644 db/pipeline/index.ts create mode 100644 db/project/index.ts create mode 100644 db/user/index.ts create mode 100644 service/index.ts delete mode 100644 service/test.ts create mode 100644 service/typings.d.ts create mode 100644 utils/pbTools.ts diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 32a8dee..70bff37 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,7 @@ "customizations": { "vscode": { "settings": { - "files.autoSave": "off", + "files.autoSave": "afterDelay", "editor.guides.bracketPairs": true, "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true, diff --git a/bun.lockb b/bun.lockb index 7d2804f363561bb0da8515d9faa337de7d930c4b..4e53bdfd53a3b9781cbd3bbc22a3e65237a32b97 100755 GIT binary patch delta 1438 zcmcIkYfMx}6rQ=ebRX=4T~-Q6gqldZ*u8uA0^(Ng?wa(6Xkt}}q-wqUn=RTAl{`DktznS}; zGv_gL=D7cad`R@4e4_j|8GaRc?wyy3eY#_-@F$oLe&3*68!KM>;IHvJJ^4%(ce1{? zv(~LG-hLP2)XrF!g6Kpn<&5Pa79u(j-H6$UTei2yTTrJt-n^kD!Pu7g`meTaV$6^K ze8{Iwnx5aR*%j&g;oi)Kd`%W&^5HybaCD-7bAMmW#Ya1oBjLq&%3h}@kBNoHlED=R zN`F$qws>`VZ0Qf@XDzuhT6+HPT_ExN`3w4Rn<+A>%Yweyt{#E0(_d)hzfUz>KKyG< z!E(lsQl$>|@F@QgGWe|qn-H}PU|y>+E(RmKN^l6USBUUUkP)IegJ@NY6dBLJ0Wtmw zbaAUE8*{8f0#jrj9MfPv|Ckt|!31_wgpWXtDJoQ&VXrCT=|Qzsy0#0oufEY9H^Ws^ zL|B#ucJpVQR^(`q5TX;RxGGZ-1Ev#EUzWlEFKk3|9rYSK1Nz3QHjfaz$*IycbInZ5qGh6wm_0qj$5qzCh z1tnG|?6NF^cEr1gw-5u?MIRXzpy>Jdk)hz3Z|+(?+!ricwL3A?HrYEt*4zml)_U%O z5o@jER3~G`y}_gKL6_|)j`mcEy;^`JcCh#A-ErCtjgH84_F7jbA5_WvI-ipRzc^$W zZ#4S=hefA}rJ~|>rc(we z*y${Tqt1#ZqqU3<5qK^tQj_@+sTqx)68LIa42g5zwyV)~$W(}-Dp3v@4Q=$LYEv$W zA*R4HXF=scEs(CWy;3-wz8FGYLNnxzry+h~?#ht{>yWyqEJ}1hq*( u7Rx1Q^+fYDB_y;-Kr0e#^+e#hH#^nnE#o+`h42q=_a~lXy|EdQc&XKUEY(p`<}ms33Mx zPl95ZO9YV;s~{+-F$aT1PeJrzPYQadhoYi~iXud?b!L;GH*sM0+nF~z^JaFY6F2*> z>iyRrtay8&*ZowwTfVXR1kb6tj7w2eQ{B)+;B zGgl)$NR=}dL`p~uHe2X-BS*%@M~nFGN5zb$Z0q=u6o)yOj5MS+(X`}Y#mHou&*EG5 z=3@$Y_$tWr5FAELJJPVgojePVd5&L$7Cxhp5jnYnwTCem11{E!9~>GY%XdLbiphE3LNxiBa2IIp&L;vbDd+w0}-_k-gwn0xyegiIZ&6O z(iCV?X=)p#WQE%)mkV~PJuB&uMnfSQ{snA zg&A#t%kWhTixC?rzR>dSj!v%KcR5<$lo9+~0>k6_fFVKN7ivel3SRsirQn6LzP!uq onC<{HeZBtjnbNV-M^BwBm5+=S*H~6f^FJ3gGnnEvE#&9F0g>{kn*aa+ diff --git a/controllers/managePipeLine/index.ts b/controllers/managePipeLine/index.ts new file mode 100644 index 0000000..ce8766e --- /dev/null +++ b/controllers/managePipeLine/index.ts @@ -0,0 +1,79 @@ +import db from "../../db"; +import { PipelineRecordModel } from "../../db/pipeline"; +import { ProjectRecordModel } from "../../db/project"; +import service from "../../service"; +import moment from "moment"; + +/** + * 获取全部的pipeline列表 + */ +const getFullPipelineList = async (project: ProjectRecordModel) => { + // 先获取最新的pipelineID + const latestOne = await db.pipeline.getLatestOne(project.id); + // 获取本次数据获取的截止时间，如果没有，则获取从20240101到现在所有pipeline信息 + const latestTime = moment( + latestOne?.created_at || "2024-01-01T00:00:00.000+08:00" + ); + // 获取pipeline列表并保存 + const fullPipelineList: GitlabPipeline[] = []; + let page = 1; + let hasBeforLatestTime = false; + while (!hasBeforLatestTime) { + const pipelines = await service.fetchPipelines(project.project_id, page++); + // 如果当前页没有数据，则直接跳出 + if (pipelines.length === 0) break; + pipelines.forEach((pipeline) => { + if (moment(pipeline.created_at).isSameOrBefore(latestTime)) { + hasBeforLatestTime = true; + } else { + fullPipelineList.push(pipeline); + } + }); + } + const fullPipelineDetailList = await Promise.all( + fullPipelineList.map(({ project_id, id, created_at }) => + service.fetchPipelineDetails(project_id, id, created_at) + ) + ); + return fullPipelineDetailList.filter( + (v) => v + ) as GitlabPipelineDetailWithCreateAt[]; +}; + +const insertFullPipelineList = async ( + fullPipelineList: GitlabPipelineDetailWithCreateAt[][], + fullUserMap: Record, + fullProjectMap: Record +) => { + const dbPipelineList: Partial = []; + + fullPipelineList.forEach((pipelineList) => { + pipelineList.forEach((pipeline) => { + dbPipelineList.push({ + project_id: fullProjectMap[pipeline.project_id], + user_id: fullUserMap[pipeline.user.id], + pipeline_id: pipeline.id, + ref: pipeline.ref, + status: pipeline.status, + web_url: pipeline.web_url, + created_at: pipeline.created_at, + started_at: pipeline.started_at, + finished_at: pipeline.finished_at, + duration: pipeline.duration, + queued_duration: pipeline.queued_duration, + }); + }); + }); + await Promise.all( + dbPipelineList.map((v: Partial) => + db.pipeline.create(v) + ) + ); +}; + +const managePipeline = { + getFullPipelineList, + insertFullPipelineList, +}; + +export default managePipeline; diff --git a/controllers/manageProject/index.ts b/controllers/manageProject/index.ts new file mode 100644 index 0000000..7f43a69 --- /dev/null +++ b/controllers/manageProject/index.ts @@ -0,0 +1,54 @@ +import db from "../../db"; +import { ProjectRecordModel } from "../../db/project"; +import service from "../../service"; + +/** + * 填充项目信息 + */ +const fillProj = async (project: ProjectRecordModel) => { + const projDetail = await service.fetchProjectDetails(project.project_id); + if (!projDetail) { + return project; + } + const useFulParams: Partial = { + ...project, + avatar_url: projDetail.avatar_url, + description: projDetail.description, + name: projDetail.name, + path_with_namespace: projDetail.path_with_namespace, + web_url: projDetail.web_url, + }; + return await db.project.update(project.id, useFulParams); +}; + +/** + * 获取到当前所有的项目列表 + * 并把信息不全的项目送给fillProj填充内容 + */ +const getFullProjList = async () => { + const fullList = await db.project.getFullList(); + // 把信息不全的项目送过去填充 + const filledProjList = await Promise.all( + fullList.filter((v) => !v.name).map((item) => fillProj(item)) + ); + // 合并成完整数据 + const filledFullProjList = fullList + .filter((v) => v.name) + .concat(filledProjList); + return filledFullProjList; +}; + +const getFullProjectMap = (fullProjList: ProjectRecordModel[]) => { + const fullProjectMap: Record = {}; + fullProjList.forEach((item) => { + fullProjectMap[item.project_id] = item.id; + }); + return fullProjectMap; +}; + +const manageProject = { + getFullProjList, + getFullProjectMap, +}; + +export default manageProject; diff --git a/controllers/manageUser/index.ts b/controllers/manageUser/index.ts new file mode 100644 index 0000000..066e605 --- /dev/null +++ b/controllers/manageUser/index.ts @@ -0,0 +1,38 @@ +import db from "../../db"; + +const getFullUserMap = async (fullPipelineList: GitlabPipelineDetail[][]) => { + const userList: GitlabUser[] = []; + fullPipelineList.forEach((fullPipeline) => { + fullPipeline.forEach((item) => { + if (item.user && !userList.find((v) => v.id === item.user?.id)) { + userList.push(item.user); + } + }); + }); + + const dbUserInfo = await Promise.all( + userList + .filter((v) => v.id !== 0) + .map((user) => + db.user.upsert({ + user_id: user.id, + username: user.username, + name: user.name, + avatar_url: user.avatar_url, + web_url: user.web_url, + }) + ) + ); + const userMap: Record = {}; + dbUserInfo.forEach((item) => { + if (!item) return; + userMap[item.user_id] = item.id; + }); + return userMap; +}; + +const manageUser = { + getFullUserMap, +}; + +export default manageUser; diff --git a/db/index.ts b/db/index.ts new file mode 100644 index 0000000..5924e81 --- /dev/null +++ b/db/index.ts @@ -0,0 +1,11 @@ +import project from "./project"; +import pipeline from "./pipeline"; +import user from "./user"; + +const db = { + project, + pipeline, + user, +}; + +export default db; diff --git a/db/pb.ts b/db/pb.ts deleted file mode 100644 index 2f9738e..0000000 --- a/db/pb.ts +++ /dev/null @@ -1,8 +0,0 @@ -import PocketBase from "pocketbase"; - -const pb = new PocketBase("https://ci-pb.xiaomiwh.cn"); - -export const getTenantAccessToken = async () => { - const { value } = await pb.collection("config").getOne("ugel8f0cpk0rut6"); - return value; -}; diff --git a/db/pbClient.ts b/db/pbClient.ts new file mode 100644 index 0000000..7999649 --- /dev/null +++ b/db/pbClient.ts @@ -0,0 +1,7 @@ +import PocketBase from "pocketbase"; + +const pbClient = new PocketBase("https://ci-pb.xiaomiwh.cn"); + +pbClient.autoCancellation(false); + +export default pbClient; diff --git a/db/pipeline/index.ts b/db/pipeline/index.ts new file mode 100644 index 0000000..d062c16 --- /dev/null +++ b/db/pipeline/index.ts @@ -0,0 +1,48 @@ +import { RecordModel } from "pocketbase"; +import pbClient from "../pbClient"; +import { managePb404 } from "../../utils/pbTools"; + +export interface PipelineRecordModel extends RecordModel { + project_id: string; + user_id: string; + pipeline_id: number; + ref: string; + status: string; + web_url: string; + // 2024-03-06 02:53:59.509Z + created_at: string; + started_at: string; + finished_at: string; + duration: number; + queued_duration: number; +} + +const getOne = (id: string) => + managePb404( + async () => await pbClient.collection("pipeline").getOne(id) + ) as Promise; + +/** + * 获取项目最新一次构建 + * @param project_id 项目id + */ +const getLatestOne = (project_id: string) => { + return managePb404( + async () => + await pbClient + .collection("pipeline") + .getFirstListItem(`project_id="${project_id}"`, { + sort: "-created_at", + }) + ) as Promise; +}; + +const create = async (data: Partial) => + await pbClient.collection("pipeline").create(data); + +const pipeline = { + create, + getOne, + getLatestOne, +}; +export default pipeline; diff --git a/db/project/index.ts b/db/project/index.ts new file mode 100644 index 0000000..59db272 --- /dev/null +++ b/db/project/index.ts @@ -0,0 +1,32 @@ +import { RecordModel } from "pocketbase"; +import { managePb404 } from "../../utils/pbTools"; +import pbClient from "../pbClient"; + +export interface ProjectRecordModel extends RecordModel { + project_id: number; + description: string; + name: string; + path_with_namespace: string; + web_url: string; + avatar_url: string; + has_new_cicd: boolean; +} + +const getOne = (id: string) => + managePb404( + async () => await pbClient.collection("project").getOne(id) + ) as Promise; + +const getFullList = async () => + await pbClient.collection("project").getFullList(); + +const update = async (id: string, data: Partial) => + await pbClient.collection("project").update(id, data); + +const project = { + getFullList, + getOne, + update, +}; + +export default project; diff --git a/db/user/index.ts b/db/user/index.ts new file mode 100644 index 0000000..41f30ce --- /dev/null +++ b/db/user/index.ts @@ -0,0 +1,46 @@ +import { RecordModel } from "pocketbase"; +import { managePb404 } from "../../utils/pbTools"; +import pbClient from "../pbClient"; + +export interface UserRecordModel extends RecordModel { + user_id: number; + username: string; + name: string; + avatar_url: string; + web_url: string; +} + +const getOne = (id: string) => + managePb404( + async () => await pbClient.collection("user").getOne(id) + ) as Promise; + +const getOneByUserId = (user_id: number) => { + return managePb404( + async () => + await pbClient + .collection("user") + .getFirstListItem(`user_id="${user_id}"`, { + sort: "-created", + }) + ) as Promise; +}; + +const create = async (data: Partial) => + await pbClient.collection("user").create(data); + +const upsert = async (data: Partial) => { + if (!data.user_id) return null; + const userInfo = await getOneByUserId(data.user_id); + if (userInfo) return userInfo; + return await create(data); +}; + +const user = { + create, + upsert, + getOne, + getOneByUserId, +}; + +export default user; diff --git a/index.ts b/index.ts index e69de29..c3d088e 100644 --- a/index.ts +++ b/index.ts @@ -0,0 +1,22 @@ +import { scheduleJob } from "node-schedule"; +import managePipeline from "./controllers/managePipeLine"; +import manageProject from "./controllers/manageProject"; +import manageUser from "./controllers/manageUser"; + +const main = async () => { + const fullProjList = await manageProject.getFullProjList(); + const fullPipelineList = await Promise.all( + fullProjList.map((v) => managePipeline.getFullPipelineList(v)) + ); + const fullUserMap = await manageUser.getFullUserMap(fullPipelineList); + const fullProjectMap = await manageProject.getFullProjectMap(fullProjList); + await managePipeline.insertFullPipelineList( + fullPipelineList, + fullUserMap, + fullProjectMap + ); +}; + +main(); + +scheduleJob("*/15 * * * *", main); diff --git a/package.json b/package.json index 12520ff..a2bf4d6 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,9 @@ }, "dependencies": { "@types/node-schedule": "^2.1.6", + "@types/lodash": "^4.14.202", + "lodash": "^4.17.21", + "moment": "^2.30.1", "node-schedule": "^2.1.1", "pocketbase": "^0.21.1" } diff --git a/readme.md b/readme.md index abfda8c..b0db53c 100644 --- a/readme.md +++ b/readme.md @@ -1 +1,63 @@ # CI 监控 + +监听新 projId，自动补全内容，获取从 20240101 到当前的所有流水线信息 + +监听功能未知原因不好用，先不做了，改手动遍历了 + +拿到 project_id 后，获取数据表中最新的 pipeline 的 Id，然后比对接口中的 ID 进行填充 + +如果没有 pipeline 的 id，直接从接口中获取 20240101 到当前的流水线信息 + +先从数据中获取用户信息填充，随后在根据填充完的用户信息获取全部的 userid 的列表，再写 pipeline 表 + +# 图表库 + +https://g2plot.antv.antgroup.com/examples + +# 数据信息 + +project 信息 + +```js +{ + id: 'aaa', + project_id: 131366, + description: "场景复现平台-展示设备（移动、音箱、小爱建议、车载、手表等设备）上小爱执行结果及相关处理流程", + name: "ai-scene-review-fe", + path_with_namespace: "miai-fe/fe/ai-scene-review-fe", + web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe", + avatar_url: null, + has_new_cicd: false, +} +``` + +pipeline 信息 + +```js +{ + id: 'bbb', + project_id: 'aaa', + user_id: 'ccc', + pipeline_id: 7646046, + ref: "preview", + status: "success", + web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe/-/pipelines/7646046", + started_at: "2024-03-01T16:47:40.192+08:00", + finished_at: "2024-03-01T16:49:30.624+08:00", + duration: 100, + queued_duration: 6, +} +``` + +user 信息 + +```js +{ + id: 'ccc', + user_id: 10011, + username: "zhaoyingbo", + name: "赵英博", + avatar_url: "https://git.n.xiaomi.com/uploads/-/system/user/avatar/10011/avatar.png", + web_url: "https://git.n.xiaomi.com/zhaoyingbo" +} +``` diff --git a/service/index.ts b/service/index.ts new file mode 100644 index 0000000..000be3e --- /dev/null +++ b/service/index.ts @@ -0,0 +1,71 @@ +const fetchGetParams = { + method: "GET", + headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" }, +}; + +/** + * 获取项目详情 + * @param id 项目id + */ +const fetchProjectDetails = async (id: number) => { + try { + const response = await fetch( + `https://git.n.xiaomi.com/api/v4/projects/${id}`, + fetchGetParams + ); + const body = (await response.json()) as GitlabProjDetail; + if (body.message === "404 Project Not Found") return null; + return body; + } catch { + return null; + } +}; + +/** + * 获取流水线列表 + */ +const fetchPipelines = async (project_id: number, page = 1) => { + try { + const response = await fetch( + `https://git.n.xiaomi.com/api/v4/projects/${project_id}/pipelines?scope=finished&status=success&per_page=100&page=${page}`, + fetchGetParams + ); + const body = (await response.json()) as GitlabPipeline[] & GitlabError; + if (body?.message === "404 Project Not Found") return []; + return body; + } catch { + return [] as GitlabPipeline[]; + } +}; + +/** + * 获取流水线详情 + */ +const fetchPipelineDetails = async ( + project_id: number, + pipeline_id: number, + created_at: string +) => { + try { + const response = await fetch( + `https://git.n.xiaomi.com/api/v4/projects/${project_id}/pipelines/${pipeline_id}`, + fetchGetParams + ); + const body = (await response.json()) as GitlabPipelineDetail; + if (body.message === "404 Project Not Found") return null; + return { + ...body, + created_at, + }; + } catch { + return null; + } +}; + +const service = { + fetchProjectDetails, + fetchPipelines, + fetchPipelineDetails, +}; + +export default service; diff --git a/service/test.ts b/service/test.ts deleted file mode 100644 index 175d5b8..0000000 --- a/service/test.ts +++ /dev/null @@ -1,39 +0,0 @@ -const fetchProjectDetails = async () => { - const response = await fetch( - "https://git.n.xiaomi.com/api/v4/projects/131366", - { - method: "GET", - headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" }, - } - ); - const body = await response.json(); - console.log(body); -}; - -const fetchPipelineDetails = async () => { - const response = await fetch( - "https://git.n.xiaomi.com/api/v4/projects/131366/pipelines/7646046", - { - method: "GET", - headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" }, - } - ); - const body = await response.json(); - console.log(body); -}; - -const fetchPipelines = async () => { - const response = await fetch( - "https://git.n.xiaomi.com/api/v4/projects/131366/pipelines", - { - method: "GET", - headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" }, - } - ); - const body = await response.json(); - console.log(body); -}; - -// fetchPipelines(); -// fetchPipelineDetails(); -fetchProjectDetails(); diff --git a/service/typings.d.ts b/service/typings.d.ts new file mode 100644 index 0000000..10f524b --- /dev/null +++ b/service/typings.d.ts @@ -0,0 +1,52 @@ +interface GitlabProjDetail { + id: number; + description: string; + name: string; + path_with_namespace: string; + web_url: string; + avatar_url?: any; + message?: string; +} + +interface GitlabPipeline { + id: number; + project_id: number; + sha: string; + ref: string; + status: string; + source: string; + created_at: string; + updated_at: string; + web_url: string; +} + +interface GitlabError { + message: string; +} + +interface GitlabUser { + id: number; + username: string; + name: string; + state: string; + avatar_url: string; + web_url: string; +} + +interface GitlabPipelineDetail { + id: number; + project_id: number; + ref: string; + status: string; + web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe/-/pipelines/7646046"; + user: GitlabUser; + started_at: string; + finished_at: string; + duration: number; + queued_duration: number; + message?: string; +} + +interface GitlabPipelineDetailWithCreateAt extends GitlabPipelineDetail { + created_at: string; +} diff --git a/utils/pbTools.ts b/utils/pbTools.ts new file mode 100644 index 0000000..01d3186 --- /dev/null +++ b/utils/pbTools.ts @@ -0,0 +1,10 @@ +export const managePb404 = async (dbFunc: Function) => { + try { + return await dbFunc(); + } catch (err: any) { + // 没有这个提醒就返回空 + if (err?.message === "The requested resource wasn't found.") { + return null; + } else throw err; + } +};