diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 32a8dee..70bff37 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,7 @@ "customizations": { "vscode": { "settings": { - "files.autoSave": "off", + "files.autoSave": "afterDelay", "editor.guides.bracketPairs": true, "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true, diff --git a/bun.lockb b/bun.lockb index 7d2804f..4e53bdf 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/controllers/managePipeLine/index.ts b/controllers/managePipeLine/index.ts new file mode 100644 index 0000000..ce8766e --- /dev/null +++ b/controllers/managePipeLine/index.ts @@ -0,0 +1,79 @@ +import db from "../../db"; +import { PipelineRecordModel } from "../../db/pipeline"; +import { ProjectRecordModel } from "../../db/project"; +import service from "../../service"; +import moment from "moment"; + +/** + * 获取全部的pipeline列表 + */ +const getFullPipelineList = async (project: ProjectRecordModel) => { + // 先获取最新的pipelineID + const latestOne = await db.pipeline.getLatestOne(project.id); + // 获取本次数据获取的截止时间,如果没有,则获取从20240101到现在所有pipeline信息 + const latestTime = moment( + latestOne?.created_at || "2024-01-01T00:00:00.000+08:00" + ); + // 获取pipeline列表并保存 + const fullPipelineList: GitlabPipeline[] = []; + let page = 1; + let hasBeforLatestTime = false; + while (!hasBeforLatestTime) { + const pipelines = await service.fetchPipelines(project.project_id, page++); + // 如果当前页没有数据,则直接跳出 + if (pipelines.length === 0) break; + pipelines.forEach((pipeline) => { + if (moment(pipeline.created_at).isSameOrBefore(latestTime)) { + hasBeforLatestTime = true; + } else { + fullPipelineList.push(pipeline); + } + }); + } + const fullPipelineDetailList = await Promise.all( + fullPipelineList.map(({ project_id, id, created_at }) => + service.fetchPipelineDetails(project_id, id, created_at) + ) + ); + return fullPipelineDetailList.filter( + (v) => v + ) as GitlabPipelineDetailWithCreateAt[]; +}; + +const insertFullPipelineList = async ( + fullPipelineList: GitlabPipelineDetailWithCreateAt[][], + fullUserMap: Record, + fullProjectMap: Record +) => { + const dbPipelineList: Partial = []; + + fullPipelineList.forEach((pipelineList) => { + pipelineList.forEach((pipeline) => { + dbPipelineList.push({ + project_id: fullProjectMap[pipeline.project_id], + user_id: fullUserMap[pipeline.user.id], + pipeline_id: pipeline.id, + ref: pipeline.ref, + status: pipeline.status, + web_url: pipeline.web_url, + created_at: pipeline.created_at, + started_at: pipeline.started_at, + finished_at: pipeline.finished_at, + duration: pipeline.duration, + queued_duration: pipeline.queued_duration, + }); + }); + }); + await Promise.all( + dbPipelineList.map((v: Partial) => + db.pipeline.create(v) + ) + ); +}; + +const managePipeline = { + getFullPipelineList, + insertFullPipelineList, +}; + +export default managePipeline; diff --git a/controllers/manageProject/index.ts b/controllers/manageProject/index.ts new file mode 100644 index 0000000..7f43a69 --- /dev/null +++ b/controllers/manageProject/index.ts @@ -0,0 +1,54 @@ +import db from "../../db"; +import { ProjectRecordModel } from "../../db/project"; +import service from "../../service"; + +/** + * 填充项目信息 + */ +const fillProj = async (project: ProjectRecordModel) => { + const projDetail = await service.fetchProjectDetails(project.project_id); + if (!projDetail) { + return project; + } + const useFulParams: Partial = { + ...project, + avatar_url: projDetail.avatar_url, + description: projDetail.description, + name: projDetail.name, + path_with_namespace: projDetail.path_with_namespace, + web_url: projDetail.web_url, + }; + return await db.project.update(project.id, useFulParams); +}; + +/** + * 获取到当前所有的项目列表 + * 并把信息不全的项目送给fillProj填充内容 + */ +const getFullProjList = async () => { + const fullList = await db.project.getFullList(); + // 把信息不全的项目送过去填充 + const filledProjList = await Promise.all( + fullList.filter((v) => !v.name).map((item) => fillProj(item)) + ); + // 合并成完整数据 + const filledFullProjList = fullList + .filter((v) => v.name) + .concat(filledProjList); + return filledFullProjList; +}; + +const getFullProjectMap = (fullProjList: ProjectRecordModel[]) => { + const fullProjectMap: Record = {}; + fullProjList.forEach((item) => { + fullProjectMap[item.project_id] = item.id; + }); + return fullProjectMap; +}; + +const manageProject = { + getFullProjList, + getFullProjectMap, +}; + +export default manageProject; diff --git a/controllers/manageUser/index.ts b/controllers/manageUser/index.ts new file mode 100644 index 0000000..066e605 --- /dev/null +++ b/controllers/manageUser/index.ts @@ -0,0 +1,38 @@ +import db from "../../db"; + +const getFullUserMap = async (fullPipelineList: GitlabPipelineDetail[][]) => { + const userList: GitlabUser[] = []; + fullPipelineList.forEach((fullPipeline) => { + fullPipeline.forEach((item) => { + if (item.user && !userList.find((v) => v.id === item.user?.id)) { + userList.push(item.user); + } + }); + }); + + const dbUserInfo = await Promise.all( + userList + .filter((v) => v.id !== 0) + .map((user) => + db.user.upsert({ + user_id: user.id, + username: user.username, + name: user.name, + avatar_url: user.avatar_url, + web_url: user.web_url, + }) + ) + ); + const userMap: Record = {}; + dbUserInfo.forEach((item) => { + if (!item) return; + userMap[item.user_id] = item.id; + }); + return userMap; +}; + +const manageUser = { + getFullUserMap, +}; + +export default manageUser; diff --git a/db/index.ts b/db/index.ts new file mode 100644 index 0000000..5924e81 --- /dev/null +++ b/db/index.ts @@ -0,0 +1,11 @@ +import project from "./project"; +import pipeline from "./pipeline"; +import user from "./user"; + +const db = { + project, + pipeline, + user, +}; + +export default db; diff --git a/db/pb.ts b/db/pb.ts deleted file mode 100644 index 2f9738e..0000000 --- a/db/pb.ts +++ /dev/null @@ -1,8 +0,0 @@ -import PocketBase from "pocketbase"; - -const pb = new PocketBase("https://ci-pb.xiaomiwh.cn"); - -export const getTenantAccessToken = async () => { - const { value } = await pb.collection("config").getOne("ugel8f0cpk0rut6"); - return value; -}; diff --git a/db/pbClient.ts b/db/pbClient.ts new file mode 100644 index 0000000..7999649 --- /dev/null +++ b/db/pbClient.ts @@ -0,0 +1,7 @@ +import PocketBase from "pocketbase"; + +const pbClient = new PocketBase("https://ci-pb.xiaomiwh.cn"); + +pbClient.autoCancellation(false); + +export default pbClient; diff --git a/db/pipeline/index.ts b/db/pipeline/index.ts new file mode 100644 index 0000000..d062c16 --- /dev/null +++ b/db/pipeline/index.ts @@ -0,0 +1,48 @@ +import { RecordModel } from "pocketbase"; +import pbClient from "../pbClient"; +import { managePb404 } from "../../utils/pbTools"; + +export interface PipelineRecordModel extends RecordModel { + project_id: string; + user_id: string; + pipeline_id: number; + ref: string; + status: string; + web_url: string; + // 2024-03-06 02:53:59.509Z + created_at: string; + started_at: string; + finished_at: string; + duration: number; + queued_duration: number; +} + +const getOne = (id: string) => + managePb404( + async () => await pbClient.collection("pipeline").getOne(id) + ) as Promise; + +/** + * 获取项目最新一次构建 + * @param project_id 项目id + */ +const getLatestOne = (project_id: string) => { + return managePb404( + async () => + await pbClient + .collection("pipeline") + .getFirstListItem(`project_id="${project_id}"`, { + sort: "-created_at", + }) + ) as Promise; +}; + +const create = async (data: Partial) => + await pbClient.collection("pipeline").create(data); + +const pipeline = { + create, + getOne, + getLatestOne, +}; +export default pipeline; diff --git a/db/project/index.ts b/db/project/index.ts new file mode 100644 index 0000000..59db272 --- /dev/null +++ b/db/project/index.ts @@ -0,0 +1,32 @@ +import { RecordModel } from "pocketbase"; +import { managePb404 } from "../../utils/pbTools"; +import pbClient from "../pbClient"; + +export interface ProjectRecordModel extends RecordModel { + project_id: number; + description: string; + name: string; + path_with_namespace: string; + web_url: string; + avatar_url: string; + has_new_cicd: boolean; +} + +const getOne = (id: string) => + managePb404( + async () => await pbClient.collection("project").getOne(id) + ) as Promise; + +const getFullList = async () => + await pbClient.collection("project").getFullList(); + +const update = async (id: string, data: Partial) => + await pbClient.collection("project").update(id, data); + +const project = { + getFullList, + getOne, + update, +}; + +export default project; diff --git a/db/user/index.ts b/db/user/index.ts new file mode 100644 index 0000000..41f30ce --- /dev/null +++ b/db/user/index.ts @@ -0,0 +1,46 @@ +import { RecordModel } from "pocketbase"; +import { managePb404 } from "../../utils/pbTools"; +import pbClient from "../pbClient"; + +export interface UserRecordModel extends RecordModel { + user_id: number; + username: string; + name: string; + avatar_url: string; + web_url: string; +} + +const getOne = (id: string) => + managePb404( + async () => await pbClient.collection("user").getOne(id) + ) as Promise; + +const getOneByUserId = (user_id: number) => { + return managePb404( + async () => + await pbClient + .collection("user") + .getFirstListItem(`user_id="${user_id}"`, { + sort: "-created", + }) + ) as Promise; +}; + +const create = async (data: Partial) => + await pbClient.collection("user").create(data); + +const upsert = async (data: Partial) => { + if (!data.user_id) return null; + const userInfo = await getOneByUserId(data.user_id); + if (userInfo) return userInfo; + return await create(data); +}; + +const user = { + create, + upsert, + getOne, + getOneByUserId, +}; + +export default user; diff --git a/index.ts b/index.ts index e69de29..c3d088e 100644 --- a/index.ts +++ b/index.ts @@ -0,0 +1,22 @@ +import { scheduleJob } from "node-schedule"; +import managePipeline from "./controllers/managePipeLine"; +import manageProject from "./controllers/manageProject"; +import manageUser from "./controllers/manageUser"; + +const main = async () => { + const fullProjList = await manageProject.getFullProjList(); + const fullPipelineList = await Promise.all( + fullProjList.map((v) => managePipeline.getFullPipelineList(v)) + ); + const fullUserMap = await manageUser.getFullUserMap(fullPipelineList); + const fullProjectMap = await manageProject.getFullProjectMap(fullProjList); + await managePipeline.insertFullPipelineList( + fullPipelineList, + fullUserMap, + fullProjectMap + ); +}; + +main(); + +scheduleJob("*/15 * * * *", main); diff --git a/package.json b/package.json index 12520ff..a2bf4d6 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,9 @@ }, "dependencies": { "@types/node-schedule": "^2.1.6", + "@types/lodash": "^4.14.202", + "lodash": "^4.17.21", + "moment": "^2.30.1", "node-schedule": "^2.1.1", "pocketbase": "^0.21.1" } diff --git a/readme.md b/readme.md index abfda8c..b0db53c 100644 --- a/readme.md +++ b/readme.md @@ -1 +1,63 @@ # CI 监控 + +监听新 projId,自动补全内容,获取从 20240101 到当前的所有流水线信息 + +监听功能未知原因不好用,先不做了,改手动遍历了 + +拿到 project_id 后,获取数据表中最新的 pipeline 的 Id,然后比对接口中的 ID 进行填充 + +如果没有 pipeline 的 id,直接从接口中获取 20240101 到当前的流水线信息 + +先从数据中获取用户信息填充,随后在根据填充完的用户信息获取全部的 userid 的列表,再写 pipeline 表 + +# 图表库 + +https://g2plot.antv.antgroup.com/examples + +# 数据信息 + +project 信息 + +```js +{ + id: 'aaa', + project_id: 131366, + description: "场景复现平台-展示设备(移动、音箱、小爱建议、车载、手表等设备)上小爱执行结果及相关处理流程", + name: "ai-scene-review-fe", + path_with_namespace: "miai-fe/fe/ai-scene-review-fe", + web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe", + avatar_url: null, + has_new_cicd: false, +} +``` + +pipeline 信息 + +```js +{ + id: 'bbb', + project_id: 'aaa', + user_id: 'ccc', + pipeline_id: 7646046, + ref: "preview", + status: "success", + web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe/-/pipelines/7646046", + started_at: "2024-03-01T16:47:40.192+08:00", + finished_at: "2024-03-01T16:49:30.624+08:00", + duration: 100, + queued_duration: 6, +} +``` + +user 信息 + +```js +{ + id: 'ccc', + user_id: 10011, + username: "zhaoyingbo", + name: "赵英博", + avatar_url: "https://git.n.xiaomi.com/uploads/-/system/user/avatar/10011/avatar.png", + web_url: "https://git.n.xiaomi.com/zhaoyingbo" +} +``` diff --git a/service/index.ts b/service/index.ts new file mode 100644 index 0000000..000be3e --- /dev/null +++ b/service/index.ts @@ -0,0 +1,71 @@ +const fetchGetParams = { + method: "GET", + headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" }, +}; + +/** + * 获取项目详情 + * @param id 项目id + */ +const fetchProjectDetails = async (id: number) => { + try { + const response = await fetch( + `https://git.n.xiaomi.com/api/v4/projects/${id}`, + fetchGetParams + ); + const body = (await response.json()) as GitlabProjDetail; + if (body.message === "404 Project Not Found") return null; + return body; + } catch { + return null; + } +}; + +/** + * 获取流水线列表 + */ +const fetchPipelines = async (project_id: number, page = 1) => { + try { + const response = await fetch( + `https://git.n.xiaomi.com/api/v4/projects/${project_id}/pipelines?scope=finished&status=success&per_page=100&page=${page}`, + fetchGetParams + ); + const body = (await response.json()) as GitlabPipeline[] & GitlabError; + if (body?.message === "404 Project Not Found") return []; + return body; + } catch { + return [] as GitlabPipeline[]; + } +}; + +/** + * 获取流水线详情 + */ +const fetchPipelineDetails = async ( + project_id: number, + pipeline_id: number, + created_at: string +) => { + try { + const response = await fetch( + `https://git.n.xiaomi.com/api/v4/projects/${project_id}/pipelines/${pipeline_id}`, + fetchGetParams + ); + const body = (await response.json()) as GitlabPipelineDetail; + if (body.message === "404 Project Not Found") return null; + return { + ...body, + created_at, + }; + } catch { + return null; + } +}; + +const service = { + fetchProjectDetails, + fetchPipelines, + fetchPipelineDetails, +}; + +export default service; diff --git a/service/test.ts b/service/test.ts deleted file mode 100644 index 175d5b8..0000000 --- a/service/test.ts +++ /dev/null @@ -1,39 +0,0 @@ -const fetchProjectDetails = async () => { - const response = await fetch( - "https://git.n.xiaomi.com/api/v4/projects/131366", - { - method: "GET", - headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" }, - } - ); - const body = await response.json(); - console.log(body); -}; - -const fetchPipelineDetails = async () => { - const response = await fetch( - "https://git.n.xiaomi.com/api/v4/projects/131366/pipelines/7646046", - { - method: "GET", - headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" }, - } - ); - const body = await response.json(); - console.log(body); -}; - -const fetchPipelines = async () => { - const response = await fetch( - "https://git.n.xiaomi.com/api/v4/projects/131366/pipelines", - { - method: "GET", - headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" }, - } - ); - const body = await response.json(); - console.log(body); -}; - -// fetchPipelines(); -// fetchPipelineDetails(); -fetchProjectDetails(); diff --git a/service/typings.d.ts b/service/typings.d.ts new file mode 100644 index 0000000..10f524b --- /dev/null +++ b/service/typings.d.ts @@ -0,0 +1,52 @@ +interface GitlabProjDetail { + id: number; + description: string; + name: string; + path_with_namespace: string; + web_url: string; + avatar_url?: any; + message?: string; +} + +interface GitlabPipeline { + id: number; + project_id: number; + sha: string; + ref: string; + status: string; + source: string; + created_at: string; + updated_at: string; + web_url: string; +} + +interface GitlabError { + message: string; +} + +interface GitlabUser { + id: number; + username: string; + name: string; + state: string; + avatar_url: string; + web_url: string; +} + +interface GitlabPipelineDetail { + id: number; + project_id: number; + ref: string; + status: string; + web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe/-/pipelines/7646046"; + user: GitlabUser; + started_at: string; + finished_at: string; + duration: number; + queued_duration: number; + message?: string; +} + +interface GitlabPipelineDetailWithCreateAt extends GitlabPipelineDetail { + created_at: string; +} diff --git a/utils/pbTools.ts b/utils/pbTools.ts new file mode 100644 index 0000000..01d3186 --- /dev/null +++ b/utils/pbTools.ts @@ -0,0 +1,10 @@ +export const managePb404 = async (dbFunc: Function) => { + try { + return await dbFunc(); + } catch (err: any) { + // 没有这个提醒就返回空 + if (err?.message === "The requested resource wasn't found.") { + return null; + } else throw err; + } +};