feat: 完成数据监控

.devcontainer/devcontainer.json

@@ -6,7 +6,7 @@
   "customizations": {
     "vscode": {
       "settings": {
-        "files.autoSave": "off",
+        "files.autoSave": "afterDelay",
         "editor.guides.bracketPairs": true,
         "editor.defaultFormatter": "esbenp.prettier-vscode",
         "editor.formatOnSave": true,

controllers/managePipeLine/index.ts

@@ -0,0 +1,79 @@
+import db from "../../db";
+import { PipelineRecordModel } from "../../db/pipeline";
+import { ProjectRecordModel } from "../../db/project";
+import service from "../../service";
+import moment from "moment";
+
+/**
+ * 获取全部的pipeline列表
+ */
+const getFullPipelineList = async (project: ProjectRecordModel) => {
+  // 先获取最新的pipelineID
+  const latestOne = await db.pipeline.getLatestOne(project.id);
+  // 获取本次数据获取的截止时间，如果没有，则获取从20240101到现在所有pipeline信息
+  const latestTime = moment(
+    latestOne?.created_at || "2024-01-01T00:00:00.000+08:00"
+  );
+  // 获取pipeline列表并保存
+  const fullPipelineList: GitlabPipeline[] = [];
+  let page = 1;
+  let hasBeforLatestTime = false;
+  while (!hasBeforLatestTime) {
+    const pipelines = await service.fetchPipelines(project.project_id, page++);
+    // 如果当前页没有数据，则直接跳出
+    if (pipelines.length === 0) break;
+    pipelines.forEach((pipeline) => {
+      if (moment(pipeline.created_at).isSameOrBefore(latestTime)) {
+        hasBeforLatestTime = true;
+      } else {
+        fullPipelineList.push(pipeline);
+      }
+    });
+  }
+  const fullPipelineDetailList = await Promise.all(
+    fullPipelineList.map(({ project_id, id, created_at }) =>
+      service.fetchPipelineDetails(project_id, id, created_at)
+    )
+  );
+  return fullPipelineDetailList.filter(
+    (v) => v
+  ) as GitlabPipelineDetailWithCreateAt[];
+};
+
+const insertFullPipelineList = async (
+  fullPipelineList: GitlabPipelineDetailWithCreateAt[][],
+  fullUserMap: Record<string, string>,
+  fullProjectMap: Record<string, string>
+) => {
+  const dbPipelineList: Partial<PipelineRecordModel> = [];
+
+  fullPipelineList.forEach((pipelineList) => {
+    pipelineList.forEach((pipeline) => {
+      dbPipelineList.push({
+        project_id: fullProjectMap[pipeline.project_id],
+        user_id: fullUserMap[pipeline.user.id],
+        pipeline_id: pipeline.id,
+        ref: pipeline.ref,
+        status: pipeline.status,
+        web_url: pipeline.web_url,
+        created_at: pipeline.created_at,
+        started_at: pipeline.started_at,
+        finished_at: pipeline.finished_at,
+        duration: pipeline.duration,
+        queued_duration: pipeline.queued_duration,
+      });
+    });
+  });
+  await Promise.all(
+    dbPipelineList.map((v: Partial<PipelineRecordModel>) =>
+      db.pipeline.create(v)
+    )
+  );
+};
+
+const managePipeline = {
+  getFullPipelineList,
+  insertFullPipelineList,
+};
+
+export default managePipeline;

controllers/manageProject/index.ts

@@ -0,0 +1,54 @@
+import db from "../../db";
+import { ProjectRecordModel } from "../../db/project";
+import service from "../../service";
+
+/**
+ * 填充项目信息
+ */
+const fillProj = async (project: ProjectRecordModel) => {
+  const projDetail = await service.fetchProjectDetails(project.project_id);
+  if (!projDetail) {
+    return project;
+  }
+  const useFulParams: Partial<ProjectRecordModel> = {
+    ...project,
+    avatar_url: projDetail.avatar_url,
+    description: projDetail.description,
+    name: projDetail.name,
+    path_with_namespace: projDetail.path_with_namespace,
+    web_url: projDetail.web_url,
+  };
+  return await db.project.update(project.id, useFulParams);
+};
+
+/**
+ * 获取到当前所有的项目列表
+ * 并把信息不全的项目送给fillProj填充内容
+ */
+const getFullProjList = async () => {
+  const fullList = await db.project.getFullList();
+  // 把信息不全的项目送过去填充
+  const filledProjList = await Promise.all(
+    fullList.filter((v) => !v.name).map((item) => fillProj(item))
+  );
+  // 合并成完整数据
+  const filledFullProjList = fullList
+    .filter((v) => v.name)
+    .concat(filledProjList);
+  return filledFullProjList;
+};
+
+const getFullProjectMap = (fullProjList: ProjectRecordModel[]) => {
+  const fullProjectMap: Record<string, string> = {};
+  fullProjList.forEach((item) => {
+    fullProjectMap[item.project_id] = item.id;
+  });
+  return fullProjectMap;
+};
+
+const manageProject = {
+  getFullProjList,
+  getFullProjectMap,
+};
+
+export default manageProject;

controllers/manageUser/index.ts

@@ -0,0 +1,38 @@
+import db from "../../db";
+
+const getFullUserMap = async (fullPipelineList: GitlabPipelineDetail[][]) => {
+  const userList: GitlabUser[] = [];
+  fullPipelineList.forEach((fullPipeline) => {
+    fullPipeline.forEach((item) => {
+      if (item.user && !userList.find((v) => v.id === item.user?.id)) {
+        userList.push(item.user);
+      }
+    });
+  });
+
+  const dbUserInfo = await Promise.all(
+    userList
+      .filter((v) => v.id !== 0)
+      .map((user) =>
+        db.user.upsert({
+          user_id: user.id,
+          username: user.username,
+          name: user.name,
+          avatar_url: user.avatar_url,
+          web_url: user.web_url,
+        })
+      )
+  );
+  const userMap: Record<string, string> = {};
+  dbUserInfo.forEach((item) => {
+    if (!item) return;
+    userMap[item.user_id] = item.id;
+  });
+  return userMap;
+};
+
+const manageUser = {
+  getFullUserMap,
+};
+
+export default manageUser;

db/index.ts

@@ -0,0 +1,11 @@
+import project from "./project";
+import pipeline from "./pipeline";
+import user from "./user";
+
+const db = {
+  project,
+  pipeline,
+  user,
+};
+
+export default db;

db/pb.ts

@@ -1,8 +0,0 @@
-import PocketBase from "pocketbase";
-
-const pb = new PocketBase("https://ci-pb.xiaomiwh.cn");
-
-export const getTenantAccessToken = async () => {
-  const { value } = await pb.collection("config").getOne("ugel8f0cpk0rut6");
-  return value;
-};

db/pbClient.ts

@@ -0,0 +1,7 @@
+import PocketBase from "pocketbase";
+
+const pbClient = new PocketBase("https://ci-pb.xiaomiwh.cn");
+
+pbClient.autoCancellation(false);
+
+export default pbClient;

db/pipeline/index.ts

@@ -0,0 +1,48 @@
+import { RecordModel } from "pocketbase";
+import pbClient from "../pbClient";
+import { managePb404 } from "../../utils/pbTools";
+
+export interface PipelineRecordModel extends RecordModel {
+  project_id: string;
+  user_id: string;
+  pipeline_id: number;
+  ref: string;
+  status: string;
+  web_url: string;
+  // 2024-03-06 02:53:59.509Z
+  created_at: string;
+  started_at: string;
+  finished_at: string;
+  duration: number;
+  queued_duration: number;
+}
+
+const getOne = (id: string) =>
+  managePb404(
+    async () => await pbClient.collection("pipeline").getOne(id)
+  ) as Promise<PipelineRecordModel>;
+
+/**
+ * 获取项目最新一次构建
+ * @param project_id 项目id
+ */
+const getLatestOne = (project_id: string) => {
+  return managePb404(
+    async () =>
+      await pbClient
+        .collection("pipeline")
+        .getFirstListItem(`project_id="${project_id}"`, {
+          sort: "-created_at",
+        })
+  ) as Promise<PipelineRecordModel>;
+};
+
+const create = async (data: Partial<PipelineRecordModel>) =>
+  await pbClient.collection("pipeline").create<PipelineRecordModel>(data);
+
+const pipeline = {
+  create,
+  getOne,
+  getLatestOne,
+};
+export default pipeline;

db/project/index.ts

@@ -0,0 +1,32 @@
+import { RecordModel } from "pocketbase";
+import { managePb404 } from "../../utils/pbTools";
+import pbClient from "../pbClient";
+
+export interface ProjectRecordModel extends RecordModel {
+  project_id: number;
+  description: string;
+  name: string;
+  path_with_namespace: string;
+  web_url: string;
+  avatar_url: string;
+  has_new_cicd: boolean;
+}
+
+const getOne = (id: string) =>
+  managePb404(
+    async () => await pbClient.collection("project").getOne(id)
+  ) as Promise<ProjectRecordModel>;
+
+const getFullList = async () =>
+  await pbClient.collection("project").getFullList<ProjectRecordModel>();
+
+const update = async (id: string, data: Partial<ProjectRecordModel>) =>
+  await pbClient.collection("project").update<ProjectRecordModel>(id, data);
+
+const project = {
+  getFullList,
+  getOne,
+  update,
+};
+
+export default project;

db/user/index.ts

@@ -0,0 +1,46 @@
+import { RecordModel } from "pocketbase";
+import { managePb404 } from "../../utils/pbTools";
+import pbClient from "../pbClient";
+
+export interface UserRecordModel extends RecordModel {
+  user_id: number;
+  username: string;
+  name: string;
+  avatar_url: string;
+  web_url: string;
+}
+
+const getOne = (id: string) =>
+  managePb404(
+    async () => await pbClient.collection("user").getOne(id)
+  ) as Promise<UserRecordModel>;
+
+const getOneByUserId = (user_id: number) => {
+  return managePb404(
+    async () =>
+      await pbClient
+        .collection("user")
+        .getFirstListItem(`user_id="${user_id}"`, {
+          sort: "-created",
+        })
+  ) as Promise<UserRecordModel>;
+};
+
+const create = async (data: Partial<UserRecordModel>) =>
+  await pbClient.collection("user").create<UserRecordModel>(data);
+
+const upsert = async (data: Partial<UserRecordModel>) => {
+  if (!data.user_id) return null;
+  const userInfo = await getOneByUserId(data.user_id);
+  if (userInfo) return userInfo;
+  return await create(data);
+};
+
+const user = {
+  create,
+  upsert,
+  getOne,
+  getOneByUserId,
+};
+
+export default user;

index.ts

@@ -0,0 +1,22 @@
+import { scheduleJob } from "node-schedule";
+import managePipeline from "./controllers/managePipeLine";
+import manageProject from "./controllers/manageProject";
+import manageUser from "./controllers/manageUser";
+
+const main = async () => {
+  const fullProjList = await manageProject.getFullProjList();
+  const fullPipelineList = await Promise.all(
+    fullProjList.map((v) => managePipeline.getFullPipelineList(v))
+  );
+  const fullUserMap = await manageUser.getFullUserMap(fullPipelineList);
+  const fullProjectMap = await manageProject.getFullProjectMap(fullProjList);
+  await managePipeline.insertFullPipelineList(
+    fullPipelineList,
+    fullUserMap,
+    fullProjectMap
+  );
+};
+
+main();
+
+scheduleJob("*/15 * * * *", main);

package.json

@@ -13,6 +13,9 @@
   },
   "dependencies": {
     "@types/node-schedule": "^2.1.6",
+    "@types/lodash": "^4.14.202",
+    "lodash": "^4.17.21",
+    "moment": "^2.30.1",
     "node-schedule": "^2.1.1",
     "pocketbase": "^0.21.1"
   }

readme.md

@@ -1 +1,63 @@
 # CI 监控
+
+监听新 projId，自动补全内容，获取从 20240101 到当前的所有流水线信息
+
+监听功能未知原因不好用，先不做了，改手动遍历了
+
+拿到 project_id 后，获取数据表中最新的 pipeline 的 Id，然后比对接口中的 ID 进行填充
+
+如果没有 pipeline 的 id，直接从接口中获取 20240101 到当前的流水线信息
+
+先从数据中获取用户信息填充，随后在根据填充完的用户信息获取全部的 userid 的列表，再写 pipeline 表
+
+# 图表库
+
+https://g2plot.antv.antgroup.com/examples
+
+# 数据信息
+
+project 信息
+
+```js
+{
+  id: 'aaa',
+  project_id: 131366,
+  description: "场景复现平台-展示设备（移动、音箱、小爱建议、车载、手表等设备）上小爱执行结果及相关处理流程",
+  name: "ai-scene-review-fe",
+  path_with_namespace: "miai-fe/fe/ai-scene-review-fe",
+  web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe",
+  avatar_url: null,
+  has_new_cicd: false,
+}
+```
+
+pipeline 信息
+
+```js
+{
+  id: 'bbb',
+  project_id: 'aaa',
+  user_id: 'ccc',
+  pipeline_id: 7646046,
+  ref: "preview",
+  status: "success",
+  web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe/-/pipelines/7646046",
+  started_at: "2024-03-01T16:47:40.192+08:00",
+  finished_at: "2024-03-01T16:49:30.624+08:00",
+  duration: 100,
+  queued_duration: 6,
+}
+```
+
+user 信息
+
+```js
+{
+  id: 'ccc',
+  user_id: 10011,
+  username: "zhaoyingbo",
+  name: "赵英博",
+  avatar_url: "https://git.n.xiaomi.com/uploads/-/system/user/avatar/10011/avatar.png",
+  web_url: "https://git.n.xiaomi.com/zhaoyingbo"
+}
+```

service/index.ts

@@ -0,0 +1,71 @@
+const fetchGetParams = {
+  method: "GET",
+  headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" },
+};
+
+/**
+ * 获取项目详情
+ * @param id 项目id
+ */
+const fetchProjectDetails = async (id: number) => {
+  try {
+    const response = await fetch(
+      `https://git.n.xiaomi.com/api/v4/projects/${id}`,
+      fetchGetParams
+    );
+    const body = (await response.json()) as GitlabProjDetail;
+    if (body.message === "404 Project Not Found") return null;
+    return body;
+  } catch {
+    return null;
+  }
+};
+
+/**
+ * 获取流水线列表
+ */
+const fetchPipelines = async (project_id: number, page = 1) => {
+  try {
+    const response = await fetch(
+      `https://git.n.xiaomi.com/api/v4/projects/${project_id}/pipelines?scope=finished&status=success&per_page=100&page=${page}`,
+      fetchGetParams
+    );
+    const body = (await response.json()) as GitlabPipeline[] & GitlabError;
+    if (body?.message === "404 Project Not Found") return [];
+    return body;
+  } catch {
+    return [] as GitlabPipeline[];
+  }
+};
+
+/**
+ * 获取流水线详情
+ */
+const fetchPipelineDetails = async (
+  project_id: number,
+  pipeline_id: number,
+  created_at: string
+) => {
+  try {
+    const response = await fetch(
+      `https://git.n.xiaomi.com/api/v4/projects/${project_id}/pipelines/${pipeline_id}`,
+      fetchGetParams
+    );
+    const body = (await response.json()) as GitlabPipelineDetail;
+    if (body.message === "404 Project Not Found") return null;
+    return {
+      ...body,
+      created_at,
+    };
+  } catch {
+    return null;
+  }
+};
+
+const service = {
+  fetchProjectDetails,
+  fetchPipelines,
+  fetchPipelineDetails,
+};
+
+export default service;

service/test.ts

@@ -1,39 +0,0 @@
-const fetchProjectDetails = async () => {
-  const response = await fetch(
-    "https://git.n.xiaomi.com/api/v4/projects/131366",
-    {
-      method: "GET",
-      headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" },
-    }
-  );
-  const body = await response.json();
-  console.log(body);
-};
-
-const fetchPipelineDetails = async () => {
-  const response = await fetch(
-    "https://git.n.xiaomi.com/api/v4/projects/131366/pipelines/7646046",
-    {
-      method: "GET",
-      headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" },
-    }
-  );
-  const body = await response.json();
-  console.log(body);
-};
-
-const fetchPipelines = async () => {
-  const response = await fetch(
-    "https://git.n.xiaomi.com/api/v4/projects/131366/pipelines",
-    {
-      method: "GET",
-      headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" },
-    }
-  );
-  const body = await response.json();
-  console.log(body);
-};
-
-// fetchPipelines();
-// fetchPipelineDetails();
-fetchProjectDetails();

service/typings.d.ts

@@ -0,0 +1,52 @@
+interface GitlabProjDetail {
+  id: number;
+  description: string;
+  name: string;
+  path_with_namespace: string;
+  web_url: string;
+  avatar_url?: any;
+  message?: string;
+}
+
+interface GitlabPipeline {
+  id: number;
+  project_id: number;
+  sha: string;
+  ref: string;
+  status: string;
+  source: string;
+  created_at: string;
+  updated_at: string;
+  web_url: string;
+}
+
+interface GitlabError {
+  message: string;
+}
+
+interface GitlabUser {
+  id: number;
+  username: string;
+  name: string;
+  state: string;
+  avatar_url: string;
+  web_url: string;
+}
+
+interface GitlabPipelineDetail {
+  id: number;
+  project_id: number;
+  ref: string;
+  status: string;
+  web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe/-/pipelines/7646046";
+  user: GitlabUser;
+  started_at: string;
+  finished_at: string;
+  duration: number;
+  queued_duration: number;
+  message?: string;
+}
+
+interface GitlabPipelineDetailWithCreateAt extends GitlabPipelineDetail {
+  created_at: string;
+}

utils/pbTools.ts

@@ -0,0 +1,10 @@
+export const managePb404 = async (dbFunc: Function) => {
+  try {
+    return await dbFunc();
+  } catch (err: any) {
+    // 没有这个提醒就返回空
+    if (err?.message === "The requested resource wasn't found.") {
+      return null;
+    } else throw err;
+  }
+};