From bc441827ecb5e4192239f793046dd6053e060c4a Mon Sep 17 00:00:00 2001 From: zhaoyingbo Date: Thu, 7 Mar 2024 12:05:10 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E4=BA=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- controllers/manageRobot/index.ts | 121 +++++++++++++++++++++++++++++ db/index.ts | 2 + db/view/index.ts | 44 +++++++++++ docker-compose.yml | 2 + index.ts | 11 +++ readme.md | 128 +++++++++++++------------------ service/index.ts | 23 +++++- utils/robotTools.ts | 61 +++++++++++++++ utils/timeTools.ts | 22 ++++++ 9 files changed, 339 insertions(+), 75 deletions(-) create mode 100644 controllers/manageRobot/index.ts create mode 100644 db/view/index.ts create mode 100644 utils/robotTools.ts create mode 100644 utils/timeTools.ts diff --git a/controllers/manageRobot/index.ts b/controllers/manageRobot/index.ts new file mode 100644 index 0000000..c913db1 --- /dev/null +++ b/controllers/manageRobot/index.ts @@ -0,0 +1,121 @@ +import db from "../../db"; +import service from "../../service"; +import { calculateWeeklyRate } from "../../utils/robotTools"; +import { + getPrevWeekWithYear, + getWeekTimeWithYear, +} from "../../utils/timeTools"; + +const getNewCicdStatus = async () => { + const fullProjList = await db.project.getFullList(); + const has_new_cicd_count = String( + fullProjList.filter((item) => { + return item.has_new_cicd === true; + }).length + ); + const without_new_cicd_count = String( + fullProjList.filter((item) => { + return item.has_new_cicd === false; + }).length + ); + return { + has_new_cicd_count, + without_new_cicd_count, + }; +}; + +const getStatisticsInfo = async () => { + const curWeekInfo = await db.view.getFullStatisticsByWeek( + getWeekTimeWithYear() + ); + const prevWeekInfo = await db.view.getFullStatisticsByWeek( + getPrevWeekWithYear() + ); + return { + total_count: String(curWeekInfo.total_count), + weekly_count_rate: calculateWeeklyRate( + curWeekInfo.total_count, + prevWeekInfo.total_count + ).text, + duration: String(curWeekInfo.duration), + weekly_duration_rate: calculateWeeklyRate( + curWeekInfo.duration, + prevWeekInfo.duration + ).text, + success_rate: String(curWeekInfo.success_rate), + weekly_success_rate: calculateWeeklyRate( + curWeekInfo.success_rate, + prevWeekInfo.success_rate + ).text, + }; +}; + +const getProjDiffInfo = async () => { + const curWeekInfo = await db.view.getProjStatisticsByWeek( + getWeekTimeWithYear() + ); + const prevWeekInfo = await db.view.getProjStatisticsByWeek( + getPrevWeekWithYear() + ); + + const group: { + project_name: string; + project_ref: string; + project_duration: string; + project_duration_rate: string; + percentage: string; + }[] = []; + + curWeekInfo.forEach((curWeekProjInfo) => { + const prevWeekProjInfo = prevWeekInfo.find( + (info) => + info.ref === curWeekProjInfo.ref && info.name === curWeekProjInfo.name + ); + if (!prevWeekProjInfo) return; + + const { text: project_duration_rate, percentage } = calculateWeeklyRate( + curWeekProjInfo.duration, + prevWeekProjInfo.duration, + false + ); + + if (percentage === "0") return; + + group.push({ + project_name: curWeekProjInfo.name, + project_ref: curWeekProjInfo.ref, + project_duration: String(curWeekProjInfo.duration), + project_duration_rate, + percentage, + }); + }); + + // 排序 + group.sort((a, b) => Number(b.percentage) - Number(a.percentage)); + + // 取前五个 + return group.slice(0, 5); +}; + +const sendRobotMsg = async () => { + const msgContent = { + type: "template", + data: { + template_id: "ctp_AAyVLS6Q37cL", + template_variable: { + ...(await getNewCicdStatus()), + ...(await getStatisticsInfo()), + group_table: await getProjDiffInfo(), + }, + }, + }; + + const res = await service.sendMessage(JSON.stringify(msgContent)); + console.log(res); +}; + +const manageRobot = { + sendRobotMsg, +}; + +export default manageRobot; diff --git a/db/index.ts b/db/index.ts index 5924e81..164175f 100644 --- a/db/index.ts +++ b/db/index.ts @@ -1,11 +1,13 @@ import project from "./project"; import pipeline from "./pipeline"; import user from "./user"; +import view from "./view"; const db = { project, pipeline, user, + view, }; export default db; diff --git a/db/view/index.ts b/db/view/index.ts new file mode 100644 index 0000000..060667f --- /dev/null +++ b/db/view/index.ts @@ -0,0 +1,44 @@ +import { RecordModel } from "pocketbase"; +import { managePb404 } from "../../utils/pbTools"; +import pbClient from "../pbClient"; + +export interface StatisticsPerWeekRecordModel extends RecordModel { + week: string; + total_count: number; + failed_count: number; + success_count: number; + success_rate: number; + duration: number; +} + +export interface StatisticsPerProjRecordModel extends RecordModel { + week: string; + name: string; + duration: number; + ref: string; +} + +const getFullStatisticsByWeek = (week: string) => { + return managePb404( + async () => + await pbClient + .collection("statisticsPerWeek") + .getFirstListItem(`week="${week}"`) + ) as Promise; +}; + +const getProjStatisticsByWeek = (week: string) => { + return managePb404( + async () => + await pbClient + .collection("statisticsPerProj") + .getFullList({ filter: `week="${week}"` }) + ) as Promise; +}; + +const view = { + getFullStatisticsByWeek, + getProjStatisticsByWeek, +}; + +export default view; diff --git a/docker-compose.yml b/docker-compose.yml index 52a3259..da5cffc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,3 +5,5 @@ services: image: git.yingbo.im:333/zhaoyingbo/ci_monitor:sha container_name: ci_monitor restart: always + ports: + - 3001:3000 diff --git a/index.ts b/index.ts index c3d088e..a7e7ea3 100644 --- a/index.ts +++ b/index.ts @@ -2,6 +2,7 @@ import { scheduleJob } from "node-schedule"; import managePipeline from "./controllers/managePipeLine"; import manageProject from "./controllers/manageProject"; import manageUser from "./controllers/manageUser"; +import manageRobot from "./controllers/manageRobot"; const main = async () => { const fullProjList = await manageProject.getFullProjList(); @@ -20,3 +21,13 @@ const main = async () => { main(); scheduleJob("*/15 * * * *", main); + +scheduleJob("0 10 * * 5", manageRobot.sendRobotMsg); + +Bun.serve({ + async fetch() { + await manageRobot.sendRobotMsg(); + return new Response("OK"); + }, + port: 3000, +}); diff --git a/readme.md b/readme.md index ac4bf84..ac2512f 100644 --- a/readme.md +++ b/readme.md @@ -10,7 +10,7 @@ 先从数据中获取用户信息填充，随后在根据填充完的用户信息获取全部的 userid 的列表，再写 pipeline 表 -# 图表库 +# 图表库（不用了） https://g2plot.antv.antgroup.com/examples @@ -62,92 +62,33 @@ user 信息 } ``` -date 视图 SQL +每周每个项目按分支的运行时长统计 SQL `statisticsPerProj` ```SQL SELECT (ROW_NUMBER() OVER()) as id, - p.has_new_cicd, - p.name AS project_name, - strftime('%Y-%m-%d', datetime(pip.started_at)) AS date, - pip.ref, - AVG(pip.duration) AS avg_duration -FROM project p -JOIN pipeline pip ON p.id = pip.project_id -GROUP BY project_name, date, pip.ref; -``` - -week 视图 SQL - -```SQL -SELECT - (ROW_NUMBER() OVER()) as id, - p.has_new_cicd, - p.name AS project_name, strftime('%Y-%W', datetime(pip.started_at)) AS week, - pip.ref, - AVG(pip.duration) AS avg_duration + p.name AS name, + ROUND(AVG(pip.duration/60.0), 1) AS duration, + pip.ref FROM project p JOIN pipeline pip ON p.id = pip.project_id -GROUP BY project_name, week, pip.ref; +GROUP BY name, week, pip.ref; ``` -本周平均用时视图 SQL +每周流水线运行统计 SQL `statisticsPerWeek` ```SQL SELECT (ROW_NUMBER() OVER()) as id, - p.has_new_cicd, - pip.ref, - AVG(pip.duration) AS avg_duration -FROM project p -JOIN pipeline pip ON p.id = pip.project_id -WHERE strftime('%Y-%W', datetime(pip.started_at)) = strftime('%Y-%W', 'now') -GROUP BY p.has_new_cicd, pip.ref; -``` - -上周平均用时视图 SQL - -```SQL -SELECT - (ROW_NUMBER() OVER()) as id, - p.has_new_cicd, - pip.ref, - AVG(pip.duration) AS avg_duration -FROM project p -JOIN pipeline pip ON p.id = pip.project_id -WHERE strftime('%Y-%W', datetime(pip.started_at)) = strftime('%Y-%W', 'now', '-7 days') -GROUP BY p.has_new_cicd, pip.ref; -``` - -本周每个项目平均用时视图 SQL - -```SQL -SELECT - (ROW_NUMBER() OVER()) as id, - p.has_new_cicd, - p.name AS project_name, - pip.ref, - AVG(pip.duration) AS avg_duration -FROM project p -JOIN pipeline pip ON p.id = pip.project_id -WHERE strftime('%Y-%W', datetime(pip.started_at)) = strftime('%Y-%W', 'now') -GROUP BY p.has_new_cicd, p.name, pip.ref; -``` - -上周每个项目平均用时视图 SQL - -```SQL -SELECT - (ROW_NUMBER() OVER()) as id, - p.has_new_cicd, - p.name AS project_name, - pip.ref, - AVG(pip.duration) AS avg_duration -FROM project p -JOIN pipeline pip ON p.id = pip.project_id -WHERE strftime('%Y-%W', datetime(pip.started_at)) = strftime('%Y-%W', 'now', '-7 days') -GROUP BY p.has_new_cicd, p.name, pip.ref; + strftime('%Y-%W', datetime(started_at)) AS week, + COUNT(*) AS total_count, + SUM(CASE WHEN status = 'success' THEN 0 ELSE 1 END) AS failed_count, + SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS success_count, + ROUND(SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS success_rate, + ROUND(AVG(duration/60.0), 1) AS duration +FROM pipeline +GROUP BY week; ``` # GPT @@ -200,3 +141,42 @@ user 表 ``` 我想按天展示每个项目的 pipline 按 ref 区分的平均 duration，如何创建视图 + +# 机器人 + +卡片 ID：ctp_AAyVLS6Q37cL + +JSON 示例 + +```json +{ + "total_count": "29", // OK + "group_table": [ + { + "project_name": "ai-ak-fe", + "project_ref": "master", + "project_duration": "1.4", + "project_duration_rate": "↓12%" + }, + { + "project_name": "ai-class-schedule-fe", + "project_ref": "preview", + "project_duration": "3.2", + "project_duration_rate": "↑5%" + }, + { + "project_name": "ai-scene-review-fe", + "project_ref": "staging", + "project_duration": "5.4", + "project_duration_rate": "↓6%" + } + ], + "weekly_count_rate": "较上周 ↑5%", // OK + "weekly_duration_rate": "较上周 ↓12%", // OK + "weekly_success_rate": "较上周 ↑12%", // OK + "duration": "0.9", // OK + "success_rate": "100", // OK + "has_new_cicd_count": "15", // OK + "without_new_cicd_count": "20" // OK +} +``` diff --git a/service/index.ts b/service/index.ts index 000be3e..b2d9785 100644 --- a/service/index.ts +++ b/service/index.ts @@ -27,7 +27,7 @@ const fetchProjectDetails = async (id: number) => { const fetchPipelines = async (project_id: number, page = 1) => { try { const response = await fetch( - `https://git.n.xiaomi.com/api/v4/projects/${project_id}/pipelines?scope=finished&status=success&per_page=100&page=${page}`, + `https://git.n.xiaomi.com/api/v4/projects/${project_id}/pipelines?scope=finished&per_page=100&page=${page}`, fetchGetParams ); const body = (await response.json()) as GitlabPipeline[] & GitlabError; @@ -62,10 +62,31 @@ const fetchPipelineDetails = async ( } }; +const sendMessage = async (content: string) => { + try { + const response = await fetch("https://egg.imoaix.cn/message", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + group_id: "52usf3w8l6z4vs1", + msg_type: "interactive", + content, + }), + }); + const body = await response.json(); + return body; + } catch { + return null; + } +}; + const service = { fetchProjectDetails, fetchPipelines, fetchPipelineDetails, + sendMessage, }; export default service; diff --git a/utils/robotTools.ts b/utils/robotTools.ts new file mode 100644 index 0000000..43df940 --- /dev/null +++ b/utils/robotTools.ts @@ -0,0 +1,61 @@ +/** + * 计算百分比变化 + * @param a + * @param b + */ +export const calculatePercentageChange = (cur: number, prev: number) => { + // 计算差值 + const diff = cur - prev; + + if (diff === 0) + return { + diff, + percentage: "0", + }; + + // 计算百分比 + const percentage = Math.abs((diff / prev) * 100).toFixed(1); + + return { + diff, + percentage, + }; +}; + +/** + * 计算周同比 + * @param cur + * @param prev + * @param needCN + */ +export const calculateWeeklyRate = ( + cur: string | number, + prev: string | number, + needCN = true +) => { + const { diff, percentage } = calculatePercentageChange( + Number(cur), + Number(prev) + ); + if (diff > 0) + return { + text: `${ + needCN ? "较上周 " : "" + }↑${percentage}%`, + diff, + percentage, + }; + if (diff < 0) + return { + text: `${ + needCN ? "较上周 " : "" + }↓${percentage}%`, + diff, + percentage, + }; + return { + text: `${needCN ? "较上周 " : ""}0%`, + diff, + percentage, + }; +}; diff --git a/utils/timeTools.ts b/utils/timeTools.ts new file mode 100644 index 0000000..cece517 --- /dev/null +++ b/utils/timeTools.ts @@ -0,0 +1,22 @@ +import moment from "moment"; + +/** + * 获取今天是今年的第几周，like 2024-05 + */ +export const getWeekTimeWithYear = () => { + return moment().format("YYYY-WW"); +}; + +/** + * 获取上周是今年的第几周，like 2024-04 + */ +export const getPrevWeekWithYear = () => { + return moment().subtract(1, "weeks").format("YYYY-WW"); +}; + +/** + * 秒转分钟，保留一位小数 + */ +export const sec2min = (sec: number) => { + return (sec / 60).toFixed(1); +};