commit 260aa8c350acc7308d644a8941b0f26847dac586 Author: zhaoyingbo Date: Mon May 6 03:28:58 2024 +0000 init diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..f49d6f5 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +{ + "name": "replace_me", + "image": "micr.cloud.mioffice.cn/zhaoyingbo/dev:bun", + "remoteUser": "bun", + "containerUser": "bun", + "customizations": { + "vscode": { + "settings": { + "files.autoSave": "afterDelay", + "editor.guides.bracketPairs": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.stylelint": true + } + }, + "extensions": [ + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode", + "eamodio.gitlens", + "unifiedjs.vscode-mdx", + "oderwat.indent-rainbow", + "jock.svg", + "ChakrounAnas.turbo-console-log", + "Gruntfuggly.todo-tree", + "MS-CEINTL.vscode-language-pack-zh-hans", + "stylelint.vscode-stylelint", + "GitHub.copilot" + ] + } + } +} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..81c05ed --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.lockb binary diff=lockb diff --git a/.gitea/workflows/cicd.yaml.template b/.gitea/workflows/cicd.yaml.template new file mode 100644 index 0000000..216ee74 --- /dev/null +++ b/.gitea/workflows/cicd.yaml.template @@ -0,0 +1,57 @@ +name: CI Monitor CI/CD +on: [push] + +jobs: + build-image: + runs-on: ubuntu-latest + container: catthehacker/ubuntu:act-latest + steps: + - name: Check out repository code + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: git.yingbo.im:333 + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build and push + uses: docker/build-push-action@v4 + with: + push: true + tags: git.yingbo.im:333/zhaoyingbo/ci_monitor:${{ github.sha }} + + deploy: + needs: build-image + runs-on: ubuntu-latest + container: catthehacker/ubuntu:act-latest + steps: + # 检出代码 + - name: Check out repository code + uses: actions/checkout@v3 + # 使用scp命令将docker-compose.yml文件上传到服务器 + - name: Upload docker-compose.yml to server + uses: appleboy/scp-action@master + with: + host: ${{ secrets.SERVER_HOST }} + username: ${{ secrets.SERVER_USERNAME }} + key: ${{ secrets.SERVER_KEY }} + port: ${{ secrets.SERVER_PORT }} + source: docker-compose.yml + target: /home/${{ secrets.SERVER_USERNAME }}/docker/ci_monitor + # 登录服务器，执行docker-compose命令 + - name: Login to the server and execute docker-compose command + uses: appleboy/ssh-action@master + with: + host: ${{ secrets.SERVER_HOST }} + username: ${{ secrets.SERVER_USERNAME }} + key: ${{ secrets.SERVER_KEY }} + port: ${{ secrets.SERVER_PORT }} + script: | + docker login -u ${{ secrets.DOCKER_USERNAME }} -p ${{ secrets.DOCKER_PASSWORD }} git.yingbo.im:333 + cd /home/${{ secrets.SERVER_USERNAME }}/docker/ci_monitor + sed -i "s/sha/${{ github.sha }}/g" docker-compose.yml + docker compose up -d --force-recreate --no-deps ci_monitor diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..52962c2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,58 @@ +# Logs +logs +*.log +npm-debug.log* + +# Runtime data +pids +*.pid +*.seed + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules +jspm_packages + +# Optional npm cache directory +.npm + +# Optional REPL history +.node_repl_history + +# 0x +profile-* + +# mac files +.DS_Store + +# vim swap files +*.swp + +# webstorm +.idea + +# vscode +.vscode +*code-workspace + +# clinic +profile* +*clinic* +*flamegraph* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..90e78e6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM micr.cloud.mioffice.cn/zhaoyingbo/bun:alpine-cn + +WORKDIR /app + +COPY package*.json ./ + +COPY bun.lockb ./ + +RUN bun install + +COPY . . + +CMD ["bun", "start"] \ No newline at end of file diff --git a/bun.lockb b/bun.lockb new file mode 100755 index 0000000..9961da9 Binary files /dev/null and b/bun.lockb differ diff --git a/controllers/template/index.ts b/controllers/template/index.ts new file mode 100644 index 0000000..437b535 --- /dev/null +++ b/controllers/template/index.ts @@ -0,0 +1,7 @@ +const templateFunc = () => {} + +const template = { + templateFunc, +}; + +export default template; diff --git a/db/index.ts b/db/index.ts new file mode 100644 index 0000000..8570c84 --- /dev/null +++ b/db/index.ts @@ -0,0 +1,7 @@ +import user from "./user"; + +const db = { + user, +}; + +export default db; diff --git a/db/pbClient.ts b/db/pbClient.ts new file mode 100644 index 0000000..7999649 --- /dev/null +++ b/db/pbClient.ts @@ -0,0 +1,7 @@ +import PocketBase from "pocketbase"; + +const pbClient = new PocketBase("https://ci-pb.xiaomiwh.cn"); + +pbClient.autoCancellation(false); + +export default pbClient; diff --git a/db/user/index.ts b/db/user/index.ts new file mode 100644 index 0000000..41f30ce --- /dev/null +++ b/db/user/index.ts @@ -0,0 +1,46 @@ +import { RecordModel } from "pocketbase"; +import { managePb404 } from "../../utils/pbTools"; +import pbClient from "../pbClient"; + +export interface UserRecordModel extends RecordModel { + user_id: number; + username: string; + name: string; + avatar_url: string; + web_url: string; +} + +const getOne = (id: string) => + managePb404( + async () => await pbClient.collection("user").getOne(id) + ) as Promise; + +const getOneByUserId = (user_id: number) => { + return managePb404( + async () => + await pbClient + .collection("user") + .getFirstListItem(`user_id="${user_id}"`, { + sort: "-created", + }) + ) as Promise; +}; + +const create = async (data: Partial) => + await pbClient.collection("user").create(data); + +const upsert = async (data: Partial) => { + if (!data.user_id) return null; + const userInfo = await getOneByUserId(data.user_id); + if (userInfo) return userInfo; + return await create(data); +}; + +const user = { + create, + upsert, + getOne, + getOneByUserId, +}; + +export default user; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..876edd6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,9 @@ +version: "3" + +services: + replace_me: + image: git.yingbo.im:333/zhaoyingbo/replace_me:sha + container_name: replace_me + restart: always + ports: + - 3001:3000 diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..ac8d057 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,26 @@ +export default { + languageOptions: { + ecmaVersion: 2021, + sourceType: 'module', + }, + ignores: ['*.js', '*.cjs', '*.mjs', '/src/Backup/*'], + plugins: ['@typescript-eslint', 'simple-import-sort'], + extends: [ + 'eslint:recommended', + 'plugin:@typescript-eslint/recommended', + 'prettier', + ], + rules: { + '@typescript-eslint/no-explicit-any': 'off', + 'simple-import-sort/imports': 'error', + 'simple-import-sort/exports': 'error', + }, + overrides: [ + { + files: ['**/*.ts', '**/*.tsx'], + languageOptions: { + parser: '@typescript-eslint/parser', + }, + }, + ], +}; \ No newline at end of file diff --git a/index.ts b/index.ts new file mode 100644 index 0000000..e67874e --- /dev/null +++ b/index.ts @@ -0,0 +1,14 @@ +import { initSchedule } from "./schedule"; + +initSchedule() + +Bun.serve({ + async fetch(req) { + const url = new URL(req.url); + // 根路由 + if (url.pathname === "/") return new Response("hello, glade to see you!"); + if (url.pathname === '/ci') return new Response("OK") + return new Response("OK"); + }, + port: 3000, +}); diff --git a/package.json b/package.json new file mode 100644 index 0000000..7287580 --- /dev/null +++ b/package.json @@ -0,0 +1,25 @@ +{ + "name": "replace_me", + "module": "index.ts", + "type": "module", + "scripts": { + "start": "bun run index.ts" + }, + "devDependencies": { + "eslint": "^9.2.0", + "bun-types": "latest", + "@types/lodash": "^4.14.202", + "@types/node-schedule": "^2.1.6", + "eslint-plugin-simple-import-sort": "^12.1.0", + "@typescript-eslint/eslint-plugin": "^7.8.0", + "@typescript-eslint/parser": "^7.8.0" + }, + "peerDependencies": { + "typescript": "^5.0.0" + }, + "dependencies": { + "lodash": "^4.17.21", + "node-schedule": "^2.1.1", + "pocketbase": "^0.21.1" + } +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..ac2512f --- /dev/null +++ b/readme.md @@ -0,0 +1,182 @@ +# CI 监控 + +监听新 projId，自动补全内容，获取从 20240101 到当前的所有流水线信息 + +监听功能未知原因不好用，先不做了，改手动遍历了 + +拿到 project_id 后，获取数据表中最新的 pipeline 的 Id，然后比对接口中的 ID 进行填充 + +如果没有 pipeline 的 id，直接从接口中获取 20240101 到当前的流水线信息 + +先从数据中获取用户信息填充，随后在根据填充完的用户信息获取全部的 userid 的列表，再写 pipeline 表 + +# 图表库（不用了） + +https://g2plot.antv.antgroup.com/examples + +# 数据信息 + +project 信息 + +```js +{ + id: 'aaa', + project_id: 131366, + description: "场景复现平台-展示设备（移动、音箱、小爱建议、车载、手表等设备）上小爱执行结果及相关处理流程", + name: "ai-scene-review-fe", + path_with_namespace: "miai-fe/fe/ai-scene-review-fe", + web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe", + avatar_url: null, + has_new_cicd: false, +} +``` + +pipeline 信息 + +```js +{ + id: 'bbb', + project_id: 'aaa', + user_id: 'ccc', + pipeline_id: 7646046, + ref: "preview", + status: "success", + web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe/-/pipelines/7646046", + started_at: "2024-03-01T16:47:40.192+08:00", + finished_at: "2024-03-01T16:49:30.624+08:00", + duration: 100, + queued_duration: 6, +} +``` + +user 信息 + +```js +{ + id: 'ccc', + user_id: 10011, + username: "zhaoyingbo", + name: "赵英博", + avatar_url: "https://git.n.xiaomi.com/uploads/-/system/user/avatar/10011/avatar.png", + web_url: "https://git.n.xiaomi.com/zhaoyingbo" +} +``` + +每周每个项目按分支的运行时长统计 SQL `statisticsPerProj` + +```SQL +SELECT + (ROW_NUMBER() OVER()) as id, + strftime('%Y-%W', datetime(pip.started_at)) AS week, + p.name AS name, + ROUND(AVG(pip.duration/60.0), 1) AS duration, + pip.ref +FROM project p +JOIN pipeline pip ON p.id = pip.project_id +GROUP BY name, week, pip.ref; +``` + +每周流水线运行统计 SQL `statisticsPerWeek` + +```SQL +SELECT + (ROW_NUMBER() OVER()) as id, + strftime('%Y-%W', datetime(started_at)) AS week, + COUNT(*) AS total_count, + SUM(CASE WHEN status = 'success' THEN 0 ELSE 1 END) AS failed_count, + SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS success_count, + ROUND(SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS success_rate, + ROUND(AVG(duration/60.0), 1) AS duration +FROM pipeline +GROUP BY week; +``` + +# GPT + +我有一个 sqlite 数据库，表如下 +project 表 + +```js +{ + id: 'aaa', + project_id: 131366, + description: "场景复现平台-展示设备（移动、音箱、小爱建议、车载、手表等设备）上小爱执行结果及相关处理流程", + name: "ai-scene-review-fe", + path_with_namespace: "miai-fe/fe/ai-scene-review-fe", + web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe", + avatar_url: null, + has_new_cicd: false, +} +``` + +pipeline 表 + +```js +{ + id: 'bbb', + project_id: 'aaa', + user_id: 'ccc', + pipeline_id: 7646046, + ref: "preview", + status: "success", + web_url: "https://git.n.xiaomi.com/miai-fe/fe/ai-scene-review-fe/-/pipelines/7646046", + started_at: "2024-03-01T16:47:40.192+08:00", + finished_at: "2024-03-01T16:49:30.624+08:00", + duration: 100, + queued_duration: 6, +} +``` + +user 表 + +```js +{ + id: 'ccc', + user_id: 10011, + username: "zhaoyingbo", + name: "赵英博", + avatar_url: "https://git.n.xiaomi.com/uploads/-/system/user/avatar/10011/avatar.png", + web_url: "https://git.n.xiaomi.com/zhaoyingbo" +} +``` + +我想按天展示每个项目的 pipline 按 ref 区分的平均 duration，如何创建视图 + +# 机器人 + +卡片 ID：ctp_AAyVLS6Q37cL + +JSON 示例 + +```json +{ + "total_count": "29", // OK + "group_table": [ + { + "project_name": "ai-ak-fe", + "project_ref": "master", + "project_duration": "1.4", + "project_duration_rate": "↓12%" + }, + { + "project_name": "ai-class-schedule-fe", + "project_ref": "preview", + "project_duration": "3.2", + "project_duration_rate": "↑5%" + }, + { + "project_name": "ai-scene-review-fe", + "project_ref": "staging", + "project_duration": "5.4", + "project_duration_rate": "↓6%" + } + ], + "weekly_count_rate": "较上周 ↑5%", // OK + "weekly_duration_rate": "较上周 ↓12%", // OK + "weekly_success_rate": "较上周 ↑12%", // OK + "duration": "0.9", // OK + "success_rate": "100", // OK + "has_new_cicd_count": "15", // OK + "without_new_cicd_count": "20" // OK +} +``` diff --git a/schedule/index.ts b/schedule/index.ts new file mode 100644 index 0000000..6a3ceec --- /dev/null +++ b/schedule/index.ts @@ -0,0 +1,10 @@ +import schedule from 'node-schedule' + +const func = () => {} + +export const initSchedule = async () => { + // 定时任务，每15分钟刷新一次token + schedule.scheduleJob('*/15 * * * *', func); + // 立即执行一次 + func() +} diff --git a/service/index.ts b/service/index.ts new file mode 100644 index 0000000..6fcaa20 --- /dev/null +++ b/service/index.ts @@ -0,0 +1,28 @@ +const fetchGetParams = { + method: "GET", + headers: { "PRIVATE-TOKEN": "Zd1UASPcMwVox5tNS6ep" }, +}; + +/** + * 获取项目详情 + * @param id 项目id + */ +const fetchTemplate = async (id: number) => { + try { + const response = await fetch( + `https://git.n.xiaomi.com/api/v4/projects/${id}`, + fetchGetParams + ); + const body = (await response.json()) as any; + if (body.message === "404 Project Not Found") return null; + return body; + } catch { + return null; + } +}; + +const service = { + fetchTemplate, +}; + +export default service; diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..7556e1d --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "lib": ["ESNext"], + "module": "esnext", + "target": "esnext", + "moduleResolution": "bundler", + "moduleDetection": "force", + "allowImportingTsExtensions": true, + "noEmit": true, + "composite": true, + "strict": true, + "downlevelIteration": true, + "skipLibCheck": true, + "jsx": "react-jsx", + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true, + "allowJs": true, + "types": [ + "bun-types" // add Bun global + ] + } +} diff --git a/types/index.ts b/types/index.ts new file mode 100644 index 0000000..e69de29 diff --git a/utils/pbTools.ts b/utils/pbTools.ts new file mode 100644 index 0000000..01d3186 --- /dev/null +++ b/utils/pbTools.ts @@ -0,0 +1,10 @@ +export const managePb404 = async (dbFunc: Function) => { + try { + return await dbFunc(); + } catch (err: any) { + // 没有这个提醒就返回空 + if (err?.message === "The requested resource wasn't found.") { + return null; + } else throw err; + } +};