From 163657a411645562af19ee4a4ec01e7a0f5f17b1 Mon Sep 17 00:00:00 2001 From: Katsumi Kato Date: Tue, 19 Nov 2024 13:38:21 +0900 Subject: [PATCH] ci: Display performance measurement results as custom metrics (#3491) * Debug result * Fix * Refactoring to convert standard input to json format for custom metrics output * Custom metrics are now output as comments in Pull Reqeust * Fix `undefined is not an object (evaluating 'line.split')` * Add check to make sure line is not undefined * Debug result * Fix * Refactoring to convert standard input to json format for custom metrics output * Custom metrics are now output as comments in Pull Reqeust * Fix `undefined is not an object (evaluating 'line.split')` * Add check to make sure line is not undefined * Move path to parent directory to consolidate multiple metrics and output results * Refactoring to convert to octocov custom metrics format * The two performance measurement jobs were merged and compared output by octocov. * Append unit * bonsai * Review of title * Integrate bundling and size check processes * Fix code style * bonsai * Fix * refs https://github.com/k2tzumi/hono/actions/runs/11903022493/job/33169186026#step:7:96, Fix `no such file or directory` * Fix lint & code style --- .github/workflows/ci.yml | 103 ++++++------------ perf-measures/.octocov.perf-measures.main.yml | 13 +++ perf-measures/.octocov.perf-measures.yml | 15 +++ perf-measures/bundle-check/.gitignore | 1 + .../bundle-check/scripts/check-bundle-size.ts | 43 ++++++++ .../bundle-check/scripts/process-results.ts | 14 --- perf-measures/type-check/.gitignore | 1 + .../type-check/scripts/process-results.ts | 49 +++++++-- 8 files changed, 144 insertions(+), 95 deletions(-) create mode 100644 perf-measures/.octocov.perf-measures.main.yml create mode 100644 perf-measures/.octocov.perf-measures.yml create mode 100644 perf-measures/bundle-check/scripts/check-bundle-size.ts delete mode 100644 perf-measures/bundle-check/scripts/process-results.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a6b109e..ba18c775 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,88 +181,53 @@ jobs: name: coverage-lambda-edge path: coverage/ - perf-measures-type-check-on-pr: - name: 'Type Check on PR' + perf-measures-type-and-bundle-check-on-pr: + name: 'Type & Bundle size Check on PR' runs-on: ubuntu-latest if: github.event_name == 'pull_request' steps: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 - run: bun install - - uses: actions/cache/restore@v4 + - name: Performance measurement of type check + run: | + bun scripts/generate-app.ts + bun tsc -p tsconfig.build.json --diagnostics | bun scripts/process-results.ts > diagnostics.json + working-directory: perf-measures/type-check + - name: Performance measurement of bundle check + run: | + bun run build + bun perf-measures/bundle-check/scripts/check-bundle-size.ts > perf-measures/bundle-check/size.json + - name: Run octocov + uses: k1LoW/octocov-action@v1 with: - path: perf-measures/type-check/previous-result.txt - restore-keys: | - type-check-perf-previous-result- - key: type-check-perf-previous-result- - - run: bun scripts/generate-app.ts - working-directory: perf-measures/type-check - - run: bun tsc -p tsconfig.build.json --diagnostics > result.txt - working-directory: perf-measures/type-check - - run: | - { - echo 'COMPARISON<> "$GITHUB_ENV" - working-directory: perf-measures/type-check - - run: echo "$COMPARISON" - name: display comparison + config: perf-measures/.octocov.perf-measures.yml + env: + OCTOCOV_CUSTOM_METRICS_BENCHMARK_BUNDLE: perf-measures/bundle-check/size.json + OCTOCOV_CUSTOM_METRICS_BENCHMARK_TYPE: perf-measures/type-check/diagnostics.json - perf-measures-type-check-on-main: - name: 'Type Check on Main' + perf-measures-type-and-bundle-check-on-main: + name: 'Type & Bundle size Check on Main' runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' steps: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 - run: bun install - - run: bun scripts/generate-app.ts + - name: Performance measurement of type check + run: | + bun scripts/generate-app.ts + bun tsc -p tsconfig.build.json --diagnostics | bun scripts/process-results.ts > diagnostics.json working-directory: perf-measures/type-check - - run: bun tsc -p tsconfig.build.json --diagnostics > previous-result.txt - working-directory: perf-measures/type-check - - uses: actions/cache/save@v4 + - name: Performance measurement of bundle check + run: | + bun run build + bun perf-measures/bundle-check/scripts/check-bundle-size.ts > perf-measures/bundle-check/size.json + - name: Run octocov (main) + uses: k1LoW/octocov-action@v1 with: - path: perf-measures/type-check/previous-result.txt - key: type-check-perf-previous-result-${{ github.sha }} - - perf-measures-bundle-check-on-pr: - name: 'Bundle Check on PR' - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - uses: actions/checkout@v4 - - uses: oven-sh/setup-bun@v2 - - run: bun install - - run: bun run build - - run: bunx esbuild --minify --bundle dist/index.js --format=esm --outfile=perf-measures/bundle-check/generated/after.js - - uses: actions/cache/restore@v4 - with: - path: perf-measures/bundle-check/generated/before.js - restore-keys: | - perf-measures-bundle-check-previous-file- - key: perf-measures-bundle-check-previous-file- - - run: | - { - echo 'COMPARISON<> "$GITHUB_ENV" - working-directory: perf-measures/bundle-check - - run: echo "$COMPARISON" - name: display comparison - - perf-measures-bundle-check-on-main: - name: 'Bundle Check on Main' - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' - steps: - - uses: actions/checkout@v4 - - uses: oven-sh/setup-bun@v2 - - run: bun install - - run: bun run build - - run: bunx esbuild --minify --bundle dist/index.js --format=esm --outfile=perf-measures/bundle-check/generated/before.js - - uses: actions/cache/save@v4 - with: - path: perf-measures/bundle-check/generated/before.js - key: perf-measures-bundle-check-previous-file-${{ github.sha }} \ No newline at end of file + config: perf-measures/.octocov.perf-measures.main.yml + env: + OCTOCOV_GITHUB_REF: refs/heads/main + OCTOCOV_CUSTOM_METRICS_BENCHMARK_BUNDLE: perf-measures/bundle-check/size.json + OCTOCOV_CUSTOM_METRICS_BENCHMARK_TYPE: perf-measures/type-check/diagnostics.json diff --git a/perf-measures/.octocov.perf-measures.main.yml b/perf-measures/.octocov.perf-measures.main.yml new file mode 100644 index 00000000..17691555 --- /dev/null +++ b/perf-measures/.octocov.perf-measures.main.yml @@ -0,0 +1,13 @@ +locale: "en" +repository: ${GITHUB_REPOSITORY}/perf-measures +coverage: + if: false +codeToTestRatio: + if: false +testExecutionTime: + if: false +report: + datastores: + - artifact://${GITHUB_REPOSITORY} +summary: + if: true diff --git a/perf-measures/.octocov.perf-measures.yml b/perf-measures/.octocov.perf-measures.yml new file mode 100644 index 00000000..63577e84 --- /dev/null +++ b/perf-measures/.octocov.perf-measures.yml @@ -0,0 +1,15 @@ +locale: "en" +repository: ${GITHUB_REPOSITORY}/perf-measures +coverage: + if: false +codeToTestRatio: + if: false +testExecutionTime: + if: false +diff: + datastores: + - artifact://${GITHUB_REPOSITORY} +comment: + if: is_pull_request +summary: + if: true diff --git a/perf-measures/bundle-check/.gitignore b/perf-measures/bundle-check/.gitignore index 49795c58..2d86ef13 100644 --- a/perf-measures/bundle-check/.gitignore +++ b/perf-measures/bundle-check/.gitignore @@ -1,2 +1,3 @@ generated !generated/.gitkeep +size.json diff --git a/perf-measures/bundle-check/scripts/check-bundle-size.ts b/perf-measures/bundle-check/scripts/check-bundle-size.ts new file mode 100644 index 00000000..646ce931 --- /dev/null +++ b/perf-measures/bundle-check/scripts/check-bundle-size.ts @@ -0,0 +1,43 @@ +import * as esbuild from 'esbuild' +import * as fs from 'node:fs' +import * as os from 'os' +import * as path from 'path' + +async function main() { + const tempDir = os.tmpdir() + const tempFilePath = path.join(tempDir, 'bundle.tmp.js') + + try { + await esbuild.build({ + entryPoints: ['dist/index.js'], + bundle: true, + minify: true, + format: 'esm' as esbuild.Format, + target: 'es2022', + outfile: tempFilePath, + }) + + const bundleSize = fs.statSync(tempFilePath).size + const metrics = [] + metrics.push({ + key: 'bundle-size', + name: 'Bundle Size', + value: bundleSize, + unit: 'B', + }) + const benchmark = { + key: 'bundle-size-check', + name: 'Bundle size check', + metrics, + } + console.log(JSON.stringify(benchmark, null, 2)) + } catch (error) { + console.error('Build failed:', error) + } finally { + if (fs.existsSync(tempFilePath)) { + fs.unlinkSync(tempFilePath) + } + } +} + +main() diff --git a/perf-measures/bundle-check/scripts/process-results.ts b/perf-measures/bundle-check/scripts/process-results.ts deleted file mode 100644 index 0dc3b216..00000000 --- a/perf-measures/bundle-check/scripts/process-results.ts +++ /dev/null @@ -1,14 +0,0 @@ -import * as fs from 'node:fs/promises' - -async function main() { - const currentResult = (await fs.readFile('./generated/after.js')).byteLength - let previousResult: number | null = null - try { - previousResult = (await fs.readFile('./generated/before.js')).byteLength - } catch (e) {} - const table = ['| | Current | Previous |', '| --- | --- | --- |'] - table.push(`| Bundle Size | ${currentResult} | ${previousResult || 'N/A'} |`) - console.log(table.join('\n')) -} - -main() diff --git a/perf-measures/type-check/.gitignore b/perf-measures/type-check/.gitignore index 79bd69e2..5fbaa717 100644 --- a/perf-measures/type-check/.gitignore +++ b/perf-measures/type-check/.gitignore @@ -2,3 +2,4 @@ generated !generated/.gitkeep trace *result.txt +diagnostics.json diff --git a/perf-measures/type-check/scripts/process-results.ts b/perf-measures/type-check/scripts/process-results.ts index 99f56d0e..8c8ef95e 100644 --- a/perf-measures/type-check/scripts/process-results.ts +++ b/perf-measures/type-check/scripts/process-results.ts @@ -1,21 +1,46 @@ -import * as fs from 'node:fs/promises' +import * as readline from 'node:readline' async function main() { - const currentResult = (await fs.readFile('./result.txt')).toString().split('\n') - const previousResult = await fs - .readFile('./previous-result.txt') - .then((data) => data.toString().split('\n')) - .catch(() => null) - const table = ['| | Current | Previous |', '| --- | --- | --- |'] - for (const [i, line] of currentResult.entries()) { - if (line === '') { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + terminal: false, + }) + const toKebabCase = (str: string): string => { + return str + .replace(/([a-z])([A-Z])/g, '$1-$2') + .replace(/[\s_\/]+/g, '-') + .toLowerCase() + } + const metrics = [] + for await (const line of rl) { + if (!line || line.trim() === '') { continue } const [name, value] = line.split(':') - const mainValue = previousResult?.[i]?.split(':')?.[1] - table.push(`| ${name?.trim()} | ${value?.trim()} | ${mainValue ? mainValue.trim() : 'N/A'} |`) + const unitMatch = value?.trim().match(/^(\d+(\.\d+)?)([a-zA-Z]*)$/) + if (unitMatch) { + const [, number, , unit] = unitMatch + metrics.push({ + key: toKebabCase(name?.trim()), + name: name?.trim(), + value: parseFloat(number), + unit: unit || undefined, + }) + } else { + metrics.push({ + key: toKebabCase(name?.trim()), + name: name?.trim(), + value: parseFloat(value?.trim()), + }) + } } - console.log(table.join('\n')) + const benchmark = { + key: 'diagnostics', + name: 'Compiler Diagnostics', + metrics, + } + console.log(JSON.stringify(benchmark, null, 2)) } main()