ci: Display performance measurement results as custom metrics (#3491)

* Debug result * Fix * Refactoring to convert standard input to json format for custom metrics output * Custom metrics are now output as comments in Pull Reqeust * Fix `undefined is not an object (evaluating 'line.split')` * Add check to make sure line is not undefined * Debug result * Fix * Refactoring to convert standard input to json format for custom metrics output * Custom metrics are now output as comments in Pull Reqeust * Fix `undefined is not an object (evaluating 'line.split')` * Add check to make sure line is not undefined * Move path to parent directory to consolidate multiple metrics and output results * Refactoring to convert to octocov custom metrics format * The two performance measurement jobs were merged and compared output by octocov. * Append unit * bonsai * Review of title * Integrate bundling and size check processes * Fix code style * bonsai * Fix * refs https://github.com/k2tzumi/hono/actions/runs/11903022493/job/33169186026#step:7:96, Fix `no such file or directory` * Fix lint & code style
2024-11-24 02:07:30 +01:00 · 2024-11-19 13:38:21 +09:00 · 2024-11-19 13:38:21 +09:00 · 163657a411
commit 163657a411
parent f5bad77a63
8 changed files with 144 additions and 95 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -181,88 +181,53 @@ jobs:
          name: coverage-lambda-edge
          path: coverage/

-  perf-measures-type-check-on-pr:
-    name: 'Type Check on PR'
+  perf-measures-type-and-bundle-check-on-pr:
+    name: 'Type & Bundle size Check on PR'
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    steps:
      - uses: actions/checkout@v4
      - uses: oven-sh/setup-bun@v2
      - run: bun install
-      - uses: actions/cache/restore@v4
+      - name: Performance measurement of type check
+        run: |
+            bun scripts/generate-app.ts
+            bun tsc -p tsconfig.build.json --diagnostics | bun scripts/process-results.ts > diagnostics.json
+        working-directory: perf-measures/type-check
+      - name: Performance measurement of bundle check
+        run: |
+          bun run build
+          bun perf-measures/bundle-check/scripts/check-bundle-size.ts > perf-measures/bundle-check/size.json
+      - name: Run octocov
+        uses: k1LoW/octocov-action@v1
        with:
-          path: perf-measures/type-check/previous-result.txt
-          restore-keys: |
-            type-check-perf-previous-result-
-          key: type-check-perf-previous-result-
-      - run: bun scripts/generate-app.ts
-        working-directory: perf-measures/type-check
-      - run: bun tsc -p tsconfig.build.json --diagnostics > result.txt
-        working-directory: perf-measures/type-check
-      - run: |
-          {
-            echo 'COMPARISON<<EOF'
-            bun scripts/process-results.ts | column -s '|' -t
-            echo 'EOF'
-          } >> "$GITHUB_ENV"
-        working-directory: perf-measures/type-check
-      - run: echo "$COMPARISON"
-        name: display comparison
+          config: perf-measures/.octocov.perf-measures.yml
+        env:
+          OCTOCOV_CUSTOM_METRICS_BENCHMARK_BUNDLE: perf-measures/bundle-check/size.json
+          OCTOCOV_CUSTOM_METRICS_BENCHMARK_TYPE: perf-measures/type-check/diagnostics.json

-  perf-measures-type-check-on-main:
-    name: 'Type Check on Main'
+  perf-measures-type-and-bundle-check-on-main:
+    name: 'Type & Bundle size Check on Main'
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v4
      - uses: oven-sh/setup-bun@v2
      - run: bun install
-      - run: bun scripts/generate-app.ts
+      - name: Performance measurement of type check
+        run: |
+            bun scripts/generate-app.ts
+            bun tsc -p tsconfig.build.json --diagnostics | bun scripts/process-results.ts > diagnostics.json
        working-directory: perf-measures/type-check
-      - run: bun tsc -p tsconfig.build.json --diagnostics > previous-result.txt
-        working-directory: perf-measures/type-check
-      - uses: actions/cache/save@v4
+      - name: Performance measurement of bundle check
+        run: |
+          bun run build
+          bun perf-measures/bundle-check/scripts/check-bundle-size.ts > perf-measures/bundle-check/size.json
+      - name: Run octocov (main)
+        uses: k1LoW/octocov-action@v1
        with:
-          path: perf-measures/type-check/previous-result.txt
-          key: type-check-perf-previous-result-${{ github.sha }}
-
-  perf-measures-bundle-check-on-pr:
-    name: 'Bundle Check on PR'
-    runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
-    steps:
-      - uses: actions/checkout@v4
-      - uses: oven-sh/setup-bun@v2
-      - run: bun install
-      - run: bun run build
-      - run: bunx esbuild --minify --bundle dist/index.js --format=esm --outfile=perf-measures/bundle-check/generated/after.js
-      - uses: actions/cache/restore@v4
-        with:
-          path: perf-measures/bundle-check/generated/before.js
-          restore-keys: |
-            perf-measures-bundle-check-previous-file-
-          key: perf-measures-bundle-check-previous-file-
-      - run: |
-          {
-            echo 'COMPARISON<<EOF'
-            bun scripts/process-results.ts | column -s '|' -t
-            echo 'EOF'
-          } >> "$GITHUB_ENV"
-        working-directory: perf-measures/bundle-check
-      - run: echo "$COMPARISON"
-        name: display comparison
-
-  perf-measures-bundle-check-on-main:
-    name: 'Bundle Check on Main'
-    runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/main'
-    steps:
-      - uses: actions/checkout@v4
-      - uses: oven-sh/setup-bun@v2
-      - run: bun install
-      - run: bun run build
-      - run: bunx esbuild --minify --bundle dist/index.js --format=esm --outfile=perf-measures/bundle-check/generated/before.js
-      - uses: actions/cache/save@v4
-        with:
-          path: perf-measures/bundle-check/generated/before.js
-          key: perf-measures-bundle-check-previous-file-${{ github.sha }}
+          config: perf-measures/.octocov.perf-measures.main.yml
+        env:
+          OCTOCOV_GITHUB_REF: refs/heads/main
+          OCTOCOV_CUSTOM_METRICS_BENCHMARK_BUNDLE: perf-measures/bundle-check/size.json
+          OCTOCOV_CUSTOM_METRICS_BENCHMARK_TYPE: perf-measures/type-check/diagnostics.json
--- a/perf-measures/.octocov.perf-measures.main.yml
+++ b/perf-measures/.octocov.perf-measures.main.yml
@ -0,0 +1,13 @@
+locale: "en"
+repository: ${GITHUB_REPOSITORY}/perf-measures
+coverage:
+  if: false
+codeToTestRatio:
+  if: false
+testExecutionTime:
+  if: false
+report:
+  datastores:
+    - artifact://${GITHUB_REPOSITORY}
+summary:
+  if: true
--- a/perf-measures/.octocov.perf-measures.yml
+++ b/perf-measures/.octocov.perf-measures.yml
@ -0,0 +1,15 @@
+locale: "en"
+repository: ${GITHUB_REPOSITORY}/perf-measures
+coverage:
+  if: false
+codeToTestRatio:
+  if: false
+testExecutionTime:
+  if: false
+diff:
+  datastores:
+    - artifact://${GITHUB_REPOSITORY}
+comment:
+  if: is_pull_request
+summary:
+  if: true
--- a/perf-measures/bundle-check/.gitignore
+++ b/perf-measures/bundle-check/.gitignore
@ -1,2 +1,3 @@
 generated
 !generated/.gitkeep
+size.json
--- a/perf-measures/bundle-check/scripts/check-bundle-size.ts
+++ b/perf-measures/bundle-check/scripts/check-bundle-size.ts
@ -0,0 +1,43 @@
+import * as esbuild from 'esbuild'
+import * as fs from 'node:fs'
+import * as os from 'os'
+import * as path from 'path'
+
+async function main() {
+  const tempDir = os.tmpdir()
+  const tempFilePath = path.join(tempDir, 'bundle.tmp.js')
+
+  try {
+    await esbuild.build({
+      entryPoints: ['dist/index.js'],
+      bundle: true,
+      minify: true,
+      format: 'esm' as esbuild.Format,
+      target: 'es2022',
+      outfile: tempFilePath,
+    })
+
+    const bundleSize = fs.statSync(tempFilePath).size
+    const metrics = []
+    metrics.push({
+      key: 'bundle-size',
+      name: 'Bundle Size',
+      value: bundleSize,
+      unit: 'B',
+    })
+    const benchmark = {
+      key: 'bundle-size-check',
+      name: 'Bundle size check',
+      metrics,
+    }
+    console.log(JSON.stringify(benchmark, null, 2))
+  } catch (error) {
+    console.error('Build failed:', error)
+  } finally {
+    if (fs.existsSync(tempFilePath)) {
+      fs.unlinkSync(tempFilePath)
+    }
+  }
+}
+
+main()
--- a/perf-measures/bundle-check/scripts/process-results.ts
+++ b/perf-measures/bundle-check/scripts/process-results.ts
@ -1,14 +0,0 @@
-import * as fs from 'node:fs/promises'
-
-async function main() {
-  const currentResult = (await fs.readFile('./generated/after.js')).byteLength
-  let previousResult: number | null = null
-  try {
-    previousResult = (await fs.readFile('./generated/before.js')).byteLength
-  } catch (e) {}
-  const table = ['| | Current | Previous |', '| --- | --- | --- |']
-  table.push(`| Bundle Size | ${currentResult} | ${previousResult || 'N/A'} |`)
-  console.log(table.join('\n'))
-}
-
-main()
--- a/perf-measures/type-check/.gitignore
+++ b/perf-measures/type-check/.gitignore
@ -2,3 +2,4 @@ generated
 !generated/.gitkeep
 trace
 *result.txt
+diagnostics.json
--- a/perf-measures/type-check/scripts/process-results.ts
+++ b/perf-measures/type-check/scripts/process-results.ts
@ -1,21 +1,46 @@
-import * as fs from 'node:fs/promises'
+import * as readline from 'node:readline'

 async function main() {
-  const currentResult = (await fs.readFile('./result.txt')).toString().split('\n')
-  const previousResult = await fs
-    .readFile('./previous-result.txt')
-    .then((data) => data.toString().split('\n'))
-    .catch(() => null)
-  const table = ['| | Current | Previous |', '| --- | --- | --- |']
-  for (const [i, line] of currentResult.entries()) {
-    if (line === '') {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+    terminal: false,
+  })
+  const toKebabCase = (str: string): string => {
+    return str
+      .replace(/([a-z])([A-Z])/g, '$1-$2')
+      .replace(/[\s_\/]+/g, '-')
+      .toLowerCase()
+  }
+  const metrics = []
+  for await (const line of rl) {
+    if (!line || line.trim() === '') {
      continue
    }
    const [name, value] = line.split(':')
-    const mainValue = previousResult?.[i]?.split(':')?.[1]
-    table.push(`| ${name?.trim()} | ${value?.trim()} | ${mainValue ? mainValue.trim() : 'N/A'} |`)
+    const unitMatch = value?.trim().match(/^(\d+(\.\d+)?)([a-zA-Z]*)$/)
+    if (unitMatch) {
+      const [, number, , unit] = unitMatch
+      metrics.push({
+        key: toKebabCase(name?.trim()),
+        name: name?.trim(),
+        value: parseFloat(number),
+        unit: unit || undefined,
+      })
+    } else {
+      metrics.push({
+        key: toKebabCase(name?.trim()),
+        name: name?.trim(),
+        value: parseFloat(value?.trim()),
+      })
    }
-  console.log(table.join('\n'))
+  }
+  const benchmark = {
+    key: 'diagnostics',
+    name: 'Compiler Diagnostics',
+    metrics,
+  }
+  console.log(JSON.stringify(benchmark, null, 2))
 }

 main()