ci: Display performance measurement results as custom metrics (#3491)

* Debug result

* Fix

* Refactoring to convert standard input to json format for custom metrics output

* Custom metrics are now output as comments in Pull Reqeust

* Fix `undefined is not an object (evaluating 'line.split')`

* Add check to make sure line is not undefined

* Debug result

* Fix

* Refactoring to convert standard input to json format for custom metrics output

* Custom metrics are now output as comments in Pull Reqeust

* Fix `undefined is not an object (evaluating 'line.split')`

* Add check to make sure line is not undefined

* Move path to parent directory to consolidate multiple metrics and output results

* Refactoring to convert to octocov custom metrics format

* The two performance measurement jobs were merged and compared output by octocov.

* Append unit

* bonsai

* Review of title

* Integrate bundling and size check processes

* Fix code style

* bonsai

* Fix

* refs https://github.com/k2tzumi/hono/actions/runs/11903022493/job/33169186026#step:7:96, Fix `no such file or directory`

* Fix lint & code style

.github/workflows/ci.yml

@@ -181,88 +181,53 @@ jobs:
           name: coverage-lambda-edge
           path: coverage/
 
-  perf-measures-type-check-on-pr:
+  perf-measures-type-and-bundle-check-on-pr:
-    name: 'Type Check on PR'
+    name: 'Type & Bundle size Check on PR'
     runs-on: ubuntu-latest
     if: github.event_name == 'pull_request'
     steps:
       - uses: actions/checkout@v4
       - uses: oven-sh/setup-bun@v2
       - run: bun install
-      - uses: actions/cache/restore@v4
+      - name: Performance measurement of type check
+        run: |
+            bun scripts/generate-app.ts
+            bun tsc -p tsconfig.build.json --diagnostics | bun scripts/process-results.ts > diagnostics.json
+        working-directory: perf-measures/type-check
+      - name: Performance measurement of bundle check
+        run: |
+          bun run build
+          bun perf-measures/bundle-check/scripts/check-bundle-size.ts > perf-measures/bundle-check/size.json
+      - name: Run octocov
+        uses: k1LoW/octocov-action@v1
         with:
-          path: perf-measures/type-check/previous-result.txt
+          config: perf-measures/.octocov.perf-measures.yml
-          restore-keys: |
+        env:
-            type-check-perf-previous-result-
+          OCTOCOV_CUSTOM_METRICS_BENCHMARK_BUNDLE: perf-measures/bundle-check/size.json
-          key: type-check-perf-previous-result-
+          OCTOCOV_CUSTOM_METRICS_BENCHMARK_TYPE: perf-measures/type-check/diagnostics.json
-      - run: bun scripts/generate-app.ts
-        working-directory: perf-measures/type-check
-      - run: bun tsc -p tsconfig.build.json --diagnostics > result.txt
-        working-directory: perf-measures/type-check
-      - run: |
-          {
-            echo 'COMPARISON<<EOF'
-            bun scripts/process-results.ts | column -s '|' -t
-            echo 'EOF'
-          } >> "$GITHUB_ENV"
-        working-directory: perf-measures/type-check
-      - run: echo "$COMPARISON"
-        name: display comparison
 
-  perf-measures-type-check-on-main:
+  perf-measures-type-and-bundle-check-on-main:
-    name: 'Type Check on Main'
+    name: 'Type & Bundle size Check on Main'
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/main'
     steps:
       - uses: actions/checkout@v4
       - uses: oven-sh/setup-bun@v2
       - run: bun install
-      - run: bun scripts/generate-app.ts
+      - name: Performance measurement of type check
+        run: |
+            bun scripts/generate-app.ts
+            bun tsc -p tsconfig.build.json --diagnostics | bun scripts/process-results.ts > diagnostics.json
         working-directory: perf-measures/type-check
-      - run: bun tsc -p tsconfig.build.json --diagnostics > previous-result.txt
+      - name: Performance measurement of bundle check
-        working-directory: perf-measures/type-check
+        run: |
-      - uses: actions/cache/save@v4
+          bun run build
+          bun perf-measures/bundle-check/scripts/check-bundle-size.ts > perf-measures/bundle-check/size.json
+      - name: Run octocov (main)
+        uses: k1LoW/octocov-action@v1
         with:
-          path: perf-measures/type-check/previous-result.txt
+          config: perf-measures/.octocov.perf-measures.main.yml
-          key: type-check-perf-previous-result-${{ github.sha }}
+        env:
-
+          OCTOCOV_GITHUB_REF: refs/heads/main
-  perf-measures-bundle-check-on-pr:
+          OCTOCOV_CUSTOM_METRICS_BENCHMARK_BUNDLE: perf-measures/bundle-check/size.json
-    name: 'Bundle Check on PR'
+          OCTOCOV_CUSTOM_METRICS_BENCHMARK_TYPE: perf-measures/type-check/diagnostics.json
-    runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
-    steps:
-      - uses: actions/checkout@v4
-      - uses: oven-sh/setup-bun@v2
-      - run: bun install
-      - run: bun run build
-      - run: bunx esbuild --minify --bundle dist/index.js --format=esm --outfile=perf-measures/bundle-check/generated/after.js
-      - uses: actions/cache/restore@v4
-        with:
-          path: perf-measures/bundle-check/generated/before.js
-          restore-keys: |
-            perf-measures-bundle-check-previous-file-
-          key: perf-measures-bundle-check-previous-file-
-      - run: |
-          {
-            echo 'COMPARISON<<EOF'
-            bun scripts/process-results.ts | column -s '|' -t
-            echo 'EOF'
-          } >> "$GITHUB_ENV"
-        working-directory: perf-measures/bundle-check
-      - run: echo "$COMPARISON"
-        name: display comparison
-
-  perf-measures-bundle-check-on-main:
-    name: 'Bundle Check on Main'
-    runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/main'
-    steps:
-      - uses: actions/checkout@v4
-      - uses: oven-sh/setup-bun@v2
-      - run: bun install
-      - run: bun run build
-      - run: bunx esbuild --minify --bundle dist/index.js --format=esm --outfile=perf-measures/bundle-check/generated/before.js
-      - uses: actions/cache/save@v4
-        with:
-          path: perf-measures/bundle-check/generated/before.js
-          key: perf-measures-bundle-check-previous-file-${{ github.sha }}

perf-measures/.octocov.perf-measures.main.yml

@@ -0,0 +1,13 @@
+locale: "en"
+repository: ${GITHUB_REPOSITORY}/perf-measures
+coverage:
+  if: false
+codeToTestRatio:
+  if: false
+testExecutionTime:
+  if: false
+report:
+  datastores:
+    - artifact://${GITHUB_REPOSITORY}
+summary:
+  if: true

perf-measures/.octocov.perf-measures.yml

@@ -0,0 +1,15 @@
+locale: "en"
+repository: ${GITHUB_REPOSITORY}/perf-measures
+coverage:
+  if: false
+codeToTestRatio:
+  if: false
+testExecutionTime:
+  if: false
+diff:
+  datastores:
+    - artifact://${GITHUB_REPOSITORY}
+comment:
+  if: is_pull_request
+summary:
+  if: true

perf-measures/bundle-check/.gitignore

@@ -1,2 +1,3 @@
 generated
 !generated/.gitkeep
+size.json

perf-measures/bundle-check/scripts/check-bundle-size.ts

@@ -0,0 +1,43 @@
+import * as esbuild from 'esbuild'
+import * as fs from 'node:fs'
+import * as os from 'os'
+import * as path from 'path'
+
+async function main() {
+  const tempDir = os.tmpdir()
+  const tempFilePath = path.join(tempDir, 'bundle.tmp.js')
+
+  try {
+    await esbuild.build({
+      entryPoints: ['dist/index.js'],
+      bundle: true,
+      minify: true,
+      format: 'esm' as esbuild.Format,
+      target: 'es2022',
+      outfile: tempFilePath,
+    })
+
+    const bundleSize = fs.statSync(tempFilePath).size
+    const metrics = []
+    metrics.push({
+      key: 'bundle-size',
+      name: 'Bundle Size',
+      value: bundleSize,
+      unit: 'B',
+    })
+    const benchmark = {
+      key: 'bundle-size-check',
+      name: 'Bundle size check',
+      metrics,
+    }
+    console.log(JSON.stringify(benchmark, null, 2))
+  } catch (error) {
+    console.error('Build failed:', error)
+  } finally {
+    if (fs.existsSync(tempFilePath)) {
+      fs.unlinkSync(tempFilePath)
+    }
+  }
+}
+
+main()

perf-measures/bundle-check/scripts/process-results.ts

@@ -1,14 +0,0 @@
-import * as fs from 'node:fs/promises'
-
-async function main() {
-  const currentResult = (await fs.readFile('./generated/after.js')).byteLength
-  let previousResult: number | null = null
-  try {
-    previousResult = (await fs.readFile('./generated/before.js')).byteLength
-  } catch (e) {}
-  const table = ['| | Current | Previous |', '| --- | --- | --- |']
-  table.push(`| Bundle Size | ${currentResult} | ${previousResult || 'N/A'} |`)
-  console.log(table.join('\n'))
-}
-
-main()

perf-measures/type-check/.gitignore

@@ -2,3 +2,4 @@ generated
 !generated/.gitkeep
 trace
 *result.txt
+diagnostics.json

perf-measures/type-check/scripts/process-results.ts

@@ -1,21 +1,46 @@
-import * as fs from 'node:fs/promises'
+import * as readline from 'node:readline'
 
 async function main() {
-  const currentResult = (await fs.readFile('./result.txt')).toString().split('\n')
+  const rl = readline.createInterface({
-  const previousResult = await fs
+    input: process.stdin,
-    .readFile('./previous-result.txt')
+    output: process.stdout,
-    .then((data) => data.toString().split('\n'))
+    terminal: false,
-    .catch(() => null)
+  })
-  const table = ['| | Current | Previous |', '| --- | --- | --- |']
+  const toKebabCase = (str: string): string => {
-  for (const [i, line] of currentResult.entries()) {
+    return str
-    if (line === '') {
+      .replace(/([a-z])([A-Z])/g, '$1-$2')
+      .replace(/[\s_\/]+/g, '-')
+      .toLowerCase()
+  }
+  const metrics = []
+  for await (const line of rl) {
+    if (!line || line.trim() === '') {
       continue
     }
     const [name, value] = line.split(':')
-    const mainValue = previousResult?.[i]?.split(':')?.[1]
+    const unitMatch = value?.trim().match(/^(\d+(\.\d+)?)([a-zA-Z]*)$/)
-    table.push(`| ${name?.trim()} | ${value?.trim()} | ${mainValue ? mainValue.trim() : 'N/A'} |`)
+    if (unitMatch) {
+      const [, number, , unit] = unitMatch
+      metrics.push({
+        key: toKebabCase(name?.trim()),
+        name: name?.trim(),
+        value: parseFloat(number),
+        unit: unit || undefined,
+      })
+    } else {
+      metrics.push({
+        key: toKebabCase(name?.trim()),
+        name: name?.trim(),
+        value: parseFloat(value?.trim()),
+      })
+    }
   }
-  console.log(table.join('\n'))
+  const benchmark = {
+    key: 'diagnostics',
+    name: 'Compiler Diagnostics',
+    metrics,
+  }
+  console.log(JSON.stringify(benchmark, null, 2))
 }
 
 main()