ci: Report coverage with CodeCov (#2862)

* Output coverage per each test

* Collect coverage

* Exclude test-only files

* update comment

* exclude types

* Add config

* Update bun comment

* Generate coverage in Deno

.github/workflows/ci.yml

@@ -12,6 +12,31 @@ on:
       - 'LICENSE'
 
 jobs:
+  coverage:
+    name: 'Coverage'
+    runs-on: ubuntu-latest
+    needs:
+      # bun excluded because it uses built-in test tool which does not support coverage in v8/istanbul format.
+      # refer https://github.com/oven-sh/bun/issues/4015
+      - main
+      - fastly
+      - node
+      - wrangler
+      - lambda
+      - lambda-edge
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-*
+          merge-multiple: true
+          path: ./coverage
+      - uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: true
+          directory: ./coverage
+          token: ${{ secrets.CODECOV_TOKEN }}
+
   main:
     name: 'Main'
     runs-on: ubuntu-latest
@@ -28,6 +53,10 @@ jobs:
       - run: bun run lint
       - run: bun run build
       - run: bun run test
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage-main
+          path: coverage/
 
   jsr-dry-run:
     name: "Checking if it's valid for JSR"
@@ -47,9 +76,13 @@ jobs:
       - uses: denoland/setup-deno@v1
         with:
           deno-version: v1.x
-      - run: env NAME=Deno deno test --allow-read --allow-env --allow-write -c runtime_tests/deno/deno.json runtime_tests/deno
+      - run: env NAME=Deno deno test --coverage=coverage/raw/deno-runtime --allow-read --allow-env --allow-write -c runtime_tests/deno/deno.json runtime_tests/deno
-      - run: deno test -c runtime_tests/deno-jsx/deno.precompile.json runtime_tests/deno-jsx
+      - run: deno test -c runtime_tests/deno-jsx/deno.precompile.json --coverage=coverage/raw/deno-precompile-jsx runtime_tests/deno-jsx
-      - run: deno test -c runtime_tests/deno-jsx/deno.react-jsx.json runtime_tests/deno-jsx
+      - run: deno test -c runtime_tests/deno-jsx/deno.react-jsx.json --coverage=coverage/raw/deno-react-jsx runtime_tests/deno-jsx
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage-deno
+          path: coverage/
 
   bun:
     name: 'Bun'
@@ -70,6 +103,10 @@ jobs:
       - run: bun install
       - run: bun run build
       - run: bun run test:fastly
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage-fastly
+          path: coverage/
 
   node:
     name: 'Node.js v${{ matrix.node }}'
@@ -86,6 +123,11 @@ jobs:
       - run: bun install
       - run: bun run build
       - run: bun run test:node
+      - uses: actions/upload-artifact@v4
+        if: matrix.node == '22.x'
+        with:
+          name: coverage-node
+          path: coverage/
 
   wrangler:
     name: 'Cloudflare Workers'
@@ -96,6 +138,10 @@ jobs:
       - run: bun install
       - run: bun run build
       - run: bun run test:wrangler
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage-wrangler
+          path: coverage/
 
   lambda:
     name: 'AWS Lambda'
@@ -106,6 +152,10 @@ jobs:
       - run: bun install
       - run: bun run build
       - run: bun run test:lambda
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage-lambda
+          path: coverage/
 
   lambda-edge:
     name: 'Lambda@Edge'
@@ -116,3 +166,7 @@ jobs:
       - run: bun install --frozen-lockfile
       - run: bun run build
       - run: bun run test:lambda-edge
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage-lambda-edge
+          path: coverage/

.vitest.config/jsx-runtime-default.ts

@@ -8,5 +8,8 @@ if (config.test) {
     '**/src/jsx/dom/**/(*.)+(spec|test).+(ts|tsx|js)',
     'src/jsx/hooks/dom.test.tsx',
   ]
+  if (config.test.coverage) {
+    config.test.coverage.reportsDirectory = './coverage/raw/jsx-runtime'
+  }
 }
 export default config

.vitest.config/jsx-runtime-dom.ts

@@ -8,5 +8,8 @@ if (config.test) {
     '**/src/jsx/dom/**/(*.)+(spec|test).+(ts|tsx|js)',
     'src/jsx/hooks/dom.test.tsx',
   ]
+  if (config.test.coverage) {
+    config.test.coverage.reportsDirectory = './coverage/raw/jsx-dom'
+  }
 }
 export default config

codecov.yml

@@ -0,0 +1,13 @@
+# Edit "test.coverage.exclude" option in vitest.config.ts to exclude specific files from coverage reports.
+# We can also use "ignore" option in codecov.yml, but it is not recognized by vitest, so results may differ on local.
+
+coverage:
+  status:
+    patch:
+      default:
+        target: 80%
+        informational: true # Don't fail the build even if coverage is below target
+    project:
+      default:
+        target: 75%
+        threshold: 1%

runtime_tests/bun/vitest.config.ts

@@ -1,9 +1,14 @@
 /// <reference types="vitest" />
 import { defineConfig } from 'vitest/config'
+import config from '../../vitest.config'
 
 export default defineConfig({
   test: {
     globals: true,
     include: ['**/runtime_tests/bun/**/*.+(ts|tsx|js)'],
+    coverage: {
+      ...config.test?.coverage,
+      reportsDirectory: './coverage/raw/runtime-bun',
+    },
   },
 })

runtime_tests/fastly/vitest.config.ts

@@ -1,6 +1,7 @@
 /// <reference types="vitest" />
 import fastlyCompute from 'vite-plugin-fastly-js-compute'
 import { defineConfig } from 'vitest/config'
+import config from '../../vitest.config'
 
 export default defineConfig({
   plugins: [fastlyCompute()],
@@ -8,5 +9,9 @@ export default defineConfig({
     globals: true,
     include: ['**/runtime_tests/fastly/**/(*.)+(test).+(ts|tsx)'],
     exclude: ['**/runtime_tests/fastly/vitest.config.ts'],
+    coverage: {
+      ...config.test?.coverage,
+      reportsDirectory: './coverage/raw/runtime-fastly',
+    },
   },
 })

runtime_tests/lambda-edge/vitest.config.ts

@@ -1,5 +1,6 @@
 /// <reference types="vitest" />
 import { defineConfig } from 'vitest/config'
+import config from '../../vitest.config'
 
 export default defineConfig({
   test: {
@@ -9,5 +10,9 @@ export default defineConfig({
     globals: true,
     include: ['**/runtime_tests/lambda-edge/**/*.+(ts|tsx|js)'],
     exclude: ['**/runtime_tests/lambda-edge/vitest.config.ts'],
+    coverage: {
+      ...config.test?.coverage,
+      reportsDirectory: './coverage/raw/runtime-lambda-edge',
+    },
   },
 })

runtime_tests/lambda/vitest.config.ts

@@ -1,5 +1,6 @@
 /// <reference types="vitest" />
 import { defineConfig } from 'vitest/config'
+import config from '../../vitest.config'
 
 export default defineConfig({
   test: {
@@ -9,5 +10,9 @@ export default defineConfig({
     globals: true,
     include: ['**/runtime_tests/lambda/**/*.+(ts|tsx|js)'],
     exclude: ['**/runtime_tests/lambda/vitest.config.ts', '**/runtime_tests/lambda/mock.ts'],
+    coverage: {
+      ...config.test?.coverage,
+      reportsDirectory: './coverage/raw/runtime-lambda',
+    },
   },
 })

runtime_tests/node/vitest.config.ts

@@ -1,5 +1,6 @@
 /// <reference types="vitest" />
 import { defineConfig } from 'vitest/config'
+import config from '../../vitest.config'
 
 export default defineConfig({
   test: {
@@ -9,5 +10,9 @@ export default defineConfig({
     globals: true,
     include: ['**/runtime_tests/node/**/*.+(ts|tsx|js)'],
     exclude: ['**/runtime_tests/node/vitest.config.ts'],
+    coverage: {
+      ...config.test?.coverage,
+      reportsDirectory: './coverage/raw/runtime-node',
+    },
   },
 })

runtime_tests/wrangler/vitest.config.ts

@@ -1,10 +1,15 @@
 /// <reference types="vitest" />
 import { defineConfig } from 'vitest/config'
+import config from '../../vitest.config'
 
 export default defineConfig({
   test: {
     globals: true,
     include: ['**/runtime_tests/wrangler/**/(*.)+(test).+(ts|tsx)'],
     exclude: ['**/runtime_tests/wrangler/vitest.config.ts'],
+    coverage: {
+      ...config.test?.coverage,
+      reportsDirectory: './coverage/raw/runtime-wrangler',
+    },
   },
 })

vitest.config.ts

@@ -12,8 +12,18 @@ export default defineConfig({
     exclude: [...configDefaults.exclude, '**/sandbox/**', '**/*.case.test.+(ts|tsx|js)'],
     setupFiles: ['./src/test-utils/setup-vitest.ts'],
     coverage: {
+      enabled: true,
       provider: 'v8',
-      reporter: ['text'],
+      reportsDirectory: './coverage/raw/default',
+      reporter: ['json'],
+      exclude: [
+        ...(configDefaults.coverage.exclude ?? []),
+        'benchmarks',
+        'runtime_tests',
+        'build.ts',
+        'src/test-utils',
+        'src/**/types.ts', // types are compile-time only, so their coverage cannot be measured
+      ]
     },
     pool: 'forks',
   },