From 8cc8a05c8982d50c3528d57b325500645a9773eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?TATSUNO=20=E2=80=9CTaz=E2=80=9D=20Yasuhiro?= Date: Sat, 1 Jun 2024 04:17:47 +0900 Subject: [PATCH] ci: Report coverage with CodeCov (#2862) * Output coverage per each test * Collect coverage * Exclude test-only files * update comment * exclude types * Add config * Update bun comment * Generate coverage in Deno --- .github/workflows/ci.yml | 60 ++++++++++++++++++++-- .vitest.config/jsx-runtime-default.ts | 3 ++ .vitest.config/jsx-runtime-dom.ts | 3 ++ codecov.yml | 13 +++++ runtime_tests/bun/vitest.config.ts | 5 ++ runtime_tests/fastly/vitest.config.ts | 5 ++ runtime_tests/lambda-edge/vitest.config.ts | 5 ++ runtime_tests/lambda/vitest.config.ts | 5 ++ runtime_tests/node/vitest.config.ts | 5 ++ runtime_tests/wrangler/vitest.config.ts | 5 ++ vitest.config.ts | 12 ++++- 11 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 codecov.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7488baba..ff3f3e09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,31 @@ on: - 'LICENSE' jobs: + coverage: + name: 'Coverage' + runs-on: ubuntu-latest + needs: + # bun excluded because it uses built-in test tool which does not support coverage in v8/istanbul format. + # refer https://github.com/oven-sh/bun/issues/4015 + - main + - fastly + - node + - wrangler + - lambda + - lambda-edge + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: coverage-* + merge-multiple: true + path: ./coverage + - uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: true + directory: ./coverage + token: ${{ secrets.CODECOV_TOKEN }} + main: name: 'Main' runs-on: ubuntu-latest @@ -28,6 +53,10 @@ jobs: - run: bun run lint - run: bun run build - run: bun run test + - uses: actions/upload-artifact@v4 + with: + name: coverage-main + path: coverage/ jsr-dry-run: name: "Checking if it's valid for JSR" @@ -47,9 +76,13 @@ jobs: - uses: denoland/setup-deno@v1 with: deno-version: v1.x - - run: env NAME=Deno deno test --allow-read --allow-env --allow-write -c runtime_tests/deno/deno.json runtime_tests/deno - - run: deno test -c runtime_tests/deno-jsx/deno.precompile.json runtime_tests/deno-jsx - - run: deno test -c runtime_tests/deno-jsx/deno.react-jsx.json runtime_tests/deno-jsx + - run: env NAME=Deno deno test --coverage=coverage/raw/deno-runtime --allow-read --allow-env --allow-write -c runtime_tests/deno/deno.json runtime_tests/deno + - run: deno test -c runtime_tests/deno-jsx/deno.precompile.json --coverage=coverage/raw/deno-precompile-jsx runtime_tests/deno-jsx + - run: deno test -c runtime_tests/deno-jsx/deno.react-jsx.json --coverage=coverage/raw/deno-react-jsx runtime_tests/deno-jsx + - uses: actions/upload-artifact@v4 + with: + name: coverage-deno + path: coverage/ bun: name: 'Bun' @@ -70,6 +103,10 @@ jobs: - run: bun install - run: bun run build - run: bun run test:fastly + - uses: actions/upload-artifact@v4 + with: + name: coverage-fastly + path: coverage/ node: name: 'Node.js v${{ matrix.node }}' @@ -86,6 +123,11 @@ jobs: - run: bun install - run: bun run build - run: bun run test:node + - uses: actions/upload-artifact@v4 + if: matrix.node == '22.x' + with: + name: coverage-node + path: coverage/ wrangler: name: 'Cloudflare Workers' @@ -96,6 +138,10 @@ jobs: - run: bun install - run: bun run build - run: bun run test:wrangler + - uses: actions/upload-artifact@v4 + with: + name: coverage-wrangler + path: coverage/ lambda: name: 'AWS Lambda' @@ -106,6 +152,10 @@ jobs: - run: bun install - run: bun run build - run: bun run test:lambda + - uses: actions/upload-artifact@v4 + with: + name: coverage-lambda + path: coverage/ lambda-edge: name: 'Lambda@Edge' @@ -116,3 +166,7 @@ jobs: - run: bun install --frozen-lockfile - run: bun run build - run: bun run test:lambda-edge + - uses: actions/upload-artifact@v4 + with: + name: coverage-lambda-edge + path: coverage/ diff --git a/.vitest.config/jsx-runtime-default.ts b/.vitest.config/jsx-runtime-default.ts index 9885a19b..a325bc04 100644 --- a/.vitest.config/jsx-runtime-default.ts +++ b/.vitest.config/jsx-runtime-default.ts @@ -8,5 +8,8 @@ if (config.test) { '**/src/jsx/dom/**/(*.)+(spec|test).+(ts|tsx|js)', 'src/jsx/hooks/dom.test.tsx', ] + if (config.test.coverage) { + config.test.coverage.reportsDirectory = './coverage/raw/jsx-runtime' + } } export default config diff --git a/.vitest.config/jsx-runtime-dom.ts b/.vitest.config/jsx-runtime-dom.ts index 11adf6cc..c9a4a3a7 100644 --- a/.vitest.config/jsx-runtime-dom.ts +++ b/.vitest.config/jsx-runtime-dom.ts @@ -8,5 +8,8 @@ if (config.test) { '**/src/jsx/dom/**/(*.)+(spec|test).+(ts|tsx|js)', 'src/jsx/hooks/dom.test.tsx', ] + if (config.test.coverage) { + config.test.coverage.reportsDirectory = './coverage/raw/jsx-dom' + } } export default config diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..b4889fe9 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,13 @@ +# Edit "test.coverage.exclude" option in vitest.config.ts to exclude specific files from coverage reports. +# We can also use "ignore" option in codecov.yml, but it is not recognized by vitest, so results may differ on local. + +coverage: + status: + patch: + default: + target: 80% + informational: true # Don't fail the build even if coverage is below target + project: + default: + target: 75% + threshold: 1% diff --git a/runtime_tests/bun/vitest.config.ts b/runtime_tests/bun/vitest.config.ts index 5d244e3f..6b1cae41 100644 --- a/runtime_tests/bun/vitest.config.ts +++ b/runtime_tests/bun/vitest.config.ts @@ -1,9 +1,14 @@ /// import { defineConfig } from 'vitest/config' +import config from '../../vitest.config' export default defineConfig({ test: { globals: true, include: ['**/runtime_tests/bun/**/*.+(ts|tsx|js)'], + coverage: { + ...config.test?.coverage, + reportsDirectory: './coverage/raw/runtime-bun', + }, }, }) diff --git a/runtime_tests/fastly/vitest.config.ts b/runtime_tests/fastly/vitest.config.ts index 89baa357..c0480b13 100644 --- a/runtime_tests/fastly/vitest.config.ts +++ b/runtime_tests/fastly/vitest.config.ts @@ -1,6 +1,7 @@ /// import fastlyCompute from 'vite-plugin-fastly-js-compute' import { defineConfig } from 'vitest/config' +import config from '../../vitest.config' export default defineConfig({ plugins: [fastlyCompute()], @@ -8,5 +9,9 @@ export default defineConfig({ globals: true, include: ['**/runtime_tests/fastly/**/(*.)+(test).+(ts|tsx)'], exclude: ['**/runtime_tests/fastly/vitest.config.ts'], + coverage: { + ...config.test?.coverage, + reportsDirectory: './coverage/raw/runtime-fastly', + }, }, }) diff --git a/runtime_tests/lambda-edge/vitest.config.ts b/runtime_tests/lambda-edge/vitest.config.ts index 568ecc05..105ac950 100644 --- a/runtime_tests/lambda-edge/vitest.config.ts +++ b/runtime_tests/lambda-edge/vitest.config.ts @@ -1,5 +1,6 @@ /// import { defineConfig } from 'vitest/config' +import config from '../../vitest.config' export default defineConfig({ test: { @@ -9,5 +10,9 @@ export default defineConfig({ globals: true, include: ['**/runtime_tests/lambda-edge/**/*.+(ts|tsx|js)'], exclude: ['**/runtime_tests/lambda-edge/vitest.config.ts'], + coverage: { + ...config.test?.coverage, + reportsDirectory: './coverage/raw/runtime-lambda-edge', + }, }, }) diff --git a/runtime_tests/lambda/vitest.config.ts b/runtime_tests/lambda/vitest.config.ts index 0fd446dc..f030cf41 100644 --- a/runtime_tests/lambda/vitest.config.ts +++ b/runtime_tests/lambda/vitest.config.ts @@ -1,5 +1,6 @@ /// import { defineConfig } from 'vitest/config' +import config from '../../vitest.config' export default defineConfig({ test: { @@ -9,5 +10,9 @@ export default defineConfig({ globals: true, include: ['**/runtime_tests/lambda/**/*.+(ts|tsx|js)'], exclude: ['**/runtime_tests/lambda/vitest.config.ts', '**/runtime_tests/lambda/mock.ts'], + coverage: { + ...config.test?.coverage, + reportsDirectory: './coverage/raw/runtime-lambda', + }, }, }) diff --git a/runtime_tests/node/vitest.config.ts b/runtime_tests/node/vitest.config.ts index a2c74bbb..5387c11f 100644 --- a/runtime_tests/node/vitest.config.ts +++ b/runtime_tests/node/vitest.config.ts @@ -1,5 +1,6 @@ /// import { defineConfig } from 'vitest/config' +import config from '../../vitest.config' export default defineConfig({ test: { @@ -9,5 +10,9 @@ export default defineConfig({ globals: true, include: ['**/runtime_tests/node/**/*.+(ts|tsx|js)'], exclude: ['**/runtime_tests/node/vitest.config.ts'], + coverage: { + ...config.test?.coverage, + reportsDirectory: './coverage/raw/runtime-node', + }, }, }) diff --git a/runtime_tests/wrangler/vitest.config.ts b/runtime_tests/wrangler/vitest.config.ts index 4b8af9b1..7eb5c620 100644 --- a/runtime_tests/wrangler/vitest.config.ts +++ b/runtime_tests/wrangler/vitest.config.ts @@ -1,10 +1,15 @@ /// import { defineConfig } from 'vitest/config' +import config from '../../vitest.config' export default defineConfig({ test: { globals: true, include: ['**/runtime_tests/wrangler/**/(*.)+(test).+(ts|tsx)'], exclude: ['**/runtime_tests/wrangler/vitest.config.ts'], + coverage: { + ...config.test?.coverage, + reportsDirectory: './coverage/raw/runtime-wrangler', + }, }, }) diff --git a/vitest.config.ts b/vitest.config.ts index 90470c8b..575f1e11 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -12,8 +12,18 @@ export default defineConfig({ exclude: [...configDefaults.exclude, '**/sandbox/**', '**/*.case.test.+(ts|tsx|js)'], setupFiles: ['./src/test-utils/setup-vitest.ts'], coverage: { + enabled: true, provider: 'v8', - reporter: ['text'], + reportsDirectory: './coverage/raw/default', + reporter: ['json'], + exclude: [ + ...(configDefaults.coverage.exclude ?? []), + 'benchmarks', + 'runtime_tests', + 'build.ts', + 'src/test-utils', + 'src/**/types.ts', // types are compile-time only, so their coverage cannot be measured + ] }, pool: 'forks', },