GeezerVison is showing github.com — feat: introduce `--since-last-run` flag (#11516) · vitest-dev/vitest@08bd93b
File treeExpand file treeCollapse file tree Open diff view settings Expand file treeCollapse file tree Open diff view settings Collapse file Copy file name to clipboardExpand all lines: docs/.vitepress/config.ts +4Lines changed: 4 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -1096,6 +1096,10 @@ export default ({ mode }: { mode: string }) => { | 1096 | 1096 | text: 'Parallel and Sequential Test Files', | 1097 | 1097 | link: '/guide/recipes/parallel-sequential', | 1098 | 1098 | }, | | 1099 | + { | | 1100 | + text: 'Running Only Affected Tests', | | 1101 | + link: '/guide/recipes/affected-tests', | | 1102 | + }, | 1099 | 1103 | ], | 1100 | 1104 | }, | 1101 | 1105 | { | |
Collapse file Copy file name to clipboardExpand all lines: docs/.vitepress/scripts/cli-generator.ts +1-1Lines changed: 1 addition & 1 deletion | Original file line number | Diff line number | Diff line change |
|---|
@@ -9,7 +9,7 @@ const cliTablePath = resolve(docsDir, './guide/cli-generated.md') | 9 | 9 |
| 10 | 10 | const nonNullable = <T>(value: T): value is NonNullable<T> => value !== null && value !== undefined | 11 | 11 |
| 12 | | -const skipCli = new Set(['mergeReports', 'shard']) | | 12 | +const skipCli = new Set(['mergeReports', 'shard', 'sinceLastRun']) | 13 | 13 |
| 14 | 14 | const skipConfig = new Set([ | 15 | 15 | 'config', | |
Collapse file Copy file name to clipboardExpand all lines: docs/config/experimental.md +48-6Lines changed: 48 additions & 6 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
@@ -316,7 +316,7 @@ If you are running tests in Deno, TypeScript files are processed by the runtime | 316 | 316 |
| 317 | 317 | ## experimental.vcsProvider <Version type="experimental">4.1.1</Version> {#experimental-vcsprovider} | 318 | 318 |
| 319 | | -- **Type:** `VCSProvider | string` | | 319 | +- **Type:** `'git' | 'mtime' | VCSProvider | string` | 320 | 320 |
| 321 | 321 | ```ts | 322 | 322 | interface VCSProvider { | @@ -326,14 +326,55 @@ interface VCSProvider { | 326 | 326 | interface VCSProviderOptions { | 327 | 327 | root: string | 328 | 328 | changedSince?: string | boolean | | 329 | + /** Resolves the dependencies of the test files of the run. */ | | 330 | + resolver: ModulesResolver | | 331 | +} | | 332 | + | | 333 | +interface ModulesResolver { | | 334 | + /** | | 335 | + * Every file whose change can affect the test files of the run. | | 336 | + */ | | 337 | + getDependencies(): Promise<ModuleDependency[]> | | 338 | +} | | 339 | + | | 340 | +interface ModuleDependency { | | 341 | + /** Absolute path of the file. */ | | 342 | + file: string | | 343 | + /** | | 344 | + * Start time of the oldest run that recorded the file, in milliseconds. | | 345 | + * Not set when the dependency comes from the static module graph. | | 346 | + */ | | 347 | + recordedAt?: number | 329 | 348 | } | 330 | 349 | ``` | 331 | 350 |
| 332 | 351 | - **Default:** `'git'` | 333 | 352 |
| 334 | | -Custom provider for detecting changed files. Used with the [`--changed`](/guide/cli#changed) flag to determine which files have been modified. | | 353 | +Provider for detecting changed files. Used with the [`--changed`](/guide/cli#changed) flag to determine which files have been modified. | | 354 | + | | 355 | +Vitest has two built-in providers: | | 356 | + | | 357 | +- `'git'` returns the files that differ from the `--changed` reference, plus staged and untracked files. | | 358 | +- `'mtime'` <Version type="experimental">5.1.0</Version> returns the recorded dependencies whose modification time is newer than the run that recorded them, and deleted dependencies. It needs [`experimental.recordDependencies`](#experimental-recorddependencies): a dependency without a record is always reported as changed, so every test runs until the records exist. It does not need Git and ignores the `--changed` reference. A `--changed` run updates the records of the tests it ran, so the next `--changed` run selects only the tests whose dependencies changed since, and the test files that failed in their last run. | | 359 | + | | 360 | +```ts [vitest.config.ts] | | 361 | +import { defineConfig } from 'vitest/config' | | 362 | + | | 363 | +export default defineConfig({ | | 364 | + test: { | | 365 | + experimental: { | | 366 | + recordDependencies: true, | | 367 | + vcsProvider: 'mtime', | | 368 | + }, | | 369 | + }, | | 370 | +}) | | 371 | +``` | | 372 | + | | 373 | +::: warning | | 374 | +The `mtime` provider compares file times with the clock of the machine that ran the tests. A checkout or a copy that sets file times in the past is not detected. | | 375 | +::: | 335 | 376 |
| 336 | | -By default, Vitest uses Git to detect changed files. You can provide a custom implementation of the `VCSProvider` interface to use a different version control system: | | 377 | +You can provide a custom implementation of the `VCSProvider` interface to use a different version control system. `options.resolver.getDependencies()` returns every file the test files of the run depend on, so a provider can limit its work to these files: | 337 | 378 |
| 338 | 379 | ```ts [vitest.config.ts] | 339 | 380 | import { defineConfig } from 'vitest/config' | @@ -342,8 +383,9 @@ export default defineConfig({ | 342 | 383 | test: { | 343 | 384 | experimental: { | 344 | 385 | vcsProvider: { | 345 | | - async findChangedFiles({ root, changedSince }) { | 346 | | - // return paths of changed files | | 386 | + async findChangedFiles({ root, changedSince, resolver }) { | | 387 | + const dependencies = await resolver.getDependencies() | | 388 | + // return the paths of the changed files | 347 | 389 | return [] | 348 | 390 | }, | 349 | 391 | }, | @@ -375,7 +417,7 @@ export default { | 375 | 417 | } | 376 | 418 | ``` | 377 | 419 |
| 378 | | -## experimental.recordDependencies <Version type="experimental">5.0.4</Version> {#experimental-recorddependencies} | | 420 | +## experimental.recordDependencies <Version type="experimental">5.1.0</Version> {#experimental-recorddependencies} | 379 | 421 |
| 380 | 422 | - **Type:** `boolean` | 381 | 423 | - **Default:** `false` | |
Collapse file Copy file name to clipboardExpand all lines: docs/guide/cli-generated.md +1-1Lines changed: 1 addition & 1 deletion - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
@@ -1018,7 +1018,7 @@ Controls whether Vitest will use Node.js Loader API to process in-source or mock | 1018 | 1018 | - **CLI:** `--experimental.vcsProvider <path>` | 1019 | 1019 | - **Config:** [experimental.vcsProvider](/config/experimental#experimental-vcsprovider) | 1020 | 1020 |
| 1021 | | -Custom provider for detecting changed files. (default: `git`) | | 1021 | +Provider for detecting changed files: `git`, `mtime` (requires `experimental.recordDependencies`) or a path to a custom provider. (default: `git`) | 1022 | 1022 |
| 1023 | 1023 | ### experimental.recordDependencies | 1024 | 1024 |
| |
Collapse file Copy file name to clipboardExpand all lines: docs/guide/cli.md +21Lines changed: 21 additions & 0 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
@@ -272,6 +272,27 @@ Merges every blob report located in the specified folder (`.vitest/blob/` by def | 272 | 272 | vitest --merge-reports --reporter=junit | 273 | 273 | ``` | 274 | 274 |
| | 275 | +### since-last-run <Version type="experimental">5.1.0</Version> {#since-last-run} | | 276 | + | | 277 | +- **Type:** `boolean` | | 278 | +- **Default:** `false` | | 279 | + | | 280 | +Run only the tests whose dependencies were modified since the last run of the test. A test is selected when one of the files it loaded during its last run has a newer modification time than that run, or was deleted. A test that never ran with this flag has no record and always runs. A test file that failed in its last run also runs, until it passes, because the cause of a failure can be outside the recorded files, for example an environment variable or a server that was down. | | 281 | + | | 282 | +```sh | | 283 | +vitest run --since-last-run | | 284 | +``` | | 285 | + | | 286 | +The flag enables [`changed`](/config/changed), [`experimental.recordDependencies`](/config/experimental#experimental-recorddependencies) and the `mtime` [provider](/config/experimental#experimental-vcsprovider). Every run with this flag updates the records of the tests it ran, so the next run selects only the tests affected by changes made in between. To run every test again, run Vitest without the flag, or delete the cache with [`--clearCache`](#clearcache). | | 287 | + | | 288 | +The flag requires [`cache`](/config/cache). Vitest throws an error if the cache is disabled. | | 289 | + | | 290 | +::: warning | | 291 | +The modification time comes from the file system. A `git checkout` or an unpacked archive that sets old modification times is not detected. Files that a test did not load during its last run, for example a new `.env` file or a new `__mocks__` file, are not detected either. See [`experimental.recordDependencies`](/config/experimental#experimental-recorddependencies) for the full list of recorded files. | | 292 | +::: | | 293 | + | | 294 | +See [Running Only Affected Tests](/guide/recipes/affected-tests) for a comparison with `--changed`. | | 295 | + | 275 | 296 | ### last-failed | 276 | 297 |
| 277 | 298 | - **Type:** `boolean` | |
Collapse file Copy file name to clipboard +66Lines changed: 66 additions & 0 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
| | 1 | +--- | | 2 | +title: Running Only Affected Tests | Recipes | | 3 | +--- | | 4 | + | | 5 | +# Running Only Affected Tests | | 6 | + | | 7 | +A full run of a large suite is the right check before a merge, but it is slow feedback for a small change. If you edited one module, only the tests that load that module can change their result. Vitest has two flags that select those tests. They differ in one question: changed compared to what? | | 8 | + | | 9 | +- [`--changed`](/config/changed) compares the working tree with a Git reference. | | 10 | +- [`--since-last-run`](/guide/cli#since-last-run) compares every file a test loaded with the time of the test's last run. | | 11 | + | | 12 | +## `--changed`: what differs from a Git reference | | 13 | + | | 14 | +```sh | | 15 | +vitest run --changed # uncommitted changes, staged and unstaged | | 16 | +vitest run --changed HEAD~1 # the last commit and uncommitted changes | | 17 | +vitest run --changed origin/main # everything on this branch | | 18 | +``` | | 19 | + | | 20 | +Vitest asks Git for the changed files, then finds the test files that import them, directly or through other modules. A test runs when any file in its import graph is in the list. | | 21 | + | | 22 | +The reference is the baseline, so the same command gives the same selection until the baseline moves. This fits checks that are defined in terms of Git: a pull request against `origin/main`, or a pre-push hook that covers the commits you are about to push. It requires a Git repository, and a change that is committed and merged stops being "changed", even if the tests never ran against it on your machine. | | 23 | + | | 24 | +## `--since-last-run`: what changed since the test last ran | | 25 | + | | 26 | +```sh | | 27 | +vitest run --since-last-run | | 28 | +``` | | 29 | + | | 30 | +The first run with `--since-last-run` runs every test and records the files each test loaded, including dynamic imports, mocks, setup files and snapshot files. Every later run compares the modification time of those files with the time of the run that recorded them. A test runs again when one of its files is newer or was deleted, or when the test has no record yet. A test file that failed in its last run also runs again, until it passes, even if the fix was outside the recorded files. | | 31 | + | | 32 | +Each run updates the records of the tests it ran, so the baseline moves with you: after a run, only the next edit selects tests again. Git is not involved, so the flag works in a fresh container, in a worktree with an unrelated history, or in a directory that is not a repository. A new test file, or one that was never recorded, always runs. | | 33 | + | | 34 | +The records live in the [cache](/config/cache). `vitest run --clearCache` deletes them, and the next `--since-last-run` run is a full run again. | | 35 | + | | 36 | +## Which one to pick | | 37 | + | | 38 | +| You want to run the tests affected by… | Use | | | 39 | +| ---------------------------------------------------------------------- | --------------------------------- | | | 40 | +| a branch, before opening or updating a pull request | `--changed origin/main` | | | 41 | +| the commits you are about to push | `--changed <upstream>` | | | 42 | +| your uncommitted edits, in a Git repository | `--changed` or `--since-last-run` | | | 43 | +| your edits since you last ran the tests, committed or not | `--since-last-run` | | | 44 | +| edits in a directory without Git, or with a history you do not control | `--since-last-run` | | | 45 | + | | 46 | +`--changed` is the better fit when the baseline is shared with other people: the reference is the same on every machine and in CI. `--since-last-run` is the better fit when the baseline is your own last run: it does not matter whether you committed in between, or whether Git is there at all. | | 47 | + | | 48 | +## Agents | | 49 | + | | 50 | +AI coding agents run tests after almost every edit. They often work in a container or a temporary worktree, commit in the middle of a task, and start a new process for every command. A Git baseline fits none of that: `--changed` is empty right after a commit, and it needs a repository. `--since-last-run` tracks what the agent actually ran, so each `vitest run --since-last-run` covers exactly the edits since the previous one, and the first command in a fresh environment is a full run that builds the records. | | 51 | + | | 52 | +Tell the agent about it in the instructions file of your project: | | 53 | + | | 54 | +```md [AGENTS.md] | | 55 | +- After a change, run `vitest run --since-last-run` to test only what the change affects. | | 56 | +- Before you finish, run `vitest run` once for the whole suite. | | 57 | +``` | | 58 | + | | 59 | +Two things the records cannot see: a file that no test loaded during the last run (a new `.env` file, a new `__mocks__` file), and modification times set in the past by a checkout or an unpacked archive. The full run at the end covers both. | | 60 | + | | 61 | +## See also | | 62 | + | | 63 | +- [`changed`](/config/changed) | | 64 | +- [`--since-last-run`](/guide/cli#since-last-run) | | 65 | +- [`experimental.recordDependencies`](/config/experimental#experimental-recorddependencies) | | 66 | +- [Test Filtering](/guide/filtering) |
Collapse file Copy file name to clipboardExpand all lines: packages/vitest/src/node/affected-modules.ts +21-4Lines changed: 21 additions & 4 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -2,6 +2,7 @@ import type { DevEnvironment } from 'vite' | 2 | 2 | import type { Vitest } from './core' | 3 | 3 | import type { TestProject } from './project' | 4 | 4 | import type { TestSpecification } from './test-specification' | | 5 | +import type { ModuleDependency, ModulesResolver } from './vcs/vcs' | 5 | 6 | import { existsSync } from 'node:fs' | 6 | 7 | import os from 'node:os' | 7 | 8 | import { cleanUrl } from '@vitest/utils/helpers' | @@ -25,18 +26,32 @@ interface ModuleNode { | 25 | 26 | /** | 26 | 27 | * Finds test specifications that statically depend on any of the changed files. | 27 | 28 | */ | 28 | | -export class AffectedModulesResolver { | | 29 | +export class AffectedModulesResolver implements ModulesResolver { | 29 | 30 | private existsCache = new Map<string, boolean>() | 30 | 31 | private transformConcurrency = os.availableParallelism() | 31 | 32 | private activeTransforms = 0 | 32 | 33 | private transformQueue: Array<() => void> = [] | | 34 | + private related: string[] = [] | | 35 | + // set while `getDependencies` walks the graph | | 36 | + walked: Set<string> | undefined | 33 | 37 |
| 34 | 38 | constructor( | 35 | 39 | private vitest: Vitest, | 36 | | - private related: string[], | | 40 | + private specs: TestSpecification[], | 37 | 41 | ) {} | 38 | 42 |
| 39 | | - async resolve(specs: TestSpecification[]): Promise<TestSpecification[]> { | | 43 | + // the graph is only known after transforming it, so it is walked once without changes | | 44 | + async getDependencies(): Promise<ModuleDependency[]> { | | 45 | + this.walked = new Set() | | 46 | + await this.getAffectedSpecifications([]) | | 47 | + const files = new Set(Array.from(this.walked, (id) => cleanUrl(id))) | | 48 | + this.walked = undefined | | 49 | + return Array.from(files, (file) => ({ file })) | | 50 | + } | | 51 | + | | 52 | + async getAffectedSpecifications(related: string[]): Promise<TestSpecification[]> { | | 53 | + this.related = related | | 54 | + const specs = this.specs | 40 | 55 | if (this.dependsOnConfig(this.vitest.vite.config.configFileDependencies)) { | 41 | 56 | return specs | 42 | 57 | } | @@ -286,7 +301,7 @@ class ProjectGraph { | 286 | 301 | constructor( | 287 | 302 | private project: TestProject, | 288 | 303 | private environment: DevEnvironment, | 289 | | - private resolver: AffectedModulesResolver, | | 304 | + public resolver: AffectedModulesResolver, | 290 | 305 | ) {} | 291 | 306 |
| 292 | 307 | getModule(id: string): Promise<ModuleNode | null> { | @@ -443,6 +458,8 @@ class GraphWalk { | 443 | 458 | if (!node) { | 444 | 459 | return | 445 | 460 | } | | 461 | + this.graph.resolver.walked?.add(id) | | 462 | + node.watchedFiles.forEach((file) => this.graph.resolver.walked?.add(file)) | 446 | 463 | // a file imported with a query is a separate module | 447 | 464 | if (node.failed || this.changed.has(cleanUrl(id))) { | 448 | 465 | this.changed.add(id) | |
|