Files
overlookmotel b4f6593c4f ci(lexer): run lint, tests and conformance on both lexer implementations (#26298)
#26263 added the ability to run tests and conformance for both versions of `oxc_lexer` (SIMD and scalar fallback) on any platform.

Run tests and conformance on CI (both SIMD and scalar) on any PR which touches `oxc_lexer` crate or other files which affect conformance runner.

Additionally, run Clippy on the SIMD version (main Lint CI task doesn't lint it, due to the `target_feature = "avx2"` gate on the code).

All of this sets up `oxc_lexer` crate for further development, catching any regressions in CI.
2026-09-03 16:40:21 +00:00

382 lines
13 KiB
Makefile
Executable File

#!/usr/bin/env -S just --justfile
set windows-shell := ["powershell.exe", "-NoLogo", "-Command"]
set shell := ["bash", "-cu"]
_default:
@just --list -u
# ==================== ALIASES ====================
alias r := ready
alias c := conformance
alias f := fix
# ==================== SETUP & INITIALIZATION ====================
# Initialize the project by installing all necessary tools
init:
# Rust related init
cargo binstall watchexec-cli cargo-insta typos-cli cargo-shear@1.13.1 -y
# Node.js related init
pnpm install
# Clone or update submodules
submodules:
node .github/scripts/clone-parallel.mjs
just update-transformer-fixtures
# Install git pre-commit hook to format files
install-hook:
echo -e "#!/bin/sh\njust fmt" > .git/hooks/pre-commit
chmod +x .git/hooks/pre-commit
# ==================== CORE DEVELOPMENT ====================
# When ready, run the same CI commands
ready:
git diff --exit-code --quiet
pnpm install
typos
cargo lintgen
just fmt
just check
just test
just lint
just doc
just ast
git status
# Run cargo check
check *args:
cargo ck {{args}}
# Run all the tests
test *args:
cargo test --all-features {{args}}
# Lint the whole project
[unix]
lint *args:
CARGO_BUILD_WARNINGS=deny cargo lint {{args}}
[windows]
lint *args:
$Env:CARGO_BUILD_WARNINGS='deny'; cargo lint {{args}}
# Format all files
fmt:
-cargo shear --fix --check-test-targets # remove all unused dependencies
cargo fmt
node --run fmt
[unix]
doc *args:
RUSTDOCFLAGS='-D warnings' cargo doc --no-deps --document-private-items --all-features {{args}}
[windows]
doc *args:
$Env:RUSTDOCFLAGS='-D warnings'; cargo doc --no-deps --document-private-items --all-features {{args}}
# Fix all auto-fixable format and lint issues
fix:
cargo clippy --fix --allow-staged --no-deps
just fmt
typos -w
git status
# ==================== DEVELOPMENT TOOLS ====================
watch *args='':
watchexec --no-vcs-ignore {{args}}
watch-check:
just watch "'cargo check; cargo clippy'"
watch-example *args='':
just watch 'just example {{args}}'
# Run examples in parser, formatter, linter
example tool *args='':
cargo run -p oxc_{{tool}} --example {{tool}} -- {{args}}
# Run the benchmarks
benchmark:
cargo benchmark
# Run benchmarks for a single component
benchmark-one *args:
cargo benchmark --bench {{args}} --no-default-features --features {{ if args == "linter" { "linter" } else { "compiler" } }}
# ==================== TESTING & CONFORMANCE ====================
# Run all conformance tests
coverage:
cargo coverage
cargo run -p oxc_transform_conformance -- --exec
# Run Test262, Babel and TypeScript conformance suite
conformance *args='':
cargo coverage -- {{args}}
# Test ESTree
test-estree *args='':
cargo run -p oxc_coverage --profile coverage -- estree {{args}}
test-estree-tokens *args='':
cargo run -p oxc_coverage --profile coverage -- estree_tokens {{args}}
# Get code coverage
codecov:
cargo codecov --html
# ==================== AST & CODEGEN ====================
# Generate AST related boilerplate code.
# If fails first time, run with JS generators disabled first, and then again with JS generators enabled.
# This is necessary because JS generators use `oxc_*` crates (e.g. `oxc_minifier`), and those crates may not compile
# unless Rust code is generated first.
# See: https://github.com/oxc-project/oxc/issues/15564
[unix]
ast:
cargo run -p oxc_ast_tools || { cargo run -p oxc_ast_tools --no-default-features && cargo run -p oxc_ast_tools; }
[windows]
ast:
try { cargo run -p oxc_ast_tools } catch { cargo run -p oxc_ast_tools --no-default-features; cargo run -p oxc_ast_tools }
# ==================== PARSER ====================
# Parser-specific commands will be added here as needed
# ==================== LEXER ====================
# `oxc_lexer` compiles to one of two implementations, chosen at build time:
#
# * SIMD core, on x86_64 with `avx2` + `bmi2` enabled
# * Scalar fallback, everywhere else
#
# Both need testing, and they should produce identical results. Neither `avx2` nor `bmi2` is in the x86_64 baseline
# on any platform, so reaching the SIMD core always means asking for them explicitly - even on an x86_64 host.
# The flags live in `.cargo/lexer-simd.toml`, passed with `cargo --config`, which avoids shell quoting entirely
# (this justfile runs PowerShell on Windows).
#
# `--target` is always passed, so that the flags land on the triple the config file names,
# and so that the two builds get separate target dirs instead of invalidating each other.
#
# On an ARM host the SIMD build is cross-compiled and run under emulation - macOS provides that via Rosetta 2.
# On Mac, ensure x86_64 target is installed: `rustup target add x86_64-apple-darwin`.
# If it's not, you'll get a "can't find crate for std" error.
#
# On ARM Linux or ARM Windows it will fail, loudly, when linking or running.
_lexer-simd-target := if os() == "macos" { "x86_64-apple-darwin" } else if os() == "windows" { "x86_64-pc-windows-msvc" } else { "x86_64-unknown-linux-gnu" }
_lexer-simd := "--target " + _lexer-simd-target + " --config .cargo/lexer-simd.toml"
# Run `oxc_lexer`'s tests against the scalar fallback
test-lexer *args='':
cargo test -p oxc_lexer {{args}}
# Run `oxc_lexer`'s tests against the SIMD core
test-lexer-simd *args='':
cargo test -p oxc_lexer {{_lexer-simd}} {{args}}
# Run lexer conformance against the scalar fallback
conformance-lexer *args='':
cargo run -p oxc_coverage --profile coverage --features lexer -- lexer {{args}}
# Run lexer conformance against the SIMD core
conformance-lexer-simd *args='':
cargo run -p oxc_coverage --profile coverage {{_lexer-simd}} --features lexer -- lexer {{args}}
# Lint `oxc_lexer` and the conformance harness against the scalar fallback
[unix]
lint-lexer *args='':
CARGO_BUILD_WARNINGS=deny cargo clippy -p oxc_lexer -p oxc_coverage --all-targets --all-features {{args}}
[windows]
lint-lexer *args='':
$Env:CARGO_BUILD_WARNINGS='deny'; cargo clippy -p oxc_lexer -p oxc_coverage --all-targets --all-features {{args}}
# Lint `oxc_lexer` and the conformance harness against the SIMD core
lint-lexer-simd *args='':
just lint-lexer {{_lexer-simd}} {{args}}
# ==================== LINTER ====================
# oxlint release build
oxlint:
cargo build -p oxlint --release --features allocator
# watch oxlint, e.g. `just watch-oxlint test.js`
watch-oxlint *args='':
just watch 'cargo run -p oxlint -- --disable-nested-config {{args}}'
# oxlint release build for node.js
# After building, you can run oxlint with `node <oxc-root>/apps/oxlint/dist/cli.js`
oxlint-node:
pnpm -C apps/oxlint run build
# oxlint dev build, for testing with Node.js locally.
# This uses a non-release Rust build without the `allocator` feature (no mimalloc) and sets DEBUG options for the JS bundle,
# which mainly affects build time, performance, and debug assertions rather than available linting functionality.
# After building, you can run oxlint with `node <oxc-root>/apps/oxlint/dist/cli.js`
oxlint-node-dev:
pnpm -C apps/oxlint run build-dev
watch-oxlint-node *args='':
just watch 'pnpm run -C apps/oxlint build-dev && node apps/oxlint/dist/cli.js --disable-nested-config {{args}}'
# Create a new lint rule for any plugin
new-rule name plugin='eslint':
cargo run -p rulegen {{name}} {{plugin}}
just linter-schema-json
just linter-config-ts
just fmt
# Update test cases for an existing lint rule from upstream
update-rule-tests name plugin='eslint':
cargo run -p rulegen {{name}} {{plugin}} --update-tests
just fmt
# Legacy aliases for backward compatibility
new-eslint-rule name: (new-rule name "eslint")
new-jest-rule name: (new-rule name "jest")
new-ts-rule name: (new-rule name "typescript")
new-unicorn-rule name: (new-rule name "unicorn")
new-import-rule name: (new-rule name "import")
new-react-rule name: (new-rule name "react")
new-jsx-a11y-rule name: (new-rule name "jsx-a11y")
new-oxc-rule name: (new-rule name "oxc")
new-nextjs-rule name: (new-rule name "nextjs")
new-jsdoc-rule name: (new-rule name "jsdoc")
new-react-perf-rule name: (new-rule name "react-perf")
new-n-rule name: (new-rule name "n")
new-promise-rule name: (new-rule name "promise")
new-vitest-rule name: (new-rule name "vitest")
new-vue-rule name: (new-rule name "vue")
# Alias for backward compatibility
alias new-typescript-rule := new-ts-rule
# ==================== FORMATTER ====================
# oxfmt release build
oxfmt:
cargo build -p oxfmt --release --features allocator
# watch oxfmt, e.g. `just watch-oxfmt test.js`
watch-oxfmt *args='':
just watch 'cargo run -p oxfmt -- {{args}}'
# Build oxfmt in release build
# After building, you can run oxfmt with `node <oxc-root>/apps/oxfmt/dist/cli.js`
oxfmt-node:
pnpm -C apps/oxfmt run build
# oxfmt dev build, for testing with Node.js locally.
# This builds faster than the release build and may differ in performance or behavior.
# After building, you can run oxfmt with `node <oxc-root>/apps/oxfmt/dist/cli.js`
oxfmt-node-dev:
pnpm -C apps/oxfmt run build-dev
watch-oxfmt-node *args='':
just watch 'pnpm run -C apps/oxfmt build-dev && node apps/oxfmt/dist/cli.js {{args}}'
# ==================== TRANSFORMER ====================
# Test Transform
test-transform *args='':
cargo run -p oxc_transform_conformance -- --exec {{args}}
# Update transformer conformance test fixtures
update-transformer-fixtures:
cd tasks/coverage/babel; git reset --hard HEAD; git clean -f -q
node tasks/transform_conformance/update_fixtures.ts
# ==================== MINIFIER ====================
# Update minifier size snapshots
minsize:
cargo minsize
just allocs
# Update memory allocation snapshots
allocs:
cargo allocs
# Update linter timing snapshots
lint-timings:
cargo lint-timings
# Generate minifier size comparison
minifier-diff:
#!/usr/bin/env bash
cargo minsize --compress-only pr
git checkout main
cargo minsize --compress-only main
for file in antd bundle.min d3 echarts jquery lodash moment react.development three typescript victory vue
do
echo $file.js >> diff
diff target/minifier/main/$file.js target/minifier/pr/$file.js >> diff
done
git checkout -
# ==================== PLAYGROUND ====================
# Install wasm32-wasip1-threads for playground
install-wasm:
rustup target add wasm32-wasip1-threads
build-playground:
pnpm --filter oxc-playground build
watch-playground:
just watch 'pnpm --filter oxc-playground build-dev'
# ==================== UTILITIES & ADVANCED ====================
# Generate website documentation, intended for updating the oxc.rs website.
# Path should be the path to your clone of https://github.com/oxc-project/website
# When testing changes to the website documentation, you may also want to run `pnpm run fmt`
# in the website directory.
website path:
cargo run -p website_linter rules --rules-json {{path}}/.vitepress/data/rules.json --rule-docs {{path}}/src/docs/guide/usage/linter/rules --git-ref $(git rev-parse HEAD) --rule-count {{path}}/src/docs/guide/usage
cargo run -p website_linter cli > {{path}}/src/docs/guide/usage/linter/generated-cli.md
cargo run -p website_linter schema-markdown > {{path}}/src/docs/guide/usage/linter/generated-config.md
cargo run -p website_linter schema-markdown-lsp > {{path}}/src/docs/guide/usage/linter/generated-lsp-config.md
cargo run -p website_formatter cli > {{path}}/src/docs/guide/usage/formatter/generated-cli.md
cargo run -p website_formatter schema-markdown > {{path}}/src/docs/guide/usage/formatter/generated-config.md
# Generate linter schema json for `npm/oxlint/configuration_schema.json`
linter-schema-json:
cargo run -p website_linter schema-json > npm/oxlint/configuration_schema.json
# Generate linter config TypeScript types for `apps/oxlint/src-js/package/config.generated.ts`
linter-config-ts:
pnpm --filter oxlint-app generate-config-types
# Generate formatter schema json for `npm/oxfmt/configuration_schema.json`
formatter-schema-json:
cargo run -p website_formatter schema-json > npm/oxfmt/configuration_schema.json
# Generate formatter config TypeScript types for `apps/oxfmt/src-js/config.generated.ts`
formatter-config-ts:
pnpm --filter oxfmt-app generate-config-types
# Automatically DRY up Cargo.toml manifests in a workspace
autoinherit:
cargo binstall cargo-autoinherit
cargo autoinherit
# ==================== PLATFORM HELPERS ====================
[unix]
clone-submodule dir url sha:
cd {{dir}} || git init {{dir}}
cd {{dir}} && git remote add origin {{url}} || true
cd {{dir}} && git fetch --depth=1 origin {{sha}} && git reset --hard {{sha}} && git clean -f -q
[windows]
clone-submodule dir url sha:
if (-not (Test-Path {{dir}}/.git)) { git init {{dir}} }
cd {{dir}} ; if ((git remote) -notcontains 'origin') { git remote add origin {{url}} } else { git remote set-url origin {{url}} }
cd {{dir}} ; git fetch --depth=1 origin {{sha}} ; git reset --hard {{sha}} ; git clean -f -q