From 746aba201e6ceef7fa54f37740dc6449290c2dfe Mon Sep 17 00:00:00 2001 From: Zeina Migeed Date: Thu, 5 Mar 2026 15:34:57 -0800 Subject: [PATCH] Add mypy_primer comparison script for pyrefly vs pyright Adds two scripts under scripts/: - projects.py: project list sourced from mypy_primer with pyrefly_cmd additions - compare_typecheckers.py: clones projects, sets up venvs with deps, runs pyrefly and pyright, and produces a comparison table of error counts and times Supports two-phase usage (clone-only then run), persistent caching, CSV output, and per-project filtering via -k. --- scripts/compare_typecheckers.py | 478 ++++++++++ scripts/projects.py | 1490 +++++++++++++++++++++++++++++++ 2 files changed, 1968 insertions(+) create mode 100644 scripts/compare_typecheckers.py create mode 100644 scripts/projects.py diff --git a/scripts/compare_typecheckers.py b/scripts/compare_typecheckers.py new file mode 100644 index 0000000000..00fb1d06ea --- /dev/null +++ b/scripts/compare_typecheckers.py @@ -0,0 +1,478 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +"""Compare pyrefly vs pyright error counts on mypy_primer projects. + +Local-only script. Two-phase usage: + + Prerequisites: + - A conda/venv environment with pyright installed: pip install pyright + + Phase 1 — Clone projects (run outside conda, proxy works): + python3 compare_typecheckers.py --clone-only --cache-dir /tmp/primer_cache + + Phase 2 — Run checkers on cached repos (run inside conda): + python3 compare_typecheckers.py --cache-dir /tmp/primer_cache --reuse-cache -o /tmp/results.txt + + Or if cloning works in your env, do it all at once: + python3 compare_typecheckers.py --pyrefly /path/to/pyrefly --cache-dir /tmp/primer_cache -o /tmp/results.txt + + Cleanup: + python3 compare_typecheckers.py --cleanup --cache-dir /tmp/primer_cache +""" + +import argparse +import configparser +import csv +import logging +import os +import re +import shutil +import subprocess +import tempfile +import time +import venv + +from projects import get_mypy_primer_projects, Project + + +def run( + cmd: str | list[str], debug: bool, **kwargs: object +) -> subprocess.CompletedProcess[str]: + """Run a command, suppressing output unless debug is set.""" + logging.debug(cmd) + if not debug and "capture_output" not in kwargs: + kwargs.setdefault("stdout", subprocess.DEVNULL) + kwargs.setdefault("stderr", subprocess.DEVNULL) + return subprocess.run(cmd, **kwargs) # noqa + + +def build_pyrefly() -> str: + """Build pyrefly via cargo in release mode and return the absolute binary path.""" + logging.info("Building pyrefly via cargo (release mode)...") + result = subprocess.run( + ["cargo", "build", "--release", "-p", "pyrefly"], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError(f"Failed to build pyrefly: {result.stderr}") + # Walk up from this script to find the repo root (where Cargo.toml lives) + script_dir = os.path.dirname(os.path.abspath(__file__)) + repo_root = os.path.dirname(script_dir) + path = os.path.join(repo_root, "target", "release", "pyrefly") + if not os.path.exists(path): + raise RuntimeError(f"Built binary not found at {path}") + logging.info(f"Built pyrefly at {path}") + return path + + +def clone_projects(projects: list[Project], cache_dir: str, debug: bool) -> None: + """Clone all projects into cache_dir.""" + os.makedirs(cache_dir, exist_ok=True) + for project in projects: + repo_dir = os.path.join(cache_dir, project.name) + if os.path.exists(repo_dir): + logging.info(f"Skipping {project.name} (already exists)") + continue + logging.info(f"Cloning {project.name}...") + run( + ["git", "clone", "--depth=1", project.location, repo_dir], + debug, + ) + if project.revision: + run(["git", "checkout", project.revision], debug, cwd=repo_dir) + + +def setup_project( + project: Project, repo_dir: str, debug: bool, reuse: bool +) -> str | None: + """Clone project, create venv, install deps + runtime deps. + + Returns site-package-path flags for pyrefly, or None if no deps. + """ + if not reuse: + run( + ["git", "clone", "--depth=1", project.location, repo_dir], + debug, + ) + if project.revision: + run(["git", "checkout", project.revision], debug, cwd=repo_dir) + + deps = [dep.format(repo=repo_dir) for dep in project.deps or []] + venv_dir = os.path.join(repo_dir, "_primer_venv") + activate = os.path.join(venv_dir, "bin", "activate") + + if not reuse or not os.path.exists(venv_dir): + venv.create(venv_dir, with_pip=True, clear=True) + if deps: + result = run( + f"source {activate} && pip install " + " ".join(deps), + debug, + shell=True, + capture_output=True, + text=True, + ) + if result.returncode: + logging.warning( + f"Failed to install deps for {project.name}: {result.stderr}" + ) + + # Install runtime deps via pip install ., then uninstall just the project package + _install_and_uninstall_project(activate, repo_dir, project.name, debug) + + # Get site-package paths for pyrefly + site_paths = run( + f"source {activate} && python -c \"import site; print(' '.join('--site-package-path=' + p for p in site.getsitepackages()))\"", + debug, + shell=True, + capture_output=True, + text=True, + ).stdout.strip() + return site_paths or None + + +def _install_and_uninstall_project( + activate: str, repo_dir: str, project_name: str, debug: bool +) -> None: + """Install project via pip to pull in runtime deps, then uninstall just the project package. + + Leaves transitive dependencies in the venv so type checkers can resolve + imports, without the project's own source shadowing the repo checkout. + """ + logging.debug( + f"Installing runtime deps for {project_name} (pip install . then uninstall)" + ) + result = run( + f"source {activate} && pip install .", + debug, + cwd=repo_dir, + shell=True, + capture_output=True, + text=True, + ) + if result.returncode: + logging.debug( + f"pip install . failed for {project_name} (non-fatal): {result.stderr}" + ) + return + + pkg_name = _get_package_name(repo_dir) + if pkg_name: + logging.debug(f"Uninstalling {pkg_name} (keeping transitive deps)") + run( + f"source {activate} && pip uninstall -y {pkg_name}", + debug, + cwd=repo_dir, + shell=True, + ) + + # Clean build artifacts + for d in ["build", "dist"]: + p = os.path.join(repo_dir, d) + if os.path.isdir(p): + shutil.rmtree(p) + for entry in os.listdir(repo_dir): + if entry.endswith(".egg-info"): + shutil.rmtree(os.path.join(repo_dir, entry)) + + +def _get_package_name(repo_dir: str) -> str | None: + """Extract the installed package name from pyproject.toml or setup.cfg. + + Needed because the package name often differs from the repo name + (e.g. repo 'python-chess' installs package 'chess'), and pip uninstall + requires the exact package name. + """ + pyproject = os.path.join(repo_dir, "pyproject.toml") + if os.path.exists(pyproject): + with open(pyproject) as f: + for line in f: + m = re.match(r'\s*name\s*=\s*["\'](.+?)["\']', line) + if m: + return m.group(1) + setup_cfg = os.path.join(repo_dir, "setup.cfg") + if os.path.exists(setup_cfg): + cfg = configparser.ConfigParser() + cfg.read(setup_cfg) + if cfg.has_option("metadata", "name"): + return cfg.get("metadata", "name") + return None + + +def parse_error_count(output: str) -> int: + """Parse error count from pyrefly or pyright output. + + Pyrefly: 'INFO 4 errors (11 suppressed)' or 'INFO 3,418 errors' or 'INFO No errors' + Pyright: '2 errors, 0 warnings, 0 informations' + """ + for line in reversed(output.splitlines()): + if "No errors" in line: + return 0 + m = re.search(r"([\d,]+)\s+errors?", line) + if m: + return int(m.group(1).replace(",", "")) + return -1 + + +def generate_table( + results: list[dict[str, object]], output_file: str | None, csv_file: str | None +) -> None: + """Generate and print summary table, optionally write to file and CSV.""" + lines = [] + header = f"{'Project':<35} {'Pyrefly Err':>12} {'Pyright Err':>12} {'Pyrefly (s)':>12} {'Pyright (s)':>12}" + separator = "-" * 87 + lines.append(header) + lines.append(separator) + for r in results: + lines.append( + f"{r['project']:<35} {r['pyrefly_errors']:>12} {r['pyright_errors']:>12} {r['pyrefly_time']:>12} {r['pyright_time']:>12}" + ) + + total_pyrefly = sum( + r["pyrefly_errors"] + for r in results + if isinstance(r["pyrefly_errors"], int) and r["pyrefly_errors"] >= 0 + ) + total_pyright = sum( + r["pyright_errors"] + for r in results + if isinstance(r["pyright_errors"], int) and r["pyright_errors"] >= 0 + ) + lines.append(separator) + lines.append(f"{'TOTAL':<35} {total_pyrefly:>12} {total_pyright:>12}") + + table = "\n" + "\n".join(lines) + "\n" + print(table) + + if output_file: + with open(output_file, "w") as f: + f.write(table) + logging.info(f"Table written to {output_file}") + + if csv_file: + with open(csv_file, "w", newline="") as f: + writer = csv.DictWriter( + f, + fieldnames=[ + "project", + "pyrefly_errors", + "pyright_errors", + "pyrefly_time", + "pyright_time", + ], + ) + writer.writeheader() + writer.writerows(results) + logging.info(f"Results written to {csv_file}") + + +def extract_paths_from_cmd(cmd: str) -> list[str]: + """Extract file/directory paths from a checker command like '{pyrefly} src tests'.""" + parts = cmd.split() + return [p for p in parts[1:] if not p.startswith("-") and not p.startswith("{")] + + +def check_project( + project: Project, repo_dir: str, pyrefly_bin: str, debug: bool +) -> dict[str, object]: + """Run pyrefly and pyright on a project, return a results row.""" + site_paths = setup_project(project, repo_dir, debug, reuse=os.path.exists(repo_dir)) + + paths = extract_paths_from_cmd(project.pyrefly_cmd) + # Filter to paths that actually exist — repos restructure over time and + # stale entries in projects.py cause pyright to exit without parseable output. + existing = [p for p in paths if os.path.exists(os.path.join(repo_dir, p))] + if paths and not existing: + logging.warning( + f" {project.name}: none of the target paths exist ({paths}), falling back to '.'" + ) + path_args = " ".join(existing) if existing else "." + + # Pyrefly — init to auto-detect configs, exclude venv via CLI flag + # (config-file project-excludes is unreliable, CLI flag works) + # Try migrating from pyright first for apples-to-apples comparison. + # If that fails (no pyright config), fall back to default init. + result = run( + f"{pyrefly_bin} init --non-interactive --migrate-from pyright", + debug, + cwd=repo_dir, + shell=True, + capture_output=True, + text=True, + ) + if result.returncode != 0: + logging.info(" No pyright config found, falling back to default init") + run(f"{pyrefly_bin} init --non-interactive", debug, cwd=repo_dir, shell=True) + pyrefly_cmd = project.pyrefly_cmd.format( + pyrefly=f'{pyrefly_bin} check --project-excludes "**/_primer_venv/**"' + ) + if site_paths: + pyrefly_cmd += " " + site_paths + + start = time.time() + pr = run( + pyrefly_cmd, debug, cwd=repo_dir, shell=True, capture_output=True, text=True + ) + pyrefly_time = time.time() - start + pyrefly_output = (pr.stdout or "") + (pr.stderr or "") + pyrefly_errors = parse_error_count(pyrefly_output) + + # Pyright — use same paths as pyrefly for apples-to-apples + start = time.time() + pp = run( + f"pyright {path_args}", + debug, + cwd=repo_dir, + shell=True, + capture_output=True, + text=True, + ) + pyright_time = time.time() - start + pyright_output = (pp.stdout or "") + (pp.stderr or "") + pyright_errors = parse_error_count(pyright_output) + + logging.info( + f" pyrefly: {pyrefly_errors} errors in {pyrefly_time:.1f}s | " + f"pyright: {pyright_errors} errors in {pyright_time:.1f}s" + ) + return { + "project": project.name, + "pyrefly_errors": pyrefly_errors, + "pyright_errors": pyright_errors, + "pyrefly_time": round(pyrefly_time, 2), + "pyright_time": round(pyright_time, 2), + } + + +def get_projects(names: list[str] | None) -> list[Project]: + """Get filtered list of projects that have both pyrefly and pyright commands.""" + projects = [ + p + for p in get_mypy_primer_projects() + if p.pyrefly_cmd and p.pyright_cmd and not p.skip_pyrefly + ] + if names: + lower_names = {n.lower() for n in names} + projects = [p for p in projects if p.name.lower() in lower_names] + return projects + + +def resolve_repo_dir( + project: Project, + cache_dir: str | None, + tmp_dir: str | None, + reuse_cache: bool, +) -> str: + """Determine the repo directory for a project, cleaning stale dirs if needed.""" + if cache_dir: + repo_dir = os.path.join(cache_dir, project.name) + if not reuse_cache and os.path.exists(repo_dir): + shutil.rmtree(repo_dir) + os.makedirs(cache_dir, exist_ok=True) + else: + repo_dir = os.path.join(tmp_dir, project.name) + return repo_dir + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Compare pyrefly vs pyright on mypy_primer projects" + ) + parser.add_argument( + "--pyrefly", help="Path to pyrefly binary (default: build via cargo)" + ) + parser.add_argument("-k", "--project", nargs="+", help="Only run these projects") + parser.add_argument("--cache-dir", help="Persistent cache dir for cloned repos") + parser.add_argument( + "--reuse-cache", + action="store_true", + help="Reuse existing cached repos (skip clone, reuse venv if present)", + ) + parser.add_argument( + "--clone-only", + action="store_true", + help="Only clone projects into --cache-dir, don't run checkers", + ) + parser.add_argument( + "--cleanup", + action="store_true", + help="Remove --cache-dir and all cloned repos/venvs, then exit", + ) + parser.add_argument("--debug", action="store_true") + parser.add_argument("--csv", help="Write results to CSV file") + parser.add_argument("--output", "-o", help="Write summary table to a file") + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + ) + + projects = get_projects(args.project) + + if not projects: + logging.info("No matching projects found") + return + + # Cleanup mode: remove cache dir and exit + if args.cleanup: + if not args.cache_dir: + parser.error("--cleanup requires --cache-dir") + if os.path.exists(args.cache_dir): + shutil.rmtree(args.cache_dir) + logging.info(f"Removed {args.cache_dir}") + else: + logging.info(f"{args.cache_dir} does not exist, nothing to clean") + return + + # Clone-only mode: just clone repos and exit + if args.clone_only: + if not args.cache_dir: + parser.error("--clone-only requires --cache-dir") + clone_projects(projects, args.cache_dir, args.debug) + logging.info(f"Cloned {len(projects)} projects into {args.cache_dir}") + return + + if not shutil.which("pyright"): + parser.error( + "pyright not found on PATH. Install it with: pip install pyright (in a conda env with its own pip)" + ) + + pyrefly_bin = os.path.abspath(args.pyrefly) if args.pyrefly else build_pyrefly() + results: list[dict[str, object]] = [] + tmp_dir = None if args.cache_dir else tempfile.mkdtemp() + + try: + for project in projects: + logging.info(f"=== {project.name} ===") + try: + repo_dir = resolve_repo_dir( + project, args.cache_dir, tmp_dir, args.reuse_cache + ) + results.append( + check_project(project, repo_dir, pyrefly_bin, args.debug) + ) + except Exception as e: + logging.error(f" FAILED: {e}") + results.append( + { + "project": project.name, + "pyrefly_errors": "ERR", + "pyright_errors": "ERR", + "pyrefly_time": 0, + "pyright_time": 0, + } + ) + finally: + if tmp_dir: + shutil.rmtree(tmp_dir, ignore_errors=True) + + generate_table(results, args.output, args.csv) + + +if __name__ == "__main__": + main() diff --git a/scripts/projects.py b/scripts/projects.py new file mode 100644 index 0000000000..0ddbe5103c --- /dev/null +++ b/scripts/projects.py @@ -0,0 +1,1490 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass(kw_only=True) +class Project: + location: str + name_override: str | None = None + pyrefly_cmd: str | None + deps: list[str] | None = None + + # BELOW ARE UNUSED FIELDS + mypy_cmd: str + pyright_cmd: str | None + + install_cmd: str | None = None + needs_mypy_plugins: bool = False + + # if expected_success, there is a recent version of mypy which passes cleanly + expected_mypy_success: bool = False + expected_pyright_success: bool = False + expected_pyrefly_success: bool = False + + # cost is vaguely proportional to type check time + # for mypy we use the compiled times + cost: dict[str, int] = field(default_factory=dict) + + revision: str | None = None + min_python_version: tuple[int, int] | None = None + supported_platforms: list[str] | None = None + + skip_pyrefly: bool = False + skip_determinism_check: bool = False + + @property + def name(self) -> str: + if self.name_override is not None: + return self.name_override + return Path(self.location).name + + +# Project list is sourced from: +# https://github.com/hauntsaninja/mypy_primer/blob/master/mypy_primer/projects.py +def get_mypy_primer_projects() -> list[Project]: + return [ + Project( + location="https://github.com/python/mypy", + mypy_cmd="{mypy} --config-file mypy_self_check.ini -p mypy -p mypyc", + pyright_cmd="{pyright} mypy mypyc", + pyrefly_cmd="{pyrefly} mypy mypyc", + deps=["pytest", "types-psutil", "types-setuptools", "filelock", "tomli"], + expected_mypy_success=True, + cost={"mypy": 15, "pyright": 50}, + ), + Project( + location="https://github.com/hauntsaninja/mypy_primer", + mypy_cmd="{mypy} -p mypy_primer --strict", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} mypy_primer", + expected_mypy_success=True, + ), + Project( + location="https://github.com/psf/black", + mypy_cmd="{mypy} src", + pyright_cmd="{pyright} src", + pyrefly_cmd="{pyrefly} src", + deps=["aiohttp", "click", "pathspec", "tomli", "platformdirs", "packaging"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/hauntsaninja/pyp", + mypy_cmd="{mypy} --strict -m pyp", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + expected_mypy_success=True, + ), + Project( + location="https://github.com/pytest-dev/pytest", + mypy_cmd="{mypy} src testing", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} src testing", + deps=["attrs", "pluggy", "py", "types-setuptools"], + expected_mypy_success=True, + skip_determinism_check=True, + ), + Project( + location="https://github.com/pandas-dev/pandas", + mypy_cmd="{mypy} pandas", + pyright_cmd="{pyright} pandas", + pyrefly_cmd="{pyrefly} pandas", + deps=[ + "numpy", + "types-python-dateutil", + "types-pytz", + "types-PyMySQL", + "types-setuptools", + "pytest", + ], + expected_mypy_success=True, + cost={"mypy": 60}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/pycqa/pylint", + mypy_cmd="{mypy} pylint/checkers --ignore-missing-imports", + pyright_cmd="{pyright} pylint/checkers", + pyrefly_cmd="{pyrefly} pylint/checkers", + deps=["types-toml"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/aio-libs/aiohttp", + mypy_cmd="{mypy} aiohttp", + pyright_cmd="{pyright} aiohttp", + pyrefly_cmd="{pyrefly} aiohttp", + deps=["pytest"], + install_cmd="AIOHTTP_NO_EXTENSIONS=1 {install} -e .", + expected_mypy_success=True, + supported_platforms=["linux", "darwin"], + ), + Project( + location="https://github.com/python-attrs/attrs", + mypy_cmd=( + "{mypy} src/attr/__init__.pyi src/attr/_version_info.pyi src/attr/converters.pyi" + " src/attr/exceptions.pyi src/attr/filters.pyi src/attr/setters.pyi" + " src/attr/validators.pyi tests/typing_example.py" + ), + pyright_cmd="{pyright}", + pyrefly_cmd=( + "{pyrefly} src/attr/__init__.pyi src/attr/_version_info.pyi src/attr/converters.pyi" + " src/attr/exceptions.pyi src/attr/filters.pyi src/attr/setters.pyi" + " src/attr/validators.pyi tests/typing_example.py" + ), + expected_mypy_success=True, + ), + Project( + location="https://github.com/sphinx-doc/sphinx", + mypy_cmd="{mypy} sphinx", + pyright_cmd="{pyright} sphinx", + pyrefly_cmd="{pyrefly} sphinx", + deps=["babel", "docutils-stubs", "types-requests", "packaging", "jinja2"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/scikit-learn/scikit-learn", + mypy_cmd="{mypy} sklearn", + pyright_cmd="{pyright} sklearn", + pyrefly_cmd="{pyrefly} sklearn", + deps=["numpy"], + expected_mypy_success=True, + cost={"mypy": 15, "pyright": 240}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/pypa/bandersnatch", + mypy_cmd="{mypy} src", + pyright_cmd="{pyright} src", + pyrefly_cmd="{pyrefly} src", + deps=["types-filelock", "types-freezegun", "types-setuptools"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/hauntsaninja/boostedblob", + mypy_cmd="{mypy} boostedblob", + pyright_cmd="{pyright} boostedblob", + pyrefly_cmd="{pyrefly} boostedblob", + deps=["aiohttp", "uvloop", "pycryptodome"], + expected_mypy_success=True, + supported_platforms=["linux", "darwin"], + ), + Project( + location="https://github.com/quora/asynq", + mypy_cmd="{mypy} asynq", + pyright_cmd="{pyright} asynq", + pyrefly_cmd="{pyrefly} asynq", + deps=["qcore"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/scrapy/scrapy", + mypy_cmd="{mypy} scrapy tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} scrapy tests", + deps=["attrs", "types-pyOpenSSL", "types-setuptools"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/pypa/twine", + mypy_cmd="{mypy} twine", + pyright_cmd="{pyright} twine", + pyrefly_cmd="{pyrefly} twine", + deps=["keyring", "types-requests", "rich"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/more-itertools/more-itertools", + mypy_cmd="{mypy} more_itertools", + pyright_cmd="{pyright} more_itertools", + pyrefly_cmd="{pyrefly} more_itertools", + expected_mypy_success=True, + ), + Project( + location="https://github.com/pydata/xarray", + mypy_cmd="{mypy} --exclude conftest.py", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly}", + deps=["types-PyYAML", "types-python-dateutil", "types-pytz", "numpy"], + expected_mypy_success=True, + cost={"mypy": 25, "pyright": 170}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/pallets/werkzeug", + mypy_cmd="{mypy} src/werkzeug tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} src/werkzeug tests", + deps=["types-setuptools", "pytest", "markupsafe"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/pallets/jinja", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["markupsafe"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/mystor/git-revise", + mypy_cmd="{mypy} gitrevise", + pyright_cmd="{pyright} gitrevise", + pyrefly_cmd="{pyrefly} gitrevise", + expected_mypy_success=True, + ), + Project( + location="https://github.com/PyGithub/PyGithub", + mypy_cmd="{mypy} github tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} github tests", + deps=["types-requests", "pyjwt"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/we-like-parsers/pegen", + mypy_cmd="{mypy} src/pegen", + pyright_cmd="{pyright} src/pegen", + pyrefly_cmd="{pyrefly} src/pegen", + expected_mypy_success=True, + ), + Project( + location="https://github.com/zulip/zulip", + mypy_cmd=( + "{mypy} zerver zilencer zproject tools analytics corporate scripts --platform=linux" + ), + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} zerver zilencer zproject tools analytics corporate scripts", + deps=[ + "types-PyYAML", + "types-polib", + "types-redis", + "types-Markdown", + "types-decorator", + "types-pytz", + "types-requests", + "types-python-dateutil", + "types-orjson", + "cryptography", + "django-stubs", + ], + # TODO: the plugin here is a little involved and might only work on linux + # figure out what it would take to make it actually work + # needs_mypy_plugins=True, + expected_mypy_success=True, + cost={"pyright": 60}, + ), + Project( + location="https://github.com/dropbox/stone", + mypy_cmd="{mypy} stone test", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} stone test", + deps=["types-six"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/yelp/paasta", + mypy_cmd="{mypy} paasta_tools", + pyright_cmd="{pyright} paasta_tools", + pyrefly_cmd="{pyrefly} paasta_tools", + deps=[ + "types-retry", + "types-tzlocal", + "types-ujson", + "types-python-dateutil", + "types-pytz", + "types-PyYAML", + "types-requests", + ], + expected_mypy_success=True, + supported_platforms=["linux", "darwin"], + ), + # mypy plugin + Project( + location="https://github.com/PrefectHQ/prefect", + mypy_cmd="{mypy} src/prefect --exclude conftest.py", + pyright_cmd="{pyright} src", + pyrefly_cmd="{pyrefly} src/prefect", + deps=[ + "types-python-dateutil", + "types-requests", + "types-simplejson", + "types-toml", + "types-croniter", + "types-PyYAML", + "types-python-slugify", + "types-pytz", + "cryptography", + "SQLAlchemy", + "pydantic", + ], + needs_mypy_plugins=True, + expected_mypy_success=True, + cost={"mypy": 10, "pyright": 60}, + ), + Project( + location="https://github.com/pallets/itsdangerous", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["pytest"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/jab/bidict", + mypy_cmd="{mypy} bidict", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} bidict", + expected_mypy_success=True, + ), + Project( + location="https://github.com/jaraco/zipp", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + expected_mypy_success=True, + ), + Project( + location="https://github.com/aaugustin/websockets", + mypy_cmd="{mypy} --strict src", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} src", + expected_mypy_success=True, + ), + Project( + location="https://github.com/pycqa/isort", + mypy_cmd="{mypy} --ignore-missing-imports isort", + pyright_cmd="{pyright} isort", + pyrefly_cmd="{pyrefly} isort", + deps=["types-setuptools"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/aio-libs/aioredis", + mypy_cmd="{mypy} aioredis --ignore-missing-imports", + pyright_cmd="{pyright} aioredis", + pyrefly_cmd="{pyrefly} aioredis", + expected_mypy_success=True, + ), + Project( + location="https://github.com/agronholm/anyio", + mypy_cmd="{mypy} src", + pyright_cmd="{pyright} src", + pyrefly_cmd="{pyrefly} src", + expected_mypy_success=True, + ), + Project( + location="https://github.com/aio-libs/yarl", + mypy_cmd="{mypy} --show-error-codes yarl tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} yarl tests", + deps=["multidict"], + expected_mypy_success=True, + ), + # mypy plugin + Project( + location="https://github.com/freqtrade/freqtrade", + mypy_cmd="{mypy} freqtrade scripts", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} freqtrade scripts", + deps=[ + "types-cachetools", + "types-requests", + "types-python-dateutil", + "types-tabulate", + "types-filelock", + "pydantic", + "sqlalchemy", + ], + needs_mypy_plugins=True, + expected_mypy_success=True, + cost={"mypy": 15}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/google/jax", + mypy_cmd="{mypy} jax", + pyright_cmd="{pyright} jax", + pyrefly_cmd="{pyrefly} jax", + deps=["types-requests", "numpy"], + expected_mypy_success=True, + cost={"mypy": 30, "pyright": 90}, + ), + Project( + location="https://github.com/dulwich/dulwich", + mypy_cmd="{mypy} dulwich", + pyright_cmd="{pyright} dulwich", + pyrefly_cmd="{pyrefly} dulwich", + deps=["types-certifi", "types-paramiko"], + expected_mypy_success=True, + skip_determinism_check=True, + ), + Project( + location="https://github.com/optuna/optuna", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly}", + deps=[ + "types-PyYAML", + "types-redis", + "types-setuptools", + "SQLAlchemy", + "numpy", + ], + expected_mypy_success=True, + cost={"mypy": 25, "pyright": 70}, + ), + Project( + location="https://github.com/trailofbits/manticore", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["types-protobuf", "types-PyYAML", "types-redis", "types-setuptools"], + expected_mypy_success=True, + cost={"mypy": 15, "pyright": 75}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/aiortc/aiortc", + mypy_cmd="{mypy} src", + pyright_cmd="{pyright} src", + pyrefly_cmd="{pyrefly} src", + deps=["cryptography"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/Textualize/rich", + mypy_cmd="{mypy} -p rich --ignore-missing-imports --warn-unreachable", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} rich", + deps=["attrs"], + expected_mypy_success=True, + ), + # mypy plugin + Project( + location="https://github.com/dedupeio/dedupe", + mypy_cmd="{mypy} --ignore-missing-imports dedupe", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} dedupe", + deps=["numpy"], + needs_mypy_plugins=True, + expected_mypy_success=True, + ), + Project( + location="https://github.com/schemathesis/schemathesis", + mypy_cmd="{mypy} src/schemathesis", + pyright_cmd="{pyright} src/schemathesis", + pyrefly_cmd="{pyrefly} src/schemathesis", + deps=["attrs", "types-requests", "types-PyYAML", "hypothesis"], + expected_mypy_success=True, + supported_platforms=["linux", "darwin"], + skip_determinism_check=True, + ), + Project( + location="https://github.com/graphql-python/graphql-core", + mypy_cmd="{mypy} src tests", + pyright_cmd="{pyright} src tests", + pyrefly_cmd="{pyrefly} src tests", + expected_mypy_success=True, + cost={"mypy": 40}, + ), + Project( + location="https://github.com/Legrandin/pycryptodome", + mypy_cmd="{mypy} lib", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} lib", + expected_mypy_success=True, + ), + Project( + location="https://github.com/niklasf/python-chess", + mypy_cmd="{mypy} --strict chess", + pyright_cmd="{pyright} chess", + pyrefly_cmd="{pyrefly} chess", + expected_mypy_success=True, + ), + Project( + location="https://github.com/pytorch/ignite", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + expected_mypy_success=True, + cost={"pyright": 50}, + ), + Project( + location="https://github.com/pypa/packaging", + mypy_cmd="{mypy} src", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} src", + expected_mypy_success=True, + ), + Project( + location="https://github.com/pydantic/pydantic", + mypy_cmd="{mypy} pydantic", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} pydantic", + deps=["types-toml"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/encode/starlette", + mypy_cmd="{mypy} starlette tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} starlette tests", + deps=["types-requests", "types-PyYAML"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/aio-libs/janus", + mypy_cmd="{mypy} janus --disallow-untyped-calls --disallow-incomplete-defs --strict", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} janus", + expected_mypy_success=True, + ), + Project( + location="https://github.com/alerta/alerta", + mypy_cmd="{mypy} alerta tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} alerta tests", + deps=["types-PyYAML", "types-setuptools", "types-requests", "types-pytz"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/nolar/kopf", + mypy_cmd="{mypy} kopf", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} kopf", + deps=["types-setuptools", "types-PyYAML"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/davidhalter/parso", + mypy_cmd="{mypy} parso", + pyright_cmd="{pyright} parso", + pyrefly_cmd="{pyrefly} parso", + expected_mypy_success=True, + cost={"pyright": 75}, + ), + Project( + location="https://github.com/konradhalas/dacite", + mypy_cmd="{mypy} dacite", + pyright_cmd="{pyright} dacite", + pyrefly_cmd="{pyrefly} dacite", + expected_mypy_success=True, + ), + Project( + location="https://github.com/ilevkivskyi/com2ann", + mypy_cmd="{mypy} --python-version=3.8 src/com2ann.py src/test_com2ann.py", + pyright_cmd="{pyright} com2ann", + pyrefly_cmd="{pyrefly} src", + expected_mypy_success=True, + ), + Project( + location="https://github.com/srittau/python-htmlgen", + mypy_cmd="{mypy} htmlgen test_htmlgen", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} htmlgen test_htmlgen", + deps=["asserts"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/mitmproxy/mitmproxy", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + expected_mypy_success=True, + ), + Project( + location="https://github.com/jpadilla/pyjwt", + mypy_cmd="{mypy} jwt", + pyright_cmd="{pyright} jwt", + pyrefly_cmd="{pyrefly} jwt", + deps=["cryptography"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/apache/spark", + mypy_cmd="{mypy} --config python/mypy.ini python/pyspark", + pyright_cmd="{pyright} python/pyspark", + pyrefly_cmd="{pyrefly} python/pyspark", + deps=["numpy"], + expected_mypy_success=True, + cost={"mypy": 20, "pyright": 110}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/laowantong/paroxython", + mypy_cmd="{mypy} paroxython", + pyright_cmd="{pyright} paroxython", + pyrefly_cmd="{pyrefly} paroxython", + expected_mypy_success=True, + ), + Project( + location="https://github.com/Akuli/porcupine", + mypy_cmd="{mypy} --config-file= porcupine", + pyright_cmd="{pyright} porcupine", + pyrefly_cmd="{pyrefly} porcupine", + expected_mypy_success=True, + ), + Project( + location="https://github.com/dropbox/mypy-protobuf", + mypy_cmd="{mypy} mypy_protobuf/", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} mypy_protobuf", + deps=["types-protobuf"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/spack/spack", + mypy_cmd="{mypy} -p spack -p llnl", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} lib/spack/llnl lib/spack/spack", + expected_mypy_success=True, + cost={"mypy": 20, "pyright": 100}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/johtso/httpx-caching", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["types-freezegun", "types-mock", "httpx", "anyio", "pytest"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/python-poetry/poetry", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["types-requests"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/awslabs/sockeye", + mypy_cmd=( + "{mypy} --ignore-missing-imports --follow-imports=silent @typechecked-files" + " --no-strict-optional" + ), + pyright_cmd=None, + pyrefly_cmd=None, + deps=["types-PyYAML"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/wntrblm/nox", + mypy_cmd="{mypy} nox", + pyright_cmd="{pyright} nox", + pyrefly_cmd="{pyrefly} nox", + deps=["jinja2", "packaging", "importlib_metadata"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/pandera-dev/pandera", + mypy_cmd="{mypy} pandera tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} pandera tests", + deps=["types-click", "types-PyYAML", "types-setuptools", "types-requests"], + expected_mypy_success=True, + ), + Project( + location="https://gitlab.com/cki-project/cki-lib", + mypy_cmd="{mypy} --strict .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["types-PyYAML", "types-requests"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/python-jsonschema/check-jsonschema", + mypy_cmd="{mypy} src", + pyright_cmd="{pyright} src", + pyrefly_cmd="{pyrefly} src", + deps=["types-jsonschema", "types-requests"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/pybind/pybind11", + mypy_cmd="{mypy} --exclude '^(tests|docs)/' .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} pybind11", + deps=["nox", "rich", "types-setuptools"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/rpdelaney/downforeveryone", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["types-requests", "types-requests", "pytest"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/DataDog/dd-trace-py", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd=None, + deps=[ + "attrs", + "types-six", + "types-setuptools", + "types-docutils", + "types-PyYAML", + "types-protobuf", + "envier", + ], + needs_mypy_plugins=True, + expected_mypy_success=True, + cost={"pyright": 75}, + ), + Project( + location="https://github.com/systemd/mkosi", + mypy_cmd="{mypy} mkosi", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} mkosi", + deps=["cryptography"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/sympy/sympy", + mypy_cmd="{mypy} sympy", + pyright_cmd="{pyright} sympy", + pyrefly_cmd="{pyrefly} sympy", + expected_mypy_success=True, + cost={"mypy": 35, "pyright": 240}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/nion-software/nionutils", + mypy_cmd="{mypy} --strict -p nion.utils", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} nion/utils", + expected_mypy_success=True, + ), + Project( + location="https://github.com/PyCQA/flake8-pyi", + mypy_cmd="{mypy} flake8_pyi", + pyright_cmd="{pyright} flake8_pyi", + pyrefly_cmd="{pyrefly} flake8_pyi", + deps=["types-pyflakes"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/internetarchive/openlibrary", + mypy_cmd="{mypy} openlibrary", + pyright_cmd="{pyright} openlibrary", + pyrefly_cmd="{pyrefly} openlibrary", + deps=[ + "types-PyYAML", + "types-python-dateutil", + "types-requests", + "types-simplejson", + "types-Deprecated", + ], + expected_mypy_success=True, + skip_determinism_check=True, + ), + Project( + location="https://github.com/JohannesBuchner/imagehash", + mypy_cmd="{mypy} imagehash", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} imagehash", + deps=["numpy", "types-Pillow"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/Kalmat/PyWinCtl", + mypy_cmd="{mypy} src/pywinctl", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} src/pywinctl", + deps=["types-setuptools", "types-pywin32", "types-python-xlib"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/mesonbuild/meson", + mypy_cmd="./run_mypy.py --mypy {mypy}", + pyright_cmd=None, + pyrefly_cmd=None, + deps=[ + "types-PyYAML", + "coverage", + "types-chevron", + "types-PyYAML", + "types-tqdm", + ], + expected_mypy_success=True, + ), + Project( + location="https://github.com/aio-libs/aiohttp-devtools", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["aiohttp", "watchfiles", "types-pygments"], + expected_mypy_success=True, + ), + Project( + location="https://github.com/sco1/pylox", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["attrs", "pytest"], + expected_mypy_success=True, + min_python_version=(3, 10), + ), + Project( + location="https://github.com/ppb/ppb-vector", + mypy_cmd="{mypy} ppb_vector tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} ppb_vector tests", + deps=["hypothesis"], + expected_mypy_success=True, + min_python_version=(3, 10), + ), + Project( + location="https://github.com/mkdocs/mkdocs", + mypy_cmd="{mypy} mkdocs", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} mkdocs", + deps=[ + "babel", + "types-Markdown", + "types-pytz", + "types-PyYAML", + "types-setuptools", + "jinja2", + "click", + "watchdog", + "pathspec", + "platformdirs", + "packaging", + ], + expected_mypy_success=True, + ), + Project( + location="https://github.com/pyppeteer/pyppeteer", + mypy_cmd="{mypy} pyppeteer --config-file tox.ini", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} pyppeteer", + install_cmd="{install} .", + ), + Project( + location="https://github.com/pypa/pip", + mypy_cmd="{mypy} src", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} src", + cost={"pyright": 45}, + ), + Project( + location="https://github.com/pytorch/vision", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + cost={"pyright": 50}, + ), + Project( + location="https://github.com/tornadoweb/tornado", + mypy_cmd="{mypy} tornado", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} tornado", + deps=["types-contextvars", "types-pycurl"], + ), + Project( + location="https://github.com/scipy/scipy-stubs", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["numpy", "optype", "packaging", "scipy"], + expected_mypy_success=True, + expected_pyright_success=True, + ), + Project( + location="https://github.com/pycqa/flake8", + mypy_cmd="{mypy} src tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} src tests", + deps=["pytest"], + ), + Project( + location="https://github.com/home-assistant/core", + mypy_cmd="{mypy} homeassistant", + pyright_cmd="{pyright} homeassistant", + pyrefly_cmd="{pyrefly} homeassistant", + deps=[ + "attrs", + "pydantic", + "types-setuptools", + "types-atomicwrites", + "types-certifi", + "types-croniter", + "types-PyYAML", + "types-requests", + "types-python-slugify", + "types-backports", + ], + needs_mypy_plugins=True, + cost={"mypy": 40, "pyright": 240}, + ), + Project( + location="https://github.com/kornia/kornia", + mypy_cmd="{mypy} kornia", + pyright_cmd="{pyright} kornia", + pyrefly_cmd="{pyrefly} kornia", + ), + Project( + location="https://github.com/ibis-project/ibis", + mypy_cmd="{mypy} --ignore-missing-imports ibis", + pyright_cmd="{pyright} ibis", + pyrefly_cmd="{pyrefly} ibis", + deps=[ + "types-setuptools", + "types-requests", + "types-python-dateutil", + "types-pytz", + "SQLAlchemy", + ], + cost={"mypy": 15, "pyright": 60}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/streamlit/streamlit", + mypy_cmd="{mypy} --config-file=lib/mypy.ini lib scripts", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} lib scripts", + deps=[ + "attrs", + "tornado", + "packaging", + "types-toml", + "types-python-dateutil", + "types-setuptools", + "types-protobuf", + "types-pytz", + "types-requests", + "types-cffi", + "click", + "pytest", + ], + cost={"mypy": 10, "pyright": 50}, + ), + Project( + location="https://github.com/dragonchain/dragonchain", + mypy_cmd="{mypy} dragonchain --error-summary", + pyright_cmd="{pyright} dragonchain", + pyrefly_cmd="{pyrefly} dragonchain", + deps=["types-redis", "types-requests"], + ), + Project( + location="https://github.com/mikeshardmind/SinbadCogs", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["attrs", "types-pytz", "types-python-dateutil", "types-PyYAML"], + ), + Project( + location="https://github.com/rotki/rotki", + mypy_cmd="{mypy} rotkehlchen/ tools/data_faker", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} rotkehlchen", + deps=["eth-typing", "types-requests", "types-setuptools"], + cost={"pyright": 60}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/arviz-devs/arviz", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["pytest", "types-setuptools", "types-ujson", "numpy", "xarray"], + cost={"pyright": 45}, + ), + Project( + location="https://github.com/urllib3/urllib3", + mypy_cmd="{mypy} . --exclude setup.py", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=[ + "idna>=2.0.0", + "cryptography>=1.3.4", + "tornado>=6.1", + "pytest", + "trustme==0.9.0", + "types-backports", + "types-requests", + ], + ), + Project( + location="https://github.com/common-workflow-language/schema_salad", + mypy_cmd="MYPYPATH=$MYPYPATH:mypy-stubs {mypy} schema_salad", + pyright_cmd=None, + pyrefly_cmd=None, + install_cmd="{install} $(grep -v mypy mypy-requirements.txt)" + " -r requirements.txt", + expected_mypy_success=True, + supported_platforms=["linux", "darwin"], + ), + Project( + location="https://github.com/common-workflow-language/cwltool", + mypy_cmd="MYPYPATH=$MYPYPATH:mypy-stubs {mypy} cwltool tests/*.py setup.py", + pyright_cmd=None, + pyrefly_cmd=None, + install_cmd="{install} $(grep -v mypy mypy-requirements.txt) -r requirements.txt", + expected_mypy_success=True, + cost={"mypy": 15}, + supported_platforms=["linux", "darwin"], + ), + Project( + location="https://github.com/AlexWaygood/typeshed-stats", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=[ + "attrs", + "cattrs", + "pytest", + "aiohttp", + "pathspec", + "jinja2", + "tomli", + ], + ), + Project( + location="https://github.com/FasterSpeeding/Tanjun", + mypy_cmd="{mypy} tanjun", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} tanjun", + deps=["hikari", "alluka"], + ), + Project( + location="https://github.com/joerick/pyinstrument", + mypy_cmd="{mypy} pyinstrument", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} pyinstrument", + ), + Project( + location="https://github.com/Gobot1234/steam.py", + mypy_cmd="{mypy} steam", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} steam", + deps=["cryptography"], + skip_determinism_check=True, + ), + Project( + location="https://github.com/cpitclaudel/alectryon", + mypy_cmd="{mypy} alectryon.py", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} alectryon.py", + expected_pyrefly_success=True, + ), + Project( + location="https://github.com/yurijmikhalevich/rclip", + mypy_cmd="{mypy} rclip --explicit-package-bases", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} rclip", + deps=["numpy", "types-Pillow", "types-requests", "types-tqdm"], + ), + Project( + location="https://github.com/psycopg/psycopg", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["pytest", "pproxy"], + ), + Project( + location="https://gitlab.com/dkg/python-sop", + mypy_cmd="{mypy} --strict sop", + pyright_cmd="{pyright} sop", + pyrefly_cmd="{pyrefly} sop", + deps=["argcomplete"], + ), + Project( + location="https://github.com/Rapptz/discord.py", + mypy_cmd="{mypy} discord", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} discord", + deps=["types-requests", "types-setuptools", "aiohttp"], + cost={"mypy": 20}, + ), + Project( + location="https://github.com/canonical/cloud-init", + mypy_cmd="{mypy} cloudinit/ tests/ tools/", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} cloudinit tests tools", + deps=[ + "jinja2", + "pytest", + "types-jsonschema", + "types-oauthlib", + "types-pyyaml", + "types-requests", + "types-setuptools", + ], + cost={"mypy": 15, "pyright": 50}, + ), + Project( + location="https://github.com/mongodb/mongo-python-driver", + mypy_cmd="{mypy} bson gridfs tools pymongo", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} bson gridfs tools pymongo", + deps=["types-requests", "types-pyOpenSSL", "cryptography", "certifi"], + ), + Project( + location="https://github.com/artigraph/artigraph", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly}", + deps=["pydantic", "numpy", "pytest"], + needs_mypy_plugins=True, + ), + Project( + location="https://github.com/MaterializeInc/materialize", + mypy_cmd=( + "MYPYPATH=$MYPYPATH:misc/python {mypy} --explicit-package-bases ci misc/python" + ), + pyright_cmd="{pyright}", + pyrefly_cmd=None, + install_cmd="{install} -r ci/builder/requirements.txt", + ), + Project( + location="https://github.com/canonical/operator", + mypy_cmd="{mypy} ops", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} ops", + deps=["types-PyYAML"], + ), + Project( + location="https://github.com/caronc/apprise", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=[ + "types-six", + "types-mock", + "cryptography", + "types-requests", + "types-Markdown", + "pytest", + "certifi", + "babel", + ], + ), + Project( + location="https://github.com/Finistere/antidote", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["pytest"], + ), + Project( + location="https://github.com/cognitedata/Expression", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["pytest"], + ), + Project( + location="https://github.com/pyodide/pyodide", + mypy_cmd="{mypy} src pyodide-build --exclude 'setup.py|^src/tests|conftest.py'", + pyright_cmd="{pyright} src pyodide-build", + pyrefly_cmd="{pyrefly} src", + deps=[ + "packaging", + "types-docutils", + "types-pyyaml", + "types-setuptools", + "numpy", + "pydantic", + ], + needs_mypy_plugins=True, + ), + Project( + location="https://github.com/bokeh/bokeh", + mypy_cmd="{mypy} src release", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} src release", + deps=["types-boto", "tornado", "numpy", "jinja2", "selenium"], + cost={"pyright": 60}, + ), + Project( + location="https://github.com/pandas-dev/pandas-stubs", + mypy_cmd="{mypy} pandas-stubs tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} pandas-stubs tests", + deps=[ + "numpy", + "types-pytz", + "matplotlib", + "xarray", + "pyarrow", + "jinja2", + "pytest", + "SQLAlchemy", + ], + expected_pyright_success=True, + cost={"mypy": 40, "pyright": 75}, + skip_determinism_check=True, + ), + # mypy checks w/ windows platform + Project( + location="https://github.com/enthought/comtypes", + mypy_cmd="{mypy} comtypes --platform win32", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} comtypes", + deps=["numpy"], + skip_determinism_check=True, + ), + Project( + location="https://github.com/mit-ll-responsible-ai/hydra-zen", + mypy_cmd="{mypy} src tests/annotations/mypy_checks.py", + pyright_cmd="{pyright} tests/annotations src", + pyrefly_cmd="{pyrefly} tests/annotations src", + deps=["pydantic", "beartype", "hydra-core"], + cost={"mypy": 10}, + ), + Project( + location="https://github.com/Toufool/AutoSplit", + mypy_cmd="{mypy} src", + pyright_cmd="{pyright} src", + pyrefly_cmd="{pyrefly} src", + deps=[ + "certifi", + "ImageHash", + "numpy", + "packaging", + "PyWinCtl", + "PySide6-Essentials", + "types-D3DShot", + "types-keyboard", + "types-Pillow", + "types-psutil", + "types-PyAutoGUI", + "types-pyinstaller", + "types-pywin32", + "types-requests", + "types-toml", + ], + ), + Project( + location="https://github.com/Avasam/speedrun.com_global_scoreboard_webapp", + mypy_cmd="{mypy} backend", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} backend", + deps=[ + "Flask", + "PyJWT", + "requests-cache", + "types-Flask-SQLAlchemy", + "types-httplib2", + "types-requests", + ], + ), + Project( + location="https://github.com/pwndbg/pwndbg", + mypy_cmd="{mypy} pwndbg", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} pwndbg", + deps=["types-gdb"], + cost={"mypy": 10, "pyright": 75}, + skip_determinism_check=True, + ), + Project( + location="https://github.com/keithasaurus/koda-validate", + mypy_cmd="{mypy} koda_validate --strict", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} koda_validate", + deps=["koda"], + ), + Project( + location="https://github.com/python/cpython", + mypy_cmd="{mypy} --config-file Tools/clinic/mypy.ini", + pyright_cmd=None, + pyrefly_cmd=None, + name_override="CPython (Argument Clinic)", + ), + Project( + location="https://github.com/python/cpython", + mypy_cmd="{mypy} --config-file Tools/cases_generator/mypy.ini", + pyright_cmd=None, + pyrefly_cmd=None, + name_override="CPython (cases_generator)", + ), + Project( + location="https://github.com/python/cpython", + mypy_cmd="{mypy} --config-file Tools/peg_generator/mypy.ini", + pyright_cmd=None, + pyrefly_cmd=None, + deps=["types-setuptools", "types-psutil"], + name_override="CPython (peg_generator)", + ), + Project( + location="https://github.com/python-trio/trio", + mypy_cmd="{mypy} src/trio", + pyright_cmd="{pyright} src/trio", + pyrefly_cmd="{pyrefly} src/trio", + deps=[ + "types-pyOpenSSL", + "types-cffi", + "attrs", + "outcome", + "exceptiongroup", + "pytest", + "sniffio", + ], + ), + Project( + location="https://github.com/pypa/setuptools", + mypy_cmd="{mypy} setuptools pkg_resources", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} setuptools pkg_resources", + deps=[ + "pytest", + "filelock", + "ini2toml", + "packaging", + "tomli", + "tomli-w", + ], + ), + Project( + location="https://github.com/detachhead/pytest-robotframework", + mypy_cmd="{mypy} -p pytest_robotframework", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} pytest_robotframework", + ), + Project( + location="https://github.com/mhammond/pywin32", + mypy_cmd="{mypy} .", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} .", + deps=["types-pywin32", "types-regex", "types-setuptools"], + ), + Project( + location="https://github.com/beartype/beartype", + mypy_cmd="{mypy} beartype", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} beartype", + ), + Project( + location="https://github.com/colour-science/colour", + mypy_cmd="{mypy} colour", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} colour", + deps=["numpy", "pytest", "matplotlib", "pandas-stubs"], + cost={"mypy": 45, "pyright": 180}, + ), + Project( + location="https://github.com/vega/altair", + mypy_cmd="{mypy} altair tests", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} altair tests", + deps=[ + "types-jsonschema", + "pandas-stubs", + "pytest", + "narwhals", + "packaging", + "jinja2", + ], + ), + Project( + location="https://github.com/hydpy-dev/hydpy", + mypy_cmd="{mypy} hydpy", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} hydpy", + deps=["numpy", "types-docutils"], + ), + Project( + location="https://github.com/static-frame/static-frame", + mypy_cmd="{mypy} static_frame", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly} static_frame", + deps=["numpy", "arraykit"], + ), + Project( + location="https://github.com/Instagram/Fixit", + mypy_cmd="{mypy} src/fixit", + pyright_cmd="{pyright} src/fixit", + pyrefly_cmd="{pyrefly} src/fixit", + deps=[ + "click", + "libcst", + "moreorless", + "packaging", + "tomli", + "trailrunner", + "attribution", + "black", + "flake8", + "flake8-bugbear", + "mypy", + "ufmt", + "usort", + "pygls[ws]", + "rich", + ], + ), + Project( + location="https://github.com/narwhals-dev/narwhals", + mypy_cmd="{mypy} narwhals tests", + pyright_cmd="{pyright} narwhals tests", + pyrefly_cmd="{pyrefly} narwhals tests", + deps=[ + "pandas-stubs", + "pyarrow-stubs", + "sqlframe", + "polars", + "hypothesis", + "pytest", + "uv", + ], + skip_determinism_check=True, + ), + Project( + location="https://github.com/pola-rs/polars", + mypy_cmd="{mypy} py-polars/src/polars", + pyright_cmd="{pyright} py-polars/src/polars", + pyrefly_cmd="{pyrefly} py-polars/src/polars", + deps=[ + "pandas-stubs", + "boto3-stubs", + "google-auth-stubs", + "polars-cloud", + "numpy", + "pyarrow", + "pydantic", + "tzdata", + "sqlalchemy", + "adbc-driver-manager", + "adbc-driver-sqlite", + "aiosqlite", + "connectorx", + "kuzu", + "cloudpickle", + "fsspec", + "fastexcel", + "openpyxl", + "xlsx2csv", + "xlsxwriter", + "deltalake", + "zstandard", + "altair", + "great-tables", + "gevent", + "matplotlib", + "hypothesis", + "orjson", + ], + ), + Project( + location="https://github.com/pytorch/helion", + mypy_cmd="{mypy}", + pyright_cmd="{pyright}", + pyrefly_cmd="{pyrefly}", + deps=[ + "torch", + "{repo}'[dev,surrogate]'", + ], + expected_pyrefly_success=False, + ), + ]