mirror of
https://github.com/mongodb/mongo.git
synced 2024-12-01 09:32:32 +01:00
211 lines
7.5 KiB
Python
Executable File
211 lines
7.5 KiB
Python
Executable File
#!/usr/bin/env python2
|
|
"""Extensible script to run one or more Python Linters across a subset of files in parallel."""
|
|
from __future__ import absolute_import
|
|
from __future__ import print_function
|
|
|
|
import argparse
|
|
import logging
|
|
import os
|
|
import sys
|
|
import threading
|
|
from abc import ABCMeta, abstractmethod
|
|
from typing import Any, Dict, List
|
|
|
|
# Get relative imports to work when the package is not installed on the PYTHONPATH.
|
|
if __name__ == "__main__" and __package__ is None:
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__)))))
|
|
|
|
from buildscripts.linter import base # pylint: disable=wrong-import-position
|
|
from buildscripts.linter import git # pylint: disable=wrong-import-position
|
|
from buildscripts.linter import mypy # pylint: disable=wrong-import-position
|
|
from buildscripts.linter import parallel # pylint: disable=wrong-import-position
|
|
from buildscripts.linter import pydocstyle # pylint: disable=wrong-import-position
|
|
from buildscripts.linter import pylint # pylint: disable=wrong-import-position
|
|
from buildscripts.linter import runner # pylint: disable=wrong-import-position
|
|
from buildscripts.linter import yapf # pylint: disable=wrong-import-position
|
|
|
|
# List of supported linters
|
|
_LINTERS = [
|
|
yapf.YapfLinter(),
|
|
pylint.PyLintLinter(),
|
|
pydocstyle.PyDocstyleLinter(),
|
|
mypy.MypyLinter(),
|
|
]
|
|
|
|
|
|
def get_py_linter(linter_filter):
|
|
# type: (str) -> List[base.LinterBase]
|
|
"""
|
|
Get a list of linters to use.
|
|
|
|
'all' or None - select all linters
|
|
'a,b,c' - a comma delimited list is describes a list of linters to choose
|
|
"""
|
|
if linter_filter is None or linter_filter == "all":
|
|
return _LINTERS
|
|
|
|
linter_list = linter_filter.split(",")
|
|
|
|
linter_candidates = [linter for linter in _LINTERS if linter.cmd_name in linter_list]
|
|
|
|
if len(linter_candidates) == 0:
|
|
raise ValueError("No linters found for filter '%s'" % (linter_filter))
|
|
|
|
return linter_candidates
|
|
|
|
|
|
def is_interesting_file(file_name):
|
|
# type: (str) -> bool
|
|
""""Return true if this file should be checked."""
|
|
return file_name.endswith(".py") and (file_name.startswith("buildscripts/idl") or
|
|
file_name.startswith("buildscripts/linter") or
|
|
file_name.startswith("buildscripts/pylinters.py"))
|
|
|
|
|
|
def _get_build_dir():
|
|
# type: () -> str
|
|
"""Get the location of the scons' build directory in case we need to download clang-format."""
|
|
return os.path.join(git.get_base_dir(), "build")
|
|
|
|
|
|
def _lint_files(linters, config_dict, file_names):
|
|
# type: (str, Dict[str, str], List[str]) -> None
|
|
"""Lint a list of files with clang-format."""
|
|
linter_list = get_py_linter(linters)
|
|
|
|
lint_runner = runner.LintRunner()
|
|
|
|
linter_instances = runner.find_linters(linter_list, config_dict)
|
|
if not linter_instances:
|
|
sys.exit(1)
|
|
|
|
for linter in linter_instances:
|
|
run_fix = lambda param1: lint_runner.run_lint(linter, param1) # pylint: disable=cell-var-from-loop
|
|
lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_fix)
|
|
|
|
if not lint_clean:
|
|
print("ERROR: Code Style does not match coding style")
|
|
sys.exit(1)
|
|
|
|
|
|
def lint_patch(linters, config_dict, file_name):
|
|
# type: (str, Dict[str, str], List[str]) -> None
|
|
"""Lint patch command entry point."""
|
|
file_names = git.get_files_to_check_from_patch(file_name, is_interesting_file)
|
|
|
|
# Patch may have files that we do not want to check which is fine
|
|
if file_names:
|
|
_lint_files(linters, config_dict, file_names)
|
|
|
|
|
|
def lint(linters, config_dict, file_names):
|
|
# type: (str, Dict[str, str], List[str]) -> None
|
|
"""Lint files command entry point."""
|
|
all_file_names = git.get_files_to_check(file_names, is_interesting_file)
|
|
|
|
_lint_files(linters, config_dict, all_file_names)
|
|
|
|
|
|
def lint_all(linters, config_dict, file_names):
|
|
# type: (str, Dict[str, str], List[str]) -> None
|
|
# pylint: disable=unused-argument
|
|
"""Lint files command entry point based on working tree."""
|
|
all_file_names = git.get_files_to_check_working_tree(is_interesting_file)
|
|
|
|
_lint_files(linters, config_dict, all_file_names)
|
|
|
|
|
|
def _fix_files(linters, config_dict, file_names):
|
|
# type: (str, Dict[str, str], List[str]) -> None
|
|
"""Fix a list of files with linters if possible."""
|
|
linter_list = get_py_linter(linters)
|
|
|
|
# Get a list of linters which return a valid command for get_fix_cmd()
|
|
fix_list = [fixer for fixer in linter_list if fixer.get_fix_cmd_args("ignore")]
|
|
|
|
if len(fix_list) == 0:
|
|
raise ValueError("Cannot find any linters '%s' that support fixing." % (linters))
|
|
|
|
lint_runner = runner.LintRunner()
|
|
|
|
linter_instances = runner.find_linters(fix_list, config_dict)
|
|
if not linter_instances:
|
|
sys.exit(1)
|
|
|
|
for linter in linter_instances:
|
|
run_linter = lambda param1: lint_runner.run(linter.cmd_path + linter.linter.get_fix_cmd_args(param1)) # pylint: disable=cell-var-from-loop
|
|
|
|
lint_clean = parallel.parallel_process([os.path.abspath(f) for f in file_names], run_linter)
|
|
|
|
if not lint_clean:
|
|
print("ERROR: Code Style does not match coding style")
|
|
sys.exit(1)
|
|
|
|
|
|
def fix_func(linters, config_dict, file_names):
|
|
# type: (str, Dict[str, str], List[str]) -> None
|
|
"""Fix files command entry point."""
|
|
all_file_names = git.get_files_to_check(file_names, is_interesting_file)
|
|
|
|
_fix_files(linters, config_dict, all_file_names)
|
|
|
|
|
|
def main():
|
|
# type: () -> None
|
|
"""Main entry point."""
|
|
|
|
parser = argparse.ArgumentParser(description='PyLinter frontend.')
|
|
|
|
linters = get_py_linter(None)
|
|
|
|
dest_prefix = "linter_"
|
|
for linter1 in linters:
|
|
msg = 'Path to linter %s' % (linter1.cmd_name)
|
|
parser.add_argument(
|
|
'--' + linter1.cmd_name, type=str, help=msg, dest=dest_prefix + linter1.cmd_name)
|
|
|
|
parser.add_argument(
|
|
'--linters',
|
|
type=str,
|
|
help="Comma separated list of filters to use, defaults to 'all'",
|
|
default="all")
|
|
|
|
parser.add_argument('-v', "--verbose", action='store_true', help="Enable verbose logging")
|
|
|
|
sub = parser.add_subparsers(title="Linter subcommands", help="sub-command help")
|
|
|
|
parser_lint = sub.add_parser('lint', help='Lint only Git files')
|
|
parser_lint.add_argument("file_names", nargs="*", help="Globs of files to check")
|
|
parser_lint.set_defaults(func=lint)
|
|
|
|
parser_lint_all = sub.add_parser('lint-all', help='Lint All files')
|
|
parser_lint_all.add_argument("file_names", nargs="*", help="Globs of files to check")
|
|
parser_lint_all.set_defaults(func=lint_all)
|
|
|
|
parser_lint_patch = sub.add_parser('lint-patch', help='Lint the files in a patch')
|
|
parser_lint_patch.add_argument("file_names", nargs="*", help="Globs of files to check")
|
|
parser_lint_patch.set_defaults(func=lint_patch)
|
|
|
|
parser_fix = sub.add_parser('fix', help='Fix files if possible')
|
|
parser_fix.add_argument("file_names", nargs="*", help="Globs of files to check")
|
|
parser_fix.set_defaults(func=fix_func)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Create a dictionary of linter locations if the user needs to override the location of a
|
|
# linter. This is common for mypy on Windows for instance.
|
|
config_dict = {}
|
|
for key in args.__dict__:
|
|
if key.startswith("linter_"):
|
|
name = key.replace(dest_prefix, "")
|
|
config_dict[name] = args.__dict__[key]
|
|
|
|
if args.verbose:
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
|
|
args.func(args.linters, config_dict, args.file_names)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|