#!/usr/bin/env python3 """ESLint module. Will download a prebuilt ESLint binary if necessary (i.e. it isn't installed, isn't in the current path, or is the wrong version). It works in much the same way as clang_format.py. In lint mode, it will lint the files or directory paths passed. In lint-patch mode, for upload.py, it will see if there are any candidate files in the supplied patch. Fix mode will run ESLint with the --fix option, and that will update the files with missing semicolons and similar repairable issues. There is also a -d mode that assumes you only want to run one copy of ESLint per file / directory parameter supplied. This lets ESLint search for candidate files to lint. """ import logging import os import platform import shutil import string import subprocess import sys import tarfile import tempfile import threading import urllib.error import urllib.parse import urllib.request from distutils import spawn from optparse import OptionParser from typing import Optional import structlog # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) # pylint: disable=wrong-import-position from buildscripts.linter import git, parallel from buildscripts.linter.filediff import gather_changed_files_for_lint # pylint: enable=wrong-import-position ############################################################################## # # Constants for ESLint # # # Expected version of ESLint. # If you want to update the version, please refer to `buildscripts/eslint/README.md` ESLINT_VERSION = "8.28.0" # Name of ESLint as a binary. ESLINT_PROGNAME = "eslint" # Arch of running system ARCH = platform.machine() if platform.machine() != "aarch64" else "arm64" # URL location of our provided ESLint binaries. ESLINT_HTTP_LINUX_CACHE = ( "https://s3.amazonaws.com/boxes.10gen.com/build/eslint-" + ESLINT_VERSION + "-linux-" + ARCH + ".tar.gz" ) ESLINT_HTTP_DARWIN_CACHE = ( "https://s3.amazonaws.com/boxes.10gen.com/build/eslint-" + ESLINT_VERSION + "-darwin.tar.gz" ) # Path in the tarball to the ESLint binary. ESLINT_SOURCE_TAR_BASE = string.Template(ESLINT_PROGNAME + "-$operating_system-$arch") LOGGER = structlog.get_logger(__name__) def callo(args): """Call a program, and capture its output.""" return subprocess.check_output(args).decode("utf-8") def extract_eslint(tar_path, target_file): """Extract ESLint tar file.""" tarfp = tarfile.open(tar_path) for name in tarfp.getnames(): if name == target_file: tarfp.extract(name) tarfp.close() def get_eslint_from_cache(dest_file, operating_system, arch): """Get ESLint binary from mongodb's cache.""" # Get URL if operating_system == "Linux": url = ESLINT_HTTP_LINUX_CACHE elif operating_system == "Darwin": url = ESLINT_HTTP_DARWIN_CACHE else: raise ValueError("ESLint is not available as a binary for " + operating_system) dest_dir = tempfile.gettempdir() temp_tar_file = os.path.join(dest_dir, "temp.tar.gz") # Download the file print("Downloading ESLint %s from %s, saving to %s" % (ESLINT_VERSION, url, temp_tar_file)) urllib.request.urlretrieve(url, temp_tar_file) print("Extracting ESLint %s to %s" % (ESLINT_VERSION, dest_file)) eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute( operating_system=operating_system, arch=arch ) extract_eslint(temp_tar_file, eslint_distfile) shutil.move(eslint_distfile, dest_file) class ESLint(object): """Class encapsulates finding a suitable copy of ESLint, and linting an individual file.""" def __init__(self, path, cache_dir): """Initialize ESLint.""" eslint_progname = ESLINT_PROGNAME # Initialize ESLint configuration information if sys.platform.startswith("linux"): self.arch = ARCH self.tar_path = None elif sys.platform == "darwin": self.arch = "x86_64" self.tar_path = None self.path = None # Find ESLint now if path is not None: if os.path.isfile(path): self.path = path else: print("WARNING: Could not find ESLint at %s" % path) # Check the environment variable if "MONGO_ESLINT" in os.environ: self.path = os.environ["MONGO_ESLINT"] if self.path and not self._validate_version(warn=True): self.path = None # Check the user's PATH environment variable now if self.path is None: self.path = spawn.find_executable(eslint_progname) if self.path and not self._validate_version(warn=True): self.path = None # Have not found it yet, download it from the web if self.path is None: if not os.path.isdir(cache_dir): os.makedirs(cache_dir) self.path = os.path.join(cache_dir, eslint_progname) if os.path.isfile(self.path) and not self._validate_version(warn=True): print( "WARNING: removing ESLint from %s to download the correct version" % self.path ) os.remove(self.path) if not os.path.isfile(self.path): if sys.platform.startswith("linux"): get_eslint_from_cache(self.path, "Linux", self.arch) elif sys.platform == "darwin": get_eslint_from_cache(self.path, "Darwin", self.arch) else: print( "ERROR: eslint.py does not support downloading ESLint " "on this platform, please install ESLint " + ESLINT_VERSION ) # Validate we have the correct version if not self._validate_version(): raise ValueError("correct version of ESLint was not found.") self.print_lock = threading.Lock() def _validate_version(self, warn=False): """Validate ESLint is the expected version.""" esl_version = callo([self.path, "--version"]).rstrip() # Ignore the leading v in the version string. if ESLINT_VERSION == esl_version[1:]: return True if warn: print( "WARNING: ESLint found in path %s, but the version is incorrect: %s" % (self.path, esl_version) ) return False def _lint(self, file_name, print_diff): """Check the specified file for linting errors.""" # ESLint returns non-zero on a linting error. That's all we care about # so only enter the printing logic if we have an error. try: callo([self.path, "-f", "unix", file_name]) except subprocess.CalledProcessError as err: if print_diff: # Take a lock to ensure error messages do not get mixed when printed to the screen with self.print_lock: print("ERROR: ESLint found errors in " + file_name) print(err.output) return False return True def lint(self, file_name): """Check the specified file has no linting errors.""" return self._lint(file_name, print_diff=True) def autofix(self, file_name): """Run ESLint in fix mode.""" return not subprocess.call([self.path, "--fix", file_name]) def is_interesting_file(file_name): """Return true if this file should be checked.""" return ( file_name.startswith("src/mongo") or file_name.startswith("jstests") ) and file_name.endswith(".js") def _get_build_dir(): """Get the location of the scons build directory in case we need to download ESLint.""" return os.path.join(git.get_base_dir(), "build") def _lint_files(eslint, files): """Lint a list of files with ESLint.""" eslint = ESLint(eslint, _get_build_dir()) print("Running ESLint %s at %s" % (ESLINT_VERSION, eslint.path)) lint_clean = parallel.parallel_process([os.path.abspath(f) for f in files], eslint.lint) if not lint_clean: print( "ERROR: ESLint found errors. Run ESLint manually to see errors in " "files that were skipped" ) sys.exit(1) return True def lint_patch(eslint, infile): """Lint patch command entry point.""" files = git.get_files_to_check_from_patch(infile, is_interesting_file) # Patch may have files that we do not want to check which is fine if files: return _lint_files(eslint, files) return True def lint_git_diff(eslint: Optional[str]) -> bool: """ Lint the files that have changes since the last git commit. :param eslint: Path to eslint command. :return: True if lint was successful. """ files = gather_changed_files_for_lint(is_interesting_file) # Patch may have files that we do not want to check which is fine if files: return _lint_files(eslint, files) return True def lint(eslint, dirmode, glob): """Lint files command entry point.""" if dirmode and glob: files = glob else: files = git.get_files_to_check(glob, is_interesting_file) _lint_files(eslint, files) return True def _autofix_files(eslint, files): """Auto-fix the specified files with ESLint.""" eslint = ESLint(eslint, _get_build_dir()) print("Running ESLint %s at %s" % (ESLINT_VERSION, eslint.path)) autofix_clean = parallel.parallel_process([os.path.abspath(f) for f in files], eslint.autofix) if not autofix_clean: print("ERROR: failed to auto-fix files") return False return True def autofix_func(eslint, dirmode, glob): """Auto-fix files command entry point.""" if dirmode: files = glob else: files = git.get_files_to_check(glob, is_interesting_file) return _autofix_files(eslint, files) def main(): """Execute Main entry point.""" success = False usage = "%prog [-e ] [-d] lint|lint-patch|fix [glob patterns] " description = ( "The script will try to find ESLint version %s on your system and run it. " "If it won't find the version it will try to download it and then run it. " "Commands description: lint runs ESLint on provided patterns or all .js " "files under `jstests/` and `src/mongo`; " "lint-patch runs ESLint against .js files modified in the " "provided patch file (for upload.py); " "fix runs ESLint with --fix on provided patterns " "or files under jstests/ and src/mongo." % ESLINT_VERSION ) epilog = "*Unless you specify -d a separate ESLint process will be launched for every file" parser = OptionParser(usage=usage, description=description, epilog=epilog) parser.add_option( "-e", "--eslint", type="string", dest="eslint", help="Fully qualified path to eslint executable", ) parser.add_option( "-d", "--dirmode", action="store_true", default=True, dest="dirmode", help="Considers the glob patterns as directories and runs ESLint process " "against each pattern", ) (options, args) = parser.parse_args(args=sys.argv) logging.basicConfig(stream=sys.stdout, level=logging.INFO) structlog.configure(logger_factory=structlog.stdlib.LoggerFactory()) if len(args) > 1: command = args[1] searchlist = args[2:] if not searchlist: searchlist = ["jstests/", "src/mongo/"] if command == "lint": success = lint(options.eslint, options.dirmode, searchlist) elif command == "lint-patch": if not args[2:]: success = False print("You must provide the patch's fully qualified file name with lint-patch") else: success = lint_patch(options.eslint, searchlist) elif command == "lint-git-diff": success = lint_git_diff(options.eslint) elif command == "fix": success = autofix_func(options.eslint, options.dirmode, searchlist) else: parser.print_help() else: parser.print_help() sys.exit(0 if success else 1) if __name__ == "__main__": main()