diff --git a/.clang-format b/.clang-format index 40241c181e1..0261864a922 100644 --- a/.clang-format +++ b/.clang-format @@ -1,11 +1,10 @@ +--- Language: Cpp -# BasedOnStyle: Google -# --- AccessModifierOffset: -4 AlignAfterOpenBracket: Align AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false -AlignEscapedNewlinesLeft: true +AlignEscapedNewlines: Left AlignOperands: false AlignTrailingComments: true AllowAllParametersOfDeclarationOnNextLine: true @@ -17,10 +16,10 @@ AllowShortLoopsOnASingleLine: false AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: true -AlwaysBreakTemplateDeclarations: true +AlwaysBreakTemplateDeclarations: Yes BinPackArguments: false BinPackParameters: false -BraceWrapping: +BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false @@ -29,15 +28,25 @@ BraceWrapping: AfterObjCDeclaration: false AfterStruct: false AfterUnion: false + AfterExternBlock: false BeforeCatch: false BeforeElse: false IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true BreakBeforeBinaryOperators: None BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true ColumnLimit: 100 CommentPragmas: '' +CompactNamespaces: false ConstructorInitializerAllOnOneLineOrOnePerLine: true ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 @@ -45,33 +54,55 @@ Cpp11BracedListStyle: true DerivePointerAlignment: false DisableFormat: false ExperimentalAutoDetectBinPacking: false -ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] -IncludeCategories: +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: - Regex: '.*' Priority: 1 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + - Regex: '.*' + Priority: 1 +IncludeIsMainRegex: '(Test)?$' IndentCaseLabels: true +IndentPPDirectives: None IndentWidth: 4 IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: true MacroBlockBegin: '' MacroBlockEnd: '' MaxEmptyLinesToKeep: 2 NamespaceIndentation: None +ObjCBinPackProtocolList: Auto ObjCBlockIndentWidth: 2 ObjCSpaceAfterProperty: false ObjCSpaceBeforeProtocolList: false +PenaltyBreakAssignment: 2 PenaltyBreakBeforeFirstCallParameter: 19 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left ReflowComments: true SortIncludes: true +SortUsingDeclarations: true SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 2 SpacesInAngles: false @@ -82,16 +113,15 @@ SpacesInSquareBrackets: false Standard: Cpp11 TabWidth: 4 UseTab: Never ---- +... +--- Language: JavaScript -# BasedOnStyle: Google -# --- AccessModifierOffset: -1 AlignAfterOpenBracket: Align AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false -AlignEscapedNewlinesLeft: true +AlignEscapedNewlines: Left AlignOperands: false AlignTrailingComments: true AllowAllParametersOfDeclarationOnNextLine: true @@ -103,10 +133,10 @@ AllowShortLoopsOnASingleLine: false AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: true -AlwaysBreakTemplateDeclarations: true +AlwaysBreakTemplateDeclarations: Yes BinPackArguments: false BinPackParameters: false -BraceWrapping: +BraceWrapping: AfterClass: false AfterControlStatement: false AfterEnum: false @@ -115,15 +145,25 @@ BraceWrapping: AfterObjCDeclaration: false AfterStruct: false AfterUnion: false + AfterExternBlock: false BeforeCatch: false BeforeElse: false IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true BreakBeforeBinaryOperators: None BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true ColumnLimit: 100 CommentPragmas: '' +CompactNamespaces: false ConstructorInitializerAllOnOneLineOrOnePerLine: true ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 @@ -131,33 +171,55 @@ Cpp11BracedListStyle: true DerivePointerAlignment: true DisableFormat: false ExperimentalAutoDetectBinPacking: false -ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] -IncludeCategories: +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: - Regex: '.*' Priority: 1 + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + - Regex: '.*' + Priority: 1 +IncludeIsMainRegex: '(Test)?$' IndentCaseLabels: true +IndentPPDirectives: None IndentWidth: 4 IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: false MacroBlockBegin: '' MacroBlockEnd: '' MaxEmptyLinesToKeep: 1 NamespaceIndentation: None +ObjCBinPackProtocolList: Auto ObjCBlockIndentWidth: 2 ObjCSpaceAfterProperty: false ObjCSpaceBeforeProtocolList: false +PenaltyBreakAssignment: 2 PenaltyBreakBeforeFirstCallParameter: 1 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left ReflowComments: true SortIncludes: true +SortUsingDeclarations: true SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 2 SpacesInAngles: false @@ -168,3 +230,5 @@ SpacesInSquareBrackets: false Standard: Auto TabWidth: 4 UseTab: Never +... + diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py index 4fe639b3c10..6acdcaff192 100755 --- a/buildscripts/clang_format.py +++ b/buildscripts/clang_format.py @@ -10,6 +10,7 @@ import difflib import glob +from io import StringIO import os import re import shutil @@ -41,17 +42,20 @@ from buildscripts.linter import parallel # pylint: disable=wrong-import-positio # # Expected version of clang-format -CLANG_FORMAT_VERSION = "3.8.0" -CLANG_FORMAT_SHORT_VERSION = "3.8" +CLANG_FORMAT_VERSION = "7.0.1" +CLANG_FORMAT_SHORT_VERSION = "7.0" +CLANG_FORMAT_SHORTER_VERSION = "70" # Name of clang-format as a binary CLANG_FORMAT_PROGNAME = "clang-format" # URL location of the "cached" copy of clang-format to download # for users which do not have clang-format installed -CLANG_FORMAT_HTTP_LINUX_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/clang-format-3.8-rhel55.tar.gz" +CLANG_FORMAT_HTTP_LINUX_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/clang-format-7.0.1-rhel70.tar.gz" -CLANG_FORMAT_HTTP_DARWIN_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/clang%2Bllvm-3.8.0-x86_64-apple-darwin.tar.xz" +CLANG_FORMAT_HTTP_DARWIN_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/clang-format-7.0.1-x86_64-apple-darwin.tar.gz" + +CLANG_FORMAT_TOOLCHAIN_PATH = "/opt/mongodbtoolchain/v3/bin/clang-format" # Path in the tarball to the clang-format binary CLANG_FORMAT_SOURCE_TAR_BASE = string.Template("clang+llvm-$version-$tar_path/bin/" + @@ -59,9 +63,9 @@ CLANG_FORMAT_SOURCE_TAR_BASE = string.Template("clang+llvm-$version-$tar_path/bi ############################################################################## -def callo(args): +def callo(args, **kwargs): """Call a program, and capture its output.""" - return subprocess.check_output(args).decode('utf-8') + return subprocess.check_output(args, **kwargs).decode('utf-8') def get_tar_path(version, tar_path): @@ -129,7 +133,8 @@ def get_clang_format_from_linux_cache(dest_file): class ClangFormat(object): """ClangFormat class.""" - def __init__(self, path, cache_dir): # pylint: disable=too-many-branches + def __init__(self, path, cache_dir): + # pylint: disable=too-many-branches,too-many-statements """Initialize ClangFormat.""" self.path = None clang_format_progname_ext = "" @@ -158,6 +163,7 @@ class ClangFormat(object): programs = [ CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_VERSION, CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_SHORT_VERSION, + CLANG_FORMAT_PROGNAME + CLANG_FORMAT_SHORTER_VERSION, CLANG_FORMAT_PROGNAME, ] @@ -165,18 +171,26 @@ class ClangFormat(object): for i, _ in enumerate(programs): programs[i] += '.exe' - for program in programs: - self.path = spawn.find_executable(program) - - if self.path: + # Check for the binary in the expected toolchain directory on non-windows systems. + if sys.platform != "win32": + if os.path.exists(CLANG_FORMAT_TOOLCHAIN_PATH): + self.path = CLANG_FORMAT_TOOLCHAIN_PATH if not self._validate_version(): self.path = None - else: - break + + if self.path is None: + for program in programs: + self.path = spawn.find_executable(program) + + if self.path: + if not self._validate_version(): + self.path = None + else: + break # If Windows, try to grab it from Program Files # Check both native Program Files and WOW64 version - if sys.platform == "win32": + if self.path is None and sys.platform == "win32": programfiles = [ os.environ["ProgramFiles"], os.environ["ProgramFiles(x86)"], @@ -233,8 +247,13 @@ class ClangFormat(object): with open(file_name, 'rb') as original_text: original_file = original_text.read().decode('utf-8') - # Get formatted file as clang-format would format the file - formatted_file = callo([self.path, "--style=file", file_name]) + original_text.seek(0) + + # Get formatted file as clang-format would format the file + formatted_file = callo([ + self.path, "--assume-filename=" + + (file_name if not file_name.endswith(".h") else file_name + "pp"), "--style=file" + ], stdin=original_text) if original_file != formatted_file: if print_diff: @@ -245,8 +264,7 @@ class ClangFormat(object): # Take a lock to ensure diffs do not get mixed when printed to the screen with self.print_lock: print("ERROR: Found diff for " + file_name) - print("To fix formatting errors, run %s --style=file -i %s" % (self.path, - file_name)) + print("To fix formatting errors, run `buildscripts/clang-format.py format`") for line in result: print(line.rstrip()) @@ -264,7 +282,25 @@ class ClangFormat(object): return True # Update the file with clang-format - formatted = not subprocess.call([self.path, "--style=file", "-i", file_name]) + # We have to tell `clang-format` to format on standard input due to its file type + # determiner. `--assume-filename` doesn't work directly on files, but only on standard + # input. Thus we have to open the file as the subprocess's standard input. Then we record + # that formatted standard output back into the file. We can't use the `-i` option, due to + # the fact that `clang-format` believes that many of our C++ headers are Objective-C code. + formatted = True + with open(file_name, 'rb') as source_stream: + try: + reformatted_text = subprocess.check_output([ + self.path, "--assume-filename=" + + (file_name if not file_name.endswith(".h") else file_name + "pp"), + "--style=file" + ], stdin=source_stream) + except subprocess.CalledProcessError: + formatted = False + + if formatted: + with open(file_name, "wb") as output_stream: + output_stream.write(reformatted_text) # Version 3.8 generates files like foo.cpp~RF83372177.TMP when it formats foo.cpp # on Windows, we must clean these up @@ -281,8 +317,8 @@ FILES_RE = re.compile('\\.(h|hpp|ipp|cpp|js)$') def is_interesting_file(file_name): """Return true if this file should be checked.""" - return ((file_name.startswith("jstests") or file_name.startswith("src")) - and not file_name.startswith("src/third_party/") + return (file_name.startswith("jstests") + or file_name.startswith("src") and not file_name.startswith("src/third_party/") and not file_name.startswith("src/mongo/gotools/")) and FILES_RE.search(file_name) @@ -303,7 +339,7 @@ def _lint_files(clang_format, files): lint_clean = parallel.parallel_process([os.path.abspath(f) for f in files], clang_format.lint) if not lint_clean: - print("ERROR: Code Style does not match coding style") + print("ERROR: Source code does not match required source formatting style") sys.exit(1) @@ -421,22 +457,22 @@ def reformat_branch( # pylint: disable=too-many-branches,too-many-locals,too-ma % (new_branch, new_branch)) commits = get_list_from_lines( - repo.log(["--reverse", "--pretty=format:%H", - "%s..HEAD" % commit_prior_to_reformat])) + repo.git_log([ + "--reverse", "--no-show-signature", "--pretty=format:%H", + "%s..HEAD" % commit_prior_to_reformat + ])) previous_commit_base = commit_after_reformat - files_match = re.compile('\\.(h|cpp|js)$') - # Go through all the commits the user made on the local branch and migrate to a new branch # that is based on post_reformat commits instead for commit_hash in commits: - repo.checkout(["--quiet", commit_hash]) + repo.git_checkout(["--quiet", commit_hash]) deleted_files = [] # Format each of the files by checking out just a single commit from the user's branch - commit_files = get_list_from_lines(repo.diff(["HEAD~", "--name-only"])) + commit_files = get_list_from_lines(repo.git_diff(["HEAD~", "--name-only"])) for commit_file in commit_files: @@ -447,7 +483,7 @@ def reformat_branch( # pylint: disable=too-many-branches,too-many-locals,too-ma deleted_files.append(commit_file) continue - if files_match.search(commit_file): + if is_interesting_file(commit_file): clang_format.format(commit_file) else: print("Skipping file '%s' since it is not a file clang_format should format" % @@ -457,44 +493,44 @@ def reformat_branch( # pylint: disable=too-many-branches,too-many-locals,too-ma if not repo.is_working_tree_dirty(): print("Commit %s needed no reformatting" % commit_hash) else: - repo.commit(["--all", "--amend", "--no-edit"]) + repo.git_commit(["--all", "--amend", "--no-edit"]) # Rebase our new commit on top the post-reformat commit - previous_commit = repo.rev_parse(["HEAD"]) + previous_commit = repo.git_rev_parse(["HEAD"]) # Checkout the new branch with the reformatted commits # Note: we will not name as a branch until we are done with all commits on the local branch - repo.checkout(["--quiet", previous_commit_base]) + repo.git_checkout(["--quiet", previous_commit_base]) # Copy each file from the reformatted commit on top of the post reformat diff_files = get_list_from_lines( - repo.diff(["%s~..%s" % (previous_commit, previous_commit), "--name-only"])) + repo.git_diff(["%s~..%s" % (previous_commit, previous_commit), "--name-only"])) for diff_file in diff_files: # If the file was deleted in the commit we are reformatting, we need to delete it again if diff_file in deleted_files: - repo.rm([diff_file]) + repo.git_rm(["--ignore-unmatch", diff_file]) continue # The file has been added or modified, continue as normal - file_contents = repo.show(["%s:%s" % (previous_commit, diff_file)]) + file_contents = repo.git_show(["%s:%s" % (previous_commit, diff_file)]) root_dir = os.path.dirname(diff_file) if root_dir and not os.path.exists(root_dir): os.makedirs(root_dir) - with open(diff_file, "w+") as new_file: + with open(diff_file, "w+", encoding="utf-8") as new_file: new_file.write(file_contents) - repo.add([diff_file]) + repo.git_add([diff_file]) # Create a new commit onto clang-formatted branch - repo.commit(["--reuse-message=%s" % previous_commit]) + repo.git_commit(["--reuse-message=%s" % previous_commit, "--no-gpg-sign", "--allow-empty"]) - previous_commit_base = repo.rev_parse(["HEAD"]) + previous_commit_base = repo.git_rev_parse(["HEAD"]) # Create a new branch to mark the hashes we have been using - repo.checkout(["-b", new_branch]) + repo.git_checkout(["-b", new_branch]) print("reformat-branch is done running.\n") print("A copy of your branch has been made named '%s', and formatted with clang-format.\n" % @@ -531,7 +567,6 @@ def main(): elif command == "format": format_func(options.clang_format) elif command == "format-my": - format_my_func(options.clang_format, args[2] if len(args) > 2 else "origin/master") elif command == "reformat-branch": diff --git a/buildscripts/linter/git_base.py b/buildscripts/linter/git_base.py index 6af56ff29db..f7eaecae95c 100644 --- a/buildscripts/linter/git_base.py +++ b/buildscripts/linter/git_base.py @@ -25,6 +25,10 @@ class Repository(object): # pylint: disable=too-many-public-methods """Run a git commit command.""" return self._callgito("commit", args) + def git_checkout(self, args): + """Run a git checkout command.""" + return self._callgito("checkout", args) + def git_diff(self, args): """Run a git diff command.""" return self._callgito("diff", args)