From 2c1f27d0d0346889e3af3b1be03e39e7117b1165 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 27 Aug 2024 21:31:07 +0200 Subject: [PATCH] Dropped safeguards against very old versions of gettext. gettext 0.19 was released in 2014. --- .../management/commands/compilemessages.py | 4 +- .../core/management/commands/makemessages.py | 27 +- django/utils/jslex.py | 250 ----------- docs/releases/5.2.txt | 2 + docs/topics/i18n/translation.txt | 2 +- tests/i18n/test_compilation.py | 4 - tests/i18n/test_extraction.py | 27 +- tests/utils_tests/test_jslex.py | 401 ------------------ 8 files changed, 12 insertions(+), 705 deletions(-) delete mode 100644 django/utils/jslex.py delete mode 100644 tests/utils_tests/test_jslex.py diff --git a/django/core/management/commands/compilemessages.py b/django/core/management/commands/compilemessages.py index eddf31b794..c56e2a237c 100644 --- a/django/core/management/commands/compilemessages.py +++ b/django/core/management/commands/compilemessages.py @@ -79,8 +79,8 @@ class Command(BaseCommand): if find_command(self.program) is None: raise CommandError( - "Can't find %s. Make sure you have GNU gettext " - "tools 0.15 or newer installed." % self.program + f"Can't find {self.program}. Make sure you have GNU gettext " + "tools 0.19 or newer installed." ) basedirs = [os.path.join("conf", "locale"), "locale"] diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py index dc3d4026e2..076667d41a 100644 --- a/django/core/management/commands/makemessages.py +++ b/django/core/management/commands/makemessages.py @@ -19,7 +19,6 @@ from django.core.management.utils import ( ) from django.utils.encoding import DEFAULT_LOCALE_ENCODING from django.utils.functional import cached_property -from django.utils.jslex import prepare_js_for_gettext from django.utils.regex_helper import _lazy_re_compile from django.utils.text import get_text_list from django.utils.translation import templatize @@ -35,8 +34,8 @@ def check_programs(*programs): for program in programs: if find_command(program) is None: raise CommandError( - "Can't find %s. Make sure you have GNU gettext tools 0.15 or " - "newer installed." % program + f"Can't find {program}. Make sure you have GNU gettext tools " + "0.19 or newer installed." ) @@ -80,9 +79,7 @@ class BuildFile: @cached_property def is_templatized(self): - if self.domain == "djangojs": - return self.command.gettext_version < (0, 18, 3) - elif self.domain == "django": + if self.domain == "django": file_ext = os.path.splitext(self.translatable.file)[1] return file_ext != ".py" return False @@ -99,11 +96,7 @@ class BuildFile: """ if not self.is_templatized: return self.path - extension = { - "djangojs": "c", - "django": "py", - }.get(self.domain) - filename = "%s.%s" % (self.translatable.file, extension) + filename = f"{self.translatable.file}.py" return os.path.join(self.translatable.dirpath, filename) def preprocess(self): @@ -117,9 +110,7 @@ class BuildFile: with open(self.path, encoding="utf-8") as fp: src_data = fp.read() - if self.domain == "djangojs": - content = prepare_js_for_gettext(src_data) - elif self.domain == "django": + if self.domain == "django": content = templatize(src_data, origin=self.path[2:]) with open(self.work_path, "w", encoding="utf-8") as fp: @@ -349,11 +340,6 @@ class Command(BaseCommand): self.msgattrib_options = self.msgattrib_options[:] + ["--no-location"] self.xgettext_options = self.xgettext_options[:] + ["--no-location"] if options["add_location"]: - if self.gettext_version < (0, 19): - raise CommandError( - "The --add-location option requires gettext 0.19 or later. " - "You have %s." % ".".join(str(x) for x in self.gettext_version) - ) arg_add_location = "--add-location=%s" % options["add_location"] self.msgmerge_options = self.msgmerge_options[:] + [arg_add_location] self.msguniq_options = self.msguniq_options[:] + [arg_add_location] @@ -636,12 +622,11 @@ class Command(BaseCommand): build_files.append(build_file) if self.domain == "djangojs": - is_templatized = build_file.is_templatized args = [ "xgettext", "-d", self.domain, - "--language=%s" % ("C" if is_templatized else "JavaScript",), + "--language=JavaScript", "--keyword=gettext_noop", "--keyword=gettext_lazy", "--keyword=ngettext_lazy:1,2", diff --git a/django/utils/jslex.py b/django/utils/jslex.py deleted file mode 100644 index fc46a686c7..0000000000 --- a/django/utils/jslex.py +++ /dev/null @@ -1,250 +0,0 @@ -"""JsLex: a lexer for JavaScript""" - -# Originally from https://bitbucket.org/ned/jslex -import re - - -class Tok: - """ - A specification for a token class. - """ - - num = 0 - - def __init__(self, name, regex, next=None): - self.id = Tok.num - Tok.num += 1 - self.name = name - self.regex = regex - self.next = next - - -def literals(choices, prefix="", suffix=""): - """ - Create a regex from a space-separated list of literal `choices`. - - If provided, `prefix` and `suffix` will be attached to each choice - individually. - """ - return "|".join(prefix + re.escape(c) + suffix for c in choices.split()) - - -class Lexer: - """ - A generic multi-state regex-based lexer. - """ - - def __init__(self, states, first): - self.regexes = {} - self.toks = {} - - for state, rules in states.items(): - parts = [] - for tok in rules: - groupid = "t%d" % tok.id - self.toks[groupid] = tok - parts.append("(?P<%s>%s)" % (groupid, tok.regex)) - self.regexes[state] = re.compile("|".join(parts), re.MULTILINE | re.VERBOSE) - - self.state = first - - def lex(self, text): - """ - Lexically analyze `text`. - - Yield pairs (`name`, `tokentext`). - """ - end = len(text) - state = self.state - regexes = self.regexes - toks = self.toks - start = 0 - - while start < end: - for match in regexes[state].finditer(text, start): - name = match.lastgroup - tok = toks[name] - toktext = match[name] - start += len(toktext) - yield (tok.name, toktext) - - if tok.next: - state = tok.next - break - - self.state = state - - -class JsLexer(Lexer): - """ - A JavaScript lexer - - >>> lexer = JsLexer() - >>> list(lexer.lex("a = 1")) - [('id', 'a'), ('ws', ' '), ('punct', '='), ('ws', ' '), ('dnum', '1')] - - This doesn't properly handle non-ASCII characters in the JavaScript source. - """ - - # Because these tokens are matched as alternatives in a regex, longer - # possibilities must appear in the list before shorter ones, for example, - # '>>' before '>'. - # - # Note that we don't have to detect malformed JavaScript, only properly - # lex correct JavaScript, so much of this is simplified. - - # Details of JavaScript lexical structure are taken from - # https://www.ecma-international.org/publications-and-standards/standards/ecma-262/ - - # A useful explanation of automatic semicolon insertion is at - # http://inimino.org/~inimino/blog/javascript_semicolons - - both_before = [ - Tok("comment", r"/\*(.|\n)*?\*/"), - Tok("linecomment", r"//.*?$"), - Tok("ws", r"\s+"), - Tok( - "keyword", - literals( - """ - break case catch class const continue debugger - default delete do else enum export extends - finally for function if import in instanceof - new return super switch this throw try typeof - var void while with - """, - suffix=r"\b", - ), - next="reg", - ), - Tok("reserved", literals("null true false", suffix=r"\b"), next="div"), - Tok( - "id", - r""" - ([a-zA-Z_$ ]|\\u[0-9a-fA-Z]{4}) # first char - ([a-zA-Z_$0-9]|\\u[0-9a-fA-F]{4})* # rest chars - """, - next="div", - ), - Tok("hnum", r"0[xX][0-9a-fA-F]+", next="div"), - Tok("onum", r"0[0-7]+"), - Tok( - "dnum", - r""" - ( (0|[1-9][0-9]*) # DecimalIntegerLiteral - \. # dot - [0-9]* # DecimalDigits-opt - ([eE][-+]?[0-9]+)? # ExponentPart-opt - | - \. # dot - [0-9]+ # DecimalDigits - ([eE][-+]?[0-9]+)? # ExponentPart-opt - | - (0|[1-9][0-9]*) # DecimalIntegerLiteral - ([eE][-+]?[0-9]+)? # ExponentPart-opt - ) - """, - next="div", - ), - Tok( - "punct", - literals( - """ - >>>= === !== >>> <<= >>= <= >= == != << >> && - || += -= *= %= &= |= ^= - """ - ), - next="reg", - ), - Tok("punct", literals("++ -- ) ]"), next="div"), - Tok("punct", literals("{ } ( [ . ; , < > + - * % & | ^ ! ~ ? : ="), next="reg"), - Tok("string", r'"([^"\\]|(\\(.|\n)))*?"', next="div"), - Tok("string", r"'([^'\\]|(\\(.|\n)))*?'", next="div"), - ] - - both_after = [ - Tok("other", r"."), - ] - - states = { - # slash will mean division - "div": both_before - + [ - Tok("punct", literals("/= /"), next="reg"), - ] - + both_after, - # slash will mean regex - "reg": both_before - + [ - Tok( - "regex", - r""" - / # opening slash - # First character is.. - ( [^*\\/[] # anything but * \ / or [ - | \\. # or an escape sequence - | \[ # or a class, which has - ( [^\]\\] # anything but \ or ] - | \\. # or an escape sequence - )* # many times - \] - ) - # Following characters are same, except for excluding a star - ( [^\\/[] # anything but \ / or [ - | \\. # or an escape sequence - | \[ # or a class, which has - ( [^\]\\] # anything but \ or ] - | \\. # or an escape sequence - )* # many times - \] - )* # many times - / # closing slash - [a-zA-Z0-9]* # trailing flags - """, - next="div", - ), - ] - + both_after, - } - - def __init__(self): - super().__init__(self.states, "reg") - - -def prepare_js_for_gettext(js): - """ - Convert the JavaScript source `js` into something resembling C for - xgettext. - - What actually happens is that all the regex literals are replaced with - "REGEX". - """ - - def escape_quotes(m): - """Used in a regex to properly escape double quotes.""" - s = m[0] - if s == '"': - return r"\"" - else: - return s - - lexer = JsLexer() - c = [] - for name, tok in lexer.lex(js): - if name == "regex": - # C doesn't grok regexes, and they aren't needed for gettext, - # so just output a string instead. - tok = '"REGEX"' - elif name == "string": - # C doesn't have single-quoted strings, so make all strings - # double-quoted. - if tok.startswith("'"): - guts = re.sub(r"\\.|.", escape_quotes, tok[1:-1]) - tok = '"' + guts + '"' - elif name == "id": - # C can't deal with Unicode escapes in identifiers. We don't - # need them for gettext anyway, so replace them with something - # innocuous - tok = tok.replace("\\", "U") - c.append(tok) - return "".join(c) diff --git a/docs/releases/5.2.txt b/docs/releases/5.2.txt index 0caeef01cf..f910416716 100644 --- a/docs/releases/5.2.txt +++ b/docs/releases/5.2.txt @@ -306,6 +306,8 @@ Miscellaneous * Adding :attr:`.EmailMultiAlternatives.alternatives` is now only supported via the :meth:`~.EmailMultiAlternatives.attach_alternative` method. +* The minimum supported version of ``gettext`` is increased from 0.15 to 0.19. + .. _deprecated-features-5.2: Features deprecated in 5.2 diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 3ac254de98..04dad034cc 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1551,7 +1551,7 @@ Django comes with a tool, :djadmin:`django-admin makemessages commands from the GNU gettext toolset: ``xgettext``, ``msgfmt``, ``msgmerge`` and ``msguniq``. - The minimum version of the ``gettext`` utilities supported is 0.15. + The minimum version of the ``gettext`` utilities supported is 0.19. To create or update a message file, run this command: diff --git a/tests/i18n/test_compilation.py b/tests/i18n/test_compilation.py index 7da95ba9e9..7b02776dbe 100644 --- a/tests/i18n/test_compilation.py +++ b/tests/i18n/test_compilation.py @@ -8,7 +8,6 @@ from subprocess import run from unittest import mock from django.core.management import CommandError, call_command, execute_from_command_line -from django.core.management.commands.makemessages import Command as MakeMessagesCommand from django.core.management.utils import find_command from django.test import SimpleTestCase, override_settings from django.test.utils import captured_stderr, captured_stdout @@ -269,9 +268,6 @@ class CompilationErrorHandling(MessageCompilationTests): "django.core.management.utils.run", lambda *args, **kwargs: run(*args, env=env, **kwargs), ): - cmd = MakeMessagesCommand() - if cmd.gettext_version < (0, 18, 3): - self.skipTest("python-brace-format is a recent gettext addition.") stderr = StringIO() with self.assertRaisesMessage( CommandError, "compilemessages generated one or more errors" diff --git a/tests/i18n/test_extraction.py b/tests/i18n/test_extraction.py index 226d51ce11..7aa600c4c1 100644 --- a/tests/i18n/test_extraction.py +++ b/tests/i18n/test_extraction.py @@ -6,7 +6,7 @@ import time import warnings from io import StringIO from pathlib import Path -from unittest import mock, skipIf, skipUnless +from unittest import mock, skipUnless from admin_scripts.tests import AdminScriptTestCase @@ -25,10 +25,6 @@ from .utils import POFileAssertionMixin, RunInTmpDirMixin, copytree LOCALE = "de" has_xgettext = find_command("xgettext") -gettext_version = MakeMessagesCommand().gettext_version if has_xgettext else None -requires_gettext_019 = skipIf( - has_xgettext and gettext_version < (0, 19), "gettext 0.19 required" -) @skipUnless(has_xgettext, "xgettext is mandatory for extraction tests") @@ -836,7 +832,6 @@ class LocationCommentsTests(ExtractorTests): self.assertLocationCommentNotPresent(self.PO_FILE, None, ".html.py") self.assertLocationCommentPresent(self.PO_FILE, 5, "templates", "test.html") - @requires_gettext_019 def test_add_location_full(self): """makemessages --add-location=full""" management.call_command( @@ -848,7 +843,6 @@ class LocationCommentsTests(ExtractorTests): self.PO_FILE, "Translatable literal #6b", "templates", "test.html" ) - @requires_gettext_019 def test_add_location_file(self): """makemessages --add-location=file""" management.call_command( @@ -862,7 +856,6 @@ class LocationCommentsTests(ExtractorTests): self.PO_FILE, "Translatable literal #6b", "templates", "test.html" ) - @requires_gettext_019 def test_add_location_never(self): """makemessages --add-location=never""" management.call_command( @@ -871,24 +864,6 @@ class LocationCommentsTests(ExtractorTests): self.assertTrue(os.path.exists(self.PO_FILE)) self.assertLocationCommentNotPresent(self.PO_FILE, None, "test.html") - @mock.patch( - "django.core.management.commands.makemessages.Command.gettext_version", - new=(0, 18, 99), - ) - def test_add_location_gettext_version_check(self): - """ - CommandError is raised when using makemessages --add-location with - gettext < 0.19. - """ - msg = ( - "The --add-location option requires gettext 0.19 or later. You have " - "0.18.99." - ) - with self.assertRaisesMessage(CommandError, msg): - management.call_command( - "makemessages", locale=[LOCALE], verbosity=0, add_location="full" - ) - class NoObsoleteExtractorTests(ExtractorTests): work_subdir = "obsolete_translations" diff --git a/tests/utils_tests/test_jslex.py b/tests/utils_tests/test_jslex.py deleted file mode 100644 index 59551930c6..0000000000 --- a/tests/utils_tests/test_jslex.py +++ /dev/null @@ -1,401 +0,0 @@ -"""Tests for jslex.""" - -# originally from https://bitbucket.org/ned/jslex - -from django.test import SimpleTestCase -from django.utils.jslex import JsLexer, prepare_js_for_gettext - - -class JsTokensTest(SimpleTestCase): - LEX_CASES = [ - # ids - ("a ABC $ _ a123", ["id a", "id ABC", "id $", "id _", "id a123"]), - ( - "\\u1234 abc\\u0020 \\u0065_\\u0067", - ["id \\u1234", "id abc\\u0020", "id \\u0065_\\u0067"], - ), - # numbers - ( - "123 1.234 0.123e-3 0 1E+40 1e1 .123", - [ - "dnum 123", - "dnum 1.234", - "dnum 0.123e-3", - "dnum 0", - "dnum 1E+40", - "dnum 1e1", - "dnum .123", - ], - ), - ("0x1 0xabCD 0XABcd", ["hnum 0x1", "hnum 0xabCD", "hnum 0XABcd"]), - ("010 0377 090", ["onum 010", "onum 0377", "dnum 0", "dnum 90"]), - ("0xa123ghi", ["hnum 0xa123", "id ghi"]), - # keywords - ( - "function Function FUNCTION", - ["keyword function", "id Function", "id FUNCTION"], - ), - ( - "const constructor in inherits", - ["keyword const", "id constructor", "keyword in", "id inherits"], - ), - ("true true_enough", ["reserved true", "id true_enough"]), - # strings - (""" 'hello' "hello" """, ["string 'hello'", 'string "hello"']), - ( - r""" 'don\'t' "don\"t" '"' "'" '\'' "\"" """, - [ - r"""string 'don\'t'""", - r'''string "don\"t"''', - r"""string '"'""", - r'''string "'"''', - r"""string '\''""", - r'''string "\""''', - ], - ), - (r'"ƃuıxǝ⅂ ʇdıɹɔsɐʌɐſ\""', [r'string "ƃuıxǝ⅂ ʇdıɹɔsɐʌɐſ\""']), - # comments - ("a//b", ["id a", "linecomment //b"]), - ( - "/****/a/=2//hello", - ["comment /****/", "id a", "punct /=", "dnum 2", "linecomment //hello"], - ), - ( - "/*\n * Header\n */\na=1;", - ["comment /*\n * Header\n */", "id a", "punct =", "dnum 1", "punct ;"], - ), - # punctuation - ("a+++b", ["id a", "punct ++", "punct +", "id b"]), - # regex - (r"a=/a*/,1", ["id a", "punct =", "regex /a*/", "punct ,", "dnum 1"]), - (r"a=/a*[^/]+/,1", ["id a", "punct =", "regex /a*[^/]+/", "punct ,", "dnum 1"]), - (r"a=/a*\[^/,1", ["id a", "punct =", r"regex /a*\[^/", "punct ,", "dnum 1"]), - (r"a=/\//,1", ["id a", "punct =", r"regex /\//", "punct ,", "dnum 1"]), - # next two are from https://www-archive.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions # NOQA - ( - 'for (var x = a in foo && "" || mot ? z:/x:3;x<5;y"', - "punct ||", - "id mot", - "punct ?", - "id z", - "punct :", - "regex /x:3;x<5;y" || mot ? z/x:3;x<5;y"', - "punct ||", - "id mot", - "punct ?", - "id z", - "punct /", - "id x", - "punct :", - "dnum 3", - "punct ;", - "id x", - "punct <", - "dnum 5", - "punct ;", - "id y", - "punct <", - "regex /g/i", - "punct )", - "punct {", - "id xyz", - "punct (", - "id x", - "punct ++", - "punct )", - "punct ;", - "punct }", - ], - ), - # Various "illegal" regexes that are valid according to the std. - ( - r"""/????/, /++++/, /[----]/ """, - ["regex /????/", "punct ,", "regex /++++/", "punct ,", "regex /[----]/"], - ), - # Stress cases from https://stackoverflow.com/questions/5533925/what-javascript-constructs-does-jslex-incorrectly-lex/5573409#5573409 # NOQA - (r"""/\[/""", [r"""regex /\[/"""]), - (r"""/[i]/""", [r"""regex /[i]/"""]), - (r"""/[\]]/""", [r"""regex /[\]]/"""]), - (r"""/a[\]]/""", [r"""regex /a[\]]/"""]), - (r"""/a[\]]b/""", [r"""regex /a[\]]b/"""]), - (r"""/[\]/]/gi""", [r"""regex /[\]/]/gi"""]), - (r"""/\[[^\]]+\]/gi""", [r"""regex /\[[^\]]+\]/gi"""]), - ( - r""" - rexl.re = { - NAME: /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/, - UNQUOTED_LITERAL: /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/, - QUOTED_LITERAL: /^'(?:[^']|'')*'/, - NUMERIC_LITERAL: /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/, - SYMBOL: /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/ - }; - """, # NOQA - [ - "id rexl", - "punct .", - "id re", - "punct =", - "punct {", - "id NAME", - "punct :", - r"""regex /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/""", - "punct ,", - "id UNQUOTED_LITERAL", - "punct :", - r"""regex /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/""", - "punct ,", - "id QUOTED_LITERAL", - "punct :", - r"""regex /^'(?:[^']|'')*'/""", - "punct ,", - "id NUMERIC_LITERAL", - "punct :", - r"""regex /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/""", - "punct ,", - "id SYMBOL", - "punct :", - r"""regex /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/""", # NOQA - "punct }", - "punct ;", - ], - ), - ( - r""" - rexl.re = { - NAME: /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/, - UNQUOTED_LITERAL: /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/, - QUOTED_LITERAL: /^'(?:[^']|'')*'/, - NUMERIC_LITERAL: /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/, - SYMBOL: /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/ - }; - str = '"'; - """, # NOQA - [ - "id rexl", - "punct .", - "id re", - "punct =", - "punct {", - "id NAME", - "punct :", - r"""regex /^(?![0-9])(?:\w)+|^"(?:[^"]|"")+"/""", - "punct ,", - "id UNQUOTED_LITERAL", - "punct :", - r"""regex /^@(?:(?![0-9])(?:\w|\:)+|^"(?:[^"]|"")+")\[[^\]]+\]/""", - "punct ,", - "id QUOTED_LITERAL", - "punct :", - r"""regex /^'(?:[^']|'')*'/""", - "punct ,", - "id NUMERIC_LITERAL", - "punct :", - r"""regex /^[0-9]+(?:\.[0-9]*(?:[eE][-+][0-9]+)?)?/""", - "punct ,", - "id SYMBOL", - "punct :", - r"""regex /^(?:==|=|<>|<=|<|>=|>|!~~|!~|~~|~|!==|!=|!~=|!~|!|&|\||\.|\:|,|\(|\)|\[|\]|\{|\}|\?|\:|;|@|\^|\/\+|\/|\*|\+|-)/""", # NOQA - "punct }", - "punct ;", - "id str", - "punct =", - """string '"'""", - "punct ;", - ], - ), - ( - r' this._js = "e.str(\"" + this.value.replace(/\\/g, "\\\\")' - r'.replace(/"/g, "\\\"") + "\")"; ', - [ - "keyword this", - "punct .", - "id _js", - "punct =", - r'''string "e.str(\""''', - "punct +", - "keyword this", - "punct .", - "id value", - "punct .", - "id replace", - "punct (", - r"regex /\\/g", - "punct ,", - r'string "\\\\"', - "punct )", - "punct .", - "id replace", - "punct (", - r'regex /"/g', - "punct ,", - r'string "\\\""', - "punct )", - "punct +", - r'string "\")"', - "punct ;", - ], - ), - ] - - -def make_function(input, toks): - def test_func(self): - lexer = JsLexer() - result = [ - "%s %s" % (name, tok) for name, tok in lexer.lex(input) if name != "ws" - ] - self.assertEqual(result, toks) - - return test_func - - -for i, (input, toks) in enumerate(JsTokensTest.LEX_CASES): - setattr(JsTokensTest, "test_case_%d" % i, make_function(input, toks)) - - -GETTEXT_CASES = ( - ( - r""" - a = 1; /* /[0-9]+/ */ - b = 0x2a0b / 1; // /[0-9]+/ - c = 3; - """, - r""" - a = 1; /* /[0-9]+/ */ - b = 0x2a0b / 1; // /[0-9]+/ - c = 3; - """, - ), - ( - r""" - a = 1.234e-5; - /* - * /[0-9+/ - */ - b = .0123; - """, - r""" - a = 1.234e-5; - /* - * /[0-9+/ - */ - b = .0123; - """, - ), - ( - r""" - x = y / z; - alert(gettext("hello")); - x /= 3; - """, - r""" - x = y / z; - alert(gettext("hello")); - x /= 3; - """, - ), - ( - r""" - s = "Hello \"th/foo/ere\""; - s = 'He\x23llo \'th/foo/ere\''; - s = 'slash quote \", just quote "'; - """, - r""" - s = "Hello \"th/foo/ere\""; - s = "He\x23llo \'th/foo/ere\'"; - s = "slash quote \", just quote \""; - """, - ), - ( - r""" - s = "Line continuation\ - continued /hello/ still the string";/hello/; - """, - r""" - s = "Line continuation\ - continued /hello/ still the string";"REGEX"; - """, - ), - ( - r""" - var regex = /pattern/; - var regex2 = /matter/gm; - var regex3 = /[*/]+/gm.foo("hey"); - """, - r""" - var regex = "REGEX"; - var regex2 = "REGEX"; - var regex3 = "REGEX".foo("hey"); - """, - ), - ( - r""" - for (var x = a in foo && "" || mot ? z:/x:3;x<5;y" || mot ? z/x:3;x<5;y" || mot ? z:"REGEX"/i) {xyz(x++);} - for (var x = a in foo && "" || mot ? z/x:3;x<5;y<"REGEX") {xyz(x++);} - """, - ), - ( - """ - \\u1234xyz = gettext('Hello there'); - """, - r""" - Uu1234xyz = gettext("Hello there"); - """, - ), -) - - -class JsToCForGettextTest(SimpleTestCase): - pass - - -def make_function(js, c): - def test_func(self): - self.assertEqual(prepare_js_for_gettext(js), c) - - return test_func - - -for i, pair in enumerate(GETTEXT_CASES): - setattr(JsToCForGettextTest, "test_case_%d" % i, make_function(*pair))