Code

Ticket #17375: issue_17375.2.diff

File issue_17375.2.diff, 15.6 KB (added by pigletto, 2 years ago)
Line 
1diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py
2index 95c1da0..d6209fc 100644
3--- a/django/core/management/commands/makemessages.py
4+++ b/django/core/management/commands/makemessages.py
5@@ -3,6 +3,7 @@ import glob
6 import os
7 import re
8 import sys
9+import tempfile
10 from itertools import dropwhile
11 from optparse import make_option
12 from subprocess import PIPE, Popen
13@@ -13,6 +14,7 @@ from django.utils.jslex import prepare_js_for_gettext
14 
15 plural_forms_re = re.compile(r'^(?P<value>"Plural-Forms.+?\\n")\s*$', re.MULTILINE | re.DOTALL)
16 
17+
18 def handle_extensions(extensions=('html',), ignored=('py',)):
19     """
20     Organizes multiple extensions that are separated with commas or passed by
21@@ -37,6 +39,7 @@ def handle_extensions(extensions=('html',), ignored=('py',)):
22             ext_list[i] = '.%s' % ext_list[i]
23     return set([x for x in ext_list if x.strip('.') not in ignored])
24 
25+
26 def _popen(cmd):
27     """
28     Friendly wrapper around Popen for Windows
29@@ -44,6 +47,7 @@ def _popen(cmd):
30     p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=os.name != 'nt', universal_newlines=True)
31     return p.communicate()
32 
33+
34 def walk(root, topdown=True, onerror=None, followlinks=False):
35     """
36     A version of os.walk that can follow symlinks for Python < 2.6
37@@ -57,6 +61,7 @@ def walk(root, topdown=True, onerror=None, followlinks=False):
38                     for link_dirpath, link_dirnames, link_filenames in walk(p):
39                         yield (link_dirpath, link_dirnames, link_filenames)
40 
41+
42 def is_ignored(path, ignore_patterns):
43     """
44     Helper function to check if the given path should be ignored or not.
45@@ -66,6 +71,7 @@ def is_ignored(path, ignore_patterns):
46             return True
47     return False
48 
49+
50 def find_files(root, ignore_patterns, verbosity, symlinks=False):
51     """
52     Helper function to get all files in the given root.
53@@ -82,6 +88,7 @@ def find_files(root, ignore_patterns, verbosity, symlinks=False):
54     all_files.sort()
55     return all_files
56 
57+
58 def copy_plural_forms(msgs, locale, domain, verbosity):
59     """
60     Copies plural forms header contents from a Django catalog of locale to
61@@ -112,15 +119,17 @@ def copy_plural_forms(msgs, locale, domain, verbosity):
62                 break
63     return msgs
64 
65-def write_pot_file(potfile, msgs, file, work_file, is_templatized):
66+
67+def write_pot_file(potfile, msgs, all_files):
68     """
69     Write the :param potfile: POT file with the :param msgs: contents,
70     previously making sure its format is valid.
71     """
72-    if is_templatized:
73-        old = '#: ' + work_file[2:]
74-        new = '#: ' + file[2:]
75-        msgs = msgs.replace(old, new)
76+    for f_data in all_files:
77+        if f_data['is_templatized']:
78+            old = '#: ' + f_data['work'][2:]
79+            new = '#: ' + f_data['orig'][2:]
80+            msgs = msgs.replace(old, new)
81     if os.path.exists(potfile):
82         # Strip the header
83         msgs = '\n'.join(dropwhile(len, msgs.split('\n')))
84@@ -132,79 +141,6 @@ def write_pot_file(potfile, msgs, file, work_file, is_templatized):
85     finally:
86         f.close()
87 
88-def process_file(file, dirpath, potfile, domain, verbosity, extensions, wrap,
89-        location):
90-    """
91-    Extract translatable literals from :param file: for :param domain:
92-    creating or updating the :param potfile: POT file.
93-
94-    Uses the xgettext GNU gettext utility.
95-    """
96-
97-    from django.utils.translation import templatize
98-
99-    if verbosity > 1:
100-        sys.stdout.write('processing file %s in %s\n' % (file, dirpath))
101-    _, file_ext = os.path.splitext(file)
102-    if domain == 'djangojs' and file_ext in extensions:
103-        is_templatized = True
104-        orig_file = os.path.join(dirpath, file)
105-        src_data = open(orig_file).read()
106-        src_data = prepare_js_for_gettext(src_data)
107-        thefile = '%s.c' % file
108-        work_file = os.path.join(dirpath, thefile)
109-        f = open(work_file, "w")
110-        try:
111-            f.write(src_data)
112-        finally:
113-            f.close()
114-        cmd = (
115-            'xgettext -d %s -L C %s %s --keyword=gettext_noop '
116-            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
117-            '--keyword=pgettext:1c,2 --keyword=npgettext:1c,2,3 '
118-            '--from-code UTF-8 --add-comments=Translators -o - "%s"' % (
119-                domain, wrap, location, work_file
120-            )
121-        )
122-    elif domain == 'django' and (file_ext == '.py' or file_ext in extensions):
123-        thefile = file
124-        orig_file = os.path.join(dirpath, file)
125-        is_templatized = file_ext in extensions
126-        if is_templatized:
127-            src_data = open(orig_file, "rU").read()
128-            thefile = '%s.py' % file
129-            content = templatize(src_data, orig_file[2:])
130-            f = open(os.path.join(dirpath, thefile), "w")
131-            try:
132-                f.write(content)
133-            finally:
134-                f.close()
135-        work_file = os.path.join(dirpath, thefile)
136-        cmd = (
137-            'xgettext -d %s -L Python %s %s --keyword=gettext_noop '
138-            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
139-            '--keyword=ugettext_noop --keyword=ugettext_lazy '
140-            '--keyword=ungettext_lazy:1,2 --keyword=pgettext:1c,2 '
141-            '--keyword=npgettext:1c,2,3 --keyword=pgettext_lazy:1c,2 '
142-            '--keyword=npgettext_lazy:1c,2,3 --from-code UTF-8 '
143-            '--add-comments=Translators -o - "%s"' % (
144-                domain, wrap, location, work_file)
145-        )
146-    else:
147-        return
148-    msgs, errors = _popen(cmd)
149-    if errors:
150-        if is_templatized:
151-            os.unlink(work_file)
152-        if os.path.exists(potfile):
153-            os.unlink(potfile)
154-        raise CommandError(
155-            "errors happened while running xgettext on %s\n%s" %
156-            (file, errors))
157-    if msgs:
158-        write_pot_file(potfile, msgs, orig_file, work_file, is_templatized)
159-    if is_templatized:
160-        os.unlink(work_file)
161 
162 def write_po_file(pofile, potfile, domain, locale, verbosity,
163         copy_pforms, wrap, location, no_obsolete):
164@@ -248,6 +184,7 @@ def write_po_file(pofile, potfile, domain, locale, verbosity,
165             raise CommandError(
166                 "errors happened while running msgattrib\n%s" % errors)
167 
168+
169 def make_messages(locale=None, domain='django', verbosity=1, all=False,
170         extensions=None, symlinks=False, ignore_patterns=None, no_wrap=False,
171         no_location=False, no_obsolete=False):
172@@ -323,16 +260,129 @@ def make_messages(locale=None, domain='django', verbosity=1, all=False,
173         if os.path.exists(potfile):
174             os.unlink(potfile)
175 
176-        for dirpath, file in find_files(".", ignore_patterns, verbosity,
177-                symlinks=symlinks):
178-            process_file(file, dirpath, potfile, domain, verbosity, extensions,
179-                    wrap, location)
180+        process_files(potfile, domain, verbosity, extensions,
181+                      wrap, location, ignore_patterns, symlinks)
182 
183         if os.path.exists(potfile):
184             write_po_file(pofile, potfile, domain, locale, verbosity,
185                     not invoked_for_django, wrap, location, no_obsolete)
186 
187 
188+def process_files(potfile, domain, verbosity, extensions, wrap, location,
189+                  ignore_patterns, symlinks):
190+    """
191+    Extract translatable literals from current directory for :param domain:
192+    creating or updating the :param potfile: POT file.
193+
194+    Uses the xgettext GNU gettext utility.
195+    """
196+    from django.utils.translation import templatize
197+
198+    # stores information about all processed files
199+    all_files = []
200+
201+    # prepare command to run xgettext depending on domain type
202+    if domain == 'djangojs':
203+        cmd = (
204+            'xgettext -d %s -L C %s %s --keyword=gettext_noop '
205+            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
206+            '--keyword=pgettext:1c,2 --keyword=npgettext:1c,2,3 '
207+            '--from-code UTF-8 --add-comments=Translators -o - -f %s'
208+        )
209+    elif domain == 'django':
210+        cmd = (
211+            'xgettext -d %s -L Python %s %s --keyword=gettext_noop '
212+            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
213+            '--keyword=ugettext_noop --keyword=ugettext_lazy '
214+            '--keyword=ungettext_lazy:1,2 --keyword=pgettext:1c,2 '
215+            '--keyword=npgettext:1c,2,3 --keyword=pgettext_lazy:1c,2 '
216+            '--keyword=npgettext_lazy:1c,2,3 --from-code UTF-8 '
217+            '--add-comments=Translators -o - -f %s'
218+        )
219+
220+    for dirpath, file in find_files(".", ignore_patterns, verbosity,
221+                                    symlinks=symlinks):
222+        if verbosity > 1:
223+            sys.stdout.write('processing file %s in %s\n' % (file, dirpath))
224+        _, file_ext = os.path.splitext(file)
225+
226+        is_templatized = file_ext in extensions
227+
228+        # path to the processed file
229+        orig_file = os.path.join(dirpath, file)
230+
231+        # store information about processed file;
232+        # 'orig' - path to original file
233+        # 'work' - path to either original file or templatized version if one is created
234+        # 'is_templatized' - defines if file was templatized
235+        file_dict = {'orig': orig_file, 'work': orig_file, 'is_templatized': is_templatized}
236+
237+        # not really necessary to initialize these here but added for clarity
238+        thefile = ''
239+        content = ''
240+
241+        if domain == 'djangojs' and is_templatized:
242+            src_data = open(orig_file).read()
243+            content = prepare_js_for_gettext(src_data)
244+            thefile = '%s.c' % file
245+        elif domain == 'django' and (file_ext == '.py' or is_templatized):
246+            if is_templatized:
247+                src_data = open(orig_file, "rU").read()
248+                content = templatize(src_data, orig_file[2:])
249+                thefile = '%s.py' % file
250+        else:
251+            continue
252+
253+        if is_templatized:
254+            # for templatized ones create work file
255+            work_file = os.path.join(dirpath, thefile)
256+            file_dict['work'] = work_file
257+
258+            f = open(work_file, "w")
259+            try:
260+                f.write(content)
261+            finally:
262+                f.close()
263+
264+        all_files.append(file_dict)
265+
266+    if all_files:
267+        # prepare command that runs xgettext with path to file that
268+        # contains paths of all files to process
269+        input_files = '\n'.join([f_data['work'] for f_data in all_files])
270+
271+        # create temporary file for xgettext
272+        f_handle, input_files_path = tempfile.mkstemp()
273+        f = open(input_files_path, 'w')
274+        try:
275+            f.write(input_files)
276+        finally:
277+            f.close()
278+
279+        cmd = cmd % (domain, wrap, location, input_files_path)
280+
281+        msgs, errors = _popen(cmd)
282+
283+        os.unlink(input_files_path)
284+
285+        if errors:
286+            # check if file exists because when symlinks are used it might
287+            # happen that it was already removed
288+            [os.unlink(f_data['work']) for f_data in all_files
289+               if f_data['is_templatized'] and os.path.exists(f_data['work'])]
290+
291+            input_files = ', '.join([f_data['work'] for f_data in all_files])
292+            raise CommandError(
293+                "errors happened while running xgettext on %s\n%s" %
294+                (input_files, errors))
295+        if msgs:
296+            write_pot_file(potfile, msgs, all_files)
297+        # check if file exists because when symlinks are used it might
298+        # happen that it was already removed
299+        [os.unlink(f_data['work']) for f_data in all_files
300+               if f_data['is_templatized'] and os.path.exists(f_data['work'])]
301+
302+
303 class Command(NoArgsCommand):
304     option_list = NoArgsCommand.option_list + (
305         make_option('--locale', '-l', default=None, dest='locale',
306diff --git a/tests/regressiontests/i18n/commands/extraction.py b/tests/regressiontests/i18n/commands/extraction.py
307index b1fc002..7afded3 100644
308--- a/tests/regressiontests/i18n/commands/extraction.py
309+++ b/tests/regressiontests/i18n/commands/extraction.py
310@@ -41,6 +41,15 @@ class ExtractorTests(TestCase):
311         msgid = re.escape(msgid)
312         return self.assertTrue(re.search('^msgid %s' % msgid, s, re.MULTILINE), 'Could not find %(q)s%(n)s%(q)s in generated PO file' % {'n':needle, 'q':q})
313 
314+    def assertMsgIdPlural(self, msgid, s, use_quotes=True):
315+        q = '"'
316+        if use_quotes:
317+            msgid = '"%s"' % msgid
318+            q = "'"
319+        needle = 'msgid_plural %s' % msgid
320+        msgid = re.escape(msgid)
321+        return self.assertTrue(re.search('^msgid_plural %s' % msgid, s, re.MULTILINE), 'Could not find %(q)s%(n)s%(q)s in generated PO file' % {'n':needle, 'q':q})
322+
323     def assertNotMsgId(self, msgid, s, use_quotes=True):
324         if use_quotes:
325             msgid = '"%s"' % msgid
326@@ -263,3 +272,38 @@ class NoLocationExtractorTests(ExtractorTests):
327         with open(self.PO_FILE, 'r') as fp:
328             po_contents = fp.read()
329             self.assertTrue('#: templates/test.html:55' in po_contents)
330+
331+
332+class MultipleFilesPluralExtractorTests(ExtractorTests):
333+
334+    def setUp(self):
335+        self._cwd = os.getcwd()
336+        self.test_dir = os.path.abspath(os.path.dirname(__file__))
337+        self.second_test_template = os.path.join(self.test_dir, 'templates', 'plural_test_template.html')
338+
339+        second_test_template = '''{% load i18n %}{% trans 'My string' %}'''
340+
341+        if not os.path.exists(self.second_test_template):
342+            f = open(self.second_test_template, 'w+')
343+            try:
344+                f.write(second_test_template)
345+            finally:
346+                f.close()
347+
348+    def tearDown(self):
349+        super(MultipleFilesPluralExtractorTests, self).tearDown()
350+        os.chdir(self.test_dir)
351+        try:
352+            os.remove(self.second_test_template)
353+        except OSError:
354+            pass
355+        os.chdir(self._cwd)
356+
357+    def test_multiple_files_plural_extraction(self):
358+        os.chdir(self.test_dir)
359+        management.call_command('makemessages', locale=LOCALE, verbosity=0, symlinks=True)
360+        self.assertTrue(os.path.exists(self.PO_FILE))
361+        with open(self.PO_FILE, 'r') as fp:
362+            po_contents = fp.read()
363+            self.assertMsgId('My string', po_contents)
364+            self.assertMsgIdPlural('My strings', po_contents)
365diff --git a/tests/regressiontests/i18n/commands/templates/test.html b/tests/regressiontests/i18n/commands/templates/test.html
366index 5789346..1e00a80 100644
367--- a/tests/regressiontests/i18n/commands/templates/test.html
368+++ b/tests/regressiontests/i18n/commands/templates/test.html
369@@ -77,3 +77,4 @@ continued here.{% endcomment %}
370 {% trans "Shouldn't double escape this sequence %% either" context "ctx1" %}
371 {% trans "Looks like a str fmt spec %s but shouldn't be interpreted as such" %}
372 {% trans "Looks like a str fmt spec % o but shouldn't be interpreted as such" %}
373+{% blocktrans count counter=mylist|length %}My string{% plural %}My strings{% endblocktrans %}
374diff --git a/tests/regressiontests/i18n/tests.py b/tests/regressiontests/i18n/tests.py
375index 77bd35b..35f9060 100644
376--- a/tests/regressiontests/i18n/tests.py
377+++ b/tests/regressiontests/i18n/tests.py
378@@ -29,7 +29,7 @@ if can_run_extraction_tests:
379     from .commands.extraction import (ExtractorTests, BasicExtractorTests,
380         JavascriptExtractorTests, IgnoredExtractorTests, SymlinkExtractorTests,
381         CopyPluralFormsExtractorTests, NoWrapExtractorTests,
382-        NoLocationExtractorTests)
383+        NoLocationExtractorTests, MultipleFilesPluralExtractorTests)
384 if can_run_compilation_tests:
385     from .commands.compilation import (PoFileTests, PoFileContentsTests,
386         PercentRenderingTests)