Ticket #17375: issue_17375.2.diff

File issue_17375.2.diff, 15.6 KB (added by pigletto, 3 years ago)
  • django/core/management/commands/makemessages.py

    diff --git a/django/core/management/commands/makemessages.py b/django/core/management/commands/makemessages.py
    index 95c1da0..d6209fc 100644
    a b import glob 
    33import os
    44import re
    55import sys
     6import tempfile
    67from itertools import dropwhile
    78from optparse import make_option
    89from subprocess import PIPE, Popen
    from django.utils.jslex import prepare_js_for_gettext 
    1314
    1415plural_forms_re = re.compile(r'^(?P<value>"Plural-Forms.+?\\n")\s*$', re.MULTILINE | re.DOTALL)
    1516
     17
    1618def handle_extensions(extensions=('html',), ignored=('py',)):
    1719    """
    1820    Organizes multiple extensions that are separated with commas or passed by
    def handle_extensions(extensions=('html',), ignored=('py',)): 
    3739            ext_list[i] = '.%s' % ext_list[i]
    3840    return set([x for x in ext_list if x.strip('.') not in ignored])
    3941
     42
    4043def _popen(cmd):
    4144    """
    4245    Friendly wrapper around Popen for Windows
    def _popen(cmd): 
    4447    p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=os.name != 'nt', universal_newlines=True)
    4548    return p.communicate()
    4649
     50
    4751def walk(root, topdown=True, onerror=None, followlinks=False):
    4852    """
    4953    A version of os.walk that can follow symlinks for Python < 2.6
    def walk(root, topdown=True, onerror=None, followlinks=False): 
    5761                    for link_dirpath, link_dirnames, link_filenames in walk(p):
    5862                        yield (link_dirpath, link_dirnames, link_filenames)
    5963
     64
    6065def is_ignored(path, ignore_patterns):
    6166    """
    6267    Helper function to check if the given path should be ignored or not.
    def is_ignored(path, ignore_patterns): 
    6671            return True
    6772    return False
    6873
     74
    6975def find_files(root, ignore_patterns, verbosity, symlinks=False):
    7076    """
    7177    Helper function to get all files in the given root.
    def find_files(root, ignore_patterns, verbosity, symlinks=False): 
    8288    all_files.sort()
    8389    return all_files
    8490
     91
    8592def copy_plural_forms(msgs, locale, domain, verbosity):
    8693    """
    8794    Copies plural forms header contents from a Django catalog of locale to
    def copy_plural_forms(msgs, locale, domain, verbosity): 
    112119                break
    113120    return msgs
    114121
    115 def write_pot_file(potfile, msgs, file, work_file, is_templatized):
     122
     123def write_pot_file(potfile, msgs, all_files):
    116124    """
    117125    Write the :param potfile: POT file with the :param msgs: contents,
    118126    previously making sure its format is valid.
    119127    """
    120     if is_templatized:
    121         old = '#: ' + work_file[2:]
    122         new = '#: ' + file[2:]
    123         msgs = msgs.replace(old, new)
     128    for f_data in all_files:
     129        if f_data['is_templatized']:
     130            old = '#: ' + f_data['work'][2:]
     131            new = '#: ' + f_data['orig'][2:]
     132            msgs = msgs.replace(old, new)
    124133    if os.path.exists(potfile):
    125134        # Strip the header
    126135        msgs = '\n'.join(dropwhile(len, msgs.split('\n')))
    def write_pot_file(potfile, msgs, file, work_file, is_templatized): 
    132141    finally:
    133142        f.close()
    134143
    135 def process_file(file, dirpath, potfile, domain, verbosity, extensions, wrap,
    136         location):
    137     """
    138     Extract translatable literals from :param file: for :param domain:
    139     creating or updating the :param potfile: POT file.
    140 
    141     Uses the xgettext GNU gettext utility.
    142     """
    143 
    144     from django.utils.translation import templatize
    145 
    146     if verbosity > 1:
    147         sys.stdout.write('processing file %s in %s\n' % (file, dirpath))
    148     _, file_ext = os.path.splitext(file)
    149     if domain == 'djangojs' and file_ext in extensions:
    150         is_templatized = True
    151         orig_file = os.path.join(dirpath, file)
    152         src_data = open(orig_file).read()
    153         src_data = prepare_js_for_gettext(src_data)
    154         thefile = '%s.c' % file
    155         work_file = os.path.join(dirpath, thefile)
    156         f = open(work_file, "w")
    157         try:
    158             f.write(src_data)
    159         finally:
    160             f.close()
    161         cmd = (
    162             'xgettext -d %s -L C %s %s --keyword=gettext_noop '
    163             '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
    164             '--keyword=pgettext:1c,2 --keyword=npgettext:1c,2,3 '
    165             '--from-code UTF-8 --add-comments=Translators -o - "%s"' % (
    166                 domain, wrap, location, work_file
    167             )
    168         )
    169     elif domain == 'django' and (file_ext == '.py' or file_ext in extensions):
    170         thefile = file
    171         orig_file = os.path.join(dirpath, file)
    172         is_templatized = file_ext in extensions
    173         if is_templatized:
    174             src_data = open(orig_file, "rU").read()
    175             thefile = '%s.py' % file
    176             content = templatize(src_data, orig_file[2:])
    177             f = open(os.path.join(dirpath, thefile), "w")
    178             try:
    179                 f.write(content)
    180             finally:
    181                 f.close()
    182         work_file = os.path.join(dirpath, thefile)
    183         cmd = (
    184             'xgettext -d %s -L Python %s %s --keyword=gettext_noop '
    185             '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
    186             '--keyword=ugettext_noop --keyword=ugettext_lazy '
    187             '--keyword=ungettext_lazy:1,2 --keyword=pgettext:1c,2 '
    188             '--keyword=npgettext:1c,2,3 --keyword=pgettext_lazy:1c,2 '
    189             '--keyword=npgettext_lazy:1c,2,3 --from-code UTF-8 '
    190             '--add-comments=Translators -o - "%s"' % (
    191                 domain, wrap, location, work_file)
    192         )
    193     else:
    194         return
    195     msgs, errors = _popen(cmd)
    196     if errors:
    197         if is_templatized:
    198             os.unlink(work_file)
    199         if os.path.exists(potfile):
    200             os.unlink(potfile)
    201         raise CommandError(
    202             "errors happened while running xgettext on %s\n%s" %
    203             (file, errors))
    204     if msgs:
    205         write_pot_file(potfile, msgs, orig_file, work_file, is_templatized)
    206     if is_templatized:
    207         os.unlink(work_file)
    208144
    209145def write_po_file(pofile, potfile, domain, locale, verbosity,
    210146        copy_pforms, wrap, location, no_obsolete):
    def write_po_file(pofile, potfile, domain, locale, verbosity, 
    248184            raise CommandError(
    249185                "errors happened while running msgattrib\n%s" % errors)
    250186
     187
    251188def make_messages(locale=None, domain='django', verbosity=1, all=False,
    252189        extensions=None, symlinks=False, ignore_patterns=None, no_wrap=False,
    253190        no_location=False, no_obsolete=False):
    def make_messages(locale=None, domain='django', verbosity=1, all=False, 
    323260        if os.path.exists(potfile):
    324261            os.unlink(potfile)
    325262
    326         for dirpath, file in find_files(".", ignore_patterns, verbosity,
    327                 symlinks=symlinks):
    328             process_file(file, dirpath, potfile, domain, verbosity, extensions,
    329                     wrap, location)
     263        process_files(potfile, domain, verbosity, extensions,
     264                      wrap, location, ignore_patterns, symlinks)
    330265
    331266        if os.path.exists(potfile):
    332267            write_po_file(pofile, potfile, domain, locale, verbosity,
    333268                    not invoked_for_django, wrap, location, no_obsolete)
    334269
    335270
     271def process_files(potfile, domain, verbosity, extensions, wrap, location,
     272                  ignore_patterns, symlinks):
     273    """
     274    Extract translatable literals from current directory for :param domain:
     275    creating or updating the :param potfile: POT file.
     276
     277    Uses the xgettext GNU gettext utility.
     278    """
     279    from django.utils.translation import templatize
     280
     281    # stores information about all processed files
     282    all_files = []
     283
     284    # prepare command to run xgettext depending on domain type
     285    if domain == 'djangojs':
     286        cmd = (
     287            'xgettext -d %s -L C %s %s --keyword=gettext_noop '
     288            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
     289            '--keyword=pgettext:1c,2 --keyword=npgettext:1c,2,3 '
     290            '--from-code UTF-8 --add-comments=Translators -o - -f %s'
     291        )
     292    elif domain == 'django':
     293        cmd = (
     294            'xgettext -d %s -L Python %s %s --keyword=gettext_noop '
     295            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
     296            '--keyword=ugettext_noop --keyword=ugettext_lazy '
     297            '--keyword=ungettext_lazy:1,2 --keyword=pgettext:1c,2 '
     298            '--keyword=npgettext:1c,2,3 --keyword=pgettext_lazy:1c,2 '
     299            '--keyword=npgettext_lazy:1c,2,3 --from-code UTF-8 '
     300            '--add-comments=Translators -o - -f %s'
     301        )
     302
     303    for dirpath, file in find_files(".", ignore_patterns, verbosity,
     304                                    symlinks=symlinks):
     305        if verbosity > 1:
     306            sys.stdout.write('processing file %s in %s\n' % (file, dirpath))
     307        _, file_ext = os.path.splitext(file)
     308
     309        is_templatized = file_ext in extensions
     310
     311        # path to the processed file
     312        orig_file = os.path.join(dirpath, file)
     313
     314        # store information about processed file;
     315        # 'orig' - path to original file
     316        # 'work' - path to either original file or templatized version if one is created
     317        # 'is_templatized' - defines if file was templatized
     318        file_dict = {'orig': orig_file, 'work': orig_file, 'is_templatized': is_templatized}
     319
     320        # not really necessary to initialize these here but added for clarity
     321        thefile = ''
     322        content = ''
     323
     324        if domain == 'djangojs' and is_templatized:
     325            src_data = open(orig_file).read()
     326            content = prepare_js_for_gettext(src_data)
     327            thefile = '%s.c' % file
     328        elif domain == 'django' and (file_ext == '.py' or is_templatized):
     329            if is_templatized:
     330                src_data = open(orig_file, "rU").read()
     331                content = templatize(src_data, orig_file[2:])
     332                thefile = '%s.py' % file
     333        else:
     334            continue
     335
     336        if is_templatized:
     337            # for templatized ones create work file
     338            work_file = os.path.join(dirpath, thefile)
     339            file_dict['work'] = work_file
     340
     341            f = open(work_file, "w")
     342            try:
     343                f.write(content)
     344            finally:
     345                f.close()
     346
     347        all_files.append(file_dict)
     348
     349    if all_files:
     350        # prepare command that runs xgettext with path to file that
     351        # contains paths of all files to process
     352        input_files = '\n'.join([f_data['work'] for f_data in all_files])
     353
     354        # create temporary file for xgettext
     355        f_handle, input_files_path = tempfile.mkstemp()
     356        f = open(input_files_path, 'w')
     357        try:
     358            f.write(input_files)
     359        finally:
     360            f.close()
     361
     362        cmd = cmd % (domain, wrap, location, input_files_path)
     363
     364        msgs, errors = _popen(cmd)
     365
     366        os.unlink(input_files_path)
     367
     368        if errors:
     369            # check if file exists because when symlinks are used it might
     370            # happen that it was already removed
     371            [os.unlink(f_data['work']) for f_data in all_files
     372               if f_data['is_templatized'] and os.path.exists(f_data['work'])]
     373
     374            input_files = ', '.join([f_data['work'] for f_data in all_files])
     375            raise CommandError(
     376                "errors happened while running xgettext on %s\n%s" %
     377                (input_files, errors))
     378        if msgs:
     379            write_pot_file(potfile, msgs, all_files)
     380        # check if file exists because when symlinks are used it might
     381        # happen that it was already removed
     382        [os.unlink(f_data['work']) for f_data in all_files
     383               if f_data['is_templatized'] and os.path.exists(f_data['work'])]
     384
     385
    336386class Command(NoArgsCommand):
    337387    option_list = NoArgsCommand.option_list + (
    338388        make_option('--locale', '-l', default=None, dest='locale',
  • tests/regressiontests/i18n/commands/extraction.py

    diff --git a/tests/regressiontests/i18n/commands/extraction.py b/tests/regressiontests/i18n/commands/extraction.py
    index b1fc002..7afded3 100644
    a b class ExtractorTests(TestCase): 
    4141        msgid = re.escape(msgid)
    4242        return self.assertTrue(re.search('^msgid %s' % msgid, s, re.MULTILINE), 'Could not find %(q)s%(n)s%(q)s in generated PO file' % {'n':needle, 'q':q})
    4343
     44    def assertMsgIdPlural(self, msgid, s, use_quotes=True):
     45        q = '"'
     46        if use_quotes:
     47            msgid = '"%s"' % msgid
     48            q = "'"
     49        needle = 'msgid_plural %s' % msgid
     50        msgid = re.escape(msgid)
     51        return self.assertTrue(re.search('^msgid_plural %s' % msgid, s, re.MULTILINE), 'Could not find %(q)s%(n)s%(q)s in generated PO file' % {'n':needle, 'q':q})
     52
    4453    def assertNotMsgId(self, msgid, s, use_quotes=True):
    4554        if use_quotes:
    4655            msgid = '"%s"' % msgid
    class NoLocationExtractorTests(ExtractorTests): 
    263272        with open(self.PO_FILE, 'r') as fp:
    264273            po_contents = fp.read()
    265274            self.assertTrue('#: templates/test.html:55' in po_contents)
     275
     276
     277class MultipleFilesPluralExtractorTests(ExtractorTests):
     278
     279    def setUp(self):
     280        self._cwd = os.getcwd()
     281        self.test_dir = os.path.abspath(os.path.dirname(__file__))
     282        self.second_test_template = os.path.join(self.test_dir, 'templates', 'plural_test_template.html')
     283
     284        second_test_template = '''{% load i18n %}{% trans 'My string' %}'''
     285
     286        if not os.path.exists(self.second_test_template):
     287            f = open(self.second_test_template, 'w+')
     288            try:
     289                f.write(second_test_template)
     290            finally:
     291                f.close()
     292
     293    def tearDown(self):
     294        super(MultipleFilesPluralExtractorTests, self).tearDown()
     295        os.chdir(self.test_dir)
     296        try:
     297            os.remove(self.second_test_template)
     298        except OSError:
     299            pass
     300        os.chdir(self._cwd)
     301
     302    def test_multiple_files_plural_extraction(self):
     303        os.chdir(self.test_dir)
     304        management.call_command('makemessages', locale=LOCALE, verbosity=0, symlinks=True)
     305        self.assertTrue(os.path.exists(self.PO_FILE))
     306        with open(self.PO_FILE, 'r') as fp:
     307            po_contents = fp.read()
     308            self.assertMsgId('My string', po_contents)
     309            self.assertMsgIdPlural('My strings', po_contents)
  • tests/regressiontests/i18n/commands/templates/test.html

    diff --git a/tests/regressiontests/i18n/commands/templates/test.html b/tests/regressiontests/i18n/commands/templates/test.html
    index 5789346..1e00a80 100644
    a b continued here.{% endcomment %} 
    7777{% trans "Shouldn't double escape this sequence %% either" context "ctx1" %}
    7878{% trans "Looks like a str fmt spec %s but shouldn't be interpreted as such" %}
    7979{% trans "Looks like a str fmt spec % o but shouldn't be interpreted as such" %}
     80{% blocktrans count counter=mylist|length %}My string{% plural %}My strings{% endblocktrans %}
  • tests/regressiontests/i18n/tests.py

    diff --git a/tests/regressiontests/i18n/tests.py b/tests/regressiontests/i18n/tests.py
    index 77bd35b..35f9060 100644
    a b if can_run_extraction_tests: 
    2929    from .commands.extraction import (ExtractorTests, BasicExtractorTests,
    3030        JavascriptExtractorTests, IgnoredExtractorTests, SymlinkExtractorTests,
    3131        CopyPluralFormsExtractorTests, NoWrapExtractorTests,
    32         NoLocationExtractorTests)
     32        NoLocationExtractorTests, MultipleFilesPluralExtractorTests)
    3333if can_run_compilation_tests:
    3434    from .commands.compilation import (PoFileTests, PoFileContentsTests,
    3535        PercentRenderingTests)
Back to Top