Code

Ticket #18533: makemessages.py

File makemessages.py, 16.9 KB (added by pasha.savchenko@…, 22 months ago)

fixed makemessages.py

Line 
1import fnmatch
2import glob
3import os
4import re
5import sys
6from itertools import dropwhile
7from optparse import make_option
8from subprocess import PIPE, Popen
9
10import django
11from django.core.management.base import CommandError, NoArgsCommand
12from django.utils.text import get_text_list
13from django.utils.jslex import prepare_js_for_gettext
14
15plural_forms_re = re.compile(r'^(?P<value>"Plural-Forms.+?\\n")\s*$', re.MULTILINE | re.DOTALL)
16
17def handle_extensions(extensions=('html',), ignored=('py',)):
18    """
19    Organizes multiple extensions that are separated with commas or passed by
20    using --extension/-e multiple times. Note that the .py extension is ignored
21    here because of the way non-*.py files are handled in make_messages() (they
22    are copied to file.ext.py files to trick xgettext to parse them as Python
23    files).
24
25    For example: running 'django-admin makemessages -e js,txt -e xhtml -a'
26    would result in an extension list: ['.js', '.txt', '.xhtml']
27
28    >>> handle_extensions(['.html', 'html,js,py,py,py,.py', 'py,.py'])
29    set(['.html', '.js'])
30    >>> handle_extensions(['.html, txt,.tpl'])
31    set(['.html', '.tpl', '.txt'])
32    """
33    ext_list = []
34    for ext in extensions:
35        ext_list.extend(ext.replace(' ', '').split(','))
36    for i, ext in enumerate(ext_list):
37        if not ext.startswith('.'):
38            ext_list[i] = '.%s' % ext_list[i]
39    return set([x for x in ext_list if x.strip('.') not in ignored])
40
41def _popen(cmd):
42    """
43    Friendly wrapper around Popen for Windows
44    """
45    p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=os.name != 'nt', universal_newlines=True)
46    return p.communicate()
47
48def walk(root, topdown=True, onerror=None, followlinks=False,
49         ignore_patterns=None, verbosity=0, stdout=sys.stdout):
50    """
51    A version of os.walk that can follow symlinks for Python < 2.6
52    """
53    if ignore_patterns is None:
54        ignore_patterns = []
55    dir_suffix = '%s*' % os.sep
56    norm_patterns = map(lambda p: p.endswith(dir_suffix)
57                        and p[:-len(dir_suffix)] or p, ignore_patterns)
58    for dirpath, dirnames, filenames in os.walk(root, topdown, onerror):
59        remove_dirs = []
60        for dirname in dirnames:
61            if is_ignored(os.path.normpath(os.path.join(dirpath, dirname)), norm_patterns):
62                remove_dirs.append(dirname)
63        for dirname in remove_dirs:
64            dirnames.remove(dirname)
65            if verbosity > 1:
66                stdout.write('ignoring directory %s\n' % dirname)
67        yield (dirpath, dirnames, filenames)
68        if followlinks:
69            for d in dirnames:
70                p = os.path.join(dirpath, d)
71                if os.path.islink(p):
72                    for link_dirpath, link_dirnames, link_filenames in walk(p):
73                        yield (link_dirpath, link_dirnames, link_filenames)
74
75def is_ignored(path, ignore_patterns):
76    """
77    Helper function to check if the given path should be ignored or not.
78    """
79    for pattern in ignore_patterns:
80        if fnmatch.fnmatchcase(path, pattern):
81            return True
82    return False
83
84def find_files(root, ignore_patterns, verbosity, stdout=sys.stdout, symlinks=False):
85    """
86    Helper function to get all files in the given root.
87    """
88    all_files = []
89    for (dirpath, dirnames, filenames) in walk(root, followlinks=symlinks,
90            ignore_patterns=ignore_patterns, verbosity=verbosity, stdout=stdout):
91        for filename in filenames:
92            norm_filepath = os.path.normpath(os.path.join(dirpath, filename))
93            if is_ignored(norm_filepath, ignore_patterns):
94                if verbosity > 1:
95                    stdout.write('ignoring file %s in %s\n' % (filename, dirpath))
96            else:
97                all_files.extend([(dirpath, filename)])
98    all_files.sort()
99    return all_files
100
101def copy_plural_forms(msgs, locale, domain, verbosity, stdout=sys.stdout):
102    """
103    Copies plural forms header contents from a Django catalog of locale to
104    the msgs string, inserting it at the right place. msgs should be the
105    contents of a newly created .po file.
106    """
107    django_dir = os.path.normpath(os.path.join(os.path.dirname(django.__file__)))
108    if domain == 'djangojs':
109        domains = ('djangojs', 'django')
110    else:
111        domains = ('django',)
112    for domain in domains:
113        django_po = os.path.join(django_dir, 'conf', 'locale', locale, 'LC_MESSAGES', '%s.po' % domain)
114        if os.path.exists(django_po):
115            m = plural_forms_re.search(open(django_po, 'rU').read())
116            if m:
117                if verbosity > 1:
118                    stdout.write("copying plural forms: %s\n" % m.group('value'))
119                lines = []
120                seen = False
121                for line in msgs.split('\n'):
122                    if not line and not seen:
123                        line = '%s\n' % m.group('value')
124                        seen = True
125                    lines.append(line)
126                msgs = '\n'.join(lines)
127                break
128    return msgs
129
130def write_pot_file(potfile, msgs, file, work_file, is_templatized):
131    """
132    Write the :param potfile: POT file with the :param msgs: contents,
133    previously making sure its format is valid.
134    """
135    if is_templatized:
136        old = '#: ' + work_file[2:]
137        new = '#: ' + file[2:]
138        msgs = msgs.replace(old, new)
139    if os.path.exists(potfile):
140        # Strip the header
141        msgs = '\n'.join(dropwhile(len, msgs.split('\n')))
142    else:
143        msgs = msgs.replace('charset=CHARSET', 'charset=UTF-8')
144    f = open(potfile, 'ab')
145    try:
146        f.write(msgs)
147    finally:
148        f.close()
149
150def process_file(file, dirpath, potfile, domain, verbosity,
151                 extensions, wrap, location, stdout=sys.stdout):
152    """
153    Extract translatable literals from :param file: for :param domain:
154    creating or updating the :param potfile: POT file.
155
156    Uses the xgettext GNU gettext utility.
157    """
158
159    from django.utils.translation import templatize
160
161    if verbosity > 1:
162        stdout.write('processing file %s in %s\n' % (file, dirpath))
163    _, file_ext = os.path.splitext(file)
164    if domain == 'djangojs' and file_ext in extensions:
165        is_templatized = True
166        orig_file = os.path.join(dirpath, file)
167        src_data = open(orig_file).read()
168        src_data = prepare_js_for_gettext(src_data)
169        thefile = '%s.c' % file
170        work_file = os.path.join(dirpath, thefile)
171        f = open(work_file, "w")
172        try:
173            f.write(src_data)
174        finally:
175            f.close()
176        cmd = (
177            'xgettext -d %s -L C %s %s --keyword=gettext_noop '
178            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
179            '--keyword=pgettext:1c,2 --keyword=npgettext:1c,2,3 '
180            '--from-code UTF-8 --add-comments=Translators -o - "%s"' %
181            (domain, wrap, location, work_file))
182    elif domain == 'django' and (file_ext == '.py' or file_ext in extensions):
183        thefile = file
184        orig_file = os.path.join(dirpath, file)
185        is_templatized = file_ext in extensions
186        if is_templatized:
187            src_data = open(orig_file, "rU").read()
188            thefile = '%s.py' % file
189            content = templatize(src_data, orig_file[2:])
190            f = open(os.path.join(dirpath, thefile), "w")
191            try:
192                f.write(content)
193            finally:
194                f.close()
195        work_file = os.path.abspath(os.path.join(dirpath, thefile))
196        cmd = (
197            'xgettext -d %s -L Python %s %s --keyword=gettext_noop '
198            '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
199            '--keyword=ugettext_noop --keyword=ugettext_lazy '
200            '--keyword=ungettext_lazy:1,2 --keyword=pgettext:1c,2 '
201            '--keyword=npgettext:1c,2,3 --keyword=pgettext_lazy:1c,2 '
202            '--keyword=npgettext_lazy:1c,2,3 --from-code UTF-8 '
203            '--add-comments=Translators -o - "%s"' %
204            (domain, wrap, location, work_file))
205    else:
206        return
207    msgs, errors = _popen(cmd)
208    if errors:
209        if is_templatized:
210            os.unlink(work_file)
211        if os.path.exists(potfile):
212            os.unlink(potfile)
213        raise CommandError(
214            "errors happened while running xgettext on %s\n%s" %
215            (file, errors))
216    if msgs:
217        write_pot_file(potfile, msgs, orig_file, work_file, is_templatized)
218    if is_templatized:
219        os.unlink(work_file)
220
221def write_po_file(pofile, potfile, domain, locale, verbosity, stdout,
222                  copy_pforms, wrap, location, no_obsolete):
223    """
224    Creates of updates the :param pofile: PO file for :param domain: and :param
225    locale:.  Uses contents of the existing :param potfile:.
226
227    Uses mguniq, msgmerge, and msgattrib GNU gettext utilities.
228    """
229    msgs, errors = _popen('msguniq %s %s --to-code=utf-8 "%s"' %
230                            (wrap, location, potfile))
231    if errors:
232        os.unlink(potfile)
233        raise CommandError("errors happened while running msguniq\n%s" % errors)
234    if os.path.exists(pofile):
235        f = open(potfile, 'w')
236        try:
237            f.write(msgs)
238        finally:
239            f.close()
240        msgs, errors = _popen('msgmerge %s %s -q "%s" "%s"' %
241                                (wrap, location, pofile, potfile))
242        if errors:
243            os.unlink(potfile)
244            raise CommandError(
245                "errors happened while running msgmerge\n%s" % errors)
246    elif copy_pforms:
247        msgs = copy_plural_forms(msgs, locale, domain, verbosity, stdout)
248    msgs = msgs.replace(
249        "#. #-#-#-#-#  %s.pot (PACKAGE VERSION)  #-#-#-#-#\n" % domain, "")
250    f = open(pofile, 'wb')
251    try:
252        f.write(msgs)
253    finally:
254        f.close()
255    os.unlink(potfile)
256    if no_obsolete:
257        msgs, errors = _popen('msgattrib %s %s -o "%s" --no-obsolete "%s"' %
258                                (wrap, location, pofile, pofile))
259        if errors:
260            raise CommandError(
261                "errors happened while running msgattrib\n%s" % errors)
262
263def make_messages(locale=None, domain='django', verbosity=1, all=False,
264        extensions=None, symlinks=False, ignore_patterns=None, no_wrap=False,
265        no_location=False, no_obsolete=False, stdout=sys.stdout):
266    """
267    Uses the ``locale/`` directory from the Django SVN tree or an
268    application/project to process all files with translatable literals for
269    the :param domain: domain and :param locale: locale.
270    """
271    # Need to ensure that the i18n framework is enabled
272    from django.conf import settings
273    if settings.configured:
274        settings.USE_I18N = True
275    else:
276        settings.configure(USE_I18N = True)
277
278    if ignore_patterns is None:
279        ignore_patterns = []
280
281    invoked_for_django = False
282    if os.path.isdir(os.path.join('conf', 'locale')):
283        localedir = os.path.abspath(os.path.join('conf', 'locale'))
284        invoked_for_django = True
285        # Ignoring all contrib apps
286        ignore_patterns += ['contrib/*']
287    elif os.path.isdir('locale'):
288        localedir = os.path.abspath('locale')
289    else:
290        raise CommandError("This script should be run from the Django SVN "
291                "tree or your project or app tree. If you did indeed run it "
292                "from the SVN checkout or your project or application, "
293                "maybe you are just missing the conf/locale (in the django "
294                "tree) or locale (for project and application) directory? It "
295                "is not created automatically, you have to create it by hand "
296                "if you want to enable i18n for your project or application.")
297
298    if domain not in ('django', 'djangojs'):
299        raise CommandError("currently makemessages only supports domains 'django' and 'djangojs'")
300
301    if (locale is None and not all) or domain is None:
302        message = "Type '%s help %s' for usage information." % (os.path.basename(sys.argv[0]), sys.argv[1])
303        raise CommandError(message)
304
305    # We require gettext version 0.15 or newer.
306    output = _popen('xgettext --version')[0]
307    match = re.search(r'(?P<major>\d+)\.(?P<minor>\d+)', output)
308    if match:
309        xversion = (int(match.group('major')), int(match.group('minor')))
310        if xversion < (0, 15):
311            raise CommandError("Django internationalization requires GNU "
312                    "gettext 0.15 or newer. You are using version %s, please "
313                    "upgrade your gettext toolset." % match.group())
314
315    locales = []
316    if locale is not None:
317        locales.append(locale)
318    elif all:
319        locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir))
320        locales = [os.path.basename(l) for l in locale_dirs]
321
322    wrap = '--no-wrap' if no_wrap else ''
323    location = '--no-location' if no_location else ''
324
325    for locale in locales:
326        if verbosity > 0:
327            stdout.write("processing language %s\n" % locale)
328        basedir = os.path.join(localedir, locale, 'LC_MESSAGES')
329        if not os.path.isdir(basedir):
330            os.makedirs(basedir)
331
332        pofile = os.path.join(basedir, '%s.po' % domain)
333        potfile = os.path.join(basedir, '%s.pot' % domain)
334
335        if os.path.exists(potfile):
336            os.unlink(potfile)
337
338        for dirpath, file in find_files(".", ignore_patterns, verbosity,
339                stdout, symlinks=symlinks):
340            process_file(file, dirpath, potfile, domain, verbosity, extensions,
341                    wrap, location, stdout)
342
343        if os.path.exists(potfile):
344            write_po_file(pofile, potfile, domain, locale, verbosity, stdout,
345                    not invoked_for_django, wrap, location, no_obsolete)
346
347
348class Command(NoArgsCommand):
349    option_list = NoArgsCommand.option_list + (
350        make_option('--locale', '-l', default=None, dest='locale',
351            help='Creates or updates the message files for the given locale (e.g. pt_BR).'),
352        make_option('--domain', '-d', default='django', dest='domain',
353            help='The domain of the message files (default: "django").'),
354        make_option('--all', '-a', action='store_true', dest='all',
355            default=False, help='Updates the message files for all existing locales.'),
356        make_option('--extension', '-e', dest='extensions',
357            help='The file extension(s) to examine (default: "html,txt", or "js" if the domain is "djangojs"). Separate multiple extensions with commas, or use -e multiple times.',
358            action='append'),
359        make_option('--symlinks', '-s', action='store_true', dest='symlinks',
360            default=False, help='Follows symlinks to directories when examining source code and templates for translation strings.'),
361        make_option('--ignore', '-i', action='append', dest='ignore_patterns',
362            default=[], metavar='PATTERN', help='Ignore files or directories matching this glob-style pattern. Use multiple times to ignore more.'),
363        make_option('--no-default-ignore', action='store_false', dest='use_default_ignore_patterns',
364            default=True, help="Don't ignore the common glob-style patterns 'CVS', '.*' and '*~'."),
365        make_option('--no-wrap', action='store_true', dest='no_wrap',
366            default=False, help="Don't break long message lines into several lines"),
367        make_option('--no-location', action='store_true', dest='no_location',
368            default=False, help="Don't write '#: filename:line' lines"),
369        make_option('--no-obsolete', action='store_true', dest='no_obsolete',
370            default=False, help="Remove obsolete message strings"),
371    )
372    help = ("Runs over the entire source tree of the current directory and "
373"pulls out all strings marked for translation. It creates (or updates) a message "
374"file in the conf/locale (in the django tree) or locale (for projects and "
375"applications) directory.\n\nYou must run this command with one of either the "
376"--locale or --all options.")
377
378    requires_model_validation = False
379    can_import_settings = False
380
381    def handle_noargs(self, *args, **options):
382        locale = options.get('locale')
383        domain = options.get('domain')
384        verbosity = int(options.get('verbosity'))
385        process_all = options.get('all')
386        extensions = options.get('extensions')
387        symlinks = options.get('symlinks')
388        ignore_patterns = options.get('ignore_patterns')
389        if options.get('use_default_ignore_patterns'):
390            ignore_patterns += ['CVS', '.*', '*~']
391        ignore_patterns = list(set(ignore_patterns))
392        no_wrap = options.get('no_wrap')
393        no_location = options.get('no_location')
394        no_obsolete = options.get('no_obsolete')
395        if domain == 'djangojs':
396            exts = extensions if extensions else ['js']
397        else:
398            exts = extensions if extensions else ['html', 'txt']
399        extensions = handle_extensions(exts)
400
401        if verbosity > 1:
402            self.stdout.write('examining files with the extensions: %s\n'
403                             % get_text_list(list(extensions), 'and'))
404
405        make_messages(locale, domain, verbosity, process_all, extensions,
406            symlinks, ignore_patterns, no_wrap, no_location, no_obsolete, self.stdout)