Ticket #1327: nightly3.diff

File nightly3.diff, 7.4 KB (added by Jannis Leidel <jl@…>, 8 years ago)

Adds version labels to archive filenames to differ them when creating archives with the included cron utility

  • django_website/apps/nightlies/views.py

     
     1"""
     2This is a simple module to get and create nightly snapshots of Django releases
     3as defined in the documentation model.
     4
     5The archive files are by default saved in the media root in a automatically
     6created directory "nightlies" with filename: "django-nightly-YYYY-MM-DD.EXT"
     7where EXT can be "zip", "tar.gz" or "tar.bz2".
     8
     9There are currently two possible ways of creating the nightlies:
     10
     111. The first visitor to the url "/nightlies/latest/" will create a gzipped
     12tarfile of the trunk (the default) and save it in the nightlies directory.
     13"/nightlies/latest.zip" will create a zipfile, "/nightlies/latest.tar.bz2" a
     14bzip2'ed tarball accordingly. A version number can be passed too:
     15"/nightlies/0.95/latest.zip" and "/nightlies/0.96/latest.tar.gz"
     16
     172. Using the make_nightlies() utility from build_nightlies.py as a cronjob
     18creates a archive in every format (see ARCHIVE_FORMATS) from every repository
     19from the documentation model.
     20"""
     21
     22import os
     23import sys
     24import stat
     25import pysvn
     26import shutil
     27import tarfile
     28import zipfile
     29import urlparse
     30from datetime import datetime
     31from tempfile import NamedTemporaryFile, mkdtemp
     32from django_website.apps.docs.models import DocumentRelease
     33
     34from django.conf import settings
     35from django.http import Http404, HttpResponseRedirect
     36
     37def make_tarball (base_name, tmpdir, save_path, compress="gz"):
     38    archive_name = os.path.join(save_path, base_name, ".tar.%s" % compress)
     39    tar = tarfile.open(archive_name, 'w:%s' % compress)
     40    tar.posix = False
     41    tar.add(tmpdir,"")
     42    tar.close()
     43    return archive_name
     44
     45def make_zipfile (base_name, tmpdir, save_path, compress=None):
     46    archive_name = os.path.join(save_path, base_name, ".zip")
     47    z = zipfile.ZipFile(archive_name, "w", compression = zipfile.ZIP_DEFLATED)
     48    for root, dirs, files in os.walk(tmpdir):
     49        for f in files:
     50            filepath = os.path.join(root, f)
     51            nice_filepath = "/".join(filepath.split("/")[len(tmpdir.split("/")):])
     52            if os.path.isfile(filepath):
     53                z.write(filepath, nice_filepath)
     54    z.close()
     55    return archive_name
     56
     57ARCHIVE_FORMATS = {
     58    'tar.gz': (make_tarball, [('compress', 'gz')], "gzip'ed tar-file"),
     59    'tar.bz2': (make_tarball, [('compress', 'bz2')], "bzip2'ed tar-file"),
     60    'zip':   (make_zipfile, [],"ZIP file")
     61    }
     62
     63def make_archive (base_name, ext, tmpdir, kwargs = {}):
     64    try:
     65        format_info = ARCHIVE_FORMATS[ext]
     66    except KeyError:
     67        raise ValueError, "unknown archive format '%s'" % ext
     68
     69    func = format_info[0]
     70    for (arg,val) in format_info[1]:
     71        kwargs[arg] = val
     72
     73    save_path = os.path.join(settings.MEDIA_ROOT, "nightlies")
     74    if not os.path.exists(save_path):
     75        os.makedirs(save_path)
     76
     77    filename = apply(func, (base_name, tmpdir, save_path), kwargs)
     78    return filename
     79
     80def get_nightly(request, ext="tar.gz", version=None):
     81    nightly_label, filename, nightly_path, nightly_url = _nightly_vars(ext)
     82
     83    if not os.path.exists(nightly_path):
     84        client, version, svnroot, tmpdir = _export_svn(version)
     85        try:
     86            make_archive("%s-%s" % (nightly_label, version), ext, tmpdir)
     87        except:
     88            raise Http404("Error while saving: %s" % filename)
     89        shutil.rmtree(tmpdir)
     90    return HttpResponseRedirect(nightly_url)
     91
     92def _nightly_vars(ext):
     93    label = datetime.now().strftime("django-nightly-%Y-%m-%d")
     94    filename = "%s.%s" % (label, ext)
     95    path = os.path.join(settings.MEDIA_ROOT, "nightlies", filename)
     96    url = urlparse.urljoin(settings.MEDIA_URL, "nightlies", filename)
     97
     98    return label, filename, path, url
     99
     100def _export_svn(version):
     101    tmpdir = mkdtemp()
     102    client = pysvn.Client()
     103
     104    if version is None:
     105        version = "trunk"
     106        subpath = "trunk/"
     107    else:
     108        rel = get_object_or_404(DocumentRelease, version=version)
     109        subpath = rel.repository_path
     110    svnroot = urlparse.urljoin(settings.DJANGO_SVN_ROOT, subpath)
     111
     112    try:
     113        client.export(svnroot, os.path.join(tmpdir, "django"))
     114    except pysvn.ClientError:
     115        raise Http404("Bad SVN path: %s" % svnroot)
     116
     117    return client, version, svnroot, tmpdir
     118
  • django_website/apps/nightlies/build_nightlies.py

     
     1#!/usr/bin/env python
     2
     3"""
     4Build nightly snapshots of django.
     5
     6Can be run as a cronjob to fetch nightly snapshots of every django release in
     7the documentation model.
     8"""
     9from django.conf import settings
     10from django_website.apps.docs.models import DocumentRelease
     11from django_website.apps.docs.views import *
     12
     13def make_nightlies():
     14    for rel in DocumentRelease.objects.all():
     15        client, version, svnroot, tmpdir = _export_svn(rel.version)
     16        for ext in ARCHIVE_FORMATS:
     17            nightly_label, filename, nightly_path, nightly_url = _nightly_vars(ext)
     18            make_archive("%s-%s" % (nightly_label, version), ext, tmpdir)
     19            shutil.rmtree(tmpdir)
     20            if settings.DEBUG:
     21                print "Successfully archived version %s (%s)." % (rel.version, nightly_url)
     22   
     23if __name__ == "__main__":
     24    make_nightlies()
  • django_website/apps/nightlies/urls.py

     
     1from django.conf.urls.defaults import *
     2
     3urlpatterns = patterns('django_website.apps.nightlies.views',
     4    (r'^latest/$',                                      'get_nightly'),
     5    (r'^latest.(?P<ext>[\w\.]+)$',                      'get_nightly'),
     6    (r'^(?P<version>[\d.]+)/latest/$',                  'get_nightly'),
     7    (r'^(?P<version>[\d.]+)/latest.(?P<ext>[\w\.]+)$',  'get_nightly'),
     8)
  • django_website/settings.py

     
    4444    'django_website.apps.blog',
    4545    'django_website.apps.docs',
    4646    'django_website.apps.aggregator',
     47#    'django_website.apps.nightlies',
    4748)
    4849ADMIN_MEDIA_PREFIX = 'http://media.djangoproject.com/admin/'
    4950MEDIA_ROOT = "/home/html/djangoproject.com/m/"
  • django_website/urls.py

     
    3232urlpatterns = patterns('',
    3333    (r'^weblog/', include('django_website.apps.blog.urls')),
    3434    (r'^documentation/', include('django_website.apps.docs.urls')),
     35    (r'^nightlies/', include('django_website.apps.nightlies.urls')),
    3536    (r'^comments/$', 'django.views.generic.list_detail.object_list', comments_info_dict),
    3637    (r'^comments/', include('django.contrib.comments.urls.comments')),
    3738    (r'^community/$', 'django.views.generic.list_detail.object_list', aggregator_info_dict),
Back to Top