Ticket #1327: nightly3.diff

django_website/apps/nightlies/views.py

@@ +1 @@
+"""
+This is a simple module to get and create nightly snapshots of Django releases
+as defined in the documentation model.
+
+The archive files are by default saved in the media root in a automatically
+created directory "nightlies" with filename: "django-nightly-YYYY-MM-DD.EXT"
+where EXT can be "zip", "tar.gz" or "tar.bz2".
+
+There are currently two possible ways of creating the nightlies:
+
+1. The first visitor to the url "/nightlies/latest/" will create a gzipped
+tarfile of the trunk (the default) and save it in the nightlies directory.
+"/nightlies/latest.zip" will create a zipfile, "/nightlies/latest.tar.bz2" a
+bzip2'ed tarball accordingly. A version number can be passed too:
+"/nightlies/0.95/latest.zip" and "/nightlies/0.96/latest.tar.gz"
+
+2. Using the make_nightlies() utility from build_nightlies.py as a cronjob
+creates a archive in every format (see ARCHIVE_FORMATS) from every repository
+from the documentation model.
+"""
+
+import os
+import sys
+import stat
+import pysvn
+import shutil
+import tarfile
+import zipfile
+import urlparse
+from datetime import datetime
+from tempfile import NamedTemporaryFile, mkdtemp
+from django_website.apps.docs.models import DocumentRelease
+
+from django.conf import settings
+from django.http import Http404, HttpResponseRedirect
+
+def make_tarball (base_name, tmpdir, save_path, compress="gz"):
+    archive_name = os.path.join(save_path, base_name, ".tar.%s" % compress)
+    tar = tarfile.open(archive_name, 'w:%s' % compress)
+    tar.posix = False
+    tar.add(tmpdir,"")
+    tar.close()
+    return archive_name
+
+def make_zipfile (base_name, tmpdir, save_path, compress=None):
+    archive_name = os.path.join(save_path, base_name, ".zip")
+    z = zipfile.ZipFile(archive_name, "w", compression = zipfile.ZIP_DEFLATED)
+    for root, dirs, files in os.walk(tmpdir):
+        for f in files:
+            filepath = os.path.join(root, f)
+            nice_filepath = "/".join(filepath.split("/")[len(tmpdir.split("/")):])
+            if os.path.isfile(filepath):
+                z.write(filepath, nice_filepath)
+    z.close()
+    return archive_name
+
+ARCHIVE_FORMATS = {
+    'tar.gz': (make_tarball, [('compress', 'gz')], "gzip'ed tar-file"),
+    'tar.bz2': (make_tarball, [('compress', 'bz2')], "bzip2'ed tar-file"),
+    'zip':   (make_zipfile, [],"ZIP file")
+    }
+
+def make_archive (base_name, ext, tmpdir, kwargs = {}):
+    try:
+        format_info = ARCHIVE_FORMATS[ext]
+    except KeyError:
+        raise ValueError, "unknown archive format '%s'" % ext
+
+    func = format_info[0]
+    for (arg,val) in format_info[1]:
+        kwargs[arg] = val
+
+    save_path = os.path.join(settings.MEDIA_ROOT, "nightlies")
+    if not os.path.exists(save_path):
+        os.makedirs(save_path)
+
+    filename = apply(func, (base_name, tmpdir, save_path), kwargs)
+    return filename
+
+def get_nightly(request, ext="tar.gz", version=None):
+    nightly_label, filename, nightly_path, nightly_url = _nightly_vars(ext)
+
+    if not os.path.exists(nightly_path):
+        client, version, svnroot, tmpdir = _export_svn(version)
+        try:
+            make_archive("%s-%s" % (nightly_label, version), ext, tmpdir)
+        except:
+            raise Http404("Error while saving: %s" % filename)
+        shutil.rmtree(tmpdir)
+    return HttpResponseRedirect(nightly_url)
+
+def _nightly_vars(ext):
+    label = datetime.now().strftime("django-nightly-%Y-%m-%d")
+    filename = "%s.%s" % (label, ext)
+    path = os.path.join(settings.MEDIA_ROOT, "nightlies", filename)
+    url = urlparse.urljoin(settings.MEDIA_URL, "nightlies", filename)
+
+    return label, filename, path, url
+
+def _export_svn(version):
+    tmpdir = mkdtemp()
+    client = pysvn.Client()
+
+    if version is None:
+        version = "trunk"
+        subpath = "trunk/"
+    else:
+        rel = get_object_or_404(DocumentRelease, version=version)
+        subpath = rel.repository_path
+    svnroot = urlparse.urljoin(settings.DJANGO_SVN_ROOT, subpath)
+
+    try:
+        client.export(svnroot, os.path.join(tmpdir, "django"))
+    except pysvn.ClientError:
+        raise Http404("Bad SVN path: %s" % svnroot)
+
+    return client, version, svnroot, tmpdir
+

django_website/apps/nightlies/build_nightlies.py

@@ +1 @@
+#!/usr/bin/env python
+
+"""
+Build nightly snapshots of django.
+
+Can be run as a cronjob to fetch nightly snapshots of every django release in
+the documentation model.
+"""
+from django.conf import settings
+from django_website.apps.docs.models import DocumentRelease
+from django_website.apps.docs.views import *
+
+def make_nightlies():
+    for rel in DocumentRelease.objects.all():
+        client, version, svnroot, tmpdir = _export_svn(rel.version)
+        for ext in ARCHIVE_FORMATS:
+            nightly_label, filename, nightly_path, nightly_url = _nightly_vars(ext)
+            make_archive("%s-%s" % (nightly_label, version), ext, tmpdir)
+            shutil.rmtree(tmpdir)
+            if settings.DEBUG:
+                print "Successfully archived version %s (%s)." % (rel.version, nightly_url)
+    
+if __name__ == "__main__":
+    make_nightlies()

django_website/apps/nightlies/urls.py

@@ +1 @@
+from django.conf.urls.defaults import *
+
+urlpatterns = patterns('django_website.apps.nightlies.views',
+    (r'^latest/$',                                      'get_nightly'),
+    (r'^latest.(?P<ext>[\w\.]+)$',                      'get_nightly'),
+    (r'^(?P<version>[\d.]+)/latest/$',                  'get_nightly'),
+    (r'^(?P<version>[\d.]+)/latest.(?P<ext>[\w\.]+)$',  'get_nightly'),
+)

django_website/settings.py

@@ -44 +44 @@
     'django_website.apps.blog',
     'django_website.apps.docs',
     'django_website.apps.aggregator',
+#    'django_website.apps.nightlies',
 )
 ADMIN_MEDIA_PREFIX = 'http://media.djangoproject.com/admin/'
 MEDIA_ROOT = "/home/html/djangoproject.com/m/"

django_website/urls.py

@@ -32 +32 @@
 urlpatterns = patterns('',
     (r'^weblog/', include('django_website.apps.blog.urls')),
     (r'^documentation/', include('django_website.apps.docs.urls')),
+    (r'^nightlies/', include('django_website.apps.nightlies.urls')),
     (r'^comments/$', 'django.views.generic.list_detail.object_list', comments_info_dict),
     (r'^comments/', include('django.contrib.comments.urls.comments')),
     (r'^community/$', 'django.views.generic.list_detail.object_list', aggregator_info_dict),