Index: django_website/apps/nightlies/views.py =================================================================== --- django_website/apps/nightlies/views.py (revision 0) +++ django_website/apps/nightlies/views.py (revision 0) @@ -0,0 +1,118 @@ +""" +This is a simple module to get and create nightly snapshots of Django releases +as defined in the documentation model. + +The archive files are by default saved in the media root in a automatically +created directory "nightlies" with filename: "django-nightly-YYYY-MM-DD.EXT" +where EXT can be "zip", "tar.gz" or "tar.bz2". + +There are currently two possible ways of creating the nightlies: + +1. The first visitor to the url "/nightlies/latest/" will create a gzipped +tarfile of the trunk (the default) and save it in the nightlies directory. +"/nightlies/latest.zip" will create a zipfile, "/nightlies/latest.tar.bz2" a +bzip2'ed tarball accordingly. A version number can be passed too: +"/nightlies/0.95/latest.zip" and "/nightlies/0.96/latest.tar.gz" + +2. Using the make_nightlies() utility from build_nightlies.py as a cronjob +creates a archive in every format (see ARCHIVE_FORMATS) from every repository +from the documentation model. +""" + +import os +import sys +import stat +import pysvn +import shutil +import tarfile +import zipfile +import urlparse +from datetime import datetime +from tempfile import NamedTemporaryFile, mkdtemp +from django_website.apps.docs.models import DocumentRelease + +from django.conf import settings +from django.http import Http404, HttpResponseRedirect + +def make_tarball (base_name, tmpdir, save_path, compress="gz"): + archive_name = os.path.join(save_path, base_name, ".tar.%s" % compress) + tar = tarfile.open(archive_name, 'w:%s' % compress) + tar.posix = False + tar.add(tmpdir,"") + tar.close() + return archive_name + +def make_zipfile (base_name, tmpdir, save_path, compress=None): + archive_name = os.path.join(save_path, base_name, ".zip") + z = zipfile.ZipFile(archive_name, "w", compression = zipfile.ZIP_DEFLATED) + for root, dirs, files in os.walk(tmpdir): + for f in files: + filepath = os.path.join(root, f) + nice_filepath = "/".join(filepath.split("/")[len(tmpdir.split("/")):]) + if os.path.isfile(filepath): + z.write(filepath, nice_filepath) + z.close() + return archive_name + +ARCHIVE_FORMATS = { + 'tar.gz': (make_tarball, [('compress', 'gz')], "gzip'ed tar-file"), + 'tar.bz2': (make_tarball, [('compress', 'bz2')], "bzip2'ed tar-file"), + 'zip': (make_zipfile, [],"ZIP file") + } + +def make_archive (base_name, ext, tmpdir, kwargs = {}): + try: + format_info = ARCHIVE_FORMATS[ext] + except KeyError: + raise ValueError, "unknown archive format '%s'" % ext + + func = format_info[0] + for (arg,val) in format_info[1]: + kwargs[arg] = val + + save_path = os.path.join(settings.MEDIA_ROOT, "nightlies") + if not os.path.exists(save_path): + os.makedirs(save_path) + + filename = apply(func, (base_name, tmpdir, save_path), kwargs) + return filename + +def get_nightly(request, ext="tar.gz", version=None): + nightly_label, filename, nightly_path, nightly_url = _nightly_vars(ext) + + if not os.path.exists(nightly_path): + client, version, svnroot, tmpdir = _export_svn(version) + try: + make_archive("%s-%s" % (nightly_label, version), ext, tmpdir) + except: + raise Http404("Error while saving: %s" % filename) + shutil.rmtree(tmpdir) + return HttpResponseRedirect(nightly_url) + +def _nightly_vars(ext): + label = datetime.now().strftime("django-nightly-%Y-%m-%d") + filename = "%s.%s" % (label, ext) + path = os.path.join(settings.MEDIA_ROOT, "nightlies", filename) + url = urlparse.urljoin(settings.MEDIA_URL, "nightlies", filename) + + return label, filename, path, url + +def _export_svn(version): + tmpdir = mkdtemp() + client = pysvn.Client() + + if version is None: + version = "trunk" + subpath = "trunk/" + else: + rel = get_object_or_404(DocumentRelease, version=version) + subpath = rel.repository_path + svnroot = urlparse.urljoin(settings.DJANGO_SVN_ROOT, subpath) + + try: + client.export(svnroot, os.path.join(tmpdir, "django")) + except pysvn.ClientError: + raise Http404("Bad SVN path: %s" % svnroot) + + return client, version, svnroot, tmpdir + Index: django_website/apps/nightlies/__init__.py =================================================================== Index: django_website/apps/nightlies/build_nightlies.py =================================================================== --- django_website/apps/nightlies/build_nightlies.py (revision 0) +++ django_website/apps/nightlies/build_nightlies.py (revision 0) @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +""" +Build nightly snapshots of django. + +Can be run as a cronjob to fetch nightly snapshots of every django release in +the documentation model. +""" +from django.conf import settings +from django_website.apps.docs.models import DocumentRelease +from django_website.apps.docs.views import * + +def make_nightlies(): + for rel in DocumentRelease.objects.all(): + client, version, svnroot, tmpdir = _export_svn(rel.version) + for ext in ARCHIVE_FORMATS: + nightly_label, filename, nightly_path, nightly_url = _nightly_vars(ext) + make_archive("%s-%s" % (nightly_label, version), ext, tmpdir) + shutil.rmtree(tmpdir) + if settings.DEBUG: + print "Successfully archived version %s (%s)." % (rel.version, nightly_url) + +if __name__ == "__main__": + make_nightlies() Index: django_website/apps/nightlies/urls.py =================================================================== --- django_website/apps/nightlies/urls.py (revision 0) +++ django_website/apps/nightlies/urls.py (revision 0) @@ -0,0 +1,8 @@ +from django.conf.urls.defaults import * + +urlpatterns = patterns('django_website.apps.nightlies.views', + (r'^latest/$', 'get_nightly'), + (r'^latest.(?P[\w\.]+)$', 'get_nightly'), + (r'^(?P[\d.]+)/latest/$', 'get_nightly'), + (r'^(?P[\d.]+)/latest.(?P[\w\.]+)$', 'get_nightly'), +) Index: django_website/settings.py =================================================================== --- django_website/settings.py (revision 4829) +++ django_website/settings.py (working copy) @@ -44,6 +44,7 @@ 'django_website.apps.blog', 'django_website.apps.docs', 'django_website.apps.aggregator', +# 'django_website.apps.nightlies', ) ADMIN_MEDIA_PREFIX = 'http://media.djangoproject.com/admin/' MEDIA_ROOT = "/home/html/djangoproject.com/m/" Index: django_website/urls.py =================================================================== --- django_website/urls.py (revision 4829) +++ django_website/urls.py (working copy) @@ -32,6 +32,7 @@ urlpatterns = patterns('', (r'^weblog/', include('django_website.apps.blog.urls')), (r'^documentation/', include('django_website.apps.docs.urls')), + (r'^nightlies/', include('django_website.apps.nightlies.urls')), (r'^comments/$', 'django.views.generic.list_detail.object_list', comments_info_dict), (r'^comments/', include('django.contrib.comments.urls.comments')), (r'^community/$', 'django.views.generic.list_detail.object_list', aggregator_info_dict),