Code

Ticket #1327: nightly.diff

File nightly.diff, 7.3 KB (added by Jannis Leidel <jl@…>, 7 years ago)

Initial approach of a nightly serving app for django_website using pysvn

Line 
1Index: django_website/apps/nightlies/views.py
2===================================================================
3--- django_website/apps/nightlies/views.py      (revision 0)
4+++ django_website/apps/nightlies/views.py      (revision 0)
5@@ -0,0 +1,117 @@
6+"""
7+This is a simple module to get and create nightly snapshots of Django releases
8+as defined in the documentation model.
9+
10+The archive files are by default saved in the media root in a automatically
11+created directory "nightlies" with filename: "django-nightly-YYYY-MM-DD.EXT"
12+where EXT can be "zip", "tar.gz" or "tar.bz2".
13+
14+There are currently two possible ways of creating the nightlies:
15+
16+1. The first visitor to the url "/nightlies/latest/" will create a gzipped
17+tarfile of the trunk (the default) and save it in the nightlies directory.
18+"/nightlies/latest.zip" will create a zipfile, "/nightlies/latest.tar.bz2" a
19+bzip2'ed tarball accordingly. A version number can be passed too:
20+"/nightlies/0.95/latest.zip" and "/nightlies/0.96/latest.tar.gz"
21+
22+2. Using the make_nightlies() utility from build_nightlies.py as a cronjob
23+creates a archive in every format (see ARCHIVE_FORMATS) from every repository
24+from the documentation model.
25+"""
26+
27+import os
28+import sys
29+import stat
30+import pysvn
31+import shutil
32+import tarfile
33+import zipfile
34+import urlparse
35+from datetime import datetime
36+from tempfile import NamedTemporaryFile, mkdtemp
37+from django_website.apps.docs.models import DocumentRelease
38+
39+from django.conf import settings
40+from django.http import Http404, HttpResponseRedirect
41+
42+def make_tarball (base_name, tmpdir, save_path, compress="gz"):
43+    archive_name = os.path.join(save_path, base_name, ".tar.%s" % compress)
44+    tar = tarfile.open(archive_name, 'w:%s' % compress)
45+    tar.posix = False
46+    tar.add(tmpdir,"")
47+    tar.close()
48+    return archive_name
49+
50+def make_zipfile (base_name, tmpdir, save_path, compress=None):
51+    archive_name = os.path.join(save_path, base_name, ".zip")
52+    z = zipfile.ZipFile(archive_name, "w", compression = zipfile.ZIP_DEFLATED)
53+    for root, dirs, files in os.walk(tmpdir):
54+        for f in files:
55+            filepath = os.path.join(root, f)
56+            nice_filepath = "/".join(filepath.split("/")[len(tmpdir.split("/")):])
57+            if os.path.isfile(filepath):
58+                z.write(filepath, nice_filepath)
59+    z.close()
60+    return archive_name
61+
62+ARCHIVE_FORMATS = {
63+    'tar.gz': (make_tarball, [('compress', 'gz')], "gzip'ed tar-file"),
64+    'tar.bz2': (make_tarball, [('compress', 'bz2')], "bzip2'ed tar-file"),
65+    'zip':   (make_zipfile, [],"ZIP file")
66+    }
67+
68+def make_archive (base_name, ext, tmpdir, kwargs = {}):
69+    try:
70+        format_info = ARCHIVE_FORMATS[ext]
71+    except KeyError:
72+        raise ValueError, "unknown archive format '%s'" % ext
73+
74+    func = format_info[0]
75+    for (arg,val) in format_info[1]:
76+        kwargs[arg] = val
77+
78+    save_path = os.path.join(settings.MEDIA_ROOT, "nightlies")
79+    if not os.path.exists(save_path):
80+        os.makedirs(save_path)
81+
82+    filename = apply(func, (base_name, tmpdir, save_path), kwargs)
83+    return filename
84+
85+def get_nightly(request, ext="tar.gz", version=None):
86+    nightly_label, filename, nightly_path, nightly_url = _get_nightly_vars(ext)
87+
88+    if not os.path.exists(nightly_path):
89+        client, version, svnroot, tmpdir = _export_svn(version)
90+        try:
91+            make_archive(nightly_label, ext, tmpdir)
92+        except:
93+            raise Http404("Error while saving: %s" % filename)
94+        shutil.rmtree(tmpdir)
95+    return HttpResponseRedirect(nightly_url)
96+
97+def _nightly_vars(ext):
98+    label = datetime.now().strftime("django-nightly-%Y-%m-%d")
99+    filename = "%s.%s" % (label, ext)
100+    path = os.path.join(settings.MEDIA_ROOT, "nightlies", filename)
101+    url = urlparse.urljoin(settings.MEDIA_URL, "nightlies", filename)
102+
103+    return label, filename, path, url
104+
105+def _export_svn(version):
106+    tmpdir = mkdtemp()
107+    client = pysvn.Client()
108+
109+    if version is None:
110+        subpath = "trunk/"
111+    else:
112+        rel = get_object_or_404(DocumentRelease, version=version)
113+        subpath = rel.repository_path
114+    svnroot = urlparse.urljoin(settings.DJANGO_SVN_ROOT, subpath)
115+
116+    try:
117+        client.export(svnroot, os.path.join(tmpdir, "django"))
118+    except pysvn.ClientError:
119+        raise Http404("Bad SVN path: %s" % svnroot)
120+
121+    return client, version, svnroot, tmpdir
122+
123Index: django_website/apps/nightlies/__init__.py
124===================================================================
125Index: django_website/apps/nightlies/build_nightlies.py
126===================================================================
127--- django_website/apps/nightlies/build_nightlies.py    (revision 0)
128+++ django_website/apps/nightlies/build_nightlies.py    (revision 0)
129@@ -0,0 +1,24 @@
130+#!/usr/bin/env python
131+
132+"""
133+Build nightly snapshots of django.
134+
135+Can be run as a cronjob to fetch nightly snapshots of every django release in
136+the documentation model.
137+"""
138+from django.conf import settings
139+from django_website.apps.docs.models import DocumentRelease
140+from django_website.apps.docs.views import *
141+
142+def make_nightlies():
143+    for rel in DocumentRelease.objects.all():
144+        client, version, svnroot, tmpdir = _export_svn(rel.version)
145+        for ext in ARCHIVE_FORMATS:
146+            nightly_label, filename, nightly_path, nightly_url = _nightly_vars(ext)
147+            make_archive(nightly_label, ext, tmpdir)
148+            shutil.rmtree(tmpdir)
149+            if settings.DEBUG:
150+                print "Successfully archived version %s (%s)." % (rel.version, nightly_url)
151+   
152+if __name__ == "__main__":
153+    make_nightlies()
154Index: django_website/apps/nightlies/urls.py
155===================================================================
156--- django_website/apps/nightlies/urls.py       (revision 0)
157+++ django_website/apps/nightlies/urls.py       (revision 0)
158@@ -0,0 +1,8 @@
159+from django.conf.urls.defaults import *
160+
161+urlpatterns = patterns('django_website.apps.nightlies.views',
162+    (r'^latest/$',                                      'get_nightly'),
163+    (r'^latest.(?P<ext>[\w\.]+)$',                      'get_nightly'),
164+    (r'^(?P<version>[\d.]+)/latest/$',                  'get_nightly'),
165+    (r'^(?P<version>[\d.]+)/latest.(?P<ext>[\w\.]+)$',  'get_nightly'),
166+)
167Index: django_website/settings.py
168===================================================================
169--- django_website/settings.py  (revision 4829)
170+++ django_website/settings.py  (working copy)
171@@ -44,6 +44,7 @@
172     'django_website.apps.blog',
173     'django_website.apps.docs',
174     'django_website.apps.aggregator',
175+    'django_website.apps.nightlies',
176 )
177 ADMIN_MEDIA_PREFIX = 'http://media.djangoproject.com/admin/'
178 MEDIA_ROOT = "/home/html/djangoproject.com/m/"
179Index: django_website/urls.py
180===================================================================
181--- django_website/urls.py      (revision 4829)
182+++ django_website/urls.py      (working copy)
183@@ -32,6 +32,7 @@
184 urlpatterns = patterns('',
185     (r'^weblog/', include('django_website.apps.blog.urls')),
186     (r'^documentation/', include('django_website.apps.docs.urls')),
187+    (r'^nightlies/', include('django_website.apps.nightlies.urls')),
188     (r'^comments/$', 'django.views.generic.list_detail.object_list', comments_info_dict),
189     (r'^comments/', include('django.contrib.comments.urls.comments')),
190     (r'^community/$', 'django.views.generic.list_detail.object_list', aggregator_info_dict),