Ticket #5012: negative_sliced_querysets.patch

File negative_sliced_querysets.patch, 16.6 KB (added by Chris Beaven, 17 years ago)

  • django/db/backends/mysql/base.py

     
    177177def get_datetime_cast_sql():
    178178    return None
    179179
    180 def get_limit_offset_sql(limit, offset=None):
     180def get_limit_offset_sql(limit=None, offset=None):
     181    if not limit and not offset:
     182        return ''
    181183    sql = "LIMIT "
    182     if offset and offset != 0:
     184    if not limit:
     185        # Per mySQL manual:
     186        # To retrieve all rows from a certain offset up to the end of the result
     187        # set, you can use some large number for the second parameter.
     188        limit = 18446744073709551615
     189    if offset:
    183190        sql += "%s," % offset
    184191    return sql + str(limit)
    185192

  • django/db/backends/mysql_old/base.py

     
    192192def get_datetime_cast_sql():
    193193    return None
    194194
    195 def get_limit_offset_sql(limit, offset=None):
     195def get_limit_offset_sql(limit=None, offset=None):
     196    if not limit and not offset:
     197        return ''
    196198    sql = "LIMIT "
    197     if offset and offset != 0:
     199    if not limit:
     200        # Per mySQL manual:
     201        # To retrieve all rows from a certain offset up to the end of the result
     202        # set, you can use some large number for the second parameter.
     203        limit = 18446744073709551615
     204    if offset:
    198205        sql += "%s," % offset
    199206    return sql + str(limit)
    200207

  • django/db/backends/postgresql/base.py

     
    158158def get_datetime_cast_sql():
    159159    return None
    160160
    161 def get_limit_offset_sql(limit, offset=None):
    162     sql = "LIMIT %s" % limit
    163     if offset and offset != 0:
     161def get_limit_offset_sql(limit=None, offset=None):
     162    if not limit and not offset:
     163        return ''
     164    # From PostgreSQL manual: "LIMIT { count | ALL } OFFSET start"
     165    sql = "LIMIT %s" % (limit or 'ALL')
     166    if offset:
    164167        sql += " OFFSET %s" % offset
    165168    return sql
    166169

  • django/db/backends/postgresql_psycopg2/base.py

     
    112112def get_datetime_cast_sql():
    113113    return None
    114114
    115 def get_limit_offset_sql(limit, offset=None):
    116     sql = "LIMIT %s" % limit
    117     if offset and offset != 0:
     115def get_limit_offset_sql(limit=None, offset=None):
     116    if not limit and not offset:
     117        return ''
     118    sql = "LIMIT %s" % (limit or 'ALL')
     119    if offset:
    118120        sql += " OFFSET %s" % offset
    119121    return sql
    120122

  • django/db/backends/sqlite3/base.py

     
    142142def get_datetime_cast_sql():
    143143    return None
    144144
    145 def get_limit_offset_sql(limit, offset=None):
    146     sql = "LIMIT %s" % limit
     145def get_limit_offset_sql(limit=None, offset=None):
     146    if not limit and not offset:
     147        return ''
     148    # From sqlite manual: "A negative LIMIT indicates no upper bound."
     149    sql = "LIMIT %s" % (limit or -1)
    147150    if offset and offset != 0:
    148151        sql += " OFFSET %s" % offset
    149152    return sql

  • django/db/models/query.py

     
    1515except NameError:
    1616    from sets import Set as set   # Python 2.3 fallback
    1717
     18try:
     19    reversed
     20except NameError:
     21    from django.utils.itercompat import reversed   # Python 2.3 fallback
     22
    1823# The string constant used to separate query parts
    1924LOOKUP_SEPARATOR = '__'
    2025
    … …  
    8287    else:
    8388        return backend.quote_name(word)
    8489
     90def negative_slice(k):
     91    return (k.start and k.start < 0) or (k.stop and k.stop < 0)
     92
     93def invert_slice(k):
     94    """
     95    Reverse the slice, inverting the negation and swapping the start and stop
     96    slice positions.
     97    """
     98    new_start = k.stop and -k.stop or None
     99    if k.stop == 0:
     100        # Edge case of [-x:0] should remain empty.
     101        new_stop = 0
     102    else:
     103        new_stop = k.start and -k.start or None
     104    return slice(new_start, new_stop)
     105
     106def invert_item(k):
     107    "Reverse the __getitem__ integer item position."
     108    return -k - 1
     109
    85110class _QuerySet(object):
    86111    "Represents a lazy database lookup for a set of objects"
    87112    def __init__(self, model=None):
    … …  
    97122        self._tables = []            # List of extra tables to use.
    98123        self._offset = None          # OFFSET clause.
    99124        self._limit = None           # LIMIT clause.
     125        self._reversed = None        # Reverse the db results (used for negative slicing)
    100126        self._result_cache = None
    101127
    102128    ########################
    … …  
    116142        "Retrieve an item or slice from the set of results."
    117143        if not isinstance(k, (slice, int)):
    118144            raise TypeError
    119         assert (not isinstance(k, slice) and (k >= 0)) \
    120             or (isinstance(k, slice) and (k.start is None or k.start >= 0) and (k.stop is None or k.stop >= 0)), \
    121             "Negative indexing is not supported."
     145        if isinstance(k, slice):
     146            if k.start and k.stop:
     147                assert not ((k.start < 0) ^ (k.stop < 0)), \
     148                    "Slicing with both positive and negative indexes is not supported."
     149            if k.start and k.start < 0 or k.stop and k.stop < 0:
     150                assert not k.step or k.step == 1, \
     151                    "Slicing with a negative index and a step is not supported."
    122152        if self._result_cache is None:
     153            extra_clone_args = {}
    123154            if isinstance(k, slice):
     155                if self._reversed:
     156                    # Because we will be dealing with a reversed QuerySet, the
     157                    # slice should be inverted.
     158                    if not negative_slice(k):
     159                        # Can't negative slice an already reversed QuerySet, so
     160                        # just slice it as a list. Note, we haven't done the
     161                        # inversion yet, hence reverse logic in the condition.
     162                        return self._get_data()[k]
     163                    k = invert_slice(k)
     164                elif negative_slice(k):
     165                    extra_clone_args['_reversed'] = True
     166                    k = invert_slice(k)
    124167                # Offset:
    125168                if self._offset is None:
    126169                    offset = k.start
    … …  
    137180                # Limit:
    138181                if k.stop is not None and k.start is not None:
    139182                    if limit is None:
    140                         limit = k.stop - k.start
     183                        limit = max(k.stop - k.start, 0)
    141184                    else:
    142185                        limit = min((k.stop - k.start), limit)
    143186                else:
    … …  
    147190                        if k.stop is not None:
    148191                            limit = min(k.stop, limit)
    149192
    150                 if k.step is None:
    151                     return self._clone(_offset=offset, _limit=limit)
     193                if k.step is None or k.step == 1:
     194                    return self._clone(_offset=offset, _limit=limit,
     195                                       **extra_clone_args)
    152196                else:
    153                     return list(self._clone(_offset=offset, _limit=limit))[::k.step]
     197                    return list(self._clone(_offset=offset, _limit=limit,
     198                                            **extra_clone_args))[::k.step]
    154199            else:
     200                if self._reversed:
     201                    # Because we will be dealing with a reversed QuerySet, the
     202                    # item position should be inverted.
     203                    if k >= 0:
     204                        # Can't negate already offset reverse slice via the
     205                        # database way, just have to get it as a list. Note, we
     206                        # haven't done the inversion yet, hence reverse logic in
     207                        # the condition.
     208                        return self._get_data()[k]
     209                    # Reversed queryset lookups need to take the current offset
     210                    # into consideration.
     211                    k += self._offset
     212                    k = invert_item(k)
     213                elif k < 0:
     214                    k = invert_item(k)
     215                    extra_clone_args['_reversed'] = True
    155216                try:
    156                     return list(self._clone(_offset=k, _limit=1))[0]
     217                    return list(self._clone(_offset=k, _limit=1,
     218                                            **extra_clone_args))[0]
    157219                except self.model.DoesNotExist, e:
    158220                    raise IndexError, e.args
    159221        else:
    … …  
    174236    ####################################
    175237
    176238    def iterator(self):
     239        if self._reversed:
     240            # Being a reverse slice, the order will be reversed from what the
     241            # user expects.
     242            return reversed(list(self._iterator()))
     243        return self._iterator()
     244    def _iterator(self):
    177245        "Performs the SELECT database lookup of this QuerySet."
    178246        try:
    179247            select, sql, params = self._get_sql_clause()
    180248        except EmptyResultSet:
    181249            raise StopIteration
     250        if self._limit == 0:
     251            raise StopIteration
    182252
    183253        # self._select is a dictionary, and dictionaries' key order is
    184254        # undefined, so we convert it to a list of tuples.
    … …  
    452522        c._tables = self._tables[:]
    453523        c._offset = self._offset
    454524        c._limit = self._limit
     525        c._reversed = self._reversed
    455526        c.__dict__.update(kwargs)
    456527        return c
    457528
    … …  
    525596
    526597        # ORDER BY clause
    527598        order_by = []
    528         if self._order_by is not None:
    529             ordering_to_use = self._order_by
    530         else:
    531             ordering_to_use = opts.ordering
    532         for f in handle_legacy_orderlist(ordering_to_use):
     599        for f in handle_legacy_orderlist(self._get_ordering()):
    533600            if f == '?': # Special case.
    534601                order_by.append(backend.get_random_function_sql())
    535602            else:
    … …  
    554621            sql.append("ORDER BY " + ", ".join(order_by))
    555622
    556623        # LIMIT and OFFSET clauses
    557         if self._limit is not None:
     624        if self._limit is not None or self._offset is not None:
    558625            sql.append("%s " % backend.get_limit_offset_sql(self._limit, self._offset))
    559         else:
    560             assert self._offset is None, "'offset' is not allowed without 'limit'"
    561626
    562627        return select, " ".join(sql), params
    563628
     629    def _get_ordering(self):
     630        if self._order_by is not None:
     631            ordering = self._order_by
     632        else:
     633            ordering = self.model._meta.ordering
     634        if not self._reversed:
     635            return ordering
     636        # Negative indexing requires inverting the ordering.
     637        assert ordering, 'Negative indexing requires explicit ordering'
     638        reverse_ordering = lambda f: (f.startswith('-') and f[1:]
     639                                      or '-%s' % f)
     640        return [reverse_ordering(f) for f in ordering]
     641
    564642# Use the backend's QuerySet class if it defines one, otherwise use _QuerySet.
    565643if hasattr(backend, 'get_query_set_class'):
    566644    QuerySet = backend.get_query_set_class(_QuerySet)
    … …  
    573651        # select_related isn't supported in values().
    574652        self._select_related = False
    575653
    576     def iterator(self):
     654    def _iterator(self):
    577655        try:
    578656            select, sql, params = self._get_sql_clause()
    579657        except EmptyResultSet:
    … …  
    621699        return c
    622700
    623701class DateQuerySet(QuerySet):
    624     def iterator(self):
     702    def _iterator(self):
    625703        from django.db.backends.util import typecast_timestamp
    626704        from django.db.models.fields import DateTimeField
    627705        self._order_by = () # Clear this because it'll mess things up otherwise.

  • docs/db-api.txt

     
    434434Note, however, that the first of these will raise ``IndexError`` while the
    435435second will raise ``DoesNotExist`` if no objects match the given criteria.
    436436
     437**New in Django development version**
     438
     439You can also use negative slicing and negative indexes, provided the
     440``QuerySet``s has an explicit order (either via ``.order_by()`` or the model's
     441meta ``ordering`` option). For example::
     442
     443    entries_by_date_order = Entry.objects.order_by('pub_date')
     444    oldest_entry = entries_by_date_order[0]
     445    latest_entry = entries_by_date_order[-1]
     446    latest_three_entries = entries_by_date_order[:-3]
     447
    437448QuerySet methods that return new QuerySets
    438449------------------------------------------
    439450

  • tests/modeltests/basic/models.py

     
    246246>>> s3 = Article.objects.filter(id__exact=3)
    247247>>> (s1 | s2 | s3)[::2]
    248248[<Article: Area woman programs in Python>, <Article: Third article>]
     249>>> Article.objects.all()[2:]
     250[<Article: Third article>, <Article: Article 6>, <Article: Default headline>, <Article: Fourth article>, <Article: Article 7>, <Article: Updated article 8>]
    249251
     252
    250253# Slices (without step) are lazy:
    251254>>> Article.objects.all()[0:5].filter()
    252255[<Article: Area woman programs in Python>, <Article: Second article>, <Article: Third article>, <Article: Article 6>, <Article: Default headline>]
    … …  
    268271[<Article: Third article>, <Article: Article 6>]
    269272>>> Article.objects.all()[2:][2:3]
    270273[<Article: Default headline>]
     274>>> Article.objects.all()[-5:-2]
     275[<Article: Article 6>, <Article: Default headline>, <Article: Fourth article>]
     276>>> Article.objects.all()[-5:-2][1]
     277<Article: Default headline>
     278>>> Article.objects.all()[-5:-2][-1]
     279<Article: Fourth article>
     280>>> Article.objects.all()[-5:-2][1:]
     281[<Article: Default headline>, <Article: Fourth article>]
     282>>> Article.objects.all()[-5:-2][-2:]
     283[<Article: Default headline>, <Article: Fourth article>]
     284>>> Article.objects.all()[-5:-2][:2]
     285[<Article: Article 6>, <Article: Default headline>]
     286>>> Article.objects.all()[-5:-2][:-2]
     287[<Article: Article 6>]
     288>>> Article.objects.all()[-5:-2][:-10]
     289[]
     290>>> Article.objects.all()[-5:-2][-10:]
     291[<Article: Article 6>, <Article: Default headline>, <Article: Fourth article>]
    271292
    272 # Note that you can't use 'offset' without 'limit' (on some dbs), so this doesn't work:
    273 >>> Article.objects.all()[2:]
    274 Traceback (most recent call last):
    275     ...
    276 AssertionError: 'offset' is not allowed without 'limit'
    277 
    278 # Also, once you have sliced you can't filter, re-order or combine
     293# Once you have sliced you can't filter, re-order or combine
    279294>>> Article.objects.all()[0:5].filter(id=1)
    280295Traceback (most recent call last):
    281296    ...
    … …  
    291306    ...
    292307AssertionError: Cannot combine queries once a slice has been taken.
    293308
    294 # Negative slices are not supported, due to database constraints.
    295 # (hint: inverting your ordering might do what you need).
     309# Negative slices are possible, provided if the QuerySet has explicit ordering.
     310# (done by internally inverting the ordering).
    296311>>> Article.objects.all()[-1]
     312<Article: Updated article 8>
     313>>> Article.objects.all()[0:-5]
     314[<Article: Area woman programs in Python>, <Article: Second article>, <Article: Third article>]
     315>>> Article.objects.all()[-2:]
     316[<Article: Article 7>, <Article: Updated article 8>]
     317>>> Article.objects.all()[:-7]
     318[<Article: Area woman programs in Python>]
     319>>> Article.objects.all()[-2:-1]
     320[<Article: Article 7>]
     321>>> Article.objects.all()[-2:0]
     322[]
     323>>> Article.objects.all()[-2:-3]
     324[]
     325
     326# Can't mix negative and positive positions when slicing.
     327>>> Article.objects.all()[1:-1]
    297328Traceback (most recent call last):
    298329    ...
    299 AssertionError: Negative indexing is not supported.
    300 >>> Article.objects.all()[0:-5]
     330AssertionError: Slicing with both positive and negative indexes is not supported.
     331>>> Article.objects.all()[-5:2]
    301332Traceback (most recent call last):
    302333    ...
    303 AssertionError: Negative indexing is not supported.
     334AssertionError: Slicing with both positive and negative indexes is not supported.
    304335
    305336# An Article instance doesn't have access to the "objects" attribute.
    306337# That's only available on the class.
Back to Top