support pagination

README.rst

@@ -113,6 +113,32 @@ The ``Database`` class also supports counting records easily::
     >>> db.count(Person, conditions="height > 1.90")
     6
 
+Pagination
+----------
+
+It is possible to paginate through model instances::
+
+    >>> order_by = 'first_name, last_name'
+    >>> page = db.paginate(Person, order_by, page_num=1, page_size=100)
+    >>> print page.number_of_objects
+    2507
+    >>> print page.pages_total
+    251
+    >>> for person in page.objects:
+    >>>     # do something
+
+The ``paginate`` method returns a ``namedtuple`` containing the following fields:
+
+- ``objects`` - the list of objects in this page
+- ``number_of_objects`` - total number of objects in all pages
+- ``pages_total`` - total number of pages
+- ``number`` - the page number
+- ``page_size`` - the number of objects per page
+
+You can optionally pass conditions to the query::
+
+    >>> page = db.paginate(Person, order_by, page_num=1, page_size=100, conditions='height > 1.90')
+
 Field Types
 -----------
 
@@ -178,4 +204,4 @@ After cloning the project, run the following commands::
 
 To run the tests, ensure that the ClickHouse server is running on http://localhost:8123/ (this is the default), and run::
 
-    bin/nosetests
+    bin/nosetests

src/infi/clickhouse_orm/database.py

@@ -1,6 +1,11 @@
 import requests
+from collections import namedtuple
 from models import ModelBase
 from utils import escape, parse_tsv
+from math import ceil
+
+
+Page = namedtuple('Page', 'objects number_of_objects pages_total number page_size')
 
 
 class DatabaseException(Exception):
@@ -59,6 +64,23 @@ class Database(object):
         for line in lines:
             yield model_class.from_tsv(line, field_names)
 
+    def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None):
+        count = self.count(model_class, conditions)
+        pages_total = int(ceil(count / float(page_size)))
+        offset = (page_num - 1) * page_size
+        query = 'SELECT * FROM `%s`.`%s`' % (self.db_name, model_class.table_name())
+        if conditions:
+            query += ' WHERE ' + conditions
+        query += ' ORDER BY %s' % order_by
+        query += ' LIMIT %d, %d' % (offset, page_size)
+        return Page(
+            objects=list(self.select(query, model_class, settings)),
+            number_of_objects=count,
+            pages_total=pages_total,
+            number=page_num,
+            page_size=page_size
+        )
+
     def _send(self, data, settings=None):
         params = self._build_params(settings)
         r = requests.post(self.db_url, params=params, data=data, stream=True)

tests/test_database.py

@@ -5,6 +5,9 @@ from infi.clickhouse_orm.models import Model
 from infi.clickhouse_orm.fields import *
 from infi.clickhouse_orm.engines import *
 
+import logging
+logging.getLogger("requests").setLevel(logging.WARNING)
+
 
 class DatabaseTestCase(unittest.TestCase):
 
@@ -70,6 +73,24 @@ class DatabaseTestCase(unittest.TestCase):
         self.assertEquals(results[1].last_name, 'Scott')
         self.assertEquals(results[1].height, 1.70)
 
+    def test_pagination(self):
+        self._insert_and_check(self._sample_data(), len(data))
+        # Try different page sizes
+        for page_size in (1, 2, 7, 10, 30, 100, 150):
+            # Iterate over pages and collect all intances
+            page_num = 1
+            instances = set()
+            while True:
+                page = self.database.paginate(Person, 'first_name, last_name', page_num, page_size)
+                self.assertEquals(page.number_of_objects, len(data))
+                self.assertGreater(page.pages_total, 0)
+                [instances.add(obj.to_tsv()) for obj in page.objects]
+                if page.pages_total == page_num:
+                    break
+                page_num += 1
+            # Verify that all instances were returned
+            self.assertEquals(len(instances), len(data))
+
     def _sample_data(self):
         for entry in data:
             yield Person(**entry)