support pagination

This commit is contained in:
Itai Shirav 2016-07-03 15:39:08 +03:00
parent a3bd212133
commit b4f4af8c75
3 changed files with 70 additions and 1 deletions

View File

@ -113,6 +113,32 @@ The ``Database`` class also supports counting records easily::
>>> db.count(Person, conditions="height > 1.90") >>> db.count(Person, conditions="height > 1.90")
6 6
Pagination
----------
It is possible to paginate through model instances::
>>> order_by = 'first_name, last_name'
>>> page = db.paginate(Person, order_by, page_num=1, page_size=100)
>>> print page.number_of_objects
2507
>>> print page.pages_total
251
>>> for person in page.objects:
>>> # do something
The ``paginate`` method returns a ``namedtuple`` containing the following fields:
- ``objects`` - the list of objects in this page
- ``number_of_objects`` - total number of objects in all pages
- ``pages_total`` - total number of pages
- ``number`` - the page number
- ``page_size`` - the number of objects per page
You can optionally pass conditions to the query::
>>> page = db.paginate(Person, order_by, page_num=1, page_size=100, conditions='height > 1.90')
Field Types Field Types
----------- -----------

View File

@ -1,6 +1,11 @@
import requests import requests
from collections import namedtuple
from models import ModelBase from models import ModelBase
from utils import escape, parse_tsv from utils import escape, parse_tsv
from math import ceil
Page = namedtuple('Page', 'objects number_of_objects pages_total number page_size')
class DatabaseException(Exception): class DatabaseException(Exception):
@ -59,6 +64,23 @@ class Database(object):
for line in lines: for line in lines:
yield model_class.from_tsv(line, field_names) yield model_class.from_tsv(line, field_names)
def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None):
count = self.count(model_class, conditions)
pages_total = int(ceil(count / float(page_size)))
offset = (page_num - 1) * page_size
query = 'SELECT * FROM `%s`.`%s`' % (self.db_name, model_class.table_name())
if conditions:
query += ' WHERE ' + conditions
query += ' ORDER BY %s' % order_by
query += ' LIMIT %d, %d' % (offset, page_size)
return Page(
objects=list(self.select(query, model_class, settings)),
number_of_objects=count,
pages_total=pages_total,
number=page_num,
page_size=page_size
)
def _send(self, data, settings=None): def _send(self, data, settings=None):
params = self._build_params(settings) params = self._build_params(settings)
r = requests.post(self.db_url, params=params, data=data, stream=True) r = requests.post(self.db_url, params=params, data=data, stream=True)

View File

@ -5,6 +5,9 @@ from infi.clickhouse_orm.models import Model
from infi.clickhouse_orm.fields import * from infi.clickhouse_orm.fields import *
from infi.clickhouse_orm.engines import * from infi.clickhouse_orm.engines import *
import logging
logging.getLogger("requests").setLevel(logging.WARNING)
class DatabaseTestCase(unittest.TestCase): class DatabaseTestCase(unittest.TestCase):
@ -70,6 +73,24 @@ class DatabaseTestCase(unittest.TestCase):
self.assertEquals(results[1].last_name, 'Scott') self.assertEquals(results[1].last_name, 'Scott')
self.assertEquals(results[1].height, 1.70) self.assertEquals(results[1].height, 1.70)
def test_pagination(self):
self._insert_and_check(self._sample_data(), len(data))
# Try different page sizes
for page_size in (1, 2, 7, 10, 30, 100, 150):
# Iterate over pages and collect all intances
page_num = 1
instances = set()
while True:
page = self.database.paginate(Person, 'first_name, last_name', page_num, page_size)
self.assertEquals(page.number_of_objects, len(data))
self.assertGreater(page.pages_total, 0)
[instances.add(obj.to_tsv()) for obj in page.objects]
if page.pages_total == page_num:
break
page_num += 1
# Verify that all instances were returned
self.assertEquals(len(instances), len(data))
def _sample_data(self): def _sample_data(self):
for entry in data: for entry in data:
yield Person(**entry) yield Person(**entry)