Finished Release v0.5.5

README.rst

@@ -113,6 +113,35 @@ The ``Database`` class also supports counting records easily::
     >>> db.count(Person, conditions="height > 1.90")
     6
 
+Pagination
+----------
+
+It is possible to paginate through model instances::
+
+    >>> order_by = 'first_name, last_name'
+    >>> page = db.paginate(Person, order_by, page_num=1, page_size=100)
+    >>> print page.number_of_objects
+    2507
+    >>> print page.pages_total
+    251
+    >>> for person in page.objects:
+    >>>     # do something
+
+The ``paginate`` method returns a ``namedtuple`` containing the following fields:
+
+- ``objects`` - the list of objects in this page
+- ``number_of_objects`` - total number of objects in all pages
+- ``pages_total`` - total number of pages
+- ``number`` - the page number
+- ``page_size`` - the number of objects per page
+
+You can optionally pass conditions to the query::
+
+    >>> page = db.paginate(Person, order_by, page_num=1, page_size=100, conditions='height > 1.90')
+
+Note that ``order_by`` must be chosen so that the ordering is unique, otherwise there might be
+inconsistencies in the pagination (such as an instance that appears on two different pages).
+
 Field Types
 -----------
 

src/infi/clickhouse_orm/database.py

@@ -1,6 +1,11 @@
 import requests
+from collections import namedtuple
 from models import ModelBase
 from utils import escape, parse_tsv
+from math import ceil
+
+
+Page = namedtuple('Page', 'objects number_of_objects pages_total number page_size')
 
 
 class DatabaseException(Exception):
@@ -14,7 +19,7 @@ class Database(object):
         self.db_url = db_url
         self.username = username
         self.password = password
-        self._send('CREATE DATABASE IF NOT EXISTS ' + db_name)
+        self._send('CREATE DATABASE IF NOT EXISTS `%s`' % db_name)
 
     def create_table(self, model_class):
         # TODO check that model has an engine
@@ -24,7 +29,7 @@ class Database(object):
         self._send(model_class.drop_table_sql(self.db_name))
 
     def drop_database(self):
-        self._send('DROP DATABASE ' + self.db_name)
+        self._send('DROP DATABASE `%s`' % self.db_name)
 
     def insert(self, model_instances):
         i = iter(model_instances)
@@ -34,7 +39,7 @@ class Database(object):
             return # model_instances is empty
         model_class = first_instance.__class__
         def gen():
-            yield 'INSERT INTO %s.%s FORMAT TabSeparated\n' % (self.db_name, model_class.table_name())
+            yield 'INSERT INTO `%s`.`%s` FORMAT TabSeparated\n' % (self.db_name, model_class.table_name())
             yield first_instance.to_tsv()
             yield '\n'
             for instance in i:
@@ -43,7 +48,7 @@ class Database(object):
         self._send(gen())
 
     def count(self, model_class, conditions=None):
-        query = 'SELECT count() FROM %s.%s' % (self.db_name, model_class.table_name())
+        query = 'SELECT count() FROM `%s`.`%s`' % (self.db_name, model_class.table_name())
         if conditions:
             query += ' WHERE ' + conditions
         r = self._send(query)
@@ -59,6 +64,23 @@ class Database(object):
         for line in lines:
             yield model_class.from_tsv(line, field_names)
 
+    def paginate(self, model_class, order_by, page_num=1, page_size=100, conditions=None, settings=None):
+        count = self.count(model_class, conditions)
+        pages_total = int(ceil(count / float(page_size)))
+        offset = (page_num - 1) * page_size
+        query = 'SELECT * FROM `%s`.`%s`' % (self.db_name, model_class.table_name())
+        if conditions:
+            query += ' WHERE ' + conditions
+        query += ' ORDER BY %s' % order_by
+        query += ' LIMIT %d, %d' % (offset, page_size)
+        return Page(
+            objects=list(self.select(query, model_class, settings)),
+            number_of_objects=count,
+            pages_total=pages_total,
+            number=page_num,
+            page_size=page_size
+        )
+
     def _send(self, data, settings=None):
         params = self._build_params(settings)
         r = requests.post(self.db_url, params=params, data=data, stream=True)

src/infi/clickhouse_orm/fields.py

@@ -73,7 +73,8 @@ class DateField(Field):
         if isinstance(value, int):
             return DateField.class_default + datetime.timedelta(days=value)
         if isinstance(value, basestring):
-            # TODO parse '0000-00-00'
+            if value == '0000-00-00':
+                return DateField.min_value
             return datetime.datetime.strptime(value, '%Y-%m-%d').date()
         raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
 
@@ -97,7 +98,7 @@ class DateTimeField(Field):
         if isinstance(value, int):
             return datetime.datetime.fromtimestamp(value, pytz.utc)
         if isinstance(value, basestring):
-            return datetime.datetime.strptime(value, '%Y-%m-%d %H-%M-%S')
+            return datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S')
         raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
 
     def get_db_prep_value(self, value):
@@ -107,11 +108,10 @@ class DateTimeField(Field):
 class BaseIntField(Field):
 
     def to_python(self, value):
-        if isinstance(value, int):
+        try:
-            return value
-        if isinstance(value, basestring):
             return int(value)
-        raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
+        except:
+            raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
 
     def validate(self, value):
         self._range_check(value, self.min_value, self.max_value)
@@ -176,11 +176,10 @@ class Int64Field(BaseIntField):
 class BaseFloatField(Field):
 
     def to_python(self, value):
-        if isinstance(value, float):
+        try:
-            return value
-        if isinstance(value, basestring) or isinstance(value, int):
             return float(value)
-        raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
+        except:
+            raise ValueError('Invalid value for %s - %r' % (self.__class__.__name__, value))
 
 
 class Float32Field(BaseFloatField):

src/infi/clickhouse_orm/models.py

@@ -92,7 +92,7 @@ class Model(object):
         '''
         Returns the SQL command for creating a table for this model.
         '''
-        parts = ['CREATE TABLE IF NOT EXISTS %s.%s (' % (db_name, cls.table_name())]
+        parts = ['CREATE TABLE IF NOT EXISTS `%s`.`%s` (' % (db_name, cls.table_name())]
         cols = []
         for name, field in cls._fields:
             default = field.get_db_prep_value(field.default)
@@ -107,7 +107,7 @@ class Model(object):
         '''
         Returns the SQL command for deleting this model's table.
         '''
-        return 'DROP TABLE IF EXISTS %s.%s' % (db_name, cls.table_name())
+        return 'DROP TABLE IF EXISTS `%s`.`%s`' % (db_name, cls.table_name())
 
     @classmethod
     def from_tsv(cls, line, field_names=None):

tests/test_database.py

@@ -5,11 +5,14 @@ from infi.clickhouse_orm.models import Model
 from infi.clickhouse_orm.fields import *
 from infi.clickhouse_orm.engines import *
 
+import logging
+logging.getLogger("requests").setLevel(logging.WARNING)
+
 
 class DatabaseTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.database = Database('test_db')
+        self.database = Database('test-db')
         self.database.create_table(Person)
 
     def tearDown(self):
@@ -41,7 +44,7 @@ class DatabaseTestCase(unittest.TestCase):
 
     def test_select(self):
         self._insert_and_check(self._sample_data(), len(data))
-        query = "SELECT * FROM test_db.person WHERE first_name = 'Whitney' ORDER BY last_name"
+        query = "SELECT * FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name"
         results = list(self.database.select(query, Person))
         self.assertEquals(len(results), 2)
         self.assertEquals(results[0].last_name, 'Durham')
@@ -51,7 +54,7 @@ class DatabaseTestCase(unittest.TestCase):
 
     def test_select_partial_fields(self):
         self._insert_and_check(self._sample_data(), len(data))
-        query = "SELECT first_name, last_name FROM test_db.person WHERE first_name = 'Whitney' ORDER BY last_name"
+        query = "SELECT first_name, last_name FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name"
         results = list(self.database.select(query, Person))
         self.assertEquals(len(results), 2)
         self.assertEquals(results[0].last_name, 'Durham')
@@ -61,7 +64,7 @@ class DatabaseTestCase(unittest.TestCase):
 
     def test_select_ad_hoc_model(self):
         self._insert_and_check(self._sample_data(), len(data))
-        query = "SELECT * FROM test_db.person WHERE first_name = 'Whitney' ORDER BY last_name"
+        query = "SELECT * FROM `test-db`.person WHERE first_name = 'Whitney' ORDER BY last_name"
         results = list(self.database.select(query))
         self.assertEquals(len(results), 2)
         self.assertEquals(results[0].__class__.__name__, 'AdHocModel')
@@ -70,6 +73,24 @@ class DatabaseTestCase(unittest.TestCase):
         self.assertEquals(results[1].last_name, 'Scott')
         self.assertEquals(results[1].height, 1.70)
 
+    def test_pagination(self):
+        self._insert_and_check(self._sample_data(), len(data))
+        # Try different page sizes
+        for page_size in (1, 2, 7, 10, 30, 100, 150):
+            # Iterate over pages and collect all intances
+            page_num = 1
+            instances = set()
+            while True:
+                page = self.database.paginate(Person, 'first_name, last_name', page_num, page_size)
+                self.assertEquals(page.number_of_objects, len(data))
+                self.assertGreater(page.pages_total, 0)
+                [instances.add(obj.to_tsv()) for obj in page.objects]
+                if page.pages_total == page_num:
+                    break
+                page_num += 1
+            # Verify that all instances were returned
+            self.assertEquals(len(instances), len(data))
+
     def _sample_data(self):
         for entry in data:
             yield Person(**entry)