mirror of
https://github.com/Infinidat/infi.clickhouse_orm.git
synced 2024-11-10 19:36:33 +03:00
Added usage examples
This commit is contained in:
parent
80b220c1e3
commit
97773d6dc7
36
examples/db_explorer/README.md
Normal file
36
examples/db_explorer/README.md
Normal file
|
@ -0,0 +1,36 @@
|
||||||
|
# DB Explorer
|
||||||
|
|
||||||
|
This is a simple Flask web application that connects to ClickHouse and displays the list of existing databases. Clicking on a database name drills down into it, showing its list of tables. Clicking on a table drills down further, showing details about the table and its columns.
|
||||||
|
|
||||||
|
For each table or column, the application displays the compressed size on disk, the uncompressed size, and the ratio between them. Additionally, several pie charts are shown - top tables by size, top tables by rows, and top columns by size (in a table).
|
||||||
|
|
||||||
|
The pie charts are generated using the `pygal` charting library.
|
||||||
|
|
||||||
|
ORM concepts that are demonstrated by this example:
|
||||||
|
|
||||||
|
- Creating ORM models from existing tables using `Database.get_model_for_table`
|
||||||
|
- Queryset filtering
|
||||||
|
- Queryset aggregation
|
||||||
|
|
||||||
|
## Running the code
|
||||||
|
|
||||||
|
Create a virtualenv and install the required libraries:
|
||||||
|
```
|
||||||
|
virtualenv -p python3.6 env
|
||||||
|
source env/bin/activate
|
||||||
|
pip install -r requirements.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the server and open http://127.0.0.1:5000/ in your browser:
|
||||||
|
```
|
||||||
|
python server.py
|
||||||
|
```
|
||||||
|
|
||||||
|
By default the server connects to ClickHouse running on http://localhost:8123/ without a username or password, but you can change this using command line arguments:
|
||||||
|
```
|
||||||
|
python server.py http://myclickhouse:8123/
|
||||||
|
```
|
||||||
|
or:
|
||||||
|
```
|
||||||
|
python server.py http://myclickhouse:8123/ admin secret123
|
||||||
|
```
|
|
@ -3,11 +3,18 @@ from pygal.style import RotateStyle
|
||||||
from jinja2.filters import do_filesizeformat
|
from jinja2.filters import do_filesizeformat
|
||||||
|
|
||||||
|
|
||||||
|
# Formatting functions
|
||||||
number_formatter = lambda v: '{:,}'.format(v)
|
number_formatter = lambda v: '{:,}'.format(v)
|
||||||
bytes_formatter = lambda v: do_filesizeformat(v, True)
|
bytes_formatter = lambda v: do_filesizeformat(v, True)
|
||||||
|
|
||||||
|
|
||||||
def tables_piechart(db, by_field, value_formatter):
|
def tables_piechart(db, by_field, value_formatter):
|
||||||
|
'''
|
||||||
|
Generate a pie chart of the top n tables in the database.
|
||||||
|
`db` - the database instance
|
||||||
|
`by_field` - the field name to sort by
|
||||||
|
`value_formatter` - a function to use for formatting the numeric values
|
||||||
|
'''
|
||||||
Tables = db.get_model_for_table('tables', system_table=True)
|
Tables = db.get_model_for_table('tables', system_table=True)
|
||||||
qs = Tables.objects_in(db).filter(database=db.db_name, is_temporary=False).exclude(engine='Buffer')
|
qs = Tables.objects_in(db).filter(database=db.db_name, is_temporary=False).exclude(engine='Buffer')
|
||||||
tuples = [(getattr(table, by_field), table.name) for table in qs]
|
tuples = [(getattr(table, by_field), table.name) for table in qs]
|
||||||
|
@ -15,6 +22,13 @@ def tables_piechart(db, by_field, value_formatter):
|
||||||
|
|
||||||
|
|
||||||
def columns_piechart(db, tbl_name, by_field, value_formatter):
|
def columns_piechart(db, tbl_name, by_field, value_formatter):
|
||||||
|
'''
|
||||||
|
Generate a pie chart of the top n columns in the table.
|
||||||
|
`db` - the database instance
|
||||||
|
`tbl_name` - the table name
|
||||||
|
`by_field` - the field name to sort by
|
||||||
|
`value_formatter` - a function to use for formatting the numeric values
|
||||||
|
'''
|
||||||
ColumnsTable = db.get_model_for_table('columns', system_table=True)
|
ColumnsTable = db.get_model_for_table('columns', system_table=True)
|
||||||
qs = ColumnsTable.objects_in(db).filter(database=db.db_name, table=tbl_name)
|
qs = ColumnsTable.objects_in(db).filter(database=db.db_name, table=tbl_name)
|
||||||
tuples = [(getattr(col, by_field), col.name) for col in qs]
|
tuples = [(getattr(col, by_field), col.name) for col in qs]
|
||||||
|
@ -22,6 +36,11 @@ def columns_piechart(db, tbl_name, by_field, value_formatter):
|
||||||
|
|
||||||
|
|
||||||
def _get_top_tuples(tuples, n=15):
|
def _get_top_tuples(tuples, n=15):
|
||||||
|
'''
|
||||||
|
Given a list of tuples (value, name), this function sorts
|
||||||
|
the list and returns only the top n results. All other tuples
|
||||||
|
are aggregated to a single "others" tuple.
|
||||||
|
'''
|
||||||
non_zero_tuples = [t for t in tuples if t[0]]
|
non_zero_tuples = [t for t in tuples if t[0]]
|
||||||
sorted_tuples = sorted(non_zero_tuples, reverse=True)
|
sorted_tuples = sorted(non_zero_tuples, reverse=True)
|
||||||
if len(sorted_tuples) > n:
|
if len(sorted_tuples) > n:
|
||||||
|
@ -31,6 +50,11 @@ def _get_top_tuples(tuples, n=15):
|
||||||
|
|
||||||
|
|
||||||
def _generate_piechart(tuples, value_formatter):
|
def _generate_piechart(tuples, value_formatter):
|
||||||
|
'''
|
||||||
|
Generates a pie chart.
|
||||||
|
`tuples` - a list of (value, name) tuples to include in the chart
|
||||||
|
`value_formatter` - a function to use for formatting the values
|
||||||
|
'''
|
||||||
style = RotateStyle('#9e6ffe', background='white', legend_font_family='Roboto', legend_font_size=18, tooltip_font_family='Roboto', tooltip_font_size=24)
|
style = RotateStyle('#9e6ffe', background='white', legend_font_family='Roboto', legend_font_size=18, tooltip_font_family='Roboto', tooltip_font_size=24)
|
||||||
chart = pygal.Pie(style=style, margin=0, title=' ', value_formatter=value_formatter, truncate_legend=-1)
|
chart = pygal.Pie(style=style, margin=0, title=' ', value_formatter=value_formatter, truncate_legend=-1)
|
||||||
for t in _get_top_tuples(tuples):
|
for t in _get_top_tuples(tuples):
|
||||||
|
|
|
@ -10,21 +10,34 @@ app = Flask(__name__)
|
||||||
|
|
||||||
@app.route('/')
|
@app.route('/')
|
||||||
def homepage_view():
|
def homepage_view():
|
||||||
|
'''
|
||||||
|
Root view that lists all databases.
|
||||||
|
'''
|
||||||
db = _get_db('system')
|
db = _get_db('system')
|
||||||
|
# Get all databases in the system.databases table
|
||||||
DatabasesTable = db.get_model_for_table('databases', system_table=True)
|
DatabasesTable = db.get_model_for_table('databases', system_table=True)
|
||||||
databases = DatabasesTable.objects_in(db).exclude(name='system').order_by(F.lower(DatabasesTable.name))
|
databases = DatabasesTable.objects_in(db).exclude(name='system')
|
||||||
|
databases = databases.order_by(F.lower(DatabasesTable.name))
|
||||||
|
# Generate the page
|
||||||
return render_template('homepage.html', db=db, databases=databases)
|
return render_template('homepage.html', db=db, databases=databases)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/<db_name>/')
|
@app.route('/<db_name>/')
|
||||||
def database_view(db_name):
|
def database_view(db_name):
|
||||||
|
'''
|
||||||
|
A view that displays information about a single database.
|
||||||
|
'''
|
||||||
db = _get_db(db_name)
|
db = _get_db(db_name)
|
||||||
|
# Get all the tables in the database, by aggregating information from system.columns
|
||||||
ColumnsTable = db.get_model_for_table('columns', system_table=True)
|
ColumnsTable = db.get_model_for_table('columns', system_table=True)
|
||||||
tables = ColumnsTable.objects_in(db).filter(database=db_name).aggregate(ColumnsTable.table,
|
tables = ColumnsTable.objects_in(db).filter(database=db_name).aggregate(
|
||||||
|
ColumnsTable.table,
|
||||||
compressed_size=F.sum(ColumnsTable.data_compressed_bytes),
|
compressed_size=F.sum(ColumnsTable.data_compressed_bytes),
|
||||||
uncompressed_size=F.sum(ColumnsTable.data_uncompressed_bytes),
|
uncompressed_size=F.sum(ColumnsTable.data_uncompressed_bytes),
|
||||||
ratio=F.sum(ColumnsTable.data_uncompressed_bytes) / F.sum(ColumnsTable.data_compressed_bytes)
|
ratio=F.sum(ColumnsTable.data_uncompressed_bytes) / F.sum(ColumnsTable.data_compressed_bytes)
|
||||||
).order_by(F.lower(ColumnsTable.table))
|
)
|
||||||
|
tables = tables.order_by(F.lower(ColumnsTable.table))
|
||||||
|
# Generate the page
|
||||||
return render_template('database.html',
|
return render_template('database.html',
|
||||||
db=db,
|
db=db,
|
||||||
tables=tables,
|
tables=tables,
|
||||||
|
@ -35,12 +48,19 @@ def database_view(db_name):
|
||||||
|
|
||||||
@app.route('/<db_name>/<tbl_name>/')
|
@app.route('/<db_name>/<tbl_name>/')
|
||||||
def table_view(db_name, tbl_name):
|
def table_view(db_name, tbl_name):
|
||||||
|
'''
|
||||||
|
A view that displays information about a single table.
|
||||||
|
'''
|
||||||
db = _get_db(db_name)
|
db = _get_db(db_name)
|
||||||
|
# Get table information from system.tables
|
||||||
TablesTable = db.get_model_for_table('tables', system_table=True)
|
TablesTable = db.get_model_for_table('tables', system_table=True)
|
||||||
tbl_info = TablesTable.objects_in(db).filter(database=db_name, name=tbl_name)[0]
|
tbl_info = TablesTable.objects_in(db).filter(database=db_name, name=tbl_name)[0]
|
||||||
|
# Get the SQL used for creating the table
|
||||||
create_table_sql = db.raw('SHOW CREATE TABLE %s FORMAT TabSeparatedRaw' % tbl_name)
|
create_table_sql = db.raw('SHOW CREATE TABLE %s FORMAT TabSeparatedRaw' % tbl_name)
|
||||||
|
# Get all columns in the table from system.columns
|
||||||
ColumnsTable = db.get_model_for_table('columns', system_table=True)
|
ColumnsTable = db.get_model_for_table('columns', system_table=True)
|
||||||
columns = ColumnsTable.objects_in(db).filter(database=db_name, table=tbl_name)
|
columns = ColumnsTable.objects_in(db).filter(database=db_name, table=tbl_name)
|
||||||
|
# Generate the page
|
||||||
return render_template('table.html',
|
return render_template('table.html',
|
||||||
db=db,
|
db=db,
|
||||||
tbl_name=tbl_name,
|
tbl_name=tbl_name,
|
||||||
|
@ -52,6 +72,10 @@ def table_view(db_name, tbl_name):
|
||||||
|
|
||||||
|
|
||||||
def _get_db(db_name):
|
def _get_db(db_name):
|
||||||
|
'''
|
||||||
|
Returns a Database instance using connection information
|
||||||
|
from the command line arguments (optional).
|
||||||
|
'''
|
||||||
db_url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8123/'
|
db_url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8123/'
|
||||||
username = sys.argv[2] if len(sys.argv) > 2 else None
|
username = sys.argv[2] if len(sys.argv) > 2 else None
|
||||||
password = sys.argv[3] if len(sys.argv) > 3 else None
|
password = sys.argv[3] if len(sys.argv) > 3 else None
|
||||||
|
|
Loading…
Reference in New Issue
Block a user