Added usage examples

This commit is contained in:
Itai Shirav 2020-07-15 23:42:40 +03:00
parent 80b220c1e3
commit 97773d6dc7
3 changed files with 87 additions and 3 deletions

View File

@ -0,0 +1,36 @@
# DB Explorer
This is a simple Flask web application that connects to ClickHouse and displays the list of existing databases. Clicking on a database name drills down into it, showing its list of tables. Clicking on a table drills down further, showing details about the table and its columns.
For each table or column, the application displays the compressed size on disk, the uncompressed size, and the ratio between them. Additionally, several pie charts are shown - top tables by size, top tables by rows, and top columns by size (in a table).
The pie charts are generated using the `pygal` charting library.
ORM concepts that are demonstrated by this example:
- Creating ORM models from existing tables using `Database.get_model_for_table`
- Queryset filtering
- Queryset aggregation
## Running the code
Create a virtualenv and install the required libraries:
```
virtualenv -p python3.6 env
source env/bin/activate
pip install -r requirements.txt
```
Run the server and open http://127.0.0.1:5000/ in your browser:
```
python server.py
```
By default the server connects to ClickHouse running on http://localhost:8123/ without a username or password, but you can change this using command line arguments:
```
python server.py http://myclickhouse:8123/
```
or:
```
python server.py http://myclickhouse:8123/ admin secret123
```

View File

@ -3,11 +3,18 @@ from pygal.style import RotateStyle
from jinja2.filters import do_filesizeformat from jinja2.filters import do_filesizeformat
# Formatting functions
number_formatter = lambda v: '{:,}'.format(v) number_formatter = lambda v: '{:,}'.format(v)
bytes_formatter = lambda v: do_filesizeformat(v, True) bytes_formatter = lambda v: do_filesizeformat(v, True)
def tables_piechart(db, by_field, value_formatter): def tables_piechart(db, by_field, value_formatter):
'''
Generate a pie chart of the top n tables in the database.
`db` - the database instance
`by_field` - the field name to sort by
`value_formatter` - a function to use for formatting the numeric values
'''
Tables = db.get_model_for_table('tables', system_table=True) Tables = db.get_model_for_table('tables', system_table=True)
qs = Tables.objects_in(db).filter(database=db.db_name, is_temporary=False).exclude(engine='Buffer') qs = Tables.objects_in(db).filter(database=db.db_name, is_temporary=False).exclude(engine='Buffer')
tuples = [(getattr(table, by_field), table.name) for table in qs] tuples = [(getattr(table, by_field), table.name) for table in qs]
@ -15,6 +22,13 @@ def tables_piechart(db, by_field, value_formatter):
def columns_piechart(db, tbl_name, by_field, value_formatter): def columns_piechart(db, tbl_name, by_field, value_formatter):
'''
Generate a pie chart of the top n columns in the table.
`db` - the database instance
`tbl_name` - the table name
`by_field` - the field name to sort by
`value_formatter` - a function to use for formatting the numeric values
'''
ColumnsTable = db.get_model_for_table('columns', system_table=True) ColumnsTable = db.get_model_for_table('columns', system_table=True)
qs = ColumnsTable.objects_in(db).filter(database=db.db_name, table=tbl_name) qs = ColumnsTable.objects_in(db).filter(database=db.db_name, table=tbl_name)
tuples = [(getattr(col, by_field), col.name) for col in qs] tuples = [(getattr(col, by_field), col.name) for col in qs]
@ -22,6 +36,11 @@ def columns_piechart(db, tbl_name, by_field, value_formatter):
def _get_top_tuples(tuples, n=15): def _get_top_tuples(tuples, n=15):
'''
Given a list of tuples (value, name), this function sorts
the list and returns only the top n results. All other tuples
are aggregated to a single "others" tuple.
'''
non_zero_tuples = [t for t in tuples if t[0]] non_zero_tuples = [t for t in tuples if t[0]]
sorted_tuples = sorted(non_zero_tuples, reverse=True) sorted_tuples = sorted(non_zero_tuples, reverse=True)
if len(sorted_tuples) > n: if len(sorted_tuples) > n:
@ -31,6 +50,11 @@ def _get_top_tuples(tuples, n=15):
def _generate_piechart(tuples, value_formatter): def _generate_piechart(tuples, value_formatter):
'''
Generates a pie chart.
`tuples` - a list of (value, name) tuples to include in the chart
`value_formatter` - a function to use for formatting the values
'''
style = RotateStyle('#9e6ffe', background='white', legend_font_family='Roboto', legend_font_size=18, tooltip_font_family='Roboto', tooltip_font_size=24) style = RotateStyle('#9e6ffe', background='white', legend_font_family='Roboto', legend_font_size=18, tooltip_font_family='Roboto', tooltip_font_size=24)
chart = pygal.Pie(style=style, margin=0, title=' ', value_formatter=value_formatter, truncate_legend=-1) chart = pygal.Pie(style=style, margin=0, title=' ', value_formatter=value_formatter, truncate_legend=-1)
for t in _get_top_tuples(tuples): for t in _get_top_tuples(tuples):

View File

@ -10,21 +10,34 @@ app = Flask(__name__)
@app.route('/') @app.route('/')
def homepage_view(): def homepage_view():
'''
Root view that lists all databases.
'''
db = _get_db('system') db = _get_db('system')
# Get all databases in the system.databases table
DatabasesTable = db.get_model_for_table('databases', system_table=True) DatabasesTable = db.get_model_for_table('databases', system_table=True)
databases = DatabasesTable.objects_in(db).exclude(name='system').order_by(F.lower(DatabasesTable.name)) databases = DatabasesTable.objects_in(db).exclude(name='system')
databases = databases.order_by(F.lower(DatabasesTable.name))
# Generate the page
return render_template('homepage.html', db=db, databases=databases) return render_template('homepage.html', db=db, databases=databases)
@app.route('/<db_name>/') @app.route('/<db_name>/')
def database_view(db_name): def database_view(db_name):
'''
A view that displays information about a single database.
'''
db = _get_db(db_name) db = _get_db(db_name)
# Get all the tables in the database, by aggregating information from system.columns
ColumnsTable = db.get_model_for_table('columns', system_table=True) ColumnsTable = db.get_model_for_table('columns', system_table=True)
tables = ColumnsTable.objects_in(db).filter(database=db_name).aggregate(ColumnsTable.table, tables = ColumnsTable.objects_in(db).filter(database=db_name).aggregate(
ColumnsTable.table,
compressed_size=F.sum(ColumnsTable.data_compressed_bytes), compressed_size=F.sum(ColumnsTable.data_compressed_bytes),
uncompressed_size=F.sum(ColumnsTable.data_uncompressed_bytes), uncompressed_size=F.sum(ColumnsTable.data_uncompressed_bytes),
ratio=F.sum(ColumnsTable.data_uncompressed_bytes) / F.sum(ColumnsTable.data_compressed_bytes) ratio=F.sum(ColumnsTable.data_uncompressed_bytes) / F.sum(ColumnsTable.data_compressed_bytes)
).order_by(F.lower(ColumnsTable.table)) )
tables = tables.order_by(F.lower(ColumnsTable.table))
# Generate the page
return render_template('database.html', return render_template('database.html',
db=db, db=db,
tables=tables, tables=tables,
@ -35,12 +48,19 @@ def database_view(db_name):
@app.route('/<db_name>/<tbl_name>/') @app.route('/<db_name>/<tbl_name>/')
def table_view(db_name, tbl_name): def table_view(db_name, tbl_name):
'''
A view that displays information about a single table.
'''
db = _get_db(db_name) db = _get_db(db_name)
# Get table information from system.tables
TablesTable = db.get_model_for_table('tables', system_table=True) TablesTable = db.get_model_for_table('tables', system_table=True)
tbl_info = TablesTable.objects_in(db).filter(database=db_name, name=tbl_name)[0] tbl_info = TablesTable.objects_in(db).filter(database=db_name, name=tbl_name)[0]
# Get the SQL used for creating the table
create_table_sql = db.raw('SHOW CREATE TABLE %s FORMAT TabSeparatedRaw' % tbl_name) create_table_sql = db.raw('SHOW CREATE TABLE %s FORMAT TabSeparatedRaw' % tbl_name)
# Get all columns in the table from system.columns
ColumnsTable = db.get_model_for_table('columns', system_table=True) ColumnsTable = db.get_model_for_table('columns', system_table=True)
columns = ColumnsTable.objects_in(db).filter(database=db_name, table=tbl_name) columns = ColumnsTable.objects_in(db).filter(database=db_name, table=tbl_name)
# Generate the page
return render_template('table.html', return render_template('table.html',
db=db, db=db,
tbl_name=tbl_name, tbl_name=tbl_name,
@ -52,6 +72,10 @@ def table_view(db_name, tbl_name):
def _get_db(db_name): def _get_db(db_name):
'''
Returns a Database instance using connection information
from the command line arguments (optional).
'''
db_url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8123/' db_url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8123/'
username = sys.argv[2] if len(sys.argv) > 2 else None username = sys.argv[2] if len(sys.argv) > 2 else None
password = sys.argv[3] if len(sys.argv) > 3 else None password = sys.argv[3] if len(sys.argv) > 3 else None