From 97773d6dc76adb8304c57b948ca3bf3b48186447 Mon Sep 17 00:00:00 2001 From: Itai Shirav Date: Wed, 15 Jul 2020 23:42:40 +0300 Subject: [PATCH] Added usage examples --- examples/db_explorer/README.md | 36 ++++++++++++++++++++++++++++++++++ examples/db_explorer/charts.py | 24 +++++++++++++++++++++++ examples/db_explorer/server.py | 30 +++++++++++++++++++++++++--- 3 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 examples/db_explorer/README.md diff --git a/examples/db_explorer/README.md b/examples/db_explorer/README.md new file mode 100644 index 0000000..1ef0bff --- /dev/null +++ b/examples/db_explorer/README.md @@ -0,0 +1,36 @@ +# DB Explorer + +This is a simple Flask web application that connects to ClickHouse and displays the list of existing databases. Clicking on a database name drills down into it, showing its list of tables. Clicking on a table drills down further, showing details about the table and its columns. + +For each table or column, the application displays the compressed size on disk, the uncompressed size, and the ratio between them. Additionally, several pie charts are shown - top tables by size, top tables by rows, and top columns by size (in a table). + +The pie charts are generated using the `pygal` charting library. + +ORM concepts that are demonstrated by this example: + +- Creating ORM models from existing tables using `Database.get_model_for_table` +- Queryset filtering +- Queryset aggregation + +## Running the code + +Create a virtualenv and install the required libraries: +``` +virtualenv -p python3.6 env +source env/bin/activate +pip install -r requirements.txt +``` + +Run the server and open http://127.0.0.1:5000/ in your browser: +``` +python server.py +``` + +By default the server connects to ClickHouse running on http://localhost:8123/ without a username or password, but you can change this using command line arguments: +``` +python server.py http://myclickhouse:8123/ +``` +or: +``` +python server.py http://myclickhouse:8123/ admin secret123 +``` diff --git a/examples/db_explorer/charts.py b/examples/db_explorer/charts.py index 0def735..1a690e6 100644 --- a/examples/db_explorer/charts.py +++ b/examples/db_explorer/charts.py @@ -3,11 +3,18 @@ from pygal.style import RotateStyle from jinja2.filters import do_filesizeformat +# Formatting functions number_formatter = lambda v: '{:,}'.format(v) bytes_formatter = lambda v: do_filesizeformat(v, True) def tables_piechart(db, by_field, value_formatter): + ''' + Generate a pie chart of the top n tables in the database. + `db` - the database instance + `by_field` - the field name to sort by + `value_formatter` - a function to use for formatting the numeric values + ''' Tables = db.get_model_for_table('tables', system_table=True) qs = Tables.objects_in(db).filter(database=db.db_name, is_temporary=False).exclude(engine='Buffer') tuples = [(getattr(table, by_field), table.name) for table in qs] @@ -15,6 +22,13 @@ def tables_piechart(db, by_field, value_formatter): def columns_piechart(db, tbl_name, by_field, value_formatter): + ''' + Generate a pie chart of the top n columns in the table. + `db` - the database instance + `tbl_name` - the table name + `by_field` - the field name to sort by + `value_formatter` - a function to use for formatting the numeric values + ''' ColumnsTable = db.get_model_for_table('columns', system_table=True) qs = ColumnsTable.objects_in(db).filter(database=db.db_name, table=tbl_name) tuples = [(getattr(col, by_field), col.name) for col in qs] @@ -22,6 +36,11 @@ def columns_piechart(db, tbl_name, by_field, value_formatter): def _get_top_tuples(tuples, n=15): + ''' + Given a list of tuples (value, name), this function sorts + the list and returns only the top n results. All other tuples + are aggregated to a single "others" tuple. + ''' non_zero_tuples = [t for t in tuples if t[0]] sorted_tuples = sorted(non_zero_tuples, reverse=True) if len(sorted_tuples) > n: @@ -31,6 +50,11 @@ def _get_top_tuples(tuples, n=15): def _generate_piechart(tuples, value_formatter): + ''' + Generates a pie chart. + `tuples` - a list of (value, name) tuples to include in the chart + `value_formatter` - a function to use for formatting the values + ''' style = RotateStyle('#9e6ffe', background='white', legend_font_family='Roboto', legend_font_size=18, tooltip_font_family='Roboto', tooltip_font_size=24) chart = pygal.Pie(style=style, margin=0, title=' ', value_formatter=value_formatter, truncate_legend=-1) for t in _get_top_tuples(tuples): diff --git a/examples/db_explorer/server.py b/examples/db_explorer/server.py index 3336eb2..6241ed9 100644 --- a/examples/db_explorer/server.py +++ b/examples/db_explorer/server.py @@ -10,21 +10,34 @@ app = Flask(__name__) @app.route('/') def homepage_view(): + ''' + Root view that lists all databases. + ''' db = _get_db('system') + # Get all databases in the system.databases table DatabasesTable = db.get_model_for_table('databases', system_table=True) - databases = DatabasesTable.objects_in(db).exclude(name='system').order_by(F.lower(DatabasesTable.name)) + databases = DatabasesTable.objects_in(db).exclude(name='system') + databases = databases.order_by(F.lower(DatabasesTable.name)) + # Generate the page return render_template('homepage.html', db=db, databases=databases) @app.route('//') def database_view(db_name): + ''' + A view that displays information about a single database. + ''' db = _get_db(db_name) + # Get all the tables in the database, by aggregating information from system.columns ColumnsTable = db.get_model_for_table('columns', system_table=True) - tables = ColumnsTable.objects_in(db).filter(database=db_name).aggregate(ColumnsTable.table, + tables = ColumnsTable.objects_in(db).filter(database=db_name).aggregate( + ColumnsTable.table, compressed_size=F.sum(ColumnsTable.data_compressed_bytes), uncompressed_size=F.sum(ColumnsTable.data_uncompressed_bytes), ratio=F.sum(ColumnsTable.data_uncompressed_bytes) / F.sum(ColumnsTable.data_compressed_bytes) - ).order_by(F.lower(ColumnsTable.table)) + ) + tables = tables.order_by(F.lower(ColumnsTable.table)) + # Generate the page return render_template('database.html', db=db, tables=tables, @@ -35,12 +48,19 @@ def database_view(db_name): @app.route('///') def table_view(db_name, tbl_name): + ''' + A view that displays information about a single table. + ''' db = _get_db(db_name) + # Get table information from system.tables TablesTable = db.get_model_for_table('tables', system_table=True) tbl_info = TablesTable.objects_in(db).filter(database=db_name, name=tbl_name)[0] + # Get the SQL used for creating the table create_table_sql = db.raw('SHOW CREATE TABLE %s FORMAT TabSeparatedRaw' % tbl_name) + # Get all columns in the table from system.columns ColumnsTable = db.get_model_for_table('columns', system_table=True) columns = ColumnsTable.objects_in(db).filter(database=db_name, table=tbl_name) + # Generate the page return render_template('table.html', db=db, tbl_name=tbl_name, @@ -52,6 +72,10 @@ def table_view(db_name, tbl_name): def _get_db(db_name): + ''' + Returns a Database instance using connection information + from the command line arguments (optional). + ''' db_url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8123/' username = sys.argv[2] if len(sys.argv) > 2 else None password = sys.argv[3] if len(sys.argv) > 3 else None