diff --git a/examples/db_explorer/.gitignore b/examples/db_explorer/.gitignore new file mode 100644 index 0000000..47def24 --- /dev/null +++ b/examples/db_explorer/.gitignore @@ -0,0 +1 @@ +/env/ diff --git a/examples/db_explorer/charts.py b/examples/db_explorer/charts.py new file mode 100644 index 0000000..0def735 --- /dev/null +++ b/examples/db_explorer/charts.py @@ -0,0 +1,38 @@ +import pygal +from pygal.style import RotateStyle +from jinja2.filters import do_filesizeformat + + +number_formatter = lambda v: '{:,}'.format(v) +bytes_formatter = lambda v: do_filesizeformat(v, True) + + +def tables_piechart(db, by_field, value_formatter): + Tables = db.get_model_for_table('tables', system_table=True) + qs = Tables.objects_in(db).filter(database=db.db_name, is_temporary=False).exclude(engine='Buffer') + tuples = [(getattr(table, by_field), table.name) for table in qs] + return _generate_piechart(tuples, value_formatter) + + +def columns_piechart(db, tbl_name, by_field, value_formatter): + ColumnsTable = db.get_model_for_table('columns', system_table=True) + qs = ColumnsTable.objects_in(db).filter(database=db.db_name, table=tbl_name) + tuples = [(getattr(col, by_field), col.name) for col in qs] + return _generate_piechart(tuples, value_formatter) + + +def _get_top_tuples(tuples, n=15): + non_zero_tuples = [t for t in tuples if t[0]] + sorted_tuples = sorted(non_zero_tuples, reverse=True) + if len(sorted_tuples) > n: + others = (sum(t[0] for t in sorted_tuples[n:]), 'others') + sorted_tuples = sorted_tuples[:n] + [others] + return sorted_tuples + + +def _generate_piechart(tuples, value_formatter): + style = RotateStyle('#9e6ffe', background='white', legend_font_family='Roboto', legend_font_size=18, tooltip_font_family='Roboto', tooltip_font_size=24) + chart = pygal.Pie(style=style, margin=0, title=' ', value_formatter=value_formatter, truncate_legend=-1) + for t in _get_top_tuples(tuples): + chart.add(t[1], t[0]) + return chart.render(is_unicode=True, disable_xml_declaration=True) diff --git a/examples/db_explorer/requirements.txt b/examples/db_explorer/requirements.txt new file mode 100644 index 0000000..8dee9f8 --- /dev/null +++ b/examples/db_explorer/requirements.txt @@ -0,0 +1,15 @@ +certifi==2020.4.5.2 +chardet==3.0.4 +click==7.1.2 +Flask==1.1.2 +idna==2.9 +infi.clickhouse-orm==2.0.1 +iso8601==0.1.12 +itsdangerous==1.1.0 +Jinja2==2.11.2 +MarkupSafe==1.1.1 +pygal==2.4.0 +pytz==2020.1 +requests==2.23.0 +urllib3==1.25.9 +Werkzeug==1.0.1 diff --git a/examples/db_explorer/server.py b/examples/db_explorer/server.py new file mode 100644 index 0000000..3336eb2 --- /dev/null +++ b/examples/db_explorer/server.py @@ -0,0 +1,63 @@ +from infi.clickhouse_orm import Database, F +from charts import tables_piechart, columns_piechart, number_formatter, bytes_formatter +from flask import Flask +from flask import render_template +import sys + + +app = Flask(__name__) + + +@app.route('/') +def homepage_view(): + db = _get_db('system') + DatabasesTable = db.get_model_for_table('databases', system_table=True) + databases = DatabasesTable.objects_in(db).exclude(name='system').order_by(F.lower(DatabasesTable.name)) + return render_template('homepage.html', db=db, databases=databases) + + +@app.route('//') +def database_view(db_name): + db = _get_db(db_name) + ColumnsTable = db.get_model_for_table('columns', system_table=True) + tables = ColumnsTable.objects_in(db).filter(database=db_name).aggregate(ColumnsTable.table, + compressed_size=F.sum(ColumnsTable.data_compressed_bytes), + uncompressed_size=F.sum(ColumnsTable.data_uncompressed_bytes), + ratio=F.sum(ColumnsTable.data_uncompressed_bytes) / F.sum(ColumnsTable.data_compressed_bytes) + ).order_by(F.lower(ColumnsTable.table)) + return render_template('database.html', + db=db, + tables=tables, + tables_piechart_by_rows=tables_piechart(db, 'total_rows', value_formatter=number_formatter), + tables_piechart_by_size=tables_piechart(db, 'total_bytes', value_formatter=bytes_formatter), + ) + + +@app.route('///') +def table_view(db_name, tbl_name): + db = _get_db(db_name) + TablesTable = db.get_model_for_table('tables', system_table=True) + tbl_info = TablesTable.objects_in(db).filter(database=db_name, name=tbl_name)[0] + create_table_sql = db.raw('SHOW CREATE TABLE %s FORMAT TabSeparatedRaw' % tbl_name) + ColumnsTable = db.get_model_for_table('columns', system_table=True) + columns = ColumnsTable.objects_in(db).filter(database=db_name, table=tbl_name) + return render_template('table.html', + db=db, + tbl_name=tbl_name, + tbl_info=tbl_info, + create_table_sql=create_table_sql, + columns=columns, + piechart=columns_piechart(db, tbl_name, 'data_compressed_bytes', value_formatter=bytes_formatter), + ) + + +def _get_db(db_name): + db_url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8123/' + username = sys.argv[2] if len(sys.argv) > 2 else None + password = sys.argv[3] if len(sys.argv) > 3 else None + return Database(db_name, db_url, username, password, readonly=True) + + +if __name__ == '__main__': + _get_db('system') # fail early on db connection problems + app.run(debug=True) diff --git a/examples/db_explorer/templates/base.html b/examples/db_explorer/templates/base.html new file mode 100644 index 0000000..0f343d7 --- /dev/null +++ b/examples/db_explorer/templates/base.html @@ -0,0 +1,22 @@ + + + + + ClickHouse Explorer + + + + + + + + +
+ + {% block contents %} + {% endblock %} + +
+ + + \ No newline at end of file diff --git a/examples/db_explorer/templates/database.html b/examples/db_explorer/templates/database.html new file mode 100644 index 0000000..9b72b7d --- /dev/null +++ b/examples/db_explorer/templates/database.html @@ -0,0 +1,54 @@ +{% extends "base.html" %} + +{% block contents %} + +

{{ db.db_name }}

+ +

+ Home + » + {{ db.db_name }} +

+ +
+ +
+

Top Tables by Size

+ {% autoescape false %} + {{ tables_piechart_by_size }} + {% endautoescape %} +
+ +
+

Top Tables by Rows

+ {% autoescape false %} + {{ tables_piechart_by_rows }} + {% endautoescape %} +
+ +
+ +

Tables ({{ tables.count() }})

+ + + + + + + + + + + + {% for table in tables %} + + + + + + {% endfor %} + +
NameUncompressed SizeCompressed SizeCompression Ratio
{{ table.table }} + {{ table.uncompressed_size|filesizeformat(true) }}{{ table.compressed_size|filesizeformat(true) }}{% if table.uncompressed_size %} {{ "%.2f" % table.ratio }} {% else %} 1 {% endif %} : 1
+ +{% endblock %} diff --git a/examples/db_explorer/templates/homepage.html b/examples/db_explorer/templates/homepage.html new file mode 100644 index 0000000..79997f7 --- /dev/null +++ b/examples/db_explorer/templates/homepage.html @@ -0,0 +1,41 @@ +{% extends "base.html" %} + +{% block contents %} + + +
+ +
+ +

ClickHouse Explorer

+ + + + + + + + + + + + + + +
URL{{ db.db_url }}
Version{{ db.server_version|join('.') }}
Timezone{{ db.server_timezone }}
+ +

Databases ({{ databases.count() }})

+
    + {% for d in databases %} +
  • + {{ d.name }} +
    • + {% endfor %} +
+ +
+ +
+ + +{% endblock %} diff --git a/examples/db_explorer/templates/table.html b/examples/db_explorer/templates/table.html new file mode 100644 index 0000000..682c275 --- /dev/null +++ b/examples/db_explorer/templates/table.html @@ -0,0 +1,79 @@ +{% extends "base.html" %} + +{% block contents %} + + +

+ Home + » + {{ db.db_name }} + » + {{ tbl_name }} +

+ +

{{ tbl_name }}

+ +
+ +
+

Details

+ + + + + + + + + + {% if tbl_info.total_rows %} + + + + + {% endif %} + + + + +
Total rows{{ "{:,}".format(tbl_info.total_rows) }}
Total size{{ tbl_info.total_bytes|filesizeformat(true) }}
Average row size{{ (tbl_info.total_bytes / tbl_info.total_rows)|filesizeformat(true) }}
Engine{{ tbl_info.engine }}
+
+ +
+

Top Columns by Size

+ {% autoescape false %} + {{ piechart }} + {% endautoescape %} +
+ +
+ +

Columns ({{ columns.count() }})

+ + + + + + + + + + + + + {% for col in columns %} + + + + + + + + {% endfor %} + +
NameTypeUncompressed SizeCompressed SizeCompression Ratio
{{ col.name }}{{ col.type }}{{ col.data_uncompressed_bytes|filesizeformat(true) }}{{ col.data_compressed_bytes|filesizeformat(true) }}{% if col.data_compressed_bytes %} {{ "%.2f" % (col.data_uncompressed_bytes / col.data_compressed_bytes) }} {% else %} 1 {% endif %} : 1
+ +

Table Definition

+ 
{{ create_table_sql }}
+ +{% endblock %}