LonamiWebs/Telethon
1
1
Fork 0
mirror of https://github.com/LonamiWebs/Telethon.git synced 2024-11-22 17:36:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Make use of pathlib nearly everywhere (breaks docs gen)

Browse Source 
Python 3.6 introduced support for the os.PathLike interface,
which means Python 3.5 did not have it yet and attempting to
use it in os functions would fail. Instead we can use pathlib
for everything, but not all work is done yet.
This commit is contained in:
Lonami Exo 2018-12-21 13:24:16 +01:00
parent b9d4eb5449
commit 8224e5aabf
7 changed files with 71 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
setup.py
View File

@ -16,6 +16,7 @@ import re
import shutil import shutil
from os import chdir from os import chdir
from pathlib import Path from pathlib import Path
from subprocess import run
from sys import argv from sys import argv
from setuptools import find_packages, setup from setuptools import find_packages, setup
@ -30,11 +31,11 @@ class TempWorkDir:
def __enter__(self): def __enter__(self):
self.original = Path('.') self.original = Path('.')
chdir(Path(__file__).parent) chdir(str(Path(__file__).parent))
return self return self
def __exit__(self, *args): def __exit__(self, *args):
chdir(self.original) chdir(str(self.original))
GENERATOR_DIR = Path('telethon_generator') GENERATOR_DIR = Path('telethon_generator')
@ -96,7 +97,7 @@ def generate(which):
if ERRORS_OUT.is_file(): if ERRORS_OUT.is_file():
ERRORS_OUT.unlink() ERRORS_OUT.unlink()
else: else:
with open(ERRORS_OUT, 'w', encoding='utf-8') as file: with ERRORS_OUT.open('w') as file:
generate_errors(errors, file) generate_errors(errors, file)
if 'docs' in which: if 'docs' in which:
@ -104,7 +105,7 @@ def generate(which):
print(action, 'documentation...') print(action, 'documentation...')
if clean: if clean:
if DOCS_OUT.is_dir(): if DOCS_OUT.is_dir():
shutil.rmtree(DOCS_OUT) shutil.rmtree(str(DOCS_OUT))
else: else:
generate_docs(tlobjects, methods, layer, DOCS_IN_RES, DOCS_OUT) generate_docs(tlobjects, methods, layer, DOCS_IN_RES, DOCS_OUT)
@ -154,18 +155,13 @@ def main():
print('Packaging for PyPi aborted, importing the module failed.') print('Packaging for PyPi aborted, importing the module failed.')
return return
# Need python3.5 or higher, but Telethon is supposed to support 3.x
# Place it here since noone should be running ./setup.py pypi anyway
from subprocess import run
from shutil import rmtree
for x in ('build', 'dist', 'Telethon.egg-info'): for x in ('build', 'dist', 'Telethon.egg-info'):
rmtree(x, ignore_errors=True) shutil.rmtree(x, ignore_errors=True)
run('python3 setup.py sdist', shell=True) run('python3 setup.py sdist', shell=True)
run('python3 setup.py bdist_wheel', shell=True) run('python3 setup.py bdist_wheel', shell=True)
run('twine upload dist/*', shell=True) run('twine upload dist/*', shell=True)
for x in ('build', 'dist', 'Telethon.egg-info'): for x in ('build', 'dist', 'Telethon.egg-info'):
rmtree(x, ignore_errors=True) shutil.rmtree(x, ignore_errors=True)
else: else:
# e.g. install from GitHub # e.g. install from GitHub

7
telethon_generator/docswriter.py
View File

@ -54,7 +54,7 @@ class DocsWriter:
<body> <body>
<div id="main_div">''', <div id="main_div">''',
title=title, title=title,
rel_css=relative_css_path.rstrip('/'), rel_css=str(relative_css_path).rstrip('/'),
def_css=default_css def_css=default_css
) )
@ -278,10 +278,7 @@ class DocsWriter:
# With block # With block
def __enter__(self): def __enter__(self):
# Sanity check # Sanity check
parent = os.path.dirname(self.filename) self.filename.parent.mkdir(parents=True, exist_ok=True)
if parent:
os.makedirs(parent, exist_ok=True)
self.handle = open(self.filename, 'w', encoding='utf-8') self.handle = open(self.filename, 'w', encoding='utf-8')
return self return self

84
telethon_generator/generators/docs.py
View File

@ -1,10 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import csv
import functools import functools
import os import os
import re import re
import shutil import shutil
from collections import defaultdict from collections import defaultdict
from pathlib import Path
from ..docswriter import DocsWriter from ..docswriter import DocsWriter
from ..parsers import TLObject, Usability from ..parsers import TLObject, Usability
@ -35,14 +35,14 @@ def get_import_code(tlobject):
def _get_create_path_for(root, tlobject, make=True): def _get_create_path_for(root, tlobject, make=True):
"""Creates and returns the path for the given TLObject at root.""" """Creates and returns the path for the given TLObject at root."""
out_dir = 'methods' if tlobject.is_function else 'constructors' out_dir = root / ('methods' if tlobject.is_function else 'constructors')
if tlobject.namespace: if tlobject.namespace:
out_dir = os.path.join(out_dir, tlobject.namespace) out_dir /= tlobject.namespace
out_dir = os.path.join(root, out_dir)
if make: if make:
os.makedirs(out_dir, exist_ok=True) out_dir.mkdir(parents=True, exist_ok=True)
return os.path.join(out_dir, _get_file_name(tlobject))
return out_dir / _get_file_name(tlobject)
def _get_path_for_type(root, type_, relative_to='.'): def _get_path_for_type(root, type_, relative_to='.'):
@ -55,15 +55,17 @@ def _get_path_for_type(root, type_, relative_to='.'):
else: else:
path = 'types/%s' % _get_file_name(type_) path = 'types/%s' % _get_file_name(type_)
return _get_relative_path(os.path.join(root, path), relative_to) return _get_relative_path(root / path, relative_to)
def _get_relative_path(destination, relative_to, folder=False): def _get_relative_path(destination, relative_to, folder=False):
"""Return the relative path to destination from relative_to.""" """Return the relative path to destination from relative_to."""
relative_to = Path(relative_to)
if not folder: if not folder:
relative_to = os.path.dirname(relative_to) relative_to = relative_to.parent
return os.path.relpath(destination, start=relative_to) # TODO Use pathlib here
return Path(os.path.relpath(destination, start=relative_to))
def _find_title(html_file): def _find_title(html_file):
@ -83,7 +85,7 @@ def _build_menu(docs, filename, root, relative_main_index):
filename = _get_relative_path(filename, root) filename = _get_relative_path(filename, root)
docs.add_menu('API', relative_main_index) docs.add_menu('API', relative_main_index)
items = filename.split('/') items = str(filename).split('/')
for i in range(len(items) - 1): for i in range(len(items) - 1):
item = items[i] item = items[i]
link = '../' * (len(items) - (i + 2)) link = '../' * (len(items) - (i + 2))
@ -106,8 +108,8 @@ def _generate_index(folder, original_paths, root,
BOT_INDEX = 'botindex.html' BOT_INDEX = 'botindex.html'
if not bots_index: if not bots_index:
for item in os.listdir(folder): for item in folder.iterdir():
if os.path.isdir(os.path.join(folder, item)): if item.is_dir():
namespaces.append(item) namespaces.append(item)
elif item not in (INDEX, BOT_INDEX): elif item not in (INDEX, BOT_INDEX):
files.append(item) files.append(item)
@ -115,7 +117,7 @@ def _generate_index(folder, original_paths, root,
# bots_index_paths should be a list of "namespace/method.html" # bots_index_paths should be a list of "namespace/method.html"
# or "method.html" # or "method.html"
for item in bots_index_paths: for item in bots_index_paths:
dirname = os.path.dirname(item) dirname = item.parent
if dirname and dirname not in namespaces: if dirname and dirname not in namespaces:
namespaces.append(dirname) namespaces.append(dirname)
elif not dirname and item not in (INDEX, BOT_INDEX): elif not dirname and item not in (INDEX, BOT_INDEX):
@ -125,10 +127,10 @@ def _generate_index(folder, original_paths, root,
for k, v in original_paths.items()} for k, v in original_paths.items()}
# Now that everything is setup, write the index.html file # Now that everything is setup, write the index.html file
filename = os.path.join(folder, BOT_INDEX if bots_index else INDEX) filename = folder / (BOT_INDEX if bots_index else INDEX)
with DocsWriter(filename, type_to_path=_get_path_for_type) as docs: with DocsWriter(filename, type_to_path=_get_path_for_type) as docs:
# Title should be the current folder name # Title should be the current folder name
docs.write_head(folder.title(), docs.write_head(str(folder).title(),
relative_css_path=paths['css'], relative_css_path=paths['css'],
default_css=original_paths['default_css']) default_css=original_paths['default_css'])
@ -136,7 +138,9 @@ def _generate_index(folder, original_paths, root,
_build_menu(docs, filename, root, _build_menu(docs, filename, root,
relative_main_index=paths['index_all']) relative_main_index=paths['index_all'])
docs.write_title(_get_relative_path(folder, root, folder=True).title()) docs.write_title(str(
_get_relative_path(folder, root, folder=True)).title())
if bots_index: if bots_index:
docs.write_text('These are the methods that you may be able to ' docs.write_text('These are the methods that you may be able to '
'use as a bot. Click <a href="{}">here</a> to ' 'use as a bot. Click <a href="{}">here</a> to '
@ -153,24 +157,23 @@ def _generate_index(folder, original_paths, root,
namespace_paths = [] namespace_paths = []
if bots_index: if bots_index:
for item in bots_index_paths: for item in bots_index_paths:
if os.path.dirname(item) == namespace: # TODO .name? or not
namespace_paths.append(os.path.basename(item)) if item.parent.name == namespace:
_generate_index(os.path.join(folder, namespace), namespace_paths.append(item.name)
_generate_index(folder / namespace,
original_paths, root, original_paths, root,
bots_index, namespace_paths) bots_index, namespace_paths)
if bots_index: if bots_index:
docs.add_row(namespace.title(), docs.add_row(namespace.title(), link=namespace / BOT_INDEX)
link=os.path.join(namespace, BOT_INDEX))
else: else:
docs.add_row(namespace.title(), docs.add_row(namespace.title(), link=namespace / INDEX)
link=os.path.join(namespace, INDEX))
docs.end_table() docs.end_table()
docs.write_title('Available items') docs.write_title('Available items')
docs.begin_table(2) docs.begin_table(2)
files = [(f, _find_title(os.path.join(folder, f))) for f in files] files = [(f, _find_title(folder / f)) for f in files]
files.sort(key=lambda t: t[1]) files.sort(key=lambda t: t[1])
for file, title in files: for file, title in files:
@ -250,9 +253,7 @@ def _write_html_pages(tlobjects, methods, layer, input_res, output_dir):
'index_methods': 'methods/index.html', 'index_methods': 'methods/index.html',
'index_constructors': 'constructors/index.html' 'index_constructors': 'constructors/index.html'
} }
original_paths = {k: os.path.join(output_dir, v) original_paths = {k: output_dir / v for k, v in original_paths.items()}
for k, v in original_paths.items()}
original_paths['default_css'] = 'light' # docs.<name>.css, local path original_paths['default_css'] = 'light' # docs.<name>.css, local path
type_to_constructors = defaultdict(list) type_to_constructors = defaultdict(list)
type_to_functions = defaultdict(list) type_to_functions = defaultdict(list)
@ -443,16 +444,17 @@ def _write_html_pages(tlobjects, methods, layer, input_res, output_dir):
temp = [] temp = []
for item in bot_docs_paths: for item in bot_docs_paths:
temp.append(os.path.sep.join(item.split(os.path.sep)[2:])) # TODO What?
temp.append(os.path.sep.join(str(item).split(os.path.sep)[2:]))
bot_docs_paths = temp bot_docs_paths = temp
# Find all the available types (which are not the same as the constructors) # Find all the available types (which are not the same as the constructors)
# Each type has a list of constructors associated to it, hence is a map # Each type has a list of constructors associated to it, hence is a map
for t, cs in type_to_constructors.items(): for t, cs in type_to_constructors.items():
filename = path_for_type(t) filename = path_for_type(t)
out_dir = os.path.dirname(filename) out_dir = filename.parent
if out_dir: if out_dir:
os.makedirs(out_dir, exist_ok=True) out_dir.mkdir(parents=True, exist_ok=True)
# Since we don't have access to the full TLObject, split the type # Since we don't have access to the full TLObject, split the type
if '.' in t: if '.' in t:
@ -570,10 +572,10 @@ def _write_html_pages(tlobjects, methods, layer, input_res, output_dir):
# information that we have available, simply a file listing all the others # information that we have available, simply a file listing all the others
# accessible by clicking on their title # accessible by clicking on their title
for folder in ['types', 'methods', 'constructors']: for folder in ['types', 'methods', 'constructors']:
_generate_index(os.path.join(output_dir, folder), original_paths, _generate_index(output_dir / folder, original_paths,
output_dir) output_dir)
_generate_index(os.path.join(output_dir, 'methods'), original_paths, _generate_index(output_dir / 'methods', original_paths,
output_dir, True, bot_docs_paths) output_dir, True, bot_docs_paths)
# Write the final core index, the main index for the rest of files # Write the final core index, the main index for the rest of files
@ -596,8 +598,8 @@ def _write_html_pages(tlobjects, methods, layer, input_res, output_dir):
methods = sorted(methods, key=lambda m: m.name) methods = sorted(methods, key=lambda m: m.name)
cs = sorted(cs, key=lambda c: c.name) cs = sorted(cs, key=lambda c: c.name)
shutil.copy(os.path.join(input_res, '404.html'), original_paths['404']) shutil.copy(str(input_res / '404.html'), str(original_paths['404']))
_copy_replace(os.path.join(input_res, 'core.html'), _copy_replace(input_res / 'core.html',
original_paths['index_all'], { original_paths['index_all'], {
'{type_count}': len(types), '{type_count}': len(types),
'{method_count}': len(methods), '{method_count}': len(methods),
@ -630,10 +632,8 @@ def _write_html_pages(tlobjects, methods, layer, input_res, output_dir):
type_urls = fmt(types, path_for_type) type_urls = fmt(types, path_for_type)
constructor_urls = fmt(cs, create_path_for) constructor_urls = fmt(cs, create_path_for)
os.makedirs(os.path.abspath(os.path.join( original_paths['search.js'].parent.mkdir(parents=True, exist_ok=True)
original_paths['search.js'], os.path.pardir _copy_replace(input_res / 'js/search.js',
)), exist_ok=True)
_copy_replace(os.path.join(input_res, 'js', 'search.js'),
original_paths['search.js'], { original_paths['search.js'], {
'{request_names}': request_names, '{request_names}': request_names,
'{type_names}': type_names, '{type_names}': type_names,
@ -647,13 +647,13 @@ def _write_html_pages(tlobjects, methods, layer, input_res, output_dir):
def _copy_resources(res_dir, out_dir): def _copy_resources(res_dir, out_dir):
for dirname, files in [('css', ['docs.light.css', 'docs.dark.css']), for dirname, files in [('css', ['docs.light.css', 'docs.dark.css']),
('img', ['arrow.svg'])]: ('img', ['arrow.svg'])]:
dirpath = os.path.join(out_dir, dirname) dirpath = out_dir / dirname
os.makedirs(dirpath, exist_ok=True) dirpath.mkdir(parents=True, exist_ok=True)
for file in files: for file in files:
shutil.copy(os.path.join(res_dir, dirname, file), dirpath) shutil.copy(str(res_dir / dirname / file), str(dirpath))
def generate_docs(tlobjects, methods, layer, input_res, output_dir): def generate_docs(tlobjects, methods, layer, input_res, output_dir):
os.makedirs(output_dir, exist_ok=True) output_dir.mkdir(parents=True, exist_ok=True)
_write_html_pages(tlobjects, methods, layer, input_res, output_dir) _write_html_pages(tlobjects, methods, layer, input_res, output_dir)
_copy_resources(input_res, output_dir) _copy_resources(input_res, output_dir)

38
telethon_generator/generators/tlobject.py
View File

@ -48,11 +48,10 @@ PATCHED_TYPES = {
def _write_modules( def _write_modules(
out_dir, depth, kind, namespace_tlobjects, type_constructors): out_dir, depth, kind, namespace_tlobjects, type_constructors):
# namespace_tlobjects: {'namespace', [TLObject]} # namespace_tlobjects: {'namespace', [TLObject]}
os.makedirs(out_dir, exist_ok=True) out_dir.mkdir(parents=True, exist_ok=True)
for ns, tlobjects in namespace_tlobjects.items(): for ns, tlobjects in namespace_tlobjects.items():
file = os.path.join(out_dir, '{}.py'.format(ns or '__init__')) file = out_dir / '{}.py'.format(ns or '__init__')
with open(file, 'w', encoding='utf-8') as f,\ with file.open('w') as f, SourceBuilder(f) as builder:
SourceBuilder(f) as builder:
builder.writeln(AUTO_GEN_NOTICE) builder.writeln(AUTO_GEN_NOTICE)
builder.writeln('from {}.tl.tlobject import TLObject', '.' * depth) builder.writeln('from {}.tl.tlobject import TLObject', '.' * depth)
@ -635,11 +634,10 @@ def _write_arg_read_code(builder, arg, args, name):
def _write_patched(out_dir, namespace_tlobjects): def _write_patched(out_dir, namespace_tlobjects):
os.makedirs(out_dir, exist_ok=True) out_dir.mkdir(parents=True, exist_ok=True)
for ns, tlobjects in namespace_tlobjects.items(): for ns, tlobjects in namespace_tlobjects.items():
file = os.path.join(out_dir, '{}.py'.format(ns or '__init__')) file = out_dir / '{}.py'.format(ns or '__init__')
with open(file, 'w', encoding='utf-8') as f,\ with file.open('w') as f, SourceBuilder(f) as builder:
SourceBuilder(f) as builder:
builder.writeln(AUTO_GEN_NOTICE) builder.writeln(AUTO_GEN_NOTICE)
builder.writeln('import struct') builder.writeln('import struct')
@ -715,26 +713,24 @@ def generate_tlobjects(tlobjects, layer, import_depth, output_dir):
if tlobject.fullname in PATCHED_TYPES: if tlobject.fullname in PATCHED_TYPES:
namespace_patched[tlobject.namespace].append(tlobject) namespace_patched[tlobject.namespace].append(tlobject)
get_file = functools.partial(os.path.join, output_dir) _write_modules(output_dir / 'functions', import_depth, 'TLRequest',
_write_modules(get_file('functions'), import_depth, 'TLRequest',
namespace_functions, type_constructors) namespace_functions, type_constructors)
_write_modules(get_file('types'), import_depth, 'TLObject', _write_modules(output_dir / 'types', import_depth, 'TLObject',
namespace_types, type_constructors) namespace_types, type_constructors)
_write_patched(get_file('patched'), namespace_patched) _write_patched(output_dir / 'patched', namespace_patched)
filename = os.path.join(get_file('alltlobjects.py')) filename = output_dir / 'alltlobjects.py'
with open(filename, 'w', encoding='utf-8') as file: with filename.open('w') as file:
with SourceBuilder(file) as builder: with SourceBuilder(file) as builder:
_write_all_tlobjects(tlobjects, layer, builder) _write_all_tlobjects(tlobjects, layer, builder)
def clean_tlobjects(output_dir): def clean_tlobjects(output_dir):
get_file = functools.partial(os.path.join, output_dir)
for d in ('functions', 'types'): for d in ('functions', 'types'):
d = get_file(d) d = output_dir / d
if os.path.isdir(d): if d.is_dir():
shutil.rmtree(d) shutil.rmtree(str(d))
tl = get_file('alltlobjects.py') tl = output_dir / 'alltlobjects.py'
if os.path.isfile(tl): if tl.is_file():
os.remove(tl) tl.unlink()

2
telethon_generator/parsers/errors.py
View File

@ -57,7 +57,7 @@ def parse_errors(csv_file):
Parses the input CSV file with columns (name, error codes, description) Parses the input CSV file with columns (name, error codes, description)
and yields `Error` instances as a result. and yields `Error` instances as a result.
""" """
with open(csv_file, newline='') as f: with csv_file.open(newline='') as f:
f = csv.reader(f) f = csv.reader(f)
next(f, None) # header next(f, None) # header
for line, (name, codes, description) in enumerate(f, start=2): for line, (name, codes, description) in enumerate(f, start=2):

2
telethon_generator/parsers/methods.py
View File

@ -30,7 +30,7 @@ def parse_methods(csv_file, errors_dict):
Parses the input CSV file with columns (method, usability, errors) Parses the input CSV file with columns (method, usability, errors)
and yields `MethodInfo` instances as a result. and yields `MethodInfo` instances as a result.
""" """
with open(csv_file, newline='') as f: with csv_file.open(newline='') as f:
f = csv.reader(f) f = csv.reader(f)
next(f, None) # header next(f, None) # header
for line, (method, usability, errors) in enumerate(f, start=2): for line, (method, usability, errors) in enumerate(f, start=2):

2
telethon_generator/parsers/tlobject/parser.py
View File

@ -86,7 +86,7 @@ def parse_tl(file_path, layer, methods=None, ignored_ids=CORE_TYPES):
obj_all = [] obj_all = []
obj_by_name = {} obj_by_name = {}
obj_by_type = collections.defaultdict(list) obj_by_type = collections.defaultdict(list)
with open(file_path, 'r', encoding='utf-8') as file: with file_path.open() as file:
is_function = False is_function = False
for line in file: for line in file:
comment_index = line.find('//') comment_index = line.find('//')