Enhance setup.py gen command

readthedocs/extra/basic/installation.rst

@@ -54,7 +54,7 @@ Manual Installation
 
 3. Enter the cloned repository: ``cd Telethon``
 
-4. Run the code generator: ``python3 setup.py gen_tl``
+4. Run the code generator: ``python3 setup.py gen tl errors``
 
 5. Done!
 

setup.py

@@ -6,16 +6,16 @@ https://packaging.python.org/en/latest/distributing.html
 https://github.com/pypa/sampleproject
 
 Extra supported commands are:
-* gen_tl, to generate the classes required for Telethon to run
+* gen, to generate the classes required for Telethon to run or docs
-* clean_tl, to clean these generated classes
 * pypi, to generate sdist, bdist_wheel, and push to PyPi
 """
 
-# To use a consistent encoding
-from codecs import open
-from sys import argv, version_info
 import os
 import re
+# To use a consistent encoding
+import shutil
+from codecs import open
+from sys import argv, version_info
 
 # Always prefer setuptools over distutils
 from setuptools import find_packages, setup
@@ -37,45 +37,85 @@ class TempWorkDir:
         os.chdir(self.original)
 
 
-ERROR_LIST = 'telethon/errors/rpc_error_list.py'
+GENERATOR_DIR = 'telethon_generator'
-ERRORS_JSON = 'telethon_generator/errors.json'
+LIBRARY_DIR = 'telethon'
-ERRORS_DESC = 'telethon_generator/error_descriptions'
+
-SCHEME_TL = 'telethon_generator/scheme.tl'
+ERRORS_IN_JSON = os.path.join(GENERATOR_DIR, 'data', 'errors.json')
-GENERATOR_DIR = 'telethon/tl'
+ERRORS_IN_DESC = os.path.join(GENERATOR_DIR, 'data', 'error_descriptions')
+ERRORS_OUT = os.path.join(LIBRARY_DIR, 'errors', 'rpc_error_list.py')
+
+TLOBJECT_IN_TL = os.path.join(GENERATOR_DIR, 'data', 'scheme.tl')
+TLOBJECT_OUT = os.path.join(LIBRARY_DIR, 'tl')
 IMPORT_DEPTH = 2
 
+DOCS_IN_RES = os.path.join(GENERATOR_DIR, 'data', 'html')
+DOCS_OUT = 'docs'
 
-def gen_tl(force=True):
-    from telethon_generator.tl_generator import TLGenerator
-    # TODO Generate errors again
-    #from telethon_generator.error_generator import generate_code
-    generator = TLGenerator(GENERATOR_DIR)
-    if generator.tlobjects_exist():
-        if not force:
-            return
-        print('Detected previous TLObjects. Cleaning...')
-        generator.clean_tlobjects()
 
-    print('Generating TLObjects...')
+def generate(which):
-    generator.generate_tlobjects(SCHEME_TL, import_depth=IMPORT_DEPTH)
+    from telethon_generator.parsers import parse_errors, parse_tl, find_layer
-    print('Generating errors...')
+    from telethon_generator.generators import\
-    #generate_code(ERROR_LIST, json_file=ERRORS_JSON, errors_desc=ERRORS_DESC)
+        generate_errors, generate_tlobjects, generate_docs, clean_tlobjects
-    print('Done.')
+
+    tlobjects = list(parse_tl(TLOBJECT_IN_TL, ignore_core=True))
+    errors = list(parse_errors(ERRORS_IN_JSON, ERRORS_IN_DESC))
+    layer = find_layer(TLOBJECT_IN_TL)
+
+    if not which:
+        which.append('(empty)')
+
+    clean = 'clean' in which
+    action = 'Cleaning' if clean else 'Generating'
+    if clean:
+        which.remove('clean')
+
+    if 'all' in which:
+        which.remove('all')
+        for x in ('tl', 'errors', 'docs'):
+            if x not in which:
+                which.append(x)
+
+    if 'tl' in which:
+        which.remove('tl')
+        print(action, 'TLObjects...')
+        if clean:
+            clean_tlobjects(TLOBJECT_OUT)
+        else:
+            generate_tlobjects(tlobjects, layer, IMPORT_DEPTH, TLOBJECT_OUT)
+
+    if 'errors' in which:
+        which.remove('errors')
+        print(action, 'RPCErrors...')
+        if clean:
+            if os.path.isfile(ERRORS_OUT):
+                os.remove(ERRORS_OUT)
+        else:
+            with open(ERRORS_OUT, 'w', encoding='utf-8') as file:
+                generate_errors(errors, file)
+
+    if 'docs' in which:
+        which.remove('docs')
+        print(action, 'documentation...')
+        if clean:
+            if os.path.isdir(DOCS_OUT):
+                shutil.rmtree(DOCS_OUT)
+        else:
+            generate_docs(tlobjects, errors, layer, DOCS_IN_RES, DOCS_OUT)
+
+    if which:
+        print('The following items were not understood:', which)
+        print('  Consider using only "tl", "errors" and/or "docs".')
+        print('  Using only "clean" will clean them. "all" to act on all.')
+        print('  For instance "gen tl errors".')
 
 
 def main():
-    if len(argv) >= 2 and argv[1] == 'gen_tl':
+    if len(argv) >= 2 and argv[1] == 'gen':
-        gen_tl()
+        generate(argv[2:])
-
-    elif len(argv) >= 2 and argv[1] == 'clean_tl':
-        from telethon_generator.tl_generator import TLGenerator
-        print('Cleaning...')
-        TLGenerator(GENERATOR_DIR).clean_tlobjects()
-        print('Done.')
 
     elif len(argv) >= 2 and argv[1] == 'pypi':
         # (Re)generate the code to make sure we don't push without it
-        gen_tl()
+        generate(['clean', 'tl', 'errors'])
 
         # Try importing the telethon module to assert it has no errors
         try:
@@ -97,14 +137,10 @@ def main():
         for x in ('build', 'dist', 'Telethon.egg-info'):
             rmtree(x, ignore_errors=True)
 
-    elif len(argv) >= 2 and argv[1] == 'fetch_errors':
-        from telethon_generator.error_generator import fetch_errors
-        fetch_errors(ERRORS_JSON)
-
     else:
-        # Call gen_tl() if the scheme.tl file exists, e.g. install from GitHub
+        # e.g. install from GitHub
-        if os.path.isfile(SCHEME_TL):
+        if os.path.isfile(GENERATOR_DIR):
-            gen_tl(force=False)
+            generate(['clean', 'tl', 'errors'])
 
         # Get the long description from the README file
         with open('README.rst', encoding='utf-8') as f:

telethon_generator/generators/__init__.py

@@ -1,3 +1,3 @@
 from .errors import generate_errors
-from .tlobject import generate_tlobjects
+from .tlobject import generate_tlobjects, clean_tlobjects
 from .docs import generate_docs

telethon_generator/generators/tlobject.py

@@ -1,5 +1,7 @@
+import functools
 import os
 import re
+import shutil
 import struct
 from collections import defaultdict
 from zlib import crc32
@@ -617,10 +619,8 @@ def _write_all_tlobjects(tlobjects, layer, builder):
     builder.writeln('}')
 
 
-def generate_tlobjects(tlobjects, layer, output_dir):
+def generate_tlobjects(tlobjects, layer, import_depth, output_dir):
-    def get_file(*paths):
+    get_file = functools.partial(os.path.join, output_dir)
-        return os.path.join(output_dir, *paths)
-
     os.makedirs(get_file('functions'), exist_ok=True)
     os.makedirs(get_file('types'), exist_ok=True)
 
@@ -637,7 +637,6 @@ def generate_tlobjects(tlobjects, layer, output_dir):
             namespace_types[tlobject.namespace].append(tlobject)
             type_constructors[tlobject.result].append(tlobject)
 
-    import_depth = 2
     _write_modules(get_file('functions'), import_depth,
                    namespace_functions, type_constructors)
     _write_modules(get_file('types'), import_depth,
@@ -647,3 +646,15 @@ def generate_tlobjects(tlobjects, layer, output_dir):
     with open(filename, 'w', encoding='utf-8') as file:
         with SourceBuilder(file) as builder:
             _write_all_tlobjects(tlobjects, layer, builder)
+
+
+def clean_tlobjects(output_dir):
+    get_file = functools.partial(os.path.join, output_dir)
+    for d in ('functions', 'types'):
+        d = get_file(d)
+        if os.path.isdir(d):
+            shutil.rmtree(d)
+
+    tl = get_file('all_tlobjects.py')
+    if os.path.isfile(tl):
+        os.remove(tl)