mirror of
https://github.com/LonamiWebs/Telethon.git
synced 2024-11-10 19:46:36 +03:00
Implemented init and write code on TLObjects Generator
The code generated by the generator now classifies the output files in their corresponding categories, also writing their __init__(...) with documented arguments, and the on_send(...) method
This commit is contained in:
parent
1974569927
commit
06832f8108
21
README.md
21
README.md
|
@ -1,5 +1,7 @@
|
||||||
# Telethon
|
# Telethon
|
||||||
**Telethon** is Telegram client implementation in Python. This project is **completely based** on [TLSharp](https://github.com/sochix/TLSharp), so please, also have a look to the original project!
|
**Telethon** is Telegram client implementation in Python. This project's _core_ is **completely based** on [TLSharp](https://github.com/sochix/TLSharp), so please, also have a look to the original project!
|
||||||
|
|
||||||
|
Other parts, such as the request themselves, the .tl tokenizer and code generator, or some ported C# utilities such as `BinaryWriter`, `BinaryReader`, `TCPClient` and so on, are no longer part of TLSharp itself.
|
||||||
|
|
||||||
### Requirements
|
### Requirements
|
||||||
This project requires the following Python modules, which can be installed by issuing `sudo -H pip install <module>` on a Linux terminal:
|
This project requires the following Python modules, which can be installed by issuing `sudo -H pip install <module>` on a Linux terminal:
|
||||||
|
@ -7,3 +9,20 @@ This project requires the following Python modules, which can be installed by is
|
||||||
|
|
||||||
### We need your help!
|
### We need your help!
|
||||||
As of now, the project is fully **untested** and with many pending things to do. If you know both Python and C#, please don't think it twice and help us (me)!
|
As of now, the project is fully **untested** and with many pending things to do. If you know both Python and C#, please don't think it twice and help us (me)!
|
||||||
|
|
||||||
|
### Code generator limitations
|
||||||
|
The current code generator is not complete, yet adding the missing features would only over-complicate an already hard-to-read code.
|
||||||
|
Some parts of the .tl file _should_ be omitted, because they're "built-in" in the generated code (such as writing booleans, etc.).
|
||||||
|
|
||||||
|
In order to make sure that all the generated files will work, please make sure to **always** comment out these lines in `scheme.tl`
|
||||||
|
(the latest version can always be found [here](https://github.com/telegramdesktop/tdesktop/blob/master/Telegram/SourceFiles/mtproto/scheme.tl)):
|
||||||
|
|
||||||
|
```tl
|
||||||
|
// boolFalse#bc799737 = Bool;
|
||||||
|
// boolTrue#997275b5 = Bool;
|
||||||
|
// true#3fedd339 = True;
|
||||||
|
// vector#1cb5c415 {t:Type} # [ t ] = Vector t;
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
7
main.py
7
main.py
|
@ -1,8 +1,5 @@
|
||||||
from parser.tl_parser import TLParser
|
import tlobjects_generator
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
tlobjects_generator.generate_tlobjecs()
|
||||||
parser = TLParser()
|
|
||||||
for tlobject in parser.parse_file('parser/scheme.tl'):
|
|
||||||
print(tlobject)
|
|
||||||
|
|
|
@ -50,7 +50,7 @@ class MtProtoSender:
|
||||||
# TODO Is there any difference with unsigned long and long?
|
# TODO Is there any difference with unsigned long and long?
|
||||||
writer.write_long(self._session.salt, signed=False)
|
writer.write_long(self._session.salt, signed=False)
|
||||||
writer.write_long(self._session.id, signed=False)
|
writer.write_long(self._session.id, signed=False)
|
||||||
writer.write_long(request.message_id)
|
writer.write_long(request.msg_id)
|
||||||
writer.write_int(self.generate_sequence(request.confirmed))
|
writer.write_int(self.generate_sequence(request.confirmed))
|
||||||
writer.write_int(len(packet))
|
writer.write_int(len(packet))
|
||||||
writer.write(packet)
|
writer.write(packet)
|
||||||
|
|
|
@ -1,13 +1,18 @@
|
||||||
|
from io import StringIO
|
||||||
|
|
||||||
|
|
||||||
class SourceBuilder:
|
class SourceBuilder:
|
||||||
"""This class should be used to build .py source files"""
|
"""This class should be used to build .py source files"""
|
||||||
|
|
||||||
def __init__(self, indent_size=4):
|
def __init__(self, out_stream=None, indent_size=4):
|
||||||
self.current_indent = 0
|
self.current_indent = 0
|
||||||
self.on_new_line = False
|
self.on_new_line = False
|
||||||
self.indent_size = indent_size
|
self.indent_size = indent_size
|
||||||
|
|
||||||
self.buffer = []
|
if out_stream is None:
|
||||||
|
self.out_stream = StringIO()
|
||||||
|
else:
|
||||||
|
self.out_stream = out_stream
|
||||||
|
|
||||||
def indent(self):
|
def indent(self):
|
||||||
self.write(' ' * (self.current_indent * self.indent_size))
|
self.write(' ' * (self.current_indent * self.indent_size))
|
||||||
|
@ -15,9 +20,10 @@ class SourceBuilder:
|
||||||
def write(self, string):
|
def write(self, string):
|
||||||
if self.on_new_line:
|
if self.on_new_line:
|
||||||
self.on_new_line = False # We're not on a new line anymore
|
self.on_new_line = False # We're not on a new line anymore
|
||||||
self.indent()
|
if string.strip(): # If the string was not empty, indent; Else it probably was a new line
|
||||||
|
self.indent()
|
||||||
|
|
||||||
self.buffer += list(string)
|
self.out_stream.write(string)
|
||||||
|
|
||||||
def writeln(self, string=''):
|
def writeln(self, string=''):
|
||||||
self.write(string + '\n')
|
self.write(string + '\n')
|
||||||
|
@ -32,7 +38,12 @@ class SourceBuilder:
|
||||||
self.writeln()
|
self.writeln()
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
if self.buffer:
|
self.out_stream.seek(0)
|
||||||
return ''.join(self.buffer)
|
return self.out_stream.read()
|
||||||
else:
|
|
||||||
return ''
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
|
self.out_stream.flush()
|
||||||
|
self.out_stream.close()
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
// Core types (no need to gen)
|
// Core types (no need to gen)
|
||||||
|
|
||||||
|
// We handle some types in a special way
|
||||||
|
//boolFalse#bc799737 = Bool;
|
||||||
|
//boolTrue#997275b5 = Bool;
|
||||||
|
//true#3fedd339 = True;
|
||||||
//vector#1cb5c415 {t:Type} # [ t ] = Vector t;
|
//vector#1cb5c415 {t:Type} # [ t ] = Vector t;
|
||||||
|
|
||||||
///////////////////////////////
|
///////////////////////////////
|
||||||
|
@ -119,11 +123,6 @@ contest.saveDeveloperInfo#9a5f6e95 vk_id:int name:string phone_number:string age
|
||||||
|
|
||||||
---types---
|
---types---
|
||||||
|
|
||||||
boolFalse#bc799737 = Bool;
|
|
||||||
boolTrue#997275b5 = Bool;
|
|
||||||
|
|
||||||
true#3fedd339 = True;
|
|
||||||
|
|
||||||
error#c4b9f9bb code:int text:string = Error;
|
error#c4b9f9bb code:int text:string = Error;
|
||||||
|
|
||||||
null#56730bcc = Null;
|
null#56730bcc = Null;
|
|
@ -96,7 +96,8 @@ class TLArg:
|
||||||
"""
|
"""
|
||||||
self.name = name
|
self.name = name
|
||||||
|
|
||||||
# Default flag values
|
# Default values
|
||||||
|
self.is_vector = False
|
||||||
self.is_flag = False
|
self.is_flag = False
|
||||||
self.flag_index = -1
|
self.flag_index = -1
|
||||||
|
|
||||||
|
@ -119,15 +120,27 @@ class TLArg:
|
||||||
self.flag_index = int(flag_match.group(1))
|
self.flag_index = int(flag_match.group(1))
|
||||||
self.type = flag_match.group(2) # Update the type to match the exact type, not the "flagged" one
|
self.type = flag_match.group(2) # Update the type to match the exact type, not the "flagged" one
|
||||||
|
|
||||||
|
# Then check if the type is a Vector<REAL_TYPE>
|
||||||
|
vector_match = re.match(r'vector<(\w+)>', self.type, re.IGNORECASE)
|
||||||
|
if vector_match:
|
||||||
|
self.is_vector = True
|
||||||
|
self.type = vector_match.group(1) # Update the type to match the one inside the vector
|
||||||
|
|
||||||
self.generic_definition = generic_definition
|
self.generic_definition = generic_definition
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
type = ('!{}'.format(self.type) if self.is_generic
|
# Find the real type representation by updating it as required
|
||||||
else
|
real_type = self.type
|
||||||
('flags.{}?{}'.format(self.flag_index, self.type) if self.is_flag
|
if self.is_vector:
|
||||||
else self.type))
|
real_type = 'Vector<{}>'.format(real_type)
|
||||||
|
|
||||||
|
if self.is_generic:
|
||||||
|
real_type = '!{}'.format(real_type)
|
||||||
|
|
||||||
|
if self.is_flag:
|
||||||
|
real_type = 'flags.{}?{}'.format(self.flag_index, real_type)
|
||||||
|
|
||||||
if self.generic_definition:
|
if self.generic_definition:
|
||||||
return '{{{}:{}}}'.format(self.name, type)
|
return '{{{}:{}}}'.format(self.name, real_type)
|
||||||
else:
|
else:
|
||||||
return '{}:{}'.format(self.name, type)
|
return '{}:{}'.format(self.name, real_type)
|
||||||
|
|
217
tlobjects_generator.py
Normal file
217
tlobjects_generator.py
Normal file
|
@ -0,0 +1,217 @@
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from parser.tl_parser import TLParser
|
||||||
|
from parser.source_builder import SourceBuilder
|
||||||
|
|
||||||
|
|
||||||
|
def generate_tlobjecs():
|
||||||
|
"""Generates all the TLObjects from scheme.tl to tl/functions and tl/types"""
|
||||||
|
|
||||||
|
# First ensure that the required parent directories exist
|
||||||
|
os.makedirs('tl/functions', exist_ok=True)
|
||||||
|
os.makedirs('tl/types', exist_ok=True)
|
||||||
|
for tlobject in TLParser.parse_file('scheme.tl'):
|
||||||
|
|
||||||
|
# Determine the output directory and create it
|
||||||
|
out_dir = os.path.join('tl',
|
||||||
|
'functions' if tlobject.is_function
|
||||||
|
else 'types')
|
||||||
|
|
||||||
|
if tlobject.namespace is not None:
|
||||||
|
out_dir = os.path.join(out_dir, tlobject.namespace)
|
||||||
|
|
||||||
|
os.makedirs(out_dir, exist_ok=True)
|
||||||
|
|
||||||
|
init_py = os.path.join(out_dir, '__init__.py')
|
||||||
|
# Also create __init__.py
|
||||||
|
if not os.path.isfile(init_py):
|
||||||
|
open(init_py, 'a').close()
|
||||||
|
|
||||||
|
# Create the file
|
||||||
|
filename = os.path.join(out_dir, get_file_name(tlobject))
|
||||||
|
with open(filename, 'w', encoding='utf-8') as file:
|
||||||
|
|
||||||
|
# Let's build the source code!
|
||||||
|
with SourceBuilder(file) as builder:
|
||||||
|
builder.writeln('from requests.mtproto_request import MTProtoRequest')
|
||||||
|
builder.writeln()
|
||||||
|
builder.writeln()
|
||||||
|
builder.writeln('class {}(MTProtoRequest):'.format(get_class_name(tlobject)))
|
||||||
|
|
||||||
|
# Write the original .tl definition, along with a "generated automatically" message
|
||||||
|
builder.writeln('"""Class generated by TLObjects\' generator. '
|
||||||
|
'All changes will be ERASED. Original .tl definition below.')
|
||||||
|
builder.writeln('{}"""'.format(tlobject))
|
||||||
|
builder.writeln()
|
||||||
|
|
||||||
|
# First sort the arguments so that those not being a flag come first
|
||||||
|
args = sorted([arg for arg in tlobject.args if not arg.flag_indicator],
|
||||||
|
key=lambda x: x.is_flag)
|
||||||
|
|
||||||
|
# Then convert the args to string parameters, the flags having =None
|
||||||
|
args = [(arg.name if not arg.is_flag
|
||||||
|
else '{}=None'.format(arg.name)) for arg in args
|
||||||
|
if not arg.flag_indicator and not arg.generic_definition]
|
||||||
|
|
||||||
|
# Write the __init__ function
|
||||||
|
if args:
|
||||||
|
builder.writeln('def __init__(self, {}):'.format(', '.join(args)))
|
||||||
|
else:
|
||||||
|
builder.writeln('def __init__(self):')
|
||||||
|
|
||||||
|
# Now update args to have the TLObject arguments, _except_
|
||||||
|
# those which are generated automatically: flag indicator and generic definitions.
|
||||||
|
# We don't need the generic definitions in Python because arguments can be any type
|
||||||
|
args = [arg for arg in tlobject.args
|
||||||
|
if not arg.flag_indicator and not arg.generic_definition]
|
||||||
|
|
||||||
|
if args:
|
||||||
|
# Write the docstring, so we know the type of the arguments
|
||||||
|
builder.writeln('"""')
|
||||||
|
for arg in args:
|
||||||
|
if not arg.flag_indicator:
|
||||||
|
builder.write(':param {}: Telegram type: «{}».'.format(arg.name, arg.type))
|
||||||
|
if arg.is_vector:
|
||||||
|
builder.write(' Must be a list.'.format(arg.name))
|
||||||
|
if arg.is_generic:
|
||||||
|
builder.write(' This should be another MTProtoRequest.')
|
||||||
|
builder.writeln()
|
||||||
|
builder.writeln('"""')
|
||||||
|
|
||||||
|
builder.writeln('super().__init__()')
|
||||||
|
# Leave an empty line if there are any args
|
||||||
|
if args:
|
||||||
|
builder.writeln()
|
||||||
|
|
||||||
|
for arg in args:
|
||||||
|
builder.writeln('self.{0} = {0}'.format(arg.name))
|
||||||
|
builder.end_block()
|
||||||
|
|
||||||
|
# Write the on_send(self, writer) function
|
||||||
|
builder.writeln('def on_send(self, writer):')
|
||||||
|
builder.writeln("writer.write_int({}) # {}'s constructor ID"
|
||||||
|
.format(hex(tlobject.id), tlobject.name))
|
||||||
|
|
||||||
|
for arg in tlobject.args:
|
||||||
|
write_onsend_code(builder, arg, tlobject.args)
|
||||||
|
builder.end_block()
|
||||||
|
|
||||||
|
|
||||||
|
def get_class_name(tlobject):
|
||||||
|
# Courtesy of http://stackoverflow.com/a/31531797/4759433
|
||||||
|
# Also, '_' could be replaced for ' ', then use .title(), and then remove ' '
|
||||||
|
result = re.sub(r'_([a-z])', lambda m: m.group(1).upper(), tlobject.name)
|
||||||
|
return result[:1].upper() + result[1:].replace('_', '') # Replace again to fully ensure!
|
||||||
|
|
||||||
|
|
||||||
|
def get_file_name(tlobject):
|
||||||
|
# Courtesy of http://stackoverflow.com/a/1176023/4759433
|
||||||
|
s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', tlobject.name)
|
||||||
|
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + '.py'
|
||||||
|
|
||||||
|
|
||||||
|
foundEver = set()
|
||||||
|
def write_onsend_code(builder, arg, args, name=None):
|
||||||
|
"""
|
||||||
|
Writes the write code for the given argument
|
||||||
|
|
||||||
|
:param builder: The source code builder
|
||||||
|
:param arg: The argument to write
|
||||||
|
:param args: All the other arguments in TLObject same on_send. This is required to determine the flags value
|
||||||
|
:param name: The name of the argument. Defaults to «self.argname»
|
||||||
|
This argument is an option because it's required when writing Vectors<>
|
||||||
|
"""
|
||||||
|
|
||||||
|
if arg.generic_definition:
|
||||||
|
return # Do nothing, this only specifies a later type
|
||||||
|
|
||||||
|
if name is None:
|
||||||
|
name = 'self.{}'.format(arg.name)
|
||||||
|
|
||||||
|
# The argument may be a flag, only write if it's not None!
|
||||||
|
if arg.is_flag:
|
||||||
|
builder.writeln('if {} is not None:'.format(name))
|
||||||
|
|
||||||
|
if arg.is_vector:
|
||||||
|
builder.writeln("writer.write_int(0x1cb5c415) # Vector's constructor ID")
|
||||||
|
builder.writeln('writer.write_int(len({}))'.format(name))
|
||||||
|
builder.writeln('for {}_item in {}:'.format(arg.name, name))
|
||||||
|
# Temporary disable .is_vector, not to enter this if again
|
||||||
|
arg.is_vector = False
|
||||||
|
write_onsend_code(builder, arg, args, name='{}_item'.format(arg.name))
|
||||||
|
arg.is_vector = True
|
||||||
|
|
||||||
|
elif arg.flag_indicator:
|
||||||
|
# Calculate the flags with those items which are not None
|
||||||
|
builder.writeln('# Calculate the flags. This equals to those flag arguments which are NOT None')
|
||||||
|
builder.writeln('flags = 0')
|
||||||
|
for flag in args:
|
||||||
|
if flag.is_flag:
|
||||||
|
builder.writeln('flags |= (1 << {}) if {} is not None else 0'
|
||||||
|
.format(flag.flag_index, 'self.{}'.format(flag.name)))
|
||||||
|
|
||||||
|
builder.writeln('writer.write_int(flags)')
|
||||||
|
builder.writeln()
|
||||||
|
|
||||||
|
elif 'int' == arg.type:
|
||||||
|
builder.writeln('writer.write_int({})'.format(name))
|
||||||
|
|
||||||
|
elif 'long' == arg.type:
|
||||||
|
builder.writeln('writer.write_long({})'.format(name))
|
||||||
|
|
||||||
|
elif 'int128' == arg.type:
|
||||||
|
builder.writeln('writer.write_large_int({}, bits=128)'.format(name))
|
||||||
|
|
||||||
|
elif 'int256' == arg.type:
|
||||||
|
builder.writeln('writer.write_large_int({}, bits=256)'.format(name))
|
||||||
|
|
||||||
|
elif 'double' == arg.type:
|
||||||
|
builder.writeln('writer.write_double({})'.format(name))
|
||||||
|
|
||||||
|
elif 'string' == arg.type:
|
||||||
|
builder.writeln('writer.tgwrite_string({})'.format(name))
|
||||||
|
|
||||||
|
elif 'Bool' == arg.type:
|
||||||
|
builder.writeln('writer.tgwrite_bool({})'.format(name))
|
||||||
|
|
||||||
|
elif 'true' == arg.type: # Awkwardly enough, Telegram has both bool and "true", used in flags
|
||||||
|
builder.writeln('writer.write_int(0x3fedd339) # true')
|
||||||
|
|
||||||
|
elif 'bytes' == arg.type:
|
||||||
|
builder.writeln('writer.write({})'.format(name))
|
||||||
|
|
||||||
|
else:
|
||||||
|
# Else it may be a custom type
|
||||||
|
builder.writeln('{}.write(writer)'.format(name))
|
||||||
|
if arg.type not in foundEver:
|
||||||
|
foundEver.add(arg.type)
|
||||||
|
print('{}: {}'.format(arg.type, arg))
|
||||||
|
|
||||||
|
# End vector and flag blocks if required (if we opened them before)
|
||||||
|
if arg.is_vector:
|
||||||
|
builder.end_block()
|
||||||
|
|
||||||
|
if arg.is_flag:
|
||||||
|
builder.end_block()
|
||||||
|
|
||||||
|
|
||||||
|
''' SourceBuilder generated file example:
|
||||||
|
|
||||||
|
class Example(MTProtoRequest):
|
||||||
|
def __init__(self, some, parameter):
|
||||||
|
"""
|
||||||
|
.tl definition: Example#12345678 some:int parameter:int = Exmpl
|
||||||
|
:param some: [type=Vector<int>] Cannot be NONE
|
||||||
|
:param parameter: [type=int] Cannot be NONE
|
||||||
|
"""
|
||||||
|
|
||||||
|
def on_send(self, writer):
|
||||||
|
writer.write_int(0x62d6b459) # example's constructor ID
|
||||||
|
writer.write_int(0x1cb5c415) # vector code
|
||||||
|
writer.write_int(len(self.msgs))
|
||||||
|
for some_item in self.some:
|
||||||
|
writer.write_int(some_item)
|
||||||
|
|
||||||
|
def on_response(self, reader):
|
||||||
|
pass
|
||||||
|
'''
|
|
@ -17,18 +17,31 @@ class BinaryWriter:
|
||||||
|
|
||||||
# region Writing
|
# region Writing
|
||||||
|
|
||||||
def write_byte(self, byte):
|
def write_byte(self, value):
|
||||||
self.writer.write(pack('B', byte))
|
self.writer.write(pack('B', value))
|
||||||
|
|
||||||
def write_int(self, integer, signed=True):
|
def write_int(self, value, signed=True):
|
||||||
if not signed:
|
if signed:
|
||||||
integer &= 0xFFFFFFFF # Ensure it's unsigned (see http://stackoverflow.com/a/30092291/4759433)
|
self.writer.write(pack('i', value))
|
||||||
self.writer.write(pack('I', integer))
|
else:
|
||||||
|
value &= 0xFFFFFFFF # Ensure it's unsigned (see http://stackoverflow.com/a/30092291/4759433)
|
||||||
|
self.writer.write(pack('I', value))
|
||||||
|
|
||||||
def write_long(self, long, signed=True):
|
def write_long(self, value, signed=True):
|
||||||
if not signed:
|
if signed:
|
||||||
long &= 0xFFFFFFFFFFFFFFFF
|
self.writer.write(pack('q', value))
|
||||||
self.writer.write(pack('Q', long))
|
else:
|
||||||
|
value &= 0xFFFFFFFFFFFFFFFF
|
||||||
|
self.writer.write(pack('Q', value))
|
||||||
|
|
||||||
|
def write_float(self, value):
|
||||||
|
self.writer.write(pack('f', value))
|
||||||
|
|
||||||
|
def write_double(self, value):
|
||||||
|
self.writer.write(pack('d', value))
|
||||||
|
|
||||||
|
def write_large_int(self, value, bits):
|
||||||
|
self.writer.write(pack('{}B'.format(bits // 8), value))
|
||||||
|
|
||||||
def write(self, data):
|
def write(self, data):
|
||||||
self.writer.write(data)
|
self.writer.write(data)
|
||||||
|
@ -71,6 +84,10 @@ class BinaryWriter:
|
||||||
def tgwrite_string(self, string):
|
def tgwrite_string(self, string):
|
||||||
return self.tgwrite_bytes(string.encode('utf-8'))
|
return self.tgwrite_bytes(string.encode('utf-8'))
|
||||||
|
|
||||||
|
def tgwrite_bool(self, bool):
|
||||||
|
# boolTrue boolFalse
|
||||||
|
return self.write_int(0x997275b5 if bool else 0xbc799737, signed=False)
|
||||||
|
|
||||||
# endregion
|
# endregion
|
||||||
|
|
||||||
def flush(self):
|
def flush(self):
|
||||||
|
|
Loading…
Reference in New Issue
Block a user