From 2c5c6745317864830d89b80864fa42cada4e5af7 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 18 Sep 2017 20:19:39 +0200 Subject: [PATCH 001/121] Stop using TLObject.__repr__ to show the TL definition --- telethon_generator/tl_generator.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index e0d207ac..f40f7f1d 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -352,11 +352,7 @@ class TLGenerator: builder.writeln('pass') builder.end_block() - # Write the __repr__(self) and __str__(self) functions - builder.writeln('def __repr__(self):') - builder.writeln("return '{}'".format(repr(tlobject))) - builder.end_block() - + # Write the __str__(self) and stringify(self) functions builder.writeln('def __str__(self):') builder.writeln('return TLObject.pretty_format(self)') builder.end_block() From 2595d45bd7d288d01b72674449a84dd464d79bea Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 18 Sep 2017 20:23:03 +0200 Subject: [PATCH 002/121] Import os only once on the generated code --- telethon_generator/tl_generator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index f40f7f1d..ebc865f3 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -137,13 +137,17 @@ class TLGenerator: x for x in namespace_tlobjects.keys() if x ))) + # Import 'os' for those needing access to 'os.urandom()' + # Currently only 'random_id' needs 'os' to be imported, + # for all those TLObjects with arg.can_be_inferred. + builder.writeln('import os') + # Generate the class for every TLObject for t in sorted(tlobjects, key=lambda x: x.name): TLGenerator._write_source_code( t, builder, depth, type_constructors ) - while builder.current_indent != 0: - builder.end_block() + builder.current_indent = 0 @staticmethod def _write_source_code(tlobject, builder, depth, type_constructors): @@ -170,10 +174,6 @@ class TLGenerator: builder.writeln('from {}.utils import {}'.format( '.' * depth, ', '.join(util_imports))) - if any(a for a in tlobject.args if a.can_be_inferred): - # Currently only 'random_id' needs 'os' to be imported - builder.writeln('import os') - builder.writeln() builder.writeln() builder.writeln('class {}(TLObject):'.format( From f7e4f3f678af1ca89416ac5c076c61950d0cbc4d Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 18 Sep 2017 21:00:06 +0200 Subject: [PATCH 003/121] Make type hinting on the generated code more IDE-friendly --- telethon_generator/parser/tl_object.py | 31 +++++++++++++++-- telethon_generator/tl_generator.py | 47 +++++++------------------- 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/telethon_generator/parser/tl_object.py b/telethon_generator/parser/tl_object.py index c8ccae83..a90428b4 100644 --- a/telethon_generator/parser/tl_object.py +++ b/telethon_generator/parser/tl_object.py @@ -96,6 +96,17 @@ class TLObject: result=match.group(3), is_function=is_function) + def class_name(self): + """Gets the class name following the Python style guidelines""" + + # Courtesy of http://stackoverflow.com/a/31531797/4759433 + result = re.sub(r'_([a-z])', lambda m: m.group(1).upper(), self.name) + result = result[:1].upper() + result[1:].replace('_', '') + # If it's a function, let it end with "Request" to identify them + if self.is_function: + result += 'Request' + return result + def sorted_args(self): """Returns the arguments properly sorted and ready to plug-in into a Python's method header (i.e., flags and those which @@ -197,8 +208,8 @@ class TLArg: else: self.flag_indicator = False self.is_generic = arg_type.startswith('!') - self.type = arg_type.lstrip( - '!') # Strip the exclamation mark always to have only the name + # Strip the exclamation mark always to have only the name + self.type = arg_type.lstrip('!') # The type may be a flag (flags.IDX?REAL_TYPE) # Note that 'flags' is NOT the flags name; this is determined by a previous argument @@ -233,6 +244,22 @@ class TLArg: self.generic_definition = generic_definition + def type_hint(self): + result = { + 'int': 'int', + 'long': 'int', + 'string': 'str', + 'date': 'datetime', + 'bytes': 'bytes', + 'true': 'bool', + }.get(self.type, 'TLObject') + if self.is_vector: + result = 'list[{}]'.format(result) + if self.is_flag: + result += ' | None' + + return result + def __str__(self): # Find the real type representation by updating it as required real_type = self.type diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index ebc865f3..705710d4 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -107,8 +107,7 @@ class TLGenerator: if tlobject.namespace: builder.write('.' + tlobject.namespace) - builder.writeln('.{},'.format( - TLGenerator.get_class_name(tlobject))) + builder.writeln('.{},'.format(tlobject.class_name())) builder.current_indent -= 1 builder.writeln('}') @@ -176,8 +175,7 @@ class TLGenerator: builder.writeln() builder.writeln() - builder.writeln('class {}(TLObject):'.format( - TLGenerator.get_class_name(tlobject))) + builder.writeln('class {}(TLObject):'.format(tlobject.class_name())) # Class-level variable to store its Telegram's constructor ID builder.writeln('constructor_id = {}'.format(hex(tlobject.id))) @@ -221,17 +219,10 @@ class TLGenerator: builder.writeln('"""') for arg in args: if not arg.flag_indicator: - builder.write( - ':param {}: Telegram type: "{}".' - .format(arg.name, arg.type) - ) - if arg.is_vector: - builder.write(' Must be a list.'.format(arg.name)) - - if arg.is_generic: - builder.write(' Must be another TLObject request.') - - builder.writeln() + builder.writeln(':param {} {}:'.format( + arg.type_hint(), arg.name + )) + builder.current_indent -= 1 # It will auto-indent (':') # We also want to know what type this request returns # or to which type this constructor belongs to @@ -246,12 +237,11 @@ class TLGenerator: builder.writeln('This type has no constructors.') elif len(constructors) == 1: builder.writeln('Instance of {}.'.format( - TLGenerator.get_class_name(constructors[0]) + constructors[0].class_name() )) else: builder.writeln('Instance of either {}.'.format( - ', '.join(TLGenerator.get_class_name(c) - for c in constructors) + ', '.join(c.class_name() for c in constructors) )) builder.writeln('"""') @@ -319,7 +309,8 @@ class TLGenerator: builder.writeln('def on_send(self, writer):') builder.writeln( 'writer.write_int({}.constructor_id, signed=False)' - .format(TLGenerator.get_class_name(tlobject))) + .format(tlobject.class_name()) + ) for arg in tlobject.args: TLGenerator.write_onsend_code(builder, arg, @@ -331,8 +322,8 @@ class TLGenerator: builder.writeln('@staticmethod') builder.writeln('def empty():') builder.writeln('return {}({})'.format( - TLGenerator.get_class_name(tlobject), ', '.join( - 'None' for _ in range(len(args))))) + tlobject.class_name(), ', '.join('None' for _ in range(len(args))) + )) builder.end_block() # Write the on_response(self, reader) function @@ -414,20 +405,6 @@ class TLGenerator: 'self.{0} = {1}({0})'.format(arg.name, get_input_code) ) - @staticmethod - def get_class_name(tlobject): - """Gets the class name following the Python style guidelines""" - - # Courtesy of http://stackoverflow.com/a/31531797/4759433 - result = re.sub(r'_([a-z])', lambda m: m.group(1).upper(), - tlobject.name) - result = result[:1].upper() + result[1:].replace( - '_', '') # Replace again to fully ensure! - # If it's a function, let it end with "Request" to identify them - if tlobject.is_function: - result += 'Request' - return result - @staticmethod def get_file_name(tlobject, add_extension=False): """Gets the file name in file_name_format.py for the given TLObject""" From 060b8a8b9c03a8c0b7f3acd72789b4a6ab6f6c8a Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 19 Sep 2017 10:16:41 +0200 Subject: [PATCH 004/121] Fix setup.py import subprocess.run (not supported on < py3.5) --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 679d068b..7a48911e 100755 --- a/setup.py +++ b/setup.py @@ -12,8 +12,6 @@ Extra supported commands are: """ # To use a consistent encoding -from subprocess import run -from shutil import rmtree from codecs import open from sys import argv from os import path @@ -48,6 +46,11 @@ if __name__ == '__main__': print('Done.') elif len(argv) >= 2 and argv[1] == 'pypi': + # Need python3.5 or higher, but Telethon is supposed to support 3.x + # Place it here since noone should be running ./setup.py pypi anyway + from subprocess import run + from shutil import rmtree + for x in ('build', 'dist', 'Telethon.egg-info'): rmtree(x, ignore_errors=True) run('python3 setup.py sdist', shell=True) From 1d3273a3065744f9ee6162f305c633b2e35416cc Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 19 Sep 2017 13:17:40 +0200 Subject: [PATCH 005/121] Fix UpdateState calling handlers with updates with lower pts --- telethon/update_state.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/telethon/update_state.py b/telethon/update_state.py index 2f313dea..8c160184 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -75,12 +75,16 @@ class UpdateState: with self._updates_lock: if isinstance(update, tl.updates.State): self._state = update - elif not hasattr(update, 'pts') or update.pts > self._state.pts: - self._state.pts = getattr(update, 'pts', self._state.pts) + return # Nothing else to be done - if self._polling: - self._updates.append(update) - self._updates_available.set() + pts = getattr(update, 'pts', self._state.pts) + if pts <= self._state.pts: + return # We already handled this update + + self._state.pts = pts + if self._polling: + self._updates.append(update) + self._updates_available.set() for handler in self.handlers: handler(update) From 81d4d76d16cf24fad65d9a514d4df6ee6b5ba2a4 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 19 Sep 2017 13:20:34 +0200 Subject: [PATCH 006/121] Fix generated code showing the incorrent type hint for datetime --- telethon_generator/parser/tl_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon_generator/parser/tl_object.py b/telethon_generator/parser/tl_object.py index a90428b4..ceb83f83 100644 --- a/telethon_generator/parser/tl_object.py +++ b/telethon_generator/parser/tl_object.py @@ -249,7 +249,7 @@ class TLArg: 'int': 'int', 'long': 'int', 'string': 'str', - 'date': 'datetime', + 'date': 'datetime.datetime', 'bytes': 'bytes', 'true': 'bool', }.get(self.type, 'TLObject') From 9ae4f7f6415c877e825c992416dfc423f55d5c3b Mon Sep 17 00:00:00 2001 From: jiang gau Date: Tue, 19 Sep 2017 22:27:10 +0800 Subject: [PATCH 007/121] Fix wrong check on .send_message (closes #250) --- telethon/telegram_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 5d3f5ae2..05b4af7e 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -427,7 +427,7 @@ class TelegramClient(TelegramBareClient): reply_to_msg_id=self._get_reply_to(reply_to) ) result = self(request) - if isinstance(request, UpdateShortSentMessage): + if isinstance(result, UpdateShortSentMessage): return Message( id=result.id, to_id=entity, From b8d7b1c8af15231a27f816a8118780cfb7fc7cf4 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 20 Sep 2017 12:47:19 +0200 Subject: [PATCH 008/121] Phone number shouldn't actually start with '+' --- telethon/telegram_client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 05b4af7e..1c4c152c 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -282,8 +282,12 @@ class TelegramClient(TelegramBareClient): def send_code_request(self, phone): """Sends a code request to the specified phone number""" - result = self( - SendCodeRequest(phone, self.api_id, self.api_hash)) + if isinstance(phone, int): + phone = str(phone) + elif phone.startswith('+'): + phone = phone.strip('+') + + result = self(SendCodeRequest(phone, self.api_id, self.api_hash)) self._phone = phone self._phone_code_hash = result.phone_code_hash return result From c22224f516ea851782adecd537699a8ac4f85118 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 20 Sep 2017 13:22:56 +0200 Subject: [PATCH 009/121] Attempt at handling ProxyConnectionError on .connect() --- telethon/telegram_client.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 1c4c152c..f6c9cf9d 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -4,6 +4,10 @@ from datetime import datetime, timedelta from functools import lru_cache from mimetypes import guess_type from threading import Thread +try: + import socks +except ImportError: + socks = None from . import TelegramBareClient from . import helpers as utils @@ -147,7 +151,19 @@ class TelegramClient(TelegramBareClient): if self._sender and self._sender.is_connected(): return - ok = super().connect(exported_auth=exported_auth) + if socks and self._recv_thread: + # Treat proxy errors specially since they're not related to + # Telegram itself, but rather to the proxy. If any happens on + # the read thread forward it to the main thread. + try: + ok = super().connect(exported_auth=exported_auth) + except socks.ProxyConnectionError as e: + ok = False + # Report the exception to the main thread + self.updates.set_error(e) + else: + ok = super().connect(exported_auth=exported_auth) + # The main TelegramClient is the only one that will have # constant_read, since it's also the only one who receives # updates and need to be processed as soon as they occur. @@ -193,6 +209,10 @@ class TelegramClient(TelegramBareClient): # region Working with different connections + def _on_read_thread(self): + return self._recv_thread is not None and \ + threading.get_ident() == self._recv_thread.ident + def create_new_connection(self, on_dc=None, timeout=timedelta(seconds=5)): """Creates a new connection which can be used in parallel with the original TelegramClient. A TelegramBareClient @@ -226,8 +246,7 @@ class TelegramClient(TelegramBareClient): *args will be ignored. """ - if self._recv_thread is not None and \ - threading.get_ident() == self._recv_thread.ident: + if self._on_read_thread(): raise AssertionError('Cannot invoke requests from the ReadThread') self.updates.check_error() From 446174c7dea9fe976fac1c944aeaa4853cd814f0 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 21 Sep 2017 14:35:06 +0300 Subject: [PATCH 010/121] Catching WinError 10038 While client.connect() there were OSError: [WinError 10038] an operation was attempted on something that is not a socket --- telethon/extensions/tcp_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index 8453af5e..2ccdb0f0 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -100,7 +100,7 @@ class TcpClient: except socket.timeout as e: raise TimeoutError() from e except OSError as e: - if e.errno == errno.EBADF: + if e.errno == errno.EBADF or e.errno == errno.ENOTSOCK: self._raise_connection_reset() else: raise From 4777b8dad43c210400915493c7dd2422a137113c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 21 Sep 2017 12:37:05 +0200 Subject: [PATCH 011/121] Handle .connect() method more gracefully --- telethon/extensions/tcp_client.py | 30 ++++++++++++++++++++++-------- telethon/network/connection.py | 12 ++++++++++-- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index 2ccdb0f0..edf6cb4b 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -30,19 +30,33 @@ class TcpClient: else: # tuple, list, etc. self._socket.set_proxy(*self._proxy) + self._socket.settimeout(self._timeout) + def connect(self, ip, port): """Connects to the specified IP and port number. 'timeout' must be given in seconds """ - if not self.connected: - if ':' in ip: # IPv6 - mode, address = socket.AF_INET6, (ip, port, 0, 0) - else: - mode, address = socket.AF_INET, (ip, port) + if ':' in ip: # IPv6 + mode, address = socket.AF_INET6, (ip, port, 0, 0) + else: + mode, address = socket.AF_INET, (ip, port) - self._recreate_socket(mode) - self._socket.settimeout(self._timeout) - self._socket.connect(address) + while True: + try: + if not self._socket: + self._recreate_socket(mode) + + self._socket.connect(address) + break # Successful connection, stop retrying to connect + except OSError as e: + # There are some errors that we know how to handle, and + # the loop will allow us to retry + if e.errno == errno.EBADF: + # Bad file descriptor, i.e. socket was closed, set it + # to none to recreate it on the next iteration + self._socket = None + else: + raise def _get_connected(self): return self._socket is not None diff --git a/telethon/network/connection.py b/telethon/network/connection.py index 1426ce78..c79703cf 100644 --- a/telethon/network/connection.py +++ b/telethon/network/connection.py @@ -3,6 +3,8 @@ from datetime import timedelta from zlib import crc32 from enum import Enum +import errno + from ..crypto import AESModeCTR from ..extensions import BinaryWriter, TcpClient from ..errors import InvalidChecksumError @@ -75,9 +77,15 @@ class Connection: setattr(self, 'read', self._read_plain) def connect(self): - self._send_counter = 0 - self.conn.connect(self.ip, self.port) + try: + self.conn.connect(self.ip, self.port) + except OSError as e: + if e.errno == errno.EISCONN: + return # Already connected, no need to re-set everything up + else: + raise + self._send_counter = 0 if self._mode == ConnectionMode.TCP_ABRIDGED: self.conn.write(b'\xef') elif self._mode == ConnectionMode.TCP_INTERMEDIATE: From 2b2da843a1835ef896874f2e6fc67f1f66b4d5ed Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 21 Sep 2017 13:43:33 +0200 Subject: [PATCH 012/121] Create a Connection only once and avoid no-op if was "connected" --- telethon/extensions/tcp_client.py | 11 ++++++--- telethon/network/authenticator.py | 3 ++- telethon/network/connection.py | 3 +++ telethon/network/mtproto_sender.py | 16 ++++++++---- telethon/telegram_bare_client.py | 39 +++++++++--------------------- telethon/telegram_client.py | 10 ++------ 6 files changed, 38 insertions(+), 44 deletions(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index edf6cb4b..4f8e9d35 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -13,9 +13,9 @@ class TcpClient: self._closing_lock = Lock() if isinstance(timeout, timedelta): - self._timeout = timeout.seconds + self.timeout = timeout.seconds elif isinstance(timeout, int) or isinstance(timeout, float): - self._timeout = float(timeout) + self.timeout = float(timeout) else: raise ValueError('Invalid timeout type', type(timeout)) @@ -30,7 +30,7 @@ class TcpClient: else: # tuple, list, etc. self._socket.set_proxy(*self._proxy) - self._socket.settimeout(self._timeout) + self._socket.settimeout(self.timeout) def connect(self, ip, port): """Connects to the specified IP and port number. @@ -81,6 +81,8 @@ class TcpClient: def write(self, data): """Writes (sends) the specified bytes to the connected peer""" + if self._socket is None: + raise ConnectionResetError() # TODO Timeout may be an issue when sending the data, Changed in v3.5: # The socket timeout is now the maximum total duration to send all data. @@ -105,6 +107,9 @@ class TcpClient: and it's waiting for more, the timeout will NOT cancel the operation. Set to None for no timeout """ + if self._socket is None: + raise ConnectionResetError() + # TODO Remove the timeout from this method, always use previous one with BufferedWriter(BytesIO(), buffer_size=size) as buffer: bytes_left = size diff --git a/telethon/network/authenticator.py b/telethon/network/authenticator.py index 7d600fb3..bea8a759 100644 --- a/telethon/network/authenticator.py +++ b/telethon/network/authenticator.py @@ -2,6 +2,8 @@ import os import time from hashlib import sha1 +import errno + from .. import helpers as utils from ..crypto import AES, AuthKey, Factorization from ..crypto import rsa @@ -30,7 +32,6 @@ def _do_authentication(connection): time offset. """ sender = MtProtoPlainSender(connection) - sender.connect() # Step 1 sending: PQ Request nonce = os.urandom(16) diff --git a/telethon/network/connection.py b/telethon/network/connection.py index c79703cf..dbd76f68 100644 --- a/telethon/network/connection.py +++ b/telethon/network/connection.py @@ -93,6 +93,9 @@ class Connection: elif self._mode == ConnectionMode.TCP_OBFUSCATED: self._setup_obfuscation() + def get_timeout(self): + return self.conn.timeout + def _setup_obfuscation(self): # Obfuscated messages secrets cannot start with any of these keywords = (b'PVrG', b'GET ', b'POST', b'\xee' * 4) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 5729c578..ee7ecac1 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -17,12 +17,12 @@ class MtProtoSender: (https://core.telegram.org/mtproto/description) """ - def __init__(self, connection, session): + def __init__(self, session, connection): """Creates a new MtProtoSender configured to send messages through 'connection' and using the parameters from 'session'. """ - self.connection = connection self.session = session + self.connection = connection self._logger = logging.getLogger(__name__) self._need_confirmation = [] # Message IDs that need confirmation @@ -47,6 +47,9 @@ class MtProtoSender: def disconnect(self): """Disconnects from the server""" self.connection.close() + self._need_confirmation.clear() + self._clear_all_pending() + self.logging_out = False # region Send and receive @@ -97,9 +100,7 @@ class MtProtoSender: # "This packet should be skipped"; since this may have # been a result for a request, invalidate every request # and just re-invoke them to avoid problems - for r in self._pending_receive: - r.confirm_received.set() - self._pending_receive.clear() + self._clear_all_pending() return message, remote_msg_id, remote_seq = self._decode_msg(body) @@ -245,6 +246,11 @@ class MtProtoSender: if self._pending_receive[i].request_msg_id == request_msg_id: return self._pending_receive.pop(i) + def _clear_all_pending(self): + for r in self._pending_receive: + r.confirm_received.set() + self._pending_receive.clear() + def _handle_pong(self, msg_id, sequence, reader): self._logger.debug('Handling pong') reader.read_int(signed=False) # code diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 3a8b27dc..e0e4673b 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -72,11 +72,13 @@ class TelegramBareClient: self.api_id = int(api_id) self.api_hash = api_hash if self.api_id < 20: # official apps must use obfuscated - self._connection_mode = ConnectionMode.TCP_OBFUSCATED - else: - self._connection_mode = connection_mode - self.proxy = proxy - self._timeout = timeout + connection_mode = ConnectionMode.TCP_OBFUSCATED + + self._sender = MtProtoSender(self.session, Connection( + self.session.server_address, self.session.port, + mode=connection_mode, proxy=proxy, timeout=timeout + )) + self._logger = logging.getLogger(__name__) # Cache "exported" senders 'dc_id: TelegramBareClient' and @@ -88,9 +90,6 @@ class TelegramBareClient: # One may change self.updates.enabled at any later point. self.updates = UpdateState(process_updates) - # These will be set later - self._sender = None - # endregion # region Connecting @@ -104,21 +103,14 @@ class TelegramBareClient: If 'exported_auth' is not None, it will be used instead to determine the authorization key for the current session. """ - if self.is_connected(): - return True - - connection = Connection( - self.session.server_address, self.session.port, - mode=self._connection_mode, proxy=self.proxy, timeout=self._timeout - ) - try: + self._sender.connect() if not self.session.auth_key: # New key, we need to tell the server we're going to use # the latest layer try: self.session.auth_key, self.session.time_offset = \ - authenticator.do_authentication(connection) + authenticator.do_authentication(self._sender.connection) except BrokenAuthKeyError: return False @@ -128,8 +120,6 @@ class TelegramBareClient: else: init_connection = self.session.layer != LAYER - self._sender = MtProtoSender(connection, self.session) - self._sender.connect() if init_connection: if exported_auth is not None: @@ -166,7 +156,7 @@ class TelegramBareClient: return False def is_connected(self): - return self._sender is not None and self._sender.is_connected() + return self._sender.is_connected() def _init_connection(self, query=None): result = self(InvokeWithLayerRequest(LAYER, InitConnectionRequest( @@ -185,9 +175,7 @@ class TelegramBareClient: def disconnect(self): """Disconnects from the Telegram server""" - if self._sender: - self._sender.disconnect() - self._sender = None + self._sender.disconnect() def reconnect(self, new_dc=None): """Disconnects and connects again (effectively reconnecting). @@ -274,7 +262,7 @@ class TelegramBareClient: session.port = dc.port client = TelegramBareClient( session, self.api_id, self.api_hash, - timeout=self._timeout + timeout=self._connection.get_timeout() ) client.connect(exported_auth=export_auth) @@ -300,9 +288,6 @@ class TelegramBareClient: if not isinstance(request, TLObject) and not request.content_related: raise ValueError('You can only invoke requests, not types!') - if not self._sender: - raise ValueError('You must be connected to invoke requests!') - if retries <= 0: raise ValueError('Number of retries reached 0.') diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index f6c9cf9d..b291bc0c 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -148,9 +148,6 @@ class TelegramClient(TelegramBareClient): exported_auth is meant for internal purposes and can be ignored. """ - if self._sender and self._sender.is_connected(): - return - if socks and self._recv_thread: # Treat proxy errors specially since they're not related to # Telegram itself, but rather to the proxy. If any happens on @@ -173,7 +170,7 @@ class TelegramClient(TelegramBareClient): # read constantly or not for updates needs to be known before hand, # and further updates won't be able to be added unless allowing to # switch the mode on the fly. - if ok: + if ok and self._recv_thread is None: self._recv_thread = Thread( name='ReadThread', daemon=True, target=self._recv_thread_impl @@ -187,9 +184,6 @@ class TelegramClient(TelegramBareClient): def disconnect(self): """Disconnects from the Telegram server and stops all the spawned threads""" - if not self._sender or not self._sender.is_connected(): - return - # The existing thread will close eventually, since it's # only running while the MtProtoSender.is_connected() self._recv_thread = None @@ -1035,7 +1029,7 @@ class TelegramClient(TelegramBareClient): # # This way, sending and receiving will be completely independent. def _recv_thread_impl(self): - while self._sender and self._sender.is_connected(): + while self._sender.is_connected(): try: if datetime.now() > self._last_ping + self._ping_delay: self._sender.send(PingRequest( From 7f83a6109f60c15fc43394e2ec6725fb3788e279 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 21 Sep 2017 13:54:44 +0200 Subject: [PATCH 013/121] Fix authenticator was disconnecting when it shouldn't --- telethon/network/authenticator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/telethon/network/authenticator.py b/telethon/network/authenticator.py index bea8a759..f6881a4b 100644 --- a/telethon/network/authenticator.py +++ b/telethon/network/authenticator.py @@ -196,9 +196,6 @@ def _do_authentication(connection): # Step 3 response: Complete DH Exchange with BinaryReader(sender.receive()) as reader: - # Everything read from the server, disconnect now - sender.disconnect() - code = reader.read_int(signed=False) if code == 0x3bcbf734: # DH Gen OK nonce_from_server = reader.read(16) From 4ba12e717f16412dc7aa0bb11b4117f4e7660d6d Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 21 Sep 2017 13:58:57 +0200 Subject: [PATCH 014/121] Fix reconnect to new_dc wasn't changing connection.ip --- telethon/telegram_bare_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index e0e4673b..2f2f0df9 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -120,7 +120,6 @@ class TelegramBareClient: else: init_connection = self.session.layer != LAYER - if init_connection: if exported_auth is not None: self._init_connection(ImportAuthorizationRequest( @@ -188,8 +187,9 @@ class TelegramBareClient: if new_dc is not None: self.session.auth_key = None # Force creating new auth_key dc = self._get_dc(new_dc) - self.session.server_address = dc.ip_address - self.session.port = dc.port + ip = dc.ip_address + self._sender.connection.ip = self.session.server_address = ip + self._sender.connection.port = self.session.port = dc.port self.session.save() self.connect() From 12c6d4d3ac34cef7e39768350c82b2299cc6c194 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 21 Sep 2017 15:36:20 +0200 Subject: [PATCH 015/121] Start the background thread only if the user is authorized --- telethon/telegram_client.py | 68 +++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index b291bc0c..7baac88e 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -126,6 +126,9 @@ class TelegramClient(TelegramBareClient): self._phone_code_hash = None self._phone = None + # Save whether the user is authorized here (a.k.a. logged in) + self._authorized = False + # Uploaded files cache so subsequent calls are instant self._upload_cache = {} @@ -161,25 +164,16 @@ class TelegramClient(TelegramBareClient): else: ok = super().connect(exported_auth=exported_auth) - # The main TelegramClient is the only one that will have - # constant_read, since it's also the only one who receives - # updates and need to be processed as soon as they occur. - # - # TODO Allow to disable this to avoid the creation of a new thread - # if the user is not going to work with updates at all? Whether to - # read constantly or not for updates needs to be known before hand, - # and further updates won't be able to be added unless allowing to - # switch the mode on the fly. - if ok and self._recv_thread is None: - self._recv_thread = Thread( - name='ReadThread', daemon=True, - target=self._recv_thread_impl - ) - self._recv_thread.start() - if self.updates.polling: - self.sync_updates() + if not ok: + return False - return ok + try: + self.sync_updates() + self._set_connected_and_authorized() + except UnauthorizedError: + self._authorized = False + + return True def disconnect(self): """Disconnects from the Telegram server @@ -291,7 +285,7 @@ class TelegramClient(TelegramBareClient): def is_user_authorized(self): """Has the user been authorized yet (code request sent and confirmed)?""" - return self.session and self.get_me() is not None + return self._authorized def send_code_request(self, phone): """Sends a code request to the specified phone number""" @@ -337,33 +331,37 @@ class TelegramClient(TelegramBareClient): except (PhoneCodeEmptyError, PhoneCodeExpiredError, PhoneCodeHashEmptyError, PhoneCodeInvalidError): return None - elif password: salt = self(GetPasswordRequest()).current_salt - result = self( - CheckPasswordRequest(utils.get_password_hash(password, salt))) - + result = self(CheckPasswordRequest( + utils.get_password_hash(password, salt) + )) elif bot_token: result = self(ImportBotAuthorizationRequest( flags=0, bot_auth_token=bot_token, - api_id=self.api_id, api_hash=self.api_hash)) - + api_id=self.api_id, api_hash=self.api_hash + )) else: raise ValueError( 'You must provide a phone and a code the first time, ' - 'and a password only if an RPCError was raised before.') + 'and a password only if an RPCError was raised before.' + ) + self._set_connected_and_authorized() return result.user def sign_up(self, code, first_name, last_name=''): """Signs up to Telegram. Make sure you sent a code request first!""" - return self(SignUpRequest( + result = self(SignUpRequest( phone_number=self._phone, phone_code_hash=self._phone_code_hash, phone_code=code, first_name=first_name, last_name=last_name - )).user + )) + + self._set_connected_and_authorized() + return result.user def log_out(self): """Logs out and deletes the current session. @@ -997,11 +995,7 @@ class TelegramClient(TelegramBareClient): called automatically on connection if self.updates.enabled = True, otherwise it should be called manually after enabling updates. """ - try: - self.updates.process(self(GetStateRequest())) - return True - except UnauthorizedError: - return False + self.updates.process(self(GetStateRequest())) def add_update_handler(self, handler): """Adds an update handler (a function which takes a TLObject, @@ -1021,6 +1015,14 @@ class TelegramClient(TelegramBareClient): # Constant read + def _set_connected_and_authorized(self): + self._authorized = True + self._recv_thread = Thread( + name='ReadThread', daemon=True, + target=self._recv_thread_impl + ) + self._recv_thread.start() + # By using this approach, another thread will be # created and started upon connection to constantly read # from the other end. Otherwise, manual calls to .receive() From ffadcd029f5505e68e441a37b5079994a5095c10 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 21 Sep 2017 19:12:46 +0200 Subject: [PATCH 016/121] Save the session much less often (doable because 151e162) --- telethon/network/mtproto_sender.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index ee7ecac1..d8dabc68 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -66,9 +66,6 @@ class MtProtoSender: self._send_packet(writer.get_bytes(), request) self._pending_receive.append(request) - # And update the saved session - self.session.save() - def _send_acknowledges(self): """Sends a messages acknowledge for all those who _need_confirmation""" if self._need_confirmation: @@ -312,7 +309,6 @@ class MtProtoSender: # sent msg_id too low or too high (respectively). # Use the current msg_id to determine the right time offset. self.session.update_time_offset(correct_msg_id=msg_id) - self.session.save() self._logger.debug('Read Bad Message error: ' + str(error)) self._logger.debug('Attempting to use the correct time offset.') return True From bc15b451b5db541b5132449aafc40cb47a2854bb Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 12:20:38 +0200 Subject: [PATCH 017/121] Use a safer reconnect behaviour (respect multithread too) --- telethon/telegram_bare_client.py | 28 ++++++++++++++------ telethon/telegram_client.py | 44 ++++++++++++++++++++------------ 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 2f2f0df9..a403c36d 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -3,6 +3,7 @@ from datetime import timedelta from hashlib import md5 from io import BytesIO from os import path +from threading import RLock from . import helpers as utils from .crypto import rsa, CdnDecrypter @@ -81,6 +82,10 @@ class TelegramBareClient: self._logger = logging.getLogger(__name__) + # Two threads may be calling reconnect() when the connection is lost, + # we only want one to actually perform the reconnection. + self._connect_lock = RLock() + # Cache "exported" senders 'dc_id: TelegramBareClient' and # their corresponding sessions not to recreate them all # the time since it's a (somewhat expensive) process. @@ -177,22 +182,29 @@ class TelegramBareClient: self._sender.disconnect() def reconnect(self, new_dc=None): - """Disconnects and connects again (effectively reconnecting). + """If 'new_dc' is not set, only a call to .connect() will be made + since it's assumed that the connection has been lost and the + library is reconnecting. - If 'new_dc' is not None, the current authorization key is - removed, the DC used is switched, and a new connection is made. + If 'new_dc' is set, the client is first disconnected from the + current data center, clears the auth key for the old DC, and + connects to the new data center. """ - self.disconnect() - - if new_dc is not None: + if new_dc is None: + # Assume we are disconnected due to some error, so connect again + with self._connect_lock: + # Another thread may have connected again, so check that first + if not self.is_connected(): + self.connect() + else: + self.disconnect() self.session.auth_key = None # Force creating new auth_key dc = self._get_dc(new_dc) ip = dc.ip_address self._sender.connection.ip = self.session.server_address = ip self._sender.connection.port = self.session.port = dc.port self.session.save() - - self.connect() + self.connect() # endregion diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 7baac88e..61dbff26 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -126,6 +126,15 @@ class TelegramClient(TelegramBareClient): self._phone_code_hash = None self._phone = None + # Despite the state of the real connection, keep track of whether + # the user has explicitly called .connect() or .disconnect() here. + # This information is required by the read thread, who will be the + # one attempting to reconnect on the background *while* the user + # doesn't explicitly call .disconnect(), thus telling it to stop + # retrying. The main thread, knowing there is a background thread + # attempting reconnection as soon as it happens, will just sleep. + self._user_connected = False + # Save whether the user is authorized here (a.k.a. logged in) self._authorized = False @@ -167,6 +176,7 @@ class TelegramClient(TelegramBareClient): if not ok: return False + self._user_connected = True try: self.sync_updates() self._set_connected_and_authorized() @@ -178,8 +188,7 @@ class TelegramClient(TelegramBareClient): def disconnect(self): """Disconnects from the Telegram server and stops all the spawned threads""" - # The existing thread will close eventually, since it's - # only running while the MtProtoSender.is_connected() + self._user_connected = False self._recv_thread = None # This will trigger a "ConnectionResetError", usually, the background @@ -255,9 +264,20 @@ class TelegramClient(TelegramBareClient): 'attempting to reconnect at DC {}' .format(e.new_dc)) + # TODO What happens with the background thread here? + # For normal use cases, this won't happen, because this will only + # be on the very first connection (not authorized, not running), + # but may be an issue for people who actually travel? self.reconnect(new_dc=e.new_dc) return self.invoke(request) + except ConnectionResetError: + if self._connect_lock.locked(): + # We are connecting and we don't want to reconnect there... + raise + while self._user_connected and not self.reconnect(): + pass # Retry forever until we finally can send the request + # Let people use client(SomeRequest()) instead client.invoke(...) __call__ = invoke @@ -1031,7 +1051,7 @@ class TelegramClient(TelegramBareClient): # # This way, sending and receiving will be completely independent. def _recv_thread_impl(self): - while self._sender.is_connected(): + while self._user_connected: try: if datetime.now() > self._last_ping + self._ping_delay: self._sender.send(PingRequest( @@ -1040,24 +1060,14 @@ class TelegramClient(TelegramBareClient): self._last_ping = datetime.now() self._sender.receive(update_state=self.updates) - except AttributeError: - # 'NoneType' object has no attribute 'receive'. - # The only moment when this can happen is reconnection - # was triggered from another thread and the ._sender - # was set to None, so close this thread and exit by return. - self._recv_thread = None - return except TimeoutError: # No problem. pass except ConnectionResetError: - if self._recv_thread is not None: - # Do NOT attempt reconnecting unless the connection was - # finished by the user -> ._recv_thread is None - self._logger.debug('Server disconnected us. Reconnecting...') - self._recv_thread = None # Not running anymore - self.reconnect() - return + self._logger.debug('Server disconnected us. Reconnecting...') + while self._user_connected and not self.reconnect(): + pass # Retry forever, this is instant messaging + except Exception as e: # Unknown exception, pass it to the main thread self.updates.set_error(e) From 4245ec5abc75195f2e034d5df45d5d4b5cf4c14b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 12:31:41 +0200 Subject: [PATCH 018/121] Make reconnect private --- telethon/telegram_bare_client.py | 4 ++-- telethon/telegram_client.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index a403c36d..e25cc52d 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -181,7 +181,7 @@ class TelegramBareClient: """Disconnects from the Telegram server""" self._sender.disconnect() - def reconnect(self, new_dc=None): + def _reconnect(self, new_dc=None): """If 'new_dc' is not set, only a call to .connect() will be made since it's assumed that the connection has been lost and the library is reconnecting. @@ -325,7 +325,7 @@ class TelegramBareClient: except ConnectionResetError: self._logger.debug('Server disconnected us. Reconnecting and ' 'resending request...') - self.reconnect() + self._reconnect() except FloodWaitError: self.disconnect() diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 61dbff26..91224417 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -268,14 +268,14 @@ class TelegramClient(TelegramBareClient): # For normal use cases, this won't happen, because this will only # be on the very first connection (not authorized, not running), # but may be an issue for people who actually travel? - self.reconnect(new_dc=e.new_dc) + self._reconnect(new_dc=e.new_dc) return self.invoke(request) except ConnectionResetError: if self._connect_lock.locked(): # We are connecting and we don't want to reconnect there... raise - while self._user_connected and not self.reconnect(): + while self._user_connected and not self._reconnect(): pass # Retry forever until we finally can send the request # Let people use client(SomeRequest()) instead client.invoke(...) @@ -1065,7 +1065,7 @@ class TelegramClient(TelegramBareClient): pass except ConnectionResetError: self._logger.debug('Server disconnected us. Reconnecting...') - while self._user_connected and not self.reconnect(): + while self._user_connected and not self._reconnect(): pass # Retry forever, this is instant messaging except Exception as e: From 4d5f16f2aa6ee1ffe0c3d26055e1851081239c67 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 12:44:09 +0200 Subject: [PATCH 019/121] Fix background thread could not reconnect properly --- telethon/telegram_bare_client.py | 4 ++-- telethon/telegram_client.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index e25cc52d..3ee96835 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -3,7 +3,7 @@ from datetime import timedelta from hashlib import md5 from io import BytesIO from os import path -from threading import RLock +from threading import Lock from . import helpers as utils from .crypto import rsa, CdnDecrypter @@ -84,7 +84,7 @@ class TelegramBareClient: # Two threads may be calling reconnect() when the connection is lost, # we only want one to actually perform the reconnection. - self._connect_lock = RLock() + self._connect_lock = Lock() # Cache "exported" senders 'dc_id: TelegramBareClient' and # their corresponding sessions not to recreate them all diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 91224417..637309d3 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -243,7 +243,10 @@ class TelegramClient(TelegramBareClient): *args will be ignored. """ - if self._on_read_thread(): + # This is only valid when the read thread is reconnecting, + # that is, the connection lock is locked. + on_read_thread = self._on_read_thread() + if on_read_thread and not self._connect_lock.locked(): raise AssertionError('Cannot invoke requests from the ReadThread') self.updates.check_error() @@ -254,8 +257,9 @@ class TelegramClient(TelegramBareClient): # will be the one which should be reading (but is invoking the # request) thus not being available to read it "in the background" # and it's needed to call receive. + call_receive = on_read_thread or self._recv_thread is None return super().invoke( - request, call_receive=self._recv_thread is None, + request, call_receive=call_receive, retries=kwargs.get('retries', 5) ) @@ -271,7 +275,7 @@ class TelegramClient(TelegramBareClient): self._reconnect(new_dc=e.new_dc) return self.invoke(request) - except ConnectionResetError: + except ConnectionResetError as e: if self._connect_lock.locked(): # We are connecting and we don't want to reconnect there... raise From d8bf8bb2ebd2cd5f8d3e6aecc472f1216cad2987 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 12:45:14 +0200 Subject: [PATCH 020/121] Fix reconnect always returning False -> infinite loop --- telethon/telegram_bare_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 3ee96835..401d3982 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -195,7 +195,9 @@ class TelegramBareClient: with self._connect_lock: # Another thread may have connected again, so check that first if not self.is_connected(): - self.connect() + return self.connect() + else: + return True else: self.disconnect() self.session.auth_key = None # Force creating new auth_key @@ -204,7 +206,7 @@ class TelegramBareClient: self._sender.connection.ip = self.session.server_address = ip self._sender.connection.port = self.session.port = dc.port self.session.save() - self.connect() + return self.connect() # endregion From 6d60e83adc28561bb467e397fb3121acc5b3425c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 13:13:41 +0200 Subject: [PATCH 021/121] Start background thread only if it was None --- telethon/telegram_client.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 637309d3..0deb9030 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1041,11 +1041,12 @@ class TelegramClient(TelegramBareClient): def _set_connected_and_authorized(self): self._authorized = True - self._recv_thread = Thread( - name='ReadThread', daemon=True, - target=self._recv_thread_impl - ) - self._recv_thread.start() + if self._recv_thread is None: + self._recv_thread = Thread( + name='ReadThread', daemon=True, + target=self._recv_thread_impl + ) + self._recv_thread.start() # By using this approach, another thread will be # created and started upon connection to constantly read @@ -1075,7 +1076,8 @@ class TelegramClient(TelegramBareClient): except Exception as e: # Unknown exception, pass it to the main thread self.updates.set_error(e) - self._recv_thread = None - return + break + + self._recv_thread = None # endregion From a992f427815112536eec352bcba6104b83a0ffb4 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 13:14:56 +0200 Subject: [PATCH 022/121] Attempt at fixing socket is None on .connect() --- telethon/extensions/tcp_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index 4f8e9d35..ef11f1f0 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -43,7 +43,7 @@ class TcpClient: while True: try: - if not self._socket: + while not self._socket: self._recreate_socket(mode) self._socket.connect(address) From 4a8e5c865a79f09a59129dd1eb38d5d387db9822 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 13:32:00 +0200 Subject: [PATCH 023/121] Detect BrokenAuthKeyError on MtProtoSender._decode_msg --- telethon/network/mtproto_sender.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index d8dabc68..33876cdf 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -4,7 +4,10 @@ from threading import RLock from .. import helpers as utils from ..crypto import AES -from ..errors import BadMessageError, InvalidChecksumError, rpc_message_to_error +from ..errors import ( + BadMessageError, InvalidChecksumError, BrokenAuthKeyError, + rpc_message_to_error +) from ..extensions import BinaryReader, BinaryWriter from ..tl.all_tlobjects import tlobjects from ..tl.types import MsgsAck @@ -147,7 +150,10 @@ class MtProtoSender: with BinaryReader(body) as reader: if len(body) < 8: - raise BufferError("Can't decode packet ({})".format(body)) + if body == b'l\xfe\xff\xff': + raise BrokenAuthKeyError() + else: + raise BufferError("Can't decode packet ({})".format(body)) # TODO Check for both auth key ID and msg_key correctness reader.read_long() # remote_auth_key_id From 9dfb5d493cfe2c3c330aad5a696f5599d995fd16 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 13:51:11 +0200 Subject: [PATCH 024/121] Fix BrokenPipeError was instance of OSError --- telethon/extensions/tcp_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index ef11f1f0..3879bcd3 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -90,13 +90,13 @@ class TcpClient: self._socket.sendall(data) except socket.timeout as e: raise TimeoutError() from e + except BrokenPipeError: + self._raise_connection_reset() except OSError as e: if e.errno == errno.EBADF: self._raise_connection_reset() else: raise - except BrokenPipeError: - self._raise_connection_reset() def read(self, size): """Reads (receives) a whole block of 'size bytes From bfa3001f871566976dc427f9b34805eab63f99c1 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 16:02:10 +0200 Subject: [PATCH 025/121] Fix MainThread would lock when reconnecting This is because it was thinking that the ReadThread would be ready to read the result, but actually, this thread is also locked trying to reconnect at the same time --- telethon/telegram_client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 0deb9030..2f848434 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -245,19 +245,19 @@ class TelegramClient(TelegramBareClient): """ # This is only valid when the read thread is reconnecting, # that is, the connection lock is locked. - on_read_thread = self._on_read_thread() - if on_read_thread and not self._connect_lock.locked(): + if self._on_read_thread() and not self._connect_lock.locked(): raise AssertionError('Cannot invoke requests from the ReadThread') self.updates.check_error() try: - # Users may call this method from within some update handler. - # If this is the case, then the thread invoking the request - # will be the one which should be reading (but is invoking the - # request) thus not being available to read it "in the background" - # and it's needed to call receive. - call_receive = on_read_thread or self._recv_thread is None + # We should call receive from this thread if there's no background + # thread reading or if the server disconnected us and we're trying + # to reconnect. This is because the read thread may either be + # locked also trying to reconnect or we may be said thread already. + call_receive = \ + self._recv_thread is None or self._connect_lock.locked() + return super().invoke( request, call_receive=call_receive, retries=kwargs.get('retries', 5) From b0f6b23e526209ee4832cbee3d113653266741ce Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 16:11:07 +0200 Subject: [PATCH 026/121] Fix creating new connections accessing invalid attributes --- telethon/extensions/tcp_client.py | 10 +++++----- telethon/telegram_bare_client.py | 3 ++- telethon/telegram_client.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index 3879bcd3..6feb9841 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -8,7 +8,7 @@ from threading import Lock class TcpClient: def __init__(self, proxy=None, timeout=timedelta(seconds=5)): - self._proxy = proxy + self.proxy = proxy self._socket = None self._closing_lock = Lock() @@ -20,15 +20,15 @@ class TcpClient: raise ValueError('Invalid timeout type', type(timeout)) def _recreate_socket(self, mode): - if self._proxy is None: + if self.proxy is None: self._socket = socket.socket(mode, socket.SOCK_STREAM) else: import socks self._socket = socks.socksocket(mode, socket.SOCK_STREAM) - if type(self._proxy) is dict: - self._socket.set_proxy(**self._proxy) + if type(self.proxy) is dict: + self._socket.set_proxy(**self.proxy) else: # tuple, list, etc. - self._socket.set_proxy(*self._proxy) + self._socket.set_proxy(*self.proxy) self._socket.settimeout(self.timeout) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 401d3982..4fb1e60f 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -276,7 +276,8 @@ class TelegramBareClient: session.port = dc.port client = TelegramBareClient( session, self.api_id, self.api_hash, - timeout=self._connection.get_timeout() + proxy=self._sender.connection.conn.proxy, + timeout=self._sender.connection.get_timeout() ) client.connect(exported_auth=export_auth) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 2f848434..18f305ac 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -225,7 +225,7 @@ class TelegramClient(TelegramBareClient): if on_dc is None: client = TelegramBareClient( self.session, self.api_id, self.api_hash, - proxy=self.proxy, timeout=timeout + proxy=self._sender.connection.conn.proxy, timeout=timeout ) client.connect() else: From 9ce43073044adb1ba3e16c171d29077f3250c87c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 16:12:43 +0200 Subject: [PATCH 027/121] Update to v0.13.5 --- telethon/telegram_bare_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 4fb1e60f..b2c5f156 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -53,7 +53,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.13.4' + __version__ = '0.13.5' # TODO Make this thread-safe, all connections share the same DC _dc_options = None From f39f9c33a0e8b29ed1fc999627e8dd572d7b6338 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 22 Sep 2017 19:20:37 +0200 Subject: [PATCH 028/121] Fix timeout on confirm_received.wait not being added (fix #257) --- telethon/telegram_bare_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index b2c5f156..7b9b0c15 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -317,7 +317,9 @@ class TelegramBareClient: # switching between constant read or not on the fly. # Must also watch out for calling .read() from two places, # in which case a Lock would be required for .receive(). - request.confirm_received.wait() # TODO Socket's timeout here? + request.confirm_received.wait( + self._sender.connection.get_timeout() + ) else: while not request.confirm_received.is_set(): self._sender.receive(update_state=self.updates) From 73fbfde7effb47690275623adc520a89281ec5eb Mon Sep 17 00:00:00 2001 From: Andrey Egorov Date: Sat, 23 Sep 2017 00:12:36 +0300 Subject: [PATCH 029/121] Process messages without pts --- telethon/update_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/update_state.py b/telethon/update_state.py index 8c160184..e92dac1f 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -78,7 +78,7 @@ class UpdateState: return # Nothing else to be done pts = getattr(update, 'pts', self._state.pts) - if pts <= self._state.pts: + if hasattr(update, 'pts') and pts <= self._state.pts: return # We already handled this update self._state.pts = pts From 0b48b1ec8a13d020e2058bce522e3107a8848f59 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 23 Sep 2017 10:41:36 +0200 Subject: [PATCH 030/121] Add support for non-recursive TLObject.to_dict() --- telethon_generator/tl_generator.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 705710d4..30313411 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -265,7 +265,7 @@ class TLGenerator: # Write the to_dict(self) method if args: - builder.writeln('def to_dict(self):') + builder.writeln('def to_dict(self, recursive=True):') builder.writeln('return {') builder.current_indent += 1 @@ -285,13 +285,15 @@ class TLGenerator: else: if arg.is_vector: builder.write( - '[] if self.{0} is None else [None ' + '([] if self.{0} is None else [None ' 'if x is None else x.to_dict() for x in self.{0}]' + ') if recursive else self.{0}' .format(arg.name) ) else: builder.write( - 'None if self.{0} is None else self.{0}.to_dict()' + '(None if self.{0} is None else ' + 'self.{0}.to_dict()) if recursive else self.{0}' .format(arg.name) ) builder.writeln(',') @@ -299,8 +301,7 @@ class TLGenerator: builder.current_indent -= 1 builder.writeln("}") else: - builder.writeln('@staticmethod') - builder.writeln('def to_dict():') + builder.writeln('def to_dict(self, recursive=True):') builder.writeln('return {}') builder.end_block() From 5701029fbf2a575c7b13e273293f1cf4ddcdc123 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 23 Sep 2017 10:59:08 +0200 Subject: [PATCH 031/121] Stop treating .to_dict() with no args as a special case --- telethon_generator/tl_generator.py | 67 ++++++++++++++---------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 30313411..53f95741 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -264,45 +264,40 @@ class TLGenerator: builder.end_block() # Write the to_dict(self) method + builder.writeln('def to_dict(self, recursive=True):') if args: - builder.writeln('def to_dict(self, recursive=True):') builder.writeln('return {') - builder.current_indent += 1 - - base_types = ('string', 'bytes', 'int', 'long', 'int128', - 'int256', 'double', 'Bool', 'true', 'date') - - for arg in args: - builder.write("'{}': ".format(arg.name)) - if arg.type in base_types: - if arg.is_vector: - builder.write( - '[] if self.{0} is None else self.{0}[:]' - .format(arg.name) - ) - else: - builder.write('self.{}'.format(arg.name)) - else: - if arg.is_vector: - builder.write( - '([] if self.{0} is None else [None ' - 'if x is None else x.to_dict() for x in self.{0}]' - ') if recursive else self.{0}' - .format(arg.name) - ) - else: - builder.write( - '(None if self.{0} is None else ' - 'self.{0}.to_dict()) if recursive else self.{0}' - .format(arg.name) - ) - builder.writeln(',') - - builder.current_indent -= 1 - builder.writeln("}") else: - builder.writeln('def to_dict(self, recursive=True):') - builder.writeln('return {}') + builder.write('return {') + builder.current_indent += 1 + + base_types = ('string', 'bytes', 'int', 'long', 'int128', + 'int256', 'double', 'Bool', 'true', 'date') + + for arg in args: + builder.write("'{}': ".format(arg.name)) + if arg.type in base_types: + if arg.is_vector: + builder.write('[] if self.{0} is None else self.{0}[:]' + .format(arg.name)) + else: + builder.write('self.{}'.format(arg.name)) + else: + if arg.is_vector: + builder.write( + '([] if self.{0} is None else [None' + ' if x is None else x.to_dict() for x in self.{0}]' + ') if recursive else self.{0}'.format(arg.name) + ) + else: + builder.write( + '(None if self.{0} is None else self.{0}.to_dict())' + ' if recursive else self.{0}'.format(arg.name) + ) + builder.writeln(',') + + builder.current_indent -= 1 + builder.writeln("}") builder.end_block() From 80e987725645a4ee5b92a739b474d16d9abda551 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 23 Sep 2017 11:01:25 +0200 Subject: [PATCH 032/121] Show the type of children TLObjects on .stringify() --- telethon/tl/tlobject.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/telethon/tl/tlobject.py b/telethon/tl/tlobject.py index 5f587b37..d701fa1e 100644 --- a/telethon/tl/tlobject.py +++ b/telethon/tl/tlobject.py @@ -20,10 +20,13 @@ class TLObject: """ if indent is None: if isinstance(obj, TLObject): - return '{{{}: {}}}'.format( - type(obj).__name__, - TLObject.pretty_format(obj.to_dict()) - ) + children = obj.to_dict(recursive=False) + if children: + return '{}: {}'.format( + type(obj).__name__, TLObject.pretty_format(children) + ) + else: + return type(obj).__name__ if isinstance(obj, dict): return '{{{}}}'.format(', '.join( '{}: {}'.format( @@ -41,12 +44,13 @@ class TLObject: else: result = [] if isinstance(obj, TLObject): - result.append('{') result.append(type(obj).__name__) - result.append(': ') - result.append(TLObject.pretty_format( - obj.to_dict(), indent - )) + children = obj.to_dict(recursive=False) + if children: + result.append(': ') + result.append(TLObject.pretty_format( + obj.to_dict(recursive=False), indent + )) elif isinstance(obj, dict): result.append('{\n') @@ -81,7 +85,7 @@ class TLObject: return ''.join(result) # These should be overrode - def to_dict(self): + def to_dict(self, recursive=True): return {} def on_send(self, writer): From b04607e7ba933291537516188803618571e6d4ef Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 23 Sep 2017 11:31:58 +0200 Subject: [PATCH 033/121] Ignore requests invoked from ReadThread instead raising Any unhandled exception on the ReadThread would cause it to stop, and handling the exception to the main thread on the next invoke or poll. Instead causing the thread to stop, simply ignore it. --- telethon/telegram_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 18f305ac..40ff5401 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -246,7 +246,7 @@ class TelegramClient(TelegramBareClient): # This is only valid when the read thread is reconnecting, # that is, the connection lock is locked. if self._on_read_thread() and not self._connect_lock.locked(): - raise AssertionError('Cannot invoke requests from the ReadThread') + return # Just ignore, we would be raising and crashing the thread self.updates.check_error() From 0f10e1419f1466b2e36e2dae352a3de0c9051836 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 23 Sep 2017 11:44:04 +0200 Subject: [PATCH 034/121] Update to v0.13.6 --- telethon/telegram_bare_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 7b9b0c15..883d92e6 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -53,7 +53,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.13.5' + __version__ = '0.13.6' # TODO Make this thread-safe, all connections share the same DC _dc_options = None From 1593a4415e91475ff2638efefc219d35c7b4d66c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 23 Sep 2017 20:07:40 +0200 Subject: [PATCH 035/121] Fix None datetime being type hinted as invalid --- telethon_generator/parser/tl_object.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telethon_generator/parser/tl_object.py b/telethon_generator/parser/tl_object.py index ceb83f83..31c6c4ad 100644 --- a/telethon_generator/parser/tl_object.py +++ b/telethon_generator/parser/tl_object.py @@ -249,13 +249,13 @@ class TLArg: 'int': 'int', 'long': 'int', 'string': 'str', - 'date': 'datetime.datetime', + 'date': 'datetime.datetime | None', # None date = 0 timestamp 'bytes': 'bytes', 'true': 'bool', }.get(self.type, 'TLObject') if self.is_vector: result = 'list[{}]'.format(result) - if self.is_flag: + if self.is_flag and self.type != 'date': result += ' | None' return result From d0e66c104a69e92c0bb8f03ded806de04a1e2a3e Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 23 Sep 2017 21:00:01 +0200 Subject: [PATCH 036/121] Fix generating documentation failing on root directory --- docs/generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/generate.py b/docs/generate.py index 80408a47..e66d6f6d 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -360,7 +360,8 @@ def generate_documentation(scheme_file): for tltype, constructors in tltypes.items(): filename = get_path_for_type(tltype) out_dir = os.path.dirname(filename) - os.makedirs(out_dir, exist_ok=True) + if out_dir: + os.makedirs(out_dir, exist_ok=True) # Since we don't have access to the full TLObject, split the type if '.' in tltype: From 56302552fbf67aed6794d547a26cd0bc558f7f31 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 23 Sep 2017 21:01:06 +0200 Subject: [PATCH 037/121] Fix formatting URLs on the documentation failing for types --- docs/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generate.py b/docs/generate.py index e66d6f6d..7b27831c 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -512,7 +512,7 @@ def generate_documentation(scheme_file): constructor_names = fmt(constructors, get_class_name) request_urls = fmt(methods, get_create_path_for) - type_urls = fmt(types, get_create_path_for) + type_urls = fmt(types, get_path_for_type) constructor_urls = fmt(constructors, get_create_path_for) replace_dict = { From 40d80f73ee35bdcc54fa300b883adb56875f593c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 23 Sep 2017 21:01:49 +0200 Subject: [PATCH 038/121] Enhance docs search to show namespaces if names clash --- docs/generate.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/generate.py b/docs/generate.py index 7b27831c..be807586 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -504,13 +504,24 @@ def generate_documentation(scheme_file): methods = sorted(methods, key=lambda m: m.name) constructors = sorted(constructors, key=lambda c: c.name) + def fmt(xs): + ys = {x: get_class_name(x) for x in xs} # cache TLObject: display + zs = {} # create a dict to hold those which have duplicated keys + for y in ys.values(): + zs[y] = y in zs + return ', '.join( + '"{}.{}"'.format(x.namespace, ys[x]) + if zs[ys[x]] and getattr(x, 'namespace', None) + else '"{}"'.format(ys[x]) for x in xs + ) + + request_names = fmt(methods) + type_names = fmt(types) + constructor_names = fmt(constructors) + def fmt(xs, formatter): return ', '.join('"{}"'.format(formatter(x)) for x in xs) - request_names = fmt(methods, get_class_name) - type_names = fmt(types, get_class_name) - constructor_names = fmt(constructors, get_class_name) - request_urls = fmt(methods, get_create_path_for) type_urls = fmt(types, get_path_for_type) constructor_urls = fmt(constructors, get_create_path_for) From d5ba259d4e3694626e0552fc2bf5cd7ce4a3fea6 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 23 Sep 2017 21:02:32 +0200 Subject: [PATCH 039/121] Ensure the working directory is unaltered after generating docs --- docs/generate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/generate.py b/docs/generate.py index be807586..94fff181 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -563,5 +563,8 @@ def copy_resources(): if __name__ == '__main__': os.makedirs('generated', exist_ok=True) os.chdir('generated') - generate_documentation('../../telethon_generator/scheme.tl') - copy_resources() + try: + generate_documentation('../../telethon_generator/scheme.tl') + copy_resources() + finally: + os.chdir(os.pardir) From 9787fb8c46eb2e9599f72737b5cfd0f98121f318 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 24 Sep 2017 13:40:54 +0200 Subject: [PATCH 040/121] Add and copy 404.html to the generated docs --- docs/generate.py | 15 +++++++++------ docs/res/404.html | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 docs/res/404.html diff --git a/docs/generate.py b/docs/generate.py index 94fff181..928eacf1 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -218,6 +218,7 @@ def generate_documentation(scheme_file): original_paths = { 'css': 'css/docs.css', 'arrow': 'img/arrow.svg', + '404': '404.html', 'index_all': 'index.html', 'index_types': 'types/index.html', 'index_methods': 'methods/index.html', @@ -540,13 +541,15 @@ def generate_documentation(scheme_file): 'constructor_urls': constructor_urls } - with open('../res/core.html') as infile: - with open(original_paths['index_all'], 'w') as outfile: - text = infile.read() - for key, value in replace_dict.items(): - text = text.replace('{' + key + '}', str(value)) + shutil.copy('../res/404.html', original_paths['404']) - outfile.write(text) + with open('../res/core.html') as infile,\ + open(original_paths['index_all'], 'w') as outfile: + text = infile.read() + for key, value in replace_dict.items(): + text = text.replace('{' + key + '}', str(value)) + + outfile.write(text) # Everything done print('Documentation generated.') diff --git a/docs/res/404.html b/docs/res/404.html new file mode 100644 index 00000000..8eb3d37d --- /dev/null +++ b/docs/res/404.html @@ -0,0 +1,44 @@ + + + Oopsie! | Telethon + + + + + + + +
+

You seem a bit lost…

+

You seem to be lost! Don't worry, that's just Telegram's API being + itself. Shall we go back to the Main Page?

+
+ + From c0ee1231565153da83e2b7c58335231046b061fd Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 25 Sep 2017 11:04:09 +0200 Subject: [PATCH 041/121] Add a small sleep when trying to reconnect forever --- telethon/telegram_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 40ff5401..bcc199f5 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -4,6 +4,8 @@ from datetime import datetime, timedelta from functools import lru_cache from mimetypes import guess_type from threading import Thread +from time import sleep + try: import socks except ImportError: @@ -280,7 +282,7 @@ class TelegramClient(TelegramBareClient): # We are connecting and we don't want to reconnect there... raise while self._user_connected and not self._reconnect(): - pass # Retry forever until we finally can send the request + sleep(0.1) # Retry forever until we can send the request # Let people use client(SomeRequest()) instead client.invoke(...) __call__ = invoke @@ -1071,7 +1073,7 @@ class TelegramClient(TelegramBareClient): except ConnectionResetError: self._logger.debug('Server disconnected us. Reconnecting...') while self._user_connected and not self._reconnect(): - pass # Retry forever, this is instant messaging + sleep(0.1) # Retry forever, this is instant messaging except Exception as e: # Unknown exception, pass it to the main thread From 01c91bb89582bcd7b36c556ba80c5a1220f9a653 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 25 Sep 2017 11:56:44 +0200 Subject: [PATCH 042/121] Report errors in the background not to interfer with users (#262) --- telethon/errors/__init__.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/telethon/errors/__init__.py b/telethon/errors/__init__.py index f2579f57..0d02545b 100644 --- a/telethon/errors/__init__.py +++ b/telethon/errors/__init__.py @@ -1,5 +1,6 @@ import urllib.request import re +from threading import Thread from .common import ( ReadCancelledError, InvalidParameterError, TypeNotFoundError, @@ -18,21 +19,28 @@ from .rpc_errors_401 import * from .rpc_errors_420 import * +def report_error(code, message, report_method): + try: + # Ensure it's signed + report_method = int.from_bytes( + report_method.to_bytes(4, 'big'), 'big', signed=True + ) + url = urllib.request.urlopen( + 'https://rpc.pwrtelegram.xyz?code={}&error={}&method={}' + .format(code, message, report_method) + ) + url.read() + url.close() + except: + "We really don't want to crash when just reporting an error" + + def rpc_message_to_error(code, message, report_method=None): if report_method is not None: - try: - # Ensure it's signed - report_method = int.from_bytes( - report_method.to_bytes(4, 'big'), 'big', signed=True - ) - url = urllib.request.urlopen( - 'https://rpc.pwrtelegram.xyz?code={}&error={}&method={}' - .format(code, message, report_method) - ) - url.read() - url.close() - except: - "We really don't want to crash when just reporting an error" + Thread( + target=report_method, + args=(code, message, report_method) + ).start() errors = { 303: rpc_errors_303_all, From 493f5cec1f443b738538f3e7f741ad13fef8848c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 25 Sep 2017 11:57:42 +0200 Subject: [PATCH 043/121] Fix-up target method typo --- telethon/errors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/errors/__init__.py b/telethon/errors/__init__.py index 0d02545b..a78ad5eb 100644 --- a/telethon/errors/__init__.py +++ b/telethon/errors/__init__.py @@ -38,7 +38,7 @@ def report_error(code, message, report_method): def rpc_message_to_error(code, message, report_method=None): if report_method is not None: Thread( - target=report_method, + target=report_error, args=(code, message, report_method) ).start() From edcd23f94c4c834490645e3844f137899c75e1a8 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 25 Sep 2017 12:00:09 +0200 Subject: [PATCH 044/121] Making timeout when reporting an error shorter --- telethon/errors/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telethon/errors/__init__.py b/telethon/errors/__init__.py index a78ad5eb..d65d426c 100644 --- a/telethon/errors/__init__.py +++ b/telethon/errors/__init__.py @@ -27,7 +27,8 @@ def report_error(code, message, report_method): ) url = urllib.request.urlopen( 'https://rpc.pwrtelegram.xyz?code={}&error={}&method={}' - .format(code, message, report_method) + .format(code, message, report_method), + timeout=5 ) url.read() url.close() From 1d50bba8bcefef9934a79ce981535f3308558ab2 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 25 Sep 2017 13:43:03 +0200 Subject: [PATCH 045/121] Add get_input_* methods for Media and such --- telethon/utils.py | 168 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 2 deletions(-) diff --git a/telethon/utils.py b/telethon/utils.py index aa82c472..cd1927a1 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -10,8 +10,17 @@ from .tl.types import ( ChatPhoto, InputPeerChannel, InputPeerChat, InputPeerUser, InputPeerEmpty, MessageMediaDocument, MessageMediaPhoto, PeerChannel, InputChannel, UserEmpty, InputUser, InputUserEmpty, InputUserSelf, InputPeerSelf, - PeerChat, PeerUser, User, UserFull, UserProfilePhoto, Document -) + PeerChat, PeerUser, User, UserFull, UserProfilePhoto, Document, + MessageMediaContact, MessageMediaEmpty, MessageMediaGame, MessageMediaGeo, + MessageMediaUnsupported, MessageMediaVenue, InputMediaContact, + InputMediaDocument, InputMediaEmpty, InputMediaGame, + InputMediaGeoPoint, InputMediaPhoto, InputMediaVenue, InputDocument, + DocumentEmpty, InputDocumentEmpty, Message, GeoPoint, InputGeoPoint, + GeoPointEmpty, InputGeoPointEmpty, Photo, InputPhoto, PhotoEmpty, + InputPhotoEmpty, FileLocation, ChatPhotoEmpty, UserProfilePhotoEmpty, + FileLocationUnavailable, InputMediaUploadedDocument, + InputMediaUploadedPhoto, + DocumentAttributeFilename) def get_display_name(entity): @@ -156,6 +165,161 @@ def get_input_user(entity): _raise_cast_fail(entity, 'InputUser') +def get_input_document(document): + """Similar to get_input_peer, but for documents""" + if document is None: + return None + + if not isinstance(document, TLObject): + _raise_cast_fail(document, 'InputDocument') + + if type(document).subclass_of_id == 0xf33fdb68: # crc32(b'InputDocument') + return document + + if isinstance(document, Document): + return InputDocument(id=document.id, access_hash=document.access_hash) + + if isinstance(document, DocumentEmpty): + return InputDocumentEmpty() + + if isinstance(document, MessageMediaDocument): + return get_input_document(document.document) + + if isinstance(document, Message): + return get_input_document(document.media) + + _raise_cast_fail(document, 'InputDocument') + + +def get_input_photo(photo): + """Similar to get_input_peer, but for documents""" + if photo is None: + return None + + if not isinstance(photo, TLObject): + _raise_cast_fail(photo, 'InputPhoto') + + if type(photo).subclass_of_id == 0x846363e0: # crc32(b'InputPhoto') + return photo + + if isinstance(photo, Photo): + return InputPhoto(id=photo.id, access_hash=photo.access_hash) + + if isinstance(photo, PhotoEmpty): + return InputPhotoEmpty() + + _raise_cast_fail(photo, 'InputPhoto') + + +def get_input_geo(geo): + """Similar to get_input_peer, but for geo points""" + if geo is None: + return None + + if not isinstance(geo, TLObject): + _raise_cast_fail(geo, 'InputGeoPoint') + + if type(geo).subclass_of_id == 0x430d225: # crc32(b'InputGeoPoint') + return geo + + if isinstance(geo, GeoPoint): + return InputGeoPoint(lat=geo.lat, long=geo.long) + + if isinstance(geo, GeoPointEmpty): + return InputGeoPointEmpty() + + if isinstance(geo, MessageMediaGeo): + return get_input_geo(geo.geo) + + if isinstance(geo, Message): + return get_input_geo(geo.media) + + _raise_cast_fail(geo, 'InputGeoPoint') + + +def get_input_media(media, user_caption=None, is_photo=False): + """Similar to get_input_peer, but for media. + + If the media is a file location and is_photo is known to be True, + it will be treated as an InputMediaUploadedPhoto. + """ + if media is None: + return None + + if not isinstance(media, TLObject): + _raise_cast_fail(media, 'InputMedia') + + if type(media).subclass_of_id == 0xfaf846f4: # crc32(b'InputMedia') + return media + + if isinstance(media, MessageMediaPhoto): + return InputMediaPhoto( + id=get_input_photo(media.photo), + caption=media.caption if user_caption is None else user_caption, + ttl_seconds=media.ttl_seconds + ) + + if isinstance(media, MessageMediaDocument): + return InputMediaDocument( + id=get_input_document(media.document), + caption=media.caption if user_caption is None else user_caption, + ttl_seconds=media.ttl_seconds + ) + + if isinstance(media, FileLocation): + if is_photo: + return InputMediaUploadedPhoto( + file=media, + caption=user_caption or '' + ) + else: + return InputMediaUploadedDocument( + file=media, + mime_type='application/octet-stream', # unknown, assume bytes + attributes=[DocumentAttributeFilename('unnamed')], + caption=user_caption or '' + ) + + if isinstance(media, MessageMediaGame): + return InputMediaGame(id=media.game.id) + + if isinstance(media, ChatPhoto) or isinstance(media, UserProfilePhoto): + if isinstance(media.photo_big, FileLocationUnavailable): + return get_input_media(media.photo_small, is_photo=True) + else: + return get_input_media(media.photo_big, is_photo=True) + + if isinstance(media, MessageMediaContact): + return InputMediaContact( + phone_number=media.phone_number, + first_name=media.first_name, + last_name=media.last_name + ) + + if isinstance(media, MessageMediaGeo): + return InputMediaGeoPoint(geo_point=get_input_geo(media.geo)) + + if isinstance(media, MessageMediaVenue): + return InputMediaVenue( + geo_point=get_input_geo(media.geo), + title=media.title, + address=media.address, + provider=media.provider, + venue_id=media.venue_id + ) + + if any(isinstance(media, t) for t in ( + MessageMediaEmpty, MessageMediaUnsupported, + FileLocationUnavailable, ChatPhotoEmpty, + UserProfilePhotoEmpty)): + return InputMediaEmpty() + + if isinstance(media, Message): + return get_input_media(media.media) + + _raise_cast_fail(media, 'InputMedia') + + def find_user_or_chat(peer, users, chats): """Finds the corresponding user or chat given a peer. Returns None if it was not found""" From dc73744fc4c976f9e4f6e5acf90b9280ef460b7b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 25 Sep 2017 13:47:25 +0200 Subject: [PATCH 046/121] Add get_input_media calls on the generated code --- telethon_generator/tl_generator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 53f95741..6455d3a4 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -168,6 +168,8 @@ class TLGenerator: util_imports.add('get_input_channel') elif a.type == 'InputUser': util_imports.add('get_input_user') + elif a.type == 'InputMedia': + util_imports.add('get_input_media') if util_imports: builder.writeln('from {}.utils import {}'.format( @@ -381,6 +383,8 @@ class TLGenerator: TLGenerator.write_get_input(builder, arg, 'get_input_channel') elif arg.type == 'InputUser' and tlobject.is_function: TLGenerator.write_get_input(builder, arg, 'get_input_user') + elif arg.type == 'InputMedia' and tlobject.is_function: + TLGenerator.write_get_input(builder, arg, 'get_input_media') else: builder.writeln('self.{0} = {0}'.format(arg.name)) From b40708a8c76dad773f8a171bc5f4617714173d72 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 25 Sep 2017 14:05:13 +0200 Subject: [PATCH 047/121] Import get_input_* only once on the generated code --- telethon_generator/tl_generator.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 6455d3a4..bb9796aa 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -136,6 +136,15 @@ class TLGenerator: x for x in namespace_tlobjects.keys() if x ))) + # Import 'get_input_*' utils + # TODO Support them on types too + if 'functions' in out_dir: + builder.writeln( + 'from {}.utils import get_input_peer, ' + 'get_input_channel, get_input_user, ' + 'get_input_media'.format('.' * depth) + ) + # Import 'os' for those needing access to 'os.urandom()' # Currently only 'random_id' needs 'os' to be imported, # for all those TLObjects with arg.can_be_inferred. @@ -157,24 +166,6 @@ class TLGenerator: the Type: [Constructors] must be given for proper importing and documentation strings. """ - if tlobject.is_function: - util_imports = set() - for a in tlobject.args: - # We can automatically convert some "full" types to - # "input only" (like User -> InputPeerUser, etc.) - if a.type == 'InputPeer': - util_imports.add('get_input_peer') - elif a.type == 'InputChannel': - util_imports.add('get_input_channel') - elif a.type == 'InputUser': - util_imports.add('get_input_user') - elif a.type == 'InputMedia': - util_imports.add('get_input_media') - - if util_imports: - builder.writeln('from {}.utils import {}'.format( - '.' * depth, ', '.join(util_imports))) - builder.writeln() builder.writeln() builder.writeln('class {}(TLObject):'.format(tlobject.class_name())) From f2331107329c9316bfc10b2334d87bd82a8219ae Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 25 Sep 2017 20:52:27 +0200 Subject: [PATCH 048/121] Allow sending several requests at once through new MessageContainer --- telethon/network/mtproto_sender.py | 22 +++++++++------- telethon/telegram_bare_client.py | 42 ++++++++++++++++++------------ telethon/telegram_client.py | 14 +++++----- telethon/tl/__init__.py | 1 + telethon/tl/message_container.py | 41 +++++++++++++++++++++++++++++ 5 files changed, 86 insertions(+), 34 deletions(-) create mode 100644 telethon/tl/message_container.py diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 33876cdf..1fe6067e 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -9,6 +9,7 @@ from ..errors import ( rpc_message_to_error ) from ..extensions import BinaryReader, BinaryWriter +from ..tl import MessageContainer from ..tl.all_tlobjects import tlobjects from ..tl.types import MsgsAck @@ -56,14 +57,20 @@ class MtProtoSender: # region Send and receive - def send(self, request): + def send(self, *requests): """Sends the specified MTProtoRequest, previously sending any message which needed confirmation.""" # If any message needs confirmation send an AckRequest first self._send_acknowledges() - # Finally send our packed request + # Finally send our packed request(s) + self._pending_receive.extend(requests) + if len(requests) == 1: + request = requests[0] + else: + request = MessageContainer(self.session, requests) + with BinaryWriter() as writer: request.on_send(writer) self._send_packet(writer.get_bytes(), request) @@ -268,22 +275,17 @@ class MtProtoSender: def _handle_container(self, msg_id, sequence, reader, state): self._logger.debug('Handling container') - reader.read_int(signed=False) # code - size = reader.read_int() - for _ in range(size): - inner_msg_id = reader.read_long() - reader.read_int() # inner_sequence - inner_length = reader.read_int() + for inner_msg_id, _, inner_len in MessageContainer.iter_read(reader): begin_position = reader.tell_position() # Note that this code is IMPORTANT for skipping RPC results of # lost requests (i.e., ones from the previous connection session) try: if not self._process_msg(inner_msg_id, sequence, reader, state): - reader.set_position(begin_position + inner_length) + reader.set_position(begin_position + inner_len) except: # If any error is raised, something went wrong; skip the packet - reader.set_position(begin_position + inner_length) + reader.set_position(begin_position + inner_len) raise return True diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 883d92e6..64bc8c5e 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -290,7 +290,7 @@ class TelegramBareClient: # region Invoking Telegram requests - def invoke(self, request, call_receive=True, retries=5): + def invoke(self, *requests, call_receive=True, retries=5): """Invokes (sends) a MTProtoRequest and returns (receives) its result. If 'updates' is not None, all read update object will be put @@ -300,7 +300,8 @@ class TelegramBareClient: thread calling to 'self._sender.receive()' running or this method will lock forever. """ - if not isinstance(request, TLObject) and not request.content_related: + if not all(isinstance(x, TLObject) and + x.content_related for x in requests): raise ValueError('You can only invoke requests, not types!') if retries <= 0: @@ -308,20 +309,22 @@ class TelegramBareClient: try: # Ensure that we start with no previous errors (i.e. resending) - request.confirm_received.clear() - request.rpc_error = None + for x in requests: + x.confirm_received.clear() + x.rpc_error = None - self._sender.send(request) + self._sender.send(*requests) if not call_receive: # TODO This will be slightly troublesome if we allow # switching between constant read or not on the fly. # Must also watch out for calling .read() from two places, # in which case a Lock would be required for .receive(). - request.confirm_received.wait( - self._sender.connection.get_timeout() - ) + for x in requests: + x.confirm_received.wait( + self._sender.connection.get_timeout() + ) else: - while not request.confirm_received.is_set(): + while not all(x.confirm_received.is_set() for x in requests): self._sender.receive(update_state=self.updates) except TimeoutError: @@ -336,14 +339,19 @@ class TelegramBareClient: self.disconnect() raise - if request.rpc_error: - raise request.rpc_error - if request.result is None: - return self.invoke( - request, call_receive=call_receive, retries=(retries - 1) - ) - else: - return request.result + try: + raise next(x.rpc_error for x in requests if x.rpc_error) + except StopIteration: + if any(x.result is None for x in requests): + # "A container may only be accepted or + # rejected by the other party as a whole." + return self.invoke( + *requests, call_receive=call_receive, retries=(retries - 1) + ) + elif len(requests) == 1: + return requests[0].result + else: + return [x.result for x in requests] # Let people use client(SomeRequest()) instead client.invoke(...) __call__ = invoke diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index bcc199f5..fe140652 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -239,11 +239,10 @@ class TelegramClient(TelegramBareClient): # region Telegram requests functions - def invoke(self, request, *args, **kwargs): - """Invokes (sends) a MTProtoRequest and returns (receives) its result. - An optional 'retries' parameter can be set. - - *args will be ignored. + def invoke(self, *requests, **kwargs): + """Invokes (sends) one or several MTProtoRequest and returns + (receives) their result. An optional named 'retries' parameter + can be used, indicating how many times it should retry. """ # This is only valid when the read thread is reconnecting, # that is, the connection lock is locked. @@ -261,7 +260,8 @@ class TelegramClient(TelegramBareClient): self._recv_thread is None or self._connect_lock.locked() return super().invoke( - request, call_receive=call_receive, + *requests, + call_receive=call_receive, retries=kwargs.get('retries', 5) ) @@ -275,7 +275,7 @@ class TelegramClient(TelegramBareClient): # be on the very first connection (not authorized, not running), # but may be an issue for people who actually travel? self._reconnect(new_dc=e.new_dc) - return self.invoke(request) + return self.invoke(*requests) except ConnectionResetError as e: if self._connect_lock.locked(): diff --git a/telethon/tl/__init__.py b/telethon/tl/__init__.py index 9ee6a979..e022408b 100644 --- a/telethon/tl/__init__.py +++ b/telethon/tl/__init__.py @@ -1,2 +1,3 @@ from .tlobject import TLObject from .session import Session +from .message_container import MessageContainer diff --git a/telethon/tl/message_container.py b/telethon/tl/message_container.py new file mode 100644 index 00000000..a5e4398a --- /dev/null +++ b/telethon/tl/message_container.py @@ -0,0 +1,41 @@ +from . import TLObject +from ..extensions import BinaryWriter + + +class MessageContainer(TLObject): + constructor_id = 0x8953ad37 + + # TODO Currently it's a bit of a hack, since the container actually holds + # messages (message id, sequence number, request body), not requests. + # Probably create a proper "Message" class + def __init__(self, session, requests): + super().__init__() + self.content_related = False + self.session = session + self.requests = requests + + def on_send(self, writer): + writer.write_int(0x73f1f8dc, signed=False) + writer.write_int(len(self.requests)) + for x in self.requests: + with BinaryWriter() as aux: + x.on_send(aux) + x.request_msg_id = self.session.get_new_msg_id() + + writer.write_long(x.request_msg_id) + writer.write_int( + self.session.generate_sequence(x.content_related) + ) + packet = aux.get_bytes() + writer.write_int(len(packet)) + writer.write(packet) + + @staticmethod + def iter_read(reader): + reader.read_int(signed=False) # code + size = reader.read_int() + for _ in range(size): + inner_msg_id = reader.read_long() + inner_sequence = reader.read_int() + inner_length = reader.read_int() + yield inner_msg_id, inner_sequence, inner_length From 2bb26d6389fccdf069a261a2671645b9d86f3472 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 26 Sep 2017 14:29:35 +0200 Subject: [PATCH 049/121] Modify setup.py to work even if generated code was wrong --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a48911e..02b37495 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,8 @@ from setuptools import find_packages, setup try: from telethon import TelegramClient -except ImportError: +except Exception as e: + print('Failed to import TelegramClient due to', e) TelegramClient = None From b83cd98ba02ee7b6e47c879cf8958044861e1b77 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 26 Sep 2017 14:36:02 +0200 Subject: [PATCH 050/121] Replace TLObject.on_send with the new .to_bytes() This also replaces some int.to_bytes() calls with a faster struct.pack (up to x4 faster). This approach is also around x3 faster than creating a BinaryWriter just to serialize a TLObject as bytes. --- telethon/extensions/binary_writer.py | 2 +- telethon/network/mtproto_sender.py | 11 +-- telethon/tl/message_container.py | 24 ++++--- telethon/tl/tlobject.py | 34 ++++++++- telethon_generator/tl_generator.py | 103 ++++++++++++++++----------- 5 files changed, 110 insertions(+), 64 deletions(-) diff --git a/telethon/extensions/binary_writer.py b/telethon/extensions/binary_writer.py index a1934a63..8147e1fd 100644 --- a/telethon/extensions/binary_writer.py +++ b/telethon/extensions/binary_writer.py @@ -110,7 +110,7 @@ class BinaryWriter: def tgwrite_object(self, tlobject): """Writes a Telegram object""" - tlobject.on_send(self) + self.write(tlobject.to_bytes()) def tgwrite_vector(self, vector): """Writes a vector of Telegram objects""" diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 1fe6067e..6a43538a 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -71,19 +71,14 @@ class MtProtoSender: else: request = MessageContainer(self.session, requests) - with BinaryWriter() as writer: - request.on_send(writer) - self._send_packet(writer.get_bytes(), request) - self._pending_receive.append(request) + self._send_packet(request.to_bytes(), request) + self._pending_receive.append(request) def _send_acknowledges(self): """Sends a messages acknowledge for all those who _need_confirmation""" if self._need_confirmation: msgs_ack = MsgsAck(self._need_confirmation) - with BinaryWriter() as writer: - msgs_ack.on_send(writer) - self._send_packet(writer.get_bytes(), msgs_ack) - + self._send_packet(msgs_ack.to_bytes(), msgs_ack) del self._need_confirmation[:] def receive(self, update_state): diff --git a/telethon/tl/message_container.py b/telethon/tl/message_container.py index a5e4398a..ac78b287 100644 --- a/telethon/tl/message_container.py +++ b/telethon/tl/message_container.py @@ -18,17 +18,21 @@ class MessageContainer(TLObject): writer.write_int(0x73f1f8dc, signed=False) writer.write_int(len(self.requests)) for x in self.requests: - with BinaryWriter() as aux: - x.on_send(aux) - x.request_msg_id = self.session.get_new_msg_id() + x.request_msg_id = self.session.get_new_msg_id() - writer.write_long(x.request_msg_id) - writer.write_int( - self.session.generate_sequence(x.content_related) - ) - packet = aux.get_bytes() - writer.write_int(len(packet)) - writer.write(packet) + writer.write_long(x.request_msg_id) + writer.write_int( + self.session.generate_sequence(x.content_related) + ) + packet = x.to_bytes() + writer.write_int(len(packet)) + writer.write(packet) + + def to_bytes(self): + # TODO Change this to delete the on_send from this class + with BinaryWriter() as writer: + self.on_send(writer) + return writer.get_bytes() @staticmethod def iter_read(reader): diff --git a/telethon/tl/tlobject.py b/telethon/tl/tlobject.py index d701fa1e..609ae143 100644 --- a/telethon/tl/tlobject.py +++ b/telethon/tl/tlobject.py @@ -84,12 +84,42 @@ class TLObject: return ''.join(result) + @staticmethod + def serialize_bytes(data): + """Write bytes by using Telegram guidelines""" + r = [] + if len(data) < 254: + padding = (len(data) + 1) % 4 + if padding != 0: + padding = 4 - padding + + r.append(bytes([len(data)])) + r.append(data) + + else: + padding = len(data) % 4 + if padding != 0: + padding = 4 - padding + + r.append(bytes([254])) + r.append(bytes([len(data) % 256])) + r.append(bytes([(len(data) >> 8) % 256])) + r.append(bytes([(len(data) >> 16) % 256])) + r.append(data) + + r.append(bytes(padding)) + return b''.join(r) + + @staticmethod + def serialize_string(string): + return TLObject.serialize_bytes(string.encode('utf-8')) + # These should be overrode def to_dict(self, recursive=True): return {} - def on_send(self, writer): - pass + def to_bytes(self): + return b'' def on_response(self, reader): pass diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index bb9796aa..49c8f5bc 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -1,6 +1,7 @@ import os import re import shutil +import struct from zlib import crc32 from collections import defaultdict @@ -150,6 +151,9 @@ class TLGenerator: # for all those TLObjects with arg.can_be_inferred. builder.writeln('import os') + # Import struct for the .to_bytes(self) serialization + builder.writeln('import struct') + # Generate the class for every TLObject for t in sorted(tlobjects, key=lambda x: x.name): TLGenerator._write_source_code( @@ -294,16 +298,18 @@ class TLGenerator: builder.end_block() - # Write the on_send(self, writer) function - builder.writeln('def on_send(self, writer):') - builder.writeln( - 'writer.write_int({}.constructor_id, signed=False)' - .format(tlobject.class_name()) - ) + # Write the .to_bytes() function + builder.writeln('def to_bytes(self):') + builder.write("return b''.join((") + + # First constructor code, we already know its bytes + builder.write('{},'.format(repr(struct.pack(' """ - if arg.generic_definition: return # Do nothing, this only specifies a later type @@ -434,73 +439,85 @@ class TLGenerator: if arg.is_flag: if arg.type == 'true': return # Exit, since True type is never written + elif arg.is_vector: + # Vector flags are special since they consist of 3 values, + # so we need an extra join here. Note that empty vector flags + # should NOT be sent either! + builder.write("b'' if not {} else b''.join((".format(name)) else: - builder.writeln('if {}:'.format(name)) + builder.write("b'' if not {} else (".format(name)) if arg.is_vector: if arg.use_vector_id: - builder.writeln('writer.write_int(0x1cb5c415, signed=False)') + # vector code, unsigned 0x1cb5c415 as little endian + builder.write(r"b'\x15\xc4\xb5\x1c',") + + builder.write("struct.pack(' Date: Tue, 26 Sep 2017 14:41:11 +0200 Subject: [PATCH 051/121] Replace 4 .append calls with a single one when serializing bytes --- telethon/tl/tlobject.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/telethon/tl/tlobject.py b/telethon/tl/tlobject.py index 609ae143..b67fdc62 100644 --- a/telethon/tl/tlobject.py +++ b/telethon/tl/tlobject.py @@ -101,10 +101,12 @@ class TLObject: if padding != 0: padding = 4 - padding - r.append(bytes([254])) - r.append(bytes([len(data) % 256])) - r.append(bytes([(len(data) >> 8) % 256])) - r.append(bytes([(len(data) >> 16) % 256])) + r.append(bytes([ + 254, + len(data) % 256, + (len(data) >> 8) % 256, + (len(data) >> 16) % 256 + ])) r.append(data) r.append(bytes(padding)) From 9b7733d6b919673e5545ef44066bfff309790cb5 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 27 Sep 2017 12:05:35 +0200 Subject: [PATCH 052/121] Avoid the use of starred expressions on autogen code (fix #266) --- telethon_generator/tl_generator.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 49c8f5bc..66e210c8 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -454,8 +454,9 @@ class TLGenerator: builder.write("struct.pack('3.5 feature, so add another join. + builder.write("b''.join(") # Temporary disable .is_vector, not to enter this if again # Also disable .is_flag since it's not needed per element @@ -465,7 +466,7 @@ class TLGenerator: arg.is_vector = True arg.is_flag = old_flag - builder.write(') for x in {}]'.format(name)) + builder.write(' for x in {})'.format(name)) elif arg.flag_indicator: # Calculate the flags with those items which are not None @@ -506,7 +507,10 @@ class TLGenerator: elif 'date' == arg.type: # Custom format # 0 if datetime is None else int(datetime.timestamp()) - builder.write(r"b'\0\0\0\0' if {0} is None else struct.pack(' Date: Wed, 27 Sep 2017 12:08:37 +0200 Subject: [PATCH 053/121] Fix CdnDecrypter incorrectly accessing clients' timeout (closes #265) --- telethon/crypto/cdn_decrypter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/crypto/cdn_decrypter.py b/telethon/crypto/cdn_decrypter.py index 000cfc30..772267f5 100644 --- a/telethon/crypto/cdn_decrypter.py +++ b/telethon/crypto/cdn_decrypter.py @@ -42,7 +42,7 @@ class CdnDecrypter: session.port = dc.port cdn_client = client_cls( # Avoid importing TelegramBareClient session, client.api_id, client.api_hash, - timeout=client._timeout + timeout=client._sender.connection.get_timeout() ) # This will make use of the new RSA keys for this specific CDN. # From 0c1196723202655fcce5ae93db9f93997abf1b94 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 27 Sep 2017 12:11:16 +0200 Subject: [PATCH 054/121] Remove unused .shaes attribute from CdnDecrypter --- telethon/crypto/cdn_decrypter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/telethon/crypto/cdn_decrypter.py b/telethon/crypto/cdn_decrypter.py index 772267f5..d6628d58 100644 --- a/telethon/crypto/cdn_decrypter.py +++ b/telethon/crypto/cdn_decrypter.py @@ -17,7 +17,6 @@ class CdnDecrypter: self.file_token = file_token self.cdn_aes = cdn_aes self.cdn_file_hashes = cdn_file_hashes - self.shaes = [sha256() for _ in range(len(cdn_file_hashes))] @staticmethod def prepare_decrypter(client, client_cls, cdn_redirect): From efca98131264920f5bd8806f3467b7853d7bc3b3 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 27 Sep 2017 13:26:35 +0200 Subject: [PATCH 055/121] Fix adding a request twice (or container) to ._pending_receive --- telethon/network/mtproto_sender.py | 1 - 1 file changed, 1 deletion(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 6a43538a..43c870f5 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -72,7 +72,6 @@ class MtProtoSender: request = MessageContainer(self.session, requests) self._send_packet(request.to_bytes(), request) - self._pending_receive.append(request) def _send_acknowledges(self): """Sends a messages acknowledge for all those who _need_confirmation""" From 7b736aa6ef1a8d39c1ad1509daaadaae682f17a5 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 27 Sep 2017 13:46:02 +0200 Subject: [PATCH 056/121] Fix MessageContainer having the wrong constructor id --- telethon/tl/message_container.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telethon/tl/message_container.py b/telethon/tl/message_container.py index ac78b287..ce7be98d 100644 --- a/telethon/tl/message_container.py +++ b/telethon/tl/message_container.py @@ -3,7 +3,7 @@ from ..extensions import BinaryWriter class MessageContainer(TLObject): - constructor_id = 0x8953ad37 + constructor_id = 0x73f1f8dc # TODO Currently it's a bit of a hack, since the container actually holds # messages (message id, sequence number, request body), not requests. @@ -15,7 +15,7 @@ class MessageContainer(TLObject): self.requests = requests def on_send(self, writer): - writer.write_int(0x73f1f8dc, signed=False) + writer.write_int(MessageContainer.constructor_id, signed=False) writer.write_int(len(self.requests)) for x in self.requests: x.request_msg_id = self.session.get_new_msg_id() From 6df9fc558ef8f1a9cf7ae8b4fe59b96b20832317 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 27 Sep 2017 13:46:53 +0200 Subject: [PATCH 057/121] Create and use a new GzipPacked class, also when sending --- telethon/network/mtproto_sender.py | 12 ++++----- telethon/tl/__init__.py | 1 + telethon/tl/gzip_packed.py | 40 ++++++++++++++++++++++++++++++ telethon/tl/message_container.py | 5 ++-- 4 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 telethon/tl/gzip_packed.py diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 43c870f5..a63e936e 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -9,7 +9,7 @@ from ..errors import ( rpc_message_to_error ) from ..extensions import BinaryReader, BinaryWriter -from ..tl import MessageContainer +from ..tl import MessageContainer, GzipPacked from ..tl.all_tlobjects import tlobjects from ..tl.types import MsgsAck @@ -68,10 +68,12 @@ class MtProtoSender: self._pending_receive.extend(requests) if len(requests) == 1: request = requests[0] + data = GzipPacked.gzip_if_smaller(request) else: request = MessageContainer(self.session, requests) + data = request.to_bytes() - self._send_packet(request.to_bytes(), request) + self._send_packet(data, request) def _send_acknowledges(self): """Sends a messages acknowledge for all those who _need_confirmation""" @@ -376,11 +378,7 @@ class MtProtoSender: def _handle_gzip_packed(self, msg_id, sequence, reader, state): self._logger.debug('Handling gzip packed data') - reader.read_int(signed=False) # code - packed_data = reader.tgread_bytes() - unpacked_data = gzip.decompress(packed_data) - - with BinaryReader(unpacked_data) as compressed_reader: + with BinaryReader(GzipPacked.read(reader)) as compressed_reader: return self._process_msg(msg_id, sequence, compressed_reader, state) # endregion diff --git a/telethon/tl/__init__.py b/telethon/tl/__init__.py index e022408b..abd27a35 100644 --- a/telethon/tl/__init__.py +++ b/telethon/tl/__init__.py @@ -1,3 +1,4 @@ from .tlobject import TLObject from .session import Session +from .gzip_packed import GzipPacked from .message_container import MessageContainer diff --git a/telethon/tl/gzip_packed.py b/telethon/tl/gzip_packed.py new file mode 100644 index 00000000..c2fe17c7 --- /dev/null +++ b/telethon/tl/gzip_packed.py @@ -0,0 +1,40 @@ +import gzip + +from . import TLObject +from ..extensions import BinaryWriter + + +class GzipPacked(TLObject): + constructor_id = 0x3072cfa1 + + def __init__(self, data): + super().__init__() + self.data = data + + @staticmethod + def gzip_if_smaller(request): + """Calls request.to_bytes(), and based on a certain threshold, + optionally gzips the resulting data. If the gzipped data is + smaller than the original byte array, this is returned instead. + + Note that this only applies to content related requests. + """ + data = request.to_bytes() + # TODO This threshold could be configurable + if request.content_related and len(data) > 512: + gzipped = GzipPacked(data).to_bytes() + return gzipped if len(gzipped) < len(data) else data + else: + return data + + def to_bytes(self): + # TODO Maybe compress level could be an option + with BinaryWriter() as writer: + writer.write_int(GzipPacked.constructor_id, signed=False) + writer.tgwrite_bytes(gzip.compress(self.data)) + return writer.get_bytes() + + @staticmethod + def read(reader): + reader.read_int(signed=False) # code + return gzip.decompress(reader.tgread_bytes()) diff --git a/telethon/tl/message_container.py b/telethon/tl/message_container.py index ce7be98d..7686773b 100644 --- a/telethon/tl/message_container.py +++ b/telethon/tl/message_container.py @@ -1,4 +1,4 @@ -from . import TLObject +from . import TLObject, GzipPacked from ..extensions import BinaryWriter @@ -24,7 +24,8 @@ class MessageContainer(TLObject): writer.write_int( self.session.generate_sequence(x.content_related) ) - packet = x.to_bytes() + + packet = GzipPacked.gzip_if_smaller(x) writer.write_int(len(packet)) writer.write(packet) From 795cb989099a940a1bd065f8212f5fe650c2bf18 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 27 Sep 2017 13:49:58 +0200 Subject: [PATCH 058/121] Move MessageContainer.on_send inside its .to_bytes --- telethon/tl/message_container.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/telethon/tl/message_container.py b/telethon/tl/message_container.py index 7686773b..dc98cfc0 100644 --- a/telethon/tl/message_container.py +++ b/telethon/tl/message_container.py @@ -14,25 +14,22 @@ class MessageContainer(TLObject): self.session = session self.requests = requests - def on_send(self, writer): - writer.write_int(MessageContainer.constructor_id, signed=False) - writer.write_int(len(self.requests)) - for x in self.requests: - x.request_msg_id = self.session.get_new_msg_id() - - writer.write_long(x.request_msg_id) - writer.write_int( - self.session.generate_sequence(x.content_related) - ) - - packet = GzipPacked.gzip_if_smaller(x) - writer.write_int(len(packet)) - writer.write(packet) - def to_bytes(self): # TODO Change this to delete the on_send from this class with BinaryWriter() as writer: - self.on_send(writer) + writer.write_int(MessageContainer.constructor_id, signed=False) + writer.write_int(len(self.requests)) + for x in self.requests: + x.request_msg_id = self.session.get_new_msg_id() + + writer.write_long(x.request_msg_id) + writer.write_int( + self.session.generate_sequence(x.content_related) + ) + + packet = GzipPacked.gzip_if_smaller(x) + writer.write_int(len(packet)) + writer.write(packet) return writer.get_bytes() @staticmethod From b0839a028ea90bd434e68e35c5f60ea791680de7 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 27 Sep 2017 13:51:59 +0200 Subject: [PATCH 059/121] Update to v0.14 --- telethon/telegram_bare_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 64bc8c5e..d5d929d3 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -53,7 +53,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.13.6' + __version__ = '0.14' # TODO Make this thread-safe, all connections share the same DC _dc_options = None From bd3dd371a23bfbe0253f8115c324094b3cc6b9e7 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 27 Sep 2017 21:01:20 +0200 Subject: [PATCH 060/121] Create a proper Message class (msg_id, seqno, body; only .to_bytes()) --- telethon/network/mtproto_sender.py | 99 +++++++++++++----------------- telethon/tl/__init__.py | 1 + telethon/tl/message.py | 17 +++++ telethon/tl/message_container.py | 22 ++----- 4 files changed, 67 insertions(+), 72 deletions(-) create mode 100644 telethon/tl/message.py diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index a63e936e..5474334e 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -1,5 +1,6 @@ import gzip import logging +import struct from threading import RLock from .. import helpers as utils @@ -8,8 +9,8 @@ from ..errors import ( BadMessageError, InvalidChecksumError, BrokenAuthKeyError, rpc_message_to_error ) -from ..extensions import BinaryReader, BinaryWriter -from ..tl import MessageContainer, GzipPacked +from ..extensions import BinaryReader +from ..tl import Message, MessageContainer, GzipPacked from ..tl.all_tlobjects import tlobjects from ..tl.types import MsgsAck @@ -29,8 +30,11 @@ class MtProtoSender: self.connection = connection self._logger = logging.getLogger(__name__) - self._need_confirmation = [] # Message IDs that need confirmation - self._pending_receive = [] # Requests sent waiting to be received + # Message IDs that need confirmation + self._need_confirmation = [] + + # Requests (as msg_id: Message) sent waiting to be received + self._pending_receive = {} # Sending and receiving are independent, but two threads cannot # send or receive at the same time no matter what. @@ -65,21 +69,22 @@ class MtProtoSender: self._send_acknowledges() # Finally send our packed request(s) - self._pending_receive.extend(requests) - if len(requests) == 1: - request = requests[0] - data = GzipPacked.gzip_if_smaller(request) - else: - request = MessageContainer(self.session, requests) - data = request.to_bytes() + messages = [Message(self.session, r) for r in requests] + self._pending_receive.update({m.msg_id: m for m in messages}) - self._send_packet(data, request) + if len(messages) == 1: + message = messages[0] + else: + message = Message(self.session, MessageContainer(messages)) + + self._send_message(message) def _send_acknowledges(self): """Sends a messages acknowledge for all those who _need_confirmation""" if self._need_confirmation: - msgs_ack = MsgsAck(self._need_confirmation) - self._send_packet(msgs_ack.to_bytes(), msgs_ack) + self._send_message( + Message(self.session, MsgsAck(self._need_confirmation)) + ) del self._need_confirmation[:] def receive(self, update_state): @@ -114,36 +119,21 @@ class MtProtoSender: # region Low level processing - def _send_packet(self, packet, request): - """Sends the given packet bytes with the additional - information of the original request. - """ - request.request_msg_id = self.session.get_new_msg_id() + def _send_message(self, message): + """Sends the given Message(TLObject) encrypted through the network""" - # First calculate plain_text to encrypt it - with BinaryWriter() as plain_writer: - plain_writer.write_long(self.session.salt, signed=False) - plain_writer.write_long(self.session.id, signed=False) - plain_writer.write_long(request.request_msg_id) - plain_writer.write_int( - self.session.generate_sequence(request.content_related)) + plain_text = \ + struct.pack(' Date: Wed, 27 Sep 2017 21:04:52 +0200 Subject: [PATCH 061/121] Fix auth_key is unsigned --- telethon/network/mtproto_sender.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 5474334e..11352de1 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -127,7 +127,7 @@ class MtProtoSender: + message.to_bytes() msg_key = utils.calc_msg_key(plain_text) - key_id = struct.pack(' Date: Wed, 27 Sep 2017 21:06:01 +0200 Subject: [PATCH 062/121] Remove BinaryWriter dependency on MessageContainer --- telethon/tl/message_container.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/telethon/tl/message_container.py b/telethon/tl/message_container.py index a86134bf..d0e1aa92 100644 --- a/telethon/tl/message_container.py +++ b/telethon/tl/message_container.py @@ -1,5 +1,6 @@ -from . import TLObject, GzipPacked -from ..extensions import BinaryWriter +import struct + +from . import TLObject class MessageContainer(TLObject): @@ -11,14 +12,9 @@ class MessageContainer(TLObject): self.messages = messages def to_bytes(self): - # TODO Change this to delete the on_send from this class - with BinaryWriter() as writer: - writer.write_int(MessageContainer.constructor_id, signed=False) - writer.write_int(len(self.messages)) - for m in self.messages: - writer.write(m.to_bytes()) - - return writer.get_bytes() + return struct.pack( + ' Date: Wed, 27 Sep 2017 21:23:59 +0200 Subject: [PATCH 063/121] Avoid using BinaryWriter where possible --- telethon/crypto/auth_key.py | 12 ++++-------- telethon/crypto/rsa.py | 11 +++++------ telethon/network/connection.py | 35 ++++++++++++++-------------------- telethon/tl/gzip_packed.py | 8 +++----- 4 files changed, 26 insertions(+), 40 deletions(-) diff --git a/telethon/crypto/auth_key.py b/telethon/crypto/auth_key.py index 02774d58..710ebed8 100644 --- a/telethon/crypto/auth_key.py +++ b/telethon/crypto/auth_key.py @@ -1,7 +1,8 @@ +import struct from hashlib import sha1 from .. import helpers as utils -from ..extensions import BinaryReader, BinaryWriter +from ..extensions import BinaryReader class AuthKey: @@ -17,10 +18,5 @@ class AuthKey: """Calculates the new nonce hash based on the current class fields' values """ - with BinaryWriter() as writer: - writer.write(new_nonce) - writer.write_byte(number) - writer.write_long(self.aux_hash, signed=False) - - new_nonce_hash = utils.calc_msg_key(writer.get_bytes()) - return new_nonce_hash + data = new_nonce + struct.pack('> 2 - if length < 127: - writer.write_byte(length) - else: - writer.write_byte(127) - writer.write(int.to_bytes(length, 3, 'little')) - writer.write(message) - self.write(writer.get_bytes()) + length = len(message) >> 2 + if length < 127: + length = struct.pack('B', length) + else: + length = b'\x7f' + int.to_bytes(length, 3, 'little') + + self.write(length + message) # endregion diff --git a/telethon/tl/gzip_packed.py b/telethon/tl/gzip_packed.py index c2fe17c7..9a0d547a 100644 --- a/telethon/tl/gzip_packed.py +++ b/telethon/tl/gzip_packed.py @@ -1,7 +1,7 @@ import gzip +import struct from . import TLObject -from ..extensions import BinaryWriter class GzipPacked(TLObject): @@ -29,10 +29,8 @@ class GzipPacked(TLObject): def to_bytes(self): # TODO Maybe compress level could be an option - with BinaryWriter() as writer: - writer.write_int(GzipPacked.constructor_id, signed=False) - writer.tgwrite_bytes(gzip.compress(self.data)) - return writer.get_bytes() + return struct.pack(' Date: Thu, 28 Sep 2017 09:30:47 +0200 Subject: [PATCH 064/121] Use .appendleft(x) for deque instead .insert(0, x) (fix #268) --- telethon/update_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/update_state.py b/telethon/update_state.py index e92dac1f..c38d6ebf 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -57,7 +57,7 @@ class UpdateState: with self._updates_lock: # Insert at the beginning so the very next poll causes an error # TODO Should this reset the pts and such? - self._updates.insert(0, error) + self._updates.appendleft(error) self._updates_available.set() def check_error(self): From fb0898b9cb11322999128d8eed052d1b473d58ba Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 28 Sep 2017 09:55:29 +0200 Subject: [PATCH 065/121] Don't distinguish between str and bytes when serializing This makes it easier to use some requests like ReqPqRequest which needs a string of bytes, not a valid utf-8 string per se. --- telethon/tl/tlobject.py | 7 +++---- telethon_generator/tl_generator.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/telethon/tl/tlobject.py b/telethon/tl/tlobject.py index b67fdc62..a99b30b0 100644 --- a/telethon/tl/tlobject.py +++ b/telethon/tl/tlobject.py @@ -87,6 +87,9 @@ class TLObject: @staticmethod def serialize_bytes(data): """Write bytes by using Telegram guidelines""" + if isinstance(data, str): + data = data.encode('utf-8') + r = [] if len(data) < 254: padding = (len(data) + 1) % 4 @@ -112,10 +115,6 @@ class TLObject: r.append(bytes(padding)) return b''.join(r) - @staticmethod - def serialize_string(string): - return TLObject.serialize_bytes(string.encode('utf-8')) - # These should be overrode def to_dict(self, recursive=True): return {} diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 66e210c8..47729ceb 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -493,7 +493,7 @@ class TLGenerator: builder.write("struct.pack(' Date: Thu, 28 Sep 2017 10:01:09 +0200 Subject: [PATCH 066/121] Fix auto-generated code didn't actually write int128/int256 --- telethon_generator/tl_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 47729ceb..5f12be1c 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -484,10 +484,10 @@ class TLGenerator: builder.write("struct.pack(' Date: Thu, 28 Sep 2017 10:12:39 +0200 Subject: [PATCH 067/121] Fix wrong type hint for int128 and int256 --- telethon_generator/parser/tl_object.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/telethon_generator/parser/tl_object.py b/telethon_generator/parser/tl_object.py index 31c6c4ad..416bc587 100644 --- a/telethon_generator/parser/tl_object.py +++ b/telethon_generator/parser/tl_object.py @@ -248,6 +248,8 @@ class TLArg: result = { 'int': 'int', 'long': 'int', + 'int128': 'int', + 'int256': 'int', 'string': 'str', 'date': 'datetime.datetime | None', # None date = 0 timestamp 'bytes': 'bytes', From 4bedbfe945476ef44f81137ee83191b901a0b5b0 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 28 Sep 2017 10:32:57 +0200 Subject: [PATCH 068/121] Use bytes instead string type on the auth-key methods --- telethon_generator/scheme.tl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/telethon_generator/scheme.tl b/telethon_generator/scheme.tl index c0b41dc9..5e949239 100644 --- a/telethon_generator/scheme.tl +++ b/telethon_generator/scheme.tl @@ -32,16 +32,16 @@ /// Authorization key creation /////////////////////////////// -resPQ#05162463 nonce:int128 server_nonce:int128 pq:string server_public_key_fingerprints:Vector = ResPQ; +resPQ#05162463 nonce:int128 server_nonce:int128 pq:bytes server_public_key_fingerprints:Vector = ResPQ; -p_q_inner_data#83c95aec pq:string p:string q:string nonce:int128 server_nonce:int128 new_nonce:int256 = P_Q_inner_data; +p_q_inner_data#83c95aec pq:bytes p:bytes q:bytes nonce:int128 server_nonce:int128 new_nonce:int256 = P_Q_inner_data; server_DH_params_fail#79cb045d nonce:int128 server_nonce:int128 new_nonce_hash:int128 = Server_DH_Params; -server_DH_params_ok#d0e8075c nonce:int128 server_nonce:int128 encrypted_answer:string = Server_DH_Params; +server_DH_params_ok#d0e8075c nonce:int128 server_nonce:int128 encrypted_answer:bytes = Server_DH_Params; -server_DH_inner_data#b5890dba nonce:int128 server_nonce:int128 g:int dh_prime:string g_a:string server_time:int = Server_DH_inner_data; +server_DH_inner_data#b5890dba nonce:int128 server_nonce:int128 g:int dh_prime:bytes g_a:bytes server_time:int = Server_DH_inner_data; -client_DH_inner_data#6643b654 nonce:int128 server_nonce:int128 retry_id:long g_b:string = Client_DH_Inner_Data; +client_DH_inner_data#6643b654 nonce:int128 server_nonce:int128 retry_id:long g_b:bytes = Client_DH_Inner_Data; dh_gen_ok#3bcbf734 nonce:int128 server_nonce:int128 new_nonce_hash1:int128 = Set_client_DH_params_answer; dh_gen_retry#46dc1fb9 nonce:int128 server_nonce:int128 new_nonce_hash2:int128 = Set_client_DH_params_answer; @@ -55,9 +55,9 @@ destroy_auth_key_fail#ea109b13 = DestroyAuthKeyRes; req_pq#60469778 nonce:int128 = ResPQ; -req_DH_params#d712e4be nonce:int128 server_nonce:int128 p:string q:string public_key_fingerprint:long encrypted_data:string = Server_DH_Params; +req_DH_params#d712e4be nonce:int128 server_nonce:int128 p:bytes q:bytes public_key_fingerprint:long encrypted_data:bytes = Server_DH_Params; -set_client_DH_params#f5045f1f nonce:int128 server_nonce:int128 encrypted_data:string = Set_client_DH_params_answer; +set_client_DH_params#f5045f1f nonce:int128 server_nonce:int128 encrypted_data:bytes = Set_client_DH_params_answer; destroy_auth_key#d1435160 = DestroyAuthKeyRes; From fe30f63d5d6f3a295e233cc2720d54ee0a61b4d5 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 28 Sep 2017 11:36:51 +0200 Subject: [PATCH 069/121] Use autogen code on the authenticator instead hardcoding requests --- telethon/crypto/auth_key.py | 1 + telethon/crypto/rsa.py | 3 +- telethon/helpers.py | 8 +- telethon/network/authenticator.py | 269 +++++++++++++----------------- 4 files changed, 125 insertions(+), 156 deletions(-) diff --git a/telethon/crypto/auth_key.py b/telethon/crypto/auth_key.py index 710ebed8..17a7f8ca 100644 --- a/telethon/crypto/auth_key.py +++ b/telethon/crypto/auth_key.py @@ -18,5 +18,6 @@ class AuthKey: """Calculates the new nonce hash based on the current class fields' values """ + new_nonce = new_nonce.to_bytes(32, 'little', signed=True) data = new_nonce + struct.pack(' Date: Thu, 28 Sep 2017 11:43:06 +0200 Subject: [PATCH 070/121] Stop using BinaryWriter on MtProtoPlainSender --- telethon/network/mtproto_plain_sender.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/telethon/network/mtproto_plain_sender.py b/telethon/network/mtproto_plain_sender.py index 5ced50a9..c7c021be 100644 --- a/telethon/network/mtproto_plain_sender.py +++ b/telethon/network/mtproto_plain_sender.py @@ -1,7 +1,8 @@ +import struct import time from ..errors import BrokenAuthKeyError -from ..extensions import BinaryReader, BinaryWriter +from ..extensions import BinaryReader class MtProtoPlainSender: @@ -25,14 +26,9 @@ class MtProtoPlainSender: """Sends a plain packet (auth_key_id = 0) containing the given message body (data) """ - with BinaryWriter(known_length=len(data) + 20) as writer: - writer.write_long(0) - writer.write_long(self._get_new_msg_id()) - writer.write_int(len(data)) - writer.write(data) - - packet = writer.get_bytes() - self._connection.send(packet) + self._connection.send( + struct.pack(' Date: Thu, 28 Sep 2017 11:49:45 +0200 Subject: [PATCH 071/121] Completely remove BinaryWriter from the project --- telethon/extensions/__init__.py | 4 +- telethon/extensions/binary_writer.py | 152 --------------------------- telethon_tests/utils_test.py | 77 +++++--------- 3 files changed, 25 insertions(+), 208 deletions(-) delete mode 100644 telethon/extensions/binary_writer.py diff --git a/telethon/extensions/__init__.py b/telethon/extensions/__init__.py index 06e2f087..0e932eac 100644 --- a/telethon/extensions/__init__.py +++ b/telethon/extensions/__init__.py @@ -1,9 +1,7 @@ """ Several extensions Python is missing, such as a proper class to handle a TCP communication with support for cancelling the operation, and an utility class -to work with arbitrary binary data in a more comfortable way (writing ints, -strings, bytes, etc.) +to read arbitrary binary data in a more comfortable way, with int/strings/etc. """ -from .binary_writer import BinaryWriter from .binary_reader import BinaryReader from .tcp_client import TcpClient diff --git a/telethon/extensions/binary_writer.py b/telethon/extensions/binary_writer.py deleted file mode 100644 index 8147e1fd..00000000 --- a/telethon/extensions/binary_writer.py +++ /dev/null @@ -1,152 +0,0 @@ -from io import BufferedWriter, BytesIO, DEFAULT_BUFFER_SIZE -from struct import pack - - -class BinaryWriter: - """ - Small utility class to write binary data. - Also creates a "Memory Stream" if necessary - """ - - def __init__(self, stream=None, known_length=None): - if not stream: - stream = BytesIO() - - if known_length is None: - # On some systems, DEFAULT_BUFFER_SIZE defaults to 8192 - # That's over 16 times as big as necessary for most messages - known_length = max(DEFAULT_BUFFER_SIZE, 1024) - - self.writer = BufferedWriter(stream, buffer_size=known_length) - self.written_count = 0 - - # region Writing - - # "All numbers are written as little endian." - # https://core.telegram.org/mtproto - def write_byte(self, value): - """Writes a single byte value""" - self.writer.write(pack('B', value)) - self.written_count += 1 - - def write_int(self, value, signed=True): - """Writes an integer value (4 bytes), optionally signed""" - self.writer.write( - int.to_bytes( - value, length=4, byteorder='little', signed=signed)) - self.written_count += 4 - - def write_long(self, value, signed=True): - """Writes a long integer value (8 bytes), optionally signed""" - self.writer.write( - int.to_bytes( - value, length=8, byteorder='little', signed=signed)) - self.written_count += 8 - - def write_float(self, value): - """Writes a floating point value (4 bytes)""" - self.writer.write(pack('> 8) % 256])) - self.write(bytes([(len(data) >> 16) % 256])) - self.write(data) - - self.write(bytes(padding)) - - def tgwrite_string(self, string): - """Write a string by using Telegram guidelines""" - self.tgwrite_bytes(string.encode('utf-8')) - - def tgwrite_bool(self, boolean): - """Write a boolean value by using Telegram guidelines""" - # boolTrue boolFalse - self.write_int(0x997275b5 if boolean else 0xbc799737, signed=False) - - def tgwrite_date(self, datetime): - """Converts a Python datetime object into Unix time - (used by Telegram) and writes it - """ - value = 0 if datetime is None else int(datetime.timestamp()) - self.write_int(value) - - def tgwrite_object(self, tlobject): - """Writes a Telegram object""" - self.write(tlobject.to_bytes()) - - def tgwrite_vector(self, vector): - """Writes a vector of Telegram objects""" - self.write_int(0x1cb5c415, signed=False) # Vector's constructor ID - self.write_int(len(vector)) - for item in vector: - self.tgwrite_object(item) - - # endregion - - def flush(self): - """Flush the current stream to "update" changes""" - self.writer.flush() - - def close(self): - """Close the current stream""" - self.writer.close() - - def get_bytes(self, flush=True): - """Get the current bytes array content from the buffer, - optionally flushing first - """ - if flush: - self.writer.flush() - return self.writer.raw.getvalue() - - def get_written_bytes_count(self): - """Gets the count of bytes written in the buffer. - This may NOT be equal to the stream length if one - was provided when initializing the writer - """ - return self.written_count - - # with block - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() diff --git a/telethon_tests/utils_test.py b/telethon_tests/utils_test.py index 5c53740c..790f3f4d 100644 --- a/telethon_tests/utils_test.py +++ b/telethon_tests/utils_test.py @@ -1,90 +1,61 @@ import os import unittest -from telethon.extensions import BinaryReader, BinaryWriter +from telethon.tl import TLObject +from telethon.extensions import BinaryReader class UtilsTests(unittest.TestCase): @staticmethod def test_binary_writer_reader(): - # Test that we can write and read properly - with BinaryWriter() as writer: - writer.write_byte(1) - writer.write_int(5) - writer.write_long(13) - writer.write_float(17.0) - writer.write_double(25.0) - writer.write(bytes([26, 27, 28, 29, 30, 31, 32])) - writer.write_large_int(2**127, 128, signed=False) - - data = writer.get_bytes() - expected = b'\x01\x05\x00\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\x88A\x00\x00\x00\x00\x00\x00' \ - b'9@\x1a\x1b\x1c\x1d\x1e\x1f \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80' - - assert data == expected, 'Retrieved data does not match the expected value' + # Test that we can read properly + data = b'\x01\x05\x00\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x00\x00' \ + b'\x88A\x00\x00\x00\x00\x00\x009@\x1a\x1b\x1c\x1d\x1e\x1f ' \ + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' \ + b'\x00\x80' with BinaryReader(data) as reader: value = reader.read_byte() - assert value == 1, 'Example byte should be 1 but is {}'.format( - value) + assert value == 1, 'Example byte should be 1 but is {}'.format(value) value = reader.read_int() - assert value == 5, 'Example integer should be 5 but is {}'.format( - value) + assert value == 5, 'Example integer should be 5 but is {}'.format(value) value = reader.read_long() - assert value == 13, 'Example long integer should be 13 but is {}'.format( - value) + assert value == 13, 'Example long integer should be 13 but is {}'.format(value) value = reader.read_float() - assert value == 17.0, 'Example float should be 17.0 but is {}'.format( - value) + assert value == 17.0, 'Example float should be 17.0 but is {}'.format(value) value = reader.read_double() - assert value == 25.0, 'Example double should be 25.0 but is {}'.format( - value) + assert value == 25.0, 'Example double should be 25.0 but is {}'.format(value) value = reader.read(7) assert value == bytes([26, 27, 28, 29, 30, 31, 32]), 'Example bytes should be {} but is {}' \ .format(bytes([26, 27, 28, 29, 30, 31, 32]), value) value = reader.read_large_int(128, signed=False) - assert value == 2**127, 'Example large integer should be {} but is {}'.format( - 2**127, value) - - # Test Telegram that types are written right - with BinaryWriter() as writer: - writer.write_int(0x60469778) - buffer = writer.get_bytes() - valid = b'\x78\x97\x46\x60' # Tested written bytes using C#'s MemoryStream - - assert buffer == valid, 'Written type should be {} but is {}'.format( - list(valid), list(buffer)) + assert value == 2**127, 'Example large integer should be {} but is {}'.format(2**127, value) @staticmethod def test_binary_tgwriter_tgreader(): small_data = os.urandom(33) - small_data_padded = os.urandom( - 19) # +1 byte for length = 20 (evenly divisible by 4) + small_data_padded = os.urandom(19) # +1 byte for length = 20 (%4 = 0) large_data = os.urandom(999) large_data_padded = os.urandom(1024) data = (small_data, small_data_padded, large_data, large_data_padded) string = 'Testing Telegram strings, this should work properly!' + serialized = b''.join(TLObject.serialize_bytes(d) for d in data) + \ + TLObject.serialize_bytes(string) - with BinaryWriter() as writer: - # First write the data + with BinaryReader(serialized) as reader: + # And then try reading it without errors (it should be unharmed!) for datum in data: - writer.tgwrite_bytes(datum) - writer.tgwrite_string(string) + value = reader.tgread_bytes() + assert value == datum, 'Example bytes should be {} but is {}'.format( + datum, value) - with BinaryReader(writer.get_bytes()) as reader: - # And then try reading it without errors (it should be unharmed!) - for datum in data: - value = reader.tgread_bytes() - assert value == datum, 'Example bytes should be {} but is {}'.format( - datum, value) - - value = reader.tgread_string() - assert value == string, 'Example string should be {} but is {}'.format( - string, value) + value = reader.tgread_string() + assert value == string, 'Example string should be {} but is {}'.format( + string, value) From cc337328c6e0077346e53a8bd981ad204ac36d4c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 28 Sep 2017 11:59:24 +0200 Subject: [PATCH 072/121] Rename handmade Message class to TLMessage to avoid confusion --- telethon/network/mtproto_sender.py | 8 ++++---- telethon/tl/__init__.py | 2 +- telethon/tl/{message.py => tl_message.py} | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename telethon/tl/{message.py => tl_message.py} (95%) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 11352de1..acda8f05 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -10,7 +10,7 @@ from ..errors import ( rpc_message_to_error ) from ..extensions import BinaryReader -from ..tl import Message, MessageContainer, GzipPacked +from ..tl import TLMessage, MessageContainer, GzipPacked from ..tl.all_tlobjects import tlobjects from ..tl.types import MsgsAck @@ -69,13 +69,13 @@ class MtProtoSender: self._send_acknowledges() # Finally send our packed request(s) - messages = [Message(self.session, r) for r in requests] + messages = [TLMessage(self.session, r) for r in requests] self._pending_receive.update({m.msg_id: m for m in messages}) if len(messages) == 1: message = messages[0] else: - message = Message(self.session, MessageContainer(messages)) + message = TLMessage(self.session, MessageContainer(messages)) self._send_message(message) @@ -83,7 +83,7 @@ class MtProtoSender: """Sends a messages acknowledge for all those who _need_confirmation""" if self._need_confirmation: self._send_message( - Message(self.session, MsgsAck(self._need_confirmation)) + TLMessage(self.session, MsgsAck(self._need_confirmation)) ) del self._need_confirmation[:] diff --git a/telethon/tl/__init__.py b/telethon/tl/__init__.py index bdcbcd6c..403e481a 100644 --- a/telethon/tl/__init__.py +++ b/telethon/tl/__init__.py @@ -1,5 +1,5 @@ from .tlobject import TLObject from .session import Session from .gzip_packed import GzipPacked -from .message import Message +from .tl_message import TLMessage from .message_container import MessageContainer diff --git a/telethon/tl/message.py b/telethon/tl/tl_message.py similarity index 95% rename from telethon/tl/message.py rename to telethon/tl/tl_message.py index 24bd0ab5..c994004e 100644 --- a/telethon/tl/message.py +++ b/telethon/tl/tl_message.py @@ -3,7 +3,7 @@ import struct from . import TLObject, GzipPacked -class Message(TLObject): +class TLMessage(TLObject): """https://core.telegram.org/mtproto/service_messages#simple-container""" def __init__(self, session, request): super().__init__() From dc5bbc17193be1bd18ce733e284f8c24b8225062 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 28 Sep 2017 12:02:47 +0200 Subject: [PATCH 073/121] Update to v0.14.1 --- telethon/telegram_bare_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index d5d929d3..2d7c994a 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -53,7 +53,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.14' + __version__ = '0.14.1' # TODO Make this thread-safe, all connections share the same DC _dc_options = None From 3fda7efeb9b1682197ed48f519c0e3673d350ce7 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 28 Sep 2017 12:20:56 +0200 Subject: [PATCH 074/121] More friendly issue template --- .github/ISSUE_TEMPLATE.md | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index c9b27a9f..eb08ea77 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,26 +1,9 @@ - -### What went wrong -Describe what happened or what the error you have is. - -``` -// paste the crash log here if any -``` - -### What I've done -Either a code example of what you were trying to do, or steps to reproduce, or methods you have tried invoking. - -```python -# Add your Python code here -``` - -### More information -If you think other information can be relevant (e.g. operative system or other variables), add it here. From 423efc436050a6378e17103c97c0327dc094e3d9 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 11:56:43 +0200 Subject: [PATCH 075/121] Fix optional vector arguments failing --- telethon_generator/tl_generator.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 5f12be1c..c6f56bbb 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -392,15 +392,14 @@ class TLGenerator: a parameter upon creating the request. Returns False otherwise """ if arg.is_vector: - builder.writeln( - 'self.{0} = [{1}(_x) for _x in {0}]' - .format(arg.name, get_input_code) - ) - pass + builder.write('self.{0} = [{1}(_x) for _x in {0}]' + .format(arg.name, get_input_code)) else: - builder.writeln( - 'self.{0} = {1}({0})'.format(arg.name, get_input_code) - ) + builder.write('self.{0} = {1}({0})' + .format(arg.name, get_input_code)) + builder.writeln( + ' if {} else None'.format(arg.name) if arg.is_flag else '' + ) @staticmethod def get_file_name(tlobject, add_extension=False): From 3c7dd2598295e16786047ae3cfe4e66c385aecb9 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 12:02:26 +0200 Subject: [PATCH 076/121] Revert "if * is None" check (8bff10d) on get_input_* due to 423efc4 --- telethon/utils.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/telethon/utils.py b/telethon/utils.py index cd1927a1..0b564236 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -74,9 +74,6 @@ def _raise_cast_fail(entity, target): def get_input_peer(entity): """Gets the input peer for the given "entity" (user, chat or channel). A ValueError is raised if the given entity isn't a supported type.""" - if entity is None: - return None - if not isinstance(entity, TLObject): _raise_cast_fail(entity, 'InputPeer') @@ -118,9 +115,6 @@ def get_input_peer(entity): def get_input_channel(entity): """Similar to get_input_peer, but for InputChannel's alone""" - if entity is None: - return None - if not isinstance(entity, TLObject): _raise_cast_fail(entity, 'InputChannel') @@ -138,9 +132,6 @@ def get_input_channel(entity): def get_input_user(entity): """Similar to get_input_peer, but for InputUser's alone""" - if entity is None: - return None - if not isinstance(entity, TLObject): _raise_cast_fail(entity, 'InputUser') @@ -167,9 +158,6 @@ def get_input_user(entity): def get_input_document(document): """Similar to get_input_peer, but for documents""" - if document is None: - return None - if not isinstance(document, TLObject): _raise_cast_fail(document, 'InputDocument') @@ -193,9 +181,6 @@ def get_input_document(document): def get_input_photo(photo): """Similar to get_input_peer, but for documents""" - if photo is None: - return None - if not isinstance(photo, TLObject): _raise_cast_fail(photo, 'InputPhoto') @@ -213,9 +198,6 @@ def get_input_photo(photo): def get_input_geo(geo): """Similar to get_input_peer, but for geo points""" - if geo is None: - return None - if not isinstance(geo, TLObject): _raise_cast_fail(geo, 'InputGeoPoint') @@ -243,9 +225,6 @@ def get_input_media(media, user_caption=None, is_photo=False): If the media is a file location and is_photo is known to be True, it will be treated as an InputMediaUploadedPhoto. """ - if media is None: - return None - if not isinstance(media, TLObject): _raise_cast_fail(media, 'InputMedia') From f39d8f132f5e9c66c7872f62ba615e5ede65e9c7 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 12:14:09 +0200 Subject: [PATCH 077/121] Make generated description on the docs more friendly --- docs/generate.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/generate.py b/docs/generate.py index 928eacf1..4feb1518 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -199,14 +199,27 @@ def generate_index(folder, original_paths): def get_description(arg): """Generates a proper description for the given argument""" desc = [] + otherwise = False if arg.can_be_inferred: - desc.append('If left to None, it will be inferred automatically.') - if arg.is_vector: - desc.append('A list must be supplied for this argument.') - if arg.is_generic: - desc.append('A different Request must be supplied for this argument.') - if arg.is_flag: + desc.append('If left unspecified, it will be inferred automatically.') + otherwise = True + elif arg.is_flag: desc.append('This argument can be omitted.') + otherwise = True + + if arg.is_vector: + if arg.is_generic: + desc.append('A list of other Requests must be supplied.') + else: + desc.append('A list must be supplied.') + elif arg.is_generic: + desc.append('A different Request must be supplied for this argument.') + else: + otherwise = False # Always reset to false if no other text is added + + if otherwise: + desc.insert(1, 'Otherwise,') + desc[-1] = desc[-1][:1].lower() + desc[-1][1:] return ' '.join(desc) From 7838f8561b0b9d6faa686382e0c0f7c2c9e69693 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 12:38:53 +0200 Subject: [PATCH 078/121] Allow running setup.py from other directories --- setup.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 02b37495..509c2064 100755 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ Extra supported commands are: # To use a consistent encoding from codecs import open from sys import argv -from os import path +import os # Always prefer setuptools over distutils from setuptools import find_packages, setup @@ -26,7 +26,23 @@ except Exception as e: TelegramClient = None -if __name__ == '__main__': +class TempWorkDir: + """Switches the working directory to be the one on which this file lives, + while within the 'with' block. + """ + def __init__(self): + self.original = None + + def __enter__(self): + self.original = os.path.abspath(os.path.curdir) + os.chdir(os.path.abspath(os.path.dirname(__file__))) + return self + + def __exit__(self, *args): + os.chdir(self.original) + + +def main(): if len(argv) >= 2 and argv[1] == 'gen_tl': from telethon_generator.tl_generator import TLGenerator generator = TLGenerator('telethon/tl') @@ -65,10 +81,8 @@ if __name__ == '__main__': print('Run `python3', argv[0], 'gen_tl` first.') quit() - here = path.abspath(path.dirname(__file__)) - # Get the long description from the README file - with open(path.join(here, 'README.rst'), encoding='utf-8') as f: + with open('README.rst', encoding='utf-8') as f: long_description = f.read() setup( @@ -112,3 +126,8 @@ if __name__ == '__main__': ]), install_requires=['pyaes', 'rsa'] ) + + +if __name__ == '__main__': + with TempWorkDir(): # Could just use a try/finally but this is + reusable + main() From c134d9ba2714ad84cf0338a28b3e27f116771031 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 12:40:03 +0200 Subject: [PATCH 079/121] Run setup.py gen_tl when installing (may fix #271) --- setup.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 509c2064..695ad1a5 100755 --- a/setup.py +++ b/setup.py @@ -42,19 +42,23 @@ class TempWorkDir: os.chdir(self.original) +def gen_tl(): + from telethon_generator.tl_generator import TLGenerator + generator = TLGenerator('telethon/tl') + if generator.tlobjects_exist(): + print('Detected previous TLObjects. Cleaning...') + generator.clean_tlobjects() + + print('Generating TLObjects...') + generator.generate_tlobjects( + 'telethon_generator/scheme.tl', import_depth=2 + ) + print('Done.') + + def main(): if len(argv) >= 2 and argv[1] == 'gen_tl': - from telethon_generator.tl_generator import TLGenerator - generator = TLGenerator('telethon/tl') - if generator.tlobjects_exist(): - print('Detected previous TLObjects. Cleaning...') - generator.clean_tlobjects() - - print('Generating TLObjects...') - generator.generate_tlobjects( - 'telethon_generator/scheme.tl', import_depth=2 - ) - print('Done.') + gen_tl() elif len(argv) >= 2 and argv[1] == 'clean_tl': from telethon_generator.tl_generator import TLGenerator @@ -78,8 +82,11 @@ def main(): else: if not TelegramClient: - print('Run `python3', argv[0], 'gen_tl` first.') - quit() + gen_tl() + from telethon import TelegramClient as TgClient + version = TgClient.__version__ + else: + version = TelegramClient.__version__ # Get the long description from the README file with open('README.rst', encoding='utf-8') as f: @@ -89,7 +96,7 @@ def main(): name='Telethon', # Versions should comply with PEP440. - version=TelegramClient.__version__, + version=version, description="Full-featured Telegram client library for Python 3", long_description=long_description, From 195bba6f2193c55ad5142563532dcbe3bb14077d Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 12:57:53 +0200 Subject: [PATCH 080/121] Fix bug where booleans were always serialized as false --- telethon_generator/tl_generator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index c6f56bbb..861261e6 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -496,7 +496,9 @@ class TLGenerator: elif 'Bool' == arg.type: # 0x997275b5 if boolean else 0xbc799737 - builder.write(r"b'\xb5ur\x99' if {} else b'7\x97y\xbc'") + builder.write( + r"b'\xb5ur\x99' if {} else b'7\x97y\xbc'".format(name) + ) elif 'true' == arg.type: pass # These are actually NOT written! Only used for flags From 76d14b2dd9bae8fbe80b3351fbab75dd930ef883 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 13:00:22 +0200 Subject: [PATCH 081/121] Make generated .to_bytes() more readable --- telethon_generator/tl_generator.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 861261e6..84598999 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -300,15 +300,17 @@ class TLGenerator: # Write the .to_bytes() function builder.writeln('def to_bytes(self):') - builder.write("return b''.join((") + builder.writeln("return b''.join((") + builder.current_indent += 1 # First constructor code, we already know its bytes - builder.write('{},'.format(repr(struct.pack(' Date: Fri, 29 Sep 2017 13:03:35 +0200 Subject: [PATCH 082/121] Precompute the flag index on the generated code --- telethon_generator/tl_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 84598999..76de12e5 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -472,8 +472,8 @@ class TLGenerator: elif arg.flag_indicator: # Calculate the flags with those items which are not None builder.write("struct.pack(' Date: Fri, 29 Sep 2017 13:07:21 +0200 Subject: [PATCH 083/121] Raise AssertionError instead TypeNotFoundError on authenticator --- telethon/network/authenticator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/telethon/network/authenticator.py b/telethon/network/authenticator.py index 1847931d..1081897a 100644 --- a/telethon/network/authenticator.py +++ b/telethon/network/authenticator.py @@ -9,7 +9,7 @@ from ..tl.types import ( from .. import helpers as utils from ..crypto import AES, AuthKey, Factorization from ..crypto import rsa -from ..errors import SecurityError, TypeNotFoundError +from ..errors import SecurityError from ..extensions import BinaryReader from ..network import MtProtoPlainSender from ..tl.functions import ( @@ -25,7 +25,7 @@ def do_authentication(connection, retries=5): while retries: try: return _do_authentication(connection) - except (SecurityError, TypeNotFoundError, NotImplementedError) as e: + except (SecurityError, AssertionError, NotImplementedError) as e: last_error = e retries -= 1 raise last_error @@ -48,7 +48,7 @@ def _do_authentication(connection): res_pq = req_pq_request.result if not isinstance(res_pq, ResPQ): - raise TypeNotFoundError(type(res_pq).constructor_id) + raise AssertionError(res_pq) if res_pq.nonce != req_pq_request.nonce: raise SecurityError('Invalid nonce from server') @@ -101,7 +101,7 @@ def _do_authentication(connection): raise SecurityError('Server DH params fail: TODO') if not isinstance(server_dh_params, ServerDHParamsOk): - raise TypeNotFoundError(type(server_dh_params).constructor_id) + raise AssertionError(server_dh_params) if server_dh_params.nonce != res_pq.nonce: raise SecurityError('Invalid nonce from server') @@ -121,7 +121,7 @@ def _do_authentication(connection): reader.read(20) # hash sum server_dh_inner = reader.tgread_object() if not isinstance(server_dh_inner, ServerDHInnerData): - raise TypeNotFoundError(server_dh_inner) + raise AssertionError(server_dh_inner) if server_dh_inner.nonce != res_pq.nonce: print(server_dh_inner.nonce, res_pq.nonce) From afc4bd9cabacc80435e017af7a991bf9a0bf52e0 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 13:11:33 +0200 Subject: [PATCH 084/121] Rename constructor/subclass_of_id to upper case, keep only static --- telethon/network/mtproto_sender.py | 2 +- telethon/telegram_client.py | 2 +- telethon/tl/gzip_packed.py | 4 ++-- telethon/tl/message_container.py | 4 ++-- telethon/tl/tlobject.py | 1 - telethon/utils.py | 14 +++++++------- telethon_generator/tl_generator.py | 7 ++++--- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index acda8f05..6ce8a167 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -329,7 +329,7 @@ class MtProtoSender: if self.session.report_errors and request: error = rpc_message_to_error( reader.read_int(), reader.tgread_string(), - report_method=type(request).constructor_id + report_method=type(request).CONSTRUCTOR_ID ) else: error = rpc_message_to_error( diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index fe140652..0d04ff27 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -581,7 +581,7 @@ class TelegramClient(TelegramBareClient): return reply_to if isinstance(reply_to, TLObject) and \ - type(reply_to).subclass_of_id == 0x790009e3: + type(reply_to).SUBCLASS_OF_ID == 0x790009e3: # hex(crc32(b'Message')) = 0x790009e3 return reply_to.id diff --git a/telethon/tl/gzip_packed.py b/telethon/tl/gzip_packed.py index 9a0d547a..05453d4b 100644 --- a/telethon/tl/gzip_packed.py +++ b/telethon/tl/gzip_packed.py @@ -5,7 +5,7 @@ from . import TLObject class GzipPacked(TLObject): - constructor_id = 0x3072cfa1 + CONSTRUCTOR_ID = 0x3072cfa1 def __init__(self, data): super().__init__() @@ -29,7 +29,7 @@ class GzipPacked(TLObject): def to_bytes(self): # TODO Maybe compress level could be an option - return struct.pack(' Date: Fri, 29 Sep 2017 13:13:05 +0200 Subject: [PATCH 085/121] Update to v0.14.2 --- telethon/telegram_bare_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 2d7c994a..f4d3d8da 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -53,7 +53,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.14.1' + __version__ = '0.14.2' # TODO Make this thread-safe, all connections share the same DC _dc_options = None From 56103845de58591149c7fdad1841a14510aa2b9e Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 13:58:15 +0200 Subject: [PATCH 086/121] Fix LogOutRequest consuming all retries (#270) --- telethon/network/mtproto_sender.py | 30 ++++++++++++++++++------------ telethon/telegram_client.py | 11 ++--------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 6ce8a167..83ac4a43 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -13,6 +13,7 @@ from ..extensions import BinaryReader from ..tl import TLMessage, MessageContainer, GzipPacked from ..tl.all_tlobjects import tlobjects from ..tl.types import MsgsAck +from ..tl.functions.auth import LogOutRequest logging.getLogger(__name__).addHandler(logging.NullHandler()) @@ -41,10 +42,6 @@ class MtProtoSender: self._send_lock = RLock() self._recv_lock = RLock() - # Used when logging out, the only request that seems to use 'ack' - # TODO There might be a better way to handle msgs_ack requests - self.logging_out = False - def connect(self): """Connects to the server""" self.connection.connect() @@ -57,7 +54,6 @@ class MtProtoSender: self.connection.close() self._need_confirmation.clear() self._clear_all_pending() - self.logging_out = False # region Send and receive @@ -201,13 +197,15 @@ class MtProtoSender: # msgs_ack, it may handle the request we wanted if code == 0x62d6b459: ack = reader.tgread_object() - # We only care about ack requests if we're logging out - if self.logging_out: - for msg_id in ack.msg_ids: - r = self._pop_request(msg_id) - if r: - self._logger.debug('Message ack confirmed', r) - r.confirm_received.set() + # Ignore every ack request *unless* when logging out, when it's + # when it seems to only make sense. We also need to set a non-None + # result since Telegram doesn't send the response for these. + for msg_id in ack.msg_ids: + r = self._pop_request_of_type(msg_id, LogOutRequest) + if r: + r.result = True # Telegram won't send this value + r.confirm_received() + self._logger.debug('Message ack confirmed', r) return True @@ -241,6 +239,14 @@ class MtProtoSender: if message: return message.request + def _pop_request_of_type(self, msg_id, t): + """Pops a pending REQUEST from self._pending_receive if it matches + the given type, or returns None if it's not found/doesn't match. + """ + message = self._pending_receive.get(msg_id, None) + if isinstance(message.request, t): + return self._pending_receive.pop(msg_id).request + def _clear_all_pending(self): for r in self._pending_receive.values(): r.confirm_received.set() diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 0d04ff27..6092c70c 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -392,19 +392,12 @@ class TelegramClient(TelegramBareClient): def log_out(self): """Logs out and deletes the current session. Returns True if everything went okay.""" - # Special flag when logging out (so the ack request confirms it) - self._sender.logging_out = True - try: self(LogOutRequest()) - # The server may have already disconnected us, we still - # try to disconnect to make sure. - self.disconnect() - except (RPCError, ConnectionError): - # Something happened when logging out, restore the state back - self._sender.logging_out = False + except RPCError: return False + self.disconnect() self.session.delete() self.session = None return True From 6df38da793d14e48865d3180c983244e0a2aa2f4 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 19:54:40 +0200 Subject: [PATCH 087/121] Fix session failing to load if no auth_key present --- telethon/tl/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/tl/session.py b/telethon/tl/session.py index e72c2a21..b8d89598 100644 --- a/telethon/tl/session.py +++ b/telethon/tl/session.py @@ -118,7 +118,7 @@ class Session: # FIXME We need to import the AuthKey here or otherwise # we get cyclic dependencies. from ..crypto import AuthKey - if data['auth_key_data'] is not None: + if data.get('auth_key_data', None) is not None: key = b64decode(data['auth_key_data']) result.auth_key = AuthKey(data=key) From ee082324738b4bfec721b85fae73803f9bcdb06f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 10:27:16 +0200 Subject: [PATCH 088/121] Fix UpdateState.check_error popping the wrong side --- telethon/update_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/update_state.py b/telethon/update_state.py index c38d6ebf..4f3ebce7 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -63,7 +63,7 @@ class UpdateState: def check_error(self): with self._updates_lock: if self._updates and isinstance(self._updates[0], Exception): - raise self._updates.pop() + raise self._updates.popleft() def process(self, update): """Processes an update object. This method is normally called by From 0a693c705a13c0e5146296768a7fd6c3d876a0b1 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 19:55:14 +0200 Subject: [PATCH 089/121] Create a new connection when called from a different thread This allows to invoke several requests in parallel while not waiting for other requests to be written to the network. --- telethon/telegram_bare_client.py | 17 ++++++++++----- telethon/telegram_client.py | 36 ++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index f4d3d8da..a6d78c14 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -290,7 +290,7 @@ class TelegramBareClient: # region Invoking Telegram requests - def invoke(self, *requests, call_receive=True, retries=5): + def invoke(self, *requests, call_receive=True, retries=5, sender=None): """Invokes (sends) a MTProtoRequest and returns (receives) its result. If 'updates' is not None, all read update object will be put @@ -307,13 +307,16 @@ class TelegramBareClient: if retries <= 0: raise ValueError('Number of retries reached 0.') + if sender is None: + sender = self._sender + try: # Ensure that we start with no previous errors (i.e. resending) for x in requests: x.confirm_received.clear() x.rpc_error = None - self._sender.send(*requests) + sender.send(*requests) if not call_receive: # TODO This will be slightly troublesome if we allow # switching between constant read or not on the fly. @@ -321,11 +324,11 @@ class TelegramBareClient: # in which case a Lock would be required for .receive(). for x in requests: x.confirm_received.wait( - self._sender.connection.get_timeout() + sender.connection.get_timeout() ) else: while not all(x.confirm_received.is_set() for x in requests): - self._sender.receive(update_state=self.updates) + sender.receive(update_state=self.updates) except TimeoutError: pass # We will just retry @@ -333,9 +336,13 @@ class TelegramBareClient: except ConnectionResetError: self._logger.debug('Server disconnected us. Reconnecting and ' 'resending request...') - self._reconnect() + if sender != self._sender: + sender.connect() + else: + self._reconnect() except FloodWaitError: + sender.disconnect() self.disconnect() raise diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 6092c70c..4c686c83 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -18,7 +18,7 @@ from .errors import ( PhoneMigrateError, NetworkMigrateError, UserMigrateError, PhoneCodeExpiredError, PhoneCodeHashEmptyError, PhoneCodeInvalidError ) -from .network import ConnectionMode +from .network import Connection, ConnectionMode, MtProtoSender from .tl import Session, TLObject from .tl.functions import PingRequest from .tl.functions.account import ( @@ -146,6 +146,11 @@ class TelegramClient(TelegramBareClient): # Constantly read for results and updates from within the main client self._recv_thread = None + # Identifier of the main thread (the one that called .connect()). + # This will be used to create new connections from any other thread, + # so that requests can be sent in parallel. + self._main_thread_ident = None + # Default PingRequest delay self._last_ping = datetime.now() self._ping_delay = timedelta(minutes=1) @@ -162,6 +167,8 @@ class TelegramClient(TelegramBareClient): exported_auth is meant for internal purposes and can be ignored. """ + self._main_thread_ident = threading.get_ident() + if socks and self._recv_thread: # Treat proxy errors specially since they're not related to # Telegram itself, but rather to the proxy. If any happens on @@ -246,23 +253,40 @@ class TelegramClient(TelegramBareClient): """ # This is only valid when the read thread is reconnecting, # that is, the connection lock is locked. - if self._on_read_thread() and not self._connect_lock.locked(): + on_read_thread = self._on_read_thread() + if on_read_thread and not self._connect_lock.locked(): return # Just ignore, we would be raising and crashing the thread self.updates.check_error() + # Determine the sender to be used (main or a new connection) + # TODO Polish this so it's nicer + on_main_thread = threading.get_ident() == self._main_thread_ident + if on_main_thread or on_read_thread: + sender = self._sender + else: + conn = Connection( + self.session.server_address, self.session.port, + mode=self._sender.connection._mode, + proxy=self._sender.connection.conn.proxy, + timeout=self._sender.connection.get_timeout() + ) + sender = MtProtoSender(self.session, conn) + sender.connect() + try: # We should call receive from this thread if there's no background # thread reading or if the server disconnected us and we're trying # to reconnect. This is because the read thread may either be # locked also trying to reconnect or we may be said thread already. - call_receive = \ + call_receive = not on_main_thread or \ self._recv_thread is None or self._connect_lock.locked() return super().invoke( *requests, call_receive=call_receive, - retries=kwargs.get('retries', 5) + retries=kwargs.get('retries', 5), + sender=sender ) except (PhoneMigrateError, NetworkMigrateError, UserMigrateError) as e: @@ -284,6 +308,10 @@ class TelegramClient(TelegramBareClient): while self._user_connected and not self._reconnect(): sleep(0.1) # Retry forever until we can send the request + finally: + if sender != self._sender: + sender.disconnect() + # Let people use client(SomeRequest()) instead client.invoke(...) __call__ = invoke From b61deb5cfb8cbe8e6b5d9436d9d2ab18f6b83f01 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 20:10:16 +0200 Subject: [PATCH 090/121] Delete methods to create_new_connection and invoke_on_dc --- telethon/telegram_client.py | 42 ------------------------------------- 1 file changed, 42 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 4c686c83..9db87684 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -219,29 +219,6 @@ class TelegramClient(TelegramBareClient): return self._recv_thread is not None and \ threading.get_ident() == self._recv_thread.ident - def create_new_connection(self, on_dc=None, timeout=timedelta(seconds=5)): - """Creates a new connection which can be used in parallel - with the original TelegramClient. A TelegramBareClient - will be returned already connected, and the caller is - responsible to disconnect it. - - If 'on_dc' is None, the new client will run on the same - data center as the current client (most common case). - - If the client is meant to be used on a different data - center, the data center ID should be specified instead. - """ - if on_dc is None: - client = TelegramBareClient( - self.session, self.api_id, self.api_hash, - proxy=self._sender.connection.conn.proxy, timeout=timeout - ) - client.connect() - else: - client = self._get_exported_client(on_dc, bypass_cache=True) - - return client - # endregion # region Telegram requests functions @@ -315,25 +292,6 @@ class TelegramClient(TelegramBareClient): # Let people use client(SomeRequest()) instead client.invoke(...) __call__ = invoke - def invoke_on_dc(self, request, dc_id, reconnect=False): - """Invokes the given request on a different DC - by making use of the exported MtProtoSenders. - - If 'reconnect=True', then the a reconnection will be performed and - ConnectionResetError will be raised if it occurs a second time. - """ - try: - client = self._get_exported_client( - dc_id, init_connection=reconnect) - - return client.invoke(request) - - except ConnectionResetError: - if reconnect: - raise - else: - return self.invoke_on_dc(request, dc_id, reconnect=True) - # region Authorization requests def is_user_authorized(self): From 479afddf507efac009ae28f8b6569e2549a06637 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 29 Sep 2017 20:50:27 +0200 Subject: [PATCH 091/121] Move core functionality to the TelegramBareClient Rationale: the intended behaviour of the TelegramClient will now be to focus on abstracting the users from manually importing requests and types to work with Telegram's API. Thus, all the core functionality has been moved to the TelegramBareClient, which will now be responsible of spawning new threads or connections and even handling updates. This way there is a clear distinction between the two clients, TelegramClient is the one meant to be exposed to the end user, since it provides all the mentioned abstractions, while the TelegramBareClient is the "basic" client needed to work with the API in a comfortable way. There is still a need for an MtProtoSender, which still even lower level, and knows as little as possible of what requests are. This handles parsing the messages received from the server so that their result can be understood. --- telethon/telegram_bare_client.py | 252 ++++++++++++++++++++++++++--- telethon/telegram_client.py | 267 +------------------------------ 2 files changed, 232 insertions(+), 287 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index a6d78c14..38fd1555 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -1,21 +1,24 @@ import logging -from datetime import timedelta +import os +import threading +from datetime import timedelta, datetime from hashlib import md5 from io import BytesIO -from os import path from threading import Lock +from time import sleep from . import helpers as utils from .crypto import rsa, CdnDecrypter from .errors import ( RPCError, BrokenAuthKeyError, - FloodWaitError, FileMigrateError, TypeNotFoundError + FloodWaitError, FileMigrateError, TypeNotFoundError, + UnauthorizedError, PhoneMigrateError, NetworkMigrateError, UserMigrateError ) from .network import authenticator, MtProtoSender, Connection, ConnectionMode from .tl import TLObject, Session from .tl.all_tlobjects import LAYER from .tl.functions import ( - InitConnectionRequest, InvokeWithLayerRequest + InitConnectionRequest, InvokeWithLayerRequest, PingRequest ) from .tl.functions.auth import ( ImportAuthorizationRequest, ExportAuthorizationRequest @@ -23,6 +26,7 @@ from .tl.functions.auth import ( from .tl.functions.help import ( GetCdnConfigRequest, GetConfigRequest ) +from .tl.functions.updates import GetStateRequest from .tl.functions.upload import ( GetFileRequest, SaveBigFilePartRequest, SaveFilePartRequest ) @@ -64,11 +68,22 @@ class TelegramBareClient: connection_mode=ConnectionMode.TCP_FULL, proxy=None, process_updates=False, - timeout=timedelta(seconds=5)): - """Initializes the Telegram client with the specified API ID and Hash. - Session must always be a Session instance, and an optional proxy - can also be specified to be used on the connection. - """ + timeout=timedelta(seconds=5), + **kwargs): + """Refer to TelegramClient.__init__ for docs on this method""" + if not api_id or not api_hash: + raise PermissionError( + "Your API ID or Hash cannot be empty or None. " + "Refer to Telethon's README.rst for more information.") + + # Determine what session object we have + if isinstance(session, str) or session is None: + session = Session.try_load_or_create_new(session) + elif not isinstance(session, Session): + raise ValueError( + 'The given session must be a str or a Session instance.' + ) + self.session = session self.api_id = int(api_id) self.api_hash = api_hash @@ -95,6 +110,39 @@ class TelegramBareClient: # One may change self.updates.enabled at any later point. self.updates = UpdateState(process_updates) + # Used on connection - the user may modify these and reconnect + kwargs['app_version'] = kwargs.get('app_version', self.__version__) + for name, value in kwargs.items(): + if hasattr(self.session, name): + setattr(self.session, name, value) + + # Despite the state of the real connection, keep track of whether + # the user has explicitly called .connect() or .disconnect() here. + # This information is required by the read thread, who will be the + # one attempting to reconnect on the background *while* the user + # doesn't explicitly call .disconnect(), thus telling it to stop + # retrying. The main thread, knowing there is a background thread + # attempting reconnection as soon as it happens, will just sleep. + self._user_connected = False + + # Save whether the user is authorized here (a.k.a. logged in) + self._authorized = False + + # Uploaded files cache so subsequent calls are instant + self._upload_cache = {} + + # Constantly read for results and updates from within the main client + self._recv_thread = None + + # Identifier of the main thread (the one that called .connect()). + # This will be used to create new connections from any other thread, + # so that requests can be sent in parallel. + self._main_thread_ident = None + + # Default PingRequest delay + self._last_ping = datetime.now() + self._ping_delay = timedelta(minutes=1) + # endregion # region Connecting @@ -108,6 +156,8 @@ class TelegramBareClient: If 'exported_auth' is not None, it will be used instead to determine the authorization key for the current session. """ + self._main_thread_ident = threading.get_ident() + try: self._sender.connect() if not self.session.auth_key: @@ -143,6 +193,15 @@ class TelegramBareClient: TelegramBareClient._dc_options = \ self(GetConfigRequest()).dc_options + # Connection was successful! Try syncing the update state + # to also assert whether the user is logged in or not. + self._user_connected = True + try: + self.sync_updates() + self._set_connected_and_authorized() + except UnauthorizedError: + self._authorized = False + return True except TypeNotFoundError as e: @@ -178,9 +237,23 @@ class TelegramBareClient: return result def disconnect(self): - """Disconnects from the Telegram server""" + """Disconnects from the Telegram server + and stops all the spawned threads""" + self._user_connected = False + self._recv_thread = None + + # This will trigger a "ConnectionResetError", for subsequent calls + # to read or send (from another thread) and usually, the background + # thread would try restarting the connection but since the + # ._recv_thread = None, it knows it doesn't have to. self._sender.disconnect() + # Also disconnect all the cached senders + for sender in self._cached_clients.values(): + sender.disconnect() + + self._cached_clients.clear() + def _reconnect(self, new_dc=None): """If 'new_dc' is not set, only a call to .connect() will be made since it's assumed that the connection has been lost and the @@ -210,7 +283,11 @@ class TelegramBareClient: # endregion - # region Working with different Data Centers + # region Working with different connections/Data Centers + + def _on_read_thread(self): + return self._recv_thread is not None and \ + threading.get_ident() == self._recv_thread.ident def _get_dc(self, dc_id, ipv6=False, cdn=False): """Gets the Data Center (DC) associated to 'dc_id'""" @@ -290,16 +367,21 @@ class TelegramBareClient: # region Invoking Telegram requests - def invoke(self, *requests, call_receive=True, retries=5, sender=None): + def invoke(self, *requests, call_receive=True, retries=5): """Invokes (sends) a MTProtoRequest and returns (receives) its result. - If 'updates' is not None, all read update object will be put - in such list. Otherwise, update objects will be ignored. - - If 'call_receive' is set to False, then there should be another - thread calling to 'self._sender.receive()' running or this method - will lock forever. + The invoke will be retried up to 'retries' times before raising + ValueError(). """ + # This is only valid when the read thread is reconnecting, + # that is, the connection lock is locked. + on_read_thread = self._on_read_thread() + if on_read_thread and not self._connect_lock.locked(): + return # Just ignore, we would be raising and crashing the thread + + # Any error from a background thread will be "posted" and checked here + self.updates.check_error() + if not all(isinstance(x, TLObject) and x.content_related for x in requests): raise ValueError('You can only invoke requests, not types!') @@ -307,8 +389,20 @@ class TelegramBareClient: if retries <= 0: raise ValueError('Number of retries reached 0.') - if sender is None: + # Determine the sender to be used (main or a new connection) + # TODO Polish this so it's nicer + on_main_thread = threading.get_ident() == self._main_thread_ident + if on_main_thread or on_read_thread: sender = self._sender + else: + conn = Connection( + self.session.server_address, self.session.port, + mode=self._sender.connection._mode, + proxy=self._sender.connection.conn.proxy, + timeout=self._sender.connection.get_timeout() + ) + sender = MtProtoSender(self.session, conn) + sender.connect() try: # Ensure that we start with no previous errors (i.e. resending) @@ -317,6 +411,14 @@ class TelegramBareClient: x.rpc_error = None sender.send(*requests) + + # We should call receive from this thread if there's no background + # thread reading or if the server disconnected us and we're trying + # to reconnect. This is because the read thread may either be + # locked also trying to reconnect or we may be said thread already. + call_receive = not on_main_thread or \ + self._recv_thread is None or self._connect_lock.locked() + if not call_receive: # TODO This will be slightly troublesome if we allow # switching between constant read or not on the fly. @@ -330,22 +432,49 @@ class TelegramBareClient: while not all(x.confirm_received.is_set() for x in requests): sender.receive(update_state=self.updates) + except (PhoneMigrateError, NetworkMigrateError, + UserMigrateError) as e: + self._logger.debug( + 'DC error when invoking request, ' + 'attempting to reconnect at DC {}'.format(e.new_dc) + ) + + # TODO What happens with the background thread here? + # For normal use cases, this won't happen, because this will only + # be on the very first connection (not authorized, not running), + # but may be an issue for people who actually travel? + self._reconnect(new_dc=e.new_dc) + return self.invoke( + *requests, call_receive=call_receive, retries=(retries - 1) + ) + except TimeoutError: pass # We will just retry except ConnectionResetError: + if self._connect_lock.locked(): + # We are connecting and we don't want to reconnect there... + raise + self._logger.debug('Server disconnected us. Reconnecting and ' 'resending request...') + if sender != self._sender: + # TODO Try reconnecting forever too? sender.connect() else: - self._reconnect() + while self._user_connected and not self._reconnect(): + sleep(0.1) # Retry forever until we can send the request except FloodWaitError: sender.disconnect() self.disconnect() raise + finally: + if sender != self._sender: + sender.disconnect() + try: raise next(x.rpc_error for x in requests if x.rpc_error) except StopIteration: @@ -363,6 +492,13 @@ class TelegramBareClient: # Let people use client(SomeRequest()) instead client.invoke(...) __call__ = invoke + # Some really basic functionality + + def is_user_authorized(self): + """Has the user been authorized yet + (code request sent and confirmed)?""" + return self._authorized + # endregion # region Uploading media @@ -388,10 +524,10 @@ class TelegramBareClient: Default values for the optional parameters if left as None are: part_size_kb = get_appropriated_part_size(file_size) - file_name = path.basename(file_path) + file_name = os.path.basename(file_path) """ if isinstance(file, str): - file_size = path.getsize(file) + file_size = os.path.getsize(file) elif isinstance(file, bytes): file_size = len(file) else: @@ -447,7 +583,7 @@ class TelegramBareClient: # Set a default file name if None was specified if not file_name: if isinstance(file, str): - file_name = path.basename(file) + file_name = os.path.basename(file) else: file_name = str(file_id) @@ -544,3 +680,73 @@ class TelegramBareClient: f.close() # endregion + + # region Updates handling + + def sync_updates(self): + """Synchronizes self.updates to their initial state. Will be + called automatically on connection if self.updates.enabled = True, + otherwise it should be called manually after enabling updates. + """ + self.updates.process(self(GetStateRequest())) + + def add_update_handler(self, handler): + """Adds an update handler (a function which takes a TLObject, + an update, as its parameter) and listens for updates""" + sync = not self.updates.handlers + self.updates.handlers.append(handler) + if sync: + self.sync_updates() + + def remove_update_handler(self, handler): + self.updates.handlers.remove(handler) + + def list_update_handlers(self): + return self.updates.handlers[:] + + # endregion + + # Constant read + + def _set_connected_and_authorized(self): + self._authorized = True + if self._recv_thread is None: + self._recv_thread = threading.Thread( + name='ReadThread', daemon=True, + target=self._recv_thread_impl + ) + self._recv_thread.start() + + # By using this approach, another thread will be + # created and started upon connection to constantly read + # from the other end. Otherwise, manual calls to .receive() + # must be performed. The MtProtoSender cannot be connected, + # or an error will be thrown. + # + # This way, sending and receiving will be completely independent. + def _recv_thread_impl(self): + while self._user_connected: + try: + if datetime.now() > self._last_ping + self._ping_delay: + self._sender.send(PingRequest( + int.from_bytes(os.urandom(8), 'big', signed=True) + )) + self._last_ping = datetime.now() + + self._sender.receive(update_state=self.updates) + except TimeoutError: + # No problem. + pass + except ConnectionResetError: + self._logger.debug('Server disconnected us. Reconnecting...') + while self._user_connected and not self._reconnect(): + sleep(0.1) # Retry forever, this is instant messaging + + except Exception as e: + # Unknown exception, pass it to the main thread + self.updates.set_error(e) + break + + self._recv_thread = None + + # endregion diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 9db87684..a28d2c62 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1,10 +1,7 @@ import os -import threading from datetime import datetime, timedelta from functools import lru_cache from mimetypes import guess_type -from threading import Thread -from time import sleep try: import socks @@ -15,12 +12,10 @@ from . import TelegramBareClient from . import helpers as utils from .errors import ( RPCError, UnauthorizedError, InvalidParameterError, PhoneCodeEmptyError, - PhoneMigrateError, NetworkMigrateError, UserMigrateError, PhoneCodeExpiredError, PhoneCodeHashEmptyError, PhoneCodeInvalidError ) -from .network import Connection, ConnectionMode, MtProtoSender -from .tl import Session, TLObject -from .tl.functions import PingRequest +from .network import ConnectionMode +from .tl import TLObject from .tl.functions.account import ( GetPasswordRequest ) @@ -35,9 +30,6 @@ from .tl.functions.messages import ( GetDialogsRequest, GetHistoryRequest, ReadHistoryRequest, SendMediaRequest, SendMessageRequest ) -from .tl.functions.updates import ( - GetStateRequest -) from .tl.functions.users import ( GetUsersRequest ) @@ -98,18 +90,6 @@ class TelegramClient(TelegramBareClient): system_lang_code = lang_code report_errors = True """ - if not api_id or not api_hash: - raise PermissionError( - "Your API ID or Hash cannot be empty or None. " - "Refer to Telethon's README.rst for more information.") - - # Determine what session object we have - if isinstance(session, str) or session is None: - session = Session.try_load_or_create_new(session) - elif not isinstance(session, Session): - raise ValueError( - 'The given session must be a str or a Session instance.') - super().__init__( session, api_id, api_hash, connection_mode=connection_mode, @@ -118,187 +98,16 @@ class TelegramClient(TelegramBareClient): timeout=timeout ) - # Used on connection - the user may modify these and reconnect - kwargs['app_version'] = kwargs.get('app_version', self.__version__) - for name, value in kwargs.items(): - if hasattr(self.session, name): - setattr(self.session, name, value) - - self._updates_thread = None + # Some fields to easy signing in self._phone_code_hash = None self._phone = None - # Despite the state of the real connection, keep track of whether - # the user has explicitly called .connect() or .disconnect() here. - # This information is required by the read thread, who will be the - # one attempting to reconnect on the background *while* the user - # doesn't explicitly call .disconnect(), thus telling it to stop - # retrying. The main thread, knowing there is a background thread - # attempting reconnection as soon as it happens, will just sleep. - self._user_connected = False - - # Save whether the user is authorized here (a.k.a. logged in) - self._authorized = False - - # Uploaded files cache so subsequent calls are instant - self._upload_cache = {} - - # Constantly read for results and updates from within the main client - self._recv_thread = None - - # Identifier of the main thread (the one that called .connect()). - # This will be used to create new connections from any other thread, - # so that requests can be sent in parallel. - self._main_thread_ident = None - - # Default PingRequest delay - self._last_ping = datetime.now() - self._ping_delay = timedelta(minutes=1) - - # endregion - - # region Connecting - - def connect(self, exported_auth=None): - """Connects to the Telegram servers, executing authentication if - required. Note that authenticating to the Telegram servers is - not the same as authenticating the desired user itself, which - may require a call (or several) to 'sign_in' for the first time. - - exported_auth is meant for internal purposes and can be ignored. - """ - self._main_thread_ident = threading.get_ident() - - if socks and self._recv_thread: - # Treat proxy errors specially since they're not related to - # Telegram itself, but rather to the proxy. If any happens on - # the read thread forward it to the main thread. - try: - ok = super().connect(exported_auth=exported_auth) - except socks.ProxyConnectionError as e: - ok = False - # Report the exception to the main thread - self.updates.set_error(e) - else: - ok = super().connect(exported_auth=exported_auth) - - if not ok: - return False - - self._user_connected = True - try: - self.sync_updates() - self._set_connected_and_authorized() - except UnauthorizedError: - self._authorized = False - - return True - - def disconnect(self): - """Disconnects from the Telegram server - and stops all the spawned threads""" - self._user_connected = False - self._recv_thread = None - - # This will trigger a "ConnectionResetError", usually, the background - # thread would try restarting the connection but since the - # ._recv_thread = None, it knows it doesn't have to. - super().disconnect() - - # Also disconnect all the cached senders - for sender in self._cached_clients.values(): - sender.disconnect() - - self._cached_clients.clear() - - # endregion - - # region Working with different connections - - def _on_read_thread(self): - return self._recv_thread is not None and \ - threading.get_ident() == self._recv_thread.ident - # endregion # region Telegram requests functions - def invoke(self, *requests, **kwargs): - """Invokes (sends) one or several MTProtoRequest and returns - (receives) their result. An optional named 'retries' parameter - can be used, indicating how many times it should retry. - """ - # This is only valid when the read thread is reconnecting, - # that is, the connection lock is locked. - on_read_thread = self._on_read_thread() - if on_read_thread and not self._connect_lock.locked(): - return # Just ignore, we would be raising and crashing the thread - - self.updates.check_error() - - # Determine the sender to be used (main or a new connection) - # TODO Polish this so it's nicer - on_main_thread = threading.get_ident() == self._main_thread_ident - if on_main_thread or on_read_thread: - sender = self._sender - else: - conn = Connection( - self.session.server_address, self.session.port, - mode=self._sender.connection._mode, - proxy=self._sender.connection.conn.proxy, - timeout=self._sender.connection.get_timeout() - ) - sender = MtProtoSender(self.session, conn) - sender.connect() - - try: - # We should call receive from this thread if there's no background - # thread reading or if the server disconnected us and we're trying - # to reconnect. This is because the read thread may either be - # locked also trying to reconnect or we may be said thread already. - call_receive = not on_main_thread or \ - self._recv_thread is None or self._connect_lock.locked() - - return super().invoke( - *requests, - call_receive=call_receive, - retries=kwargs.get('retries', 5), - sender=sender - ) - - except (PhoneMigrateError, NetworkMigrateError, UserMigrateError) as e: - self._logger.debug('DC error when invoking request, ' - 'attempting to reconnect at DC {}' - .format(e.new_dc)) - - # TODO What happens with the background thread here? - # For normal use cases, this won't happen, because this will only - # be on the very first connection (not authorized, not running), - # but may be an issue for people who actually travel? - self._reconnect(new_dc=e.new_dc) - return self.invoke(*requests) - - except ConnectionResetError as e: - if self._connect_lock.locked(): - # We are connecting and we don't want to reconnect there... - raise - while self._user_connected and not self._reconnect(): - sleep(0.1) # Retry forever until we can send the request - - finally: - if sender != self._sender: - sender.disconnect() - - # Let people use client(SomeRequest()) instead client.invoke(...) - __call__ = invoke - # region Authorization requests - def is_user_authorized(self): - """Has the user been authorized yet - (code request sent and confirmed)?""" - return self._authorized - def send_code_request(self, phone): """Sends a code request to the specified phone number""" if isinstance(phone, int): @@ -992,73 +801,3 @@ class TelegramClient(TelegramBareClient): ) # endregion - - # region Updates handling - - def sync_updates(self): - """Synchronizes self.updates to their initial state. Will be - called automatically on connection if self.updates.enabled = True, - otherwise it should be called manually after enabling updates. - """ - self.updates.process(self(GetStateRequest())) - - def add_update_handler(self, handler): - """Adds an update handler (a function which takes a TLObject, - an update, as its parameter) and listens for updates""" - sync = not self.updates.handlers - self.updates.handlers.append(handler) - if sync: - self.sync_updates() - - def remove_update_handler(self, handler): - self.updates.handlers.remove(handler) - - def list_update_handlers(self): - return self.updates.handlers[:] - - # endregion - - # Constant read - - def _set_connected_and_authorized(self): - self._authorized = True - if self._recv_thread is None: - self._recv_thread = Thread( - name='ReadThread', daemon=True, - target=self._recv_thread_impl - ) - self._recv_thread.start() - - # By using this approach, another thread will be - # created and started upon connection to constantly read - # from the other end. Otherwise, manual calls to .receive() - # must be performed. The MtProtoSender cannot be connected, - # or an error will be thrown. - # - # This way, sending and receiving will be completely independent. - def _recv_thread_impl(self): - while self._user_connected: - try: - if datetime.now() > self._last_ping + self._ping_delay: - self._sender.send(PingRequest( - int.from_bytes(os.urandom(8), 'big', signed=True) - )) - self._last_ping = datetime.now() - - self._sender.receive(update_state=self.updates) - except TimeoutError: - # No problem. - pass - except ConnectionResetError: - self._logger.debug('Server disconnected us. Reconnecting...') - while self._user_connected and not self._reconnect(): - sleep(0.1) # Retry forever, this is instant messaging - - except Exception as e: - # Unknown exception, pass it to the main thread - self.updates.set_error(e) - break - - self._recv_thread = None - - # endregion From b87a798dd5678b5e2b71f7b572f5ff2b59f3edde Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 10:12:01 +0200 Subject: [PATCH 092/121] Spawn new worker threads to handle updates instead using ReadThread --- telethon/update_state.py | 75 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/telethon/update_state.py b/telethon/update_state.py index 4f3ebce7..314511d6 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -1,6 +1,7 @@ +import logging from collections import deque from datetime import datetime -from threading import RLock, Event +from threading import RLock, Event, Thread from .tl import types as tl @@ -11,14 +12,24 @@ class UpdateState: """ def __init__(self, polling): self._polling = polling + self._workers = 4 + self._worker_threads = [] + self.handlers = [] self._updates_lock = RLock() self._updates_available = Event() self._updates = deque() + self._logger = logging.getLogger(__name__) + # https://core.telegram.org/api/updates self._state = tl.updates.State(0, 0, datetime.now(), 0, 0) + # TODO Rename "polling" to some other variable + # that signifies "running background threads". + if polling: + self._setup_workers() + def can_poll(self): """Returns True if a call to .poll() won't lock""" return self._updates_available.is_set() @@ -39,17 +50,74 @@ class UpdateState: return update + # TODO How should this be handled with background worker threads? def get_polling(self): return self._polling def set_polling(self, polling): self._polling = polling - if not polling: + if polling: + self._setup_workers() + else: with self._updates_lock: self._updates.clear() + self._stop_workers() polling = property(fget=get_polling, fset=set_polling) + def get_workers(self): + return self._workers + + def set_workers(self, n): + self._stop_workers() + self._workers = n + self._setup_workers() + + workers = property(fget=get_workers, fset=set_workers) + + def _stop_workers(self): + """Raises "StopIterationException" on the worker threads to stop them, + and also clears all of them off the list + """ + if self._worker_threads: + pass + + self.set_error(StopIteration()) + for t in self._worker_threads: + t.join() + + self._worker_threads.clear() + + def _setup_workers(self): + if self._worker_threads: + # There already are workers + return + + for i in range(self._workers): + thread = Thread( + target=UpdateState._worker_loop, + name='UpdateWorker{}'.format(i), + daemon=True, + args=(self, i) + ) + self._worker_threads.append(thread) + thread.start() + + def _worker_loop(self, wid): + while True: + try: + update = self.poll() + # TODO Maybe people can add different handlers per update type + for handler in self.handlers: + handler(update) + except StopIteration: + break + except Exception as e: + # We don't want to crash a worker thread due to any reason + self._logger.debug( + '[ERROR] Unhandled exception on worker {}'.format(wid), e + ) + def set_error(self, error): """Sets an error, so that the next call to .poll() will raise it. Can be (and is) used to pass exceptions between threads. @@ -85,6 +153,3 @@ class UpdateState: if self._polling: self._updates.append(update) self._updates_available.set() - - for handler in self.handlers: - handler(update) From 9560bcc3240b7f94006f9476d79e49f2dad71f05 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 10:24:54 +0200 Subject: [PATCH 093/121] Remove "if background thread" check as it isn't exposed anymore --- telethon/telegram_bare_client.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 38fd1555..e9eae399 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -373,12 +373,6 @@ class TelegramBareClient: The invoke will be retried up to 'retries' times before raising ValueError(). """ - # This is only valid when the read thread is reconnecting, - # that is, the connection lock is locked. - on_read_thread = self._on_read_thread() - if on_read_thread and not self._connect_lock.locked(): - return # Just ignore, we would be raising and crashing the thread - # Any error from a background thread will be "posted" and checked here self.updates.check_error() @@ -392,7 +386,7 @@ class TelegramBareClient: # Determine the sender to be used (main or a new connection) # TODO Polish this so it's nicer on_main_thread = threading.get_ident() == self._main_thread_ident - if on_main_thread or on_read_thread: + if on_main_thread or self._on_read_thread(): sender = self._sender else: conn = Connection( From 72b7e99222053e3a252a517dffa84f19751983e3 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 10:59:33 +0200 Subject: [PATCH 094/121] Ensure the worker threads have updates once they acquire the lock --- telethon/update_state.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/telethon/update_state.py b/telethon/update_state.py index 314511d6..9239bd03 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -1,4 +1,5 @@ import logging +import threading from collections import deque from datetime import datetime from threading import RLock, Event, Thread @@ -41,6 +42,9 @@ class UpdateState: self._updates_available.wait() with self._updates_lock: + if not self._updates_available.is_set(): + return + update = self._updates.popleft() if not self._updates: self._updates_available.clear() From 7cef5885fa35d92a8b4c276e80a7d56a223e5ee2 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 11:17:31 +0200 Subject: [PATCH 095/121] Rename process_updates/polling to workers --- telethon/telegram_bare_client.py | 4 +-- telethon/telegram_client.py | 14 ++++---- telethon/update_state.py | 61 ++++++++++++-------------------- 3 files changed, 32 insertions(+), 47 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index e9eae399..1a6cfebd 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -67,7 +67,7 @@ class TelegramBareClient: def __init__(self, session, api_id, api_hash, connection_mode=ConnectionMode.TCP_FULL, proxy=None, - process_updates=False, + update_workers=None, timeout=timedelta(seconds=5), **kwargs): """Refer to TelegramClient.__init__ for docs on this method""" @@ -108,7 +108,7 @@ class TelegramBareClient: # This member will process updates if enabled. # One may change self.updates.enabled at any later point. - self.updates = UpdateState(process_updates) + self.updates = UpdateState(workers=update_workers) # Used on connection - the user may modify these and reconnect kwargs['app_version'] = kwargs.get('app_version', self.__version__) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index a28d2c62..b78fc68d 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -57,7 +57,7 @@ class TelegramClient(TelegramBareClient): def __init__(self, session, api_id, api_hash, connection_mode=ConnectionMode.TCP_FULL, proxy=None, - process_updates=False, + update_workers=None, timeout=timedelta(seconds=5), **kwargs): """Initializes the Telegram client with the specified API ID and Hash. @@ -71,11 +71,11 @@ class TelegramClient(TelegramBareClient): This will only affect how messages are sent over the network and how much processing is required before sending them. - If 'process_updates' is set to True, incoming updates will be - processed and you must manually call 'self.updates.poll()' from - another thread to retrieve the saved update objects, or your - memory will fill with these. You may modify the value of - 'self.updates.polling' at any later point. + The integer 'update_workers' represents depending on its value: + is None: Updates will *not* be stored in memory. + = 0: Another thread is responsible for calling self.updates.poll() + > 0: 'update_workers' background threads will be spawned, any + any of them will invoke all the self.updates.handlers. Despite the value of 'process_updates', if you later call '.add_update_handler(...)', updates will also be processed @@ -94,7 +94,7 @@ class TelegramClient(TelegramBareClient): session, api_id, api_hash, connection_mode=connection_mode, proxy=proxy, - process_updates=process_updates, + update_workers=update_workers, timeout=timeout ) diff --git a/telethon/update_state.py b/telethon/update_state.py index 9239bd03..cced0e1c 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -1,5 +1,4 @@ import logging -import threading from collections import deque from datetime import datetime from threading import RLock, Event, Thread @@ -11,9 +10,15 @@ class UpdateState: """Used to hold the current state of processed updates. To retrieve an update, .poll() should be called. """ - def __init__(self, polling): - self._polling = polling - self._workers = 4 + def __init__(self, workers=None): + """ + :param workers: This integer parameter has three possible cases: + workers is None: Updates will *not* be stored on self. + workers = 0: Another thread is responsible for calling self.poll() + workers > 0: 'workers' background threads will be spawned, any + any of them will invoke all the self.handlers. + """ + self._workers = workers self._worker_threads = [] self.handlers = [] @@ -25,11 +30,7 @@ class UpdateState: # https://core.telegram.org/api/updates self._state = tl.updates.State(0, 0, datetime.now(), 0, 0) - - # TODO Rename "polling" to some other variable - # that signifies "running background threads". - if polling: - self._setup_workers() + self._setup_workers() def can_poll(self): """Returns True if a call to .poll() won't lock""" @@ -37,9 +38,6 @@ class UpdateState: def poll(self): """Polls an update or blocks until an update object is available""" - if not self._polling: - raise ValueError('Updates are not being polled hence not saved.') - self._updates_available.wait() with self._updates_lock: if not self._updates_available.is_set(): @@ -54,28 +52,19 @@ class UpdateState: return update - # TODO How should this be handled with background worker threads? - def get_polling(self): - return self._polling - - def set_polling(self, polling): - self._polling = polling - if polling: - self._setup_workers() - else: - with self._updates_lock: - self._updates.clear() - self._stop_workers() - - polling = property(fget=get_polling, fset=set_polling) - def get_workers(self): return self._workers def set_workers(self, n): + """Changes the number of workers running. + If 'n is None', clears all pending updates from memory. + """ self._stop_workers() self._workers = n - self._setup_workers() + if n is None: + self._updates.clear() + else: + self._setup_workers() workers = property(fget=get_workers, fset=set_workers) @@ -83,9 +72,6 @@ class UpdateState: """Raises "StopIterationException" on the worker threads to stop them, and also clears all of them off the list """ - if self._worker_threads: - pass - self.set_error(StopIteration()) for t in self._worker_threads: t.join() @@ -93,8 +79,8 @@ class UpdateState: self._worker_threads.clear() def _setup_workers(self): - if self._worker_threads: - # There already are workers + if self._worker_threads or not self._workers: + # There already are workers, or workers is None or 0. Do nothing. return for i in range(self._workers): @@ -141,8 +127,8 @@ class UpdateState: """Processes an update object. This method is normally called by the library itself. """ - if not self._polling and not self.handlers: - return + if self._workers is None: + return # No processing needs to be done if nobody's working with self._updates_lock: if isinstance(update, tl.updates.State): @@ -154,6 +140,5 @@ class UpdateState: return # We already handled this update self._state.pts = pts - if self._polling: - self._updates.append(update) - self._updates_available.set() + self._updates.append(update) + self._updates_available.set() From a3ae56ca9e670d606785f0a93dbaf052ed32d5f2 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 11:21:07 +0200 Subject: [PATCH 096/121] Use a timeout when worker threads are polling --- telethon/update_state.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/telethon/update_state.py b/telethon/update_state.py index cced0e1c..4402fb14 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -10,6 +10,8 @@ class UpdateState: """Used to hold the current state of processed updates. To retrieve an update, .poll() should be called. """ + WORKER_POLL_TIMEOUT = 5.0 # Avoid waiting forever on the workers + def __init__(self, workers=None): """ :param workers: This integer parameter has three possible cases: @@ -36,9 +38,14 @@ class UpdateState: """Returns True if a call to .poll() won't lock""" return self._updates_available.is_set() - def poll(self): - """Polls an update or blocks until an update object is available""" - self._updates_available.wait() + def poll(self, timeout=None): + """Polls an update or blocks until an update object is available. + If 'timeout is not None', it should be a floating point value, + and the method will 'return None' if waiting times out. + """ + if not self._updates_available.wait(timeout=timeout): + return + with self._updates_lock: if not self._updates_available.is_set(): return @@ -96,10 +103,11 @@ class UpdateState: def _worker_loop(self, wid): while True: try: - update = self.poll() + update = self.poll(timeout=UpdateState.WORKER_POLL_TIMEOUT) # TODO Maybe people can add different handlers per update type - for handler in self.handlers: - handler(update) + if update: + for handler in self.handlers: + handler(update) except StopIteration: break except Exception as e: From 61033b2f564171302868baa5b021f96be3b54c68 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 11:28:15 +0200 Subject: [PATCH 097/121] Allow disabling spawning a second thread --- telethon/telegram_bare_client.py | 7 +++++-- telethon/telegram_client.py | 14 +++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 1a6cfebd..251c927e 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -68,6 +68,7 @@ class TelegramBareClient: connection_mode=ConnectionMode.TCP_FULL, proxy=None, update_workers=None, + spawn_read_thread=True, timeout=timedelta(seconds=5), **kwargs): """Refer to TelegramClient.__init__ for docs on this method""" @@ -131,7 +132,9 @@ class TelegramBareClient: # Uploaded files cache so subsequent calls are instant self._upload_cache = {} - # Constantly read for results and updates from within the main client + # Constantly read for results and updates from within the main client, + # if the user has left enabled such option. + self._spawn_read_thread = spawn_read_thread self._recv_thread = None # Identifier of the main thread (the one that called .connect()). @@ -704,7 +707,7 @@ class TelegramBareClient: def _set_connected_and_authorized(self): self._authorized = True - if self._recv_thread is None: + if self._spawn_read_thread and self._recv_thread is None: self._recv_thread = threading.Thread( name='ReadThread', daemon=True, target=self._recv_thread_impl diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index b78fc68d..a053ed95 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -59,6 +59,7 @@ class TelegramClient(TelegramBareClient): proxy=None, update_workers=None, timeout=timedelta(seconds=5), + spawn_read_thread=True, **kwargs): """Initializes the Telegram client with the specified API ID and Hash. @@ -77,9 +78,15 @@ class TelegramClient(TelegramBareClient): > 0: 'update_workers' background threads will be spawned, any any of them will invoke all the self.updates.handlers. - Despite the value of 'process_updates', if you later call - '.add_update_handler(...)', updates will also be processed - and the update objects will be passed to the handlers you added. + If 'spawn_read_thread', a background thread will be started once + an authorized user has been logged in to Telegram to read items + (such as updates and responses) from the network as soon as they + occur, which will speed things up. + + If you don't want to spawn any additional threads, pending updates + will be read and processed accordingly after invoking a request + and not immediately. This is useful if you don't care about updates + at all and have set 'update_workers=None'. If more named arguments are provided as **kwargs, they will be used to update the Session instance. Most common settings are: @@ -95,6 +102,7 @@ class TelegramClient(TelegramBareClient): connection_mode=connection_mode, proxy=proxy, update_workers=update_workers, + spawn_read_thread=spawn_read_thread, timeout=timeout ) From 003e23123960b8877960cd814ebd14b50dbba88b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 11:45:35 +0200 Subject: [PATCH 098/121] Attempt at cleaning up TelegramBareClient.invoke() --- telethon/telegram_bare_client.py | 45 +++++++++++++++++++------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 251c927e..010ae086 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -91,6 +91,10 @@ class TelegramBareClient: if self.api_id < 20: # official apps must use obfuscated connection_mode = ConnectionMode.TCP_OBFUSCATED + # This is the main sender, which will be used from the thread + # that calls .connect(). Every other thread will spawn a new + # temporary connection. The connection on this one is always + # kept open so Telegram can send us updates. self._sender = MtProtoSender(self.session, Connection( self.session.server_address, self.session.port, mode=connection_mode, proxy=proxy, timeout=timeout @@ -370,7 +374,7 @@ class TelegramBareClient: # region Invoking Telegram requests - def invoke(self, *requests, call_receive=True, retries=5): + def invoke(self, *requests, retries=5): """Invokes (sends) a MTProtoRequest and returns (receives) its result. The invoke will be retried up to 'retries' times before raising @@ -383,9 +387,6 @@ class TelegramBareClient: x.content_related for x in requests): raise ValueError('You can only invoke requests, not types!') - if retries <= 0: - raise ValueError('Number of retries reached 0.') - # Determine the sender to be used (main or a new connection) # TODO Polish this so it's nicer on_main_thread = threading.get_ident() == self._main_thread_ident @@ -401,6 +402,25 @@ class TelegramBareClient: sender = MtProtoSender(self.session, conn) sender.connect() + # We should call receive from this thread if there's no background + # thread reading or if the server disconnected us and we're trying + # to reconnect. This is because the read thread may either be + # locked also trying to reconnect or we may be said thread already. + call_receive = not on_main_thread or \ + self._recv_thread is None or self._connect_lock.locked() + try: + for _ in range(retries): + result = self._invoke(sender, call_receive, *requests) + if result: + return result + + if retries <= 0: + raise ValueError('Number of retries reached 0.') + finally: + if sender != self._sender: + sender.disconnect() # Close temporary connections + + def _invoke(self, sender, call_receive, *requests): try: # Ensure that we start with no previous errors (i.e. resending) for x in requests: @@ -409,13 +429,6 @@ class TelegramBareClient: sender.send(*requests) - # We should call receive from this thread if there's no background - # thread reading or if the server disconnected us and we're trying - # to reconnect. This is because the read thread may either be - # locked also trying to reconnect or we may be said thread already. - call_receive = not on_main_thread or \ - self._recv_thread is None or self._connect_lock.locked() - if not call_receive: # TODO This will be slightly troublesome if we allow # switching between constant read or not on the fly. @@ -441,9 +454,7 @@ class TelegramBareClient: # be on the very first connection (not authorized, not running), # but may be an issue for people who actually travel? self._reconnect(new_dc=e.new_dc) - return self.invoke( - *requests, call_receive=call_receive, retries=(retries - 1) - ) + return self._invoke(sender, call_receive, *requests) except TimeoutError: pass # We will just retry @@ -477,10 +488,8 @@ class TelegramBareClient: except StopIteration: if any(x.result is None for x in requests): # "A container may only be accepted or - # rejected by the other party as a whole." - return self.invoke( - *requests, call_receive=call_receive, retries=(retries - 1) - ) + # rejected by the other party as a whole." + return None elif len(requests) == 1: return requests[0].result else: From 5da300ca8412686c0d6cfff2755eb1bc15cab7e7 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 11:49:38 +0200 Subject: [PATCH 099/121] Make MtProtoSender not thread-safe Rationale: a new connection should be spawned if one desires to send and receive requests in parallel, which would otherwise cause one of either threads to lock. --- telethon/network/mtproto_sender.py | 41 ++++++++++++++---------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 83ac4a43..772aa213 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -1,7 +1,6 @@ import gzip import logging import struct -from threading import RLock from .. import helpers as utils from ..crypto import AES @@ -20,7 +19,12 @@ logging.getLogger(__name__).addHandler(logging.NullHandler()) class MtProtoSender: """MTProto Mobile Protocol sender - (https://core.telegram.org/mtproto/description) + (https://core.telegram.org/mtproto/description). + + Note that this class is not thread-safe, and calling send/receive + from two or more threads at the same time is undefined behaviour. + Rationale: a new connection should be spawned to send/receive requests + in parallel, so thread-safety (hence locking) isn't needed. """ def __init__(self, session, connection): @@ -37,11 +41,6 @@ class MtProtoSender: # Requests (as msg_id: Message) sent waiting to be received self._pending_receive = {} - # Sending and receiving are independent, but two threads cannot - # send or receive at the same time no matter what. - self._send_lock = RLock() - self._recv_lock = RLock() - def connect(self): """Connects to the server""" self.connection.connect() @@ -93,19 +92,18 @@ class MtProtoSender: Any unhandled object (likely updates) will be passed to update_state.process(TLObject). """ - with self._recv_lock: - try: - body = self.connection.recv() - except (BufferError, InvalidChecksumError): - # TODO BufferError, we should spot the cause... - # "No more bytes left"; something wrong happened, clear - # everything to be on the safe side, or: - # - # "This packet should be skipped"; since this may have - # been a result for a request, invalidate every request - # and just re-invoke them to avoid problems - self._clear_all_pending() - return + try: + body = self.connection.recv() + except (BufferError, InvalidChecksumError): + # TODO BufferError, we should spot the cause... + # "No more bytes left"; something wrong happened, clear + # everything to be on the safe side, or: + # + # "This packet should be skipped"; since this may have + # been a result for a request, invalidate every request + # and just re-invoke them to avoid problems + self._clear_all_pending() + return message, remote_msg_id, remote_seq = self._decode_msg(body) with BinaryReader(message) as reader: @@ -128,8 +126,7 @@ class MtProtoSender: cipher_text = AES.encrypt_ige(plain_text, key, iv) result = key_id + msg_key + cipher_text - with self._send_lock: - self.connection.send(result) + self.connection.send(result) def _decode_msg(self, body): """Decodes an received encrypted message body bytes""" From 0a567fcd7c39dfb3d8e590d37c38ac0c248bdd1d Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 12:08:06 +0200 Subject: [PATCH 100/121] Make creating a new sender cleaner --- telethon/network/connection.py | 7 +++++++ telethon/network/mtproto_sender.py | 4 ++++ telethon/telegram_bare_client.py | 9 +-------- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/telethon/network/connection.py b/telethon/network/connection.py index d333b8ff..28c548eb 100644 --- a/telethon/network/connection.py +++ b/telethon/network/connection.py @@ -130,6 +130,13 @@ class Connection: def close(self): self.conn.close() + def clone(self): + """Creates a copy of this Connection""" + return Connection(self.ip, self.port, + mode=self._mode, + proxy=self.conn.proxy, + timeout=self.conn.timeout) + # region Receive message implementations def recv(self): diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 772aa213..6558a20c 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -54,6 +54,10 @@ class MtProtoSender: self._need_confirmation.clear() self._clear_all_pending() + def clone(self): + """Creates a copy of this MtProtoSender as a new connection""" + return MtProtoSender(self.session, self.connection.clone()) + # region Send and receive def send(self, *requests): diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 010ae086..04398826 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -388,18 +388,11 @@ class TelegramBareClient: raise ValueError('You can only invoke requests, not types!') # Determine the sender to be used (main or a new connection) - # TODO Polish this so it's nicer on_main_thread = threading.get_ident() == self._main_thread_ident if on_main_thread or self._on_read_thread(): sender = self._sender else: - conn = Connection( - self.session.server_address, self.session.port, - mode=self._sender.connection._mode, - proxy=self._sender.connection.conn.proxy, - timeout=self._sender.connection.get_timeout() - ) - sender = MtProtoSender(self.session, conn) + sender = self._sender.clone() sender.connect() # We should call receive from this thread if there's no background From 18e485ded2cae368ed86bb173df5c9c2a04abada Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 15:53:47 +0200 Subject: [PATCH 101/121] Set default TelegramBareClient behaviour to not spawn ReadThread --- telethon/telegram_bare_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 04398826..4355be3c 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -68,7 +68,7 @@ class TelegramBareClient: connection_mode=ConnectionMode.TCP_FULL, proxy=None, update_workers=None, - spawn_read_thread=True, + spawn_read_thread=False, timeout=timedelta(seconds=5), **kwargs): """Refer to TelegramClient.__init__ for docs on this method""" From 8ecd2c2e060db034900a18e434fa89a3e3334821 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 16:11:16 +0200 Subject: [PATCH 102/121] Call .sync_updates on .connect iff exported_auth is None Calling this method on exported clients would trigger a UserMigrateError because it was being used on a non-native data center. For .connects like this, don't attempt syncing updates. --- telethon/telegram_bare_client.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 4355be3c..cf26c465 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -201,13 +201,16 @@ class TelegramBareClient: self(GetConfigRequest()).dc_options # Connection was successful! Try syncing the update state + # IF we don't have an exported authorization (hence we're + # not in our NATIVE data center or we'd get UserMigrateError) # to also assert whether the user is logged in or not. self._user_connected = True - try: - self.sync_updates() - self._set_connected_and_authorized() - except UnauthorizedError: - self._authorized = False + if not exported_auth: + try: + self.sync_updates() + self._set_connected_and_authorized() + except UnauthorizedError: + self._authorized = False return True From c1c6df9fd0bfc9a20c7eb6864c697d9f38ce81b6 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 16:18:16 +0200 Subject: [PATCH 103/121] Fix invoke not raising ValueError when retries reach 0 --- telethon/telegram_bare_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index cf26c465..d027a358 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -410,8 +410,7 @@ class TelegramBareClient: if result: return result - if retries <= 0: - raise ValueError('Number of retries reached 0.') + raise ValueError('Number of retries reached 0.') finally: if sender != self._sender: sender.disconnect() # Close temporary connections From a35c4b15dba34415617ee798d4d38c993f6ab97a Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 16:32:10 +0200 Subject: [PATCH 104/121] Cache exported Sessions instead whole clients --- telethon/telegram_bare_client.py | 97 ++++++++++++++++---------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index d027a358..f7a3933d 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -106,10 +106,10 @@ class TelegramBareClient: # we only want one to actually perform the reconnection. self._connect_lock = Lock() - # Cache "exported" senders 'dc_id: TelegramBareClient' and - # their corresponding sessions not to recreate them all - # the time since it's a (somewhat expensive) process. - self._cached_clients = {} + # Cache "exported" sessions as 'dc_id: Session' not to recreate + # them all the time since generating a new key is a relatively + # expensive operation. + self._exported_sessions = {} # This member will process updates if enabled. # One may change self.updates.enabled at any later point. @@ -154,14 +154,22 @@ class TelegramBareClient: # region Connecting - def connect(self, exported_auth=None): + def connect(self, _exported_auth=None, _sync_updates=True): """Connects to the Telegram servers, executing authentication if required. Note that authenticating to the Telegram servers is not the same as authenticating the desired user itself, which may require a call (or several) to 'sign_in' for the first time. - If 'exported_auth' is not None, it will be used instead to + Note that the optional parameters are meant for internal use. + + If '_exported_auth' is not None, it will be used instead to determine the authorization key for the current session. + + If '_sync_updates', sync_updates() will be called and a + second thread will be started if necessary. Note that this + will FAIL if the client is not connected to the user's + native data center, raising a "UserMigrateError", and + calling .disconnect() in the process. """ self._main_thread_ident = threading.get_ident() @@ -183,17 +191,17 @@ class TelegramBareClient: init_connection = self.session.layer != LAYER if init_connection: - if exported_auth is not None: + if _exported_auth is not None: self._init_connection(ImportAuthorizationRequest( - exported_auth.id, exported_auth.bytes + _exported_auth.id, _exported_auth.bytes )) else: TelegramBareClient._dc_options = \ self._init_connection(GetConfigRequest()).dc_options - elif exported_auth is not None: + elif _exported_auth is not None: self(ImportAuthorizationRequest( - exported_auth.id, exported_auth.bytes + _exported_auth.id, _exported_auth.bytes )) if TelegramBareClient._dc_options is None: @@ -201,11 +209,11 @@ class TelegramBareClient: self(GetConfigRequest()).dc_options # Connection was successful! Try syncing the update state - # IF we don't have an exported authorization (hence we're - # not in our NATIVE data center or we'd get UserMigrateError) + # UNLESS '_sync_updates' is False (we probably are in + # another data center and this would raise UserMigrateError) # to also assert whether the user is logged in or not. self._user_connected = True - if not exported_auth: + if _sync_updates: try: self.sync_updates() self._set_connected_and_authorized() @@ -218,7 +226,10 @@ class TelegramBareClient: # This is fine, probably layer migration self._logger.debug('Found invalid item, probably migrating', e) self.disconnect() - return self.connect(exported_auth=exported_auth) + return self.connect( + _exported_auth=_exported_auth, + _sync_updates=_sync_updates + ) except (RPCError, ConnectionError) as error: # Probably errors from the previous session, ignore them @@ -258,11 +269,8 @@ class TelegramBareClient: # ._recv_thread = None, it knows it doesn't have to. self._sender.disconnect() - # Also disconnect all the cached senders - for sender in self._cached_clients.values(): - sender.disconnect() - - self._cached_clients.clear() + # TODO Shall we clear the _exported_sessions, or may be reused? + pass def _reconnect(self, new_dc=None): """If 'new_dc' is not set, only a call to .connect() will be made @@ -324,30 +332,22 @@ class TelegramBareClient: TelegramBareClient._dc_options = self(GetConfigRequest()).dc_options return self._get_dc(dc_id, ipv6=ipv6, cdn=cdn) - def _get_exported_client(self, dc_id, - init_connection=False, - bypass_cache=False): - """Gets a cached exported TelegramBareClient for the desired DC. + def _get_exported_client(self, dc_id): + """Creates and connects a new TelegramBareClient for the desired DC. - If it's the first time retrieving the TelegramBareClient, the - current authorization is exported to the new DC so that - it can be used there, and the connection is initialized. - - If after using the sender a ConnectionResetError is raised, - this method should be called again with init_connection=True - in order to perform the reconnection. - - If bypass_cache is True, a new client will be exported and - it will not be cached. + If it's the first time calling the method with a given dc_id, + a new session will be first created, and its auth key generated. + Exporting/Importing the authorization will also be done so that + the auth is bound with the key. """ # Thanks badoualy/kotlogram on /telegram/api/DefaultTelegramClient.kt # for clearly showing how to export the authorization! ^^ - client = self._cached_clients.get(dc_id) - if client and not bypass_cache: - if init_connection: - client.reconnect() - return client + session = self._exported_sessions.get(dc_id) + if session: + export_auth = None # Already bound with the auth key else: + # TODO Add a lock, don't allow two threads to create an auth key + # for the same data center. dc = self._get_dc(dc_id) # Export the current authorization to the new DC. @@ -361,17 +361,15 @@ class TelegramBareClient: session = Session(self.session) session.server_address = dc.ip_address session.port = dc.port - client = TelegramBareClient( - session, self.api_id, self.api_hash, - proxy=self._sender.connection.conn.proxy, - timeout=self._sender.connection.get_timeout() - ) - client.connect(exported_auth=export_auth) + self._exported_sessions[dc_id] = session - if not bypass_cache: - # Don't go through this expensive process every time. - self._cached_clients[dc_id] = client - return client + client = TelegramBareClient( + session, self.api_id, self.api_hash, + proxy=self._sender.connection.conn.proxy, + timeout=self._sender.connection.get_timeout() + ) + client.connect(_exported_auth=export_auth, _sync_updates=False) + return client # endregion @@ -672,6 +670,9 @@ class TelegramBareClient: if progress_callback: progress_callback(f.tell(), file_size) finally: + if client != self: + client.disconnect() + if cdn_decrypter: try: cdn_decrypter.client.disconnect() From d28f370ab6bc2b7e409c6d87882cedd08cdb0bb3 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 17:51:07 +0200 Subject: [PATCH 105/121] Add ._get_cdn_client as alternative ._get_exported_client version --- telethon/crypto/cdn_decrypter.py | 32 ++++++--------------------- telethon/telegram_bare_client.py | 37 +++++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/telethon/crypto/cdn_decrypter.py b/telethon/crypto/cdn_decrypter.py index d6628d58..f4b0ef3e 100644 --- a/telethon/crypto/cdn_decrypter.py +++ b/telethon/crypto/cdn_decrypter.py @@ -10,7 +10,7 @@ from ..errors import CdnFileTamperedError class CdnDecrypter: """Used when downloading a file results in a 'FileCdnRedirect' to both prepare the redirect, decrypt the file as it downloads, and - ensure the file hasn't been tampered. + ensure the file hasn't been tampered. https://core.telegram.org/cdn """ def __init__(self, cdn_client, file_token, cdn_aes, cdn_file_hashes): self.client = cdn_client @@ -19,46 +19,26 @@ class CdnDecrypter: self.cdn_file_hashes = cdn_file_hashes @staticmethod - def prepare_decrypter(client, client_cls, cdn_redirect): + def prepare_decrypter(client, cdn_client, cdn_redirect): """Prepares a CDN decrypter, returning (decrypter, file data). - 'client' should be the original TelegramBareClient that - tried to download the file. - - 'client_cls' should be the class of the TelegramBareClient. + 'client' should be an existing client not connected to a CDN. + 'cdn_client' should be an already-connected TelegramBareClient + with the auth key already created. """ - # TODO Avoid the need for 'client_cls=TelegramBareClient' - # https://core.telegram.org/cdn cdn_aes = AESModeCTR( key=cdn_redirect.encryption_key, # 12 first bytes of the IV..4 bytes of the offset (0, big endian) iv=cdn_redirect.encryption_iv[:12] + bytes(4) ) - # Create a new client on said CDN - dc = client._get_dc(cdn_redirect.dc_id, cdn=True) - session = Session(client.session) - session.server_address = dc.ip_address - session.port = dc.port - cdn_client = client_cls( # Avoid importing TelegramBareClient - session, client.api_id, client.api_hash, - timeout=client._sender.connection.get_timeout() - ) - # This will make use of the new RSA keys for this specific CDN. - # # We assume that cdn_redirect.cdn_file_hashes are ordered by offset, # and that there will be enough of these to retrieve the whole file. - # - # This relies on the fact that TelegramBareClient._dc_options is - # static and it won't be called from this DC (it would fail). - cdn_client.connect() - - # CDN client is ready, create the resulting CdnDecrypter decrypter = CdnDecrypter( cdn_client, cdn_redirect.file_token, cdn_aes, cdn_redirect.cdn_file_hashes ) - cdn_file = client(GetCdnFileRequest( + cdn_file = cdn_client(GetCdnFileRequest( file_token=cdn_redirect.file_token, offset=cdn_redirect.cdn_file_hashes[0].offset, limit=cdn_redirect.cdn_file_hashes[0].limit diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index f7a3933d..dd075c9b 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -154,7 +154,7 @@ class TelegramBareClient: # region Connecting - def connect(self, _exported_auth=None, _sync_updates=True): + def connect(self, _exported_auth=None, _sync_updates=True, cdn=False): """Connects to the Telegram servers, executing authentication if required. Note that authenticating to the Telegram servers is not the same as authenticating the desired user itself, which @@ -170,6 +170,9 @@ class TelegramBareClient: will FAIL if the client is not connected to the user's native data center, raising a "UserMigrateError", and calling .disconnect() in the process. + + If 'cdn' is False, methods that are not allowed on such data + centers won't be invoked. """ self._main_thread_ident = threading.get_ident() @@ -195,7 +198,7 @@ class TelegramBareClient: self._init_connection(ImportAuthorizationRequest( _exported_auth.id, _exported_auth.bytes )) - else: + elif not cdn: TelegramBareClient._dc_options = \ self._init_connection(GetConfigRequest()).dc_options @@ -204,7 +207,7 @@ class TelegramBareClient: _exported_auth.id, _exported_auth.bytes )) - if TelegramBareClient._dc_options is None: + if TelegramBareClient._dc_options is None and not cdn: TelegramBareClient._dc_options = \ self(GetConfigRequest()).dc_options @@ -213,7 +216,7 @@ class TelegramBareClient: # another data center and this would raise UserMigrateError) # to also assert whether the user is logged in or not. self._user_connected = True - if _sync_updates: + if _sync_updates and not cdn: try: self.sync_updates() self._set_connected_and_authorized() @@ -347,6 +350,7 @@ class TelegramBareClient: export_auth = None # Already bound with the auth key else: # TODO Add a lock, don't allow two threads to create an auth key + # (when calling .connect() if there wasn't a previous session). # for the same data center. dc = self._get_dc(dc_id) @@ -371,6 +375,29 @@ class TelegramBareClient: client.connect(_exported_auth=export_auth, _sync_updates=False) return client + def _get_cdn_client(self, cdn_redirect): + """Similar to ._get_exported_client, but for CDNs""" + session = self._exported_sessions.get(cdn_redirect.dc_id) + if not session: + dc = self._get_dc(cdn_redirect.dc_id, cdn=True) + session = Session(self.session) + session.server_address = dc.ip_address + session.port = dc.port + self._exported_sessions[cdn_redirect.dc_id] = session + + client = TelegramBareClient( + session, self.api_id, self.api_hash, + proxy=self._sender.connection.conn.proxy, + timeout=self._sender.connection.get_timeout() + ) + + # This will make use of the new RSA keys for this specific CDN. + # + # This relies on the fact that TelegramBareClient._dc_options is + # static and it won't be called from this DC (it would fail). + client.connect(cdn=True) # Avoid invoking non-CDN specific methods + return client + # endregion # region Invoking Telegram requests @@ -651,7 +678,7 @@ class TelegramBareClient: if isinstance(result, FileCdnRedirect): cdn_decrypter, result = \ CdnDecrypter.prepare_decrypter( - client, TelegramBareClient, result + client, self._get_cdn_client(result), result ) except FileMigrateError as e: From 4cd7e1711e01c201bd268a7860d557fdedf481ef Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 17:56:42 +0200 Subject: [PATCH 106/121] Rename cdn parameter to _cdn --- telethon/telegram_bare_client.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index dd075c9b..54ebe095 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -154,7 +154,7 @@ class TelegramBareClient: # region Connecting - def connect(self, _exported_auth=None, _sync_updates=True, cdn=False): + def connect(self, _exported_auth=None, _sync_updates=True, _cdn=False): """Connects to the Telegram servers, executing authentication if required. Note that authenticating to the Telegram servers is not the same as authenticating the desired user itself, which @@ -171,7 +171,7 @@ class TelegramBareClient: native data center, raising a "UserMigrateError", and calling .disconnect() in the process. - If 'cdn' is False, methods that are not allowed on such data + If '_cdn' is False, methods that are not allowed on such data centers won't be invoked. """ self._main_thread_ident = threading.get_ident() @@ -198,7 +198,7 @@ class TelegramBareClient: self._init_connection(ImportAuthorizationRequest( _exported_auth.id, _exported_auth.bytes )) - elif not cdn: + elif not _cdn: TelegramBareClient._dc_options = \ self._init_connection(GetConfigRequest()).dc_options @@ -207,7 +207,7 @@ class TelegramBareClient: _exported_auth.id, _exported_auth.bytes )) - if TelegramBareClient._dc_options is None and not cdn: + if TelegramBareClient._dc_options is None and not _cdn: TelegramBareClient._dc_options = \ self(GetConfigRequest()).dc_options @@ -216,7 +216,7 @@ class TelegramBareClient: # another data center and this would raise UserMigrateError) # to also assert whether the user is logged in or not. self._user_connected = True - if _sync_updates and not cdn: + if _sync_updates and not _cdn: try: self.sync_updates() self._set_connected_and_authorized() @@ -231,7 +231,8 @@ class TelegramBareClient: self.disconnect() return self.connect( _exported_auth=_exported_auth, - _sync_updates=_sync_updates + _sync_updates=_sync_updates, + _cdn=_cdn ) except (RPCError, ConnectionError) as error: From 671ac1cdb7413688da81ca757a5d12f87872e92c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 18:02:08 +0200 Subject: [PATCH 107/121] Fix **kwargs not being passed to TelegramBareClient --- telethon/telegram_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index a053ed95..0e264a3e 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -103,7 +103,8 @@ class TelegramClient(TelegramBareClient): proxy=proxy, update_workers=update_workers, spawn_read_thread=spawn_read_thread, - timeout=timeout + timeout=timeout, + **kwargs ) # Some fields to easy signing in From b6bc9ac39b923073f7febc4fbe527f26d9b2e858 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 18:25:09 +0200 Subject: [PATCH 108/121] Rename ._connect_lock to ._reconnect_lock --- telethon/telegram_bare_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 54ebe095..d2e7595b 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -104,7 +104,7 @@ class TelegramBareClient: # Two threads may be calling reconnect() when the connection is lost, # we only want one to actually perform the reconnection. - self._connect_lock = Lock() + self._reconnect_lock = Lock() # Cache "exported" sessions as 'dc_id: Session' not to recreate # them all the time since generating a new key is a relatively @@ -287,7 +287,7 @@ class TelegramBareClient: """ if new_dc is None: # Assume we are disconnected due to some error, so connect again - with self._connect_lock: + with self._reconnect_lock: # Another thread may have connected again, so check that first if not self.is_connected(): return self.connect() @@ -428,8 +428,8 @@ class TelegramBareClient: # thread reading or if the server disconnected us and we're trying # to reconnect. This is because the read thread may either be # locked also trying to reconnect or we may be said thread already. - call_receive = not on_main_thread or \ - self._recv_thread is None or self._connect_lock.locked() + call_receive = not on_main_thread or self._recv_thread is None \ + or self._reconnect_lock.locked() try: for _ in range(retries): result = self._invoke(sender, call_receive, *requests) @@ -481,7 +481,7 @@ class TelegramBareClient: pass # We will just retry except ConnectionResetError: - if self._connect_lock.locked(): + if self._reconnect_lock.locked(): # We are connecting and we don't want to reconnect there... raise From 933ae01d854715a185edb258f0d3aef897299bf9 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 18:33:34 +0200 Subject: [PATCH 109/121] Change condition to perform automatic reconnection --- telethon/telegram_bare_client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index d2e7595b..d173a1c1 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -374,6 +374,7 @@ class TelegramBareClient: timeout=self._sender.connection.get_timeout() ) client.connect(_exported_auth=export_auth, _sync_updates=False) + client._authorized = True # We exported the auth, so we got auth return client def _get_cdn_client(self, cdn_redirect): @@ -396,7 +397,8 @@ class TelegramBareClient: # # This relies on the fact that TelegramBareClient._dc_options is # static and it won't be called from this DC (it would fail). - client.connect(cdn=True) # Avoid invoking non-CDN specific methods + client.connect(_cdn=True) # Avoid invoking non-CDN specific methods + client._authorized = self._authorized return client # endregion @@ -481,8 +483,9 @@ class TelegramBareClient: pass # We will just retry except ConnectionResetError: - if self._reconnect_lock.locked(): - # We are connecting and we don't want to reconnect there... + if not self._authorized or self._reconnect_lock.locked(): + # Only attempt reconnecting if we're authorized and not + # reconnecting already. raise self._logger.debug('Server disconnected us. Reconnecting and ' From 8c3c990e7415e4e72f9feaf3d0083745b1cfde9d Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 18:39:31 +0200 Subject: [PATCH 110/121] Remove UpdateState .set and .check error --- telethon/telegram_bare_client.py | 14 +++++++++----- telethon/update_state.py | 24 +++++++----------------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index d173a1c1..75928b69 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -411,9 +411,6 @@ class TelegramBareClient: The invoke will be retried up to 'retries' times before raising ValueError(). """ - # Any error from a background thread will be "posted" and checked here - self.updates.check_error() - if not all(isinstance(x, TLObject) and x.content_related for x in requests): raise ValueError('You can only invoke requests, not types!') @@ -775,9 +772,16 @@ class TelegramBareClient: while self._user_connected and not self._reconnect(): sleep(0.1) # Retry forever, this is instant messaging - except Exception as e: + except Exception as error: # Unknown exception, pass it to the main thread - self.updates.set_error(e) + self._logger.debug( + '[ERROR] Unknown error on the read thread, please report', + error + ) + # If something strange happens we don't want to enter an + # infinite loop where all we do is raise an exception, so + # add a little sleep to avoid the CPU usage going mad. + sleep(0.1) break self._recv_thread = None diff --git a/telethon/update_state.py b/telethon/update_state.py index 4402fb14..7e25549f 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -55,7 +55,7 @@ class UpdateState: self._updates_available.clear() if isinstance(update, Exception): - raise update # Some error was set through .set_error() + raise update # Some error was set through (surely StopIteration) return update @@ -79,7 +79,12 @@ class UpdateState: """Raises "StopIterationException" on the worker threads to stop them, and also clears all of them off the list """ - self.set_error(StopIteration()) + with self._updates_lock: + # Insert at the beginning so the very next poll causes an error + # TODO Should this reset the pts and such? + self._updates.appendleft(StopIteration()) + self._updates_available.set() + for t in self._worker_threads: t.join() @@ -116,21 +121,6 @@ class UpdateState: '[ERROR] Unhandled exception on worker {}'.format(wid), e ) - def set_error(self, error): - """Sets an error, so that the next call to .poll() will raise it. - Can be (and is) used to pass exceptions between threads. - """ - with self._updates_lock: - # Insert at the beginning so the very next poll causes an error - # TODO Should this reset the pts and such? - self._updates.appendleft(error) - self._updates_available.set() - - def check_error(self): - with self._updates_lock: - if self._updates and isinstance(self._updates[0], Exception): - raise self._updates.popleft() - def process(self, update): """Processes an update object. This method is normally called by the library itself. From f1bca0fd06caa38dc7bed5d50ed138c5f26bd29c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 18:44:37 +0200 Subject: [PATCH 111/121] Fix setting None update workers not causing all threads to stop --- telethon/update_state.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/telethon/update_state.py b/telethon/update_state.py index 7e25549f..fa1963a3 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -79,11 +79,14 @@ class UpdateState: """Raises "StopIterationException" on the worker threads to stop them, and also clears all of them off the list """ - with self._updates_lock: - # Insert at the beginning so the very next poll causes an error - # TODO Should this reset the pts and such? - self._updates.appendleft(StopIteration()) - self._updates_available.set() + if self._workers: + with self._updates_lock: + # Insert at the beginning so the very next poll causes an error + # on all the worker threads + # TODO Should this reset the pts and such? + for _ in range(self._workers): + self._updates.appendleft(StopIteration()) + self._updates_available.set() for t in self._worker_threads: t.join() From 76e5206acc04e835e13746865fe687afe0a87a2d Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 18:45:56 +0200 Subject: [PATCH 112/121] Raise ValueError if an unknown **kwarg is given --- telethon/telegram_bare_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 75928b69..bec5d8ee 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -118,8 +118,9 @@ class TelegramBareClient: # Used on connection - the user may modify these and reconnect kwargs['app_version'] = kwargs.get('app_version', self.__version__) for name, value in kwargs.items(): - if hasattr(self.session, name): - setattr(self.session, name, value) + if not hasattr(self.session, name): + raise ValueError('Unknown named parameter', name) + setattr(self.session, name, value) # Despite the state of the real connection, keep track of whether # the user has explicitly called .connect() or .disconnect() here. From c9dff552b8e7bb2ae0a0f95453344ea8802545f1 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 18:51:25 +0200 Subject: [PATCH 113/121] Except ServerErrors and sleep a bit before retrying --- telethon/telegram_bare_client.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index bec5d8ee..157ab866 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -10,7 +10,7 @@ from time import sleep from . import helpers as utils from .crypto import rsa, CdnDecrypter from .errors import ( - RPCError, BrokenAuthKeyError, + RPCError, BrokenAuthKeyError, ServerError, FloodWaitError, FileMigrateError, TypeNotFoundError, UnauthorizedError, PhoneMigrateError, NetworkMigrateError, UserMigrateError ) @@ -496,6 +496,13 @@ class TelegramBareClient: while self._user_connected and not self._reconnect(): sleep(0.1) # Retry forever until we can send the request + except ServerError as e: + # Telegram is having some issues, sleep a tiny bit and retry + self._logger.debug( + '[ERROR] Telegram is having some internal issues', e + ) + sleep(2) + except FloodWaitError: sender.disconnect() self.disconnect() From 33dbac6350aa08811c2362ef99941a23ecea4058 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 19:00:32 +0200 Subject: [PATCH 114/121] Add some missing BadRequest errors --- telethon/errors/rpc_errors_400.py | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/telethon/errors/rpc_errors_400.py b/telethon/errors/rpc_errors_400.py index 68874199..6c96ce7e 100644 --- a/telethon/errors/rpc_errors_400.py +++ b/telethon/errors/rpc_errors_400.py @@ -18,6 +18,15 @@ class BotMethodInvalidError(BadRequestError): ) +class CdnMethodInvalidError(BadRequestError): + def __init__(self, **kwargs): + super(Exception, self).__init__( + self, + 'This method cannot be invoked on a CDN server. Refer to ' + 'https://core.telegram.org/cdn#schema for available methods.' + ) + + class ChannelInvalidError(BadRequestError): def __init__(self, **kwargs): super(Exception, self).__init__( @@ -134,6 +143,16 @@ class InputMethodInvalidError(BadRequestError): ) +class InputRequestTooLongError(BadRequestError): + def __init__(self, **kwargs): + super(Exception, self).__init__( + self, + 'The input request was too long. This may be a bug in the library ' + 'as it can occur when serializing more bytes than it should (like' + 'appending the vector constructor code at the end of a message).' + ) + + class LastNameInvalidError(BadRequestError): def __init__(self, **kwargs): super(Exception, self).__init__( @@ -142,6 +161,24 @@ class LastNameInvalidError(BadRequestError): ) +class LimitInvalidError(BadRequestError): + def __init__(self, **kwargs): + super(Exception, self).__init__( + self, + 'An invalid limit was provided. See ' + 'https://core.telegram.org/api/files#downloading-files' + ) + + +class LocationInvalidError(BadRequestError): + def __init__(self, **kwargs): + super(Exception, self).__init__( + self, + 'The location given for a file was invalid. See ' + 'https://core.telegram.org/api/files#downloading-files' + ) + + class Md5ChecksumInvalidError(BadRequestError): def __init__(self, **kwargs): super(Exception, self).__init__( @@ -191,6 +228,16 @@ class MsgWaitFailedError(BadRequestError): ) +class OffsetInvalidError(BadRequestError): + def __init__(self, **kwargs): + super(Exception, self).__init__( + self, + 'The given offset was invalid, it must be divisible by 1KB. ' + 'See https://core.telegram.org/api/files#downloading-files' + ) + + + class PasswordHashInvalidError(BadRequestError): def __init__(self, **kwargs): super(Exception, self).__init__( @@ -350,6 +397,7 @@ class UserIdInvalidError(BadRequestError): rpc_errors_400_all = { 'API_ID_INVALID': ApiIdInvalidError, 'BOT_METHOD_INVALID': BotMethodInvalidError, + 'CDN_METHOD_INVALID': CdnMethodInvalidError, 'CHANNEL_INVALID': ChannelInvalidError, 'CHAT_ADMIN_REQUIRED': ChatAdminRequiredError, 'CHAT_ID_INVALID': ChatIdInvalidError, @@ -362,13 +410,17 @@ rpc_errors_400_all = { 'FILE_PART_INVALID': FilePartInvalidError, 'FIRSTNAME_INVALID': FirstNameInvalidError, 'INPUT_METHOD_INVALID': InputMethodInvalidError, + 'INPUT_REQUEST_TOO_LONG': InputRequestTooLongError, 'LASTNAME_INVALID': LastNameInvalidError, + 'LIMIT_INVALID': LimitInvalidError, + 'LOCATION_INVALID': LocationInvalidError, 'MD5_CHECKSUM_INVALID': Md5ChecksumInvalidError, 'MESSAGE_EMPTY': MessageEmptyError, 'MESSAGE_ID_INVALID': MessageIdInvalidError, 'MESSAGE_TOO_LONG': MessageTooLongError, 'MESSAGE_NOT_MODIFIED': MessageNotModifiedError, 'MSG_WAIT_FAILED': MsgWaitFailedError, + 'OFFSET_INVALID': OffsetInvalidError, 'PASSWORD_HASH_INVALID': PasswordHashInvalidError, 'PEER_ID_INVALID': PeerIdInvalidError, 'PHONE_CODE_EMPTY': PhoneCodeEmptyError, From a5ab49c7074a2ba88e8a600a512b8d1d7b25c8bb Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 19:02:12 +0200 Subject: [PATCH 115/121] Remove sleep on ServerError as per @danog's recommendation --- telethon/telegram_bare_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 157ab866..b672c842 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -497,11 +497,10 @@ class TelegramBareClient: sleep(0.1) # Retry forever until we can send the request except ServerError as e: - # Telegram is having some issues, sleep a tiny bit and retry + # Telegram is having some issues, just retry self._logger.debug( '[ERROR] Telegram is having some internal issues', e ) - sleep(2) except FloodWaitError: sender.disconnect() From ce48c9752eaeddf8981b57db486ee6f4198bf945 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 30 Sep 2017 19:47:19 +0200 Subject: [PATCH 116/121] Assert flag params with same flag index are all set/unset --- telethon_generator/tl_generator.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 7b1924fd..1ee0831a 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -301,6 +301,26 @@ class TLGenerator: # Write the .to_bytes() function builder.writeln('def to_bytes(self):') + + # Some objects require more than one flag parameter to be set + # at the same time. In this case, add an assertion. + repeated_args = defaultdict(list) + for arg in tlobject.args: + if arg.is_flag: + repeated_args[arg.flag_index].append(arg) + + for args in repeated_args.values(): + if len(args) > 1: + cnd1 = ('self.{} is None'.format(a.name) for a in args) + cnd2 = ('self.{} is not None'.format(a.name) for a in args) + builder.writeln( + "assert ({}) or ({}), '{} parameters must all " + "be None or neither be None'".format( + ' and '.join(cnd1), ' and '.join(cnd2), + ', '.join(a.name for a in args) + ) + ) + builder.writeln("return b''.join((") builder.current_indent += 1 From 06bb09b95c86362ade6608bfcd36b38140f9ca60 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 1 Oct 2017 10:45:35 +0200 Subject: [PATCH 117/121] Fix wrong .empty() due to variable shadowing (from ce48c97) --- telethon_generator/tl_generator.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 1ee0831a..e76dffaa 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -309,15 +309,15 @@ class TLGenerator: if arg.is_flag: repeated_args[arg.flag_index].append(arg) - for args in repeated_args.values(): - if len(args) > 1: - cnd1 = ('self.{} is None'.format(a.name) for a in args) - cnd2 = ('self.{} is not None'.format(a.name) for a in args) + for ra in repeated_args.values(): + if len(ra) > 1: + cnd1 = ('self.{} is None'.format(a.name) for a in ra) + cnd2 = ('self.{} is not None'.format(a.name) for a in ra) builder.writeln( "assert ({}) or ({}), '{} parameters must all " "be None or neither be None'".format( ' and '.join(cnd1), ' and '.join(cnd2), - ', '.join(a.name for a in args) + ', '.join(a.name for a in ra) ) ) @@ -354,7 +354,8 @@ class TLGenerator: if tlobject.args: for arg in tlobject.args: TLGenerator.write_onresponse_code( - builder, arg, tlobject.args) + builder, arg, tlobject.args + ) else: # If there were no arguments, we still need an # on_response method, and hence "pass" if empty From 68e7d481f435dd0cbea5711fc10a611d585934f6 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 1 Oct 2017 10:50:37 +0200 Subject: [PATCH 118/121] Add support to get all dialogs at once --- telethon/telegram_client.py | 56 +++++++++++++++++++++++++++++++------ telethon/utils.py | 29 ++++++++++--------- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 0e264a3e..810ce903 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -41,6 +41,7 @@ from .tl.types import ( InputUserSelf, UserProfilePhoto, ChatPhoto, UpdateMessageID, UpdateNewMessage, UpdateShortSentMessage ) +from .tl.types.messages import DialogsSlice from .utils import find_user_or_chat, get_extension @@ -224,22 +225,61 @@ class TelegramClient(TelegramBareClient): offset_id=0, offset_peer=InputPeerEmpty()): """Returns a tuple of lists ([dialogs], [entities]) - with at least 'limit' items each. + with at least 'limit' items each unless all dialogs were consumed. + + If `limit` is None, all dialogs will be retrieved (from the given + offset) will be retrieved. - If `limit` is 0, all dialogs will (should) retrieved. The `entities` represent the user, chat or channel - corresponding to that dialog. + corresponding to that dialog. If it's an integer, not + all dialogs may be retrieved at once. """ + if limit is None: + limit = float('inf') - r = self( - GetDialogsRequest( + dialogs = {} # Use Dialog.top_message as identifier to avoid dupes + messages = {} # Used later for sorting TODO also return these? + entities = {} + while len(dialogs) < limit: + r = self(GetDialogsRequest( offset_date=offset_date, offset_id=offset_id, offset_peer=offset_peer, - limit=limit)) + limit=0 # limit 0 often means "as much as possible" + )) + if not r.dialogs: + break + + for d in r.dialogs: + dialogs[d.top_message] = d + for m in r.messages: + messages[m.id] = m + + # We assume users can't have the same ID as a chat + for u in r.users: + entities[u.id] = u + for c in r.chats: + entities[c.id] = c + + if isinstance(r, DialogsSlice): + # Don't enter next iteration if we already got all + break + + offset_date = r.messages[-1].date + offset_peer = find_user_or_chat(r.dialogs[-1].peer, entities, + entities) + offset_id = r.messages[-1].id & 4294967296 # Telegram/danog magic + + # Sort by message date + no_date = datetime.fromtimestamp(0) + dialogs = sorted( + list(dialogs.values()), + key=lambda d: getattr(messages[d.top_message], 'date', no_date) + ) return ( - r.dialogs, - [find_user_or_chat(d.peer, r.users, r.chats) for d in r.dialogs]) + dialogs, + [find_user_or_chat(d.peer, entities, entities) for d in dialogs] + ) # endregion diff --git a/telethon/utils.py b/telethon/utils.py index 25104d90..273dc962 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -302,24 +302,23 @@ def get_input_media(media, user_caption=None, is_photo=False): def find_user_or_chat(peer, users, chats): """Finds the corresponding user or chat given a peer. Returns None if it was not found""" - try: - if isinstance(peer, PeerUser): - return next(u for u in users if u.id == peer.user_id) - - elif isinstance(peer, PeerChat): - return next(c for c in chats if c.id == peer.chat_id) - + if isinstance(peer, PeerUser): + peer, where = peer.user_id, users + else: + where = chats + if isinstance(peer, PeerChat): + peer = peer.chat_id elif isinstance(peer, PeerChannel): - return next(c for c in chats if c.id == peer.channel_id) - - except StopIteration: return + peer = peer.channel_id if isinstance(peer, int): - try: return next(u for u in users if u.id == peer) - except StopIteration: pass - - try: return next(c for c in chats if c.id == peer) - except StopIteration: pass + if isinstance(where, dict): + return where.get(peer) + else: + try: + return next(x for x in where if x.id == peer) + except StopIteration: + pass def get_appropriated_part_size(file_size): From 1d159908c731a7525bc51f46faa455884fcb382e Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 1 Oct 2017 11:25:56 +0200 Subject: [PATCH 119/121] Fix RPC excepts (e.g. UserMigrate) being in the wrong try --- telethon/telegram_bare_client.py | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index b672c842..d41a8b8e 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -463,20 +463,6 @@ class TelegramBareClient: while not all(x.confirm_received.is_set() for x in requests): sender.receive(update_state=self.updates) - except (PhoneMigrateError, NetworkMigrateError, - UserMigrateError) as e: - self._logger.debug( - 'DC error when invoking request, ' - 'attempting to reconnect at DC {}'.format(e.new_dc) - ) - - # TODO What happens with the background thread here? - # For normal use cases, this won't happen, because this will only - # be on the very first connection (not authorized, not running), - # but may be an issue for people who actually travel? - self._reconnect(new_dc=e.new_dc) - return self._invoke(sender, call_receive, *requests) - except TimeoutError: pass # We will just retry @@ -496,17 +482,6 @@ class TelegramBareClient: while self._user_connected and not self._reconnect(): sleep(0.1) # Retry forever until we can send the request - except ServerError as e: - # Telegram is having some issues, just retry - self._logger.debug( - '[ERROR] Telegram is having some internal issues', e - ) - - except FloodWaitError: - sender.disconnect() - self.disconnect() - raise - finally: if sender != self._sender: sender.disconnect() @@ -523,6 +498,31 @@ class TelegramBareClient: else: return [x.result for x in requests] + except (PhoneMigrateError, NetworkMigrateError, + UserMigrateError) as e: + self._logger.debug( + 'DC error when invoking request, ' + 'attempting to reconnect at DC {}'.format(e.new_dc) + ) + + # TODO What happens with the background thread here? + # For normal use cases, this won't happen, because this will only + # be on the very first connection (not authorized, not running), + # but may be an issue for people who actually travel? + self._reconnect(new_dc=e.new_dc) + return self._invoke(sender, call_receive, *requests) + + except ServerError as e: + # Telegram is having some issues, just retry + self._logger.debug( + '[ERROR] Telegram is having some internal issues', e + ) + + except FloodWaitError: + sender.disconnect() + self.disconnect() + raise + # Let people use client(SomeRequest()) instead client.invoke(...) __call__ = invoke From 1d250a544165956d369f53ccb7db4d68e2cb06dd Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 1 Oct 2017 11:31:26 +0200 Subject: [PATCH 120/121] Allow overriding phone on .sign_in (fix #278) --- telethon/telegram_client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 810ce903..aed7ec76 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -148,9 +148,12 @@ class TelegramClient(TelegramBareClient): if phone and not code: return self.send_code_request(phone) elif code: - if self._phone is None: + if not self._phone: + self._phone = phone + if not self._phone: raise ValueError( - 'Please make sure to call send_code_request first.') + 'Please make sure to call send_code_request first.' + ) try: if isinstance(code, int): From 9445d2ba535ed7d214a7e6e68b85e7f3af1a690e Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 1 Oct 2017 11:37:18 +0200 Subject: [PATCH 121/121] More enhancements on .sign_in from different clients (#278) --- telethon/telegram_client.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index aed7ec76..85346b42 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -131,13 +131,22 @@ class TelegramClient(TelegramBareClient): return result def sign_in(self, phone=None, code=None, - password=None, bot_token=None): + password=None, bot_token=None, phone_code_hash=None): """Completes the sign in process with the phone number + code pair. If no phone or code is provided, then the sole password will be used. The password should be used after a normal authorization attempt has happened and an SessionPasswordNeededError was raised. + If you're calling .sign_in() on two completely different clients + (for example, through an API that creates a new client per phone), + you must first call .sign_in(phone) to receive the code, and then + with the result such method results, call + .sign_in(phone, code, phone_code_hash=result.phone_code_hash). + + If this is done on the same client, the client will fill said values + for you. + To login as a bot, only `bot_token` should be provided. This should equal to the bot access hash provided by https://t.me/BotFather during your bot creation. @@ -148,18 +157,20 @@ class TelegramClient(TelegramBareClient): if phone and not code: return self.send_code_request(phone) elif code: - if not self._phone: - self._phone = phone - if not self._phone: + phone = phone or self._phone + phone_code_hash = phone_code_hash or self._phone_code_hash + if not phone: raise ValueError( 'Please make sure to call send_code_request first.' ) + if not phone_code_hash: + raise ValueError('You also need to provide a phone_code_hash.') try: if isinstance(code, int): code = str(code) result = self(SignInRequest( - self._phone, self._phone_code_hash, code + phone, phone_code_hash, code )) except (PhoneCodeEmptyError, PhoneCodeExpiredError,