From 697434be3795efc750c5a8b2c686f23690d8fa39 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 19 Jun 2017 09:58:03 +0200 Subject: [PATCH 01/52] Don't do anything on .connect() if it's already connected --- telethon/network/mtproto_sender.py | 3 +++ telethon/network/tcp_transport.py | 3 +++ telethon/telegram_bare_client.py | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 516a381c..e911af1f 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -35,6 +35,9 @@ class MtProtoSender: """Connects to the server""" self._transport.connect() + def is_connected(self): + return self._transport.is_connected() + def disconnect(self): """Disconnects from the server""" self._transport.close() diff --git a/telethon/network/tcp_transport.py b/telethon/network/tcp_transport.py index 9b35455b..c5f7a9df 100644 --- a/telethon/network/tcp_transport.py +++ b/telethon/network/tcp_transport.py @@ -18,6 +18,9 @@ class TcpTransport: self.send_counter = 0 self.tcp_client.connect(self.ip, self.port) + def is_connected(self): + return self.tcp_client.connected + # Original reference: https://core.telegram.org/mtproto#tcp-transport # The packets are encoded as: total length, sequence number, packet and checksum (CRC32) def send(self, packet): diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 9e28075e..d4338169 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -79,6 +79,12 @@ class TelegramBareClient: If 'exported_auth' is not None, it will be used instead to determine the authorization key for the current session. """ + if self.sender and self.sender.is_connected(): + self._logger.warning( + 'Attempted to connect when the client was already connected.' + ) + return + transport = TcpTransport(self.session.server_address, self.session.port, proxy=self.proxy) From 86358d7805d1973595a9fb0260c65f19478cec81 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 20 Jun 2017 09:46:20 +0200 Subject: [PATCH 02/52] Add periodic pings if an updates thread was started (closes #138) --- telethon/telegram_client.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 09d0092e..f8e739bd 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1,7 +1,7 @@ from datetime import timedelta from mimetypes import guess_type from threading import Event, RLock, Thread -from time import sleep +from time import sleep, time from . import TelegramBareClient @@ -35,6 +35,9 @@ from .tl.functions.messages import ( # For .get_me() and ensuring we're authorized from .tl.functions.users import GetUsersRequest +# So the server doesn't stop sending updates to us +from .tl.functions import PingRequest + # All the types we need to work with from .tl.types import ( ChatPhotoEmpty, DocumentAttributeAudio, DocumentAttributeFilename, @@ -94,11 +97,14 @@ class TelegramClient(TelegramBareClient): # Safety across multiple threads (for the updates thread) self._lock = RLock() - # Methods to be called when an update is received + # Updates-related members self._update_handlers = [] self._updates_thread_running = Event() self._updates_thread_receiving = Event() + self._next_ping_at = 0 + self.ping_interval = 60 # Seconds + # Used on connection - the user may modify these and reconnect if device_model: self.session.device_model = device_model @@ -805,6 +811,10 @@ class TelegramClient(TelegramBareClient): 'Trying to receive updates from the updates thread' ) + if time() > self._next_ping_at: + self._next_ping_at = time() + self.ping_interval + self.invoke(PingRequest(utils.generate_random_long())) + updates = self.sender.receive_updates(timeout=timeout) self._updates_thread_receiving.clear() From d58222d00714500e2f4d333e599ed90bac2c1573 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 20 Jun 2017 11:03:46 +0200 Subject: [PATCH 03/52] Use crc32 from zlib instead from binascii It seems to be a bit faster stackoverflow.com/q/44502855 --- telethon/network/tcp_transport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/network/tcp_transport.py b/telethon/network/tcp_transport.py index c5f7a9df..98cb25e7 100644 --- a/telethon/network/tcp_transport.py +++ b/telethon/network/tcp_transport.py @@ -1,4 +1,4 @@ -from binascii import crc32 +from zlib import crc32 from datetime import timedelta from ..errors import InvalidChecksumError From a9a5c7e254f8a37582093e47fe3cc334de05b8b3 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 21 Jun 2017 10:20:39 +0200 Subject: [PATCH 04/52] Use signed longs for every request_id (#138 follow-up) --- telethon/network/mtproto_sender.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index e911af1f..012f2d3d 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -309,7 +309,7 @@ class MtProtoSender: def _handle_bad_msg_notification(self, msg_id, sequence, reader): self._logger.debug('Handling bad message notification') reader.read_int(signed=False) # code - reader.read_long(signed=False) # request_id + reader.read_long() # request_id reader.read_int() # request_sequence error_code = reader.read_int() @@ -328,7 +328,7 @@ class MtProtoSender: def _handle_rpc_result(self, msg_id, sequence, reader): self._logger.debug('Handling RPC result') reader.read_int(signed=False) # code - request_id = reader.read_long(signed=False) + request_id = reader.read_long() inner_code = reader.read_int(signed=False) try: From b0b814bdb9b53ca3fe80b0740879e9df0bedee04 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 21 Jun 2017 11:27:22 +0200 Subject: [PATCH 05/52] Use `setup.py` to run `tl_generator.py` instead (plus some cleanup) --- README.rst | 4 +- setup.py | 125 ++++++++++++++++------------- telethon_generator/tl_generator.py | 73 +++++++---------- 3 files changed, 98 insertions(+), 104 deletions(-) mode change 100755 => 100644 telethon_generator/tl_generator.py diff --git a/README.rst b/README.rst index 9636df6b..88251354 100755 --- a/README.rst +++ b/README.rst @@ -71,7 +71,7 @@ Installing Telethon manually (`GitHub `_, `package index `_) 2. Clone Telethon's GitHub repository: ``git clone https://github.com/LonamiWebs/Telethon.git`` 3. Enter the cloned repository: ``cd Telethon`` -4. Run the code generator: ``cd telethon_generator && python3 tl_generator.py`` +4. Run the code generator: ``python3 setup.py gen_tl`` 5. Done! Running Telethon @@ -232,7 +232,7 @@ Have you found a more updated version of the ``scheme.tl`` file? Those are great as grabbing the `latest version `_ and replacing the one you can find in this same directory by the updated one. -Don't forget to run ``python3 tl_generator.py``. +Don't forget to run ``python3 setup.py gen_tl``. If the changes weren't too big, everything should still work the same way as it did before; but with extra features. diff --git a/setup.py b/setup.py index dd755a32..86021e9c 100644 --- a/setup.py +++ b/setup.py @@ -3,85 +3,94 @@ See: https://packaging.python.org/en/latest/distributing.html https://github.com/pypa/sampleproject + +Extra supported commands are: +* gen_tl, to generate the classes required for Telethon to run +* clean_tl, to clean these generated classes """ # To use a consistent encoding from codecs import open +from sys import argv from os import path # Always prefer setuptools over distutils from setuptools import find_packages, setup -from telethon import TelegramClient +from telethon_generator.tl_generator import TLGenerator +try: + from telethon import TelegramClient +except ModuleNotFoundError: + TelegramClient = None -here = path.abspath(path.dirname(__file__)) -# Get the long description from the README file -with open(path.join(here, 'README.rst'), encoding='utf-8') as f: - long_description = f.read() +if __name__ == '__main__': + if len(argv) >= 2 and argv[1] == 'gen_tl': + generator = TLGenerator('telethon/tl') + if generator.tlobjects_exist(): + print('Detected previous TLObjects. Cleaning...') + generator.clean_tlobjects() -setup( - name='Telethon', + print('Generating TLObjects...') + generator.generate_tlobjects( + 'telethon_generator/scheme.tl', import_depth=2 + ) + print('Done.') - # Versions should comply with PEP440. - version=TelegramClient.__version__, - description="Python3 Telegram's client implementation with full access to its API", - long_description=long_description, + elif len(argv) >= 2 and argv[1] == 'clean_tl': + print('Cleaning...') + TLGenerator('telethon/tl').clean_tlobjects() + print('Done.') - # The project's main homepage. - url='https://github.com/LonamiWebs/Telethon', - download_url='https://github.com/LonamiWebs/Telethon/releases', + else: + if not TelegramClient: + print('Run `python3', argv[0], 'gen_tl` first.') + quit() - # Author details - author='Lonami Exo', - author_email='totufals@hotmail.com', + here = path.abspath(path.dirname(__file__)) - # Choose your license - license='MIT', + # Get the long description from the README file + with open(path.join(here, 'README.rst'), encoding='utf-8') as f: + long_description = f.read() - # See https://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - 'Development Status :: 3 - Alpha', + setup( + name='Telethon', - # Indicate who your project is intended for - 'Intended Audience :: Developers', - 'Topic :: Communications :: Chat', + # Versions should comply with PEP440. + version=TelegramClient.__version__, + description="Full-featured Telegram client library for Python 3", + long_description=long_description, - # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: MIT License', + url='https://github.com/LonamiWebs/Telethon', + download_url='https://github.com/LonamiWebs/Telethon/releases', - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6' - ], + author='Lonami Exo', + author_email='totufals@hotmail.com', - # What does your project relate to? - keywords='Telegram API chat client MTProto', + license='MIT', - # You can just specify the packages manually here if your project is - # simple. Or you can use find_packages(). - packages=find_packages(exclude=[ - 'telethon_generator', 'telethon_tests', 'run_tests.py', - 'try_telethon.py' - ]), + # See https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', - # List run-time dependencies here. These will be installed by pip when - # your project is installed. - install_requires=['pyaes'], + 'Intended Audience :: Developers', + 'Topic :: Communications :: Chat', - # To provide executable scripts, use entry points in preference to the - # "scripts" keyword. Entry points provide cross-platform support and allow - # pip to create the appropriate form of executable for the target platform. - entry_points={ - 'console_scripts': [ - 'gen_tl = tl_generator:clean_and_generate', - ], - }) + 'License :: OSI Approved :: MIT License', + + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6' + ], + keywords='telegram api chat client library messaging mtproto', + packages=find_packages(exclude=[ + 'telethon_generator', 'telethon_tests', 'run_tests.py', + 'try_telethon.py' + ]), + install_requires=['pyaes'] + ) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py old mode 100755 new mode 100644 index 8ccc4d65..ae8fdb68 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -1,51 +1,46 @@ -#!/usr/bin/env python3 import os import re import shutil from zlib import crc32 from collections import defaultdict -try: - from .parser import SourceBuilder, TLParser -except (ImportError, SystemError): - from parser import SourceBuilder, TLParser - - -def get_output_path(normal_path): - return os.path.join('../telethon/tl', normal_path) - -output_base_depth = 2 # telethon/tl/ +from .parser import SourceBuilder, TLParser class TLGenerator: - @staticmethod - def tlobjects_exist(): + def __init__(self, output_dir): + self.output_dir = output_dir + + def _get_file(self, *paths): + return os.path.join(self.output_dir, *paths) + + def _rm_if_exists(self, filename): + file = self._get_file(filename) + if os.path.exists(file): + if os.path.isdir(file): + shutil.rmtree(file) + else: + os.remove(file) + + def tlobjects_exist(self): """Determines whether the TLObjects were previously generated (hence exist) or not """ - return os.path.isfile(get_output_path('all_tlobjects.py')) + return os.path.isfile(self._get_file('all_tlobjects.py')) - @staticmethod - def clean_tlobjects(): + def clean_tlobjects(self): """Cleans the automatically generated TLObjects from disk""" - if os.path.isdir(get_output_path('functions')): - shutil.rmtree(get_output_path('functions')) + for name in ('functions', 'types', 'all_tlobjects.py'): + self._rm_if_exists(name) - if os.path.isdir(get_output_path('types')): - shutil.rmtree(get_output_path('types')) - - if os.path.isfile(get_output_path('all_tlobjects.py')): - os.remove(get_output_path('all_tlobjects.py')) - - @staticmethod - def generate_tlobjects(scheme_file): + def generate_tlobjects(self, scheme_file, import_depth): """Generates all the TLObjects from scheme.tl to tl/functions and tl/types """ # First ensure that the required parent directories exist - os.makedirs(get_output_path('functions'), exist_ok=True) - os.makedirs(get_output_path('types'), exist_ok=True) + os.makedirs(self._get_file('functions'), exist_ok=True) + os.makedirs(self._get_file('types'), exist_ok=True) # Step 0: Cache the parsed file on a tuple tlobjects = tuple(TLParser.parse_file(scheme_file)) @@ -91,11 +86,11 @@ class TLGenerator: continue # Determine the output directory and create it - out_dir = get_output_path('functions' - if tlobject.is_function else 'types') + out_dir = self._get_file('functions' + if tlobject.is_function else 'types') # Path depth to perform relative import - depth = output_base_depth + depth = import_depth if tlobject.namespace: depth += 1 out_dir = os.path.join(out_dir, tlobject.namespace) @@ -121,19 +116,19 @@ class TLGenerator: tlobject, builder, depth, type_constructors) # Step 3: Add the relative imports to the namespaces on __init__.py's - init_py = os.path.join(get_output_path('functions'), '__init__.py') + init_py = self._get_file('functions', '__init__.py') with open(init_py, 'a') as file: file.write('from . import {}\n' .format(', '.join(function_namespaces))) - init_py = os.path.join(get_output_path('types'), '__init__.py') + init_py = self._get_file('types', '__init__.py') with open(init_py, 'a') as file: file.write('from . import {}\n' .format(', '.join(type_namespaces))) # Step 4: Once all the objects have been generated, # we can now group them in a single file - filename = os.path.join(get_output_path('all_tlobjects.py')) + filename = os.path.join(self._get_file('all_tlobjects.py')) with open(filename, 'w', encoding='utf-8') as file: with SourceBuilder(file) as builder: builder.writeln( @@ -658,13 +653,3 @@ class TLGenerator: builder.writeln('self.result = reader.tgread_vector()') else: builder.writeln('self.result = reader.tgread_object()') - - -if __name__ == '__main__': - if TLGenerator.tlobjects_exist(): - print('Detected previous TLObjects. Cleaning...') - TLGenerator.clean_tlobjects() - - print('Generating TLObjects...') - TLGenerator.generate_tlobjects('scheme.tl') - print('Done.') From e7fac8e254159fff8452a6abcccb071d4740e0d0 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 21 Jun 2017 19:18:22 +0200 Subject: [PATCH 06/52] Add shebang to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 86021e9c..099cbafb --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """A setuptools based setup module. See: From 52a42661ee376cd3767a5b3abb8275a64ac50d43 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 22 Jun 2017 10:39:00 +0200 Subject: [PATCH 07/52] Add timeout to connect() --- telethon/telegram_bare_client.py | 11 +++++++---- telethon/telegram_client.py | 4 +++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index d4338169..25e3de05 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -70,12 +70,14 @@ class TelegramBareClient: # region Connecting - def connect(self, exported_auth=None): + def connect(self, timeout=timedelta(seconds=5), exported_auth=None): """Connects to the Telegram servers, executing authentication if required. Note that authenticating to the Telegram servers is not the same as authenticating the desired user itself, which may require a call (or several) to 'sign_in' for the first time. + The specified timeout will be used on internal .invoke()'s. + If 'exported_auth' is not None, it will be used instead to determine the authorization key for the current session. """ @@ -115,13 +117,14 @@ class TelegramBareClient: query=query) result = self.invoke( - InvokeWithLayerRequest( - layer=layer, query=request)) + InvokeWithLayerRequest(layer=layer, query=request), + timeout=timeout + ) if exported_auth is not None: # TODO Don't actually need this for exported authorizations, # they're only valid on such data center. - result = self.invoke(GetConfigRequest()) + result = self.invoke(GetConfigRequest(), timeout=timeout) # We're only interested in the DC options, # although many other options are available! diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index f8e739bd..c8efbe7a 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -129,12 +129,14 @@ class TelegramClient(TelegramBareClient): # region Connecting - def connect(self, *args): + def connect(self, timeout=timedelta(seconds=5), *args): """Connects to the Telegram servers, executing authentication if required. Note that authenticating to the Telegram servers is not the same as authenticating the desired user itself, which may require a call (or several) to 'sign_in' for the first time. + The specified timeout will be used on internal .invoke()'s. + *args will be ignored. """ return super(TelegramClient, self).connect() From e4fbd87c7551ae8775f2b893cc00a2a0fa6637b9 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 22 Jun 2017 11:43:42 +0200 Subject: [PATCH 08/52] Turn timeout into a property instead leaving it as a parameter --- telethon/extensions/tcp_client.py | 4 +- telethon/network/mtproto_sender.py | 29 ++++++------- telethon/network/tcp_transport.py | 19 ++++++--- telethon/telegram_bare_client.py | 68 ++++++++++++++++++++---------- telethon/telegram_client.py | 44 ++++++++++--------- 5 files changed, 97 insertions(+), 67 deletions(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index a63c1331..089066e9 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -80,7 +80,7 @@ class TcpClient: # Set the starting time so we can # calculate whether the timeout should fire - start_time = datetime.now() if timeout else None + start_time = datetime.now() if timeout is not None else None with BufferedWriter(BytesIO(), buffer_size=size) as buffer: bytes_left = size @@ -104,7 +104,7 @@ class TcpClient: time.sleep(self.delay) # Check if the timeout finished - if timeout: + if timeout is not None: time_passed = datetime.now() - start_time if time_passed > timeout: raise TimeoutError( diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 012f2d3d..37042e13 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -17,7 +17,7 @@ class MtProtoSender: """MTProto Mobile Protocol sender (https://core.telegram.org/mtproto/description)""" def __init__(self, transport, session): - self._transport = transport + self.transport = transport self.session = session self._logger = logging.getLogger(__name__) @@ -33,14 +33,14 @@ class MtProtoSender: def connect(self): """Connects to the server""" - self._transport.connect() + self.transport.connect() def is_connected(self): - return self._transport.is_connected() + return self.transport.is_connected() def disconnect(self): """Disconnects from the server""" - self._transport.close() + self.transport.close() # region Send and receive @@ -76,11 +76,12 @@ class MtProtoSender: del self._need_confirmation[:] - def receive(self, request=None, timeout=timedelta(seconds=5), updates=None): + def receive(self, request=None, updates=None, **kwargs): """Receives the specified MTProtoRequest ("fills in it" the received data). This also restores the updates thread. - An optional timeout can be specified to cancel the operation - if no data has been read after its time delta. + + An optional named parameter 'timeout' can be specified if + one desires to override 'self.transport.timeout'. If 'request' is None, a single item will be read into the 'updates' list (which cannot be None). @@ -100,7 +101,7 @@ class MtProtoSender: while (request and not request.confirm_received) or \ (not request and not updates): self._logger.info('Trying to .receive() the request result...') - seq, body = self._transport.receive(timeout) + seq, body = self.transport.receive(**kwargs) message, remote_msg_id, remote_seq = self._decode_msg(body) with BinaryReader(message) as reader: @@ -116,18 +117,16 @@ class MtProtoSender: self._logger.info('Request result received') self._logger.debug('receive() released the lock') - def receive_updates(self, timeout=timedelta(seconds=5)): - """Receives one or more update objects - and returns them as a list - """ + def receive_updates(self, **kwargs): + """Wrapper for .receive(request=None, updates=[])""" updates = [] - self.receive(timeout=timeout, updates=updates) + self.receive(updates=updates, **kwargs) return updates def cancel_receive(self): """Cancels any pending receive operation by raising a ReadCancelledError""" - self._transport.cancel_receive() + self.transport.cancel_receive() # endregion @@ -160,7 +159,7 @@ class MtProtoSender: self.session.auth_key.key_id, signed=False) cipher_writer.write(msg_key) cipher_writer.write(cipher_text) - self._transport.send(cipher_writer.get_bytes()) + self.transport.send(cipher_writer.get_bytes()) def _decode_msg(self, body): """Decodes an received encrypted message body bytes""" diff --git a/telethon/network/tcp_transport.py b/telethon/network/tcp_transport.py index 98cb25e7..702d2633 100644 --- a/telethon/network/tcp_transport.py +++ b/telethon/network/tcp_transport.py @@ -7,10 +7,12 @@ from ..extensions import BinaryWriter class TcpTransport: - def __init__(self, ip_address, port, proxy=None): + def __init__(self, ip_address, port, + proxy=None, timeout=timedelta(seconds=5)): self.ip = ip_address self.port = port self.tcp_client = TcpClient(proxy) + self.timeout = timeout self.send_counter = 0 def connect(self): @@ -22,7 +24,8 @@ class TcpTransport: return self.tcp_client.connected # Original reference: https://core.telegram.org/mtproto#tcp-transport - # The packets are encoded as: total length, sequence number, packet and checksum (CRC32) + # The packets are encoded as: + # total length, sequence number, packet and checksum (CRC32) def send(self, packet): """Sends the given packet (bytes array) to the connected peer""" if not self.tcp_client.connected: @@ -39,10 +42,14 @@ class TcpTransport: self.send_counter += 1 self.tcp_client.write(writer.get_bytes()) - def receive(self, timeout=timedelta(seconds=5)): - """Receives a TCP message (tuple(sequence number, body)) from the connected peer. - There is a default timeout of 5 seconds before the operation is cancelled. - Timeout can be set to None for no timeout""" + def receive(self, **kwargs): + """Receives a TCP message (tuple(sequence number, body)) from the + connected peer. + + If a named 'timeout' parameter is present, it will override + 'self.timeout', and this can be a 'timedelta' or 'None'. + """ + timeout = kwargs.get('timeout', self.timeout) # First read everything we need packet_length_bytes = self.tcp_client.read(4, timeout) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 25e3de05..9cd60915 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -51,7 +51,8 @@ class TelegramBareClient: # region Initialization - def __init__(self, session, api_id, api_hash, proxy=None): + def __init__(self, session, api_id, api_hash, + proxy=None, timeout=timedelta(seconds=5)): """Initializes the Telegram client with the specified API ID and Hash. Session must always be a Session instance, and an optional proxy can also be specified to be used on the connection. @@ -60,35 +61,36 @@ class TelegramBareClient: self.api_id = int(api_id) self.api_hash = api_hash self.proxy = proxy + self._timeout = timeout self._logger = logging.getLogger(__name__) # These will be set later self.dc_options = None - self.sender = None + self._sender = None # endregion # region Connecting - def connect(self, timeout=timedelta(seconds=5), exported_auth=None): + def connect(self, exported_auth=None): """Connects to the Telegram servers, executing authentication if required. Note that authenticating to the Telegram servers is not the same as authenticating the desired user itself, which may require a call (or several) to 'sign_in' for the first time. - The specified timeout will be used on internal .invoke()'s. - If 'exported_auth' is not None, it will be used instead to determine the authorization key for the current session. """ - if self.sender and self.sender.is_connected(): + if self._sender and self._sender.is_connected(): self._logger.warning( 'Attempted to connect when the client was already connected.' ) return transport = TcpTransport(self.session.server_address, - self.session.port, proxy=self.proxy) + self.session.port, + proxy=self.proxy, + timeout=self._timeout) try: if not self.session.auth_key: @@ -97,8 +99,8 @@ class TelegramBareClient: self.session.save() - self.sender = MtProtoSender(transport, self.session) - self.sender.connect() + self._sender = MtProtoSender(transport, self.session) + self._sender.connect() # Now it's time to send an InitConnectionRequest # This must always be invoked with the layer we'll be using @@ -117,14 +119,13 @@ class TelegramBareClient: query=query) result = self.invoke( - InvokeWithLayerRequest(layer=layer, query=request), - timeout=timeout + InvokeWithLayerRequest(layer=layer, query=request) ) if exported_auth is not None: # TODO Don't actually need this for exported authorizations, # they're only valid on such data center. - result = self.invoke(GetConfigRequest(), timeout=timeout) + result = self.invoke(GetConfigRequest()) # We're only interested in the DC options, # although many other options are available! @@ -140,9 +141,9 @@ class TelegramBareClient: def disconnect(self): """Disconnects from the Telegram server""" - if self.sender: - self.sender.disconnect() - self.sender = None + if self._sender: + self._sender.disconnect() + self._sender = None def reconnect(self, new_dc=None): """Disconnects and connects again (effectively reconnecting). @@ -163,6 +164,30 @@ class TelegramBareClient: # endregion + # region Properties + + def set_timeout(self, timeout): + if timeout is None: + self._timeout = None + elif isinstance(timeout, int) or isinstance(timeout, float): + self._timeout = timedelta(seconds=timeout) + elif isinstance(timeout, timedelta): + self._timeout = timeout + else: + raise ValueError( + '{} is not a valid type for a timeout'.format(type(timeout)) + ) + + if self._sender: + self._sender.transport.timeout = self._timeout + + def get_timeout(self): + return self._timeout + + timeout = property(get_timeout, set_timeout) + + # endregion + # region Working with different Data Centers def _get_dc(self, dc_id): @@ -178,31 +203,28 @@ class TelegramBareClient: # region Invoking Telegram requests - def invoke(self, request, timeout=timedelta(seconds=5), updates=None): + def invoke(self, request, updates=None): """Invokes (sends) a MTProtoRequest and returns (receives) its result. - An optional timeout can be specified to cancel the operation if no - result is received within such time, or None to disable any timeout. - If 'updates' is not None, all read update object will be put in such list. Otherwise, update objects will be ignored. """ if not isinstance(request, MTProtoRequest): raise ValueError('You can only invoke MtProtoRequests') - if not self.sender: + if not self._sender: raise ValueError('You must be connected to invoke requests!') try: - self.sender.send(request) - self.sender.receive(request, timeout, updates=updates) + self._sender.send(request) + self._sender.receive(request, updates=updates) return request.result except ConnectionResetError: self._logger.info('Server disconnected us. Reconnecting and ' 'resending request...') self.reconnect() - return self.invoke(request, timeout=timeout) + return self.invoke(request) except FloodWaitError: self.disconnect() diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index c8efbe7a..6d3860dd 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -64,7 +64,8 @@ class TelegramClient(TelegramBareClient): def __init__(self, session, api_id, api_hash, proxy=None, device_model=None, system_version=None, - app_version=None, lang_code=None): + app_version=None, lang_code=None, + timeout=timedelta(seconds=5)): """Initializes the Telegram client with the specified API ID and Hash. Session can either be a `str` object (filename for the .session) @@ -78,7 +79,6 @@ class TelegramClient(TelegramBareClient): app_version = TelegramClient.__version__ lang_code = 'en' """ - if not api_id or not api_hash: raise PermissionError( "Your API ID or Hash cannot be empty or None. " @@ -92,7 +92,7 @@ class TelegramClient(TelegramBareClient): raise ValueError( 'The given session must be a str or a Session instance.') - super().__init__(session, api_id, api_hash, proxy) + super().__init__(session, api_id, api_hash, proxy, timeout=timeout) # Safety across multiple threads (for the updates thread) self._lock = RLock() @@ -129,7 +129,7 @@ class TelegramClient(TelegramBareClient): # region Connecting - def connect(self, timeout=timedelta(seconds=5), *args): + def connect(self, *args): """Connects to the Telegram servers, executing authentication if required. Note that authenticating to the Telegram servers is not the same as authenticating the desired user itself, which @@ -195,7 +195,10 @@ class TelegramClient(TelegramBareClient): session = JsonSession(self.session) session.server_address = dc.ip_address session.port = dc.port - client = TelegramBareClient(session, self.api_id, self.api_hash) + client = TelegramBareClient( + session, self.api_id, self.api_hash, + timeout=self._timeout + ) client.connect(exported_auth=export_auth) if not bypass_cache: @@ -233,7 +236,7 @@ class TelegramClient(TelegramBareClient): # region Telegram requests functions - def invoke(self, request, timeout=timedelta(seconds=5), *args): + def invoke(self, request, *args): """Invokes (sends) a MTProtoRequest and returns (receives) its result. An optional timeout can be specified to cancel the operation if no @@ -244,18 +247,19 @@ class TelegramClient(TelegramBareClient): if not issubclass(type(request), MTProtoRequest): raise ValueError('You can only invoke MtProtoRequests') - if not self.sender: + if not self._sender: raise ValueError('You must be connected to invoke requests!') if self._updates_thread_receiving.is_set(): - self.sender.cancel_receive() + self._sender.cancel_receive() try: self._lock.acquire() updates = [] if self._update_handlers else None result = super(TelegramClient, self).invoke( - request, timeout=timeout, updates=updates) + request, updates=updates + ) if updates: for update in updates: @@ -271,13 +275,12 @@ class TelegramClient(TelegramBareClient): .format(e.new_dc)) self.reconnect(new_dc=e.new_dc) - return self.invoke(request, timeout=timeout) + return self.invoke(request) finally: self._lock.release() - def invoke_on_dc(self, request, dc_id, - timeout=timedelta(seconds=5), reconnect=False): + def invoke_on_dc(self, request, dc_id, reconnect=False): """Invokes the given request on a different DC by making use of the exported MtProtoSenders. @@ -294,8 +297,7 @@ class TelegramClient(TelegramBareClient): if reconnect: raise else: - return self.invoke_on_dc(request, dc_id, - timeout=timeout, reconnect=True) + return self.invoke_on_dc(request, dc_id, reconnect=True) # region Authorization requests @@ -374,7 +376,7 @@ class TelegramClient(TelegramBareClient): Returns True if everything went okay.""" # Special flag when logging out (so the ack request confirms it) - self.sender.logging_out = True + self._sender.logging_out = True try: self.invoke(LogOutRequest()) self.disconnect() @@ -385,7 +387,7 @@ class TelegramClient(TelegramBareClient): return True except (RPCError, ConnectionError): # Something happened when logging out, restore the state back - self.sender.logging_out = False + self._sender.logging_out = False return False def get_me(self): @@ -756,7 +758,7 @@ class TelegramClient(TelegramBareClient): def add_update_handler(self, handler): """Adds an update handler (a function which takes a TLObject, an update, as its parameter) and listens for updates""" - if not self.sender: + if not self._sender: raise RuntimeError("You can't add update handlers until you've " "successfully connected to the server.") @@ -791,7 +793,7 @@ class TelegramClient(TelegramBareClient): else: self._updates_thread_running.clear() if self._updates_thread_receiving.is_set(): - self.sender.cancel_receive() + self._sender.cancel_receive() def _updates_thread_method(self): """This method will run until specified and listen for incoming updates""" @@ -817,7 +819,7 @@ class TelegramClient(TelegramBareClient): self._next_ping_at = time() + self.ping_interval self.invoke(PingRequest(utils.generate_random_long())) - updates = self.sender.receive_updates(timeout=timeout) + updates = self._sender.receive_updates(timeout=timeout) self._updates_thread_receiving.clear() self._logger.info( @@ -848,9 +850,9 @@ class TelegramClient(TelegramBareClient): except OSError: self._logger.warning('OSError on updates thread, %s logging out', - 'was' if self.sender.logging_out else 'was not') + 'was' if self._sender.logging_out else 'was not') - if self.sender.logging_out: + if self._sender.logging_out: # This error is okay when logging out, means we got disconnected # TODO Not sure why this happens because we call disconnect()... self._set_updates_thread(running=False) From 843c16215c2bd88ea80ee32acf5eb2c992c0b0da Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 22 Jun 2017 19:21:33 +0200 Subject: [PATCH 09/52] Add timeout parameter on TcpClient.connect() too --- telethon/extensions/tcp_client.py | 7 +++++-- telethon/network/tcp_transport.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index 089066e9..3600ac88 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -30,9 +30,12 @@ class TcpClient: else: # tuple, list, etc. self._socket.set_proxy(*self._proxy) - def connect(self, ip, port): - """Connects to the specified IP and port number""" + def connect(self, ip, port, timeout): + """Connects to the specified IP and port number. + 'timeout' must be given in seconds + """ if not self.connected: + self._socket.settimeout(timeout) self._socket.connect((ip, port)) self._socket.setblocking(False) self.connected = True diff --git a/telethon/network/tcp_transport.py b/telethon/network/tcp_transport.py index 702d2633..76b6b6b9 100644 --- a/telethon/network/tcp_transport.py +++ b/telethon/network/tcp_transport.py @@ -18,7 +18,8 @@ class TcpTransport: def connect(self): """Connects to the specified IP address and port""" self.send_counter = 0 - self.tcp_client.connect(self.ip, self.port) + self.tcp_client.connect(self.ip, self.port, + timeout=round(self.timeout.seconds)) def is_connected(self): return self.tcp_client.connected From 20956b23d1bea8f12f63ef29f18d9460f9c27c5b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 23 Jun 2017 10:15:11 +0200 Subject: [PATCH 10/52] Replace super calls with simply super() --- telethon/telegram_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 6d3860dd..21e0527c 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -139,13 +139,13 @@ class TelegramClient(TelegramBareClient): *args will be ignored. """ - return super(TelegramClient, self).connect() + return super().connect() def disconnect(self): """Disconnects from the Telegram server and stops all the spawned threads""" self._set_updates_thread(running=False) - super(TelegramClient, self).disconnect() + super().disconnect() # Also disconnect all the cached senders for sender in self._cached_clients.values(): @@ -257,7 +257,7 @@ class TelegramClient(TelegramBareClient): self._lock.acquire() updates = [] if self._update_handlers else None - result = super(TelegramClient, self).invoke( + result = super().invoke( request, updates=updates ) @@ -729,7 +729,7 @@ class TelegramClient(TelegramBareClient): """ if on_dc is None: try: - super(TelegramClient, self).download_file( + super().download_file( input_location, file, part_size_kb=part_size_kb, From 459e988ff53f12139c2b4d768c50355cffcd9c31 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 24 Jun 2017 18:10:05 +0200 Subject: [PATCH 11/52] Rename rpc_n_errors dictionary to rpc_errors_n_all --- telethon/errors/__init__.py | 8 ++++---- telethon/errors/rpc_errors_303.py | 2 +- telethon/errors/rpc_errors_400.py | 2 +- telethon/errors/rpc_errors_401.py | 2 +- telethon/errors/rpc_errors_420.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/telethon/errors/__init__.py b/telethon/errors/__init__.py index 08ccee20..cf1093ea 100644 --- a/telethon/errors/__init__.py +++ b/telethon/errors/__init__.py @@ -18,10 +18,10 @@ from .rpc_errors_420 import * def rpc_message_to_error(code, message): errors = { - 303: rpc_303_errors, - 400: rpc_400_errors, - 401: rpc_401_errors, - 420: rpc_420_errors + 303: rpc_errors_303_all, + 400: rpc_errors_400_all, + 401: rpc_errors_401_all, + 420: rpc_errors_420_all }.get(code, None) if errors is not None: diff --git a/telethon/errors/rpc_errors_303.py b/telethon/errors/rpc_errors_303.py index fcef7825..21963154 100644 --- a/telethon/errors/rpc_errors_303.py +++ b/telethon/errors/rpc_errors_303.py @@ -43,7 +43,7 @@ class UserMigrateError(InvalidDCError): ) -rpc_303_errors = { +rpc_errors_303_all = { 'FILE_MIGRATE_(\d+)': FileMigrateError, 'PHONE_MIGRATE_(\d+)': PhoneMigrateError, 'NETWORK_MIGRATE_(\d+)': NetworkMigrateError, diff --git a/telethon/errors/rpc_errors_400.py b/telethon/errors/rpc_errors_400.py index 57966fd6..66e9bfe7 100644 --- a/telethon/errors/rpc_errors_400.py +++ b/telethon/errors/rpc_errors_400.py @@ -321,7 +321,7 @@ class UserIdInvalidError(BadRequestError): ) -rpc_400_errors = { +rpc_errors_400_all = { 'API_ID_INVALID': ApiIdInvalidError, 'BOT_METHOD_INVALID': BotMethodInvalidError, 'CHANNEL_INVALID': ChannelInvalidError, diff --git a/telethon/errors/rpc_errors_401.py b/telethon/errors/rpc_errors_401.py index 03ebb3fb..5b22cb73 100644 --- a/telethon/errors/rpc_errors_401.py +++ b/telethon/errors/rpc_errors_401.py @@ -84,7 +84,7 @@ class UserDeactivatedError(UnauthorizedError): ) -rpc_401_errors = { +rpc_errors_401_all = { 'ACTIVE_USER_REQUIRED': ActiveUserRequiredError, 'AUTH_KEY_INVALID': AuthKeyInvalidError, 'AUTH_KEY_PERM_EMPTY': AuthKeyPermEmptyError, diff --git a/telethon/errors/rpc_errors_420.py b/telethon/errors/rpc_errors_420.py index 3c760bc8..8106cc5c 100644 --- a/telethon/errors/rpc_errors_420.py +++ b/telethon/errors/rpc_errors_420.py @@ -11,6 +11,6 @@ class FloodWaitError(FloodError): ) -rpc_420_errors = { +rpc_errors_420_all = { 'FLOOD_WAIT_(\d+)': FloodWaitError } From a5ce375358361b5f53ce845e71ac0695919dd206 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 24 Jun 2017 18:15:31 +0200 Subject: [PATCH 12/52] Update to v0.11.1 and fix setup.py --- setup.py | 2 +- telethon/telegram_bare_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 099cbafb..ea98e1cd 100755 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ if __name__ == '__main__': else: if not TelegramClient: print('Run `python3', argv[0], 'gen_tl` first.') - quit() + quit() here = path.abspath(path.dirname(__file__)) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 9cd60915..4c2ee9d6 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -47,7 +47,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.11' + __version__ = '0.11.1' # region Initialization From aa7e8dba8a0b86d464bbb567cb1f59e4dbfe21ba Mon Sep 17 00:00:00 2001 From: "Dmitry D. Chernov" Date: Sun, 25 Jun 2017 05:39:43 +1000 Subject: [PATCH 13/52] Fix setup.py failing on Python < 3.6 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ea98e1cd..05f2a9cd 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ from setuptools import find_packages, setup from telethon_generator.tl_generator import TLGenerator try: from telethon import TelegramClient -except ModuleNotFoundError: +except ImportError: TelegramClient = None From b0173c3ec2e19d82cf179e0f1852b2ce71e737da Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 26 Jun 2017 11:00:43 +0200 Subject: [PATCH 14/52] Use more accurate values for msg_ids --- telethon/network/mtproto_plain_sender.py | 18 ++++-------- telethon/tl/session.py | 35 +++++++++--------------- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/telethon/network/mtproto_plain_sender.py b/telethon/network/mtproto_plain_sender.py index 7fae3c20..bbb3e297 100644 --- a/telethon/network/mtproto_plain_sender.py +++ b/telethon/network/mtproto_plain_sender.py @@ -1,4 +1,3 @@ -import random import time from ..extensions import BinaryReader, BinaryWriter @@ -42,17 +41,12 @@ class MtProtoPlainSender: return response def _get_new_msg_id(self): - """Generates a new message ID based on the current time (in ms) since epoch""" - # See https://core.telegram.org/mtproto/description#message-identifier-msg-id - ms_time = int(time.time() * 1000) - new_msg_id = (((ms_time // 1000) << 32) - | # "must approximately equal unix time*2^32" - ((ms_time % 1000) << 22) - | # "approximate moment in time the message was created" - random.randint(0, 524288) - << 2) # "message identifiers are divisible by 4" - - # Ensure that we always return a message ID which is higher than the previous one + """Generates a new message ID based on the current time since epoch""" + # See core.telegram.org/mtproto/description#message-identifier-msg-id + now = time.time() + nanoseconds = int((now - int(now)) * 1e+9) + # "message identifiers are divisible by 4" + new_msg_id = (int(now) << 32) | (nanoseconds << 2) if self._last_msg_id >= new_msg_id: new_msg_id = self._last_msg_id + 4 diff --git a/telethon/tl/session.py b/telethon/tl/session.py index ee5f1091..3dfc2ba6 100644 --- a/telethon/tl/session.py +++ b/telethon/tl/session.py @@ -2,7 +2,6 @@ import json import os import pickle import platform -import random import time from threading import Lock from base64 import b64encode, b64decode @@ -65,15 +64,10 @@ class Session: return self.sequence * 2 def get_new_msg_id(self): - """Generates a new message ID based on the current time (in ms) since epoch""" - # Refer to mtproto_plain_sender.py for the original method, this is a simple copy - ms_time = int(time.time() * 1000) - new_msg_id = (((ms_time // 1000 + self.time_offset) << 32) - | # "must approximately equal unix time*2^32" - ((ms_time % 1000) << 22) - | # "approximate moment in time the message was created" - random.randint(0, 524288) - << 2) # "message identifiers are divisible by 4" + now = time.time() + nanoseconds = int((now - int(now)) * 1e+9) + # "message identifiers are divisible by 4" + new_msg_id = (int(now) << 32) | (nanoseconds << 2) if self.last_message_id >= new_msg_id: new_msg_id = self.last_message_id + 4 @@ -133,7 +127,7 @@ class JsonSession: self._sequence = 0 self.salt = 0 # Unsigned long self.time_offset = 0 - self.last_message_id = 0 # Long + self._last_msg_id = 0 # Long def save(self): """Saves the current session object as session_user_id.session""" @@ -229,19 +223,16 @@ class JsonSession: def get_new_msg_id(self): """Generates a new unique message ID based on the current time (in ms) since epoch""" - # Refer to mtproto_plain_sender.py for the original method, - ms_time = int(time.time() * 1000) - new_msg_id = (((ms_time // 1000 + self.time_offset) << 32) - | # "must approximately equal unix time*2^32" - ((ms_time % 1000) << 22) - | # "approximate moment in time the message was created" - random.randint(0, 524288) - << 2) # "message identifiers are divisible by 4" + # Refer to mtproto_plain_sender.py for the original method + now = time.time() + nanoseconds = int((now - int(now)) * 1e+9) + # "message identifiers are divisible by 4" + new_msg_id = (int(now) << 32) | (nanoseconds << 2) - if self.last_message_id >= new_msg_id: - new_msg_id = self.last_message_id + 4 + if self._last_msg_id >= new_msg_id: + new_msg_id = self._last_msg_id + 4 - self.last_message_id = new_msg_id + self._last_msg_id = new_msg_id return new_msg_id def update_time_offset(self, correct_msg_id): From 0cfbf63eaf9bf706d1609b08f88ed08219008229 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 27 Jun 2017 10:18:35 +0200 Subject: [PATCH 15/52] Fix ConnectionResetError not flagging the socket as disconnected --- telethon/extensions/tcp_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index 3600ac88..27cc0d93 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -96,8 +96,9 @@ class TcpClient: try: partial = self._socket.recv(bytes_left) if len(partial) == 0: + self.connected = False raise ConnectionResetError( - 'The server has closed the connection (recv() returned 0 bytes).') + 'The server has closed the connection.') buffer.write(partial) bytes_left -= len(partial) From 83c8e9844893bab219b4b0b48d99053fb779d485 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 27 Jun 2017 18:45:52 +0200 Subject: [PATCH 16/52] Ensure that message ids are signed once again --- telethon/network/mtproto_sender.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 37042e13..0266e7b4 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -257,7 +257,7 @@ class MtProtoSender: def _handle_pong(self, msg_id, sequence, reader): self._logger.debug('Handling pong') reader.read_int(signed=False) # code - received_msg_id = reader.read_long(signed=False) + received_msg_id = reader.read_long() try: request = next(r for r in self._pending_receive @@ -274,7 +274,7 @@ class MtProtoSender: reader.read_int(signed=False) # code size = reader.read_int() for _ in range(size): - inner_msg_id = reader.read_long(signed=False) + inner_msg_id = reader.read_long() reader.read_int() # inner_sequence inner_length = reader.read_int() begin_position = reader.tell_position() @@ -290,7 +290,7 @@ class MtProtoSender: def _handle_bad_server_salt(self, msg_id, sequence, reader): self._logger.debug('Handling bad server salt') reader.read_int(signed=False) # code - bad_msg_id = reader.read_long(signed=False) + bad_msg_id = reader.read_long() reader.read_int() # bad_msg_seq_no reader.read_int() # error_code new_salt = reader.read_long(signed=False) From 79ee98a4ddc6972b75296ffda6b0db091c7cbd3f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 30 Jun 2017 11:28:54 +0200 Subject: [PATCH 17/52] Infer the object ID from its TL definition when not given --- telethon_generator/parser/tl_object.py | 58 +++++++++++++++++++++----- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/telethon_generator/parser/tl_object.py b/telethon_generator/parser/tl_object.py index 3d83d192..57cc341d 100644 --- a/telethon_generator/parser/tl_object.py +++ b/telethon_generator/parser/tl_object.py @@ -1,4 +1,5 @@ import re +from zlib import crc32 class TLObject: @@ -24,12 +25,18 @@ class TLObject: self.namespace = None self.name = fullname - # The ID should be an hexadecimal string - self.id = int(object_id, base=16) self.args = args self.result = result self.is_function = is_function + # The ID should be an hexadecimal string or None to be inferred + if object_id is None: + self.id = self.infer_id() + else: + self.id = int(object_id, base=16) + assert self.id == self.infer_id(),\ + 'Invalid inferred ID for ' + repr(self) + @staticmethod def from_tl(tl, is_function): """Returns a TL object from the given TL scheme line""" @@ -38,8 +45,10 @@ class TLObject: match = re.match(r''' ^ # We want to match from the beginning to the end ([\w.]+) # The .tl object can contain alpha_name or namespace.alpha_name - \# # After the name, comes the ID of the object - ([0-9a-f]+) # The constructor ID is in hexadecimal form + (?: + \# # After the name, comes the ID of the object + ([0-9a-f]+) # The constructor ID is in hexadecimal form + )? # If no constructor ID was given, CRC32 the 'tl' to determine it (?:\s # After that, we want to match its arguments (name:type) {? # For handling the start of the '{X:Type}' case @@ -91,16 +100,39 @@ class TLObject: (and thus should be embedded in the generated code) or not""" return self.id in TLObject.CORE_TYPES - def __repr__(self): + def __repr__(self, ignore_id=False): fullname = ('{}.{}'.format(self.namespace, self.name) if self.namespace is not None else self.name) - hex_id = hex(self.id)[2:].rjust(8, - '0') # Skip 0x and add 0's for padding + if getattr(self, 'id', None) is None or ignore_id: + hex_id = '' + else: + # Skip 0x and add 0's for padding + hex_id = '#' + hex(self.id)[2:].rjust(8, '0') - return '{}#{} {} = {}'.format( - fullname, hex_id, ' '.join([str(arg) for arg in self.args]), - self.result) + if self.args: + args = ' ' + ' '.join([repr(arg) for arg in self.args]) + else: + args = '' + + return '{}{}{} = {}'.format(fullname, hex_id, args, self.result) + + def infer_id(self): + representation = self.__repr__(ignore_id=True) + + # Clean the representation + representation = representation\ + .replace(':bytes ', ':string ')\ + .replace('?bytes ', '?string ')\ + .replace('<', ' ').replace('>', '')\ + .replace('{', '').replace('}', '') + + representation = re.sub( + r' \w+:flags\.\d+\?true', + r'', + representation + ) + return crc32(representation.encode('ascii')) def __str__(self): fullname = ('{}.{}'.format(self.namespace, self.name) @@ -214,3 +246,9 @@ class TLArg: return '{{{}:{}}}'.format(self.name, real_type) else: return '{}:{}'.format(self.name, real_type) + + def __repr__(self): + # Get rid of our special type + return str(self)\ + .replace(':date', ':int')\ + .replace('?date', '?int') From 23e280221548d8170ef6fd08af9d8816ceae0c6a Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 30 Jun 2017 11:48:45 +0200 Subject: [PATCH 18/52] Update to layer 68 --- telethon/errors/rpc_errors_400.py | 8 +++ telethon/telegram_bare_client.py | 2 + telethon/telegram_client.py | 13 +++-- telethon/tl/session.py | 4 ++ telethon_generator/scheme.tl | 93 ++++++++++++++++++++++++------- 5 files changed, 95 insertions(+), 25 deletions(-) diff --git a/telethon/errors/rpc_errors_400.py b/telethon/errors/rpc_errors_400.py index 66e9bfe7..d55e0053 100644 --- a/telethon/errors/rpc_errors_400.py +++ b/telethon/errors/rpc_errors_400.py @@ -44,6 +44,14 @@ class ChatIdInvalidError(BadRequestError): ) +class ConnectionLangPackInvalid(BadRequestError): + def __init__(self, **kwargs): + super(Exception, self).__init__( + self, + 'The specified language pack is not valid.' + ) + + class ConnectionLayerInvalidError(BadRequestError): def __init__(self, **kwargs): super(Exception, self).__init__( diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 4c2ee9d6..d2e9407a 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -116,6 +116,8 @@ class TelegramBareClient: system_version=self.session.system_version, app_version=self.session.app_version, lang_code=self.session.lang_code, + system_lang_code=self.session.system_lang_code, + lang_pack='', # "langPacks are for official apps only" query=query) result = self.invoke( diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 21e0527c..3ef10512 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -65,6 +65,7 @@ class TelegramClient(TelegramBareClient): def __init__(self, session, api_id, api_hash, proxy=None, device_model=None, system_version=None, app_version=None, lang_code=None, + system_lang_code=None, timeout=timedelta(seconds=5)): """Initializes the Telegram client with the specified API ID and Hash. @@ -74,10 +75,11 @@ class TelegramClient(TelegramBareClient): session - remember to '.log_out()'! Default values for the optional parameters if left as None are: - device_model = platform.node() - system_version = platform.system() - app_version = TelegramClient.__version__ - lang_code = 'en' + device_model = platform.node() + system_version = platform.system() + app_version = TelegramClient.__version__ + lang_code = 'en' + system_lang_code = lang_code """ if not api_id or not api_hash: raise PermissionError( @@ -118,6 +120,9 @@ class TelegramClient(TelegramBareClient): if lang_code: self.session.lang_code = lang_code + self.session.system_lang_code = \ + system_lang_code if system_lang_code else self.session.lang_code + # Cache "exported" senders 'dc_id: MtProtoSender' and # their corresponding sessions not to recreate them all # the time since it's a (somewhat expensive) process. diff --git a/telethon/tl/session.py b/telethon/tl/session.py index 3dfc2ba6..22638806 100644 --- a/telethon/tl/session.py +++ b/telethon/tl/session.py @@ -107,6 +107,8 @@ class JsonSession: self.system_version = session.system_version self.app_version = session.app_version self.lang_code = session.lang_code + self.system_lang_code = session.system_lang_code + self.lang_pack = session.lang_pack else: # str / None self.session_user_id = session_user_id @@ -115,6 +117,8 @@ class JsonSession: self.system_version = platform.system() self.app_version = '1.0' # note: '0' will provoke error self.lang_code = 'en' + self.system_lang_code = self.lang_code + self.lang_pack = '' # Cross-thread safety self._lock = Lock() diff --git a/telethon_generator/scheme.tl b/telethon_generator/scheme.tl index 3807a626..b2e1774e 100644 --- a/telethon_generator/scheme.tl +++ b/telethon_generator/scheme.tl @@ -106,6 +106,9 @@ new_session_created#9ec20908 first_msg_id:long unique_id:long server_salt:long = http_wait#9299359f max_delay:int wait_after:int max_wait:int = HttpWait; +ipPort ipv4:int port:int = IpPort; +help.configSimple#d997c3c5 date:int expires:int dc_id:int ip_port_list:Vector = help.ConfigSimple; + ---functions--- rpc_drop_answer#58e4a740 req_msg_id:long = RpcDropAnswer; @@ -216,11 +219,11 @@ userStatusLastMonth#77ebc742 = UserStatus; chatEmpty#9ba2d800 id:int = Chat; chat#d91cdd54 flags:# creator:flags.0?true kicked:flags.1?true left:flags.2?true admins_enabled:flags.3?true admin:flags.4?true deactivated:flags.5?true id:int title:string photo:ChatPhoto participants_count:int date:int version:int migrated_to:flags.6?InputChannel = Chat; chatForbidden#7328bdb id:int title:string = Chat; -channel#a14dca52 flags:# creator:flags.0?true kicked:flags.1?true left:flags.2?true editor:flags.3?true moderator:flags.4?true broadcast:flags.5?true verified:flags.7?true megagroup:flags.8?true restricted:flags.9?true democracy:flags.10?true signatures:flags.11?true min:flags.12?true id:int access_hash:flags.13?long title:string username:flags.6?string photo:ChatPhoto date:int version:int restriction_reason:flags.9?string = Chat; -channelForbidden#8537784f flags:# broadcast:flags.5?true megagroup:flags.8?true id:int access_hash:long title:string = Chat; +channel#cb44b1c flags:# creator:flags.0?true left:flags.2?true broadcast:flags.5?true verified:flags.7?true megagroup:flags.8?true restricted:flags.9?true democracy:flags.10?true signatures:flags.11?true min:flags.12?true id:int access_hash:flags.13?long title:string username:flags.6?string photo:ChatPhoto date:int version:int restriction_reason:flags.9?string admin_rights:flags.14?ChannelAdminRights banned_rights:flags.15?ChannelBannedRights = Chat; +channelForbidden#289da732 flags:# broadcast:flags.5?true megagroup:flags.8?true id:int access_hash:long title:string until_date:flags.16?int = Chat; chatFull#2e02a614 id:int participants:ChatParticipants chat_photo:Photo notify_settings:PeerNotifySettings exported_invite:ExportedChatInvite bot_info:Vector = ChatFull; -channelFull#c3d5512f flags:# can_view_participants:flags.3?true can_set_username:flags.6?true id:int about:string participants_count:flags.0?int admins_count:flags.1?int kicked_count:flags.2?int read_inbox_max_id:int read_outbox_max_id:int unread_count:int chat_photo:Photo notify_settings:PeerNotifySettings exported_invite:ExportedChatInvite bot_info:Vector migrated_from_chat_id:flags.4?int migrated_from_max_id:flags.4?int pinned_msg_id:flags.5?int = ChatFull; +channelFull#95cb5f57 flags:# can_view_participants:flags.3?true can_set_username:flags.6?true id:int about:string participants_count:flags.0?int admins_count:flags.1?int kicked_count:flags.2?int banned_count:flags.2?int read_inbox_max_id:int read_outbox_max_id:int unread_count:int chat_photo:Photo notify_settings:PeerNotifySettings exported_invite:ExportedChatInvite bot_info:Vector migrated_from_chat_id:flags.4?int migrated_from_max_id:flags.4?int pinned_msg_id:flags.5?int = ChatFull; chatParticipant#c8d7493e user_id:int inviter_id:int date:int = ChatParticipant; chatParticipantCreator#da13538a user_id:int = ChatParticipant; @@ -420,6 +423,8 @@ updateBotWebhookJSONQuery#9b9240a6 query_id:long data:DataJSON timeout:int = Upd updateBotShippingQuery#e0cdc940 query_id:long user_id:int payload:bytes shipping_address:PostAddress = Update; updateBotPrecheckoutQuery#5d2f3aa9 flags:# query_id:long user_id:int payload:bytes info:flags.0?PaymentRequestedInfo shipping_option_id:flags.1?string currency:string total_amount:long = Update; updatePhoneCall#ab0f6b1e phone_call:PhoneCall = Update; +updateLangPackTooLong#10c2404b = Update; +updateLangPack#56022f4d difference:LangPackDifference = Update; updates.state#a56c2a3e pts:int qts:int date:int seq:int unread_count:int = updates.State; @@ -444,9 +449,9 @@ photos.photo#20212ca8 photo:Photo users:Vector = photos.Photo; upload.file#96a18d5 type:storage.FileType mtime:int bytes:bytes = upload.File; upload.fileCdnRedirect#1508485a dc_id:int file_token:bytes encryption_key:bytes encryption_iv:bytes = upload.File; -dcOption#5d8c6cc flags:# ipv6:flags.0?true media_only:flags.1?true tcpo_only:flags.2?true cdn:flags.3?true id:int ip_address:string port:int = DcOption; +dcOption#5d8c6cc flags:# ipv6:flags.0?true media_only:flags.1?true tcpo_only:flags.2?true cdn:flags.3?true static:flags.4?true id:int ip_address:string port:int = DcOption; -config#cb601684 flags:# phonecalls_enabled:flags.1?true date:int expires:int test_mode:Bool this_dc:int dc_options:Vector chat_size_max:int megagroup_size_max:int forwarded_count_max:int online_update_period_ms:int offline_blur_timeout_ms:int offline_idle_timeout_ms:int online_cloud_timeout_ms:int notify_cloud_delay_ms:int notify_default_delay_ms:int chat_big_size:int push_chat_period_ms:int push_chat_limit:int saved_gifs_limit:int edit_time_limit:int rating_e_decay:int stickers_recent_limit:int tmp_sessions:flags.0?int pinned_dialogs_count_max:int call_receive_timeout_ms:int call_ring_timeout_ms:int call_connect_timeout_ms:int call_packet_timeout_ms:int me_url_prefix:string disabled_features:Vector = Config; +config#7feec888 flags:# phonecalls_enabled:flags.1?true date:int expires:int test_mode:Bool this_dc:int dc_options:Vector chat_size_max:int megagroup_size_max:int forwarded_count_max:int online_update_period_ms:int offline_blur_timeout_ms:int offline_idle_timeout_ms:int online_cloud_timeout_ms:int notify_cloud_delay_ms:int notify_default_delay_ms:int chat_big_size:int push_chat_period_ms:int push_chat_limit:int saved_gifs_limit:int edit_time_limit:int rating_e_decay:int stickers_recent_limit:int tmp_sessions:flags.0?int pinned_dialogs_count_max:int call_receive_timeout_ms:int call_ring_timeout_ms:int call_connect_timeout_ms:int call_packet_timeout_ms:int me_url_prefix:string suggested_lang_code:flags.2?string lang_pack_version:flags.2?int disabled_features:Vector = Config; nearestDc#8e1a1775 country:string this_dc:int nearest_dc:int = NearestDc; @@ -644,19 +649,16 @@ channelMessagesFilter#cd77d957 flags:# exclude_new_messages:flags.1?true ranges: channelParticipant#15ebac1d user_id:int date:int = ChannelParticipant; channelParticipantSelf#a3289a6d user_id:int inviter_id:int date:int = ChannelParticipant; -channelParticipantModerator#91057fef user_id:int inviter_id:int date:int = ChannelParticipant; -channelParticipantEditor#98192d61 user_id:int inviter_id:int date:int = ChannelParticipant; -channelParticipantKicked#8cc5e69a user_id:int kicked_by:int date:int = ChannelParticipant; channelParticipantCreator#e3e2e1f9 user_id:int = ChannelParticipant; +channelParticipantAdmin#a82fa898 flags:# can_edit:flags.0?true user_id:int inviter_id:int promoted_by:int date:int admin_rights:ChannelAdminRights = ChannelParticipant; +channelParticipantBanned#222c1886 flags:# left:flags.0?true user_id:int kicked_by:int date:int banned_rights:ChannelBannedRights = ChannelParticipant; channelParticipantsRecent#de3f3c79 = ChannelParticipantsFilter; channelParticipantsAdmins#b4608969 = ChannelParticipantsFilter; -channelParticipantsKicked#3c37bb7a = ChannelParticipantsFilter; +channelParticipantsKicked#a3b54985 q:string = ChannelParticipantsFilter; channelParticipantsBots#b0d1865b = ChannelParticipantsFilter; - -channelRoleEmpty#b285a0c6 = ChannelParticipantRole; -channelRoleModerator#9618d975 = ChannelParticipantRole; -channelRoleEditor#820bfe8c = ChannelParticipantRole; +channelParticipantsBanned#1427a5e1 q:string = ChannelParticipantsFilter; +channelParticipantsSearch#656ac4b q:string = ChannelParticipantsFilter; channels.channelParticipants#f56ee2a8 count:int participants:Vector users:Vector = channels.ChannelParticipants; @@ -725,6 +727,7 @@ topPeerCategoryBotsInline#148677e2 = TopPeerCategory; topPeerCategoryCorrespondents#637b7ed = TopPeerCategory; topPeerCategoryGroups#bd17a14a = TopPeerCategory; topPeerCategoryChannels#161d9628 = TopPeerCategory; +topPeerCategoryPhoneCalls#1e76a78c = TopPeerCategory; topPeerCategoryPeers#fb834291 category:TopPeerCategory count:int peers:Vector = TopPeerCategoryPeers; @@ -795,9 +798,10 @@ pageBlockEmbedPost#292c7be9 url:string webpage_id:long author_photo_id:long auth pageBlockCollage#8b31c4f items:Vector caption:RichText = PageBlock; pageBlockSlideshow#130c8963 items:Vector caption:RichText = PageBlock; pageBlockChannel#ef1751b5 channel:Chat = PageBlock; +pageBlockAudio#31b81a7f audio_id:long caption:RichText = PageBlock; -pagePart#8dee6c44 blocks:Vector photos:Vector videos:Vector = Page; -pageFull#d7a19d69 blocks:Vector photos:Vector videos:Vector = Page; +pagePart#8e3f9ebe blocks:Vector photos:Vector documents:Vector = Page; +pageFull#556ec7aa blocks:Vector photos:Vector documents:Vector = Page; phoneCallDiscardReasonMissed#85e42301 = PhoneCallDiscardReason; phoneCallDiscardReasonDisconnect#e095c1a0 = PhoneCallDiscardReason; @@ -844,6 +848,8 @@ account.tmpPassword#db64fd34 tmp_password:bytes valid_until:int = account.TmpPas shippingOption#b6213cdf id:string title:string prices:Vector = ShippingOption; +inputStickerSetItem#ffa0a496 flags:# document:InputDocument emoji:string mask_coords:flags.0?MaskCoords = InputStickerSetItem; + inputPhoneCall#1e36fded id:long access_hash:long = InputPhoneCall; phoneCallEmpty#5366c915 id:long = PhoneCall; @@ -866,11 +872,44 @@ cdnPublicKey#c982eaba dc_id:int public_key:string = CdnPublicKey; cdnConfig#5725e40a public_keys:Vector = CdnConfig; +langPackString#cad181f6 key:string value:string = LangPackString; +langPackStringPluralized#6c47ac9f flags:# key:string zero_value:flags.0?string one_value:flags.1?string two_value:flags.2?string few_value:flags.3?string many_value:flags.4?string other_value:string = LangPackString; +langPackStringDeleted#2979eeb2 key:string = LangPackString; + +langPackDifference#f385c1f6 lang_code:string from_version:int version:int strings:Vector = LangPackDifference; + +langPackLanguage#117698f1 name:string native_name:string lang_code:string = LangPackLanguage; + +channelAdminRights#5d7ceba5 flags:# change_info:flags.0?true post_messages:flags.1?true edit_messages:flags.2?true delete_messages:flags.3?true ban_users:flags.4?true invite_users:flags.5?true invite_link:flags.6?true pin_messages:flags.7?true add_admins:flags.9?true = ChannelAdminRights; + +channelBannedRights#58cf4249 flags:# view_messages:flags.0?true send_messages:flags.1?true send_media:flags.2?true send_stickers:flags.3?true send_gifs:flags.4?true send_games:flags.5?true send_inline:flags.6?true embed_links:flags.7?true until_date:int = ChannelBannedRights; + +channelAdminLogEventActionChangeTitle#e6dfb825 prev_value:string new_value:string = ChannelAdminLogEventAction; +channelAdminLogEventActionChangeAbout#55188a2e prev_value:string new_value:string = ChannelAdminLogEventAction; +channelAdminLogEventActionChangeUsername#6a4afc38 prev_value:string new_value:string = ChannelAdminLogEventAction; +channelAdminLogEventActionChangePhoto#b82f55c3 prev_photo:ChatPhoto new_photo:ChatPhoto = ChannelAdminLogEventAction; +channelAdminLogEventActionToggleInvites#1b7907ae new_value:Bool = ChannelAdminLogEventAction; +channelAdminLogEventActionToggleSignatures#26ae0971 new_value:Bool = ChannelAdminLogEventAction; +channelAdminLogEventActionUpdatePinned#e9e82c18 message:Message = ChannelAdminLogEventAction; +channelAdminLogEventActionEditMessage#709b2405 prev_message:Message new_message:Message = ChannelAdminLogEventAction; +channelAdminLogEventActionDeleteMessage#42e047bb message:Message = ChannelAdminLogEventAction; +channelAdminLogEventActionParticipantJoin#183040d3 = ChannelAdminLogEventAction; +channelAdminLogEventActionParticipantLeave#f89777f2 = ChannelAdminLogEventAction; +channelAdminLogEventActionParticipantInvite#e31c34d8 participant:ChannelParticipant = ChannelAdminLogEventAction; +channelAdminLogEventActionParticipantToggleBan#e6d83d7e prev_participant:ChannelParticipant new_participant:ChannelParticipant = ChannelAdminLogEventAction; +channelAdminLogEventActionParticipantToggleAdmin#d5676710 prev_participant:ChannelParticipant new_participant:ChannelParticipant = ChannelAdminLogEventAction; + +channelAdminLogEvent#3b5a3e40 id:long date:int user_id:int action:ChannelAdminLogEventAction = ChannelAdminLogEvent; + +channels.adminLogResults#ed8af74d events:Vector chats:Vector users:Vector = channels.AdminLogResults; + +channelAdminLogEventsFilter#ea107ae4 flags:# join:flags.0?true leave:flags.1?true invite:flags.2?true ban:flags.3?true unban:flags.4?true kick:flags.5?true unkick:flags.6?true promote:flags.7?true demote:flags.8?true info:flags.9?true settings:flags.10?true pinned:flags.11?true edit:flags.12?true delete:flags.13?true = ChannelAdminLogEventsFilter; + ---functions--- invokeAfterMsg#cb9f372d {X:Type} msg_id:long query:!X = X; invokeAfterMsgs#3dc4b4f0 {X:Type} msg_ids:Vector query:!X = X; -initConnection#69796de9 {X:Type} api_id:int device_model:string system_version:string app_version:string lang_code:string query:!X = X; +initConnection#c7481da6 {X:Type} api_id:int device_model:string system_version:string app_version:string system_lang_code:string lang_pack:string lang_code:string query:!X = X; invokeWithLayer#da9b0d0d {X:Type} layer:int query:!X = X; invokeWithoutUpdates#bf9459b7 {X:Type} query:!X = X; @@ -935,13 +974,13 @@ contacts.exportCard#84e53737 = Vector; contacts.importCard#4fe196fe export_card:Vector = User; contacts.search#11f812d8 q:string limit:int = contacts.Found; contacts.resolveUsername#f93ccba3 username:string = contacts.ResolvedPeer; -contacts.getTopPeers#d4982db5 flags:# correspondents:flags.0?true bots_pm:flags.1?true bots_inline:flags.2?true groups:flags.10?true channels:flags.15?true offset:int limit:int hash:int = contacts.TopPeers; +contacts.getTopPeers#d4982db5 flags:# correspondents:flags.0?true bots_pm:flags.1?true bots_inline:flags.2?true phone_calls:flags.3?true groups:flags.10?true channels:flags.15?true offset:int limit:int hash:int = contacts.TopPeers; contacts.resetTopPeerRating#1ae373ac category:TopPeerCategory peer:InputPeer = Bool; messages.getMessages#4222fa74 id:Vector = messages.Messages; messages.getDialogs#191ba9c5 flags:# exclude_pinned:flags.0?true offset_date:int offset_id:int offset_peer:InputPeer limit:int = messages.Dialogs; messages.getHistory#afa92846 peer:InputPeer offset_id:int offset_date:int add_offset:int limit:int max_id:int min_id:int = messages.Messages; -messages.search#d4569248 flags:# peer:InputPeer q:string filter:MessagesFilter min_date:int max_date:int offset:int max_id:int limit:int = messages.Messages; +messages.search#f288a275 flags:# peer:InputPeer q:string from_id:flags.0?InputUser filter:MessagesFilter min_date:int max_date:int offset:int max_id:int limit:int = messages.Messages; messages.readHistory#e306d3a peer:InputPeer max_id:int = messages.AffectedMessages; messages.deleteHistory#1c015b09 flags:# just_clear:flags.0?true peer:InputPeer max_id:int = messages.AffectedHistory; messages.deleteMessages#e58e95d2 flags:# revoke:flags.0?true id:Vector = messages.AffectedMessages; @@ -1023,6 +1062,7 @@ messages.reorderPinnedDialogs#959ff644 flags:# force:flags.0?true order:Vector = Bool; messages.setBotPrecheckoutResults#9c2dd95 flags:# success:flags.1?true query_id:long error:flags.0?string = Bool; +messages.uploadMedia#519bc2b1 peer:InputPeer media:InputMedia = MessageMedia; updates.getState#edd4882a = updates.State; updates.getDifference#25939651 flags:# pts:int pts_total_limit:flags.0?int date:int qts:int = updates.Difference; @@ -1062,7 +1102,7 @@ channels.getChannels#a7f6bbb id:Vector = messages.Chats; channels.getFullChannel#8736a09 channel:InputChannel = messages.ChatFull; channels.createChannel#f4893d7f flags:# broadcast:flags.0?true megagroup:flags.1?true title:string about:string = Updates; channels.editAbout#13e27f1e channel:InputChannel about:string = Bool; -channels.editAdmin#eb7611d0 channel:InputChannel user_id:InputUser role:ChannelParticipantRole = Updates; +channels.editAdmin#20b88214 channel:InputChannel user_id:InputUser admin_rights:ChannelAdminRights = Updates; channels.editTitle#566decd0 channel:InputChannel title:string = Updates; channels.editPhoto#f12e57c9 channel:InputChannel photo:InputChatPhoto = Updates; channels.checkUsername#10e6bd2c channel:InputChannel username:string = Bool; @@ -1070,7 +1110,6 @@ channels.updateUsername#3514b3de channel:InputChannel username:string = Bool; channels.joinChannel#24b524c5 channel:InputChannel = Updates; channels.leaveChannel#f836aa95 channel:InputChannel = Updates; channels.inviteToChannel#199f3a6c channel:InputChannel users:Vector = Updates; -channels.kickFromChannel#a672de14 channel:InputChannel user_id:InputUser kicked:Bool = Updates; channels.exportInvite#c7560885 channel:InputChannel = ExportedChatInvite; channels.deleteChannel#c0111fe3 channel:InputChannel = Updates; channels.toggleInvites#49609307 channel:InputChannel enabled:Bool = Updates; @@ -1078,6 +1117,8 @@ channels.exportMessageLink#c846d22d channel:InputChannel id:int = ExportedMessag channels.toggleSignatures#1f69b606 channel:InputChannel enabled:Bool = Updates; channels.updatePinnedMessage#a72ded52 flags:# silent:flags.0?true channel:InputChannel id:int = Updates; channels.getAdminedPublicChannels#8d8d82d7 = messages.Chats; +channels.editBanned#bfd915cd channel:InputChannel user_id:InputUser banned_rights:ChannelBannedRights = Updates; +channels.getAdminLog#33ddf480 flags:# channel:InputChannel q:string events_filter:flags.0?ChannelAdminLogEventsFilter admins:flags.1?Vector max_id:long min_id:long limit:int = channels.AdminLogResults; bots.sendCustomRequest#aa2769ed custom_method:string params:DataJSON = DataJSON; bots.answerWebhookJSONQuery#e6213f4d query_id:long data:DataJSON = Bool; @@ -1089,6 +1130,11 @@ payments.sendPaymentForm#2b8879b3 flags:# msg_id:int requested_info_id:flags.0?s payments.getSavedInfo#227d824b = payments.SavedInfo; payments.clearSavedInfo#d83d70c1 flags:# credentials:flags.0?true info:flags.1?true = Bool; +stickers.createStickerSet#9bd86e6a flags:# masks:flags.0?true user_id:InputUser title:string short_name:string stickers:Vector = messages.StickerSet; +stickers.removeStickerFromSet#4255934 sticker:InputDocument = Bool; +stickers.changeStickerPosition#4ed705ca sticker:InputDocument position:int = Bool; +stickers.addStickerToSet#8653febe stickerset:InputStickerSet sticker:InputStickerSetItem = messages.StickerSet; + phone.getCallConfig#55451fa9 = DataJSON; phone.requestCall#5b95b3d4 user_id:InputUser random_id:int g_a_hash:bytes protocol:PhoneCallProtocol = phone.PhoneCall; phone.acceptCall#3bd2b4a0 peer:InputPhoneCall g_b:bytes protocol:PhoneCallProtocol = phone.PhoneCall; @@ -1098,4 +1144,9 @@ phone.discardCall#78d413a6 peer:InputPhoneCall duration:int reason:PhoneCallDisc phone.setCallRating#1c536a34 peer:InputPhoneCall rating:int comment:string = Updates; phone.saveCallDebug#277add7e peer:InputPhoneCall debug:DataJSON = Bool; -// LAYER 66 +langpack.getLangPack#9ab5c58e lang_code:string = LangPackDifference; +langpack.getStrings#2e1ee318 lang_code:string keys:Vector = Vector; +langpack.getDifference#b2e4d7d from_version:int = LangPackDifference; +langpack.getLanguages#800fd57d = Vector; + +// LAYER 68 From 15673d9f776ec0809dc264f2ba522c3dcdd3af76 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 2 Jul 2017 11:56:40 +0200 Subject: [PATCH 19/52] Let __call__ = invoke, and encourage this new way to invoke requests --- README.rst | 7 ++-- telethon/telegram_bare_client.py | 15 +++++---- telethon/telegram_client.py | 57 +++++++++++++++++--------------- 3 files changed, 45 insertions(+), 34 deletions(-) diff --git a/README.rst b/README.rst index 88251354..1565670f 100755 --- a/README.rst +++ b/README.rst @@ -160,13 +160,16 @@ The ``TelegramClient`` class should be used to provide a quick, well-documented It is **not** meant to be a place for *all* the available Telegram ``Request``'s, because there are simply too many. However, this doesn't mean that you cannot ``invoke`` all the power of Telegram's API. -Whenever you need to ``invoke`` a Telegram ``Request``, all you need to do is the following: +Whenever you need to ``call`` a Telegram ``Request``, all you need to do is the following: .. code:: python + result = client(SomeRequest(...)) + + # Or the old way: result = client.invoke(SomeRequest(...)) -You have just ``invoke``'d ``SomeRequest`` and retrieved its ``result``! That wasn't hard at all, was it? +You have just called ``SomeRequest`` and retrieved its ``result``! That wasn't hard at all, was it? Now you may wonder, what's the deal with *all the power of Telegram's API*? Have a look under ``tl/functions/``. That is *everything* you can do. You have **over 200 API** ``Request``'s at your disposal. diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index d2e9407a..11ef5483 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -120,14 +120,14 @@ class TelegramBareClient: lang_pack='', # "langPacks are for official apps only" query=query) - result = self.invoke( - InvokeWithLayerRequest(layer=layer, query=request) - ) + result = self(InvokeWithLayerRequest( + layer=layer, query=request + )) if exported_auth is not None: # TODO Don't actually need this for exported authorizations, # they're only valid on such data center. - result = self.invoke(GetConfigRequest()) + result = self(GetConfigRequest()) # We're only interested in the DC options, # although many other options are available! @@ -232,6 +232,9 @@ class TelegramBareClient: self.disconnect() raise + # Let people use client(SomeRequest()) instead client.invoke(...) + __call__ = invoke + # endregion # region Uploading media @@ -283,7 +286,7 @@ class TelegramBareClient: else: request = SaveFilePartRequest(file_id, part_index, part) - result = self.invoke(request) + result = self(request) if result: if not is_large: # No need to update the hash if it's a large file @@ -342,7 +345,7 @@ class TelegramBareClient: offset_index = 0 while True: offset = offset_index * part_size - result = self.invoke( + result = self( GetFileRequest(input_location, offset, part_size)) offset_index += 1 diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 3ef10512..5d0362b0 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -190,7 +190,7 @@ class TelegramClient(TelegramBareClient): dc = self._get_dc(dc_id) # Export the current authorization to the new DC. - export_auth = self.invoke(ExportAuthorizationRequest(dc_id)) + export_auth = self(ExportAuthorizationRequest(dc_id)) # Create a temporary session for this IP address, which needs # to be different because each auth_key is unique per DC. @@ -285,6 +285,9 @@ class TelegramClient(TelegramBareClient): finally: self._lock.release() + # Let people use client(SomeRequest()) instead client.invoke(...) + __call__ = invoke + def invoke_on_dc(self, request, dc_id, reconnect=False): """Invokes the given request on a different DC by making use of the exported MtProtoSenders. @@ -313,7 +316,7 @@ class TelegramClient(TelegramBareClient): def send_code_request(self, phone_number): """Sends a code request to the specified phone number""" - result = self.invoke( + result = self( SendCodeRequest(phone_number, self.api_id, self.api_hash)) self._phone_code_hashes[phone_number] = result.phone_code_hash @@ -339,7 +342,7 @@ class TelegramClient(TelegramBareClient): 'Please make sure to call send_code_request first.') try: - result = self.invoke(SignInRequest( + result = self(SignInRequest( phone_number, self._phone_code_hashes[phone_number], code)) except (PhoneCodeEmptyError, PhoneCodeExpiredError, @@ -347,12 +350,12 @@ class TelegramClient(TelegramBareClient): return None elif password: - salt = self.invoke(GetPasswordRequest()).current_salt - result = self.invoke( + salt = self(GetPasswordRequest()).current_salt + result = self( CheckPasswordRequest(utils.get_password_hash(password, salt))) elif bot_token: - result = self.invoke(ImportBotAuthorizationRequest( + result = self(ImportBotAuthorizationRequest( flags=0, bot_auth_token=bot_token, api_id=self.api_id, api_hash=self.api_hash)) @@ -365,7 +368,7 @@ class TelegramClient(TelegramBareClient): def sign_up(self, phone_number, code, first_name, last_name=''): """Signs up to Telegram. Make sure you sent a code request first!""" - result = self.invoke( + result = self( SignUpRequest( phone_number=phone_number, phone_code_hash=self._phone_code_hashes[phone_number], @@ -383,7 +386,7 @@ class TelegramClient(TelegramBareClient): # Special flag when logging out (so the ack request confirms it) self._sender.logging_out = True try: - self.invoke(LogOutRequest()) + self(LogOutRequest()) self.disconnect() if not self.session.delete(): return False @@ -399,7 +402,7 @@ class TelegramClient(TelegramBareClient): """Gets "me" (the self user) which is currently authenticated, or None if the request fails (hence, not authenticated).""" try: - return self.invoke(GetUsersRequest([InputUserSelf()]))[0] + return self(GetUsersRequest([InputUserSelf()]))[0] except UnauthorizedError: return None @@ -420,7 +423,7 @@ class TelegramClient(TelegramBareClient): corresponding to that dialog. """ - r = self.invoke( + r = self( GetDialogsRequest( offset_date=offset_date, offset_id=offset_id, @@ -440,14 +443,13 @@ class TelegramClient(TelegramBareClient): no_web_page=False): """Sends a message to the given entity (or input peer) and returns the sent message ID""" - request = SendMessageRequest( + result = self(SendMessageRequest( peer=get_input_peer(entity), message=message, entities=[], no_webpage=no_web_page - ) - self.invoke(request) - return request.random_id + )) + return result.random_id def get_message_history(self, entity, @@ -471,15 +473,15 @@ class TelegramClient(TelegramBareClient): :return: A tuple containing total message count and two more lists ([messages], [senders]). Note that the sender can be null if it was not found! """ - result = self.invoke( - GetHistoryRequest( - get_input_peer(entity), - limit=limit, - offset_date=offset_date, - offset_id=offset_id, - max_id=max_id, - min_id=min_id, - add_offset=add_offset)) + result = self(GetHistoryRequest( + get_input_peer(entity), + limit=limit, + offset_date=offset_date, + offset_id=offset_id, + max_id=max_id, + min_id=min_id, + add_offset=add_offset + )) # The result may be a messages slice (not all messages were retrieved) # or simply a messages TLObject. In the later case, no "count" @@ -513,7 +515,10 @@ class TelegramClient(TelegramBareClient): else: max_id = messages.id - return self.invoke(ReadHistoryRequest(peer=get_input_peer(entity), max_id=max_id)) + return self(ReadHistoryRequest( + peer=get_input_peer(entity), + max_id=max_id + )) # endregion @@ -552,7 +557,7 @@ class TelegramClient(TelegramBareClient): def send_media_file(self, input_media, entity): """Sends any input_media (contact, document, photo...) to the given entity""" - self.invoke(SendMediaRequest( + self(SendMediaRequest( peer=get_input_peer(entity), media=input_media )) @@ -822,7 +827,7 @@ class TelegramClient(TelegramBareClient): if time() > self._next_ping_at: self._next_ping_at = time() + self.ping_interval - self.invoke(PingRequest(utils.generate_random_long())) + self(PingRequest(utils.generate_random_long())) updates = self._sender.receive_updates(timeout=timeout) From 1f3aec589bfe2ef7a4f0784ec126228ea410fce8 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 4 Jul 2017 10:21:15 +0200 Subject: [PATCH 20/52] Let TelegramBareClient handle FileMigrateErrors instead (closes #148) --- telethon/telegram_bare_client.py | 79 +++++++++++++++++++++--- telethon/telegram_client.py | 101 ++----------------------------- 2 files changed, 75 insertions(+), 105 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 11ef5483..9708fe21 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -5,22 +5,25 @@ from os import path # Import some externalized utilities to work with the Telegram types and more from . import helpers as utils -from .errors import RPCError, FloodWaitError +from .errors import RPCError, FloodWaitError, FileMigrateError from .network import authenticator, MtProtoSender, TcpTransport from .utils import get_appropriated_part_size # For sending and receiving requests -from .tl import MTProtoRequest +from .tl import MTProtoRequest, JsonSession from .tl.all_tlobjects import layer from .tl.functions import (InitConnectionRequest, InvokeWithLayerRequest) # Initial request from .tl.functions.help import GetConfigRequest -from .tl.functions.auth import ImportAuthorizationRequest +from .tl.functions.auth import ( + ImportAuthorizationRequest, ExportAuthorizationRequest +) # Easier access for working with media from .tl.functions.upload import ( - GetFileRequest, SaveBigFilePartRequest, SaveFilePartRequest) + GetFileRequest, SaveBigFilePartRequest, SaveFilePartRequest +) # All the types we need to work with from .tl.types import InputFile, InputFileBig @@ -64,6 +67,11 @@ class TelegramBareClient: self._timeout = timeout self._logger = logging.getLogger(__name__) + # Cache "exported" senders 'dc_id: TelegramBareClient' and + # their corresponding sessions not to recreate them all + # the time since it's a (somewhat expensive) process. + self._cached_clients = {} + # These will be set later self.dc_options = None self._sender = None @@ -125,8 +133,6 @@ class TelegramBareClient: )) if exported_auth is not None: - # TODO Don't actually need this for exported authorizations, - # they're only valid on such data center. result = self(GetConfigRequest()) # We're only interested in the DC options, @@ -201,6 +207,54 @@ class TelegramBareClient: return next(dc for dc in self.dc_options if dc.id == dc_id) + def _get_exported_client(self, dc_id, + init_connection=False, + bypass_cache=False): + """Gets a cached exported TelegramBareClient for the desired DC. + + If it's the first time retrieving the TelegramBareClient, the + current authorization is exported to the new DC so that + it can be used there, and the connection is initialized. + + If after using the sender a ConnectionResetError is raised, + this method should be called again with init_connection=True + in order to perform the reconnection. + + If bypass_cache is True, a new client will be exported and + it will not be cached. + """ + # Thanks badoualy/kotlogram on /telegram/api/DefaultTelegramClient.kt + # for clearly showing how to export the authorization! ^^ + client = self._cached_clients.get(dc_id) + if client and not bypass_cache: + if init_connection: + client.reconnect() + return client + else: + dc = self._get_dc(dc_id) + + # Export the current authorization to the new DC. + export_auth = self(ExportAuthorizationRequest(dc_id)) + + # Create a temporary session for this IP address, which needs + # to be different because each auth_key is unique per DC. + # + # Construct this session with the connection parameters + # (system version, device model...) from the current one. + session = JsonSession(self.session) + session.server_address = dc.ip_address + session.port = dc.port + client = TelegramBareClient( + session, self.api_id, self.api_hash, + timeout=self._timeout + ) + client.connect(exported_auth=export_auth) + + if not bypass_cache: + # Don't go through this expensive process every time. + self._cached_clients[dc_id] = client + return client + # endregion # region Invoking Telegram requests @@ -341,12 +395,21 @@ class TelegramBareClient: else: f = file + # The used client will change if FileMigrateError occurs + client = self + try: offset_index = 0 while True: offset = offset_index * part_size - result = self( - GetFileRequest(input_location, offset, part_size)) + + try: + result = client( + GetFileRequest(input_location, offset, part_size)) + except FileMigrateError as e: + client = self._get_exported_client(e.new_dc) + continue + offset_index += 1 # If we have received no data (0 bytes), the file is over diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 5d0362b0..64e269aa 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -8,8 +8,8 @@ from . import TelegramBareClient # Import some externalized utilities to work with the Telegram types and more from . import helpers as utils from .errors import (RPCError, UnauthorizedError, InvalidParameterError, - ReadCancelledError, FileMigrateError, PhoneMigrateError, - NetworkMigrateError, UserMigrateError, PhoneCodeEmptyError, + ReadCancelledError, PhoneCodeEmptyError, + PhoneMigrateError, NetworkMigrateError, UserMigrateError, PhoneCodeExpiredError, PhoneCodeHashEmptyError, PhoneCodeInvalidError, InvalidChecksumError) @@ -123,10 +123,6 @@ class TelegramClient(TelegramBareClient): self.session.system_lang_code = \ system_lang_code if system_lang_code else self.session.lang_code - # Cache "exported" senders 'dc_id: MtProtoSender' and - # their corresponding sessions not to recreate them all - # the time since it's a (somewhat expensive) process. - self._cached_clients = {} self._updates_thread = None self._phone_code_hashes = {} @@ -162,55 +158,6 @@ class TelegramClient(TelegramBareClient): # region Working with different connections - def _get_exported_client(self, dc_id, - init_connection=False, - bypass_cache=False): - """Gets a cached exported TelegramBareClient for the desired DC. - - If it's the first time retrieving the TelegramBareClient, the - current authorization is exported to the new DC so that - it can be used there, and the connection is initialized. - - If after using the sender a ConnectionResetError is raised, - this method should be called again with init_connection=True - in order to perform the reconnection. - - If bypass_cache is True, a new client will be exported and - it will not be cached. - """ - # Thanks badoualy/kotlogram on /telegram/api/DefaultTelegramClient.kt - # for clearly showing how to export the authorization! ^^ - - client = self._cached_clients.get(dc_id) - if client and not bypass_cache: - if init_connection: - client.reconnect() - return client - else: - dc = self._get_dc(dc_id) - - # Export the current authorization to the new DC. - export_auth = self(ExportAuthorizationRequest(dc_id)) - - # Create a temporary session for this IP address, which needs - # to be different because each auth_key is unique per DC. - # - # Construct this session with the connection parameters - # (system version, device model...) from the current one. - session = JsonSession(self.session) - session.server_address = dc.ip_address - session.port = dc.port - client = TelegramBareClient( - session, self.api_id, self.api_hash, - timeout=self._timeout - ) - client.connect(exported_auth=export_auth) - - if not bypass_cache: - # Don't go through this expensive process every time. - self._cached_clients[dc_id] = client - return client - def create_new_connection(self, on_dc=None): """Creates a new connection which can be used in parallel with the original TelegramClient. A TelegramBareClient @@ -222,15 +169,10 @@ class TelegramClient(TelegramBareClient): If the client is meant to be used on a different data center, the data center ID should be specified instead. - - Note that TelegramBareClients will not handle automatic - reconnection (i.e. switching to another data center to - download media), and InvalidDCError will be raised in - such case. """ if on_dc is None: - client = TelegramBareClient(self.session, self.api_id, self.api_hash, - proxy=self.proxy) + client = TelegramBareClient( + self.session, self.api_id, self.api_hash, proxy=self.proxy) client.connect() else: client = self._get_exported_client(on_dc, bypass_cache=True) @@ -724,41 +666,6 @@ class TelegramClient(TelegramBareClient): return file_path - def download_file(self, - input_location, - file, - part_size_kb=None, - file_size=None, - progress_callback=None, - on_dc=None): - """Downloads the given InputFileLocation to file (a stream or str). - - If 'progress_callback' is not None, it should be a function that - takes two parameters, (bytes_downloaded, total_bytes). Note that - 'total_bytes' simply equals 'file_size', and may be None. - """ - if on_dc is None: - try: - super().download_file( - input_location, - file, - part_size_kb=part_size_kb, - file_size=file_size, - progress_callback=progress_callback - ) - except FileMigrateError as e: - on_dc = e.new_dc - - if on_dc is not None: - client = self._get_exported_client(on_dc) - client.download_file( - input_location, - file, - part_size_kb=part_size_kb, - file_size=file_size, - progress_callback=progress_callback - ) - # endregion # endregion From 127e5f70d85a2e4b03f758c86fcad56f038d82b8 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 4 Jul 2017 10:39:57 +0200 Subject: [PATCH 21/52] Update to v0.11.2 --- telethon/telegram_bare_client.py | 2 +- telethon/telegram_client.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 9708fe21..39e08ff2 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -50,7 +50,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.11.1' + __version__ = '0.11.2' # region Initialization diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 64e269aa..a45139f9 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -24,9 +24,6 @@ from .tl.functions.auth import (CheckPasswordRequest, LogOutRequest, SendCodeRequest, SignInRequest, SignUpRequest, ImportBotAuthorizationRequest) -# Required to work with different data centers -from .tl.functions.auth import ExportAuthorizationRequest - # Easier access to common methods from .tl.functions.messages import ( GetDialogsRequest, GetHistoryRequest, ReadHistoryRequest, SendMediaRequest, From 3585fb8cc6e1e4cabb572076a29931b444dd634b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 4 Jul 2017 11:02:32 +0200 Subject: [PATCH 22/52] Fix setup.py for source distributions --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 05f2a9cd..6a80d95b 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ from os import path # Always prefer setuptools over distutils from setuptools import find_packages, setup -from telethon_generator.tl_generator import TLGenerator try: from telethon import TelegramClient except ImportError: @@ -27,6 +26,7 @@ except ImportError: if __name__ == '__main__': if len(argv) >= 2 and argv[1] == 'gen_tl': + from telethon_generator.tl_generator import TLGenerator generator = TLGenerator('telethon/tl') if generator.tlobjects_exist(): print('Detected previous TLObjects. Cleaning...') @@ -39,6 +39,7 @@ if __name__ == '__main__': print('Done.') elif len(argv) >= 2 and argv[1] == 'clean_tl': + from telethon_generator.tl_generator import TLGenerator print('Cleaning...') TLGenerator('telethon/tl').clean_tlobjects() print('Done.') From 9bb6353fa3ae52b8400517ba1f4895e7d3019295 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 4 Jul 2017 16:53:07 +0200 Subject: [PATCH 23/52] Fix send_message using the incorrect type to return the msg_id (#156) --- telethon/telegram_bare_client.py | 2 +- telethon/telegram_client.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 39e08ff2..6076d6ca 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -50,7 +50,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.11.2' + __version__ = '0.11.3' # region Initialization diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index a45139f9..fe9d6c2c 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -382,13 +382,16 @@ class TelegramClient(TelegramBareClient): no_web_page=False): """Sends a message to the given entity (or input peer) and returns the sent message ID""" - result = self(SendMessageRequest( + request = SendMessageRequest( peer=get_input_peer(entity), message=message, entities=[], no_webpage=no_web_page - )) - return result.random_id + ) + result = self(request) + for handler in self._update_handlers: + handler(result) + return request.random_id def get_message_history(self, entity, From 632fcb7c009a293ebfac761ce55af7830608c55c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 4 Jul 2017 19:47:55 +0200 Subject: [PATCH 24/52] Ensure device model is non-empty (closes #154) --- telethon/tl/session.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/telethon/tl/session.py b/telethon/tl/session.py index 22638806..24a9c8ec 100644 --- a/telethon/tl/session.py +++ b/telethon/tl/session.py @@ -113,9 +113,10 @@ class JsonSession: else: # str / None self.session_user_id = session_user_id - self.device_model = platform.node() - self.system_version = platform.system() - self.app_version = '1.0' # note: '0' will provoke error + system = platform.uname() + self.device_model = system.system if system.system else 'Unknown' + self.system_version = system.release if system.release else '1.0' + self.app_version = '1.0' # '0' will provoke error self.lang_code = 'en' self.system_lang_code = self.lang_code self.lang_pack = '' From 8fd0d7eadd944ff42e18aaf06228adc7aba794b5 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 4 Jul 2017 21:15:47 +0200 Subject: [PATCH 25/52] Add a new .stringify() function to visualize TLObjects more easily --- telethon/tl/mtproto_request.py | 72 ++++++++++++++++++++++++++++++ telethon_generator/tl_generator.py | 8 +++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/telethon/tl/mtproto_request.py b/telethon/tl/mtproto_request.py index 391eeb07..8a103e16 100644 --- a/telethon/tl/mtproto_request.py +++ b/telethon/tl/mtproto_request.py @@ -30,7 +30,79 @@ class MTProtoRequest: self.confirmed and not self.confirm_received and datetime.now() - self.send_time > timedelta(seconds=3)) + @staticmethod + def pretty_format(obj, indent=None): + """Pretty formats the given object as a string which is returned. + If indent is None, a single line will be returned. + """ + if indent is None: + if isinstance(obj, MTProtoRequest): + return '{{{}: {}}}'.format( + type(obj).__name__, + MTProtoRequest.pretty_format(obj.to_dict()) + ) + if isinstance(obj, dict): + return '{{{}}}'.format(', '.join( + '{}: {}'.format( + k, MTProtoRequest.pretty_format(v) + ) for k, v in obj.items() + )) + elif isinstance(obj, str): + return '"{}"'.format(obj) + elif hasattr(obj, '__iter__'): + return '[{}]'.format( + ', '.join(MTProtoRequest.pretty_format(x) for x in obj) + ) + else: + return str(obj) + else: + result = [] + if isinstance(obj, MTProtoRequest): + result.append('{') + result.append(type(obj).__name__) + result.append(': ') + result.append(MTProtoRequest.pretty_format( + obj.to_dict(), indent + )) + + elif isinstance(obj, dict): + result.append('{\n') + indent += 1 + for k, v in obj.items(): + result.append('\t' * indent) + result.append(k) + result.append(': ') + result.append(MTProtoRequest.pretty_format(v, indent)) + result.append(',\n') + indent -= 1 + result.append('\t' * indent) + result.append('}') + + elif isinstance(obj, str): + result.append('"') + result.append(obj) + result.append('"') + + elif hasattr(obj, '__iter__'): + result.append('[\n') + indent += 1 + for x in obj: + result.append('\t' * indent) + result.append(MTProtoRequest.pretty_format(x, indent)) + result.append(',\n') + indent -= 1 + result.append('\t' * indent) + result.append(']') + + else: + result.append(str(obj)) + + return ''.join(result) + # These should be overrode + def to_dict(self): + return {} + def on_send(self, writer): pass diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index ae8fdb68..0d55a8a3 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -179,6 +179,7 @@ class TLGenerator: # Both types and functions inherit from # MTProtoRequest so they all can be sent + # TODO MTProtoRequest is not the best name for a type builder.writeln('from {}.tl.mtproto_request import MTProtoRequest' .format('.' * depth)) @@ -408,9 +409,12 @@ class TLGenerator: builder.end_block() builder.writeln('def __str__(self):') - builder.writeln('return {}'.format(str(tlobject))) - # builder.end_block() # No need to end the last block + builder.writeln('return MTProtoRequest.pretty_format(self)') + builder.end_block() + builder.writeln('def stringify(self):') + builder.writeln('return MTProtoRequest.pretty_format(self, indent=0)') + # builder.end_block() # No need to end the last block @staticmethod def get_class_name(tlobject): From f88efa7f493aa17cb72941b1468d2e1934fba84d Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 4 Jul 2017 21:18:35 +0200 Subject: [PATCH 26/52] Let PeerChat be casted to InputPeerChat automatically --- telethon/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/telethon/utils.py b/telethon/utils.py index d94734a2..485e9034 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -6,8 +6,8 @@ from mimetypes import add_type, guess_extension from .tl.types import ( Channel, ChannelForbidden, Chat, ChatEmpty, ChatForbidden, ChatFull, - ChatPhoto, InputPeerChannel, InputPeerChat, InputPeerUser, InputPeerEmpty, - InputPeerSelf, MessageMediaDocument, MessageMediaPhoto, PeerChannel, + ChatPhoto, InputPeerChannel, InputPeerChat, InputPeerUser, + MessageMediaDocument, MessageMediaPhoto, PeerChannel, PeerChat, PeerUser, User, UserFull, UserProfilePhoto) @@ -73,6 +73,9 @@ def get_input_peer(entity): if isinstance(entity, ChatFull): return InputPeerChat(entity.id) + if isinstance(entity, PeerChat): + return InputPeerChat(entity.chat_id) + raise ValueError('Cannot cast {} to any kind of InputPeer.' .format(type(entity).__name__)) From 95a989be2c708d98be8feff70d33149c14441879 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 7 Jul 2017 09:48:06 +0200 Subject: [PATCH 27/52] Automatically cast Channel to InputChannel (similar to InputPeer) --- telethon/utils.py | 17 +++++++++++++++-- telethon_generator/tl_generator.py | 24 +++++++++++++++++------- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/telethon/utils.py b/telethon/utils.py index 485e9034..411db593 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -7,7 +7,7 @@ from mimetypes import add_type, guess_extension from .tl.types import ( Channel, ChannelForbidden, Chat, ChatEmpty, ChatForbidden, ChatFull, ChatPhoto, InputPeerChannel, InputPeerChat, InputPeerUser, - MessageMediaDocument, MessageMediaPhoto, PeerChannel, + MessageMediaDocument, MessageMediaPhoto, PeerChannel, InputChannel, PeerChat, PeerUser, User, UserFull, UserProfilePhoto) @@ -52,7 +52,7 @@ def get_extension(media): def get_input_peer(entity): """Gets the input peer for the given "entity" (user, chat or channel). A ValueError is raised if the given entity isn't a supported type.""" - if type(entity).subclass_of_id == 0xc91c90b6: # crc32('InputUser') + if type(entity).subclass_of_id == 0xc91c90b6: # crc32(b'InputPeer') return entity if isinstance(entity, User): @@ -80,6 +80,19 @@ def get_input_peer(entity): .format(type(entity).__name__)) +def get_input_channel(entity): + """Gets the input peer for the given "entity" (user, chat or channel). + A ValueError is raised if the given entity isn't a supported type.""" + if type(entity).subclass_of_id == 0x40f202fd: # crc32(b'InputChannel') + return entity + + if isinstance(entity, Channel) or isinstance(entity, ChannelForbidden): + return InputChannel(entity.id, entity.access_hash) + + raise ValueError('Cannot cast {} to any kind of InputChannel.' + .format(type(entity).__name__)) + + def find_user_or_chat(peer, users, chats): """Finds the corresponding user or chat given a peer. Returns None if it was not found""" diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 0d55a8a3..0fffe1b8 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -183,12 +183,17 @@ class TLGenerator: builder.writeln('from {}.tl.mtproto_request import MTProtoRequest' .format('.' * depth)) - if tlobject.is_function and \ - any(a for a in tlobject.args if a.type == 'InputPeer'): - # We can automatically convert a normal peer to an InputPeer, - # it will make invoking a lot of requests a lot simpler. - builder.writeln('from {}.utils import get_input_peer' - .format('.' * depth)) + if tlobject.is_function: + if any(a for a in tlobject.args if a.type == 'InputPeer'): + # We can automatically convert a normal peer to an InputPeer, + # it will make invoking a lot of requests a lot simpler. + builder.writeln('from {}.utils import get_input_peer' + .format('.' * depth)) + + if any(a for a in tlobject.args if a.type == 'InputChannel'): + # Same applies to channels + builder.writeln('from {}.utils import get_input_channel' + .format('.' * depth)) if any(a for a in tlobject.args if a.can_be_inferred): # Currently only 'random_id' needs 'os' to be imported @@ -316,7 +321,12 @@ class TLGenerator: elif arg.type == 'InputPeer' and tlobject.is_function: # Well-known case, auto-cast it to the right type builder.writeln( - 'self.{0} = get_input_peer({0})'.format(arg.name)) + 'self.{0} = get_input_peer({0})'.format(arg.name) + ) + elif arg.type == 'InputChannel' and tlobject.is_function: + builder.writeln( + 'self.{0} = get_input_channel({0})'.format(arg.name) + ) else: builder.writeln('self.{0} = {0}'.format(arg.name)) From 0119a006585acd1a1a9a8901a21bb2f193142cfe Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 7 Jul 2017 10:37:19 +0200 Subject: [PATCH 28/52] Rename no_webpage to link_preview for clarity --- telethon/telegram_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index fe9d6c2c..9447e8ae 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -379,14 +379,14 @@ class TelegramClient(TelegramBareClient): def send_message(self, entity, message, - no_web_page=False): + link_preview=True): """Sends a message to the given entity (or input peer) and returns the sent message ID""" request = SendMessageRequest( peer=get_input_peer(entity), message=message, entities=[], - no_webpage=no_web_page + no_webpage=not link_preview ) result = self(request) for handler in self._update_handlers: From 5061e22c66d9fd2cc17837002929f947a33a833a Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 7 Jul 2017 11:33:24 +0200 Subject: [PATCH 29/52] Update documentation index to reflect __call__ and enhance search --- docs/res/core.html | 65 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/docs/res/core.html b/docs/res/core.html index 713c16bf..b7875a82 100644 --- a/docs/res/core.html +++ b/docs/res/core.html @@ -14,6 +14,7 @@
+ @@ -146,7 +147,6 @@
#!/usr/bin/python3
 from telethon import TelegramClient
 from telethon.tl.functions.messages import GetHistoryRequest
-from telethon.utils import get_input_peer
 
 # (1) Use your own values here
 api_id   = 12345
@@ -167,25 +167,28 @@ dialogs, entities = client.get_dialogs(10)
 entity = entities[0]
 
 # (4) !! Invoking a request manually !!
-result = client.invoke(
-    GetHistoryRequest(
-        get_input_peer(entity),
-        limit=20,
-        offset_date=None,
-        offset_id=0,
-        max_id=0,
-        min_id=0,
-        add_offset=0))
+result = client(GetHistoryRequest(
+    entity,
+    limit=20,
+    offset_date=None,
+    offset_id=0,
+    max_id=0,
+    min_id=0,
+    add_offset=0
+))
 
 # Now you have access to the first 20 messages
 messages = result.messages
-

As it can be seen, manually invoking requests with - client.invoke() is way more verbose than using the built-in - methods (such as client.get_dialogs(). However, and given - that there are so many methods available, it's impossible to provide a nice - interface to things that may change over time. To get full access, however, - you're still able to invoke these methods manually.

+

As it can be seen, manually calling requests with + client(request) (or using the old way, by calling + client.invoke(request)) is way more verbose than using the + built-in methods (such as client.get_dialogs()).

+ +

However, and + given that there are so many methods available, it's impossible to provide + a nice interface to things that may change over time. To get full access, + however, you're still able to invoke these methods manually.

@@ -195,11 +198,18 @@ searchDiv = document.getElementById("searchDiv"); searchBox = document.getElementById("searchBox"); searchTable = document.getElementById("searchTable"); -requests = [{request_names}]; -types = [{type_names}]; +try { + requests = [{request_names}]; + types = [{type_names}]; -requestsu = [{request_urls}]; -typesu = [{type_urls}]; + requestsu = [{request_urls}]; + typesu = [{type_urls}]; +} catch (e) { + requests = []; + types = []; + requetsu = []; + typesu = []; +} function updateSearch() { if (searchBox.value) { @@ -253,6 +263,21 @@ function updateSearch() { } } +function getQuery(name) { + var query = window.location.search.substring(1); + var vars = query.split("&"); + for (var i = 0; i != vars.length; ++i) { + var pair = vars[i].split("="); + if (pair[0] == name) + return pair[1]; + } +} + +var query = getQuery('q'); +if (query) { + searchBox.value = query; +} + updateSearch(); From 4563875ab5694248412e9b025a71a48865953915 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 8 Jul 2017 13:28:23 +0200 Subject: [PATCH 30/52] Let constructors be searched on the docs, and allow collapsing types --- docs/generate.py | 10 +++- docs/res/core.html | 112 ++++++++++++++++++++++++++---------------- docs/res/css/docs.css | 24 ++++++++- 3 files changed, 102 insertions(+), 44 deletions(-) diff --git a/docs/generate.py b/docs/generate.py index 1b467727..5c607460 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -433,20 +433,26 @@ def generate_documentation(scheme_file): layer = TLParser.find_layer(scheme_file) types = set() methods = [] + constructors = [] for tlobject in tlobjects: if tlobject.is_function: methods.append(tlobject) + else: + constructors.append(tlobject) types.add(tlobject.result) types = sorted(types) methods = sorted(methods, key=lambda m: m.name) + constructors = sorted(constructors, key=lambda c: c.name) request_names = ', '.join('"' + get_class_name(m) + '"' for m in methods) type_names = ', '.join('"' + get_class_name(t) + '"' for t in types) + constructor_names = ', '.join('"' + get_class_name(t) + '"' for t in constructors) request_urls = ', '.join('"' + get_create_path_for(m) + '"' for m in methods) type_urls = ', '.join('"' + get_path_for_type(t) + '"' for t in types) + constructor_urls = ', '.join('"' + get_create_path_for(t) + '"' for t in constructors) replace_dict = { 'type_count': len(types), @@ -456,8 +462,10 @@ def generate_documentation(scheme_file): 'request_names': request_names, 'type_names': type_names, + 'constructor_names': constructor_names, 'request_urls': request_urls, - 'type_urls': type_urls + 'type_urls': type_urls, + 'constructor_urls': constructor_urls } with open('../res/core.html') as infile: diff --git a/docs/res/core.html b/docs/res/core.html index b7875a82..bc5c04b3 100644 --- a/docs/res/core.html +++ b/docs/res/core.html @@ -19,7 +19,21 @@ placeholder="Search for requests and types…" />
-
+ +
Methods (0) +
    +
+
+ +
Types (0) +
    +
+
+ +
Constructors (0) +
    +
+
@@ -196,19 +210,65 @@ messages = result.messages contentDiv = document.getElementById("contentDiv"); searchDiv = document.getElementById("searchDiv"); searchBox = document.getElementById("searchBox"); -searchTable = document.getElementById("searchTable"); + +// Search lists +methodsList = document.getElementById("methodsList"); +methodsCount = document.getElementById("methodsCount"); + +typesList = document.getElementById("typesList"); +typesCount = document.getElementById("typesCount"); + +constructorsList = document.getElementById("constructorsList"); +constructorsCount = document.getElementById("constructorsCount"); try { requests = [{request_names}]; types = [{type_names}]; + constructors = [{constructor_names}]; requestsu = [{request_urls}]; typesu = [{type_urls}]; + constructorsu = [{constructor_urls}]; } catch (e) { requests = []; types = []; - requetsu = []; + constructors = []; + requestsu = []; typesu = []; + constructorsu = []; +} + +// Given two input arrays "original" and "original urls" and a query, +// return a pair of arrays with matching "query" elements from "original". +// +// TODO Perhaps return an array of pairs instead a pair of arrays (for cache). +function getSearchArray(original, originalu, query) { + var destination = []; + var destinationu = []; + + for (var i = 0; i < original.length; ++i) { + if (original[i].toLowerCase().indexOf(query) != -1) { + destination.push(original[i]); + destinationu.push(originalu[i]); + } + } + + return [destination, destinationu]; +} + +// Modify "countSpan" and "resultList" accordingly based on the elements +// given as [[elements], [element urls]] (both with the same length) +function buildList(countSpan, resultList, foundElements) { + var result = ""; + for (var i = 0; i < foundElements[0].length; ++i) { + result += '
  • '; + result += ''; + result += foundElements[0][i]; + result += '
  • '; + } + + countSpan.innerHTML = "" + foundElements[0].length; + resultList.innerHTML = result; } function updateSearch() { @@ -217,46 +277,16 @@ function updateSearch() { searchDiv.style.display = ""; var query = searchBox.value.toLowerCase(); - var foundRequests = []; - var foundRequestsu = []; - for (var i = 0; i < requests.length; ++i) { - if (requests[i].toLowerCase().indexOf(query) != -1) { - foundRequests.push(requests[i]); - foundRequestsu.push(requestsu[i]); - } - } - var foundTypes = []; - var foundTypesu = []; - for (var i = 0; i < types.length; ++i) { - if (types[i].toLowerCase().indexOf(query) != -1) { - foundTypes.push(types[i]); - foundTypesu.push(typesu[i]); - } - } + var foundRequests = getSearchArray(requests, requestsu, query); + var foundTypes = getSearchArray(types, typesu, query); + var foundConstructors = getSearchArray( + constructors, constructorsu, query + ); - var top = foundRequests.length > foundTypes.length ? - foundRequests.length : foundTypes.length; - - result = ""; - for (var i = 0; i <= top; ++i) { - result += ""; - - if (i < foundRequests.length) { - result += - ''+foundRequests[i]+''; - } - - result += ""; - - if (i < foundTypes.length) { - result += - ''+foundTypes[i]+''; - } - - result += ""; - } - searchTable.innerHTML = result; + buildList(methodsCount, methodsList, foundRequests); + buildList(typesCount, typesList, foundTypes); + buildList(constructorsCount, constructorsList, foundConstructors); } else { contentDiv.style.display = ""; searchDiv.style.display = "none"; diff --git a/docs/res/css/docs.css b/docs/res/css/docs.css index 9a9b710b..05c61c9f 100644 --- a/docs/res/css/docs.css +++ b/docs/res/css/docs.css @@ -52,7 +52,7 @@ table td { margin: 0 8px -2px 0; } -h1 { +h1, summary.title { font-size: 24px; } @@ -137,6 +137,26 @@ button:hover { color: #fff; } +/* https://www.w3schools.com/css/css_navbar.asp */ +ul.together { + list-style-type: none; + margin: 0; + padding: 0; + overflow: hidden; +} + +ul.together li { + float: left; +} + +ul.together li a { + display: block; + border-radius: 8px; + background: #f0f4f8; + padding: 4px 8px; + margin: 8px; +} + /* https://stackoverflow.com/a/30810322 */ .invisible { left: 0; @@ -153,7 +173,7 @@ button:hover { } @media (max-width: 640px) { - h1 { + h1, summary.title { font-size: 18px; } h3 { From 5f19f22d4636a7f296c279b6b41883ebe0bdf5bd Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 8 Jul 2017 13:35:10 +0200 Subject: [PATCH 31/52] Docs should not let core types be searched --- docs/generate.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/generate.py b/docs/generate.py index 5c607460..46c7a5f9 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -75,12 +75,19 @@ def get_create_path_for(tlobject): return os.path.join(out_dir, get_file_name(tlobject, add_extension=True)) +def is_core_type(type_): + """Returns "true" if the type is considered a core type""" + return type_.lower() in { + 'int', 'long', 'int128', 'int256', 'double', + 'vector', 'string', 'bool', 'true', 'bytes', 'date' + } + + def get_path_for_type(type_, relative_to='.'): """Similar to getting the path for a TLObject, it might not be possible to have the TLObject itself but rather its name (the type); this method works in the same way, returning a relative path""" - if type_.lower() in {'int', 'long', 'int128', 'int256', 'double', - 'vector', 'string', 'bool', 'true', 'bytes', 'date'}: + if is_core_type(type_): path = 'index.html#%s' % type_.lower() elif '.' in type_: @@ -440,7 +447,11 @@ def generate_documentation(scheme_file): else: constructors.append(tlobject) - types.add(tlobject.result) + if not is_core_type(tlobject.result): + if re.search('^vector<', tlobject.result, re.IGNORECASE): + types.add(tlobject.result.split('<')[1].strip('>')) + else: + types.add(tlobject.result) types = sorted(types) methods = sorted(methods, key=lambda m: m.name) From eab44af4c0ab198fd8a48ac7557bb416b2d2afd2 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 8 Jul 2017 17:31:57 +0200 Subject: [PATCH 32/52] Show "Methods accepting this type as input" on the docs --- docs/generate.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/docs/generate.py b/docs/generate.py index 46c7a5f9..2fc35426 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -403,11 +403,39 @@ def generate_documentation(scheme_file): docs.add_row(get_class_name(func), link=link) docs.end_table() + # List all the methods which take this type as input + docs.write_title('Methods accepting this type as input', level=3) + other_methods = sorted( + (t for t in tlobjects + if any(tltype == a.type for a in t.args) and t.is_function), + key=lambda t: t.name + ) + if not other_methods: + docs.write_text( + 'No methods accept this type as an input parameter.') + elif len(other_methods) == 1: + docs.write_text( + 'Only this method has a parameter with this type.') + else: + docs.write_text( + 'The following %d methods accept this type as an input ' + 'parameter.' % len(other_methods)) + + docs.begin_table(2) + for ot in other_methods: + link = get_create_path_for(ot) + link = get_relative_path(link, relative_to=filename) + docs.add_row(get_class_name(ot), link=link) + docs.end_table() + # List every other type which has this type as a member docs.write_title('Other types containing this type', level=3) - other_types = sorted((t for t in tlobjects - if any(tltype == a.type for a in t.args)), - key=lambda t: t.name) + other_types = sorted( + (t for t in tlobjects + if any(tltype == a.type for a in t.args) + and not t.is_function + ), key=lambda t: t.name + ) if not other_types: docs.write_text( From 1f7ac7118750ed84e2165dce9c6aca2e6ea0c6a4 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 10 Jul 2017 15:21:20 +0200 Subject: [PATCH 33/52] Debug level should always be used for logging since it's a library --- telethon/extensions/threaded_tcp_client.py | 94 ++++++++++++++++++++++ telethon/network/mtproto_sender.py | 20 ++--- telethon/telegram_bare_client.py | 6 +- telethon/telegram_client.py | 16 ++-- 4 files changed, 115 insertions(+), 21 deletions(-) create mode 100644 telethon/extensions/threaded_tcp_client.py diff --git a/telethon/extensions/threaded_tcp_client.py b/telethon/extensions/threaded_tcp_client.py new file mode 100644 index 00000000..d97ce6ee --- /dev/null +++ b/telethon/extensions/threaded_tcp_client.py @@ -0,0 +1,94 @@ +import socket +import time +from datetime import datetime, timedelta +from io import BytesIO, BufferedWriter +from threading import Event, Lock, Thread, Condition + +from ..errors import ReadCancelledError + + +class ThreadedTcpClient: + """The main difference with the TcpClient class is that this one + will spawn a secondary thread that will be constantly reading + from the network and putting everything on another buffer. + """ + def __init__(self, proxy=None): + self.connected = False + self._proxy = proxy + self._recreate_socket() + + # Support for multi-threading advantages and safety + self.cancelled = Event() # Has the read operation been cancelled? + self.delay = 0.1 # Read delay when there was no data available + self._lock = Lock() + + self._buffer = [] + self._read_thread = Thread(target=self._reading_thread, daemon=True) + self._cv = Condition() # Condition Variable + + def _recreate_socket(self): + if self._proxy is None: + self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + else: + import socks + self._socket = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) + if type(self._proxy) is dict: + self._socket.set_proxy(**self._proxy) + else: # tuple, list, etc. + self._socket.set_proxy(*self._proxy) + + def connect(self, ip, port, timeout): + """Connects to the specified IP and port number. + 'timeout' must be given in seconds + """ + if not self.connected: + self._socket.settimeout(timeout) + self._socket.connect((ip, port)) + self._socket.setblocking(False) + self.connected = True + + def close(self): + """Closes the connection""" + if self.connected: + self._socket.shutdown(socket.SHUT_RDWR) + self._socket.close() + self.connected = False + self._recreate_socket() + + def write(self, data): + """Writes (sends) the specified bytes to the connected peer""" + self._socket.sendall(data) + + def read(self, size, timeout=timedelta(seconds=5)): + """Reads (receives) a whole block of 'size bytes + from the connected peer. + + A timeout can be specified, which will cancel the operation if + no data has been read in the specified time. If data was read + and it's waiting for more, the timeout will NOT cancel the + operation. Set to None for no timeout + """ + with self._cv: + print('wait for...') + self._cv.wait_for(lambda: len(self._buffer) >= size, timeout=timeout.seconds) + print('got', size) + result, self._buffer = self._buffer[:size], self._buffer[size:] + return result + + def _reading_thread(self): + while True: + partial = self._socket.recv(4096) + if len(partial) == 0: + self.connected = False + raise ConnectionResetError( + 'The server has closed the connection.') + + with self._cv: + print('extended', len(partial)) + self._buffer.extend(partial) + self._cv.notify() + + def cancel_read(self): + """Cancels the read operation IF it hasn't yet + started, raising a ReadCancelledError""" + self.cancelled.set() diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 0266e7b4..32059986 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -100,7 +100,7 @@ class MtProtoSender: # or, if there is no request, until we read an update while (request and not request.confirm_received) or \ (not request and not updates): - self._logger.info('Trying to .receive() the request result...') + self._logger.debug('Trying to .receive() the request result...') seq, body = self.transport.receive(**kwargs) message, remote_msg_id, remote_seq = self._decode_msg(body) @@ -114,7 +114,7 @@ class MtProtoSender: self._pending_receive.remove(request) except ValueError: pass - self._logger.info('Request result received') + self._logger.debug('Request result received') self._logger.debug('receive() released the lock') def receive_updates(self, **kwargs): @@ -226,10 +226,10 @@ class MtProtoSender: ack = reader.tgread_object() for r in self._pending_receive: if r.request_msg_id in ack.msg_ids: - self._logger.warning('Ack found for the a request') + self._logger.debug('Ack found for the a request') if self.logging_out: - self._logger.info('Message ack confirmed a request') + self._logger.debug('Message ack confirmed a request') r.confirm_received = True return True @@ -247,7 +247,7 @@ class MtProtoSender: return True - self._logger.warning('Unknown message: {}'.format(hex(code))) + self._logger.debug('Unknown message: {}'.format(hex(code))) return False # endregion @@ -263,7 +263,7 @@ class MtProtoSender: request = next(r for r in self._pending_receive if r.request_msg_id == received_msg_id) - self._logger.warning('Pong confirmed a request') + self._logger.debug('Pong confirmed a request') request.confirm_received = True except StopIteration: pass @@ -318,8 +318,8 @@ class MtProtoSender: # Use the current msg_id to determine the right time offset. self.session.update_time_offset(correct_msg_id=msg_id) self.session.save() - self._logger.warning('Read Bad Message error: ' + str(error)) - self._logger.info('Attempting to use the correct time offset.') + self._logger.debug('Read Bad Message error: ' + str(error)) + self._logger.debug('Attempting to use the correct time offset.') return True else: raise error @@ -346,7 +346,7 @@ class MtProtoSender: self._need_confirmation.append(request_id) self._send_acknowledges() - self._logger.warning('Read RPC error: %s', str(error)) + self._logger.debug('Read RPC error: %s', str(error)) if isinstance(error, InvalidDCError): # Must resend this request, if any if request: @@ -368,7 +368,7 @@ class MtProtoSender: else: # If it's really a result for RPC from previous connection # session, it will be skipped by the handle_container() - self._logger.warning('Lost request will be skipped.') + self._logger.debug('Lost request will be skipped.') return False def _handle_gzip_packed(self, msg_id, sequence, reader, updates): diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 6076d6ca..c45498cf 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -90,7 +90,7 @@ class TelegramBareClient: determine the authorization key for the current session. """ if self._sender and self._sender.is_connected(): - self._logger.warning( + self._logger.debug( 'Attempted to connect when the client was already connected.' ) return @@ -143,7 +143,7 @@ class TelegramBareClient: except (RPCError, ConnectionError) as error: # Probably errors from the previous session, ignore them self.disconnect() - self._logger.warning('Could not stabilise initial connection: {}' + self._logger.debug('Could not stabilise initial connection: {}' .format(error)) return False @@ -277,7 +277,7 @@ class TelegramBareClient: return request.result except ConnectionResetError: - self._logger.info('Server disconnected us. Reconnecting and ' + self._logger.debug('Server disconnected us. Reconnecting and ' 'resending request...') self.reconnect() return self.invoke(request) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 9447e8ae..7f7ba729 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -214,7 +214,7 @@ class TelegramClient(TelegramBareClient): return result except (PhoneMigrateError, NetworkMigrateError, UserMigrateError) as e: - self._logger.info('DC error when invoking request, ' + self._logger.debug('DC error when invoking request, ' 'attempting to reconnect at DC {}' .format(e.new_dc)) @@ -698,7 +698,7 @@ class TelegramClient(TelegramBareClient): return # Different state, update the saved value and behave as required - self._logger.info('Changing updates thread running status to %s', running) + self._logger.debug('Changing updates thread running status to %s', running) if running: self._updates_thread_running.set() if not self._updates_thread: @@ -739,7 +739,7 @@ class TelegramClient(TelegramBareClient): updates = self._sender.receive_updates(timeout=timeout) self._updates_thread_receiving.clear() - self._logger.info( + self._logger.debug( 'Received {} update(s) from the updates thread' .format(len(updates)) ) @@ -748,25 +748,25 @@ class TelegramClient(TelegramBareClient): handler(update) except ConnectionResetError: - self._logger.info('Server disconnected us. Reconnecting...') + self._logger.debug('Server disconnected us. Reconnecting...') self.reconnect() except TimeoutError: self._logger.debug('Receiving updates timed out') except ReadCancelledError: - self._logger.info('Receiving updates cancelled') + self._logger.debug('Receiving updates cancelled') except BrokenPipeError: - self._logger.info('Tcp session is broken. Reconnecting...') + self._logger.debug('Tcp session is broken. Reconnecting...') self.reconnect() except InvalidChecksumError: - self._logger.info('MTProto session is broken. Reconnecting...') + self._logger.debug('MTProto session is broken. Reconnecting...') self.reconnect() except OSError: - self._logger.warning('OSError on updates thread, %s logging out', + self._logger.debug('OSError on updates thread, %s logging out', 'was' if self._sender.logging_out else 'was not') if self._sender.logging_out: From 84bb3bb32527eec736f19a5b24a0c5f605a06abc Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 10 Jul 2017 15:22:19 +0200 Subject: [PATCH 34/52] Fix interactive example not using a new parameter name --- telethon_examples/interactive_telegram_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon_examples/interactive_telegram_client.py b/telethon_examples/interactive_telegram_client.py index f4c6ba8a..0002b17d 100644 --- a/telethon_examples/interactive_telegram_client.py +++ b/telethon_examples/interactive_telegram_client.py @@ -217,7 +217,7 @@ class InteractiveTelegramClient(TelegramClient): # Send chat message (if any) elif msg: self.send_message( - entity, msg, no_web_page=True) + entity, msg, link_preview=False) def send_photo(self, path, entity): print('Uploading {}...'.format(path)) From bdee94eaf3332f3c32828c0e70feee81a7084db7 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 10 Jul 2017 16:04:10 +0200 Subject: [PATCH 35/52] Implement automatic cast to InputUser too (closes #159) --- telethon/utils.py | 25 +++++++++++++++++++++++-- telethon_generator/tl_generator.py | 26 +++++++++++++++++--------- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/telethon/utils.py b/telethon/utils.py index 411db593..362fe6bb 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -8,6 +8,7 @@ from .tl.types import ( Channel, ChannelForbidden, Chat, ChatEmpty, ChatForbidden, ChatFull, ChatPhoto, InputPeerChannel, InputPeerChat, InputPeerUser, MessageMediaDocument, MessageMediaPhoto, PeerChannel, InputChannel, + UserEmpty, InputUser, InputUserEmpty, InputUserSelf, PeerChat, PeerUser, User, UserFull, UserProfilePhoto) @@ -81,8 +82,7 @@ def get_input_peer(entity): def get_input_channel(entity): - """Gets the input peer for the given "entity" (user, chat or channel). - A ValueError is raised if the given entity isn't a supported type.""" + """Similar to get_input_peer, but for InputChannel's alone""" if type(entity).subclass_of_id == 0x40f202fd: # crc32(b'InputChannel') return entity @@ -93,6 +93,27 @@ def get_input_channel(entity): .format(type(entity).__name__)) +def get_input_user(entity): + """Similar to get_input_peer, but for InputUser's alone""" + if type(entity).subclass_of_id == 0xe669bf46: # crc32(b'InputUser') + return entity + + if isinstance(entity, User): + if entity.is_self: + return InputUserSelf() + else: + return InputUser(entity.id, entity.access_hash) + + if isinstance(entity, UserEmpty): + return InputUserEmpty() + + if isinstance(entity, UserFull): + return get_input_user(entity.user) + + raise ValueError('Cannot cast {} to any kind of InputUser.' + .format(type(entity).__name__)) + + def find_user_or_chat(peer, users, chats): """Finds the corresponding user or chat given a peer. Returns None if it was not found""" diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index 0fffe1b8..e6e3dc4d 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -184,16 +184,20 @@ class TLGenerator: .format('.' * depth)) if tlobject.is_function: - if any(a for a in tlobject.args if a.type == 'InputPeer'): - # We can automatically convert a normal peer to an InputPeer, - # it will make invoking a lot of requests a lot simpler. - builder.writeln('from {}.utils import get_input_peer' - .format('.' * depth)) + util_imports = set() + for a in tlobject.args: + # We can automatically convert some "full" types to + # "input only" (like User -> InputPeerUser, etc.) + if a.type == 'InputPeer': + util_imports.add('get_input_peer') + elif a.type == 'InputChannel': + util_imports.add('get_input_channel') + elif a.type == 'InputUser': + util_imports.add('get_input_user') - if any(a for a in tlobject.args if a.type == 'InputChannel'): - # Same applies to channels - builder.writeln('from {}.utils import get_input_channel' - .format('.' * depth)) + if util_imports: + builder.writeln('from {}.utils import {}'.format( + '.' * depth, ', '.join(util_imports))) if any(a for a in tlobject.args if a.can_be_inferred): # Currently only 'random_id' needs 'os' to be imported @@ -327,6 +331,10 @@ class TLGenerator: builder.writeln( 'self.{0} = get_input_channel({0})'.format(arg.name) ) + elif arg.type == 'InputUser' and tlobject.is_function: + builder.writeln( + 'self.{0} = get_input_user({0})'.format(arg.name) + ) else: builder.writeln('self.{0} = {0}'.format(arg.name)) From 88c4cdfb52cdaa49fa8c11a87c401debb6ef1987 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 10 Jul 2017 16:09:20 +0200 Subject: [PATCH 36/52] Make get_input_* methods slightly smarter --- telethon/utils.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/telethon/utils.py b/telethon/utils.py index 362fe6bb..60dfd76e 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -6,9 +6,9 @@ from mimetypes import add_type, guess_extension from .tl.types import ( Channel, ChannelForbidden, Chat, ChatEmpty, ChatForbidden, ChatFull, - ChatPhoto, InputPeerChannel, InputPeerChat, InputPeerUser, + ChatPhoto, InputPeerChannel, InputPeerChat, InputPeerUser, InputPeerEmpty, MessageMediaDocument, MessageMediaPhoto, PeerChannel, InputChannel, - UserEmpty, InputUser, InputUserEmpty, InputUserSelf, + UserEmpty, InputUser, InputUserEmpty, InputUserSelf, InputPeerSelf, PeerChat, PeerUser, User, UserFull, UserProfilePhoto) @@ -57,7 +57,10 @@ def get_input_peer(entity): return entity if isinstance(entity, User): - return InputPeerUser(entity.id, entity.access_hash) + if entity.is_self: + return InputPeerSelf() + else: + return InputPeerUser(entity.id, entity.access_hash) if any(isinstance(entity, c) for c in ( Chat, ChatEmpty, ChatForbidden)): @@ -68,8 +71,14 @@ def get_input_peer(entity): return InputPeerChannel(entity.id, entity.access_hash) # Less common cases + if isinstance(entity, UserEmpty): + return InputPeerEmpty() + + if isinstance(entity, InputUser): + return InputPeerUser(entity.user_id, entity.access_hash) + if isinstance(entity, UserFull): - return InputPeerUser(entity.user.id, entity.user.access_hash) + return get_input_peer(entity.user) if isinstance(entity, ChatFull): return InputPeerChat(entity.id) @@ -110,6 +119,9 @@ def get_input_user(entity): if isinstance(entity, UserFull): return get_input_user(entity.user) + if isinstance(entity, InputPeerUser): + return InputUser(entity.user_id, entity.access_hash) + raise ValueError('Cannot cast {} to any kind of InputUser.' .format(type(entity).__name__)) From 5ded836437eb0a744e2bbc0614dcedfaa2ef495b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 10 Jul 2017 16:13:45 +0200 Subject: [PATCH 37/52] Update to v0.11.4 --- telethon/telegram_bare_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index c45498cf..e81f7d10 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -50,7 +50,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.11.3' + __version__ = '0.11.4' # region Initialization From cfea0f80dae73e9db80e9617fafb85727d3248f5 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 11 Jul 2017 11:14:58 +0200 Subject: [PATCH 38/52] Consider vector attributes for get_input_* utils (closes #166) --- telethon_generator/tl_generator.py | 32 ++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index e6e3dc4d..f7ab886f 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -322,19 +322,15 @@ class TLGenerator: ) else: raise ValueError('Cannot infer a value for ', arg) + + # Well-known cases, auto-cast it to the right type elif arg.type == 'InputPeer' and tlobject.is_function: - # Well-known case, auto-cast it to the right type - builder.writeln( - 'self.{0} = get_input_peer({0})'.format(arg.name) - ) + TLGenerator.write_get_input(builder, arg, 'get_input_peer') elif arg.type == 'InputChannel' and tlobject.is_function: - builder.writeln( - 'self.{0} = get_input_channel({0})'.format(arg.name) - ) + TLGenerator.write_get_input(builder, arg, 'get_input_channel') elif arg.type == 'InputUser' and tlobject.is_function: - builder.writeln( - 'self.{0} = get_input_user({0})'.format(arg.name) - ) + TLGenerator.write_get_input(builder, arg, 'get_input_user') + else: builder.writeln('self.{0} = {0}'.format(arg.name)) @@ -434,6 +430,22 @@ class TLGenerator: builder.writeln('return MTProtoRequest.pretty_format(self, indent=0)') # builder.end_block() # No need to end the last block + @staticmethod + def write_get_input(builder, arg, get_input_code): + """Returns "True" if the get_input_* code was written when assigning + a parameter upon creating the request. Returns False otherwise + """ + if arg.is_vector: + builder.writeln( + 'self.{0} = [{1}({0}_item) for {0}_item in {0}]' + .format(arg.name, get_input_code) + ) + pass + else: + builder.writeln( + 'self.{0} = {1}({0})'.format(arg.name, get_input_code) + ) + @staticmethod def get_class_name(tlobject): """Gets the class name following the Python style guidelines, in ThisClassFormat""" From 9b17888e3e9baa2a5cb6b19478042192b74c18b4 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 11 Jul 2017 11:36:00 +0200 Subject: [PATCH 39/52] Make generated code slightly smaller --- telethon_generator/tl_generator.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index f7ab886f..c5bb4f3c 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -437,7 +437,7 @@ class TLGenerator: """ if arg.is_vector: builder.writeln( - 'self.{0} = [{1}({0}_item) for {0}_item in {0}]' + 'self.{0} = [{1}(_x) for _x in {0}]' .format(arg.name, get_input_code) ) pass @@ -504,11 +504,10 @@ class TLGenerator: "writer.write_int(0x1cb5c415, signed=False) # Vector's constructor ID") builder.writeln('writer.write_int(len({}))'.format(name)) - builder.writeln('for {}_item in {}:'.format(arg.name, name)) + builder.writeln('for _x in {}:'.format(name)) # Temporary disable .is_vector, not to enter this if again arg.is_vector = False - TLGenerator.write_onsend_code( - builder, arg, args, name='{}_item'.format(arg.name)) + TLGenerator.write_onsend_code(builder, arg, args, name='_x') arg.is_vector = True elif arg.flag_indicator: @@ -599,13 +598,12 @@ class TLGenerator: builder.writeln("reader.read_int() # Vector's constructor ID") builder.writeln('{} = [] # Initialize an empty list'.format(name)) - builder.writeln('{}_len = reader.read_int()'.format(arg.name)) - builder.writeln('for _ in range({}_len):'.format(arg.name)) + builder.writeln('_len = reader.read_int()') + builder.writeln('for _ in range(_len):') # Temporary disable .is_vector, not to enter this if again arg.is_vector = False - TLGenerator.write_onresponse_code( - builder, arg, args, name='{}_item'.format(arg.name)) - builder.writeln('{}.append({}_item)'.format(name, arg.name)) + TLGenerator.write_onresponse_code(builder, arg, args, name='_x') + builder.writeln('{}.append(_x)'.format(name)) arg.is_vector = True elif arg.flag_indicator: @@ -620,12 +618,14 @@ class TLGenerator: builder.writeln('{} = reader.read_long()'.format(name)) elif 'int128' == arg.type: - builder.writeln('{} = reader.read_large_int(bits=128)'.format( - name)) + builder.writeln( + '{} = reader.read_large_int(bits=128)'.format(name) + ) elif 'int256' == arg.type: - builder.writeln('{} = reader.read_large_int(bits=256)'.format( - name)) + builder.writeln( + '{} = reader.read_large_int(bits=256)'.format(name) + ) elif 'double' == arg.type: builder.writeln('{} = reader.read_double()'.format(name)) From 3be995b5a33de5edf9ecac9b8264767bc9b413de Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 11 Jul 2017 11:38:17 +0200 Subject: [PATCH 40/52] Update to v0.11.5 --- telethon/telegram_bare_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index e81f7d10..933ca084 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -50,7 +50,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.11.4' + __version__ = '0.11.5' # region Initialization From c9e566342eb4007d1f2ac93106bb35f6330dbe37 Mon Sep 17 00:00:00 2001 From: hnikaein Date: Thu, 20 Jul 2017 12:07:19 +0430 Subject: [PATCH 41/52] All download_* methods now accept streams --- telethon/telegram_client.py | 73 +++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 7f7ba729..79adb8aa 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -542,33 +542,33 @@ class TelegramClient(TelegramBareClient): def download_msg_media(self, message_media, - file_path, + file, add_extension=True, progress_callback=None): """Downloads the given MessageMedia (Photo, Document or Contact) - into the desired file_path, optionally finding its extension automatically + into the desired file(a stream or str), optionally finding its extension automatically The progress_callback should be a callback function which takes two parameters, uploaded size (in bytes) and total file size (in bytes). This will be called every time a part is downloaded""" if type(message_media) == MessageMediaPhoto: - return self.download_photo(message_media, file_path, add_extension, + return self.download_photo(message_media, file, add_extension, progress_callback) elif type(message_media) == MessageMediaDocument: - return self.download_document(message_media, file_path, + return self.download_document(message_media, file, add_extension, progress_callback) elif type(message_media) == MessageMediaContact: - return self.download_contact(message_media, file_path, + return self.download_contact(message_media, file, add_extension) def download_photo(self, message_media_photo, - file_path, + file, add_extension=False, progress_callback=None): """Downloads MessageMediaPhoto's largest size into the desired - file_path, optionally finding its extension automatically + file(a stream or str), optionally finding its extension automatically The progress_callback should be a callback function which takes two parameters, uploaded size (in bytes) and total file size (in bytes). This will be called every time a part is downloaded""" @@ -579,8 +579,8 @@ class TelegramClient(TelegramBareClient): file_size = largest_size.size largest_size = largest_size.location - if add_extension: - file_path += get_extension(message_media_photo) + if isinstance(file, str) and add_extension: + file += get_extension(message_media_photo) # Download the media with the largest size input file location self.download_file( @@ -589,19 +589,19 @@ class TelegramClient(TelegramBareClient): local_id=largest_size.local_id, secret=largest_size.secret ), - file_path, + file, file_size=file_size, progress_callback=progress_callback ) - return file_path + return file def download_document(self, message_media_document, - file_path=None, + file=None, add_extension=True, progress_callback=None): """Downloads the given MessageMediaDocument into the desired - file_path, optionally finding its extension automatically. + file(a stream or str), optionally finding its extension automatically. If no file_path is given, it will try to be guessed from the document The progress_callback should be a callback function which takes two parameters, uploaded size (in bytes) and total file size (in bytes). @@ -610,21 +610,21 @@ class TelegramClient(TelegramBareClient): file_size = document.size # If no file path was given, try to guess it from the attributes - if file_path is None: + if file is None: for attr in document.attributes: if type(attr) == DocumentAttributeFilename: - file_path = attr.file_name + file = attr.file_name break # This attribute has higher preference elif type(attr) == DocumentAttributeAudio: - file_path = '{} - {}'.format(attr.performer, attr.title) + file = '{} - {}'.format(attr.performer, attr.title) - if file_path is None: + if file is None: raise ValueError('Could not infer a file_path for the document' '. Please provide a valid file_path manually') - if add_extension: - file_path += get_extension(message_media_document) + if isinstance(file, str) and add_extension: + file += get_extension(message_media_document) self.download_file( InputDocumentFileLocation( @@ -632,39 +632,50 @@ class TelegramClient(TelegramBareClient): access_hash=document.access_hash, version=document.version ), - file_path, + file, file_size=file_size, progress_callback=progress_callback ) - return file_path + return file @staticmethod - def download_contact(message_media_contact, file_path, add_extension=True): + def download_contact(message_media_contact, file, add_extension=True): """Downloads a media contact using the vCard 4.0 format""" first_name = message_media_contact.first_name last_name = message_media_contact.last_name phone_number = message_media_contact.phone_number - # The only way we can save a contact in an understandable - # way by phones is by using the .vCard format - if add_extension: - file_path += '.vcard' + if isinstance(file, str): + # The only way we can save a contact in an understandable + # way by phones is by using the .vCard format + if add_extension: + file += '.vcard' - # Ensure that we'll be able to download the contact - utils.ensure_parent_dir_exists(file_path) + # Ensure that we'll be able to download the contact + utils.ensure_parent_dir_exists(file) - with open(file_path, 'w', encoding='utf-8') as file: + with open(file, 'w', encoding='utf-8') as f: + f.write('BEGIN:VCARD\n') + f.write('VERSION:4.0\n') + f.write('N:{};{};;;\n'.format(first_name, last_name + if last_name else '')) + f.write('FN:{}\n'.format(' '.join((first_name, last_name)))) + f.write('TEL;TYPE=cell;VALUE=uri:tel:+{}\n'.format( + phone_number)) + f.write('END:VCARD\n') + + else: file.write('BEGIN:VCARD\n') file.write('VERSION:4.0\n') file.write('N:{};{};;;\n'.format(first_name, last_name - if last_name else '')) + if last_name else '')) file.write('FN:{}\n'.format(' '.join((first_name, last_name)))) file.write('TEL;TYPE=cell;VALUE=uri:tel:+{}\n'.format( phone_number)) file.write('END:VCARD\n') - return file_path + return file # endregion From fe2e9f335b62abd2fc967cf1630286803999c7d0 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 23 Jul 2017 17:08:04 +0200 Subject: [PATCH 42/52] Style enhancements for PR #173 --- telethon/telegram_client.py | 45 +++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 79adb8aa..a5928715 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -546,10 +546,13 @@ class TelegramClient(TelegramBareClient): add_extension=True, progress_callback=None): """Downloads the given MessageMedia (Photo, Document or Contact) - into the desired file(a stream or str), optionally finding its extension automatically - The progress_callback should be a callback function which takes two parameters, - uploaded size (in bytes) and total file size (in bytes). - This will be called every time a part is downloaded""" + into the desired file (a stream or str), optionally finding its + extension automatically. + + The progress_callback should be a callback function which takes + two parameters, uploaded size and total file size (both in bytes). + This will be called every time a part is downloaded + """ if type(message_media) == MessageMediaPhoto: return self.download_photo(message_media, file, add_extension, progress_callback) @@ -654,26 +657,24 @@ class TelegramClient(TelegramBareClient): # Ensure that we'll be able to download the contact utils.ensure_parent_dir_exists(file) - - with open(file, 'w', encoding='utf-8') as f: - f.write('BEGIN:VCARD\n') - f.write('VERSION:4.0\n') - f.write('N:{};{};;;\n'.format(first_name, last_name - if last_name else '')) - f.write('FN:{}\n'.format(' '.join((first_name, last_name)))) - f.write('TEL;TYPE=cell;VALUE=uri:tel:+{}\n'.format( - phone_number)) - f.write('END:VCARD\n') - + f = open(file, 'w', encoding='utf-8') else: - file.write('BEGIN:VCARD\n') - file.write('VERSION:4.0\n') - file.write('N:{};{};;;\n'.format(first_name, last_name - if last_name else '')) - file.write('FN:{}\n'.format(' '.join((first_name, last_name)))) - file.write('TEL;TYPE=cell;VALUE=uri:tel:+{}\n'.format( + f = file + + try: + f.write('BEGIN:VCARD\n') + f.write('VERSION:4.0\n') + f.write('N:{};{};;;\n'.format( + first_name, last_name if last_name else '') + ) + f.write('FN:{}\n'.format(' '.join((first_name, last_name)))) + f.write('TEL;TYPE=cell;VALUE=uri:tel:+{}\n'.format( phone_number)) - file.write('END:VCARD\n') + f.write('END:VCARD\n') + finally: + # Only close the stream if we opened it + if isinstance(file, str): + f.close() return file From 38e0888ea00cd16904e9589dbd7806e4695b0304 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 23 Jul 2017 17:12:29 +0200 Subject: [PATCH 43/52] Update to layer 70 --- telethon_generator/scheme.tl | 42 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/telethon_generator/scheme.tl b/telethon_generator/scheme.tl index b2e1774e..b502b7b9 100644 --- a/telethon_generator/scheme.tl +++ b/telethon_generator/scheme.tl @@ -155,17 +155,16 @@ inputFile#f52ff27f id:long parts:int name:string md5_checksum:string = InputFile inputFileBig#fa4f0bb5 id:long parts:int name:string = InputFile; inputMediaEmpty#9664f57f = InputMedia; -inputMediaUploadedPhoto#630c9af1 flags:# file:InputFile caption:string stickers:flags.0?Vector = InputMedia; -inputMediaPhoto#e9bfb4f3 id:InputPhoto caption:string = InputMedia; +inputMediaUploadedPhoto#2f37e231 flags:# file:InputFile caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; +inputMediaPhoto#81fa373a flags:# id:InputPhoto caption:string ttl_seconds:flags.0?int = InputMedia; inputMediaGeoPoint#f9c44144 geo_point:InputGeoPoint = InputMedia; inputMediaContact#a6e45987 phone_number:string first_name:string last_name:string = InputMedia; -inputMediaUploadedDocument#d070f1e9 flags:# file:InputFile mime_type:string attributes:Vector caption:string stickers:flags.0?Vector = InputMedia; -inputMediaUploadedThumbDocument#50d88cae flags:# file:InputFile thumb:InputFile mime_type:string attributes:Vector caption:string stickers:flags.0?Vector = InputMedia; -inputMediaDocument#1a77f29c id:InputDocument caption:string = InputMedia; +inputMediaUploadedDocument#e39621fd flags:# file:InputFile thumb:flags.2?InputFile mime_type:string attributes:Vector caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; +inputMediaDocument#5acb668e flags:# id:InputDocument caption:string ttl_seconds:flags.0?int = InputMedia; inputMediaVenue#2827a81a geo_point:InputGeoPoint title:string address:string provider:string venue_id:string = InputMedia; inputMediaGifExternal#4843b0fd url:string q:string = InputMedia; -inputMediaPhotoExternal#b55f4f18 url:string caption:string = InputMedia; -inputMediaDocumentExternal#e5e9607c url:string caption:string = InputMedia; +inputMediaPhotoExternal#922aec1 flags:# url:string caption:string ttl_seconds:flags.0?int = InputMedia; +inputMediaDocumentExternal#b6f74335 flags:# url:string caption:string ttl_seconds:flags.0?int = InputMedia; inputMediaGame#d33f43f3 id:InputGame = InputMedia; inputMediaInvoice#92153685 flags:# title:string description:string photo:flags.0?InputWebDocument invoice:Invoice payload:bytes provider:string start_param:string = InputMedia; @@ -219,7 +218,7 @@ userStatusLastMonth#77ebc742 = UserStatus; chatEmpty#9ba2d800 id:int = Chat; chat#d91cdd54 flags:# creator:flags.0?true kicked:flags.1?true left:flags.2?true admins_enabled:flags.3?true admin:flags.4?true deactivated:flags.5?true id:int title:string photo:ChatPhoto participants_count:int date:int version:int migrated_to:flags.6?InputChannel = Chat; chatForbidden#7328bdb id:int title:string = Chat; -channel#cb44b1c flags:# creator:flags.0?true left:flags.2?true broadcast:flags.5?true verified:flags.7?true megagroup:flags.8?true restricted:flags.9?true democracy:flags.10?true signatures:flags.11?true min:flags.12?true id:int access_hash:flags.13?long title:string username:flags.6?string photo:ChatPhoto date:int version:int restriction_reason:flags.9?string admin_rights:flags.14?ChannelAdminRights banned_rights:flags.15?ChannelBannedRights = Chat; +channel#cb44b1c flags:# creator:flags.0?true left:flags.2?true editor:flags.3?true broadcast:flags.5?true verified:flags.7?true megagroup:flags.8?true restricted:flags.9?true democracy:flags.10?true signatures:flags.11?true min:flags.12?true id:int access_hash:flags.13?long title:string username:flags.6?string photo:ChatPhoto date:int version:int restriction_reason:flags.9?string admin_rights:flags.14?ChannelAdminRights banned_rights:flags.15?ChannelBannedRights = Chat; channelForbidden#289da732 flags:# broadcast:flags.5?true megagroup:flags.8?true id:int access_hash:long title:string until_date:flags.16?int = Chat; chatFull#2e02a614 id:int participants:ChatParticipants chat_photo:Photo notify_settings:PeerNotifySettings exported_invite:ExportedChatInvite bot_info:Vector = ChatFull; @@ -236,15 +235,15 @@ chatPhotoEmpty#37c1011c = ChatPhoto; chatPhoto#6153276a photo_small:FileLocation photo_big:FileLocation = ChatPhoto; messageEmpty#83e5de54 id:int = Message; -message#c09be45f flags:# out:flags.1?true mentioned:flags.4?true media_unread:flags.5?true silent:flags.13?true post:flags.14?true id:int from_id:flags.8?int to_id:Peer fwd_from:flags.2?MessageFwdHeader via_bot_id:flags.11?int reply_to_msg_id:flags.3?int date:int message:string media:flags.9?MessageMedia reply_markup:flags.6?ReplyMarkup entities:flags.7?Vector views:flags.10?int edit_date:flags.15?int = Message; +message#90dddc11 flags:# out:flags.1?true mentioned:flags.4?true media_unread:flags.5?true silent:flags.13?true post:flags.14?true id:int from_id:flags.8?int to_id:Peer fwd_from:flags.2?MessageFwdHeader via_bot_id:flags.11?int reply_to_msg_id:flags.3?int date:int message:string media:flags.9?MessageMedia reply_markup:flags.6?ReplyMarkup entities:flags.7?Vector views:flags.10?int edit_date:flags.15?int post_author:flags.16?string = Message; messageService#9e19a1f6 flags:# out:flags.1?true mentioned:flags.4?true media_unread:flags.5?true silent:flags.13?true post:flags.14?true id:int from_id:flags.8?int to_id:Peer reply_to_msg_id:flags.3?int date:int action:MessageAction = Message; messageMediaEmpty#3ded6320 = MessageMedia; -messageMediaPhoto#3d8ce53d photo:Photo caption:string = MessageMedia; +messageMediaPhoto#b5223b0f flags:# photo:flags.0?Photo caption:flags.1?string ttl_seconds:flags.2?int = MessageMedia; messageMediaGeo#56e0d474 geo:GeoPoint = MessageMedia; messageMediaContact#5e7d2f39 phone_number:string first_name:string last_name:string user_id:int = MessageMedia; messageMediaUnsupported#9f84f49e = MessageMedia; -messageMediaDocument#f3e02ea8 document:Document caption:string = MessageMedia; +messageMediaDocument#7c4414d3 flags:# document:flags.0?Document caption:flags.1?string ttl_seconds:flags.2?int = MessageMedia; messageMediaWebPage#a32dd600 webpage:WebPage = MessageMedia; messageMediaVenue#7912b71f geo:GeoPoint title:string address:string provider:string venue_id:string = MessageMedia; messageMediaGame#fdb19008 game:Game = MessageMedia; @@ -267,6 +266,7 @@ messageActionGameScore#92a72876 game_id:long score:int = MessageAction; messageActionPaymentSentMe#8f31b327 flags:# currency:string total_amount:long payload:bytes info:flags.0?PaymentRequestedInfo shipping_option_id:flags.1?string charge:PaymentCharge = MessageAction; messageActionPaymentSent#40699cd0 currency:string total_amount:long = MessageAction; messageActionPhoneCall#80e11a7f flags:# call_id:long reason:flags.0?PhoneCallDiscardReason duration:flags.1?int = MessageAction; +messageActionScreenshotTaken#4792929b = MessageAction; dialog#66ffba14 flags:# pinned:flags.2?true peer:Peer top_message:int read_inbox_max_id:int read_outbox_max_id:int unread_count:int notify_settings:PeerNotifySettings pts:flags.0?int draft:flags.1?DraftMessage = Dialog; @@ -329,7 +329,7 @@ contacts.link#3ace484c my_link:ContactLink foreign_link:ContactLink user:User = contacts.contactsNotModified#b74ba9d2 = contacts.Contacts; contacts.contacts#6f8b8cb2 contacts:Vector users:Vector = contacts.Contacts; -contacts.importedContacts#ad524315 imported:Vector retry_contacts:Vector users:Vector = contacts.ImportedContacts; +contacts.importedContacts#77d01c3b imported:Vector popular_invites:Vector retry_contacts:Vector users:Vector = contacts.ImportedContacts; contacts.blocked#1c138d15 blocked:Vector users:Vector = contacts.Blocked; contacts.blockedSlice#900802a1 count:int blocked:Vector users:Vector = contacts.Blocked; @@ -447,7 +447,7 @@ photos.photosSlice#15051f54 count:int photos:Vector users:Vector = photos.photo#20212ca8 photo:Photo users:Vector = photos.Photo; upload.file#96a18d5 type:storage.FileType mtime:int bytes:bytes = upload.File; -upload.fileCdnRedirect#1508485a dc_id:int file_token:bytes encryption_key:bytes encryption_iv:bytes = upload.File; +upload.fileCdnRedirect#ea52fe5a dc_id:int file_token:bytes encryption_key:bytes encryption_iv:bytes cdn_file_hashes:Vector = upload.File; dcOption#5d8c6cc flags:# ipv6:flags.0?true media_only:flags.1?true tcpo_only:flags.2?true cdn:flags.3?true static:flags.4?true id:int ip_address:string port:int = DcOption; @@ -699,7 +699,7 @@ messages.botResults#ccd3563d flags:# gallery:flags.0?true query_id:long next_off exportedMessageLink#1f486803 link:string = ExportedMessageLink; -messageFwdHeader#c786ddcb flags:# from_id:flags.0?int date:int channel_id:flags.1?int channel_post:flags.2?int = MessageFwdHeader; +messageFwdHeader#fadff4ac flags:# from_id:flags.0?int date:int channel_id:flags.1?int channel_post:flags.2?int post_author:flags.3?string = MessageFwdHeader; auth.codeTypeSms#72a3158c = auth.CodeType; auth.codeTypeCall#741cd3e3 = auth.CodeType; @@ -905,6 +905,10 @@ channels.adminLogResults#ed8af74d events:Vector chats:Vect channelAdminLogEventsFilter#ea107ae4 flags:# join:flags.0?true leave:flags.1?true invite:flags.2?true ban:flags.3?true unban:flags.4?true kick:flags.5?true unkick:flags.6?true promote:flags.7?true demote:flags.8?true info:flags.9?true settings:flags.10?true pinned:flags.11?true edit:flags.12?true delete:flags.13?true = ChannelAdminLogEventsFilter; +popularContact#5ce14175 client_id:long importers:int = PopularContact; + +cdnFileHash#77eec38f offset:int limit:int hash:bytes = CdnFileHash; + ---functions--- invokeAfterMsg#cb9f372d {X:Type} msg_id:long query:!X = X; @@ -1063,6 +1067,7 @@ messages.getPinnedDialogs#e254d64e = messages.PeerDialogs; messages.setBotShippingResults#e5f672fa flags:# query_id:long error:flags.0?string shipping_options:flags.1?Vector = Bool; messages.setBotPrecheckoutResults#9c2dd95 flags:# success:flags.1?true query_id:long error:flags.0?string = Bool; messages.uploadMedia#519bc2b1 peer:InputPeer media:InputMedia = MessageMedia; +messages.sendScreenshotNotification#c97df020 peer:InputPeer reply_to_msg_id:int random_id:long = Updates; updates.getState#edd4882a = updates.State; updates.getDifference#25939651 flags:# pts:int pts_total_limit:flags.0?int date:int qts:int = updates.Difference; @@ -1078,7 +1083,8 @@ upload.getFile#e3a6cfb5 location:InputFileLocation offset:int limit:int = upload upload.saveBigFilePart#de7b673d file_id:long file_part:int file_total_parts:int bytes:bytes = Bool; upload.getWebFile#24e6818d location:InputWebFileLocation offset:int limit:int = upload.WebFile; upload.getCdnFile#2000bcc3 file_token:bytes offset:int limit:int = upload.CdnFile; -upload.reuploadCdnFile#2e7a2020 file_token:bytes request_token:bytes = Bool; +upload.reuploadCdnFile#1af91c09 file_token:bytes request_token:bytes = Vector; +upload.getCdnFileHashes#f715c87b file_token:bytes offset:int = Vector; help.getConfig#c4f9186b = Config; help.getNearestDc#1fb33026 = NearestDc; @@ -1131,8 +1137,8 @@ payments.getSavedInfo#227d824b = payments.SavedInfo; payments.clearSavedInfo#d83d70c1 flags:# credentials:flags.0?true info:flags.1?true = Bool; stickers.createStickerSet#9bd86e6a flags:# masks:flags.0?true user_id:InputUser title:string short_name:string stickers:Vector = messages.StickerSet; -stickers.removeStickerFromSet#4255934 sticker:InputDocument = Bool; -stickers.changeStickerPosition#4ed705ca sticker:InputDocument position:int = Bool; +stickers.removeStickerFromSet#f7760f51 sticker:InputDocument = messages.StickerSet; +stickers.changeStickerPosition#ffb6d4ca sticker:InputDocument position:int = messages.StickerSet; stickers.addStickerToSet#8653febe stickerset:InputStickerSet sticker:InputStickerSetItem = messages.StickerSet; phone.getCallConfig#55451fa9 = DataJSON; @@ -1149,4 +1155,4 @@ langpack.getStrings#2e1ee318 lang_code:string keys:Vector = Vector; -// LAYER 68 +// LAYER 70 From 773376ee215b6e73a39089f23c931f9d5930f580 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 23 Jul 2017 18:38:27 +0200 Subject: [PATCH 44/52] Fix two more spelling mistakes --- telethon/telegram_client.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index a5928715..f9986711 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -570,11 +570,13 @@ class TelegramClient(TelegramBareClient): file, add_extension=False, progress_callback=None): - """Downloads MessageMediaPhoto's largest size into the desired - file(a stream or str), optionally finding its extension automatically - The progress_callback should be a callback function which takes two parameters, - uploaded size (in bytes) and total file size (in bytes). - This will be called every time a part is downloaded""" + """Downloads MessageMediaPhoto's largest size into the desired file + (a stream or str), optionally finding its extension automatically. + + The progress_callback should be a callback function which takes + two parameters, uploaded size and total file size (both in bytes). + This will be called every time a part is downloaded + """ # Determine the photo and its largest size photo = message_media_photo.photo @@ -603,12 +605,15 @@ class TelegramClient(TelegramBareClient): file=None, add_extension=True, progress_callback=None): - """Downloads the given MessageMediaDocument into the desired - file(a stream or str), optionally finding its extension automatically. - If no file_path is given, it will try to be guessed from the document - The progress_callback should be a callback function which takes two parameters, - uploaded size (in bytes) and total file size (in bytes). - This will be called every time a part is downloaded""" + """Downloads the given MessageMediaDocument into the desired file + (a stream or str), optionally finding its extension automatically. + + If no file_path is given it will try to be guessed from the document. + + The progress_callback should be a callback function which takes + two parameters, uploaded size and total file size (both in bytes). + This will be called every time a part is downloaded + """ document = message_media_document.document file_size = document.size From 160a3699ac07ba22469eafe13cce2dcff4a996ff Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 24 Jul 2017 16:54:48 +0200 Subject: [PATCH 45/52] Fix confusing names "MtProtoRequest" and ".confirmed" (#176) This also fixes the annoyingly confusing message: "Odd msg_seqno expected (relevant message), but even received." --- telethon/network/mtproto_sender.py | 2 +- telethon/telegram_bare_client.py | 6 ++--- telethon/telegram_client.py | 8 +------ telethon/tl/__init__.py | 2 +- .../tl/{mtproto_request.py => tlobject.py} | 22 +++++++++---------- telethon_generator/tl_generator.py | 18 +++++++-------- 6 files changed, 26 insertions(+), 32 deletions(-) rename telethon/tl/{mtproto_request.py => tlobject.py} (82%) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 32059986..38bfd917 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -143,7 +143,7 @@ class MtProtoSender: plain_writer.write_long(self.session.id, signed=False) plain_writer.write_long(request.request_msg_id) plain_writer.write_int( - self.session.generate_sequence(request.confirmed)) + self.session.generate_sequence(request.content_related)) plain_writer.write_int(len(packet)) plain_writer.write(packet) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 933ca084..aa27b67c 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -10,7 +10,7 @@ from .network import authenticator, MtProtoSender, TcpTransport from .utils import get_appropriated_part_size # For sending and receiving requests -from .tl import MTProtoRequest, JsonSession +from .tl import TLObject, JsonSession from .tl.all_tlobjects import layer from .tl.functions import (InitConnectionRequest, InvokeWithLayerRequest) @@ -265,8 +265,8 @@ class TelegramBareClient: If 'updates' is not None, all read update object will be put in such list. Otherwise, update objects will be ignored. """ - if not isinstance(request, MTProtoRequest): - raise ValueError('You can only invoke MtProtoRequests') + if not isinstance(request, TLObject) and not request.content_related: + raise ValueError('You can only invoke requests, not types!') if not self._sender: raise ValueError('You must be connected to invoke requests!') diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index f9986711..703f3b3f 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -14,7 +14,7 @@ from .errors import (RPCError, UnauthorizedError, InvalidParameterError, PhoneCodeInvalidError, InvalidChecksumError) # For sending and receiving requests -from .tl import MTProtoRequest, Session, JsonSession +from .tl import Session, JsonSession # Required to get the password salt from .tl.functions.account import GetPasswordRequest @@ -188,12 +188,6 @@ class TelegramClient(TelegramBareClient): *args will be ignored. """ - if not issubclass(type(request), MTProtoRequest): - raise ValueError('You can only invoke MtProtoRequests') - - if not self._sender: - raise ValueError('You must be connected to invoke requests!') - if self._updates_thread_receiving.is_set(): self._sender.cancel_receive() diff --git a/telethon/tl/__init__.py b/telethon/tl/__init__.py index 8f58f5c2..404850ba 100644 --- a/telethon/tl/__init__.py +++ b/telethon/tl/__init__.py @@ -1,2 +1,2 @@ -from .mtproto_request import MTProtoRequest +from .tlobject import TLObject from .session import Session, JsonSession diff --git a/telethon/tl/mtproto_request.py b/telethon/tl/tlobject.py similarity index 82% rename from telethon/tl/mtproto_request.py rename to telethon/tl/tlobject.py index 8a103e16..d1cc63ba 100644 --- a/telethon/tl/mtproto_request.py +++ b/telethon/tl/tlobject.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta -class MTProtoRequest: +class TLObject: def __init__(self): self.sent = False @@ -14,7 +14,7 @@ class MTProtoRequest: # These should be overrode self.constructor_id = 0 - self.confirmed = False + self.content_related = False # Only requests/functions/queries are self.responded = False # These should not be overrode @@ -27,7 +27,7 @@ class MTProtoRequest: def need_resend(self): return self.dirty or ( - self.confirmed and not self.confirm_received and + self.content_related and not self.confirm_received and datetime.now() - self.send_time > timedelta(seconds=3)) @staticmethod @@ -36,32 +36,32 @@ class MTProtoRequest: If indent is None, a single line will be returned. """ if indent is None: - if isinstance(obj, MTProtoRequest): + if isinstance(obj, TLObject): return '{{{}: {}}}'.format( type(obj).__name__, - MTProtoRequest.pretty_format(obj.to_dict()) + TLObject.pretty_format(obj.to_dict()) ) if isinstance(obj, dict): return '{{{}}}'.format(', '.join( '{}: {}'.format( - k, MTProtoRequest.pretty_format(v) + k, TLObject.pretty_format(v) ) for k, v in obj.items() )) elif isinstance(obj, str): return '"{}"'.format(obj) elif hasattr(obj, '__iter__'): return '[{}]'.format( - ', '.join(MTProtoRequest.pretty_format(x) for x in obj) + ', '.join(TLObject.pretty_format(x) for x in obj) ) else: return str(obj) else: result = [] - if isinstance(obj, MTProtoRequest): + if isinstance(obj, TLObject): result.append('{') result.append(type(obj).__name__) result.append(': ') - result.append(MTProtoRequest.pretty_format( + result.append(TLObject.pretty_format( obj.to_dict(), indent )) @@ -72,7 +72,7 @@ class MTProtoRequest: result.append('\t' * indent) result.append(k) result.append(': ') - result.append(MTProtoRequest.pretty_format(v, indent)) + result.append(TLObject.pretty_format(v, indent)) result.append(',\n') indent -= 1 result.append('\t' * indent) @@ -88,7 +88,7 @@ class MTProtoRequest: indent += 1 for x in obj: result.append('\t' * indent) - result.append(MTProtoRequest.pretty_format(x, indent)) + result.append(TLObject.pretty_format(x, indent)) result.append(',\n') indent -= 1 result.append('\t' * indent) diff --git a/telethon_generator/tl_generator.py b/telethon_generator/tl_generator.py index c5bb4f3c..3f4973a4 100644 --- a/telethon_generator/tl_generator.py +++ b/telethon_generator/tl_generator.py @@ -177,10 +177,10 @@ class TLGenerator: importing and documentation strings. '""" - # Both types and functions inherit from - # MTProtoRequest so they all can be sent - # TODO MTProtoRequest is not the best name for a type - builder.writeln('from {}.tl.mtproto_request import MTProtoRequest' + # Both types and functions inherit from the TLObject class so they + # all can be serialized and sent, however, only the functions are + # "content_related". + builder.writeln('from {}.tl.tlobject import TLObject' .format('.' * depth)) if tlobject.is_function: @@ -205,7 +205,7 @@ class TLGenerator: builder.writeln() builder.writeln() - builder.writeln('class {}(MTProtoRequest):'.format( + builder.writeln('class {}(TLObject):'.format( TLGenerator.get_class_name(tlobject))) # Write the original .tl definition, @@ -269,7 +269,7 @@ class TLGenerator: builder.write(' Must be a list.'.format(arg.name)) if arg.is_generic: - builder.write(' Must be another MTProtoRequest.') + builder.write(' Must be another TLObject request.') builder.writeln() @@ -301,7 +301,7 @@ class TLGenerator: if tlobject.is_function: builder.writeln('self.result = None') builder.writeln( - 'self.confirmed = True # Confirmed by default') + 'self.content_related = True') # Set the arguments if args: @@ -423,11 +423,11 @@ class TLGenerator: builder.end_block() builder.writeln('def __str__(self):') - builder.writeln('return MTProtoRequest.pretty_format(self)') + builder.writeln('return TLObject.pretty_format(self)') builder.end_block() builder.writeln('def stringify(self):') - builder.writeln('return MTProtoRequest.pretty_format(self, indent=0)') + builder.writeln('return TLObject.pretty_format(self, indent=0)') # builder.end_block() # No need to end the last block @staticmethod From 9d35a836d1577948dc96fb2036120e11f8647008 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 24 Jul 2017 16:56:05 +0200 Subject: [PATCH 46/52] Fix interactive example not working after #173 --- telethon_examples/interactive_telegram_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon_examples/interactive_telegram_client.py b/telethon_examples/interactive_telegram_client.py index 0002b17d..ee2c67c4 100644 --- a/telethon_examples/interactive_telegram_client.py +++ b/telethon_examples/interactive_telegram_client.py @@ -250,7 +250,7 @@ class InteractiveTelegramClient(TelegramClient): print('Downloading media with name {}...'.format(output)) output = self.download_msg_media( msg.media, - file_path=output, + file=output, progress_callback=self.download_progress_callback) print('Media downloaded to {}!'.format(output)) From 7844cd358ebccd858f36c94cb5ac22b37e92bf4f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 26 Jul 2017 16:10:45 +0200 Subject: [PATCH 47/52] Attempt at making layer migrations more smooth (#158) --- telethon/network/mtproto_sender.py | 9 +++++++-- telethon/telegram_bare_client.py | 10 +++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 38bfd917..1dce4b05 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -281,9 +281,14 @@ class MtProtoSender: # Note that this code is IMPORTANT for skipping RPC results of # lost requests (i.e., ones from the previous connection session) - if not self._process_msg( - inner_msg_id, sequence, reader, updates): + try: + if not self._process_msg( + inner_msg_id, sequence, reader, updates): + reader.set_position(begin_position + inner_length) + except: + # If any error is raised, something went wrong; skip the packet reader.set_position(begin_position + inner_length) + raise return True diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index aa27b67c..8945d6f5 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -5,7 +5,9 @@ from os import path # Import some externalized utilities to work with the Telegram types and more from . import helpers as utils -from .errors import RPCError, FloodWaitError, FileMigrateError +from .errors import ( + RPCError, FloodWaitError, FileMigrateError, TypeNotFoundError +) from .network import authenticator, MtProtoSender, TcpTransport from .utils import get_appropriated_part_size @@ -140,6 +142,12 @@ class TelegramBareClient: self.dc_options = result.dc_options return True + except TypeNotFoundError as e: + # This is fine, probably layer migration + self._logger.debug('Found invalid item, probably migrating', e) + self.disconnect() + self.connect(exported_auth=exported_auth) + except (RPCError, ConnectionError) as error: # Probably errors from the previous session, ignore them self.disconnect() From 18e65cbf322b6903d0304994dbaccaa3b98f87ab Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 29 Jul 2017 10:38:31 +0200 Subject: [PATCH 48/52] Mention SyntaxError when installing via pip on the README --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 1565670f..aa5a8290 100755 --- a/README.rst +++ b/README.rst @@ -58,11 +58,11 @@ On a terminal, issue the following command: sudo -H pip install telethon -You're ready to go. Oh, and upgrading is just as easy: +If you get something like "SyntaxError: invalid syntax" on the ``from error`` +line, it's because ``pip`` defaults to Python 2. Use `pip3` instead. -.. code:: sh - - sudo -H pip install --upgrade telethon +If you already have Telethon installed, +upgrade with ``pip install --upgrade telethon``! Installing Telethon manually ---------------------------- From 9e88d9d219e9254c239fe50240b42af8ad8c6670 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 5 Aug 2017 09:36:07 +0200 Subject: [PATCH 49/52] Replace "type is Type" check with "isinstance" --- telethon_examples/interactive_telegram_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telethon_examples/interactive_telegram_client.py b/telethon_examples/interactive_telegram_client.py index ee2c67c4..a6346aaa 100644 --- a/telethon_examples/interactive_telegram_client.py +++ b/telethon_examples/interactive_telegram_client.py @@ -275,7 +275,7 @@ class InteractiveTelegramClient(TelegramClient): @staticmethod def update_handler(update_object): - if type(update_object) is UpdateShortMessage: + if isinstance(update_object, UpdateShortMessage): if update_object.out: sprint('You sent {} to user #{}'.format( update_object.message, update_object.user_id)) @@ -283,7 +283,7 @@ class InteractiveTelegramClient(TelegramClient): sprint('[User #{} sent {}]'.format( update_object.user_id, update_object.message)) - elif type(update_object) is UpdateShortChatMessage: + elif isinstance(update_object, UpdateShortChatMessage): if update_object.out: sprint('You sent {} to chat #{}'.format( update_object.message, update_object.chat_id)) From 83c346ccc53ad47068cd0b2cf5660102e67b8af4 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 5 Aug 2017 09:37:34 +0200 Subject: [PATCH 50/52] Let InputPeerChannel be casted automatically into InputChannel --- telethon/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/telethon/utils.py b/telethon/utils.py index 60dfd76e..c4811a6f 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -98,6 +98,9 @@ def get_input_channel(entity): if isinstance(entity, Channel) or isinstance(entity, ChannelForbidden): return InputChannel(entity.id, entity.access_hash) + if isinstance(entity, InputPeerChannel): + return InputChannel(entity.channel_id, entity.access_hash) + raise ValueError('Cannot cast {} to any kind of InputChannel.' .format(type(entity).__name__)) From 1794acdfece98c8441805175aa2ff35de4e0de2d Mon Sep 17 00:00:00 2001 From: MeytiGHG Date: Mon, 7 Aug 2017 03:24:23 +0430 Subject: [PATCH 51/52] Check for isinstance(x, JsonSession) instead crashing during transition --- telethon/telegram_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 703f3b3f..bae29e49 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -87,7 +87,7 @@ class TelegramClient(TelegramBareClient): # TODO JsonSession until migration is complete (by v1.0) if isinstance(session, str) or session is None: session = JsonSession.try_load_or_create_new(session) - elif not isinstance(session, Session): + elif not isinstance(session, Session) and not isinstance(session, JsonSession): raise ValueError( 'The given session must be a str or a Session instance.') From 7e85a3cda487df9546fcf136700d590a0a2cdd14 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 14 Aug 2017 15:15:18 +0200 Subject: [PATCH 52/52] Attempt at making get_new_msg_id thread-safe (#195) --- telethon/tl/session.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/telethon/tl/session.py b/telethon/tl/session.py index 24a9c8ec..818f4b5b 100644 --- a/telethon/tl/session.py +++ b/telethon/tl/session.py @@ -234,10 +234,12 @@ class JsonSession: # "message identifiers are divisible by 4" new_msg_id = (int(now) << 32) | (nanoseconds << 2) - if self._last_msg_id >= new_msg_id: - new_msg_id = self._last_msg_id + 4 + with self._lock: + if self._last_msg_id >= new_msg_id: + new_msg_id = self._last_msg_id + 4 + + self._last_msg_id = new_msg_id - self._last_msg_id = new_msg_id return new_msg_id def update_time_offset(self, correct_msg_id):