From 623c1bd7d178c2876d3dabada4095ff4dd0cfdbc Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 25 Feb 2018 20:34:40 +0100 Subject: [PATCH 01/38] Add missing parameters to TelegramClient.send_voice_note --- telethon/telegram_client.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 1e418904..562a6d7e 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1278,13 +1278,9 @@ class TelegramClient(TelegramBareClient): return msg - def send_voice_note(self, entity, file, caption=None, - progress_callback=None, reply_to=None): - """Wrapper method around .send_file() with is_voice_note=()""" - return self.send_file(entity, file, caption, - progress_callback=progress_callback, - reply_to=reply_to, - is_voice_note=()) # empty tuple is enough + def send_voice_note(self, *args, **kwargs): + """Wrapper method around .send_file() with is_voice_note=True""" + return self.send_file(*args, **kwargs, is_voice_note=True) def _send_album(self, entity, files, caption=None, progress_callback=None, reply_to=None): From 3b0ab7794b7399f126a0088d8d3ad9e3b5ec1118 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 25 Feb 2018 20:35:55 +0100 Subject: [PATCH 02/38] Get name attribute from streams instead always 'unnamed' --- telethon/telegram_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 562a6d7e..a2bf5e85 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1226,8 +1226,8 @@ class TelegramClient(TelegramBareClient): attr_dict[DocumentAttributeVideo] = doc else: attr_dict = { - DocumentAttributeFilename: - DocumentAttributeFilename('unnamed') + DocumentAttributeFilename: DocumentAttributeFilename( + getattr(file, 'name', None) or 'unnamed') } if 'is_voice_note' in kwargs: From 9604161c91b18169ba6de6e02d349d2091d8c29b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 26 Feb 2018 12:14:21 +0100 Subject: [PATCH 03/38] Fix incoming private messages not working with whitelists For some reason this was only happening with bots and not actual private messages. The fix doesn't seem to affect previous behaviour with actual users in private messages. --- telethon/events/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/telethon/events/__init__.py b/telethon/events/__init__.py index 204609c0..1f3b15f2 100644 --- a/telethon/events/__init__.py +++ b/telethon/events/__init__.py @@ -277,7 +277,14 @@ class NewMessage(_EventBuilder): Whether the message is a reply to some other or not. """ def __init__(self, message): - super().__init__(chat_peer=message.to_id, + if not message.out and isinstance(message.to_id, types.PeerUser): + # Incoming message (e.g. from a bot) has to_id=us, and + # from_id=bot (the actual "chat" from an user's perspective). + chat_peer = types.PeerUser(message.from_id) + else: + chat_peer = message.to_id + + super().__init__(chat_peer=chat_peer, msg_id=message.id, broadcast=bool(message.post)) self.message = message From 6f16aeb55340ab1442efe9cd2b468a6abc85a69c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 26 Feb 2018 13:41:07 +0100 Subject: [PATCH 04/38] Add logging calls on the TcpClient --- telethon/extensions/tcp_client.py | 35 +++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index dd177aa2..d9cea2f0 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -2,6 +2,7 @@ This module holds a rough implementation of the C# TCP client. """ import errno +import logging import socket import time from datetime import timedelta @@ -19,6 +20,8 @@ CONN_RESET_ERRNOS = { errno.EINVAL, errno.ENOTCONN } +__log__ = logging.getLogger(__name__) + class TcpClient: """A simple TCP client to ease the work with sockets and proxies.""" @@ -75,6 +78,7 @@ class TcpClient: self._socket.connect(address) break # Successful connection, stop retrying to connect except OSError as e: + __log__.info('OSError "%s" raised while connecting', e) # Stop retrying to connect if proxy connection error occurred if socks and isinstance(e, socks.ProxyConnectionError): raise @@ -120,19 +124,22 @@ class TcpClient: :param data: the data to send. """ if self._socket is None: - self._raise_connection_reset() + self._raise_connection_reset(None) # TODO Timeout may be an issue when sending the data, Changed in v3.5: # The socket timeout is now the maximum total duration to send all data. try: self._socket.sendall(data) except socket.timeout as e: + __log__.debug('socket.timeout "%s" while writing data', e) raise TimeoutError() from e - except ConnectionError: - self._raise_connection_reset() + except ConnectionError as e: + __log__.info('ConnectionError "%s" while writing data', e) + self._raise_connection_reset(e) except OSError as e: + __log__.info('OSError "%s" while writing data', e) if e.errno in CONN_RESET_ERRNOS: - self._raise_connection_reset() + self._raise_connection_reset(e) else: raise @@ -144,7 +151,7 @@ class TcpClient: :return: the read data with len(data) == size. """ if self._socket is None: - self._raise_connection_reset() + self._raise_connection_reset(None) # TODO Remove the timeout from this method, always use previous one with BufferedWriter(BytesIO(), buffer_size=size) as buffer: @@ -153,17 +160,22 @@ class TcpClient: try: partial = self._socket.recv(bytes_left) except socket.timeout as e: + # These are somewhat common if the server has nothing + # to send to us, so use a lower logging priority. + __log__.debug('socket.timeout "%s" while reading data', e) raise TimeoutError() from e - except ConnectionError: - self._raise_connection_reset() + except ConnectionError as e: + __log__.info('ConnectionError "%s" while reading data', e) + self._raise_connection_reset(e) except OSError as e: + __log__.info('OSError "%s" while reading data', e) if e.errno in CONN_RESET_ERRNOS: - self._raise_connection_reset() + self._raise_connection_reset(e) else: raise if len(partial) == 0: - self._raise_connection_reset() + self._raise_connection_reset(None) buffer.write(partial) bytes_left -= len(partial) @@ -172,7 +184,8 @@ class TcpClient: buffer.flush() return buffer.raw.getvalue() - def _raise_connection_reset(self): + def _raise_connection_reset(self, original): """Disconnects the client and raises ConnectionResetError.""" self.close() # Connection reset -> flag as socket closed - raise ConnectionResetError('The server has closed the connection.') + raise ConnectionResetError('The server has closed the connection.')\ + from original From 5a54e2279fecc3bbfd44694ad68a8fdb52593a6a Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 26 Feb 2018 14:12:21 +0100 Subject: [PATCH 05/38] Avoid relying on .__iter__ to tell iterators apart .send_file() would fail with stream objects (those from open()) since they are iterable, and asserting that they weren't bytes or str was not enough. --- telethon/events/__init__.py | 2 +- telethon/session.py | 6 +++--- telethon/telegram_client.py | 6 +++--- telethon/utils.py | 12 ++++++++++++ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/telethon/events/__init__.py b/telethon/events/__init__.py index 1f3b15f2..48b26004 100644 --- a/telethon/events/__init__.py +++ b/telethon/events/__init__.py @@ -14,7 +14,7 @@ def _into_id_set(client, chats): if chats is None: return None - if not hasattr(chats, '__iter__') or isinstance(chats, str): + if not utils.is_list_like(chats): chats = (chats,) result = set() diff --git a/telethon/session.py b/telethon/session.py index 4658df2b..faa1516f 100644 --- a/telethon/session.py +++ b/telethon/session.py @@ -355,14 +355,14 @@ class Session: if not self.save_entities: return - if not isinstance(tlo, TLObject) and hasattr(tlo, '__iter__'): + if not isinstance(tlo, TLObject) and utils.is_list_like(tlo): # This may be a list of users already for instance entities = tlo else: entities = [] - if hasattr(tlo, 'chats') and hasattr(tlo.chats, '__iter__'): + if hasattr(tlo, 'chats') and utils.is_list_like(tlo.chats): entities.extend(tlo.chats) - if hasattr(tlo, 'users') and hasattr(tlo.users, '__iter__'): + if hasattr(tlo, 'users') and utils.is_list_like(tlo.users): entities.extend(tlo.users) if not entities: return diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index a2bf5e85..1dad2716 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -975,7 +975,7 @@ class TelegramClient(TelegramBareClient): """ if max_id is None: if message: - if hasattr(message, '__iter__'): + if utils.is_list_like(message): max_id = max(msg.id for msg in message) else: max_id = message.id @@ -1140,7 +1140,7 @@ class TelegramClient(TelegramBareClient): """ # First check if the user passed an iterable, in which case # we may want to send as an album if all are photo files. - if hasattr(file, '__iter__') and not isinstance(file, (str, bytes)): + if utils.is_list_like(file): # Convert to tuple so we can iterate several times file = tuple(x for x in file) if all(utils.is_image(x) for x in file): @@ -1960,7 +1960,7 @@ class TelegramClient(TelegramBareClient): ``User``, ``Chat`` or ``Channel`` corresponding to the input entity. """ - if hasattr(entity, '__iter__') and not isinstance(entity, str): + if utils.is_list_like(entity): single = False else: single = True diff --git a/telethon/utils.py b/telethon/utils.py index fdadbd1c..8f38563a 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -5,6 +5,7 @@ to convert between an entity like an User, Chat, etc. into its Input version) import math import mimetypes import re +import types from mimetypes import add_type, guess_extension from .tl.types import ( @@ -341,6 +342,17 @@ def is_video(file): (mimetypes.guess_type(file)[0] or '').startswith('video/')) +def is_list_like(obj): + """ + Returns True if the given object looks like a list. + + Checking if hasattr(obj, '__iter__') and ignoring str/bytes is not + enough. Things like open() are also iterable (and probably many + other things), so just support the commonly known list-like objects. + """ + return isinstance(obj, (list, tuple, set, dict, types.GeneratorType)) + + def parse_phone(phone): """Parses the given phone, or returns None if it's invalid""" if isinstance(phone, int): From 8d1b6629cb7e68ac14737ec533aa48475fa71efc Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 26 Feb 2018 14:14:44 +0100 Subject: [PATCH 06/38] Sending open()'ed files would make their name the entire path --- telethon/telegram_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 1dad2716..158855ad 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1227,7 +1227,8 @@ class TelegramClient(TelegramBareClient): else: attr_dict = { DocumentAttributeFilename: DocumentAttributeFilename( - getattr(file, 'name', None) or 'unnamed') + os.path.basename( + getattr(file, 'name', None) or 'unnamed')) } if 'is_voice_note' in kwargs: From 29f10f277179f89082e0e9bd43bb942b7d04da9e Mon Sep 17 00:00:00 2001 From: Lucas Yuji Suguinoshita Aciole Date: Tue, 27 Feb 2018 06:05:27 -0300 Subject: [PATCH 07/38] Fix named arguments after kwargs (#646) In Python3, you're unable to send named parameters after **kwargs * Use single quotes --- telethon/telegram_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 158855ad..a8270aaf 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1281,7 +1281,8 @@ class TelegramClient(TelegramBareClient): def send_voice_note(self, *args, **kwargs): """Wrapper method around .send_file() with is_voice_note=True""" - return self.send_file(*args, **kwargs, is_voice_note=True) + kwargs['is_voice_note'] = True + return self.send_file(*args, **kwargs) def _send_album(self, entity, files, caption=None, progress_callback=None, reply_to=None): From 0b662f3b043907e872e49931201dc0fd923c63dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joscha=20G=C3=B6tzer?= Date: Tue, 27 Feb 2018 11:30:42 +0100 Subject: [PATCH 08/38] Support stopping propagation of events (#622) --- .../extra/basic/working-with-updates.rst | 27 +++++++++++++++++++ telethon/events/__init__.py | 23 ++++++++++++++++ telethon/telegram_client.py | 10 ++++++- 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/readthedocs/extra/basic/working-with-updates.rst b/readthedocs/extra/basic/working-with-updates.rst index a6c0a529..df971d7b 100644 --- a/readthedocs/extra/basic/working-with-updates.rst +++ b/readthedocs/extra/basic/working-with-updates.rst @@ -121,6 +121,33 @@ random number, while if you say ``'eval 4+4'``, you will reply with the solution. Try it! +Stopping propagation of Updates +******************************* + +There might be cases when an event handler is supposed to be used solitary and +it makes no sense to process any other handlers in the chain. For this case, +it is possible to raise a ``StopPropagation`` exception which will cause the +propagation of the update through your handlers to stop: + + .. code-block:: python + + from telethon.events import StopPropagation + + @client.on(events.NewMessage) + def _(event): + # ... some conditions + event.delete() + + # Other handlers won't have an event to work with + raise StopPropagation + + @client.on(events.NewMessage) + def _(event): + # Will never be reached, because it is the second handler + # in the chain. + pass + + Events module ************* diff --git a/telethon/events/__init__.py b/telethon/events/__init__.py index 48b26004..c5c87fbd 100644 --- a/telethon/events/__init__.py +++ b/telethon/events/__init__.py @@ -873,3 +873,26 @@ class MessageChanged(_EventBuilder): self.edited = bool(edit_msg) self.deleted = bool(deleted_ids) self.deleted_ids = deleted_ids or [] + + +class StopPropagation(Exception): + """ + If this Exception is found to be raised in any of the handlers for a + given update, it will stop the execution of all other registered + event handlers in the chain. + Think of it like a ``StopIteration`` exception in a for loop. + + Example usage: + ``` + @client.on(events.NewMessage) + def delete(event): + event.delete() + # Other handlers won't have an event to work with + raise StopPropagation + + @client.on(events.NewMessage) + def _(event): + # Will never be reached, because it is the second handler in the chain. + pass + ``` + """ diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index a8270aaf..00763000 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1885,7 +1885,15 @@ class TelegramClient(TelegramBareClient): event = builder.build(update) if event: event._client = self - callback(event) + try: + callback(event) + except events.StopPropagation: + __log__.debug( + "Event handler '{}' stopped chain of " + "propagation for event {}." + .format(callback.__name__, type(event).__name__) + ) + break def add_event_handler(self, callback, event=None): """ From d5832e4f3be8ae8afcabc14b3075fa19990cacf4 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 27 Feb 2018 11:41:45 +0100 Subject: [PATCH 09/38] Fix time offset failing if system time was ahead of time While the offset was working, the last message ID was never reset, so it would always pick an higher message ID (safety check), which completely defeated the purpose of negative time offsets. Should close #496. --- telethon/session.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/telethon/session.py b/telethon/session.py index faa1516f..6b374c39 100644 --- a/telethon/session.py +++ b/telethon/session.py @@ -325,10 +325,10 @@ class Session: """Generates a new unique message ID based on the current time (in ms) since epoch""" # Refer to mtproto_plain_sender.py for the original method - now = time.time() + now = time.time() + self.time_offset nanoseconds = int((now - int(now)) * 1e+9) # "message identifiers are divisible by 4" - new_msg_id = ((int(now) + self.time_offset) << 32) | (nanoseconds << 2) + new_msg_id = (int(now) << 32) | (nanoseconds << 2) with self._msg_id_lock: if self._last_msg_id >= new_msg_id: @@ -343,6 +343,7 @@ class Session: now = int(time.time()) correct = correct_msg_id >> 32 self.time_offset = correct - now + self._last_msg_id = 0 # Entity processing From 229cd78df08a0778bf09511fd2014f2d198dfbce Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 27 Feb 2018 14:10:02 +0100 Subject: [PATCH 10/38] Fix markdown's URL regex not acceping newlines --- telethon/extensions/markdown.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index e1fd04b7..a5dde5c6 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -21,10 +21,7 @@ DEFAULT_DELIMITERS = { '```': MessageEntityPre } -# Regex used to match r'\[(.+?)\]\((.+?)\)' (for URLs. -DEFAULT_URL_RE = re.compile(r'\[(.+?)\]\((.+?)\)') - -# Reverse operation for DEFAULT_URL_RE. {0} for text, {1} for URL. +DEFAULT_URL_RE = re.compile(r'\[([^\]]+)\]\((.+?)\)') DEFAULT_URL_FORMAT = '[{0}]({1})' From 057c6a0b12f7ae36bc073c5383e9cc143afd7679 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 28 Feb 2018 13:24:44 +0100 Subject: [PATCH 11/38] Support getting more than 10k members on .get_participants() Discussed on #580, original PR made on #639. --- telethon/telegram_client.py | 70 ++++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 00763000..ce3dbbc3 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1015,48 +1015,70 @@ class TelegramClient(TelegramBareClient): raise TypeError('Invalid message type: {}'.format(type(message))) - def get_participants(self, entity, limit=None, search=''): + def get_participants(self, entity, limit=None, search='', + aggressive=False): """ - Gets the list of participants from the specified entity + Gets the list of participants from the specified entity. Args: entity (:obj:`entity`): The entity from which to retrieve the participants list. - limit (:obj: `int`): + limit (:obj:`int`): Limits amount of participants fetched. - search (:obj: `str`, optional): + search (:obj:`str`, optional): Look for participants with this string in name/username. + aggressive (:obj:`bool`, optional): + Aggressively looks for all participants in the chat in + order to get more than 10,000 members (a hard limit + imposed by Telegram). Note that this might make over + 20 times more requests and take over 5 minutes in some + cases, but often returns well over 10,000 members. + + This has no effect for groups or channels with less than + 10,000 members. + Returns: - A list of participants with an additional .total variable on the list - indicating the total amount of members in this group/channel. + A list of participants with an additional .total variable on the + list indicating the total amount of members in this group/channel. """ entity = self.get_input_entity(entity) limit = float('inf') if limit is None else int(limit) if isinstance(entity, InputPeerChannel): - offset = 0 + total = self(GetFullChannelRequest( + entity + )).full_chat.participants_count + all_participants = {} - search = ChannelParticipantsSearch(search) - while True: - loop_limit = min(limit - offset, 200) - participants = self(GetParticipantsRequest( - entity, search, offset, loop_limit, hash=0 - )) - if not participants.users: - break - for user in participants.users: - if len(all_participants) < limit: - all_participants[user.id] = user - offset += len(participants.users) - if offset > limit: - break + if total > 10000 and aggressive: + searches = tuple(chr(x) for x in range(ord('a'), ord('z') + 1)) + else: + searches = ('',) + + for extra_search in searches: + req = GetParticipantsRequest( + channel=entity, + filter=ChannelParticipantsSearch(search + extra_search), + offset=0, + limit=0, + hash=0 + ) + while True: + req.limit = min(limit - req.offset, 200) + participants = self(req) + if not participants.users: + break + for user in participants.users: + if len(all_participants) < limit: + all_participants[user.id] = user + req.offset += len(participants.users) + if req.offset > limit: + break users = UserList(all_participants.values()) - users.total = self(GetFullChannelRequest( - entity)).full_chat.participants_count - + users.total = total elif isinstance(entity, InputPeerChat): users = self(GetFullChatRequest(entity.chat_id)).users if len(users) > limit: From 3afd7dca8435bf9402632331892695977eb89a1b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 28 Feb 2018 17:10:44 +0100 Subject: [PATCH 12/38] Invoke multiple getParticipant's at the same time (#580) --- telethon/telegram_client.py | 66 +++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index ce3dbbc3..f2c1074b 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1033,9 +1033,9 @@ class TelegramClient(TelegramBareClient): aggressive (:obj:`bool`, optional): Aggressively looks for all participants in the chat in order to get more than 10,000 members (a hard limit - imposed by Telegram). Note that this might make over - 20 times more requests and take over 5 minutes in some - cases, but often returns well over 10,000 members. + imposed by Telegram). Note that this might take a long + time (over 5 minutes), but is able to return over 90,000 + participants on groups with 100,000 members. This has no effect for groups or channels with less than 10,000 members. @@ -1053,31 +1053,49 @@ class TelegramClient(TelegramBareClient): all_participants = {} if total > 10000 and aggressive: - searches = tuple(chr(x) for x in range(ord('a'), ord('z') + 1)) - else: - searches = ('',) - - for extra_search in searches: - req = GetParticipantsRequest( + requests = [GetParticipantsRequest( channel=entity, - filter=ChannelParticipantsSearch(search + extra_search), + filter=ChannelParticipantsSearch(search + chr(x)), offset=0, - limit=0, + limit=200, hash=0 - ) - while True: - req.limit = min(limit - req.offset, 200) - participants = self(req) - if not participants.users: - break - for user in participants.users: - if len(all_participants) < limit: - all_participants[user.id] = user - req.offset += len(participants.users) - if req.offset > limit: - break + ) for x in range(ord('a'), ord('z') + 1)] + else: + requests = [GetParticipantsRequest( + channel=entity, + filter=ChannelParticipantsSearch(search), + offset=0, + limit=200, + hash=0 + )] - users = UserList(all_participants.values()) + while requests: + # Only care about the limit for the first request + # (small amount of people, won't be aggressive). + # + # Most people won't care about getting exactly 12,345 + # members so it doesn't really matter not to be 100% + # precise with being out of the offset/limit here. + requests[0].limit = min(limit - requests[0].offset, 200) + if requests[0].offset > limit: + break + + results = self(*requests) + for i in reversed(range(len(requests))): + participants = results[i] + if not participants.users: + requests.pop(i) + else: + requests[i].offset += len(participants.users) + for user in participants.users: + if len(all_participants) < limit: + all_participants[user.id] = user + if limit < float('inf'): + values = all_participants.values() + else: + values = itertools.islice(all_participants.values(), limit) + + users = UserList(values) users.total = total elif isinstance(entity, InputPeerChat): users = self(GetFullChatRequest(entity.chat_id)).users From 3655df50dcd894dbddb9f3b19227087684c7a2d3 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 28 Feb 2018 21:09:05 +0100 Subject: [PATCH 13/38] Postpone events resolution --- telethon/telegram_client.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index f2c1074b..73cdbfe9 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -174,6 +174,7 @@ class TelegramClient(TelegramBareClient): ) self._event_builders = [] + self._events_pending_resolve = [] # Some fields to easy signing in. Let {phone: hash} be # a dictionary because the user may change their mind. @@ -288,6 +289,7 @@ class TelegramClient(TelegramBareClient): self.connect() if self.is_user_authorized(): + self._check_events_pending_resolve() return self if bot_token: @@ -344,6 +346,7 @@ class TelegramClient(TelegramBareClient): # We won't reach here if any step failed (exit by exception) print('Signed in successfully as', utils.get_display_name(me)) + self._check_events_pending_resolve() return self def sign_in(self, phone=None, code=None, @@ -377,6 +380,9 @@ class TelegramClient(TelegramBareClient): The signed in user, or the information about :meth:`.send_code_request()`. """ + if self.is_user_authorized(): + self._check_events_pending_resolve() + return self.get_me() if phone and not code and not password: return self.send_code_request(phone) @@ -435,6 +441,10 @@ class TelegramClient(TelegramBareClient): Returns: The new created user. """ + if self.is_user_authorized(): + self._check_events_pending_resolve() + return self.get_me() + result = self(SignUpRequest( phone_number=self._phone, phone_code_hash=self._phone_code_hash.get(self._phone, ''), @@ -1920,6 +1930,12 @@ class TelegramClient(TelegramBareClient): return decorator + def _check_events_pending_resolve(self): + if self._events_pending_resolve: + for event in self._events_pending_resolve: + event.resolve(self) + self._events_pending_resolve.clear() + def _on_handler(self, update): for builder, callback in self._event_builders: event = builder.build(update) @@ -1963,7 +1979,12 @@ class TelegramClient(TelegramBareClient): elif not event: event = events.Raw() - event.resolve(self) + if self.is_user_authorized(): + event.resolve(self) + self._check_events_pending_resolve() + else: + self._events_pending_resolve.append(event) + self._event_builders.append((event, callback)) def add_update_handler(self, handler): @@ -1987,6 +2008,10 @@ class TelegramClient(TelegramBareClient): # region Small utilities to make users' life easier + def _set_connected_and_authorized(self): + super()._set_connected_and_authorized() + self._check_events_pending_resolve() + def get_entity(self, entity): """ Turns the given entity into a valid Telegram user or chat. From 31846415490fe90d79e84b8c9e38b7b528207746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joscha=20G=C3=B6tzer?= Date: Thu, 1 Mar 2018 00:15:30 +0100 Subject: [PATCH 14/38] Allow access to events' pattern match (#654) --- telethon/events/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/telethon/events/__init__.py b/telethon/events/__init__.py index c5c87fbd..5966a120 100644 --- a/telethon/events/__init__.py +++ b/telethon/events/__init__.py @@ -77,6 +77,8 @@ class _EventCommon(abc.ABC): self._input_chat = None self._chat = None + self.pattern_match = None + self.is_private = isinstance(chat_peer, types.PeerUser) self.is_group = ( isinstance(chat_peer, (types.PeerChat, types.PeerChannel)) @@ -251,8 +253,12 @@ class NewMessage(_EventBuilder): return if self.outgoing and not event.message.out: return - if self.pattern and not self.pattern(event.message.message or ''): - return + + if self.pattern: + match = self.pattern(event.message.message or '') + if not match: + return + event.pattern_match = match return self._filter_event(event) From 835ff51e2520ea44ec57dc57a11d46d88a1e1007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joscha=20G=C3=B6tzer?= Date: Thu, 1 Mar 2018 13:21:28 +0100 Subject: [PATCH 15/38] Multiple small changes/fixed typos to docs/error messages (#623) --- .../extra/advanced-usage/update-modes.rst | 12 +++---- readthedocs/extra/basic/entities.rst | 31 ++++++++++--------- readthedocs/extra/basic/installation.rst | 8 +++-- .../extra/basic/working-with-updates.rst | 10 ++++++ .../telegram-api-in-other-languages.rst | 17 +++++++--- .../extra/examples/chats-and-channels.rst | 8 +++++ telethon/telegram_bare_client.py | 4 ++- telethon/telegram_client.py | 4 +-- telethon/tl/custom/draft.py | 6 ++-- telethon/utils.py | 6 ++-- 10 files changed, 71 insertions(+), 35 deletions(-) diff --git a/readthedocs/extra/advanced-usage/update-modes.rst b/readthedocs/extra/advanced-usage/update-modes.rst index 83495ef7..942af9c9 100644 --- a/readthedocs/extra/advanced-usage/update-modes.rst +++ b/readthedocs/extra/advanced-usage/update-modes.rst @@ -37,7 +37,7 @@ an `Update`__ arrives: def callback(update): print('I received', update) - client.add_update_handler(callback) + client.add_event_handler(callback) # do more work here, or simply sleep! That's it! This is the old way to listen for raw updates, with no further @@ -56,7 +56,7 @@ let's reply to them with the same text reversed: client.send_message(PeerUser(update.user_id), update.message[::-1]) - client.add_update_handler(replier) + client.add_event_handler(replier) input('Press enter to stop this!') client.disconnect() @@ -96,9 +96,9 @@ additional workers: ``client = TelegramClient('session', api_id, api_hash, update_workers=0)`` -You **must** set it to ``0`` (or other number), as it defaults to ``None`` -and there is a different. ``None`` workers means updates won't be processed -*at all*, so you must set it to some value (``0`` or greater) if you want +You **must** set it to ``0`` (or higher), as it defaults to ``None`` and that +has a different meaning. ``None`` workers means updates won't be processed +*at all*, so you must set it to some integer value if you want ``client.updates.poll()`` to work. @@ -134,7 +134,7 @@ As a complete example: update_workers=1, spawn_read_thread=False) client.connect() - client.add_update_handler(callback) + client.add_event_handler(callback) client.idle() # ends with Ctrl+C diff --git a/readthedocs/extra/basic/entities.rst b/readthedocs/extra/basic/entities.rst index 472942a7..b68a74d7 100644 --- a/readthedocs/extra/basic/entities.rst +++ b/readthedocs/extra/basic/entities.rst @@ -43,14 +43,15 @@ you're able to just do this: my_channel = client.get_entity(PeerChannel(some_id)) -All methods in the :ref:`telegram-client` call ``.get_input_entity()`` to -further save you from the hassle of doing so manually, so doing things like -``client.send_message('lonami', 'hi!')`` is possible. +All methods in the :ref:`telegram-client` call ``.get_input_entity()`` prior +to sending the requst to save you from the hassle of doing so manually. +That way, convenience calls such as ``client.send_message('lonami', 'hi!')`` +become possible. -Every entity the library "sees" (in any response to any call) will by -default be cached in the ``.session`` file, to avoid performing -unnecessary API calls. If the entity cannot be found, some calls -like ``ResolveUsernameRequest`` or ``GetContactsRequest`` may be +Every entity the library encounters (in any response to any call) will by +default be cached in the ``.session`` file (an SQLite database), to avoid +performing unnecessary API calls. If the entity cannot be found, additonal +calls like ``ResolveUsernameRequest`` or ``GetContactsRequest`` may be made to obtain the required information. @@ -61,16 +62,18 @@ Entities vs. Input Entities Don't worry if you don't understand this section, just remember some of the details listed here are important. When you're calling a method, - don't call ``.get_entity()`` before, just use the username or phone, + don't call ``.get_entity()`` beforehand, just use the username or phone, or the entity retrieved by other means like ``.get_dialogs()``. -To save bandwidth, the API also makes use of their "input" versions. -The input version of an entity (e.g. ``InputPeerUser``, ``InputChat``, -etc.) only contains the minimum required information that's required -for Telegram to be able to identify who you're referring to: their ID -and hash. This ID/hash pair is unique per user, so if you use the pair -given by another user **or bot** it will **not** work. +On top of the normal types, the API also make use of what they call their +``Input*`` versions of objects. The input version of an entity (e.g. +``InputPeerUser``, ``InputChat``, etc.) only contains the minimum +information that's required from Telegram to be able to identify +who you're referring to: a ``Peer``'s **ID** and **hash**. + +This ID/hash pair is unique per user, so if you use the pair given by another +user **or bot** it will **not** work. To save *even more* bandwidth, the API also makes use of the ``Peer`` versions, which just have an ID. This serves to identify them, but diff --git a/readthedocs/extra/basic/installation.rst b/readthedocs/extra/basic/installation.rst index 0f812127..c00ea79c 100644 --- a/readthedocs/extra/basic/installation.rst +++ b/readthedocs/extra/basic/installation.rst @@ -65,9 +65,10 @@ To generate the `method documentation`__, ``cd docs`` and then Optional dependencies ********************* -If ``libssl`` is available on your system, it will be used wherever encryption -is needed, but otherwise it will fall back to pure Python implementation so it -will also work without it. +If the `cryptg`__ is installed, you might notice a speed-up in the download +and upload speed, since these are the most cryptographic-heavy part of the +library and said module is a C extension. Otherwise, the ``pyaes`` fallback +will be used. __ https://github.com/ricmoo/pyaes @@ -75,3 +76,4 @@ __ https://pypi.python.org/pypi/pyaes __ https://github.com/sybrenstuvel/python-rsa __ https://pypi.python.org/pypi/rsa/3.4.2 __ https://lonamiwebs.github.io/Telethon +__ https://github.com/Lonami/cryptg diff --git a/readthedocs/extra/basic/working-with-updates.rst b/readthedocs/extra/basic/working-with-updates.rst index df971d7b..652f6000 100644 --- a/readthedocs/extra/basic/working-with-updates.rst +++ b/readthedocs/extra/basic/working-with-updates.rst @@ -10,6 +10,16 @@ over what Telegram calls `updates`__, and are meant to ease simple and common usage when dealing with them, since there are many updates. Let's dive in! +.. note:: + + The library logs by default no output, and any exception that occurs + inside your handlers will be "hidden" from you to prevent the thread + from terminating (so it can still deliver events). You should enable + logging (``import logging; logging.basicConfig(level=logging.ERROR)``) + when working with events, at least the error level, to see if this is + happening so you can debug the error. + + .. contents:: diff --git a/readthedocs/extra/developing/telegram-api-in-other-languages.rst b/readthedocs/extra/developing/telegram-api-in-other-languages.rst index 44e45d51..7637282e 100644 --- a/readthedocs/extra/developing/telegram-api-in-other-languages.rst +++ b/readthedocs/extra/developing/telegram-api-in-other-languages.rst @@ -18,6 +18,14 @@ there by `@vysheng `__, `telegram-cli `__. Latest development has been moved to `BitBucket `__. +C++ +*** + +The newest (and official) library, written from scratch, is called +`tdlib `__ and is what the Telegram X +uses. You can find more information in the official documentation, +published `here `__. + JavaScript ********** @@ -52,13 +60,14 @@ Python A fairly new (as of the end of 2017) Telegram library written from the ground up in Python by `@delivrance `__ and his -`Pyrogram `__ library! No hard -feelings Dan and good luck dealing with some of your users ;) +`Pyrogram `__ library. +There isn't really a reason to pick it over Telethon and it'd be kinda +sad to see you go, but it would be nice to know what you miss from each +other library in either one so both can improve. Rust **** Yet another work-in-progress implementation, this time for Rust thanks to `@JuanPotato `__ under the fancy -name of `Vail `__. This one is very -early still, but progress is being made at a steady rate. +name of `Vail `__. diff --git a/readthedocs/extra/examples/chats-and-channels.rst b/readthedocs/extra/examples/chats-and-channels.rst index 44ee6112..95fa1b1e 100644 --- a/readthedocs/extra/examples/chats-and-channels.rst +++ b/readthedocs/extra/examples/chats-and-channels.rst @@ -121,6 +121,13 @@ a fixed limit: offset += len(participants.users) +.. note:: + + It is **not** possible to get more than 10,000 members from a + group. It's a hard limit impossed by Telegram and there is + nothing you can do about it. Refer to `issue 573`__ for more. + + Note that ``GetParticipantsRequest`` returns `ChannelParticipants`__, which may have more information you need (like the role of the participants, total count of members, etc.) @@ -130,6 +137,7 @@ __ https://lonamiwebs.github.io/Telethon/methods/channels/get_participants.html __ https://lonamiwebs.github.io/Telethon/types/channel_participants_filter.html __ https://lonamiwebs.github.io/Telethon/constructors/channel_participants_search.html __ https://lonamiwebs.github.io/Telethon/constructors/channels/channel_participants.html +__ https://github.com/LonamiWebs/Telethon/issues/573 Recent Actions diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 23fd4ee4..8a15476e 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -455,7 +455,9 @@ class TelegramBareClient: with self._reconnect_lock: self._reconnect() - raise RuntimeError('Number of retries reached 0.') + raise RuntimeError('Number of retries reached 0 for {}.'.format( + [type(x).__name__ for x in requests] + )) # Let people use client.invoke(SomeRequest()) instead client(...) invoke = __call__ diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 73cdbfe9..6cd1d6eb 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -586,7 +586,7 @@ class TelegramClient(TelegramBareClient): Returns: A list of custom ``Draft`` objects that are easy to work with: - You can call :meth:`draft.set_message('text')` to change the message, + You can call ``draft.set_message('text')`` to change the message, or delete it through :meth:`draft.delete()`. """ response = self(GetAllDraftsRequest()) @@ -2193,7 +2193,7 @@ class TelegramClient(TelegramBareClient): return utils.get_input_peer(entity) raise TypeError( - 'Could not find the input entity corresponding to "{}".' + 'Could not find the input entity corresponding to "{}". ' 'Make sure you have encountered this peer before.'.format(peer) ) diff --git a/telethon/tl/custom/draft.py b/telethon/tl/custom/draft.py index ae08403a..9b800d4c 100644 --- a/telethon/tl/custom/draft.py +++ b/telethon/tl/custom/draft.py @@ -42,7 +42,7 @@ class Draft: """ Changes the draft message on the Telegram servers. The changes are reflected in this object. Changing only individual attributes like for - example the `reply_to_msg_id` should be done by providing the current + example the ``reply_to_msg_id`` should be done by providing the current values of this object, like so: draft.set_message( @@ -56,7 +56,7 @@ class Draft: :param bool no_webpage: Whether to attach a web page preview :param int reply_to_msg_id: Message id to reply to :param list entities: A list of formatting entities - :return bool: `True` on success + :return bool: ``True`` on success """ result = self._client(SaveDraftRequest( peer=self._peer, @@ -77,6 +77,6 @@ class Draft: def delete(self): """ Deletes this draft - :return bool: `True` on success + :return bool: ``True`` on success """ return self.set_message(text='') diff --git a/telethon/utils.py b/telethon/utils.py index 8f38563a..ed93bdfa 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -35,8 +35,10 @@ VALID_USERNAME_RE = re.compile(r'^[a-zA-Z][\w\d]{3,30}[a-zA-Z\d]$') def get_display_name(entity): - """Gets the input peer for the given "entity" (user, chat or channel) - Returns None if it was not found""" + """ + Gets the display name for the given entity, if it's an ``User``, + ``Chat`` or ``Channel``. Returns an empty string otherwise. + """ if isinstance(entity, User): if entity.last_name and entity.first_name: return '{} {}'.format(entity.first_name, entity.last_name) From 3d68c879dd08fdff5d8df0aab90fc750c1b7668b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 1 Mar 2018 13:25:38 +0100 Subject: [PATCH 16/38] Avoid using undefined variable in the README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index febc43cd..a2e0d3de 100755 --- a/README.rst +++ b/README.rst @@ -46,12 +46,12 @@ Doing stuff .. code:: python - print(me.stringify()) + print(client.get_me().stringify()) client.send_message('username', 'Hello! Talking to you from Telethon') client.send_file('username', '/home/myself/Pictures/holidays.jpg') - client.download_profile_photo(me) + client.download_profile_photo('me') messages = client.get_message_history('username') client.download_media(messages[0]) From 771c573db1304292492e9a65288cd4802ace40dc Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 1 Mar 2018 13:31:39 +0100 Subject: [PATCH 17/38] Better attempt at joining update worker threads --- telethon/update_state.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/telethon/update_state.py b/telethon/update_state.py index 6fa0b12a..1ac2e00d 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -62,23 +62,23 @@ class UpdateState: """ self.stop_workers() self._workers = n - if n is None: - while self._updates: - self._updates.get() - else: + if n is not None: self.setup_workers() workers = property(fget=get_workers, fset=set_workers) def stop_workers(self): - """Raises "StopIterationException" on the worker threads to stop them, - and also clears all of them off the list + """ + Raises "StopIterationException" on the worker threads to stop + them, and also clears all the workers/updates from the lists. """ if self._workers: with self._updates_lock: # Insert at the beginning so the very next poll causes an error # on all the worker threads # TODO Should this reset the pts and such? + while self._updates: + self._updates.get() for _ in range(self._workers): self._updates.put(StopIteration()) From 3a3f221bd15fe312aacfd981ac07fd7714ea09e8 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 1 Mar 2018 20:03:51 +0100 Subject: [PATCH 18/38] Look in all dialogs when getting entities by peer ID --- telethon/telegram_client.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 6cd1d6eb..9690fcb4 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -572,7 +572,7 @@ class TelegramClient(TelegramBareClient): offset_date = r.messages[-1].date offset_peer = entities[utils.get_peer_id(r.dialogs[-1].peer)] - offset_id = r.messages[-1].id & 4294967296 # Telegram/danog magic + offset_id = r.messages[-1].id dialogs = UserList( itertools.islice(dialogs.values(), min(limit, len(dialogs))) @@ -2175,22 +2175,32 @@ class TelegramClient(TelegramBareClient): 'Cannot turn "{}" into an input entity.'.format(peer) ) - # Not found, look in the latest dialogs. - # This is useful if for instance someone just sent a message but - # the updates didn't specify who, as this person or chat should - # be in the latest dialogs. - dialogs = self(GetDialogsRequest( + # Not found, look in the dialogs with the hope to find it. + target_id = utils.get_peer_id(peer) + req = GetDialogsRequest( offset_date=None, offset_id=0, offset_peer=InputPeerEmpty(), - limit=0, - exclude_pinned=True - )) + limit=100 + ) + while True: + result = self(req) + entities = {} + for x in itertools.chain(result.users, result.chats): + x_id = utils.get_peer_id(x) + if x_id == target_id: + return utils.get_input_peer(x) + else: + entities[x_id] = x + if len(result.dialogs) < req.limit: + break - target = utils.get_peer_id(peer) - for entity in itertools.chain(dialogs.users, dialogs.chats): - if utils.get_peer_id(entity) == target: - return utils.get_input_peer(entity) + req.offset_id = result.messages[-1].id + req.offset_date = result.messages[-1].date + req.offset_peer = entities[utils.get_peer_id( + result.dialogs[-1].peer + )] + time.sleep(1) raise TypeError( 'Could not find the input entity corresponding to "{}". ' From f09ab6c6b6cc1ff84cc0d285a6c88c4f32705cc2 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 1 Mar 2018 20:13:21 +0100 Subject: [PATCH 19/38] Fix-up 771c573 to properly stop background update workers The "special" StopIteration object didn't actually make any sense. Instead looping forever, workers now loop while there are workers, so that they stop looping once the count is cleared. Dummy values are still inserted so that they don't need to timeout on the queue before exiting (these values are None) so in essence, this keeps the best of both of worlds. --- telethon/update_state.py | 43 +++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/telethon/update_state.py b/telethon/update_state.py index 1ac2e00d..98345cdf 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -39,19 +39,15 @@ class UpdateState: return not self._updates.empty() def poll(self, timeout=None): - """Polls an update or blocks until an update object is available. - If 'timeout is not None', it should be a floating point value, - and the method will 'return None' if waiting times out. + """ + Polls an update or blocks until an update object is available. + If 'timeout is not None', it should be a floating point value, + and the method will 'return None' if waiting times out. """ try: - update = self._updates.get(timeout=timeout) + return self._updates.get(timeout=timeout) except Empty: - return - - if isinstance(update, Exception): - raise update # Some error was set through (surely StopIteration) - - return update + return None def get_workers(self): return self._workers @@ -60,27 +56,24 @@ class UpdateState: """Changes the number of workers running. If 'n is None', clears all pending updates from memory. """ - self.stop_workers() - self._workers = n - if n is not None: + if n is None: + self.stop_workers() + else: + self._workers = n self.setup_workers() workers = property(fget=get_workers, fset=set_workers) def stop_workers(self): """ - Raises "StopIterationException" on the worker threads to stop - them, and also clears all the workers/updates from the lists. + Waits for all the worker threads to stop. """ - if self._workers: - with self._updates_lock: - # Insert at the beginning so the very next poll causes an error - # on all the worker threads - # TODO Should this reset the pts and such? - while self._updates: - self._updates.get() - for _ in range(self._workers): - self._updates.put(StopIteration()) + # Put dummy ``None`` objects so that they don't need to timeout. + n = self._workers + self._workers = None + with self._updates_lock: + for _ in range(n): + self._updates.put(None) for t in self._worker_threads: t.join() @@ -103,7 +96,7 @@ class UpdateState: thread.start() def _worker_loop(self, wid): - while True: + while self._workers is not None: try: update = self.poll(timeout=UpdateState.WORKER_POLL_TIMEOUT) if update and self.handler: From c5e6f7e265227702ad39f3c9cf632524ff1274bf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 1 Mar 2018 23:34:32 +0200 Subject: [PATCH 20/38] Split Session into three parts and make a module for sessions --- telethon/network/mtproto_sender.py | 4 +- telethon/sessions/__init__.py | 3 + telethon/sessions/abstract.py | 136 +++++++++ telethon/sessions/memory.py | 297 ++++++++++++++++++++ telethon/{session.py => sessions/sqlite.py} | 234 ++++----------- telethon/telegram_bare_client.py | 10 +- 6 files changed, 491 insertions(+), 193 deletions(-) create mode 100644 telethon/sessions/__init__.py create mode 100644 telethon/sessions/abstract.py create mode 100644 telethon/sessions/memory.py rename telethon/{session.py => sessions/sqlite.py} (59%) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 43b5e803..cbcdc76d 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -402,13 +402,13 @@ class MtProtoSender: elif bad_msg.error_code == 32: # msg_seqno too low, so just pump it up by some "large" amount # TODO A better fix would be to start with a new fresh session ID - self.session._sequence += 64 + self.session.sequence += 64 __log__.info('Attempting to set the right higher sequence') self._resend_request(bad_msg.bad_msg_id) return True elif bad_msg.error_code == 33: # msg_seqno too high never seems to happen but just in case - self.session._sequence -= 16 + self.session.sequence -= 16 __log__.info('Attempting to set the right lower sequence') self._resend_request(bad_msg.bad_msg_id) return True diff --git a/telethon/sessions/__init__.py b/telethon/sessions/__init__.py new file mode 100644 index 00000000..af3423f3 --- /dev/null +++ b/telethon/sessions/__init__.py @@ -0,0 +1,3 @@ +from .abstract import Session +from .memory import MemorySession +from .sqlite import SQLiteSession diff --git a/telethon/sessions/abstract.py b/telethon/sessions/abstract.py new file mode 100644 index 00000000..c7392ffc --- /dev/null +++ b/telethon/sessions/abstract.py @@ -0,0 +1,136 @@ +from abc import ABC, abstractmethod + + +class Session(ABC): + @abstractmethod + def clone(self): + raise NotImplementedError + + @abstractmethod + def set_dc(self, dc_id, server_address, port): + raise NotImplementedError + + @property + @abstractmethod + def server_address(self): + raise NotImplementedError + + @property + @abstractmethod + def port(self): + raise NotImplementedError + + @property + @abstractmethod + def auth_key(self): + raise NotImplementedError + + @auth_key.setter + @abstractmethod + def auth_key(self, value): + raise NotImplementedError + + @property + @abstractmethod + def time_offset(self): + raise NotImplementedError + + @time_offset.setter + @abstractmethod + def time_offset(self, value): + raise NotImplementedError + + @property + @abstractmethod + def salt(self): + raise NotImplementedError + + @salt.setter + @abstractmethod + def salt(self, value): + raise NotImplementedError + + @property + @abstractmethod + def device_model(self): + raise NotImplementedError + + @property + @abstractmethod + def system_version(self): + raise NotImplementedError + + @property + @abstractmethod + def app_version(self): + raise NotImplementedError + + @property + @abstractmethod + def lang_code(self): + raise NotImplementedError + + @property + @abstractmethod + def system_lang_code(self): + raise NotImplementedError + + @property + @abstractmethod + def report_errors(self): + raise NotImplementedError + + @property + @abstractmethod + def sequence(self): + raise NotImplementedError + + @property + @abstractmethod + def flood_sleep_threshold(self): + raise NotImplementedError + + @abstractmethod + def close(self): + raise NotImplementedError + + @abstractmethod + def save(self): + raise NotImplementedError + + @abstractmethod + def delete(self): + raise NotImplementedError + + @classmethod + @abstractmethod + def list_sessions(cls): + raise NotImplementedError + + @abstractmethod + def get_new_msg_id(self): + raise NotImplementedError + + @abstractmethod + def update_time_offset(self, correct_msg_id): + raise NotImplementedError + + @abstractmethod + def generate_sequence(self, content_related): + raise NotImplementedError + + @abstractmethod + def process_entities(self, tlo): + raise NotImplementedError + + @abstractmethod + def get_input_entity(self, key): + raise NotImplementedError + + @abstractmethod + def cache_file(self, md5_digest, file_size, instance): + raise NotImplementedError + + @abstractmethod + def get_file(self, md5_digest, file_size, cls): + raise NotImplementedError diff --git a/telethon/sessions/memory.py b/telethon/sessions/memory.py new file mode 100644 index 00000000..66558829 --- /dev/null +++ b/telethon/sessions/memory.py @@ -0,0 +1,297 @@ +from enum import Enum +import time +import platform + +from .. import utils +from .abstract import Session +from ..tl import TLObject + +from ..tl.types import ( + PeerUser, PeerChat, PeerChannel, + InputPeerUser, InputPeerChat, InputPeerChannel, + InputPhoto, InputDocument +) + + +class _SentFileType(Enum): + DOCUMENT = 0 + PHOTO = 1 + + @staticmethod + def from_type(cls): + if cls == InputDocument: + return _SentFileType.DOCUMENT + elif cls == InputPhoto: + return _SentFileType.PHOTO + else: + raise ValueError('The cls must be either InputDocument/InputPhoto') + + +class MemorySession(Session): + def __init__(self): + self._dc_id = None + self._server_address = None + self._port = None + self._salt = None + self._auth_key = None + self._sequence = 0 + self._last_msg_id = 0 + self._time_offset = 0 + self._flood_sleep_threshold = 60 + + system = platform.uname() + self._device_model = system.system or 'Unknown' + self._system_version = system.release or '1.0' + self._app_version = '1.0' + self._lang_code = 'en' + self._system_lang_code = self.lang_code + self._report_errors = True + self._flood_sleep_threshold = 60 + + self._files = {} + self._entities = set() + + def clone(self): + cloned = MemorySession() + cloned._device_model = self.device_model + cloned._system_version = self.system_version + cloned._app_version = self.app_version + cloned._lang_code = self.lang_code + cloned._system_lang_code = self.system_lang_code + cloned._report_errors = self.report_errors + cloned._flood_sleep_threshold = self.flood_sleep_threshold + + def set_dc(self, dc_id, server_address, port): + self._dc_id = dc_id + self._server_address = server_address + self._port = port + + @property + def server_address(self): + return self._server_address + + @property + def port(self): + return self._port + + @property + def auth_key(self): + return self._auth_key + + @auth_key.setter + def auth_key(self, value): + self._auth_key = value + + @property + def time_offset(self): + return self._time_offset + + @time_offset.setter + def time_offset(self, value): + self._time_offset = value + + @property + def salt(self): + return self._salt + + @salt.setter + def salt(self, value): + self._salt = value + + @property + def device_model(self): + return self._device_model + + @property + def system_version(self): + return self._system_version + + @property + def app_version(self): + return self._app_version + + @property + def lang_code(self): + return self._lang_code + + @property + def system_lang_code(self): + return self._system_lang_code + + @property + def report_errors(self): + return self._report_errors + + @property + def sequence(self): + return self._sequence + + @property + def flood_sleep_threshold(self): + return self._flood_sleep_threshold + + def close(self): + pass + + def save(self): + pass + + def delete(self): + pass + + @classmethod + def list_sessions(cls): + raise NotImplementedError + + def get_new_msg_id(self): + """Generates a new unique message ID based on the current + time (in ms) since epoch""" + now = time.time() + self._time_offset + nanoseconds = int((now - int(now)) * 1e+9) + new_msg_id = (int(now) << 32) | (nanoseconds << 2) + + if self._last_msg_id >= new_msg_id: + new_msg_id = self._last_msg_id + 4 + + self._last_msg_id = new_msg_id + + return new_msg_id + + def update_time_offset(self, correct_msg_id): + now = int(time.time()) + correct = correct_msg_id >> 32 + self._time_offset = correct - now + self._last_msg_id = 0 + + def generate_sequence(self, content_related): + if content_related: + result = self._sequence * 2 + 1 + self._sequence += 1 + return result + else: + return self._sequence * 2 + + @staticmethod + def _entities_to_rows(tlo): + if not isinstance(tlo, TLObject) and utils.is_list_like(tlo): + # This may be a list of users already for instance + entities = tlo + else: + entities = [] + if hasattr(tlo, 'chats') and utils.is_list_like(tlo.chats): + entities.extend(tlo.chats) + if hasattr(tlo, 'users') and utils.is_list_like(tlo.users): + entities.extend(tlo.users) + if not entities: + return + + rows = [] # Rows to add (id, hash, username, phone, name) + for e in entities: + if not isinstance(e, TLObject): + continue + try: + p = utils.get_input_peer(e, allow_self=False) + marked_id = utils.get_peer_id(p) + except ValueError: + continue + + if isinstance(p, (InputPeerUser, InputPeerChannel)): + if not p.access_hash: + # Some users and channels seem to be returned without + # an 'access_hash', meaning Telegram doesn't want you + # to access them. This is the reason behind ensuring + # that the 'access_hash' is non-zero. See issue #354. + # Note that this checks for zero or None, see #392. + continue + else: + p_hash = p.access_hash + elif isinstance(p, InputPeerChat): + p_hash = 0 + else: + continue + + username = getattr(e, 'username', None) or None + if username is not None: + username = username.lower() + phone = getattr(e, 'phone', None) + name = utils.get_display_name(e) or None + rows.append((marked_id, p_hash, username, phone, name)) + return rows + + def process_entities(self, tlo): + self._entities += set(self._entities_to_rows(tlo)) + + def get_entity_rows_by_phone(self, phone): + rows = [(id, hash) for id, hash, _, found_phone, _ + in self._entities if found_phone == phone] + return rows[0] if rows else None + + def get_entity_rows_by_username(self, username): + rows = [(id, hash) for id, hash, found_username, _, _ + in self._entities if found_username == username] + return rows[0] if rows else None + + def get_entity_rows_by_name(self, name): + rows = [(id, hash) for id, hash, _, _, found_name + in self._entities if found_name == name] + return rows[0] if rows else None + + def get_entity_rows_by_id(self, id): + rows = [(id, hash) for found_id, hash, _, _, _ + in self._entities if found_id == id] + return rows[0] if rows else None + + def get_input_entity(self, key): + try: + if key.SUBCLASS_OF_ID in (0xc91c90b6, 0xe669bf46, 0x40f202fd): + # hex(crc32(b'InputPeer', b'InputUser' and b'InputChannel')) + # We already have an Input version, so nothing else required + return key + # Try to early return if this key can be casted as input peer + return utils.get_input_peer(key) + except (AttributeError, TypeError): + # Not a TLObject or can't be cast into InputPeer + if isinstance(key, TLObject): + key = utils.get_peer_id(key) + + result = None + if isinstance(key, str): + phone = utils.parse_phone(key) + if phone: + result = self.get_entity_rows_by_phone(phone) + else: + username, _ = utils.parse_username(key) + if username: + result = self.get_entity_rows_by_username(username) + + if isinstance(key, int): + result = self.get_entity_rows_by_id(key) + + if not result and isinstance(key, str): + result = self.get_entity_rows_by_name(key) + + if result: + i, h = result # unpack resulting tuple + i, k = utils.resolve_id(i) # removes the mark and returns kind + if k == PeerUser: + return InputPeerUser(i, h) + elif k == PeerChat: + return InputPeerChat(i) + elif k == PeerChannel: + return InputPeerChannel(i, h) + else: + raise ValueError('Could not find input entity with key ', key) + + def cache_file(self, md5_digest, file_size, instance): + if not isinstance(instance, (InputDocument, InputPhoto)): + raise TypeError('Cannot cache %s instance' % type(instance)) + key = (md5_digest, file_size, _SentFileType.from_type(instance)) + value = (instance.id, instance.access_hash) + self._files[key] = value + + def get_file(self, md5_digest, file_size, cls): + key = (md5_digest, file_size, _SentFileType.from_type(cls)) + try: + return self._files[key] + except KeyError: + return None diff --git a/telethon/session.py b/telethon/sessions/sqlite.py similarity index 59% rename from telethon/session.py rename to telethon/sessions/sqlite.py index 6b374c39..66a0c887 100644 --- a/telethon/session.py +++ b/telethon/sessions/sqlite.py @@ -5,14 +5,15 @@ import sqlite3 import struct import time from base64 import b64decode -from enum import Enum from os.path import isfile as file_exists from threading import Lock, RLock -from . import utils -from .crypto import AuthKey -from .tl import TLObject -from .tl.types import ( +from .. import utils +from .abstract import Session +from .memory import MemorySession, _SentFileType +from ..crypto import AuthKey +from ..tl import TLObject +from ..tl.types import ( PeerUser, PeerChat, PeerChannel, InputPeerUser, InputPeerChat, InputPeerChannel, InputPhoto, InputDocument @@ -22,21 +23,7 @@ EXTENSION = '.session' CURRENT_VERSION = 3 # database version -class _SentFileType(Enum): - DOCUMENT = 0 - PHOTO = 1 - - @staticmethod - def from_type(cls): - if cls == InputDocument: - return _SentFileType.DOCUMENT - elif cls == InputPhoto: - return _SentFileType.PHOTO - else: - raise ValueError('The cls must be either InputDocument/InputPhoto') - - -class Session: +class SQLiteSession(MemorySession): """This session contains the required information to login into your Telegram account. NEVER give the saved JSON file to anyone, since they would gain instant access to all your messages and contacts. @@ -44,7 +31,9 @@ class Session: If you think the session has been compromised, close all the sessions through an official Telegram client to revoke the authorization. """ + def __init__(self, session_id): + super().__init__() """session_user_id should either be a string or another Session. Note that if another session is given, only parameters like those required to init a connection will be copied. @@ -54,15 +43,15 @@ class Session: # For connection purposes if isinstance(session_id, Session): - self.device_model = session_id.device_model - self.system_version = session_id.system_version - self.app_version = session_id.app_version - self.lang_code = session_id.lang_code - self.system_lang_code = session_id.system_lang_code - self.lang_pack = session_id.lang_pack - self.report_errors = session_id.report_errors - self.save_entities = session_id.save_entities - self.flood_sleep_threshold = session_id.flood_sleep_threshold + self._device_model = session_id.device_model + self._system_version = session_id.system_version + self._app_version = session_id.app_version + self._lang_code = session_id.lang_code + self._system_lang_code = session_id.system_lang_code + self._report_errors = session_id.report_errors + self._flood_sleep_threshold = session_id.flood_sleep_threshold + if isinstance(session_id, SQLiteSession): + self.save_entities = session_id.save_entities else: # str / None if session_id: self.filename = session_id @@ -70,15 +59,14 @@ class Session: self.filename += EXTENSION system = platform.uname() - self.device_model = system.system or 'Unknown' - self.system_version = system.release or '1.0' - self.app_version = '1.0' # '0' will provoke error - self.lang_code = 'en' - self.system_lang_code = self.lang_code - self.lang_pack = '' - self.report_errors = True + self._device_model = system.system or 'Unknown' + self._system_version = system.release or '1.0' + self._app_version = '1.0' # '0' will provoke error + self._lang_code = 'en' + self._system_lang_code = self.lang_code + self._report_errors = True self.save_entities = True - self.flood_sleep_threshold = 60 + self._flood_sleep_threshold = 60 self.id = struct.unpack('q', os.urandom(8))[0] self._sequence = 0 @@ -163,6 +151,9 @@ class Session: c.close() self.save() + def clone(self): + return SQLiteSession(self) + def _check_migrate_json(self): if file_exists(self.filename): try: @@ -233,19 +224,7 @@ class Session: self._auth_key = None c.close() - @property - def server_address(self): - return self._server_address - - @property - def port(self): - return self._port - - @property - def auth_key(self): - return self._auth_key - - @auth_key.setter + @Session.auth_key.setter def auth_key(self, value): self._auth_key = value self._update_session_table() @@ -298,53 +277,14 @@ class Session: except OSError: return False - @staticmethod - def list_sessions(): + @classmethod + def list_sessions(cls): """Lists all the sessions of the users who have ever connected using this client and never logged out """ return [os.path.splitext(os.path.basename(f))[0] for f in os.listdir('.') if f.endswith(EXTENSION)] - def generate_sequence(self, content_related): - """Thread safe method to generates the next sequence number, - based on whether it was confirmed yet or not. - - Note that if confirmed=True, the sequence number - will be increased by one too - """ - with self._seq_no_lock: - if content_related: - result = self._sequence * 2 + 1 - self._sequence += 1 - return result - else: - return self._sequence * 2 - - def get_new_msg_id(self): - """Generates a new unique message ID based on the current - time (in ms) since epoch""" - # Refer to mtproto_plain_sender.py for the original method - now = time.time() + self.time_offset - nanoseconds = int((now - int(now)) * 1e+9) - # "message identifiers are divisible by 4" - new_msg_id = (int(now) << 32) | (nanoseconds << 2) - - with self._msg_id_lock: - if self._last_msg_id >= new_msg_id: - new_msg_id = self._last_msg_id + 4 - - self._last_msg_id = new_msg_id - - return new_msg_id - - def update_time_offset(self, correct_msg_id): - """Updates the time offset based on a known correct message ID""" - now = int(time.time()) - correct = correct_msg_id >> 32 - self.time_offset = correct - now - self._last_msg_id = 0 - # Entity processing def process_entities(self, tlo): @@ -356,49 +296,7 @@ class Session: if not self.save_entities: return - if not isinstance(tlo, TLObject) and utils.is_list_like(tlo): - # This may be a list of users already for instance - entities = tlo - else: - entities = [] - if hasattr(tlo, 'chats') and utils.is_list_like(tlo.chats): - entities.extend(tlo.chats) - if hasattr(tlo, 'users') and utils.is_list_like(tlo.users): - entities.extend(tlo.users) - if not entities: - return - - rows = [] # Rows to add (id, hash, username, phone, name) - for e in entities: - if not isinstance(e, TLObject): - continue - try: - p = utils.get_input_peer(e, allow_self=False) - marked_id = utils.get_peer_id(p) - except ValueError: - continue - - if isinstance(p, (InputPeerUser, InputPeerChannel)): - if not p.access_hash: - # Some users and channels seem to be returned without - # an 'access_hash', meaning Telegram doesn't want you - # to access them. This is the reason behind ensuring - # that the 'access_hash' is non-zero. See issue #354. - # Note that this checks for zero or None, see #392. - continue - else: - p_hash = p.access_hash - elif isinstance(p, InputPeerChat): - p_hash = 0 - else: - continue - - username = getattr(e, 'username', None) or None - if username is not None: - username = username.lower() - phone = getattr(e, 'phone', None) - name = utils.get_display_name(e) or None - rows.append((marked_id, p_hash, username, phone, name)) + rows = self._entities_to_rows(tlo) if not rows: return @@ -408,62 +306,26 @@ class Session: ) self.save() - def get_input_entity(self, key): - """Parses the given string, integer or TLObject key into a - marked entity ID, which is then used to fetch the hash - from the database. - - If a callable key is given, every row will be fetched, - and passed as a tuple to a function, that should return - a true-like value when the desired row is found. - - Raises ValueError if it cannot be found. - """ - try: - if key.SUBCLASS_OF_ID in (0xc91c90b6, 0xe669bf46, 0x40f202fd): - # hex(crc32(b'InputPeer', b'InputUser' and b'InputChannel')) - # We already have an Input version, so nothing else required - return key - # Try to early return if this key can be casted as input peer - return utils.get_input_peer(key) - except (AttributeError, TypeError): - # Not a TLObject or can't be cast into InputPeer - if isinstance(key, TLObject): - key = utils.get_peer_id(key) - + def _fetchone_entity(self, query, args): c = self._cursor() - if isinstance(key, str): - phone = utils.parse_phone(key) - if phone: - c.execute('select id, hash from entities where phone=?', - (phone,)) - else: - username, _ = utils.parse_username(key) - if username: - c.execute('select id, hash from entities where username=?', + c.execute(query, args) + return c.fetchone() + + def get_entity_rows_by_phone(self, phone): + return self._fetchone_entity( + 'select id, hash from entities where phone=?', (phone,)) + + def get_entity_rows_by_username(self, username): + self._fetchone_entity('select id, hash from entities where username=?', (username,)) - if isinstance(key, int): - c.execute('select id, hash from entities where id=?', (key,)) + def get_entity_rows_by_name(self, name): + self._fetchone_entity('select id, hash from entities where name=?', + (name,)) - result = c.fetchone() - if not result and isinstance(key, str): - # Try exact match by name if phone/username failed - c.execute('select id, hash from entities where name=?', (key,)) - result = c.fetchone() - - c.close() - if result: - i, h = result # unpack resulting tuple - i, k = utils.resolve_id(i) # removes the mark and returns kind - if k == PeerUser: - return InputPeerUser(i, h) - elif k == PeerChat: - return InputPeerChat(i) - elif k == PeerChannel: - return InputPeerChannel(i, h) - else: - raise ValueError('Could not find input entity with key ', key) + def get_entity_rows_by_id(self, id): + self._fetchone_entity('select id, hash from entities where id=?', + (id,)) # File processing diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 8a15476e..3a5b2bd0 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -14,7 +14,7 @@ from .errors import ( PhoneMigrateError, NetworkMigrateError, UserMigrateError ) from .network import authenticator, MtProtoSender, Connection, ConnectionMode -from .session import Session +from .sessions import Session, SQLiteSession from .tl import TLObject from .tl.all_tlobjects import LAYER from .tl.functions import ( @@ -81,10 +81,10 @@ class TelegramBareClient: "Refer to telethon.rtfd.io for more information.") self._use_ipv6 = use_ipv6 - + # Determine what session object we have if isinstance(session, str) or session is None: - session = Session(session) + session = SQLiteSession(session) elif not isinstance(session, Session): raise TypeError( 'The given session must be a str or a Session instance.' @@ -361,7 +361,7 @@ class TelegramBareClient: # # Construct this session with the connection parameters # (system version, device model...) from the current one. - session = Session(self.session) + session = self.session.clone() session.set_dc(dc.id, dc.ip_address, dc.port) self._exported_sessions[dc_id] = session @@ -387,7 +387,7 @@ class TelegramBareClient: session = self._exported_sessions.get(cdn_redirect.dc_id) if not session: dc = self._get_dc(cdn_redirect.dc_id, cdn=True) - session = Session(self.session) + session = self.session.clone() session.set_dc(dc.id, dc.ip_address, dc.port) self._exported_sessions[cdn_redirect.dc_id] = session From 4c64d53e7178b3555b9c9346b93c7b1f4ab3366f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 11:10:11 +0200 Subject: [PATCH 21/38] Move non-persistent stuff to base Session class --- telethon/sessions/abstract.py | 144 ++++++++++++++++++++-------------- telethon/sessions/memory.py | 62 +-------------- 2 files changed, 87 insertions(+), 119 deletions(-) diff --git a/telethon/sessions/abstract.py b/telethon/sessions/abstract.py index c7392ffc..ff0fd16d 100644 --- a/telethon/sessions/abstract.py +++ b/telethon/sessions/abstract.py @@ -1,10 +1,32 @@ from abc import ABC, abstractmethod +import time +import platform class Session(ABC): - @abstractmethod + def __init__(self): + self._sequence = 0 + self._last_msg_id = 0 + self._time_offset = 0 + + system = platform.uname() + self._device_model = system.system or 'Unknown' + self._system_version = system.release or '1.0' + self._app_version = '1.0' + self._lang_code = 'en' + self._system_lang_code = self.lang_code + self._report_errors = True + self._flood_sleep_threshold = 60 + def clone(self): - raise NotImplementedError + cloned = self.__class__() + cloned._device_model = self.device_model + cloned._system_version = self.system_version + cloned._app_version = self.app_version + cloned._lang_code = self.lang_code + cloned._system_lang_code = self.system_lang_code + cloned._report_errors = self.report_errors + cloned._flood_sleep_threshold = self.flood_sleep_threshold @abstractmethod def set_dc(self, dc_id, server_address, port): @@ -31,14 +53,12 @@ class Session(ABC): raise NotImplementedError @property - @abstractmethod def time_offset(self): - raise NotImplementedError + return self._time_offset @time_offset.setter - @abstractmethod def time_offset(self, value): - raise NotImplementedError + self._time_offset = value @property @abstractmethod @@ -50,46 +70,6 @@ class Session(ABC): def salt(self, value): raise NotImplementedError - @property - @abstractmethod - def device_model(self): - raise NotImplementedError - - @property - @abstractmethod - def system_version(self): - raise NotImplementedError - - @property - @abstractmethod - def app_version(self): - raise NotImplementedError - - @property - @abstractmethod - def lang_code(self): - raise NotImplementedError - - @property - @abstractmethod - def system_lang_code(self): - raise NotImplementedError - - @property - @abstractmethod - def report_errors(self): - raise NotImplementedError - - @property - @abstractmethod - def sequence(self): - raise NotImplementedError - - @property - @abstractmethod - def flood_sleep_threshold(self): - raise NotImplementedError - @abstractmethod def close(self): raise NotImplementedError @@ -107,18 +87,6 @@ class Session(ABC): def list_sessions(cls): raise NotImplementedError - @abstractmethod - def get_new_msg_id(self): - raise NotImplementedError - - @abstractmethod - def update_time_offset(self, correct_msg_id): - raise NotImplementedError - - @abstractmethod - def generate_sequence(self, content_related): - raise NotImplementedError - @abstractmethod def process_entities(self, tlo): raise NotImplementedError @@ -134,3 +102,63 @@ class Session(ABC): @abstractmethod def get_file(self, md5_digest, file_size, cls): raise NotImplementedError + + @property + def device_model(self): + return self._device_model + + @property + def system_version(self): + return self._system_version + + @property + def app_version(self): + return self._app_version + + @property + def lang_code(self): + return self._lang_code + + @property + def system_lang_code(self): + return self._system_lang_code + + @property + def report_errors(self): + return self._report_errors + + @property + def flood_sleep_threshold(self): + return self._flood_sleep_threshold + + @property + def sequence(self): + return self._sequence + + def get_new_msg_id(self): + """Generates a new unique message ID based on the current + time (in ms) since epoch""" + now = time.time() + self._time_offset + nanoseconds = int((now - int(now)) * 1e+9) + new_msg_id = (int(now) << 32) | (nanoseconds << 2) + + if self._last_msg_id >= new_msg_id: + new_msg_id = self._last_msg_id + 4 + + self._last_msg_id = new_msg_id + + return new_msg_id + + def update_time_offset(self, correct_msg_id): + now = int(time.time()) + correct = correct_msg_id >> 32 + self._time_offset = correct - now + self._last_msg_id = 0 + + def generate_sequence(self, content_related): + if content_related: + result = self._sequence * 2 + 1 + self._sequence += 1 + return result + else: + return self._sequence * 2 diff --git a/telethon/sessions/memory.py b/telethon/sessions/memory.py index 66558829..09aa1fa0 100644 --- a/telethon/sessions/memory.py +++ b/telethon/sessions/memory.py @@ -1,6 +1,4 @@ from enum import Enum -import time -import platform from .. import utils from .abstract import Session @@ -29,38 +27,16 @@ class _SentFileType(Enum): class MemorySession(Session): def __init__(self): + super().__init__() self._dc_id = None self._server_address = None self._port = None self._salt = None self._auth_key = None - self._sequence = 0 - self._last_msg_id = 0 - self._time_offset = 0 - self._flood_sleep_threshold = 60 - - system = platform.uname() - self._device_model = system.system or 'Unknown' - self._system_version = system.release or '1.0' - self._app_version = '1.0' - self._lang_code = 'en' - self._system_lang_code = self.lang_code - self._report_errors = True - self._flood_sleep_threshold = 60 self._files = {} self._entities = set() - def clone(self): - cloned = MemorySession() - cloned._device_model = self.device_model - cloned._system_version = self.system_version - cloned._app_version = self.app_version - cloned._lang_code = self.lang_code - cloned._system_lang_code = self.system_lang_code - cloned._report_errors = self.report_errors - cloned._flood_sleep_threshold = self.flood_sleep_threshold - def set_dc(self, dc_id, server_address, port): self._dc_id = dc_id self._server_address = server_address @@ -82,14 +58,6 @@ class MemorySession(Session): def auth_key(self, value): self._auth_key = value - @property - def time_offset(self): - return self._time_offset - - @time_offset.setter - def time_offset(self, value): - self._time_offset = value - @property def salt(self): return self._salt @@ -143,34 +111,6 @@ class MemorySession(Session): def list_sessions(cls): raise NotImplementedError - def get_new_msg_id(self): - """Generates a new unique message ID based on the current - time (in ms) since epoch""" - now = time.time() + self._time_offset - nanoseconds = int((now - int(now)) * 1e+9) - new_msg_id = (int(now) << 32) | (nanoseconds << 2) - - if self._last_msg_id >= new_msg_id: - new_msg_id = self._last_msg_id + 4 - - self._last_msg_id = new_msg_id - - return new_msg_id - - def update_time_offset(self, correct_msg_id): - now = int(time.time()) - correct = correct_msg_id >> 32 - self._time_offset = correct - now - self._last_msg_id = 0 - - def generate_sequence(self, content_related): - if content_related: - result = self._sequence * 2 + 1 - self._sequence += 1 - return result - else: - return self._sequence * 2 - @staticmethod def _entities_to_rows(tlo): if not isinstance(tlo, TLObject) and utils.is_list_like(tlo): From 4c7224e56aad2b9fc2f1b84ab4404a12b121339f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 2 Mar 2018 10:10:59 +0100 Subject: [PATCH 22/38] Fix n might be None when stopping workers --- telethon/update_state.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/telethon/update_state.py b/telethon/update_state.py index 98345cdf..6a496603 100644 --- a/telethon/update_state.py +++ b/telethon/update_state.py @@ -71,9 +71,10 @@ class UpdateState: # Put dummy ``None`` objects so that they don't need to timeout. n = self._workers self._workers = None - with self._updates_lock: - for _ in range(n): - self._updates.put(None) + if n: + with self._updates_lock: + for _ in range(n): + self._updates.put(None) for t in self._worker_threads: t.join() From df3faaeb7f4f2694d9e68074fbb91314dad21af7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 11:11:59 +0200 Subject: [PATCH 23/38] Fix abstract Session method ordering --- telethon/sessions/abstract.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/telethon/sessions/abstract.py b/telethon/sessions/abstract.py index ff0fd16d..ec4f649f 100644 --- a/telethon/sessions/abstract.py +++ b/telethon/sessions/abstract.py @@ -52,14 +52,6 @@ class Session(ABC): def auth_key(self, value): raise NotImplementedError - @property - def time_offset(self): - return self._time_offset - - @time_offset.setter - def time_offset(self, value): - self._time_offset = value - @property @abstractmethod def salt(self): @@ -127,6 +119,14 @@ class Session(ABC): def report_errors(self): return self._report_errors + @property + def time_offset(self): + return self._time_offset + + @time_offset.setter + def time_offset(self, value): + self._time_offset = value + @property def flood_sleep_threshold(self): return self._flood_sleep_threshold From d9a73744a49031d823fdd1da671476ddfc56c9d8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 12:36:39 +0200 Subject: [PATCH 24/38] Remove old sqlite session variables and clone code --- telethon/sessions/sqlite.py | 47 +++++++------------------------------ 1 file changed, 9 insertions(+), 38 deletions(-) diff --git a/telethon/sessions/sqlite.py b/telethon/sessions/sqlite.py index 66a0c887..64f3cbf6 100644 --- a/telethon/sessions/sqlite.py +++ b/telethon/sessions/sqlite.py @@ -32,7 +32,7 @@ class SQLiteSession(MemorySession): through an official Telegram client to revoke the authorization. """ - def __init__(self, session_id): + def __init__(self, session_id=None): super().__init__() """session_user_id should either be a string or another Session. Note that if another session is given, only parameters like @@ -40,51 +40,20 @@ class SQLiteSession(MemorySession): """ # These values will NOT be saved self.filename = ':memory:' + self.save_entities = True - # For connection purposes - if isinstance(session_id, Session): - self._device_model = session_id.device_model - self._system_version = session_id.system_version - self._app_version = session_id.app_version - self._lang_code = session_id.lang_code - self._system_lang_code = session_id.system_lang_code - self._report_errors = session_id.report_errors - self._flood_sleep_threshold = session_id.flood_sleep_threshold - if isinstance(session_id, SQLiteSession): - self.save_entities = session_id.save_entities - else: # str / None - if session_id: - self.filename = session_id - if not self.filename.endswith(EXTENSION): - self.filename += EXTENSION - - system = platform.uname() - self._device_model = system.system or 'Unknown' - self._system_version = system.release or '1.0' - self._app_version = '1.0' # '0' will provoke error - self._lang_code = 'en' - self._system_lang_code = self.lang_code - self._report_errors = True - self.save_entities = True - self._flood_sleep_threshold = 60 + if session_id: + self.filename = session_id + if not self.filename.endswith(EXTENSION): + self.filename += EXTENSION self.id = struct.unpack('q', os.urandom(8))[0] - self._sequence = 0 - self.time_offset = 0 - self._last_msg_id = 0 # Long - self.salt = 0 # Long # Cross-thread safety self._seq_no_lock = Lock() self._msg_id_lock = Lock() self._db_lock = RLock() - # These values will be saved - self._dc_id = 0 - self._server_address = None - self._port = None - self._auth_key = None - # Migrating from .json -> SQL entities = self._check_migrate_json() @@ -152,7 +121,9 @@ class SQLiteSession(MemorySession): self.save() def clone(self): - return SQLiteSession(self) + cloned = super().clone() + cloned.save_entities = self.save_entities + return cloned def _check_migrate_json(self): if file_exists(self.filename): From 118d9b10e869f04c818541b1cea41ccd801fd800 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 13:20:11 +0200 Subject: [PATCH 25/38] Add more abstraction --- telethon/sessions/abstract.py | 5 ++- telethon/sessions/memory.py | 69 +++++++++++++++++++---------------- telethon/sessions/sqlite.py | 23 ++++++------ 3 files changed, 53 insertions(+), 44 deletions(-) diff --git a/telethon/sessions/abstract.py b/telethon/sessions/abstract.py index ec4f649f..89f80b7a 100644 --- a/telethon/sessions/abstract.py +++ b/telethon/sessions/abstract.py @@ -18,8 +18,8 @@ class Session(ABC): self._report_errors = True self._flood_sleep_threshold = 60 - def clone(self): - cloned = self.__class__() + def clone(self, to_instance=None): + cloned = to_instance or self.__class__() cloned._device_model = self.device_model cloned._system_version = self.system_version cloned._app_version = self.app_version @@ -27,6 +27,7 @@ class Session(ABC): cloned._system_lang_code = self.system_lang_code cloned._report_errors = self.report_errors cloned._flood_sleep_threshold = self.flood_sleep_threshold + return cloned @abstractmethod def set_dc(self, dc_id, server_address, port): diff --git a/telethon/sessions/memory.py b/telethon/sessions/memory.py index 09aa1fa0..92674fa6 100644 --- a/telethon/sessions/memory.py +++ b/telethon/sessions/memory.py @@ -111,8 +111,41 @@ class MemorySession(Session): def list_sessions(cls): raise NotImplementedError - @staticmethod - def _entities_to_rows(tlo): + def _entity_values_to_row(self, id, hash, username, phone, name): + return id, hash, username, phone, name + + def _entity_to_row(self, e): + if not isinstance(e, TLObject): + return + try: + p = utils.get_input_peer(e, allow_self=False) + marked_id = utils.get_peer_id(p) + except ValueError: + return + + if isinstance(p, (InputPeerUser, InputPeerChannel)): + if not p.access_hash: + # Some users and channels seem to be returned without + # an 'access_hash', meaning Telegram doesn't want you + # to access them. This is the reason behind ensuring + # that the 'access_hash' is non-zero. See issue #354. + # Note that this checks for zero or None, see #392. + return + else: + p_hash = p.access_hash + elif isinstance(p, InputPeerChat): + p_hash = 0 + else: + return + + username = getattr(e, 'username', None) or None + if username is not None: + username = username.lower() + phone = getattr(e, 'phone', None) + name = utils.get_display_name(e) or None + return self._entity_values_to_row(marked_id, p_hash, username, phone, name) + + def _entities_to_rows(self, tlo): if not isinstance(tlo, TLObject) and utils.is_list_like(tlo): # This may be a list of users already for instance entities = tlo @@ -127,35 +160,9 @@ class MemorySession(Session): rows = [] # Rows to add (id, hash, username, phone, name) for e in entities: - if not isinstance(e, TLObject): - continue - try: - p = utils.get_input_peer(e, allow_self=False) - marked_id = utils.get_peer_id(p) - except ValueError: - continue - - if isinstance(p, (InputPeerUser, InputPeerChannel)): - if not p.access_hash: - # Some users and channels seem to be returned without - # an 'access_hash', meaning Telegram doesn't want you - # to access them. This is the reason behind ensuring - # that the 'access_hash' is non-zero. See issue #354. - # Note that this checks for zero or None, see #392. - continue - else: - p_hash = p.access_hash - elif isinstance(p, InputPeerChat): - p_hash = 0 - else: - continue - - username = getattr(e, 'username', None) or None - if username is not None: - username = username.lower() - phone = getattr(e, 'phone', None) - name = utils.get_display_name(e) or None - rows.append((marked_id, p_hash, username, phone, name)) + row = self._entity_to_row(e) + if row: + rows.append(row) return rows def process_entities(self, tlo): diff --git a/telethon/sessions/sqlite.py b/telethon/sessions/sqlite.py index 64f3cbf6..0ea26ae5 100644 --- a/telethon/sessions/sqlite.py +++ b/telethon/sessions/sqlite.py @@ -120,8 +120,8 @@ class SQLiteSession(MemorySession): c.close() self.save() - def clone(self): - cloned = super().clone() + def clone(self, to_instance=None): + cloned = super().clone(to_instance) cloned.save_entities = self.save_entities return cloned @@ -180,9 +180,7 @@ class SQLiteSession(MemorySession): # Data from sessions should be kept as properties # not to fetch the database every time we need it def set_dc(self, dc_id, server_address, port): - self._dc_id = dc_id - self._server_address = server_address - self._port = port + super().set_dc(dc_id, server_address, port) self._update_session_table() # Fetch the auth_key corresponding to this data center @@ -287,16 +285,19 @@ class SQLiteSession(MemorySession): 'select id, hash from entities where phone=?', (phone,)) def get_entity_rows_by_username(self, username): - self._fetchone_entity('select id, hash from entities where username=?', - (username,)) + return self._fetchone_entity( + 'select id, hash from entities where username=?', + (username,)) def get_entity_rows_by_name(self, name): - self._fetchone_entity('select id, hash from entities where name=?', - (name,)) + return self._fetchone_entity( + 'select id, hash from entities where name=?', + (name,)) def get_entity_rows_by_id(self, id): - self._fetchone_entity('select id, hash from entities where id=?', - (id,)) + return self._fetchone_entity( + 'select id, hash from entities where id=?', + (id,)) # File processing From 07c2fc50ec3f841b9366777bbd0bb63164875c98 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 13:22:30 +0200 Subject: [PATCH 26/38] Add SQLAlchemy-based session --- optional-requirements.txt | 1 + setup.py | 3 +- telethon/sessions/sqlalchemy.py | 177 ++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 telethon/sessions/sqlalchemy.py diff --git a/optional-requirements.txt b/optional-requirements.txt index 55bfc014..fb83c1ab 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,3 +1,4 @@ cryptg pysocks hachoir3 +sqlalchemy diff --git a/setup.py b/setup.py index 00dd7446..0e052d31 100755 --- a/setup.py +++ b/setup.py @@ -151,7 +151,8 @@ def main(): ]), install_requires=['pyaes', 'rsa'], extras_require={ - 'cryptg': ['cryptg'] + 'cryptg': ['cryptg'], + 'sqlalchemy': ['sqlalchemy'] } ) diff --git a/telethon/sessions/sqlalchemy.py b/telethon/sessions/sqlalchemy.py new file mode 100644 index 00000000..94f24a1e --- /dev/null +++ b/telethon/sessions/sqlalchemy.py @@ -0,0 +1,177 @@ +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, String, Integer, Blob, orm +import sqlalchemy as sql + +from ..tl.types import InputPhoto, InputDocument + +from .memory import MemorySession, _SentFileType + +Base = declarative_base() +LATEST_VERSION = 1 + + +class DBVersion(Base): + __tablename__ = "version" + version = Column(Integer, primary_key=True) + + +class DBSession(Base): + __tablename__ = "sessions" + + session_id = Column(String, primary_key=True) + dc_id = Column(Integer, primary_key=True) + server_address = Column(String) + port = Column(Integer) + auth_key = Column(Blob) + + +class DBEntity(Base): + __tablename__ = "entities" + + session_id = Column(String, primary_key=True) + id = Column(Integer, primary_key=True) + hash = Column(Integer, nullable=False) + username = Column(String) + phone = Column(Integer) + name = Column(String) + + +class DBSentFile(Base): + __tablename__ = "sent_files" + + session_id = Column(String, primary_key=True) + md5_digest = Column(Blob, primary_key=True) + file_size = Column(Integer, primary_key=True) + type = Column(Integer, primary_key=True) + id = Column(Integer) + hash = Column(Integer) + + +class AlchemySessionContainer: + def __init__(self, database): + if not isinstance(database, sql.Engine): + database = sql.create_engine(database) + + self.db_engine = database + db_factory = orm.sessionmaker(bind=self.db_engine) + self.db = orm.scoping.scoped_session(db_factory) + + if not self.db_engine.dialect.has_table(self.db_engine, + DBVersion.__tablename__): + Base.metadata.create_all(bind=self.db_engine) + self.db.add(DBVersion(version=LATEST_VERSION)) + self.db.commit() + else: + self.check_and_upgrade_database() + + DBVersion.query = self.db.query_property() + DBSession.query = self.db.query_property() + DBEntity.query = self.db.query_property() + DBSentFile.query = self.db.query_property() + + def check_and_upgrade_database(self): + row = DBVersion.query.get() + version = row.version if row else 1 + if version == LATEST_VERSION: + return + + DBVersion.query.delete() + + # Implement table schema updates here and increase version + + self.db.add(DBVersion(version=version)) + self.db.commit() + + def new_session(self, session_id): + return AlchemySession(self, session_id) + + def list_sessions(self): + return + + def save(self): + self.db.commit() + + +class AlchemySession(MemorySession): + def __init__(self, container, session_id): + super().__init__() + self.container = container + self.db = container.db + self.session_id = session_id + + def clone(self, to_instance=None): + cloned = to_instance or self.__class__(self.container, self.session_id) + return super().clone(cloned) + + def set_dc(self, dc_id, server_address, port): + super().set_dc(dc_id, server_address, port) + + def _update_session_table(self): + self.db.query(DBSession).filter( + DBSession.session_id == self.session_id).delete() + new = DBSession(session_id=self.session_id, dc_id=self._dc_id, + server_address=self._server_address, port=self._port, + auth_key=self._auth_key.key if self._auth_key else b'') + self.db.merge(new) + + def _db_query(self, dbclass, *args): + return self.db.query(dbclass).filter( + dbclass.session_id == self.session_id, + *args) + + def save(self): + self.container.save() + + def close(self): + # Nothing to do here, connection is managed by AlchemySessionContainer. + pass + + def delete(self): + self._db_query(DBSession).delete() + self._db_query(DBEntity).delete() + self._db_query(DBSentFile).delete() + + def _entity_values_to_row(self, id, hash, username, phone, name): + return DBEntity(session_id=self.session_id, id=id, hash=hash, + username=username, phone=phone, name=name) + + def process_entities(self, tlo): + rows = self._entities_to_rows(tlo) + if not rows: + return + + self.db.add_all(rows) + self.save() + + def get_entity_rows_by_phone(self, key): + row = self._db_query(DBEntity, DBEntity.phone == key).one_or_none() + return row.id, row.hash if row else None + + def get_entity_rows_by_username(self, key): + row = self._db_query(DBEntity, DBEntity.username == key).one_or_none() + return row.id, row.hash if row else None + + def get_entity_rows_by_name(self, key): + row = self._db_query(DBEntity, DBEntity.name == key).one_or_none() + return row.id, row.hash if row else None + + def get_entity_rows_by_id(self, key): + row = self._db_query(DBEntity, DBEntity.id == key).one_or_none() + return row.id, row.hash if row else None + + def get_file(self, md5_digest, file_size, cls): + row = self._db_query(DBSentFile, DBSentFile.md5_digest == md5_digest, + DBSentFile.file_size == file_size, + DBSentFile.type == _SentFileType.from_type( + cls).value).one_or_none() + return row.id, row.hash if row else None + + def cache_file(self, md5_digest, file_size, instance): + if not isinstance(instance, (InputDocument, InputPhoto)): + raise TypeError('Cannot cache %s instance' % type(instance)) + + self.db.merge( + DBSentFile(session_id=self.session_id, md5_digest=md5_digest, + type=_SentFileType.from_type(type(instance)).value, + id=instance.id, hash=instance.access_hash)) + self.save() From 03d4ab37657123e4ba3258ccf1867e645630b111 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 13:25:40 +0200 Subject: [PATCH 27/38] Fix create_engine check --- telethon/sessions/sqlalchemy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/sessions/sqlalchemy.py b/telethon/sessions/sqlalchemy.py index 94f24a1e..782f811f 100644 --- a/telethon/sessions/sqlalchemy.py +++ b/telethon/sessions/sqlalchemy.py @@ -49,7 +49,7 @@ class DBSentFile(Base): class AlchemySessionContainer: def __init__(self, database): - if not isinstance(database, sql.Engine): + if isinstance(database, str): database = sql.create_engine(database) self.db_engine = database From e1d7cc541f878a22a46dbd3f8cc2d0dd5115f16d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 18:23:12 +0200 Subject: [PATCH 28/38] Add setters for non-persistent values that apps might change --- telethon/sessions/abstract.py | 32 ++++++++++++++++++++++++++++++++ telethon/sessions/memory.py | 32 -------------------------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/telethon/sessions/abstract.py b/telethon/sessions/abstract.py index 89f80b7a..dd1541ab 100644 --- a/telethon/sessions/abstract.py +++ b/telethon/sessions/abstract.py @@ -100,26 +100,50 @@ class Session(ABC): def device_model(self): return self._device_model + @device_model.setter + def device_model(self, value): + self._device_model = value + @property def system_version(self): return self._system_version + @system_version.setter + def system_version(self, value): + self._system_version = value + @property def app_version(self): return self._app_version + @app_version.setter + def app_version(self, value): + self._app_version = value + @property def lang_code(self): return self._lang_code + @lang_code.setter + def lang_code(self, value): + self._lang_code = value + @property def system_lang_code(self): return self._system_lang_code + @system_lang_code.setter + def system_lang_code(self, value): + self._system_lang_code = value + @property def report_errors(self): return self._report_errors + @report_errors.setter + def report_errors(self, value): + self._report_errors = value + @property def time_offset(self): return self._time_offset @@ -132,10 +156,18 @@ class Session(ABC): def flood_sleep_threshold(self): return self._flood_sleep_threshold + @flood_sleep_threshold.setter + def flood_sleep_threshold(self, value): + self._flood_sleep_threshold = value + @property def sequence(self): return self._sequence + @sequence.setter + def sequence(self, value): + self._sequence = value + def get_new_msg_id(self): """Generates a new unique message ID based on the current time (in ms) since epoch""" diff --git a/telethon/sessions/memory.py b/telethon/sessions/memory.py index 92674fa6..71d6e551 100644 --- a/telethon/sessions/memory.py +++ b/telethon/sessions/memory.py @@ -66,38 +66,6 @@ class MemorySession(Session): def salt(self, value): self._salt = value - @property - def device_model(self): - return self._device_model - - @property - def system_version(self): - return self._system_version - - @property - def app_version(self): - return self._app_version - - @property - def lang_code(self): - return self._lang_code - - @property - def system_lang_code(self): - return self._system_lang_code - - @property - def report_errors(self): - return self._report_errors - - @property - def sequence(self): - return self._sequence - - @property - def flood_sleep_threshold(self): - return self._flood_sleep_threshold - def close(self): pass From a7f98fd3cc2abdd6935eef69c5a8251b0756ca4a Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 2 Mar 2018 17:26:42 +0100 Subject: [PATCH 29/38] Ignore bad file descriptor while closing --- telethon/extensions/tcp_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index d9cea2f0..d335e57a 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -168,7 +168,10 @@ class TcpClient: __log__.info('ConnectionError "%s" while reading data', e) self._raise_connection_reset(e) except OSError as e: - __log__.info('OSError "%s" while reading data', e) + if e.errno != errno.EBADF and self._closing_lock.locked(): + # Ignore bad file descriptor while closing + __log__.info('OSError "%s" while reading data', e) + if e.errno in CONN_RESET_ERRNOS: self._raise_connection_reset(e) else: From dc2229fdba10f0140a7e159737091d82852cdef3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 18:39:04 +0200 Subject: [PATCH 30/38] Move salt and ID to base session and remove unused imports --- telethon/sessions/abstract.py | 23 +++++++++++++---------- telethon/sessions/memory.py | 10 +--------- telethon/sessions/sqlite.py | 12 +----------- 3 files changed, 15 insertions(+), 30 deletions(-) diff --git a/telethon/sessions/abstract.py b/telethon/sessions/abstract.py index dd1541ab..d92e0754 100644 --- a/telethon/sessions/abstract.py +++ b/telethon/sessions/abstract.py @@ -1,13 +1,18 @@ from abc import ABC, abstractmethod import time import platform +import struct +import os class Session(ABC): def __init__(self): + self.id = struct.unpack('q', os.urandom(8))[0] + self._sequence = 0 self._last_msg_id = 0 self._time_offset = 0 + self._salt = 0 system = platform.uname() self._device_model = system.system or 'Unknown' @@ -53,16 +58,6 @@ class Session(ABC): def auth_key(self, value): raise NotImplementedError - @property - @abstractmethod - def salt(self): - raise NotImplementedError - - @salt.setter - @abstractmethod - def salt(self, value): - raise NotImplementedError - @abstractmethod def close(self): raise NotImplementedError @@ -96,6 +91,14 @@ class Session(ABC): def get_file(self, md5_digest, file_size, cls): raise NotImplementedError + @property + def salt(self): + return self._salt + + @salt.setter + def salt(self, value): + self._salt = value + @property def device_model(self): return self._device_model diff --git a/telethon/sessions/memory.py b/telethon/sessions/memory.py index 71d6e551..7ab31b21 100644 --- a/telethon/sessions/memory.py +++ b/telethon/sessions/memory.py @@ -28,10 +28,10 @@ class _SentFileType(Enum): class MemorySession(Session): def __init__(self): super().__init__() + self._dc_id = None self._server_address = None self._port = None - self._salt = None self._auth_key = None self._files = {} @@ -58,14 +58,6 @@ class MemorySession(Session): def auth_key(self, value): self._auth_key = value - @property - def salt(self): - return self._salt - - @salt.setter - def salt(self, value): - self._salt = value - def close(self): pass diff --git a/telethon/sessions/sqlite.py b/telethon/sessions/sqlite.py index 0ea26ae5..0423d88a 100644 --- a/telethon/sessions/sqlite.py +++ b/telethon/sessions/sqlite.py @@ -1,21 +1,13 @@ import json import os -import platform import sqlite3 -import struct -import time from base64 import b64decode from os.path import isfile as file_exists from threading import Lock, RLock -from .. import utils -from .abstract import Session from .memory import MemorySession, _SentFileType from ..crypto import AuthKey -from ..tl import TLObject from ..tl.types import ( - PeerUser, PeerChat, PeerChannel, - InputPeerUser, InputPeerChat, InputPeerChannel, InputPhoto, InputDocument ) @@ -47,8 +39,6 @@ class SQLiteSession(MemorySession): if not self.filename.endswith(EXTENSION): self.filename += EXTENSION - self.id = struct.unpack('q', os.urandom(8))[0] - # Cross-thread safety self._seq_no_lock = Lock() self._msg_id_lock = Lock() @@ -193,7 +183,7 @@ class SQLiteSession(MemorySession): self._auth_key = None c.close() - @Session.auth_key.setter + @MemorySession.auth_key.setter def auth_key(self, value): self._auth_key = value self._update_session_table() From c1a8896faa43dfa798fc9223283eae42dbf5afd6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 20:14:11 +0200 Subject: [PATCH 31/38] Fix SQLAlchemy implementation --- telethon/sessions/__init__.py | 1 + telethon/sessions/sqlalchemy.py | 197 ++++++++++++++++++-------------- 2 files changed, 114 insertions(+), 84 deletions(-) diff --git a/telethon/sessions/__init__.py b/telethon/sessions/__init__.py index af3423f3..a487a4bd 100644 --- a/telethon/sessions/__init__.py +++ b/telethon/sessions/__init__.py @@ -1,3 +1,4 @@ from .abstract import Session from .memory import MemorySession from .sqlite import SQLiteSession +from .sqlalchemy import AlchemySessionContainer, AlchemySession diff --git a/telethon/sessions/sqlalchemy.py b/telethon/sessions/sqlalchemy.py index 782f811f..aa618e4c 100644 --- a/telethon/sessions/sqlalchemy.py +++ b/telethon/sessions/sqlalchemy.py @@ -1,85 +1,95 @@ from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import Column, String, Integer, Blob, orm +from sqlalchemy import Column, String, Integer, BLOB, orm import sqlalchemy as sql +from ..crypto import AuthKey from ..tl.types import InputPhoto, InputDocument from .memory import MemorySession, _SentFileType -Base = declarative_base() LATEST_VERSION = 1 -class DBVersion(Base): - __tablename__ = "version" - version = Column(Integer, primary_key=True) - - -class DBSession(Base): - __tablename__ = "sessions" - - session_id = Column(String, primary_key=True) - dc_id = Column(Integer, primary_key=True) - server_address = Column(String) - port = Column(Integer) - auth_key = Column(Blob) - - -class DBEntity(Base): - __tablename__ = "entities" - - session_id = Column(String, primary_key=True) - id = Column(Integer, primary_key=True) - hash = Column(Integer, nullable=False) - username = Column(String) - phone = Column(Integer) - name = Column(String) - - -class DBSentFile(Base): - __tablename__ = "sent_files" - - session_id = Column(String, primary_key=True) - md5_digest = Column(Blob, primary_key=True) - file_size = Column(Integer, primary_key=True) - type = Column(Integer, primary_key=True) - id = Column(Integer) - hash = Column(Integer) - - class AlchemySessionContainer: - def __init__(self, database): - if isinstance(database, str): - database = sql.create_engine(database) + def __init__(self, engine=None, session=None, table_prefix="", + table_base=None, manage_tables=True): + if isinstance(engine, str): + engine = sql.create_engine(engine) - self.db_engine = database - db_factory = orm.sessionmaker(bind=self.db_engine) - self.db = orm.scoping.scoped_session(db_factory) - - if not self.db_engine.dialect.has_table(self.db_engine, - DBVersion.__tablename__): - Base.metadata.create_all(bind=self.db_engine) - self.db.add(DBVersion(version=LATEST_VERSION)) - self.db.commit() + self.db_engine = engine + if not session: + db_factory = orm.sessionmaker(bind=self.db_engine) + self.db = orm.scoping.scoped_session(db_factory) else: - self.check_and_upgrade_database() + self.db = session - DBVersion.query = self.db.query_property() - DBSession.query = self.db.query_property() - DBEntity.query = self.db.query_property() - DBSentFile.query = self.db.query_property() + table_base = table_base or declarative_base() + (self.Version, self.Session, self.Entity, + self.SentFile) = self.create_table_classes(self.db, table_prefix, + table_base) + + if manage_tables: + table_base.metadata.bind = self.db_engine + if not self.db_engine.dialect.has_table(self.db_engine, + self.Version.__tablename__): + table_base.metadata.create_all() + self.db.add(self.Version(version=LATEST_VERSION)) + self.db.commit() + else: + self.check_and_upgrade_database() + + @staticmethod + def create_table_classes(db, prefix, Base): + class Version(Base): + query = db.query_property() + __tablename__ = "{prefix}version".format(prefix=prefix) + version = Column(Integer, primary_key=True) + + class Session(Base): + query = db.query_property() + __tablename__ = "{prefix}sessions".format(prefix=prefix) + + session_id = Column(String, primary_key=True) + dc_id = Column(Integer, primary_key=True) + server_address = Column(String) + port = Column(Integer) + auth_key = Column(BLOB) + + class Entity(Base): + query = db.query_property() + __tablename__ = "{prefix}entities".format(prefix=prefix) + + session_id = Column(String, primary_key=True) + id = Column(Integer, primary_key=True) + hash = Column(Integer, nullable=False) + username = Column(String) + phone = Column(Integer) + name = Column(String) + + class SentFile(Base): + query = db.query_property() + __tablename__ = "{prefix}sent_files".format(prefix=prefix) + + session_id = Column(String, primary_key=True) + md5_digest = Column(BLOB, primary_key=True) + file_size = Column(Integer, primary_key=True) + type = Column(Integer, primary_key=True) + id = Column(Integer) + hash = Column(Integer) + + return Version, Session, Entity, SentFile def check_and_upgrade_database(self): - row = DBVersion.query.get() - version = row.version if row else 1 + row = self.Version.query.all() + version = row[0].version if row else 1 if version == LATEST_VERSION: return - DBVersion.query.delete() + self.Version.query.delete() # Implement table schema updates here and increase version - self.db.add(DBVersion(version=version)) + self.db.add(self.Version(version=version)) self.db.commit() def new_session(self, session_id): @@ -97,7 +107,20 @@ class AlchemySession(MemorySession): super().__init__() self.container = container self.db = container.db + self.Version, self.Session, self.Entity, self.SentFile = ( + container.Version, container.Session, container.Entity, + container.SentFile) self.session_id = session_id + self.load_session() + + def load_session(self): + sessions = self._db_query(self.Session).all() + session = sessions[0] if sessions else None + if session: + self._dc_id = session.dc_id + self._server_address = session.server_address + self._port = session.port + self._auth_key = AuthKey(data=session.auth_key) def clone(self, to_instance=None): cloned = to_instance or self.__class__(self.container, self.session_id) @@ -107,17 +130,18 @@ class AlchemySession(MemorySession): super().set_dc(dc_id, server_address, port) def _update_session_table(self): - self.db.query(DBSession).filter( - DBSession.session_id == self.session_id).delete() - new = DBSession(session_id=self.session_id, dc_id=self._dc_id, - server_address=self._server_address, port=self._port, - auth_key=self._auth_key.key if self._auth_key else b'') + self.Session.query.filter( + self.Session.session_id == self.session_id).delete() + new = self.Session(session_id=self.session_id, dc_id=self._dc_id, + server_address=self._server_address, + port=self._port, + auth_key=(self._auth_key.key + if self._auth_key else b'')) self.db.merge(new) def _db_query(self, dbclass, *args): - return self.db.query(dbclass).filter( - dbclass.session_id == self.session_id, - *args) + return dbclass.query.filter(dbclass.session_id == self.session_id, + *args) def save(self): self.container.save() @@ -127,42 +151,47 @@ class AlchemySession(MemorySession): pass def delete(self): - self._db_query(DBSession).delete() - self._db_query(DBEntity).delete() - self._db_query(DBSentFile).delete() + self._db_query(self.Session).delete() + self._db_query(self.Entity).delete() + self._db_query(self.SentFile).delete() def _entity_values_to_row(self, id, hash, username, phone, name): - return DBEntity(session_id=self.session_id, id=id, hash=hash, - username=username, phone=phone, name=name) + return self.Entity(session_id=self.session_id, id=id, hash=hash, + username=username, phone=phone, name=name) def process_entities(self, tlo): rows = self._entities_to_rows(tlo) if not rows: return - self.db.add_all(rows) + for row in rows: + self.db.merge(row) self.save() def get_entity_rows_by_phone(self, key): - row = self._db_query(DBEntity, DBEntity.phone == key).one_or_none() + row = self._db_query(self.Entity, + self.Entity.phone == key).one_or_none() return row.id, row.hash if row else None def get_entity_rows_by_username(self, key): - row = self._db_query(DBEntity, DBEntity.username == key).one_or_none() + row = self._db_query(self.Entity, + self.Entity.username == key).one_or_none() return row.id, row.hash if row else None def get_entity_rows_by_name(self, key): - row = self._db_query(DBEntity, DBEntity.name == key).one_or_none() + row = self._db_query(self.Entity, + self.Entity.name == key).one_or_none() return row.id, row.hash if row else None def get_entity_rows_by_id(self, key): - row = self._db_query(DBEntity, DBEntity.id == key).one_or_none() + row = self._db_query(self.Entity, self.Entity.id == key).one_or_none() return row.id, row.hash if row else None def get_file(self, md5_digest, file_size, cls): - row = self._db_query(DBSentFile, DBSentFile.md5_digest == md5_digest, - DBSentFile.file_size == file_size, - DBSentFile.type == _SentFileType.from_type( + row = self._db_query(self.SentFile, + self.SentFile.md5_digest == md5_digest, + self.SentFile.file_size == file_size, + self.SentFile.type == _SentFileType.from_type( cls).value).one_or_none() return row.id, row.hash if row else None @@ -171,7 +200,7 @@ class AlchemySession(MemorySession): raise TypeError('Cannot cache %s instance' % type(instance)) self.db.merge( - DBSentFile(session_id=self.session_id, md5_digest=md5_digest, - type=_SentFileType.from_type(type(instance)).value, - id=instance.id, hash=instance.access_hash)) + self.SentFile(session_id=self.session_id, md5_digest=md5_digest, + type=_SentFileType.from_type(type(instance)).value, + id=instance.id, hash=instance.access_hash)) self.save() From f805914c80e034058789ec6d7fef43dd4762e051 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 20:40:03 +0200 Subject: [PATCH 32/38] Handle SQLAlchemy import errors --- telethon/sessions/sqlalchemy.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/telethon/sessions/sqlalchemy.py b/telethon/sessions/sqlalchemy.py index aa618e4c..0b028c02 100644 --- a/telethon/sessions/sqlalchemy.py +++ b/telethon/sessions/sqlalchemy.py @@ -1,6 +1,10 @@ -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import Column, String, Integer, BLOB, orm -import sqlalchemy as sql +try: + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy import Column, String, Integer, BLOB, orm + import sqlalchemy as sql +except ImportError: + sql = None + pass from ..crypto import AuthKey from ..tl.types import InputPhoto, InputDocument @@ -13,6 +17,8 @@ LATEST_VERSION = 1 class AlchemySessionContainer: def __init__(self, engine=None, session=None, table_prefix="", table_base=None, manage_tables=True): + if not sql: + raise ImportError("SQLAlchemy not imported") if isinstance(engine, str): engine = sql.create_engine(engine) From 5e88b21aa9ce5bd51ecd7fcd3627797c58d996ee Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 20:42:51 +0200 Subject: [PATCH 33/38] Use single quotes --- telethon/sessions/sqlalchemy.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/telethon/sessions/sqlalchemy.py b/telethon/sessions/sqlalchemy.py index 0b028c02..0fd76fe3 100644 --- a/telethon/sessions/sqlalchemy.py +++ b/telethon/sessions/sqlalchemy.py @@ -15,10 +15,10 @@ LATEST_VERSION = 1 class AlchemySessionContainer: - def __init__(self, engine=None, session=None, table_prefix="", + def __init__(self, engine=None, session=None, table_prefix='', table_base=None, manage_tables=True): if not sql: - raise ImportError("SQLAlchemy not imported") + raise ImportError('SQLAlchemy not imported') if isinstance(engine, str): engine = sql.create_engine(engine) @@ -48,12 +48,12 @@ class AlchemySessionContainer: def create_table_classes(db, prefix, Base): class Version(Base): query = db.query_property() - __tablename__ = "{prefix}version".format(prefix=prefix) + __tablename__ = '{prefix}version'.format(prefix=prefix) version = Column(Integer, primary_key=True) class Session(Base): query = db.query_property() - __tablename__ = "{prefix}sessions".format(prefix=prefix) + __tablename__ = '{prefix}sessions'.format(prefix=prefix) session_id = Column(String, primary_key=True) dc_id = Column(Integer, primary_key=True) @@ -63,7 +63,7 @@ class AlchemySessionContainer: class Entity(Base): query = db.query_property() - __tablename__ = "{prefix}entities".format(prefix=prefix) + __tablename__ = '{prefix}entities'.format(prefix=prefix) session_id = Column(String, primary_key=True) id = Column(Integer, primary_key=True) @@ -74,7 +74,7 @@ class AlchemySessionContainer: class SentFile(Base): query = db.query_property() - __tablename__ = "{prefix}sent_files".format(prefix=prefix) + __tablename__ = '{prefix}sent_files'.format(prefix=prefix) session_id = Column(String, primary_key=True) md5_digest = Column(BLOB, primary_key=True) From 47cdcda9e2b8c17d3f4e842509a67fcf76b2548c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 21:05:09 +0200 Subject: [PATCH 34/38] Move device info out of Session --- telethon/sessions/abstract.py | 53 -------------------------------- telethon/telegram_bare_client.py | 30 ++++++++++-------- 2 files changed, 18 insertions(+), 65 deletions(-) diff --git a/telethon/sessions/abstract.py b/telethon/sessions/abstract.py index d92e0754..647a87c1 100644 --- a/telethon/sessions/abstract.py +++ b/telethon/sessions/abstract.py @@ -1,6 +1,5 @@ from abc import ABC, abstractmethod import time -import platform import struct import os @@ -13,23 +12,11 @@ class Session(ABC): self._last_msg_id = 0 self._time_offset = 0 self._salt = 0 - - system = platform.uname() - self._device_model = system.system or 'Unknown' - self._system_version = system.release or '1.0' - self._app_version = '1.0' - self._lang_code = 'en' - self._system_lang_code = self.lang_code self._report_errors = True self._flood_sleep_threshold = 60 def clone(self, to_instance=None): cloned = to_instance or self.__class__() - cloned._device_model = self.device_model - cloned._system_version = self.system_version - cloned._app_version = self.app_version - cloned._lang_code = self.lang_code - cloned._system_lang_code = self.system_lang_code cloned._report_errors = self.report_errors cloned._flood_sleep_threshold = self.flood_sleep_threshold return cloned @@ -99,46 +86,6 @@ class Session(ABC): def salt(self, value): self._salt = value - @property - def device_model(self): - return self._device_model - - @device_model.setter - def device_model(self, value): - self._device_model = value - - @property - def system_version(self): - return self._system_version - - @system_version.setter - def system_version(self, value): - self._system_version = value - - @property - def app_version(self): - return self._app_version - - @app_version.setter - def app_version(self, value): - self._app_version = value - - @property - def lang_code(self): - return self._lang_code - - @lang_code.setter - def lang_code(self, value): - self._lang_code = value - - @property - def system_lang_code(self): - return self._system_lang_code - - @system_lang_code.setter - def system_lang_code(self, value): - self._system_lang_code = value - @property def report_errors(self): return self._report_errors diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 3a5b2bd0..bf33a7dc 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -1,11 +1,11 @@ import logging import os +import platform import threading from datetime import timedelta, datetime from signal import signal, SIGINT, SIGTERM, SIGABRT from threading import Lock from time import sleep - from . import version, utils from .crypto import rsa from .errors import ( @@ -73,7 +73,12 @@ class TelegramBareClient: update_workers=None, spawn_read_thread=False, timeout=timedelta(seconds=5), - **kwargs): + loop=None, + device_model=None, + system_version=None, + app_version=None, + lang_code='en', + system_lang_code='en'): """Refer to TelegramClient.__init__ for docs on this method""" if not api_id or not api_hash: raise ValueError( @@ -125,11 +130,12 @@ class TelegramBareClient: self.updates = UpdateState(workers=update_workers) # Used on connection - the user may modify these and reconnect - kwargs['app_version'] = kwargs.get('app_version', self.__version__) - for name, value in kwargs.items(): - if not hasattr(self.session, name): - raise ValueError('Unknown named parameter', name) - setattr(self.session, name, value) + system = platform.uname() + self.device_model = device_model or system.system or 'Unknown' + self.system_version = system_version or system.release or '1.0' + self.app_version = app_version or self.__version__ + self.lang_code = lang_code + self.system_lang_code = system_lang_code # Despite the state of the real connection, keep track of whether # the user has explicitly called .connect() or .disconnect() here. @@ -233,11 +239,11 @@ class TelegramBareClient: """Wraps query around InvokeWithLayerRequest(InitConnectionRequest())""" return InvokeWithLayerRequest(LAYER, InitConnectionRequest( api_id=self.api_id, - device_model=self.session.device_model, - system_version=self.session.system_version, - app_version=self.session.app_version, - lang_code=self.session.lang_code, - system_lang_code=self.session.system_lang_code, + device_model=self.device_model, + system_version=self.system_version, + app_version=self.app_version, + lang_code=self.lang_code, + system_lang_code=self.system_lang_code, lang_pack='', # "langPacks are for official apps only" query=query )) From 290afd85fc5f91688e4e18355ac207e8a2c2f1ec Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 2 Mar 2018 21:58:16 +0200 Subject: [PATCH 35/38] Fix AlchemySession session table updating --- telethon/sessions/sqlalchemy.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/telethon/sessions/sqlalchemy.py b/telethon/sessions/sqlalchemy.py index 0fd76fe3..933f44c2 100644 --- a/telethon/sessions/sqlalchemy.py +++ b/telethon/sessions/sqlalchemy.py @@ -117,9 +117,9 @@ class AlchemySession(MemorySession): container.Version, container.Session, container.Entity, container.SentFile) self.session_id = session_id - self.load_session() + self._load_session() - def load_session(self): + def _load_session(self): sessions = self._db_query(self.Session).all() session = sessions[0] if sessions else None if session: @@ -134,6 +134,19 @@ class AlchemySession(MemorySession): def set_dc(self, dc_id, server_address, port): super().set_dc(dc_id, server_address, port) + self._update_session_table() + + sessions = self._db_query(self.Session).all() + session = sessions[0] if sessions else None + if session and session.auth_key: + self._auth_key = AuthKey(data=session.auth_key) + else: + self._auth_key = None + + @MemorySession.auth_key.setter + def auth_key(self, value): + self._auth_key = value + self._update_session_table() def _update_session_table(self): self.Session.query.filter( From 2e31a686e86699387e2875804f732b5cb513f368 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 2 Mar 2018 21:28:33 +0100 Subject: [PATCH 36/38] Upgrade to layer 75 Captions are now "messages" and also support message entities. --- .../extra/examples/working-with-messages.rst | 3 +- telethon/telegram_client.py | 36 +++++++++++++------ telethon/utils.py | 24 ++++--------- .../interactive_telegram_client.py | 4 +-- telethon_generator/scheme.tl | 36 +++++++++---------- 5 files changed, 52 insertions(+), 51 deletions(-) diff --git a/readthedocs/extra/examples/working-with-messages.rst b/readthedocs/extra/examples/working-with-messages.rst index ab38788c..e2471a25 100644 --- a/readthedocs/extra/examples/working-with-messages.rst +++ b/readthedocs/extra/examples/working-with-messages.rst @@ -114,8 +114,7 @@ send yourself the very first sticker you have: id=InputDocument( id=stickers.documents[0].id, access_hash=stickers.documents[0].access_hash - ), - caption='' + ) ) )) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 9690fcb4..24615ace 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1128,6 +1128,7 @@ class TelegramClient(TelegramBareClient): attributes=None, thumb=None, allow_cache=True, + parse_mode='md', **kwargs): """ Sends a file to the specified entity. @@ -1177,6 +1178,9 @@ class TelegramClient(TelegramBareClient): Must be ``False`` if you wish to use different attributes or thumb than those that were used when the file was cached. + parse_mode (:obj:`str`, optional): + The parse mode for the caption message. + Kwargs: If "is_voice_note" in kwargs, despite its value, and the file is sent as a document, it will be sent as a voice note. @@ -1210,18 +1214,21 @@ class TelegramClient(TelegramBareClient): entity = self.get_input_entity(entity) reply_to = self._get_message_id(reply_to) + caption, msg_entities = self._parse_message_text(caption, parse_mode) if not isinstance(file, (str, bytes, io.IOBase)): # The user may pass a Message containing media (or the media, # or anything similar) that should be treated as a file. Try # getting the input media for whatever they passed and send it. try: - media = utils.get_input_media(file, user_caption=caption) + media = utils.get_input_media(file) except TypeError: pass # Can't turn whatever was given into media else: request = SendMediaRequest(entity, media, - reply_to_msg_id=reply_to) + reply_to_msg_id=reply_to, + message=caption, + entities=msg_entities) return self._get_response_message(request, self(request)) as_image = utils.is_image(file) and not force_document @@ -1234,11 +1241,11 @@ class TelegramClient(TelegramBareClient): if isinstance(file_handle, use_cache): # File was cached, so an instance of use_cache was returned if as_image: - media = InputMediaPhoto(file_handle, caption or '') + media = InputMediaPhoto(file_handle) else: - media = InputMediaDocument(file_handle, caption or '') + media = InputMediaDocument(file_handle) elif as_image: - media = InputMediaUploadedPhoto(file_handle, caption or '') + media = InputMediaUploadedPhoto(file_handle) else: mime_type = None if isinstance(file, str): @@ -1309,13 +1316,13 @@ class TelegramClient(TelegramBareClient): file=file_handle, mime_type=mime_type, attributes=list(attr_dict.values()), - caption=caption or '', **input_kw ) # Once the media type is properly specified and the file uploaded, # send the media message to the desired entity. - request = SendMediaRequest(entity, media, reply_to_msg_id=reply_to) + request = SendMediaRequest(entity, media, reply_to_msg_id=reply_to, + message=caption, entities=msg_entities) msg = self._get_response_message(request, self(request)) if msg and isinstance(file_handle, InputSizedFile): # There was a response message and we didn't use cached @@ -1335,15 +1342,18 @@ class TelegramClient(TelegramBareClient): return self.send_file(*args, **kwargs) def _send_album(self, entity, files, caption=None, - progress_callback=None, reply_to=None): + progress_callback=None, reply_to=None, + parse_mode='md'): """Specialized version of .send_file for albums""" # We don't care if the user wants to avoid cache, we will use it # anyway. Why? The cached version will be exactly the same thing # we need to produce right now to send albums (uploadMedia), and # cache only makes a difference for documents where the user may - # want the attributes used on them to change. Caption's ignored. + # want the attributes used on them to change. + # TODO Support a different captions for each file entity = self.get_input_entity(entity) caption = caption or '' + caption, msg_entities = self._parse_message_text(caption, parse_mode) reply_to = self._get_message_id(reply_to) # Need to upload the media first, but only if they're not cached yet @@ -1353,11 +1363,15 @@ class TelegramClient(TelegramBareClient): fh = self.upload_file(file, use_cache=InputPhoto) if not isinstance(fh, InputPhoto): input_photo = utils.get_input_photo(self(UploadMediaRequest( - entity, media=InputMediaUploadedPhoto(fh, caption) + entity, media=InputMediaUploadedPhoto(fh) )).photo) self.session.cache_file(fh.md5, fh.size, input_photo) fh = input_photo - media.append(InputSingleMedia(InputMediaPhoto(fh, caption))) + media.append(InputSingleMedia( + InputMediaPhoto(fh), + message=caption, + entities=msg_entities + )) # Now we can construct the multi-media request result = self(SendMultiMediaRequest( diff --git a/telethon/utils.py b/telethon/utils.py index ed93bdfa..faf69649 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -241,7 +241,7 @@ def get_input_geo(geo): _raise_cast_fail(geo, 'InputGeoPoint') -def get_input_media(media, user_caption=None, is_photo=False): +def get_input_media(media, is_photo=False): """Similar to get_input_peer, but for media. If the media is a file location and is_photo is known to be True, @@ -256,31 +256,23 @@ def get_input_media(media, user_caption=None, is_photo=False): if isinstance(media, MessageMediaPhoto): return InputMediaPhoto( id=get_input_photo(media.photo), - ttl_seconds=media.ttl_seconds, - caption=((media.caption if user_caption is None else user_caption) - or '') + ttl_seconds=media.ttl_seconds ) if isinstance(media, MessageMediaDocument): return InputMediaDocument( id=get_input_document(media.document), - ttl_seconds=media.ttl_seconds, - caption=((media.caption if user_caption is None else user_caption) - or '') + ttl_seconds=media.ttl_seconds ) if isinstance(media, FileLocation): if is_photo: - return InputMediaUploadedPhoto( - file=media, - caption=user_caption or '' - ) + return InputMediaUploadedPhoto(file=media) else: return InputMediaUploadedDocument( file=media, mime_type='application/octet-stream', # unknown, assume bytes - attributes=[DocumentAttributeFilename('unnamed')], - caption=user_caption or '' + attributes=[DocumentAttributeFilename('unnamed')] ) if isinstance(media, MessageMediaGame): @@ -291,7 +283,7 @@ def get_input_media(media, user_caption=None, is_photo=False): media = media.photo_small else: media = media.photo_big - return get_input_media(media, user_caption=user_caption, is_photo=True) + return get_input_media(media, is_photo=True) if isinstance(media, MessageMediaContact): return InputMediaContact( @@ -319,9 +311,7 @@ def get_input_media(media, user_caption=None, is_photo=False): return InputMediaEmpty() if isinstance(media, Message): - return get_input_media( - media.media, user_caption=user_caption, is_photo=is_photo - ) + return get_input_media(media.media, is_photo=is_photo) _raise_cast_fail(media, 'InputMedia') diff --git a/telethon_examples/interactive_telegram_client.py b/telethon_examples/interactive_telegram_client.py index f6986370..44185995 100644 --- a/telethon_examples/interactive_telegram_client.py +++ b/telethon_examples/interactive_telegram_client.py @@ -222,10 +222,8 @@ class InteractiveTelegramClient(TelegramClient): # Format the message content if getattr(msg, 'media', None): self.found_media[msg.id] = msg - # The media may or may not have a caption - caption = getattr(msg.media, 'caption', '') content = '<{}> {}'.format( - type(msg.media).__name__, caption) + type(msg.media).__name__, msg.message) elif hasattr(msg, 'message'): content = msg.message diff --git a/telethon_generator/scheme.tl b/telethon_generator/scheme.tl index 491f0c9e..a736b066 100644 --- a/telethon_generator/scheme.tl +++ b/telethon_generator/scheme.tl @@ -158,16 +158,16 @@ inputFile#f52ff27f id:long parts:int name:string md5_checksum:string = InputFile inputFileBig#fa4f0bb5 id:long parts:int name:string = InputFile; inputMediaEmpty#9664f57f = InputMedia; -inputMediaUploadedPhoto#2f37e231 flags:# file:InputFile caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; -inputMediaPhoto#81fa373a flags:# id:InputPhoto caption:string ttl_seconds:flags.0?int = InputMedia; +inputMediaUploadedPhoto#1e287d04 flags:# file:InputFile stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; +inputMediaPhoto#b3ba0635 flags:# id:InputPhoto ttl_seconds:flags.0?int = InputMedia; inputMediaGeoPoint#f9c44144 geo_point:InputGeoPoint = InputMedia; inputMediaContact#a6e45987 phone_number:string first_name:string last_name:string = InputMedia; -inputMediaUploadedDocument#e39621fd flags:# nosound_video:flags.3?true file:InputFile thumb:flags.2?InputFile mime_type:string attributes:Vector caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; -inputMediaDocument#5acb668e flags:# id:InputDocument caption:string ttl_seconds:flags.0?int = InputMedia; +inputMediaUploadedDocument#5b38c6c1 flags:# nosound_video:flags.3?true file:InputFile thumb:flags.2?InputFile mime_type:string attributes:Vector stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; +inputMediaDocument#23ab23d2 flags:# id:InputDocument ttl_seconds:flags.0?int = InputMedia; inputMediaVenue#c13d1c11 geo_point:InputGeoPoint title:string address:string provider:string venue_id:string venue_type:string = InputMedia; inputMediaGifExternal#4843b0fd url:string q:string = InputMedia; -inputMediaPhotoExternal#922aec1 flags:# url:string caption:string ttl_seconds:flags.0?int = InputMedia; -inputMediaDocumentExternal#b6f74335 flags:# url:string caption:string ttl_seconds:flags.0?int = InputMedia; +inputMediaPhotoExternal#e5bbfe1a flags:# url:string ttl_seconds:flags.0?int = InputMedia; +inputMediaDocumentExternal#fb52dc99 flags:# url:string ttl_seconds:flags.0?int = InputMedia; inputMediaGame#d33f43f3 id:InputGame = InputMedia; inputMediaInvoice#f4e096c3 flags:# title:string description:string photo:flags.0?InputWebDocument invoice:Invoice payload:bytes provider:string provider_data:DataJSON start_param:string = InputMedia; inputMediaGeoLive#7b1a118f geo_point:InputGeoPoint period:int = InputMedia; @@ -243,11 +243,11 @@ message#44f9b43d flags:# out:flags.1?true mentioned:flags.4?true media_unread:fl messageService#9e19a1f6 flags:# out:flags.1?true mentioned:flags.4?true media_unread:flags.5?true silent:flags.13?true post:flags.14?true id:int from_id:flags.8?int to_id:Peer reply_to_msg_id:flags.3?int date:int action:MessageAction = Message; messageMediaEmpty#3ded6320 = MessageMedia; -messageMediaPhoto#b5223b0f flags:# photo:flags.0?Photo caption:flags.1?string ttl_seconds:flags.2?int = MessageMedia; +messageMediaPhoto#695150d7 flags:# photo:flags.0?Photo ttl_seconds:flags.2?int = MessageMedia; messageMediaGeo#56e0d474 geo:GeoPoint = MessageMedia; messageMediaContact#5e7d2f39 phone_number:string first_name:string last_name:string user_id:int = MessageMedia; messageMediaUnsupported#9f84f49e = MessageMedia; -messageMediaDocument#7c4414d3 flags:# document:flags.0?Document caption:flags.1?string ttl_seconds:flags.2?int = MessageMedia; +messageMediaDocument#9cb070d7 flags:# document:flags.0?Document ttl_seconds:flags.2?int = MessageMedia; messageMediaWebPage#a32dd600 webpage:WebPage = MessageMedia; messageMediaVenue#2ec0533f geo:GeoPoint title:string address:string provider:string venue_id:string venue_type:string = MessageMedia; messageMediaGame#fdb19008 game:Game = MessageMedia; @@ -688,7 +688,7 @@ messages.foundGifs#450a1c0a next_offset:int results:Vector = messages. messages.savedGifsNotModified#e8025ca2 = messages.SavedGifs; messages.savedGifs#2e0709a5 hash:int gifs:Vector = messages.SavedGifs; -inputBotInlineMessageMediaAuto#292fed13 flags:# caption:string reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; +inputBotInlineMessageMediaAuto#3380c786 flags:# message:string entities:flags.1?Vector reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; inputBotInlineMessageText#3dcd7a87 flags:# no_webpage:flags.0?true message:string entities:flags.1?Vector reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; inputBotInlineMessageMediaGeo#c1b15d65 flags:# geo_point:InputGeoPoint period:int reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; inputBotInlineMessageMediaVenue#aaafadc8 flags:# geo_point:InputGeoPoint title:string address:string provider:string venue_id:string reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; @@ -700,7 +700,7 @@ inputBotInlineResultPhoto#a8d864a7 id:string type:string photo:InputPhoto send_m inputBotInlineResultDocument#fff8fdc4 flags:# id:string type:string title:flags.1?string description:flags.2?string document:InputDocument send_message:InputBotInlineMessage = InputBotInlineResult; inputBotInlineResultGame#4fa417f2 id:string short_name:string send_message:InputBotInlineMessage = InputBotInlineResult; -botInlineMessageMediaAuto#a74b15b flags:# caption:string reply_markup:flags.2?ReplyMarkup = BotInlineMessage; +botInlineMessageMediaAuto#764cf810 flags:# message:string entities:flags.1?Vector reply_markup:flags.2?ReplyMarkup = BotInlineMessage; botInlineMessageText#8c7f65e2 flags:# no_webpage:flags.0?true message:string entities:flags.1?Vector reply_markup:flags.2?ReplyMarkup = BotInlineMessage; botInlineMessageMediaGeo#b722de65 flags:# geo:GeoPoint period:int reply_markup:flags.2?ReplyMarkup = BotInlineMessage; botInlineMessageMediaVenue#4366232e flags:# geo:GeoPoint title:string address:string provider:string venue_id:string reply_markup:flags.2?ReplyMarkup = BotInlineMessage; @@ -711,7 +711,7 @@ botInlineMediaResult#17db940b flags:# id:string type:string photo:flags.0?Photo messages.botResults#947ca848 flags:# gallery:flags.0?true query_id:long next_offset:flags.1?string switch_pm:flags.2?InlineBotSwitchPM results:Vector cache_time:int users:Vector = messages.BotResults; -exportedMessageLink#1f486803 link:string = ExportedMessageLink; +exportedMessageLink#5dab1af4 link:string html:string = ExportedMessageLink; messageFwdHeader#559ebe6d flags:# from_id:flags.0?int date:int channel_id:flags.1?int channel_post:flags.2?int post_author:flags.3?string saved_from_peer:flags.4?Peer saved_from_msg_id:flags.4?int = MessageFwdHeader; @@ -896,7 +896,7 @@ langPackDifference#f385c1f6 lang_code:string from_version:int version:int string langPackLanguage#117698f1 name:string native_name:string lang_code:string = LangPackLanguage; -channelAdminRights#5d7ceba5 flags:# change_info:flags.0?true post_messages:flags.1?true edit_messages:flags.2?true delete_messages:flags.3?true ban_users:flags.4?true invite_users:flags.5?true invite_link:flags.6?true pin_messages:flags.7?true add_admins:flags.9?true manage_call:flags.10?true = ChannelAdminRights; +channelAdminRights#5d7ceba5 flags:# change_info:flags.0?true post_messages:flags.1?true edit_messages:flags.2?true delete_messages:flags.3?true ban_users:flags.4?true invite_users:flags.5?true invite_link:flags.6?true pin_messages:flags.7?true add_admins:flags.9?true = ChannelAdminRights; channelBannedRights#58cf4249 flags:# view_messages:flags.0?true send_messages:flags.1?true send_media:flags.2?true send_stickers:flags.3?true send_gifs:flags.4?true send_games:flags.5?true send_inline:flags.6?true embed_links:flags.7?true until_date:int = ChannelBannedRights; @@ -938,7 +938,7 @@ recentMeUrlStickerSet#bc0a57dc url:string set:StickerSetCovered = RecentMeUrl; help.recentMeUrls#e0310d7 urls:Vector chats:Vector users:Vector = help.RecentMeUrls; -inputSingleMedia#5eaa7809 media:InputMedia random_id:long = InputSingleMedia; +inputSingleMedia#31bc3d25 media:InputMedia flags:# random_id:long message:string entities:flags.0?Vector = InputSingleMedia; ---functions--- @@ -966,7 +966,7 @@ auth.resendCode#3ef1a9bf phone_number:string phone_code_hash:string = auth.SentC auth.cancelCode#1f040578 phone_number:string phone_code_hash:string = Bool; auth.dropTempAuthKeys#8e48a188 except_auth_keys:Vector = Bool; -account.registerDevice#f75874d1 token_type:int token:string other_uids:Vector = Bool; +account.registerDevice#1389cc token_type:int token:string app_sandbox:Bool other_uids:Vector = Bool; account.unregisterDevice#3076c4bf token_type:int token:string other_uids:Vector = Bool; account.updateNotifySettings#84be5b93 peer:InputNotifyPeer settings:InputPeerNotifySettings = Bool; account.getNotifySettings#12b3ad31 peer:InputNotifyPeer = PeerNotifySettings; @@ -1023,7 +1023,7 @@ messages.deleteMessages#e58e95d2 flags:# revoke:flags.0?true id:Vector = me messages.receivedMessages#5a954c0 max_id:int = Vector; messages.setTyping#a3825e50 peer:InputPeer action:SendMessageAction = Bool; messages.sendMessage#fa88427a flags:# no_webpage:flags.1?true silent:flags.5?true background:flags.6?true clear_draft:flags.7?true peer:InputPeer reply_to_msg_id:flags.0?int message:string random_id:long reply_markup:flags.2?ReplyMarkup entities:flags.3?Vector = Updates; -messages.sendMedia#c8f16791 flags:# silent:flags.5?true background:flags.6?true clear_draft:flags.7?true peer:InputPeer reply_to_msg_id:flags.0?int media:InputMedia random_id:long reply_markup:flags.2?ReplyMarkup = Updates; +messages.sendMedia#b8d1262b flags:# silent:flags.5?true background:flags.6?true clear_draft:flags.7?true peer:InputPeer reply_to_msg_id:flags.0?int media:InputMedia message:string random_id:long reply_markup:flags.2?ReplyMarkup entities:flags.3?Vector = Updates; messages.forwardMessages#708e0195 flags:# silent:flags.5?true background:flags.6?true with_my_score:flags.8?true grouped:flags.9?true from_peer:InputPeer id:Vector random_id:Vector to_peer:InputPeer = Updates; messages.reportSpam#cf1592db peer:InputPeer = Bool; messages.hideReportSpam#a8f1709b peer:InputPeer = Bool; @@ -1035,7 +1035,6 @@ messages.editChatPhoto#ca4c79d8 chat_id:int photo:InputChatPhoto = Updates; messages.addChatUser#f9a0aa09 chat_id:int user_id:InputUser fwd_limit:int = Updates; messages.deleteChatUser#e0611f16 chat_id:int user_id:InputUser = Updates; messages.createChat#9cb126e users:Vector title:string = Updates; -messages.forwardMessage#33963bf9 peer:InputPeer id:int random_id:long = Updates; messages.getDhConfig#26cf8950 version:int random_length:int = messages.DhConfig; messages.requestEncryption#f64daf43 user_id:InputUser random_id:int g_a:bytes = EncryptedChat; messages.acceptEncryption#3dbc0415 peer:InputEncryptedChat g_b:bytes key_fingerprint:long = EncryptedChat; @@ -1048,8 +1047,9 @@ messages.sendEncryptedService#32d439a4 peer:InputEncryptedChat random_id:long da messages.receivedQueue#55a5bb66 max_qts:int = Vector; messages.reportEncryptedSpam#4b0c8c0f peer:InputEncryptedChat = Bool; messages.readMessageContents#36a73f77 id:Vector = messages.AffectedMessages; +messages.getStickers#ae22e045 emoticon:string hash:string = messages.Stickers; messages.getAllStickers#1c9618b1 hash:int = messages.AllStickers; -messages.getWebPagePreview#25223e24 message:string = MessageMedia; +messages.getWebPagePreview#8b68b0cc flags:# message:string entities:flags.3?Vector = MessageMedia; messages.exportChatInvite#7d885289 chat_id:int = ExportedChatInvite; messages.checkChatInvite#3eadb1bb hash:string = ChatInvite; messages.importChatInvite#6c50051c hash:string = Updates; @@ -1199,4 +1199,4 @@ langpack.getStrings#2e1ee318 lang_code:string keys:Vector = Vector; -// LAYER 74 +// LAYER 75 From 74bffd2ae3e79c18bed379f0c3183801dfc98a1e Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 2 Mar 2018 21:32:38 +0100 Subject: [PATCH 37/38] Support multiple captions when sending albums --- telethon/telegram_client.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 24615ace..5323d13a 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1200,7 +1200,8 @@ class TelegramClient(TelegramBareClient): if all(utils.is_image(x) for x in file): return self._send_album( entity, file, caption=caption, - progress_callback=progress_callback, reply_to=reply_to + progress_callback=progress_callback, reply_to=reply_to, + parse_mode=parse_mode ) # Not all are images, so send all the files one by one return [ @@ -1350,10 +1351,13 @@ class TelegramClient(TelegramBareClient): # we need to produce right now to send albums (uploadMedia), and # cache only makes a difference for documents where the user may # want the attributes used on them to change. - # TODO Support a different captions for each file entity = self.get_input_entity(entity) - caption = caption or '' - caption, msg_entities = self._parse_message_text(caption, parse_mode) + if not utils.is_list_like(caption): + caption = (caption,) + captions = [ + self._parse_message_text(caption or '', parse_mode) + for caption in reversed(caption) # Pop from the end (so reverse) + ] reply_to = self._get_message_id(reply_to) # Need to upload the media first, but only if they're not cached yet @@ -1367,11 +1371,13 @@ class TelegramClient(TelegramBareClient): )).photo) self.session.cache_file(fh.md5, fh.size, input_photo) fh = input_photo - media.append(InputSingleMedia( - InputMediaPhoto(fh), - message=caption, - entities=msg_entities - )) + + if captions: + caption, msg_entities = captions.pop() + else: + caption, msg_entities = '', None + media.append(InputSingleMedia(InputMediaPhoto(fh), message=caption, + entities=msg_entities)) # Now we can construct the multi-media request result = self(SendMultiMediaRequest( From 9bf5cb7ed8d15021bd73e5051a7dd01e7af8a33b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 3 Mar 2018 12:28:18 +0200 Subject: [PATCH 38/38] Add new sessions docs --- readthedocs/extra/advanced-usage/sessions.rst | 96 +++++++++++++++---- 1 file changed, 78 insertions(+), 18 deletions(-) diff --git a/readthedocs/extra/advanced-usage/sessions.rst b/readthedocs/extra/advanced-usage/sessions.rst index fca7828e..ad824837 100644 --- a/readthedocs/extra/advanced-usage/sessions.rst +++ b/readthedocs/extra/advanced-usage/sessions.rst @@ -25,29 +25,89 @@ file, so that you can quickly access them by username or phone number. If you're not going to work with updates, or don't need to cache the ``access_hash`` associated with the entities' ID, you can disable this -by setting ``client.session.save_entities = False``, or pass it as a -parameter to the ``TelegramClient``. +by setting ``client.session.save_entities = False``. -If you don't want to save the files as a database, you can also create -your custom ``Session`` subclass and override the ``.save()`` and ``.load()`` -methods. For example, you could save it on a database: +Custom Session Storage +---------------------- + +If you don't want to use the default SQLite session storage, you can also use +one of the other implementations or implement your own storage. + +To use a custom session storage, simply pass the custom session instance to +``TelegramClient`` instead of the session name. + +Currently, there are three implementations of the abstract ``Session`` class: +* MemorySession. Stores session data in Python variables. +* SQLiteSession, the default. Stores sessions in their own SQLite databases. +* AlchemySession. Stores all sessions in a single database via SQLAlchemy. + +Using AlchemySession +~~~~~~~~~~~~~~~~~~~~ +The AlchemySession implementation can store multiple Sessions in the same +database, but to do this, each session instance needs to have access to the +same models and database session. + +To get started, you need to create an ``AlchemySessionContainer`` which will +contain that shared data. The simplest way to use ``AlchemySessionContainer`` +is to simply pass it the database URL: .. code-block:: python - class DatabaseSession(Session): - def save(): - # serialize relevant data to the database + container = AlchemySessionContainer('mysql://user:pass@localhost/telethon') - def load(): - # load relevant data to the database +If you already have SQLAlchemy set up for your own project, you can also pass +the engine separately: + + .. code-block:: python + + my_sqlalchemy_engine = sqlalchemy.create_engine('...') + container = AlchemySessionContainer(engine=my_sqlalchemy_engine) + +By default, the session container will manage table creation/schema updates/etc +automatically. If you want to manage everything yourself, you can pass your +SQLAlchemy Session and ``declarative_base`` instances and set ``manage_tables`` +to ``False``: + + .. code-block:: python + + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy import orm + import sqlalchemy + + ... + + session_factory = orm.sessionmaker(bind=my_sqlalchemy_engine) + session = session_factory() + my_base = declarative_base() + + ... + + container = AlchemySessionContainer(session=session, table_base=my_base, manage_tables=False) + +You always need to provide either ``engine`` or ``session`` to the container. +If you set ``manage_tables=False`` and provide a ``session``, ``engine`` is not +needed. In any other case, ``engine`` is always required. + +After you have your ``AlchemySessionContainer`` instance created, you can +create new sessions by calling ``new_session``: + + .. code-block:: python + + session = container.new_session('some session id') + client = TelegramClient(session) + +where ``some session id`` is an unique identifier for the session. + +Creating your own storage +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The easiest way to create your own implementation is to use MemorySession as +the base and check out how ``SQLiteSession`` or ``AlchemySession`` work. You +can find the relevant Python files under the ``sessions`` directory. -You should read the ````session.py```` source file to know what "relevant -data" you need to keep track of. - - -Sessions and Heroku -------------------- +SQLite Sessions and Heroku +-------------------------- You probably have a newer version of SQLite installed (>= 3.8.2). Heroku uses SQLite 3.7.9 which does not support ``WITHOUT ROWID``. So, if you generated @@ -59,8 +119,8 @@ session file on your Heroku dyno itself. The most complicated is creating a custom buildpack to install SQLite >= 3.8.2. -Generating a Session File on a Heroku Dyno -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Generating a SQLite Session File on a Heroku Dyno +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. note:: Due to Heroku's ephemeral filesystem all dynamically generated