From 624067783182522051ebc354488bf9aa0b94cf6b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 9 Feb 2018 19:39:20 +0100 Subject: [PATCH 01/17] Fix sign up method not accepting integer codes --- telethon/telegram_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index c4f5d722..1e90824f 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -416,7 +416,7 @@ class TelegramClient(TelegramBareClient): result = self(SignUpRequest( phone_number=self._phone, phone_code_hash=self._phone_code_hash.get(self._phone, ''), - phone_code=code, + phone_code=str(code), first_name=first_name, last_name=last_name )) From 7d8d86c5f1b5ac0aa6a9a4d9485786d3d53fc0e7 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 10 Feb 2018 10:26:58 +0100 Subject: [PATCH 02/17] Support inline mentions (bot API style or username/phone) --- telethon/telegram_client.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 1e90824f..f4efaa55 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -3,6 +3,7 @@ import io import itertools import logging import os +import re import sys import time from collections import OrderedDict, UserList @@ -68,7 +69,8 @@ from .tl.types import ( PeerUser, InputPeerUser, InputPeerChat, InputPeerChannel, MessageEmpty, ChatInvite, ChatInviteAlready, PeerChannel, Photo, InputPeerSelf, InputSingleMedia, InputMediaPhoto, InputPhoto, InputFile, InputFileBig, - InputDocument, InputMediaDocument, Document + InputDocument, InputMediaDocument, Document, MessageEntityTextUrl, + InputMessageEntityMentionName ) from .tl.types.messages import DialogsSlice from .extensions import markdown, html @@ -603,6 +605,20 @@ class TelegramClient(TelegramBareClient): message, msg_entities = html.parse(message) else: raise ValueError('Unknown parsing mode: {}'.format(parse_mode)) + + for i, e in enumerate(msg_entities): + if isinstance(e, MessageEntityTextUrl): + m = re.match(r'^@|\+|tg://user\?id=(\d+)', e.url) + if m: + try: + msg_entities[i] = InputMessageEntityMentionName( + e.offset, e.length, self.get_input_entity( + int(m.group(1)) if m.group(1) else e.url + ) + ) + except (ValueError, TypeError): + # Make no replacement + pass else: msg_entities = [] From eca1e8ec877812c5f7b699c3b7daaccf49ceeda0 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 10 Feb 2018 10:45:43 +0100 Subject: [PATCH 03/17] Default to markdown parse mode on send_message This is consistent with official clients and also provide the expected result when replying to NewMessage events. --- telethon/telegram_client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index f4efaa55..e5689fad 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -570,7 +570,7 @@ class TelegramClient(TelegramBareClient): if update.message.id == msg_id: return update.message - def send_message(self, entity, message, reply_to=None, parse_mode=None, + def send_message(self, entity, message, reply_to=None, parse_mode='md', link_preview=True): """ Sends the given message to the specified entity (user/chat/channel). @@ -587,8 +587,10 @@ class TelegramClient(TelegramBareClient): it should be the ID of the message that it should reply to. parse_mode (:obj:`str`, optional): - Can be 'md' or 'markdown' for markdown-like parsing, in a similar - fashion how official clients work. + Can be 'md' or 'markdown' for markdown-like parsing (default), + or 'htm' or 'html' for HTML-like parsing. If ``None`` or any + other false-y value is provided, the message will be sent with + no formatting. link_preview (:obj:`bool`, optional): Should the link preview be shown? From 0633e204c219a86665ba515124959af9edccada7 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 11 Feb 2018 10:30:45 +0100 Subject: [PATCH 04/17] Fix whitelisting multiple chats on events not working --- telethon/events/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/telethon/events/__init__.py b/telethon/events/__init__.py index 40678998..a22a757e 100644 --- a/telethon/events/__init__.py +++ b/telethon/events/__init__.py @@ -149,8 +149,8 @@ class NewMessage(_EventBuilder): def resolve(self, client): if hasattr(self.chats, '__iter__') and not isinstance(self.chats, str): - self.chats = set(utils.get_peer_id(x) - for x in client.get_input_entity(self.chats)) + self.chats = set(utils.get_peer_id(client.get_input_entity(x)) + for x in self.chats) elif self.chats is not None: self.chats = {utils.get_peer_id( client.get_input_entity(self.chats))} @@ -397,8 +397,8 @@ class ChatAction(_EventBuilder): def resolve(self, client): if hasattr(self.chats, '__iter__') and not isinstance(self.chats, str): - self.chats = set(utils.get_peer_id(x) - for x in client.get_input_entity(self.chats)) + self.chats = set(utils.get_peer_id(client.get_input_entity(x)) + for x in self.chats) elif self.chats is not None: self.chats = {utils.get_peer_id( client.get_input_entity(self.chats))} From 9abeefac7f276336b9235bd3b160427e227ccea6 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 12 Feb 2018 10:33:51 +0100 Subject: [PATCH 05/17] Send video files as video by default instead as document (#601) --- telethon/telegram_client.py | 9 ++++++--- telethon/utils.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index e5689fad..c0c1bb1e 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -70,7 +70,7 @@ from .tl.types import ( ChatInvite, ChatInviteAlready, PeerChannel, Photo, InputPeerSelf, InputSingleMedia, InputMediaPhoto, InputPhoto, InputFile, InputFileBig, InputDocument, InputMediaDocument, Document, MessageEntityTextUrl, - InputMessageEntityMentionName + InputMessageEntityMentionName, DocumentAttributeVideo ) from .tl.types.messages import DialogsSlice from .extensions import markdown, html @@ -922,8 +922,8 @@ class TelegramClient(TelegramBareClient): force_document (:obj:`bool`, optional): If left to ``False`` and the file is a path that ends with - ``.png``, ``.jpg`` and such, the file will be sent as a photo. - Otherwise always as a document. + the extension of an image file or a video file, it will be + sent as such. Otherwise always as a document. progress_callback (:obj:`callable`, optional): A callback function accepting two parameters: @@ -1015,6 +1015,9 @@ class TelegramClient(TelegramBareClient): # TODO If the input file is an audio, find out: # Performer and song title and add DocumentAttributeAudio } + if not force_document and utils.is_video(file): + attr_dict[DocumentAttributeVideo] = \ + DocumentAttributeVideo(0, 0, 0) else: attr_dict = { DocumentAttributeFilename: diff --git a/telethon/utils.py b/telethon/utils.py index 3e310d3d..9460986c 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -3,10 +3,10 @@ Utilities for working with the Telegram API itself (such as handy methods to convert between an entity like an User, Chat, etc. into its Input version) """ import math +import mimetypes import re from mimetypes import add_type, guess_extension -from .tl.types.contacts import ResolvedPeer from .tl.types import ( Channel, ChannelForbidden, Chat, ChatEmpty, ChatForbidden, ChatFull, ChatPhoto, InputPeerChannel, InputPeerChat, InputPeerUser, InputPeerEmpty, @@ -24,6 +24,7 @@ from .tl.types import ( InputMediaUploadedPhoto, DocumentAttributeFilename, photos, TopPeer, InputNotifyPeer ) +from .tl.types.contacts import ResolvedPeer USERNAME_RE = re.compile( r'@|(?:https?://)?(?:telegram\.(?:me|dog)|t\.me)/(joinchat/)?' @@ -322,8 +323,12 @@ def get_input_media(media, user_caption=None, is_photo=False): def is_image(file): """Returns True if the file extension looks like an image file""" - return (isinstance(file, str) and - bool(re.search(r'\.(png|jpe?g|gif)$', file, re.IGNORECASE))) + return (mimetypes.guess_type(file)[0] or '').startswith('image/') + + +def is_video(file): + """Returns True if the file extension looks like a video file""" + return (mimetypes.guess_type(file)[0] or '').startswith('video/') def parse_phone(phone): From c83638ed0e168985f8edefd2cb2b1c266a288174 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 12 Feb 2018 13:40:00 +0100 Subject: [PATCH 06/17] Add further logging calls to better spot lost requests --- telethon/network/mtproto_sender.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 877611df..43b5e803 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -86,6 +86,11 @@ class MtProtoSender: messages = [TLMessage(self.session, r) for r in requests] self._pending_receive.update({m.msg_id: m for m in messages}) + __log__.debug('Sending requests with IDs: %s', ', '.join( + '{}: {}'.format(m.request.__class__.__name__, m.msg_id) + for m in messages + )) + # Pack everything in the same container if we need to send AckRequests if self._need_confirmation: messages.append( @@ -465,6 +470,7 @@ class MtProtoSender: request_id = reader.read_long() inner_code = reader.read_int(signed=False) + __log__.debug('Received response for request with ID %d', request_id) request = self._pop_request(request_id) if inner_code == 0x2144ca19: # RPC Error @@ -502,8 +508,18 @@ class MtProtoSender: return True # If it's really a result for RPC from previous connection - # session, it will be skipped by the handle_container() - __log__.warning('Lost request will be skipped') + # session, it will be skipped by the handle_container(). + # For some reason this also seems to happen when downloading + # photos, where the server responds with FileJpeg(). + try: + obj = reader.tgread_object() + except Exception as e: + obj = '(failed to read: %s)' % e + + __log__.warning( + 'Lost request (ID %d) with code %s will be skipped, contents: %s', + request_id, hex(inner_code), obj + ) return False def _handle_gzip_packed(self, msg_id, sequence, reader, state): From 08b9d7c4ef034370cd6733643cda167e0b1c6d48 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 13 Feb 2018 10:24:35 +0100 Subject: [PATCH 07/17] Add more logic to better retrieve input_sender on events --- telethon/events/__init__.py | 44 +++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/telethon/events/__init__.py b/telethon/events/__init__.py index a22a757e..a02385c1 100644 --- a/telethon/events/__init__.py +++ b/telethon/events/__init__.py @@ -136,6 +136,11 @@ class NewMessage(_EventBuilder): blacklist_chats (:obj:`bool`, optional): Whether to treat the the list of chats as a blacklist (if it matches it will NOT be handled) or a whitelist (default). + + Notes: + The ``message.from_id`` might not only be an integer or ``None``, + but also ``InputPeerSelf()`` for short private messages (the API + would not return such thing, this is a custom modification). """ def __init__(self, incoming=None, outgoing=None, chats=None, blacklist_chats=False): @@ -169,6 +174,7 @@ class NewMessage(_EventBuilder): silent=update.silent, id=update.id, to_id=types.PeerUser(update.user_id), + from_id=types.InputPeerSelf() if update.out else update.user_id, message=update.message, date=update.date, fwd_from=update.fwd_from, @@ -257,21 +263,23 @@ class NewMessage(_EventBuilder): things like username or similar, but still useful in some cases. Note that this might not be available if the library can't - find the input chat. + find the input chat, or if the message a broadcast on a channel. """ if self._input_sender is None: + if self.is_channel and not self.is_group: + return None + try: self._input_sender = self._client.get_input_entity( self.message.from_id ) except (ValueError, TypeError): - if isinstance(self.message.to_id, types.PeerChannel): - # We can rely on self.input_chat for this - self._input_sender = self._get_input_entity( - self.message.id, - self.message.from_id, - chat=self.input_chat - ) + # We can rely on self.input_chat for this + self._input_sender = self._get_input_entity( + self.message.id, + self.message.from_id, + chat=self.input_chat + ) return self._input_sender @@ -835,22 +843,24 @@ class MessageChanged(_EventBuilder): things like username or similar, but still useful in some cases. Note that this might not be available if the library can't - find the input chat. + find the input chat, or if the message a broadcast on a channel. """ # TODO Code duplication - if self._input_sender is None and self.message: + if self._input_sender is None: + if self.is_channel and not self.is_group: + return None + try: self._input_sender = self._client.get_input_entity( self.message.from_id ) except (ValueError, TypeError): - if isinstance(self.message.to_id, types.PeerChannel): - # We can rely on self.input_chat for this - self._input_sender = self._get_input_entity( - self.message.id, - self.message.from_id, - chat=self.input_chat - ) + # We can rely on self.input_chat for this + self._input_sender = self._get_input_entity( + self.message.id, + self.message.from_id, + chat=self.input_chat + ) return self._input_sender From 55bcc29ae0834e202a0040dd72c1116cb4a9f5c7 Mon Sep 17 00:00:00 2001 From: "Dmitry D. Chernov" Date: Wed, 14 Feb 2018 17:09:22 +1000 Subject: [PATCH 08/17] Errors: Fix passing 'self' to the constructors of the superclasses This is necessary only if the superclass name is specified explicitly instead of super() call. --- telethon/errors/common.py | 8 +++----- telethon/errors/rpc_base_errors.py | 8 ++++---- telethon_generator/error_generator.py | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/telethon/errors/common.py b/telethon/errors/common.py index 46b0b52e..0c03aee6 100644 --- a/telethon/errors/common.py +++ b/telethon/errors/common.py @@ -4,7 +4,7 @@ class ReadCancelledError(Exception): """Occurs when a read operation was cancelled.""" def __init__(self): - super().__init__(self, 'The read operation was cancelled.') + super().__init__('The read operation was cancelled.') class TypeNotFoundError(Exception): @@ -14,7 +14,7 @@ class TypeNotFoundError(Exception): """ def __init__(self, invalid_constructor_id): super().__init__( - self, 'Could not find a matching Constructor ID for the TLObject ' + 'Could not find a matching Constructor ID for the TLObject ' 'that was supposed to be read with ID {}. Most likely, a TLObject ' 'was trying to be read when it should not be read.' .format(hex(invalid_constructor_id))) @@ -29,7 +29,6 @@ class InvalidChecksumError(Exception): """ def __init__(self, checksum, valid_checksum): super().__init__( - self, 'Invalid checksum ({} when {} was expected). ' 'This packet should be skipped.' .format(checksum, valid_checksum)) @@ -44,7 +43,6 @@ class BrokenAuthKeyError(Exception): """ def __init__(self): super().__init__( - self, 'The authorization key is broken, and it must be reset.' ) @@ -56,7 +54,7 @@ class SecurityError(Exception): def __init__(self, *args): if not args: args = ['A security check failed.'] - super().__init__(self, *args) + super().__init__(*args) class CdnFileTamperedError(SecurityError): diff --git a/telethon/errors/rpc_base_errors.py b/telethon/errors/rpc_base_errors.py index 9e6eed1a..467b256c 100644 --- a/telethon/errors/rpc_base_errors.py +++ b/telethon/errors/rpc_base_errors.py @@ -40,7 +40,7 @@ class ForbiddenError(RPCError): message = 'FORBIDDEN' def __init__(self, message): - super().__init__(self, message) + super().__init__(message) self.message = message @@ -52,7 +52,7 @@ class NotFoundError(RPCError): message = 'NOT_FOUND' def __init__(self, message): - super().__init__(self, message) + super().__init__(message) self.message = message @@ -77,7 +77,7 @@ class ServerError(RPCError): message = 'INTERNAL' def __init__(self, message): - super().__init__(self, message) + super().__init__(message) self.message = message @@ -121,7 +121,7 @@ class BadMessageError(Exception): } def __init__(self, code): - super().__init__(self, self.ErrorMessages.get( + super().__init__(self.ErrorMessages.get( code, 'Unknown error code (this should not happen): {}.'.format(code))) diff --git a/telethon_generator/error_generator.py b/telethon_generator/error_generator.py index a56d4b91..73fb5c5a 100644 --- a/telethon_generator/error_generator.py +++ b/telethon_generator/error_generator.py @@ -68,7 +68,7 @@ def write_error(f, code, name, desc, capture_name): f.write( "self.{} = int(kwargs.get('capture', 0))\n ".format(capture_name) ) - f.write('super(Exception, self).__init__(self, {}'.format(repr(desc))) + f.write('super(Exception, self).__init__({}'.format(repr(desc))) if capture_name: f.write('.format(self.{})'.format(capture_name)) f.write(')\n') From e9dd93f09cc2232c388479fb1fa366adef51af51 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 14 Feb 2018 14:06:45 +0100 Subject: [PATCH 09/17] Further clarify the documentation (flood wait, lists and inputs) --- docs/generate.py | 13 ++++++++++++- docs/res/css/docs.css | 4 ++++ readthedocs/extra/basic/creating-a-client.rst | 5 +++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/generate.py b/docs/generate.py index ae2bd43c..75ab3091 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -207,6 +207,13 @@ def get_description(arg): desc.append('This argument can be omitted.') otherwise = True + if arg.type in {'InputPeer', 'InputUser', 'InputChannel'}: + desc.append( + 'Anything entity-like will work if the library can find its ' + 'Input version (e.g., usernames, Peer, ' + 'User or Channel objects, etc.).' + ) + if arg.is_vector: if arg.is_generic: desc.append('A list of other Requests must be supplied.') @@ -221,7 +228,11 @@ def get_description(arg): desc.insert(1, 'Otherwise,') desc[-1] = desc[-1][:1].lower() + desc[-1][1:] - return ' '.join(desc) + return ' '.join(desc).replace( + 'list', + 'list' + ) def copy_replace(src, dst, replacements): diff --git a/docs/res/css/docs.css b/docs/res/css/docs.css index 05c61c9f..cd67af70 100644 --- a/docs/res/css/docs.css +++ b/docs/res/css/docs.css @@ -108,6 +108,10 @@ span.sh4 { color: #06c; } +span.tooltip { + border-bottom: 1px dashed #444; +} + #searchBox { width: 100%; border: none; diff --git a/readthedocs/extra/basic/creating-a-client.rst b/readthedocs/extra/basic/creating-a-client.rst index bf565bb0..e68f170b 100644 --- a/readthedocs/extra/basic/creating-a-client.rst +++ b/readthedocs/extra/basic/creating-a-client.rst @@ -95,6 +95,11 @@ is just a matter of taste, and how much control you need. Remember that you can get yourself at any time with ``client.get_me()``. +.. warning:: + Please note that if you fail to login around 5 times (or change the first + parameter of the ``TelegramClient``, which is the session name) you will + receive a ``FloodWaitError`` of around 22 hours, so be careful not to mess + this up! This shouldn't happen if you're doing things as explained, though. .. note:: If you want to use a **proxy**, you have to `install PySocks`__ From 80f918956a147c53e7878323f0f9b03d27475397 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 15 Feb 2018 10:02:12 +0100 Subject: [PATCH 10/17] Revert "official apps must be obfuscated" (29471f3) It didn't really make any sense and @danog keeps changing it. --- telethon/telegram_bare_client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 5984bb2e..db07f321 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -103,8 +103,6 @@ class TelegramBareClient: self.session = session self.api_id = int(api_id) self.api_hash = api_hash - if self.api_id < 20: # official apps must use obfuscated - connection_mode = ConnectionMode.TCP_OBFUSCATED # This is the main sender, which will be used from the thread # that calls .connect(). Every other thread will spawn a new From 62c057a0582247bc9b6787a318e4481181bf66af Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 15 Feb 2018 11:19:34 +0100 Subject: [PATCH 11/17] Add edit_message convenience method and refactor to accomodate it --- telethon/telegram_client.py | 147 ++++++++++++++++++++++++++---------- 1 file changed, 109 insertions(+), 38 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index c0c1bb1e..e54690e9 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -46,8 +46,8 @@ from .tl.functions.contacts import ( from .tl.functions.messages import ( GetDialogsRequest, GetHistoryRequest, SendMediaRequest, SendMessageRequest, GetChatsRequest, GetAllDraftsRequest, - CheckChatInviteRequest, ReadMentionsRequest, - SendMultiMediaRequest, UploadMediaRequest + CheckChatInviteRequest, ReadMentionsRequest, SendMultiMediaRequest, + UploadMediaRequest, EditMessageRequest ) from .tl.functions import channels @@ -70,7 +70,8 @@ from .tl.types import ( ChatInvite, ChatInviteAlready, PeerChannel, Photo, InputPeerSelf, InputSingleMedia, InputMediaPhoto, InputPhoto, InputFile, InputFileBig, InputDocument, InputMediaDocument, Document, MessageEntityTextUrl, - InputMessageEntityMentionName, DocumentAttributeVideo + InputMessageEntityMentionName, DocumentAttributeVideo, + UpdateEditMessage, UpdateEditChannelMessage, UpdateShort, Updates ) from .tl.types.messages import DialogsSlice from .extensions import markdown, html @@ -565,11 +566,60 @@ class TelegramClient(TelegramBareClient): msg_id = update.id break - for update in result.updates: + if isinstance(result, UpdateShort): + updates = [result.update] + elif isinstance(result, Updates): + updates = result.updates + else: + return + + for update in updates: if isinstance(update, (UpdateNewChannelMessage, UpdateNewMessage)): if update.message.id == msg_id: return update.message + elif (isinstance(update, UpdateEditMessage) and + not isinstance(request.peer, InputPeerChannel)): + if request.id == update.message.id: + return update.message + + elif (isinstance(update, UpdateEditChannelMessage) and + utils.get_peer_id(request.peer) == + utils.get_peer_id(update.message.to_id)): + if request.id == update.message.id: + return update.message + + def _parse_message_text(self, message, parse_mode): + """ + Returns a (parsed message, entities) tuple depending on parse_mode. + """ + if not parse_mode: + return message, [] + + parse_mode = parse_mode.lower() + if parse_mode in {'md', 'markdown'}: + message, msg_entities = markdown.parse(message) + elif parse_mode.startswith('htm'): + message, msg_entities = html.parse(message) + else: + raise ValueError('Unknown parsing mode: {}'.format(parse_mode)) + + for i, e in enumerate(msg_entities): + if isinstance(e, MessageEntityTextUrl): + m = re.match(r'^@|\+|tg://user\?id=(\d+)', e.url) + if m: + try: + msg_entities[i] = InputMessageEntityMentionName( + e.offset, e.length, self.get_input_entity( + int(m.group(1)) if m.group(1) else e.url + ) + ) + except (ValueError, TypeError): + # Make no replacement + pass + + return message, msg_entities + def send_message(self, entity, message, reply_to=None, parse_mode='md', link_preview=True): """ @@ -599,37 +649,14 @@ class TelegramClient(TelegramBareClient): the sent message """ entity = self.get_input_entity(entity) - if parse_mode: - parse_mode = parse_mode.lower() - if parse_mode in {'md', 'markdown'}: - message, msg_entities = markdown.parse(message) - elif parse_mode.startswith('htm'): - message, msg_entities = html.parse(message) - else: - raise ValueError('Unknown parsing mode: {}'.format(parse_mode)) - - for i, e in enumerate(msg_entities): - if isinstance(e, MessageEntityTextUrl): - m = re.match(r'^@|\+|tg://user\?id=(\d+)', e.url) - if m: - try: - msg_entities[i] = InputMessageEntityMentionName( - e.offset, e.length, self.get_input_entity( - int(m.group(1)) if m.group(1) else e.url - ) - ) - except (ValueError, TypeError): - # Make no replacement - pass - else: - msg_entities = [] + message, msg_entities = self._parse_message_text(message, parse_mode) request = SendMessageRequest( peer=entity, message=message, entities=msg_entities, no_webpage=not link_preview, - reply_to_msg_id=self._get_reply_to(reply_to) + reply_to_msg_id=self._get_message_id(reply_to) ) result = self(request) if isinstance(result, UpdateShortSentMessage): @@ -645,6 +672,50 @@ class TelegramClient(TelegramBareClient): return self._get_response_message(request, result) + def edit_message(self, entity, message_id, message=None, parse_mode='md', + link_preview=True): + """ + Edits the given message ID (to change its contents or disable preview). + + Args: + entity (:obj:`entity`): + From which chat to edit the message. + + message_id (:obj:`str`): + The ID of the message (or ``Message`` itself) to be edited. + + message (:obj:`str`, optional): + The new text of the message. + + parse_mode (:obj:`str`, optional): + Can be 'md' or 'markdown' for markdown-like parsing (default), + or 'htm' or 'html' for HTML-like parsing. If ``None`` or any + other false-y value is provided, the message will be sent with + no formatting. + + link_preview (:obj:`bool`, optional): + Should the link preview be shown? + + Raises: + ``MessageAuthorRequiredError`` if you're not the author of the + message but try editing it anyway. + + ``MessageNotModifiedError`` if the contents of the message were + not modified at all. + + Returns: + the edited message + """ + message, msg_entities = self._parse_message_text(message, parse_mode) + request = EditMessageRequest( + peer=self.get_input_entity(entity), + id=self._get_message_id(message_id), + message=message, + no_webpage=not link_preview + ) + result = self(request) + return self._get_response_message(request, result) + def delete_messages(self, entity, message_ids, revoke=True): """ Deletes a message from a chat, optionally "for everyone". @@ -869,22 +940,22 @@ class TelegramClient(TelegramBareClient): return False @staticmethod - def _get_reply_to(reply_to): + def _get_message_id(message): """Sanitizes the 'reply_to' parameter a user may send""" - if reply_to is None: + if message is None: return None - if isinstance(reply_to, int): - return reply_to + if isinstance(message, int): + return message try: - if reply_to.SUBCLASS_OF_ID == 0x790009e3: + if message.SUBCLASS_OF_ID == 0x790009e3: # hex(crc32(b'Message')) = 0x790009e3 - return reply_to.id + return message.id except AttributeError: pass - raise TypeError('Invalid reply_to type: {}'.format(type(reply_to))) + raise TypeError('Invalid message type: {}'.format(type(message))) # endregion @@ -973,7 +1044,7 @@ class TelegramClient(TelegramBareClient): ] entity = self.get_input_entity(entity) - reply_to = self._get_reply_to(reply_to) + reply_to = self._get_message_id(reply_to) if not isinstance(file, (str, bytes, io.IOBase)): # The user may pass a Message containing media (or the media, @@ -1086,7 +1157,7 @@ class TelegramClient(TelegramBareClient): # cache only makes a difference for documents where the user may # want the attributes used on them to change. Caption's ignored. entity = self.get_input_entity(entity) - reply_to = self._get_reply_to(reply_to) + reply_to = self._get_message_id(reply_to) # Need to upload the media first, but only if they're not cached yet media = [] From 196275e9c8e3aa1e48ee258be5557dfc02e91090 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 15 Feb 2018 11:35:12 +0100 Subject: [PATCH 12/17] Add edit and delete shorthand methods to events.NewMessage --- telethon/events/__init__.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/telethon/events/__init__.py b/telethon/events/__init__.py index a02385c1..3a888200 100644 --- a/telethon/events/__init__.py +++ b/telethon/events/__init__.py @@ -255,6 +255,32 @@ class NewMessage(_EventBuilder): reply_to=self.message.id, *args, **kwargs) + def edit(self, *args, **kwargs): + """ + Edits the message iff it's outgoing. This is a shorthand for + ``client.edit_message(event.chat, event.message, ...)``. + + Returns ``None`` if the message was incoming, + or the edited message otherwise. + """ + if not self.message.out: + return None + + return self._client.edit_message(self.input_chat, + self.message, + *args, **kwargs) + + def delete(self, *args, **kwargs): + """ + Deletes the message. You're responsible for checking whether you + have the permission to do so, or to except the error otherwise. + This is a shorthand for + ``client.delete_messages(event.chat, event.message, ...)``. + """ + return self._client.delete_messages(self.input_chat, + [self.message], + *args, **kwargs) + @property def input_sender(self): """ From 178643d3a1951688ee16a32ee33d89ed53d4f14a Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 15 Feb 2018 11:41:32 +0100 Subject: [PATCH 13/17] Periodically send getState even without disconnect (341fb38) After some more tests, even if the server doesn't drop the connection, it might also just stop sending updates at all. --- telethon/telegram_bare_client.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index db07f321..31a3f7d9 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -158,6 +158,14 @@ class TelegramBareClient: self._last_ping = datetime.now() self._ping_delay = timedelta(minutes=1) + # Also have another delay for GetStateRequest. + # + # If the connection is kept alive for long without invoking any + # high level request the server simply stops sending updates. + # TODO maybe we can have ._last_request instead if any req works? + self._last_state = datetime.now() + self._state_delay = timedelta(hours=1) + # Some errors are known but there's nothing we can do from the # background thread. If any of these happens, call .disconnect(), # and raise them next time .invoke() is tried to be called. @@ -579,6 +587,7 @@ class TelegramBareClient: otherwise it should be called manually after enabling updates. """ self.updates.process(self(GetStateRequest())) + self._last_state = datetime.now() def add_update_handler(self, handler): """Adds an update handler (a function which takes a TLObject, @@ -650,6 +659,10 @@ class TelegramBareClient: )) self._last_ping = datetime.now() + if datetime.now() > self._last_state + self._state_delay: + self._sender.send(GetStateRequest()) + self._last_state = datetime.now() + __log__.debug('Receiving items from the network...') self._sender.receive(update_state=self.updates) except TimeoutError: From 75d99fbb53c1bb7d46f9b9909c056b6e00eab4d8 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 15 Feb 2018 11:52:46 +0100 Subject: [PATCH 14/17] Fix HTML entity parsing failing when needing surrogates --- telethon/extensions/html.py | 23 +++++++++++++++++++---- telethon/extensions/markdown.py | 3 +++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/telethon/extensions/html.py b/telethon/extensions/html.py index 8cd170cb..bcbd13cc 100644 --- a/telethon/extensions/html.py +++ b/telethon/extensions/html.py @@ -1,9 +1,10 @@ """ Simple HTML -> Telegram entity parser. """ +import struct +from collections import deque from html import escape, unescape from html.parser import HTMLParser -from collections import deque from ..tl.types import ( MessageEntityBold, MessageEntityItalic, MessageEntityCode, @@ -12,6 +13,18 @@ from ..tl.types import ( ) +# Helpers from markdown.py +def _add_surrogate(text): + return ''.join( + ''.join(chr(y) for y in struct.unpack(' Date: Thu, 15 Feb 2018 12:11:26 +0100 Subject: [PATCH 15/17] Update to v0.17.2 --- readthedocs/extra/changelog.rst | 35 +++++++++++++++++++++++++++++++++ telethon/version.py | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/readthedocs/extra/changelog.rst b/readthedocs/extra/changelog.rst index 57b11bec..34609615 100644 --- a/readthedocs/extra/changelog.rst +++ b/readthedocs/extra/changelog.rst @@ -14,6 +14,41 @@ it can take advantage of new goodies! .. contents:: List of All Versions +New small convenience functions (v0.17.2) +========================================= + +*Published at 2018/02/15* + +Primarily bug fixing and a few welcomed additions. + +Additions +~~~~~~~~~ + +- New convenience ``.edit_message()`` method on the ``TelegramClient``. +- New ``.edit()`` and ``.delete()`` shorthands on the ``NewMessage`` event. +- Default to markdown parsing when sending and editing messages. +- Support for inline mentions when sending and editing messages. They work + like inline urls (e.g. ``[text](@username)``) and also support the Bot-API + style (see `here `__). + +Bug fixes +~~~~~~~~~ + +- Periodically send ``GetStateRequest`` automatically to keep the server + sending updates even if you're not invoking any request yourself. +- HTML parsing was failing due to not handling surrogates properly. +- ``.sign_up`` was not accepting ``int`` codes. +- Whitelisting more than one chat on ``events`` wasn't working. +- Video files are sent as a video by default unless ``force_document``. + +Internal changes +~~~~~~~~~~~~~~~~ + +- More ``logging`` calls to help spot some bugs in the future. +- Some more logic to retrieve input entities on events. +- Clarified a few parts of the documentation. + + Updates as Events (v0.17.1) =========================== diff --git a/telethon/version.py b/telethon/version.py index 6cf35eba..4ead720b 100644 --- a/telethon/version.py +++ b/telethon/version.py @@ -1,3 +1,3 @@ # Versions should comply with PEP440. # This line is parsed in setup.py: -__version__ = '0.17.1' +__version__ = '0.17.2' From c11aefa95be711540b167abc3442101ed628c6f2 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 16 Feb 2018 13:42:12 +0100 Subject: [PATCH 16/17] Fix message entities being ignored by edit_message --- telethon/telegram_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index e54690e9..7de7b59a 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -711,7 +711,8 @@ class TelegramClient(TelegramBareClient): peer=self.get_input_entity(entity), id=self._get_message_id(message_id), message=message, - no_webpage=not link_preview + no_webpage=not link_preview, + entities=msg_entities ) result = self(request) return self._get_response_message(request, result) From 030f29220328693e08bc9bc7148b07ddc22f7aef Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 16 Feb 2018 18:24:44 +0100 Subject: [PATCH 17/17] Dump libssl bindings in favour of the new optional cryptg module --- telethon/crypto/aes.py | 124 ++++++++++++++++++++------------------ telethon/crypto/libssl.py | 107 -------------------------------- 2 files changed, 64 insertions(+), 167 deletions(-) delete mode 100644 telethon/crypto/libssl.py diff --git a/telethon/crypto/aes.py b/telethon/crypto/aes.py index 191cde15..8f13b5f0 100644 --- a/telethon/crypto/aes.py +++ b/telethon/crypto/aes.py @@ -3,86 +3,90 @@ AES IGE implementation in Python. This module may use libssl if available. """ import os import pyaes -from . import libssl + +try: + import cryptg +except ImportError: + cryptg = None -if libssl.AES is not None: - # Use libssl if available, since it will be faster - AES = libssl.AES -else: - # Fallback to a pure Python implementation - class AES: +class AES: + """ + Class that servers as an interface to encrypt and decrypt + text through the AES IGE mode. + """ + @staticmethod + def decrypt_ige(cipher_text, key, iv): """ - Class that servers as an interface to encrypt and decrypt - text through the AES IGE mode. + Decrypts the given text in 16-bytes blocks by using the + given key and 32-bytes initialization vector. """ - @staticmethod - def decrypt_ige(cipher_text, key, iv): - """ - Decrypts the given text in 16-bytes blocks by using the - given key and 32-bytes initialization vector. - """ - iv1 = iv[:len(iv) // 2] - iv2 = iv[len(iv) // 2:] + if cryptg: + return cryptg.decrypt_ige(cipher_text, key, iv) - aes = pyaes.AES(key) + iv1 = iv[:len(iv) // 2] + iv2 = iv[len(iv) // 2:] - plain_text = [] - blocks_count = len(cipher_text) // 16 + aes = pyaes.AES(key) - cipher_text_block = [0] * 16 - for block_index in range(blocks_count): - for i in range(16): - cipher_text_block[i] = \ - cipher_text[block_index * 16 + i] ^ iv2[i] + plain_text = [] + blocks_count = len(cipher_text) // 16 - plain_text_block = aes.decrypt(cipher_text_block) + cipher_text_block = [0] * 16 + for block_index in range(blocks_count): + for i in range(16): + cipher_text_block[i] = \ + cipher_text[block_index * 16 + i] ^ iv2[i] - for i in range(16): - plain_text_block[i] ^= iv1[i] + plain_text_block = aes.decrypt(cipher_text_block) - iv1 = cipher_text[block_index * 16:block_index * 16 + 16] - iv2 = plain_text_block + for i in range(16): + plain_text_block[i] ^= iv1[i] - plain_text.extend(plain_text_block) + iv1 = cipher_text[block_index * 16:block_index * 16 + 16] + iv2 = plain_text_block - return bytes(plain_text) + plain_text.extend(plain_text_block) - @staticmethod - def encrypt_ige(plain_text, key, iv): - """ - Encrypts the given text in 16-bytes blocks by using the - given key and 32-bytes initialization vector. - """ + return bytes(plain_text) - # Add random padding iff it's not evenly divisible by 16 already - if len(plain_text) % 16 != 0: - padding_count = 16 - len(plain_text) % 16 - plain_text += os.urandom(padding_count) + @staticmethod + def encrypt_ige(plain_text, key, iv): + """ + Encrypts the given text in 16-bytes blocks by using the + given key and 32-bytes initialization vector. + """ + # Add random padding iff it's not evenly divisible by 16 already + if len(plain_text) % 16 != 0: + padding_count = 16 - len(plain_text) % 16 + plain_text += os.urandom(padding_count) - iv1 = iv[:len(iv) // 2] - iv2 = iv[len(iv) // 2:] + if cryptg: + return cryptg.encrypt_ige(plain_text, key, iv) - aes = pyaes.AES(key) + iv1 = iv[:len(iv) // 2] + iv2 = iv[len(iv) // 2:] - cipher_text = [] - blocks_count = len(plain_text) // 16 + aes = pyaes.AES(key) - for block_index in range(blocks_count): - plain_text_block = list( - plain_text[block_index * 16:block_index * 16 + 16] - ) - for i in range(16): - plain_text_block[i] ^= iv1[i] + cipher_text = [] + blocks_count = len(plain_text) // 16 - cipher_text_block = aes.encrypt(plain_text_block) + for block_index in range(blocks_count): + plain_text_block = list( + plain_text[block_index * 16:block_index * 16 + 16] + ) + for i in range(16): + plain_text_block[i] ^= iv1[i] - for i in range(16): - cipher_text_block[i] ^= iv2[i] + cipher_text_block = aes.encrypt(plain_text_block) - iv1 = cipher_text_block - iv2 = plain_text[block_index * 16:block_index * 16 + 16] + for i in range(16): + cipher_text_block[i] ^= iv2[i] - cipher_text.extend(cipher_text_block) + iv1 = cipher_text_block + iv2 = plain_text[block_index * 16:block_index * 16 + 16] - return bytes(cipher_text) + cipher_text.extend(cipher_text_block) + + return bytes(cipher_text) diff --git a/telethon/crypto/libssl.py b/telethon/crypto/libssl.py deleted file mode 100644 index b4735112..00000000 --- a/telethon/crypto/libssl.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -This module holds an AES IGE class, if libssl is available on the system. -""" -import os -import ctypes -from ctypes.util import find_library - -lib = find_library('ssl') -if not lib: - AES = None -else: - """ - # define AES_ENCRYPT 1 - # define AES_DECRYPT 0 - # define AES_MAXNR 14 - struct aes_key_st { - # ifdef AES_LONG - unsigned long rd_key[4 * (AES_MAXNR + 1)]; - # else - unsigned int rd_key[4 * (AES_MAXNR + 1)]; - # endif - int rounds; - }; - typedef struct aes_key_st AES_KEY; - - int AES_set_encrypt_key(const unsigned char *userKey, const int bits, - AES_KEY *key); - int AES_set_decrypt_key(const unsigned char *userKey, const int bits, - AES_KEY *key); - void AES_ige_encrypt(const unsigned char *in, unsigned char *out, - size_t length, const AES_KEY *key, - unsigned char *ivec, const int enc); - """ - _libssl = ctypes.cdll.LoadLibrary(lib) - - AES_MAXNR = 14 - AES_ENCRYPT = ctypes.c_int(1) - AES_DECRYPT = ctypes.c_int(0) - - class AES_KEY(ctypes.Structure): - """Helper class representing an AES key""" - _fields_ = [ - ('rd_key', ctypes.c_uint32 * (4*(AES_MAXNR + 1))), - ('rounds', ctypes.c_uint), - ] - - class AES: - """ - Class that servers as an interface to encrypt and decrypt - text through the AES IGE mode, using the system's libssl. - """ - @staticmethod - def decrypt_ige(cipher_text, key, iv): - """ - Decrypts the given text in 16-bytes blocks by using the - given key and 32-bytes initialization vector. - """ - aeskey = AES_KEY() - ckey = (ctypes.c_ubyte * len(key))(*key) - cklen = ctypes.c_int(len(key)*8) - cin = (ctypes.c_ubyte * len(cipher_text))(*cipher_text) - ctlen = ctypes.c_size_t(len(cipher_text)) - cout = (ctypes.c_ubyte * len(cipher_text))() - civ = (ctypes.c_ubyte * len(iv))(*iv) - - _libssl.AES_set_decrypt_key(ckey, cklen, ctypes.byref(aeskey)) - _libssl.AES_ige_encrypt( - ctypes.byref(cin), - ctypes.byref(cout), - ctlen, - ctypes.byref(aeskey), - ctypes.byref(civ), - AES_DECRYPT - ) - - return bytes(cout) - - @staticmethod - def encrypt_ige(plain_text, key, iv): - """ - Encrypts the given text in 16-bytes blocks by using the - given key and 32-bytes initialization vector. - """ - # Add random padding iff it's not evenly divisible by 16 already - if len(plain_text) % 16 != 0: - padding_count = 16 - len(plain_text) % 16 - plain_text += os.urandom(padding_count) - - aeskey = AES_KEY() - ckey = (ctypes.c_ubyte * len(key))(*key) - cklen = ctypes.c_int(len(key)*8) - cin = (ctypes.c_ubyte * len(plain_text))(*plain_text) - ctlen = ctypes.c_size_t(len(plain_text)) - cout = (ctypes.c_ubyte * len(plain_text))() - civ = (ctypes.c_ubyte * len(iv))(*iv) - - _libssl.AES_set_encrypt_key(ckey, cklen, ctypes.byref(aeskey)) - _libssl.AES_ige_encrypt( - ctypes.byref(cin), - ctypes.byref(cout), - ctlen, - ctypes.byref(aeskey), - ctypes.byref(civ), - AES_ENCRYPT - ) - - return bytes(cout)