From 33ce702ab933712e16432736038dc1be68430137 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 7 Jul 2018 11:46:21 +0200 Subject: [PATCH 01/19] Pre-pack outgoing TLMessage This has several benefits: - The message can be resent without re-calling bytes(), which for some requests may be expensive. - Converting requests to bytes early lets us detect errors early, such as OverflowError on bad requests. - Containers can't exceed 1044456 bytes so knowing their length is important. This can now be done in O(1). But also several drawbacks: - If the object is modified the bytes won't reflect this. This isn't an issue because it's only done for in msgs. - Incoming messages can no longer be reconverted into bytes but this was never needed anyway. --- telethon/network/mtprotosender.py | 6 ++-- telethon/network/mtprotostate.py | 3 +- telethon/tl/core/tlmessage.py | 55 ++++++++++++++++++++++++------- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/telethon/network/mtprotosender.py b/telethon/network/mtprotosender.py index 5eff89cf..38ace40b 100644 --- a/telethon/network/mtprotosender.py +++ b/telethon/network/mtprotosender.py @@ -507,7 +507,6 @@ class MTProtoSender: rpc_result.req_msg_id) if rpc_result.error: - # TODO Report errors if possible/enabled error = rpc_message_to_error(rpc_result.error) self._send_queue.put_nowait(self.state.create_message( MsgsAck([message.msg_id]) @@ -517,10 +516,13 @@ class MTProtoSender: message.future.set_exception(error) return elif message: + # TODO Would be nice to avoid accessing a per-obj read_result + # Instead have a variable that indicated how the result should + # be read (an enum) and dispatch to read the result, mostly + # always it's just a normal TLObject. with BinaryReader(rpc_result.body) as reader: result = message.obj.read_result(reader) - # TODO Process entities if not message.future.cancelled(): message.future.set_result(result) return diff --git a/telethon/network/mtprotostate.py b/telethon/network/mtprotostate.py index 4516c820..6ea13ae2 100644 --- a/telethon/network/mtprotostate.py +++ b/telethon/network/mtprotostate.py @@ -46,7 +46,8 @@ class MTProtoState: msg_id=self._get_new_msg_id(), seq_no=self._get_seq_no(isinstance(obj, TLRequest)), obj=obj, - after_id=after.msg_id if after else None + after_id=after.msg_id if after else None, + out=True # Pre-convert the request into bytes ) def update_message_id(self, message): diff --git a/telethon/tl/core/tlmessage.py b/telethon/tl/core/tlmessage.py index 50ab4ddf..d0345cb2 100644 --- a/telethon/tl/core/tlmessage.py +++ b/telethon/tl/core/tlmessage.py @@ -21,9 +21,7 @@ class TLMessage(TLObject): sent `TLMessage`, and this result can be represented as a `Future` that will eventually be set with either a result, error or cancelled. """ - def __init__(self, msg_id, seq_no, obj=None, after_id=0): - self.msg_id = msg_id - self.seq_no = seq_no + def __init__(self, msg_id, seq_no, obj, out=False, after_id=0): self.obj = obj self.container_msg_id = None self.future = asyncio.Future() @@ -31,23 +29,56 @@ class TLMessage(TLObject): # After which message ID this one should run. We do this so # InvokeAfterMsgRequest is transparent to the user and we can # easily invoke after while confirming the original request. + # TODO Currently we don't update this if another message ID changes self.after_id = after_id + # There are two use-cases for the TLMessage, outgoing and incoming. + # Outgoing messages are meant to be serialized and sent across the + # network so it makes sense to pack them as early as possible and + # avoid this computation if it needs to be resent, and also shows + # serializing-errors as early as possible (foreground task). + # + # We assume obj won't change so caching the bytes is safe to do. + # Caching bytes lets us get the size in a fast way, necessary for + # knowing whether a container can be sent (<1MB) or not (too big). + # + # Incoming messages don't really need this body, but we save the + # msg_id and seq_no inside the body for consistency and raise if + # one tries to bytes()-ify the entire message (len == 12). + if not out: + self._body = struct.pack(' Date: Sat, 7 Jul 2018 11:58:48 +0200 Subject: [PATCH 02/19] Avoid exceeding maximum container size This issue would likely be triggered when automatically merging multiple requests into a single one while having their size exceed 1044456 bytes like SaveFilePartRequest. This commit avoids such issue by keeping track of the current size, and if it exceeds the limit, avoid merge. --- telethon/network/mtprotosender.py | 7 +++++-- telethon/tl/core/messagecontainer.py | 5 +++++ telethon/tl/core/tlmessage.py | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/telethon/network/mtprotosender.py b/telethon/network/mtprotosender.py index 38ace40b..81deae64 100644 --- a/telethon/network/mtprotosender.py +++ b/telethon/network/mtprotosender.py @@ -749,14 +749,17 @@ class _ContainerQueue(asyncio.Queue): isinstance(result.obj, MessageContainer): return result + size = result.size() result = [result] while not self.empty(): item = self.get_nowait() - if item == _reconnect_sentinel or\ - isinstance(item.obj, MessageContainer): + if (item == _reconnect_sentinel or + isinstance(item.obj, MessageContainer) + or size + item.size() > MessageContainer.MAXIMUM_SIZE): self.put_nowait(item) break else: + size += item.size() result.append(item) return result diff --git a/telethon/tl/core/messagecontainer.py b/telethon/tl/core/messagecontainer.py index f5c3c378..fc36fd5e 100644 --- a/telethon/tl/core/messagecontainer.py +++ b/telethon/tl/core/messagecontainer.py @@ -10,6 +10,11 @@ __log__ = logging.getLogger(__name__) class MessageContainer(TLObject): CONSTRUCTOR_ID = 0x73f1f8dc + # Maximum size in bytes for the inner payload of the container. + # Telegram will close the connection if the payload is bigger. + # The overhead of the container itself is subtracted. + MAXIMUM_SIZE = 1044456 - 8 + def __init__(self, messages): self.messages = messages diff --git a/telethon/tl/core/tlmessage.py b/telethon/tl/core/tlmessage.py index d0345cb2..5d032c7c 100644 --- a/telethon/tl/core/tlmessage.py +++ b/telethon/tl/core/tlmessage.py @@ -82,3 +82,6 @@ class TLMessage(TLObject): raise TypeError('Incoming messages should not be bytes()-ed') return self._body + + def size(self): + return len(self._body) From dfda61a1b50d846e19593263e5b363d4cfbfe00f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 7 Jul 2018 12:01:42 +0200 Subject: [PATCH 03/19] Correct thumb parameter documentation --- telethon/client/uploads.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telethon/client/uploads.py b/telethon/client/uploads.py index 17b361ef..52cc8203 100644 --- a/telethon/client/uploads.py +++ b/telethon/client/uploads.py @@ -72,7 +72,8 @@ class UploadMethods(MessageParseMethods, UserMethods): :tl:`DocumentAttributeFilename` and so on. thumb (`str` | `bytes` | `file`, optional): - Optional thumbnail (for videos). + Optional JPEG thumbnail (for documents). **Telegram will + ignore this parameter** unless you pass a ``.jpg`` file! allow_cache (`bool`, optional): Whether to allow using the cached version stored in the From bd878acbdec09b12e4d962b2cdf0f9f22285f811 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 7 Jul 2018 12:14:03 +0200 Subject: [PATCH 04/19] Support InputNotifyPeer autocast --- telethon/client/users.py | 14 ++++++++++++++ telethon_generator/generators/tlobject.py | 1 + 2 files changed, 15 insertions(+) diff --git a/telethon/client/users.py b/telethon/client/users.py index 393069ed..e4bec48b 100644 --- a/telethon/client/users.py +++ b/telethon/client/users.py @@ -337,4 +337,18 @@ class UserMethods(TelegramBaseClient): 'Cannot find any entity corresponding to "{}"'.format(string) ) + async def _get_input_notify(self, notify): + """ + Returns a :tl:`InputNotifyPeer`. This is a bit tricky because + it may or not need access to the client to convert what's given + into an input entity. + """ + try: + if notify.SUBCLASS_OF_ID == 0x58981615: + if isinstance(notify, types.InputNotifyPeer): + notify.peer = await self.get_input_entity(notify.peer) + return notify + except AttributeError: + return types.InputNotifyPeer(await self.get_input_entity(notify)) + # endregion diff --git a/telethon_generator/generators/tlobject.py b/telethon_generator/generators/tlobject.py index 111b0c75..43571c27 100644 --- a/telethon_generator/generators/tlobject.py +++ b/telethon_generator/generators/tlobject.py @@ -23,6 +23,7 @@ AUTO_CASTS = { 'InputDialogPeer': 'utils.get_input_dialog(await client.get_input_entity({}))', + 'InputNotifyPeer': 'await client._get_input_notify({})', 'InputMedia': 'utils.get_input_media({})', 'InputPhoto': 'utils.get_input_photo({})', 'InputMessage': 'utils.get_input_message({})' From bb4ed4019ffaa8bd2afb16f7da7d14243001861d Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 7 Jul 2018 12:14:50 +0200 Subject: [PATCH 05/19] Revert forward_messages requiring named from_peer arg --- telethon/client/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/client/messages.py b/telethon/client/messages.py index 79d0db92..3149a8d6 100644 --- a/telethon/client/messages.py +++ b/telethon/client/messages.py @@ -450,7 +450,7 @@ class MessageMethods(UploadMethods, MessageParseMethods): return self._get_response_message(request, result, entity) - async def forward_messages(self, entity, messages, *, from_peer=None): + async def forward_messages(self, entity, messages, from_peer=None): """ Forwards the given message(s) to the specified entity. From 5d4b8a283d073631cbbc80fd2414898dee02079f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 7 Jul 2018 12:17:29 +0200 Subject: [PATCH 06/19] Don't generate def resolve() for types This would require nested resolving which could be quite expensive. Instead there will just be a single level for resolve() and it will belong in the requests. --- telethon_generator/generators/tlobject.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telethon_generator/generators/tlobject.py b/telethon_generator/generators/tlobject.py index 43571c27..e3107336 100644 --- a/telethon_generator/generators/tlobject.py +++ b/telethon_generator/generators/tlobject.py @@ -233,7 +233,8 @@ def _write_class_init(tlobject, kind, type_constructors, builder): def _write_resolve(tlobject, builder): - if any(arg.type in AUTO_CASTS for arg in tlobject.real_args): + if tlobject.is_function and\ + any(arg.type in AUTO_CASTS for arg in tlobject.real_args): builder.writeln('async def resolve(self, client, utils):') for arg in tlobject.real_args: ac = AUTO_CASTS.get(arg.type, None) From 3b3b148a4389feedbccaf6ce062e867143aaaf1c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 7 Jul 2018 12:44:05 +0200 Subject: [PATCH 07/19] Revert add_mark parameter on utils.get_peer_id And a fix for -1000 IDs that wasn't being accounted for. --- telethon/utils.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/telethon/utils.py b/telethon/utils.py index 13dbfb91..18aab14d 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -575,17 +575,6 @@ def parse_username(username): return None, False -def _fix_peer_id(peer_id): - """ - Fixes the peer ID for chats and channels, in case the users - mix marking the ID with the :tl:`Peer` constructors. - """ - peer_id = abs(peer_id) - if str(peer_id).startswith('100'): - peer_id = str(peer_id)[3:] - return int(peer_id) - - def get_inner_text(text, entities): """ Gets the inner text that's surrounded by the given entities. @@ -605,7 +594,7 @@ def get_inner_text(text, entities): return result -def get_peer_id(peer): +def get_peer_id(peer, add_mark=True): """ Finds the ID of the given peer, and converts it to the "bot api" format so it the peer can be identified back. User ID is left unmodified, @@ -616,7 +605,7 @@ def get_peer_id(peer): """ # First we assert it's a Peer TLObject, or early return for integers if isinstance(peer, int): - return peer + return peer if add_mark else resolve_id(peer)[0] try: if peer.SUBCLASS_OF_ID not in (0x2d45687, 0xc91c90b6): @@ -634,9 +623,9 @@ def get_peer_id(peer): elif isinstance(peer, (PeerChat, InputPeerChat)): # Check in case the user mixed things up to avoid blowing up if not (0 < peer.chat_id <= 0x7fffffff): - peer.chat_id = _fix_peer_id(peer.chat_id) + peer.chat_id = resolve_id(peer.chat_id)[0] - return -peer.chat_id + return -peer.chat_id if add_mark else peer.chat_id elif isinstance(peer, (PeerChannel, InputPeerChannel, ChannelFull)): if isinstance(peer, ChannelFull): # Special case: .get_input_peer can't return InputChannel from @@ -647,15 +636,18 @@ def get_peer_id(peer): # Check in case the user mixed things up to avoid blowing up if not (0 < i <= 0x7fffffff): - i = _fix_peer_id(i) + i = resolve_id(i)[0] if isinstance(peer, ChannelFull): peer.id = i else: peer.channel_id = i - # Concat -100 through math tricks, .to_supergroup() on Madeline - # IDs will be strictly positive -> log works - return -(i + pow(10, math.floor(math.log10(i) + 3))) + if add_mark: + # Concat -100 through math tricks, .to_supergroup() on + # Madeline IDs will be strictly positive -> log works. + return -(i + pow(10, math.floor(math.log10(i) + 3))) + else: + return i _raise_cast_fail(peer, 'int') From cb3846cb7f29997a6b144c7d9deb37c6d802cf09 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 7 Jul 2018 12:45:50 +0200 Subject: [PATCH 08/19] Add client.get_peer_id --- telethon/client/users.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/telethon/client/users.py b/telethon/client/users.py index e4bec48b..413657b2 100644 --- a/telethon/client/users.py +++ b/telethon/client/users.py @@ -276,6 +276,32 @@ class UserMethods(TelegramBaseClient): .format(peer) ) + async def get_peer_id(self, peer, add_mark=True): + """ + Gets the ID for the given peer, which may be anything entity-like. + + This method needs to be ``async`` because `peer` supports usernames, + invite-links, phone numbers, etc. + + If ``add_mark is False``, then a positive ID will be returned + instead. By default, bot-API style IDs (signed) are returned. + """ + if isinstance(peer, int): + return utils.get_peer_id(peer, add_mark=add_mark) + + try: + if peer.SUBCLASS_OF_ID in (0x2d45687, 0xc91c90b6): + # 0x2d45687, 0xc91c90b6 == crc32(b'Peer') and b'InputPeer' + return utils.get_peer_id(peer) + except AttributeError: + pass + + peer = await self.get_input_entity(peer) + if isinstance(peer, types.InputPeerSelf): + peer = await self.get_me(input_peer=True) + + return utils.get_peer_id(peer, add_mark=add_mark) + # endregion # region Private methods From 066004acd1cc1dcca312faba5aefac6b9185b6d0 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 7 Jul 2018 12:53:46 +0200 Subject: [PATCH 09/19] Support chat_id autocast --- telethon_generator/generators/tlobject.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/telethon_generator/generators/tlobject.py b/telethon_generator/generators/tlobject.py index e3107336..52affb66 100644 --- a/telethon_generator/generators/tlobject.py +++ b/telethon_generator/generators/tlobject.py @@ -29,6 +29,10 @@ AUTO_CASTS = { 'InputMessage': 'utils.get_input_message({})' } +NAMED_AUTO_CASTS = { + ('chat_id', 'int'): 'await client.get_peer_id({})' +} + BASE_TYPES = ('string', 'bytes', 'int', 'long', 'int128', 'int256', 'double', 'Bool', 'true', 'date') @@ -233,13 +237,18 @@ def _write_class_init(tlobject, kind, type_constructors, builder): def _write_resolve(tlobject, builder): - if tlobject.is_function and\ - any(arg.type in AUTO_CASTS for arg in tlobject.real_args): + if tlobject.is_function and any( + (arg.type in AUTO_CASTS + or (arg.name, arg.type in NAMED_AUTO_CASTS)) + for arg in tlobject.real_args + ): builder.writeln('async def resolve(self, client, utils):') for arg in tlobject.real_args: - ac = AUTO_CASTS.get(arg.type, None) + ac = AUTO_CASTS.get(arg.type) if not ac: - continue + ac = NAMED_AUTO_CASTS.get((arg.name, arg.type)) + if not ac: + continue if arg.is_flag: builder.writeln('if self.{}:', arg.name) From 61f9dc1cd7fe8f21e27a96ae40da9203ea305692 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 7 Jul 2018 13:03:46 +0200 Subject: [PATCH 10/19] Fix-up missing parenthesis from 066004a --- telethon_generator/generators/tlobject.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon_generator/generators/tlobject.py b/telethon_generator/generators/tlobject.py index 52affb66..6b371486 100644 --- a/telethon_generator/generators/tlobject.py +++ b/telethon_generator/generators/tlobject.py @@ -239,7 +239,7 @@ def _write_class_init(tlobject, kind, type_constructors, builder): def _write_resolve(tlobject, builder): if tlobject.is_function and any( (arg.type in AUTO_CASTS - or (arg.name, arg.type in NAMED_AUTO_CASTS)) + or ((arg.name, arg.type) in NAMED_AUTO_CASTS)) for arg in tlobject.real_args ): builder.writeln('async def resolve(self, client, utils):') From 8ca2e56aeec2de95d3f744d83c6e8ffa7553dca2 Mon Sep 17 00:00:00 2001 From: Lonami Date: Sat, 7 Jul 2018 19:42:47 +0200 Subject: [PATCH 11/19] Fix ID autocast should not add the mark --- telethon_generator/generators/tlobject.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon_generator/generators/tlobject.py b/telethon_generator/generators/tlobject.py index 6b371486..5190c6f8 100644 --- a/telethon_generator/generators/tlobject.py +++ b/telethon_generator/generators/tlobject.py @@ -30,7 +30,7 @@ AUTO_CASTS = { } NAMED_AUTO_CASTS = { - ('chat_id', 'int'): 'await client.get_peer_id({})' + ('chat_id', 'int'): 'await client.get_peer_id({}, add_mark=False)' } BASE_TYPES = ('string', 'bytes', 'int', 'long', 'int128', From 54bffb30d8adcfaf34ec1230cda368c5e0805b1c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 8 Jul 2018 00:01:18 +0200 Subject: [PATCH 12/19] Fix send_message('me', Message) --- telethon/client/messages.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/telethon/client/messages.py b/telethon/client/messages.py index 3149a8d6..5aaf8d12 100644 --- a/telethon/client/messages.py +++ b/telethon/client/messages.py @@ -407,10 +407,17 @@ class MessageMethods(UploadMethods, MessageParseMethods): if reply_to is not None: reply_id = utils.get_message_id(reply_to) - elif utils.get_peer_id(entity) == utils.get_peer_id(message.to_id): - reply_id = message.reply_to_msg_id else: - reply_id = None + if isinstance(entity, types.InputPeerSelf): + eid = utils.get_peer_id(await self.get_me(input_peer=True)) + else: + eid = utils.get_peer_id(entity) + + if eid == utils.get_peer_id(message.to_id): + reply_id = message.reply_to_msg_id + else: + reply_id = None + request = functions.messages.SendMessageRequest( peer=entity, message=message.message or '', From d02cb84abe2be4df4adfa72043a1fb80aa486690 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 8 Jul 2018 00:04:50 +0200 Subject: [PATCH 13/19] Fix end of sync with block warning on disconnect --- telethon/client/auth.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/telethon/client/auth.py b/telethon/client/auth.py index 25c0444c..90d3da05 100644 --- a/telethon/client/auth.py +++ b/telethon/client/auth.py @@ -1,5 +1,6 @@ import getpass import hashlib +import inspect import os import sys @@ -469,7 +470,12 @@ class AuthMethods(MessageParseMethods, UserMethods): return await self.start() def __exit__(self, *args): - self.disconnect() + if self._loop.is_running(): + self._loop.create_task(self.disconnect()) + elif inspect.iscoroutinefunction(self.disconnect): + self._loop.run_until_complete(self.disconnect()) + else: + self.disconnect() async def __aexit__(self, *args): await self.disconnect() From 3a9cce8720ee19fb13a3eec99e8557dc2b768af5 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 8 Jul 2018 15:11:10 +0200 Subject: [PATCH 14/19] Add missing async/await to events' documentation --- readthedocs/extra/basic/getting-started.rst | 4 ++-- telethon/events/__init__.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/readthedocs/extra/basic/getting-started.rst b/readthedocs/extra/basic/getting-started.rst index a3dec33d..ddfa9fbe 100644 --- a/readthedocs/extra/basic/getting-started.rst +++ b/readthedocs/extra/basic/getting-started.rst @@ -79,8 +79,8 @@ Handling Updates from telethon import events @client.on(events.NewMessage(incoming=True, pattern='(?i)hi')) - def handler(event): - event.reply('Hello!') + async def handler(event): + await event.reply('Hello!') client.run_until_disconnected() diff --git a/telethon/events/__init__.py b/telethon/events/__init__.py index 7614dbfe..08ef1701 100644 --- a/telethon/events/__init__.py +++ b/telethon/events/__init__.py @@ -18,13 +18,13 @@ class StopPropagation(Exception): >>> client = TelegramClient(...) >>> >>> @client.on(events.NewMessage) - ... def delete(event): - ... event.delete() + ... async def delete(event): + ... await event.delete() ... # No other event handler will have a chance to handle this event ... raise StopPropagation ... >>> @client.on(events.NewMessage) - ... def _(event): + ... async def _(event): ... # Will never be reached, because it is the second handler ... pass """ From e0513e10df1d199889c4d579c6c5b7af36cb4c66 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 8 Jul 2018 15:24:33 +0200 Subject: [PATCH 15/19] Remove debug print --- telethon/client/messages.py | 1 - 1 file changed, 1 deletion(-) diff --git a/telethon/client/messages.py b/telethon/client/messages.py index 5aaf8d12..49d7a369 100644 --- a/telethon/client/messages.py +++ b/telethon/client/messages.py @@ -139,7 +139,6 @@ class MessageMethods(UploadMethods, MessageParseMethods): offset_id = max(offset_id, min_id) if offset_id and max_id: if max_id - offset_id <= 1: - print('suck lol') return if not max_id: From 128053750d6cf504c8aa368cfddff36dfa3aef46 Mon Sep 17 00:00:00 2001 From: Lonami Date: Sun, 8 Jul 2018 17:45:49 +0200 Subject: [PATCH 16/19] Implement HTTP(S) mode (closes #112) (#883) --- telethon/extensions/tcpclient.py | 9 +++- telethon/network/connection/__init__.py | 1 + telethon/network/connection/http.py | 62 +++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 telethon/network/connection/http.py diff --git a/telethon/extensions/tcpclient.py b/telethon/extensions/tcpclient.py index ace0d38c..2b9ee03d 100644 --- a/telethon/extensions/tcpclient.py +++ b/telethon/extensions/tcpclient.py @@ -11,6 +11,7 @@ import asyncio import errno import logging import socket +import ssl from io import BytesIO CONN_RESET_ERRNOS = { @@ -28,6 +29,7 @@ try: except ImportError: socks = None +SSL_PORT = 443 __log__ = logging.getLogger(__name__) @@ -37,15 +39,18 @@ class TcpClient: class SocketClosed(ConnectionError): pass - def __init__(self, *, loop, timeout, proxy=None): + def __init__(self, *, loop, timeout, ssl=None, proxy=None): """ Initializes the TCP client. :param proxy: the proxy to be used, if any. :param timeout: the timeout for connect, read and write operations. + :param ssl: ssl.wrap_socket keyword arguments to use when connecting + if port == SSL_PORT, or do nothing if not present. """ self._loop = loop self.proxy = proxy + self.ssl = ssl self._socket = None self._closed = asyncio.Event(loop=self._loop) self._closed.set() @@ -87,6 +92,8 @@ class TcpClient: try: if self._socket is None: self._socket = self._create_socket(mode, self.proxy) + if self.ssl and port == SSL_PORT: + self._socket = ssl.wrap_socket(self._socket, **self.ssl) await asyncio.wait_for( self._loop.sock_connect(self._socket, address), diff --git a/telethon/network/connection/__init__.py b/telethon/network/connection/__init__.py index 0c7a07d0..262aaa3a 100644 --- a/telethon/network/connection/__init__.py +++ b/telethon/network/connection/__init__.py @@ -2,3 +2,4 @@ from .tcpfull import ConnectionTcpFull from .tcpabridged import ConnectionTcpAbridged from .tcpobfuscated import ConnectionTcpObfuscated from .tcpintermediate import ConnectionTcpIntermediate +from .http import ConnectionHttp diff --git a/telethon/network/connection/http.py b/telethon/network/connection/http.py new file mode 100644 index 00000000..955b9ab3 --- /dev/null +++ b/telethon/network/connection/http.py @@ -0,0 +1,62 @@ +import errno +import ssl + +from .common import Connection +from ...extensions import TcpClient + + +class ConnectionHttp(Connection): + def __init__(self, *, loop, timeout, proxy=None): + super().__init__(loop=loop, timeout=timeout, proxy=proxy) + self.conn = TcpClient( + timeout=self._timeout, loop=self._loop, proxy=self._proxy, + ssl=dict(ssl_version=ssl.PROTOCOL_SSLv23, ciphers='ADH-AES256-SHA') + ) + self.read = self.conn.read + self.write = self.conn.write + self._host = None + + async def connect(self, ip, port): + self._host = '{}:{}'.format(ip, port) + try: + await self.conn.connect(ip, port) + except OSError as e: + if e.errno == errno.EISCONN: + return # Already connected, no need to re-set everything up + else: + raise + + def get_timeout(self): + return self.conn.timeout + + def is_connected(self): + return self.conn.is_connected + + async def close(self): + self.conn.close() + + async def recv(self): + while True: + line = await self._read_line() + if line.lower().startswith(b'content-length: '): + await self.read(2) + length = int(line[16:-2]) + return await self.read(length) + + async def _read_line(self): + newline = ord('\n') + line = await self.read(1) + while line[-1] != newline: + line += await self.read(1) + return line + + async def send(self, message): + await self.write( + 'POST /api HTTP/1.1\r\n' + 'Host: {}\r\n' + 'Content-Type: application/x-www-form-urlencoded\r\n' + 'Connection: keep-alive\r\n' + 'Keep-Alive: timeout=100000, max=10000000\r\n' + 'Content-Length: {}\r\n\r\n'.format(self._host, len(message)) + .encode('ascii') + message + ) From 7b6e65a7a50460514ca138b279d6e96e91cc0fb2 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 8 Jul 2018 17:48:27 +0200 Subject: [PATCH 17/19] Re-export ConnectionHttp from the network package --- telethon/network/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/network/__init__.py b/telethon/network/__init__.py index f4bd72d0..e8070b21 100644 --- a/telethon/network/__init__.py +++ b/telethon/network/__init__.py @@ -7,5 +7,5 @@ from .authenticator import do_authentication from .mtprotosender import MTProtoSender from .connection import ( ConnectionTcpFull, ConnectionTcpAbridged, ConnectionTcpObfuscated, - ConnectionTcpIntermediate + ConnectionTcpIntermediate, ConnectionHttp ) From 1b22d0eb12134d3206698997c67fe106fd1df67e Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 8 Jul 2018 23:44:56 +0200 Subject: [PATCH 18/19] Add missing await --- telethon/tl/custom/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/tl/custom/message.py b/telethon/tl/custom/message.py index 64c83121..40236c6c 100644 --- a/telethon/tl/custom/message.py +++ b/telethon/tl/custom/message.py @@ -604,7 +604,7 @@ class Message: if not self.original_message.out: if not isinstance(self.original_message.to_id, types.PeerUser): return None - me = self._client.get_me(input_peer=True) + me = await self._client.get_me(input_peer=True) if self.original_message.to_id.user_id != me.user_id: return None From 1437b69829894dbeb11fd1d8516d00bb53a2629b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 9 Jul 2018 12:32:47 +0200 Subject: [PATCH 19/19] Update to v1.0.4 --- readthedocs/extra/changelog.rst | 52 +++++++++++++++++++++++++++++++++ telethon/version.py | 2 +- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/readthedocs/extra/changelog.rst b/readthedocs/extra/changelog.rst index e0d01b7d..fe0b50a2 100644 --- a/readthedocs/extra/changelog.rst +++ b/readthedocs/extra/changelog.rst @@ -14,6 +14,58 @@ it can take advantage of new goodies! .. contents:: List of All Versions +New HTTP(S) Connection Mode (v1.0.4) +==================================== + +*Published at 2018/07/09* + +This release implements the HTTP connection mode to the library, which +means certain proxies that only allow HTTP connections should now work +properly. You can use it doing the following, like any other mode: + +.. code-block:: python + + from telethon import TelegramClient, sync + from telethon.network import ConnectionHttp + + client = TelegramClient(..., connection=ConnectionHttp) + with client: + client.send_message('me', 'Hi!') + + +Additions +~~~~~~~~~ + +- ``add_mark=`` is now back on ``utils.get_input_peer`` and also on + `client.get_input_entity `. +- New `client.get_peer_id ` + convenience for ``utils.get_peer_id(await client.get_input_entity(peer))``. + + +Bug fixes +~~~~~~~~~ + +- If several `TLMessage` in a `MessageContainer` exceeds 1MB, it will no + longer be automatically turned into one. This basically means that e.g. + uploading 10 file parts at once will work properly again. +- Documentation fixes and some missing ``await``. +- Revert named argument for `client.forward_messages + ` + +Enhancements +~~~~~~~~~~~~ + +- New auto-casts to :tl:`InputNotifyPeer` and ``chat_id``. + +Internal changes +~~~~~~~~~~~~~~~~ + +- Outgoing `TLMessage` are now pre-packed so if there's an error when + serializing the raw requests, the library will no longer swallow it. + This also means re-sending packets doesn't need to re-pack their bytes. + + + Iterate Messages in Reverse (v1.0.3) ==================================== diff --git a/telethon/version.py b/telethon/version.py index eb620e48..85dfd0b3 100644 --- a/telethon/version.py +++ b/telethon/version.py @@ -1,3 +1,3 @@ # Versions should comply with PEP440. # This line is parsed in setup.py: -__version__ = '1.0.3' +__version__ = '1.0.4'