Merge branch 'master' into sync

telethon/client/auth.py

@@ -1,5 +1,6 @@
 import getpass
 import hashlib
+import inspect
 import os
 import sys
 

telethon/client/messages.py

@@ -136,7 +136,6 @@ class MessageMethods(UploadMethods, MessageParseMethods):
             offset_id = max(offset_id, min_id)
             if offset_id and max_id:
                 if max_id - offset_id <= 1:
-                    print('suck lol')
                     return
 
             if not max_id:
@@ -404,10 +403,17 @@ class MessageMethods(UploadMethods, MessageParseMethods):
 
             if reply_to is not None:
                 reply_id = utils.get_message_id(reply_to)
-            elif utils.get_peer_id(entity) == utils.get_peer_id(message.to_id):
-                reply_id = message.reply_to_msg_id
             else:
-                reply_id = None
+                if isinstance(entity, types.InputPeerSelf):
+                    eid = utils.get_peer_id(self.get_me(input_peer=True))
+                else:
+                    eid = utils.get_peer_id(entity)
+
+                if eid == utils.get_peer_id(message.to_id):
+                    reply_id = message.reply_to_msg_id
+                else:
+                    reply_id = None
+
             request = functions.messages.SendMessageRequest(
                 peer=entity,
                 message=message.message or '',
@@ -447,7 +453,7 @@ class MessageMethods(UploadMethods, MessageParseMethods):
 
         return self._get_response_message(request, result, entity)
 
-    def forward_messages(self, entity, messages, *, from_peer=None):
+    def forward_messages(self, entity, messages, from_peer=None):
         """
         Forwards the given message(s) to the specified entity.
 

telethon/client/uploads.py

@@ -72,7 +72,8 @@ class UploadMethods(MessageParseMethods, UserMethods):
                 :tl:`DocumentAttributeFilename` and so on.
 
             thumb (`str` | `bytes` | `file`, optional):
-                Optional thumbnail (for videos).
+                Optional JPEG thumbnail (for documents). **Telegram will
+                ignore this parameter** unless you pass a ``.jpg`` file!
 
             allow_cache (`bool`, optional):
                 Whether to allow using the cached version stored in the

telethon/client/users.py

@@ -273,6 +273,32 @@ class UserMethods(TelegramBaseClient):
             .format(peer)
         )
 
+    def get_peer_id(self, peer, add_mark=True):
+        """
+        Gets the ID for the given peer, which may be anything entity-like.
+
+        This method needs to be ``async`` because `peer` supports usernames,
+        invite-links, phone numbers, etc.
+
+        If ``add_mark is False``, then a positive ID will be returned
+        instead. By default, bot-API style IDs (signed) are returned.
+        """
+        if isinstance(peer, int):
+            return utils.get_peer_id(peer, add_mark=add_mark)
+
+        try:
+            if peer.SUBCLASS_OF_ID in (0x2d45687, 0xc91c90b6):
+                # 0x2d45687, 0xc91c90b6 == crc32(b'Peer') and b'InputPeer'
+                return utils.get_peer_id(peer)
+        except AttributeError:
+            pass
+
+        peer = self.get_input_entity(peer)
+        if isinstance(peer, types.InputPeerSelf):
+            peer = self.get_me(input_peer=True)
+
+        return utils.get_peer_id(peer, add_mark=add_mark)
+
     # endregion
 
     # region Private methods
@@ -334,4 +360,18 @@ class UserMethods(TelegramBaseClient):
             'Cannot find any entity corresponding to "{}"'.format(string)
         )
 
+    def _get_input_notify(self, notify):
+        """
+        Returns a :tl:`InputNotifyPeer`. This is a bit tricky because
+        it may or not need access to the client to convert what's given
+        into an input entity.
+        """
+        try:
+            if notify.SUBCLASS_OF_ID == 0x58981615:
+                if isinstance(notify, types.InputNotifyPeer):
+                    notify.peer = self.get_input_entity(notify.peer)
+                return notify
+        except AttributeError:
+            return types.InputNotifyPeer(self.get_input_entity(notify))
+
     # endregion

telethon/events/__init__.py

@@ -18,13 +18,13 @@ class StopPropagation(Exception):
         >>> client = TelegramClient(...)
         >>>
         >>> @client.on(events.NewMessage)
-        ... def delete(event):
+        ... async def delete(event):
-        ...     event.delete()
+        ...     await event.delete()
         ...     # No other event handler will have a chance to handle this event
         ...     raise StopPropagation
         ...
         >>> @client.on(events.NewMessage)
-        ... def _(event):
+        ... async def _(event):
         ...     # Will never be reached, because it is the second handler
         ...     pass
     """

telethon/extensions/tcpclient.py

@@ -10,6 +10,7 @@ any sort, nor any other kind of errors such as connecting twice.
 import errno
 import logging
 import socket
+import ssl
 import threading
 from io import BytesIO
 
@@ -28,6 +29,7 @@ try:
 except ImportError:
     socks = None
 
+SSL_PORT = 443
 __log__ = logging.getLogger(__name__)
 
 
@@ -37,14 +39,17 @@ class TcpClient:
     class SocketClosed(ConnectionError):
         pass
 
-    def __init__(self, *, timeout, proxy=None):
+    def __init__(self, *, timeout, ssl=None, proxy=None):
         """
         Initializes the TCP client.
 
         :param proxy: the proxy to be used, if any.
         :param timeout: the timeout for connect, read and write operations.
+        :param ssl: ssl.wrap_socket keyword arguments to use when connecting
+                    if port == SSL_PORT, or do nothing if not present.
         """
         self.proxy = proxy
+        self.ssl = ssl
         self._socket = None
 
         self._closed = threading.Event()
@@ -87,6 +92,8 @@ class TcpClient:
         try:
             if self._socket is None:
                 self._socket = self._create_socket(mode, self.proxy)
+                if self.ssl and port == SSL_PORT:
+                    self._socket = ssl.wrap_socket(self._socket, **self.ssl)
 
             self._socket.settimeout(self.timeout)
             self._socket.connect(address)

telethon/network/__init__.py

@@ -7,5 +7,5 @@ from .authenticator import do_authentication
 from .mtprotosender import MTProtoSender
 from .connection import (
     ConnectionTcpFull, ConnectionTcpAbridged, ConnectionTcpObfuscated,
-    ConnectionTcpIntermediate
+    ConnectionTcpIntermediate, ConnectionHttp
 )

telethon/network/connection/__init__.py

@@ -2,3 +2,4 @@ from .tcpfull import ConnectionTcpFull
 from .tcpabridged import ConnectionTcpAbridged
 from .tcpobfuscated import ConnectionTcpObfuscated
 from .tcpintermediate import ConnectionTcpIntermediate
+from .http import ConnectionHttp

telethon/network/connection/http.py

@@ -0,0 +1,62 @@
+import errno
+import ssl
+
+from .common import Connection
+from ...extensions import TcpClient
+
+
+class ConnectionHttp(Connection):
+    def __init__(self, *, loop, timeout, proxy=None):
+        super().__init__(loop=loop, timeout=timeout, proxy=proxy)
+        self.conn = TcpClient(
+            timeout=self._timeout, loop=self._loop, proxy=self._proxy,
+            ssl=dict(ssl_version=ssl.PROTOCOL_SSLv23, ciphers='ADH-AES256-SHA')
+        )
+        self.read = self.conn.read
+        self.write = self.conn.write
+        self._host = None
+
+    async def connect(self, ip, port):
+        self._host = '{}:{}'.format(ip, port)
+        try:
+            await self.conn.connect(ip, port)
+        except OSError as e:
+            if e.errno == errno.EISCONN:
+                return  # Already connected, no need to re-set everything up
+            else:
+                raise
+
+    def get_timeout(self):
+        return self.conn.timeout
+
+    def is_connected(self):
+        return self.conn.is_connected
+
+    async def close(self):
+        self.conn.close()
+
+    async def recv(self):
+        while True:
+            line = await self._read_line()
+            if line.lower().startswith(b'content-length: '):
+                await self.read(2)
+                length = int(line[16:-2])
+                return await self.read(length)
+
+    async def _read_line(self):
+        newline = ord('\n')
+        line = await self.read(1)
+        while line[-1] != newline:
+            line += await self.read(1)
+        return line
+
+    async def send(self, message):
+        await self.write(
+            'POST /api HTTP/1.1\r\n'
+            'Host: {}\r\n'
+            'Content-Type: application/x-www-form-urlencoded\r\n'
+            'Connection: keep-alive\r\n'
+            'Keep-Alive: timeout=100000, max=10000000\r\n'
+            'Content-Length: {}\r\n\r\n'.format(self._host, len(message))
+            .encode('ascii') + message
+        )

telethon/network/mtprotosender.py

@@ -513,7 +513,6 @@ class MTProtoSender:
                       rpc_result.req_msg_id)
 
         if rpc_result.error:
-            # TODO Report errors if possible/enabled
             error = rpc_message_to_error(rpc_result.error)
             self._send_queue.put_nowait(self.state.create_message(
                 MsgsAck([message.msg_id])
@@ -523,10 +522,13 @@ class MTProtoSender:
                 message.future.set_exception(error)
             return
         elif message:
+            # TODO Would be nice to avoid accessing a per-obj read_result
+            # Instead have a variable that indicated how the result should
+            # be read (an enum) and dispatch to read the result, mostly
+            # always it's just a normal TLObject.
             with BinaryReader(rpc_result.body) as reader:
                 result = message.obj.read_result(reader)
 
-            # TODO Process entities
             if not message.future.cancelled():
                 message.future.set_result(result)
             return
@@ -753,6 +755,7 @@ class _ContainerQueue(queue.Queue):
                 isinstance(result.obj, MessageContainer):
             return result
 
+        size = result.size()
         result = [result]
         while not self.empty():
             # TODO Is this a bug in Python? For some reason get_nowait()
@@ -773,11 +776,13 @@ class _ContainerQueue(queue.Queue):
             if not isinstance(items, list):
                 items = [items]
             for item in items:
-                if item == _reconnect_sentinel or\
+                if (item == _reconnect_sentinel or
-                        isinstance(item.obj, MessageContainer):
+                    isinstance(item.obj, MessageContainer)
+                        or size + item.size() > MessageContainer.MAXIMUM_SIZE):
                     self.put_nowait(item)
-                    break
+                    return result  # break 2 levels
                 else:
+                    size += item.size()
                     result.append(item)
 
         return result

telethon/network/mtprotostate.py

@@ -46,7 +46,8 @@ class MTProtoState:
             msg_id=self._get_new_msg_id(),
             seq_no=self._get_seq_no(isinstance(obj, TLRequest)),
             obj=obj,
-            after_id=after.msg_id if after else None
+            after_id=after.msg_id if after else None,
+            out=True  # Pre-convert the request into bytes
         )
 
     def update_message_id(self, message):

telethon/tl/core/messagecontainer.py

@@ -10,6 +10,11 @@ __log__ = logging.getLogger(__name__)
 class MessageContainer(TLObject):
     CONSTRUCTOR_ID = 0x73f1f8dc
 
+    # Maximum size in bytes for the inner payload of the container.
+    # Telegram will close the connection if the payload is bigger.
+    # The overhead of the container itself is subtracted.
+    MAXIMUM_SIZE = 1044456 - 8
+
     def __init__(self, messages):
         self.messages = messages
 

telethon/tl/core/tlmessage.py

@@ -21,9 +21,7 @@ class TLMessage(TLObject):
     sent `TLMessage`, and this result can be represented as a `Future`
     that will eventually be set with either a result, error or cancelled.
     """
-    def __init__(self, msg_id, seq_no, obj=None, after_id=0):
+    def __init__(self, msg_id, seq_no, obj, out=False, after_id=0):
-        self.msg_id = msg_id
-        self.seq_no = seq_no
         self.obj = obj
         self.container_msg_id = None
         self.future = concurrent.futures.Future()
@@ -31,23 +29,59 @@ class TLMessage(TLObject):
         # After which message ID this one should run. We do this so
         # InvokeAfterMsgRequest is transparent to the user and we can
         # easily invoke after while confirming the original request.
+        # TODO Currently we don't update this if another message ID changes
         self.after_id = after_id
 
+        # There are two use-cases for the TLMessage, outgoing and incoming.
+        # Outgoing messages are meant to be serialized and sent across the
+        # network so it makes sense to pack them as early as possible and
+        # avoid this computation if it needs to be resent, and also shows
+        # serializing-errors as early as possible (foreground task).
+        #
+        # We assume obj won't change so caching the bytes is safe to do.
+        # Caching bytes lets us get the size in a fast way, necessary for
+        # knowing whether a container can be sent (<1MB) or not (too big).
+        #
+        # Incoming messages don't really need this body, but we save the
+        # msg_id and seq_no inside the body for consistency and raise if
+        # one tries to bytes()-ify the entire message (len == 12).
+        if not out:
+            self._body = struct.pack('<qi', msg_id, seq_no)
+        else:
+            if self.after_id is None:
+                body = GzipPacked.gzip_if_smaller(self.obj)
+            else:
+                body = GzipPacked.gzip_if_smaller(
+                    InvokeAfterMsgRequest(self.after_id, self.obj))
+
+            self._body = struct.pack('<qii', msg_id, seq_no, len(body)) + body
+
     def to_dict(self):
         return {
             '_': 'TLMessage',
             'msg_id': self.msg_id,
             'seq_no': self.seq_no,
             'obj': self.obj,
-            'container_msg_id': self.container_msg_id,
+            'container_msg_id': self.container_msg_id
-            'after_id': self.after_id
         }
 
-    def __bytes__(self):
+    @property
-        if self.after_id is None:
+    def msg_id(self):
-            body = GzipPacked.gzip_if_smaller(self.obj)
+        return struct.unpack('<q', self._body[:8])[0]
-        else:
-            body = GzipPacked.gzip_if_smaller(
-                InvokeAfterMsgRequest(self.after_id, self.obj))
 
-        return struct.pack('<qii', self.msg_id, self.seq_no, len(body)) + body
+    @msg_id.setter
+    def msg_id(self, value):
+        self._body = struct.pack('<q', value) + self._body[8:]
+
+    @property
+    def seq_no(self):
+        return struct.unpack('<i', self._body[8:12])[0]
+
+    def __bytes__(self):
+        if len(self._body) == 12:  # msg_id, seqno
+            raise TypeError('Incoming messages should not be bytes()-ed')
+
+        return self._body
+
+    def size(self):
+        return len(self._body)

telethon/utils.py

@@ -575,17 +575,6 @@ def parse_username(username):
         return None, False
 
 
-def _fix_peer_id(peer_id):
-    """
-    Fixes the peer ID for chats and channels, in case the users
-    mix marking the ID with the :tl:`Peer` constructors.
-    """
-    peer_id = abs(peer_id)
-    if str(peer_id).startswith('100'):
-        peer_id = str(peer_id)[3:]
-    return int(peer_id)
-
-
 def get_inner_text(text, entities):
     """
     Gets the inner text that's surrounded by the given entities.
@@ -605,7 +594,7 @@ def get_inner_text(text, entities):
     return result
 
 
-def get_peer_id(peer):
+def get_peer_id(peer, add_mark=True):
     """
     Finds the ID of the given peer, and converts it to the "bot api" format
     so it the peer can be identified back. User ID is left unmodified,
@@ -616,7 +605,7 @@ def get_peer_id(peer):
     """
     # First we assert it's a Peer TLObject, or early return for integers
     if isinstance(peer, int):
-        return peer
+        return peer if add_mark else resolve_id(peer)[0]
 
     try:
         if peer.SUBCLASS_OF_ID not in (0x2d45687, 0xc91c90b6):
@@ -634,9 +623,9 @@ def get_peer_id(peer):
     elif isinstance(peer, (PeerChat, InputPeerChat)):
         # Check in case the user mixed things up to avoid blowing up
         if not (0 < peer.chat_id <= 0x7fffffff):
-            peer.chat_id = _fix_peer_id(peer.chat_id)
+            peer.chat_id = resolve_id(peer.chat_id)[0]
 
-        return -peer.chat_id
+        return -peer.chat_id if add_mark else peer.chat_id
     elif isinstance(peer, (PeerChannel, InputPeerChannel, ChannelFull)):
         if isinstance(peer, ChannelFull):
             # Special case: .get_input_peer can't return InputChannel from
@@ -647,15 +636,18 @@ def get_peer_id(peer):
 
         # Check in case the user mixed things up to avoid blowing up
         if not (0 < i <= 0x7fffffff):
-            i = _fix_peer_id(i)
+            i = resolve_id(i)[0]
             if isinstance(peer, ChannelFull):
                 peer.id = i
             else:
                 peer.channel_id = i
 
-        # Concat -100 through math tricks, .to_supergroup() on Madeline
+        if add_mark:
-        # IDs will be strictly positive -> log works
+            # Concat -100 through math tricks, .to_supergroup() on
-        return -(i + pow(10, math.floor(math.log10(i) + 3)))
+            # Madeline IDs will be strictly positive -> log works.
+            return -(i + pow(10, math.floor(math.log10(i) + 3)))
+        else:
+            return i
 
     _raise_cast_fail(peer, 'int')
 

telethon/version.py

@@ -1,3 +1,3 @@
 # Versions should comply with PEP440.
 # This line is parsed in setup.py:
-__version__ = '1.0.3'
+__version__ = '1.0.4'

telethon_generator/generators/tlobject.py

@@ -23,11 +23,16 @@ AUTO_CASTS = {
     'InputDialogPeer':
         'utils.get_input_dialog(client.get_input_entity({}))',
 
+    'InputNotifyPeer': 'client._get_input_notify({})',
     'InputMedia': 'utils.get_input_media({})',
     'InputPhoto': 'utils.get_input_photo({})',
     'InputMessage': 'utils.get_input_message({})'
 }
 
+NAMED_AUTO_CASTS = {
+    ('chat_id', 'int'): 'client.get_peer_id({}, add_mark=False)'
+}
+
 BASE_TYPES = ('string', 'bytes', 'int', 'long', 'int128',
               'int256', 'double', 'Bool', 'true', 'date')
 
@@ -232,12 +237,18 @@ def _write_class_init(tlobject, kind, type_constructors, builder):
 
 
 def _write_resolve(tlobject, builder):
-    if any(arg.type in AUTO_CASTS for arg in tlobject.real_args):
+    if tlobject.is_function and any(
+            (arg.type in AUTO_CASTS
+             or ((arg.name, arg.type) in NAMED_AUTO_CASTS))
+            for arg in tlobject.real_args
+    ):
         builder.writeln('def resolve(self, client, utils):')
         for arg in tlobject.real_args:
-            ac = AUTO_CASTS.get(arg.type, None)
+            ac = AUTO_CASTS.get(arg.type)
             if not ac:
-                continue
+                ac = NAMED_AUTO_CASTS.get((arg.name, arg.type))
+                if not ac:
+                    continue
 
             if arg.is_flag:
                 builder.writeln('if self.{}:', arg.name)