Implement different mtproto proxy protocols; refactor obfuscated2

telethon/client/telegrambaseclient.py

@@ -9,7 +9,7 @@ from datetime import datetime, timezone
 from .. import version, __name__ as __base_name__
 from ..crypto import rsa
 from ..extensions import markdown
-from ..network import MTProtoSender, ConnectionTcpFull, ConnectionTcpMTProxy
+from ..network import MTProtoSender, ConnectionTcpFull, TcpMTProxy
 from ..sessions import Session, SQLiteSession, MemorySession
 from ..tl import TLObject, functions, types
 from ..tl.alltlobjects import LAYER
@@ -64,7 +64,7 @@ class TelegramBaseClient(abc.ABC):
 
         proxy (`tuple` | `list` | `dict`, optional):
             An iterable consisting of the proxy info. If `connection` is
-            `ConnectionTcpMTProxy`, then it should contain MTProxy credentials:
+            one of `MTProxy`, then it should contain MTProxy credentials:
             ``('hostname', port, 'secret')``. Otherwise, it's meant to store
             function parameters for PySocks, like ``(type, 'hostname', port)``.
             See https://github.com/Anorov/PySocks#usage-1 for more.
@@ -249,8 +249,8 @@ class TelegramBaseClient(abc.ABC):
 
         assert isinstance(connection, type)
         self._connection = connection
-        init_proxy = None if connection is not ConnectionTcpMTProxy else \
+        init_proxy = None if not issubclass(connection, TcpMTProxy) else \
-            types.InputClientProxy(*ConnectionTcpMTProxy.address_info(proxy))
+            types.InputClientProxy(*connection.address_info(proxy))
 
         # Used on connection. Capture the variables in a lambda since
         # exporting clients need to create this InvokeWithLayerRequest.
@@ -551,7 +551,7 @@ class TelegramBaseClient(abc.ABC):
             self._exported_sessions[cdn_redirect.dc_id] = session
 
         self._log[__name__].info('Creating new CDN client')
-        client = TelegramBareClient(
+        client = TelegramBaseClient(
             session, self.api_id, self.api_hash,
             proxy=self._sender.connection.conn.proxy,
             timeout=self._sender.connection.get_timeout()

telethon/network/__init__.py

@@ -7,5 +7,7 @@ from .authenticator import do_authentication
 from .mtprotosender import MTProtoSender
 from .connection import (
     ConnectionTcpFull, ConnectionTcpIntermediate, ConnectionTcpAbridged,
-    ConnectionTcpObfuscated, ConnectionTcpMTProxy, ConnectionHttp
+    ConnectionTcpObfuscated, ConnectionTcpMTProxyAbridged,
+    ConnectionTcpMTProxyIntermediate,
+    ConnectionTcpMTProxyRandomizedIntermediate, ConnectionHttp, TcpMTProxy
 )

telethon/network/connection/__init__.py

@@ -2,5 +2,10 @@ from .tcpfull import ConnectionTcpFull
 from .tcpintermediate import ConnectionTcpIntermediate
 from .tcpabridged import ConnectionTcpAbridged
 from .tcpobfuscated import ConnectionTcpObfuscated
-from .tcpmtproxy import ConnectionTcpMTProxy
+from .tcpmtproxy import (
+    TcpMTProxy,
+    ConnectionTcpMTProxyAbridged,
+    ConnectionTcpMTProxyIntermediate,
+    ConnectionTcpMTProxyRandomizedIntermediate
+)
 from .http import ConnectionHttp

telethon/network/connection/connection.py

@@ -30,6 +30,8 @@ class Connection(abc.ABC):
         self._connected = False
         self._send_task = None
         self._recv_task = None
+        self._codec = None
+        self._obfuscation = None  # TcpObfuscated and MTProxy
         self._send_queue = asyncio.Queue(1)
         self._recv_queue = asyncio.Queue(1)
 

telethon/network/connection/tcpabridged.py

@@ -9,34 +9,36 @@ class ConnectionTcpAbridged(Connection):
     only require 1 byte if the packet length is less than
     508 bytes (127 << 2, which is very common).
     """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._codec = AbridgedPacket()
+
     def _init_conn(self):
-        self._writer.write(b'\xef')
+        self._writer.write(self._codec.tag)
-
-    def _write(self, data):
-        """
-        Define wrapper write methods for `TcpObfuscated` to override.
-        """
-        self._writer.write(data)
-
-    async def _read(self, n):
-        """
-        Define wrapper read methods for `TcpObfuscated` to override.
-        """
-        return await self._reader.readexactly(n)
 
     def _send(self, data):
+        self._writer.write(self._codec.encode_packet(data))
+
+    async def _recv(self):
+        return await self._codec.read_packet(self._reader)
+
+
+class AbridgedPacket:
+    tag = b'\xef'
+    mtproto_proxy_tag = b'\xef\xef\xef\xef'
+
+    def encode_packet(self, data):
         length = len(data) >> 2
         if length < 127:
             length = struct.pack('B', length)
         else:
             length = b'\x7f' + int.to_bytes(length, 3, 'little')
+        return length + data
 
-        self._write(length + data)
+    async def read_packet(self, reader):
-
+        length = struct.unpack('<B', await reader.readexactly(1))[0]
-    async def _recv(self):
-        length = struct.unpack('<B', await self._read(1))[0]
         if length >= 127:
             length = struct.unpack(
-                '<i', await self._read(3) + b'\0')[0]
+                '<i', await reader.readexactly(3) + b'\0')[0]
 
-        return await self._read(length << 2)
+        return await reader.readexactly(length << 2)

telethon/network/connection/tcpintermediate.py

@@ -1,4 +1,6 @@
 import struct
+import random
+import os
 
 from .connection import Connection
 
@@ -8,12 +10,47 @@ class ConnectionTcpIntermediate(Connection):
     Intermediate mode between `ConnectionTcpFull` and `ConnectionTcpAbridged`.
     Always sends 4 extra bytes for the packet length.
     """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._codec = IntermediatePacket()
+
     def _init_conn(self):
-        self._writer.write(b'\xee\xee\xee\xee')
+        self._writer.write(self._codec.tag)
 
     def _send(self, data):
-        self._writer.write(struct.pack('<i', len(data)) + data)
+        self._writer.write(self._codec.encode_packet(data))
 
     async def _recv(self):
-        return await self._reader.readexactly(
+        return await self.codec.read_packet(self._reader)
-            struct.unpack('<i', await self._reader.readexactly(4))[0])
+
+
+class IntermediatePacket:
+    tag = b'\xee\xee\xee\xee'
+    mtproto_proxy_tag = tag
+
+    def encode_packet(self, data):
+        return struct.pack('<i', len(data)) + data
+
+    async def read_packet(self, reader):
+        length = struct.unpack('<i', await reader.readexactly(4))[0]
+        return await reader.readexactly(length)
+
+
+class RandomizedIntermediatePacket(IntermediatePacket):
+    """
+    Data packets are aligned to 4bytes. This codec adds random bytes of size
+    from 0 to 3 bytes, which are ignored by decoder.
+    """
+    mtproto_proxy_tag = b'\xdd\xdd\xdd\xdd'
+
+    def encode_packet(self, data):
+        pad_size = random.randint(0, 3)
+        padding = os.urandom(pad_size)
+        return super().encode_packet(data + padding)
+
+    async def read_packet(self, reader):
+        packet_with_padding = await super().read_packet(reader)
+        pad_size = len(packet_with_padding) % 4
+        if pad_size > 0:
+            return packet_with_padding[:-pad_size]
+        return packet_with_padding

telethon/network/connection/tcpmtproxy.py

@@ -1,12 +1,17 @@
 import hashlib
+import os
 
-from .tcpobfuscated import ConnectionTcpObfuscated
+from .connection import Connection
+from .tcpabridged import AbridgedPacket
+from .tcpintermediate import IntermediatePacket, RandomizedIntermediatePacket
+
+from ...crypto import AESModeCTR
 
 
-class ConnectionTcpMTProxy(ConnectionTcpObfuscated):
+class TcpMTProxy(Connection):
     """
-    Wrapper around the "obfuscated2" mode that modifies it a little and allows
+    Connector which allows user to connect to the Telegram via proxy servers
-    user to connect to the Telegram proxy servers commonly known as MTProxy.
+    commonly known as MTProxy.
     Implemented very ugly due to the leaky abstractions in Telethon networking
     classes that should be refactored later (TODO).
 
@@ -15,6 +20,8 @@ class ConnectionTcpMTProxy(ConnectionTcpObfuscated):
         The support for MTProtoProxies class is **EXPERIMENTAL** and prone to
         be changed. You shouldn't be using this class yet.
     """
+    packet_codec = None
+
     @staticmethod
     def address_info(proxy_info):
         if proxy_info is None:
@@ -25,19 +32,96 @@ class ConnectionTcpMTProxy(ConnectionTcpObfuscated):
         proxy_host, proxy_port = self.address_info(proxy)
         super().__init__(
             proxy_host, proxy_port, dc_id, loop=loop, loggers=loggers)
-
+        self._codec = self.packet_codec()
-        # TODO: Implement the dd-secret secure mode (adds noise to fool DPI)
+        secret = bytes.fromhex(proxy[2])
-        self._secret = bytes.fromhex(proxy[2])
+        is_dd = (len(secret) == 17) and (secret[0] == 0xDD)
-        if len(self._secret) != 16:
+        if is_dd and (self.packet_codec != RandomizedIntermediatePacket):
             raise ValueError(
-                "MTProxy secure mode is not implemented for now"
+                "Only RandomizedIntermediate can be used with dd-secrets")
-                if len(self._secret) == 17 and self._secret[0] == 0xDD else
+        secret = secret[:-1] if is_dd else secret
-                "MTProxy secret must be a hex-string representing 16 bytes"
+        if len(secret) != 16:
-            )
+            raise ValueError(
+                "MTProxy secret must be a hex-string representing 16 bytes")
+        self._dc_id = dc_id
+        self._secret = secret
 
-    def _compose_key(self, data):
+    def _init_conn(self):
-        return hashlib.sha256(data + self._secret).digest()
+        self._obfuscation = MTProxyIO(self._reader, self._writer,
+                                      self._codec.mtproto_proxy_tag,
+                                      self._secret, self._dc_id)
+        self._writer.write(self._obfuscation.header)
 
-    def _compose_tail(self, data):
+    def _send(self, data):
-        dc_id_bytes = self._dc_id.to_bytes(2, "little", signed=True)
+        self._obfuscation.write(self._codec.encode_packet(data))
-        return super()._compose_tail(data[:60] + dc_id_bytes + data[62:])
+
+    async def _recv(self):
+        return await self._codec.read_packet(self._obfuscation)
+
+
+class ConnectionTcpMTProxyAbridged(TcpMTProxy):
+    """
+    Connect to proxy using abridged protocol
+    """
+    packet_codec = AbridgedPacket
+
+
+class ConnectionTcpMTProxyIntermediate(TcpMTProxy):
+    """
+    Connect to proxy using intermediate protocol
+    """
+    packet_codec = IntermediatePacket
+
+
+class ConnectionTcpMTProxyRandomizedIntermediate(TcpMTProxy):
+    """
+    Connect to proxy using randomized intermediate protocol (dd-secrets)
+    """
+    packet_codec = RandomizedIntermediatePacket
+
+
+class MTProxyIO:
+    """
+    It's very similar to tcpobfuscated.ObfuscatedIO, but the way
+    encryption keys, protocol tag and dc_id are encoded is different.
+    """
+    header = None
+
+    def __init__(self, reader, writer, protocol_tag, secret, dc_id):
+        self._reader = reader
+        self._writer = writer
+        # Obfuscated messages secrets cannot start with any of these
+        keywords = (b'PVrG', b'GET ', b'POST', b'\xee\xee\xee\xee')
+        while True:
+            random = os.urandom(64)
+            if (random[0] != 0xef and
+                    random[:4] not in keywords and
+                    random[4:4] != b'\0\0\0\0'):
+                break
+
+        random = bytearray(random)
+        random_reversed = random[55:7:-1]  # Reversed (8, len=48)
+
+        # Encryption has "continuous buffer" enabled
+        encrypt_key = hashlib.sha256(
+            bytes(random[8:40]) + secret).digest()
+        encrypt_iv = bytes(random[40:56])
+        decrypt_key = hashlib.sha256(
+            bytes(random_reversed[:32]) + secret).digest()
+        decrypt_iv = bytes(random_reversed[32:48])
+
+        self._aes_encrypt = AESModeCTR(encrypt_key, encrypt_iv)
+        self._aes_decrypt = AESModeCTR(decrypt_key, decrypt_iv)
+
+        random[56:60] = protocol_tag
+
+        dc_id_bytes = dc_id.to_bytes(2, "little", signed=True)
+        random = random[:60] + dc_id_bytes + random[62:]
+        random[56:64] = self._aes_encrypt.encrypt(bytes(random))[56:64]
+
+        self.header = random
+
+    async def readexactly(self, n):
+        return self._aes_decrypt.encrypt(await self._reader.readexactly(n))
+
+    def write(self, data):
+        self._writer.write(self._aes_encrypt.encrypt(data))

telethon/network/connection/tcpobfuscated.py

@@ -1,11 +1,12 @@
 import os
 
+from .tcpabridged import AbridgedPacket
 from .connection import Connection
-from .tcpabridged import ConnectionTcpAbridged
+
 from ...crypto import AESModeCTR
 
 
-class ConnectionTcpObfuscated(ConnectionTcpAbridged):
+class ConnectionTcpObfuscated(Connection):
     """
     Mode that Telegram defines as "obfuscated2". Encodes the packet
     just like `ConnectionTcpAbridged`, but encrypts every message with
@@ -15,16 +16,26 @@ class ConnectionTcpObfuscated(ConnectionTcpAbridged):
     def __init__(self, ip, port, dc_id, *, loop, loggers, proxy=None):
         super().__init__(
             ip, port, dc_id, loop=loop, loggers=loggers, proxy=proxy)
-        self._aes_encrypt = None
+        self._codec = AbridgedPacket()
-        self._aes_decrypt = None
-
-    def _write(self, data):
-        self._writer.write(self._aes_encrypt.encrypt(data))
-
-    async def _read(self, n):
-        return self._aes_decrypt.encrypt(await self._reader.readexactly(n))
 
     def _init_conn(self):
+        self._obfuscation = ObfuscatedIO(
+            self._reader, self._writer, self._codec.mtproto_proxy_tag)
+        self._writer.write(self._obfuscation.header)
+
+    def _send(self, data):
+        self._obfuscation.write(self._codec.encode_packet(data))
+
+    async def _recv(self):
+        return await self._codec.read_packet(self._obfuscation)
+
+
+class ObfuscatedIO:
+    header = None
+
+    def __init__(self, reader, writer, protocol_tag):
+        self._reader = reader
+        self._writer = writer
         # Obfuscated messages secrets cannot start with any of these
         keywords = (b'PVrG', b'GET ', b'POST', b'\xee\xee\xee\xee')
         while True:
@@ -38,23 +49,21 @@ class ConnectionTcpObfuscated(ConnectionTcpAbridged):
         random_reversed = random[55:7:-1]  # Reversed (8, len=48)
 
         # Encryption has "continuous buffer" enabled
-        encrypt_key = self._compose_key(bytes(random[8:40]))
+        encrypt_key = bytes(random[8:40])
         encrypt_iv = bytes(random[40:56])
-        decrypt_key = self._compose_key(bytes(random_reversed[:32]))
+        decrypt_key = bytes(random_reversed[:32])
         decrypt_iv = bytes(random_reversed[32:48])
 
         self._aes_encrypt = AESModeCTR(encrypt_key, encrypt_iv)
         self._aes_decrypt = AESModeCTR(decrypt_key, decrypt_iv)
 
-        random[56:60] = b'\xef\xef\xef\xef'
+        random[56:60] = protocol_tag
-        random[56:64] = self._compose_tail(bytes(random))
+        random[56:64] = self._aes_encrypt.encrypt(bytes(random))[56:64]
 
-        self._writer.write(random)
+        self.header = random
 
-    # Next functions provide the variable parts of the connection handshake.
+    async def readexactly(self, n):
-    # This is necessary to modify obfuscated2 the way that MTProxy requires.
+        return self._aes_decrypt.encrypt(await self._reader.readexactly(n))
-    def _compose_key(self, data):
-        return data
 
-    def _compose_tail(self, data):
+    def write(self, data):
-        return self._aes_encrypt.encrypt(data)[56:64]
+        self._writer.write(self._aes_encrypt.encrypt(data))