Merge branch 'master' into asyncio

README.rst

@@ -5,14 +5,24 @@ Telethon
   ⭐️ Thanks **everyone** who has starred the project, it means a lot!
 
 **Telethon** is Telegram client implementation in **Python 3** which uses
-the latest available API of Telegram. Remember to use **pip3** to install!
+the latest available API of Telegram.
+
+
+What is this?
+-------------
+
+Telegram is a popular messaging application. This library is meant
+to make it easy for you to write Python programs that can interact
+with Telegram. Think of it as a wrapper that has already done the
+heavy job for you, so you can focus on developing an application.
+
 
 Installing
 ----------
 
 .. code:: sh
 
-  pip install telethon
+  pip3 install telethon
 
 
 Creating a client
@@ -27,14 +37,10 @@ Creating a client
   # api_hash from https://my.telegram.org, under API Development.
   api_id = 12345
   api_hash = '0123456789abcdef0123456789abcdef'
-  phone = '+34600000000'
 
   client = TelegramClient('session_name', api_id, api_hash)
   async def main():
-      await client.connect()
+      await client.start()
-      # Skip this if you already have a previous 'session_name.session' file
-      await client.sign_in(phone_number)
-      me = await client.sign_in(code=input('Code: '))
 
   asyncio.get_event_loop().run_until_complete(main())
 

docs/docs_writer.py

@@ -28,6 +28,7 @@ class DocsWriter:
         self.table_columns = 0
         self.table_columns_left = None
         self.write_copy_script = False
+        self._script = ''
 
     # High level writing
     def write_head(self, title, relative_css_path):
@@ -254,6 +255,12 @@ class DocsWriter:
         self.write('<button onclick="cp(\'{}\');">{}</button>'
                    .format(text_to_copy, text))
 
+    def add_script(self, src='', relative_src=None):
+        if relative_src:
+            self._script += '<script src="{}"></script>'.format(relative_src)
+        elif src:
+            self._script += '<script>{}</script>'.format(src)
+
     def end_body(self):
         """Ends the whole document. This should be called the last"""
         if self.write_copy_script:
@@ -268,7 +275,9 @@ class DocsWriter:
                 'catch(e){}}'
                 '</script>')
 
-        self.write('</div></body></html>')
+        self.write('</div>')
+        self.write(self._script)
+        self.write('</body></html>')
 
     # "Low" level writing
     def write(self, s):

docs/generate.py

@@ -224,6 +224,16 @@ def get_description(arg):
     return ' '.join(desc)
 
 
+def copy_replace(src, dst, replacements):
+    """Copies the src file into dst applying the replacements dict"""
+    with open(src) as infile, open(dst, 'w') as outfile:
+        outfile.write(re.sub(
+            '|'.join(re.escape(k) for k in replacements),
+            lambda m: str(replacements[m.group(0)]),
+            infile.read()
+        ))
+
+
 def generate_documentation(scheme_file):
     """Generates the documentation HTML files from from scheme.tl to
        /methods and /constructors, etc.
@@ -231,6 +241,7 @@ def generate_documentation(scheme_file):
     original_paths = {
         'css': 'css/docs.css',
         'arrow': 'img/arrow.svg',
+        'search.js': 'js/search.js',
         '404': '404.html',
         'index_all': 'index.html',
         'index_types': 'types/index.html',
@@ -366,6 +377,10 @@ def generate_documentation(scheme_file):
                 else:
                     docs.write_text('This type has no members.')
 
+            # TODO Bit hacky, make everything like this? (prepending '../')
+            depth = '../' * (2 if tlobject.namespace else 1)
+            docs.add_script(src='prependPath = "{}";'.format(depth))
+            docs.add_script(relative_src=paths['search.js'])
             docs.end_body()
 
     # Find all the available types (which are not the same as the constructors)
@@ -540,36 +555,31 @@ def generate_documentation(scheme_file):
     type_urls = fmt(types, get_path_for_type)
     constructor_urls = fmt(constructors, get_create_path_for)
 
-    replace_dict = {
-        'type_count': len(types),
-        'method_count': len(methods),
-        'constructor_count': len(tlobjects) - len(methods),
-        'layer': layer,
-
-        'request_names': request_names,
-        'type_names': type_names,
-        'constructor_names': constructor_names,
-        'request_urls': request_urls,
-        'type_urls': type_urls,
-        'constructor_urls': constructor_urls
-    }
-
     shutil.copy('../res/404.html', original_paths['404'])
-
+    copy_replace('../res/core.html', original_paths['index_all'], {
-    with open('../res/core.html') as infile,\
+        '{type_count}': len(types),
-            open(original_paths['index_all'], 'w') as outfile:
+        '{method_count}': len(methods),
-        text = infile.read()
+        '{constructor_count}': len(tlobjects) - len(methods),
-        for key, value in replace_dict.items():
+        '{layer}': layer,
-            text = text.replace('{' + key + '}', str(value))
+    })
-
+    os.makedirs(os.path.abspath(os.path.join(
-        outfile.write(text)
+        original_paths['search.js'], os.path.pardir
+    )), exist_ok=True)
+    copy_replace('../res/js/search.js', original_paths['search.js'], {
+        '{request_names}': request_names,
+        '{type_names}': type_names,
+        '{constructor_names}': constructor_names,
+        '{request_urls}': request_urls,
+        '{type_urls}': type_urls,
+        '{constructor_urls}': constructor_urls
+    })
 
     # Everything done
     print('Documentation generated.')
 
 
 def copy_resources():
-    for d in ['css', 'img']:
+    for d in ('css', 'img'):
         os.makedirs(d, exist_ok=True)
 
     shutil.copy('../res/img/arrow.svg', 'img')

docs/res/core.html

@@ -14,29 +14,7 @@
 </head>
 <body>
 <div id="main_div">
-    <!-- You can append '?q=query' to the URL to default to a search -->
+    <noscript>Please enable JavaScript if you would like to use search.</noscript>
-    <input id="searchBox" type="text" onkeyup="updateSearch()"
-           placeholder="Search for requests and types…" />
-
-<div id="searchDiv">
-
-    <details open><summary class="title">Methods (<span id="methodsCount">0</span>)</summary>
-        <ul id="methodsList" class="together">
-        </ul>
-    </details>
-
-    <details open><summary class="title">Types (<span id="typesCount">0</span>)</summary>
-        <ul id="typesList" class="together">
-        </ul>
-    </details>
-
-    <details><summary class="title">Constructors (<span id="constructorsCount">0</span>)</summary>
-        <ul id="constructorsList" class="together">
-        </ul>
-    </details>
-</div>
-
-<div id="contentDiv">
     <h1>Telethon API</h1>
     <p>This documentation was generated straight from the <code>scheme.tl</code>
     provided by Telegram. However, there is no official documentation per se
@@ -44,8 +22,15 @@
     page aims to provide easy access to all the available methods, their
     definition and parameters.</p>
 
-    <p>Although this documentation was generated for <i>Telethon</i>, it may
+    <p>Please note that when you see this:</p>
-    be useful for any other Telegram library out there.</p>
+    <pre>---functions---
+users.getUsers#0d91a548 id:Vector&lt;InputUser&gt; = Vector&lt;User&gt;</pre>
+
+    <p>This is <b>not</b> Python code. It's the "TL definition". It's
+    an easy-to-read line that gives a quick overview on the parameters
+    and its result. You don't need to worry about this. See
+    <a href="http://telethon.readthedocs.io/en/latest/extra/developing/understanding-the-type-language.html">here</a>
+    for more details on it.</p>
 
     <h3>Index</h3>
     <ul>
@@ -69,12 +54,12 @@
     <p>Currently there are <b>{method_count} methods</b> available for the layer
     {layer}. The complete list can be seen <a href="methods/index.html">here</a>.
     <br /><br />
-    Methods, also known as <i>requests</i>, are used to interact with
+    Methods, also known as <i>requests</i>, are used to interact with the
-    the Telegram API itself and are invoked with a call to <code>.invoke()</code>.
+    Telegram API itself and are invoked through <code>client(Request(...))</code>.
-    <b>Only these</b> can be passed to <code>.invoke()</code>! You cannot
+    <b>Only these</b> can be used like that! You cannot invoke types or
-    <code>.invoke()</code> types or constructors, only requests. After this,
+    constructors, only requests. After this, Telegram will return a
-    Telegram will return a <code>result</code>, which may be, for instance,
+    <code>result</code>, which may be, for instance, a bunch of messages,
-    a bunch of messages, some dialogs, users, etc.</p>
+    some dialogs, users, etc.</p>
 
     <h3 id="types">Types</h3>
     <p>Currently there are <b>{type_count} types</b>. You can see the full
@@ -145,170 +130,20 @@
     </li>
     <li id="date"><b>date</b>:
         Although this type is internally used as an <code>int</code>,
-        you can pass a <code>datetime</code> object instead to work
+        you can pass a <code>datetime</code> or <code>date</code> object
-        with date parameters.
+        instead to work with date parameters.<br />
+        Note that the library uses the date in <b>UTC+0</b>, since timezone
+        conversion is not responsibility of the library. Furthermore, this
+        eases converting into any other timezone without the need for a middle
+        step.
     </li>
     </ul>
 
     <h3 id="example">Full example</h3>
-    <p>The following example demonstrates:</p>
+    <p>Documentation for this is now
-    <ol>
+    <a href="http://telethon.readthedocs.io/en/latest/extra/advanced-usage/accessing-the-full-api.html">here</a>.
-        <li>How to create a <code>TelegramClient</code>.</li>
+    </p>
-        <li>Connecting to the Telegram servers and authorizing an user.</li>
-        <li>Retrieving a list of chats (<i>dialogs</i>).</li>
-        <li>Invoking a request without the built-in methods.</li>
-    </ol>
-    <pre><span class="sh3">#!/usr/bin/python3</span>
-<span class="sh4">from</span> telethon <span class="sh4">import</span> TelegramClient
-<span class="sh4">from</span> telethon.tl.functions.messages <span class="sh4">import</span> GetHistoryRequest
-
-<span class="sh3"># <b>(1)</b> Use your own values here</span>
-api_id   = <span class="sh1">12345</span>
-api_hash = <span class="sh2">'0123456789abcdef0123456789abcdef'</span>
-phone    = <span class="sh2">'+34600000000'</span>
-
-<span class="sh3"># <b>(2)</b> Create the client and connect</span>
-client = TelegramClient(<span class="sh2">'username'</span>, api_id, api_hash)
-client.connect()
-
-<span class="sh3"># Ensure you're authorized</span>
-if not client.is_user_authorized():
-    client.send_code_request(phone)
-    client.sign_in(phone, input(<span class="sh2">'Enter the code: '</span>))
-
-<span class="sh3"># <b>(3)</b> Using built-in methods</span>
-dialogs, entities = client.get_dialogs(<span class="sh1">10</span>)
-entity = entities[<span class="sh1">0</span>]
-
-<span class="sh3"># <b>(4)</b> !! Invoking a request manually !!</span>
-result = <b>client</b>(GetHistoryRequest(
-    entity,
-    limit=<span class="sh1">20</span>,
-    offset_date=<span class="sh1">None</span>,
-    offset_id=<span class="sh1">0</span>,
-    max_id=<span class="sh1">0</span>,
-    min_id=<span class="sh1">0</span>,
-    add_offset=<span class="sh1">0</span>
-))
-
-<span class="sh3"># Now you have access to the first 20 messages</span>
-messages = result.messages</pre>
-
-    <p>As it can be seen, manually calling requests with
-    <code>client(request)</code> (or using the old way, by calling
-    <code>client.invoke(request)</code>) is way more verbose than using the
-    built-in methods (such as <code>client.get_dialogs()</code>).</p>
-
-    <p>However, and
-    given that there are so many methods available, it's impossible to provide
-    a nice interface to things that may change over time. To get full access,
-    however, you're still able to invoke these methods manually.</p>
 </div>
-
+<script src="js/search.js"></script>
-</div>
-<script>
-contentDiv = document.getElementById("contentDiv");
-searchDiv = document.getElementById("searchDiv");
-searchBox = document.getElementById("searchBox");
-
-// Search lists
-methodsList = document.getElementById("methodsList");
-methodsCount = document.getElementById("methodsCount");
-
-typesList = document.getElementById("typesList");
-typesCount = document.getElementById("typesCount");
-
-constructorsList = document.getElementById("constructorsList");
-constructorsCount = document.getElementById("constructorsCount");
-
-try {
-    requests = [{request_names}];
-    types = [{type_names}];
-    constructors = [{constructor_names}];
-
-    requestsu = [{request_urls}];
-    typesu = [{type_urls}];
-    constructorsu = [{constructor_urls}];
-} catch (e) {
-    requests = [];
-    types = [];
-    constructors = [];
-    requestsu = [];
-    typesu = [];
-    constructorsu = [];
-}
-
-// Given two input arrays "original" and "original urls" and a query,
-// return a pair of arrays with matching "query" elements from "original".
-//
-// TODO Perhaps return an array of pairs instead a pair of arrays (for cache).
-function getSearchArray(original, originalu, query) {
-    var destination = [];
-    var destinationu = [];
-
-    for (var i = 0; i < original.length; ++i) {
-        if (original[i].toLowerCase().indexOf(query) != -1) {
-            destination.push(original[i]);
-            destinationu.push(originalu[i]);
-        }
-    }
-
-    return [destination, destinationu];
-}
-
-// Modify "countSpan" and "resultList" accordingly based on the elements
-// given as [[elements], [element urls]] (both with the same length)
-function buildList(countSpan, resultList, foundElements) {
-    var result = "";
-    for (var i = 0; i < foundElements[0].length; ++i) {
-        result += '<li>';
-        result += '<a href="' + foundElements[1][i] + '">';
-        result += foundElements[0][i];
-        result += '</a></li>';
-    }
-
-    countSpan.innerHTML = "" + foundElements[0].length;
-    resultList.innerHTML = result;
-}
-
-function updateSearch() {
-    if (searchBox.value) {
-        contentDiv.style.display = "none";
-        searchDiv.style.display = "";
-
-        var query = searchBox.value.toLowerCase();
-
-        var foundRequests = getSearchArray(requests, requestsu, query);
-        var foundTypes = getSearchArray(types, typesu, query);
-        var foundConstructors = getSearchArray(
-            constructors, constructorsu, query
-        );
-
-        buildList(methodsCount, methodsList, foundRequests);
-        buildList(typesCount, typesList, foundTypes);
-        buildList(constructorsCount, constructorsList, foundConstructors);
-    } else {
-        contentDiv.style.display = "";
-        searchDiv.style.display = "none";
-    }
-}
-
-function getQuery(name) {
-    var query = window.location.search.substring(1);
-    var vars = query.split("&");
-    for (var i = 0; i != vars.length; ++i) {
-        var pair = vars[i].split("=");
-        if (pair[0] == name)
-            return pair[1];
-    }
-}
-
-var query = getQuery('q');
-if (query) {
-    searchBox.value = query;
-}
-
-updateSearch();
-</script>
 </body>
 </html>

docs/res/js/search.js

@@ -0,0 +1,172 @@
+root = document.getElementById("main_div");
+root.innerHTML = `
+<!-- You can append '?q=query' to the URL to default to a search -->
+<input id="searchBox" type="text" onkeyup="updateSearch()"
+       placeholder="Search for requests and types…" />
+
+<div id="searchDiv">
+    <div id="exactMatch" style="display:none;">
+        <b>Exact match:</b>
+        <ul id="exactList" class="together">
+        </ul>
+    </div>
+
+    <details open><summary class="title">Methods (<span id="methodsCount">0</span>)</summary>
+        <ul id="methodsList" class="together">
+        </ul>
+    </details>
+
+    <details open><summary class="title">Types (<span id="typesCount">0</span>)</summary>
+        <ul id="typesList" class="together">
+        </ul>
+    </details>
+
+    <details><summary class="title">Constructors (<span id="constructorsCount">0</span>)</summary>
+        <ul id="constructorsList" class="together">
+        </ul>
+    </details>
+</div>
+<div id="contentDiv">
+` + root.innerHTML + "</div>";
+
+// HTML modified, now load documents
+contentDiv = document.getElementById("contentDiv");
+searchDiv = document.getElementById("searchDiv");
+searchBox = document.getElementById("searchBox");
+
+// Search lists
+methodsList = document.getElementById("methodsList");
+methodsCount = document.getElementById("methodsCount");
+
+typesList = document.getElementById("typesList");
+typesCount = document.getElementById("typesCount");
+
+constructorsList = document.getElementById("constructorsList");
+constructorsCount = document.getElementById("constructorsCount");
+
+// Exact match
+exactMatch = document.getElementById("exactMatch");
+exactList = document.getElementById("exactList");
+
+try {
+    requests = [{request_names}];
+    types = [{type_names}];
+    constructors = [{constructor_names}];
+
+    requestsu = [{request_urls}];
+    typesu = [{type_urls}];
+    constructorsu = [{constructor_urls}];
+} catch (e) {
+    requests = [];
+    types = [];
+    constructors = [];
+    requestsu = [];
+    typesu = [];
+    constructorsu = [];
+}
+
+if (typeof prependPath !== 'undefined') {
+    for (var i = 0; i != requestsu.length; ++i) {
+        requestsu[i] = prependPath + requestsu[i];
+    }
+    for (var i = 0; i != typesu.length; ++i) {
+        typesu[i] = prependPath + typesu[i];
+    }
+    for (var i = 0; i != constructorsu.length; ++i) {
+        constructorsu[i] = prependPath + constructorsu[i];
+    }
+}
+
+// Given two input arrays "original" and "original urls" and a query,
+// return a pair of arrays with matching "query" elements from "original".
+//
+// TODO Perhaps return an array of pairs instead a pair of arrays (for cache).
+function getSearchArray(original, originalu, query) {
+    var destination = [];
+    var destinationu = [];
+
+    for (var i = 0; i < original.length; ++i) {
+        if (original[i].toLowerCase().indexOf(query) != -1) {
+            destination.push(original[i]);
+            destinationu.push(originalu[i]);
+        }
+    }
+
+    return [destination, destinationu];
+}
+
+// Modify "countSpan" and "resultList" accordingly based on the elements
+// given as [[elements], [element urls]] (both with the same length)
+function buildList(countSpan, resultList, foundElements) {
+    var result = "";
+    for (var i = 0; i < foundElements[0].length; ++i) {
+        result += '<li>';
+        result += '<a href="' + foundElements[1][i] + '">';
+        result += foundElements[0][i];
+        result += '</a></li>';
+    }
+
+    if (countSpan) {
+        countSpan.innerHTML = "" + foundElements[0].length;
+    }
+    resultList.innerHTML = result;
+}
+
+function updateSearch() {
+    if (searchBox.value) {
+        contentDiv.style.display = "none";
+        searchDiv.style.display = "";
+
+        var query = searchBox.value.toLowerCase();
+
+        var foundRequests = getSearchArray(requests, requestsu, query);
+        var foundTypes = getSearchArray(types, typesu, query);
+        var foundConstructors = getSearchArray(
+            constructors, constructorsu, query
+        );
+
+        buildList(methodsCount, methodsList, foundRequests);
+        buildList(typesCount, typesList, foundTypes);
+        buildList(constructorsCount, constructorsList, foundConstructors);
+
+        // Now look for exact matches
+        var original = requests.concat(constructors);
+        var originalu = requestsu.concat(constructorsu);
+        var destination = [];
+        var destinationu = [];
+
+        for (var i = 0; i < original.length; ++i) {
+            if (original[i].toLowerCase().replace("request", "") == query) {
+                destination.push(original[i]);
+                destinationu.push(originalu[i]);
+            }
+        }
+
+        if (destination.length == 0) {
+            exactMatch.style.display = "none";
+        } else {
+            exactMatch.style.display = "";
+            buildList(null, exactList, [destination, destinationu]);
+        }
+    } else {
+        contentDiv.style.display = "";
+        searchDiv.style.display = "none";
+    }
+}
+
+function getQuery(name) {
+    var query = window.location.search.substring(1);
+    var vars = query.split("&");
+    for (var i = 0; i != vars.length; ++i) {
+        var pair = vars[i].split("=");
+        if (pair[0] == name)
+            return pair[1];
+    }
+}
+
+var query = getQuery('q');
+if (query) {
+    searchBox.value = query;
+}
+
+updateSearch();

readthedocs/conf.py

@@ -20,6 +20,11 @@
 # import os
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
+import os
+import re
+
+
+root = os.path.abspath(os.path.join(__file__, os.path.pardir, os.path.pardir))
 
 
 # -- General configuration ------------------------------------------------
@@ -55,9 +60,12 @@ author = 'Lonami'
 # built documents.
 #
 # The short X.Y version.
-version = '0.15'
+with open(os.path.join(root, 'telethon', 'version.py')) as f:
+    version = re.search(r"^__version__\s+=\s+'(.*)'$",
+                        f.read(), flags=re.MULTILINE).group(1)
+
 # The full version, including alpha/beta/rc tags.
-release = '0.15.5'
+release = version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

readthedocs/extra/advanced-usage/accessing-the-full-api.rst

@@ -14,8 +14,10 @@ through a sorted list of everything you can do.
 
 .. note::
 
-    Removing the hand crafted documentation for methods is still
+    The reason to keep both https://lonamiwebs.github.io/Telethon and this
-    a work in progress!
+    documentation alive is that the former allows instant search results
+    as you type, and a "Copy import" button. If you like namespaces, you
+    can also do ``from telethon.tl import types, functions``. Both work.
 
 
 You should also refer to the documentation to see what the objects
@@ -39,8 +41,8 @@ If you're going to use a lot of these, you may do:
 
     .. code-block:: python
     
-        import telethon.tl.functions as tl
+        from telethon.tl import types, functions
-        # We now have access to 'tl.messages.SendMessageRequest'
+        # We now have access to 'functions.messages.SendMessageRequest'
 
 We see that this request must take at least two parameters, a ``peer``
 of type `InputPeer`__, and a ``message`` which is just a Python
@@ -82,6 +84,14 @@ every time its used, simply call ``.get_input_peer``:
         from telethon import utils
         peer = utils.get_input_user(entity)
 
+
+.. note::
+
+    Since ``v0.16.2`` this is further simplified. The ``Request`` itself
+    will call ``client.get_input_entity()`` for you when required, but
+    it's good to remember what's happening.
+
+
 After this small parenthesis about ``.get_entity`` versus
 ``.get_input_entity``, we have everything we need. To ``.invoke()`` our
 request we do:

readthedocs/extra/advanced-usage/sessions.rst

@@ -4,7 +4,7 @@
 Session Files
 ==============
 
-The first parameter you pass the the constructor of the ``TelegramClient`` is
+The first parameter you pass to the constructor of the ``TelegramClient`` is
 the ``session``, and defaults to be the session name (or full path). That is,
 if you create a ``TelegramClient('anon')`` instance and connect, an
 ``anon.session`` file will be created on the working directory.
@@ -44,3 +44,70 @@ methods. For example, you could save it on a database:
 
 You should read the ````session.py```` source file to know what "relevant
 data" you need to keep track of.
+
+
+Sessions and Heroku
+-------------------
+
+You probably have a newer version of SQLite installed (>= 3.8.2). Heroku uses
+SQLite 3.7.9 which does not support ``WITHOUT ROWID``. So, if you generated
+your session file on a system with SQLite >= 3.8.2 your session file will not
+work on Heroku's platform and will throw a corrupted schema error.
+
+There are multiple ways to solve this, the easiest of which is generating a
+session file on your Heroku dyno itself. The most complicated is creating
+a custom buildpack to install SQLite >= 3.8.2.
+
+
+Generating a Session File on a Heroku Dyno
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. note::
+    Due to Heroku's ephemeral filesystem all dynamically generated
+    files not part of your applications buildpack or codebase are destroyed
+    upon each restart.
+
+.. warning::
+    Do not restart your application Dyno at any point prior to retrieving your
+    session file. Constantly creating new session files from Telegram's API
+    will result in a 24 hour rate limit ban.
+
+Due to Heroku's ephemeral filesystem all dynamically generated
+files not part of your applications buildpack or codebase are destroyed upon
+each restart.
+
+Using this scaffolded code we can start the authentication process:
+
+    .. code-block:: python
+
+        client = TelegramClient('login.session', api_id, api_hash).start()
+
+At this point your Dyno will crash because you cannot access stdin. Open your
+Dyno's control panel on the Heroku website and "Run console" from the "More"
+dropdown at the top right. Enter ``bash`` and wait for it to load.
+
+You will automatically be placed into your applications working directory.
+So run your application ``python app.py`` and now you can complete the input
+requests such as "what is your phone number" etc.
+
+Once you're successfully authenticated exit your application script with
+CTRL + C and ``ls`` to confirm ``login.session`` exists in your current
+directory. Now you can create a git repo on your account and commit
+``login.session`` to that repo.
+
+You cannot ``ssh`` into your Dyno instance because it has crashed, so unless
+you programatically upload this file to a server host this is the only way to
+get it off of your Dyno.
+
+You now have a session file compatible with SQLite <= 3.8.2. Now you can
+programatically fetch this file from an external host (Firebase, S3 etc.)
+and login to your session using the following scaffolded code:
+
+    .. code-block:: python
+
+        fileName, headers = urllib.request.urlretrieve(file_url, 'login.session')
+        client = TelegramClient(os.path.abspath(fileName), api_id, api_hash).start()
+
+.. note::
+    - ``urlretrieve`` will be depreciated, consider using ``requests``.
+    - ``file_url`` represents the location of your file.

readthedocs/extra/advanced-usage/update-modes.rst

@@ -0,0 +1,144 @@
+.. _update-modes:
+
+============
+Update Modes
+============
+
+
+The library can run in four distinguishable modes:
+
+- With no extra threads at all.
+- With an extra thread that receives everything as soon as possible (default).
+- With several worker threads that run your update handlers.
+- A mix of the above.
+
+Since this section is about updates, we'll describe the simplest way to
+work with them.
+
+
+Using multiple workers
+**********************
+
+When you create your client, simply pass a number to the
+``update_workers`` parameter:
+
+    ``client = TelegramClient('session', api_id, api_hash, update_workers=2)``
+
+You can set any amount of workers you want. The more you put, the more
+update handlers that can be called "at the same time". One or two should
+suffice most of the time, since setting more will not make things run
+faster most of the times (actually, it could slow things down).
+
+The next thing you want to do is to add a method that will be called when
+an `Update`__ arrives:
+
+    .. code-block:: python
+
+        def callback(update):
+            print('I received', update)
+
+        client.add_update_handler(callback)
+        # do more work here, or simply sleep!
+
+That's it! This is the old way to listen for raw updates, with no further
+processing. If this feels annoying for you, remember that you can always
+use :ref:`working-with-updates` but maybe use this for some other cases.
+
+Now let's do something more interesting. Every time an user talks to use,
+let's reply to them with the same text reversed:
+
+    .. code-block:: python
+
+        from telethon.tl.types import UpdateShortMessage, PeerUser
+
+        def replier(update):
+            if isinstance(update, UpdateShortMessage) and not update.out:
+                client.send_message(PeerUser(update.user_id), update.message[::-1])
+
+
+        client.add_update_handler(replier)
+        input('Press enter to stop this!')
+        client.disconnect()
+
+We only ask you one thing: don't keep this running for too long, or your
+contacts will go mad.
+
+
+Spawning no worker at all
+*************************
+
+All the workers do is loop forever and poll updates from a queue that is
+filled from the ``ReadThread``, responsible for reading every item off
+the network. If you only need a worker and the ``MainThread`` would be
+doing no other job, this is the preferred way. You can easily do the same
+as the workers like so:
+
+    .. code-block:: python
+
+        while True:
+            try:
+                update = client.updates.poll()
+                if not update:
+                    continue
+
+                print('I received', update)
+            except KeyboardInterrupt:
+                break
+
+        client.disconnect()
+
+Note that ``poll`` accepts a ``timeout=`` parameter, and it will return
+``None`` if other thread got the update before you could or if the timeout
+expired, so it's important to check ``if not update``.
+
+This can coexist with the rest of ``N`` workers, or you can set it to ``0``
+additional workers:
+
+    ``client = TelegramClient('session', api_id, api_hash, update_workers=0)``
+
+You **must** set it to ``0`` (or other number), as it defaults to ``None``
+and there is a different. ``None`` workers means updates won't be processed
+*at all*, so you must set it to some value (``0`` or greater) if you want
+``client.updates.poll()`` to work.
+
+
+Using the main thread instead the ``ReadThread``
+************************************************
+
+If you have no work to do on the ``MainThread`` and you were planning to have
+a ``while True: sleep(1)``, don't do that. Instead, don't spawn the secondary
+``ReadThread`` at all like so:
+
+    .. code-block:: python
+
+        client = TelegramClient(
+            ...
+            spawn_read_thread=False
+        )
+
+And then ``.idle()`` from the ``MainThread``:
+
+    ``client.idle()``
+
+You can stop it with :kbd:`Control+C`, and you can configure the signals
+to be used in a similar fashion to `Python Telegram Bot`__.
+
+As a complete example:
+
+    .. code-block:: python
+
+        def callback(update):
+            print('I received', update)
+
+        client = TelegramClient('session', api_id, api_hash,
+                                update_workers=1, spawn_read_thread=False)
+
+        client.connect()
+        client.add_update_handler(callback)
+        client.idle()  # ends with Ctrl+C
+
+
+This is the preferred way to use if you're simply going to listen for updates.
+
+__ https://lonamiwebs.github.io/Telethon/types/update.html
+__ https://github.com/python-telegram-bot/python-telegram-bot/blob/4b3315db6feebafb94edcaa803df52bb49999ced/telegram/ext/updater.py#L460

readthedocs/extra/basic/creating-a-client.rst

@@ -31,7 +31,6 @@ one is very simple:
         # Use your own values here
         api_id = 12345
         api_hash = '0123456789abcdef0123456789abcdef'
-        phone_number = '+34600000000'
 
         client = TelegramClient('some_name', api_id, api_hash)
 
@@ -54,6 +53,7 @@ If you're not authorized, you need to ``.sign_in()``:
 
     .. code-block:: python
 
+        phone_number = '+34600000000'
         client.send_code_request(phone_number)
         myself = client.sign_in(phone_number, input('Enter code: '))
         # If .sign_in raises PhoneNumberUnoccupiedError, use .sign_up instead
@@ -76,6 +76,26 @@ As a full example:
             me = client.sign_in(phone_number, input('Enter code: '))
 
 
+All of this, however, can be done through a call to ``.start()``:
+
+    .. code-block:: python
+
+        client = TelegramClient('anon', api_id, api_hash)
+        client.start()
+
+
+The code shown is just what ``.start()`` will be doing behind the scenes
+(with a few extra checks), so that you know how to sign in case you want
+to avoid using ``input()`` (the default) for whatever reason. If no phone
+or bot token is provided, you will be asked one through ``input()``. The
+method also accepts a ``phone=`` and ``bot_token`` parameters.
+
+You can use either, as both will work. Determining which
+is just a matter of taste, and how much control you need.
+
+Remember that you can get yourself at any time with ``client.get_me()``.
+
+
 .. note::
     If you want to use a **proxy**, you have to `install PySocks`__
     (via pip or manual) and then set the appropriated parameters:
@@ -113,6 +133,9 @@ account, calling :meth:`telethon.TelegramClient.sign_in` will raise a
             client.sign_in(password=getpass.getpass())
 
 
+The mentioned ``.start()`` method will handle this for you as well, but
+you must set the ``password=`` parameter beforehand (it won't be asked).
+
 If you don't have 2FA enabled, but you would like to do so through the library,
 take as example the following code snippet:
 

readthedocs/extra/basic/entities.rst

@@ -10,21 +10,6 @@ The library widely uses the concept of "entities". An entity will refer
 to any ``User``, ``Chat`` or ``Channel`` object that the API may return
 in response to certain methods, such as ``GetUsersRequest``.
 
-To save bandwidth, the API also makes use of their "input" versions.
-The input version of an entity (e.g. ``InputPeerUser``, ``InputChat``,
-etc.) only contains the minimum required information that's required
-for Telegram to be able to identify who you're referring to: their ID
-and hash. This ID/hash pair is unique per user, so if you use the pair
-given by another user **or bot** it will **not** work.
-
-To save *even more* bandwidth, the API also makes use of the ``Peer``
-versions, which just have an ID. This serves to identify them, but
-peers alone are not enough to use them. You need to know their hash
-before you can "use them".
-
-Luckily, the library tries to simplify this mess the best it can.
-
-
 Getting entities
 ****************
 
@@ -58,8 +43,8 @@ you're able to just do this:
         my_channel = client.get_entity(PeerChannel(some_id))
 
 
-All methods in the :ref:`telegram-client` call ``.get_entity()`` to further
+All methods in the :ref:`telegram-client` call ``.get_input_entity()`` to
-save you from the hassle of doing so manually, so doing things like
+further save you from the hassle of doing so manually, so doing things like
 ``client.send_message('lonami', 'hi!')`` is possible.
 
 Every entity the library "sees" (in any response to any call) will by
@@ -72,7 +57,27 @@ made to obtain the required information.
 Entities vs. Input Entities
 ***************************
 
-As we mentioned before, API calls don't need to know the whole information
+.. note::
+
+    Don't worry if you don't understand this section, just remember some
+    of the details listed here are important. When you're calling a method,
+    don't call ``.get_entity()`` before, just use the username or phone,
+    or the entity retrieved by other means like ``.get_dialogs()``.
+
+
+To save bandwidth, the API also makes use of their "input" versions.
+The input version of an entity (e.g. ``InputPeerUser``, ``InputChat``,
+etc.) only contains the minimum required information that's required
+for Telegram to be able to identify who you're referring to: their ID
+and hash. This ID/hash pair is unique per user, so if you use the pair
+given by another user **or bot** it will **not** work.
+
+To save *even more* bandwidth, the API also makes use of the ``Peer``
+versions, which just have an ID. This serves to identify them, but
+peers alone are not enough to use them. You need to know their hash
+before you can "use them".
+
+As we just mentioned, API calls don't need to know the whole information
 about the entities, only their ID and hash. For this reason, another method,
 ``.get_input_entity()`` is available. This will always use the cache while
 possible, making zero API calls most of the time. When a request is made,
@@ -85,3 +90,15 @@ the most recent information about said entity, but invoking requests don't
 need this information, just the ``InputPeer``. Only use ``.get_entity()``
 if you need to get actual information, like the username, name, title, etc.
 of the entity.
+
+To further simplify the workflow, since the version ``0.16.2`` of the
+library, the raw requests you make to the API are also able to call
+``.get_input_entity`` wherever needed, so you can even do things like:
+
+    .. code-block:: python
+
+        client(SendMessageRequest('username', 'hello'))
+
+The library will call the ``.resolve()`` method of the request, which will
+resolve ``'username'`` with the appropriated ``InputPeer``. Don't worry if
+you don't get this yet, but remember some of the details here are important.

readthedocs/extra/basic/getting-started.rst

@@ -1,7 +1,5 @@
-.. Telethon documentation master file, created by
+.. _getting-started:
-   sphinx-quickstart on Fri Nov 17 15:36:11 2017.
+
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
 
 ===============
 Getting Started
@@ -11,7 +9,7 @@ Getting Started
 Simple Installation
 *******************
 
-   ``pip install telethon``
+   ``pip3 install telethon``
 
    **More details**: :ref:`installation`
 
@@ -27,14 +25,9 @@ Creating a client
        # api_hash from https://my.telegram.org, under API Development.
        api_id = 12345
        api_hash = '0123456789abcdef0123456789abcdef'
-       phone = '+34600000000'
 
        client = TelegramClient('session_name', api_id, api_hash)
-       client.connect()
+       client.start()
-
-       # If you already have a previous 'session_name.session' file, skip this.
-       client.sign_in(phone=phone)
-       me = client.sign_in(code=77777)  # Put whatever code you received here.
 
    **More details**: :ref:`creating-a-client`
 
@@ -44,13 +37,36 @@ Basic Usage
 
    .. code-block:: python
 
-       print(me.stringify())
+       # Getting information about yourself
+       print(client.get_me().stringify())
 
-       client.send_message('username', 'Hello! Talking to you from Telethon')
+       # Sending a message (you can use 'me' or 'self' to message yourself)
+       client.send_message('username', 'Hello World from Telethon!')
+
+       # Sending a file
        client.send_file('username', '/home/myself/Pictures/holidays.jpg')
 
-       client.download_profile_photo(me)
+       # Retrieving messages from a chat
+       from telethon import utils
+       for message in client.get_message_history('username', limit=10):
+           print(utils.get_display_name(message.sender), message.message)
+
+       # Listing all the dialogs (conversations you have open)
+       for dialog in client.get_dialogs(limit=10):
+           print(utils.get_display_name(dialog.entity), dialog.draft.message)
+
+       # Downloading profile photos (default path is the working directory)
+       client.download_profile_photo('username')
+
+       # Once you have a message with .media (if message.media)
+       # you can download it using client.download_media():
        messages = client.get_message_history('username')
        client.download_media(messages[0])
 
    **More details**: :ref:`telegram-client`
+
+
+----------
+
+You can continue by clicking on the "More details" link below each
+snippet of code or the "Next" button at the bottom of the page.

readthedocs/extra/basic/installation.rst

@@ -10,27 +10,28 @@ Automatic Installation
 
 To install Telethon, simply do:
 
-    ``pip install telethon``
+    ``pip3 install telethon``
 
-If you get something like ``"SyntaxError: invalid syntax"`` or any other
+Needless to say, you must have Python 3 and PyPi installed in your system.
-error while installing/importing the library, it's probably because ``pip``
+See https://python.org and https://pypi.python.org/pypi/pip for more.
-defaults to Python 2, which is not supported. Use ``pip3`` instead.
 
 If you already have the library installed, upgrade with:
 
-    ``pip install --upgrade telethon``
+    ``pip3 install --upgrade telethon``
 
 You can also install the library directly from GitHub or a fork:
 
    .. code-block:: sh
 
-        # pip install git+https://github.com/LonamiWebs/Telethon.git
+        # pip3 install git+https://github.com/LonamiWebs/Telethon.git
         or
         $ git clone https://github.com/LonamiWebs/Telethon.git
         $ cd Telethon/
         # pip install -Ue .
 
-If you don't have root access, simply pass the ``--user`` flag to the pip command.
+If you don't have root access, simply pass the ``--user`` flag to the pip
+command. If you want to install a specific branch, append ``@branch`` to
+the end of the first install command.
 
 
 Manual Installation
@@ -39,7 +40,7 @@ Manual Installation
 1. Install the required ``pyaes`` (`GitHub`__ | `PyPi`__) and
    ``rsa`` (`GitHub`__ | `PyPi`__) modules:
 
-    ``sudo -H pip install pyaes rsa``
+    ``sudo -H pip3 install pyaes rsa``
 
 2. Clone Telethon's GitHub repository:
    ``git clone https://github.com/LonamiWebs/Telethon.git``
@@ -50,7 +51,8 @@ Manual Installation
 
 5. Done!
 
-To generate the documentation, ``cd docs`` and then ``python3 generate.py``.
+To generate the `method documentation`__, ``cd docs`` and then
+``python3 generate.py`` (if some pages render bad do it twice).
 
 
 Optional dependencies
@@ -63,5 +65,6 @@ will also work without it.
 
 __ https://github.com/ricmoo/pyaes
 __ https://pypi.python.org/pypi/pyaes
-__ https://github.com/sybrenstuvel/python-rsa/
+__ https://github.com/sybrenstuvel/python-rsa
 __ https://pypi.python.org/pypi/rsa/3.4.2
+__ https://lonamiwebs.github.io/Telethon

readthedocs/extra/basic/telegram-client.rst

@@ -43,30 +43,29 @@ how the library refers to either of these:
         lonami = client.get_entity('lonami')
 
 The so called "entities" are another important whole concept on its own,
-and you should
+but for now you don't need to worry about it. Simply know that they are
-Note that saving and using these entities will be more important when
+a good way to get information about an user, chat or channel.
-Accessing the Full API. For now, this is a good way to get information
-about an user or chat.
 
-Other common methods for quick scripts are also available:
+Many other common methods for quick scripts are also available:
 
     .. code-block:: python
 
-        # Sending a message (use an entity/username/etc)
+        # Note that you can use 'me' or 'self' to message yourself
-        client.send_message('TheAyyBot', 'ayy')
+        client.send_message('username', 'Hello World from Telethon!')
 
-        # Sending a photo, or a file
+        client.send_file('username', '/home/myself/Pictures/holidays.jpg')
-        client.send_file(myself, '/path/to/the/file.jpg', force_document=True)
 
-        # Downloading someone's profile photo. File is saved to 'where'
+        # The utils package has some goodies, like .get_display_name()
-        where = client.download_profile_photo(someone)
+        from telethon import utils
+        for message in client.get_message_history('username', limit=10):
+            print(utils.get_display_name(message.sender), message.message)
 
-        # Retrieving the message history
+        # Dialogs are the conversations you have open
-        messages = client.get_message_history(someone)
+        for dialog in client.get_dialogs(limit=10):
+            print(utils.get_display_name(dialog.entity), dialog.draft.message)
 
-        # Downloading the media from a specific message
+        # Default path is the working directory
-        # You can specify either a directory, a filename, or nothing at all
+        client.download_profile_photo('username')
-        where = client.download_media(message, '/path/to/output')
 
         # Call .disconnect() when you're done
         client.disconnect()

readthedocs/extra/basic/working-with-updates.rst

@@ -4,139 +4,131 @@
 Working with Updates
 ====================
 
+
+The library comes with the :mod:`events` module. *Events* are an abstraction
+over what Telegram calls `updates`__, and are meant to ease simple and common
+usage when dealing with them, since there are many updates. Let's dive in!
+
+
 .. contents::
 
 
-The library can run in four distinguishable modes:
+Getting Started
-
+***************
-- With no extra threads at all.
-- With an extra thread that receives everything as soon as possible (default).
-- With several worker threads that run your update handlers.
-- A mix of the above.
-
-Since this section is about updates, we'll describe the simplest way to
-work with them.
-
-
-Using multiple workers
-**********************
-
-When you create your client, simply pass a number to the
-``update_workers`` parameter:
-
-    ``client = TelegramClient('session', api_id, api_hash, update_workers=4)``
-
-4 workers should suffice for most cases (this is also the default on
-`Python Telegram Bot`__). You can set this value to more, or even less
-if you need.
-
-The next thing you want to do is to add a method that will be called when
-an `Update`__ arrives:
 
     .. code-block:: python
 
-        def callback(update):
+        from telethon import TelegramClient, events
-            print('I received', update)
 
-        client.add_update_handler(callback)
+        client = TelegramClient(..., update_workers=1, spawn_read_thread=False)
-        # do more work here, or simply sleep!
+        client.start()
 
-That's it! Now let's do something more interesting.
+        @client.on(events.NewMessage)
-Every time an user talks to use, let's reply to them with the same
+        def my_event_handler(event):
-text reversed:
+            if 'hello' in event.raw_text:
+                event.reply('hi!')
+
+        client.idle()
+
+
+Not much, but there might be some things unclear. What does this code do?
 
     .. code-block:: python
 
-        from telethon.tl.types import UpdateShortMessage, PeerUser
+        from telethon import TelegramClient, events
 
-        def replier(update):
+        client = TelegramClient(..., update_workers=1, spawn_read_thread=False)
-            if isinstance(update, UpdateShortMessage) and not update.out:
+        client.start()
-                client.send_message(PeerUser(update.user_id), update.message[::-1])
 
 
-        client.add_update_handler(replier)
+This is normal initialization (of course, pass session name, API ID and hash).
-        input('Press enter to stop this!')
+Nothing we don't know already.
-        client.disconnect()
-
-We only ask you one thing: don't keep this running for too long, or your
-contacts will go mad.
-
-
-Spawning no worker at all
-*************************
-
-All the workers do is loop forever and poll updates from a queue that is
-filled from the ``ReadThread``, responsible for reading every item off
-the network. If you only need a worker and the ``MainThread`` would be
-doing no other job, this is the preferred way. You can easily do the same
-as the workers like so:
 
     .. code-block:: python
 
-        while True:
+        @client.on(events.NewMessage)
-            try:
-                update = client.updates.poll()
-                if not update:
-                    continue
-
-                print('I received', update)
-            except KeyboardInterrupt:
-                break
-
-        client.disconnect()
-
-Note that ``poll`` accepts a ``timeout=`` parameter, and it will return
-``None`` if other thread got the update before you could or if the timeout
-expired, so it's important to check ``if not update``.
-
-This can coexist with the rest of ``N`` workers, or you can set it to ``0``
-additional workers:
-
-    ``client = TelegramClient('session', api_id, api_hash, update_workers=0)``
-
-You **must** set it to ``0`` (or other number), as it defaults to ``None``
-and there is a different. ``None`` workers means updates won't be processed
-*at all*, so you must set it to some value (``0`` or greater) if you want
-``client.updates.poll()`` to work.
 
 
-Using the main thread instead the ``ReadThread``
+This Python decorator will attach itself to the ``my_event_handler``
-************************************************
+definition, and basically means that *on* a ``NewMessage`` *event*,
-
+the callback function you're about to define will be called:
-If you have no work to do on the ``MainThread`` and you were planning to have
-a ``while True: sleep(1)``, don't do that. Instead, don't spawn the secondary
-``ReadThread`` at all like so:
 
     .. code-block:: python
 
-        client = TelegramClient(
+        def my_event_handler(event):
-            ...
+            if 'hello' in event.raw_text:
-            spawn_read_thread=False
+                event.reply('hi!')
-        )
 
-And then ``.idle()`` from the ``MainThread``:
 
-    ``client.idle()``
+If a ``NewMessage`` event occurs, and ``'hello'`` is in the text of the
-
+message, we ``reply`` to the event with a ``'hi!'`` message.
-You can stop it with :kbd:`Control+C`, and you can configure the signals
-to be used in a similar fashion to `Python Telegram Bot`__.
-
-As a complete example:
 
     .. code-block:: python
 
-        def callback(update):
+        client.idle()
-            print('I received', update)
+
-
+
-        client = TelegramClient('session', api_id, api_hash,
+Finally, this tells the client that we're done with our code, and want
-                                update_workers=1, spawn_read_thread=False)
+to listen for all these events to occur. Of course, you might want to
-
+do other things instead idling. For this refer to :ref:`update-modes`.
-        client.connect()
+
-        client.add_update_handler(callback)
+
-        client.idle()  # ends with Ctrl+C
+More on events
-        client.disconnect()
+**************
+
+The ``NewMessage`` event has much more than what was shown. You can access
+the ``.sender`` of the message through that member, or even see if the message
+had ``.media``, a ``.photo`` or a ``.document`` (which you could download with
+for example ``client.download_media(event.photo)``.
+
+If you don't want to ``.reply`` as a reply, you can use the ``.respond()``
+method instead. Of course, there are more events such as ``ChatAction`` or
+``UserUpdate``, and they're all used in the same way. Simply add the
+``@client.on(events.XYZ)`` decorator on the top of your handler and you're
+done! The event that will be passed always is of type ``XYZ.Event`` (for
+instance, ``NewMessage.Event``), except for the ``Raw`` event which just
+passes the ``Update`` object.
+
+You can put the same event on many handlers, and even different events on
+the same handler. You can also have a handler work on only specific chats,
+for example:
+
+
+    .. code-block:: python
+
+        import ast
+        import random
+
+
+        @client.on(events.NewMessage(chats='TelethonOffTopic', incoming=True))
+        def normal_handler(event):
+            if 'roll' in event.raw_text:
+                event.reply(str(random.randint(1, 6)))
+
+
+        @client.on(events.NewMessage(chats='TelethonOffTopic', outgoing=True))
+        def admin_handler(event):
+            if event.raw_text.startswith('eval'):
+                expression = event.raw_text.replace('eval', '').strip()
+                event.reply(str(ast.literal_eval(expression)))
+
+
+You can pass one or more chats to the ``chats`` parameter (as a list or tuple),
+and only events from there will be processed. You can also specify whether you
+want to handle incoming or outgoing messages (those you receive or those you
+send). In this example, people can say ``'roll'`` and you will reply with a
+random number, while if you say ``'eval 4+4'``, you will reply with the
+solution. Try it!
+
+
+Events module
+*************
+
+.. automodule:: telethon.events
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 
 
-__ https://python-telegram-bot.org/
 __ https://lonamiwebs.github.io/Telethon/types/update.html
-__ https://github.com/python-telegram-bot/python-telegram-bot/blob/4b3315db6feebafb94edcaa803df52bb49999ced/telegram/ext/updater.py#L460

readthedocs/extra/developing/telegram-api-in-other-languages.rst

@@ -13,18 +13,18 @@ C
 *
 
 Possibly the most well-known unofficial open source implementation out
-there by `**@vysheng** <https://github.com/vysheng>`__,
+there by `@vysheng <https://github.com/vysheng>`__,
-```tgl`` <https://github.com/vysheng/tgl>`__, and its console client
+`tgl <https://github.com/vysheng/tgl>`__, and its console client
-```telegram-cli`` <https://github.com/vysheng/tg>`__. Latest development
+`telegram-cli <https://github.com/vysheng/tg>`__. Latest development
 has been moved to `BitBucket <https://bitbucket.org/vysheng/tdcli>`__.
 
 JavaScript
 **********
 
-`**@zerobias** <https://github.com/zerobias>`__ is working on
+`@zerobias <https://github.com/zerobias>`__ is working on
-```telegram-mtproto`` <https://github.com/zerobias/telegram-mtproto>`__,
+`telegram-mtproto <https://github.com/zerobias/telegram-mtproto>`__,
 a work-in-progress JavaScript library installable via
-```npm`` <https://www.npmjs.com/>`__.
+`npm <https://www.npmjs.com/>`__.
 
 Kotlin
 ******
@@ -34,14 +34,14 @@ implementation written in Kotlin (the now
 `official <https://blog.jetbrains.com/kotlin/2017/05/kotlin-on-android-now-official/>`__
 language for
 `Android <https://developer.android.com/kotlin/index.html>`__) by
-`**@badoualy** <https://github.com/badoualy>`__, currently as a beta–
+`@badoualy <https://github.com/badoualy>`__, currently as a beta–
 yet working.
 
 PHP
 ***
 
 A PHP implementation is also available thanks to
-`**@danog** <https://github.com/danog>`__ and his
+`@danog <https://github.com/danog>`__ and his
 `MadelineProto <https://github.com/danog/MadelineProto>`__ project, with
 a very nice `online
 documentation <https://daniil.it/MadelineProto/API_docs/>`__ too.
@@ -51,7 +51,7 @@ Python
 
 A fairly new (as of the end of 2017) Telegram library written from the
 ground up in Python by
-`**@delivrance** <https://github.com/delivrance>`__ and his
+`@delivrance <https://github.com/delivrance>`__ and his
 `Pyrogram <https://github.com/pyrogram/pyrogram>`__ library! No hard
 feelings Dan and good luck dealing with some of your users ;)
 
@@ -59,6 +59,6 @@ Rust
 ****
 
 Yet another work-in-progress implementation, this time for Rust thanks
-to `**@JuanPotato** <https://github.com/JuanPotato>`__ under the fancy
+to `@JuanPotato <https://github.com/JuanPotato>`__ under the fancy
 name of `Vail <https://github.com/JuanPotato/Vail>`__. This one is very
 early still, but progress is being made at a steady rate.

readthedocs/extra/developing/understanding-the-type-language.rst

@@ -10,9 +10,7 @@ what other programming languages commonly call classes or structs.
 Every definition is written as follows for a Telegram object is defined
 as follows:
 
-.. code:: tl
+    ``name#id argument_name:argument_type = CommonType``
-
-    name#id argument_name:argument_type = CommonType
 
 This means that in a single line you know what the ``TLObject`` name is.
 You know it's unique ID, and you know what arguments it has. It really

readthedocs/extra/examples/bots.rst

@@ -3,6 +3,11 @@ Bots
 ====
 
 
+.. note::
+
+    These examples assume you have read :ref:`accessing-the-full-api`.
+
+
 Talking to Inline Bots
 **********************
 

readthedocs/extra/examples/chats-and-channels.rst

@@ -3,6 +3,11 @@ Working with Chats and Channels
 ===============================
 
 
+.. note::
+
+    These examples assume you have read :ref:`accessing-the-full-api`.
+
+
 Joining a chat or channel
 *************************
 
@@ -41,7 +46,7 @@ enough information to join! The part after the
 example, is the ``hash`` of the chat or channel. Now you can use
 `ImportChatInviteRequest`__ as follows:
 
-    .. -block:: python
+    .. code-block:: python
 
         from telethon.tl.functions.messages import ImportChatInviteRequest
         updates = client(ImportChatInviteRequest('AAAAAEHbEkejzxUjAUCfYg'))
@@ -61,7 +66,7 @@ which use is very straightforward:
         client(AddChatUserRequest(
             chat_id,
             user_to_add,
-            fwd_limit=10  # allow the user to see the 10 last messages
+            fwd_limit=10  # Allow the user to see the 10 last messages
         ))
 
 
@@ -70,7 +75,7 @@ Checking a link without joining
 
 If you don't need to join but rather check whether it's a group or a
 channel, you can use the `CheckChatInviteRequest`__, which takes in
-the `hash`__ of said channel or group.
+the hash of said channel or group.
 
 __ https://lonamiwebs.github.io/Telethon/constructors/chat.html
 __ https://lonamiwebs.github.io/Telethon/constructors/channel.html
@@ -80,7 +85,6 @@ __ https://lonamiwebs.github.io/Telethon/methods/channels/index.html
 __ https://lonamiwebs.github.io/Telethon/methods/messages/import_chat_invite.html
 __ https://lonamiwebs.github.io/Telethon/methods/messages/add_chat_user.html
 __ https://lonamiwebs.github.io/Telethon/methods/messages/check_chat_invite.html
-__ https://github.com/LonamiWebs/Telethon/wiki/Joining-a-chat-or-channel#joining-a-private-chat-or-channel
 
 
 Retrieving all chat members (channels too)
@@ -107,8 +111,9 @@ a fixed limit:
         all_participants = []
 
         while True:
-            participants = client.invoke(GetParticipantsRequest(
+            participants = client(GetParticipantsRequest(
-                channel, ChannelParticipantsSearch(''), offset, limit
+                channel, ChannelParticipantsSearch(''), offset, limit,
+                hash=0
             ))
             if not participants.users:
                 break
@@ -165,13 +170,27 @@ Giving or revoking admin permissions can be done with the `EditAdminRequest`__:
             invite_link=None,
             edit_messages=None
         ) 
+        # Equivalent to:
+        #     rights = ChannelAdminRights(
+        #         change_info=True,
+        #         delete_messages=True,
+        #         pin_messages=True
+        #     )
 
-        client(EditAdminRequest(channel, who, rights))
+        # Once you have a ChannelAdminRights, invoke it
+        client(EditAdminRequest(channel, user, rights))
 
+        # User will now be able to change group info, delete other people's
+        # messages and pin messages.
         
-Thanks to `@Kyle2142`__ for `pointing out`__ that you **cannot** set
+|  Thanks to `@Kyle2142`__ for `pointing out`__ that you **cannot** set all
-to ``True`` the ``post_messages`` and ``edit_messages`` fields. Those that
+|  parameters to ``True`` to give a user full permissions, as not all
-are ``None`` can be omitted (left here so you know `which are available`__.
+|  permissions are related to both broadcast channels/megagroups.
+|
+|  E.g. trying to set ``post_messages=True`` in a megagroup will raise an
+|  error. It is recommended to always use keyword arguments, and to set only
+|  the permissions the user needs. If you don't need to change a permission,
+|  it can be omitted (full list `here`__).
 
 __ https://lonamiwebs.github.io/Telethon/methods/channels/edit_admin.html
 __ https://github.com/Kyle2142

readthedocs/extra/examples/working-with-messages.rst

@@ -3,6 +3,11 @@ Working with messages
 =====================
 
 
+.. note::
+
+    These examples assume you have read :ref:`accessing-the-full-api`.
+
+
 Forwarding messages
 *******************
 
@@ -42,12 +47,26 @@ into issues_. A valid example would be:
 
     .. code-block:: python
 
+        from telethon.tl.functions.messages import SearchRequest
+        from telethon.tl.types import InputMessagesFilterEmpty
+
+        filter = InputMessagesFilterEmpty()
         result = client(SearchRequest(
-            entity, 'query', InputMessagesFilterEmpty(), None, None, 0, 0, 100
+            peer=peer,      # On which chat/conversation
+            q='query',      # What to search for
+            filter=filter,  # Filter to use (maybe filter for media)
+            min_date=None,  # Minimum date
+            max_date=None,  # Maximum date
+            offset_id=0,    # ID of the message to use as offset
+            add_offset=0,   # Additional offset
+            limit=10,       # How many results
+            max_id=0,       # Maximum message ID
+            min_id=0,       # Minimum message ID
+            from_id=None    # Who must have sent the message (peer)
         ))
 
-It's important to note that the optional parameter ``from_id`` has been left
+It's important to note that the optional parameter ``from_id`` could have
-omitted and thus defaults to ``None``. Changing it to InputUserEmpty_, as one
+been omitted (defaulting to ``None``). Changing it to InputUserEmpty_, as one
 could think to specify "no user", won't work because this parameter is a flag,
 and it being unspecified has a different meaning.
 

readthedocs/extra/troubleshooting/rpc-errors.rst

@@ -2,10 +2,11 @@
 RPC Errors
 ==========
 
-RPC stands for Remote Procedure Call, and when Telethon raises an
+RPC stands for Remote Procedure Call, and when the library raises
-``RPCError``, it's most likely because you have invoked some of the API
+a ``RPCError``, it's because you have invoked some of the API
 methods incorrectly (wrong parameters, wrong permissions, or even
-something went wrong on Telegram's server). The most common are:
+something went wrong on Telegram's server). All the errors are
+available in :ref:`telethon-errors-package`, but some examples are:
 
 -  ``FloodWaitError`` (420), the same request was repeated many times.
    Must wait ``.seconds`` (you can access this parameter).
@@ -17,11 +18,12 @@ something went wrong on Telegram's server). The most common are:
    said operation on a chat or channel. Try avoiding filters, i.e. when
    searching messages.
 
-The generic classes for different error codes are: \* ``InvalidDCError``
+The generic classes for different error codes are:
-(303), the request must be repeated on another DC. \*
+
-``BadRequestError`` (400), the request contained errors. \*
+- ``InvalidDCError`` (303), the request must be repeated on another DC.
-``UnauthorizedError`` (401), the user is not authorized yet. \*
+- ``BadRequestError`` (400), the request contained errors.
-``ForbiddenError`` (403), privacy violation error. \* ``NotFoundError``
+- ``UnauthorizedError`` (401), the user is not authorized yet.
-(404), make sure you're invoking ``Request``\ 's!
+- ``ForbiddenError`` (403), privacy violation error.
+- ``NotFoundError`` (404), make sure you're invoking ``Request``\ 's!
 
 If the error is not recognised, it will only be an ``RPCError``.

readthedocs/extra/wall-of-shame.rst

@@ -9,27 +9,28 @@ you to file **issues** whenever you encounter any when working with the
 library. Said section is **not** for issues on *your* program but rather
 issues with Telethon itself.
 
-If you have not made the effort to 1. `read through the
+If you have not made the effort to 1. read through the docs and 2.
-wiki <https://github.com/LonamiWebs/Telethon/wiki>`__ and 2. `look for
+`look for the method you need <https://lonamiwebs.github.io/Telethon/>`__,
-the method you need <https://lonamiwebs.github.io/Telethon/>`__, you
+you will end up on the `Wall of
-will end up on the `Wall of
 Shame <https://github.com/LonamiWebs/Telethon/issues?q=is%3Aissue+label%3ARTFM+is%3Aclosed>`__,
 i.e. all issues labeled
 `"RTFM" <http://www.urbandictionary.com/define.php?term=RTFM>`__:
 
-> > **rtfm**
+       **rtfm**
-> > Literally "Read The F\ **king Manual"; a term showing the
+       Literally "Read The F--king Manual"; a term showing the
-frustration of being bothered with questions so trivial that the asker
+       frustration of being bothered with questions so trivial that the asker
-could have quickly figured out the answer on their own with minimal
+       could have quickly figured out the answer on their own with minimal
-effort, usually by reading readily-available documents. People who
+       effort, usually by reading readily-available documents. People who
-say"RTFM!" might be considered rude, but the true rude ones are the
+       say"RTFM!" might be considered rude, but the true rude ones are the
-annoying people who take absolutely no self-responibility and expect to
+       annoying people who take absolutely no self-responibility and expect to
-have all the answers handed to them personally.
+       have all the answers handed to them personally.
-> > *"Damn, that's the twelveth time that somebody posted this question
-to the messageboard today! RTFM, already!"*
-> > **\ by Bill M. July 27, 2004*\*
 
-If you have indeed read the wiki, and have tried looking for the method,
+       *"Damn, that's the twelveth time that somebody posted this question
+       to the messageboard today! RTFM, already!"*
+
+       *by Bill M. July 27, 2004*
+
+If you have indeed read the docs, and have tried looking for the method,
 and yet you didn't find what you need, **that's fine**. Telegram's API
 can have some obscure names at times, and for this reason, there is a
 `"question"

readthedocs/index.rst

@@ -10,7 +10,21 @@ Welcome to Telethon's documentation!
 
 Pure Python 3 Telegram client library.
 Official Site `here <https://lonamiwebs.github.io/Telethon>`_.
-Please follow the links below to get you started.
+Please follow the links on the index below to navigate from here,
+or use the menu on the left. Remember to read the :ref:`changelog`
+when you upgrade!
+
+.. important::
+   If you're new here, you want to read :ref:`getting-started`.
+
+
+What is this?
+*************
+
+Telegram is a popular messaging application. This library is meant
+to make it easy for you to write Python programs that can interact
+with Telegram. Think of it as a wrapper that has already done the
+heavy job for you, so you can focus on developing an application.
 
 
 .. _installation-and-usage:
@@ -35,6 +49,7 @@ Please follow the links below to get you started.
 
    extra/advanced-usage/accessing-the-full-api
    extra/advanced-usage/sessions
+   extra/advanced-usage/update-modes
 
 
 .. _Examples:
@@ -75,19 +90,20 @@ Please follow the links below to get you started.
    extra/developing/telegram-api-in-other-languages.rst
 
 
-.. _Wall-of-shame:
+.. _More:
 
 .. toctree::
    :maxdepth: 2
-   :caption: Wall of Shame
+   :caption: More
 
+   extra/changelog
    extra/wall-of-shame.rst
 
 
 .. toctree::
    :caption: Telethon modules
 
-   telethon
+   modules
 
 
 Indices and tables

readthedocs/telethon.errors.rst

@@ -1,3 +1,6 @@
+.. _telethon-errors-package:
+
+
 telethon\.errors package
 ========================
 

readthedocs/telethon.events.rst

@@ -0,0 +1,4 @@
+telethon\.events package
+========================
+
+

readthedocs/telethon.rst

@@ -26,6 +26,14 @@ telethon\.telegram\_client module
     :undoc-members:
     :show-inheritance:
 
+telethon\.events package
+------------------------
+
+.. toctree::
+
+    telethon.events
+
+
 telethon\.update\_state module
 ------------------------------
 
@@ -42,6 +50,13 @@ telethon\.utils module
     :undoc-members:
     :show-inheritance:
 
+telethon\.session module
+------------------------
+
+.. automodule:: telethon.session
+    :members:
+    :undoc-members:
+    :show-inheritance:
 
 telethon\.cryto package
 ------------------------
@@ -58,21 +73,21 @@ telethon\.errors package
     telethon.errors
 
 telethon\.extensions package
-------------------------
+----------------------------
 
 .. toctree::
 
     telethon.extensions
 
 telethon\.network package
-------------------------
+-------------------------
 
 .. toctree::
 
     telethon.network
 
 telethon\.tl package
-------------------------
+--------------------
 
 .. toctree::
 

readthedocs/telethon.tl.rst

@@ -7,14 +7,6 @@ telethon\.tl package
     telethon.tl.custom
 
 
-telethon\.tl\.entity\_database module
--------------------------------------
-
-.. automodule:: telethon.tl.entity_database
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 telethon\.tl\.gzip\_packed module
 ---------------------------------
 
@@ -31,14 +23,6 @@ telethon\.tl\.message\_container module
     :undoc-members:
     :show-inheritance:
 
-telethon\.tl\.session module
-----------------------------
-
-.. automodule:: telethon.tl.session
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 telethon\.tl\.tl\_message module
 --------------------------------
 

setup.py

@@ -45,11 +45,13 @@ GENERATOR_DIR = 'telethon/tl'
 IMPORT_DEPTH = 2
 
 
-def gen_tl():
+def gen_tl(force=True):
     from telethon_generator.tl_generator import TLGenerator
     from telethon_generator.error_generator import generate_code
     generator = TLGenerator(GENERATOR_DIR)
     if generator.tlobjects_exist():
+        if not force:
+            return
         print('Detected previous TLObjects. Cleaning...')
         generator.clean_tlobjects()
 
@@ -99,6 +101,10 @@ def main():
         fetch_errors(ERRORS_JSON)
 
     else:
+        # Call gen_tl() if the scheme.tl file exists, e.g. install from GitHub
+        if os.path.isfile(SCHEME_TL):
+            gen_tl(force=False)
+
         # Get the long description from the README file
         with open('README.rst', encoding='utf-8') as f:
             long_description = f.read()

telethon/crypto/auth_key.py

@@ -4,7 +4,6 @@ This module holds the AuthKey class.
 import struct
 from hashlib import sha1
 
-from .. import helpers as utils
 from ..extensions import BinaryReader
 
 
@@ -36,4 +35,6 @@ class AuthKey:
         """
         new_nonce = new_nonce.to_bytes(32, 'little', signed=True)
         data = new_nonce + struct.pack('<BQ', number, self.aux_hash)
-        return utils.calc_msg_key(data)
+
+        # Calculates the message key from the given data
+        return sha1(data).digest()[4:20]

telethon/events/__init__.py

@@ -0,0 +1,868 @@
+import abc
+import datetime
+import itertools
+
+from .. import utils
+from ..errors import RPCError
+from ..extensions import markdown
+from ..tl import types, functions
+
+
+class _EventBuilder(abc.ABC):
+    @abc.abstractmethod
+    def build(self, update):
+        """Builds an event for the given update if possible, or returns None"""
+
+    @abc.abstractmethod
+    async def resolve(self, client):
+        """Helper method to allow event builders to be resolved before usage"""
+
+
+class _EventCommon(abc.ABC):
+    """Intermediate class with common things to all events"""
+
+    def __init__(self, chat_peer=None, msg_id=None, broadcast=False):
+        self._client = None
+        self._chat_peer = chat_peer
+        self._message_id = msg_id
+        self._input_chat = None
+        self._chat = None
+
+        self.is_private = isinstance(chat_peer, types.PeerUser)
+        self.is_group = (
+            isinstance(chat_peer, (types.PeerChat, types.PeerChannel))
+            and not broadcast
+        )
+        self.is_channel = isinstance(chat_peer, types.PeerChannel)
+
+    async def _get_input_entity(self, msg_id, entity_id, chat=None):
+        """
+        Helper function to call GetMessages on the give msg_id and
+        return the input entity whose ID is the given entity ID.
+
+        If ``chat`` is present it must be an InputPeer.
+        """
+        try:
+            if isinstance(chat, types.InputPeerChannel):
+                result = await self._client(
+                    functions.channels.GetMessagesRequest(chat, [msg_id])
+                )
+            else:
+                result = await self._client(
+                    functions.messages.GetMessagesRequest([msg_id])
+                )
+        except RPCError:
+            return
+        entity = {
+            utils.get_peer_id(x): x for x in itertools.chain(
+                getattr(result, 'chats', []),
+                getattr(result, 'users', []))
+        }.get(entity_id)
+        if entity:
+            return utils.get_input_peer(entity)
+
+    @property
+    async def input_chat(self):
+        """
+        The (:obj:`InputPeer`) (group, megagroup or channel) on which
+        the event occurred. This doesn't have the title or anything,
+        but is useful if you don't need those to avoid further
+        requests.
+
+        Note that this might be ``None`` if the library can't find it.
+        """
+
+        if self._input_chat is None and self._chat_peer is not None:
+            try:
+                self._input_chat = await self._client.get_input_entity(
+                    self._chat_peer
+                )
+            except (ValueError, TypeError):
+                # The library hasn't seen this chat, get the message
+                if not isinstance(self._chat_peer, types.PeerChannel):
+                    # TODO For channels, getDifference? Maybe looking
+                    # in the dialogs (which is already done) is enough.
+                    if self._message_id is not None:
+                        self._input_chat = await self._get_input_entity(
+                            self._message_id,
+                            utils.get_peer_id(self._chat_peer)
+                        )
+        return self._input_chat
+
+    @property
+    async def chat(self):
+        """
+        The (:obj:`User` | :obj:`Chat` | :obj:`Channel`, optional) on which
+        the event occurred. This property will make an API call the first time
+        to get the most up to date version of the chat, so use with care as
+        there is no caching besides local caching yet.
+        """
+        if self._chat is None and await self.input_chat:
+            self._chat = await self._client.get_entity(self._input_chat)
+        return self._chat
+
+
+class Raw(_EventBuilder):
+    """
+    Represents a raw event. The event is the update itself.
+    """
+    async def resolve(self, client):
+        pass
+
+    def build(self, update):
+        return update
+
+
+# Classes defined here are actually Event builders
+# for their inner Event classes. Inner ._client is
+# set later by the creator TelegramClient.
+class NewMessage(_EventBuilder):
+    """
+    Represents a new message event builder.
+
+    Args:
+        incoming (:obj:`bool`, optional):
+            If set to ``True``, only **incoming** messages will be handled.
+            Mutually exclusive with ``outgoing`` (can only set one of either).
+
+        outgoing (:obj:`bool`, optional):
+            If set to ``True``, only **outgoing** messages will be handled.
+            Mutually exclusive with ``incoming`` (can only set one of either).
+
+        chats (:obj:`entity`, optional):
+            May be one or more entities (username/peer/etc.). By default,
+            only matching chats will be handled.
+
+        blacklist_chats (:obj:`bool`, optional):
+            Whether to treat the the list of chats as a blacklist (if
+            it matches it will NOT be handled) or a whitelist (default).
+    """
+    def __init__(self, incoming=None, outgoing=None,
+                 chats=None, blacklist_chats=False):
+        if incoming and outgoing:
+            raise ValueError('Can only set either incoming or outgoing')
+
+        self.incoming = incoming
+        self.outgoing = outgoing
+        self.chats = chats
+        self.blacklist_chats = blacklist_chats
+
+    async def resolve(self, client):
+        if hasattr(self.chats, '__iter__') and not isinstance(self.chats, str):
+            self.chats = set(utils.get_peer_id(x)
+                             for x in await client.get_input_entity(self.chats))
+        elif self.chats is not None:
+            self.chats = {utils.get_peer_id(
+                          await client.get_input_entity(self.chats))}
+
+    def build(self, update):
+        if isinstance(update,
+                      (types.UpdateNewMessage, types.UpdateNewChannelMessage)):
+            if not isinstance(update.message, types.Message):
+                return  # We don't care about MessageService's here
+            event = NewMessage.Event(update.message)
+        elif isinstance(update, types.UpdateShortMessage):
+            event = NewMessage.Event(types.Message(
+                out=update.out,
+                mentioned=update.mentioned,
+                media_unread=update.media_unread,
+                silent=update.silent,
+                id=update.id,
+                to_id=types.PeerUser(update.user_id),
+                message=update.message,
+                date=update.date,
+                fwd_from=update.fwd_from,
+                via_bot_id=update.via_bot_id,
+                reply_to_msg_id=update.reply_to_msg_id,
+                entities=update.entities
+            ))
+        else:
+            return
+
+        # Short-circuit if we let pass all events
+        if all(x is None for x in (self.incoming, self.outgoing, self.chats)):
+            return event
+
+        if self.incoming and event.message.out:
+            return
+        if self.outgoing and not event.message.out:
+            return
+
+        if self.chats is not None:
+            inside = utils.get_peer_id(event.message.to_id) in self.chats
+            if inside == self.blacklist_chats:
+                # If this chat matches but it's a blacklist ignore.
+                # If it doesn't match but it's a whitelist ignore.
+                return
+
+        # Tests passed so return the event
+        return event
+
+    class Event(_EventCommon):
+        """
+        Represents the event of a new message.
+
+        Members:
+            message (:obj:`Message`):
+                This is the original ``Message`` object.
+
+            is_private (:obj:`bool`):
+                True if the message was sent as a private message.
+
+            is_group (:obj:`bool`):
+                True if the message was sent on a group or megagroup.
+
+            is_channel (:obj:`bool`):
+                True if the message was sent on a megagroup or channel.
+
+            is_reply (:obj:`str`):
+                Whether the message is a reply to some other or not.
+        """
+        def __init__(self, message):
+            super().__init__(chat_peer=message.to_id,
+                             msg_id=message.id, broadcast=bool(message.post))
+
+            self.message = message
+            self._text = None
+
+            self._input_chat = None
+            self._chat = None
+            self._input_sender = None
+            self._sender = None
+
+            self.is_reply = bool(message.reply_to_msg_id)
+            self._reply_message = None
+
+        async def respond(self, *args, **kwargs):
+            """
+            Responds to the message (not as a reply). This is a shorthand for
+            ``client.send_message(event.chat, ...)``.
+            """
+            return await self._client.send_message(await self.input_chat, *args, **kwargs)
+
+        async def reply(self, *args, **kwargs):
+            """
+            Replies to the message (as a reply). This is a shorthand for
+            ``client.send_message(event.chat, ..., reply_to=event.message.id)``.
+            """
+            return await self._client.send_message(await self.input_chat,
+                                                   reply_to=self.message.id,
+                                                   *args, **kwargs)
+
+        @property
+        async def input_sender(self):
+            """
+            This (:obj:`InputPeer`) is the input version of the user who
+            sent the message. Similarly to ``input_chat``, this doesn't have
+            things like username or similar, but still useful in some cases.
+
+            Note that this might not be available if the library can't
+            find the input chat.
+            """
+            if self._input_sender is None:
+                try:
+                    self._input_sender = await self._client.get_input_entity(
+                        self.message.from_id
+                    )
+                except (ValueError, TypeError):
+                    if isinstance(self.message.to_id, types.PeerChannel):
+                        # We can rely on self.input_chat for this
+                        self._input_sender = await self._get_input_entity(
+                            self.message.id,
+                            self.message.from_id,
+                            chat=await self.input_chat
+                        )
+
+            return self._input_sender
+
+        @property
+        async def sender(self):
+            """
+            This (:obj:`User`) will make an API call the first time to get
+            the most up to date version of the sender, so use with care as
+            there is no caching besides local caching yet.
+
+            ``input_sender`` needs to be available (often the case).
+            """
+            if self._sender is None and await self.input_sender:
+                self._sender = await self._client.get_entity(self._input_sender)
+            return self._sender
+
+        @property
+        def text(self):
+            """
+            The message text, markdown-formatted.
+            """
+            if self._text is None:
+                if not self.message.entities:
+                    return self.message.message
+                self._text = markdown.unparse(self.message.message,
+                                              self.message.entities or [])
+            return self._text
+
+        @property
+        def raw_text(self):
+            """
+            The raw message text, ignoring any formatting.
+            """
+            return self.message.message
+
+        @property
+        async def reply_message(self):
+            """
+            This (:obj:`Message`, optional) will make an API call the first
+            time to get the full ``Message`` object that one was replying to,
+            so use with care as there is no caching besides local caching yet.
+            """
+            if not self.message.reply_to_msg_id:
+                return None
+
+            if self._reply_message is None:
+                if isinstance(await self.input_chat, types.InputPeerChannel):
+                    r = await self._client(functions.channels.GetMessagesRequest(
+                        await self.input_chat, [self.message.reply_to_msg_id]
+                    ))
+                else:
+                    r = await self._client(functions.messages.GetMessagesRequest(
+                        [self.message.reply_to_msg_id]
+                    ))
+                if not isinstance(r, types.messages.MessagesNotModified):
+                    self._reply_message = r.messages[0]
+
+            return self._reply_message
+
+        @property
+        def forward(self):
+            """
+            The unmodified (:obj:`MessageFwdHeader`, optional).
+            """
+            return self.message.fwd_from
+
+        @property
+        def media(self):
+            """
+            The unmodified (:obj:`MessageMedia`, optional).
+            """
+            return self.message.media
+
+        @property
+        def photo(self):
+            """
+            If the message media is a photo,
+            this returns the (:obj:`Photo`) object.
+            """
+            if isinstance(self.message.media, types.MessageMediaPhoto):
+                photo = self.message.media.photo
+                if isinstance(photo, types.Photo):
+                    return photo
+
+        @property
+        def document(self):
+            """
+            If the message media is a document,
+            this returns the (:obj:`Document`) object.
+            """
+            if isinstance(self.message.media, types.MessageMediaDocument):
+                doc = self.message.media.document
+                if isinstance(doc, types.Document):
+                    return doc
+
+        @property
+        def out(self):
+            """
+            Whether the message is outgoing (i.e. you sent it from
+            another session) or incoming (i.e. someone else sent it).
+            """
+            return self.message.out
+
+
+class ChatAction(_EventBuilder):
+    """
+    Represents an action in a chat (such as user joined, left, or new pin).
+
+    Args:
+        chats (:obj:`entity`, optional):
+            May be one or more entities (username/peer/etc.). By default,
+            only matching chats will be handled.
+
+        blacklist_chats (:obj:`bool`, optional):
+            Whether to treat the the list of chats as a blacklist (if
+            it matches it will NOT be handled) or a whitelist (default).
+
+    """
+    def __init__(self, chats=None, blacklist_chats=False):
+        # TODO This can probably be reused in all builders
+        self.chats = chats
+        self.blacklist_chats = blacklist_chats
+
+    async def resolve(self, client):
+        if hasattr(self.chats, '__iter__') and not isinstance(self.chats, str):
+            self.chats = set(utils.get_peer_id(x)
+                             for x in await client.get_input_entity(self.chats))
+        elif self.chats is not None:
+            self.chats = {utils.get_peer_id(
+                          await client.get_input_entity(self.chats))}
+
+    def build(self, update):
+        if isinstance(update, types.UpdateChannelPinnedMessage):
+            # Telegram sends UpdateChannelPinnedMessage and then
+            # UpdateNewChannelMessage with MessageActionPinMessage.
+            event = ChatAction.Event(types.PeerChannel(update.channel_id),
+                                     new_pin=update.id)
+
+        elif isinstance(update, types.UpdateChatParticipantAdd):
+            event = ChatAction.Event(types.PeerChat(update.chat_id),
+                                     added_by=update.inviter_id or True,
+                                     users=update.user_id)
+
+        elif isinstance(update, types.UpdateChatParticipantDelete):
+            event = ChatAction.Event(types.PeerChat(update.chat_id),
+                                     kicked_by=True,
+                                     users=update.user_id)
+
+        elif (isinstance(update, (
+                types.UpdateNewMessage, types.UpdateNewChannelMessage))
+              and isinstance(update.message, types.MessageService)):
+            msg = update.message
+            action = update.message.action
+            if isinstance(action, types.MessageActionChatJoinedByLink):
+                event = ChatAction.Event(msg.to_id,
+                                         added_by=True,
+                                         users=msg.from_id)
+            elif isinstance(action, types.MessageActionChatAddUser):
+                event = ChatAction.Event(msg.to_id,
+                                         added_by=msg.from_id or True,
+                                         users=action.users)
+            elif isinstance(action, types.MessageActionChatDeleteUser):
+                event = ChatAction.Event(msg.to_id,
+                                         kicked_by=msg.from_id or True,
+                                         users=action.user_id)
+            elif isinstance(action, types.MessageActionChatCreate):
+                event = ChatAction.Event(msg.to_id,
+                                         users=action.users,
+                                         created=True,
+                                         new_title=action.title)
+            elif isinstance(action, types.MessageActionChannelCreate):
+                event = ChatAction.Event(msg.to_id,
+                                         created=True,
+                                         new_title=action.title)
+            elif isinstance(action, types.MessageActionChatEditTitle):
+                event = ChatAction.Event(msg.to_id,
+                                         new_title=action.title)
+            elif isinstance(action, types.MessageActionChatEditPhoto):
+                event = ChatAction.Event(msg.to_id,
+                                         new_photo=action.photo)
+            elif isinstance(action, types.MessageActionChatDeletePhoto):
+                event = ChatAction.Event(msg.to_id,
+                                         new_photo=True)
+            else:
+                return
+        else:
+            return
+
+        if self.chats is None:
+            return event
+        else:
+            inside = utils.get_peer_id(event._chat_peer) in self.chats
+            if inside == self.blacklist_chats:
+                # If this chat matches but it's a blacklist ignore.
+                # If it doesn't match but it's a whitelist ignore.
+                return
+
+        return event
+
+    class Event(_EventCommon):
+        """
+        Represents the event of a new chat action.
+
+        Members:
+            new_pin (:obj:`bool`):
+                ``True`` if the pin has changed (new pin or removed).
+
+            new_photo (:obj:`bool`):
+                ``True`` if there's a new chat photo (or it was removed).
+
+            photo (:obj:`Photo`, optional):
+                The new photo (or ``None`` if it was removed).
+
+
+            user_added (:obj:`bool`):
+                ``True`` if the user was added by some other.
+
+            user_joined (:obj:`bool`):
+                ``True`` if the user joined on their own.
+
+            user_left (:obj:`bool`):
+                ``True`` if the user left on their own.
+
+            user_kicked (:obj:`bool`):
+                ``True`` if the user was kicked by some other.
+
+            created (:obj:`bool`, optional):
+                ``True`` if this chat was just created.
+
+            new_title (:obj:`bool`, optional):
+                The new title string for the chat, if applicable.
+        """
+        def __init__(self, chat_peer, new_pin=None, new_photo=None,
+                     added_by=None, kicked_by=None, created=None,
+                     users=None, new_title=None):
+            super().__init__(chat_peer=chat_peer, msg_id=new_pin)
+
+            self.new_pin = isinstance(new_pin, int)
+            self._pinned_message = new_pin
+
+            self.new_photo = new_photo is not None
+            self.photo = \
+                new_photo if isinstance(new_photo, types.Photo) else None
+
+            self._added_by = None
+            self._kicked_by = None
+            self.user_added, self.user_joined, self.user_left,\
+                self.user_kicked = (False, False, False, False)
+
+            if added_by is True:
+                self.user_joined = True
+            elif added_by:
+                self.user_added = True
+                self._added_by = added_by
+
+            if kicked_by is True:
+                self.user_left = True
+            elif kicked_by:
+                self.user_kicked = True
+                self._kicked_by = kicked_by
+
+            self.created = bool(created)
+            self._user_peers = users if isinstance(users, list) else [users]
+            self._users = None
+            self.new_title = new_title
+
+        @property
+        async def pinned_message(self):
+            """
+            If ``new_pin`` is ``True``, this returns the (:obj:`Message`)
+            object that was pinned.
+            """
+            if self._pinned_message == 0:
+                return None
+
+            if isinstance(self._pinned_message, int) and await self.input_chat:
+                r = await self._client(functions.channels.GetMessagesRequest(
+                    self._input_chat, [self._pinned_message]
+                ))
+                try:
+                    self._pinned_message = next(
+                        x for x in r.messages
+                        if isinstance(x, types.Message)
+                        and x.id == self._pinned_message
+                    )
+                except StopIteration:
+                    pass
+
+            if isinstance(self._pinned_message, types.Message):
+                return self._pinned_message
+
+        @property
+        async def added_by(self):
+            """
+            The user who added ``users``, if applicable (``None`` otherwise).
+            """
+            if self._added_by and not isinstance(self._added_by, types.User):
+                self._added_by = await self._client.get_entity(self._added_by)
+            return self._added_by
+
+        @property
+        async def kicked_by(self):
+            """
+            The user who kicked ``users``, if applicable (``None`` otherwise).
+            """
+            if self._kicked_by and not isinstance(self._kicked_by, types.User):
+                self._kicked_by = await self._client.get_entity(self._kicked_by)
+            return self._kicked_by
+
+        @property
+        async def user(self):
+            """
+            The single user that takes part in this action (e.g. joined).
+
+            Might be ``None`` if the information can't be retrieved or
+            there is no user taking part.
+            """
+            try:
+                return next(await self.users)
+            except (StopIteration, TypeError):
+                return None
+
+        @property
+        async def users(self):
+            """
+            A list of users that take part in this action (e.g. joined).
+
+            Might be empty if the information can't be retrieved or there
+            are no users taking part.
+            """
+            if self._users is None and self._user_peers:
+                try:
+                    self._users = await self._client.get_entity(self._user_peers)
+                except (TypeError, ValueError):
+                    self._users = []
+
+            return self._users
+
+
+class UserUpdate(_EventBuilder):
+    """
+    Represents an user update (gone online, offline, joined Telegram).
+    """
+
+    def build(self, update):
+        if isinstance(update, types.UpdateUserStatus):
+            event = UserUpdate.Event(update.user_id,
+                                     status=update.status)
+        else:
+            return
+
+        return event
+
+    async def resolve(self, client):
+        pass
+
+    class Event(_EventCommon):
+        """
+        Represents the event of an user status update (last seen, joined).
+
+        Members:
+            online (:obj:`bool`, optional):
+                ``True`` if the user is currently online, ``False`` otherwise.
+                Might be ``None`` if this information is not present.
+
+            last_seen (:obj:`datetime`, optional):
+                Exact date when the user was last seen if known.
+
+            until (:obj:`datetime`, optional):
+                Until when will the user remain online.
+
+            within_months (:obj:`bool`):
+                ``True`` if the user was seen within 30 days.
+
+            within_weeks (:obj:`bool`):
+                ``True`` if the user was seen within 7 days.
+
+            recently (:obj:`bool`):
+                ``True`` if the user was seen within a day.
+
+            action (:obj:`SendMessageAction`, optional):
+                The "typing" action if any the user is performing if any.
+
+            cancel (:obj:`bool`):
+                ``True`` if the action was cancelling other actions.
+
+            typing (:obj:`bool`):
+                ``True`` if the action is typing a message.
+
+            recording (:obj:`bool`):
+                ``True`` if the action is recording something.
+
+            uploading (:obj:`bool`):
+                ``True`` if the action is uploading something.
+
+            playing (:obj:`bool`):
+                ``True`` if the action is playing a game.
+
+            audio (:obj:`bool`):
+                ``True`` if what's being recorded/uploaded is an audio.
+
+            round (:obj:`bool`):
+                ``True`` if what's being recorded/uploaded is a round video.
+
+            video (:obj:`bool`):
+                ``True`` if what's being recorded/uploaded is an video.
+
+            document (:obj:`bool`):
+                ``True`` if what's being uploaded is document.
+
+            geo (:obj:`bool`):
+                ``True`` if what's being uploaded is a geo.
+
+            photo (:obj:`bool`):
+                ``True`` if what's being uploaded is a photo.
+
+            contact (:obj:`bool`):
+                ``True`` if what's being uploaded (selected) is a contact.
+        """
+        def __init__(self, user_id, status=None, typing=None):
+            super().__init__(types.PeerUser(user_id))
+
+            self.online = None if status is None else \
+                isinstance(status, types.UserStatusOnline)
+
+            self.last_seen = status.was_online if \
+                isinstance(status, types.UserStatusOffline) else None
+
+            self.until = status.expires if \
+                isinstance(status, types.UserStatusOnline) else None
+
+            if self.last_seen:
+                diff = datetime.datetime.now() - self.last_seen
+                if diff < datetime.timedelta(days=30):
+                    self.within_months = True
+                    if diff < datetime.timedelta(days=7):
+                        self.within_weeks = True
+                        if diff < datetime.timedelta(days=1):
+                            self.recently = True
+            else:
+                self.within_months = self.within_weeks = self.recently = False
+                if isinstance(status, (types.UserStatusOnline,
+                                       types.UserStatusRecently)):
+                    self.within_months = self.within_weeks = True
+                    self.recently = True
+                elif isinstance(status, types.UserStatusLastWeek):
+                    self.within_months = self.within_weeks = True
+                elif isinstance(status, types.UserStatusLastMonth):
+                    self.within_months = True
+
+            self.action = typing
+            if typing:
+                self.cancel = self.typing = self.recording = self.uploading = \
+                    self.playing = False
+                self.audio = self.round = self.video = self.document = \
+                    self.geo = self.photo = self.contact = False
+
+                if isinstance(typing, types.SendMessageCancelAction):
+                    self.cancel = True
+                elif isinstance(typing, types.SendMessageTypingAction):
+                    self.typing = True
+                elif isinstance(typing, types.SendMessageGamePlayAction):
+                    self.playing = True
+                elif isinstance(typing, types.SendMessageGeoLocationAction):
+                    self.geo = True
+                elif isinstance(typing, types.SendMessageRecordAudioAction):
+                    self.recording = self.audio = True
+                elif isinstance(typing, types.SendMessageRecordRoundAction):
+                    self.recording = self.round = True
+                elif isinstance(typing, types.SendMessageRecordVideoAction):
+                    self.recording = self.video = True
+                elif isinstance(typing, types.SendMessageChooseContactAction):
+                    self.uploading = self.contact = True
+                elif isinstance(typing, types.SendMessageUploadAudioAction):
+                    self.uploading = self.audio = True
+                elif isinstance(typing, types.SendMessageUploadDocumentAction):
+                    self.uploading = self.document = True
+                elif isinstance(typing, types.SendMessageUploadPhotoAction):
+                    self.uploading = self.photo = True
+                elif isinstance(typing, types.SendMessageUploadRoundAction):
+                    self.uploading = self.round = True
+                elif isinstance(typing, types.SendMessageUploadVideoAction):
+                    self.uploading = self.video = True
+
+        @property
+        def user(self):
+            """Alias around the chat (conversation)."""
+            return self.chat
+
+
+class MessageChanged(_EventBuilder):
+    """
+    Represents a message changed (edited or deleted).
+    """
+
+    def build(self, update):
+        if isinstance(update, (types.UpdateEditMessage,
+                               types.UpdateEditChannelMessage)):
+            event = MessageChanged.Event(edit_msg=update.message)
+        elif isinstance(update, (types.UpdateDeleteMessages,
+                                 types.UpdateDeleteChannelMessages)):
+            event = MessageChanged.Event(
+                deleted_ids=update.messages,
+                peer=types.PeerChannel(update.channel_id)
+            )
+        else:
+            return
+
+        return event
+
+    async def resolve(self, client):
+        pass
+
+    class Event(_EventCommon):
+        """
+        Represents the event of an user status update (last seen, joined).
+
+        Members:
+            edited (:obj:`bool`):
+                ``True`` if the message was edited.
+
+            message (:obj:`Message`, optional):
+                The new edited message, if any.
+
+            deleted (:obj:`bool`):
+                ``True`` if the message IDs were deleted.
+
+            deleted_ids (:obj:`List[int]`):
+                A list containing the IDs of the messages that were deleted.
+
+            input_sender (:obj:`InputPeer`):
+                This is the input version of the user who edited the message.
+                Similarly to ``input_chat``, this doesn't have things like
+                username or similar, but still useful in some cases.
+
+                Note that this might not be available if the library can't
+                find the input chat.
+
+            sender (:obj:`User`):
+                This property will make an API call the first time to get the
+                most up to date version of the sender, so use with care as
+                there is no caching besides local caching yet.
+
+                ``input_sender`` needs to be available (often the case).
+        """
+        def __init__(self, edit_msg=None, deleted_ids=None, peer=None):
+            super().__init__(peer if not edit_msg else edit_msg.to_id)
+
+            self.edited = bool(edit_msg)
+            self.message = edit_msg
+            self.deleted = bool(deleted_ids)
+            self.deleted_ids = deleted_ids or []
+            self._input_sender = None
+            self._sender = None
+
+        @property
+        async def input_sender(self):
+            """
+            This (:obj:`InputPeer`) is the input version of the user who
+            sent the message. Similarly to ``input_chat``, this doesn't have
+            things like username or similar, but still useful in some cases.
+
+            Note that this might not be available if the library can't
+            find the input chat.
+            """
+            # TODO Code duplication
+            if self._input_sender is None and self.message:
+                try:
+                    self._input_sender = await self._client.get_input_entity(
+                        self.message.from_id
+                    )
+                except (ValueError, TypeError):
+                    if isinstance(self.message.to_id, types.PeerChannel):
+                        # We can rely on self.input_chat for this
+                        self._input_sender = await self._get_input_entity(
+                            self.message.id,
+                            self.message.from_id,
+                            chat=await self.input_chat
+                        )
+
+            return self._input_sender
+
+        @property
+        async def sender(self):
+            """
+            This (:obj:`User`) will make an API call the first time to get
+            the most up to date version of the sender, so use with care as
+            there is no caching besides local caching yet.
+
+            ``input_sender`` needs to be available (often the case).
+            """
+            if self._sender is None and await self.input_sender:
+                self._sender = await self._client.get_entity(self._input_sender)
+            return self._sender

telethon/extensions/binary_reader.py

@@ -56,8 +56,11 @@ class BinaryReader:
         return int.from_bytes(
             self.read(bits // 8), byteorder='little', signed=signed)
 
-    def read(self, length):
+    def read(self, length=None):
         """Read the given amount of bytes."""
+        if length is None:
+            return self.reader.read()
+
         result = self.reader.read(length)
         if len(result) != length:
             raise BufferError(
@@ -130,6 +133,8 @@ class BinaryReader:
                 return True
             elif value == 0xbc799737:  # boolFalse
                 return False
+            elif value == 0x1cb5c415:  # Vector
+                return [self.tgread_object() for _ in range(self.read_int())]
 
             # If there was still no luck, give up
             self.seek(-4)  # Go back

telethon/extensions/html.py

@@ -0,0 +1,167 @@
+"""
+Simple HTML -> Telegram entity parser.
+"""
+from html import escape, unescape
+from html.parser import HTMLParser
+from collections import deque
+
+from ..tl.types import (
+    MessageEntityBold, MessageEntityItalic, MessageEntityCode,
+    MessageEntityPre, MessageEntityEmail, MessageEntityUrl,
+    MessageEntityTextUrl
+)
+
+
+class HTMLToTelegramParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.text = ''
+        self.entities = []
+        self._building_entities = {}
+        self._open_tags = deque()
+        self._open_tags_meta = deque()
+
+    def handle_starttag(self, tag, attrs):
+        self._open_tags.appendleft(tag)
+        self._open_tags_meta.appendleft(None)
+
+        attrs = dict(attrs)
+        EntityType = None
+        args = {}
+        if tag == 'strong' or tag == 'b':
+            EntityType = MessageEntityBold
+        elif tag == 'em' or tag == 'i':
+            EntityType = MessageEntityItalic
+        elif tag == 'code':
+            try:
+                # If we're in the middle of a <pre> tag, this <code> tag is
+                # probably intended for syntax highlighting.
+                #
+                # Syntax highlighting is set with
+                #     <code class='language-...'>codeblock</code>
+                # inside <pre> tags
+                pre = self._building_entities['pre']
+                try:
+                    pre.language = attrs['class'][len('language-'):]
+                except KeyError:
+                    pass
+            except KeyError:
+                EntityType = MessageEntityCode
+        elif tag == 'pre':
+            EntityType = MessageEntityPre
+            args['language'] = ''
+        elif tag == 'a':
+            try:
+                url = attrs['href']
+            except KeyError:
+                return
+            if url.startswith('mailto:'):
+                url = url[len('mailto:'):]
+                EntityType = MessageEntityEmail
+            else:
+                if self.get_starttag_text() == url:
+                    EntityType = MessageEntityUrl
+                else:
+                    EntityType = MessageEntityTextUrl
+                    args['url'] = url
+                    url = None
+            self._open_tags_meta.popleft()
+            self._open_tags_meta.appendleft(url)
+
+        if EntityType and tag not in self._building_entities:
+            self._building_entities[tag] = EntityType(
+                offset=len(self.text),
+                # The length will be determined when closing the tag.
+                length=0,
+                **args)
+
+    def handle_data(self, text):
+        text = unescape(text)
+
+        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ''
+        if previous_tag == 'a':
+            url = self._open_tags_meta[0]
+            if url:
+                text = url
+
+        for tag, entity in self._building_entities.items():
+            entity.length += len(text.strip('\n'))
+
+        self.text += text
+
+    def handle_endtag(self, tag):
+        try:
+            self._open_tags.popleft()
+            self._open_tags_meta.popleft()
+        except IndexError:
+            pass
+        entity = self._building_entities.pop(tag, None)
+        if entity:
+            self.entities.append(entity)
+
+
+def parse(html):
+    """
+    Parses the given HTML message and returns its stripped representation
+    plus a list of the MessageEntity's that were found.
+
+    :param message: the message with HTML to be parsed.
+    :return: a tuple consisting of (clean message, [message entities]).
+    """
+    parser = HTMLToTelegramParser()
+    parser.feed(html)
+    return parser.text, parser.entities
+
+
+def unparse(text, entities):
+    """
+    Performs the reverse operation to .parse(), effectively returning HTML
+    given a normal text and its MessageEntity's.
+
+    :param text: the text to be reconverted into HTML.
+    :param entities: the MessageEntity's applied to the text.
+    :return: a HTML representation of the combination of both inputs.
+    """
+    if not entities:
+        return text
+    html = []
+    last_offset = 0
+    for entity in entities:
+        if entity.offset > last_offset:
+            html.append(escape(text[last_offset:entity.offset]))
+        elif entity.offset < last_offset:
+            continue
+
+        skip_entity = False
+        entity_text = escape(text[entity.offset:entity.offset + entity.length])
+        entity_type = type(entity)
+
+        if entity_type == MessageEntityBold:
+            html.append('<strong>{}</strong>'.format(entity_text))
+        elif entity_type == MessageEntityItalic:
+            html.append('<em>{}</em>'.format(entity_text))
+        elif entity_type == MessageEntityCode:
+            html.append('<code>{}</code>'.format(entity_text))
+        elif entity_type == MessageEntityPre:
+            if entity.language:
+                html.append(
+                    "<pre>\n"
+                    "    <code class='language-{}'>\n"
+                    "        {}\n"
+                    "    </code>\n"
+                    "</pre>".format(entity.language, entity_text))
+            else:
+                html.append('<pre><code>{}</code></pre>'
+                            .format(entity_text))
+        elif entity_type == MessageEntityEmail:
+            html.append('<a href="mailto:{0}">{0}</a>'.format(entity_text))
+        elif entity_type == MessageEntityUrl:
+            html.append('<a href="{0}">{0}</a>'.format(entity_text))
+        elif entity_type == MessageEntityTextUrl:
+            html.append('<a href="{}">{}</a>'
+                        .format(escape(entity.url), entity_text))
+        else:
+            skip_entity = True
+        last_offset = entity.offset + (0 if skip_entity else entity.length)
+    html.append(text[last_offset:])
+    return ''.join(html)

telethon/extensions/markdown.py

@@ -4,6 +4,7 @@ for use within the library, which attempts to handle emojies correctly,
 since they seem to count as two characters and it's a bit strange.
 """
 import re
+import struct
 
 from ..tl import TLObject
 
@@ -20,15 +21,24 @@ DEFAULT_DELIMITERS = {
     '```': MessageEntityPre
 }
 
-# Regex used to match utf-16le encoded r'\[(.+?)\]\((.+?)\)',
+# Regex used to match r'\[(.+?)\]\((.+?)\)' (for URLs.
-# reason why there's '\0' after every match-literal character.
+DEFAULT_URL_RE = re.compile(r'\[(.+?)\]\((.+?)\)')
-DEFAULT_URL_RE = re.compile(b'\\[\0(.+?)\\]\0\\(\0(.+?)\\)\0')
 
 # Reverse operation for DEFAULT_URL_RE. {0} for text, {1} for URL.
 DEFAULT_URL_FORMAT = '[{0}]({1})'
 
-# Encoding to be used
+
-ENC = 'utf-16le'
+def _add_surrogate(text):
+    return ''.join(
+        # SMP -> Surrogate Pairs (Telegram offsets are calculated with these).
+        # See https://en.wikipedia.org/wiki/Plane_(Unicode)#Overview for more.
+        ''.join(chr(y) for y in struct.unpack('<HH', x.encode('utf-16le')))
+        if (0x10000 <= ord(x) <= 0x10FFFF) else x for x in text
+    )
+
+
+def _del_surrogate(text):
+    return text.encode('utf-16', 'surrogatepass').decode('utf-16')
 
 
 def parse(message, delimiters=None, url_re=None):
@@ -43,17 +53,14 @@ def parse(message, delimiters=None, url_re=None):
     """
     if url_re is None:
         url_re = DEFAULT_URL_RE
-    elif url_re:
+    elif isinstance(url_re, str):
-        if isinstance(url_re, bytes):
+        url_re = re.compile(url_re)
-            url_re = re.compile(url_re)
 
     if not delimiters:
         if delimiters is not None:
             return message, []
         delimiters = DEFAULT_DELIMITERS
 
-    delimiters = {k.encode(ENC): v for k, v in delimiters.items()}
-
     # Cannot use a for loop because we need to skip some indices
     i = 0
     result = []
@@ -62,7 +69,7 @@ def parse(message, delimiters=None, url_re=None):
 
     # Work on byte level with the utf-16le encoding to get the offsets right.
     # The offset will just be half the index we're at.
-    message = message.encode(ENC)
+    message = _add_surrogate(message)
     while i < len(message):
         if url_re and current is None:
             # If we're not inside a previous match since Telegram doesn't allow
@@ -70,15 +77,15 @@ def parse(message, delimiters=None, url_re=None):
             url_match = url_re.match(message, pos=i)
             if url_match:
                 # Replace the whole match with only the inline URL text.
-                message = b''.join((
+                message = ''.join((
                     message[:url_match.start()],
                     url_match.group(1),
                     message[url_match.end():]
                 ))
 
                 result.append(MessageEntityTextUrl(
-                    offset=i // 2, length=len(url_match.group(1)) // 2,
+                    offset=i, length=len(url_match.group(1)),
-                    url=url_match.group(2).decode(ENC)
+                    url=_del_surrogate(url_match.group(2))
                 ))
                 i += len(url_match.group(1))
                 # Next loop iteration, don't check delimiters, since
@@ -103,16 +110,16 @@ def parse(message, delimiters=None, url_re=None):
                 message = message[:i] + message[i + len(d):]
                 if m == MessageEntityPre:
                     # Special case, also has 'lang'
-                    current = m(i // 2, None, '')
+                    current = m(i, None, '')
                 else:
-                    current = m(i // 2, None)
+                    current = m(i, None)
 
                 end_delimiter = d  # We expect the same delimiter.
                 break
 
         elif message[i:i + len(end_delimiter)] == end_delimiter:
             message = message[:i] + message[i + len(end_delimiter):]
-            current.length = (i // 2) - current.offset
+            current.length = i - current.offset
             result.append(current)
             current, end_delimiter = None, None
             # Don't increment i here as we matched a delimiter,
@@ -121,19 +128,19 @@ def parse(message, delimiters=None, url_re=None):
             # as we already know there won't be the same right after.
             continue
 
-        # Next iteration, utf-16 encoded characters need 2 bytes.
+        # Next iteration
-        i += 2
+        i += 1
 
     # We may have found some a delimiter but not its ending pair.
     # If this is the case, we want to insert the delimiter character back.
     if current is not None:
         message = (
-            message[:2 * current.offset]
+            message[:current.offset]
             + end_delimiter
-            + message[2 * current.offset:]
+            + message[current.offset:]
         )
 
-    return message.decode(ENC), result
+    return _del_surrogate(message), result
 
 
 def unparse(text, entities, delimiters=None, url_fmt=None):
@@ -158,29 +165,21 @@ def unparse(text, entities, delimiters=None, url_fmt=None):
     else:
         entities = tuple(sorted(entities, key=lambda e: e.offset, reverse=True))
 
-    # Reverse the delimiters, and encode them as utf16
+    text = _add_surrogate(text)
-    delimiters = {v: k.encode(ENC) for k, v in delimiters.items()}
-    text = text.encode(ENC)
     for entity in entities:
-        s = entity.offset * 2
+        s = entity.offset
-        e = (entity.offset + entity.length) * 2
+        e = entity.offset + entity.length
         delimiter = delimiters.get(type(entity), None)
         if delimiter:
             text = text[:s] + delimiter + text[s:e] + delimiter + text[e:]
         elif isinstance(entity, MessageEntityTextUrl) and url_fmt:
-            # If byte-strings supported .format(), we, could have converted
-            # the str url_fmt to a byte-string with the following regex:
-            # re.sub(b'{\0\s*(?:([01])\0)?\s*}\0',rb'{\1}',url_fmt.encode(ENC))
-            #
-            # This would preserve {}, {0} and {1}.
-            # Alternatively (as it's done), we can decode/encode it every time.
             text = (
                 text[:s] +
-                url_fmt.format(text[s:e].decode(ENC), entity.url).encode(ENC) +
+                _add_surrogate(url_fmt.format(text[s:e], entity.url)) +
                 text[e:]
             )
 
-    return text.decode(ENC)
+    return _del_surrogate(text)
 
 
 def get_inner_text(text, entity):
@@ -198,11 +197,11 @@ def get_inner_text(text, entity):
     else:
         multiple = True
 
-    text = text.encode(ENC)
+    text = _add_surrogate(text)
     result = []
     for e in entity:
-        start = e.offset * 2
+        start = e.offset
-        end = (e.offset + e.length) * 2
+        end = e.offset + e.length
-        result.append(text[start:end].decode(ENC))
+        result.append(_del_surrogate(text[start:end]))
 
     return result if multiple else result[0]

telethon/extensions/tcp_client.py

@@ -5,6 +5,7 @@ This module holds a rough implementation of the C# TCP client.
 import asyncio
 import errno
 import socket
+import time
 from datetime import timedelta
 from io import BytesIO, BufferedWriter
 
@@ -14,6 +15,12 @@ CONN_RESET_ERRNOS = {
     errno.EINVAL, errno.ENOTCONN
 }
 
+MAX_TIMEOUT = 15  # in seconds
+CONN_RESET_ERRNOS = {
+    errno.EBADF, errno.ENOTSOCK, errno.ENETUNREACH,
+    errno.EINVAL, errno.ENOTCONN
+}
+
 
 class TcpClient:
     """A simple TCP client to ease the work with sockets and proxies."""
@@ -56,12 +63,7 @@ class TcpClient:
         :param port: the port to connect to.
         """
         if ':' in ip:  # IPv6
-            # The address needs to be surrounded by [] as discussed on PR#425
+            ip = ip.replace('[', '').replace(']', '')
-            if not ip.startswith('['):
-                ip = '[' + ip
-            if not ip.endswith(']'):
-                ip = ip + ']'
-
             mode, address = socket.AF_INET6, (ip, port, 0, 0)
         else:
             mode, address = socket.AF_INET, (ip, port)
@@ -81,7 +83,9 @@ class TcpClient:
             except OSError as e:
                 # There are some errors that we know how to handle, and
                 # the loop will allow us to retry
-                if e.errno in (errno.EBADF, errno.ENOTSOCK, errno.EINVAL):
+                if e.errno in (errno.EBADF, errno.ENOTSOCK, errno.EINVAL,
+                               errno.ECONNREFUSED,  # Windows-specific follow
+                               getattr(errno, 'WSAEACCES', None)):
                     # Bad file descriptor, i.e. socket was closed, set it
                     # to none to recreate it on the next iteration
                     self._socket = None
@@ -139,6 +143,8 @@ class TcpClient:
         :param size: the size of the block to be read.
         :return: the read data with len(data) == size.
         """
+        if self._socket is None:
+            self._raise_connection_reset()
 
         with BufferedWriter(BytesIO(), buffer_size=size) as buffer:
             bytes_left = size

telethon/helpers.py

@@ -1,6 +1,12 @@
 """Various helpers not related to the Telegram API itself"""
-from hashlib import sha1, sha256
 import os
+import struct
+from hashlib import sha1, sha256
+
+from telethon.crypto import AES
+from telethon.errors import SecurityError
+from telethon.extensions import BinaryReader
+
 
 # region Multiple utilities
 
@@ -21,28 +27,68 @@ def ensure_parent_dir_exists(file_path):
 # region Cryptographic related utils
 
 
-def calc_key(shared_key, msg_key, client):
+def pack_message(session, message):
-    """Calculate the key based on Telegram guidelines,
+    """Packs a message following MtProto 2.0 guidelines"""
-       specifying whether it's the client or not
+    # See https://core.telegram.org/mtproto/description
+    data = struct.pack('<qq', session.salt, session.id) + bytes(message)
+    padding = os.urandom(-(len(data) + 12) % 16 + 12)
+
+    # Being substr(what, offset, length); x = 0 for client
+    # "msg_key_large = SHA256(substr(auth_key, 88+x, 32) + pt + padding)"
+    msg_key_large = sha256(
+        session.auth_key.key[88:88 + 32] + data + padding).digest()
+
+    # "msg_key = substr (msg_key_large, 8, 16)"
+    msg_key = msg_key_large[8:24]
+    aes_key, aes_iv = calc_key(session.auth_key.key, msg_key, True)
+
+    key_id = struct.pack('<Q', session.auth_key.key_id)
+    return key_id + msg_key + AES.encrypt_ige(data + padding, aes_key, aes_iv)
+
+
+def unpack_message(session, reader):
+    """Unpacks a message following MtProto 2.0 guidelines"""
+    # See https://core.telegram.org/mtproto/description
+    if reader.read_long(signed=False) != session.auth_key.key_id:
+        raise SecurityError('Server replied with an invalid auth key')
+
+    msg_key = reader.read(16)
+    aes_key, aes_iv = calc_key(session.auth_key.key, msg_key, False)
+    data = BinaryReader(AES.decrypt_ige(reader.read(), aes_key, aes_iv))
+
+    data.read_long()  # remote_salt
+    if data.read_long() != session.id:
+        raise SecurityError('Server replied with a wrong session ID')
+
+    remote_msg_id = data.read_long()
+    remote_sequence = data.read_int()
+    msg_len = data.read_int()
+    message = data.read(msg_len)
+
+    # https://core.telegram.org/mtproto/security_guidelines
+    # Sections "checking sha256 hash" and "message length"
+    if msg_key != sha256(
+            session.auth_key.key[96:96 + 32] + data.get_bytes()).digest()[8:24]:
+        raise SecurityError("Received msg_key doesn't match with expected one")
+
+    return message, remote_msg_id, remote_sequence
+
+
+def calc_key(auth_key, msg_key, client):
     """
+    Calculate the key based on Telegram guidelines
+    for MtProto 2, specifying whether it's the client or not.
+    """
+    # https://core.telegram.org/mtproto/description#defining-aes-key-and-initialization-vector
     x = 0 if client else 8
 
-    sha1a = sha1(msg_key + shared_key[x:x + 32]).digest()
+    sha256a = sha256(msg_key + auth_key[x: x + 36]).digest()
-    sha1b = sha1(shared_key[x + 32:x + 48] + msg_key +
+    sha256b = sha256(auth_key[x + 40:x + 76] + msg_key).digest()
-                 shared_key[x + 48:x + 64]).digest()
 
-    sha1c = sha1(shared_key[x + 64:x + 96] + msg_key).digest()
+    aes_key = sha256a[:8] + sha256b[8:24] + sha256a[24:32]
-    sha1d = sha1(msg_key + shared_key[x + 96:x + 128]).digest()
+    aes_iv = sha256b[:8] + sha256a[8:24] + sha256b[24:32]
 
-    key = sha1a[0:8] + sha1b[8:20] + sha1c[4:16]
+    return aes_key, aes_iv
-    iv = sha1a[8:20] + sha1b[0:8] + sha1c[16:20] + sha1d[0:8]
-
-    return key, iv
-
-
-def calc_msg_key(data):
-    """Calculates the message key from the given data"""
-    return sha1(data).digest()[4:20]
 
 
 def generate_key_data_from_nonce(server_nonce, new_nonce):

telethon/network/connection.py

@@ -3,6 +3,7 @@ This module holds both the Connection class and the ConnectionMode enum,
 which specifies the protocol to be used by the Connection.
 """
 import errno
+import logging
 import os
 import struct
 from datetime import timedelta
@@ -13,6 +14,8 @@ from ..crypto import AESModeCTR
 from ..errors import InvalidChecksumError
 from ..extensions import TcpClient
 
+__log__ = logging.getLogger(__name__)
+
 
 class ConnectionMode(Enum):
     """Represents which mode should be used to stabilise a connection.
@@ -183,6 +186,21 @@ class Connection:
         packet_len_seq = await self.read(8)  # 4 and 4
         packet_len, seq = struct.unpack('<ii', packet_len_seq)
 
+        # Sometimes Telegram seems to send a packet length of 0 (12)
+        # and after that, just a single byte. Not sure what this is.
+        # TODO Figure out what this is, and if there's a better fix.
+        if packet_len <= 12:
+            __log__.error('Read invalid packet length %d, '
+                          'reading data left:', packet_len)
+            while True:
+                try:
+                    __log__.error(repr(await self.read(1)))
+                except TimeoutError:
+                    break
+            # Connection reset and hope it's fixed after
+            self.conn.close()
+            raise ConnectionResetError()
+
         body = await self.read(packet_len - 12)
         checksum = struct.unpack('<I', await self.read(4))[0]
 

telethon/network/mtproto_sender.py

@@ -2,14 +2,12 @@
 This module contains the class used to communicate with Telegram's servers
 encrypting every packet, and relies on a valid AuthKey in the used Session.
 """
+import asyncio
 import gzip
 import logging
-import struct
-import asyncio
 from asyncio import Event
 
 from .. import helpers as utils
-from ..crypto import AES
 from ..errors import (
     BadMessageError, InvalidChecksumError, BrokenAuthKeyError,
     rpc_message_to_error
@@ -17,11 +15,11 @@ from ..errors import (
 from ..extensions import BinaryReader
 from ..tl import TLMessage, MessageContainer, GzipPacked
 from ..tl.all_tlobjects import tlobjects
+from ..tl.functions.auth import LogOutRequest
 from ..tl.types import (
-    MsgsAck, Pong, BadServerSalt, BadMsgNotification,
+    MsgsAck, Pong, BadServerSalt, BadMsgNotification, FutureSalts,
     MsgNewDetailedInfo, NewSessionCreated, MsgDetailedInfo
 )
-from ..tl.functions.auth import LogOutRequest
 
 __log__ = logging.getLogger(__name__)
 
@@ -155,17 +153,7 @@ class MtProtoSender:
 
         :param message: the TLMessage to be sent.
         """
-        plain_text = \
+        await self.connection.send(utils.pack_message(self.session, message))
-            struct.pack('<qq', self.session.salt, self.session.id) \
-            + bytes(message)
-
-        msg_key = utils.calc_msg_key(plain_text)
-        key_id = struct.pack('<Q', self.session.auth_key.key_id)
-        key, iv = utils.calc_key(self.session.auth_key.key, msg_key, True)
-        cipher_text = AES.encrypt_ige(plain_text, key, iv)
-
-        result = key_id + msg_key + cipher_text
-        await self.connection.send(result)
 
     def _decode_msg(self, body):
         """
@@ -174,34 +162,14 @@ class MtProtoSender:
         :param body: the body to be decoded.
         :return: a tuple of (decoded message, remote message id, remote seq).
         """
-        message = None
+        if len(body) < 8:
-        remote_msg_id = None
+            if body == b'l\xfe\xff\xff':
-        remote_sequence = None
+                raise BrokenAuthKeyError()
+            else:
+                raise BufferError("Can't decode packet ({})".format(body))
 
         with BinaryReader(body) as reader:
-            if len(body) < 8:
+            return utils.unpack_message(self.session, reader)
-                if body == b'l\xfe\xff\xff':
-                    raise BrokenAuthKeyError()
-                else:
-                    raise BufferError("Can't decode packet ({})".format(body))
-
-            # TODO Check for both auth key ID and msg_key correctness
-            reader.read_long()  # remote_auth_key_id
-            msg_key = reader.read(16)
-
-            key, iv = utils.calc_key(self.session.auth_key.key, msg_key, False)
-            plain_text = AES.decrypt_ige(
-                reader.read(len(body) - reader.tell_position()), key, iv)
-
-            with BinaryReader(plain_text) as plain_text_reader:
-                plain_text_reader.read_long()  # remote_salt
-                plain_text_reader.read_long()  # remote_session_id
-                remote_msg_id = plain_text_reader.read_long()
-                remote_sequence = plain_text_reader.read_int()
-                msg_len = plain_text_reader.read_int()
-                message = plain_text_reader.read(msg_len)
-
-        return message, remote_msg_id, remote_sequence
 
     async def _process_msg(self, msg_id, sequence, reader, state):
         """
@@ -271,6 +239,12 @@ class MtProtoSender:
 
             return True
 
+        if isinstance(obj, FutureSalts):
+            r = self._pop_request(obj.req_msg_id)
+            if r:
+                r.result = obj
+                r.confirm_received.set()
+
         # If the object isn't any of the above, then it should be an Update.
         self.session.process_entities(obj)
         if state:
@@ -343,7 +317,7 @@ class MtProtoSender:
         if requests:
             await self.send(*requests)
 
-    def _handle_pong(self, msg_id, sequence, pong):
+    async def _handle_pong(self, msg_id, sequence, pong):
         """
         Handles a Pong response.
 

telethon/session.py

@@ -2,19 +2,37 @@ import json
 import os
 import platform
 import sqlite3
+import struct
 import time
 from base64 import b64decode
+from enum import Enum
 from os.path import isfile as file_exists
 
-from .. import utils, helpers
+from . import utils
-from ..tl import TLObject
+from .crypto import AuthKey
-from ..tl.types import (
+from .tl import TLObject
+from .tl.types import (
     PeerUser, PeerChat, PeerChannel,
-    InputPeerUser, InputPeerChat, InputPeerChannel
+    InputPeerUser, InputPeerChat, InputPeerChannel,
+    InputPhoto, InputDocument
 )
 
 EXTENSION = '.session'
-CURRENT_VERSION = 2  # database version
+CURRENT_VERSION = 3  # database version
+
+
+class _SentFileType(Enum):
+    DOCUMENT = 0
+    PHOTO = 1
+
+    @staticmethod
+    def from_type(cls):
+        if cls == InputDocument:
+            return _SentFileType.DOCUMENT
+        elif cls == InputPhoto:
+            return _SentFileType.PHOTO
+        else:
+            raise ValueError('The cls must be either InputDocument/InputPhoto')
 
 
 class Session:
@@ -61,7 +79,7 @@ class Session:
             self.save_entities = True
             self.flood_sleep_threshold = 60
 
-        self.id = helpers.generate_random_long(signed=True)
+        self.id = struct.unpack('q', os.urandom(8))[0]
         self._sequence = 0
         self.time_offset = 0
         self._last_msg_id = 0  # Long
@@ -76,8 +94,8 @@ class Session:
         # Migrating from .json -> SQL
         entities = self._check_migrate_json()
 
-        self._conn = sqlite3.connect(self.filename, check_same_thread=False)
+        self._conn = None
-        c = self._conn.cursor()
+        c = self._cursor()
         c.execute("select name from sqlite_master "
                   "where type='table' and name='version'")
         if c.fetchone():
@@ -95,48 +113,47 @@ class Session:
             tuple_ = c.fetchone()
             if tuple_:
                 self._dc_id, self._server_address, self._port, key, = tuple_
-                from ..crypto import AuthKey
                 self._auth_key = AuthKey(data=key)
 
             c.close()
         else:
             # Tables don't exist, create new ones
-            c.execute("create table version (version integer)")
+            self._create_table(
-            c.execute("insert into version values (?)", (CURRENT_VERSION,))
+                c,
-            c.execute(
+                "version (version integer primary key)"
-                """create table sessions (
+                ,
+                """sessions (
                     dc_id integer primary key,
                     server_address text,
                     port integer,
                     auth_key blob
-                ) without rowid"""
+                )"""
-            )
+                ,
-            c.execute(
+                """entities (
-                """create table entities (
                     id integer primary key,
                     hash integer not null,
                     username text,
                     phone integer,
                     name text
-                ) without rowid"""
+                )"""
-            )
+                ,
-            # Save file_size along with md5_digest
+                """sent_files (
-            # to make collisions even more unlikely.
-            c.execute(
-                """create table sent_files (
                     md5_digest blob,
                     file_size integer,
-                    file_id integer,
+                    type integer,
-                    part_count integer,
+                    id integer,
-                    primary key(md5_digest, file_size)
+                    hash integer,
-                ) without rowid"""
+                    primary key(md5_digest, file_size, type)
+                )"""
             )
+            c.execute("insert into version values (?)", (CURRENT_VERSION,))
             # Migrating from JSON -> new table and may have entities
             if entities:
                 c.executemany(
                     'insert or replace into entities values (?,?,?,?,?)',
                     entities
                 )
+            self._update_session_table()
             c.close()
             self.save()
 
@@ -151,30 +168,46 @@ class Session:
                 self._server_address = \
                     data.get('server_address', self._server_address)
 
-                from ..crypto import AuthKey
                 if data.get('auth_key_data', None) is not None:
                     key = b64decode(data['auth_key_data'])
                     self._auth_key = AuthKey(data=key)
 
                 rows = []
                 for p_id, p_hash in data.get('entities', []):
-                    rows.append((p_id, p_hash, None, None, None))
+                    if p_hash is not None:
+                        rows.append((p_id, p_hash, None, None, None))
                 return rows
             except UnicodeDecodeError:
                 return []  # No entities
 
     def _upgrade_database(self, old):
-        if old == 1:
+        c = self._cursor()
-            self._conn.execute(
+        # old == 1 doesn't have the old sent_files so no need to drop
-                """create table sent_files (
+        if old == 2:
-                    md5_digest blob,
+            # Old cache from old sent_files lasts then a day anyway, drop
-                    file_size integer,
+            c.execute('drop table sent_files')
-                    file_id integer,
+        self._create_table(c, """sent_files (
-                    part_count integer,
+            md5_digest blob,
-                    primary key(md5_digest, file_size)
+            file_size integer,
-                ) without rowid"""
+            type integer,
-            )
+            id integer,
-            old = 2
+            hash integer,
+            primary key(md5_digest, file_size, type)
+        )""")
+        c.close()
+
+    @staticmethod
+    def _create_table(c, *definitions):
+        """
+        Creates a table given its definition 'name (columns).
+        If the sqlite version is >= 3.8.2, it will use "without rowid".
+        See http://www.sqlite.org/releaselog/3_8_2.html.
+        """
+        required = (3, 8, 2)
+        sqlite_v = tuple(int(x) for x in sqlite3.sqlite_version.split('.'))
+        extra = ' without rowid' if sqlite_v >= required else ''
+        for definition in definitions:
+            c.execute('create table {}{}'.format(definition, extra))
 
     # Data from sessions should be kept as properties
     # not to fetch the database every time we need it
@@ -185,11 +218,10 @@ class Session:
         self._update_session_table()
 
         # Fetch the auth_key corresponding to this data center
-        c = self._conn.cursor()
+        c = self._cursor()
         c.execute('select auth_key from sessions')
         tuple_ = c.fetchone()
         if tuple_:
-            from ..crypto import AuthKey
             self._auth_key = AuthKey(data=tuple_[0])
         else:
             self._auth_key = None
@@ -213,7 +245,13 @@ class Session:
         self._update_session_table()
 
     def _update_session_table(self):
-        c = self._conn.cursor()
+        c = self._cursor()
+        # While we can save multiple rows into the sessions table
+        # currently we only want to keep ONE as the tables don't
+        # tell us which auth_key's are usable and will work. Needs
+        # some more work before being able to save auth_key's for
+        # multiple DCs. Probably done differently.
+        c.execute('delete from sessions')
         c.execute('insert or replace into sessions values (?,?,?,?)', (
             self._dc_id,
             self._server_address,
@@ -226,6 +264,19 @@ class Session:
         """Saves the current session object as session_user_id.session"""
         self._conn.commit()
 
+    def _cursor(self):
+        """Asserts that the connection is open and returns a cursor"""
+        if self._conn is None:
+            self._conn = sqlite3.connect(self.filename)
+        return self._conn.cursor()
+
+    def close(self):
+        """Closes the connection unless we're working in-memory"""
+        if self.filename != ':memory:':
+            if self._conn is not None:
+                self._conn.close()
+                self._conn = None
+
     def delete(self):
         """Deletes the current session file"""
         if self.filename == ':memory:':
@@ -265,7 +316,7 @@ class Session:
         now = time.time()
         nanoseconds = int((now - int(now)) * 1e+9)
         # "message identifiers are divisible by 4"
-        new_msg_id = (int(now) << 32) | (nanoseconds << 2)
+        new_msg_id = ((int(now) + self.time_offset) << 32) | (nanoseconds << 2)
 
         if self._last_msg_id >= new_msg_id:
             new_msg_id = self._last_msg_id + 4
@@ -313,12 +364,19 @@ class Session:
             except ValueError:
                 continue
 
-            p_hash = getattr(p, 'access_hash', 0)
+            if isinstance(p, (InputPeerUser, InputPeerChannel)):
-            if p_hash is None:
+                if not p.access_hash:
-                # Some users and channels seem to be returned without
+                    # Some users and channels seem to be returned without
-                # an 'access_hash', meaning Telegram doesn't want you
+                    # an 'access_hash', meaning Telegram doesn't want you
-                # to access them. This is the reason behind ensuring
+                    # to access them. This is the reason behind ensuring
-                # that the 'access_hash' is non-zero. See issue #354.
+                    # that the 'access_hash' is non-zero. See issue #354.
+                    # Note that this checks for zero or None, see #392.
+                    continue
+                else:
+                    p_hash = p.access_hash
+            elif isinstance(p, InputPeerChat):
+                p_hash = 0
+            else:
                 continue
 
             username = getattr(e, 'username', None) or None
@@ -330,7 +388,7 @@ class Session:
         if not rows:
             return
 
-        self._conn.executemany(
+        self._cursor().executemany(
             'insert or replace into entities values (?,?,?,?,?)', rows
         )
         self.save()
@@ -346,15 +404,19 @@ class Session:
 
            Raises ValueError if it cannot be found.
         """
-        if isinstance(key, TLObject):
+        try:
-            try:
+            if key.SUBCLASS_OF_ID in (0xc91c90b6, 0xe669bf46, 0x40f202fd):
-                # Try to early return if this key can be casted as input peer
+                # hex(crc32(b'InputPeer', b'InputUser' and b'InputChannel'))
-                return utils.get_input_peer(key)
+                # We already have an Input version, so nothing else required
-            except TypeError:
+                return key
-                # Otherwise, get the ID of the peer
+            # Try to early return if this key can be casted as input peer
+            return utils.get_input_peer(key)
+        except (AttributeError, TypeError):
+            # Not a TLObject or can't be cast into InputPeer
+            if isinstance(key, TLObject):
                 key = utils.get_peer_id(key)
 
-        c = self._conn.cursor()
+        c = self._cursor()
         if isinstance(key, str):
             phone = utils.parse_phone(key)
             if phone:
@@ -384,15 +446,24 @@ class Session:
 
     # File processing
 
-    def get_file(self, md5_digest, file_size):
+    def get_file(self, md5_digest, file_size, cls):
-        return self._conn.execute(
+        tuple_ = self._cursor().execute(
-            'select * from sent_files '
+            'select id, hash from sent_files '
-            'where md5_digest = ? and file_size = ?', (md5_digest, file_size)
+            'where md5_digest = ? and file_size = ? and type = ?',
+            (md5_digest, file_size, _SentFileType.from_type(cls).value)
         ).fetchone()
+        if tuple_:
+            # Both allowed classes have (id, access_hash) as parameters
+            return cls(tuple_[0], tuple_[1])
 
-    def cache_file(self, md5_digest, file_size, file_id, part_count):
+    def cache_file(self, md5_digest, file_size, instance):
-        self._conn.execute(
+        if not isinstance(instance, (InputDocument, InputPhoto)):
-            'insert into sent_files values (?,?,?,?)',
+            raise TypeError('Cannot cache %s instance' % type(instance))
-            (md5_digest, file_size, file_id, part_count)
+
-        )
+        self._cursor().execute(
+            'insert or replace into sent_files values (?,?,?,?,?)', (
+                md5_digest, file_size,
+                _SentFileType.from_type(type(instance)).value,
+                instance.id, instance.access_hash
+        ))
         self.save()

telethon/telegram_bare_client.py

@@ -1,20 +1,19 @@
+import asyncio
 import logging
 import os
-import asyncio
-from datetime import timedelta
-from hashlib import md5
-from io import BytesIO
 from asyncio import Lock
+from datetime import timedelta
 
-from . import helpers as utils, version
+from . import version, utils
-from .crypto import rsa, CdnDecrypter
+from .crypto import rsa
 from .errors import (
-    RPCError, BrokenAuthKeyError, ServerError,
+    RPCError, BrokenAuthKeyError, ServerError, FloodWaitError,
-    FloodWaitError, FileMigrateError, TypeNotFoundError,
+    FloodTestPhoneWaitError, TypeNotFoundError, UnauthorizedError,
-    UnauthorizedError, PhoneMigrateError, NetworkMigrateError, UserMigrateError
+    PhoneMigrateError, NetworkMigrateError, UserMigrateError
 )
 from .network import authenticator, MtProtoSender, Connection, ConnectionMode
-from .tl import TLObject, Session
+from .session import Session
+from .tl import TLObject
 from .tl.all_tlobjects import LAYER
 from .tl.functions import (
     InitConnectionRequest, InvokeWithLayerRequest, PingRequest
@@ -26,15 +25,8 @@ from .tl.functions.help import (
     GetCdnConfigRequest, GetConfigRequest
 )
 from .tl.functions.updates import GetStateRequest
-from .tl.functions.upload import (
-    GetFileRequest, SaveBigFilePartRequest, SaveFilePartRequest
-)
-from .tl.types import InputFile, InputFileBig
 from .tl.types.auth import ExportedAuthorization
-from .tl.types.upload import FileCdnRedirect
 from .update_state import UpdateState
-from .utils import get_appropriated_part_size
-
 
 DEFAULT_DC_ID = 4
 DEFAULT_IPV4_IP = '149.154.167.51'
@@ -83,7 +75,7 @@ class TelegramBareClient:
         if not api_id or not api_hash:
             raise ValueError(
                 "Your API ID or Hash cannot be empty or None. "
-                "Refer to Telethon's wiki for more information.")
+                "Refer to telethon.rtfd.io for more information.")
 
         self._use_ipv6 = use_ipv6
         
@@ -160,6 +152,7 @@ class TelegramBareClient:
 
         self._recv_loop = None
         self._ping_loop = None
+        self._idling = asyncio.Event()
 
         # Default PingRequest delay
         self._ping_delay = timedelta(minutes=1)
@@ -241,9 +234,7 @@ class TelegramBareClient:
         self._sender.disconnect()
         # TODO Shall we clear the _exported_sessions, or may be reused?
         self._first_request = True  # On reconnect it will be first again
-
+        self.session.close()
-    def __del__(self):
-        self.disconnect()
 
     async def _reconnect(self, new_dc=None):
         """If 'new_dc' is not set, only a call to .connect() will be made
@@ -264,7 +255,7 @@ class TelegramBareClient:
 
                 __log__.info('Attempting reconnection...')
                 return await self.connect()
-            except ConnectionResetError:
+            except ConnectionResetError as e:
                 __log__.warning('Reconnection failed due to %s', e)
                 return False
             finally:
@@ -408,6 +399,9 @@ class TelegramBareClient:
                    x.content_related for x in requests):
             raise TypeError('You can only invoke requests, not types!')
 
+        for request in requests:
+            await request.resolve(self, utils)
+
         # For logging purposes
         if len(requests) == 1:
             which = type(requests[0]).__name__
@@ -416,12 +410,8 @@ class TelegramBareClient:
                 len(requests), [type(x).__name__ for x in requests])
 
         __log__.debug('Invoking %s', which)
-
+        call_receive = \
-        # We should call receive from this thread if there's no background
+            not self._idling.is_set() or self._reconnect_lock.locked()
-        # thread reading or if the server disconnected us and we're trying
-        # to reconnect. This is because the read thread may either be
-        # locked also trying to reconnect or we may be said thread already.
-        call_receive = self._recv_loop is None
 
         for retry in range(retries):
             result = await self._invoke(call_receive, retry, *requests)
@@ -435,7 +425,7 @@ class TelegramBareClient:
             await asyncio.sleep(retry + 1, loop=self._loop)
             if not self._reconnect_lock.locked():
                 with await self._reconnect_lock:
-                    self._reconnect()
+                    await self._reconnect()
 
         raise RuntimeError('Number of retries reached 0.')
 
@@ -482,17 +472,26 @@ class TelegramBareClient:
             __log__.error('Authorization key seems broken and was invalid!')
             self.session.auth_key = None
 
+        except TypeNotFoundError as e:
+            # Only occurs when we call receive. May happen when
+            # we need to reconnect to another DC on login and
+            # Telegram somehow sends old objects (like configOld)
+            self._first_request = True
+            __log__.warning('Read unknown TLObject code ({}). '
+                            'Setting again first_request flag.'
+                            .format(hex(e.invalid_constructor_id)))
+
         except TimeoutError:
             __log__.warning('Invoking timed out')  # We will just retry
 
-        except ConnectionResetError:
+        except ConnectionResetError as e:
             __log__.warning('Connection was reset while invoking')
             if self._user_connected:
                 # Server disconnected us, __call__ will try reconnecting.
                 return None
             else:
                 # User never called .connect(), so raise this error.
-                raise
+                raise RuntimeError('Tried to invoke without .connect()') from e
 
         # Clear the flag if we got this far
         self._first_request = False
@@ -514,13 +513,13 @@ class TelegramBareClient:
                 UserMigrateError) as e:
 
             await self._reconnect(new_dc=e.new_dc)
-            return None
+            return await self._invoke(call_receive, retry, *requests)
 
         except ServerError as e:
             # Telegram is having some issues, just retry
             __log__.error('Telegram servers are having internal errors %s', e)
 
-        except FloodWaitError as e:
+        except (FloodWaitError, FloodTestPhoneWaitError) as e:
             __log__.warning('Request invoked too often, wait %ds', e.seconds)
             if e.seconds > self.session.flood_sleep_threshold | 0:
                 raise
@@ -535,211 +534,12 @@ class TelegramBareClient:
            (code request sent and confirmed)?"""
         return self._authorized
 
-    # endregion
+    def get_input_entity(self, peer):
-
-    # region Uploading media
-
-    async def upload_file(self,
-                    file,
-                    part_size_kb=None,
-                    file_name=None,
-                    progress_callback=None):
-        """Uploads the specified file and returns a handle (an instance
-           of InputFile or InputFileBig, as required) which can be later used.
-
-           Uploading a file will simply return a "handle" to the file stored
-           remotely in the Telegram servers, which can be later used on. This
-           will NOT upload the file to your own chat.
-
-           'file' may be either a file path, a byte array, or a stream.
-           Note that if the file is a stream it will need to be read
-           entirely into memory to tell its size first.
-
-           If 'progress_callback' is not None, it should be a function that
-           takes two parameters, (bytes_uploaded, total_bytes).
-
-           Default values for the optional parameters if left as None are:
-             part_size_kb = get_appropriated_part_size(file_size)
-             file_name    = os.path.basename(file_path)
         """
-        if isinstance(file, (InputFile, InputFileBig)):
+        Stub method, no functionality so that calling
-            return file  # Already uploaded
+        ``.get_input_entity()`` from ``.resolve()`` doesn't fail.
-
-        if isinstance(file, str):
-            file_size = os.path.getsize(file)
-        elif isinstance(file, bytes):
-            file_size = len(file)
-        else:
-            file = file.read()
-            file_size = len(file)
-
-        # File will now either be a string or bytes
-        if not part_size_kb:
-            part_size_kb = get_appropriated_part_size(file_size)
-
-        if part_size_kb > 512:
-            raise ValueError('The part size must be less or equal to 512KB')
-
-        part_size = int(part_size_kb * 1024)
-        if part_size % 1024 != 0:
-            raise ValueError('The part size must be evenly divisible by 1024')
-
-        # Set a default file name if None was specified
-        file_id = utils.generate_random_long()
-        if not file_name:
-            if isinstance(file, str):
-                file_name = os.path.basename(file)
-            else:
-                file_name = str(file_id)
-
-        # Determine whether the file is too big (over 10MB) or not
-        # Telegram does make a distinction between smaller or larger files
-        is_large = file_size > 10 * 1024 * 1024
-        if not is_large:
-            # Calculate the MD5 hash before anything else.
-            # As this needs to be done always for small files,
-            # might as well do it before anything else and
-            # check the cache.
-            if isinstance(file, str):
-                with open(file, 'rb') as stream:
-                    file = stream.read()
-            hash_md5 = md5(file)
-            tuple_ = self.session.get_file(hash_md5.digest(), file_size)
-            if tuple_:
-                __log__.info('File was already cached, not uploading again')
-                return InputFile(name=file_name,
-                    md5_checksum=tuple_[0], id=tuple_[2], parts=tuple_[3])
-        else:
-            hash_md5 = None
-
-        part_count = (file_size + part_size - 1) // part_size
-        __log__.info('Uploading file of %d bytes in %d chunks of %d',
-                     file_size, part_count, part_size)
-
-        with open(file, 'rb') if isinstance(file, str) else BytesIO(file) \
-                as stream:
-            for part_index in range(part_count):
-                # Read the file by in chunks of size part_size
-                part = stream.read(part_size)
-
-                # The SavePartRequest is different depending on whether
-                # the file is too large or not (over or less than 10MB)
-                if is_large:
-                    request = SaveBigFilePartRequest(file_id, part_index,
-                                                     part_count, part)
-                else:
-                    request = SaveFilePartRequest(file_id, part_index, part)
-
-                result = await self(request)
-                if result:
-                    __log__.debug('Uploaded %d/%d', part_index + 1, part_count)
-                    if progress_callback:
-                        progress_callback(stream.tell(), file_size)
-                else:
-                    raise RuntimeError(
-                        'Failed to upload file part {}.'.format(part_index))
-
-        if is_large:
-            return InputFileBig(file_id, part_count, file_name)
-        else:
-            self.session.cache_file(
-                hash_md5.digest(), file_size, file_id, part_count)
-
-            return InputFile(file_id, part_count, file_name,
-                             md5_checksum=hash_md5.hexdigest())
-
-    # endregion
-
-    # region Downloading media
-
-    async def download_file(self,
-                      input_location,
-                      file,
-                      part_size_kb=None,
-                      file_size=None,
-                      progress_callback=None):
-        """Downloads the given InputFileLocation to file (a stream or str).
-
-           If 'progress_callback' is not None, it should be a function that
-           takes two parameters, (bytes_downloaded, total_bytes). Note that
-           'total_bytes' simply equals 'file_size', and may be None.
         """
-        if not part_size_kb:
+        return peer
-            if not file_size:
-                part_size_kb = 64  # Reasonable default
-            else:
-                part_size_kb = get_appropriated_part_size(file_size)
-
-        part_size = int(part_size_kb * 1024)
-        # https://core.telegram.org/api/files says:
-        # > part_size % 1024 = 0 (divisible by 1KB)
-        #
-        # But https://core.telegram.org/cdn (more recent) says:
-        # > limit must be divisible by 4096 bytes
-        # So we just stick to the 4096 limit.
-        if part_size % 4096 != 0:
-            raise ValueError('The part size must be evenly divisible by 4096.')
-
-        if isinstance(file, str):
-            # Ensure that we'll be able to download the media
-            utils.ensure_parent_dir_exists(file)
-            f = open(file, 'wb')
-        else:
-            f = file
-
-        # The used client will change if FileMigrateError occurs
-        client = self
-        cdn_decrypter = None
-
-        __log__.info('Downloading file in chunks of %d bytes', part_size)
-        try:
-            offset = 0
-            while True:
-                try:
-                    if cdn_decrypter:
-                        result = await cdn_decrypter.get_file()
-                    else:
-                        result = await client(GetFileRequest(
-                            input_location, offset, part_size
-                        ))
-
-                        if isinstance(result, FileCdnRedirect):
-                            __log__.info('File lives in a CDN')
-                            cdn_decrypter, result = \
-                                await CdnDecrypter.prepare_decrypter(
-                                    client,
-                                    await self._get_cdn_client(result),
-                                    result
-                                )
-
-                except FileMigrateError as e:
-                    __log__.info('File lives in another DC')
-                    client = await self._get_exported_client(e.new_dc)
-                    continue
-
-                offset += part_size
-
-                # If we have received no data (0 bytes), the file is over
-                # So there is nothing left to download and write
-                if not result.bytes:
-                    # Return some extra information, unless it's a CDN file
-                    return getattr(result, 'type', '')
-
-                f.write(result.bytes)
-                __log__.debug('Saved %d more bytes', len(result.bytes))
-                if progress_callback:
-                    progress_callback(f.tell(), file_size)
-        finally:
-            if client != self:
-                client.disconnect()
-
-            if cdn_decrypter:
-                try:
-                    cdn_decrypter.client.disconnect()
-                except:
-                    pass
-            if isinstance(file, str):
-                f.close()
 
     # endregion
 
@@ -782,6 +582,7 @@ class TelegramBareClient:
 
     async def _recv_loop_impl(self):
         __log__.info('Starting to wait for items from the network')
+        self._idling.set()
         need_reconnect = False
         while self._user_connected:
             try:
@@ -792,37 +593,27 @@ class TelegramBareClient:
                         # Retry forever, this is instant messaging
                         await asyncio.sleep(0.1, loop=self._loop)
 
+                    # Telegram seems to kick us every 1024 items received
+                    # from the network not considering things like bad salt.
+                    # We must execute some *high level* request (that's not
+                    # a ping) if we want to receive updates again.
+                    # TODO Test if getDifference works too (better alternative)
+                    await self._sender.send(GetStateRequest())
+
                 __log__.debug('Receiving items from the network...')
                 await self._sender.receive(update_state=self.updates)
             except TimeoutError:
                 # No problem.
-                __log__.info('Receiving items from the network timed out')
+                __log__.debug('Receiving items from the network timed out')
-            except ConnectionError as error:
+            except ConnectionError:
                 need_reconnect = True
                 __log__.error('Connection was reset while receiving items')
                 await asyncio.sleep(1, loop=self._loop)
-            except Exception as error:
+            except:
-                # Unknown exception, pass it to the main thread
+                self._idling.clear()
-                __log__.exception('Unknown exception in the read thread! '
+                raise
-                                  'Disconnecting and leaving it to main thread')
 
-                try:
+        self._idling.clear()
-                    import socks
+        __log__.info('Connection closed by the user, not reading anymore')
-                    if isinstance(error, (
-                            socks.GeneralProxyError,
-                            socks.ProxyConnectionError
-                    )):
-                        # This is a known error, and it's not related to
-                        # Telegram but rather to the proxy. Disconnect and
-                        # hand it over to the main thread.
-                        self._background_error = error
-                        self.disconnect()
-                        break
-                except ImportError:
-                    "Not using PySocks, so it can't be a socket error"
-
-                break
-
-        self._recv_loop = None
 
     # endregion

telethon/tl/__init__.py

@@ -1,5 +1,4 @@
 from .tlobject import TLObject
-from .session import Session
 from .gzip_packed import GzipPacked
 from .tl_message import TLMessage
 from .message_container import MessageContainer

telethon/tl/custom/__init__.py

@@ -1,2 +1,3 @@
 from .draft import Draft
 from .dialog import Dialog
+from .input_sized_file import InputSizedFile

telethon/tl/custom/dialog.py

@@ -24,10 +24,7 @@ class Dialog:
         self.unread_count = dialog.unread_count
         self.unread_mentions_count = dialog.unread_mentions_count
 
-        if dialog.draft:
+        self.draft = Draft(client, dialog.peer, dialog.draft)
-            self.draft = Draft(client, dialog.peer, dialog.draft)
-        else:
-            self.draft = None
 
     async def send_message(self, *args, **kwargs):
         """

telethon/tl/custom/draft.py

@@ -1,16 +1,18 @@
 from ..functions.messages import SaveDraftRequest
-from ..types import UpdateDraftMessage
+from ..types import UpdateDraftMessage, DraftMessage
 
 
 class Draft:
     """
     Custom class that encapsulates a draft on the Telegram servers, providing
     an abstraction to change the message conveniently. The library will return
-    instances of this class when calling `client.get_drafts()`.
+    instances of this class when calling ``client.get_drafts()``.
     """
     def __init__(self, client, peer, draft):
         self._client = client
         self._peer = peer
+        if not draft:
+            draft = DraftMessage('', None, None, None, None)
 
         self.text = draft.message
         self.date = draft.date

telethon/tl/custom/input_sized_file.py

@@ -0,0 +1,9 @@
+from ..types import InputFile
+
+
+class InputSizedFile(InputFile):
+    """InputFile class with two extra parameters: md5 (digest) and size"""
+    def __init__(self, id_, parts, name, md5, size):
+        super().__init__(id_, parts, name, md5.hexdigest())
+        self.md5 = md5.digest()
+        self.size = size

telethon/tl/tlobject.py

@@ -6,6 +6,7 @@ class TLObject:
     def __init__(self):
         self.confirm_received = None
         self.rpc_error = None
+        self.result = None
 
         # These should be overrode
         self.content_related = False  # Only requests/functions/queries are
@@ -18,14 +19,12 @@ class TLObject:
         """
         if indent is None:
             if isinstance(obj, TLObject):
-                return '{}({})'.format(type(obj).__name__, ', '.join(
+                obj = obj.to_dict()
-                    '{}={}'.format(k, TLObject.pretty_format(v))
+
-                    for k, v in obj.to_dict(recursive=False).items()
-                ))
             if isinstance(obj, dict):
-                return '{{{}}}'.format(', '.join(
+                return '{}({})'.format(obj.get('_', 'dict'), ', '.join(
-                    '{}: {}'.format(k, TLObject.pretty_format(v))
+                    '{}={}'.format(k, TLObject.pretty_format(v))
-                    for k, v in obj.items()
+                    for k, v in obj.items() if k != '_'
                 ))
             elif isinstance(obj, str) or isinstance(obj, bytes):
                 return repr(obj)
@@ -41,30 +40,28 @@ class TLObject:
                 return repr(obj)
         else:
             result = []
-            if isinstance(obj, TLObject) or isinstance(obj, dict):
+            if isinstance(obj, TLObject):
-                if isinstance(obj, dict):
+                obj = obj.to_dict()
-                    d = obj
-                    start, end, sep = '{', '}', ': '
-                else:
-                    d = obj.to_dict(recursive=False)
-                    start, end, sep = '(', ')', '='
-                    result.append(type(obj).__name__)
 
-                result.append(start)
+            if isinstance(obj, dict):
-                if d:
+                result.append(obj.get('_', 'dict'))
+                result.append('(')
+                if obj:
                     result.append('\n')
                     indent += 1
-                    for k, v in d.items():
+                    for k, v in obj.items():
+                        if k == '_':
+                            continue
                         result.append('\t' * indent)
                         result.append(k)
-                        result.append(sep)
+                        result.append('=')
                         result.append(TLObject.pretty_format(v, indent))
                         result.append(',\n')
                     result.pop()  # last ',\n'
                     indent -= 1
                     result.append('\n')
                     result.append('\t' * indent)
-                result.append(end)
+                result.append(')')
 
             elif isinstance(obj, str) or isinstance(obj, bytes):
                 result.append(repr(obj))
@@ -142,8 +139,27 @@ class TLObject:
 
         raise TypeError('Cannot interpret "{}" as a date.'.format(dt))
 
+    # These are nearly always the same for all subclasses
+    def on_response(self, reader):
+        self.result = reader.tgread_object()
+
+    def __eq__(self, o):
+        return isinstance(o, type(self)) and self.to_dict() == o.to_dict()
+
+    def __ne__(self, o):
+        return not isinstance(o, type(self)) or self.to_dict() != o.to_dict()
+
+    def __str__(self):
+        return TLObject.pretty_format(self)
+
+    def stringify(self):
+        return TLObject.pretty_format(self, indent=0)
+
     # These should be overrode
-    def to_dict(self, recursive=True):
+    async def resolve(self, client, utils):
+        pass
+
+    def to_dict(self):
         return {}
 
     def __bytes__(self):

telethon/update_state.py

@@ -17,11 +17,8 @@ class UpdateState:
 
     def __init__(self, loop=None):
         self.handlers = []
-        self._latest_updates = deque(maxlen=10)
         self._loop = loop if loop else asyncio.get_event_loop()
 
-        self._logger = logging.getLogger(__name__)
-
         # https://core.telegram.org/api/updates
         self._state = tl.updates.State(0, 0, datetime.now(), 0, 0)
 
@@ -38,44 +35,20 @@ class UpdateState:
             self._state = update
             return  # Nothing else to be done
 
-        pts = getattr(update, 'pts', self._state.pts)
+        if hasattr(update, 'pts'):
-        if hasattr(update, 'pts') and pts <= self._state.pts:
+            self._state.pts = update.pts
-            __log__.info('Ignoring %s, already have it', update)
-            return  # We already handled this update
-
-        self._state.pts = pts
-
-        # TODO There must be a better way to handle updates rather than
-        # keeping a queue with the latest updates only, and handling
-        # the 'pts' correctly should be enough. However some updates
-        # like UpdateUserStatus (even inside UpdateShort) will be called
-        # repeatedly very often if invoking anything inside an update
-        # handler. TODO Figure out why.
-        """
-        client = TelegramClient('anon', api_id, api_hash, update_workers=1)
-        client.connect()
-        def handle(u):
-            client.get_me()
-        client.add_update_handler(handle)
-        input('Enter to exit.')
-        """
-        data = pickle.dumps(update.to_dict())
-        if data in self._latest_updates:
-            __log__.info('Ignoring %s, already have it', update)
-            return  # Duplicated too
-
-        self._latest_updates.append(data)
 
+        # After running the script for over an hour and receiving over
+        # 1000 updates, the only duplicates received were users going
+        # online or offline. We can trust the server until new reports.
+        if isinstance(update, tl.UpdateShort):
+            self.handle_update(update.update)
         # Expand "Updates" into "Update", and pass these to callbacks.
         # Since .users and .chats have already been processed, we
         # don't need to care about those either.
-        if isinstance(update, tl.UpdateShort):
-            self.handle_update(update.update)
-
         elif isinstance(update, (tl.Updates, tl.UpdatesCombined)):
-            for upd in update.updates:
+            for u in update.updates:
-                self.handle_update(upd)
+                self.handle_update(u)
-
+        # TODO Handle "tl.UpdatesTooLong"
-        # TODO Handle "Updates too long"
         else:
             self.handle_update(update)

telethon/utils.py

@@ -3,11 +3,10 @@ Utilities for working with the Telegram API itself (such as handy methods
 to convert between an entity like an User, Chat, etc. into its Input version)
 """
 import math
+import re
 from mimetypes import add_type, guess_extension
 
-import re
+from .tl.types.contacts import ResolvedPeer
-
-from .tl import TLObject
 from .tl.types import (
     Channel, ChannelForbidden, Chat, ChatEmpty, ChatForbidden, ChatFull,
     ChatPhoto, InputPeerChannel, InputPeerChat, InputPeerUser, InputPeerEmpty,
@@ -22,10 +21,10 @@ from .tl.types import (
     GeoPointEmpty, InputGeoPointEmpty, Photo, InputPhoto, PhotoEmpty,
     InputPhotoEmpty, FileLocation, ChatPhotoEmpty, UserProfilePhotoEmpty,
     FileLocationUnavailable, InputMediaUploadedDocument, ChannelFull,
-    InputMediaUploadedPhoto, DocumentAttributeFilename, photos
+    InputMediaUploadedPhoto, DocumentAttributeFilename, photos,
+    TopPeer, InputNotifyPeer
 )
 
-
 USERNAME_RE = re.compile(
     r'@|(?:https?://)?(?:telegram\.(?:me|dog)|t\.me)/(joinchat/)?'
 )
@@ -62,13 +61,13 @@ def get_extension(media):
 
     # Documents will come with a mime type
     if isinstance(media, MessageMediaDocument):
-        if isinstance(media.document, Document):
+        media = media.document
-            if media.document.mime_type == 'application/octet-stream':
+    if isinstance(media, Document):
-                # Octet stream are just bytes, which have no default extension
+        if media.mime_type == 'application/octet-stream':
-                return ''
+            # Octet stream are just bytes, which have no default extension
-            else:
+            return ''
-                extension = guess_extension(media.document.mime_type)
+        else:
-                return extension if extension else ''
+            return guess_extension(media.mime_type) or ''
 
     return ''
 
@@ -81,12 +80,12 @@ def _raise_cast_fail(entity, target):
 def get_input_peer(entity, allow_self=True):
     """Gets the input peer for the given "entity" (user, chat or channel).
        A TypeError is raised if the given entity isn't a supported type."""
-    if not isinstance(entity, TLObject):
+    try:
+        if entity.SUBCLASS_OF_ID == 0xc91c90b6:  # crc32(b'InputPeer')
+            return entity
+    except AttributeError:
         _raise_cast_fail(entity, 'InputPeer')
 
-    if type(entity).SUBCLASS_OF_ID == 0xc91c90b6:  # crc32(b'InputPeer')
-        return entity
-
     if isinstance(entity, User):
         if entity.is_self and allow_self:
             return InputPeerSelf()
@@ -100,15 +99,18 @@ def get_input_peer(entity, allow_self=True):
         return InputPeerChannel(entity.id, entity.access_hash or 0)
 
     # Less common cases
-    if isinstance(entity, UserEmpty):
-        return InputPeerEmpty()
-
     if isinstance(entity, InputUser):
         return InputPeerUser(entity.user_id, entity.access_hash)
 
+    if isinstance(entity, InputChannel):
+        return InputPeerChannel(entity.channel_id, entity.access_hash)
+
     if isinstance(entity, InputUserSelf):
         return InputPeerSelf()
 
+    if isinstance(entity, UserEmpty):
+        return InputPeerEmpty()
+
     if isinstance(entity, UserFull):
         return get_input_peer(entity.user)
 
@@ -123,12 +125,12 @@ def get_input_peer(entity, allow_self=True):
 
 def get_input_channel(entity):
     """Similar to get_input_peer, but for InputChannel's alone"""
-    if not isinstance(entity, TLObject):
+    try:
+        if entity.SUBCLASS_OF_ID == 0x40f202fd:  # crc32(b'InputChannel')
+            return entity
+    except AttributeError:
         _raise_cast_fail(entity, 'InputChannel')
 
-    if type(entity).SUBCLASS_OF_ID == 0x40f202fd:  # crc32(b'InputChannel')
-        return entity
-
     if isinstance(entity, (Channel, ChannelForbidden)):
         return InputChannel(entity.id, entity.access_hash or 0)
 
@@ -140,12 +142,12 @@ def get_input_channel(entity):
 
 def get_input_user(entity):
     """Similar to get_input_peer, but for InputUser's alone"""
-    if not isinstance(entity, TLObject):
+    try:
+        if entity.SUBCLASS_OF_ID == 0xe669bf46:  # crc32(b'InputUser'):
+            return entity
+    except AttributeError:
         _raise_cast_fail(entity, 'InputUser')
 
-    if type(entity).SUBCLASS_OF_ID == 0xe669bf46:  # crc32(b'InputUser')
-        return entity
-
     if isinstance(entity, User):
         if entity.is_self:
             return InputUserSelf()
@@ -169,12 +171,12 @@ def get_input_user(entity):
 
 def get_input_document(document):
     """Similar to get_input_peer, but for documents"""
-    if not isinstance(document, TLObject):
+    try:
+        if document.SUBCLASS_OF_ID == 0xf33fdb68:  # crc32(b'InputDocument'):
+            return document
+    except AttributeError:
         _raise_cast_fail(document, 'InputDocument')
 
-    if type(document).SUBCLASS_OF_ID == 0xf33fdb68:  # crc32(b'InputDocument')
-        return document
-
     if isinstance(document, Document):
         return InputDocument(id=document.id, access_hash=document.access_hash)
 
@@ -192,12 +194,12 @@ def get_input_document(document):
 
 def get_input_photo(photo):
     """Similar to get_input_peer, but for documents"""
-    if not isinstance(photo, TLObject):
+    try:
+        if photo.SUBCLASS_OF_ID == 0x846363e0:  # crc32(b'InputPhoto'):
+            return photo
+    except AttributeError:
         _raise_cast_fail(photo, 'InputPhoto')
 
-    if type(photo).SUBCLASS_OF_ID == 0x846363e0:  # crc32(b'InputPhoto')
-        return photo
-
     if isinstance(photo, photos.Photo):
         photo = photo.photo
 
@@ -212,12 +214,12 @@ def get_input_photo(photo):
 
 def get_input_geo(geo):
     """Similar to get_input_peer, but for geo points"""
-    if not isinstance(geo, TLObject):
+    try:
+        if geo.SUBCLASS_OF_ID == 0x430d225:  # crc32(b'InputGeoPoint'):
+            return geo
+    except AttributeError:
         _raise_cast_fail(geo, 'InputGeoPoint')
 
-    if type(geo).SUBCLASS_OF_ID == 0x430d225:  # crc32(b'InputGeoPoint')
-        return geo
-
     if isinstance(geo, GeoPoint):
         return InputGeoPoint(lat=geo.lat, long=geo.long)
 
@@ -239,24 +241,26 @@ def get_input_media(media, user_caption=None, is_photo=False):
        If the media is a file location and is_photo is known to be True,
        it will be treated as an InputMediaUploadedPhoto.
     """
-    if not isinstance(media, TLObject):
+    try:
+        if media.SUBCLASS_OF_ID == 0xfaf846f4:  # crc32(b'InputMedia'):
+            return media
+    except AttributeError:
         _raise_cast_fail(media, 'InputMedia')
 
-    if type(media).SUBCLASS_OF_ID == 0xfaf846f4:  # crc32(b'InputMedia')
-        return media
-
     if isinstance(media, MessageMediaPhoto):
         return InputMediaPhoto(
             id=get_input_photo(media.photo),
-            caption=media.caption if user_caption is None else user_caption,
+            ttl_seconds=media.ttl_seconds,
-            ttl_seconds=media.ttl_seconds
+            caption=((media.caption if user_caption is None else user_caption)
+                     or '')
         )
 
     if isinstance(media, MessageMediaDocument):
         return InputMediaDocument(
             id=get_input_document(media.document),
-            caption=media.caption if user_caption is None else user_caption,
+            ttl_seconds=media.ttl_seconds,
-            ttl_seconds=media.ttl_seconds
+            caption=((media.caption if user_caption is None else user_caption)
+                     or '')
         )
 
     if isinstance(media, FileLocation):
@@ -278,9 +282,10 @@ def get_input_media(media, user_caption=None, is_photo=False):
 
     if isinstance(media, (ChatPhoto, UserProfilePhoto)):
         if isinstance(media.photo_big, FileLocationUnavailable):
-            return get_input_media(media.photo_small, is_photo=True)
+            media = media.photo_small
         else:
-            return get_input_media(media.photo_big, is_photo=True)
+            media = media.photo_big
+        return get_input_media(media, user_caption=user_caption, is_photo=True)
 
     if isinstance(media, MessageMediaContact):
         return InputMediaContact(
@@ -298,7 +303,8 @@ def get_input_media(media, user_caption=None, is_photo=False):
             title=media.title,
             address=media.address,
             provider=media.provider,
-            venue_id=media.venue_id
+            venue_id=media.venue_id,
+            venue_type=''
         )
 
     if isinstance(media, (
@@ -307,11 +313,19 @@ def get_input_media(media, user_caption=None, is_photo=False):
         return InputMediaEmpty()
 
     if isinstance(media, Message):
-        return get_input_media(media.media)
+        return get_input_media(
+            media.media, user_caption=user_caption, is_photo=is_photo
+        )
 
     _raise_cast_fail(media, 'InputMedia')
 
 
+def is_image(file):
+    """Returns True if the file extension looks like an image file"""
+    return (isinstance(file, str) and
+            bool(re.search(r'\.(png|jpe?g|gif)$', file, re.IGNORECASE)))
+
+
 def parse_phone(phone):
     """Parses the given phone, or returns None if it's invalid"""
     if isinstance(phone, int):
@@ -348,15 +362,18 @@ def get_peer_id(peer):
     a call to utils.resolve_id(marked_id).
     """
     # First we assert it's a Peer TLObject, or early return for integers
-    if not isinstance(peer, TLObject):
+    if isinstance(peer, int):
-        if isinstance(peer, int):
+        return peer
-            return peer
-        else:
-            _raise_cast_fail(peer, 'int')
 
-    elif type(peer).SUBCLASS_OF_ID not in {0x2d45687, 0xc91c90b6}:
+    try:
-        # Not a Peer or an InputPeer, so first get its Input version
+        if peer.SUBCLASS_OF_ID not in (0x2d45687, 0xc91c90b6):
-        peer = get_input_peer(peer, allow_self=False)
+            if isinstance(peer, (ResolvedPeer, InputNotifyPeer, TopPeer)):
+                peer = peer.peer
+            else:
+                # Not a Peer or an InputPeer, so first get its Input version
+                peer = get_input_peer(peer, allow_self=False)
+    except AttributeError:
+        _raise_cast_fail(peer, 'int')
 
     # Set the right ID/kind, or raise if the TLObject is not recognised
     if isinstance(peer, (PeerUser, InputPeerUser)):

telethon/version.py

@@ -1,3 +1,3 @@
 # Versions should comply with PEP440.
 # This line is parsed in setup.py:
-__version__ = '0.16'
+__version__ = '0.17.1'

telethon_examples/interactive_telegram_client.py

@@ -84,9 +84,9 @@ class InteractiveTelegramClient(TelegramClient):
             update_workers=1
         )
 
-        # Store all the found media in memory here,
+        # Store {message.id: message} map here so that we can download
-        # so it can be downloaded if the user wants
+        # media known the message ID, for every message having media.
-        self.found_media = set()
+        self.found_media = {}
 
         # Calling .connect() may return False, so you need to assert it's
         # True before continuing. Otherwise you may want to retry as done here.
@@ -204,27 +204,21 @@ class InteractiveTelegramClient(TelegramClient):
                 # History
                 elif msg == '!h':
                     # First retrieve the messages and some information
-                    total_count, messages, senders = \
+                    messages = self.get_message_history(entity, limit=10)
-                        self.get_message_history(entity, limit=10)
 
                     # Iterate over all (in reverse order so the latest appear
                     # the last in the console) and print them with format:
                     # "[hh:mm] Sender: Message"
-                    for msg, sender in zip(
+                    for msg in reversed(messages):
-                            reversed(messages), reversed(senders)):
+                        # Note that the .sender attribute is only there for
-                        # Get the name of the sender if any
+                        # convenience, the API returns it differently. But
-                        if sender:
+                        # this shouldn't concern us. See the documentation
-                            name = getattr(sender, 'first_name', None)
+                        # for .get_message_history() for more information.
-                            if not name:
+                        name = get_display_name(msg.sender)
-                                name = getattr(sender, 'title')
-                                if not name:
-                                    name = '???'
-                        else:
-                            name = '???'
 
                         # Format the message content
                         if getattr(msg, 'media', None):
-                            self.found_media.add(msg)
+                            self.found_media[msg.id] = msg
                             # The media may or may not have a caption
                             caption = getattr(msg.media, 'caption', '')
                             content = '<{}> {}'.format(
@@ -257,8 +251,7 @@ class InteractiveTelegramClient(TelegramClient):
                 elif msg.startswith('!d '):
                     # Slice the message to get message ID
                     deleted_msg = self.delete_messages(entity, msg[len('!d '):])
-                    print('Deleted. {}'.format(deleted_msg))
+                    print('Deleted {}'.format(deleted_msg))
-
 
                 # Download media
                 elif msg.startswith('!dm '):
@@ -275,12 +268,11 @@ class InteractiveTelegramClient(TelegramClient):
                             'Profile picture downloaded to {}'.format(output)
                         )
                     else:
-                        print('No profile picture found for this user.')
+                        print('No profile picture found for this user!')
 
                 # Send chat message (if any)
                 elif msg:
-                    self.send_message(
+                    self.send_message(entity, msg, link_preview=False)
-                        entity, msg, link_preview=False)
 
     def send_photo(self, path, entity):
         """Sends the file located at path to the desired entity as a photo"""
@@ -304,23 +296,20 @@ class InteractiveTelegramClient(TelegramClient):
            downloads it.
         """
         try:
-            # The user may have entered a non-integer string!
+            msg = self.found_media[int(media_id)]
-            msg_media_id = int(media_id)
+        except (ValueError, KeyError):
+            # ValueError when parsing, KeyError when accessing dictionary
+            print('Invalid media ID given or message not found!')
+            return
 
-            # Search the message ID
+        print('Downloading media to usermedia/...')
-            for msg in self.found_media:
+        os.makedirs('usermedia', exist_ok=True)
-                if msg.id == msg_media_id:
+        output = self.download_media(
-                    print('Downloading media to usermedia/...')
+            msg.media,
-                    os.makedirs('usermedia', exist_ok=True)
+            file='usermedia/',
-                    output = self.download_media(
+            progress_callback=self.download_progress_callback
-                        msg.media,
+        )
-                        file='usermedia/',
+        print('Media downloaded to {}!'.format(output))
-                        progress_callback=self.download_progress_callback
-                    )
-                    print('Media downloaded to {}!'.format(output))
-
-        except ValueError:
-            print('Invalid media ID given!')
 
     @staticmethod
     def download_progress_callback(downloaded_bytes, total_bytes):

telethon_generator/error_descriptions

@@ -63,3 +63,4 @@ SESSION_REVOKED=The authorization has been invalidated, because of the user term
 USER_ALREADY_PARTICIPANT=The authenticated user is already a participant of the chat
 USER_DEACTIVATED=The user has been deleted/deactivated
 FLOOD_WAIT_X=A wait of {} seconds is required
+FLOOD_TEST_PHONE_WAIT_X=A wait of {} seconds is required in the test servers

telethon_generator/error_generator.py

@@ -26,7 +26,9 @@ known_codes = {
 
 def fetch_errors(output, url=URL):
     print('Opening a connection to', url, '...')
-    r = urllib.request.urlopen(url)
+    r = urllib.request.urlopen(urllib.request.Request(
+        url, headers={'User-Agent' : 'Mozilla/5.0'}
+    ))
     print('Checking response...')
     data = json.loads(
         r.read().decode(r.info().get_param('charset') or 'utf-8')
@@ -34,11 +36,11 @@ def fetch_errors(output, url=URL):
     if data.get('ok'):
         print('Response was okay, saving data')
         with open(output, 'w', encoding='utf-8') as f:
-            json.dump(data, f)
+            json.dump(data, f, sort_keys=True)
         return True
     else:
         print('The data received was not okay:')
-        print(json.dumps(data, indent=4))
+        print(json.dumps(data, indent=4, sort_keys=True))
         return False
 
 
@@ -79,7 +81,9 @@ def generate_code(output, json_file, errors_desc):
     errors = defaultdict(set)
     # PWRTelegram's API doesn't return all errors, which we do need here.
     # Add some special known-cases manually first.
-    errors[420].add('FLOOD_WAIT_X')
+    errors[420].update((
+        'FLOOD_WAIT_X', 'FLOOD_TEST_PHONE_WAIT_X'
+    ))
     errors[401].update((
         'AUTH_KEY_INVALID', 'SESSION_EXPIRED', 'SESSION_REVOKED'
     ))
@@ -118,6 +122,7 @@ def generate_code(output, json_file, errors_desc):
     # Names for the captures, or 'x' if unknown
     capture_names = {
         'FloodWaitError': 'seconds',
+        'FloodTestPhoneWaitError': 'seconds',
         'FileMigrateError': 'new_dc',
         'NetworkMigrateError': 'new_dc',
         'PhoneMigrateError': 'new_dc',
@@ -161,3 +166,11 @@ def generate_code(output, json_file, errors_desc):
         for pattern, name in patterns:
             f.write('    {}: {},\n'.format(repr(pattern), name))
         f.write('}\n')
+
+
+if __name__ == '__main__':
+    if input('generate (y/n)?: ').lower() == 'y':
+        generate_code('../telethon/errors/rpc_error_list.py',
+                      'errors.json', 'error_descriptions')
+    elif input('fetch (y/n)?: ').lower() == 'y':
+        fetch_errors('errors.json')

telethon_generator/parser/tl_object.py

@@ -264,7 +264,7 @@ class TLArg:
             'date': 'datetime.datetime | None',  # None date = 0 timestamp
             'bytes': 'bytes',
             'true': 'bool',
-        }.get(self.type, 'TLObject')
+        }.get(self.type, self.type)
         if self.is_vector:
             result = 'list[{}]'.format(result)
         if self.is_flag and self.type != 'date':

telethon_generator/tl_generator.py

@@ -10,6 +10,15 @@ AUTO_GEN_NOTICE = \
     '"""File generated by TLObjects\' generator. All changes will be ERASED"""'
 
 
+AUTO_CASTS = {
+    'InputPeer': 'utils.get_input_peer(await client.get_input_entity({}))',
+    'InputChannel': 'utils.get_input_channel(await client.get_input_entity({}))',
+    'InputUser': 'utils.get_input_user(await client.get_input_entity({}))',
+    'InputMedia': 'utils.get_input_media({})',
+    'InputPhoto': 'utils.get_input_photo({})'
+}
+
+
 class TLGenerator:
     def __init__(self, output_dir):
         self.output_dir = output_dir
@@ -137,15 +146,6 @@ class TLGenerator:
                         x for x in namespace_tlobjects.keys() if x
                     )))
 
-                # Import 'get_input_*' utils
-                # TODO Support them on types too
-                if 'functions' in out_dir:
-                    builder.writeln(
-                        'from {}.utils import get_input_peer, '
-                        'get_input_channel, get_input_user, '
-                        'get_input_media, get_input_photo'.format('.' * depth)
-                    )
-
                 # Import 'os' for those needing access to 'os.urandom()'
                 # Currently only 'random_id' needs 'os' to be imported,
                 # for all those TLObjects with arg.can_be_inferred.
@@ -257,22 +257,56 @@ class TLGenerator:
             builder.writeln()
 
         for arg in args:
-            TLGenerator._write_self_assigns(builder, tlobject, arg, args)
+            if not arg.can_be_inferred:
+                builder.writeln('self.{0} = {0}'.format(arg.name))
+                continue
+
+            # Currently the only argument that can be
+            # inferred are those called 'random_id'
+            if arg.name == 'random_id':
+                # Endianness doesn't really matter, and 'big' is shorter
+                code = "int.from_bytes(os.urandom({}), 'big', signed=True)" \
+                    .format(8 if arg.type == 'long' else 4)
+
+                if arg.is_vector:
+                    # Currently for the case of "messages.forwardMessages"
+                    # Ensure we can infer the length from id:Vector<>
+                    if not next(
+                            a for a in args if a.name == 'id').is_vector:
+                        raise ValueError(
+                            'Cannot infer list of random ids for ', tlobject
+                        )
+                    code = '[{} for _ in range(len(id))]'.format(code)
+
+                builder.writeln(
+                    "self.random_id = random_id if random_id "
+                    "is not None else {}".format(code)
+                )
+            else:
+                raise ValueError('Cannot infer a value for ', arg)
 
         builder.end_block()
 
+        # Write the resolve(self, client, utils) method
+        if any(arg.type in AUTO_CASTS for arg in args):
+            builder.writeln('async def resolve(self, client, utils):')
+            for arg in args:
+                ac = AUTO_CASTS.get(arg.type, None)
+                if ac:
+                    TLGenerator._write_self_assign(builder, arg, ac)
+            builder.end_block()
+
         # Write the to_dict(self) method
-        builder.writeln('def to_dict(self, recursive=True):')
+        builder.writeln('def to_dict(self):')
-        if args:
+        builder.writeln('return {')
-            builder.writeln('return {')
-        else:
-            builder.write('return {')
         builder.current_indent += 1
 
         base_types = ('string', 'bytes', 'int', 'long', 'int128',
                       'int256', 'double', 'Bool', 'true', 'date')
 
+        builder.write("'_': '{}'".format(tlobject.class_name()))
         for arg in args:
+            builder.writeln(',')
             builder.write("'{}': ".format(arg.name))
             if arg.type in base_types:
                 if arg.is_vector:
@@ -283,17 +317,17 @@ class TLGenerator:
             else:
                 if arg.is_vector:
                     builder.write(
-                        '([] if self.{0} is None else [None'
+                        '[] if self.{0} is None else [None '
-                        ' if x is None else x.to_dict() for x in self.{0}]'
+                        'if x is None else x.to_dict() for x in self.{0}]'
-                        ') if recursive else self.{0}'.format(arg.name)
+                        .format(arg.name)
                     )
                 else:
                     builder.write(
-                        '(None if self.{0} is None else self.{0}.to_dict())'
+                        'None if self.{0} is None else self.{0}.to_dict()'
-                        ' if recursive else self.{0}'.format(arg.name)
+                        .format(arg.name)
                     )
-            builder.writeln(',')
 
+        builder.writeln()
         builder.current_indent -= 1
         builder.writeln("}")
 
@@ -351,78 +385,43 @@ class TLGenerator:
                 if not a.flag_indicator and not a.generic_definition
             )
         ))
-        builder.end_block()
 
         # Only requests can have a different response that's not their
         # serialized body, that is, we'll be setting their .result.
-        if tlobject.is_function:
+        #
+        # The default behaviour is reading a TLObject too, so no need
+        # to override it unless necessary.
+        if tlobject.is_function and not TLGenerator._is_boxed(tlobject.result):
+            builder.end_block()
             builder.writeln('def on_response(self, reader):')
             TLGenerator.write_request_result_code(builder, tlobject)
-            builder.end_block()
-
-        # Write the __str__(self) and stringify(self) functions
-        builder.writeln('def __str__(self):')
-        builder.writeln('return TLObject.pretty_format(self)')
-        builder.end_block()
-
-        builder.writeln('def stringify(self):')
-        builder.writeln('return TLObject.pretty_format(self, indent=0)')
-        # builder.end_block()  # No need to end the last block
 
     @staticmethod
-    def _write_self_assigns(builder, tlobject, arg, args):
+    def _is_boxed(type_):
-        if arg.can_be_inferred:
+        # https://core.telegram.org/mtproto/serialize#boxed-and-bare-types
-            # Currently the only argument that can be
+        # TL;DR; boxed types start with uppercase always, so we can use
-            # inferred are those called 'random_id'
+        # this to check whether everything in it is boxed or not.
-            if arg.name == 'random_id':
+        #
-                # Endianness doesn't really matter, and 'big' is shorter
+        # The API always returns a boxed type, but it may inside a Vector<>
-                code = "int.from_bytes(os.urandom({}), 'big', signed=True)"\
+        # or a namespace, and the Vector may have a not-boxed type. For this
-                    .format(8 if arg.type == 'long' else 4)
+        # reason we find whatever index, '<' or '.'. If neither are present
-
+        # we will get -1, and the 0th char is always upper case thus works.
-                if arg.is_vector:
+        # For Vector types and namespaces, it will check in the right place.
-                    # Currently for the case of "messages.forwardMessages"
+        check_after = max(type_.find('<'), type_.find('.'))
-                    # Ensure we can infer the length from id:Vector<>
+        return type_[check_after + 1].isupper()
-                    if not next(a for a in args if a.name == 'id').is_vector:
-                        raise ValueError(
-                            'Cannot infer list of random ids for ', tlobject
-                        )
-                    code = '[{} for _ in range(len(id))]'.format(code)
-
-                builder.writeln(
-                    "self.random_id = random_id if random_id "
-                    "is not None else {}".format(code)
-                )
-            else:
-                raise ValueError('Cannot infer a value for ', arg)
-
-        # Well-known cases, auto-cast it to the right type
-        elif arg.type == 'InputPeer' and tlobject.is_function:
-            TLGenerator.write_get_input(builder, arg, 'get_input_peer')
-        elif arg.type == 'InputChannel' and tlobject.is_function:
-            TLGenerator.write_get_input(builder, arg, 'get_input_channel')
-        elif arg.type == 'InputUser' and tlobject.is_function:
-            TLGenerator.write_get_input(builder, arg, 'get_input_user')
-        elif arg.type == 'InputMedia' and tlobject.is_function:
-            TLGenerator.write_get_input(builder, arg, 'get_input_media')
-        elif arg.type == 'InputPhoto' and tlobject.is_function:
-            TLGenerator.write_get_input(builder, arg, 'get_input_photo')
-
-        else:
-            builder.writeln('self.{0} = {0}'.format(arg.name))
 
     @staticmethod
-    def write_get_input(builder, arg, get_input_code):
+    def _write_self_assign(builder, arg, get_input_code):
-        """Returns "True" if the get_input_* code was written when assigning
+        """Writes self.arg = input.format(self.arg), considering vectors"""
-           a parameter upon creating the request. Returns False otherwise
-        """
         if arg.is_vector:
-            builder.write('self.{0} = [{1}(_x) for _x in {0}]'
+            builder.write('self.{0} = [{1} for _x in self.{0}]'
-                          .format(arg.name, get_input_code))
+                          .format(arg.name, get_input_code.format('_x')))
         else:
-            builder.write('self.{0} = {1}({0})'
+            builder.write('self.{} = {}'.format(
-                          .format(arg.name, get_input_code))
+                arg.name, get_input_code.format('self.' + arg.name)))
+
         builder.writeln(
-            ' if {} else None'.format(arg.name) if arg.is_flag else ''
+            ' if self.{} else None'.format(arg.name) if arg.is_flag else ''
         )
 
     @staticmethod
@@ -695,13 +694,13 @@ class TLGenerator:
             # not parsed as arguments are and it's a bit harder to tell which
             # is which.
             if tlobject.result == 'Vector<int>':
-                builder.writeln('reader.read_int()  # Vector id')
+                builder.writeln('reader.read_int()  # Vector ID')
                 builder.writeln('count = reader.read_int()')
                 builder.writeln(
                     'self.result = [reader.read_int() for _ in range(count)]'
                 )
             elif tlobject.result == 'Vector<long>':
-                builder.writeln('reader.read_int()  # Vector id')
+                builder.writeln('reader.read_int()  # Vector ID')
                 builder.writeln('count = reader.read_long()')
                 builder.writeln(
                     'self.result = [reader.read_long() for _ in range(count)]'

telethon_tests/crypto_test.py

@@ -53,6 +53,7 @@ class CryptoTests(unittest.TestCase):
 
     @staticmethod
     def test_calc_key():
+        # TODO Upgrade test for MtProto 2.0
         shared_key = b'\xbc\xd2m\xb7\xcav\xf4][\x88\x83\' \xf3\x11\x8as\xd04\x941\xae' \
                      b'*O\x03\x86\x9a/H#\x1a\x8c\xb5j\xe9$\xe0IvCm^\xe70\x1a5C\t\x16' \
                      b'\x03\xd2\x9d\xa9\x89\xd6\xce\x08P\x0fdr\xa0\xb3\xeb\xfecv\x1a' \
@@ -98,13 +99,6 @@ class CryptoTests(unittest.TestCase):
         assert iv == expected_iv, 'Invalid IV (expected ("{}"), got ("{}"))'.format(
             expected_iv, iv)
 
-    @staticmethod
-    def test_calc_msg_key():
-        value = utils.calc_msg_key(b'Some random message')
-        expected = b'\xdfAa\xfc\x10\xab\x89\xd2\xfe\x19C\xf1\xdd~\xbf\x81'
-        assert value == expected, 'Value ("{}") does not equal expected ("{}")'.format(
-            value, expected)
-
     @staticmethod
     def test_generate_key_data_from_nonce():
         server_nonce = int.from_bytes(b'The 16-bit nonce', byteorder='little')