Update to layer 200

Fixes #4576.

.coveragerc

@@ -0,0 +1,8 @@
+[run]
+branch = true
+parallel = true
+source =
+    telethon
+
+[report]
+precision = 2

.github/ISSUE_TEMPLATE/bug-report.md

@@ -1,27 +0,0 @@
----
-name: Bug Report
-about: Create a report about a bug inside the library or issues with the documentation
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Checklist**
-* [ ] The error is in the library's code, and not in my own.
-* [ ] I have searched for this issue before posting it and there isn't a duplicate.
-* [ ] I ran `pip install -U https://github.com/LonamiWebs/Telethon/archive/master.zip` and triggered the bug in the latest version.
-
-**Code that causes the issue**
-```python
-from telethon.sync import TelegramClient
-...
-
-```
-
-**Traceback**
-```
-Traceback (most recent call last):
-  File "code.py", line 1, in <code>
-
-```

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,96 @@
+name: Bug Report
+description: Create a report about a bug inside the library.
+body:
+
+  - type: textarea
+    id: reproducing-example
+    attributes:
+      label: Code that causes the issue
+      description: Provide a code example that reproduces the problem. Try to keep it short without other dependencies.
+      placeholder: |
+        ```python
+        from telethon.sync import TelegramClient
+        ...
+
+        ```
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected behavior
+      description: Explain what you should expect to happen. Include reproduction steps.
+      placeholder: |
+        "I was doing... I was expecting the following to happen..."
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual-behavior
+    attributes:
+      label: Actual behavior
+      description: Explain what actually happens.
+      placeholder: |
+        "This happened instead..."
+    validations:
+      required: true
+
+  - type: textarea
+    id: traceback
+    attributes:
+      label: Traceback
+      description: |
+        The traceback, if the problem is a crash.
+      placeholder: |
+        ```
+        Traceback (most recent call last):
+          File "code.py", line 1, in <code>
+
+        ```
+
+  - type: input
+    id: telethon-version
+    attributes:
+      label: Telethon version
+      description: The output of `python -c "import telethon; print(telethon.__version__)"`.
+      placeholder: "1.x"
+    validations:
+      required: true
+
+  - type: input
+    id: python-version
+    attributes:
+      label: Python version
+      description: The output of `python --version`.
+      placeholder: "3.x"
+    validations:
+      required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating system (including distribution name and version)
+      placeholder: Windows 11, macOS 13.4, Ubuntu 23.04...
+    validations:
+      required: true
+
+  - type: textarea
+    id: other-details
+    attributes:
+      label: Other details
+      placeholder: |
+        Additional details and attachments. Is it a server? Network condition?
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Checklist
+      description: Read this carefully, we will close and ignore your issue if you skimmed through this.
+      options:
+        - label: The error is in the library's code, and not in my own.
+          required: true
+        - label: I have searched for this issue before posting it and there isn't an open duplicate.
+          required: true
+        - label: I ran `pip install -U https://github.com/LonamiWebs/Telethon/archive/v1.zip` and triggered the bug in the latest version.
+          required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask questions in StackOverflow
+    url: https://stackoverflow.com/questions/ask?tags=telethon
+    about: Questions are not bugs. Please ask them in StackOverflow instead. Questions in the bug tracker will be closed
+  - name: Find about updates and our Telegram groups
+    url: https://t.me/s/TelethonUpdates
+    about: Be notified of updates, chat with other people about the library or ask questions in these groups

.github/ISSUE_TEMPLATE/documentation-issue.yml

@@ -0,0 +1,22 @@
+name: Documentation Issue
+description: Report a problem with the documentation.
+labels: [documentation]
+body:
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: Describe the problem in detail.
+      placeholder: This part is unclear...
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Checklist
+      description: Read this carefully, we will close and ignore your issue if you skimmed through this.
+      options:
+        - label: This is a documentation problem, not a question or a bug report.
+          required: true
+        - label: I have searched for this issue before posting it and there isn't a duplicate.
+          required: true

.github/ISSUE_TEMPLATE/feature-request.md

@@ -1,10 +0,0 @@
----
-name: Feature Request
-about: Suggest ideas, changes or other enhancements for the library
-title: ''
-labels: enhancement
-assignees: ''
-
----
-
-Please describe your idea. Would you like another friendly method? Renaming them to something more appropriated? Changing the way something works?

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,22 @@
+name: Feature Request
+description: Suggest ideas, changes or other enhancements for the library.
+labels: [enhancement]
+body:
+
+  - type: textarea
+    id: feature-description
+    attributes:
+      label: Describe your suggested feature
+      description: Please describe your idea. Would you like another friendly method? Renaming them to something more appropriate? Changing the way something works?
+      placeholder: "It should work like this..."
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: checklist
+    attributes:
+      label: Checklist
+      description: Read this carefully, we will close and ignore your issue if you skimmed through this.
+      options:
+        - label: I have searched for this issue before posting it and there isn't a duplicate.
+          required: true

.github/ISSUE_TEMPLATE/question-about-usage.md

@@ -1,20 +0,0 @@
----
-name: Question about Usage
-about: QUESTIONS DON'T BELONG HERE. Ask in StackOverflow or in @TelethonUpdates
-title: ''
-labels: RTFM
-assignees: ''
-
----
-
-QUESTIONS ARE NEITHER BUGS NOR ENHANCEMENTS AND DON'T BELONG HERE.
-
-If you DO have a question, ask in:
-* https://stackoverflow.com or
-* https://t.me/TelethonUpdates (@TelethonUpdates channel in Telegram)
-
-If you do post a question, it will be labelled "RTFM" and closed as soon as possible without any answer.
-
-If you DON'T have a question, use the right template for bugs/issues with the library or to propose an improvement/enhancement to either the code or documentation.
-
-We are not being harsh. Only clear. The issues section is not for questions, and people keep asking things over and over, which is a waste of everyone's time.

.github/pull_request_template.md

@@ -0,0 +1,5 @@
+<!--
+Thanks for the PR! Please keep in mind that v1 is *feature frozen*.
+New features very likely won't be merged, although fixes can be sent.
+All new development should happen in v2. Thanks!
+-->

.github/workflows.disabled/python.yml

@@ -0,0 +1,28 @@
+name: Python Library
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.5", "3.6", "3.7", "3.8"]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Set up env
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+    - name: Lint with flake8
+      run: |
+        tox -e flake
+    - name: Test with pytest
+      run: |
+        # use "py", which is the default python version
+        tox -e py

.gitignore

@@ -1,108 +1,23 @@
-# Docs
-_build/
-docs/
-
 # Generated code
-telethon/tl/functions/
-telethon/tl/types/
-telethon/tl/patched/
-telethon/tl/alltlobjects.py
-telethon/errors/rpcerrorlist.py
+/telethon/tl/functions/
+/telethon/tl/types/
+/telethon/tl/alltlobjects.py
+/telethon/errors/rpcerrorlist.py
 
 # User session
 *.session
-usermedia/
+/usermedia/
 
-# Quick tests should live in this file
-example.py
-
-# Byte-compiled / optimized / DLL files
+# Builds and testing
 __pycache__/
-*.py[cod]
-*$py.class
+/dist/
+/build/
+/*.egg-info/
+/readthedocs/_build/
+/.tox/
 
-# C extensions
-*.so
+# API reference docs
+/docs/
 
-# Distribution / packaging
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*,cover
-.hypothesis/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# IPython Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# dotenv
-.env
-
-# virtualenv
-.venv/
-venv/
-ENV/
-
-# Spyder project settings
-.spyderproject
-
-# Rope project settings
-.ropeproject
+# File used to manually test new changes, contains sensitive data
+/example.py

.readthedocs.yaml

@@ -0,0 +1,18 @@
+# https://docs.readthedocs.io/en/stable/config-file/v2.html
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+sphinx:
+  configuration: readthedocs/conf.py
+
+formats:
+  - pdf
+  - epub
+
+python:
+  install:
+    - requirements: readthedocs/requirements.txt

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2016-2019 LonamiWebs
+Copyright (c) 2016-Present LonamiWebs
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

MANIFEST.in

@@ -1,4 +0,0 @@
-include LICENSE
-include README.rst
-
-recursive-include telethon *

README.rst

@@ -12,6 +12,8 @@ as a user or through a bot account (bot API alternative).
 
     If you have code using Telethon before its 1.0 version, you must
     read `Compatibility and Convenience`_ to learn how to migrate.
+    As with any third-party library for Telegram, be careful not to
+    break `Telegram's ToS`_ or `Telegram can ban the account`_.
 
 What is this?
 -------------
@@ -75,7 +77,9 @@ useful information.
 .. _asyncio: https://docs.python.org/3/library/asyncio.html
 .. _MTProto: https://core.telegram.org/mtproto
 .. _Telegram: https://telegram.org
-.. _Compatibility and Convenience: https://docs.telethon.dev/en/latest/misc/compatibility-and-convenience.html
+.. _Compatibility and Convenience: https://docs.telethon.dev/en/stable/misc/compatibility-and-convenience.html
+.. _Telegram's ToS: https://core.telegram.org/api/terms
+.. _Telegram can ban the account: https://docs.telethon.dev/en/stable/quick-references/faq.html#my-account-was-deleted-limited-when-using-the-library
 .. _Read The Docs: https://docs.telethon.dev
 
 .. |logo| image:: logo.svg

dev-requirements.txt

@@ -0,0 +1,3 @@
+pytest
+pytest-cov
+pytest-asyncio

optional-requirements.txt

@@ -1,4 +1,5 @@
 cryptg
 pysocks
-hachoir3
+python-socks[asyncio]
+hachoir
 pillow

pyproject.toml

@@ -0,0 +1,36 @@
+# https://snarky.ca/what-the-heck-is-pyproject-toml/
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+# Need to use legacy format for the time being
+# https://tox.readthedocs.io/en/3.20.0/example/basic.html#pyproject-toml-tox-legacy-ini
+[tool.tox]
+legacy_tox_ini = """
+[tox]
+envlist = py35,py36,py37,py38
+
+# run with tox -e py
+[testenv]
+deps =
+    -rrequirements.txt
+    -roptional-requirements.txt
+    -rdev-requirements.txt
+commands =
+    # NOTE: you can run any command line tool here - not just tests
+    pytest {posargs}
+
+# run with tox -e flake
+[testenv:flake]
+deps =
+    -rrequirements.txt
+    -roptional-requirements.txt
+    -rdev-requirements.txt
+    flake8
+commands =
+    # stop the build if there are Python syntax errors or undefined names
+    flake8 telethon/ telethon_generator/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
+    # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+    flake8 telethon/ telethon_generator/ tests/ --count --exit-zero --exclude telethon/tl/,telethon/errors/rpcerrorlist.py --max-complexity=10 --max-line-length=127 --statistics
+
+"""

readthedocs/basic/installation.rst

@@ -6,14 +6,16 @@ Installation
 
 Telethon is a Python library, which means you need to download and install
 Python from https://www.python.org/downloads/ if you haven't already. Once
-you have Python installed, run:
+you have Python installed, `upgrade pip`__ and run:
 
 .. code-block:: sh
 
-    pip3 install -U telethon --user
+    python3 -m pip install --upgrade pip
+    python3 -m pip install --upgrade telethon
 
-To install or upgrade the library to the latest version.
+…to install or upgrade the library to the latest version.
 
+.. __: https://pythonspeed.com/articles/upgrade-pip/
 
 Installing Development Versions
 ===============================
@@ -23,7 +25,7 @@ you can run the following command instead:
 
 .. code-block:: sh
 
-    pip3 install -U https://github.com/LonamiWebs/Telethon/archive/master.zip --user
+    python3 -m pip install --upgrade https://github.com/LonamiWebs/Telethon/archive/v1.zip
 
 .. note::
 
@@ -74,7 +76,7 @@ manually.
 
     Some of the modules may require additional dependencies before being
     installed through ``pip``. If you have an ``apt``-based system, consider
-    installing the most commonly missing dependencies:
+    installing the most commonly missing dependencies (with the right ``pip``):
 
     .. code-block:: sh
 
@@ -85,6 +87,7 @@ manually.
 
     Thanks to `@bb010g`_ for writing down this nice list.
 
+
 .. _cryptg: https://github.com/cher-nov/cryptg
 .. _pyaes: https://github.com/ricmoo/pyaes
 .. _pillow: https://python-pillow.org

readthedocs/basic/next-steps.rst

@@ -20,3 +20,27 @@ that are worth learning and understanding.
 
 From now on, you can keep pressing the "Next" button if you want,
 or use the menu on the left, since some pages are quite lengthy.
+
+A note on developing applications
+=================================
+
+If you're using the library to make an actual application (and not just
+automate things), you should make sure to `comply with the ToS`__:
+
+    […] when logging in as an existing user, apps are supposed to call
+    [:tl:`GetTermsOfServiceUpdate`] to check for any updates to the Terms of
+    Service; this call should be repeated after ``expires`` seconds have
+    elapsed. If an update to the Terms Of Service is available, clients are
+    supposed to show a consent popup; if accepted, clients should call
+    [:tl:`AcceptTermsOfService`], providing the ``termsOfService id`` JSON
+    object; in case of denial, clients are to delete the account using
+    [:tl:`DeleteAccount`], providing Decline ToS update as deletion reason.
+
+.. __: https://core.telegram.org/api/config#terms-of-service
+
+However, if you use the library to automate or enhance your Telegram
+experience, it's very likely that you are using other applications doing this
+check for you (so you wouldn't run the risk of violating the ToS).
+
+The library itself will not automatically perform this check or accept the ToS
+because it should require user action (the only exception is during sign-up).

readthedocs/basic/quick-start.rst

@@ -8,17 +8,18 @@ use these if possible.
 
 .. code-block:: python
 
-    from telethon.sync import TelegramClient
+    from telethon import TelegramClient
 
     # Remember to use your own values from my.telegram.org!
     api_id = 12345
     api_hash = '0123456789abcdef0123456789abcdef'
+    client = TelegramClient('anon', api_id, api_hash)
 
-    with TelegramClient('anon', api_id, api_hash) as client:
+    async def main():
         # Getting information about yourself
-        me = client.get_me()
+        me = await client.get_me()
 
-        # "me" is an User object. You can pretty-print
+        # "me" is a user object. You can pretty-print
         # any Telegram object with the "stringify" method:
         print(me.stringify())
 
@@ -30,23 +31,23 @@ use these if possible.
         print(me.phone)
 
         # You can print all the dialogs/conversations that you are part of:
-        for dialog in client.iter_dialogs():
+        async for dialog in client.iter_dialogs():
             print(dialog.name, 'has ID', dialog.id)
 
         # You can send messages to yourself...
-        client.send_message('me', 'Hello, myself!')
+        await client.send_message('me', 'Hello, myself!')
         # ...to some chat ID
-        client.send_message(-100123456, 'Hello, group!')
+        await client.send_message(-100123456, 'Hello, group!')
         # ...to your contacts
-        client.send_message('+34600123123', 'Hello, friend!')
+        await client.send_message('+34600123123', 'Hello, friend!')
         # ...or even to any username
-        client.send_message('TelethonChat', 'Hello, Telethon!')
+        await client.send_message('username', 'Testing Telethon!')
 
         # You can, of course, use markdown in your messages:
-        message = client.send_message(
+        message = await client.send_message(
             'me',
             'This message has **bold**, `code`, __italics__ and '
-            'a [nice website](https://lonamiwebs.github.io)!',
+            'a [nice website](https://example.com)!',
             link_preview=False
         )
 
@@ -54,20 +55,23 @@ use these if possible.
         print(message.raw_text)
 
         # You can reply to messages directly if you have a message object
-        message.reply('Cool!')
+        await message.reply('Cool!')
 
         # Or send files, songs, documents, albums...
-        client.send_file('me', '/home/me/Pictures/holidays.jpg')
+        await client.send_file('me', '/home/me/Pictures/holidays.jpg')
 
         # You can print the message history of any chat:
-        for message in client.iter_messages('me'):
+        async for message in client.iter_messages('me'):
             print(message.id, message.text)
 
             # You can download media from messages, too!
             # The method will return the path where the file was saved.
             if message.photo:
-                path = message.download_media()
-                print('File saved to', path)
+                path = await message.download_media()
+                print('File saved to', path)  # printed after download is done
+
+    with client:
+        client.loop.run_until_complete(main())
 
 
 Here, we show how to sign in, get information about yourself, send
@@ -77,3 +81,31 @@ files.
 You should make sure that you understand what the code shown here
 does, take note on how methods are called and used and so on before
 proceeding. We will see all the available methods later on.
+
+.. important::
+
+    Note that Telethon is an asynchronous library, and as such, you should
+    get used to it and learn a bit of basic `asyncio`. This will help a lot.
+    As a quick start, this means you generally want to write all your code
+    inside some ``async def`` like so:
+
+    .. code-block:: python
+
+        client = ...
+
+        async def do_something(me):
+            ...
+
+        async def main():
+            # Most of your code should go here.
+            # You can of course make and use your own async def (do_something).
+            # They only need to be async if they need to await things.
+            me = await client.get_me()
+            await do_something(me)
+
+        with client:
+            client.loop.run_until_complete(main())
+
+    After you understand this, you may use the ``telethon.sync`` hack if you
+    want do so (see :ref:`compatibility-and-convenience`), but note you may
+    run into other issues (iPython, Anaconda, etc. have some issues with it).

readthedocs/basic/signing-in.rst

@@ -49,7 +49,7 @@ We can finally write some code to log into our account!
 
 .. code-block:: python
 
-    from telethon.sync import TelegramClient
+    from telethon import TelegramClient
 
     # Use your own values from my.telegram.org
     api_id = 12345
@@ -57,7 +57,7 @@ We can finally write some code to log into our account!
 
     # The first parameter is the .session file name (absolute paths allowed)
     with TelegramClient('anon', api_id, api_hash) as client:
-        client.send_message('me', 'Hello, myself!')
+        client.loop.run_until_complete(client.send_message('me', 'Hello, myself!'))
 
 
 In the first line, we import the class name so we can create an instance
@@ -68,6 +68,16 @@ At last, we create a new `TelegramClient <telethon.client.telegramclient.Telegra
 instance and call it ``client``. We can now use the client variable
 for anything that we want, such as sending a message to ourselves.
 
+.. note::
+
+    Since Telethon is an asynchronous library, you need to ``await``
+    coroutine functions to have them run (or otherwise, run the loop
+    until they are complete). In this tiny example, we don't bother
+    making an ``async def main()``.
+
+    See :ref:`mastering-asyncio` to find out more.
+
+
 Using a ``with`` block is the preferred way to use the library. It will
 automatically `start() <telethon.client.auth.AuthMethods.start>` the client,
 logging or signing up if necessary.
@@ -89,7 +99,7 @@ You will still need an API ID and hash, but the process is very similar:
 
     api_id = 12345
     api_hash = '0123456789abcdef0123456789abcdef'
-    bot_token = '12345:0123456789abcdef0123456789abcdef
+    bot_token = '12345:0123456789abcdef0123456789abcdef'
 
     # We have to manually call "start" if we want an explicit bot token
     bot = TelegramClient('bot', api_id, api_hash).start(bot_token=bot_token)
@@ -107,7 +117,12 @@ Signing In behind a Proxy
 =========================
 
 If you need to use a proxy to access Telegram,
-you will need to  `install PySocks`__ and then change:
+you will need to either:
+
+* For Python >= 3.6 : `install python-socks[asyncio]`__
+* For Python <= 3.5 : `install PySocks`__
+
+and then change
 
 .. code-block:: python
 
@@ -117,13 +132,48 @@ with
 
 .. code-block:: python
 
-    TelegramClient('anon', api_id, api_hash, proxy=(socks.SOCKS5, '127.0.0.1', 4444))
+    TelegramClient('anon', api_id, api_hash, proxy=("socks5", '127.0.0.1', 4444))
 
-(of course, replacing the IP and port with the IP and port of the proxy).
+(of course, replacing the protocol, IP and port with the protocol, IP and port of the proxy).
 
-The ``proxy=`` argument should be a tuple, a list or a dict,
+The ``proxy=`` argument should be a dict (or tuple, for backwards compatibility),
 consisting of parameters described `in PySocks usage`__.
 
+The allowed values for the argument ``proxy_type`` are:
+
+* For Python <= 3.5:
+    * ``socks.SOCKS5`` or ``'socks5'``
+    * ``socks.SOCKS4`` or ``'socks4'``
+    * ``socks.HTTP`` or ``'http'``
+
+* For Python >= 3.6:
+    * All of the above
+    * ``python_socks.ProxyType.SOCKS5``
+    * ``python_socks.ProxyType.SOCKS4``
+    * ``python_socks.ProxyType.HTTP``
+
+
+Example:
+
+.. code-block:: python
+
+    proxy = {
+        'proxy_type': 'socks5', # (mandatory) protocol to use (see above)
+        'addr': '1.1.1.1',      # (mandatory) proxy IP address
+        'port': 5555,           # (mandatory) proxy port number
+        'username': 'foo',      # (optional) username if the proxy requires auth
+        'password': 'bar',      # (optional) password if the proxy requires auth
+        'rdns': True            # (optional) whether to use remote or local resolve, default remote
+    }
+
+For backwards compatibility with ``PySocks`` the following format
+is possible (but discouraged):
+
+.. code-block:: python
+
+    proxy = (socks.SOCKS5, '1.1.1.1', 5555, True, 'foo', 'bar')
+
+.. __: https://github.com/romis2012/python-socks#installation
 .. __: https://github.com/Anorov/PySocks#installation
 .. __: https://github.com/Anorov/PySocks#usage-1
 

readthedocs/basic/updates.rst

@@ -16,7 +16,7 @@ For that, you can use **events**.
     .. code-block:: python
 
         import logging
-        logging.basicConfig(format='[%(levelname) 5s/%(asctime)s] %(name)s: %(message)s',
+        logging.basicConfig(format='[%(levelname) %(asctime)s] %(name)s: %(message)s',
                             level=logging.WARNING)
 
 

readthedocs/concepts/asyncio.rst

@@ -40,22 +40,22 @@ because tasks are smaller than threads, which are smaller than processes.
 What are asyncio basics?
 ========================
 
+The code samples below assume that you have Python 3.7 or greater installed.
+
 .. code-block:: python
 
     # First we need the asyncio library
     import asyncio
 
-    # Then we need a loop to work with
-    loop = asyncio.get_event_loop()
-
     # We also need something to run
     async def main():
         for char in 'Hello, world!\n':
             print(char, end='', flush=True)
             await asyncio.sleep(0.2)
 
-    # Then, we need to run the loop with a task
-    loop.run_until_complete(main())
+    # Then, we can create a new asyncio loop and use it to run our coroutine.
+    # The creation and tear-down of the loop is hidden away from us.
+    asyncio.run(main())
 
 
 What does telethon.sync do?
@@ -96,8 +96,12 @@ Instead of this:
 
 .. code-block:: python
 
+    me = client.loop.run_until_complete(client.get_me())
+    print(me.username)
+
+    # or, using asyncio's default loop (it's the same)
     import asyncio
-    loop = asyncio.get_event_loop()
+    loop = asyncio.get_running_loop()  # == client.loop
     me = loop.run_until_complete(client.get_me())
     print(me.username)
 
@@ -154,13 +158,10 @@ loops or use ``async with``:
 
                 print(message.sender.username)
 
-    loop = asyncio.get_event_loop()
-    # ^ this assigns the default event loop from the main thread to a variable
-
-    loop.run_until_complete(main())
-    # ^ this runs the *entire* loop until the main() function finishes.
-    #   While the main() function does not finish, the loop will be running.
-    #   While the loop is running, you can't run it again.
+    asyncio.run(main())
+    # ^ this will create a new asyncio loop behind the scenes and tear it down
+    #   once the function returns. It will run the loop untiil main finishes.
+    #   You should only use this function if there is no other loop running.
 
 
 The ``await`` keyword blocks the *current* task, and the loop can run
@@ -180,14 +181,14 @@ concurrently:
         await asyncio.sleep(delay)  # await tells the loop this task is "busy"
         print('world')  # eventually the loop finishes all tasks
 
-    loop = asyncio.get_event_loop()  # get the default loop for the main thread
-    loop.create_task(world(2))  # create the world task, passing 2 as delay
-    loop.create_task(hello(delay=1))  # another task, but with delay 1
+    async def main():
+        asyncio.create_task(world(2))  # create the world task, passing 2 as delay
+        asyncio.create_task(hello(delay=1))  # another task, but with delay 1
+        await asyncio.sleep(3)  # wait for three seconds before exiting
+
     try:
-        # run the event loop forever; ctrl+c to stop it
-        # we could also run the loop for three seconds:
-        #     loop.run_until_complete(asyncio.sleep(3))
-        loop.run_forever()
+        # create a new temporary asyncio loop and use it to run main
+        asyncio.run(main())
     except KeyboardInterrupt:
         pass
 
@@ -205,29 +206,36 @@ The same example, but without the comment noise:
         await asyncio.sleep(delay)
         print('world')
 
-    loop = asyncio.get_event_loop()
-    loop.create_task(world(2))
-    loop.create_task(hello(1))
-    loop.run_until_complete(asyncio.sleep(3))
+    async def main():
+        asyncio.create_task(world(2))
+        asyncio.create_task(hello(delay=1))
+        await asyncio.sleep(3)
+
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        pass
 
 
 Can I use threads?
 ==================
 
 Yes, you can, but you must understand that the loops themselves are
-not thread safe. and you must be sure to know what is happening. You
-may want to create a loop in a new thread and make sure to pass it to
-the client:
+not thread safe. and you must be sure to know what is happening. The
+easiest and cleanest option is to use `asyncio.run` to create and manage
+the new event loop for you:
 
 .. code-block:: python
 
     import asyncio
     import threading
 
-    def go():
-        loop = asyncio.new_event_loop()
+    async def actual_work():
         client = TelegramClient(..., loop=loop)
-        ...
+        ...  # can use `await` here
+
+    def go():
+        asyncio.run(actual_work())
 
     threading.Thread(target=go).start()
 
@@ -244,9 +252,9 @@ You may have seen this error:
 
     RuntimeError: There is no current event loop in thread 'Thread-1'.
 
-It just means you didn't create a loop for that thread, and if you don't
-pass a loop when creating the client, it uses ``asyncio.get_event_loop()``,
-which only works in the main thread.
+It just means you didn't create a loop for that thread. Please refer to
+the ``asyncio`` documentation to correctly learn how to set the event loop
+for non-main threads.
 
 
 client.run_until_disconnected() blocks!
@@ -304,27 +312,26 @@ you can run requests in parallel:
 
     async def main():
         last, sent, download_path = await asyncio.gather(
-            client.get_messages('TelethonChat', 10),
-            client.send_message('TelethonOfftopic', 'Hey guys!'),
-            client.download_profile_photo('TelethonChat')
+            client.get_messages('telegram', 10),
+            client.send_message('me', 'Using asyncio!'),
+            client.download_profile_photo('telegram')
         )
 
     loop.run_until_complete(main())
 
 
-This code will get the 10 last messages from `@TelethonChat
-<https://t.me/TelethonChat>`_, send one to `@TelethonOfftopic
-<https://t.me/TelethonOfftopic>`_, and also download the profile
-photo of the main group. `asyncio` will run all these three tasks
-at the same time. You can run all the tasks you want this way.
+This code will get the 10 last messages from `@telegram
+<https://t.me/telegram>`_, send one to the chat with yourself, and also
+download the profile photo of the channel. `asyncio` will run all these
+three tasks at the same time. You can run all the tasks you want this way.
 
 A different way would be:
 
 .. code-block:: python
 
-    loop.create_task(client.get_messages('TelethonChat', 10))
-    loop.create_task(client.send_message('TelethonOfftopic', 'Hey guys!'))
-    loop.create_task(client.download_profile_photo('TelethonChat'))
+    loop.create_task(client.get_messages('telegram', 10))
+    loop.create_task(client.send_message('me', 'Using asyncio!'))
+    loop.create_task(client.download_profile_photo('telegram'))
 
 They will run in the background as long as the loop is running too.
 
@@ -356,6 +363,6 @@ Where can I read more?
 ======================
 
 `Check out my blog post
-<https://lonamiwebs.github.io/blog/asyncio/>`_ about `asyncio`, which
+<https://lonami.dev/blog/asyncio/>`_ about `asyncio`, which
 has some more examples and pictures to help you understand what happens
 when the loop runs.

readthedocs/concepts/botapi-vs-mtproto.rst

@@ -28,6 +28,9 @@ their own Telegram bots. Quoting their main page:
 Bot API is simply an HTTP endpoint which translates your requests to it into
 MTProto calls through tdlib_, their bot backend.
 
+Configuration of your bot, such as its available commands and auto-completion,
+is configured through `@BotFather <https://t.me/BotFather>`_.
+
 
 What is MTProto?
 ================
@@ -66,8 +69,8 @@ things, you will be able to easily login as a user and even keep your bot
 without having to learn a new library.
 
 If less overhead and full control didn't convince you to use Telethon yet,
-check out the repository `HTTP Bot API vs MTProto comparison`_ with a more
-exhaustive and up-to-date list of differences.
+check out the wiki page `MTProto vs HTTP Bot API`_ with a more exhaustive
+and up-to-date list of differences.
 
 
 Migrating from Bot API to Telethon
@@ -88,7 +91,7 @@ Next, we will see some examples from the most popular libraries.
 Migrating from python-telegram-bot
 ----------------------------------
 
-Let's take their `echobot2.py`_ example and shorten it a bit:
+Let's take their `echobot.py`_ example and shorten it a bit:
 
 .. code-block:: python
 
@@ -107,7 +110,7 @@ Let's take their `echobot2.py`_ example and shorten it a bit:
         updater = Updater("TOKEN")
         dp = updater.dispatcher
         dp.add_handler(CommandHandler("start", start))
-        dp.add_handler(MessageHandler(Filters.text, echo))
+        dp.add_handler(MessageHandler(Filters.text & ~Filters.command, echo))
 
         updater.start_polling()
 
@@ -145,7 +148,7 @@ After using Telethon:
 
 Key differences:
 
-* The recommended way to do it imports less things.
+* The recommended way to do it imports fewer things.
 * All handlers trigger by default, so we need ``events.StopPropagation``.
 * Adding handlers, responding and running is a lot less verbose.
 * Telethon needs ``async def`` and ``await``.
@@ -296,7 +299,7 @@ After rewriting:
 
     class Subbot(TelegramClient):
         def __init__(self, *a, **kw):
-            await super().__init__(*a, **kw)
+            super().__init__(*a, **kw)
             self.add_event_handler(self.on_update, events.NewMessage)
 
         async def connect():
@@ -323,11 +326,11 @@ Key differences:
 .. _Bot FAQ: https://core.telegram.org/bots/faq
 .. _tdlib: https://core.telegram.org/tdlib
 .. _MTProto: https://core.telegram.org/mtproto
-.. _HTTP Bot API vs MTProto comparison: https://github.com/telegram-mtproto/botapi-comparison
+.. _MTProto vs HTTP Bot API: https://github.com/LonamiWebs/Telethon/wiki/MTProto-vs-HTTP-Bot-API
 .. _requests: https://pypi.org/project/requests/
 .. _python-telegram-bot: https://python-telegram-bot.readthedocs.io
 .. _pyTelegramBotAPI: https://github.com/eternnoir/pyTelegramBotAPI
 .. _aiohttp: https://docs.aiohttp.org/en/stable
 .. _aiogram: https://aiogram.readthedocs.io
 .. _dumbot: https://github.com/Lonami/dumbot
-.. _echobot2.py: https://github.com/python-telegram-bot/python-telegram-bot/blob/master/examples/echobot2.py
+.. _echobot.py: https://github.com/python-telegram-bot/python-telegram-bot/blob/master/examples/echobot.py

readthedocs/concepts/chats-vs-channels.rst

@@ -0,0 +1,169 @@
+.. _chats-channels:
+
+=================
+Chats vs Channels
+=================
+
+Telegram's raw API can get very confusing sometimes, in particular when it
+comes to talking about "chats", "channels", "groups", "megagroups", and all
+those concepts.
+
+This section will try to explain what each of these concepts are.
+
+
+Chats
+=====
+
+A ``Chat`` can be used to talk about either the common "subclass" that both
+chats and channels share, or the concrete :tl:`Chat` type.
+
+Technically, both :tl:`Chat` and :tl:`Channel` are a form of the `Chat type`_.
+
+**Most of the time**, the term :tl:`Chat` is used to talk about *small group
+chats*. When you create a group through an official application, this is the
+type that you get. Official applications refer to these as "Group".
+
+Both the bot API and Telethon will add a minus sign (negate) the real chat ID
+so that you can tell at a glance, with just a number, the entity type.
+
+For example, if you create a chat with :tl:`CreateChatRequest`, the real chat
+ID might be something like `123`. If you try printing it from a
+`message.chat_id` you will see `-123`. This ID helps Telethon know you're
+talking about a :tl:`Chat`.
+
+
+Channels
+========
+
+Official applications create a *broadcast* channel when you create a new
+channel (used to broadcast messages, only administrators can post messages).
+
+Official applications implicitly *migrate* an *existing* :tl:`Chat` to a
+*megagroup* :tl:`Channel` when you perform certain actions (exceed user limit,
+add a public username, set certain permissions, etc.).
+
+A ``Channel`` can be created directly with :tl:`CreateChannelRequest`, as
+either a ``megagroup`` or ``broadcast``.
+
+Official applications use the term "channel" **only** for broadcast channels.
+
+The API refers to the different types of :tl:`Channel` with certain attributes:
+
+* A **broadcast channel** is a :tl:`Channel` with the ``channel.broadcast``
+  attribute set to `True`.
+
+* A **megagroup channel** is a :tl:`Channel` with the ``channel.megagroup``
+  attribute set to `True`. Official applications refer to this as "supergroup".
+
+* A **gigagroup channel** is a :tl:`Channel` with the ``channel.gigagroup``
+  attribute set to `True`. Official applications refer to this as "broadcast
+  groups", and is used when a megagroup becomes very large and administrators
+  want to transform it into something where only they can post messages.
+
+
+Both the bot API and Telethon will "concatenate" ``-100`` to the real chat ID
+so that you can tell at a glance, with just a number, the entity type.
+
+For example, if you create a new broadcast channel, the real channel ID might
+be something like `456`. If you try printing it from a `message.chat_id` you
+will see `-1000000000456`. This ID helps Telethon know you're talking about a
+:tl:`Channel`.
+
+
+Converting IDs
+==============
+
+You can convert between the "marked" identifiers (prefixed with a minus sign)
+and the real ones with ``utils.resolve_id``. It will return a tuple with the
+real ID, and the peer type (the class):
+
+.. code-block:: python
+
+    from telethon import utils
+    real_id, peer_type = utils.resolve_id(-1000000000456)
+
+    print(real_id)  # 456
+    print(peer_type)  # <class 'telethon.tl.types.PeerChannel'>
+
+    peer = peer_type(real_id)
+    print(peer)  # PeerChannel(channel_id=456)
+
+
+The reverse operation can be done with ``utils.get_peer_id``:
+
+.. code-block:: python
+
+    print(utils.get_peer_id(types.PeerChannel(456)))  # -1000000000456
+
+
+Note that this function can also work with other types, like :tl:`Chat` or
+:tl:`Channel` instances.
+
+If you need to convert other types like usernames which might need to perform
+API calls to find out the identifier, you can use ``client.get_peer_id``:
+
+
+.. code-block:: python
+
+    print(await client.get_peer_id('me'))  # your id
+
+
+If there is no "mark" (no minus sign), Telethon will assume your identifier
+refers to a :tl:`User`. If this is **not** the case, you can manually fix it:
+
+
+.. code-block:: python
+
+    from telethon import types
+    await client.send_message(types.PeerChannel(456), 'hello')
+    #                         ^^^^^^^^^^^^^^^^^ explicit peer type
+
+
+A note on raw API
+=================
+
+Certain methods only work on a :tl:`Chat`, and some others only work on a
+:tl:`Channel` (and these may only work in broadcast, or megagroup). Your code
+likely knows what it's working with, so it shouldn't be too much of an issue.
+
+If you need to find the :tl:`Channel` from a :tl:`Chat` that migrated to it,
+access the `migrated_to` property:
+
+.. code-block:: python
+
+    # chat is a Chat
+    channel = await client.get_entity(chat.migrated_to)
+    # channel is now a Channel
+
+Channels do not have a "migrated_from", but a :tl:`ChannelFull` does. You can
+use :tl:`GetFullChannelRequest` to obtain this:
+
+.. code-block:: python
+
+    from telethon import functions
+    full = await client(functions.channels.GetFullChannelRequest(your_channel))
+    full_channel = full.full_chat
+    # full_channel is a ChannelFull
+    print(full_channel.migrated_from_chat_id)
+
+This way, you can also access the linked discussion megagroup of a broadcast channel:
+
+.. code-block:: python
+
+    print(full_channel.linked_chat_id)  # prints ID of linked discussion group or None
+
+You do not need to use ``client.get_entity`` to access the
+``migrated_from_chat_id`` :tl:`Chat` or the ``linked_chat_id`` :tl:`Channel`.
+They are in the ``full.chats`` attribute:
+
+.. code-block:: python
+
+    if full_channel.migrated_from_chat_id:
+        migrated_from_chat = next(c for c in full.chats if c.id == full_channel.migrated_from_chat_id)
+        print(migrated_from_chat.title)
+
+    if full_channel.linked_chat_id:
+        linked_group = next(c for c in full.chats if c.id == full_channel.linked_chat_id)
+        print(linked_group.username)
+
+.. _Chat type: https://tl.telethon.dev/types/chat.html

readthedocs/concepts/entities.rst

@@ -102,33 +102,35 @@ you're able to just do this:
 
 .. code-block:: python
 
+    # (These examples assume you are inside an "async def")
+    #
     # Dialogs are the "conversations you have open".
     # This method returns a list of Dialog, which
     # has the .entity attribute and other information.
     #
-    # This part is IMPORTANT, because it feels the entity cache.
-    dialogs = client.get_dialogs()
+    # This part is IMPORTANT, because it fills the entity cache.
+    dialogs = await client.get_dialogs()
 
     # All of these work and do the same.
-    lonami = client.get_entity('lonami')
-    lonami = client.get_entity('t.me/lonami')
-    lonami = client.get_entity('https://telegram.dog/lonami')
+    username = await client.get_entity('username')
+    username = await client.get_entity('t.me/username')
+    username = await client.get_entity('https://telegram.dog/username')
 
     # Other kind of entities.
-    channel = client.get_entity('telegram.me/joinchat/AAAAAEkk2WdoDrB4-Q8-gg')
-    contact = client.get_entity('+34xxxxxxxxx')
-    friend  = client.get_entity(friend_id)
+    channel = await client.get_entity('telegram.me/joinchat/AAAAAEkk2WdoDrB4-Q8-gg')
+    contact = await client.get_entity('+34xxxxxxxxx')
+    friend  = await client.get_entity(friend_id)
 
     # Getting entities through their ID (User, Chat or Channel)
-    entity = client.get_entity(some_id)
+    entity = await client.get_entity(some_id)
 
     # You can be more explicit about the type for said ID by wrapping
     # it inside a Peer instance. This is recommended but not necessary.
     from telethon.tl.types import PeerUser, PeerChat, PeerChannel
 
-    my_user    = client.get_entity(PeerUser(some_id))
-    my_chat    = client.get_entity(PeerChat(some_id))
-    my_channel = client.get_entity(PeerChannel(some_id))
+    my_user    = await client.get_entity(PeerUser(some_id))
+    my_chat    = await client.get_entity(PeerChat(some_id))
+    my_channel = await client.get_entity(PeerChannel(some_id))
 
 
 .. note::
@@ -140,7 +142,7 @@ you're able to just do this:
 All methods in the :ref:`telethon-client` call `.get_input_entity()
 <telethon.client.users.UserMethods.get_input_entity>` prior
 to sending the request to save you from the hassle of doing so manually.
-That way, convenience calls such as `client.send_message('lonami', 'hi!')
+That way, convenience calls such as `client.send_message('username', 'hi!')
 <telethon.client.messages.MessageMethods.send_message>`
 become possible.
 
@@ -176,7 +178,7 @@ exist, which just have the ID. You cannot get the hash out of them since
 you should not be needing it. The library probably has cached it before.
 
 Peers are enough to identify an entity, but they are not enough to make
-a request with them use them. You need to know their hash before you can
+a request with them. You need to know their hash before you can
 "use them", and to know the hash you need to "encounter" them, let it
 be in your dialogs, participants, message forwards, etc.
 
@@ -212,7 +214,7 @@ wherever needed, so you can even do things like:
 
 .. code-block:: python
 
-    client(SendMessageRequest('username', 'hello'))
+    await client(SendMessageRequest('username', 'hello'))
 
 The library will call the ``.resolve()`` method of the request, which will
 resolve ``'username'`` with the appropriated :tl:`InputPeer`. Don't worry if
@@ -258,7 +260,7 @@ That means you can do this:
 
     message.is_private
     message.chat_id
-    message.get_chat()
+    await message.get_chat()
     # ...etc
 
 `SenderGetter <telethon.tl.custom.sendergetter.SenderGetter>` is similar:
@@ -266,7 +268,7 @@ That means you can do this:
 .. code-block:: python
 
     message.user_id
-    message.get_input_user()
+    await message.get_input_sender()
     message.user
     # ...etc
 
@@ -285,22 +287,25 @@ applications"? Now do the same with the library. Use what applies:
 
 .. code-block:: python
 
-    with client:
-        # Does it have an username? Use it!
-        entity = client.get_entity(username)
+    # (These examples assume you are inside an "async def")
+    async with client:
+        # Does it have a username? Use it!
+        entity = await client.get_entity(username)
 
         # Do you have a conversation open with them? Get dialogs.
-        client.get_dialogs()
+        await client.get_dialogs()
 
         # Are they participant of some group? Get them.
-        client.get_participants('TelethonChat')
+        await client.get_participants('username')
 
         # Is the entity the original sender of a forwarded message? Get it.
-        client.get_messages('TelethonChat', 100)
+        await client.get_messages('username', 100)
 
         # NOW you can use the ID, anywhere!
-        entity = client.get_entity(123456)
-        client.send_message(123456, 'Hi!')
+        await client.send_message(123456, 'Hi!')
+
+        entity = await client.get_entity(123456)
+        print(entity)
 
 Once the library has "seen" the entity, you can use their **integer** ID.
 You can't use entities from IDs the library hasn't seen. You must make the

readthedocs/concepts/errors.rst

@@ -7,8 +7,59 @@ RPC Errors
 RPC stands for Remote Procedure Call, and when the library raises
 a ``RPCError``, it's because you have invoked some of the API
 methods incorrectly (wrong parameters, wrong permissions, or even
-something went wrong on Telegram's server). All the errors are
-available in :ref:`telethon-errors`, but some examples are:
+something went wrong on Telegram's server).
+
+You should import the errors from ``telethon.errors`` like so:
+
+.. code-block:: python
+
+    from telethon import errors
+
+    try:
+        async with client.takeout() as takeout:
+            ...
+
+    except errors.TakeoutInitDelayError as e:
+        #  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ here we except TAKEOUT_INIT_DELAY
+        print('Must wait', e.seconds, 'before takeout')
+
+
+There isn't any official list of all possible RPC errors, so the
+`list of known errors`_ is provided on a best-effort basis. When new methods
+are available, the list may be lacking since we simply don't know what errors
+can raise from them.
+
+Once we do find out about a new error and what causes it, the list is
+updated, so if you see an error without a specific class, do report it
+(and what method caused it)!.
+
+This list is used to generate documentation for the `raw API page`_.
+For example, if we want to know what errors can occur from
+`messages.sendMessage`_ we can simply navigate to its raw API page
+and find it has 24 known RPC errors at the time of writing.
+
+
+Base Errors
+===========
+
+All the "base" errors are listed in :ref:`telethon-errors`.
+Any other more specific error will be a subclass of these.
+
+If the library isn't aware of a specific error just yet, it will instead
+raise one of these superclasses. This means you may find stuff like this:
+
+.. code-block:: text
+
+    telethon.errors.rpcbaseerrors.BadRequestError: RPCError 400: MESSAGE_POLL_CLOSED (caused by SendVoteRequest)
+
+If you do, make sure to open an issue or send a pull request to update the
+`list of known errors`_.
+
+
+Common Errors
+=============
+
+These are some of the errors you may normally need to deal with:
 
 -  ``FloodWaitError`` (420), the same request was repeated many times.
    Must wait ``.seconds`` (you can access this attribute). For example:
@@ -19,15 +70,15 @@ available in :ref:`telethon-errors`, but some examples are:
        from telethon import errors
 
        try:
-           print(client.get_messages(chat)[0].text)
+           messages = await client.get_messages(chat)
+           print(messages[0].text)
        except errors.FloodWaitError as e:
            print('Have to sleep', e.seconds, 'seconds')
            time.sleep(e.seconds)
 
 -  ``SessionPasswordNeededError``, if you have setup two-steps
-   verification on Telegram.
--  ``CdnFileTamperedError``, if the media you were trying to download
-   from a CDN has been altered.
+   verification on Telegram and are trying to sign in.
+-  ``FilePartMissingError``, if you have tried to upload an empty file.
 -  ``ChatAdminRequiredError``, you don't have permissions to perform
    said operation on a chat or channel. Try avoiding filters, i.e. when
    searching messages.
@@ -46,6 +97,28 @@ You can refer to all errors from Python through the ``telethon.errors``
 module. If you don't know what attributes they have, try printing their
 dir (like ``print(dir(e))``).
 
+
+Attributes
+==========
+
+Some of the errors carry additional data in them. When they look like
+``EMAIL_UNCONFIRMED_X``, the ``_X`` value will be accessible from the
+error instance. The current list of errors that do this is the following:
+
+- ``EmailUnconfirmedError`` has ``.code_length``.
+- ``FileMigrateError`` has ``.new_dc``.
+- ``FilePartMissingError`` has ``.which``.
+- ``FloodTestPhoneWaitError`` has ``.seconds``.
+- ``FloodWaitError`` has ``.seconds``.
+- ``InterdcCallErrorError`` has ``.dc``.
+- ``InterdcCallRichErrorError`` has ``.dc``.
+- ``NetworkMigrateError`` has ``.new_dc``.
+- ``PhoneMigrateError`` has ``.new_dc``.
+- ``SlowModeWaitError`` has ``.seconds``.
+- ``TakeoutInitDelayError`` has ``.seconds``.
+- ``UserMigrateError`` has ``.new_dc``.
+
+
 Avoiding Limits
 ===============
 
@@ -75,3 +148,8 @@ You can also except it and act as you prefer:
         quit(1)
 
 VoIP numbers are very limited, and some countries are more limited too.
+
+
+.. _list of known errors: https://github.com/LonamiWebs/Telethon/blob/v1/telethon_generator/data/errors.csv
+.. _raw API page: https://tl.telethon.dev/
+.. _messages.sendMessage: https://tl.telethon.dev/methods/messages/send_message.html

readthedocs/concepts/full-api.rst

@@ -10,13 +10,20 @@ The Full API
     methods listed on :ref:`client-ref` unless you have a better reason
     not to, like a method not existing or you wanting more control.
 
+.. contents::
+
+
+Introduction
+============
 
 The :ref:`telethon-client` doesn't offer a method for every single request
 the Telegram API supports. However, it's very simple to *call* or *invoke*
-any request. Whenever you need something, don't forget to `check the documentation`_
-and look for the `method you need`_. There you can go through a sorted list
-of everything you can do.
+any request defined in Telegram's API.
 
+This section will teach you how to use what Telethon calls the `TL reference`_.
+The linked page contains a list and a way to search through *all* types
+generated from the definition of Telegram's API (in ``.tl`` file format,
+hence the name). These types include requests and constructors.
 
 .. note::
 
@@ -25,19 +32,193 @@ of everything you can do.
     as you type, and a "Copy import" button. If you like namespaces, you
     can also do ``from telethon.tl import types, functions``. Both work.
 
+Telegram makes these ``.tl`` files public, which other implementations, such
+as Telethon, can also use to generate code. These files are versioned under
+what's called "layers". ``.tl`` files consist of thousands of definitions,
+and newer layers often add, change, or remove them. Each definition refers
+to either a Remote Procedure Call (RPC) function, or a type (which the
+`TL reference`_ calls "constructors", as they construct particular type
+instances).
 
-.. important::
-
-    All the examples in this documentation assume that you have
-    ``from telethon import sync`` or ``import telethon.sync`` for the
-    sake of simplicity and that you understand what it does (see
-    :ref:`compatibility-and-convenience` for more). Simply add
-    either line at the beginning of your project and it will work.
+As such, the `TL reference`_ is a good place to go to learn about all possible
+requests, types, and what they look like. If you're curious about what's been
+changed between layers, you can refer to the `TL diff`_ site.
 
 
-You should also refer to the documentation to see what the objects
-(constructors) Telegram returns look like. Every constructor inherits
-from a common type, and that's the reason for this distinction.
+Navigating the TL reference
+===========================
+
+Functions
+---------
+
+"Functions" is the term used for the Remote Procedure Calls (RPC) that can be
+sent to Telegram to ask it to perform something (e.g. "send message"). These
+requests have an associated return type. These can be invoked ("called"):
+
+.. code-block:: python
+
+    client = TelegramClient(...)
+    function_instance = SomeRequest(...)
+
+    # Invoke the request
+    returned_type = await client(function_instance)
+
+Whenever you find the type for a function in the `TL reference`_, the page
+will contain the following information:
+
+* What type of account can use the method. This information is regenerated
+  from time to time (by attempting to invoke the function under both account
+  types and finding out where it fails). Some requests can only be used by
+  bot accounts, others by user accounts, and others by both.
+* The TL definition. This helps you get a feel for the what the function
+  looks like. This is not Python code. It just contains the definition in
+  a concise manner.
+* "Copy import" button. Does what it says: it will copy the necessary Python
+  code to import the function to your system's clipboard for easy access.
+* Returns. The returned type. When you invoke the function, this is what the
+  result will be. It also includes which of the constructors can be returned
+  inline, to save you a click.
+* Parameters. The parameters accepted by the function, including their type,
+  whether they expect a list, and whether they're optional.
+* Known RPC errors. A best-effort list of known errors the request may cause.
+  This list is not complete and may be out of date, but should provide an
+  overview of what could go wrong.
+* Example. Autogenerated example, showcasing how you may want to call it.
+  Bear in mind that this is *autogenerated*. It may be spitting out non-sense.
+  The goal of this example is not to show you everything you can do with the
+  request, only to give you a feel for what it looks like to use it.
+
+It is very important to click through the links and navigate to get the full
+picture. A specific page will show you what the specific function returns and
+needs as input parameters. But it may reference other types, so you need to
+navigate to those to learn what those contain or need.
+
+Types
+-----
+
+"Types" as understood by TL are not actually generated in Telethon.
+They would be the "abstract base class" of the constructors, but since Python
+is duck-typed, there is hardly any need to generate mostly unnecessary code.
+The page for a type contains:
+
+* Constructors. Every type will have one or more constructors. These
+  constructors *are* generated and can be immported and used.
+* Requests returning this type. A helpful way to find out "what requests can
+  return this?". This is how you may learn what request you need to use to
+  obtain a particular instance of a type.
+* Requests accepting this type as input. A helpful way to find out "what
+  requests can use this type as one of their input parameters?". This is how
+  you may learn where a type is used.
+* Other types containing this type. A helpful way to find out "where else
+  does this type appear?". This is how you can walk back through nested
+  objects.
+
+Constructors
+------------
+
+Constructors are used to create instances of a particular type, and are also
+returned when invoking requests. You will have to create instances yourself
+when invoking requests that need a particular type as input.
+The page for a constructor contains:
+
+* Belongs to. The parent type. This is a link back to the types page for the
+  specific constructor. It also contains the sibling constructors inline, to
+  save you a click.
+* Members. Both the input parameters *and* fields the constructor contains.
+
+
+Using the TL reference
+======================
+
+After you've found a request you want to send, a good start would be to simply
+copy and paste the autogenerated example into your script. Then you can simply
+tweak it to your needs.
+
+If you want to do it from scratch, first, make sure to import the request into
+your code (either using the "Copy import" button near the top, or by manually
+spelling out the package under ``telethon.tl.functions.*``).
+
+Then, start reading the parameters one by one. If the parameter cannot be
+omitted, you **will** need to specify it, so make sure to spell it out as
+an input parameter when constructing the request instance. Let's look at
+`PingRequest`_ for example. First, we copy the import:
+
+.. code-block:: python
+
+    from telethon.tl.functions import PingRequest
+
+Then, we look at the parameters:
+
+    ping_id - long
+
+A single parameter, and it's a long (a integer number with a large range of
+values). It doesn't say it can be omitted, so we must provide it, like so:
+
+.. code-block:: python
+
+    PingRequest(
+        ping_id=48641868471
+    )
+
+(In this case, the ping ID is a random number. You often have to guess what
+the parameter needs just by looking at the name.)
+
+Now that we have our request, we can invoke it:
+
+.. code-block:: python
+
+    response = await client(PingRequest(
+        ping_id=48641868471
+    ))
+
+To find out what ``response`` looks like, we can do as the autogenerated
+example suggests and "stringify" the result as a pretty-printed string:
+
+.. code-block:: python
+
+    print(result.stringify())
+
+This will print out the following:
+
+.. code-block:: python
+
+    Pong(
+        msg_id=781875678118,
+        ping_id=48641868471
+    )
+
+Which is a very easy way to get a feel for a response. You should nearly
+always print the stringified result, at least once, when trying out requests,
+to get a feel for what the response may look like.
+
+But of course, you don't need to do that. Without writing any code, you could
+have navigated through the "Returns" link to learn ``PingRequest`` returns a
+``Pong``, which only has one constructor, and the constructor has two members,
+``msg_id`` and ``ping_id``.
+
+If you wanted to create your own ``Pong``, you would use both members as input
+parameters:
+
+.. code-block:: python
+
+    my_pong = Pong(
+        msg_id=781875678118,
+        ping_id=48641868471
+    )
+
+(Yes, constructing object instances can use the same code that ``.stringify``
+would return!)
+
+And if you wanted to access the ``msg_id`` member, you would simply access it
+like any other attribute access in Python:
+
+.. code-block:: python
+
+    print(response.msg_id)
+
+
+Example walkthrough
+===================
 
 Say `client.send_message()
 <telethon.client.messages.MessageMethods.send_message>` didn't exist,
@@ -62,7 +243,7 @@ If you're going to use a lot of these, you may do:
 
 We see that this request must take at least two parameters, a ``peer``
 of type :tl:`InputPeer`, and a ``message`` which is just a Python
-``str``\ ing.
+`str`\ ing.
 
 How can we retrieve this :tl:`InputPeer`? We have two options. We manually
 construct one, for instance:
@@ -78,8 +259,17 @@ Or we call `client.get_input_entity()
 
 .. code-block:: python
 
-    import telethon.sync
-    peer = client.get_input_entity('someone')
+    import telethon
+
+    async def main():
+        peer = await client.get_input_entity('someone')
+
+    client.loop.run_until_complete(main())
+
+.. note::
+
+    Remember that ``await`` must occur inside an ``async def``.
+    Every full API example assumes you already know and do this.
 
 
 When you're going to invoke an API method, most require you to pass an
@@ -92,7 +282,7 @@ instead:
 
 .. code-block:: python
 
-    entity = client.get_entity('someone')
+    entity = await client.get_entity('someone')
 
 In the later case, when you use the entity, the library will cast it to
 its "input" version for you. If you already have the complete user and
@@ -120,7 +310,7 @@ request we do:
 
 .. code-block:: python
 
-    result = client(SendMessageRequest(peer, 'Hello there!'))
+    result = await client(SendMessageRequest(peer, 'Hello there!'))
 
 Message sent! Of course, this is only an example. There are over 250
 methods available as of layer 80, and you can use every single of them
@@ -128,8 +318,8 @@ as you wish. Remember to use the right types! To sum up:
 
 .. code-block:: python
 
-    result = client(SendMessageRequest(
-        client.get_input_entity('username'), 'Hello there!'
+    result = await client(SendMessageRequest(
+        await client.get_input_entity('username'), 'Hello there!'
     ))
 
 
@@ -137,9 +327,9 @@ This can further be simplified to:
 
 .. code-block:: python
 
-    result = client(SendMessageRequest('username', 'Hello there!'))
+    result = await client(SendMessageRequest('username', 'Hello there!'))
     # Or even
-    result = client(SendMessageRequest(PeerChannel(id), 'Hello there!'))
+    result = await client(SendMessageRequest(PeerChannel(id), 'Hello there!'))
 
 .. note::
 
@@ -195,7 +385,7 @@ knows all requests directly:
 
 .. code-block:: python
 
-    client([
+    await client([
         SendMessageRequest('me', 'Hello'),
         SendMessageRequest('me', ', '),
         SendMessageRequest('me', 'World'),
@@ -212,7 +402,7 @@ and still access the successful results:
     from telethon.errors import MultiError
 
     try:
-        client([
+        await client([
             SendMessageRequest('me', 'Hello'),
             SendMessageRequest('me', ''),
             SendMessageRequest('me', 'World')
@@ -224,6 +414,7 @@ and still access the successful results:
         # The second request failed.
         second = e.exceptions[1]
 
-.. _check the documentation: https://tl.telethon.dev
-.. _method you need: https://tl.telethon.dev/methods/index.html
+.. _TL reference: https://tl.telethon.dev
+.. _TL diff: https://diff.telethon.dev
+.. _PingRequest: https://tl.telethon.dev/methods/ping.html
 .. _use the search: https://tl.telethon.dev/?q=message&redirect=no

readthedocs/concepts/sessions.rst

@@ -48,8 +48,12 @@ by setting ``client.session.save_entities = False``.
 Different Session Storage
 =========================
 
-If you don't want to use the default SQLite session storage, you can also use
-one of the other implementations or implement your own storage.
+If you don't want to use the default SQLite session storage, you can also
+use one of the other implementations or implement your own storage.
+
+While it's often not the case, it's possible that SQLite is slow enough to
+be noticeable, in which case you can also use a different storage. Note that
+this is rare and most people won't have this issue, but it's worth a mention.
 
 To use a custom session storage, simply pass the custom session instance to
 :ref:`TelegramClient <telethon-client>` instead of
@@ -96,6 +100,9 @@ There are other community-maintained implementations available:
 * `Redis <https://github.com/ezdev128/telethon-session-redis>`_:
   stores all sessions in a single Redis data store.
 
+* `MongoDB <https://github.com/watzon/telethon-session-mongo>`_:
+  stores the current session in a MongoDB database.
+
 
 Creating your Own Storage
 =========================
@@ -136,7 +143,7 @@ output (likely your terminal).
 .. warning::
 
     **Keep this string safe!** Anyone with this string can use it
-    to login into your account and do anything they want to to do.
+    to login into your account and do anything they want to.
 
     This is similar to leaking your ``*.session`` files online,
     but it is easier to leak a string than it is to leak a file.
@@ -150,7 +157,7 @@ you can save it in a variable directly:
 
     string = '1aaNk8EX-YRfwoRsebUkugFvht6DUPi_Q25UOCzOAqzc...'
     with TelegramClient(StringSession(string), api_id, api_hash) as client:
-        client.send_message('me', 'Hi')
+        client.loop.run_until_complete(client.send_message('me', 'Hi'))
 
 
 These strings are really convenient for using in places like Heroku since

readthedocs/concepts/strings.rst

@@ -3,87 +3,75 @@ String-based Debugging
 ======================
 
 Debugging is *really* important. Telegram's API is really big and there
-is a lot of things that you should know. Such as, what attributes or fields
+are a lot of things that you should know. Such as, what attributes or fields
 does a result have? Well, the easiest thing to do is printing it:
 
 .. code-block:: python
 
-    user = client.get_entity('Lonami')
-    print(user)
+    entity = await client.get_entity('username')
+    print(entity)
 
 That will show a huge **string** similar to the following:
 
 .. code-block:: python
 
-    User(id=10885151, is_self=False, contact=False, mutual_contact=False, deleted=False, bot=False, bot_chat_history=False, bot_nochats=False, verified=False, restricted=False, min=False, bot_inline_geo=False, access_hash=123456789012345678, first_name='Lonami', last_name=None, username='Lonami', phone=None, photo=UserProfilePhoto(photo_id=123456789012345678, photo_small=FileLocation(dc_id=4, volume_id=1234567890, local_id=1234567890, secret=123456789012345678), photo_big=FileLocation(dc_id=4, volume_id=1234567890, local_id=1234567890, secret=123456789012345678)), status=UserStatusOffline(was_online=datetime.datetime(2018, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc)), bot_info_version=None, restriction_reason=None, bot_inline_placeholder=None, lang_code=None)
+    Channel(id=1066197625, title='Telegram Usernames', photo=ChatPhotoEmpty(), date=datetime.datetime(2016, 12, 16, 15, 15, 43, tzinfo=datetime.timezone.utc), version=0, creator=False, left=True, broadcast=True, verified=True, megagroup=False, restricted=False, signatures=False, min=False, scam=False, has_link=False, has_geo=False, slowmode_enabled=False, access_hash=-6309373984955162244, username='username', restriction_reason=[], admin_rights=None, banned_rights=None, default_banned_rights=None, participants_count=None)
 
 That's a lot of text. But as you can see, all the properties are there.
-So if you want the username you **don't use regex** or anything like
-splitting ``str(user)`` to get what you want. You just access the
+So if you want the title you **don't use regex** or anything like
+splitting ``str(entity)`` to get what you want. You just access the
 attribute you need:
 
 .. code-block:: python
 
-    username = user.username
+    title = entity.title
 
 Can we get better than the shown string, though? Yes!
 
 .. code-block:: python
 
-    print(user.stringify())
+    print(entity.stringify())
 
-Will show a much better:
+Will show a much better representation:
 
 .. code-block:: python
 
-    User(
-        id=10885151,
-        is_self=False,
-        contact=False,
-        mutual_contact=False,
-        deleted=False,
-        bot=False,
-        bot_chat_history=False,
-        bot_nochats=False,
-        verified=False,
+    Channel(
+        id=1066197625,
+        title='Telegram Usernames',
+        photo=ChatPhotoEmpty(
+        ),
+        date=datetime.datetime(2016, 12, 16, 15, 15, 43, tzinfo=datetime.timezone.utc),
+        version=0,
+        creator=False,
+        left=True,
+        broadcast=True,
+        verified=True,
+        megagroup=False,
         restricted=False,
+        signatures=False,
         min=False,
-        bot_inline_geo=False,
-        access_hash=123456789012345678,
-        first_name='Lonami',
-        last_name=None,
-        username='Lonami',
-        phone=None,
-        photo=UserProfilePhoto(
-            photo_id=123456789012345678,
-            photo_small=FileLocation(
-                dc_id=4,
-                volume_id=123456789,
-                local_id=123456789,
-                secret=-123456789012345678
-            ),
-            photo_big=FileLocation(
-                dc_id=4,
-                volume_id=123456789,
-                local_id=123456789,
-                secret=123456789012345678
-            )
-        ),
-        status=UserStatusOffline(
-            was_online=datetime.datetime(2018, 1, 2, 3, 4, 5, tzinfo=datetime.timezone.utc)
-        ),
-        bot_info_version=None,
-        restriction_reason=None,
-        bot_inline_placeholder=None,
-        lang_code=None
+        scam=False,
+        has_link=False,
+        has_geo=False,
+        slowmode_enabled=False,
+        access_hash=-6309373984955162244,
+        username='username',
+        restriction_reason=[
+        ],
+        admin_rights=None,
+        banned_rights=None,
+        default_banned_rights=None,
+        participants_count=None
     )
 
+
 Now it's easy to see how we could get, for example,
-the ``was_online`` time. It's inside ``status``:
+the ``year`` value. It's inside ``date``:
 
 .. code-block:: python
 
-    online_at = user.status.was_online
+    channel_year = entity.date.year
 
 You don't need to print everything to see what all the possible values
 can be. You can just search in http://tl.telethon.dev/.
@@ -96,5 +84,5 @@ to check the type of something. For example:
 
     from telethon import types
 
-    if isinstance(user.status, types.UserStatusOffline):
-        print(user.status.was_online)
+    if isinstance(entity.photo, types.ChatPhotoEmpty):
+        print('Channel has no photo')

readthedocs/concepts/updates.rst

@@ -12,7 +12,7 @@ can access all the properties it has, like ``.sender``.
 **However** events are different to other methods in the client, like
 `client.get_messages <telethon.client.messages.MessageMethods.get_messages>`.
 Events *may not* send information about the sender or chat, which means it
-can be ``None``, but all the methods defined in the client always have this
+can be `None`, but all the methods defined in the client always have this
 information so it doesn't need to be re-fetched. For this reason, you have
 ``get_`` methods, which will make a network call if necessary.
 
@@ -113,7 +113,7 @@ and `client.list_event_handlers
 <telethon.client.updates.UpdateMethods.list_event_handlers>`.
 
 The ``event`` argument is optional in all three methods and defaults to
-`events.Raw <telethon.events.raw.Raw>` for adding, and ``None`` when
+`events.Raw <telethon.events.raw.Raw>` for adding, and `None` when
 removing (so all callbacks would be removed).
 
 .. note::
@@ -191,8 +191,7 @@ so the code above and the following are equivalent:
     async def main():
         await client.disconnected
 
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(main())
+    asyncio.run(main())
 
 
 You could also run `client.disconnected
@@ -207,7 +206,7 @@ Notice that unlike `client.disconnected
 <telethon.client.telegrambaseclient.TelegramBaseClient.disconnected>`,
 `client.run_until_disconnected
 <telethon.client.updates.UpdateMethods.run_until_disconnected>` will
-handle ``KeyboardInterrupt`` with you. This method is special and can
+handle ``KeyboardInterrupt`` for you. This method is special and can
 also be ran while the loop is running, so you can do this:
 
 .. code-block:: python

readthedocs/conf.py

@@ -85,7 +85,7 @@ release = version
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -99,6 +99,24 @@ pygments_style = 'friendly'
 todo_include_todos = False
 
 
+def skip(app, what, name, obj, would_skip, options):
+    if name.endswith('__'):
+        # We want to show special methods names, except some which add clutter
+        return name in {
+            '__init__',
+            '__abstractmethods__',
+            '__module__',
+            '__doc__',
+            '__dict__'
+        }
+
+    return would_skip
+
+
+def setup(app):
+    app.connect("autodoc-skip-member", skip)
+
+
 # -- Options for HTML output ----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for

readthedocs/developing/project-structure.rst

@@ -45,3 +45,7 @@ an index so that they can be imported easily.
 
 Custom documentation can also be generated to easily navigate through
 the vast amount of items offered by the API.
+
+If you clone the repository, you will have to run ``python setup.py gen``
+in order to generate the code. Installing the library runs the generator
+too, but the mentioned command will just generate code.

readthedocs/developing/telegram-api-in-other-languages.rst

@@ -2,72 +2,12 @@
 Telegram API in Other Languages
 ===============================
 
+Telethon was made for **Python**, and it has inspired other libraries such as
+`gramjs <https://github.com/gram-js/gramjs>`__ (JavaScript) and `grammers
+<https://github.com/Lonami/grammers>`__ (Rust). But there is a lot more beyond
+those, made independently by different developers.
 
-Telethon was made for **Python**, and as far as I know, there is no
-*exact* port to other languages. However, there *are* other
-implementations made by awesome people (one needs to be awesome to
-understand the official Telegram documentation) on several languages
-(even more Python too), listed below:
-
-C
-*
-
-Possibly the most well-known unofficial open source implementation out
-there by `@vysheng <https://github.com/vysheng>`__,
-`tgl <https://github.com/vysheng/tgl>`__, and its console client
-`telegram-cli <https://github.com/vysheng/tg>`__. Latest development
-has been moved to `BitBucket <https://bitbucket.org/vysheng/tdcli>`__.
-
-C++
-===
-
-The newest (and official) library, written from scratch, is called
-`tdlib <https://github.com/tdlib/td>`__ and is what the Telegram X
-uses. You can find more information in the official documentation,
-published `here <https://core.telegram.org/tdlib/docs/>`__.
-
-JavaScript
-==========
-
-`@zerobias <https://github.com/zerobias>`__ is working on
-`telegram-mtproto <https://github.com/zerobias/telegram-mtproto>`__,
-a work-in-progress JavaScript library installable via
-`npm <https://www.npmjs.com/>`__.
-
-Kotlin
-======
-
-`Kotlogram <https://github.com/badoualy/kotlogram>`__ is a Telegram
-implementation written in Kotlin (one of the
-`official <https://blog.jetbrains.com/kotlin/2017/05/kotlin-on-android-now-official/>`__
-languages for
-`Android <https://developer.android.com/kotlin/index.html>`__) by
-`@badoualy <https://github.com/badoualy>`__, currently as a beta–
-yet working.
-
-PHP
-===
-
-A PHP implementation is also available thanks to
-`@danog <https://github.com/danog>`__ and his
-`MadelineProto <https://github.com/danog/MadelineProto>`__ project, with
-a very nice `online
-documentation <https://daniil.it/MadelineProto/API_docs/>`__ too.
-
-Python
-======
-
-A fairly new (as of the end of 2017) Telegram library written from the
-ground up in Python by
-`@delivrance <https://github.com/delivrance>`__ and his
-`Pyrogram <https://github.com/pyrogram/pyrogram>`__ library.
-There isn't really a reason to pick it over Telethon and it'd be kinda
-sad to see you go, but it would be nice to know what you miss from each
-other library in either one so both can improve.
-
-Rust
-====
-
-Yet another work-in-progress implementation, this time for Rust thanks
-to `@JuanPotato <https://github.com/JuanPotato>`__ under the fancy
-name of `Vail <https://github.com/JuanPotato/Vail>`__.
+If you're looking for something like Telethon but in a different programming
+language, head over to `Telegram API in Other Languages in the official wiki
+<https://github.com/LonamiWebs/Telethon/wiki/Telegram-API-in-Other-Languages>`__
+for a (mostly) up-to-date list.

readthedocs/developing/test-servers.rst

@@ -12,7 +12,7 @@ To run Telethon on a test server, use the following code:
 
 You can check your ``'test ip'`` on https://my.telegram.org.
 
-You should set ``None`` session so to ensure you're generating a new
+You should set `None` session so to ensure you're generating a new
 authorization key for it (it would fail if you used a session where you
 had previously connected to another data center).
 
@@ -35,3 +35,7 @@ times, in this case, ``22222`` so we can hardcode that:
     client.start(
         phone='9996621234', code_callback=lambda: '22222'
     )
+
+Note that Telegram has changed the length of login codes multiple times in the
+past, so if ``dc_id`` repeated five times does not work, try repeating it six
+times.

readthedocs/developing/testing.rst

@@ -0,0 +1,87 @@
+=====
+Tests
+=====
+
+Telethon uses `Pytest <https://pytest.org/>`__, for testing, `Tox
+<https://tox.readthedocs.io/en/latest/>`__ for environment setup, and
+`pytest-asyncio <https://pypi.org/project/pytest-asyncio/>`__ and `pytest-cov
+<https://pytest-cov.readthedocs.io/en/latest/>`__ for asyncio and 
+`coverage <https://coverage.readthedocs.io/>`__ integration.
+
+While reading the full documentation for these is probably a good idea, there
+is a lot to read, so a brief summary of these tools is provided below for
+convienience.
+
+Brief Introduction to Pytest
+============================
+
+`Pytest <https://pytest.org/>`__ is a tool for discovering and running python
+tests, as well as allowing modular reuse of test setup code using fixtures.
+
+Most Pytest tests will look something like this::
+
+    from module import my_thing, my_other_thing
+
+    def test_my_thing(fixture):
+        assert my_thing(fixture) == 42
+
+    @pytest.mark.asyncio
+    async def test_my_thing(event_loop):
+        assert await my_other_thing(loop=event_loop) == 42
+
+Note here:
+
+ 1. The test imports one specific function. The role of unit tests is to test
+    that the implementation of some unit, like a function or class, works.
+    It's role is not so much to test that components interact well with each
+    other. I/O, such as connecting to remote servers, should be avoided. This
+    helps with quickly identifying the source of an error, finding silent
+    breakage, and makes it easier to cover all possible code paths.
+
+    System or integration tests can also be useful, but are currently out of
+    scope of Telethon's automated testing.
+
+ 2. A function ``test_my_thing`` is declared. Pytest searches for files
+    starting with ``test_``, classes starting with ``Test`` and executes any
+    functions or methods starting with ``test_`` it finds.
+
+ 3. The function is declared with a parameter ``fixture``. Fixtures are used to
+    request things required to run the test, such as temporary directories,
+    free TCP ports, Connections, etc. Fixtures are declared by simply adding
+    the fixture name as parameter. A full list of available fixtures can be
+    found with the ``pytest --fixtures`` command.
+
+ 4. The test uses a simple ``assert`` to test some condition is valid.  Pytest
+    uses some magic to ensure that the errors from this are readable and easy
+    to debug.
+
+ 5. The ``pytest.mark.asyncio`` fixture is provided by ``pytest-asyncio``. It
+    starts a loop and executes a test function as coroutine. This should be
+    used for testing asyncio code. It also declares the ``event_loop``
+    fixture, which will request an ``asyncio`` event loop.
+
+Brief Introduction to Tox
+=========================
+
+`Tox <https://tox.readthedocs.io/en/latest/>`__ is a tool for automated setup
+of virtual environments for testing. While the tests can be run directly by
+just running ``pytest``, this only tests one specific python version in your
+existing environment, which will not catch e.g. undeclared dependencies, or
+version incompatabilities.
+
+Tox environments are declared in the ``tox.ini`` file. The default
+environments, declared at the top, can be simply run with ``tox``. The option
+``tox -e py36,flake`` can be used to request specific environments to be run.
+
+Brief Introduction to Pytest-cov
+================================
+
+Coverage is a useful metric for testing. It measures the lines of code and
+branches that are exercised by the tests. The higher the coverage, the more
+likely it is that any coding errors will be caught by the tests.
+
+A brief coverage report can be generated with the ``--cov`` option to ``tox``,
+which will be passed on to ``pytest``. Additionally, the very useful HTML
+report can be generated with ``--cov --cov-report=html``, which contains a
+browsable copy of the source code, annotated with coverage information for each
+line.

readthedocs/examples/chats-and-channels.rst

@@ -15,7 +15,7 @@ Joining a chat or channel
 
 Note that :tl:`Chat` are normal groups, and :tl:`Channel` are a
 special form of :tl:`Chat`, which can also be super-groups if
-their ``megagroup`` member is ``True``.
+their ``megagroup`` member is `True`.
 
 
 Joining a public channel
@@ -27,11 +27,11 @@ to, you can make use of the :tl:`JoinChannelRequest` to join such channel:
 .. code-block:: python
 
     from telethon.tl.functions.channels import JoinChannelRequest
-    client(JoinChannelRequest(channel))
+    await client(JoinChannelRequest(channel))
 
     # In the same way, you can also leave such channel
     from telethon.tl.functions.channels import LeaveChannelRequest
-    client(LeaveChannelRequest(input_channel))
+    await client(LeaveChannelRequest(input_channel))
 
 
 For more on channels, check the `channels namespace`__.
@@ -53,7 +53,7 @@ example, is the ``hash`` of the chat or channel. Now you can use
 .. code-block:: python
 
     from telethon.tl.functions.messages import ImportChatInviteRequest
-    updates = client(ImportChatInviteRequest('AAAAAEHbEkejzxUjAUCfYg'))
+    updates = await client(ImportChatInviteRequest('AAAAAEHbEkejzxUjAUCfYg'))
 
 
 Adding someone else to such chat or channel
@@ -70,7 +70,7 @@ use is very straightforward, or :tl:`InviteToChannelRequest` for channels:
 
     # Note that ``user_to_add`` is NOT the name of the parameter.
     # It's the user you want to add (``user_id=user_to_add``).
-    client(AddChatUserRequest(
+    await client(AddChatUserRequest(
         chat_id,
         user_to_add,
         fwd_limit=10  # Allow the user to see the 10 last messages
@@ -79,11 +79,15 @@ use is very straightforward, or :tl:`InviteToChannelRequest` for channels:
     # For channels (which includes megagroups)
     from telethon.tl.functions.channels import InviteToChannelRequest
 
-    client(InviteToChannelRequest(
+    await client(InviteToChannelRequest(
         channel,
         [users_to_add]
     ))
 
+Note that this method will only really work for friends or bot accounts.
+Trying to mass-add users with this approach will not work, and can put both
+your account and group to risk, possibly being flagged as spam and limited.
+
 
 Checking a link without joining
 ===============================
@@ -93,140 +97,6 @@ channel, you can use the :tl:`CheckChatInviteRequest`, which takes in
 the hash of said channel or group.
 
 
-Admin Permissions
-=================
-
-Giving or revoking admin permissions can be done with the :tl:`EditAdminRequest`:
-
-.. code-block:: python
-
-    from telethon.tl.functions.channels import EditAdminRequest
-    from telethon.tl.types import ChatAdminRights
-
-    # You need both the channel and who to grant permissions
-    # They can either be channel/user or input channel/input user.
-    #
-    # ChatAdminRights is a list of granted permissions.
-    # Set to True those you want to give.
-    rights = ChatAdminRights(
-        post_messages=None,
-        add_admins=None,
-        invite_users=None,
-        change_info=True,
-        ban_users=None,
-        delete_messages=True,
-        pin_messages=True,
-        invite_link=None,
-        edit_messages=None
-    )
-    # Equivalent to:
-    #     rights = ChatAdminRights(
-    #         change_info=True,
-    #         delete_messages=True,
-    #         pin_messages=True
-    #     )
-
-    # Once you have a ChatAdminRights, invoke it
-    client(EditAdminRequest(channel, user, rights))
-
-    # User will now be able to change group info, delete other people's
-    # messages and pin messages.
-    #
-    # In a normal chat, you should do this instead:
-    from telethon.tl.functions.messages import EditChatAdminRequest
-
-    client(EditChatAdminRequest(chat_id, user, is_admin=True))
-
-
-
-.. note::
-
-    Thanks to `@Kyle2142`__ for `pointing out`__ that you **cannot** set all
-    parameters to ``True`` to give a user full permissions, as not all
-    permissions are related to both broadcast channels/megagroups.
-
-    E.g. trying to set ``post_messages=True`` in a megagroup will raise an
-    error. It is recommended to always use keyword arguments, and to set only
-    the permissions the user needs. If you don't need to change a permission,
-    it can be omitted (full list `here`__).
-
-
-Restricting Users
-=================
-
-Similar to how you give or revoke admin permissions, you can edit the
-banned rights of a user through :tl:`EditBannedRequest` and its parameter
-:tl:`ChatBannedRights`:
-
-.. code-block:: python
-
-    from telethon.tl.functions.channels import EditBannedRequest
-    from telethon.tl.types import ChatBannedRights
-
-    from datetime import datetime, timedelta
-
-    # Restricting a user for 7 days, only allowing view/send messages.
-    #
-    # Note that it's "reversed". You must set to ``True`` the permissions
-    # you want to REMOVE, and leave as ``None`` those you want to KEEP.
-    rights = ChatBannedRights(
-        until_date=timedelta(days=7),
-        view_messages=None,
-        send_messages=None,
-        send_media=True,
-        send_stickers=True,
-        send_gifs=True,
-        send_games=True,
-        send_inline=True,
-        embed_links=True
-    )
-
-    # The above is equivalent to
-    rights = ChatBannedRights(
-        until_date=datetime.now() + timedelta(days=7),
-        send_media=True,
-        send_stickers=True,
-        send_gifs=True,
-        send_games=True,
-        send_inline=True,
-        embed_links=True
-    )
-
-    client(EditBannedRequest(channel, user, rights))
-
-
-You can use a `datetime.datetime` object for ``until_date=``,
-a `datetime.timedelta` or even a Unix timestamp. Note that if you ban
-someone for less than 30 seconds or for more than 366 days, Telegram
-will consider the ban to actually last forever. This is officially
-documented under https://core.telegram.org/bots/api#restrictchatmember.
-
-
-Kicking a member
-================
-
-Telegram doesn't actually have a request to kick a user from a group.
-Instead, you need to restrict them so they can't see messages. Any date
-is enough:
-
-.. code-block:: python
-
-    from telethon.tl.functions.channels import EditBannedRequest
-    from telethon.tl.types import ChatBannedRights
-
-    client(EditBannedRequest(
-        channel, user, ChatBannedRights(
-            until_date=None,
-            view_messages=True
-        )
-    ))
-
-
-__ https://github.com/Kyle2142
-__ https://github.com/LonamiWebs/Telethon/issues/490
-__ https://tl.telethon.dev/constructors/channel_admin_rights.html
-
-
 Increasing View Count in a Channel
 ==================================
 
@@ -240,7 +110,7 @@ use :tl:`GetMessagesViewsRequest`, setting ``increment=True``:
     # Obtain `channel' through dialogs or through client.get_entity() or anyhow.
     # Obtain `msg_ids' through `.get_messages()` or anyhow. Must be a list.
 
-    client(GetMessagesViewsRequest(
+    await client(GetMessagesViewsRequest(
         peer=channel,
         id=msg_ids,
         increment=True

readthedocs/examples/projects-using-telethon.rst

@@ -1,85 +0,0 @@
-.. _telethon_projects:
-
-=======================
-Projects using Telethon
-=======================
-
-This page lists some **interesting and useful** real world
-examples showcasing what can be built with the library.
-
-.. note::
-
-    Do you have an interesting project that uses the library or know of any
-    that's not listed here? Feel free to leave a comment at
-    `issue 744 <https://github.com/LonamiWebs/Telethon/issues/744>`_
-    so it can be included in the next revision of the documentation!
-
-    You can also advertise your bot and its features, in the issue, although
-    it should be a big project which can be useful for others before being
-    included here, so please don't feel offended if it can't be here!
-
-
-.. _projects-telegram-export:
-
-telethon_examples/
-==================
-
-`telethon_examples <https://github.com/LonamiWebs/Telethon/tree/master/telethon_examples>`_ /
-`LonamiWebs' site <https://lonamiwebs.github.io>`_
-
-This documentation is not the only place where you can find useful code
-snippets using the library. The main repository also has a folder with
-some cool examples (even a Tkinter GUI!) which you can download, edit
-and run to learn and play with them.
-
-@TelethonSnippets
-=================
-
-`@TelethonSnippets <https://t.me/TelethonSnippets>`_
-
-You can find useful short snippets for Telethon here.
-
-telegram-export
-===============
-
-`telegram-export <https://github.com/expectocode/telegram-export>`_ /
-`expectocode's GitHub <https://github.com/expectocode>`_
-
-A tool to download Telegram data (users, chats, messages, and media)
-into a database (and display the saved data).
-
-.. _projects-mautrix-telegram:
-
-mautrix-telegram
-================
-
-`mautrix-telegram <https://github.com/tulir/mautrix-telegram>`_ /
-`maunium's site <https://maunium.net/>`_
-
-A Matrix-Telegram hybrid puppeting/relaybot bridge.
-
-.. _projects-telegramtui:
-
-TelegramTUI
-===========
-
-`TelegramTUI <https://github.com/bad-day/TelegramTUI>`_ /
-`bad-day's GitHub <https://github.com/bad-day>`_
-
-A Telegram client on your terminal.
-
-tgcloud
-=======
-
-`tgcloud <https://github.com/SlavikMIPT/tgcloud>`_ /
-`tgcloud's site <https://dev.tgcloud.xyz/>`_
-
-Opensource Telegram based cloud storage.
-
-tgmount
-=======
-
-`tgmount <https://github.com/nktknshn/tgmount>`_ /
-`nktknshn's GitHub <https://github.com/nktknshn>`_
-
-Mount Telegram dialogs and channels as a Virtual File System.

readthedocs/examples/users.rst

@@ -21,11 +21,11 @@ you should use :tl:`GetFullUser`:
 
     from telethon.tl.functions.users import GetFullUserRequest
 
-    full = client(GetFullUserRequest(user))
+    full = await client(GetFullUserRequest(user))
     # or even
-    full = client(GetFullUserRequest('username'))
+    full = await client(GetFullUserRequest('username'))
 
-    bio = full.about
+    bio = full.full_user.about
 
 
 See :tl:`UserFull` to know what other fields you can access.
@@ -41,7 +41,7 @@ request. Omitted fields won't change after invoking :tl:`UpdateProfile`:
 
     from telethon.tl.functions.account import UpdateProfileRequest
 
-    client(UpdateProfileRequest(
+    await client(UpdateProfileRequest(
         about='This is a test from Telethon'
     ))
 
@@ -55,7 +55,7 @@ You need to use :tl:`account.UpdateUsername`:
 
     from telethon.tl.functions.account import UpdateUsernameRequest
 
-    client(UpdateUsernameRequest('new_username'))
+    await client(UpdateUsernameRequest('new_username'))
 
 
 Updating your profile photo
@@ -69,6 +69,6 @@ through :tl:`UploadProfilePhoto`:
 
     from telethon.tl.functions.photos import UploadProfilePhotoRequest
 
-    client(UploadProfilePhotoRequest(
-        client.upload_file('/path/to/some/file')
-    )))
+    await client(UploadProfilePhotoRequest(
+        await client.upload_file('/path/to/some/file')
+    ))

readthedocs/examples/word-of-warning.rst

@@ -13,4 +13,5 @@ Full API **will** break between different minor versions of the library,
 since Telegram changes very often. The friendly methods will be kept
 compatible between major versions.
 
-If you need to see real-world examples, please refer to :ref:`telethon_projects`.
+If you need to see real-world examples, please refer to the
+`wiki page of projects using Telethon <https://github.com/LonamiWebs/Telethon/wiki/Projects-using-Telethon>`__.

readthedocs/examples/working-with-messages.rst

@@ -2,44 +2,12 @@
 Working with messages
 =====================
 
-
 .. note::
 
     These examples assume you have read :ref:`full-api`.
 
-.. contents::
+This section has been `moved to the wiki`_, where it can be easily edited as new
+features arrive and the API changes. Please refer to the linked page to learn how
+to send spoilers, custom emoji, stickers, react to messages, and more things.
 
-
-Sending stickers
-================
-
-Stickers are nothing else than ``files``, and when you successfully retrieve
-the stickers for a certain sticker set, all you will have are ``handles`` to
-these files. Remember, the files Telegram holds on their servers can be
-referenced through this pair of ID/hash (unique per user), and you need to
-use this handle when sending a "document" message. This working example will
-send yourself the very first sticker you have:
-
-.. code-block:: python
-
-    # Get all the sticker sets this user has
-    from telethon.tl.functions.messages import GetAllStickersRequest
-    sticker_sets = client(GetAllStickersRequest(0))
-
-    # Choose a sticker set
-    from telethon.tl.functions.messages import GetStickerSetRequest
-    from telethon.tl.types import InputStickerSetID
-    sticker_set = sticker_sets.sets[0]
-
-    # Get the stickers for this sticker set
-    stickers = client(GetStickerSetRequest(
-        stickerset=InputStickerSetID(
-            id=sticker_set.id, access_hash=sticker_set.access_hash
-        )
-    ))
-
-    # Stickers are nothing more than files, so send that
-    client.send_file('me', stickers.documents[0])
-
-
-.. _issues: https://github.com/LonamiWebs/Telethon/issues/215
+.. _moved to the wiki: https://github.com/LonamiWebs/Telethon/wiki/Sending-more-than-just-messages

readthedocs/index.rst

@@ -68,6 +68,7 @@ You can also use the menu on the left to quickly skip over sections.
 
     concepts/strings
     concepts/entities
+    concepts/chats-vs-channels
     concepts/updates
     concepts/sessions
     concepts/full-api
@@ -83,7 +84,6 @@ You can also use the menu on the left to quickly skip over sections.
     examples/chats-and-channels
     examples/users
     examples/working-with-messages
-    examples/projects-using-telethon
 
 .. toctree::
     :hidden:
@@ -93,6 +93,7 @@ You can also use the menu on the left to quickly skip over sections.
     developing/test-servers.rst
     developing/project-structure.rst
     developing/coding-style.rst
+    developing/testing.rst
     developing/understanding-the-type-language.rst
     developing/tips-for-porting-the-project.rst
     developing/telegram-api-in-other-languages.rst

readthedocs/misc/compatibility-and-convenience.rst

@@ -161,19 +161,17 @@ just get rid of ``telethon.sync`` and work inside an ``async def``:
 
             await client.run_until_disconnected()
 
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(main())
+    asyncio.run(main())
 
 
-The ``telethon.sync`` magic module simply wraps every method behind:
+The ``telethon.sync`` magic module essentially wraps every method behind:
 
 .. code-block:: python
 
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(main())
+    asyncio.run(main())
 
-So that you don't have to write it yourself every time. That's the
-overhead you pay if you import it, and what you save if you don't.
+With some other tricks, so that you don't have to write it yourself every time.
+That's the overhead you pay if you import it, and what you save if you don't.
 
 Learning
 ========

readthedocs/modules/client.rst

@@ -14,15 +14,16 @@ Each mixin has its own methods, which you all can use.
 
 .. code-block:: python
 
-    import asyncio
     from telethon import TelegramClient
 
+    client = TelegramClient(name, api_id, api_hash)
+
     async def main():
-        client = await TelegramClient(name, api_id, api_hash).start()
         # Now you can use all client methods listed below, like for example...
         await client.send_message('me', 'Hello to myself!')
 
-    asyncio.get_event_loop().run_until_complete(main())
+    with client:
+        client.loop.run_until_complete(main())
 
 
 You **don't** need to import these `AuthMethods`, `MessageMethods`, etc.

readthedocs/modules/custom.rst

@@ -136,6 +136,24 @@ MessageButton
     :show-inheritance:
 
 
+ParticipantPermissions
+======================
+
+.. automodule:: telethon.tl.custom.participantpermissions
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+QRLogin
+=======
+
+.. automodule:: telethon.tl.custom.qrlogin
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
 SenderGetter
 ============
 

readthedocs/modules/errors.rst

@@ -6,7 +6,8 @@ API Errors
 
 These are the base errors that Telegram's API may raise.
 
-See :ref:`rpc-errors` for a more friendly explanation.
+See :ref:`rpc-errors` for a more in-depth explanation on how to handle all
+known possible errors and learning to determine what a method may raise.
 
 .. automodule:: telethon.errors.common
     :members:

readthedocs/modules/events.rst

@@ -54,6 +54,11 @@ so all the methods in it can be used from any event builder/event instance.
     :undoc-members:
     :show-inheritance:
 
+.. automodule:: telethon.events.album
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 .. automodule:: telethon.events.raw
     :members:
     :undoc-members:

readthedocs/quick-references/client-reference.rst

@@ -9,7 +9,7 @@ you may need when using Telethon. They are sorted by relevance and are not in
 alphabetical order.
 
 You should use this page to learn about which methods are available, and
-if you need an usage example or further description of the arguments, be
+if you need a usage example or further description of the arguments, be
 sure to follow the links.
 
 .. contents::
@@ -31,7 +31,7 @@ Auth
     start
     send_code_request
     sign_in
-    sign_up
+    qr_login
     log_out
     edit_2fa
 
@@ -48,6 +48,7 @@ Base
     is_connected
     disconnected
     loop
+    set_proxy
 
 Messages
 --------
@@ -64,6 +65,7 @@ Messages
     iter_messages
     get_messages
     pin_message
+    unpin_message
     send_read_acknowledge
 
 Uploads
@@ -88,6 +90,7 @@ Downloads
     download_media
     download_profile_photo
     download_file
+    iter_download
 
 Dialogs
 -------
@@ -99,7 +102,7 @@ Dialogs
 
     iter_dialogs
     get_dialogs
-    archive
+    edit_folder
     iter_drafts
     get_drafts
     delete_dialog
@@ -130,10 +133,15 @@ Chats
 
     iter_participants
     get_participants
+    kick_participant
     iter_admin_log
     get_admin_log
     iter_profile_photos
     get_profile_photos
+    edit_admin
+    edit_permissions
+    get_permissions
+    get_stats
     action
 
 Parse Mode
@@ -160,6 +168,7 @@ Updates
     remove_event_handler
     list_event_handlers
     catch_up
+    set_receive_updates
 
 Bots
 ----

readthedocs/quick-references/events-reference.rst

@@ -90,7 +90,8 @@ Full documentation for the `MessageRead
 ChatAction
 ==========
 
-Occurs whenever a user joins or leaves a chat, or a message is pinned.
+Occurs on certain chat actions, such as chat title changes,
+user join or leaves, pinned messages, photo changes, etc.
 
 Full documentation for the `ChatAction
 <telethon.events.chataction.ChatAction>`.
@@ -126,9 +127,6 @@ UserUpdate
 
 Occurs whenever a user goes online, starts typing, etc.
 
-A lot of fields are attributes and not properties, so they
-are not shown here. Please refer to its full documentation.
-
 Full documentation for the `UserUpdate
 <telethon.events.userupdate.UserUpdate>`.
 
@@ -144,6 +142,25 @@ Full documentation for the `UserUpdate
         get_user
         get_input_user
 
+        typing
+        uploading
+        recording
+        playing
+        cancel
+        geo
+        audio
+        round
+        video
+        contact
+        document
+        photo
+        last_seen
+        until
+        online
+        recently
+        within_weeks
+        within_months
+
 
 CallbackQuery
 =============
@@ -194,6 +211,34 @@ Full documentation for the `InlineQuery
 
         answer
 
+Album
+=====
+
+Occurs whenever you receive an entire album.
+
+Full documentation for the `Album
+<telethon.events.album.Album>`.
+
+.. currentmodule:: telethon.events.album.Album.Event
+
+.. autosummary::
+    :nosignatures:
+
+        grouped_id
+        text
+        raw_text
+        is_reply
+        forward
+
+        get_reply_message
+        respond
+        reply
+        forward_to
+        edit
+        delete
+        mark_read
+        pin
+
 Raw
 ===
 

readthedocs/quick-references/faq.rst

@@ -20,7 +20,7 @@ To enable logging, at the following code to the top of your main file:
 .. code-block:: python
 
     import logging
-    logging.basicConfig(format='[%(levelname) 5s/%(asctime)s] %(name)s: %(message)s',
+    logging.basicConfig(format='[%(levelname) %(asctime)s] %(name)s: %(message)s',
                         level=logging.WARNING)
 
 You can change the logging level to be something different, from less to more information:
@@ -50,7 +50,7 @@ And except them as such:
 .. code-block:: python
 
     try:
-        client.send_message(chat, 'Hi')
+        await client.send_message(chat, 'Hi')
     except errors.FloodWaitError as e:
         # e.seconds is how many seconds you have
         # to wait before making the request again.
@@ -60,6 +60,16 @@ And except them as such:
 My account was deleted/limited when using the library
 =====================================================
 
+First and foremost, **this is not a problem exclusive to Telethon.
+Any third-party library is prone to cause the accounts to appear banned.**
+Even official applications can make Telegram ban an account under certain
+circumstances. Third-party libraries such as Telethon are a lot easier to
+use, and as such, they are misused to spam, which causes Telegram to learn
+certain patterns and ban suspicious activity.
+
+There is no point in Telethon trying to circumvent this. Even if it succeeded,
+spammers would then abuse the library again, and the cycle would repeat.
+
 The library will only do things that you tell it to do. If you use
 the library with bad intentions, Telegram will hopefully ban you.
 
@@ -67,8 +77,7 @@ However, you may also be part of a limited country, such as Iran or Russia.
 In that case, we have bad news for you. Telegram is much more likely to ban
 these numbers, as they are often used to spam other accounts, likely through
 the use of libraries like this one. The best advice we can give you is to not
-abuse the API, like calling many requests really quickly, and to sign up with
-these phones through an official application.
+abuse the API, like calling many requests really quickly.
 
 We have also had reports from Kazakhstan and China, where connecting
 would fail. To solve these connection problems, you should use a proxy.
@@ -76,6 +85,16 @@ would fail. To solve these connection problems, you should use a proxy.
 Telegram may also ban virtual (VoIP) phone numbers,
 as again, they're likely to be used for spam.
 
+More recently (year 2023 onwards), Telegram has started putting a lot more
+measures to prevent spam (with even additions such as anonymous participants
+in groups or the inability to fetch group members at all). This means some
+of the anti-spam measures have gotten more aggressive.
+
+The recommendation has usually been to use the library only on well-established
+accounts (and not an account you just created), and to not perform actions that
+could be seen as abuse. Telegram decides what those actions are, and they're
+free to change how they operate at any time.
+
 If you want to check if your account has been limited,
 simply send a private message to `@SpamBot`_ through Telegram itself.
 You should notice this by getting errors like ``PeerFloodError``,
@@ -98,11 +117,11 @@ This is basic Python knowledge. You should use the dot operator:
 
 .. code-block:: python
 
-    me = client.get_me()
+    me = await client.get_me()
     print(me.username)
     #       ^ we used the dot operator to access the username attribute
 
-    result = client(functions.photos.GetUserPhotosRequest(
+    result = await client(functions.photos.GetUserPhotosRequest(
         user_id='me',
         offset=0,
         max_id=0,
@@ -179,6 +198,137 @@ won't do unnecessary work unless you need to:
         sender = await event.get_sender()
 
 
+File download is slow or sending files takes too long
+=====================================================
+
+The communication with Telegram is encrypted. Encryption requires a lot of
+math, and doing it in pure Python is very slow. ``cryptg`` is a library which
+containns the encryption functions used by Telethon. If it is installed (via
+``pip install cryptg``), it will automatically be used and should provide
+a considerable speed boost. You can know whether it's used by configuring
+``logging`` (at ``INFO`` level or lower) *before* importing ``telethon``.
+
+Note that the library does *not* download or upload files in parallel, which
+can also help with the speed of downloading or uploading a single file. There
+are snippets online implementing that. The reason why this is not built-in
+is because the limiting factor in the long run are ``FloodWaitError``, and
+using parallel download or uploads only makes them occur sooner.
+
+
+What does "Server sent a very new message with ID" mean?
+========================================================
+
+You may also see this error as "Server sent a very old message with ID".
+
+This is a security feature from Telethon that cannot be disabled and is
+meant to protect you against replay attacks.
+
+When this message is incorrectly reported as a "bug",
+the most common patterns seem to be:
+
+* Your system time is incorrect.
+* The proxy you're using may be interfering somehow.
+* The Telethon session is being used or has been used from somewhere else.
+  Make sure that you created the session from Telethon, and are not using the
+  same session anywhere else. If you need to use the same account from
+  multiple places, login and use a different session for each place you need.
+
+
+What does "Server replied with a wrong session ID" mean?
+========================================================
+
+This is a security feature from Telethon that cannot be disabled and is
+meant to protect you against unwanted session reuse.
+
+When this message is reported as a "bug", the most common patterns seem to be:
+
+* The proxy you're using may be interfering somehow.
+* The Telethon session is being used or has been used from somewhere else.
+  Make sure that you created the session from Telethon, and are not using the
+  same session anywhere else. If you need to use the same account from
+  multiple places, login and use a different session for each place you need.
+* You may be using multiple connections to the Telegram server, which seems
+  to confuse Telegram.
+
+Most of the time it should be safe to ignore this warning. If the library
+still doesn't behave correctly, make sure to check if any of the above bullet
+points applies in your case and try to work around it.
+
+If the issue persists and there is a way to reliably reproduce this error,
+please add a comment with any additional details you can provide to
+`issue 3759`_, and perhaps some additional investigation can be done
+(but it's unlikely, as Telegram *is* sending unexpected data).
+
+
+What does "Could not find a matching Constructor ID for the TLObject" mean?
+===========================================================================
+
+Telegram uses "layers", which you can think of as "versions" of the API they
+offer. When Telethon reads responses that the Telegram servers send, these
+need to be deserialized (into what Telethon calls "TLObjects").
+
+Every Telethon version understands a single Telegram layer. When Telethon
+connects to Telegram, both agree on the layer to use. If the layers don't
+match, Telegram may send certain objects which Telethon no longer understands.
+
+When this message is reported as a "bug", the most common patterns seem to be
+that the Telethon session is being used or has been used from somewhere else.
+Make sure that you created the session from Telethon, and are not using the
+same session anywhere else. If you need to use the same account from
+multiple places, login and use a different session for each place you need.
+
+
+What does "Task was destroyed but it is pending" mean?
+======================================================
+
+Your script likely finished abruptly, the ``asyncio`` event loop got
+destroyed, and the library did not get a chance to properly close the
+connection and close the session.
+
+Make sure you're either using the context manager for the client or always
+call ``await client.disconnect()`` (by e.g. using a ``try/finally``).
+
+
+What does "The asyncio event loop must not change after connection" mean?
+=========================================================================
+
+Telethon uses ``asyncio``, and makes use of things like tasks and queues
+internally to manage the connection to the server and match responses to the
+requests you make. Most of them are initialized after the client is connected.
+
+For example, if the library expects a result to a request made in loop A, but
+you attempt to get that result in loop B, you will very likely find a deadlock.
+To avoid a deadlock, the library checks to make sure the loop in use is the
+same as the one used to initialize everything, and if not, it throws an error.
+
+The most common cause is ``asyncio.run``, since it creates a new event loop.
+If you ``asyncio.run`` a function to create the client and set it up, and then
+you ``asyncio.run`` another function to do work, things won't work, so the
+library throws an error early to let you know something is wrong.
+
+Instead, it's often a good idea to have a single ``async def main`` and simply
+``asyncio.run()`` it and do all the work there. From it, you're also able to
+call other ``async def`` without having to touch ``asyncio.run`` again:
+
+.. code-block:: python
+
+    # It's fine to create the client outside as long as you don't connect
+    client = TelegramClient(...)
+
+    async def main():
+        # Now the client will connect, so the loop must not change from now on.
+        # But as long as you do all the work inside main, including calling
+        # other async functions, things will work.
+        async with client:
+            ....
+
+    if __name__ == '__main__':
+        asyncio.run(main())
+
+Be sure to read the ``asyncio`` documentation if you want a better
+understanding of event loop, tasks, and what functions you can use.
+
+
 What does "bases ChatGetter" mean?
 ==================================
 
@@ -204,6 +354,36 @@ Telegram has a lot to offer, and inheritance helps the library reduce
 boilerplate, so it's important to know this concept. For newcomers,
 this may be a problem, so we explain what it means here in the FAQ.
 
+Can I send files by ID?
+=======================
+
+When people talk about IDs, they often refer to one of two things:
+the integer ID inside media, and a random-looking long string.
+
+You cannot use the integer ID to send media. Generally speaking, sending media
+requires a combination of ID, ``access_hash`` and ``file_reference``.
+The first two are integers, while the last one is a random ``bytes`` sequence.
+
+* The integer ``id`` will always be the same for every account, so every user
+  or bot looking at a particular media file, will see a consistent ID.
+* The ``access_hash`` will always be the same for a given account, but
+  different accounts will each see their own, different ``access_hash``.
+  This makes it impossible to get media object from one account and use it in
+  another. The other account must fetch the media object itself.
+* The ``file_reference`` is random for everyone and will only work for a few
+  hours before it expires. It must be refetched before the media can be used
+  (to either resend the media or download it).
+
+The second type of "`file ID <https://core.telegram.org/bots/api#inputfile>`_"
+people refer to is a concept from the HTTP Bot API. It's a custom format which
+encodes enough information to use the media.
+
+Telethon provides an old version of these HTTP Bot API-style file IDs via
+``message.file.id``, however, this feature is no longer maintained, so it may
+not work. It will be removed in future versions. Nonetheless, it is possible
+to find a different Python package (or write your own) to parse these file IDs
+and construct the necessary input file objects to send or download the media.
+
 
 Can I use Flask with the library?
 =================================
@@ -213,6 +393,31 @@ lot of headaches to get threads and asyncio to work together. Instead,
 consider using `Quart <https://pgjones.gitlab.io/quart/>`_, an asyncio-based
 alternative to `Flask <flask.pocoo.org/>`_.
 
+Check out `quart_login.py`_ for an example web-application based on Quart.
+
+Can I use Anaconda/Spyder/IPython with the library?
+===================================================
+
+Yes, but these interpreters run the asyncio event loop implicitly,
+which interferes with the ``telethon.sync`` magic module.
+
+If you use them, you should **not** import ``sync``:
+
+.. code-block:: python
+
+    # Change any of these...:
+    from telethon import TelegramClient, sync, ...
+    from telethon.sync import TelegramClient, ...
+
+    # ...with this:
+    from telethon import TelegramClient, ...
+
+You are also more likely to get "sqlite3.OperationalError: database is locked"
+with them. If they cause too much trouble, just write your code in a ``.py``
+file and run that, or use the normal ``python`` interpreter.
+
 .. _logging: https://docs.python.org/3/library/logging.html
 .. _@SpamBot: https://t.me/SpamBot
 .. _issue 297: https://github.com/LonamiWebs/Telethon/issues/297
+.. _issue 3759: https://github.com/LonamiWebs/Telethon/issues/3759
+.. _quart_login.py: https://github.com/LonamiWebs/Telethon/tree/v1/telethon_examples#quart_loginpy

readthedocs/quick-references/objects-reference.rst

@@ -121,6 +121,7 @@ Methods
     delete
     get_reply_message
     click
+    mark_read
     pin
     download_media
     get_entities_text
@@ -142,6 +143,8 @@ its name, bot-API style file ID, etc.
 
     id
     name
+    ext
+    mime_type
     width
     height
     size
@@ -176,6 +179,7 @@ It bases `ChatGetter <telethon.tl.custom.chatgetter.ChatGetter>`.
     wait_read
     wait_event
     cancel
+    cancel_all
 
 
 AdminLogEvent
@@ -236,9 +240,11 @@ These are the static methods you can use to create instances of the markup:
     inline
     switch_inline
     url
+    auth
     text
     request_location
     request_phone
+    request_poll
     clear
     force_reply
 

readthedocs/requirements.txt

@@ -1 +1,2 @@
-telethon
+./
+sphinx-rtd-theme~=1.3.0

setup.py

@@ -12,32 +12,40 @@ Extra supported commands are:
 
 import itertools
 import json
+import os
 import re
 import shutil
-from os import chdir
+import sys
+import urllib.request
 from pathlib import Path
 from subprocess import run
-from sys import argv
 
 from setuptools import find_packages, setup
 
+# Needed since we're importing local files
+sys.path.insert(0, os.path.dirname(__file__))
 
 class TempWorkDir:
     """Switches the working directory to be the one on which this file lives,
        while within the 'with' block.
     """
-    def __init__(self):
+    def __init__(self, new=None):
         self.original = None
+        self.new = new or str(Path(__file__).parent.resolve())
 
     def __enter__(self):
-        self.original = Path('.')
-        chdir(str(Path(__file__).parent))
+        # os.chdir does not work with Path in Python 3.5.x
+        self.original = str(Path('.').resolve())
+        os.makedirs(self.new, exist_ok=True)
+        os.chdir(self.new)
         return self
 
     def __exit__(self, *args):
-        chdir(str(self.original))
+        os.chdir(self.original)
 
 
+API_REF_URL = 'https://tl.telethon.dev/'
+
 GENERATOR_DIR = Path('telethon_generator')
 LIBRARY_DIR = Path('telethon')
 
@@ -108,7 +116,9 @@ def generate(which, action='gen'):
             if DOCS_OUT.is_dir():
                 shutil.rmtree(str(DOCS_OUT))
         else:
-            generate_docs(tlobjects, methods, layer, DOCS_IN_RES, DOCS_OUT)
+            in_path = DOCS_IN_RES.resolve()
+            with TempWorkDir(DOCS_OUT):
+                generate_docs(tlobjects, methods, layer, in_path)
 
     if 'json' in which:
         which.remove('json')
@@ -135,29 +145,54 @@ def generate(which, action='gen'):
                 gen_json(*fs)
 
     if which:
-        print('The following items were not understood:', which)
-        print('  Consider using only "tl", "errors" and/or "docs".')
-        print('  Using only "clean" will clean them. "all" to act on all.')
-        print('  For instance "gen tl errors".')
+        print(
+            'The following items were not understood:', which,
+            '\n  Consider using only "tl", "errors" and/or "docs".'
+            '\n  Using only "clean" will clean them. "all" to act on all.'
+            '\n  For instance "gen tl errors".'
+        )
 
 
-def main():
+def main(argv):
     if len(argv) >= 2 and argv[1] in ('gen', 'clean'):
         generate(argv[2:], argv[1])
 
     elif len(argv) >= 2 and argv[1] == 'pypi':
+        # Make sure tl.telethon.dev is up-to-date first
+        with urllib.request.urlopen(API_REF_URL) as resp:
+            html = resp.read()
+            m = re.search(br'layer\s+(\d+)', html)
+            if not m:
+                print('Failed to check that the API reference is up to date:', API_REF_URL)
+                return
+
+            from telethon_generator.parsers import find_layer
+            layer = next(filter(None, map(find_layer, TLOBJECT_IN_TLS)))
+            published_layer = int(m[1])
+            if published_layer != layer:
+                print('Published layer', published_layer, 'does not match current layer', layer, '.')
+                print('Make sure to update the API reference site first:', API_REF_URL)
+                return
+
         # (Re)generate the code to make sure we don't push without it
         generate(['tl', 'errors'])
 
         # Try importing the telethon module to assert it has no errors
         try:
             import telethon
-        except:
+        except Exception as e:
             print('Packaging for PyPi aborted, importing the module failed.')
+            print(e)
             return
 
-        for x in ('build', 'dist', 'Telethon.egg-info'):
+        remove_dirs = ['__pycache__', 'build', 'dist', 'Telethon.egg-info']
+        for root, _dirs, _files in os.walk(LIBRARY_DIR, topdown=False):
+            # setuptools is including __pycache__ for some reason (#1605)
+            if root.endswith('/__pycache__'):
+                remove_dirs.append(root)
+        for x in remove_dirs:
             shutil.rmtree(x, ignore_errors=True)
+
         run('python3 setup.py sdist', shell=True)
         run('python3 setup.py bdist_wheel', shell=True)
         run('twine upload dist/*', shell=True)
@@ -209,11 +244,13 @@ def main():
 
                 'Programming Language :: Python :: 3',
                 'Programming Language :: Python :: 3.5',
-                'Programming Language :: Python :: 3.6'
+                'Programming Language :: Python :: 3.6',
+                'Programming Language :: Python :: 3.7',
+                'Programming Language :: Python :: 3.8',
             ],
             keywords='telegram api chat client library messaging mtproto',
             packages=find_packages(exclude=[
-                'telethon_*', 'run_tests.py', 'try_telethon.py'
+                'telethon_*', 'tests*'
             ]),
             install_requires=['pyaes', 'rsa'],
             extras_require={
@@ -223,5 +260,5 @@ def main():
 
 
 if __name__ == '__main__':
-    with TempWorkDir():  # Could just use a try/finally but this is + reusable
-        main()
+    with TempWorkDir():
+        main(sys.argv)

telethon/__init__.py

@@ -1,8 +1,8 @@
 from .client.telegramclient import TelegramClient
 from .network import connection
-from .tl import types, functions, custom
 from .tl.custom import Button
-from . import version, events, utils, errors
+from .tl import patched as _  # import for its side-effects
+from . import version, events, utils, errors, types, functions, custom
 
 __version__ = version.__version__
 

telethon/_updates/__init__.py

@@ -0,0 +1,3 @@
+from .entitycache import EntityCache
+from .messagebox import MessageBox, GapError, PrematureEndReason
+from .session import SessionState, ChannelState, Entity, EntityType

telethon/_updates/entitycache.py

@@ -0,0 +1,62 @@
+from .session import EntityType, Entity
+
+
+_sentinel = object()
+
+
+class EntityCache:
+    def __init__(
+        self,
+        hash_map: dict = _sentinel,
+        self_id: int = None,
+        self_bot: bool = None
+    ):
+        self.hash_map = {} if hash_map is _sentinel else hash_map
+        self.self_id = self_id
+        self.self_bot = self_bot
+
+    def set_self_user(self, id, bot, hash):
+        self.self_id = id
+        self.self_bot = bot
+        if hash:
+            self.hash_map[id] = (hash, EntityType.BOT if bot else EntityType.USER)
+
+    def get(self, id):
+        try:
+            hash, ty = self.hash_map[id]
+            return Entity(ty, id, hash)
+        except KeyError:
+            return None
+
+    def extend(self, users, chats):
+        # See https://core.telegram.org/api/min for "issues" with "min constructors".
+        self.hash_map.update(
+            (u.id, (
+                u.access_hash,
+                EntityType.BOT if u.bot else EntityType.USER,
+            ))
+            for u in users
+            if getattr(u, 'access_hash', None) and not u.min
+        )
+        self.hash_map.update(
+            (c.id, (
+                c.access_hash,
+                EntityType.MEGAGROUP if c.megagroup else (
+                    EntityType.GIGAGROUP if getattr(c, 'gigagroup', None) else EntityType.CHANNEL
+                ),
+            ))
+            for c in chats
+            if getattr(c, 'access_hash', None) and not getattr(c, 'min', None)
+        )
+
+    def get_all_entities(self):
+        return [Entity(ty, id, hash) for id, (hash, ty) in self.hash_map.items()]
+
+    def put(self, entity):
+        self.hash_map[entity.id] = (entity.hash, entity.ty)
+
+    def retain(self, filter):
+        self.hash_map = {k: v for k, v in self.hash_map.items() if filter(k)}
+
+    def __len__(self):
+        return len(self.hash_map)

telethon/_updates/messagebox.py

@@ -0,0 +1,833 @@
+"""
+This module deals with correct handling of updates, including gaps, and knowing when the code
+should "get difference" (the set of updates that the client should know by now minus the set
+of updates that it actually knows).
+
+Each chat has its own [`Entry`] in the [`MessageBox`] (this `struct` is the "entry point").
+At any given time, the message box may be either getting difference for them (entry is in
+[`MessageBox::getting_diff_for`]) or not. If not getting difference, a possible gap may be
+found for the updates (entry is in [`MessageBox::possible_gaps`]). Otherwise, the entry is
+on its happy path.
+
+Gaps are cleared when they are either resolved on their own (by waiting for a short time)
+or because we got the difference for the corresponding entry.
+
+While there are entries for which their difference must be fetched,
+[`MessageBox::check_deadlines`] will always return [`Instant::now`], since "now" is the time
+to get the difference.
+"""
+import asyncio
+import datetime
+import time
+import logging
+from enum import Enum
+from .session import SessionState, ChannelState
+from ..tl import types as tl, functions as fn
+from ..helpers import get_running_loop
+
+
+# Telegram sends `seq` equal to `0` when "it doesn't matter", so we use that value too.
+NO_SEQ = 0
+
+# See https://core.telegram.org/method/updates.getChannelDifference.
+BOT_CHANNEL_DIFF_LIMIT = 100000
+USER_CHANNEL_DIFF_LIMIT = 100
+
+# > It may be useful to wait up to 0.5 seconds
+POSSIBLE_GAP_TIMEOUT = 0.5
+
+# After how long without updates the client will "timeout".
+#
+# When this timeout occurs, the client will attempt to fetch updates by itself, ignoring all the
+# updates that arrive in the meantime. After all updates are fetched when this happens, the
+# client will resume normal operation, and the timeout will reset.
+#
+# Documentation recommends 15 minutes without updates (https://core.telegram.org/api/updates).
+NO_UPDATES_TIMEOUT = 15 * 60
+
+# object() but with a tag to make it easier to debug
+class Sentinel:
+    __slots__ = ('tag',)
+
+    def __init__(self, tag=None):
+        self.tag = tag or '_'
+
+    def __repr__(self):
+        return self.tag
+
+# Entry "enum".
+# Account-wide `pts` includes private conversations (one-to-one) and small group chats.
+ENTRY_ACCOUNT = Sentinel('ACCOUNT')
+# Account-wide `qts` includes only "secret" one-to-one chats.
+ENTRY_SECRET = Sentinel('SECRET')
+# Integers will be Channel-specific `pts`, and includes "megagroup", "broadcast" and "supergroup" channels.
+
+# Python's logging doesn't define a TRACE level. Pick halfway between DEBUG and NOTSET.
+# We don't define a name for this as libraries shouldn't do that though.
+LOG_LEVEL_TRACE = (logging.DEBUG - logging.NOTSET) // 2
+
+_sentinel = Sentinel()
+
+def next_updates_deadline():
+    return get_running_loop().time() + NO_UPDATES_TIMEOUT
+
+def epoch():
+    return datetime.datetime(*time.gmtime(0)[:6]).replace(tzinfo=datetime.timezone.utc)
+
+class GapError(ValueError):
+    def __repr__(self):
+        return 'GapError()'
+
+
+class PrematureEndReason(Enum):
+    TEMPORARY_SERVER_ISSUES = 'tmp'
+    BANNED = 'ban'
+
+
+# Represents the information needed to correctly handle a specific `tl::enums::Update`.
+class PtsInfo:
+    __slots__ = ('pts', 'pts_count', 'entry')
+
+    def __init__(
+        self,
+        pts: int,
+        pts_count: int,
+        entry: object
+    ):
+        self.pts = pts
+        self.pts_count = pts_count
+        self.entry = entry
+
+    @classmethod
+    def from_update(cls, update):
+        pts = getattr(update, 'pts', None)
+        if pts:
+            pts_count = getattr(update, 'pts_count', None) or 0
+            try:
+                entry = update.message.peer_id.channel_id
+            except AttributeError:
+                entry = getattr(update, 'channel_id', None) or ENTRY_ACCOUNT
+            return cls(pts=pts, pts_count=pts_count, entry=entry)
+
+        qts = getattr(update, 'qts', None)
+        if qts:
+            return cls(pts=qts, pts_count=1, entry=ENTRY_SECRET)
+
+        return None
+
+    def __repr__(self):
+        return f'PtsInfo(pts={self.pts}, pts_count={self.pts_count}, entry={self.entry})'
+
+
+# The state of a particular entry in the message box.
+class State:
+    __slots__ = ('pts', 'deadline')
+
+    def __init__(
+        self,
+        # Current local persistent timestamp.
+        pts: int,
+        # Next instant when we would get the update difference if no updates arrived before then.
+        deadline: float
+    ):
+        self.pts = pts
+        self.deadline = deadline
+
+    def __repr__(self):
+        return f'State(pts={self.pts}, deadline={self.deadline})'
+
+
+# > ### Recovering gaps
+# > […] Manually obtaining updates is also required in the following situations:
+# > • Loss of sync: a gap was found in `seq` / `pts` / `qts` (as described above).
+# >   It may be useful to wait up to 0.5 seconds in this situation and abort the sync in case a new update
+# >   arrives, that fills the gap.
+#
+# This is really easy to trigger by spamming messages in a channel (with as little as 3 members works), because
+# the updates produced by the RPC request take a while to arrive (whereas the read update comes faster alone).
+class PossibleGap:
+    __slots__ = ('deadline', 'updates')
+
+    def __init__(
+        self,
+        deadline: float,
+        # Pending updates (those with a larger PTS, producing the gap which may later be filled).
+        updates: list  # of updates
+    ):
+        self.deadline = deadline
+        self.updates = updates
+
+    def __repr__(self):
+        return f'PossibleGap(deadline={self.deadline}, update_count={len(self.updates)})'
+
+
+# Represents a "message box" (event `pts` for a specific entry).
+#
+# See https://core.telegram.org/api/updates#message-related-event-sequences.
+class MessageBox:
+    __slots__ = ('_log', 'map', 'date', 'seq', 'next_deadline', 'possible_gaps', 'getting_diff_for')
+
+    def __init__(
+        self,
+        log,
+        # Map each entry to their current state.
+        map: dict = _sentinel,  # entry -> state
+
+        # Additional fields beyond PTS needed by `ENTRY_ACCOUNT`.
+        date: datetime.datetime = epoch() + datetime.timedelta(seconds=1),
+        seq: int = NO_SEQ,
+
+        # Holds the entry with the closest deadline (optimization to avoid recalculating the minimum deadline).
+        next_deadline: object = None,  # entry
+
+        # Which entries have a gap and may soon trigger a need to get difference.
+        #
+        # If a gap is found, stores the required information to resolve it (when should it timeout and what updates
+        # should be held in case the gap is resolved on its own).
+        #
+        # Not stored directly in `map` as an optimization (else we would need another way of knowing which entries have
+        # a gap in them).
+        possible_gaps: dict = _sentinel,  # entry -> possiblegap
+
+        # For which entries are we currently getting difference.
+        getting_diff_for: set = _sentinel,  # entry
+    ):
+        self._log = log
+        self.map = {} if map is _sentinel else map
+        self.date = date
+        self.seq = seq
+        self.next_deadline = next_deadline
+        self.possible_gaps = {} if possible_gaps is _sentinel else possible_gaps
+        self.getting_diff_for = set() if getting_diff_for is _sentinel else getting_diff_for
+
+        if __debug__:
+            self._trace('MessageBox initialized')
+
+    def _trace(self, msg, *args, **kwargs):
+        # Calls to trace can't really be removed beforehand without some dark magic.
+        # So every call to trace is prefixed with `if __debug__`` instead, to remove
+        # it when using `python -O`. Probably unnecessary, but it's nice to avoid
+        # paying the cost for something that is not used.
+        self._log.log(LOG_LEVEL_TRACE, 'Current MessageBox state: seq = %r, date = %s, map = %r',
+                      self.seq, self.date.isoformat(), self.map)
+        self._log.log(LOG_LEVEL_TRACE, msg, *args, **kwargs)
+
+    # region Creation, querying, and setting base state.
+
+    def load(self, session_state, channel_states):
+        """
+        Create a [`MessageBox`] from a previously known update state.
+        """
+        if __debug__:
+            self._trace('Loading MessageBox with session_state = %r, channel_states = %r', session_state, channel_states)
+
+        deadline = next_updates_deadline()
+
+        self.map.clear()
+        if session_state.pts != NO_SEQ:
+            self.map[ENTRY_ACCOUNT] = State(pts=session_state.pts, deadline=deadline)
+        if session_state.qts != NO_SEQ:
+            self.map[ENTRY_SECRET] = State(pts=session_state.qts, deadline=deadline)
+        self.map.update((s.channel_id, State(pts=s.pts, deadline=deadline)) for s in channel_states)
+
+        self.date = datetime.datetime.fromtimestamp(session_state.date, tz=datetime.timezone.utc)
+        self.seq = session_state.seq
+        self.next_deadline = ENTRY_ACCOUNT
+
+    def session_state(self):
+        """
+        Return the current state.
+
+        This should be used for persisting the state.
+        """
+        return dict(
+            pts=self.map[ENTRY_ACCOUNT].pts if ENTRY_ACCOUNT in self.map else NO_SEQ,
+            qts=self.map[ENTRY_SECRET].pts if ENTRY_SECRET in self.map else NO_SEQ,
+            date=self.date,
+            seq=self.seq,
+        ), {id: state.pts for id, state in self.map.items() if isinstance(id, int)}
+
+    def is_empty(self) -> bool:
+        """
+        Return true if the message box is empty and has no state yet.
+        """
+        return ENTRY_ACCOUNT not in self.map
+
+    def check_deadlines(self):
+        """
+        Return the next deadline when receiving updates should timeout.
+
+        If a deadline expired, the corresponding entries will be marked as needing to get its difference.
+        While there are entries pending of getting their difference, this method returns the current instant.
+        """
+        now = get_running_loop().time()
+
+        if self.getting_diff_for:
+            return now
+
+        deadline = next_updates_deadline()
+
+        # Most of the time there will be zero or one gap in flight so finding the minimum is cheap.
+        if self.possible_gaps:
+            deadline = min(deadline, *(gap.deadline for gap in self.possible_gaps.values()))
+        elif self.next_deadline in self.map:
+            deadline = min(deadline, self.map[self.next_deadline].deadline)
+
+        # asyncio's loop time precision only seems to be about 3 decimal places, so it's possible that
+        # we find the same number again on repeated calls. Without the "or equal" part we would log the
+        # timeout for updates several times (it also makes sense to get difference if now is the deadline).
+        if now >= deadline:
+            # Check all expired entries and add them to the list that needs getting difference.
+            self.getting_diff_for.update(entry for entry, gap in self.possible_gaps.items() if now >= gap.deadline)
+            self.getting_diff_for.update(entry for entry, state in self.map.items() if now >= state.deadline)
+
+            if __debug__:
+                self._trace('Deadlines met, now getting diff for %r', self.getting_diff_for)
+
+            # When extending `getting_diff_for`, it's important to have the moral equivalent of
+            # `begin_get_diff` (that is, clear possible gaps if we're now getting difference).
+            for entry in self.getting_diff_for:
+                self.possible_gaps.pop(entry, None)
+
+        return deadline
+
+    # Reset the deadline for the periods without updates for the given entries.
+    #
+    # It also updates the next deadline time to reflect the new closest deadline.
+    def reset_deadlines(self, entries, deadline):
+        if not entries:
+            return
+        for entry in entries:
+            if entry not in self.map:
+                raise RuntimeError('Called reset_deadline on an entry for which we do not have state')
+            self.map[entry].deadline = deadline
+
+        if self.next_deadline in entries:
+            # If the updated deadline was the closest one, recalculate the new minimum.
+            self.next_deadline = min(self.map.items(), key=lambda entry_state: entry_state[1].deadline)[0]
+        elif self.next_deadline in self.map and deadline < self.map[self.next_deadline].deadline:
+            # If the updated deadline is smaller than the next deadline, change the next deadline to be the new one.
+            # Any entry will do, so the one from the last iteration is fine.
+            self.next_deadline = entry
+        # else an unrelated deadline was updated, so the closest one remains unchanged.
+
+    # Convenience to reset a channel's deadline, with optional timeout.
+    def reset_channel_deadline(self, channel_id, timeout):
+        self.reset_deadlines({channel_id}, get_running_loop().time() + (timeout or NO_UPDATES_TIMEOUT))
+
+    # Sets the update state.
+    #
+    # Should be called right after login if [`MessageBox::new`] was used, otherwise undesirable
+    # updates will be fetched.
+    def set_state(self, state, reset=True):
+        if __debug__:
+            self._trace('Setting state %s', state)
+
+        deadline = next_updates_deadline()
+
+        if state.pts != NO_SEQ or not reset:
+            self.map[ENTRY_ACCOUNT] = State(pts=state.pts, deadline=deadline)
+        else:
+            self.map.pop(ENTRY_ACCOUNT, None)
+
+        # Telegram seems to use the `qts` for bot accounts, but while applying difference,
+        # it might be reset back to 0. See issue #3873 for more details.
+        #
+        # During login, a value of zero would mean the `pts` is unknown,
+        # so the map shouldn't contain that entry.
+        # But while applying difference, if the value is zero, it (probably)
+        # truly means that's what should be used (hence the `reset` flag).
+        if state.qts != NO_SEQ or not reset:
+            self.map[ENTRY_SECRET] = State(pts=state.qts, deadline=deadline)
+        else:
+            self.map.pop(ENTRY_SECRET, None)
+
+        self.date = state.date
+        self.seq = state.seq
+
+    # Like [`MessageBox::set_state`], but for channels. Useful when getting dialogs.
+    #
+    # The update state will only be updated if no entry was known previously.
+    def try_set_channel_state(self, id, pts):
+        if __debug__:
+            self._trace('Trying to set channel state for %r: %r', id, pts)
+
+        if id not in self.map:
+            self.map[id] = State(pts=pts, deadline=next_updates_deadline())
+
+    # Try to begin getting difference for the given entry.
+    # Fails if the entry does not have a previously-known state that can be used to get its difference.
+    #
+    # Clears any previous gaps.
+    def try_begin_get_diff(self, entry, reason):
+        if entry not in self.map:
+            # Won't actually be able to get difference for this entry if we don't have a pts to start off from.
+            if entry in self.possible_gaps:
+                raise RuntimeError('Should not have a possible_gap for an entry not in the state map')
+
+            if __debug__:
+                self._trace('Should get difference for %r because %s but cannot due to missing hash', entry, reason)
+            return
+
+        if __debug__:
+            self._trace('Marking %r as needing difference because %s', entry, reason)
+        self.getting_diff_for.add(entry)
+        self.possible_gaps.pop(entry, None)
+
+    # Finish getting difference for the given entry.
+    #
+    # It also resets the deadline.
+    def end_get_diff(self, entry):
+        try:
+            self.getting_diff_for.remove(entry)
+        except KeyError:
+            raise RuntimeError('Called end_get_diff on an entry which was not getting diff for')
+
+        self.reset_deadlines({entry}, next_updates_deadline())
+        assert entry not in self.possible_gaps, "gaps shouldn't be created while getting difference"
+
+    # endregion Creation, querying, and setting base state.
+
+    # region "Normal" updates flow (processing and detection of gaps).
+
+    # Process an update and return what should be done with it.
+    #
+    # Updates corresponding to entries for which their difference is currently being fetched
+    # will be ignored. While according to the [updates' documentation]:
+    #
+    # > Implementations [have] to postpone updates received via the socket while
+    # > filling gaps in the event and `Update` sequences, as well as avoid filling
+    # > gaps in the same sequence.
+    #
+    # In practice, these updates should have also been retrieved through getting difference.
+    #
+    # [updates documentation] https://core.telegram.org/api/updates
+    def process_updates(
+        self,
+        updates,
+        chat_hashes,
+        result,  # out list of updates; returns list of user, chat, or raise if gap
+    ):
+
+        # v1 has never sent updates produced by the client itself to the handlers.
+        # However proper update handling requires those to be processed.
+        # This is an ugly workaround for that.
+        self_outgoing = getattr(updates, '_self_outgoing', False)
+        real_result = result
+        result = []
+
+        date = getattr(updates, 'date', None)
+        seq = getattr(updates, 'seq', None)
+        seq_start = getattr(updates, 'seq_start', None)
+        users = getattr(updates, 'users', None) or []
+        chats = getattr(updates, 'chats', None) or []
+
+        if __debug__:
+            self._trace('Processing updates with seq = %r, seq_start = %r, date = %s: %s',
+                        seq, seq_start, date.isoformat() if date else None, updates)
+
+        if date is None:
+            # updatesTooLong is the only one with no date (we treat it as a gap)
+            self.try_begin_get_diff(ENTRY_ACCOUNT, 'received updatesTooLong')
+            raise GapError
+        if seq is None:
+            seq = NO_SEQ
+        if seq_start is None:
+            seq_start = seq
+
+        # updateShort is the only update which cannot be dispatched directly but doesn't have 'updates' field
+        updates = getattr(updates, 'updates', None) or [updates.update if isinstance(updates, tl.UpdateShort) else updates]
+
+        for u in updates:
+            u._self_outgoing = self_outgoing
+
+        # > For all the other [not `updates` or `updatesCombined`] `Updates` type constructors
+        # > there is no need to check `seq` or change a local state.
+        if seq_start != NO_SEQ:
+            if self.seq + 1 > seq_start:
+                # Skipping updates that were already handled
+                if __debug__:
+                    self._trace('Skipping updates as they should have already been handled')
+                return (users, chats)
+            elif self.seq + 1 < seq_start:
+                # Gap detected
+                self.try_begin_get_diff(ENTRY_ACCOUNT, 'detected gap')
+                raise GapError
+            # else apply
+
+        def _sort_gaps(update):
+            pts = PtsInfo.from_update(update)
+            return pts.pts - pts.pts_count if pts else 0
+
+        reset_deadlines = set()  # temporary buffer
+        any_pts_applied = [False]  # using a list to pass "by reference"
+
+        result.extend(filter(None, (
+            self.apply_pts_info(u, reset_deadlines=reset_deadlines, any_pts_applied=any_pts_applied)
+            # Telegram can send updates out of order (e.g. ReadChannelInbox first
+            # and then NewChannelMessage, both with the same pts, but the count is
+            # 0 and 1 respectively), so we sort them first.
+            for u in sorted(updates, key=_sort_gaps))))
+
+        # > If the updates were applied, local *Updates* state must be updated
+        # > with `seq` (unless it's 0) and `date` from the constructor.
+        #
+        # By "were applied", we assume it means "some other pts was applied".
+        # Updates which can be applied in any order, such as `UpdateChat`,
+        # should not cause `seq` to be updated (or upcoming updates such as
+        # `UpdateChatParticipant` could be missed).
+        if any_pts_applied[0]:
+            if __debug__:
+                self._trace('Updating seq as local pts was updated too')
+            if date != epoch():
+                self.date = date
+            if seq != NO_SEQ:
+                self.seq = seq
+
+        self.reset_deadlines(reset_deadlines, next_updates_deadline())
+
+        if self.possible_gaps:
+            if __debug__:
+                self._trace('Trying to re-apply %r possible gaps', len(self.possible_gaps))
+
+            # For each update in possible gaps, see if the gap has been resolved already.
+            for key in list(self.possible_gaps.keys()):
+                self.possible_gaps[key].updates.sort(key=_sort_gaps)
+
+                for _ in range(len(self.possible_gaps[key].updates)):
+                    update = self.possible_gaps[key].updates.pop(0)
+
+                    # If this fails to apply, it will get re-inserted at the end.
+                    # All should fail, so the order will be preserved (it would've cycled once).
+                    update = self.apply_pts_info(update, reset_deadlines=None)
+                    if update:
+                        result.append(update)
+                        if __debug__:
+                            self._trace('Resolved gap with %r: %s', PtsInfo.from_update(update), update)
+
+            # Clear now-empty gaps.
+            self.possible_gaps = {entry: gap for entry, gap in self.possible_gaps.items() if gap.updates}
+
+        real_result.extend(u for u in result if not u._self_outgoing)
+
+        return (users, chats)
+
+    # Tries to apply the input update if its `PtsInfo` follows the correct order.
+    #
+    # If the update can be applied, it is returned; otherwise, the update is stored in a
+    # possible gap (unless it was already handled or would be handled through getting
+    # difference) and `None` is returned.
+    def apply_pts_info(
+        self,
+        update,
+        *,
+        reset_deadlines,
+        any_pts_applied=[True],  # mutable default is fine as it's write-only
+    ):
+        # This update means we need to call getChannelDifference to get the updates from the channel
+        if isinstance(update, tl.UpdateChannelTooLong):
+            self.try_begin_get_diff(update.channel_id, 'received updateChannelTooLong')
+            return None
+
+        pts = PtsInfo.from_update(update)
+        if not pts:
+            # No pts means that the update can be applied in any order.
+            if __debug__:
+                self._trace('No pts in update, so it can be applied in any order: %s', update)
+            return update
+
+        # As soon as we receive an update of any form related to messages (has `PtsInfo`),
+        # the "no updates" period for that entry is reset.
+        #
+        # Build the `HashSet` to avoid calling `reset_deadline` more than once for the same entry.
+        #
+        # By the time this method returns, self.map will have an entry for which we can reset its deadline.
+        if reset_deadlines:
+            reset_deadlines.add(pts.entry)
+
+        if pts.entry in self.getting_diff_for:
+            # Note: early returning here also prevents gap from being inserted (which they should
+            # not be while getting difference).
+            if __debug__:
+                self._trace('Skipping update with %r as its difference is being fetched', pts)
+            return None
+
+        if pts.entry in self.map:
+            local_pts = self.map[pts.entry].pts
+            if local_pts + pts.pts_count > pts.pts:
+                # Ignore
+                if __debug__:
+                    self._trace('Skipping update since local pts %r > %r: %s', local_pts, pts, update)
+                return None
+            elif local_pts + pts.pts_count < pts.pts:
+                # Possible gap
+                # TODO store chats too?
+                if __debug__:
+                    self._trace('Possible gap since local pts %r < %r: %s', local_pts, pts, update)
+                if pts.entry not in self.possible_gaps:
+                    self.possible_gaps[pts.entry] = PossibleGap(
+                        deadline=get_running_loop().time() + POSSIBLE_GAP_TIMEOUT,
+                        updates=[]
+                    )
+
+                self.possible_gaps[pts.entry].updates.append(update)
+                return None
+            else:
+                # Apply
+                any_pts_applied[0] = True
+                if __debug__:
+                    self._trace('Applying update pts since local pts %r = %r: %s', local_pts, pts, update)
+
+        # In a channel, we may immediately receive:
+        # * ReadChannelInbox (pts = X, pts_count = 0)
+        # * NewChannelMessage (pts = X, pts_count = 1)
+        #
+        # Notice how both `pts` are the same. If they were to be applied out of order, the first
+        # one however would've triggered a gap because `local_pts` + `pts_count` of 0 would be
+        # less than `remote_pts`. So there is no risk by setting the `local_pts` to match the
+        # `remote_pts` here of missing the new message.
+        #
+        # The message would however be lost if we initialized the pts with the first one, since
+        # the second one would appear "already handled". To prevent this we set the pts to be
+        # one less when the count is 0 (which might be wrong and trigger a gap later on, but is
+        # unlikely). This will prevent us from losing updates in the unlikely scenario where these
+        # two updates arrive in different packets (and therefore couldn't be sorted beforehand).
+        if pts.entry in self.map:
+            self.map[pts.entry].pts = pts.pts
+        else:
+            # When a chat is migrated to a megagroup, the first update can be a `ReadChannelInbox`
+            # with `pts = 1, pts_count = 0` followed by a `NewChannelMessage` with `pts = 2, pts_count=1`.
+            # Note how the `pts` for the message is 2 and not 1 unlike the case described before!
+            # This is likely because the `pts` cannot be 0 (or it would fail with PERSISTENT_TIMESTAMP_EMPTY),
+            # which forces the first update to be 1. But if we got difference with 1 and the second update
+            # also used 1, we would miss it, so Telegram probably uses 2 to work around that.
+            self.map[pts.entry] = State(
+                pts=(pts.pts - (0 if pts.pts_count else 1)) or 1,
+                deadline=next_updates_deadline()
+            )
+
+        return update
+
+    # endregion "Normal" updates flow (processing and detection of gaps).
+
+    # region Getting and applying account difference.
+
+    # Return the request that needs to be made to get the difference, if any.
+    def get_difference(self):
+        for entry in (ENTRY_ACCOUNT, ENTRY_SECRET):
+            if entry in self.getting_diff_for:
+                if entry not in self.map:
+                    raise RuntimeError('Should not try to get difference for an entry without known state')
+
+                gd = fn.updates.GetDifferenceRequest(
+                    pts=self.map[ENTRY_ACCOUNT].pts,
+                    pts_total_limit=None,
+                    date=self.date,
+                    qts=self.map[ENTRY_SECRET].pts if ENTRY_SECRET in self.map else NO_SEQ,
+                )
+                if __debug__:
+                    self._trace('Requesting account difference %s', gd)
+                return gd
+
+        return None
+
+    # Similar to [`MessageBox::process_updates`], but using the result from getting difference.
+    def apply_difference(
+        self,
+        diff,
+        chat_hashes,
+    ):
+        if __debug__:
+            self._trace('Applying account difference %s', diff)
+
+        finish = None
+        result = None
+
+        if isinstance(diff, tl.updates.DifferenceEmpty):
+            finish = True
+            self.date = diff.date
+            self.seq = diff.seq
+            result = [], [], []
+        elif isinstance(diff, tl.updates.Difference):
+            finish = True
+            chat_hashes.extend(diff.users, diff.chats)
+            result = self.apply_difference_type(diff, chat_hashes)
+        elif isinstance(diff, tl.updates.DifferenceSlice):
+            finish = False
+            chat_hashes.extend(diff.users, diff.chats)
+            result = self.apply_difference_type(diff, chat_hashes)
+        elif isinstance(diff, tl.updates.DifferenceTooLong):
+            finish = True
+            self.map[ENTRY_ACCOUNT].pts = diff.pts  # the deadline will be reset once the diff ends
+            result = [], [], []
+
+        if finish:
+            account = ENTRY_ACCOUNT in self.getting_diff_for
+            secret = ENTRY_SECRET in self.getting_diff_for
+
+            if not account and not secret:
+                raise RuntimeError('Should not be applying the difference when neither account or secret was diff was active')
+
+            # Both may be active if both expired at the same time.
+            if account:
+                self.end_get_diff(ENTRY_ACCOUNT)
+            if secret:
+                self.end_get_diff(ENTRY_SECRET)
+
+        return result
+
+    def apply_difference_type(
+        self,
+        diff,
+        chat_hashes,
+    ):
+        state = getattr(diff, 'intermediate_state', None) or diff.state
+        self.set_state(state, reset=False)
+
+        # diff.other_updates can contain things like UpdateChannelTooLong and UpdateNewChannelMessage.
+        # We need to process those as if they were socket updates to discard any we have already handled.
+        updates = []
+        self.process_updates(tl.Updates(
+            updates=diff.other_updates,
+            users=diff.users,
+            chats=diff.chats,
+            date=epoch(),
+            seq=NO_SEQ,  # this way date is not used
+        ), chat_hashes, updates)
+
+        updates.extend(tl.UpdateNewMessage(
+            message=m,
+            pts=NO_SEQ,
+            pts_count=NO_SEQ,
+        ) for m in diff.new_messages)
+        updates.extend(tl.UpdateNewEncryptedMessage(
+            message=m,
+            qts=NO_SEQ,
+        ) for m in diff.new_encrypted_messages)
+
+        return updates, diff.users, diff.chats
+
+    def end_difference(self):
+        if __debug__:
+            self._trace('Ending account difference')
+
+        account = ENTRY_ACCOUNT in self.getting_diff_for
+        secret = ENTRY_SECRET in self.getting_diff_for
+
+        if not account and not secret:
+            raise RuntimeError('Should not be ending get difference when neither account or secret was diff was active')
+
+        # Both may be active if both expired at the same time.
+        if account:
+            self.end_get_diff(ENTRY_ACCOUNT)
+        if secret:
+            self.end_get_diff(ENTRY_SECRET)
+
+    # endregion Getting and applying account difference.
+
+    # region Getting and applying channel difference.
+
+    # Return the request that needs to be made to get a channel's difference, if any.
+    def get_channel_difference(
+        self,
+        chat_hashes,
+    ):
+        entry = next((id for id in self.getting_diff_for if isinstance(id, int)), None)
+        if not entry:
+            return None
+
+        packed = chat_hashes.get(entry)
+        if not packed:
+            # Cannot get channel difference as we're missing its hash
+            # TODO we should probably log this
+            self.end_get_diff(entry)
+            # Remove the outdated `pts` entry from the map so that the next update can correct
+            # it. Otherwise, it will spam that the access hash is missing.
+            self.map.pop(entry, None)
+            return None
+
+        state = self.map.get(entry)
+        if not state:
+            raise RuntimeError('Should not try to get difference for an entry without known state')
+
+        gd = fn.updates.GetChannelDifferenceRequest(
+            force=False,
+            channel=tl.InputChannel(packed.id, packed.hash),
+            filter=tl.ChannelMessagesFilterEmpty(),
+            pts=state.pts,
+            limit=BOT_CHANNEL_DIFF_LIMIT if chat_hashes.self_bot else USER_CHANNEL_DIFF_LIMIT
+        )
+        if __debug__:
+            self._trace('Requesting channel difference %s', gd)
+        return gd
+
+    # Similar to [`MessageBox::process_updates`], but using the result from getting difference.
+    def apply_channel_difference(
+        self,
+        request,
+        diff,
+        chat_hashes,
+    ):
+        entry = request.channel.channel_id
+        if __debug__:
+            self._trace('Applying channel difference for %r: %s', entry, diff)
+
+        self.possible_gaps.pop(entry, None)
+
+        if isinstance(diff, tl.updates.ChannelDifferenceEmpty):
+            assert diff.final
+            self.end_get_diff(entry)
+            self.map[entry].pts = diff.pts
+            return [], [], []
+        elif isinstance(diff, tl.updates.ChannelDifferenceTooLong):
+            assert diff.final
+            self.map[entry].pts = diff.dialog.pts
+            chat_hashes.extend(diff.users, diff.chats)
+            self.reset_channel_deadline(entry, diff.timeout)
+            # This `diff` has the "latest messages and corresponding chats", but it would
+            # be strange to give the user only partial changes of these when they would
+            # expect all updates to be fetched. Instead, nothing is returned.
+            return [], [], []
+        elif isinstance(diff, tl.updates.ChannelDifference):
+            if diff.final:
+                self.end_get_diff(entry)
+
+            self.map[entry].pts = diff.pts
+            chat_hashes.extend(diff.users, diff.chats)
+
+            updates = []
+            self.process_updates(tl.Updates(
+                updates=diff.other_updates,
+                users=diff.users,
+                chats=diff.chats,
+                date=epoch(),
+                seq=NO_SEQ,  # this way date is not used
+            ), chat_hashes, updates)
+
+            updates.extend(tl.UpdateNewChannelMessage(
+                message=m,
+                pts=NO_SEQ,
+                pts_count=NO_SEQ,
+            ) for m in diff.new_messages)
+            self.reset_channel_deadline(entry, None)
+
+            return updates, diff.users, diff.chats
+
+    def end_channel_difference(self, request, reason: PrematureEndReason, chat_hashes):
+        entry = request.channel.channel_id
+        if __debug__:
+            self._trace('Ending channel difference for %r because %s', entry, reason)
+
+        if reason == PrematureEndReason.TEMPORARY_SERVER_ISSUES:
+            # Temporary issues. End getting difference without updating the pts so we can retry later.
+            self.possible_gaps.pop(entry, None)
+            self.end_get_diff(entry)
+        elif reason == PrematureEndReason.BANNED:
+            # Banned in the channel. Forget its state since we can no longer fetch updates from it.
+            self.possible_gaps.pop(entry, None)
+            self.end_get_diff(entry)
+            del self.map[entry]
+        else:
+            raise RuntimeError('Unknown reason to end channel difference')
+
+    # endregion Getting and applying channel difference.

telethon/_updates/session.py

@@ -0,0 +1,195 @@
+from typing import Optional, Tuple
+from enum import IntEnum
+from ..tl.types import InputPeerUser, InputPeerChat, InputPeerChannel
+
+
+class SessionState:
+    """
+    Stores the information needed to fetch updates and about the current user.
+
+    * user_id: 64-bit number representing the user identifier.
+    * dc_id: 32-bit number relating to the datacenter identifier where the user is.
+    * bot: is the logged-in user a bot?
+    * pts: 64-bit number holding the state needed to fetch updates.
+    * qts: alternative 64-bit number holding the state needed to fetch updates.
+    * date: 64-bit number holding the date needed to fetch updates.
+    * seq: 64-bit-number holding the sequence number needed to fetch updates.
+    * takeout_id: 64-bit-number holding the identifier of the current takeout session.
+
+    Note that some of the numbers will only use 32 out of the 64 available bits.
+    However, for future-proofing reasons, we recommend you pretend they are 64-bit long.
+    """
+    __slots__ = ('user_id', 'dc_id', 'bot', 'pts', 'qts', 'date', 'seq', 'takeout_id')
+
+    def __init__(
+        self,
+        user_id: int,
+        dc_id: int,
+        bot: bool,
+        pts: int,
+        qts: int,
+        date: int,
+        seq: int,
+        takeout_id: Optional[int]
+    ):
+        self.user_id = user_id
+        self.dc_id = dc_id
+        self.bot = bot
+        self.pts = pts
+        self.qts = qts
+        self.date = date
+        self.seq = seq
+        self.takeout_id = takeout_id
+
+    def __repr__(self):
+        return repr({k: getattr(self, k) for k in self.__slots__})
+
+
+class ChannelState:
+    """
+    Stores the information needed to fetch updates from a channel.
+
+    * channel_id: 64-bit number representing the channel identifier.
+    * pts: 64-bit number holding the state needed to fetch updates.
+    """
+    __slots__ = ('channel_id', 'pts')
+
+    def __init__(
+        self,
+        channel_id: int,
+        pts: int,
+    ):
+        self.channel_id = channel_id
+        self.pts = pts
+
+    def __repr__(self):
+        return repr({k: getattr(self, k) for k in self.__slots__})
+
+
+class EntityType(IntEnum):
+    """
+    You can rely on the type value to be equal to the ASCII character one of:
+
+    * 'U' (85): this entity belongs to a :tl:`User` who is not a ``bot``.
+    * 'B' (66): this entity belongs to a :tl:`User` who is a ``bot``.
+    * 'G' (71): this entity belongs to a small group :tl:`Chat`.
+    * 'C' (67): this entity belongs to a standard broadcast :tl:`Channel`.
+    * 'M' (77): this entity belongs to a megagroup :tl:`Channel`.
+    * 'E' (69): this entity belongs to an "enormous" "gigagroup" :tl:`Channel`.
+    """
+    USER = ord('U')
+    BOT = ord('B')
+    GROUP = ord('G')
+    CHANNEL = ord('C')
+    MEGAGROUP = ord('M')
+    GIGAGROUP = ord('E')
+
+    def canonical(self):
+        """
+        Return the canonical version of this type.
+        """
+        return _canon_entity_types[self]
+
+
+_canon_entity_types = {
+    EntityType.USER: EntityType.USER,
+    EntityType.BOT: EntityType.USER,
+    EntityType.GROUP: EntityType.GROUP,
+    EntityType.CHANNEL: EntityType.CHANNEL,
+    EntityType.MEGAGROUP: EntityType.CHANNEL,
+    EntityType.GIGAGROUP: EntityType.CHANNEL,
+}
+
+
+class Entity:
+    """
+    Stores the information needed to use a certain user, chat or channel with the API.
+
+    * ty: 8-bit number indicating the type of the entity (of type `EntityType`).
+    * id: 64-bit number uniquely identifying the entity among those of the same type.
+    * hash: 64-bit signed number needed to use this entity with the API.
+
+    The string representation of this class is considered to be stable, for as long as
+    Telegram doesn't need to add more fields to the entities. It can also be converted
+    to bytes with ``bytes(entity)``, for a more compact representation.
+    """
+    __slots__ = ('ty', 'id', 'hash')
+
+    def __init__(
+        self,
+        ty: EntityType,
+        id: int,
+        hash: int
+    ):
+        self.ty = ty
+        self.id = id
+        self.hash = hash
+
+    @property
+    def is_user(self):
+        """
+        ``True`` if the entity is either a user or a bot.
+        """
+        return self.ty in (EntityType.USER, EntityType.BOT)
+
+    @property
+    def is_group(self):
+        """
+        ``True`` if the entity is a small group chat or `megagroup`_.
+
+        .. _megagroup: https://telegram.org/blog/supergroups5k
+        """
+        return self.ty in (EntityType.GROUP, EntityType.MEGAGROUP)
+
+    @property
+    def is_broadcast(self):
+        """
+        ``True`` if the entity is a broadcast channel or `broadcast group`_.
+
+        .. _broadcast group: https://telegram.org/blog/autodelete-inv2#groups-with-unlimited-members
+        """
+        return self.ty in (EntityType.CHANNEL, EntityType.GIGAGROUP)
+
+    @classmethod
+    def from_str(cls, string: str):
+        """
+        Convert the string into an `Entity`.
+        """
+        try:
+            ty, id, hash = string.split('.')
+            ty, id, hash = ord(ty), int(id), int(hash)
+        except AttributeError:
+            raise TypeError(f'expected str, got {string!r}') from None
+        except (TypeError, ValueError):
+            raise ValueError(f'malformed entity str (must be T.id.hash), got {string!r}') from None
+
+        return cls(EntityType(ty), id, hash)
+
+    @classmethod
+    def from_bytes(cls, blob):
+        """
+        Convert the bytes into an `Entity`.
+        """
+        try:
+            ty, id, hash = struct.unpack('<Bqq', blob)
+        except struct.error:
+            raise ValueError(f'malformed entity data, got {string!r}') from None
+
+        return cls(EntityType(ty), id, hash)
+
+    def __str__(self):
+        return f'{chr(self.ty)}.{self.id}.{self.hash}'
+
+    def __bytes__(self):
+        return struct.pack('<Bqq', self.ty, self.id, self.hash)
+
+    def _as_input_peer(self):
+        if self.is_user:
+            return InputPeerUser(self.id, self.hash)
+        elif self.ty == EntityType.GROUP:
+            return InputPeerChat(self.id)
+        else:
+            return InputPeerChannel(self.id, self.hash)
+
+    def __repr__(self):
+        return repr({k: getattr(self, k) for k in self.__slots__})

telethon/client/account.py

@@ -2,7 +2,7 @@ import functools
 import inspect
 import typing
 
-from .users import UserMethods, _NOT_A_REQUEST
+from .users import _NOT_A_REQUEST
 from .. import helpers, utils
 from ..tl import functions, TLRequest
 
@@ -107,7 +107,7 @@ class _TakeoutClient:
         return setattr(self.__client, name, value)
 
 
-class AccountMethods(UserMethods):
+class AccountMethods:
     def takeout(
             self: 'TelegramClient',
             finalize: bool = True,
@@ -134,8 +134,8 @@ class AccountMethods(UserMethods):
         to adjust the `wait_time` of methods like `client.iter_messages
         <telethon.client.messages.MessageMethods.iter_messages>`.
 
-        By default, all parameters are ``None``, and you need to enable those
-        you plan to use by setting them to either ``True`` or ``False``.
+        By default, all parameters are `None`, and you need to enable those
+        you plan to use by setting them to either `True` or `False`.
 
         You should ``except errors.TakeoutInitDelayError as e``, since this
         exception will raise depending on the condition of the session. You
@@ -144,10 +144,10 @@ class AccountMethods(UserMethods):
 
         There's also a `success` property available in the takeout proxy
         object, so from the `with` body you can set the boolean result that
-        will be sent back to Telegram. But if it's left ``None`` as by
+        will be sent back to Telegram. But if it's left `None` as by
         default, then the action is based on the `finalize` parameter. If
-        it's ``True`` then the takeout will be finished, and if no exception
-        occurred during it, then ``True`` will be considered as a result.
+        it's `True` then the takeout will be finished, and if no exception
+        occurred during it, then `True` will be considered as a result.
         Otherwise, the takeout will not be finished and its ID will be
         preserved for future usage as `client.session.takeout_id
         <telethon.sessions.abstract.Session.takeout_id>`.
@@ -158,26 +158,26 @@ class AccountMethods(UserMethods):
                 exit or not.
 
             contacts (`bool`):
-                Set to ``True`` if you plan on downloading contacts.
+                Set to `True` if you plan on downloading contacts.
 
             users (`bool`):
-                Set to ``True`` if you plan on downloading information
+                Set to `True` if you plan on downloading information
                 from users and their private conversations with you.
 
             chats (`bool`):
-                Set to ``True`` if you plan on downloading information
+                Set to `True` if you plan on downloading information
                 from small group chats, such as messages and media.
 
             megagroups (`bool`):
-                Set to ``True`` if you plan on downloading information
+                Set to `True` if you plan on downloading information
                 from megagroups (channels), such as messages and media.
 
             channels (`bool`):
-                Set to ``True`` if you plan on downloading information
+                Set to `True` if you plan on downloading information
                 from broadcast channels, such as messages and media.
 
             files (`bool`):
-                Set to ``True`` if you plan on downloading media and
+                Set to `True` if you plan on downloading media and
                 you don't only wish to export messages.
 
             max_file_size (`int`):
@@ -190,11 +190,11 @@ class AccountMethods(UserMethods):
                 from telethon import errors
 
                 try:
-                    with client.takeout() as takeout:
-                        client.get_messages('me')  # normal call
-                        takeout.get_messages('me')  # wrapped through takeout (less limits)
+                    async with client.takeout() as takeout:
+                        await client.get_messages('me')  # normal call
+                        await takeout.get_messages('me')  # wrapped through takeout (less limits)
 
-                        for message in takeout.iter_messages(chat, wait_time=0):
+                        async for message in takeout.iter_messages(chat, wait_time=0):
                             ...  # Do something with the message
 
                 except errors.TakeoutInitDelayError as e:
@@ -228,12 +228,12 @@ class AccountMethods(UserMethods):
                 Whether the takeout completed successfully or not.
 
         Returns
-            ``True`` if the operation was successful, ``False`` otherwise.
+            `True` if the operation was successful, `False` otherwise.
 
         Example
             .. code-block:: python
 
-                client.end_takeout(success=False)
+                await client.end_takeout(success=False)
         """
         try:
             async with _TakeoutClient(True, self, None) as takeout:

telethon/client/auth.py

@@ -3,24 +3,24 @@ import inspect
 import os
 import sys
 import typing
+import warnings
 
-from .messageparse import MessageParseMethods
-from .users import UserMethods
 from .. import utils, helpers, errors, password as pwd_mod
-from ..tl import types, functions
+from ..tl import types, functions, custom
+from .._updates import SessionState
 
 if typing.TYPE_CHECKING:
     from .telegramclient import TelegramClient
 
 
-class AuthMethods(MessageParseMethods, UserMethods):
+class AuthMethods:
 
     # region Public methods
 
     def start(
             self: 'TelegramClient',
-            phone: typing.Callable[[], str] = lambda: input('Please enter your phone (or bot token): '),
-            password: typing.Callable[[], str] = lambda: getpass.getpass('Please enter your password: '),
+            phone: typing.Union[typing.Callable[[], str], str] = lambda: input('Please enter your phone (or bot token): '),
+            password: typing.Union[typing.Callable[[], str], str] = lambda: getpass.getpass('Please enter your password: '),
             *,
             bot_token: str = None,
             force_sms: bool = False,
@@ -34,12 +34,6 @@ class AuthMethods(MessageParseMethods, UserMethods):
         By default, this method will be interactive (asking for
         user input if needed), and will handle 2FA if enabled too.
 
-        If the phone doesn't belong to an existing account (and will hence
-        `sign_up` for a new one),  **you are agreeing to Telegram's
-        Terms of Service. This is required and your account
-        will be banned otherwise.** See https://telegram.org/tos
-        and https://core.telegram.org/api/terms.
-
         If the event loop is already running, this method returns a
         coroutine that you should await on your own code; otherwise
         the loop is ran until said coroutine completes.
@@ -91,10 +85,10 @@ class AuthMethods(MessageParseMethods, UserMethods):
                 client = TelegramClient('anon', api_id, api_hash)
 
                 # Starting as a bot account
-                client.start(bot_token=bot_token)
+                await client.start(bot_token=bot_token)
 
-                # Starting as an user account
-                client.start(phone)
+                # Starting as a user account
+                await client.start(phone)
                 # Please enter the code you received: 12345
                 # Please enter your password: *******
                 # (You are now logged in)
@@ -135,12 +129,36 @@ class AuthMethods(MessageParseMethods, UserMethods):
         )
 
     async def _start(
-            self, phone, password, bot_token, force_sms,
+            self: 'TelegramClient', phone, password, bot_token, force_sms,
             code_callback, first_name, last_name, max_attempts):
         if not self.is_connected():
             await self.connect()
 
-        if await self.is_user_authorized():
+        # Rather than using `is_user_authorized`, use `get_me`. While this is
+        # more expensive and needs to retrieve more data from the server, it
+        # enables the library to warn users trying to login to a different
+        # account. See #1172.
+        me = await self.get_me()
+        if me is not None:
+            # The warnings here are on a best-effort and may fail.
+            if bot_token:
+                # bot_token's first part has the bot ID, but it may be invalid
+                # so don't try to parse as int (instead cast our ID to string).
+                if bot_token[:bot_token.find(':')] != str(me.id):
+                    warnings.warn(
+                        'the session already had an authorized user so it did '
+                        'not login to the bot account using the provided bot_token; '
+                        'if you were expecting a different user, check whether '
+                        'you are accidentally reusing an existing session'
+                    )
+            elif phone and not callable(phone) and utils.parse_phone(phone) != me.phone:
+                warnings.warn(
+                    'the session already had an authorized user so it did '
+                    'not login to the user account using the provided phone; '
+                    'if you were expecting a different user, check whether '
+                    'you are accidentally reusing an existing session'
+                )
+
             return self
 
         if not bot_token:
@@ -165,8 +183,7 @@ class AuthMethods(MessageParseMethods, UserMethods):
         attempts = 0
         two_step_detected = False
 
-        sent_code = await self.send_code_request(phone, force_sms=force_sms)
-        sign_up = not sent_code.phone_registered
+        await self.send_code_request(phone, force_sms=force_sms)
         while attempts < max_attempts:
             try:
                 value = code_callback()
@@ -179,19 +196,12 @@ class AuthMethods(MessageParseMethods, UserMethods):
                 if not value:
                     raise errors.PhoneCodeEmptyError(request=None)
 
-                if sign_up:
-                    me = await self.sign_up(value, first_name, last_name)
-                else:
-                    # Raises SessionPasswordNeededError if 2FA enabled
-                    me = await self.sign_in(phone, code=value)
+                # Raises SessionPasswordNeededError if 2FA enabled
+                me = await self.sign_in(phone, code=value)
                 break
             except errors.SessionPasswordNeededError:
                 two_step_detected = True
                 break
-            except errors.PhoneNumberOccupiedError:
-                sign_up = False
-            except errors.PhoneNumberUnoccupiedError:
-                sign_up = True
             except (errors.PhoneCodeEmptyError,
                     errors.PhoneCodeExpiredError,
                     errors.PhoneCodeHashEmptyError,
@@ -230,13 +240,14 @@ class AuthMethods(MessageParseMethods, UserMethods):
                 me = await self.sign_in(phone=phone, password=password)
 
         # We won't reach here if any step failed (exit by exception)
-        signed, name = 'Signed in successfully as', utils.get_display_name(me)
+        signed, name = 'Signed in successfully as ', utils.get_display_name(me)
+        tos = '; remember to not break the ToS or you will risk an account ban!'
         try:
-            print(signed, name)
+            print(signed, name, tos, sep='')
         except UnicodeEncodeError:
             # Some terminals don't support certain characters
             print(signed, name.encode('utf-8', errors='ignore')
-                              .decode('ascii', errors='ignore'))
+                              .decode('ascii', errors='ignore'), tos, sep='')
 
         return self
 
@@ -263,7 +274,7 @@ class AuthMethods(MessageParseMethods, UserMethods):
             *,
             password: str = None,
             bot_token: str = None,
-            phone_code_hash: str = None) -> 'types.User':
+            phone_code_hash: str = None) -> 'typing.Union[types.User, types.auth.SentCode]':
         """
         Logs in to Telegram to an existing user or bot account.
 
@@ -298,7 +309,7 @@ class AuthMethods(MessageParseMethods, UserMethods):
 
             phone_code_hash (`str`, optional):
                 The hash returned by `send_code_request`. This can be left as
-                ``None`` to use the last hash known for the phone to be used.
+                `None` to use the last hash known for the phone to be used.
 
         Returns
             The signed in user, or the information about
@@ -308,10 +319,10 @@ class AuthMethods(MessageParseMethods, UserMethods):
             .. code-block:: python
 
                 phone = '+34 123 123 123'
-                client.sign_in(phone)  # send code
+                await client.sign_in(phone)  # send code
 
                 code = input('enter code: ')
-                client.sign_in(phone, code)
+                await client.sign_in(phone, code)
         """
         me = await self.get_me()
         if me:
@@ -325,25 +336,37 @@ class AuthMethods(MessageParseMethods, UserMethods):
 
             # May raise PhoneCodeEmptyError, PhoneCodeExpiredError,
             # PhoneCodeHashEmptyError or PhoneCodeInvalidError.
-            result = await self(functions.auth.SignInRequest(
-                phone, phone_code_hash, str(code)))
+            request = functions.auth.SignInRequest(
+                phone, phone_code_hash, str(code)
+            )
         elif password:
             pwd = await self(functions.account.GetPasswordRequest())
-            result = await self(functions.auth.CheckPasswordRequest(
+            request = functions.auth.CheckPasswordRequest(
                 pwd_mod.compute_check(pwd, password)
-            ))
+            )
         elif bot_token:
-            result = await self(functions.auth.ImportBotAuthorizationRequest(
+            request = functions.auth.ImportBotAuthorizationRequest(
                 flags=0, bot_auth_token=bot_token,
                 api_id=self.api_id, api_hash=self.api_hash
-            ))
+            )
         else:
             raise ValueError(
                 'You must provide a phone and a code the first time, '
                 'and a password only if an RPCError was raised before.'
             )
 
-        return self._on_login(result.user)
+        try:
+            result = await self(request)
+        except errors.PhoneCodeExpiredError:
+            self._phone_code_hash.pop(phone, None)
+            raise
+
+        if isinstance(result, types.auth.AuthorizationSignUpRequired):
+            # Emulate pre-layer 104 behaviour
+            self._tos = result.terms_of_service
+            raise errors.PhoneNumberUnoccupiedError(request=request)
+
+        return await self._on_login(result.user)
 
     async def sign_up(
             self: 'TelegramClient',
@@ -354,93 +377,41 @@ class AuthMethods(MessageParseMethods, UserMethods):
             phone: str = None,
             phone_code_hash: str = None) -> 'types.User':
         """
-        Signs up to Telegram as a new user account.
-
-        Use this if you don't have an account yet.
-
-        You must call `send_code_request` first.
-
-        **By using this method you're agreeing to Telegram's
-        Terms of Service. This is required and your account
-        will be banned otherwise.** See https://telegram.org/tos
-        and https://core.telegram.org/api/terms.
-
-        Arguments
-            code (`str` | `int`):
-                The code sent by Telegram
-
-            first_name (`str`):
-                The first name to be used by the new account.
-
-            last_name (`str`, optional)
-                Optional last name.
-
-            phone (`str` | `int`, optional):
-                The phone to sign up. This will be the last phone used by
-                default (you normally don't need to set this).
-
-            phone_code_hash (`str`, optional):
-                The hash returned by `send_code_request`. This can be left as
-                ``None`` to use the last hash known for the phone to be used.
-
-        Returns
-            The new created :tl:`User`.
-
-        Example
-            .. code-block:: python
-
-                phone = '+34 123 123 123'
-                client.send_code_request(phone)
-
-                code = input('enter code: ')
-                client.sign_up(code, first_name='Anna', last_name='Banana')
+        This method can no longer be used, and will immediately raise a ``ValueError``.
+        See `issue #4050 <https://github.com/LonamiWebs/Telethon/issues/4050>`_ for context.
         """
-        me = await self.get_me()
-        if me:
-            return me
+        raise ValueError('Third-party applications cannot sign up for Telegram. See https://github.com/LonamiWebs/Telethon/issues/4050 for details')
 
-        if self._tos and self._tos.text:
-            if self.parse_mode:
-                t = self.parse_mode.unparse(self._tos.text, self._tos.entities)
-            else:
-                t = self._tos.text
-            sys.stderr.write("{}\n".format(t))
-            sys.stderr.flush()
-
-        phone, phone_code_hash = \
-            self._parse_phone_and_hash(phone, phone_code_hash)
-
-        result = await self(functions.auth.SignUpRequest(
-            phone_number=phone,
-            phone_code_hash=phone_code_hash,
-            phone_code=str(code),
-            first_name=first_name,
-            last_name=last_name
-        ))
-
-        if self._tos:
-            await self(
-                functions.help.AcceptTermsOfServiceRequest(self._tos.id))
-
-        return self._on_login(result.user)
-
-    def _on_login(self, user):
+    async def _on_login(self, user):
         """
         Callback called whenever the login or sign up process completes.
 
         Returns the input user parameter.
         """
-        self._bot = bool(user.bot)
-        self._self_input_peer = utils.get_input_peer(user, allow_self=False)
+        self._mb_entity_cache.set_self_user(user.id, user.bot, user.access_hash)
         self._authorized = True
 
+        state = await self(functions.updates.GetStateRequest())
+        # the server may send an old qts in getState
+        difference = await self(functions.updates.GetDifferenceRequest(pts=state.pts, date=state.date, qts=state.qts))
+
+        if isinstance(difference, types.updates.Difference):
+            state = difference.state
+        elif isinstance(difference, types.updates.DifferenceSlice):
+            state = difference.intermediate_state
+        elif isinstance(difference, types.updates.DifferenceTooLong):
+            state.pts = difference.pts
+
+        self._message_box.load(SessionState(0, 0, 0, state.pts, state.qts, int(state.date.timestamp()), state.seq, 0), [])
+
         return user
 
     async def send_code_request(
             self: 'TelegramClient',
             phone: str,
             *,
-            force_sms: bool = False) -> 'types.auth.SentCode':
+            force_sms: bool = False,
+            _retry_count: int = 0) -> 'types.auth.SentCode':
         """
         Sends the Telegram code needed to login to the given phone number.
 
@@ -449,7 +420,8 @@ class AuthMethods(MessageParseMethods, UserMethods):
                 The phone to which the code will be sent.
 
             force_sms (`bool`, optional):
-                Whether to force sending as SMS.
+                Whether to force sending as SMS. This has been deprecated.
+                See `issue #4050 <https://github.com/LonamiWebs/Telethon/issues/4050>`_ for context.
 
         Returns
             An instance of :tl:`SentCode`.
@@ -458,14 +430,13 @@ class AuthMethods(MessageParseMethods, UserMethods):
             .. code-block:: python
 
                 phone = '+34 123 123 123'
-                sent = client.send_code_request(phone)
+                sent = await client.send_code_request(phone)
                 print(sent)
-
-                if sent.phone_registered:
-                    print('This phone has an existing account registered')
-                else:
-                    print('This phone does not have an account registered')
         """
+        if force_sms:
+            warnings.warn('force_sms has been deprecated and no longer works')
+            force_sms = False
+
         result = None
         phone = utils.parse_phone(phone) or self._phone
         phone_hash = self._phone_code_hash.get(phone)
@@ -475,48 +446,114 @@ class AuthMethods(MessageParseMethods, UserMethods):
                 result = await self(functions.auth.SendCodeRequest(
                     phone, self.api_id, self.api_hash, types.CodeSettings()))
             except errors.AuthRestartError:
-                return await self.send_code_request(phone, force_sms=force_sms)
+                if _retry_count > 2:
+                    raise
+                return await self.send_code_request(
+                    phone, force_sms=force_sms, _retry_count=_retry_count+1)
 
-            self._tos = result.terms_of_service
-            self._phone_code_hash[phone] = phone_hash = result.phone_code_hash
+            # TODO figure out when/if/how this can happen
+            if isinstance(result, types.auth.SentCodeSuccess):
+                raise RuntimeError('logged in right after sending the code')
+
+            # If we already sent a SMS, do not resend the code (hash may be empty)
+            if isinstance(result.type, types.auth.SentCodeTypeSms):
+                force_sms = False
+
+            # phone_code_hash may be empty, if it is, do not save it (#1283)
+            if result.phone_code_hash:
+                self._phone_code_hash[phone] = phone_hash = result.phone_code_hash
         else:
             force_sms = True
 
         self._phone = phone
 
         if force_sms:
-            result = await self(
-                functions.auth.ResendCodeRequest(phone, phone_hash))
+            try:
+                result = await self(
+                    functions.auth.ResendCodeRequest(phone, phone_hash))
+            except errors.PhoneCodeExpiredError:
+                if _retry_count > 2:
+                    raise
+                self._phone_code_hash.pop(phone, None)
+                self._log[__name__].info(
+                    "Phone code expired in ResendCodeRequest, requesting a new code"
+                )
+                return await self.send_code_request(
+                    phone, force_sms=False, _retry_count=_retry_count+1)
+
+            if isinstance(result, types.auth.SentCodeSuccess):
+                raise RuntimeError('logged in right after resending the code')
 
             self._phone_code_hash[phone] = result.phone_code_hash
 
         return result
 
+    async def qr_login(self: 'TelegramClient', ignored_ids: typing.List[int] = None) -> custom.QRLogin:
+        """
+        Initiates the QR login procedure.
+
+        Note that you must be connected before invoking this, as with any
+        other request.
+
+        It is up to the caller to decide how to present the code to the user,
+        whether it's the URL, using the token bytes directly, or generating
+        a QR code and displaying it by other means.
+
+        See the documentation for `QRLogin` to see how to proceed after this.
+
+        Arguments
+            ignored_ids (List[`int`]):
+                List of already logged-in user IDs, to prevent logging in
+                twice with the same user.
+
+        Returns
+            An instance of `QRLogin`.
+
+        Example
+            .. code-block:: python
+
+                def display_url_as_qr(url):
+                    pass  # do whatever to show url as a qr to the user
+
+                qr_login = await client.qr_login()
+                display_url_as_qr(qr_login.url)
+
+                # Important! You need to wait for the login to complete!
+                await qr_login.wait()
+
+                # If you have 2FA enabled, `wait` will raise `telethon.errors.SessionPasswordNeededError`.
+                # You should except that error and call `sign_in` with the password if this happens.
+        """
+        qr_login = custom.QRLogin(self, ignored_ids or [])
+        await qr_login.recreate()
+        return qr_login
+
     async def log_out(self: 'TelegramClient') -> bool:
         """
         Logs out Telegram and deletes the current ``*.session`` file.
 
+        The client is unusable after logging out and a new instance should be created.
+
         Returns
-            ``True`` if the operation was successful.
+            `True` if the operation was successful.
 
         Example
             .. code-block:: python
 
                 # Note: you will need to login again!
-                client.log_out()
+                await client.log_out()
         """
         try:
             await self(functions.auth.LogOutRequest())
         except errors.RPCError:
             return False
 
-        self._bot = None
-        self._self_input_peer = None
+        self._mb_entity_cache.set_self_user(None, None, None)
         self._authorized = False
-        self._state_cache.reset()
 
         await self.disconnect()
         self.session.delete()
+        self.session = None
         return True
 
     async def edit_2fa(
@@ -548,7 +585,7 @@ class AuthMethods(MessageParseMethods, UserMethods):
             new_password (`str`, optional):
                 The password to set as 2FA.
                 If 2FA was already enabled, ``current_password`` **must** be set.
-                Leaving this blank or ``None`` will remove the password.
+                Leaving this blank or `None` will remove the password.
 
             hint (`str`, optional):
                 Hint to be displayed by Telegram when it asks for 2FA.
@@ -569,16 +606,16 @@ class AuthMethods(MessageParseMethods, UserMethods):
                 ``CodeInvalidError``.
 
         Returns
-            ``True`` if successful, ``False`` otherwise.
+            `True` if successful, `False` otherwise.
 
         Example
             .. code-block:: python
 
                 # Setting a password for your account which didn't have
-                client.edit_2fa(new_password='I_<3_Telethon')
+                await client.edit_2fa(new_password='I_<3_Telethon')
 
                 # Removing the password
-                client.edit_2fa(current_password='I_<3_Telethon')
+                await client.edit_2fa(current_password='I_<3_Telethon')
         """
         if new_password is None and current_password is None:
             return False

telethon/client/bots.py

@@ -1,6 +1,5 @@
 import typing
 
-from .users import UserMethods
 from .. import hints
 from ..tl import types, functions, custom
 
@@ -8,16 +7,17 @@ if typing.TYPE_CHECKING:
     from .telegramclient import TelegramClient
 
 
-class BotMethods(UserMethods):
+class BotMethods:
     async def inline_query(
             self: 'TelegramClient',
             bot: 'hints.EntityLike',
             query: str,
             *,
+            entity: 'hints.EntityLike' = None,
             offset: str = None,
             geo_point: 'types.GeoPoint' = None) -> custom.InlineResults:
         """
-        Makes an inline query to the specified bot (e.g. ``@vote New Poll``).
+        Makes an inline query to the specified bot (``@vote New Poll``).
 
         Arguments
             bot (`entity`):
@@ -26,6 +26,15 @@ class BotMethods(UserMethods):
             query (`str`):
                 The query that should be made to the bot.
 
+            entity (`entity`, optional):
+                The entity where the inline query is being made from. Certain
+                bots use this to display different results depending on where
+                it's used, such as private chats, groups or channels.
+
+                If specified, it will also be the default entity where the
+                message will be sent after clicked. Otherwise, the "empty
+                peer" will be used, which some bots may not handle correctly.
+
             offset (`str`, optional):
                 The string offset to use for the bot.
 
@@ -41,18 +50,23 @@ class BotMethods(UserMethods):
             .. code-block:: python
 
                 # Make an inline query to @like
-                results = client.inline_query('like', 'Do you like Telethon?')
+                results = await client.inline_query('like', 'Do you like Telethon?')
 
                 # Send the first result to some chat
-                message = results[0].click('TelethonOffTopic')
+                message = await results[0].click('TelethonOffTopic')
         """
         bot = await self.get_input_entity(bot)
+        if entity:
+            peer = await self.get_input_entity(entity)
+        else:
+            peer = types.InputPeerEmpty()
+
         result = await self(functions.messages.GetInlineBotResultsRequest(
             bot=bot,
-            peer=types.InputPeerEmpty(),
+            peer=peer,
             query=query,
             offset=offset or '',
             geo_point=geo_point
         ))
 
-        return custom.InlineResults(self, result)
+        return custom.InlineResults(self, result, entity=peer if entity else None)

telethon/client/buttons.py

@@ -1,14 +1,10 @@
 import typing
 
-from .updates import UpdateMethods
 from .. import utils, hints
 from ..tl import types, custom
 
-if typing.TYPE_CHECKING:
-    from .telegramclient import TelegramClient
 
-
-class ButtonMethods(UpdateMethods):
+class ButtonMethods:
     @staticmethod
     def build_reply_markup(
             buttons: 'typing.Optional[hints.MarkupLike]',
@@ -39,7 +35,8 @@ class ButtonMethods(UpdateMethods):
                 from telethon import Button
 
                 markup = client.build_reply_markup(Button.inline('hi'))
-                client.send_message('click me', buttons=markup)
+                # later
+                await client.send_message(chat, 'click me', buttons=markup)
         """
         if buttons is None:
             return None
@@ -52,7 +49,7 @@ class ButtonMethods(UpdateMethods):
 
         if not utils.is_list_like(buttons):
             buttons = [[buttons]]
-        elif not utils.is_list_like(buttons[0]):
+        elif not buttons or not utils.is_list_like(buttons[0]):
             buttons = [buttons]
 
         is_inline = False

telethon/client/dialogs.py

@@ -1,9 +1,9 @@
 import asyncio
+import inspect
 import itertools
 import typing
 
-from .users import UserMethods
-from .. import utils, hints
+from .. import helpers, utils, hints, errors
 from ..requestiter import RequestIter
 from ..tl import types, functions, custom
 
@@ -13,6 +13,17 @@ if typing.TYPE_CHECKING:
     from .telegramclient import TelegramClient
 
 
+def _dialog_message_key(peer, message_id):
+    """
+    Get the key to get messages from a dialog.
+
+    We cannot just use the message ID because channels share message IDs,
+    and the peer ID is required to distinguish between them. But it is not
+    necessary in small group chats and private chats.
+    """
+    return (peer.channel_id if isinstance(peer, types.PeerChannel) else None), message_id
+
+
 class _DialogsIter(RequestIter):
     async def _init(
             self, offset_date, offset_id, offset_peer, ignore_pinned, ignore_migrated, folder
@@ -44,35 +55,45 @@ class _DialogsIter(RequestIter):
         self.total = getattr(r, 'count', len(r.dialogs))
 
         entities = {utils.get_peer_id(x): x
-                    for x in itertools.chain(r.users, r.chats)}
+                    for x in itertools.chain(r.users, r.chats)
+                    if not isinstance(x, (types.UserEmpty, types.ChatEmpty))}
+
+        self.client._mb_entity_cache.extend(r.users, r.chats)
 
         messages = {}
         for m in r.messages:
             m._finish_init(self.client, entities, None)
-            messages[m.id] = m
+            messages[_dialog_message_key(m.peer_id, m.id)] = m
 
         for d in r.dialogs:
             # We check the offset date here because Telegram may ignore it
+            message = messages.get(_dialog_message_key(d.peer, d.top_message))
             if self.offset_date:
-                date = getattr(messages.get(
-                    d.top_message, None), 'date', None)
-
+                date = getattr(message, 'date', None)
                 if not date or date.timestamp() > self.offset_date.timestamp():
                     continue
 
             peer_id = utils.get_peer_id(d.peer)
             if peer_id not in self.seen:
                 self.seen.add(peer_id)
-                cd = custom.Dialog(self.client, d, entities, messages)
+                if peer_id not in entities:
+                    # > In which case can a UserEmpty appear in the list of banned members?
+                    # > In a very rare cases. This is possible but isn't an expected behavior.
+                    # Real world example: https://t.me/TelethonChat/271471
+                    continue
+
+                cd = custom.Dialog(self.client, d, entities, message)
                 if cd.dialog.pts:
-                    self.client._channel_pts[cd.id] = cd.dialog.pts
+                    self.client._message_box.try_set_channel_state(
+                        utils.get_peer_id(d.peer, add_mark=False), cd.dialog.pts)
 
                 if not self.ignore_migrated or getattr(
                         cd.entity, 'migrated_to', None) is None:
                     self.buffer.append(cd)
 
-        if len(r.dialogs) < self.request.limit\
+        if not self.buffer or len(r.dialogs) < self.request.limit\
                 or not isinstance(r, types.messages.DialogsSlice):
+            # Buffer being empty means all returned dialogs were skipped (due to offsets).
             # Less than we requested means we reached the end, or
             # we didn't get a DialogsSlice which means we got all.
             return True
@@ -81,30 +102,45 @@ class _DialogsIter(RequestIter):
         # Why? Because pinned dialogs will mess with the order
         # in this list. Instead, we find the last dialog which
         # has a message, and use it as an offset.
-        last_message = next((
-            messages[d.top_message]
+        last_message = next(filter(None, (
+            messages.get(_dialog_message_key(d.peer, d.top_message))
             for d in reversed(r.dialogs)
-            if d.top_message in messages
-        ), None)
+        )), None)
 
         self.request.exclude_pinned = True
         self.request.offset_id = last_message.id if last_message else 0
         self.request.offset_date = last_message.date if last_message else None
-        self.request.offset_peer =\
-            entities[utils.get_peer_id(r.dialogs[-1].peer)]
+        self.request.offset_peer = self.buffer[-1].input_entity
 
 
 class _DraftsIter(RequestIter):
-    async def _init(self, **kwargs):
-        r = await self.client(functions.messages.GetAllDraftsRequest())
-        self.buffer.extend(custom.Draft._from_update(self.client, u)
-                           for u in r.updates)
+    async def _init(self, entities, **kwargs):
+        if not entities:
+            r = await self.client(functions.messages.GetAllDraftsRequest())
+            items = r.updates
+        else:
+            peers = []
+            for entity in entities:
+                peers.append(types.InputDialogPeer(
+                    await self.client.get_input_entity(entity)))
+
+            r = await self.client(functions.messages.GetPeerDialogsRequest(peers))
+            items = r.dialogs
+
+        # TODO Maybe there should be a helper method for this?
+        entities = {utils.get_peer_id(x): x
+                    for x in itertools.chain(r.users, r.chats)}
+
+        self.buffer.extend(
+            custom.Draft(self.client, entities[utils.get_peer_id(d.peer)], d.draft)
+            for d in items
+        )
 
     async def _load_next_chunk(self):
         return []
 
 
-class DialogMethods(UserMethods):
+class DialogMethods:
 
     # region Public methods
 
@@ -123,10 +159,14 @@ class DialogMethods(UserMethods):
         """
         Iterator over the dialogs (open conversations/subscribed channels).
 
+        The order is the same as the one seen in official applications
+        (first pinned, them from those with the most recent message to
+        those with the oldest message).
+
         Arguments
             limit (`int` | `None`):
                 How many dialogs to be retrieved as maximum. Can be set to
-                ``None`` to retrieve all dialogs. Note that this may take
+                `None` to retrieve all dialogs. Note that this may take
                 whole minutes if you have hundreds of dialogs, as Telegram
                 will tell the library to slow down through a
                 ``FloodWaitError``.
@@ -142,12 +182,12 @@ class DialogMethods(UserMethods):
 
             ignore_pinned (`bool`, optional):
                 Whether pinned dialogs should be ignored or not.
-                When set to ``True``, these won't be yielded at all.
+                When set to `True`, these won't be yielded at all.
 
             ignore_migrated (`bool`, optional):
                 Whether :tl:`Chat` that have ``migrated_to`` a :tl:`Channel`
                 should be included or not. By default all the chats in your
-                dialogs are returned, but setting this to ``True`` will ignore
+                dialogs are returned, but setting this to `True` will ignore
                 (i.e. skip) them in the same way official applications do.
 
             folder (`int`, optional):
@@ -168,7 +208,7 @@ class DialogMethods(UserMethods):
 
             archived (`bool`, optional):
                 Alias for `folder`. If unspecified, all will be returned,
-                ``False`` implies ``folder=0`` and ``True`` implies ``folder=1``.
+                `False` implies ``folder=0`` and `True` implies ``folder=1``.
         Yields
             Instances of `Dialog <telethon.tl.custom.dialog.Dialog>`.
 
@@ -176,7 +216,7 @@ class DialogMethods(UserMethods):
             .. code-block:: python
 
                 # Print all dialog IDs and the title, nicely formatted
-                for dialog in client.iter_dialogs():
+                async for dialog in client.iter_dialogs():
                     print('{:>14}: {}'.format(dialog.id, dialog.title))
         """
         if archived is not None:
@@ -202,26 +242,38 @@ class DialogMethods(UserMethods):
             .. code-block:: python
 
                 # Get all open conversation, print the title of the first
-                dialogs = client.get_dialogs()
+                dialogs = await client.get_dialogs()
                 first = dialogs[0]
                 print(first.title)
 
                 # Use the dialog somewhere else
-                client.send_message(first, 'hi')
+                await client.send_message(first, 'hi')
 
                 # Getting only non-archived dialogs (both equivalent)
-                non_archived = client.get_dialogs(folder=0)
-                non_archived = client.get_dialogs(archived=False)
+                non_archived = await client.get_dialogs(folder=0)
+                non_archived = await client.get_dialogs(archived=False)
 
                 # Getting only archived dialogs (both equivalent)
-                archived = client.get_dialogs(folder=1)
-                non_archived = client.get_dialogs(archived=True)
+                archived = await client.get_dialogs(folder=1)
+                archived = await client.get_dialogs(archived=True)
         """
         return await self.iter_dialogs(*args, **kwargs).collect()
 
-    def iter_drafts(self: 'TelegramClient') -> _DraftsIter:
+    get_dialogs.__signature__ = inspect.signature(iter_dialogs)
+
+    def iter_drafts(
+            self: 'TelegramClient',
+            entity: 'hints.EntitiesLike' = None
+    ) -> _DraftsIter:
         """
-        Iterator over all open draft messages.
+        Iterator over draft messages.
+
+        The order is unspecified.
+
+        Arguments
+            entity (`hints.EntitiesLike`, optional):
+                The entity or entities for which to fetch the draft messages.
+                If left unspecified, all draft messages will be returned.
 
         Yields
             Instances of `Draft <telethon.tl.custom.draft.Draft>`.
@@ -230,13 +282,23 @@ class DialogMethods(UserMethods):
             .. code-block:: python
 
                 # Clear all drafts
-                for draft in client.get_drafts():
-                    draft.delete()
-        """
-        # TODO Passing a limit here makes no sense
-        return _DraftsIter(self, None)
+                async for draft in client.get_drafts():
+                    await draft.delete()
 
-    async def get_drafts(self: 'TelegramClient') -> 'hints.TotalList':
+                # Getting the drafts with 'bot1' and 'bot2'
+                async for draft in client.iter_drafts(['bot1', 'bot2']):
+                    print(draft.text)
+        """
+        if entity and not utils.is_list_like(entity):
+            entity = (entity,)
+
+        # TODO Passing a limit here makes no sense
+        return _DraftsIter(self, None, entities=entity)
+
+    async def get_drafts(
+            self: 'TelegramClient',
+            entity: 'hints.EntitiesLike' = None
+    ) -> 'hints.TotalList':
         """
         Same as `iter_drafts()`, but returns a list instead.
 
@@ -244,10 +306,18 @@ class DialogMethods(UserMethods):
             .. code-block:: python
 
                 # Get drafts, print the text of the first
-                drafts = client.get_drafts()
+                drafts = await client.get_drafts()
                 print(drafts[0].text)
+
+                # Get the draft in your chat
+                draft = await client.get_drafts('me')
+                print(drafts.text)
         """
-        return await self.iter_drafts().collect()
+        items = await self.iter_drafts(entity).collect()
+        if not entity or utils.is_list_like(entity):
+            return items
+        else:
+            return items[0]
 
     async def edit_folder(
             self: 'TelegramClient',
@@ -293,18 +363,18 @@ class DialogMethods(UserMethods):
             .. code-block:: python
 
                 # Archiving the first 5 dialogs
-                dialogs = client.get_dialogs(5)
-                client.edit_folder(dialogs, 1)
+                dialogs = await client.get_dialogs(5)
+                await client.edit_folder(dialogs, 1)
 
                 # Un-archiving the third dialog (archiving to folder 0)
-                client.edit_folder(dialog[2], 0)
+                await client.edit_folder(dialog[2], 0)
 
                 # Moving the first dialog to folder 0 and the second to 1
-                dialogs = client.get_dialogs(2)
-                client.edit_folder(dialogs, [0, 1])
+                dialogs = await client.get_dialogs(2)
+                await client.edit_folder(dialogs, [0, 1])
 
                 # Un-archiving all dialogs
-                client.archive(unpack=1)
+                await client.edit_folder(unpack=1)
         """
         if (entity is None) == (unpack is None):
             raise ValueError('You can only set either entities or unpack, not both')
@@ -318,7 +388,7 @@ class DialogMethods(UserMethods):
             entities = [await self.get_input_entity(entity)]
         else:
             entities = await asyncio.gather(
-                *(self.get_input_entity(x) for x in entity), loop=self.loop)
+                *(self.get_input_entity(x) for x in entity))
 
         if folder is None:
             raise ValueError('You must specify a folder')
@@ -341,6 +411,10 @@ class DialogMethods(UserMethods):
         """
         Deletes a dialog (leaves a chat or channel).
 
+        This method can be used as a user and as a bot. However,
+        bots will only be able to use it to leave groups and channels
+        (trying to delete a private conversation will do nothing).
+
         See also `Dialog.delete() <telethon.tl.custom.dialog.Dialog.delete>`.
 
         Arguments
@@ -351,8 +425,11 @@ class DialogMethods(UserMethods):
 
             revoke (`bool`, optional):
                 On private chats, you may revoke the messages from
-                the other peer too. By default, it's ``False``. Set
-                it to ``True`` to delete the history for both.
+                the other peer too. By default, it's `False`. Set
+                it to `True` to delete the history for both.
+
+                This makes no difference for bot accounts, who can
+                only leave groups and channels.
 
         Returns
             The :tl:`Updates` object that the request produces,
@@ -362,23 +439,38 @@ class DialogMethods(UserMethods):
             .. code-block:: python
 
                 # Deleting the first dialog
-                dialogs = client.get_dialogs(5)
-                client.delete_dialog(dialogs[0])
+                dialogs = await client.get_dialogs(5)
+                await client.delete_dialog(dialogs[0])
 
                 # Leaving a channel by username
-                client.delete_dialog('username')
+                await client.delete_dialog('username')
         """
+        # If we have enough information (`Dialog.delete` gives it to us),
+        # then we know we don't have to kick ourselves in deactivated chats.
+        if isinstance(entity, types.Chat):
+            deactivated = entity.deactivated
+        else:
+            deactivated = False
+
         entity = await self.get_input_entity(entity)
-        if isinstance(entity, types.InputPeerChannel):
+        ty = helpers._entity_type(entity)
+        if ty == helpers._EntityType.CHANNEL:
             return await self(functions.channels.LeaveChannelRequest(entity))
 
-        if isinstance(entity, types.InputPeerChat):
-            result = await self(functions.messages.DeleteChatUserRequest(
-                entity.chat_id, types.InputUserSelf()))
+        if ty == helpers._EntityType.CHAT and not deactivated:
+            try:
+                result = await self(functions.messages.DeleteChatUserRequest(
+                    entity.chat_id, types.InputUserSelf(), revoke_history=revoke
+                ))
+            except errors.PeerIdInvalidError:
+                # Happens if we didn't have the deactivated information
+                result = None
         else:
             result = None
 
-        await self(functions.messages.DeleteHistoryRequest(entity, 0, revoke=revoke))
+        if not await self.is_bot():
+            await self(functions.messages.DeleteHistoryRequest(entity, 0, revoke=revoke))
+
         return result
 
     def conversation(
@@ -394,6 +486,16 @@ class DialogMethods(UserMethods):
         Creates a `Conversation <telethon.tl.custom.conversation.Conversation>`
         with the given entity.
 
+        .. note::
+
+            This Conversation API has certain shortcomings, such as lacking
+            persistence, poor interaction with other event handlers, and
+            overcomplicated usage for anything beyond the simplest case.
+
+            If you plan to interact with a bot without handlers, this works
+            fine, but when running a bot yourself, you may instead prefer
+            to follow the advice from https://stackoverflow.com/a/62246569/.
+
         This is not the same as just sending a message to create a "dialog"
         with them, but rather a way to easily send messages and await for
         responses or other reactions. Refer to its documentation for more.
@@ -488,10 +590,10 @@ class DialogMethods(UserMethods):
                         # <you> Your name didn't have any letters! Try again
                         conv.send_message("Your name didn't have any letters! Try again")
 
-                        # <usr> Lonami
+                        # <usr> Human
                         name = conv.get_response().raw_text
 
-                    # <you> Thanks Lonami!
+                    # <you> Thanks Human!
                     conv.send_message('Thanks {}!'.format(name))
         """
         return custom.Conversation(

telethon/client/downloads.py

@@ -3,8 +3,11 @@ import io
 import os
 import pathlib
 import typing
+import inspect
+import asyncio
+
+from ..crypto import AES
 
-from .users import UserMethods
 from .. import utils, helpers, errors, hints
 from ..requestiter import RequestIter
 from ..tl import TLObject, types, functions
@@ -17,25 +20,38 @@ except ImportError:
 if typing.TYPE_CHECKING:
     from .telegramclient import TelegramClient
 
-
 # Chunk sizes for upload.getFile must be multiples of the smallest size
 MIN_CHUNK_SIZE = 4096
 MAX_CHUNK_SIZE = 512 * 1024
 
+# 2021-01-15, users reported that `errors.TimeoutError` can occur while downloading files.
+TIMED_OUT_SLEEP = 1
+
+
+class _CdnRedirect(Exception):
+    def __init__(self, cdn_redirect=None):
+        self.cdn_redirect = cdn_redirect
+      
   
 class _DirectDownloadIter(RequestIter):
     async def _init(
-            self, file, dc_id, offset, stride, chunk_size, request_size, file_size
-    ):
+            self, file, dc_id, offset, stride, chunk_size, request_size, file_size, msg_data, cdn_redirect=None):
         self.request = functions.upload.GetFileRequest(
             file, offset=offset, limit=request_size) 
+        self._client = self.client
+        self._cdn_redirect = cdn_redirect
+        if cdn_redirect is not None:
+          self.request = functions.upload.GetCdnFileRequest(cdn_redirect.file_token, offset=offset, limit=request_size)
+          self._client = await self.client._get_cdn_client(cdn_redirect)
         
         self.total = file_size
         self._stride = stride
         self._chunk_size = chunk_size
         self._last_part = None
+        self._msg_data = msg_data
+        self._timed_out = False
         
-        self._exported = dc_id and self.client.session.dc_id != dc_id
+        self._exported = dc_id and self._client.session.dc_id != dc_id
         if not self._exported:
             # The used sender will also change if ``FileMigrateError`` occurs
             self._sender = self.client._sender
@@ -67,18 +83,58 @@ class _DirectDownloadIter(RequestIter):
 
     async def _request(self):
         try:
-            result = await self._sender.send(self.request)
+            result = await self._client._call(self._sender, self.request)
+            self._timed_out = False
             if isinstance(result, types.upload.FileCdnRedirect):
-                raise NotImplementedError  # TODO Implement
+                if self.client._mb_entity_cache.self_bot:
+                    raise ValueError('FileCdnRedirect but the GetCdnFileRequest API access for bot users is restricted. Try to change api_id to avoid FileCdnRedirect')
+                raise _CdnRedirect(result)
+            if isinstance(result, types.upload.CdnFileReuploadNeeded):
+                await self.client._call(self.client._sender, functions.upload.ReuploadCdnFileRequest(file_token=self._cdn_redirect.file_token, request_token=result.request_token))
+                result = await self._client._call(self._sender, self.request)
+                return result.bytes
             else:
                 return result.bytes
 
+        except errors.TimedOutError as e:
+            if self._timed_out:
+                self.client._log[__name__].warning('Got two timeouts in a row while downloading file')
+                raise
+
+            self._timed_out = True
+            self.client._log[__name__].info('Got timeout while downloading file, retrying once')
+            await asyncio.sleep(TIMED_OUT_SLEEP)
+            return await self._request()
+
         except errors.FileMigrateError as e:
             self.client._log[__name__].info('File lives in another DC')
             self._sender = await self.client._borrow_exported_sender(e.new_dc)
             self._exported = True
             return await self._request()
 
+        except (errors.FilerefUpgradeNeededError, errors.FileReferenceExpiredError) as e:
+            # Only implemented for documents which are the ones that may take that long to download
+            if not self._msg_data \
+                    or not isinstance(self.request.location, types.InputDocumentFileLocation) \
+                    or self.request.location.thumb_size != '':
+                raise
+
+            self.client._log[__name__].info('File ref expired during download; refetching message')
+            chat, msg_id = self._msg_data
+            msg = await self.client.get_messages(chat, ids=msg_id)
+
+            if not isinstance(msg.media, types.MessageMediaDocument):
+                raise
+
+            document = msg.media.document
+
+            # Message media may have been edited for something else
+            if document.id != self.request.location.id:
+                raise
+
+            self.request.location.file_reference = document.file_reference
+            return await self._request()
+
     async def close(self):
         if not self._sender:
             return
@@ -92,7 +148,7 @@ class _DirectDownloadIter(RequestIter):
             self._sender = None
 
     async def __aenter__(self):
-        pass
+        return self
 
     async def __aexit__(self, *args):
         await self.close()
@@ -102,12 +158,12 @@ class _DirectDownloadIter(RequestIter):
 
 
 class _GenericDownloadIter(_DirectDownloadIter):
-    async def _load_next_chunk(self, mask=MIN_CHUNK_SIZE - 1):
+    async def _load_next_chunk(self):
         # 1. Fetch enough for one chunk
         data = b''
 
         # 1.1. ``bad`` is how much into the data we have we need to offset
-        bad = self.request.offset & mask
+        bad = self.request.offset % self.request.limit
         before = self.request.offset
 
         # 1.2. We have to fetch from a valid offset, so remove that bad part
@@ -125,7 +181,7 @@ class _GenericDownloadIter(_DirectDownloadIter):
         self.request.offset = before
 
         # 2. Fill the buffer with the data we have
-        # 2.1. Slicing ``bytes`` is expensive, yield ``memoryview`` instead
+        # 2.1. Slicing `bytes` is expensive, yield `memoryview` instead
         mem = memoryview(data)
 
         # 2.2. The current chunk starts at ``bad`` offset into the data,
@@ -152,7 +208,7 @@ class _GenericDownloadIter(_DirectDownloadIter):
             self.request.offset -= self._stride
 
 
-class DownloadMethods(UserMethods):
+class DownloadMethods:
 
     # region Public methods
 
@@ -183,20 +239,21 @@ class DownloadMethods(UserMethods):
                 The output file path, directory, or stream-like object.
                 If the path exists and is a file, it will be overwritten.
                 If file is the type `bytes`, it will be downloaded in-memory
-                as a bytestring (e.g. ``file=bytes``).
+                and returned as a bytestring (i.e. ``file=bytes``, without
+                parentheses or quotes).
 
             download_big (`bool`, optional):
                 Whether to use the big version of the available photos.
 
         Returns
-            ``None`` if no photo was provided, or if it was Empty. On success
+            `None` if no photo was provided, or if it was Empty. On success
             the file path is returned since it may differ from the one given.
 
         Example
             .. code-block:: python
 
                 # Download your own profile photo
-                path = client.download_profile_photo('me')
+                path = await client.download_profile_photo('me')
                 print(path)
         """
         # hex(crc32(x.encode('ascii'))) for x in
@@ -231,11 +288,11 @@ class DownloadMethods(UserMethods):
 
         if isinstance(photo, (types.UserProfilePhoto, types.ChatPhoto)):
             dc_id = photo.dc_id
-            which = photo.photo_big if download_big else photo.photo_small
             loc = types.InputPeerPhotoFileLocation(
-                peer=await self.get_input_entity(entity),
-                local_id=which.local_id,
-                volume_id=which.volume_id,
+                # min users can be used to download profile photos
+                # self.get_input_entity would otherwise not accept those
+                peer=utils.get_input_peer(entity, check_hash=False),
+                photo_id=photo.photo_id,
                 big=download_big
             )
         else:
@@ -257,7 +314,8 @@ class DownloadMethods(UserMethods):
             # See issue #500, Android app fails as of v4.6.0 (1155).
             # The fix seems to be using the full channel chat photo.
             ie = await self.get_input_entity(entity)
-            if isinstance(ie, types.InputPeerChannel):
+            ty = helpers._entity_type(ie)
+            if ty == helpers._EntityType.CHANNEL:
                 full = await self(functions.channels.GetFullChannelRequest(ie))
                 return await self._download_photo(
                     full.full_chat.chat_photo, file,
@@ -273,7 +331,7 @@ class DownloadMethods(UserMethods):
             message: 'hints.MessageLike',
             file: 'hints.FileLike' = None,
             *,
-            thumb: 'hints.FileLike' = None,
+            thumb: 'typing.Union[int, types.TypePhotoSize]' = None,
             progress_callback: 'hints.ProgressCallback' = None) -> typing.Optional[typing.Union[str, bytes]]:
         """
         Downloads the given media from a message object.
@@ -292,7 +350,8 @@ class DownloadMethods(UserMethods):
                 The output file path, directory, or stream-like object.
                 If the path exists and is a file, it will be overwritten.
                 If file is the type `bytes`, it will be downloaded in-memory
-                as a bytestring (e.g. ``file=bytes``).
+                and returned as a bytestring (i.e. ``file=bytes``, without
+                parentheses or quotes).
 
             progress_callback (`callable`, optional):
                 A callback function accepting two parameters:
@@ -303,35 +362,58 @@ class DownloadMethods(UserMethods):
                 instead of downloading the document or photo itself.
 
                 If it's specified but the file does not have a thumbnail,
-                this method will return ``None``.
+                this method will return `None`.
 
                 The parameter should be an integer index between ``0`` and
                 ``len(sizes)``. ``0`` will download the smallest thumbnail,
                 and ``len(sizes) - 1`` will download the largest thumbnail.
-                You can also use negative indices.
+                You can also use negative indices, which work the same as
+                they do in Python's `list`.
 
                 You can also pass the :tl:`PhotoSize` instance to use.
+                Alternatively, the thumb size type `str` may be used.
 
                 In short, use ``thumb=0`` if you want the smallest thumbnail
                 and ``thumb=-1`` if you want the largest thumbnail.
 
+                .. note::
+                    The largest thumbnail may be a video instead of a photo,
+                    as they are available since layer 116 and are bigger than
+                    any of the photos.
+
         Returns
-            ``None`` if no media was provided, or if it was Empty. On success
+            `None` if no media was provided, or if it was Empty. On success
             the file path is returned since it may differ from the one given.
 
         Example
             .. code-block:: python
 
-                path = client.download_media(message)
-                client.download_media(message, filename)
+                path = await client.download_media(message)
+                await client.download_media(message, filename)
                 # or
-                path = message.download_media()
-                message.download_media(filename)
+                path = await message.download_media()
+                await message.download_media(filename)
+
+                # Downloading to memory
+                blob = await client.download_media(message, bytes)
+
+                # Printing download progress
+                def callback(current, total):
+                    print('Downloaded', current, 'out of', total,
+                          'bytes: {:.2%}'.format(current / total))
+
+                await client.download_media(message, progress_callback=callback)
         """
+        # Downloading large documents may be slow enough to require a new file reference
+        # to be obtained mid-download. Store (input chat, message id) so that the message
+        # can be re-fetched.
+        msg_data = None
+
         # TODO This won't work for messageService
         if isinstance(message, types.Message):
             date = message.date
             media = message.media
+            msg_data = (message.input_chat, message.id) if message.input_chat else None
         else:
             date = datetime.datetime.now()
             media = message
@@ -339,6 +421,11 @@ class DownloadMethods(UserMethods):
         if isinstance(media, str):
             media = utils.resolve_bot_file_id(media)
 
+        if isinstance(media, types.MessageService):
+            if isinstance(message.action,
+                          types.MessageActionChatEditPhoto):
+                media = media.photo
+
         if isinstance(media, types.MessageMediaWebPage):
             if isinstance(media.webpage, types.WebPage):
                 media = media.webpage.document or media.webpage.photo
@@ -349,7 +436,7 @@ class DownloadMethods(UserMethods):
             )
         elif isinstance(media, (types.MessageMediaDocument, types.Document)):
             return await self._download_document(
-                media, file, date, thumb, progress_callback
+                media, file, date, thumb, progress_callback, msg_data
             )
         elif isinstance(media, types.MessageMediaContact) and thumb is None:
             return self._download_contact(
@@ -368,10 +455,17 @@ class DownloadMethods(UserMethods):
             part_size_kb: float = None,
             file_size: int = None,
             progress_callback: 'hints.ProgressCallback' = None,
-            dc_id: int = None) -> typing.Optional[bytes]:
+            dc_id: int = None,
+            key: bytes = None,
+            iv: bytes = None) -> typing.Optional[bytes]:
         """
         Low-level method to download files from their input location.
 
+        .. note::
+
+            Generally, you should instead use `download_media`.
+            This method is intended to be a bit more low-level.
+
         Arguments
             input_location (:tl:`InputFileLocation`):
                 The file location from which the file will be downloaded.
@@ -382,7 +476,7 @@ class DownloadMethods(UserMethods):
                 The output file path, directory, or stream-like object.
                 If the path exists and is a file, it will be overwritten.
 
-                If the file path is ``None`` or ``bytes``, then the result
+                If the file path is `None` or `bytes`, then the result
                 will be saved in memory and returned as `bytes`.
 
             part_size_kb (`int`, optional):
@@ -402,13 +496,45 @@ class DownloadMethods(UserMethods):
                 The data center the library should connect to in order
                 to download the file. You shouldn't worry about this.
 
+            key ('bytes', optional):
+                In case of an encrypted upload (secret chats) a key is supplied
+
+            iv ('bytes', optional):
+                In case of an encrypted upload (secret chats) an iv is supplied
+
+
         Example
             .. code-block:: python
 
                 # Download a file and print its header
-                data = client.download_file(input_file, bytes)
+                data = await client.download_file(input_file, bytes)
                 print(data[:16])
         """
+        return await self._download_file(
+            input_location,
+            file,
+            part_size_kb=part_size_kb,
+            file_size=file_size,
+            progress_callback=progress_callback,
+            dc_id=dc_id,
+            key=key,
+            iv=iv,
+        )
+
+    async def _download_file(
+            self: 'TelegramClient',
+            input_location: 'hints.FileLike',
+            file: 'hints.OutFileLike' = None,
+            *,
+            part_size_kb: float = None,
+            file_size: int = None,
+            progress_callback: 'hints.ProgressCallback' = None,
+            dc_id: int = None,
+            key: bytes = None,
+            iv: bytes = None,
+            msg_data: tuple = None,
+            cdn_redirect: types.upload.FileCdnRedirect = None
+    ) -> typing.Optional[bytes]:
         if not part_size_kb:
             if not file_size:
                 part_size_kb = 64  # Reasonable default
@@ -420,6 +546,9 @@ class DownloadMethods(UserMethods):
             raise ValueError(
                 'The part size must be evenly divisible by 4096.')
 
+        if isinstance(file, pathlib.Path):
+            file = str(file.absolute())
+
         in_memory = file is None or file is bytes
         if in_memory:
             f = io.BytesIO()
@@ -431,15 +560,39 @@ class DownloadMethods(UserMethods):
             f = file
 
         try:
-            async for chunk in self.iter_download(
-                    input_location, request_size=part_size, dc_id=dc_id):
-                f.write(chunk)
-                if progress_callback:
-                    progress_callback(f.tell(), file_size)
+            async for chunk in self._iter_download(
+                    input_location, request_size=part_size, dc_id=dc_id, msg_data=msg_data, cdn_redirect=cdn_redirect):
+                if iv and key:
+                    chunk = AES.decrypt_ige(chunk, key, iv)
+                r = f.write(chunk)
+                if inspect.isawaitable(r):
+                    await r
+
+                if progress_callback:
+                    r = progress_callback(f.tell(), file_size)
+                    if inspect.isawaitable(r):
+                        await r
+
+            # Not all IO objects have flush (see #1227)
+            if callable(getattr(f, 'flush', None)):
+                f.flush()
 
-            f.flush()
             if in_memory:
                 return f.getvalue()
+        except _CdnRedirect as e:
+          self._log[__name__].info('FileCdnRedirect to CDN data center %s', e.cdn_redirect.dc_id)
+          return await self._download_file(
+              input_location=input_location,
+              file=file,
+              part_size_kb=part_size_kb,
+              file_size=file_size,
+              progress_callback=progress_callback,
+              dc_id=e.cdn_redirect.dc_id,
+              key=e.cdn_redirect.encryption_key,
+              iv=e.cdn_redirect.encryption_iv,
+              msg_data=msg_data,
+              cdn_redirect=e.cdn_redirect
+          )
         finally:
             if isinstance(file, str) or in_memory:
                 f.close()
@@ -517,8 +670,8 @@ class DownloadMethods(UserMethods):
 
         Yields
 
-            ``bytes`` objects representing the chunks of the file if the
-            right conditions are met, or ``memoryview`` objects instead.
+            `bytes` objects representing the chunks of the file if the
+            right conditions are met, or `memoryview` objects instead.
 
         Example
             .. code-block:: python
@@ -526,23 +679,53 @@ class DownloadMethods(UserMethods):
                 # Streaming `media` to an output file
                 # After the iteration ends, the sender is cleaned up
                 with open('photo.jpg', 'wb') as fd:
-                    for chunk client.iter_download(media):
+                    async for chunk in client.iter_download(media):
                         fd.write(chunk)
 
                 # Fetching only the header of a file (32 bytes)
                 # You should manually close the iterator in this case.
+                #
+                # "stream" is a common name for asynchronous generators,
+                # and iter_download will yield `bytes` (chunks of the file).
                 stream = client.iter_download(media, request_size=32)
-                header = next(stream)
-                stream.close()
+                header = await stream.__anext__()  # "manual" version of `async for`
+                await stream.close()
                 assert len(header) == 32
-
-                # Fetching only the header, inside of an ``async def``
-                async def main():
-                    stream = client.iter_download(media, request_size=32)
-                    header = await stream.__anext__()
-                    await stream.close()
-                    assert len(header) == 32
         """
+        return self._iter_download(
+            file,
+            offset=offset,
+            stride=stride,
+            limit=limit,
+            chunk_size=chunk_size,
+            request_size=request_size,
+            file_size=file_size,
+            dc_id=dc_id,
+        )
+
+    def _iter_download(
+            self: 'TelegramClient',
+            file: 'hints.FileLike',
+            *,
+            offset: int = 0,
+            stride: int = None,
+            limit: int = None,
+            chunk_size: int = None,
+            request_size: int = MAX_CHUNK_SIZE,
+            file_size: int = None,
+            dc_id: int = None,
+            msg_data: tuple = None,
+            cdn_redirect: types.upload.FileCdnRedirect = None
+    ):
+        info = utils._get_file_info(file)
+        if info.dc_id is not None:
+            dc_id = info.dc_id
+
+        if file_size is None:
+            file_size = info.size
+
+        file = info.location
+
         if chunk_size is None:
             chunk_size = request_size
 
@@ -560,14 +743,10 @@ class DownloadMethods(UserMethods):
         elif request_size > MAX_CHUNK_SIZE:
             request_size = MAX_CHUNK_SIZE
 
-        old_dc = dc_id
-        dc_id, file = utils.get_input_location(file)
-        if dc_id is None:
-            dc_id = old_dc
-
         if chunk_size == request_size \
                 and offset % MIN_CHUNK_SIZE == 0 \
-                and stride % MIN_CHUNK_SIZE == 0:
+                and stride % MIN_CHUNK_SIZE == 0 \
+                and (limit is None or offset % limit == 0):
             cls = _DirectDownloadIter
             self._log[__name__].info('Starting direct file download in chunks of '
                                      '%d at %d, stride %d', request_size, offset, stride)
@@ -585,7 +764,9 @@ class DownloadMethods(UserMethods):
             stride=stride,
             chunk_size=chunk_size,
             request_size=request_size,
-            file_size=file_size
+            file_size=file_size,
+            msg_data=msg_data,
+            cdn_redirect=cdn_redirect
         )
 
     # endregion
@@ -594,12 +775,44 @@ class DownloadMethods(UserMethods):
 
     @staticmethod
     def _get_thumb(thumbs, thumb):
+        if not thumbs:
+            return None
+
+        # Seems Telegram has changed the order and put `PhotoStrippedSize`
+        # last while this is the smallest (layer 116). Ensure we have the
+        # sizes sorted correctly with a custom function.
+        def sort_thumbs(thumb):
+            if isinstance(thumb, types.PhotoStrippedSize):
+                return 1, len(thumb.bytes)
+            if isinstance(thumb, types.PhotoCachedSize):
+                return 1, len(thumb.bytes)
+            if isinstance(thumb, types.PhotoSize):
+                return 1, thumb.size
+            if isinstance(thumb, types.PhotoSizeProgressive):
+                return 1, max(thumb.sizes)
+            if isinstance(thumb, types.VideoSize):
+                return 2, thumb.size
+
+            # Empty size or invalid should go last
+            return 0, 0
+
+        thumbs = list(sorted(thumbs, key=sort_thumbs))
+
+        for i in reversed(range(len(thumbs))):
+            # :tl:`PhotoPathSize` is used for animated stickers preview, and the thumb is actually
+            # a SVG path of the outline. Users expect thumbnails to be JPEG files, so pretend this
+            # thumb size doesn't actually exist (#1655).
+            if isinstance(thumbs[i], types.PhotoPathSize):
+                thumbs.pop(i)
+
         if thumb is None:
             return thumbs[-1]
         elif isinstance(thumb, int):
             return thumbs[thumb]
+        elif isinstance(thumb, str):
+            return next((t for t in thumbs if t.type == thumb), None)
         elif isinstance(thumb, (types.PhotoSize, types.PhotoCachedSize,
-                                types.PhotoStrippedSize)):
+                                types.PhotoStrippedSize, types.VideoSize)):
             return thumb
         else:
             return None
@@ -634,14 +847,24 @@ class DownloadMethods(UserMethods):
         if not isinstance(photo, types.Photo):
             return
 
-        size = self._get_thumb(photo.sizes, thumb)
+        # Include video sizes here (but they may be None so provide an empty list)
+        size = self._get_thumb(photo.sizes + (photo.video_sizes or []), thumb)
         if not size or isinstance(size, types.PhotoSizeEmpty):
             return
 
-        file = self._get_proper_filename(file, 'photo', '.jpg', date=date)
+        if isinstance(size, types.VideoSize):
+            file = self._get_proper_filename(file, 'video', '.mp4', date=date)
+        else:
+            file = self._get_proper_filename(file, 'photo', '.jpg', date=date)
+
         if isinstance(size, (types.PhotoCachedSize, types.PhotoStrippedSize)):
             return self._download_cached_photo_size(size, file)
 
+        if isinstance(size, types.PhotoSizeProgressive):
+            file_size = max(size.sizes)
+        else:
+            file_size = size.size
+
         result = await self.download_file(
             types.InputPhotoFileLocation(
                 id=photo.id,
@@ -650,7 +873,7 @@ class DownloadMethods(UserMethods):
                 thumb_size=size.type
             ),
             file,
-            file_size=size.size,
+            file_size=file_size,
             progress_callback=progress_callback
         )
         return result if file is bytes else file
@@ -680,27 +903,30 @@ class DownloadMethods(UserMethods):
         return kind, possible_names
 
     async def _download_document(
-            self, document, file, date, thumb, progress_callback):
+            self, document, file, date, thumb, progress_callback, msg_data):
         """Specialized version of .download_media() for documents."""
         if isinstance(document, types.MessageMediaDocument):
             document = document.document
         if not isinstance(document, types.Document):
             return
 
-        kind, possible_names = self._get_kind_and_names(document.attributes)
-        file = self._get_proper_filename(
-            file, kind, utils.get_extension(document),
-            date=date, possible_names=possible_names
-        )
-
         if thumb is None:
+            kind, possible_names = self._get_kind_and_names(document.attributes)
+            file = self._get_proper_filename(
+                file, kind, utils.get_extension(document),
+                date=date, possible_names=possible_names
+            )
             size = None
         else:
+            file = self._get_proper_filename(file, 'photo', '.jpg', date=date)
             size = self._get_thumb(document.thumbs, thumb)
+            if not size or isinstance(size, types.PhotoSizeEmpty):
+                return
+
             if isinstance(size, (types.PhotoCachedSize, types.PhotoStrippedSize)):
                 return self._download_cached_photo_size(size, file)
 
-        result = await self.download_file(
+        result = await self._download_file(
             types.InputDocumentFileLocation(
                 id=document.id,
                 access_hash=document.access_hash,
@@ -709,7 +935,8 @@ class DownloadMethods(UserMethods):
             ),
             file,
             file_size=size.size if size else document.size,
-            progress_callback=progress_callback
+            progress_callback=progress_callback,
+            msg_data=msg_data,
         )
 
         return result if file is bytes else file
@@ -736,22 +963,19 @@ class DownloadMethods(UserMethods):
             'END:VCARD\n'
         ).format(f=first_name, l=last_name, p=phone_number).encode('utf-8')
 
+        file = cls._get_proper_filename(
+            file, 'contact', '.vcard',
+            possible_names=[first_name, phone_number, last_name]
+        )
         if file is bytes:
             return result
-        elif isinstance(file, str):
-            file = cls._get_proper_filename(
-                file, 'contact', '.vcard',
-                possible_names=[first_name, phone_number, last_name]
-            )
-            f = open(file, 'wb')
-        else:
-            f = file
+        f = file if hasattr(file, 'write') else open(file, 'wb')
 
         try:
             f.write(result)
         finally:
             # Only close the stream if we opened it
-            if isinstance(file, str):
+            if f != file:
                 f.close()
 
         return file
@@ -768,21 +992,20 @@ class DownloadMethods(UserMethods):
             )
 
         # TODO Better way to get opened handles of files and auto-close
-        in_memory = file is bytes
-        if in_memory:
+        kind, possible_names = self._get_kind_and_names(web.attributes)
+        file = self._get_proper_filename(
+            file, kind, utils.get_extension(web),
+            possible_names=possible_names
+        )
+        if file is bytes:
             f = io.BytesIO()
-        elif isinstance(file, str):
-            kind, possible_names = cls._get_kind_and_names(web.attributes)
-            file = cls._get_proper_filename(
-                file, kind, utils.get_extension(web),
-                possible_names=possible_names
-            )
-            f = open(file, 'wb')
-        else:
+        elif hasattr(file, 'write'):
             f = file
+        else:
+            f = open(file, 'wb')
 
         try:
-            with aiohttp.ClientSession() as session:
+            async with aiohttp.ClientSession() as session:
                 # TODO Use progress_callback; get content length from response
                 # https://github.com/telegramdesktop/tdesktop/blob/c7e773dd9aeba94e2be48c032edc9a78bb50234e/Telegram/SourceFiles/ui/images.cpp#L1318-L1319
                 async with session.get(web.url) as response:
@@ -792,10 +1015,10 @@ class DownloadMethods(UserMethods):
                             break
                         f.write(chunk)
         finally:
-            if isinstance(file, str) or file is bytes:
+            if f != file:
                 f.close()
 
-        return f.getvalue() if in_memory else file
+        return f.getvalue() if file is bytes else file
 
     @staticmethod
     def _get_proper_filename(file, kind, extension,

telethon/client/messageparse.py

@@ -2,15 +2,14 @@ import itertools
 import re
 import typing
 
-from .users import UserMethods
-from .. import utils
+from .. import helpers, utils
 from ..tl import types
 
 if typing.TYPE_CHECKING:
     from .telegramclient import TelegramClient
 
 
-class MessageParseMethods(UserMethods):
+class MessageParseMethods:
 
     # region Public properties
 
@@ -19,14 +18,14 @@ class MessageParseMethods(UserMethods):
         """
         This property is the default parse mode used when sending messages.
         Defaults to `telethon.extensions.markdown`. It will always
-        be either ``None`` or an object with ``parse`` and ``unparse``
+        be either `None` or an object with ``parse`` and ``unparse``
         methods.
 
         When setting a different value it should be one of:
 
         * Object with ``parse`` and ``unparse`` methods.
         * A ``callable`` to act as the parse method.
-        * A ``str`` indicating the ``parse_mode``. For Markdown ``'md'``
+        * A `str` indicating the ``parse_mode``. For Markdown ``'md'``
           or ``'markdown'`` may be used. For HTML, ``'htm'`` or ``'html'``
           may be used.
 
@@ -76,7 +75,7 @@ class MessageParseMethods(UserMethods):
         """
         Returns a (parsed message, entities) tuple depending on ``parse_mode``.
         """
-        if parse_mode is ():
+        if parse_mode == ():
             parse_mode = self._parse_mode
         else:
             parse_mode = utils.sanitize_parse_mode(parse_mode)
@@ -84,10 +83,19 @@ class MessageParseMethods(UserMethods):
         if not parse_mode:
             return message, []
 
+        original_message = message
         message, msg_entities = parse_mode.parse(message)
+        if original_message and not message and not msg_entities:
+            raise ValueError("Failed to parse message")
+
         for i in reversed(range(len(msg_entities))):
             e = msg_entities[i]
-            if isinstance(e, types.MessageEntityTextUrl):
+            if not e.length:
+                # 0-length MessageEntity is no longer valid #3884.
+                # Because the user can provide their own parser (with reasonable 0-length
+                # entities), strip them here rather than fixing the built-in parsers.
+                del msg_entities[i]
+            elif isinstance(e, types.MessageEntityTextUrl):
                 m = re.match(r'^@|\+|tg://user\?id=(\d+)', e.url)
                 if m:
                     user = int(m.group(1)) if m.group(1) else e.url
@@ -131,27 +139,70 @@ class MessageParseMethods(UserMethods):
             elif isinstance(update, (
                     types.UpdateNewChannelMessage, types.UpdateNewMessage)):
                 update.message._finish_init(self, entities, input_chat)
-                id_to_message[update.message.id] = update.message
+
+                # Pinning a message with `updatePinnedMessage` seems to
+                # always produce a service message we can't map so return
+                # it directly. The same happens for kicking users.
+                #
+                # It could also be a list (e.g. when sending albums).
+                #
+                # TODO this method is getting messier and messier as time goes on
+                if hasattr(request, 'random_id') or utils.is_list_like(request):
+                    id_to_message[update.message.id] = update.message
+                else:
+                    return update.message
 
             elif (isinstance(update, types.UpdateEditMessage)
-                  and not isinstance(request.peer, types.InputPeerChannel)):
-                if request.id == update.message.id:
-                    update.message._finish_init(self, entities, input_chat)
+                  and helpers._entity_type(request.peer) != helpers._EntityType.CHANNEL):
+                update.message._finish_init(self, entities, input_chat)
+
+                # Live locations use `sendMedia` but Telegram responds with
+                # `updateEditMessage`, which means we won't have `id` field.
+                if hasattr(request, 'random_id'):
+                    id_to_message[update.message.id] = update.message
+                elif request.id == update.message.id:
                     return update.message
 
             elif (isinstance(update, types.UpdateEditChannelMessage)
                   and utils.get_peer_id(request.peer) ==
-                  utils.get_peer_id(update.message.to_id)):
+                  utils.get_peer_id(update.message.peer_id)):
                 if request.id == update.message.id:
                     update.message._finish_init(self, entities, input_chat)
                     return update.message
 
+            elif isinstance(update, types.UpdateNewScheduledMessage):
+                update.message._finish_init(self, entities, input_chat)
+                # Scheduled IDs may collide with normal IDs. However, for a
+                # single request there *shouldn't* be a mix between "some
+                # scheduled and some not".
+                id_to_message[update.message.id] = update.message
+
+            elif isinstance(update, types.UpdateMessagePoll):
+                if request.media.poll.id == update.poll_id:
+                    m = types.Message(
+                        id=request.id,
+                        peer_id=utils.get_peer(request.peer),
+                        media=types.MessageMediaPoll(
+                            poll=update.poll,
+                            results=update.results
+                        )
+                    )
+                    m._finish_init(self, entities, input_chat)
+                    return m
+
         if request is None:
             return id_to_message
 
-        random_id = request if isinstance(request, int) else request.random_id
+        random_id = request if isinstance(request, (int, list)) else getattr(request, 'random_id', None)
+        if random_id is None:
+            # Can happen when pinning a message does not actually produce a service message.
+            self._log[__name__].warning(
+                'No random_id in %s to map to, returning None message for %s', request, result)
+            return None
+
         if not utils.is_list_like(random_id):
             msg = id_to_message.get(random_to_id.get(random_id))
+
             if not msg:
                 self._log[__name__].warning(
                     'Request %s had missing message mapping %s', request, result)
@@ -165,9 +216,18 @@ class MessageParseMethods(UserMethods):
             # deleted or `WORKER_BUSY_TOO_LONG_RETRY` if there are issues at
             # Telegram), in which case we get some "missing" message mappings.
             # Log them with the hope that we can better work around them.
+            #
+            # This also happens when trying to forward messages that can't
+            # be forwarded because they don't exist (0, service, deleted)
+            # among others which could be (like deleted or existing).
             self._log[__name__].warning(
                 'Request %s had missing message mappings %s', request, result)
 
-            return [id_to_message.get(random_to_id.get(rnd)) for rnd in random_to_id]
+        return [
+            id_to_message.get(random_to_id[rnd])
+            if rnd in random_to_id
+            else None
+            for rnd in random_id
+        ]
 
     # endregion

telethon/client/telegrambaseclient.py

@@ -1,31 +1,67 @@
 import abc
+import re
 import asyncio
 import collections
 import logging
 import platform
 import time
 import typing
+import datetime
+import pathlib
 
 from .. import version, helpers, __name__ as __base_name__
 from ..crypto import rsa
-from ..entitycache import EntityCache
 from ..extensions import markdown
 from ..network import MTProtoSender, Connection, ConnectionTcpFull, TcpMTProxy
 from ..sessions import Session, SQLiteSession, MemorySession
-from ..statecache import StateCache
-from ..tl import TLObject, functions, types
+from ..tl import functions, types
 from ..tl.alltlobjects import LAYER
+from .._updates import MessageBox, EntityCache as MbEntityCache, SessionState, ChannelState, Entity, EntityType
 
-DEFAULT_DC_ID = 4
+DEFAULT_DC_ID = 2
 DEFAULT_IPV4_IP = '149.154.167.51'
-DEFAULT_IPV6_IP = '[2001:67c:4e8:f002::a]'
+DEFAULT_IPV6_IP = '2001:67c:4e8:f002::a'
 DEFAULT_PORT = 443
 
 if typing.TYPE_CHECKING:
     from .telegramclient import TelegramClient
 
-__default_log__ = logging.getLogger(__base_name__)
-__default_log__.addHandler(logging.NullHandler())
+_base_log = logging.getLogger(__base_name__)
+
+
+# In seconds, how long to wait before disconnecting a exported sender.
+_DISCONNECT_EXPORTED_AFTER = 60
+
+
+class _ExportState:
+    def __init__(self):
+        # ``n`` is the amount of borrows a given sender has;
+        # once ``n`` reaches ``0``, disconnect the sender after a while.
+        self._n = 0
+        self._zero_ts = 0
+        self._connected = False
+
+    def add_borrow(self):
+        self._n += 1
+        self._connected = True
+
+    def add_return(self):
+        self._n -= 1
+        assert self._n >= 0, 'returned sender more than it was borrowed'
+        if self._n == 0:
+            self._zero_ts = time.time()
+
+    def should_disconnect(self):
+        return (self._n == 0
+                and self._connected
+                and (time.time() - self._zero_ts) > _DISCONNECT_EXPORTED_AFTER)
+
+    def need_connect(self):
+        return not self._connected
+
+    def mark_disconnected(self):
+        assert self.should_disconnect(), 'marked as disconnected when it was borrowed'
+        self._connected = False
 
 
 # TODO How hard would it be to support both `trio` and `asyncio`?
@@ -39,7 +75,7 @@ class TelegramBaseClient(abc.ABC):
         session (`str` | `telethon.sessions.abstract.Session`, `None`):
             The file name of the session file to be used if a string is
             given (it may be a full path), or the Session instance to be
-            used otherwise. If it's ``None``, the session will not be saved,
+            used otherwise. If it's `None`, the session will not be saved,
             and you should call :meth:`.log_out()` when you're done.
 
             Note that if you pass a string it will be a file in the current
@@ -54,7 +90,7 @@ class TelegramBaseClient(abc.ABC):
             The API ID you obtained from https://my.telegram.org.
 
         api_hash (`str`):
-            The API ID you obtained from https://my.telegram.org.
+            The API hash you obtained from https://my.telegram.org.
 
         connection (`telethon.network.connection.common.Connection`, optional):
             The connection instance to be used when creating a new connection
@@ -64,7 +100,7 @@ class TelegramBaseClient(abc.ABC):
 
         use_ipv6 (`bool`, optional):
             Whether to connect to the servers through IPv6 or not.
-            By default this is ``False`` as IPv6 support is not
+            By default this is `False` as IPv6 support is not
             too widespread yet.
 
         proxy (`tuple` | `list` | `dict`, optional):
@@ -74,6 +110,11 @@ class TelegramBaseClient(abc.ABC):
             function parameters for PySocks, like ``(type, 'hostname', port)``.
             See https://github.com/Anorov/PySocks#usage-1 for more.
 
+        local_addr (`str` | `tuple`, optional):
+            Local host address (and port, optionally) used to bind the socket to locally.
+            You only need to use this if you have multiple network cards and
+            want to use a specific one.
+
         timeout (`int` | `float`, optional):
             The timeout in seconds to be used when connecting.
             This is **not** the timeout to be used when ``await``'ing for
@@ -87,14 +128,14 @@ class TelegramBaseClient(abc.ABC):
             when there is a ``errors.FloodWaitError`` less than
             `flood_sleep_threshold`, or when there's a migrate error.
 
-            May take a negative or ``None`` value for infinite retries, but
+            May take a negative or `None` value for infinite retries, but
             this is not recommended, since some requests can always trigger
             a call fail (such as searching for messages).
 
         connection_retries (`int` | `None`, optional):
             How many times the reconnection should retry, either on the
             initial connection or when Telegram disconnects us. May be
-            set to a negative or ``None`` value for infinite retries, but
+            set to a negative or `None` value for infinite retries, but
             this is not recommended, since the program can get stuck in an
             infinite loop.
 
@@ -111,26 +152,32 @@ class TelegramBaseClient(abc.ABC):
             the order in which updates are processed to be sequential, and
             this setting allows them to do so.
 
-            If set to ``True``, incoming updates will be put in a queue
+            If set to `True`, incoming updates will be put in a queue
             and processed sequentially. This means your event handlers
             should *not* perform long-running operations since new
             updates are put inside of an unbounded queue.
 
         flood_sleep_threshold (`int` | `float`, optional):
             The threshold below which the library should automatically
-            sleep on flood wait errors (inclusive). For instance, if a
+            sleep on flood wait and slow mode wait errors (inclusive). For instance, if a
             ``FloodWaitError`` for 17s occurs and `flood_sleep_threshold`
             is 20s, the library will ``sleep`` automatically. If the error
             was for 21s, it would ``raise FloodWaitError`` instead. Values
             larger than a day (like ``float('inf')``) will be changed to a day.
 
+        raise_last_call_error (`bool`, optional):
+            When API calls fail in a way that causes Telethon to retry
+            automatically, should the RPC error of the last attempt be raised
+            instead of a generic ValueError. This is mostly useful for
+            detecting when Telegram has internal issues.
+
         device_model (`str`, optional):
             "Device model" to be sent when creating the initial connection.
-            Defaults to ``platform.node()``.
+            Defaults to 'PC (n)bit' derived from ``platform.uname().machine``, or its direct value if unknown.
 
         system_version (`str`, optional):
             "System version" to be sent when creating the initial connection.
-            Defaults to ``platform.system()``.
+            Defaults to ``platform.uname().release`` stripped of everything ahead of -.
 
         app_version (`str`, optional):
             "App version" to be sent when creating the initial connection.
@@ -145,13 +192,37 @@ class TelegramBaseClient(abc.ABC):
             Defaults to `lang_code`.
 
         loop (`asyncio.AbstractEventLoop`, optional):
-            Asyncio event loop to use. Defaults to `asyncio.get_event_loop()`
+            Asyncio event loop to use. Defaults to `asyncio.get_running_loop()`.
+            This argument is ignored.
 
         base_logger (`str` | `logging.Logger`, optional):
             Base logger name or instance to use.
             If a `str` is given, it'll be passed to `logging.getLogger()`. If a
             `logging.Logger` is given, it'll be used directly. If something
             else or nothing is given, the default logger will be used.
+
+        receive_updates (`bool`, optional):
+            Whether the client will receive updates or not. By default, updates
+            will be received from Telegram as they occur.
+
+            Turning this off means that Telegram will not send updates at all
+            so event handlers, conversations, and QR login will not work.
+            However, certain scripts don't need updates, so this will reduce
+            the amount of bandwidth used.
+
+        entity_cache_limit (`int`, optional):
+            How many users, chats and channels to keep in the in-memory cache
+            at most. This limit is checked against when processing updates.
+
+            When this limit is reached or exceeded, all entities that are not
+            required for update handling will be flushed to the session file.
+
+            Note that this implies that there is a lower bound to the amount
+            of entities that must be kept in memory.
+
+            Setting this limit too low will cause the library to attempt to
+            flush entities to the session file even if no entities can be
+            removed from the in-memory cache, which will degrade performance.
     """
 
     # Current TelegramClient version
@@ -165,39 +236,44 @@ class TelegramBaseClient(abc.ABC):
 
     def __init__(
             self: 'TelegramClient',
-            session: 'typing.Union[str, Session]',
+            session: 'typing.Union[str, pathlib.Path, Session]',
             api_id: int,
             api_hash: str,
             *,
             connection: 'typing.Type[Connection]' = ConnectionTcpFull,
             use_ipv6: bool = False,
             proxy: typing.Union[tuple, dict] = None,
+            local_addr: typing.Union[str, tuple] = None,
             timeout: int = 10,
             request_retries: int = 5,
-            connection_retries: int =5,
+            connection_retries: int = 5,
             retry_delay: int = 1,
             auto_reconnect: bool = True,
             sequential_updates: bool = False,
             flood_sleep_threshold: int = 60,
+            raise_last_call_error: bool = False,
             device_model: str = None,
             system_version: str = None,
             app_version: str = None,
             lang_code: str = 'en',
             system_lang_code: str = 'en',
             loop: asyncio.AbstractEventLoop = None,
-            base_logger: typing.Union[str, logging.Logger] = None):
+            base_logger: typing.Union[str, logging.Logger] = None,
+            receive_updates: bool = True,
+            catch_up: bool = False,
+            entity_cache_limit: int = 5000
+    ):
         if not api_id or not api_hash:
             raise ValueError(
                 "Your API ID or Hash cannot be empty or None. "
                 "Refer to telethon.rtfd.io for more information.")
 
         self._use_ipv6 = use_ipv6
-        self._loop = loop or asyncio.get_event_loop()
 
         if isinstance(base_logger, str):
             base_logger = logging.getLogger(base_logger)
         elif not isinstance(base_logger, logging.Logger):
-            base_logger = __default_log__
+            base_logger = _base_log
 
         class _Loggers(dict):
             def __missing__(self, key):
@@ -209,9 +285,9 @@ class TelegramBaseClient(abc.ABC):
         self._log = _Loggers()
 
         # Determine what session object we have
-        if isinstance(session, str) or session is None:
+        if isinstance(session, (str, pathlib.Path)):
             try:
-                session = SQLiteSession(session)
+                session = SQLiteSession(str(session))
             except ImportError:
                 import warnings
                 warnings.warn(
@@ -222,6 +298,8 @@ class TelegramBaseClient(abc.ABC):
                     'you use another session storage'
                 )
                 session = MemorySession()
+        elif session is None:
+            session = MemorySession()
         elif not isinstance(session, Session):
             raise TypeError(
                 'The given session must be a str or a Session instance.'
@@ -235,11 +313,12 @@ class TelegramBaseClient(abc.ABC):
                 DEFAULT_IPV6_IP if self._use_ipv6 else DEFAULT_IPV4_IP,
                 DEFAULT_PORT
             )
+            session.save()
 
         self.flood_sleep_threshold = flood_sleep_threshold
 
         # TODO Use AsyncClassWrapper(session)
-        # ChatGetter and SenderGetter can use the in-memory _entity_cache
+        # ChatGetter and SenderGetter can use the in-memory _mb_entity_cache
         # to avoid network access and the need for await in session files.
         #
         # The session files only wants the entities to persist
@@ -247,14 +326,44 @@ class TelegramBaseClient(abc.ABC):
         # TODO Session should probably return all cached
         #      info of entities, not just the input versions
         self.session = session
-        self._entity_cache = EntityCache()
         self.api_id = int(api_id)
         self.api_hash = api_hash
 
+        # Current proxy implementation requires `sock_connect`, and some
+        # event loops lack this method. If the current loop is missing it,
+        # bail out early and suggest an alternative.
+        #
+        # TODO A better fix is obviously avoiding the use of `sock_connect`
+        #
+        # See https://github.com/LonamiWebs/Telethon/issues/1337 for details.
+        if not callable(getattr(self.loop, 'sock_connect', None)):
+            raise TypeError(
+                'Event loop of type {} lacks `sock_connect`, which is needed to use proxies.\n\n'
+                'Change the event loop in use to use proxies:\n'
+                '# https://github.com/LonamiWebs/Telethon/issues/1337\n'
+                'import asyncio\n'
+                'asyncio.set_event_loop(asyncio.SelectorEventLoop())'.format(
+                    self.loop.__class__.__name__
+                )
+            )
+
+        if local_addr is not None:
+            if use_ipv6 is False and ':' in local_addr:
+                raise TypeError(
+                    'A local IPv6 address must only be used with `use_ipv6=True`.'
+                )
+            elif use_ipv6 is True and ':' not in local_addr:
+                raise TypeError(
+                    '`use_ipv6=True` must only be used with a local IPv6 address.'
+                )
+
+        self._raise_last_call_error = raise_last_call_error
+
         self._request_retries = request_retries
         self._connection_retries = connection_retries
         self._retry_delay = retry_delay or 0
         self._proxy = proxy
+        self._local_addr = local_addr
         self._timeout = timeout
         self._auto_reconnect = auto_reconnect
 
@@ -266,67 +375,63 @@ class TelegramBaseClient(abc.ABC):
         # Used on connection. Capture the variables in a lambda since
         # exporting clients need to create this InvokeWithLayerRequest.
         system = platform.uname()
-        self._init_with = lambda x: functions.InvokeWithLayerRequest(
-            LAYER, functions.InitConnectionRequest(
-                api_id=self.api_id,
-                device_model=device_model or system.system or 'Unknown',
-                system_version=system_version or system.release or '1.0',
-                app_version=app_version or self.__version__,
-                lang_code=lang_code,
-                system_lang_code=system_lang_code,
-                lang_pack='',  # "langPacks are for official apps only"
-                query=x,
-                proxy=init_proxy
-            )
-        )
 
-        self._sender = MTProtoSender(
-            self.session.auth_key, self._loop,
-            loggers=self._log,
-            retries=self._connection_retries,
-            delay=self._retry_delay,
-            auto_reconnect=self._auto_reconnect,
-            connect_timeout=self._timeout,
-            auth_key_callback=self._auth_key_callback,
-            update_callback=self._handle_update,
-            auto_reconnect_callback=self._handle_auto_reconnect
+        if system.machine in ('x86_64', 'AMD64'):
+            default_device_model = 'PC 64bit'
+        elif system.machine in ('i386','i686','x86'):
+            default_device_model = 'PC 32bit'
+        else:
+            default_device_model = system.machine
+        default_system_version = re.sub(r'-.+','',system.release)
+
+        self._init_request = functions.InitConnectionRequest(
+            api_id=self.api_id,
+            device_model=device_model or default_device_model or 'Unknown',
+            system_version=system_version or default_system_version or '1.0',
+            app_version=app_version or self.__version__,
+            lang_code=lang_code,
+            system_lang_code=system_lang_code,
+            lang_pack='',  # "langPacks are for official apps only"
+            query=None,
+            proxy=init_proxy
         )
 
         # Remember flood-waited requests to avoid making them again
         self._flood_waited_requests = {}
 
-        # Cache ``{dc_id: (n, MTProtoSender)}`` for all borrowed senders,
-        # being ``n`` the amount of borrows a given sender has; once ``n``
-        # reaches ``0`` it should be disconnected and removed.
+        # Cache ``{dc_id: (_ExportState, MTProtoSender)}`` for all borrowed senders
         self._borrowed_senders = {}
-        self._borrow_sender_lock = asyncio.Lock(loop=self._loop)
+        self._borrow_sender_lock = asyncio.Lock()
+        self._exported_sessions = {}
 
+        self._loop = None  # only used as a sanity check
+        self._updates_error = None
         self._updates_handle = None
+        self._keepalive_handle = None
         self._last_request = time.time()
-        self._channel_pts = {}
+        self._no_updates = not receive_updates
 
-        if sequential_updates:
-            self._updates_queue = asyncio.Queue(loop=self._loop)
-            self._dispatching_updates_queue = asyncio.Event(loop=self._loop)
-        else:
-            # Use a set of pending instead of a queue so we can properly
-            # terminate all pending updates on disconnect.
-            self._updates_queue = set()
-            self._dispatching_updates_queue = None
+        # Used for non-sequential updates, in order to terminate all pending tasks on disconnect.
+        self._sequential_updates = sequential_updates
+        self._event_handler_tasks = set()
 
         self._authorized = None  # None = unknown, False = no, True = yes
 
-        # Update state (for catching up after a disconnection)
-        # TODO Get state from channels too
-        self._state_cache = StateCache(
-            self.session.get_update_state(0), self._log)
-
         # Some further state for subclasses
         self._event_builders = []
 
         # {chat_id: {Conversation}}
         self._conversations = collections.defaultdict(set)
 
+        # Hack to workaround the fact Telegram may send album updates as
+        # different Updates when being sent from a different data center.
+        # {grouped_id: AlbumHack}
+        #
+        # FIXME: We don't bother cleaning this up because it's not really
+        #        worth it, albums are pretty rare and this only holds them
+        #        for a second at most.
+        self._albums = {}
+
         # Default parse mode
         self._parse_mode = markdown
 
@@ -336,9 +441,28 @@ class TelegramBaseClient(abc.ABC):
         self._phone = None
         self._tos = None
 
-        # Sometimes we need to know who we are, cache the self peer
-        self._self_input_peer = None
-        self._bot = None
+        # A place to store if channels are a megagroup or not (see `edit_admin`)
+        self._megagroup_cache = {}
+
+        # This is backported from v2 in a very ad-hoc way just to get proper update handling
+        self._catch_up = catch_up
+        self._updates_queue = asyncio.Queue()
+        self._message_box = MessageBox(self._log['messagebox'])
+        self._mb_entity_cache = MbEntityCache()  # required for proper update handling (to know when to getDifference)
+        self._entity_cache_limit = entity_cache_limit
+
+        self._sender = MTProtoSender(
+            self.session.auth_key,
+            loggers=self._log,
+            retries=self._connection_retries,
+            delay=self._retry_delay,
+            auto_reconnect=self._auto_reconnect,
+            connect_timeout=self._timeout,
+            auth_key_callback=self._auth_key_callback,
+            updates_queue=self._updates_queue,
+            auto_reconnect_callback=self._handle_auto_reconnect
+        )
+
 
     # endregion
 
@@ -353,7 +477,7 @@ class TelegramBaseClient(abc.ABC):
             .. code-block:: python
 
                 # Download media in the background
-                task = client.loop_create_task(message.download_media())
+                task = client.loop.create_task(message.download_media())
 
                 # Do some work
                 ...
@@ -361,7 +485,7 @@ class TelegramBaseClient(abc.ABC):
                 # Join the task (wait for it to complete)
                 await task
         """
-        return self._loop
+        return helpers.get_running_loop()
 
     @property
     def disconnected(self: 'TelegramClient') -> asyncio.Future:
@@ -379,6 +503,15 @@ class TelegramBaseClient(abc.ABC):
         """
         return self._sender.disconnected
 
+    @property
+    def flood_sleep_threshold(self):
+        return self._flood_sleep_threshold
+
+    @flood_sleep_threshold.setter
+    def flood_sleep_threshold(self, value):
+        # None -> 0, negative values don't really matter
+        self._flood_sleep_threshold = min(value or 0, 24 * 60 * 60)
+
     # endregion
 
     # region Connecting
@@ -401,29 +534,80 @@ class TelegramBaseClient(abc.ABC):
             .. code-block:: python
 
                 try:
-                    client.connect()
+                    await client.connect()
                 except OSError:
                     print('Failed to connect')
         """
-        await self._sender.connect(self._connection(
+        if self.session is None:
+            raise ValueError('TelegramClient instance cannot be reused after logging out')
+
+        if self._loop is None:
+            self._loop = helpers.get_running_loop()
+        elif self._loop != helpers.get_running_loop():
+            raise RuntimeError('The asyncio event loop must not change after connection (see the FAQ for details)')
+
+        if not await self._sender.connect(self._connection(
             self.session.server_address,
             self.session.port,
             self.session.dc_id,
-            loop=self._loop,
             loggers=self._log,
-            proxy=self._proxy
-        ))
+            proxy=self._proxy,
+            local_addr=self._local_addr
+        )):
+            # We don't want to init or modify anything if we were already connected
+            return
+
         self.session.auth_key = self._sender.auth_key
         self.session.save()
 
-        await self._sender.send(self._init_with(
-            functions.help.GetConfigRequest()))
+        try:
+            # See comment when saving entities to understand this hack
+            self_id = self.session.get_input_entity(0).access_hash
+            self_user = self.session.get_input_entity(self_id)
+            self._mb_entity_cache.set_self_user(self_id, None, self_user.access_hash)
+        except ValueError:
+            pass
 
-        self._updates_handle = self._loop.create_task(self._update_loop())
+        if self._catch_up:
+            ss = SessionState(0, 0, False, 0, 0, 0, 0, None)
+            cs = []
+
+            for entity_id, state in self.session.get_update_states():
+                if entity_id == 0:
+                    # TODO current session doesn't store self-user info but adding that is breaking on downstream session impls
+                    ss = SessionState(0, 0, False, state.pts, state.qts, int(state.date.timestamp()), state.seq, None)
+                else:
+                    cs.append(ChannelState(entity_id, state.pts))
+
+            self._message_box.load(ss, cs)
+            for state in cs:
+                try:
+                    entity = self.session.get_input_entity(state.channel_id)
+                except ValueError:
+                    self._log[__name__].warning(
+                        'No access_hash in cache for channel %s, will not catch up', state.channel_id)
+                else:
+                    self._mb_entity_cache.put(Entity(EntityType.CHANNEL, entity.channel_id, entity.access_hash))
+
+        self._init_request.query = functions.help.GetConfigRequest()
+
+        req = self._init_request
+        if self._no_updates:
+            req = functions.InvokeWithoutUpdatesRequest(req)
+
+        await self._sender.send(functions.InvokeWithLayerRequest(LAYER, req))
+
+        if self._message_box.is_empty():
+            me = await self.get_me()
+            if me:
+                await self._on_login(me)  # also calls GetState to initialize the MessageBox
+
+        self._updates_handle = self.loop.create_task(self._update_loop())
+        self._keepalive_handle = self.loop.create_task(self._keepalive_loop())
 
     def is_connected(self: 'TelegramClient') -> bool:
         """
-        Returns ``True`` if the user has connected.
+        Returns `True` if the user has connected.
 
         This method is **not** asynchronous (don't use ``await`` on it).
 
@@ -444,17 +628,27 @@ class TelegramBaseClient(abc.ABC):
         coroutine that you should await on your own code; otherwise
         the loop is ran until said coroutine completes.
 
+        Event handlers which are currently running will be cancelled before
+        this function returns (in order to properly clean-up their tasks).
+        In particular, this means that using ``disconnect`` in a handler
+        will cause code after the ``disconnect`` to never run. If this is
+        needed, consider spawning a separate task to do the remaining work.
+
         Example
             .. code-block:: python
 
                 # You don't need to use this if you used "with client"
-                client.disconnect()
+                await client.disconnect()
         """
-        if self._loop.is_running():
-            return self._disconnect_coro()
+        if self.loop.is_running():
+            # Disconnect may be called from an event handler, which would
+            # cancel itself during itself and never actually complete the
+            # disconnection. Shield the task to prevent disconnect itself
+            # from being cancelled. See issue #3942 for more details.
+            return asyncio.shield(self.loop.create_task(self._disconnect_coro()))
         else:
             try:
-                self._loop.run_until_complete(self._disconnect_coro())
+                self.loop.run_until_complete(self._disconnect_coro())
             except RuntimeError:
                 # Python 3.5.x complains when called from
                 # `__aexit__` and there were pending updates with:
@@ -463,27 +657,84 @@ class TelegramBaseClient(abc.ABC):
                 # However, it doesn't really make a lot of sense.
                 pass
 
+    def set_proxy(self: 'TelegramClient', proxy: typing.Union[tuple, dict]):
+        """
+        Changes the proxy which will be used on next (re)connection.
+
+        Method has no immediate effects if the client is currently connected.
+
+        The new proxy will take it's effect on the next reconnection attempt:
+            - on a call `await client.connect()` (after complete disconnect)
+            - on auto-reconnect attempt (e.g, after previous connection was lost)
+        """
+        init_proxy = None if not issubclass(self._connection, TcpMTProxy) else \
+            types.InputClientProxy(*self._connection.address_info(proxy))
+
+        self._init_request.proxy = init_proxy
+        self._proxy = proxy
+
+        # While `await client.connect()` passes new proxy on each new call,
+        # auto-reconnect attempts use already set up `_connection` inside
+        # the `_sender`, so the only way to change proxy between those
+        # is to directly inject parameters.
+
+        connection = getattr(self._sender, "_connection", None)
+        if connection:
+            if isinstance(connection, TcpMTProxy):
+                connection._ip = proxy[0]
+                connection._port = proxy[1]
+            else:
+                connection._proxy = proxy
+
+    def _save_states_and_entities(self: 'TelegramClient'):
+        entities = self._mb_entity_cache.get_all_entities()
+
+        # Piggy-back on an arbitrary TL type with users and chats so the session can understand to read the entities.
+        # It doesn't matter if we put users in the list of chats.
+        self.session.process_entities(types.contacts.ResolvedPeer(None, [e._as_input_peer() for e in entities], []))
+
+        # As a hack to not need to change the session files, save ourselves with ``id=0`` and ``access_hash`` of our ``id``.
+        # This way it is possible to determine our own ID by querying for 0. However, whether we're a bot is not saved.
+        if self._mb_entity_cache.self_id:
+            self.session.process_entities(types.contacts.ResolvedPeer(None, [types.InputPeerUser(0, self._mb_entity_cache.self_id)], []))
+
+        ss, cs = self._message_box.session_state()
+        self.session.set_update_state(0, types.updates.State(**ss, unread_count=0))
+        now = datetime.datetime.now()  # any datetime works; channels don't need it
+        for channel_id, pts in cs.items():
+            self.session.set_update_state(channel_id, types.updates.State(pts, 0, now, 0, unread_count=0))
+
     async def _disconnect_coro(self: 'TelegramClient'):
+        if self.session is None:
+            return  # already logged out and disconnected
+
         await self._disconnect()
 
+        # Also clean-up all exported senders because we're done with them
+        async with self._borrow_sender_lock:
+            for state, sender in self._borrowed_senders.values():
+                # Note that we're not checking for `state.should_disconnect()`.
+                # If the user wants to disconnect the client, ALL connections
+                # to Telegram (including exported senders) should be closed.
+                #
+                # Disconnect should never raise, so there's no try/except.
+                await sender.disconnect()
+                # Can't use `mark_disconnected` because it may be borrowed.
+                state._connected = False
+
+            # If any was borrowed
+            self._borrowed_senders.clear()
+
         # trio's nurseries would handle this for us, but this is asyncio.
         # All tasks spawned in the background should properly be terminated.
-        if self._dispatching_updates_queue is None and self._updates_queue:
-            for task in self._updates_queue:
+        if self._event_handler_tasks:
+            for task in self._event_handler_tasks:
                 task.cancel()
 
-            await asyncio.wait(self._updates_queue, loop=self._loop)
-            self._updates_queue.clear()
+            await asyncio.wait(self._event_handler_tasks)
+            self._event_handler_tasks.clear()
 
-        pts, date = self._state_cache[None]
-        if pts and date:
-            self.session.set_update_state(0, types.updates.State(
-                pts=pts,
-                qts=0,
-                date=date,
-                seq=0,
-                unread_count=0
-            ))
+        self._save_states_and_entities()
 
         self.session.close()
 
@@ -496,7 +747,8 @@ class TelegramBaseClient(abc.ABC):
         """
         await self._sender.disconnect()
         await helpers._cancel(self._log[__name__],
-                              updates_handle=self._updates_handle)
+                              updates_handle=self._updates_handle,
+                              keepalive_handle=self._keepalive_handle)
 
     async def _switch_dc(self: 'TelegramClient', new_dc):
         """
@@ -535,13 +787,27 @@ class TelegramBaseClient(abc.ABC):
         if cdn and not self._cdn_config:
             cls._cdn_config = await self(functions.help.GetCdnConfigRequest())
             for pk in cls._cdn_config.public_keys:
-                rsa.add_key(pk.public_key)
+                if pk.dc_id == dc_id:
+                    rsa.add_key(pk.public_key, old=False)
 
-        return next(
-            dc for dc in cls._config.dc_options
-            if dc.id == dc_id
-            and bool(dc.ipv6) == self._use_ipv6 and bool(dc.cdn) == cdn
-        )
+        try:
+            return next(
+                dc for dc in cls._config.dc_options
+                if dc.id == dc_id
+                and bool(dc.ipv6) == self._use_ipv6 and bool(dc.cdn) == cdn
+            )
+        except StopIteration:
+            self._log[__name__].warning(
+                'Failed to get DC %s (cdn = %s) with use_ipv6 = %s; retrying ignoring IPv6 check',
+                dc_id, cdn, self._use_ipv6
+            )
+            try:
+                return next(
+                    dc for dc in cls._config.dc_options
+                    if dc.id == dc_id and bool(dc.cdn) == cdn
+                )
+            except StopIteration:
+                raise ValueError(f'Failed to get DC {dc_id} (cdn = {cdn})')
 
     async def _create_exported_sender(self: 'TelegramClient', dc_id):
         """
@@ -555,21 +821,19 @@ class TelegramBaseClient(abc.ABC):
         #
         # If one were to do that, Telegram would reset the connection
         # with no further clues.
-        sender = MTProtoSender(None, self._loop, loggers=self._log)
+        sender = MTProtoSender(None, loggers=self._log)
         await sender.connect(self._connection(
             dc.ip_address,
             dc.port,
             dc.id,
-            loop=self._loop,
             loggers=self._log,
-            proxy=self._proxy
+            proxy=self._proxy,
+            local_addr=self._local_addr
         ))
-        self._log[__name__].info('Exporting authorization for data center %s',
-                                 dc)
+        self._log[__name__].info('Exporting auth for new borrowed sender in %s', dc)
         auth = await self(functions.auth.ExportAuthorizationRequest(dc_id))
-        req = self._init_with(functions.auth.ImportAuthorizationRequest(
-            id=auth.id, bytes=auth.bytes
-        ))
+        self._init_request.query = functions.auth.ImportAuthorizationRequest(id=auth.id, bytes=auth.bytes)
+        req = functions.InvokeWithLayerRequest(LAYER, self._init_request)
         await sender.send(req)
         return sender
 
@@ -582,24 +846,28 @@ class TelegramBaseClient(abc.ABC):
         Once its job is over it should be `_return_exported_sender`.
         """
         async with self._borrow_sender_lock:
-            n, sender = self._borrowed_senders.get(dc_id, (0, None))
-            if not sender:
+            self._log[__name__].debug('Borrowing sender for dc_id %d', dc_id)
+            state, sender = self._borrowed_senders.get(dc_id, (None, None))
+
+            if state is None:
+                state = _ExportState()
                 sender = await self._create_exported_sender(dc_id)
                 sender.dc_id = dc_id
-            elif not n:
+                self._borrowed_senders[dc_id] = (state, sender)
+
+            elif state.need_connect():
                 dc = await self._get_dc(dc_id)
                 await sender.connect(self._connection(
                     dc.ip_address,
                     dc.port,
                     dc.id,
-                    loop=self._loop,
                     loggers=self._log,
-                    proxy=self._proxy
+                    proxy=self._proxy,
+                    local_addr=self._local_addr
                 ))
 
-            self._borrowed_senders[dc_id] = (n + 1, sender)
-
-        return sender
+            state.add_borrow()
+            return sender
 
     async def _return_exported_sender(self: 'TelegramClient', sender):
         """
@@ -607,39 +875,50 @@ class TelegramBaseClient(abc.ABC):
         been returned, the sender is cleanly disconnected.
         """
         async with self._borrow_sender_lock:
-            dc_id = sender.dc_id
-            n, _ = self._borrowed_senders[dc_id]
-            n -= 1
-            self._borrowed_senders[dc_id] = (n, sender)
-            if not n:
-                self._log[__name__].info(
-                    'Disconnecting borrowed sender for DC %d', dc_id)
-                await sender.disconnect()
+            self._log[__name__].debug('Returning borrowed sender for dc_id %d', sender.dc_id)
+            state, _ = self._borrowed_senders[sender.dc_id]
+            state.add_return()
+
+    async def _clean_exported_senders(self: 'TelegramClient'):
+        """
+        Cleans-up all unused exported senders by disconnecting them.
+        """
+        async with self._borrow_sender_lock:
+            for dc_id, (state, sender) in self._borrowed_senders.items():
+                if state.should_disconnect():
+                    self._log[__name__].info(
+                        'Disconnecting borrowed sender for DC %d', dc_id)
+
+                    # Disconnect should never raise
+                    await sender.disconnect()
+                    state.mark_disconnected()
 
     async def _get_cdn_client(self: 'TelegramClient', cdn_redirect):
         """Similar to ._borrow_exported_client, but for CDNs"""
-        # TODO Implement
-        raise NotImplementedError
         session = self._exported_sessions.get(cdn_redirect.dc_id)
         if not session:
             dc = await self._get_dc(cdn_redirect.dc_id, cdn=True)
             session = self.session.clone()
-            await session.set_dc(dc.id, dc.ip_address, dc.port)
+            session.set_dc(dc.id, dc.ip_address, dc.port)
             self._exported_sessions[cdn_redirect.dc_id] = session
 
         self._log[__name__].info('Creating new CDN client')
-        client = TelegramBareClient(
+        client = self.__class__(
             session, self.api_id, self.api_hash,
-            proxy=self._sender.connection.conn.proxy,
-            timeout=self._sender.connection.get_timeout()
+            proxy=self._proxy,
+            timeout=self._timeout,
+            loop=self.loop
         )
 
-        # This will make use of the new RSA keys for this specific CDN.
-        #
-        # We won't be calling GetConfigRequest because it's only called
-        # when needed by ._get_dc, and also it's static so it's likely
-        # set already. Avoid invoking non-CDN methods by not syncing updates.
-        client.connect(_sync_updates=False)
+        session.auth_key = self._sender.auth_key
+        await client._sender.connect(self._connection(
+            session.server_address,
+            session.port,
+            session.dc_id,
+            loggers=self._log,
+            proxy=self._proxy,
+            local_addr=self._local_addr
+        ))
         return client
 
     # endregion
@@ -661,16 +940,17 @@ class TelegramBaseClient(abc.ABC):
                 executed sequentially on the server. They run in arbitrary
                 order by default.
 
+            flood_sleep_threshold (`int` | `None`, optional):
+                The flood sleep threshold to use for this request. This overrides
+                the default value stored in
+                `client.flood_sleep_threshold <telethon.client.telegrambaseclient.TelegramBaseClient.flood_sleep_threshold>`
+
         Returns:
             The result of the request (often a `TLObject`) or a list of
             results if more than one request was given.
         """
         raise NotImplementedError
 
-    @abc.abstractmethod
-    def _handle_update(self: 'TelegramClient', update):
-        raise NotImplementedError
-
     @abc.abstractmethod
     def _update_loop(self: 'TelegramClient'):
         raise NotImplementedError

telethon/client/telegramclient.py

@@ -1,13 +1,13 @@
 from . import (
     AccountMethods, AuthMethods, DownloadMethods, DialogMethods, ChatMethods,
     BotMethods, MessageMethods, UploadMethods, ButtonMethods, UpdateMethods,
-    MessageParseMethods, UserMethods
+    MessageParseMethods, UserMethods, TelegramBaseClient
 )
 
 
 class TelegramClient(
     AccountMethods, AuthMethods, DownloadMethods, DialogMethods, ChatMethods,
     BotMethods, MessageMethods, UploadMethods, ButtonMethods, UpdateMethods,
-    MessageParseMethods, UserMethods
+    MessageParseMethods, UserMethods, TelegramBaseClient
 ):
     pass

telethon/client/updates.py

@@ -1,19 +1,31 @@
 import asyncio
+import inspect
 import itertools
 import random
+import sys
 import time
+import traceback
 import typing
+import logging
+import warnings
+from collections import deque
+import sqlite3
 
-from .users import UserMethods
 from .. import events, utils, errors
 from ..events.common import EventBuilder, EventCommon
 from ..tl import types, functions
+from .._updates import GapError, PrematureEndReason
+from ..helpers import get_running_loop
+from ..version import __version__
+
 
 if typing.TYPE_CHECKING:
     from .telegramclient import TelegramClient
 
 
-class UpdateMethods(UserMethods):
+Callback = typing.Callable[[typing.Any], typing.Any]
+
+class UpdateMethods:
 
     # region Public methods
 
@@ -21,18 +33,34 @@ class UpdateMethods(UserMethods):
         try:
             # Make a high-level request to notify that we want updates
             await self(functions.updates.GetStateRequest())
-            return await self.disconnected
+            result = await self.disconnected
+            if self._updates_error is not None:
+                raise self._updates_error
+            return result
         except KeyboardInterrupt:
             pass
         finally:
             await self.disconnect()
 
+    async def set_receive_updates(self: 'TelegramClient', receive_updates):
+        """
+        Change the value of `receive_updates`.
+
+        This is an `async` method, because in order for Telegram to start
+        sending updates again, a request must be made.
+        """
+        self._no_updates = not receive_updates
+        if receive_updates:
+            await self(functions.updates.GetStateRequest())
+
     def run_until_disconnected(self: 'TelegramClient'):
         """
         Runs the event loop until the library is disconnected.
 
         It also notifies Telegram that we want to receive updates
         as described in https://core.telegram.org/api/updates.
+        If an unexpected error occurs during update handling,
+        the client will disconnect and said error will be raised.
 
         Manual disconnections can be made by calling `disconnect()
         <telethon.client.telegrambaseclient.TelegramBaseClient.disconnect>`
@@ -60,7 +88,7 @@ class UpdateMethods(UserMethods):
                 #
                 # You will still receive updates, since this prevents the
                 # script from exiting.
-                client.run_until_disconnected()
+                await client.run_until_disconnected()
         """
         if self.loop.is_running():
             return self._run_until_disconnected()
@@ -101,7 +129,7 @@ class UpdateMethods(UserMethods):
 
     def add_event_handler(
             self: 'TelegramClient',
-            callback: callable,
+            callback: Callback,
             event: EventBuilder = None):
         """
         Registers a new event handler callback.
@@ -150,7 +178,7 @@ class UpdateMethods(UserMethods):
 
     def remove_event_handler(
             self: 'TelegramClient',
-            callback: callable,
+            callback: Callback,
             event: EventBuilder = None) -> int:
         """
         Inverse operation of `add_event_handler()`.
@@ -188,7 +216,7 @@ class UpdateMethods(UserMethods):
         return found
 
     def list_event_handlers(self: 'TelegramClient')\
-            -> 'typing.Sequence[typing.Tuple[callable, EventBuilder]]':
+            -> 'typing.Sequence[typing.Tuple[Callback, EventBuilder]]':
         """
         Lists all registered event handlers.
 
@@ -219,114 +247,244 @@ class UpdateMethods(UserMethods):
         Example
             .. code-block:: python
 
-                client.catch_up()
+                await client.catch_up()
         """
-        pts, date = self._state_cache[None]
-        if not pts:
-            return
-
-        self.session.catching_up = True
-        try:
-            while True:
-                d = await self(functions.updates.GetDifferenceRequest(
-                    pts, date, 0
-                ))
-                if isinstance(d, (types.updates.DifferenceSlice,
-                                  types.updates.Difference)):
-                    if isinstance(d, types.updates.Difference):
-                        state = d.state
-                    else:
-                        state = d.intermediate_state
-
-                    pts, date = state.pts, state.date
-                    self._handle_update(types.Updates(
-                        users=d.users,
-                        chats=d.chats,
-                        date=state.date,
-                        seq=state.seq,
-                        updates=d.other_updates + [
-                            types.UpdateNewMessage(m, 0, 0)
-                            for m in d.new_messages
-                        ]
-                    ))
-
-                    # TODO Implement upper limit (max_pts)
-                    # We don't want to fetch updates we already know about.
-                    #
-                    # We may still get duplicates because the Difference
-                    # contains a lot of updates and presumably only has
-                    # the state for the last one, but at least we don't
-                    # unnecessarily fetch too many.
-                    #
-                    # updates.getDifference's pts_total_limit seems to mean
-                    # "how many pts is the request allowed to return", and
-                    # if there is more than that, it returns "too long" (so
-                    # there would be duplicate updates since we know about
-                    # some). This can be used to detect collisions (i.e.
-                    # it would return an update we have already seen).
-                else:
-                    if isinstance(d, types.updates.DifferenceEmpty):
-                        date = d.date
-                    elif isinstance(d, types.updates.DifferenceTooLong):
-                        pts = d.pts
-                    break
-        except (ConnectionError, asyncio.CancelledError):
-            pass
-        finally:
-            # TODO Save new pts to session
-            self._state_cache._pts_date = (pts, date)
-            self.session.catching_up = False
+        await self._updates_queue.put(types.UpdatesTooLong())
 
     # endregion
 
     # region Private methods
 
-    # It is important to not make _handle_update async because we rely on
-    # the order that the updates arrive in to update the pts and date to
-    # be always-increasing. There is also no need to make this async.
-    def _handle_update(self: 'TelegramClient', update):
-        self.session.process_entities(update)
-        self._entity_cache.add(update)
-
-        if isinstance(update, (types.Updates, types.UpdatesCombined)):
-            entities = {utils.get_peer_id(x): x for x in
-                        itertools.chain(update.users, update.chats)}
-            for u in update.updates:
-                self._process_update(u, entities)
-        elif isinstance(update, types.UpdateShort):
-            self._process_update(update.update)
-        else:
-            self._process_update(update)
-
-        self._state_cache.update(update)
-
-    def _process_update(self: 'TelegramClient', update, entities=None):
-        update._entities = entities or {}
-
-        # This part is somewhat hot so we don't bother patching
-        # update with channel ID/its state. Instead we just pass
-        # arguments which is faster.
-        channel_id = self._state_cache.get_channel_id(update)
-        args = (update, channel_id, self._state_cache[channel_id])
-        if self._dispatching_updates_queue is None:
-            task = self._loop.create_task(self._dispatch_update(*args))
-            self._updates_queue.add(task)
-            task.add_done_callback(lambda _: self._updates_queue.discard(task))
-        else:
-            self._updates_queue.put_nowait(args)
-            if not self._dispatching_updates_queue.is_set():
-                self._dispatching_updates_queue.set()
-                self._loop.create_task(self._dispatch_queue_updates())
-
-        self._state_cache.update(update)
-
     async def _update_loop(self: 'TelegramClient'):
+        # If the MessageBox is not empty, the account had to be logged-in to fill in its state.
+        # This flag is used to propagate the "you got logged-out" error up (but getting logged-out
+        # can only happen if it was once logged-in).
+        was_once_logged_in = self._authorized is True or not self._message_box.is_empty()
+
+        self._updates_error = None
+        try:
+            if self._catch_up:
+                # User wants to catch up as soon as the client is up and running,
+                # so this is the best place to do it.
+                await self.catch_up()
+
+            updates_to_dispatch = deque()
+
+            while self.is_connected():
+                if updates_to_dispatch:
+                    if self._sequential_updates:
+                        await self._dispatch_update(updates_to_dispatch.popleft())
+                    else:
+                        while updates_to_dispatch:
+                            # TODO if _dispatch_update fails for whatever reason, it's not logged! this should be fixed
+                            task = self.loop.create_task(self._dispatch_update(updates_to_dispatch.popleft()))
+                            self._event_handler_tasks.add(task)
+                            task.add_done_callback(self._event_handler_tasks.discard)
+
+                    continue
+
+                if len(self._mb_entity_cache) >= self._entity_cache_limit:
+                    self._log[__name__].info(
+                        'In-memory entity cache limit reached (%s/%s), flushing to session',
+                        len(self._mb_entity_cache),
+                        self._entity_cache_limit
+                    )
+                    self._save_states_and_entities()
+                    self._mb_entity_cache.retain(lambda id: id == self._mb_entity_cache.self_id or id in self._message_box.map)
+                    if len(self._mb_entity_cache) >= self._entity_cache_limit:
+                        warnings.warn('in-memory entities exceed entity_cache_limit after flushing; consider setting a larger limit')
+
+                    self._log[__name__].info(
+                        'In-memory entity cache at %s/%s after flushing to session',
+                        len(self._mb_entity_cache),
+                        self._entity_cache_limit
+                    )
+
+
+                get_diff = self._message_box.get_difference()
+                if get_diff:
+                    self._log[__name__].debug('Getting difference for account updates')
+                    try:
+                        diff = await self(get_diff)
+                    except (
+                        errors.ServerError,
+                        errors.TimedOutError,
+                        errors.FloodWaitError,
+                        ValueError
+                    ) as e:
+                        # Telegram is having issues
+                        self._log[__name__].info('Cannot get difference since Telegram is having issues: %s', type(e).__name__)
+                        self._message_box.end_difference()
+                        continue
+                    except (errors.UnauthorizedError, errors.AuthKeyError) as e:
+                        # Not logged in or broken authorization key, can't get difference
+                        self._log[__name__].info('Cannot get difference since the account is not logged in: %s', type(e).__name__)
+                        self._message_box.end_difference()
+                        if was_once_logged_in:
+                            self._updates_error = e
+                            await self.disconnect()
+                            break
+                        continue
+                    except (errors.TypeNotFoundError, sqlite3.OperationalError) as e:
+                        # User is likely doing weird things with their account or session and Telegram gets confused as to what layer they use
+                        self._log[__name__].warning('Cannot get difference since the account is likely misusing the session: %s', e)
+                        self._message_box.end_difference()
+                        self._updates_error = e
+                        await self.disconnect()
+                        break
+                    except OSError as e:
+                        # Network is likely down, but it's unclear for how long.
+                        # If disconnect is called this task will be cancelled along with the sleep.
+                        # If disconnect is not called, getting difference should be retried after a few seconds.
+                        self._log[__name__].info('Cannot get difference since the network is down: %s: %s', type(e).__name__, e)
+                        await asyncio.sleep(5)
+                        continue
+                    updates, users, chats = self._message_box.apply_difference(diff, self._mb_entity_cache)
+                    if updates:
+                        self._log[__name__].info('Got difference for account updates')
+
+                    updates_to_dispatch.extend(self._preprocess_updates(updates, users, chats))
+                    continue
+
+                get_diff = self._message_box.get_channel_difference(self._mb_entity_cache)
+                if get_diff:
+                    self._log[__name__].debug('Getting difference for channel %s updates', get_diff.channel.channel_id)
+                    try:
+                        diff = await self(get_diff)
+                    except (errors.UnauthorizedError, errors.AuthKeyError) as e:
+                        # Not logged in or broken authorization key, can't get difference
+                        self._log[__name__].warning(
+                            'Cannot get difference for channel %s since the account is not logged in: %s',
+                            get_diff.channel.channel_id, type(e).__name__
+                        )
+                        self._message_box.end_channel_difference(
+                            get_diff,
+                            PrematureEndReason.TEMPORARY_SERVER_ISSUES,
+                            self._mb_entity_cache
+                        )
+                        if was_once_logged_in:
+                            self._updates_error = e
+                            await self.disconnect()
+                            break
+                        continue
+                    except (errors.TypeNotFoundError, sqlite3.OperationalError) as e:
+                        self._log[__name__].warning(
+                            'Cannot get difference for channel %s since the account is likely misusing the session: %s',
+                            get_diff.channel.channel_id, e
+                        )
+                        self._message_box.end_channel_difference(
+                            get_diff,
+                            PrematureEndReason.TEMPORARY_SERVER_ISSUES,
+                            self._mb_entity_cache
+                        )
+                        self._updates_error = e
+                        await self.disconnect()
+                        break
+                    except (
+                        errors.PersistentTimestampOutdatedError,
+                        errors.PersistentTimestampInvalidError,
+                        errors.ServerError,
+                        errors.TimedOutError,
+                        errors.FloodWaitError,
+                        ValueError
+                    ) as e:
+                        # According to Telegram's docs:
+                        # "Channel internal replication issues, try again later (treat this like an RPC_CALL_FAIL)."
+                        # We can treat this as "empty difference" and not update the local pts.
+                        # Then this same call will be retried when another gap is detected or timeout expires.
+                        #
+                        # Another option would be to literally treat this like an RPC_CALL_FAIL and retry after a few
+                        # seconds, but if Telegram is having issues it's probably best to wait for it to send another
+                        # update (hinting it may be okay now) and retry then.
+                        #
+                        # This is a bit hacky because MessageBox doesn't really have a way to "not update" the pts.
+                        # Instead we manually extract the previously-known pts and use that.
+                        #
+                        # For PersistentTimestampInvalidError:
+                        # Somehow our pts is either too new or the server does not know about this.
+                        # We treat this as PersistentTimestampOutdatedError for now.
+                        # TODO investigate why/when this happens and if this is the proper solution
+                        self._log[__name__].warning(
+                            'Getting difference for channel updates %s caused %s;'
+                            ' ending getting difference prematurely until server issues are resolved',
+                            get_diff.channel.channel_id, type(e).__name__
+                        )
+                        self._message_box.end_channel_difference(
+                            get_diff,
+                            PrematureEndReason.TEMPORARY_SERVER_ISSUES,
+                            self._mb_entity_cache
+                        )
+                        continue
+                    except (errors.ChannelPrivateError, errors.ChannelInvalidError):
+                        # Timeout triggered a get difference, but we have been banned in the channel since then.
+                        # Because we can no longer fetch updates from this channel, we should stop keeping track
+                        # of it entirely.
+                        self._log[__name__].info(
+                            'Account is now banned in %d so we can no longer fetch updates from it',
+                            get_diff.channel.channel_id
+                        )
+                        self._message_box.end_channel_difference(
+                            get_diff,
+                            PrematureEndReason.BANNED,
+                            self._mb_entity_cache
+                        )
+                        continue
+                    except OSError as e:
+                        self._log[__name__].info(
+                            'Cannot get difference for channel %d since the network is down: %s: %s',
+                            get_diff.channel.channel_id, type(e).__name__, e
+                        )
+                        await asyncio.sleep(5)
+                        continue
+
+                    updates, users, chats = self._message_box.apply_channel_difference(get_diff, diff, self._mb_entity_cache)
+                    if updates:
+                        self._log[__name__].info('Got difference for channel %d updates', get_diff.channel.channel_id)
+
+                    updates_to_dispatch.extend(self._preprocess_updates(updates, users, chats))
+                    continue
+
+                deadline = self._message_box.check_deadlines()
+                deadline_delay = deadline - get_running_loop().time()
+                if deadline_delay > 0:
+                    # Don't bother sleeping and timing out if the delay is already 0 (pollutes the logs).
+                    try:
+                        updates = await asyncio.wait_for(self._updates_queue.get(), deadline_delay)
+                    except asyncio.TimeoutError:
+                        self._log[__name__].debug('Timeout waiting for updates expired')
+                        continue
+                else:
+                    continue
+
+                processed = []
+                try:
+                    users, chats = self._message_box.process_updates(updates, self._mb_entity_cache, processed)
+                except GapError:
+                    continue  # get(_channel)_difference will start returning requests
+
+                updates_to_dispatch.extend(self._preprocess_updates(processed, users, chats))
+        except asyncio.CancelledError:
+            pass
+        except Exception as e:
+            self._log[__name__].exception(f'Fatal error handling updates (this is a bug in Telethon v{__version__}, please report it)')
+            self._updates_error = e
+            await self.disconnect()
+
+    def _preprocess_updates(self, updates, users, chats):
+        self._mb_entity_cache.extend(users, chats)
+        entities = {utils.get_peer_id(x): x
+                    for x in itertools.chain(users, chats)}
+        for u in updates:
+            u._entities = entities
+        return updates
+
+    async def _keepalive_loop(self: 'TelegramClient'):
         # Pings' ID don't really need to be secure, just "random"
         rnd = lambda: random.randrange(-2**63, 2**63)
         while self.is_connected():
             try:
                 await asyncio.wait_for(
-                    self.disconnected, timeout=60, loop=self._loop
+                    self.disconnected, timeout=60
                 )
                 continue  # We actually just want to act upon timeout
             except asyncio.TimeoutError:
@@ -336,10 +494,19 @@ class UpdateMethods(UserMethods):
             except Exception:
                 continue  # Any disconnected exception should be ignored
 
+            # Check if we have any exported senders to clean-up periodically
+            await self._clean_exported_senders()
+
+            # Don't bother sending pings until the low-level connection is
+            # ready, otherwise a lot of pings will be batched to be sent upon
+            # reconnect, when we really don't care about that.
+            if not self._sender._transport_connected():
+                continue
+
             # We also don't really care about their result.
             # Just send them periodically.
             try:
-                self._sender.send(functions.PingRequest(rnd()))
+                self._sender._keepalive_ping(rnd())
             except (ConnectionError, asyncio.CancelledError):
                 return
 
@@ -347,41 +514,27 @@ class UpdateMethods(UserMethods):
             # inserted because this is a rather expensive operation
             # (default's sqlite3 takes ~0.1s to commit changes). Do
             # it every minute instead. No-op if there's nothing new.
+            self._save_states_and_entities()
+
             self.session.save()
 
-            # We need to send some content-related request at least hourly
-            # for Telegram to keep delivering updates, otherwise they will
-            # just stop even if we're connected. Do so every 30 minutes.
+    async def _dispatch_update(self: 'TelegramClient', update):
+        # TODO only used for AlbumHack, and MessageBox is not really designed for this
+        others = None
+
+        if not self._mb_entity_cache.self_id:
+            # Some updates require our own ID, so we must make sure
+            # that the event builder has offline access to it. Calling
+            # `get_me()` will cache it under `self._mb_entity_cache`.
             #
-            # TODO Call getDifference instead since it's more relevant
-            if time.time() - self._last_request > 30 * 60:
-                if not await self.is_user_authorized():
-                    # What can be the user doing for so
-                    # long without being logged in...?
-                    continue
+            # It will return `None` if we haven't logged in yet which is
+            # fine, we will just retry next time anyway.
+            try:
+                await self.get_me(input_peer=True)
+            except OSError:
+                pass  # might not have connection
 
-                try:
-                    await self(functions.updates.GetStateRequest())
-                except (ConnectionError, asyncio.CancelledError):
-                    return
-
-    async def _dispatch_queue_updates(self: 'TelegramClient'):
-        while not self._updates_queue.empty():
-            await self._dispatch_update(*self._updates_queue.get_nowait())
-
-        self._dispatching_updates_queue.clear()
-
-    async def _dispatch_update(self: 'TelegramClient', update, channel_id, pts_date):
-        if not self._entity_cache.ensure_cached(update):
-            # We could add a lock to not fetch the same pts twice if we are
-            # already fetching it. However this does not happen in practice,
-            # which makes sense, because different updates have different pts.
-            if self._state_cache.update(update, check_only=True):
-                # If the update doesn't have pts, fetching won't do anything.
-                # For example, UpdateUserStatus or UpdateChatUserTyping.
-                await self._get_difference(update, channel_id, pts_date)
-
-        built = EventBuilderDict(self, update)
+        built = EventBuilderDict(self, update, others)
         for conv_set in self._conversations.values():
             for conv in conv_set:
                 ev = built[events.NewMessage]
@@ -407,7 +560,10 @@ class UpdateMethods(UserMethods):
             if not builder.resolved:
                 await builder.resolve(self)
 
-            if not builder.filter(event):
+            filter = builder.filter(event)
+            if inspect.isawaitable(filter):
+                filter = await filter
+            if not filter:
                 continue
 
             try:
@@ -427,67 +583,59 @@ class UpdateMethods(UserMethods):
             except Exception as e:
                 if not isinstance(e, asyncio.CancelledError) or self.is_connected():
                     name = getattr(callback, '__name__', repr(callback))
-                    self._log[__name__].exception('Unhandled exception on %s',
-                                                  name)
+                    self._log[__name__].exception('Unhandled exception on %s', name)
 
-    async def _get_difference(self: 'TelegramClient', update, channel_id, pts_date):
+    async def _dispatch_event(self: 'TelegramClient', event):
         """
-        Get the difference for this `channel_id` if any, then load entities.
+        Dispatches a single, out-of-order event. Used by `AlbumHack`.
+        """
+        # We're duplicating a most logic from `_dispatch_update`, but all in
+        # the name of speed; we don't want to make it worse for all updates
+        # just because albums may need it.
+        for builder, callback in self._event_builders:
+            if isinstance(builder, events.Raw):
+                continue
+            if not isinstance(event, builder.Event):
+                continue
+
+            if not builder.resolved:
+                await builder.resolve(self)
+
+            filter = builder.filter(event)
+            if inspect.isawaitable(filter):
+                filter = await filter
+            if not filter:
+                continue
 
-        Calls :tl:`updates.getDifference`, which fills the entities cache
-        (always done by `__call__`) and lets us know about the full entities.
-        """
-        # Fetch since the last known pts/date before this update arrived,
-        # in order to fetch this update at full, including its entities.
-        self._log[__name__].debug('Getting difference for entities '
-                                  'for %r', update.__class__)
-        if channel_id:
             try:
-                where = await self.get_input_entity(channel_id)
-            except ValueError:
-                # There's a high chance that this fails, since
-                # we are getting the difference to fetch entities.
-                return
-
-            if not pts_date:
-                # First-time, can't get difference. Get pts instead.
-                result = await self(functions.messages.GetPeerDialogsRequest([
-                    utils.get_input_dialog(where)
-                ]))
-                self._state_cache[channel_id] = result.dialogs[0].pts
-                return
-
-            result = await self(functions.updates.GetChannelDifferenceRequest(
-                channel=where,
-                filter=types.ChannelMessagesFilterEmpty(),
-                pts=pts_date,  # just pts
-                limit=100,
-                force=True
-            ))
-        else:
-            if not pts_date[0]:
-                # First-time, can't get difference. Get pts instead.
-                result = await self(functions.updates.GetStateRequest())
-                self._state_cache[None] = result.pts, result.date
-                return
-
-            result = await self(functions.updates.GetDifferenceRequest(
-                pts=pts_date[0],
-                date=pts_date[1],
-                qts=0
-            ))
-
-        if isinstance(result, (types.updates.Difference,
-                               types.updates.DifferenceSlice,
-                               types.updates.ChannelDifference,
-                               types.updates.ChannelDifferenceTooLong)):
-            update._entities.update({
-                utils.get_peer_id(x): x for x in
-                itertools.chain(result.users, result.chats)
-            })
+                await callback(event)
+            except errors.AlreadyInConversationError:
+                name = getattr(callback, '__name__', repr(callback))
+                self._log[__name__].debug(
+                    'Event handler "%s" already has an open conversation, '
+                    'ignoring new one', name)
+            except events.StopPropagation:
+                name = getattr(callback, '__name__', repr(callback))
+                self._log[__name__].debug(
+                    'Event handler "%s" stopped chain of propagation '
+                    'for event %s.', name, type(event).__name__
+                )
+                break
+            except Exception as e:
+                if not isinstance(e, asyncio.CancelledError) or self.is_connected():
+                    name = getattr(callback, '__name__', repr(callback))
+                    self._log[__name__].exception('Unhandled exception on %s', name)
 
     async def _handle_auto_reconnect(self: 'TelegramClient'):
         # TODO Catch-up
+        # For now we make a high-level request to let Telegram
+        # know we are still interested in receiving more updates.
+        try:
+            await self.get_me()
+        except Exception as e:
+            self._log[__name__].warning('Error executing high-level request '
+                                        'after reconnect: %s: %s', type(e), e)
+
         return
         try:
             self._log[__name__].info(
@@ -518,8 +666,8 @@ class UpdateMethods(UserMethods):
             self._log[__name__].warning('Failed to get missed updates after '
                                         'reconnect: %r', e)
         except Exception:
-            self._log[__name__].exception('Unhandled exception while getting '
-                                          'update difference after reconnect')
+            self._log[__name__].exception(
+                'Unhandled exception while getting update difference after reconnect')
 
     # endregion
 
@@ -528,17 +676,21 @@ class EventBuilderDict:
     """
     Helper "dictionary" to return events from types and cache them.
     """
-    def __init__(self, client: 'TelegramClient', update):
+    def __init__(self, client: 'TelegramClient', update, others):
         self.client = client
         self.update = update
+        self.others = others
 
     def __getitem__(self, builder):
         try:
             return self.__dict__[builder]
         except KeyError:
-            event = self.__dict__[builder] = builder.build(self.update)
+            event = self.__dict__[builder] = builder.build(
+                self.update, self.others, self.client._self_id)
+
             if isinstance(event, EventCommon):
                 event.original_update = self.update
+                event._entities = self.update._entities
                 event._set_client(self.client)
             elif event:
                 event._client = self.client

telethon/client/uploads.py

@@ -7,9 +7,8 @@ import re
 import typing
 from io import BytesIO
 
-from .buttons import ButtonMethods
-from .messageparse import MessageParseMethods
-from .users import UserMethods
+from ..crypto import AES
+
 from .. import utils, helpers, hints
 from ..tl import types, functions, custom
 
@@ -19,7 +18,6 @@ try:
 except ImportError:
     PIL = None
 
-
 if typing.TYPE_CHECKING:
     from .telegramclient import TelegramClient
 
@@ -37,7 +35,7 @@ class _CacheType:
 
 
 def _resize_photo_if_needed(
-        file, is_image, width=1280, height=1280, background=(255, 255, 255)):
+        file, is_image, width=2560, height=2560, background=(255, 255, 255)):
 
     # https://github.com/telegramdesktop/tdesktop/blob/12905f0dcb9d513378e7db11989455a1b764ef75/Telegram/SourceFiles/boxes/photo_crop_box.cpp#L254
     if (not is_image
@@ -48,43 +46,67 @@ def _resize_photo_if_needed(
     if isinstance(file, bytes):
         file = io.BytesIO(file)
 
-    before = file.tell() if isinstance(file, io.IOBase) else None
+    if isinstance(file, io.IOBase):
+        # Pillow seeks to 0 unconditionally later anyway
+        old_pos = file.tell()
+        file.seek(0, io.SEEK_END)
+        before = file.tell()
+    elif isinstance(file, str) and os.path.exists(file):
+        # Check if file exists as a path and if so, get its size on disk
+        before = os.path.getsize(file)
+    else:
+        # Would be weird...
+        before = None
 
     try:
         # Don't use a `with` block for `image`, or `file` would be closed.
         # See https://github.com/LonamiWebs/Telethon/issues/1121 for more.
         image = PIL.Image.open(file)
-        if image.width <= width and image.height <= height:
-            return file
+        try:
+            kwargs = {'exif': image.info['exif']}
+        except KeyError:
+            kwargs = {}
 
-        image.thumbnail((width, height), PIL.Image.ANTIALIAS)
+        if image.mode == 'RGB':
+            # Check if image is within acceptable bounds, if so, check if the image is at or below 10 MB, or assume it isn't if size is None or 0
+            if image.width <= width and image.height <= height and (before <= 10000000 if before else False):
+                return file
 
-        alpha_index = image.mode.find('A')
-        if alpha_index == -1:
-            # If the image mode doesn't have alpha
-            # channel then don't bother masking it away.
+            # If the image is already RGB, don't convert it
+            # certain modes such as 'P' have no alpha index but can't be saved as JPEG directly
+            image.thumbnail((width, height), PIL.Image.LANCZOS)
             result = image
         else:
             # We could save the resized image with the original format, but
             # JPEG often compresses better -> smaller size -> faster upload
             # We need to mask away the alpha channel ([3]), since otherwise
             # IOError is raised when trying to save alpha channels in JPEG.
+            image.thumbnail((width, height), PIL.Image.LANCZOS)
             result = PIL.Image.new('RGB', image.size, background)
-            result.paste(image, mask=image.split()[alpha_index])
+            mask = None
+
+            if image.has_transparency_data:
+                if image.mode == 'RGBA':
+                    mask = image.getchannel('A')
+                else:
+                    mask = image.convert('RGBA').getchannel('A')
+
+            result.paste(image, mask=mask)
 
         buffer = io.BytesIO()
-        result.save(buffer, 'JPEG')
+        result.save(buffer, 'JPEG', progressive=True, **kwargs)
         buffer.seek(0)
+        buffer.name = 'a.jpg'
         return buffer
-
     except IOError:
         return file
     finally:
-        if before is not None:
-            file.seek(before, io.SEEK_SET)
+        # The original position might matter
+        if isinstance(file, io.IOBase):
+            file.seek(old_pos)
 
 
-class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
+class UploadMethods:
 
     # region Public methods
 
@@ -95,18 +117,32 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
             *,
             caption: typing.Union[str, typing.Sequence[str]] = None,
             force_document: bool = False,
+            file_size: int = None,
+            clear_draft: bool = False,
             progress_callback: 'hints.ProgressCallback' = None,
             reply_to: 'hints.MessageIDLike' = None,
             attributes: 'typing.Sequence[types.TypeDocumentAttribute]' = None,
             thumb: 'hints.FileLike' = None,
             allow_cache: bool = True,
             parse_mode: str = (),
+            formatting_entities: typing.Optional[
+                typing.Union[
+                    typing.List[types.TypeMessageEntity], typing.List[typing.List[types.TypeMessageEntity]]
+                ]
+            ] = None,
             voice_note: bool = False,
             video_note: bool = False,
-            buttons: 'hints.MarkupLike' = None,
+            buttons: typing.Optional['hints.MarkupLike'] = None,
             silent: bool = None,
+            background: bool = None,
             supports_streaming: bool = False,
-            **kwargs) -> 'types.Message':
+            schedule: 'hints.DateLike' = None,
+            comment_to: 'typing.Union[int, types.Message]' = None,
+            ttl: int = None,
+            nosound_video: bool = None,
+            send_as: typing.Optional['hints.EntityLike'] = None,
+            message_effect_id: typing.Optional[int] = None,
+            **kwargs) -> typing.Union[typing.List[typing.Any], typing.Any]:
         """
         Sends message with the given file to the specified entity.
 
@@ -153,6 +189,10 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
 
                 * A handle to an uploaded file (from `upload_file`).
 
+                * A :tl:`InputMedia` instance. For example, if you want to
+                  send a dice use :tl:`InputMediaDice`, or if you want to
+                  send a contact use :tl:`InputMediaContact`.
+
                 To send an album, you should provide a list in this parameter.
 
                 If a list or similar is provided, the files in it will be
@@ -165,10 +205,20 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
                 assigned to the files pairwise.
 
             force_document (`bool`, optional):
-                If left to ``False`` and the file is a path that ends with
+                If left to `False` and the file is a path that ends with
                 the extension of an image file or a video file, it will be
                 sent as such. Otherwise always as a document.
 
+            file_size (`int`, optional):
+                The size of the file to be uploaded if it needs to be uploaded,
+                which will be determined automatically if not specified.
+
+                If the file size can't be determined beforehand, the entire
+                file will be read in-memory to find out how large it is.
+
+            clear_draft (`bool`, optional):
+                Whether the existing draft should be cleared or not.
+
             progress_callback (`callable`, optional):
                 A callback function accepting two parameters:
                 ``(sent bytes, total)``.
@@ -184,15 +234,19 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
                 Optional JPEG thumbnail (for documents). **Telegram will
                 ignore this parameter** unless you pass a ``.jpg`` file!
 
-                The file must also be small in dimensions and in-disk size.
-                Successful thumbnails were files below 20kb and 200x200px.
+                The file must also be small in dimensions and in disk size.
+                Successful thumbnails were files below 20kB and 320x320px.
                 Width/height and dimensions/size ratios may be important.
+                For Telegram to accept a thumbnail, you must provide the
+                dimensions of the underlying media through ``attributes=``
+                with :tl:`DocumentAttributesVideo` or by installing the
+                optional ``hachoir`` dependency.
+
 
             allow_cache (`bool`, optional):
-                Whether to allow using the cached version stored in the
-                database or not. Defaults to ``True`` to avoid re-uploads.
-                Must be ``False`` if you wish to use different attributes
-                or thumb than those that were used when the file was cached.
+                This parameter currently does nothing, but is kept for
+                backward-compatibility (and it may get its use back in
+                the future).
 
             parse_mode (`object`, optional):
                 See the `TelegramClient.parse_mode
@@ -200,19 +254,20 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
                 property for allowed values. Markdown parsing will be used by
                 default.
 
-            voice_note (`bool`, optional):
-                If ``True`` the audio will be sent as a voice note.
+            formatting_entities (`list`, optional):
+                Optional formatting entities for the sent media message. When sending an album,
+                `formatting_entities` can be a list of lists, where each inner list contains
+                `types.TypeMessageEntity`. Each inner list will be assigned to the corresponding
+                file in a pairwise manner with the caption. If provided, the ``parse_mode``
+                parameter will be ignored.
 
-                Set `allow_cache` to ``False`` if you sent the same file
-                without this setting before for it to work.
+            voice_note (`bool`, optional):
+                If `True` the audio will be sent as a voice note.
 
             video_note (`bool`, optional):
-                If ``True`` the video will be sent as a video note,
+                If `True` the video will be sent as a video note,
                 also known as a round video message.
 
-                Set `allow_cache` to ``False`` if you sent the same file
-                without this setting before for it to work.
-
             buttons (`list`, `custom.Button <telethon.tl.custom.button.Button>`, :tl:`KeyboardButton`):
                 The matrix (list of lists), row list or button to be shown
                 after sending the message. This parameter will only work if
@@ -220,9 +275,13 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
                 :tl:`ReplyMarkup` here.
 
             silent (`bool`, optional):
-                Whether the message should notify people in a broadcast
-                channel or not. Defaults to ``False``, which means it will
-                notify them. Set it to ``True`` to alter this behaviour.
+                Whether the message should notify people with sound or not.
+                Defaults to `False` (send with a notification sound unless
+                the person has the chat muted). Set it to `True` to alter
+                this behaviour.
+
+            background (`bool`, optional):
+                Whether the message should be send in background.
 
             supports_streaming (`bool`, optional):
                 Whether the sent video supports streaming or not. Note that
@@ -231,6 +290,50 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
                 these to MP4 before sending if you want them to be streamable.
                 Unsupported formats will result in ``VideoContentTypeError``.
 
+            schedule (`hints.DateLike`, optional):
+                If set, the file won't send immediately, and instead
+                it will be scheduled to be automatically sent at a later
+                time.
+
+            comment_to (`int` | `Message <telethon.tl.custom.message.Message>`, optional):
+                Similar to ``reply_to``, but replies in the linked group of a
+                broadcast channel instead (effectively leaving a "comment to"
+                the specified message).
+
+                This parameter takes precedence over ``reply_to``. If there is
+                no linked chat, `telethon.errors.sgIdInvalidError` is raised.
+
+            ttl (`int`. optional):
+                The Time-To-Live of the file (also known as "self-destruct timer"
+                or "self-destructing media"). If set, files can only be viewed for
+                a short period of time before they disappear from the message
+                history automatically.
+
+                The value must be at least 1 second, and at most 60 seconds,
+                otherwise Telegram will ignore this parameter.
+
+                Not all types of media can be used with this parameter, such
+                as text documents, which will fail with ``TtlMediaInvalidError``.
+
+            nosound_video (`bool`, optional):
+                Only applicable when sending a video file without an audio
+                track. If set to ``True``, the video will be displayed in
+                Telegram as a video. If set to ``False``, Telegram will attempt
+                to display the video as an animated gif. (It may still display
+                as a video due to other factors.) The value is ignored if set
+                on non-video files. This is set to ``True`` for albums, as gifs
+                cannot be sent in albums.
+
+            send_as (`entity`):
+                Unique identifier (int) or username (str) of the chat or channel to send the message as.
+                You can use this to send the message on behalf of a chat or channel where you have appropriate permissions.
+                Use the GetSendAs to return the list of message sender identifiers, which can be used to send messages in the chat,
+                This setting applies to the current message and will remain effective for future messages unless explicitly changed.
+                To set this behavior permanently for all messages, use SaveDefaultSendAs.
+
+            message_effect_id (`int`, optional):
+                Unique identifier of the message effect to be added to the message; for private chats only
+
         Returns
             The `Message <telethon.tl.custom.message.Message>` (or messages)
             containing the sent file, or messages if a list of them was passed.
@@ -239,98 +342,120 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
             .. code-block:: python
 
                 # Normal files like photos
-                client.send_file(chat, '/my/photos/me.jpg', caption="It's me!")
+                await client.send_file(chat, '/my/photos/me.jpg', caption="It's me!")
                 # or
-                client.send_message(chat, "It's me!", file='/my/photos/me.jpg')
+                await client.send_message(chat, "It's me!", file='/my/photos/me.jpg')
 
                 # Voice notes or round videos
-                client.send_file(chat, '/my/songs/song.mp3', voice_note=True)
-                client.send_file(chat, '/my/videos/video.mp4', video_note=True)
+                await client.send_file(chat, '/my/songs/song.mp3', voice_note=True)
+                await client.send_file(chat, '/my/videos/video.mp4', video_note=True)
 
                 # Custom thumbnails
-                client.send_file(chat, '/my/documents/doc.txt', thumb='photo.jpg')
+                await client.send_file(chat, '/my/documents/doc.txt', thumb='photo.jpg')
 
                 # Only documents
-                client.send_file(chat, '/my/photos/photo.png', force_document=True)
+                await client.send_file(chat, '/my/photos/photo.png', force_document=True)
 
                 # Albums
-                client.send_file(chat, [
+                await client.send_file(chat, [
                     '/my/photos/holiday1.jpg',
                     '/my/photos/holiday2.jpg',
                     '/my/drawings/portrait.png'
                 ])
+
+                # Printing upload progress
+                def callback(current, total):
+                    print('Uploaded', current, 'out of', total,
+                          'bytes: {:.2%}'.format(current / total))
+
+                await client.send_file(chat, file, progress_callback=callback)
+
+                # Dices, including dart and other future emoji
+                from telethon.tl import types
+                await client.send_file(chat, types.InputMediaDice(''))
+                await client.send_file(chat, types.InputMediaDice('🎯'))
+
+                # Contacts
+                await client.send_file(chat, types.InputMediaContact(
+                    phone_number='+34 123 456 789',
+                    first_name='Example',
+                    last_name='',
+                    vcard=''
+                ))
         """
-        # i.e. ``None`` was used
+        # TODO Properly implement allow_cache to reuse the sha256 of the file
+        # i.e. `None` was used
         if not file:
             raise TypeError('Cannot use {!r} as file'.format(file))
 
         if not caption:
             caption = ''
 
+        if not formatting_entities:
+            formatting_entities = []
+
+        entity = await self.get_input_entity(entity)
+        if comment_to is not None:
+            entity, reply_to = await self._get_comment_data(entity, comment_to)
+        else:
+            reply_to = utils.get_message_id(reply_to)
+
         # First check if the user passed an iterable, in which case
-        # we may want to send as an album if all are photo files.
+        # we may want to send grouped.
         if utils.is_list_like(file):
-            image_captions = []
-            document_captions = []
+            sent_count = 0
+            used_callback = None if not progress_callback else (
+                lambda s, t: progress_callback(sent_count + s, len(file))
+            )
+
             if utils.is_list_like(caption):
                 captions = caption
             else:
                 captions = [caption]
 
-            # TODO Fix progress_callback
-            images = []
-            if force_document:
-                documents = file
+            # Check that formatting_entities list is valid
+            if all(utils.is_list_like(obj) for obj in formatting_entities):
+                formatting_entities = formatting_entities
+            elif utils.is_list_like(formatting_entities):
+                formatting_entities = [formatting_entities]
             else:
-                documents = []
-                for doc, cap in itertools.zip_longest(file, captions):
-                    if utils.is_image(doc):
-                        images.append(doc)
-                        image_captions.append(cap)
-                    else:
-                        documents.append(doc)
-                        document_captions.append(cap)
+                raise TypeError('The formatting_entities argument must be a list or a sequence of lists')
+
+            # Check that all entities in all lists are of the correct type
+            if not all(isinstance(ent, types.TypeMessageEntity) for sublist in formatting_entities for ent in sublist):
+                raise TypeError('All entities must be instances of <types.TypeMessageEntity>')
 
             result = []
-            while images:
+            while file:
                 result += await self._send_album(
-                    entity, images[:10], caption=image_captions[:10],
-                    progress_callback=progress_callback, reply_to=reply_to,
-                    parse_mode=parse_mode, silent=silent
+                    entity, file[:10], caption=captions[:10], formatting_entities=formatting_entities[:10],
+                    progress_callback=used_callback, reply_to=reply_to,
+                    parse_mode=parse_mode, silent=silent, schedule=schedule,
+                    supports_streaming=supports_streaming, clear_draft=clear_draft,
+                    force_document=force_document, background=background,
+                    send_as=send_as, message_effect_id=message_effect_id
                 )
-                images = images[10:]
-                image_captions = image_captions[10:]
-
-            for doc, cap in zip(documents, captions):
-                result.append(await self.send_file(
-                    entity, doc, allow_cache=allow_cache,
-                    caption=cap, force_document=force_document,
-                    progress_callback=progress_callback, reply_to=reply_to,
-                    attributes=attributes, thumb=thumb, voice_note=voice_note,
-                    video_note=video_note, buttons=buttons, silent=silent,
-                    supports_streaming=supports_streaming,
-                    **kwargs
-                ))
+                file = file[10:]
+                captions = captions[10:]
+                formatting_entities = formatting_entities[10:]
+                sent_count += 10
 
             return result
 
-        entity = await self.get_input_entity(entity)
-        reply_to = utils.get_message_id(reply_to)
-
-        # Not document since it's subject to change.
-        # Needed when a Message is passed to send_message and it has media.
-        if 'entities' in kwargs:
-            msg_entities = kwargs['entities']
+        if formatting_entities:
+            msg_entities = formatting_entities
         else:
             caption, msg_entities =\
                 await self._parse_message_text(caption, parse_mode)
 
         file_handle, media, image = await self._file_to_media(
             file, force_document=force_document,
+            file_size=file_size,
             progress_callback=progress_callback,
-            attributes=attributes,  allow_cache=allow_cache, thumb=thumb,
+            attributes=attributes, allow_cache=allow_cache, thumb=thumb,
             voice_note=voice_note, video_note=video_note,
-            supports_streaming=supports_streaming
+            supports_streaming=supports_streaming, ttl=ttl,
+            nosound_video=nosound_video,
         )
 
         # e.g. invalid cast from :tl:`MessageMediaWebPage`
@@ -338,18 +463,25 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
             raise TypeError('Cannot use {!r} as file'.format(file))
 
         markup = self.build_reply_markup(buttons)
+        reply_to = None if reply_to is None else types.InputReplyToMessage(reply_to)
         request = functions.messages.SendMediaRequest(
-            entity, media, reply_to_msg_id=reply_to, message=caption,
-            entities=msg_entities, reply_markup=markup, silent=silent
+            entity, media, reply_to=reply_to, message=caption,
+            entities=msg_entities, reply_markup=markup, silent=silent,
+            schedule_date=schedule, clear_draft=clear_draft,
+            background=background,
+            send_as=await self.get_input_entity(send_as) if send_as else None,
+            effect=message_effect_id
         )
-        msg = self._get_response_message(request, await self(request), entity)
-        await self._cache_media(msg, file, file_handle, image=image)
-
-        return msg
+        return self._get_response_message(request, await self(request), entity)
 
     async def _send_album(self: 'TelegramClient', entity, files, caption='',
+                          formatting_entities=None,
                           progress_callback=None, reply_to=None,
-                          parse_mode=(), silent=None):
+                          parse_mode=(), silent=None, schedule=None,
+                          supports_streaming=None, clear_draft=None,
+                          force_document=False, background=None, ttl=None,
+                          send_as: typing.Optional['hints.EntityLike'] = None,
+                          message_effect_id: typing.Optional[int] = None):
         """Specialized version of .send_file for albums"""
         # We don't care if the user wants to avoid cache, we will use it
         # anyway. Why? The cached version will be exactly the same thing
@@ -357,35 +489,57 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
         # cache only makes a difference for documents where the user may
         # want the attributes used on them to change.
         #
-        # In theory documents can be sent inside the albums but they appear
+        # In theory documents can be sent inside the albums, but they appear
         # as different messages (not inside the album), and the logic to set
         # the attributes/avoid cache is already written in .send_file().
         entity = await self.get_input_entity(entity)
         if not utils.is_list_like(caption):
             caption = (caption,)
+        if not all(isinstance(obj, list) for obj in formatting_entities):
+            formatting_entities = (formatting_entities,)
 
         captions = []
-        for c in reversed(caption):  # Pop from the end (so reverse)
-            captions.append(await self._parse_message_text(c or '', parse_mode))
+        # If the formatting_entities argument is provided, we don't use parse_mode
+        if formatting_entities:
+            # Pop from the end (so reverse)
+            capt_with_ent = itertools.zip_longest(reversed(caption), reversed(formatting_entities), fillvalue=None)
+            for msg_caption, msg_entities in capt_with_ent:
+                captions.append((msg_caption, msg_entities))
+        else:
+            for c in reversed(caption):  # Pop from the end (so reverse)
+                captions.append(await self._parse_message_text(c or '', parse_mode))
 
         reply_to = utils.get_message_id(reply_to)
 
+        used_callback = None if not progress_callback else (
+            # use an integer when sent matches total, to easily determine a file has been fully sent
+            lambda s, t: progress_callback(sent_count + 1 if s == t else sent_count + s / t, len(files))
+        )
+
         # Need to upload the media first, but only if they're not cached yet
         media = []
-        for file in files:
+        for sent_count, file in enumerate(files):
             # Albums want :tl:`InputMedia` which, in theory, includes
-            # :tl:`InputMediaUploadedPhoto`. However using that will
+            # :tl:`InputMediaUploadedPhoto`. However, using that will
             # make it `raise MediaInvalidError`, so we need to upload
             # it as media and then convert that to :tl:`InputMediaPhoto`.
-            fh, fm, _ = await self._file_to_media(file)
-            if isinstance(fm, types.InputMediaUploadedPhoto):
+            fh, fm, _ = await self._file_to_media(
+                file, supports_streaming=supports_streaming,
+                force_document=force_document, ttl=ttl,
+                progress_callback=used_callback, nosound_video=True)
+            if isinstance(fm, (types.InputMediaUploadedPhoto, types.InputMediaPhotoExternal)):
                 r = await self(functions.messages.UploadMediaRequest(
                     entity, media=fm
                 ))
-                self.session.cache_file(
-                    fh.md5, fh.size, utils.get_input_photo(r.photo))
 
                 fm = utils.get_input_media(r.photo)
+            elif isinstance(fm, (types.InputMediaUploadedDocument, types.InputMediaDocumentExternal)):
+                r = await self(functions.messages.UploadMediaRequest(
+                    entity, media=fm
+                ))
+
+                fm = utils.get_input_media(
+                    r.document, supports_streaming=supports_streaming)
 
             if captions:
                 caption, msg_entities = captions.pop()
@@ -395,36 +549,40 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
                 fm,
                 message=caption,
                 entities=msg_entities
+                # random_id is autogenerated
             ))
 
         # Now we can construct the multi-media request
-        result = await self(functions.messages.SendMultiMediaRequest(
-            entity, reply_to_msg_id=reply_to, multi_media=media, silent=silent
-        ))
+        request = functions.messages.SendMultiMediaRequest(
+            entity, reply_to=None if reply_to is None else types.InputReplyToMessage(reply_to), multi_media=media,
+            silent=silent, schedule_date=schedule, clear_draft=clear_draft,
+            background=background,
+            send_as=await self.get_input_entity(send_as) if send_as else None,
+            effect=message_effect_id
+        )
+        result = await self(request)
 
-        # We never sent a `random_id` for the messages that resulted from
-        # the request so we can't pair them up with the `Updates` that we
-        # get from Telegram. However, the sent messages have a photo and
-        # the photo IDs match with those we did send.
-        #
-        # Updates -> {_: message}
-        messages = self._get_response_message(None, result, entity)
-        # {_: message} -> {photo ID: message}
-        messages = {m.photo.id: m for m in messages.values()}
-        # Sent photo IDs -> messages
-        return [messages[m.media.id.id] for m in media]
+        random_ids = [m.random_id for m in media]
+        return self._get_response_message(random_ids, result, entity)
 
     async def upload_file(
             self: 'TelegramClient',
             file: 'hints.FileLike',
             *,
             part_size_kb: float = None,
+            file_size: int = None,
             file_name: str = None,
             use_cache: type = None,
+            key: bytes = None,
+            iv: bytes = None,
             progress_callback: 'hints.ProgressCallback' = None) -> 'types.TypeInputFile':
         """
         Uploads a file to Telegram's servers, without sending it.
 
+        .. note::
+
+            Generally, you want to use `send_file` instead.
+
         This method returns a handle (an instance of :tl:`InputFile` or
         :tl:`InputFileBig`, as required) which can be later used before
         it expires (they are usable during less than a day).
@@ -444,22 +602,40 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
                 Chunk size when uploading files. The larger, the less
                 requests will be made (up to 512KB maximum).
 
+            file_size (`int`, optional):
+                The size of the file to be uploaded, which will be determined
+                automatically if not specified.
+
+                If the file size can't be determined beforehand, the entire
+                file will be read in-memory to find out how large it is.
+
             file_name (`str`, optional):
                 The file name which will be used on the resulting InputFile.
                 If not specified, the name will be taken from the ``file``
-                and if this is not a ``str``, it will be ``"unnamed"``.
+                and if this is not a `str`, it will be ``"unnamed"``.
 
             use_cache (`type`, optional):
-                The type of cache to use (currently either :tl:`InputDocument`
-                or :tl:`InputPhoto`). If present and the file is small enough
-                to need the MD5, it will be checked against the database,
-                and if a match is found, the upload won't be made. Instead,
-                an instance of type ``use_cache`` will be returned.
+                This parameter currently does nothing, but is kept for
+                backward-compatibility (and it may get its use back in
+                the future).
+
+            key ('bytes', optional):
+                In case of an encrypted upload (secret chats) a key is supplied
+
+            iv ('bytes', optional):
+                In case of an encrypted upload (secret chats) an iv is supplied
 
             progress_callback (`callable`, optional):
                 A callback function accepting two parameters:
                 ``(sent bytes, total)``.
 
+                When sending an album, the callback will receive a number
+                between 0 and the amount of files as the "sent" parameter,
+                and the amount of files as the "total". Note that the first
+                parameter will be a floating point number to indicate progress
+                within a file (e.g. ``2.5`` means it has sent 50% of the third
+                file, because it's between 2 and 3).
+
         Returns
             :tl:`InputFileBig` if the file size is larger than 10MB,
             `InputSizedFile <telethon.tl.custom.inputsizedfile.InputSizedFile>`
@@ -469,101 +645,89 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
             .. code-block:: python
 
                 # Photos as photo and document
-                file = client.upload_file('photo.jpg')
-                client.send_file(chat, file)                       # sends as photo
-                client.send_file(chat, file, force_document=True)  # sends as document
+                file = await client.upload_file('photo.jpg')
+                await client.send_file(chat, file)                       # sends as photo
+                await client.send_file(chat, file, force_document=True)  # sends as document
 
                 file.name = 'not a photo.jpg'
-                client.send_file(chat, file, force_document=True)  # document, new name
+                await client.send_file(chat, file, force_document=True)  # document, new name
 
                 # As song or as voice note
-                file = client.upload_file('song.ogg')
-                client.send_file(chat, file)                   # sends as song
-                client.send_file(chat, file, voice_note=True)  # sends as voice note
+                file = await client.upload_file('song.ogg')
+                await client.send_file(chat, file)                   # sends as song
+                await client.send_file(chat, file, voice_note=True)  # sends as voice note
         """
         if isinstance(file, (types.InputFile, types.InputFileBig)):
             return file  # Already uploaded
 
-        if not file_name and getattr(file, 'name', None):
-            file_name = file.name
+        pos = 0
+        async with helpers._FileStream(file, file_size=file_size) as stream:
+            # Opening the stream will determine the correct file size
+            file_size = stream.file_size
 
-        if isinstance(file, str):
-            file_size = os.path.getsize(file)
-        elif isinstance(file, bytes):
-            file_size = len(file)
-        else:
-            if isinstance(file, io.IOBase) and file.seekable():
-                pos = file.tell()
-            else:
-                pos = None
+            if not part_size_kb:
+                part_size_kb = utils.get_appropriated_part_size(file_size)
 
-            # TODO Don't load the entire file in memory always
-            data = file.read()
-            if pos is not None:
-                file.seek(pos)
+            if part_size_kb > 512:
+                raise ValueError('The part size must be less or equal to 512KB')
 
-            file = data
-            file_size = len(file)
+            part_size = int(part_size_kb * 1024)
+            if part_size % 1024 != 0:
+                raise ValueError(
+                    'The part size must be evenly divisible by 1024')
 
-        # File will now either be a string or bytes
-        if not part_size_kb:
-            part_size_kb = utils.get_appropriated_part_size(file_size)
+            # Set a default file name if None was specified
+            file_id = helpers.generate_random_long()
+            if not file_name:
+                file_name = stream.name or str(file_id)
 
-        if part_size_kb > 512:
-            raise ValueError('The part size must be less or equal to 512KB')
+            # If the file name lacks extension, add it if possible.
+            # Else Telegram complains with `PHOTO_EXT_INVALID_ERROR`
+            # even if the uploaded image is indeed a photo.
+            if not os.path.splitext(file_name)[-1]:
+                file_name += utils._get_extension(stream)
 
-        part_size = int(part_size_kb * 1024)
-        if part_size % 1024 != 0:
-            raise ValueError(
-                'The part size must be evenly divisible by 1024')
+            # Determine whether the file is too big (over 10MB) or not
+            # Telegram does make a distinction between smaller or larger files
+            is_big = file_size > 10 * 1024 * 1024
+            hash_md5 = hashlib.md5()
 
-        # Set a default file name if None was specified
-        file_id = helpers.generate_random_long()
-        if not file_name:
-            if isinstance(file, str):
-                file_name = os.path.basename(file)
-            else:
-                file_name = str(file_id)
+            part_count = (file_size + part_size - 1) // part_size
+            self._log[__name__].info('Uploading file of %d bytes in %d chunks of %d',
+                                     file_size, part_count, part_size)
 
-        # If the file name lacks extension, add it if possible.
-        # Else Telegram complains with `PHOTO_EXT_INVALID_ERROR`
-        # even if the uploaded image is indeed a photo.
-        if not os.path.splitext(file_name)[-1]:
-            file_name += utils._get_extension(file)
-
-        # Determine whether the file is too big (over 10MB) or not
-        # Telegram does make a distinction between smaller or larger files
-        is_large = file_size > 10 * 1024 * 1024
-        hash_md5 = hashlib.md5()
-        if not is_large:
-            # Calculate the MD5 hash before anything else.
-            # As this needs to be done always for small files,
-            # might as well do it before anything else and
-            # check the cache.
-            if isinstance(file, str):
-                with open(file, 'rb') as stream:
-                    file = stream.read()
-            hash_md5.update(file)
-            if use_cache:
-                cached = self.session.get_file(
-                    hash_md5.digest(), file_size, cls=_CacheType(use_cache)
-                )
-                if cached:
-                    return cached
-
-        part_count = (file_size + part_size - 1) // part_size
-        self._log[__name__].info('Uploading file of %d bytes in %d chunks of %d',
-                                 file_size, part_count, part_size)
-
-        with open(file, 'rb') if isinstance(file, str) else BytesIO(file)\
-                as stream:
+            pos = 0
             for part_index in range(part_count):
                 # Read the file by in chunks of size part_size
-                part = stream.read(part_size)
+                part = await helpers._maybe_await(stream.read(part_size))
+
+                if not isinstance(part, bytes):
+                    raise TypeError(
+                        'file descriptor returned {}, not bytes (you must '
+                        'open the file in bytes mode)'.format(type(part)))
+
+                # `file_size` could be wrong in which case `part` may not be
+                # `part_size` before reaching the end.
+                if len(part) != part_size and part_index < part_count - 1:
+                    raise ValueError(
+                        'read less than {} before reaching the end; either '
+                        '`file_size` or `read` are wrong'.format(part_size))
+
+                pos += len(part)
+
+                # Encryption part if needed
+                if key and iv:
+                    part = AES.encrypt_ige(part, key, iv)
+
+                if not is_big:
+                    # Bit odd that MD5 is only needed for small files and not
+                    # big ones with more chance for corruption, but that's
+                    # what Telegram wants.
+                    hash_md5.update(part)
 
                 # The SavePartRequest is different depending on whether
                 # the file is too large or not (over or less than 10MB)
-                if is_large:
+                if is_big:
                     request = functions.upload.SaveBigFilePartRequest(
                         file_id, part_index, part_count, part)
                 else:
@@ -575,12 +739,12 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
                     self._log[__name__].debug('Uploaded %d/%d',
                                               part_index + 1, part_count)
                     if progress_callback:
-                        progress_callback(stream.tell(), file_size)
+                        await helpers._maybe_await(progress_callback(pos, file_size))
                 else:
                     raise RuntimeError(
                         'Failed to upload file part {}.'.format(part_index))
 
-        if is_large:
+        if is_big:
             return types.InputFileBig(file_id, part_count, file_name)
         else:
             return custom.InputSizedFile(
@@ -590,20 +754,25 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
     # endregion
 
     async def _file_to_media(
-            self, file, force_document=False,
+            self, file, force_document=False, file_size=None,
             progress_callback=None, attributes=None, thumb=None,
             allow_cache=True, voice_note=False, video_note=False,
-            supports_streaming=False, mime_type=None, as_image=None):
+            supports_streaming=False, mime_type=None, as_image=None,
+            ttl=None, nosound_video=None):
         if not file:
             return None, None, None
 
         if isinstance(file, pathlib.Path):
             file = str(file.absolute())
 
+        is_image = utils.is_image(file)
         if as_image is None:
-            as_image = utils.is_image(file) and not force_document
+            as_image = is_image and not force_document
 
-        if not isinstance(file, (str, bytes, io.IOBase)):
+        # `aiofiles` do not base `io.IOBase` but do have `read`, so we
+        # just check for the read attribute to see if it's file-like.
+        if not isinstance(file, (str, bytes, types.InputFile, types.InputFileBig)) \
+                and not hasattr(file, 'read'):
             # The user may pass a Message containing media (or the media,
             # or anything similar) that should be treated as a file. Try
             # getting the input media for whatever they passed and send it.
@@ -618,7 +787,8 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
                     force_document=force_document,
                     voice_note=voice_note,
                     video_note=video_note,
-                    supports_streaming=supports_streaming
+                    supports_streaming=supports_streaming,
+                    ttl=ttl
                 ), as_image)
             except TypeError:
                 # Can't turn whatever was given into media
@@ -626,24 +796,24 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
 
         media = None
         file_handle = None
-        use_cache = types.InputPhoto if as_image else types.InputDocument
-        if not isinstance(file, str) or os.path.isfile(file):
+
+        if isinstance(file, (types.InputFile, types.InputFileBig)):
+            file_handle = file
+        elif not isinstance(file, str) or os.path.isfile(file):
             file_handle = await self.upload_file(
                 _resize_photo_if_needed(file, as_image),
-                progress_callback=progress_callback,
-                use_cache=use_cache if allow_cache else None
+                file_size=file_size,
+                progress_callback=progress_callback
             )
         elif re.match('https?://', file):
             if as_image:
-                media = types.InputMediaPhotoExternal(file)
-            elif not force_document and utils.is_gif(file):
-                media = types.InputMediaGifExternal(file, '')
+                media = types.InputMediaPhotoExternal(file, ttl_seconds=ttl)
             else:
-                media = types.InputMediaDocumentExternal(file)
+                media = types.InputMediaDocumentExternal(file, ttl_seconds=ttl)
         else:
             bot_file = utils.resolve_bot_file_id(file)
             if bot_file:
-                media = utils.get_input_media(bot_file)
+                media = utils.get_input_media(bot_file, ttl=ttl)
 
         if media:
             pass  # Already have media, don't check the rest
@@ -652,49 +822,40 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods):
                 'Failed to convert {} to media. Not an existing file, '
                 'an HTTP URL or a valid bot-API-like file ID'.format(file)
             )
-        elif isinstance(file_handle, use_cache):
-            # File was cached, so an instance of use_cache was returned
-            if as_image:
-                media = types.InputMediaPhoto(file_handle)
-            else:
-                media = types.InputMediaDocument(file_handle)
         elif as_image:
-            media = types.InputMediaUploadedPhoto(file_handle)
+            media = types.InputMediaUploadedPhoto(file_handle, ttl_seconds=ttl)
         else:
             attributes, mime_type = utils.get_attributes(
                 file,
                 mime_type=mime_type,
                 attributes=attributes,
-                force_document=force_document,
+                force_document=force_document and not is_image,
                 voice_note=voice_note,
                 video_note=video_note,
-                supports_streaming=supports_streaming
+                supports_streaming=supports_streaming,
+                thumb=thumb
             )
 
-            input_kw = {}
-            if thumb:
+            if not thumb:
+                thumb = None
+            else:
                 if isinstance(thumb, pathlib.Path):
                     thumb = str(thumb.absolute())
-                input_kw['thumb'] = await self.upload_file(thumb)
+                thumb = await self.upload_file(thumb, file_size=file_size)
+
+            # setting `nosound_video` to `True` doesn't affect videos with sound
+            # instead it prevents sending silent videos as GIFs
+            nosound_video = nosound_video if mime_type.split("/")[0] == 'video' else None
 
             media = types.InputMediaUploadedDocument(
                 file=file_handle,
                 mime_type=mime_type,
                 attributes=attributes,
-                **input_kw
+                thumb=thumb,
+                force_file=force_document and not is_image,
+                ttl_seconds=ttl,
+                nosound_video=nosound_video
             )
         return file_handle, media, as_image
 
-    async def _cache_media(self: 'TelegramClient', msg, file, file_handle, image):
-        if file and msg and isinstance(file_handle,
-                                       custom.InputSizedFile):
-            # There was a response message and we didn't use cached
-            # version, so cache whatever we just sent to the database.
-            md5, size = file_handle.md5, file_handle.size
-            if image:
-                to_cache = utils.get_input_photo(msg.media.photo)
-            else:
-                to_cache = utils.get_input_document(msg.media.document)
-            self.session.cache_file(md5, size, to_cache)
-
     # endregion

telethon/client/users.py

@@ -1,10 +1,10 @@
 import asyncio
+import datetime
 import itertools
 import time
 import typing
 
-from .telegrambaseclient import TelegramBaseClient
-from .. import errors, utils, hints
+from .. import errors, helpers, utils, hints
 from ..errors import MultiError, RPCError
 from ..helpers import retry_range
 from ..tl import TLRequest, types, functions
@@ -15,10 +15,30 @@ if typing.TYPE_CHECKING:
     from .telegramclient import TelegramClient
 
 
-class UserMethods(TelegramBaseClient):
-    async def __call__(self: 'TelegramClient', request, ordered=False):
-        requests = (request if utils.is_list_like(request) else (request,))
-        for r in requests:
+def _fmt_flood(delay, request, *, early=False, td=datetime.timedelta):
+    return (
+        'Sleeping%s for %ds (%s) on %s flood wait',
+        ' early' if early else '',
+        delay,
+        td(seconds=delay),
+        request.__class__.__name__
+    )
+
+
+class UserMethods:
+    async def __call__(self: 'TelegramClient', request, ordered=False, flood_sleep_threshold=None):
+        return await self._call(self._sender, request, ordered=ordered)
+
+    async def _call(self: 'TelegramClient', sender, request, ordered=False, flood_sleep_threshold=None):
+        if self._loop is not None and self._loop != helpers.get_running_loop():
+            raise RuntimeError('The asyncio event loop must not change after connection (see the FAQ for details)')
+        # if the loop is None it will fail with a connection error later on
+
+        if flood_sleep_threshold is None:
+            flood_sleep_threshold = self.flood_sleep_threshold
+        requests = list(request) if utils.is_list_like(request) else [request]
+        request = list(request) if utils.is_list_like(request) else request
+        for i, r in enumerate(requests):
             if not isinstance(r, TLRequest):
                 raise _NOT_A_REQUEST()
             await r.resolve(self, utils)
@@ -29,19 +49,27 @@ class UserMethods(TelegramBaseClient):
                 diff = round(due - time.time())
                 if diff <= 3:  # Flood waits below 3 seconds are "ignored"
                     self._flood_waited_requests.pop(r.CONSTRUCTOR_ID, None)
-                elif diff <= self.flood_sleep_threshold:
-                    self._log[__name__].info(
-                        'Sleeping early for %ds on flood wait', diff)
-                    await asyncio.sleep(diff, loop=self._loop)
+                elif diff <= flood_sleep_threshold:
+                    self._log[__name__].info(*_fmt_flood(diff, r, early=True))
+                    await asyncio.sleep(diff)
                     self._flood_waited_requests.pop(r.CONSTRUCTOR_ID, None)
                 else:
                     raise errors.FloodWaitError(request=r, capture=diff)
 
+            if self._no_updates:
+                if utils.is_list_like(request):
+                    request[i] = functions.InvokeWithoutUpdatesRequest(r)
+                else:
+                    # This should only run once as requests should be a list of 1 item
+                    request = functions.InvokeWithoutUpdatesRequest(r)
+
         request_index = 0
+        last_error = None
         self._last_request = time.time()
+
         for attempt in retry_range(self._request_retries):
             try:
-                future = self._sender.send(request, ordered=ordered)
+                future = sender.send(request, ordered=ordered)
                 if isinstance(future, list):
                     results = []
                     exceptions = []
@@ -53,7 +81,6 @@ class UserMethods(TelegramBaseClient):
                             results.append(None)
                             continue
                         self.session.process_entities(result)
-                        self._entity_cache.add(result)
                         exceptions.append(None)
                         results.append(result)
                         request_index += 1
@@ -64,30 +91,41 @@ class UserMethods(TelegramBaseClient):
                 else:
                     result = await future
                     self.session.process_entities(result)
-                    self._entity_cache.add(result)
                     return result
             except (errors.ServerError, errors.RpcCallFailError,
-                    errors.RpcMcgetFailError) as e:
+                    errors.RpcMcgetFailError, errors.InterdcCallErrorError,
+                    errors.TimedOutError,
+                    errors.InterdcCallRichErrorError) as e:
+                last_error = e
                 self._log[__name__].warning(
                     'Telegram is having internal issues %s: %s',
                     e.__class__.__name__, e)
 
                 await asyncio.sleep(2)
-            except (errors.FloodWaitError, errors.FloodTestPhoneWaitError) as e:
+            except (errors.FloodWaitError, errors.FloodPremiumWaitError,
+                    errors.SlowModeWaitError, errors.FloodTestPhoneWaitError) as e:
+                last_error = e
                 if utils.is_list_like(request):
                     request = request[request_index]
 
-                self._flood_waited_requests\
-                    [request.CONSTRUCTOR_ID] = time.time() + e.seconds
+                # SLOW_MODE_WAIT is chat-specific, not request-specific
+                if not isinstance(e, errors.SlowModeWaitError):
+                    self._flood_waited_requests\
+                        [request.CONSTRUCTOR_ID] = time.time() + e.seconds
+
+                # In test servers, FLOOD_WAIT_0 has been observed, and sleeping for
+                # such a short amount will cause retries very fast leading to issues.
+                if e.seconds == 0:
+                    e.seconds = 1
 
                 if e.seconds <= self.flood_sleep_threshold:
-                    self._log[__name__].info('Sleeping for %ds on flood wait',
-                                             e.seconds)
-                    await asyncio.sleep(e.seconds, loop=self._loop)
+                    self._log[__name__].info(*_fmt_flood(e.seconds, request))
+                    await asyncio.sleep(e.seconds)
                 else:
                     raise
             except (errors.PhoneMigrateError, errors.NetworkMigrateError,
                     errors.UserMigrateError) as e:
+                last_error = e
                 self._log[__name__].info('Phone migrated to %d', e.new_dc)
                 should_raise = isinstance(e, (
                     errors.PhoneMigrateError, errors.NetworkMigrateError
@@ -96,6 +134,8 @@ class UserMethods(TelegramBaseClient):
                     raise
                 await self._switch_dc(e.new_dc)
 
+        if self._raise_last_call_error and last_error is not None:
+            raise last_error
         raise ValueError('Request was unsuccessful {} time(s)'
                          .format(attempt))
 
@@ -106,7 +146,7 @@ class UserMethods(TelegramBaseClient):
         """
         Gets "me", the current :tl:`User` who is logged in.
 
-        If the user has not logged in yet, this method returns ``None``.
+        If the user has not logged in yet, this method returns `None`.
 
         Arguments
             input_peer (`bool`, optional):
@@ -120,53 +160,61 @@ class UserMethods(TelegramBaseClient):
         Example
             .. code-block:: python
 
-                print(client.get_me().username)
+                me = await client.get_me()
+                print(me.username)
         """
-        if input_peer and self._self_input_peer:
-            return self._self_input_peer
+        if input_peer and self._mb_entity_cache.self_id:
+            return self._mb_entity_cache.get(self._mb_entity_cache.self_id)._as_input_peer()
 
         try:
             me = (await self(
                 functions.users.GetUsersRequest([types.InputUserSelf()])))[0]
 
-            self._bot = me.bot
-            if not self._self_input_peer:
-                self._self_input_peer = utils.get_input_peer(
-                    me, allow_self=False
-                )
+            if not self._mb_entity_cache.self_id:
+                self._mb_entity_cache.set_self_user(me.id, me.bot, me.access_hash)
 
-            return self._self_input_peer if input_peer else me
+            return utils.get_input_peer(me, allow_self=False) if input_peer else me
         except errors.UnauthorizedError:
             return None
 
+    @property
+    def _self_id(self: 'TelegramClient') -> typing.Optional[int]:
+        """
+        Returns the ID of the logged-in user, if known.
+
+        This property is used in every update, and some like `updateLoginToken`
+        occur prior to login, so it gracefully handles when no ID is known yet.
+        """
+        return self._mb_entity_cache.self_id
+
     async def is_bot(self: 'TelegramClient') -> bool:
         """
-        Return ``True`` if the signed-in user is a bot, ``False`` otherwise.
+        Return `True` if the signed-in user is a bot, `False` otherwise.
 
         Example
             .. code-block:: python
 
-                if client.is_bot():
+                if await client.is_bot():
                     print('Beep')
                 else:
                     print('Hello')
         """
-        if self._bot is None:
-            self._bot = (await self.get_me()).bot
+        if self._mb_entity_cache.self_bot is None:
+            await self.get_me(input_peer=True)
 
-        return self._bot
+        return self._mb_entity_cache.self_bot
 
     async def is_user_authorized(self: 'TelegramClient') -> bool:
         """
-        Returns ``True`` if the user is authorized (i.e. has logged in).
+        Returns `True` if the user is authorized (logged in).
 
         Example
             .. code-block:: python
 
-                if not client.is_user_authorized():
-                    client.send_code_request(phone)
+                if not await client.is_user_authorized():
+                    await client.send_code_request(phone)
                     code = input('enter code: ')
-                    client.sign_in(phone, code)
+                    await client.sign_in(phone, code)
         """
         if self._authorized is None:
             try:
@@ -180,7 +228,7 @@ class UserMethods(TelegramBaseClient):
 
     async def get_entity(
             self: 'TelegramClient',
-            entity: 'hints.EntitiesLike') -> 'hints.Entity':
+            entity: 'hints.EntitiesLike') -> typing.Union['hints.Entity', typing.List['hints.Entity']]:
         """
         Turns the given entity into a valid Telegram :tl:`User`, :tl:`Chat`
         or :tl:`Channel`. You can also pass a list or iterable of entities,
@@ -219,20 +267,20 @@ class UserMethods(TelegramBaseClient):
 
                 from telethon import utils
 
-                me = client.get_entity('me')
+                me = await client.get_entity('me')
                 print(utils.get_display_name(me))
 
-                chat = client.get_input_entity('username')
-                for message in client.iter_messages(chat):
+                chat = await client.get_input_entity('username')
+                async for message in client.iter_messages(chat):
                     ...
 
                 # Note that you could have used the username directly, but it's
                 # good to use get_input_entity if you will reuse it a lot.
-                for message in client.iter_messages('username'):
+                async for message in client.iter_messages('username'):
                     ...
 
                 # Note that for this to work the phone number must be in your contacts
-                some_id = client.get_peer_id('+34123456789')
+                some_id = await client.get_peer_id('+34123456789')
         """
         single = not utils.is_list_like(entity)
         if single:
@@ -249,12 +297,20 @@ class UserMethods(TelegramBaseClient):
             else:
                 inputs.append(await self.get_input_entity(x))
 
-        users = [x for x in inputs
-                 if isinstance(x, (types.InputPeerUser, types.InputPeerSelf))]
-        chats = [x.chat_id for x in inputs
-                 if isinstance(x, types.InputPeerChat)]
-        channels = [x for x in inputs
-                    if isinstance(x, types.InputPeerChannel)]
+        lists = {
+            helpers._EntityType.USER: [],
+            helpers._EntityType.CHAT: [],
+            helpers._EntityType.CHANNEL: [],
+        }
+        for x in inputs:
+            try:
+                lists[helpers._entity_type(x)].append(x)
+            except TypeError:
+                pass
+
+        users = lists[helpers._EntityType.USER]
+        chats = lists[helpers._EntityType.CHAT]
+        channels = lists[helpers._EntityType.CHANNEL]
         if users:
             # GetUsersRequest has a limit of 200 per call
             tmp = []
@@ -264,14 +320,16 @@ class UserMethods(TelegramBaseClient):
             users = tmp
         if chats:  # TODO Handle chats slice?
             chats = (await self(
-                functions.messages.GetChatsRequest(chats))).chats
+                functions.messages.GetChatsRequest([x.chat_id for x in chats]))).chats
         if channels:
             channels = (await self(
                 functions.channels.GetChannelsRequest(channels))).chats
 
         # Merge users, chats and channels into a single dictionary
         id_entity = {
-            utils.get_peer_id(x): x
+            # `get_input_entity` might've guessed the type from a non-marked ID,
+            # so the only way to match that with the input is by not using marks here.
+            utils.get_peer_id(x, add_mark=False): x
             for x in itertools.chain(users, chats, channels)
         }
 
@@ -284,7 +342,7 @@ class UserMethods(TelegramBaseClient):
             if isinstance(x, str):
                 result.append(await self._get_entity_from_string(x))
             elif not isinstance(x, types.InputPeerSelf):
-                result.append(id_entity[utils.get_peer_id(x)])
+                result.append(id_entity[utils.get_peer_id(x, add_mark=False)])
             else:
                 result.append(next(
                     u for u in id_entity.values()
@@ -352,10 +410,10 @@ class UserMethods(TelegramBaseClient):
                 # If you're going to use "username" often in your code
                 # (make a lot of calls), consider getting its input entity
                 # once, and then using the "user" everywhere instead.
-                user = client.get_input_entity('username')
+                user = await client.get_input_entity('username')
 
                 # The same applies to IDs, chats or channels.
-                chat = client.get_input_entity(-123456789)
+                chat = await client.get_input_entity(-123456789)
         """
         # Short-circuit if the input parameter directly maps to an InputPeer
         try:
@@ -367,8 +425,8 @@ class UserMethods(TelegramBaseClient):
         try:
             # 0x2d45687 == crc32(b'Peer')
             if isinstance(peer, int) or peer.SUBCLASS_OF_ID == 0x2d45687:
-                return self._entity_cache[peer]
-        except (AttributeError, KeyError):
+                return self._mb_entity_cache.get(utils.get_peer_id(peer, add_mark=False))._as_input_peer()
+        except AttributeError:
             pass
 
         # Then come known strings that take precedence
@@ -414,12 +472,16 @@ class UserMethods(TelegramBaseClient):
                 pass
 
         raise ValueError(
-            'Could not find the input entity for {!r}. Please read https://'
-            'docs.telethon.dev/en/latest/concepts/entities.html to'
+            'Could not find the input entity for {} ({}). Please read https://'
+            'docs.telethon.dev/en/stable/concepts/entities.html to'
             ' find out more details.'
-            .format(peer)
+            .format(peer, type(peer).__name__)
         )
 
+    async def _get_peer(self: 'TelegramClient', peer: 'hints.EntityLike'):
+        i, cls = utils.resolve_id(await self.get_peer_id(peer))
+        return cls(i)
+
     async def get_peer_id(
             self: 'TelegramClient',
             peer: 'hints.EntityLike',
@@ -436,7 +498,7 @@ class UserMethods(TelegramBaseClient):
         Example
             .. code-block:: python
 
-                print(client.get_peer_id('me'))
+                print(await client.get_peer_id('me'))
         """
         if isinstance(peer, int):
             return utils.get_peer_id(peer, add_mark=add_mark)
@@ -550,6 +612,8 @@ class UserMethods(TelegramBaseClient):
                     notify.peer = await self.get_input_entity(notify.peer)
                 return notify
         except AttributeError:
-            return types.InputNotifyPeer(await self.get_input_entity(notify))
+            pass
+
+        return types.InputNotifyPeer(await self.get_input_entity(notify))
 
     # endregion

telethon/crypto/factorization.py

@@ -13,6 +13,8 @@ class Factorization:
         """
         Factorizes the given large integer.
 
+        Implementation from https://comeoncodeon.wordpress.com/2010/09/18/pollard-rho-brent-integer-factorization/.
+
         :param pq: the prime pair pq.
         :return: a tuple containing the two factors p and q.
         """

telethon/crypto/libssl.py

@@ -3,6 +3,8 @@ Helper module around the system's libssl library if available for IGE mode.
 """
 import ctypes
 import ctypes.util
+import platform
+import sys
 try:
     import ctypes.macholib.dyld
 except ImportError:
@@ -15,6 +17,22 @@ __log__ = logging.getLogger(__name__)
 
 def _find_ssl_lib():
     lib = ctypes.util.find_library('ssl')
+    # macOS 10.15 segfaults on  unversioned crypto libraries.
+    # We therefore pin the current stable version here
+    # Credit for fix goes to Sarah Harvey (@worldwise001)
+    # https://www.shh.sh/2020/01/04/python-abort-trap-6.html
+    if sys.platform == 'darwin':
+        release, _version_info, _machine = platform.mac_ver()
+        ver, major, *_ = release.split('.')
+        # macOS 10.14 "mojave" is the last known major release
+        # to support unversioned libssl.dylib. Anything above
+        # needs specific versions
+        if int(ver) > 10 or int(ver) == 10 and int(major) > 14:
+            lib = (
+                ctypes.util.find_library('libssl.46') or
+                ctypes.util.find_library('libssl.44') or
+                ctypes.util.find_library('libssl.42')
+            )
     if not lib:
         raise OSError('no library called "ssl" found')
 

telethon/crypto/rsa.py

@@ -14,7 +14,7 @@ except ImportError:
 from ..tl import TLObject
 
 
-# {fingerprint: Crypto.PublicKey.RSA._RSAobj} dictionary
+# {fingerprint: (Crypto.PublicKey.RSA._RSAobj, old)} dictionary
 _server_keys = {}
 
 
@@ -47,26 +47,27 @@ def _compute_fingerprint(key):
     return struct.unpack('<q', sha1(n + e).digest()[-8:])[0]
 
 
-def add_key(pub):
+def add_key(pub, *, old):
     """Adds a new public key to be used when encrypting new data is needed"""
     global _server_keys
     key = rsa.PublicKey.load_pkcs1(pub)
-    _server_keys[_compute_fingerprint(key)] = key
+    _server_keys[_compute_fingerprint(key)] = (key, old)
 
 
-def encrypt(fingerprint, data):
+def encrypt(fingerprint, data, *, use_old=False):
     """
     Encrypts the given data known the fingerprint to be used
     in the way Telegram requires us to do so (sha1(data) + data + padding)
 
     :param fingerprint: the fingerprint of the RSA key.
     :param data: the data to be encrypted.
+    :param use_old: whether old keys should be used.
     :return:
         the cipher text, or None if no key matching this fingerprint is found.
     """
     global _server_keys
-    key = _server_keys.get(fingerprint, None)
-    if not key:
+    key, old = _server_keys.get(fingerprint, [None, None])
+    if (not key) or (old and not use_old):
         return None
 
     # len(sha1.digest) is always 20, so we're left with 255 - 20 - x padding
@@ -82,6 +83,48 @@ def encrypt(fingerprint, data):
 
 
 # Add default keys
+# https://github.com/DrKLO/Telegram/blob/a724d96e9c008b609fe188d122aa2922e40de5fc/TMessagesProj/jni/tgnet/Handshake.cpp#L356-L436
+for pub in (
+        '''-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEAruw2yP/BCcsJliRoW5eBVBVle9dtjJw+OYED160Wybum9SXtBBLX
+riwt4rROd9csv0t0OHCaTmRqBcQ0J8fxhN6/cpR1GWgOZRUAiQxoMnlt0R93LCX/
+j1dnVa/gVbCjdSxpbrfY2g2L4frzjJvdl84Kd9ORYjDEAyFnEA7dD556OptgLQQ2
+e2iVNq8NZLYTzLp5YpOdO1doK+ttrltggTCy5SrKeLoCPPbOgGsdxJxyz5KKcZnS
+Lj16yE5HvJQn0CNpRdENvRUXe6tBP78O39oJ8BTHp9oIjd6XWXAsp2CvK45Ol8wF
+XGF710w9lwCGNbmNxNYhtIkdqfsEcwR5JwIDAQAB
+-----END RSA PUBLIC KEY-----''',
+
+        '''-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEAvfLHfYH2r9R70w8prHblWt/nDkh+XkgpflqQVcnAfSuTtO05lNPs
+pQmL8Y2XjVT4t8cT6xAkdgfmmvnvRPOOKPi0OfJXoRVylFzAQG/j83u5K3kRLbae
+7fLccVhKZhY46lvsueI1hQdLgNV9n1cQ3TDS2pQOCtovG4eDl9wacrXOJTG2990V
+jgnIKNA0UMoP+KF03qzryqIt3oTvZq03DyWdGK+AZjgBLaDKSnC6qD2cFY81UryR
+WOab8zKkWAnhw2kFpcqhI0jdV5QaSCExvnsjVaX0Y1N0870931/5Jb9ICe4nweZ9
+kSDF/gip3kWLG0o8XQpChDfyvsqB9OLV/wIDAQAB
+-----END RSA PUBLIC KEY-----''',
+
+        '''-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEAs/ditzm+mPND6xkhzwFIz6J/968CtkcSE/7Z2qAJiXbmZ3UDJPGr
+zqTDHkO30R8VeRM/Kz2f4nR05GIFiITl4bEjvpy7xqRDspJcCFIOcyXm8abVDhF+
+th6knSU0yLtNKuQVP6voMrnt9MV1X92LGZQLgdHZbPQz0Z5qIpaKhdyA8DEvWWvS
+Uwwc+yi1/gGaybwlzZwqXYoPOhwMebzKUk0xW14htcJrRrq+PXXQbRzTMynseCoP
+Ioke0dtCodbA3qQxQovE16q9zz4Otv2k4j63cz53J+mhkVWAeWxVGI0lltJmWtEY
+K6er8VqqWot3nqmWMXogrgRLggv/NbbooQIDAQAB
+-----END RSA PUBLIC KEY-----''',
+
+        '''-----BEGIN RSA PUBLIC KEY-----
+MIIBCgKCAQEAvmpxVY7ld/8DAjz6F6q05shjg8/4p6047bn6/m8yPy1RBsvIyvuD
+uGnP/RzPEhzXQ9UJ5Ynmh2XJZgHoE9xbnfxL5BXHplJhMtADXKM9bWB11PU1Eioc
+3+AXBB8QiNFBn2XI5UkO5hPhbb9mJpjA9Uhw8EdfqJP8QetVsI/xrCEbwEXe0xvi
+fRLJbY08/Gp66KpQvy7g8w7VB8wlgePexW3pT13Ap6vuC+mQuJPyiHvSxjEKHgqe
+Pji9NP3tJUFQjcECqcm0yV7/2d0t/pbCm+ZH1sadZspQCEPPrtbkQBlvHb4OLiIW
+PGHKSMeRFvp3IWcmdJqXahxLCUS1Eh6MAQIDAQAB
+-----END RSA PUBLIC KEY-----''',
+
+):
+    add_key(pub, old=False)
+
+
 for pub in (
         '''-----BEGIN RSA PUBLIC KEY-----
 MIIBCgKCAQEAwVACPi9w23mF3tBkdZz+zwrzKOaaQdr01vAbU4E1pvkfj4sqDsm6
@@ -117,6 +160,6 @@ qAqBdmI1iBGdQv/OQCBcbXIWCGDY2AsiqLhlGQfPOI7/vvKc188rTriocgUtoTUc
 /n/sIUzkgwTqRyvWYynWARWzQg0I9olLBBC2q5RQJJlnYXZwyTL3y9tdb7zOHkks
 WV9IMQmZmyZh/N7sMbGWQpt4NMchGpPGeJ2e5gHBjDnlIf2p1yZOYeUYrdbwcS0t
 UiggS4UeE8TzIuXFQxw7fzEIlmhIaq3FnwIDAQAB
------END RSA PUBLIC KEY-----'''
+-----END RSA PUBLIC KEY-----''',
 ):
-    add_key(pub)
+    add_key(pub, old=True)

telethon/custom.py

@@ -0,0 +1 @@
+from .tl.custom import *

telethon/entitycache.py

@@ -1,141 +0,0 @@
-import inspect
-import itertools
-
-from . import utils
-from .tl import types
-
-# Which updates have the following fields?
-_has_field = {
-    ('user_id', int): [],
-    ('chat_id', int): [],
-    ('channel_id', int): [],
-    ('peer', 'TypePeer'): [],
-    ('peer', 'TypeDialogPeer'): [],
-    ('message', 'TypeMessage'): [],
-}
-
-# Note: We don't bother checking for some rare:
-# * `UpdateChatParticipantAdd.inviter_id` integer.
-# * `UpdateNotifySettings.peer` dialog peer.
-# * `UpdatePinnedDialogs.order` list of dialog peers.
-# * `UpdateReadMessagesContents.messages` list of messages.
-# * `UpdateChatParticipants.participants` list of participants.
-#
-# There are also some uninteresting `update.message` of type string.
-
-
-def _fill():
-    for name in dir(types):
-        update = getattr(types, name)
-        if getattr(update, 'SUBCLASS_OF_ID', None) == 0x9f89304e:
-            cid = update.CONSTRUCTOR_ID
-            sig = inspect.signature(update.__init__)
-            for param in sig.parameters.values():
-                vec = _has_field.get((param.name, param.annotation))
-                if vec is not None:
-                    vec.append(cid)
-
-    # Future-proof check: if the documentation format ever changes
-    # then we won't be able to pick the update types we are interested
-    # in, so we must make sure we have at least an update for each field
-    # which likely means we are doing it right.
-    if not all(_has_field.values()):
-        raise RuntimeError('FIXME: Did the init signature or updates change?')
-
-
-# We use a function to avoid cluttering the globals (with name/update/cid/doc)
-_fill()
-
-
-class EntityCache:
-    """
-    In-memory input entity cache, defaultdict-like behaviour.
-    """
-    def add(self, entities):
-        """
-        Adds the given entities to the cache, if they weren't saved before.
-        """
-        if not utils.is_list_like(entities):
-            # Invariant: all "chats" and "users" are always iterables,
-            # and "user" never is (so we wrap it inside a list).
-            entities = itertools.chain(
-                getattr(entities, 'chats', []),
-                getattr(entities, 'users', []),
-                (hasattr(entities, 'user') and [entities.user]) or []
-            )
-
-        for entity in entities:
-            try:
-                pid = utils.get_peer_id(entity)
-                if pid not in self.__dict__:
-                    # Note: `get_input_peer` already checks for `access_hash`
-                    self.__dict__[pid] = utils.get_input_peer(entity)
-            except TypeError:
-                pass
-
-    def __getitem__(self, item):
-        """
-        Gets the corresponding :tl:`InputPeer` for the given ID or peer,
-        or raises ``KeyError`` on any error (i.e. cannot be found).
-        """
-        if not isinstance(item, int) or item < 0:
-            try:
-                return self.__dict__[utils.get_peer_id(item)]
-            except TypeError:
-                raise KeyError('Invalid key will not have entity') from None
-
-        for cls in (types.PeerUser, types.PeerChat, types.PeerChannel):
-            result = self.__dict__.get(utils.get_peer_id(cls(item)))
-            if result:
-                return result
-
-        raise KeyError('No cached entity for the given key')
-
-    def ensure_cached(
-            self,
-            update,
-            has_user_id=frozenset(_has_field[('user_id', int)]),
-            has_chat_id=frozenset(_has_field[('chat_id', int)]),
-            has_channel_id=frozenset(_has_field[('channel_id', int)]),
-            has_peer=frozenset(_has_field[('peer', 'TypePeer')] + _has_field[('peer', 'TypeDialogPeer')]),
-            has_message=frozenset(_has_field[('message', 'TypeMessage')])
-    ):
-        """
-        Ensures that all the relevant entities in the given update are cached.
-        """
-        # This method is called pretty often and we want it to have the lowest
-        # overhead possible. For that, we avoid `isinstance` and constantly
-        # getting attributes out of `types.` by "caching" the constructor IDs
-        # in sets inside the arguments, and using local variables.
-        dct = self.__dict__
-        cid = update.CONSTRUCTOR_ID
-        if cid in has_user_id and \
-                update.user_id not in dct:
-            return False
-
-        if cid in has_chat_id and \
-                utils.get_peer_id(types.PeerChat(update.chat_id)) not in dct:
-            return False
-
-        if cid in has_channel_id and \
-                utils.get_peer_id(types.PeerChannel(update.channel_id)) not in dct:
-            return False
-
-        if cid in has_peer and \
-                utils.get_peer_id(update.peer) not in dct:
-            return False
-
-        if cid in has_message:
-            x = update.message
-            y = getattr(x, 'to_id', None)  # handle MessageEmpty
-            if y and utils.get_peer_id(y) not in dct:
-                return False
-
-            y = getattr(x, 'from_id', None)
-            if y and y not in dct:
-                return False
-
-            # We don't quite worry about entities anywhere else.
-            # This is enough.
-
-        return True

telethon/errors/__init__.py

@@ -6,7 +6,7 @@ import re
 
 from .common import (
     ReadCancelledError, TypeNotFoundError, InvalidChecksumError,
-    InvalidBufferError, SecurityError, CdnFileTamperedError,
+    InvalidBufferError, AuthKeyNotFound, SecurityError, CdnFileTamperedError,
     AlreadyInConversationError, BadMessageError, MultiError
 )
 
@@ -24,7 +24,8 @@ def rpc_message_to_error(rpc_error, request):
     :return: the RPCError as a Python exception that represents this error.
     """
     # Try to get the error by direct look-up, otherwise regex
-    cls = rpc_errors_dict.get(rpc_error.error_message, None)
+    # Case-insensitive, for things like "timeout" which don't conform.
+    cls = rpc_errors_dict.get(rpc_error.error_message.upper(), None)
     if cls:
         return cls(request=request)
 

telethon/errors/common.py

@@ -1,5 +1,6 @@
 """Errors not related to the Telegram API itself"""
 import struct
+import textwrap
 
 from ..tl import TLRequest
 
@@ -18,8 +19,8 @@ class TypeNotFoundError(Exception):
     def __init__(self, invalid_constructor_id, remaining):
         super().__init__(
             'Could not find a matching Constructor ID for the TLObject '
-            'that was supposed to be read with ID {:08x}. Most likely, '
-            'a TLObject was trying to be read when it should not be read. '
+            'that was supposed to be read with ID {:08x}. See the FAQ '
+            'for more details. '
             'Remaining bytes: {!r}'.format(invalid_constructor_id, remaining))
 
         self.invalid_constructor_id = invalid_constructor_id
@@ -58,6 +59,22 @@ class InvalidBufferError(BufferError):
                 'Invalid response buffer (too short {})'.format(self.payload))
 
 
+class AuthKeyNotFound(Exception):
+    """
+    The server claims it doesn't know about the authorization key (session
+    file) currently being used. This might be because it either has never
+    seen this authorization key, or it used to know about the authorization
+    key but has forgotten it, either temporarily or permanently (possibly
+    due to server errors).
+
+    If the issue persists, you may need to recreate the session file and login
+    again. This is not done automatically because it is not possible to know
+    if the issue is temporary or permanent.
+    """
+    def __init__(self):
+        super().__init__(textwrap.dedent(self.__class__.__doc__))
+
+
 class SecurityError(Exception):
     """
     Generic security error, mostly used when generating a new AuthKey.

telethon/errors/rpcbaseerrors.py

@@ -1,3 +1,15 @@
+from ..tl import functions
+
+_NESTS_QUERY = (
+    functions.InvokeAfterMsgRequest,
+    functions.InvokeAfterMsgsRequest,
+    functions.InitConnectionRequest,
+    functions.InvokeWithLayerRequest,
+    functions.InvokeWithoutUpdatesRequest,
+    functions.InvokeWithMessagesRangeRequest,
+    functions.InvokeWithTakeoutRequest,
+)
+
 class RPCError(Exception):
     """Base class for all Remote Procedure Call errors."""
     code = None
@@ -7,15 +19,24 @@ class RPCError(Exception):
         super().__init__('RPCError {}: {}{}'.format(
             code or self.code, message, self._fmt_request(request)))
 
+        self.request = request
         self.code = code
         self.message = message
 
     @staticmethod
     def _fmt_request(request):
-        return ' (caused by {})'.format(request.__class__.__name__)
+        n = 0
+        reason = ''
+        while isinstance(request, _NESTS_QUERY):
+            n += 1
+            reason += request.__class__.__name__ + '('
+            request = request.query
+        reason += request.__class__.__name__ + ')' * n
+
+        return ' (caused by {})'.format(reason)
 
     def __reduce__(self):
-        return type(self), (self.code, self.message)
+        return type(self), (self.request, self.message, self.code)
 
 
 class InvalidDCError(RPCError):

telethon/events/__init__.py

@@ -1,4 +1,5 @@
 from .raw import Raw
+from .album import Album
 from .chataction import ChatAction
 from .messagedeleted import MessageDeleted
 from .messageedited import MessageEdited
@@ -118,7 +119,7 @@ def unregister(callback, event=None):
 
 def is_handler(callback):
     """
-    Returns ``True`` if the given callback is an
+    Returns `True` if the given callback is an
     event handler (i.e. you used `register` on it).
     """
     return hasattr(callback, _HANDLERS_ATTRIBUTE)
@@ -134,6 +135,6 @@ def list(callback):
 
 def _get_handlers(callback):
     """
-    Like ``list`` but returns ``None`` if the callback was never registered.
+    Like ``list`` but returns `None` if the callback was never registered.
     """
     return getattr(callback, _HANDLERS_ATTRIBUTE, None)

telethon/events/album.py

@@ -0,0 +1,343 @@
+import asyncio
+import time
+import weakref
+
+from .common import EventBuilder, EventCommon, name_inner_event
+from .. import utils
+from ..tl import types
+from ..tl.custom.sendergetter import SenderGetter
+
+_IGNORE_MAX_SIZE = 100  # len()
+_IGNORE_MAX_AGE = 5  # seconds
+
+# IDs to ignore, and when they were added. If it grows too large, we will
+# remove old entries. Although it should generally not be bigger than 10,
+# it may be possible some updates are not processed and thus not removed.
+_IGNORE_DICT = {}
+
+
+_HACK_DELAY = 0.5
+
+
+class AlbumHack:
+    """
+    When receiving an album from a different data-center, they will come in
+    separate `Updates`, so we need to temporarily remember them for a while
+    and only after produce the event.
+
+    Of course events are not designed for this kind of wizardy, so this is
+    a dirty hack that gets the job done.
+
+    When cleaning up the code base we may want to figure out a better way
+    to do this, or just leave the album problem to the users; the update
+    handling code is bad enough as it is.
+    """
+    def __init__(self, client, event):
+        # It's probably silly to use a weakref here because this object is
+        # very short-lived but might as well try to do "the right thing".
+        self._client = weakref.ref(client)
+        self._event = event  # parent event
+        self._due = client.loop.time() + _HACK_DELAY
+
+        client.loop.create_task(self.deliver_event())
+
+    def extend(self, messages):
+        client = self._client()
+        if client:  # weakref may be dead
+            self._event.messages.extend(messages)
+            self._due = client.loop.time() + _HACK_DELAY
+
+    async def deliver_event(self):
+        while True:
+            client = self._client()
+            if client is None:
+                return  # weakref is dead, nothing to deliver
+
+            diff = self._due - client.loop.time()
+            if diff <= 0:
+                # We've hit our due time, deliver event. It won't respect
+                # sequential updates but fixing that would just worsen this.
+                await client._dispatch_event(self._event)
+                return
+
+            del client  # Clear ref and sleep until our due time
+            await asyncio.sleep(diff)
+
+
+@name_inner_event
+class Album(EventBuilder):
+    """
+    Occurs whenever you receive an album. This event only exists
+    to ease dealing with an unknown amount of messages that belong
+    to the same album.
+
+    Example
+        .. code-block:: python
+
+            from telethon import events
+
+            @client.on(events.Album)
+            async def handler(event):
+                # Counting how many photos or videos the album has
+                print('Got an album with', len(event), 'items')
+
+                # Forwarding the album as a whole to some chat
+                event.forward_to(chat)
+
+                # Printing the caption
+                print(event.text)
+
+                # Replying to the fifth item in the album
+                await event.messages[4].reply('Cool!')
+    """
+
+    def __init__(
+            self, chats=None, *, blacklist_chats=False, func=None):
+        super().__init__(chats, blacklist_chats=blacklist_chats, func=func)
+
+    @classmethod
+    def build(cls, update, others=None, self_id=None):
+        # TODO normally we'd only check updates if they come with other updates
+        # but MessageBox is not designed for this so others will always be None.
+        # In essence we always rely on AlbumHack rather than returning early if not others.
+        others = [update]
+
+        if isinstance(update,
+                      (types.UpdateNewMessage, types.UpdateNewChannelMessage)):
+            if not isinstance(update.message, types.Message):
+                return  # We don't care about MessageService's here
+
+            group = update.message.grouped_id
+            if group is None:
+                return  # It must be grouped
+
+            # Check whether we are supposed to skip this update, and
+            # if we do also remove it from the ignore list since we
+            # won't need to check against it again.
+            if _IGNORE_DICT.pop(id(update), None):
+                return
+
+            # Check if the ignore list is too big, and if it is clean it
+            # TODO time could technically go backwards; time is not monotonic
+            now = time.time()
+            if len(_IGNORE_DICT) > _IGNORE_MAX_SIZE:
+                for i in [i for i, t in _IGNORE_DICT.items() if now - t > _IGNORE_MAX_AGE]:
+                    del _IGNORE_DICT[i]
+
+            # Add the other updates to the ignore list
+            for u in others:
+                if u is not update:
+                    _IGNORE_DICT[id(u)] = now
+
+            # Figure out which updates share the same group and use those
+            return cls.Event([
+                u.message for u in others
+                if (isinstance(u, (types.UpdateNewMessage, types.UpdateNewChannelMessage))
+                    and isinstance(u.message, types.Message)
+                    and u.message.grouped_id == group)
+            ])
+
+    def filter(self, event):
+        # Albums with less than two messages require a few hacks to work.
+        if len(event.messages) > 1:
+            return super().filter(event)
+
+    class Event(EventCommon, SenderGetter):
+        """
+        Represents the event of a new album.
+
+        Members:
+            messages (Sequence[`Message <telethon.tl.custom.message.Message>`]):
+                The list of messages belonging to the same album.
+        """
+        def __init__(self, messages):
+            message = messages[0]
+            super().__init__(chat_peer=message.peer_id,
+                             msg_id=message.id, broadcast=bool(message.post))
+            SenderGetter.__init__(self, message.sender_id)
+            self.messages = messages
+
+        def _set_client(self, client):
+            super()._set_client(client)
+            self._sender, self._input_sender = utils._get_entity_pair(
+                self.sender_id, self._entities, client._mb_entity_cache)
+
+            for msg in self.messages:
+                msg._finish_init(client, self._entities, None)
+
+            if len(self.messages) == 1:
+                # This will require hacks to be a proper album event
+                hack = client._albums.get(self.grouped_id)
+                if hack is None:
+                    client._albums[self.grouped_id] = AlbumHack(client, self)
+                else:
+                    hack.extend(self.messages)
+
+        @property
+        def grouped_id(self):
+            """
+            The shared ``grouped_id`` between all the messages.
+            """
+            return self.messages[0].grouped_id
+
+        @property
+        def text(self):
+            """
+            The message text of the first photo with a caption,
+            formatted using the client's default parse mode.
+            """
+            return next((m.text for m in self.messages if m.text), '')
+
+        @property
+        def raw_text(self):
+            """
+            The raw message text of the first photo
+            with a caption, ignoring any formatting.
+            """
+            return next((m.raw_text for m in self.messages if m.raw_text), '')
+
+        @property
+        def is_reply(self):
+            """
+            `True` if the album is a reply to some other message.
+
+            Remember that you can access the ID of the message
+            this one is replying to through `reply_to_msg_id`,
+            and the `Message` object with `get_reply_message()`.
+            """
+            # Each individual message in an album all reply to the same message
+            return self.messages[0].is_reply
+
+        @property
+        def forward(self):
+            """
+            The `Forward <telethon.tl.custom.forward.Forward>`
+            information for the first message in the album if it was forwarded.
+            """
+            # Each individual message in an album all reply to the same message
+            return self.messages[0].forward
+
+        # endregion Public Properties
+
+        # region Public Methods
+
+        async def get_reply_message(self):
+            """
+            The `Message <telethon.tl.custom.message.Message>`
+            that this album is replying to, or `None`.
+
+            The result will be cached after its first use.
+            """
+            return await self.messages[0].get_reply_message()
+
+        async def respond(self, *args, **kwargs):
+            """
+            Responds to the album (not as a reply). Shorthand for
+            `telethon.client.messages.MessageMethods.send_message`
+            with ``entity`` already set.
+            """
+            return await self.messages[0].respond(*args, **kwargs)
+
+        async def reply(self, *args, **kwargs):
+            """
+            Replies to the first photo in the album (as a reply). Shorthand
+            for `telethon.client.messages.MessageMethods.send_message`
+            with both ``entity`` and ``reply_to`` already set.
+            """
+            return await self.messages[0].reply(*args, **kwargs)
+
+        async def forward_to(self, *args, **kwargs):
+            """
+            Forwards the entire album. Shorthand for
+            `telethon.client.messages.MessageMethods.forward_messages`
+            with both ``messages`` and ``from_peer`` already set.
+            """
+            if self._client:
+                kwargs['messages'] = self.messages
+                kwargs['from_peer'] = await self.get_input_chat()
+                return await self._client.forward_messages(*args, **kwargs)
+
+        async def edit(self, *args, **kwargs):
+            """
+            Edits the first caption or the message, or the first messages'
+            caption if no caption is set, iff it's outgoing. Shorthand for
+            `telethon.client.messages.MessageMethods.edit_message`
+            with both ``entity`` and ``message`` already set.
+
+            Returns `None` if the message was incoming,
+            or the edited `Message` otherwise.
+
+            .. note::
+
+                This is different from `client.edit_message
+                <telethon.client.messages.MessageMethods.edit_message>`
+                and **will respect** the previous state of the message.
+                For example, if the message didn't have a link preview,
+                the edit won't add one by default, and you should force
+                it by setting it to `True` if you want it.
+
+                This is generally the most desired and convenient behaviour,
+                and will work for link previews and message buttons.
+            """
+            for msg in self.messages:
+                if msg.raw_text:
+                    return await msg.edit(*args, **kwargs)
+
+            return await self.messages[0].edit(*args, **kwargs)
+
+        async def delete(self, *args, **kwargs):
+            """
+            Deletes the entire album. You're responsible for checking whether
+            you have the permission to do so, or to except the error otherwise.
+            Shorthand for
+            `telethon.client.messages.MessageMethods.delete_messages` with
+            ``entity`` and ``message_ids`` already set.
+            """
+            if self._client:
+                return await self._client.delete_messages(
+                    await self.get_input_chat(), self.messages,
+                    *args, **kwargs
+                )
+
+        async def mark_read(self):
+            """
+            Marks the entire album as read. Shorthand for
+            `client.send_read_acknowledge()
+            <telethon.client.messages.MessageMethods.send_read_acknowledge>`
+            with both ``entity`` and ``message`` already set.
+            """
+            if self._client:
+                await self._client.send_read_acknowledge(
+                    await self.get_input_chat(), max_id=self.messages[-1].id)
+
+        async def pin(self, *, notify=False):
+            """
+            Pins the first photo in the album. Shorthand for
+            `telethon.client.messages.MessageMethods.pin_message`
+            with both ``entity`` and ``message`` already set.
+            """
+            return await self.messages[0].pin(notify=notify)
+
+        def __len__(self):
+            """
+            Return the amount of messages in the album.
+
+            Equivalent to ``len(self.messages)``.
+            """
+            return len(self.messages)
+
+        def __iter__(self):
+            """
+            Iterate over the messages in the album.
+
+            Equivalent to ``iter(self.messages)``.
+            """
+            return iter(self.messages)
+
+        def __getitem__(self, n):
+            """
+            Access the n'th message in the album.
+
+            Equivalent to ``event.messages[n]``.
+            """
+            return self.messages[n]

telethon/events/callbackquery.py

@@ -19,67 +19,109 @@ class CallbackQuery(EventBuilder):
     `chat_instance` which should be used for inline callbacks.
 
     Args:
-        data (`bytes` | `str` | `callable`, optional):
+        data (`bytes`, `str`, `callable`, optional):
             If set, the inline button payload data must match this data.
             A UTF-8 string can also be given, a regex or a callable. For
             instance, to check against ``'data_1'`` and ``'data_2'`` you
             can use ``re.compile(b'data_')``.
+
+        pattern (`bytes`, `str`, `callable`, `Pattern`, optional):
+            If set, only buttons with payload matching this pattern will be handled.
+            You can specify a regex-like string which will be matched
+            against the payload data, a callable function that returns `True`
+            if a the payload data is acceptable, or a compiled regex pattern.
+
+    Example
+        .. code-block:: python
+
+            from telethon import events, Button
+
+            # Handle all callback queries and check data inside the handler
+            @client.on(events.CallbackQuery)
+            async def handler(event):
+                if event.data == b'yes':
+                    await event.answer('Correct answer!')
+
+            # Handle only callback queries with data being b'no'
+            @client.on(events.CallbackQuery(data=b'no'))
+            async def handler(event):
+                # Pop-up message with alert
+                await event.answer('Wrong answer!', alert=True)
+
+            # Send a message with buttons users can click
+            async def main():
+                await client.send_message(user, 'Yes or no?', buttons=[
+                    Button.inline('Yes!', b'yes'),
+                    Button.inline('Nope', b'no')
+                ])
     """
     def __init__(
-            self, chats=None, *, blacklist_chats=False, func=None, data=None):
+            self, chats=None, *, blacklist_chats=False, func=None, data=None, pattern=None):
         super().__init__(chats, blacklist_chats=blacklist_chats, func=func)
 
-        if isinstance(data, bytes):
-            self.data = data
-        elif isinstance(data, str):
-            self.data = data.encode('utf-8')
-        elif not data or callable(data):
-            self.data = data
-        elif hasattr(data, 'match') and callable(data.match):
-            if not isinstance(getattr(data, 'pattern', b''), bytes):
-                data = re.compile(data.pattern.encode('utf-8'),
-                                  data.flags & (~re.UNICODE))
+        if data and pattern:
+            raise ValueError("Only pass either data or pattern not both.")
 
-            self.data = data.match
+        if isinstance(data, str):
+            data = data.encode('utf-8')
+        if isinstance(pattern, str):
+            pattern = pattern.encode('utf-8')
+
+        match = data if data else pattern
+
+        if isinstance(match, bytes):
+            self.match = data if data else re.compile(pattern).match
+        elif not match or callable(match):
+            self.match = match
+        elif hasattr(match, 'match') and callable(match.match):
+            if not isinstance(getattr(match, 'pattern', b''), bytes):
+                match = re.compile(match.pattern.encode('utf-8'),
+                                   match.flags & (~re.UNICODE))
+
+            self.match = match.match
         else:
-            raise TypeError('Invalid data type given')
+            raise TypeError('Invalid data or pattern type given')
+
+        self._no_check = all(x is None for x in (
+            self.chats, self.func, self.match,
+        ))
 
     @classmethod
-    def build(cls, update):
+    def build(cls, update, others=None, self_id=None):
         if isinstance(update, types.UpdateBotCallbackQuery):
-            event = cls.Event(update, update.peer, update.msg_id)
+            return cls.Event(update, update.peer, update.msg_id)
         elif isinstance(update, types.UpdateInlineBotCallbackQuery):
             # See https://github.com/LonamiWebs/Telethon/pull/1005
             # The long message ID is actually just msg_id + peer_id
             mid, pid = struct.unpack('<ii', struct.pack('<q', update.msg_id.id))
             peer = types.PeerChannel(-pid) if pid < 0 else types.PeerUser(pid)
-            event = cls.Event(update, peer, mid)
-        else:
-            return
-
-        event._entities = update._entities
-        return event
+            return cls.Event(update, peer, mid)
 
     def filter(self, event):
         # We can't call super().filter(...) because it ignores chat_instance
+        if self._no_check:
+            return event
+
         if self.chats is not None:
             inside = event.query.chat_instance in self.chats
             if event.chat_id:
                 inside |= event.chat_id in self.chats
 
             if inside == self.blacklist_chats:
-                return None
+                return
 
-        if self.data:
-            if callable(self.data):
-                event.data_match = self.data(event.query.data)
+        if self.match:
+            if callable(self.match):
+                event.data_match = event.pattern_match = self.match(event.query.data)
                 if not event.data_match:
-                    return None
-            elif event.query.data != self.data:
-                return None
+                    return
+            elif event.query.data != self.match:
+                return
 
-        if not self.func or self.func(event):
-            return event
+        if self.func:
+            # Return the result of func directly as it may need to be awaited
+            return self.func(event)
+        return True
 
     class Event(EventCommon, SenderGetter):
         """
@@ -93,19 +135,23 @@ class CallbackQuery(EventBuilder):
                 The object returned by the ``data=`` parameter
                 when creating the event builder, if any. Similar
                 to ``pattern_match`` for the new message event.
+
+            pattern_match (`obj`, optional):
+                Alias for ``data_match``.
         """
         def __init__(self, query, peer, msg_id):
             super().__init__(peer, msg_id=msg_id)
             SenderGetter.__init__(self, query.user_id)
             self.query = query
             self.data_match = None
+            self.pattern_match = None
             self._message = None
             self._answered = False
 
         def _set_client(self, client):
             super()._set_client(client)
             self._sender, self._input_sender = utils._get_entity_pair(
-                self.sender_id, self._entities, client._entity_cache)
+                self.sender_id, self._entities, client._mb_entity_cache)
 
         @property
         def id(self):
@@ -162,8 +208,9 @@ class CallbackQuery(EventBuilder):
             if not getattr(self._input_sender, 'access_hash', True):
                 # getattr with True to handle the InputPeerSelf() case
                 try:
-                    self._input_sender = self._client._entity_cache[self._sender_id]
-                except KeyError:
+                    self._input_sender = self._client._mb_entity_cache.get(
+                        utils.resolve_id(self._sender_id)[0])._as_input_peer()
+                except AttributeError:
                     m = await self.get_message()
                     if m:
                         self._sender = m._sender
@@ -189,7 +236,7 @@ class CallbackQuery(EventBuilder):
 
                 alert (`bool`, optional):
                     Whether an alert (a pop-up dialog) should be used
-                    instead of showing a toast. Defaults to ``False``.
+                    instead of showing a toast. Defaults to `False`.
             """
             if self._answered:
                 return
@@ -210,11 +257,11 @@ class CallbackQuery(EventBuilder):
             """
             Whether this callback was generated from an inline button sent
             via an inline query or not. If the bot sent the message itself
-            with buttons, and one of those is clicked, this will be ``False``.
+            with buttons, and one of those is clicked, this will be `False`.
             If a user sent the message coming from an inline query to the
-            bot, and one of those is clicked, this will be ``True``.
+            bot, and one of those is clicked, this will be `True`.
 
-            If it's ``True``, it's likely that the bot is **not** in the
+            If it's `True`, it's likely that the bot is **not** in the
             chat, so methods like `respond` or `delete` won't work (but
             `edit` will always work).
             """
@@ -228,7 +275,7 @@ class CallbackQuery(EventBuilder):
 
             This method also creates a task to `answer` the callback.
 
-            This method will likely fail if `via_inline` is ``True``.
+            This method will likely fail if `via_inline` is `True`.
             """
             self._client.loop.create_task(self.answer())
             return await self._client.send_message(
@@ -242,7 +289,7 @@ class CallbackQuery(EventBuilder):
 
             This method also creates a task to `answer` the callback.
 
-            This method will likely fail if `via_inline` is ``True``.
+            This method will likely fail if `via_inline` is `True`.
             """
             self._client.loop.create_task(self.answer())
             kwargs['reply_to'] = self.query.msg_id
@@ -253,9 +300,9 @@ class CallbackQuery(EventBuilder):
             """
             Edits the message. Shorthand for
             `telethon.client.messages.MessageMethods.edit_message` with
-            the ``entity`` set to the correct :tl:`InputBotInlineMessageID`.
+            the ``entity`` set to the correct :tl:`InputBotInlineMessageID` or :tl:`InputBotInlineMessageID64`.
 
-            Returns ``True`` if the edit was successful.
+            Returns `True` if the edit was successful.
 
             This method also creates a task to `answer` the callback.
 
@@ -266,7 +313,7 @@ class CallbackQuery(EventBuilder):
                 since the message object is normally not present.
             """
             self._client.loop.create_task(self.answer())
-            if isinstance(self.query.msg_id, types.InputBotInlineMessageID):
+            if isinstance(self.query.msg_id, (types.InputBotInlineMessageID, types.InputBotInlineMessageID64)):
                 return await self._client.edit_message(
                     self.query.msg_id, *args, **kwargs
                 )
@@ -288,9 +335,11 @@ class CallbackQuery(EventBuilder):
 
             This method also creates a task to `answer` the callback.
 
-            This method will likely fail if `via_inline` is ``True``.
+            This method will likely fail if `via_inline` is `True`.
             """
             self._client.loop.create_task(self.answer())
+            if isinstance(self.query.msg_id, (types.InputBotInlineMessageID, types.InputBotInlineMessageID64)):
+                raise TypeError('Inline messages cannot be deleted as there is no API request available to do so')
             return await self._client.delete_messages(
                 await self.get_input_chat(), [self.query.msg_id],
                 *args, **kwargs

telethon/events/chataction.py

@@ -1,31 +1,64 @@
 from .common import EventBuilder, EventCommon, name_inner_event
 from .. import utils
-from ..tl import types, functions
+from ..tl import types
 
 
 @name_inner_event
 class ChatAction(EventBuilder):
     """
-    Occurs whenever a user joins or leaves a chat, or a message is pinned.
+    Occurs on certain chat actions:
+
+    * Whenever a new chat is created.
+    * Whenever a chat's title or photo is changed or removed.
+    * Whenever a new message is pinned.
+    * Whenever a user scores in a game.
+    * Whenever a user joins or is added to the group.
+    * Whenever a user is removed or leaves a group if it has
+      less than 50 members or the removed user was a bot.
+
+    Note that "chat" refers to "small group, megagroup and broadcast
+    channel", whereas "group" refers to "small group and megagroup" only.
+
+    Example
+        .. code-block:: python
+
+            from telethon import events
+
+            @client.on(events.ChatAction)
+            async def handler(event):
+                # Welcome every new user
+                if event.user_joined:
+                    await event.reply('Welcome to the group!')
     """
+
     @classmethod
-    def build(cls, update):
-        if isinstance(update, types.UpdateChannelPinnedMessage) and update.id == 0:
-            # Telegram does not always send
-            # UpdateChannelPinnedMessage for new pins
-            # but always for unpin, with update.id = 0
-            event = cls.Event(types.PeerChannel(update.channel_id),
-                              unpin=True)
+    def build(cls, update, others=None, self_id=None):
+        # Rely on specific pin updates for unpins, but otherwise ignore them
+        # for new pins (we'd rather handle the new service message with pin,
+        # so that we can act on that message').
+        if isinstance(update, types.UpdatePinnedChannelMessages) and not update.pinned:
+            return cls.Event(types.PeerChannel(update.channel_id),
+                             pin_ids=update.messages,
+                             pin=update.pinned)
+
+        elif isinstance(update, types.UpdatePinnedMessages) and not update.pinned:
+            return cls.Event(update.peer,
+                             pin_ids=update.messages,
+                             pin=update.pinned)
 
         elif isinstance(update, types.UpdateChatParticipantAdd):
-            event = cls.Event(types.PeerChat(update.chat_id),
-                              added_by=update.inviter_id or True,
-                              users=update.user_id)
+            return cls.Event(types.PeerChat(update.chat_id),
+                             added_by=update.inviter_id or True,
+                             users=update.user_id)
 
         elif isinstance(update, types.UpdateChatParticipantDelete):
-            event = cls.Event(types.PeerChat(update.chat_id),
-                              kicked_by=True,
-                              users=update.user_id)
+            return cls.Event(types.PeerChat(update.chat_id),
+                             kicked_by=True,
+                             users=update.user_id)
+
+        # UpdateChannel is sent if we leave a channel, and the update._entities
+        # set by _process_update would let us make some guesses. However it's
+        # better not to rely on this. Rely only in MessageActionChatDeleteUser.
 
         elif (isinstance(update, (
                 types.UpdateNewMessage, types.UpdateNewChannelMessage))
@@ -33,53 +66,58 @@ class ChatAction(EventBuilder):
             msg = update.message
             action = update.message.action
             if isinstance(action, types.MessageActionChatJoinedByLink):
-                event = cls.Event(msg,
-                                  added_by=True,
-                                  users=msg.from_id)
+                return cls.Event(msg,
+                                 added_by=True,
+                                 users=msg.from_id)
             elif isinstance(action, types.MessageActionChatAddUser):
-                # If a user adds itself, it means they joined
-                added_by = ([msg.from_id] == action.users) or msg.from_id
-                event = cls.Event(msg,
-                                  added_by=added_by,
-                                  users=action.users)
+                # If a user adds itself, it means they joined via the public chat username
+                added_by = ([msg.sender_id] == action.users) or msg.from_id
+                return cls.Event(msg,
+                                 added_by=added_by,
+                                 users=action.users)
             elif isinstance(action, types.MessageActionChatDeleteUser):
-                event = cls.Event(msg,
-                                  kicked_by=msg.from_id or True,
-                                  users=action.user_id)
+                return cls.Event(msg,
+                                 kicked_by=utils.get_peer_id(msg.from_id) if msg.from_id else True,
+                                 users=action.user_id)
             elif isinstance(action, types.MessageActionChatCreate):
-                event = cls.Event(msg,
-                                  users=action.users,
-                                  created=True,
-                                  new_title=action.title)
+                return cls.Event(msg,
+                                 users=action.users,
+                                 created=True,
+                                 new_title=action.title)
             elif isinstance(action, types.MessageActionChannelCreate):
-                event = cls.Event(msg,
-                                  created=True,
-                                  users=msg.from_id,
-                                  new_title=action.title)
+                return cls.Event(msg,
+                                 created=True,
+                                 users=msg.from_id,
+                                 new_title=action.title)
             elif isinstance(action, types.MessageActionChatEditTitle):
-                event = cls.Event(msg,
-                                  users=msg.from_id,
-                                  new_title=action.title)
+                return cls.Event(msg,
+                                 users=msg.from_id,
+                                 new_title=action.title)
             elif isinstance(action, types.MessageActionChatEditPhoto):
-                event = cls.Event(msg,
-                                  users=msg.from_id,
-                                  new_photo=action.photo)
+                return cls.Event(msg,
+                                 users=msg.from_id,
+                                 new_photo=action.photo)
             elif isinstance(action, types.MessageActionChatDeletePhoto):
-                event = cls.Event(msg,
-                                  users=msg.from_id,
-                                  new_photo=True)
-            elif isinstance(action, types.MessageActionPinMessage):
-                # Telegram always sends this service message for new pins
-                event = cls.Event(msg,
-                                  users=msg.from_id,
-                                  new_pin=msg.reply_to_msg_id)
-            else:
-                return
-        else:
-            return
+                return cls.Event(msg,
+                                 users=msg.from_id,
+                                 new_photo=True)
+            elif isinstance(action, types.MessageActionPinMessage) and msg.reply_to:
+                return cls.Event(msg,
+                                 pin_ids=[msg.reply_to_msg_id])
+            elif isinstance(action, types.MessageActionGameScore):
+                return cls.Event(msg,
+                                 new_score=action.score)
 
-        event._entities = update._entities
-        return event
+        elif isinstance(update, types.UpdateChannelParticipant) \
+                and bool(update.new_participant) != bool(update.prev_participant):
+            # If members are hidden, bots will receive this update instead,
+            # as there won't be a service message. Promotions and demotions
+            # seem to have both new and prev participant, which are ignored
+            # by this event.
+            return cls.Event(types.PeerChannel(update.channel_id),
+                             users=update.user_id,
+                             added_by=update.actor_id if update.new_participant else None,
+                             kicked_by=update.actor_id if update.prev_participant else None)
 
     class Event(EventCommon):
         """
@@ -90,48 +128,55 @@ class ChatAction(EventBuilder):
                 The message invoked by this Chat Action.
 
             new_pin (`bool`):
-                ``True`` if there is a new pin.
+                `True` if there is a new pin.
 
             new_photo (`bool`):
-                ``True`` if there's a new chat photo (or it was removed).
+                `True` if there's a new chat photo (or it was removed).
 
             photo (:tl:`Photo`, optional):
-                The new photo (or ``None`` if it was removed).
+                The new photo (or `None` if it was removed).
 
             user_added (`bool`):
-                ``True`` if the user was added by some other.
+                `True` if the user was added by some other.
 
             user_joined (`bool`):
-                ``True`` if the user joined on their own.
+                `True` if the user joined on their own.
 
             user_left (`bool`):
-                ``True`` if the user left on their own.
+                `True` if the user left on their own.
 
             user_kicked (`bool`):
-                ``True`` if the user was kicked by some other.
+                `True` if the user was kicked by some other.
 
             created (`bool`, optional):
-                ``True`` if this chat was just created.
+                `True` if this chat was just created.
 
             new_title (`str`, optional):
                 The new title string for the chat, if applicable.
 
+            new_score (`str`, optional):
+                The new score string for the game, if applicable.
+
             unpin (`bool`):
-                ``True`` if the existing pin gets unpinned.
+                `True` if the existing pin gets unpinned.
         """
-        def __init__(self, where, new_pin=None, new_photo=None,
+
+        def __init__(self, where, new_photo=None,
                      added_by=None, kicked_by=None, created=None,
-                     users=None, new_title=None, unpin=None):
+                     users=None, new_title=None, pin_ids=None, pin=None, new_score=None):
             if isinstance(where, types.MessageService):
                 self.action_message = where
-                where = where.to_id
+                where = where.peer_id
             else:
                 self.action_message = None
 
-            super().__init__(chat_peer=where, msg_id=new_pin)
+            # TODO needs some testing (can there be more than one id, and do they follow pin order?)
+            #      same in get_pinned_message
+            super().__init__(chat_peer=where, msg_id=pin_ids[0] if pin_ids else None)
 
-            self.new_pin = isinstance(new_pin, int)
-            self._pinned_message = new_pin
+            self.new_pin = pin_ids is not None
+            self._pin_ids = pin_ids
+            self._pinned_messages = None
 
             self.new_photo = new_photo is not None
             self.photo = \
@@ -148,20 +193,28 @@ class ChatAction(EventBuilder):
                 self.user_added = True
                 self._added_by = added_by
 
-            # If `from_id` was not present (it's ``True``) or the affected
+            # If `from_id` was not present (it's `True`) or the affected
             # user was "kicked by itself", then it left. Else it was kicked.
-            if kicked_by is True or kicked_by == users:
+            if kicked_by is True or (users is not None and kicked_by == users):
                 self.user_left = True
             elif kicked_by:
                 self.user_kicked = True
                 self._kicked_by = kicked_by
 
             self.created = bool(created)
-            self._user_peers = users if isinstance(users, list) else [users]
+
+            if isinstance(users, list):
+                self._user_ids = [utils.get_peer_id(u) for u in users]
+            elif users:
+                self._user_ids = [utils.get_peer_id(users)]
+            else:
+                self._user_ids = []
+
             self._users = None
             self._input_users = None
             self.new_title = new_title
-            self.unpin = unpin
+            self.new_score = new_score
+            self.unpin = not pin
 
         def _set_client(self, client):
             super()._set_client(client)
@@ -212,33 +265,34 @@ class ChatAction(EventBuilder):
 
         async def get_pinned_message(self):
             """
-            If ``new_pin`` is ``True``, this returns the `Message
+            If ``new_pin`` is `True`, this returns the `Message
             <telethon.tl.custom.message.Message>` object that was pinned.
             """
-            if self._pinned_message == 0:
-                return None
+            if self._pinned_messages is None:
+                await self.get_pinned_messages()
 
-            if isinstance(self._pinned_message, int)\
-                    and await self.get_input_chat():
-                r = await self._client(functions.channels.GetMessagesRequest(
-                    self._input_chat, [self._pinned_message]
-                ))
-                try:
-                    self._pinned_message = next(
-                        x for x in r.messages
-                        if isinstance(x, types.Message)
-                        and x.id == self._pinned_message
-                    )
-                except StopIteration:
-                    pass
+            if self._pinned_messages:
+                return self._pinned_messages[0]
 
-            if isinstance(self._pinned_message, types.Message):
-                return self._pinned_message
+        async def get_pinned_messages(self):
+            """
+            If ``new_pin`` is `True`, this returns a `list` of `Message
+            <telethon.tl.custom.message.Message>` objects that were pinned.
+            """
+            if not self._pin_ids:
+                return self._pin_ids  # either None or empty list
+
+            chat = await self.get_input_chat()
+            if chat:
+                self._pinned_messages = await self._client.get_messages(
+                    self._input_chat, ids=self._pin_ids)
+
+            return self._pinned_messages
 
         @property
         def added_by(self):
             """
-            The user who added ``users``, if applicable (``None`` otherwise).
+            The user who added ``users``, if applicable (`None` otherwise).
             """
             if self._added_by and not isinstance(self._added_by, types.User):
                 aby = self._entities.get(utils.get_peer_id(self._added_by))
@@ -259,7 +313,7 @@ class ChatAction(EventBuilder):
         @property
         def kicked_by(self):
             """
-            The user who kicked ``users``, if applicable (``None`` otherwise).
+            The user who kicked ``users``, if applicable (`None` otherwise).
             """
             if self._kicked_by and not isinstance(self._kicked_by, types.User):
                 kby = self._entities.get(utils.get_peer_id(self._kicked_by))
@@ -280,9 +334,9 @@ class ChatAction(EventBuilder):
         @property
         def user(self):
             """
-            The first user that takes part in this action (e.g. joined).
+            The first user that takes part in this action. For example, who joined.
 
-            Might be ``None`` if the information can't be retrieved or
+            Might be `None` if the information can't be retrieved or
             there is no user taking part.
             """
             if self.users:
@@ -315,25 +369,25 @@ class ChatAction(EventBuilder):
             """
             Returns the marked signed ID of the first user, if any.
             """
-            if self._user_peers:
-                return utils.get_peer_id(self._user_peers[0])
+            if self._user_ids:
+                return self._user_ids[0]
 
         @property
         def users(self):
             """
-            A list of users that take part in this action (e.g. joined).
+            A list of users that take part in this action. For example, who joined.
 
             Might be empty if the information can't be retrieved or there
             are no users taking part.
             """
-            if not self._user_peers:
+            if not self._user_ids:
                 return []
 
             if self._users is None:
                 self._users = [
-                    self._entities[utils.get_peer_id(peer)]
-                    for peer in self._user_peers
-                    if utils.get_peer_id(peer) in self._entities
+                    self._entities[user_id]
+                    for user_id in self._user_ids
+                    if user_id in self._entities
                 ]
 
             return self._users
@@ -342,10 +396,11 @@ class ChatAction(EventBuilder):
             """
             Returns `users` but will make an API call if necessary.
             """
-            if not self._user_peers:
+            if not self._user_ids:
                 return []
 
-            if self._users is None or len(self._users) != len(self._user_peers):
+            # Note: we access the property first so that it fills if needed
+            if (self.users is None or len(self._users) != len(self._user_ids)) and self.action_message:
                 await self.action_message._reload_message()
                 self._users = [
                     u for u in self.action_message.action_entities
@@ -358,22 +413,35 @@ class ChatAction(EventBuilder):
             """
             Input version of the ``self.users`` property.
             """
-            if self._input_users is None and self._user_peers:
+            if self._input_users is None and self._user_ids:
                 self._input_users = []
-                for peer in self._user_peers:
+                for user_id in self._user_ids:
+                    # First try to get it from our entities
                     try:
-                        self._input_users.append(self._client._entity_cache[peer])
-                    except KeyError:
+                        self._input_users.append(utils.get_input_peer(self._entities[user_id]))
+                        continue
+                    except (KeyError, TypeError):
                         pass
+
+                    # If missing, try from the entity cache
+                    try:
+                        self._input_users.append(self._client._mb_entity_cache.get(
+                            utils.resolve_id(user_id)[0])._as_input_peer())
+                        continue
+                    except AttributeError:
+                        pass
+
             return self._input_users or []
 
         async def get_input_users(self):
             """
             Returns `input_users` but will make an API call if necessary.
             """
-            self._input_users = None
-            if self._input_users is None:
-                await self.action_message._reload_message()
+            if not self._user_ids:
+                return []
+
+            # Note: we access the property first so that it fills if needed
+            if (self.input_users is None or len(self._input_users) != len(self._user_ids)) and self.action_message:
                 self._input_users = [
                     utils.get_input_peer(u)
                     for u in self.action_message.action_entities
@@ -386,5 +454,5 @@ class ChatAction(EventBuilder):
             """
             Returns the marked signed ID of the users, if any.
             """
-            if self._user_peers:
-                return [utils.get_peer_id(u) for u in self._user_peers]
+            if self._user_ids:
+                return self._user_ids[:]

telethon/events/common.py

@@ -1,10 +1,9 @@
 import abc
 import asyncio
-import itertools
 import warnings
 
 from .. import utils
-from ..tl import TLObject, types, functions
+from ..tl import TLObject, types
 from ..tl.custom.chatgetter import ChatGetter
 
 
@@ -55,7 +54,7 @@ class EventBuilder(abc.ABC):
             which will be ignored if ``blacklist_chats=True``.
 
         func (`callable`, optional):
-            A callable function that should accept the event as input
+            A callable (async or not) function that should accept the event as input
             parameter, and return a value indicating whether the event
             should be dispatched or not (any truthy value will do, it
             does not need to be a `bool`). It works like a custom filter:
@@ -66,8 +65,6 @@ class EventBuilder(abc.ABC):
                 async def handler(event):
                     pass  # code here
     """
-    self_id = None
-
     def __init__(self, chats=None, *, blacklist_chats=False, func=None):
         self.chats = chats
         self.blacklist_chats = bool(blacklist_chats)
@@ -77,8 +74,17 @@ class EventBuilder(abc.ABC):
 
     @classmethod
     @abc.abstractmethod
-    def build(cls, update):
-        """Builds an event for the given update if possible, or returns None"""
+    def build(cls, update, others=None, self_id=None):
+        """
+        Builds an event for the given update if possible, or returns None.
+
+        `others` are the rest of updates that came in the same container
+        as the current `update`.
+
+        `self_id` should be the current user's ID, since it is required
+        for some events which lack this information but still need it.
+        """
+        # TODO So many parameters specific to only some update types seems dirty
 
     async def resolve(self, client):
         """Helper method to allow event builders to be resolved before usage"""
@@ -86,7 +92,7 @@ class EventBuilder(abc.ABC):
             return
 
         if not self._resolve_lock:
-            self._resolve_lock = asyncio.Lock(loop=client.loop)
+            self._resolve_lock = asyncio.Lock()
 
         async with self._resolve_lock:
             if not self.resolved:
@@ -95,18 +101,16 @@ class EventBuilder(abc.ABC):
 
     async def _resolve(self, client):
         self.chats = await _into_id_set(client, self.chats)
-        if not EventBuilder.self_id:
-            EventBuilder.self_id = await client.get_peer_id('me')
 
     def filter(self, event):
         """
-        If the ID of ``event._chat_peer`` isn't in the chats set (or it is
-        but the set is a blacklist) returns ``None``, otherwise the event.
+        Returns a truthy value if the event passed the filter and should be
+        used, or falsy otherwise. The return value may need to be awaited.
 
         The events must have been resolved before this can be called.
         """
         if not self.resolved:
-            return None
+            return
 
         if self.chats is not None:
             # Note: the `event.chat_id` property checks if it's `None` for us
@@ -114,10 +118,13 @@ class EventBuilder(abc.ABC):
             if inside == self.blacklist_chats:
                 # If this chat matches but it's a blacklist ignore.
                 # If it doesn't match but it's a whitelist ignore.
-                return None
+                return
 
-        if not self.func or self.func(event):
-            return event
+        if not self.func:
+            return True
+
+        # Return the result of func directly as it may need to be awaited
+        return self.func(event)
 
 
 class EventCommon(ChatGetter, abc.ABC):
@@ -133,7 +140,7 @@ class EventCommon(ChatGetter, abc.ABC):
     """
     _event_name = 'Event'
 
-    def __init__(self, chat_peer=None, msg_id=None, broadcast=False):
+    def __init__(self, chat_peer=None, msg_id=None, broadcast=None):
         super().__init__(chat_peer, broadcast=broadcast)
         self._entities = {}
         self._client = None
@@ -147,7 +154,7 @@ class EventCommon(ChatGetter, abc.ABC):
         self._client = client
         if self._chat_peer:
             self._chat, self._input_chat = utils._get_entity_pair(
-                self.chat_id, self._entities, client._entity_cache)
+                self.chat_id, self._entities, client._mb_entity_cache)
         else:
             self._chat = self._input_chat = None
 

telethon/events/inlinequery.py

@@ -4,7 +4,7 @@ import re
 import asyncio
 
 from .common import EventBuilder, EventCommon, name_inner_event
-from .. import utils
+from .. import utils, helpers
 from ..tl import types, functions, custom
 from ..tl.custom.sendergetter import SenderGetter
 
@@ -29,8 +29,23 @@ class InlineQuery(EventBuilder):
         pattern (`str`, `callable`, `Pattern`, optional):
             If set, only queries matching this pattern will be handled.
             You can specify a regex-like string which will be matched
-            against the message, a callable function that returns ``True``
+            against the message, a callable function that returns `True`
             if a message is acceptable, or a compiled regex pattern.
+
+    Example
+        .. code-block:: python
+
+            from telethon import events
+
+            @client.on(events.InlineQuery)
+            async def handler(event):
+                builder = event.builder
+
+                # Two options (convert user text to UPPERCASE or lowercase)
+                await event.answer([
+                    builder.article('UPPERCASE', text=event.text.upper()),
+                    builder.article('lowercase', text=event.text.lower()),
+                ])
     """
     def __init__(
             self, users=None, *, blacklist_users=False, func=None, pattern=None):
@@ -46,14 +61,9 @@ class InlineQuery(EventBuilder):
             raise TypeError('Invalid pattern type given')
 
     @classmethod
-    def build(cls, update):
+    def build(cls, update, others=None, self_id=None):
         if isinstance(update, types.UpdateBotInlineQuery):
-            event = cls.Event(update)
-        else:
-            return
-
-        event._entities = update._entities
-        return event
+            return cls.Event(update)
 
     def filter(self, event):
         if self.pattern:
@@ -69,11 +79,11 @@ class InlineQuery(EventBuilder):
         Represents the event of a new callback query.
 
         Members:
-            query (:tl:`UpdateBotCallbackQuery`):
-                The original :tl:`UpdateBotCallbackQuery`.
+            query (:tl:`UpdateBotInlineQuery`):
+                The original :tl:`UpdateBotInlineQuery`.
 
-                Make sure to access the `text` of the query if
-                that's what you want instead working with this.
+                Make sure to access the `text` property of the query if
+                you want the text rather than the actual query object.
 
             pattern_match (`obj`, optional):
                 The resulting object from calling the passed ``pattern``
@@ -89,7 +99,7 @@ class InlineQuery(EventBuilder):
         def _set_client(self, client):
             super()._set_client(client)
             self._sender, self._input_sender = utils._get_entity_pair(
-                self.sender_id, self._entities, client._entity_cache)
+                self.sender_id, self._entities, client._mb_entity_cache)
 
         @property
         def id(self):
@@ -120,7 +130,7 @@ class InlineQuery(EventBuilder):
             and the user's device is able to send it, this will return
             the :tl:`GeoPoint` with the position of the user.
             """
-            return
+            return self.query.geo
 
         @property
         def builder(self):
@@ -137,6 +147,9 @@ class InlineQuery(EventBuilder):
             """
             Answers the inline query with the given results.
 
+            See the documentation for `builder` to know what kind of answers
+            can be given.
+
             Args:
                 results (`list`, optional):
                     A list of :tl:`InputBotInlineResult` to use.
@@ -196,10 +209,9 @@ class InlineQuery(EventBuilder):
                 return
 
             if results:
-                futures = [self._as_future(x, self._client.loop)
-                           for x in results]
+                futures = [self._as_future(x) for x in results]
 
-                await asyncio.wait(futures, loop=self._client.loop)
+                await asyncio.wait(futures)
 
                 # All futures will be in the `done` *set* that `wait` returns.
                 #
@@ -226,10 +238,10 @@ class InlineQuery(EventBuilder):
             )
 
         @staticmethod
-        def _as_future(obj, loop):
+        def _as_future(obj):
             if inspect.isawaitable(obj):
-                return asyncio.ensure_future(obj, loop=loop)
+                return asyncio.ensure_future(obj)
 
-            f = loop.create_future()
+            f = helpers.get_running_loop().create_future()
             f.set_result(obj)
             return f

telethon/events/messagedeleted.py

@@ -23,24 +23,30 @@ class MessageDeleted(EventBuilder):
 
         This means that the ``chats=`` parameter will not work reliably,
         unless you intend on working with channels and super-groups only.
+
+    Example
+        .. code-block:: python
+
+            from telethon import events
+
+            @client.on(events.MessageDeleted)
+            async def handler(event):
+                # Log all deleted message IDs
+                for msg_id in event.deleted_ids:
+                    print('Message', msg_id, 'was deleted in', event.chat_id)
     """
     @classmethod
-    def build(cls, update):
+    def build(cls, update, others=None, self_id=None):
         if isinstance(update, types.UpdateDeleteMessages):
-            event = cls.Event(
+            return cls.Event(
                 deleted_ids=update.messages,
                 peer=None
             )
         elif isinstance(update, types.UpdateDeleteChannelMessages):
-            event = cls.Event(
+            return cls.Event(
                 deleted_ids=update.messages,
                 peer=types.PeerChannel(update.channel_id)
             )
-        else:
-            return
-
-        event._entities = update._entities
-        return event
 
     class Event(EventCommon):
         def __init__(self, deleted_ids, peer):

telethon/events/messageedited.py

@@ -13,7 +13,7 @@ class MessageEdited(NewMessage):
     .. warning::
 
         On channels, `Message.out <telethon.tl.custom.message.Message>`
-        will be ``True`` if you sent the message originally, **not if
+        will be `True` if you sent the message originally, **not if
         you edited it**! This can be dangerous if you run outgoing
         commands on edits.
 
@@ -31,17 +31,22 @@ class MessageEdited(NewMessage):
         Instead, consider using ``from_users='me'`` (it won't work in
         broadcast channels at all since the sender is the channel and
         not you).
+
+    Example
+        .. code-block:: python
+
+            from telethon import events
+
+            @client.on(events.MessageEdited)
+            async def handler(event):
+                # Log the date of new edits
+                print('Message', event.id, 'changed at', event.date)
     """
     @classmethod
-    def build(cls, update):
+    def build(cls, update, others=None, self_id=None):
         if isinstance(update, (types.UpdateEditMessage,
                                types.UpdateEditChannelMessage)):
-            event = cls.Event(update.message)
-        else:
-            return
-
-        event._entities = update._entities
-        return event
+            return cls.Event(update.message)
 
     class Event(NewMessage.Event):
         pass  # Required if we want a different name for it

telethon/events/messageread.py

@@ -10,9 +10,24 @@ class MessageRead(EventBuilder):
 
     Args:
         inbox (`bool`, optional):
-            If this argument is ``True``, then when you read someone else's
-            messages the event will be fired. By default (``False``) only
+            If this argument is `True`, then when you read someone else's
+            messages the event will be fired. By default (`False`) only
             when messages you sent are read by someone else will fire it.
+
+    Example
+        .. code-block:: python
+
+            from telethon import events
+
+            @client.on(events.MessageRead)
+            async def handler(event):
+                # Log when someone reads your messages
+                print('Someone has read all your messages until', event.max_id)
+
+            @client.on(events.MessageRead(inbox=True))
+            async def handler(event):
+                # Log when you read message in a chat (from your "inbox")
+                print('You have read messages until', event.max_id)
     """
     def __init__(
             self, chats=None, *, blacklist_chats=False, func=None, inbox=False):
@@ -20,29 +35,24 @@ class MessageRead(EventBuilder):
         self.inbox = inbox
 
     @classmethod
-    def build(cls, update):
+    def build(cls, update, others=None, self_id=None):
         if isinstance(update, types.UpdateReadHistoryInbox):
-            event = cls.Event(update.peer, update.max_id, False)
+            return cls.Event(update.peer, update.max_id, False)
         elif isinstance(update, types.UpdateReadHistoryOutbox):
-            event = cls.Event(update.peer, update.max_id, True)
+            return cls.Event(update.peer, update.max_id, True)
         elif isinstance(update, types.UpdateReadChannelInbox):
-            event = cls.Event(types.PeerChannel(update.channel_id),
-                                      update.max_id, False)
+            return cls.Event(types.PeerChannel(update.channel_id),
+                             update.max_id, False)
         elif isinstance(update, types.UpdateReadChannelOutbox):
-            event = cls.Event(types.PeerChannel(update.channel_id),
-                              update.max_id, True)
+            return cls.Event(types.PeerChannel(update.channel_id),
+                             update.max_id, True)
         elif isinstance(update, types.UpdateReadMessagesContents):
-            event = cls.Event(message_ids=update.messages,
-                              contents=True)
+            return cls.Event(message_ids=update.messages,
+                             contents=True)
         elif isinstance(update, types.UpdateChannelReadMessagesContents):
-            event = cls.Event(types.PeerChannel(update.channel_id),
-                              message_ids=update.messages,
-                              contents=True)
-        else:
-            return
-
-        event._entities = update._entities
-        return event
+            return cls.Event(types.PeerChannel(update.channel_id),
+                             message_ids=update.messages,
+                             contents=True)
 
     def filter(self, event):
         if self.inbox == event.outbox:
@@ -60,10 +70,10 @@ class MessageRead(EventBuilder):
                 with an ID equal or lower to it have been read.
 
             outbox (`bool`):
-                ``True`` if someone else has read your messages.
+                `True` if someone else has read your messages.
 
             contents (`bool`):
-                ``True`` if what was read were the contents of a message.
+                `True` if what was read were the contents of a message.
                 This will be the case when e.g. you play a voice note.
                 It may only be set on ``inbox`` events.
         """
@@ -79,7 +89,7 @@ class MessageRead(EventBuilder):
         @property
         def inbox(self):
             """
-            ``True`` if you have read someone else's messages.
+            `True` if you have read someone else's messages.
             """
             return not self.outbox
 
@@ -113,7 +123,7 @@ class MessageRead(EventBuilder):
 
         def is_read(self, message):
             """
-            Returns ``True`` if the given message (or its ID) has been read.
+            Returns `True` if the given message (or its ID) has been read.
 
             If a list-like argument is provided, this method will return a
             list of booleans indicating which messages have been read.
@@ -126,7 +136,7 @@ class MessageRead(EventBuilder):
                         else message.id) <= self.max_id
 
         def __contains__(self, message):
-            """``True`` if the message(s) are read message."""
+            """`True` if the message(s) are read message."""
             if utils.is_list_like(message):
                 return all(self.is_read(message))
             else:

telethon/events/newmessage.py

@@ -1,7 +1,7 @@
-import asyncio
 import re
 
 from .common import EventBuilder, EventCommon, name_inner_event, _into_id_set
+from .. import utils
 from ..tl import types
 
 
@@ -12,11 +12,11 @@ class NewMessage(EventBuilder):
 
     Args:
         incoming (`bool`, optional):
-            If set to ``True``, only **incoming** messages will be handled.
+            If set to `True`, only **incoming** messages will be handled.
             Mutually exclusive with ``outgoing`` (can only set one of either).
 
         outgoing (`bool`, optional):
-            If set to ``True``, only **outgoing** messages will be handled.
+            If set to `True`, only **outgoing** messages will be handled.
             Mutually exclusive with ``incoming`` (can only set one of either).
 
         from_users (`entity`, optional):
@@ -28,15 +28,33 @@ class NewMessage(EventBuilder):
 
         forwards (`bool`, optional):
             Whether forwarded messages should be handled or not. By default,
-            both forwarded and normal messages are included. If it's ``True``
-            *only* forwards will be handled. If it's ``False`` only messages
+            both forwarded and normal messages are included. If it's `True`
+            *only* forwards will be handled. If it's `False` only messages
             that are *not* forwards will be handled.
 
         pattern (`str`, `callable`, `Pattern`, optional):
             If set, only messages matching this pattern will be handled.
             You can specify a regex-like string which will be matched
-            against the message, a callable function that returns ``True``
+            against the message, a callable function that returns `True`
             if a message is acceptable, or a compiled regex pattern.
+
+    Example
+        .. code-block:: python
+
+            import asyncio
+            from telethon import events
+
+            @client.on(events.NewMessage(pattern='(?i)hello.+'))
+            async def handler(event):
+                # Respond whenever someone says "Hello" and something else
+                await event.reply('Hey!')
+
+            @client.on(events.NewMessage(outgoing=True, pattern='!ping'))
+            async def handler(event):
+                # Say "!pong" whenever you send "!ping", then delete both messages
+                m = await event.respond('!pong')
+                await asyncio.sleep(5)
+                await client.delete_messages(event.chat_id, [event.id, m.id])
     """
     def __init__(self, chats=None, *, blacklist_chats=False, func=None,
                  incoming=None, outgoing=None,
@@ -76,7 +94,7 @@ class NewMessage(EventBuilder):
         self.from_users = await _into_id_set(client, self.from_users)
 
     @classmethod
-    def build(cls, update):
+    def build(cls, update, others=None, self_id=None):
         if isinstance(update,
                       (types.UpdateNewMessage, types.UpdateNewChannelMessage)):
             if not isinstance(update.message, types.Message):
@@ -89,18 +107,15 @@ class NewMessage(EventBuilder):
                 media_unread=update.media_unread,
                 silent=update.silent,
                 id=update.id,
-                # Note that to_id/from_id complement each other in private
-                # messages, depending on whether the message was outgoing.
-                to_id=types.PeerUser(
-                    update.user_id if update.out else cls.self_id
-                ),
-                from_id=cls.self_id if update.out else update.user_id,
+                peer_id=types.PeerUser(update.user_id),
+                from_id=types.PeerUser(self_id if update.out else update.user_id),
                 message=update.message,
                 date=update.date,
                 fwd_from=update.fwd_from,
                 via_bot_id=update.via_bot_id,
-                reply_to_msg_id=update.reply_to_msg_id,
-                entities=update.entities
+                reply_to=update.reply_to,
+                entities=update.entities,
+                ttl_period=update.ttl_period
             ))
         elif isinstance(update, types.UpdateShortChatMessage):
             event = cls.Event(types.Message(
@@ -109,26 +124,19 @@ class NewMessage(EventBuilder):
                 media_unread=update.media_unread,
                 silent=update.silent,
                 id=update.id,
-                from_id=update.from_id,
-                to_id=types.PeerChat(update.chat_id),
+                from_id=types.PeerUser(self_id if update.out else update.from_id),
+                peer_id=types.PeerChat(update.chat_id),
                 message=update.message,
                 date=update.date,
                 fwd_from=update.fwd_from,
                 via_bot_id=update.via_bot_id,
-                reply_to_msg_id=update.reply_to_msg_id,
-                entities=update.entities
+                reply_to=update.reply_to,
+                entities=update.entities,
+                ttl_period=update.ttl_period
             ))
         else:
             return
 
-        # Make messages sent to ourselves outgoing unless they're forwarded.
-        # This makes it consistent with official client's appearance.
-        ori = event.message
-        if isinstance(ori.to_id, types.PeerUser):
-            if ori.from_id == ori.to_id.user_id and not ori.fwd_from:
-                event.message.out = True
-
-        event._entities = update._entities
         return event
 
     def filter(self, event):
@@ -144,7 +152,7 @@ class NewMessage(EventBuilder):
                 return
 
         if self.from_users is not None:
-            if event.message.from_id not in self.from_users:
+            if event.message.sender_id not in self.from_users:
                 return
 
         if self.pattern:
@@ -182,7 +190,7 @@ class NewMessage(EventBuilder):
                 >>> @client.on(events.NewMessage(pattern=r'hi (\\w+)!'))
                 ... async def handler(event):
                 ...     # In this case, the result is a ``Match`` object
-                ...     # since the ``str`` pattern was converted into
+                ...     # since the `str` pattern was converted into
                 ...     # the ``re.compile(pattern).match`` function.
                 ...     print('Welcomed', event.pattern_match.group(1))
                 ...
@@ -190,14 +198,7 @@ class NewMessage(EventBuilder):
         """
         def __init__(self, message):
             self.__dict__['_init'] = False
-            if not message.out and isinstance(message.to_id, types.PeerUser):
-                # Incoming message (e.g. from a bot) has to_id=us, and
-                # from_id=bot (the actual "chat" from a user's perspective).
-                chat_peer = types.PeerUser(message.from_id)
-            else:
-                chat_peer = message.to_id
-
-            super().__init__(chat_peer=chat_peer,
+            super().__init__(chat_peer=message.peer_id,
                              msg_id=message.id, broadcast=bool(message.post))
 
             self.pattern_match = None

telethon/events/raw.py

@@ -12,6 +12,16 @@ class Raw(EventBuilder):
         types (`list` | `tuple` | `type`, optional):
             The type or types that the :tl:`Update` instance must be.
             Equivalent to ``if not isinstance(update, types): return``.
+
+    Example
+        .. code-block:: python
+
+            from telethon import events
+
+            @client.on(events.Raw)
+            async def handler(update):
+                # Print all incoming updates
+                print(update.stringify())
     """
     def __init__(self, types=None, *, func=None):
         super().__init__(func=func)
@@ -19,12 +29,12 @@ class Raw(EventBuilder):
             self.types = None
         elif not utils.is_list_like(types):
             if not isinstance(types, type):
-                raise TypeError('Invalid input type given %s', types)
+                raise TypeError('Invalid input type given: {}'.format(types))
 
             self.types = types
         else:
             if not all(isinstance(x, type) for x in types):
-                raise TypeError('Invalid input types given %s', types)
+                raise TypeError('Invalid input types given: {}'.format(types))
 
             self.types = tuple(types)
 
@@ -32,10 +42,12 @@ class Raw(EventBuilder):
         self.resolved = True
 
     @classmethod
-    def build(cls, update):
+    def build(cls, update, others=None, self_id=None):
         return update
 
     def filter(self, event):
-        if ((not self.types or isinstance(event, self.types))
-                and (not self.func or self.func(event))):
+        if not self.types or isinstance(event, self.types):
+            if self.func:
+                # Return the result of func directly as it may need to be awaited
+                return self.func(event)
             return event

telethon/events/userupdate.py

@@ -1,4 +1,5 @@
 import datetime
+import functools
 
 from .common import EventBuilder, EventCommon, name_inner_event
 from .. import utils
@@ -6,29 +7,63 @@ from ..tl import types
 from ..tl.custom.sendergetter import SenderGetter
 
 
+# TODO Either the properties are poorly named or they should be
+#      different events, but that would be a breaking change.
+#
+# TODO There are more "user updates", but bundling them all up
+#      in a single place will make it annoying to use (since
+#      the user needs to check for the existence of `None`).
+#
+# TODO Handle UpdateUserBlocked, UpdateUserName, UpdateUserPhone, UpdateUser
+
+def _requires_action(function):
+    @functools.wraps(function)
+    def wrapped(self):
+        return None if self.action is None else function(self)
+
+    return wrapped
+
+
+def _requires_status(function):
+    @functools.wraps(function)
+    def wrapped(self):
+        return None if self.status is None else function(self)
+
+    return wrapped
+
+
 @name_inner_event
 class UserUpdate(EventBuilder):
     """
     Occurs whenever a user goes online, starts typing, etc.
+
+    Example
+        .. code-block:: python
+
+            from telethon import events
+
+            @client.on(events.UserUpdate)
+            async def handler(event):
+                # If someone is uploading, say something
+                if event.uploading:
+                    await client.send_message(event.user_id, 'What are you sending?')
     """
     @classmethod
-    def build(cls, update):
+    def build(cls, update, others=None, self_id=None):
         if isinstance(update, types.UpdateUserStatus):
-            event = cls.Event(update.user_id,
-                              status=update.status)
+            return cls.Event(types.PeerUser(update.user_id),
+                             status=update.status)
+        elif isinstance(update, types.UpdateChannelUserTyping):
+            return cls.Event(update.from_id,
+                             chat_peer=types.PeerChannel(update.channel_id),
+                             typing=update.action)
         elif isinstance(update, types.UpdateChatUserTyping):
-            # Unfortunately, we can't know whether `chat_id`'s type
-            event = cls.Event(update.user_id,
-                              chat_id=update.chat_id,
-                              typing=update.action)
+            return cls.Event(update.from_id,
+                             chat_peer=types.PeerChat(update.chat_id),
+                             typing=update.action)
         elif isinstance(update, types.UpdateUserTyping):
-            event = cls.Event(update.user_id,
-                              typing=update.action)
-        else:
-            return
-
-        event._entities = update._entities
-        return event
+            return cls.Event(update.user_id,
+                             typing=update.action)
 
     class Event(EventCommon, SenderGetter):
         """
@@ -36,156 +71,31 @@ class UserUpdate(EventBuilder):
         such as gone online, started typing, etc.
 
         Members:
-            online (`bool`, optional):
-                ``True`` if the user is currently online, ``False`` otherwise.
-                Might be ``None`` if this information is not present.
+            status (:tl:`UserStatus`, optional):
+                The user status if the update is about going online or offline.
 
-            last_seen (`datetime`, optional):
-                Exact date when the user was last seen if known.
-
-            until (`datetime`, optional):
-                Until when will the user remain online.
-
-            within_months (`bool`):
-                ``True`` if the user was seen within 30 days.
-
-            within_weeks (`bool`):
-                ``True`` if the user was seen within 7 days.
-
-            recently (`bool`):
-                ``True`` if the user was seen within a day.
+                You should check this attribute first before checking any
+                of the seen within properties, since they will all be `None`
+                if the status is not set.
 
             action (:tl:`SendMessageAction`, optional):
                 The "typing" action if any the user is performing if any.
 
-            cancel (`bool`):
-                ``True`` if the action was cancelling other actions.
-
-            typing (`bool`):
-                ``True`` if the action is typing a message.
-
-            recording (`bool`):
-                ``True`` if the action is recording something.
-
-            uploading (`bool`):
-                ``True`` if the action is uploading something.
-
-            playing (`bool`):
-                ``True`` if the action is playing a game.
-
-            audio (`bool`):
-                ``True`` if what's being recorded/uploaded is an audio.
-
-            round (`bool`):
-                ``True`` if what's being recorded/uploaded is a round video.
-
-            video (`bool`):
-                ``True`` if what's being recorded/uploaded is an video.
-
-            document (`bool`):
-                ``True`` if what's being uploaded is document.
-
-            geo (`bool`):
-                ``True`` if what's being uploaded is a geo.
-
-            photo (`bool`):
-                ``True`` if what's being uploaded is a photo.
-
-            contact (`bool`):
-                ``True`` if what's being uploaded (selected) is a contact.
+                You should check this attribute first before checking any
+                of the typing properties, since they will all be `None`
+                if the action is not set.
         """
-        def __init__(self, user_id, *, status=None, chat_id=None, typing=None):
-            if chat_id is None:
-                super().__init__(types.PeerUser(user_id))
-            else:
-                # Temporarily set the chat_peer to the ID until ._set_client.
-                # We need the client to actually figure out its type.
-                super().__init__(chat_id)
-
-            SenderGetter.__init__(self, user_id)
-
-            self.online = None if status is None else \
-                isinstance(status, types.UserStatusOnline)
-
-            self.last_seen = status.was_online if \
-                isinstance(status, types.UserStatusOffline) else None
-
-            self.until = status.expires if \
-                isinstance(status, types.UserStatusOnline) else None
-
-            if self.last_seen:
-                now = datetime.datetime.now(tz=datetime.timezone.utc)
-                diff = now - self.last_seen
-                if diff < datetime.timedelta(days=30):
-                    self.within_months = True
-                    if diff < datetime.timedelta(days=7):
-                        self.within_weeks = True
-                        if diff < datetime.timedelta(days=1):
-                            self.recently = True
-            else:
-                self.within_months = self.within_weeks = self.recently = False
-                if isinstance(status, (types.UserStatusOnline,
-                                       types.UserStatusRecently)):
-                    self.within_months = self.within_weeks = True
-                    self.recently = True
-                elif isinstance(status, types.UserStatusLastWeek):
-                    self.within_months = self.within_weeks = True
-                elif isinstance(status, types.UserStatusLastMonth):
-                    self.within_months = True
+        def __init__(self, peer, *, status=None, chat_peer=None, typing=None):
+            super().__init__(chat_peer or peer)
+            SenderGetter.__init__(self, utils.get_peer_id(peer))
 
+            self.status = status
             self.action = typing
-            if typing:
-                self.cancel = self.typing = self.recording = self.uploading = \
-                    self.playing = False
-                self.audio = self.round = self.video = self.document = \
-                    self.geo = self.photo = self.contact = False
-
-                if isinstance(typing, types.SendMessageCancelAction):
-                    self.cancel = True
-                elif isinstance(typing, types.SendMessageTypingAction):
-                    self.typing = True
-                elif isinstance(typing, types.SendMessageGamePlayAction):
-                    self.playing = True
-                elif isinstance(typing, types.SendMessageGeoLocationAction):
-                    self.geo = True
-                elif isinstance(typing, types.SendMessageRecordAudioAction):
-                    self.recording = self.audio = True
-                elif isinstance(typing, types.SendMessageRecordRoundAction):
-                    self.recording = self.round = True
-                elif isinstance(typing, types.SendMessageRecordVideoAction):
-                    self.recording = self.video = True
-                elif isinstance(typing, types.SendMessageChooseContactAction):
-                    self.uploading = self.contact = True
-                elif isinstance(typing, types.SendMessageUploadAudioAction):
-                    self.uploading = self.audio = True
-                elif isinstance(typing, types.SendMessageUploadDocumentAction):
-                    self.uploading = self.document = True
-                elif isinstance(typing, types.SendMessageUploadPhotoAction):
-                    self.uploading = self.photo = True
-                elif isinstance(typing, types.SendMessageUploadRoundAction):
-                    self.uploading = self.round = True
-                elif isinstance(typing, types.SendMessageUploadVideoAction):
-                    self.uploading = self.video = True
 
         def _set_client(self, client):
-            if isinstance(self._chat_peer, int):
-                try:
-                    chat = client._entity_cache[self._chat_peer]
-                    if isinstance(chat, types.InputPeerChat):
-                        self._chat_peer = types.PeerChat(self._chat_peer)
-                    elif isinstance(chat, types.InputPeerChannel):
-                        self._chat_peer = types.PeerChannel(self._chat_peer)
-                    else:
-                        # Should not happen
-                        self._chat_peer = types.PeerUser(self._chat_peer)
-                except KeyError:
-                    # Hope for the best. We don't know where this event
-                    # occurred but it was most likely in a channel.
-                    self._chat_peer = types.PeerChannel(self._chat_peer)
-
             super()._set_client(client)
             self._sender, self._input_sender = utils._get_entity_pair(
-                self.sender_id, self._entities, client._entity_cache)
+                self.sender_id, self._entities, client._mb_entity_cache)
 
         @property
         def user(self):
@@ -209,3 +119,192 @@ class UserUpdate(EventBuilder):
         def user_id(self):
             """Alias for `sender_id <telethon.tl.custom.sendergetter.SenderGetter.sender_id>`."""
             return self.sender_id
+
+        @property
+        @_requires_action
+        def typing(self):
+            """
+            `True` if the action is typing a message.
+            """
+            return isinstance(self.action, types.SendMessageTypingAction)
+
+        @property
+        @_requires_action
+        def uploading(self):
+            """
+            `True` if the action is uploading something.
+            """
+            return isinstance(self.action, (
+                types.SendMessageChooseContactAction,
+                types.SendMessageChooseStickerAction,
+                types.SendMessageUploadAudioAction,
+                types.SendMessageUploadDocumentAction,
+                types.SendMessageUploadPhotoAction,
+                types.SendMessageUploadRoundAction,
+                types.SendMessageUploadVideoAction
+            ))
+
+        @property
+        @_requires_action
+        def recording(self):
+            """
+            `True` if the action is recording something.
+            """
+            return isinstance(self.action, (
+                types.SendMessageRecordAudioAction,
+                types.SendMessageRecordRoundAction,
+                types.SendMessageRecordVideoAction
+            ))
+
+        @property
+        @_requires_action
+        def playing(self):
+            """
+            `True` if the action is playing a game.
+            """
+            return isinstance(self.action, types.SendMessageGamePlayAction)
+
+        @property
+        @_requires_action
+        def cancel(self):
+            """
+            `True` if the action was cancelling other actions.
+            """
+            return isinstance(self.action, types.SendMessageCancelAction)
+
+        @property
+        @_requires_action
+        def geo(self):
+            """
+            `True` if what's being uploaded is a geo.
+            """
+            return isinstance(self.action, types.SendMessageGeoLocationAction)
+
+        @property
+        @_requires_action
+        def audio(self):
+            """
+            `True` if what's being recorded/uploaded is an audio.
+            """
+            return isinstance(self.action, (
+                types.SendMessageRecordAudioAction,
+                types.SendMessageUploadAudioAction
+            ))
+
+        @property
+        @_requires_action
+        def round(self):
+            """
+            `True` if what's being recorded/uploaded is a round video.
+            """
+            return isinstance(self.action, (
+                types.SendMessageRecordRoundAction,
+                types.SendMessageUploadRoundAction
+            ))
+
+        @property
+        @_requires_action
+        def video(self):
+            """
+            `True` if what's being recorded/uploaded is an video.
+            """
+            return isinstance(self.action, (
+                types.SendMessageRecordVideoAction,
+                types.SendMessageUploadVideoAction
+            ))
+
+        @property
+        @_requires_action
+        def contact(self):
+            """
+            `True` if what's being uploaded (selected) is a contact.
+            """
+            return isinstance(self.action, types.SendMessageChooseContactAction)
+
+        @property
+        @_requires_action
+        def document(self):
+            """
+            `True` if what's being uploaded is document.
+            """
+            return isinstance(self.action, types.SendMessageUploadDocumentAction)
+
+        @property
+        @_requires_action
+        def sticker(self):
+            """
+            `True` if what's being uploaded is a sticker.
+            """
+            return isinstance(self.action, types.SendMessageChooseStickerAction)
+
+        @property
+        @_requires_action
+        def photo(self):
+            """
+            `True` if what's being uploaded is a photo.
+            """
+            return isinstance(self.action, types.SendMessageUploadPhotoAction)
+
+        @property
+        @_requires_status
+        def last_seen(self):
+            """
+            Exact `datetime.datetime` when the user was last seen if known.
+            """
+            if isinstance(self.status, types.UserStatusOffline):
+                return self.status.was_online
+
+        @property
+        @_requires_status
+        def until(self):
+            """
+            The `datetime.datetime` until when the user should appear online.
+            """
+            if isinstance(self.status, types.UserStatusOnline):
+                return self.status.expires
+
+        def _last_seen_delta(self):
+            if isinstance(self.status, types.UserStatusOffline):
+                return datetime.datetime.now(tz=datetime.timezone.utc) - self.status.was_online
+            elif isinstance(self.status, types.UserStatusOnline):
+                return datetime.timedelta(days=0)
+            elif isinstance(self.status, types.UserStatusRecently):
+                return datetime.timedelta(days=1)
+            elif isinstance(self.status, types.UserStatusLastWeek):
+                return datetime.timedelta(days=7)
+            elif isinstance(self.status, types.UserStatusLastMonth):
+                return datetime.timedelta(days=30)
+            else:
+                return datetime.timedelta(days=365)
+
+        @property
+        @_requires_status
+        def online(self):
+            """
+            `True` if the user is currently online,
+            """
+            return self._last_seen_delta() <= datetime.timedelta(days=0)
+
+        @property
+        @_requires_status
+        def recently(self):
+            """
+            `True` if the user was seen within a day.
+            """
+            return self._last_seen_delta() <= datetime.timedelta(days=1)
+
+        @property
+        @_requires_status
+        def within_weeks(self):
+            """
+            `True` if the user was seen within 7 days.
+            """
+            return self._last_seen_delta() <= datetime.timedelta(days=7)
+
+        @property
+        @_requires_status
+        def within_months(self):
+            """
+            `True` if the user was seen within 30 days.
+            """
+            return self._last_seen_delta() <= datetime.timedelta(days=30)

telethon/extensions/__init__.py

@@ -1,6 +1,6 @@
 """
 Several extensions Python is missing, such as a proper class to handle a TCP
-communication with support for cancelling the operation, and an utility class
+communication with support for cancelling the operation, and a utility class
 to read arbitrary binary data in a more comfortable way, with int/strings/etc.
 """
 from .binaryreader import BinaryReader

telethon/extensions/binaryreader.py

@@ -2,30 +2,26 @@
 This module contains the BinaryReader utility class.
 """
 import os
-from datetime import datetime, timezone
-from io import BufferedReader, BytesIO
+import time
+from datetime import datetime, timezone, timedelta
+from io import BytesIO
 from struct import unpack
 
 from ..errors import TypeNotFoundError
 from ..tl.alltlobjects import tlobjects
 from ..tl.core import core_objects
 
+_EPOCH_NAIVE = datetime(*time.gmtime(0)[:6])
+_EPOCH = _EPOCH_NAIVE.replace(tzinfo=timezone.utc)
+
 
 class BinaryReader:
     """
     Small utility class to read binary data.
-    Also creates a "Memory Stream" if necessary
     """
 
-    def __init__(self, data=None, stream=None):
-        if data:
-            self.stream = BytesIO(data)
-        elif stream:
-            self.stream = stream
-        else:
-            raise ValueError('Either bytes or a stream must be provided')
-
-        self.reader = BufferedReader(self.stream)
+    def __init__(self, data):
+        self.stream = BytesIO(data)
         self._last = None  # Should come in handy to spot -404 errors
 
     # region Reading
@@ -57,13 +53,10 @@ class BinaryReader:
         return int.from_bytes(
             self.read(bits // 8), byteorder='little', signed=signed)
 
-    def read(self, length=None):
-        """Read the given amount of bytes."""
-        if length is None:
-            return self.reader.read()
-
-        result = self.reader.read(length)
-        if len(result) != length:
+    def read(self, length=-1):
+        """Read the given amount of bytes, or -1 to read all remaining."""
+        result = self.stream.read(length)
+        if (length >= 0) and (len(result) != length):
             raise BufferError(
                 'No more data left to read (need {}, got {}: {}); last read {}'
                 .format(length, len(result), repr(result), repr(self._last))
@@ -120,10 +113,7 @@ class BinaryReader:
            into a Python datetime object.
         """
         value = self.read_int()
-        if value == 0:
-            return None
-        else:
-            return datetime.fromtimestamp(value, tz=timezone.utc)
+        return _EPOCH + timedelta(seconds=value)
 
     def tgread_object(self):
         """Reads a Telegram object."""
@@ -163,24 +153,24 @@ class BinaryReader:
 
     def close(self):
         """Closes the reader, freeing the BytesIO stream."""
-        self.reader.close()
+        self.stream.close()
 
     # region Position related
 
     def tell_position(self):
         """Tells the current position on the stream."""
-        return self.reader.tell()
+        return self.stream.tell()
 
     def set_position(self, position):
         """Sets the current position on the stream."""
-        self.reader.seek(position)
+        self.stream.seek(position)
 
     def seek(self, offset):
         """
         Seeks the stream position given an offset from the current position.
         The offset may be negative.
         """
-        self.reader.seek(offset, os.SEEK_CUR)
+        self.stream.seek(offset, os.SEEK_CUR)
 
     # endregion
 

telethon/extensions/html.py

@@ -1,31 +1,22 @@
 """
 Simple HTML -> Telegram entity parser.
 """
-import struct
 from collections import deque
-from html import escape, unescape
+from html import escape
 from html.parser import HTMLParser
+from typing import Iterable, Tuple, List
 
-from .. import helpers
+from ..helpers import add_surrogate, del_surrogate, within_surrogate, strip_text
+from ..tl import TLObject
 from ..tl.types import (
     MessageEntityBold, MessageEntityItalic, MessageEntityCode,
     MessageEntityPre, MessageEntityEmail, MessageEntityUrl,
-    MessageEntityTextUrl, MessageEntityMentionName
+    MessageEntityTextUrl, MessageEntityMentionName,
+    MessageEntityUnderline, MessageEntityStrike, MessageEntityBlockquote,
+    TypeMessageEntity
 )
 
 
-# Helpers from markdown.py
-def _add_surrogate(text):
-    return ''.join(
-        ''.join(chr(y) for y in struct.unpack('<HH', x.encode('utf-16le')))
-        if (0x10000 <= ord(x) <= 0x10FFFF) else x for x in text
-    )
-
-
-def _del_surrogate(text):
-    return text.encode('utf-16', 'surrogatepass').decode('utf-16')
-
-
 class HTMLToTelegramParser(HTMLParser):
     def __init__(self):
         super().__init__()
@@ -46,6 +37,12 @@ class HTMLToTelegramParser(HTMLParser):
             EntityType = MessageEntityBold
         elif tag == 'em' or tag == 'i':
             EntityType = MessageEntityItalic
+        elif tag == 'u':
+            EntityType = MessageEntityUnderline
+        elif tag == 'del' or tag == 's':
+            EntityType = MessageEntityStrike
+        elif tag == 'blockquote':
+            EntityType = MessageEntityBlockquote
         elif tag == 'code':
             try:
                 # If we're in the middle of a <pre> tag, this <code> tag is
@@ -77,7 +74,7 @@ class HTMLToTelegramParser(HTMLParser):
                     EntityType = MessageEntityUrl
                 else:
                     EntityType = MessageEntityTextUrl
-                    args['url'] = url
+                    args['url'] = del_surrogate(url)
                     url = None
             self._open_tags_meta.popleft()
             self._open_tags_meta.appendleft(url)
@@ -90,8 +87,6 @@ class HTMLToTelegramParser(HTMLParser):
                 **args)
 
     def handle_data(self, text):
-        text = unescape(text)
-
         previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ''
         if previous_tag == 'a':
             url = self._open_tags_meta[0]
@@ -114,24 +109,47 @@ class HTMLToTelegramParser(HTMLParser):
             self.entities.append(entity)
 
 
-def parse(html):
+def parse(html: str) -> Tuple[str, List[TypeMessageEntity]]:
     """
     Parses the given HTML message and returns its stripped representation
     plus a list of the MessageEntity's that were found.
 
-    :param message: the message with HTML to be parsed.
+    :param html: the message with HTML to be parsed.
     :return: a tuple consisting of (clean message, [message entities]).
     """
     if not html:
         return html, []
 
     parser = HTMLToTelegramParser()
-    parser.feed(_add_surrogate(html))
-    text = helpers.strip_text(parser.text, parser.entities)
-    return _del_surrogate(text), parser.entities
+    parser.feed(add_surrogate(html))
+    text = strip_text(parser.text, parser.entities)
+    parser.entities.reverse()
+    parser.entities.sort(key=lambda entity: entity.offset)
+    return del_surrogate(text), parser.entities
 
 
-def unparse(text, entities):
+ENTITY_TO_FORMATTER = {
+    MessageEntityBold: ('<strong>', '</strong>'),
+    MessageEntityItalic: ('<em>', '</em>'),
+    MessageEntityCode: ('<code>', '</code>'),
+    MessageEntityUnderline: ('<u>', '</u>'),
+    MessageEntityStrike: ('<del>', '</del>'),
+    MessageEntityBlockquote: ('<blockquote>', '</blockquote>'),
+    MessageEntityPre: lambda e, _: (
+        "<pre>\n"
+        "    <code class='language-{}'>\n"
+        "        ".format(e.language), "{}\n"
+        "    </code>\n"
+        "</pre>"
+    ),
+    MessageEntityEmail: lambda _, t: ('<a href="mailto:{}">'.format(t), '</a>'),
+    MessageEntityUrl: lambda _, t: ('<a href="{}">'.format(t), '</a>'),
+    MessageEntityTextUrl: lambda e, _: ('<a href="{}">'.format(escape(e.url)), '</a>'),
+    MessageEntityMentionName: lambda e, _: ('<a href="tg://user?id={}">'.format(e.user_id), '</a>'),
+}
+
+
+def unparse(text: str, entities: Iterable[TypeMessageEntity]) -> str:
     """
     Performs the reverse operation to .parse(), effectively returning HTML
     given a normal text and its MessageEntity's.
@@ -140,51 +158,37 @@ def unparse(text, entities):
     :param entities: the MessageEntity's applied to the text.
     :return: a HTML representation of the combination of both inputs.
     """
-    if not text or not entities:
+    if not text:
         return text
+    elif not entities:
+        return escape(text)
 
-    text = _add_surrogate(text)
-    html = []
-    last_offset = 0
-    for entity in entities:
-        if entity.offset > last_offset:
-            html.append(escape(text[last_offset:entity.offset]))
-        elif entity.offset < last_offset:
-            continue
+    if isinstance(entities, TLObject):
+        entities = (entities,)
 
-        skip_entity = False
-        entity_text = escape(text[entity.offset:entity.offset + entity.length])
-        entity_type = type(entity)
+    text = add_surrogate(text)
+    insert_at = []
+    for i, entity in enumerate(entities):
+        s = entity.offset
+        e = entity.offset + entity.length
+        delimiter = ENTITY_TO_FORMATTER.get(type(entity), None)
+        if delimiter:
+            if callable(delimiter):
+                delimiter = delimiter(entity, text[s:e])
+            insert_at.append((s, i, delimiter[0]))
+            insert_at.append((e, -i, delimiter[1]))
 
-        if entity_type == MessageEntityBold:
-            html.append('<strong>{}</strong>'.format(entity_text))
-        elif entity_type == MessageEntityItalic:
-            html.append('<em>{}</em>'.format(entity_text))
-        elif entity_type == MessageEntityCode:
-            html.append('<code>{}</code>'.format(entity_text))
-        elif entity_type == MessageEntityPre:
-            if entity.language:
-                html.append(
-                    "<pre>\n"
-                    "    <code class='language-{}'>\n"
-                    "        {}\n"
-                    "    </code>\n"
-                    "</pre>".format(entity.language, entity_text))
-            else:
-                html.append('<pre><code>{}</code></pre>'
-                            .format(entity_text))
-        elif entity_type == MessageEntityEmail:
-            html.append('<a href="mailto:{0}">{0}</a>'.format(entity_text))
-        elif entity_type == MessageEntityUrl:
-            html.append('<a href="{0}">{0}</a>'.format(entity_text))
-        elif entity_type == MessageEntityTextUrl:
-            html.append('<a href="{}">{}</a>'
-                        .format(escape(entity.url), entity_text))
-        elif entity_type == MessageEntityMentionName:
-            html.append('<a href="tg://user?id={}">{}</a>'
-                        .format(entity.user_id, entity_text))
-        else:
-            skip_entity = True
-        last_offset = entity.offset + (0 if skip_entity else entity.length)
-    html.append(text[last_offset:])
-    return _del_surrogate(''.join(html))
+    insert_at.sort(key=lambda t: (t[0], t[1]))
+    next_escape_bound = len(text)
+    while insert_at:
+        # Same logic as markdown.py
+        at, _, what = insert_at.pop()
+        while within_surrogate(text, at):
+            at += 1
+
+        text = text[:at] + what + escape(text[at:next_escape_bound]) + text[next_escape_bound:]
+        next_escape_bound = at
+
+    text = escape(text[:next_escape_bound]) + text[next_escape_bound:]
+
+    return del_surrogate(text)

telethon/extensions/markdown.py

@@ -4,22 +4,25 @@ for use within the library, which attempts to handle emojies correctly,
 since they seem to count as two characters and it's a bit strange.
 """
 import re
+import warnings
 
-from ..helpers import add_surrogate, del_surrogate, strip_text
+from ..helpers import add_surrogate, del_surrogate, within_surrogate, strip_text
 from ..tl import TLObject
 from ..tl.types import (
     MessageEntityBold, MessageEntityItalic, MessageEntityCode,
-    MessageEntityPre, MessageEntityTextUrl, MessageEntityMentionName
+    MessageEntityPre, MessageEntityTextUrl, MessageEntityMentionName,
+    MessageEntityStrike
 )
 
 DEFAULT_DELIMITERS = {
     '**': MessageEntityBold,
     '__': MessageEntityItalic,
+    '~~': MessageEntityStrike,
     '`': MessageEntityCode,
     '```': MessageEntityPre
 }
 
-DEFAULT_URL_RE = re.compile(r'\[([\S\s]+?)\]\((.+?)\)')
+DEFAULT_URL_RE = re.compile(r'\[([^]]*?)\]\(([\s\S]*?)\)')
 DEFAULT_URL_FORMAT = '[{0}]({1})'
 
 
@@ -46,85 +49,87 @@ def parse(message, delimiters=None, url_re=None):
             return message, []
         delimiters = DEFAULT_DELIMITERS
 
+    # Build a regex to efficiently test all delimiters at once.
+    # Note that the largest delimiter should go first, we don't
+    # want ``` to be interpreted as a single back-tick in a code block.
+    delim_re = re.compile('|'.join('({})'.format(re.escape(k))
+                                   for k in sorted(delimiters, key=len, reverse=True)))
+
     # Cannot use a for loop because we need to skip some indices
     i = 0
     result = []
-    current = None
-    end_delimiter = None
 
     # Work on byte level with the utf-16le encoding to get the offsets right.
     # The offset will just be half the index we're at.
     message = add_surrogate(message)
     while i < len(message):
-        if url_re and current is None:
-            # If we're not inside a previous match since Telegram doesn't allow
-            # nested message entities, try matching the URL from the i'th pos.
-            url_match = url_re.match(message, pos=i)
-            if url_match:
-                # Replace the whole match with only the inline URL text.
+        m = delim_re.match(message, pos=i)
+
+        # Did we find some delimiter here at `i`?
+        if m:
+            delim = next(filter(None, m.groups()))
+
+            # +1 to avoid matching right after (e.g. "****")
+            end = message.find(delim, i + len(delim) + 1)
+
+            # Did we find the earliest closing tag?
+            if end != -1:
+
+                # Remove the delimiter from the string
                 message = ''.join((
-                    message[:url_match.start()],
-                    url_match.group(1),
-                    message[url_match.end():]
+                        message[:i],
+                        message[i + len(delim):end],
+                        message[end + len(delim):]
                 ))
 
-                result.append(MessageEntityTextUrl(
-                    offset=url_match.start(), length=len(url_match.group(1)),
-                    url=del_surrogate(url_match.group(2))
-                ))
-                i += len(url_match.group(1))
-                # Next loop iteration, don't check delimiters, since
-                # a new inline URL might be right after this one.
+                # Check other affected entities
+                for ent in result:
+                    # If the end is after our start, it is affected
+                    if ent.offset + ent.length > i:
+                        # If the old start is before ours and the old end is after ours, we are fully enclosed
+                        if ent.offset <= i and ent.offset + ent.length >= end + len(delim):
+                            ent.length -= len(delim) * 2
+                        else:
+                            ent.length -= len(delim)
+
+                # Append the found entity
+                ent = delimiters[delim]
+                if ent == MessageEntityPre:
+                    result.append(ent(i, end - i - len(delim), ''))  # has 'lang'
+                else:
+                    result.append(ent(i, end - i - len(delim)))
+
+                # No nested entities inside code blocks
+                if ent in (MessageEntityCode, MessageEntityPre):
+                    i = end - len(delim)
+
                 continue
 
-        if end_delimiter is None:
-            # We're not expecting any delimiter, so check them all
-            for d, m in delimiters.items():
-                # Slice the string at the current i'th position to see if
-                # it matches the current delimiter d, otherwise skip it.
-                if message[i:i + len(d)] != d:
-                    continue
+        elif url_re:
+            m = url_re.match(message, pos=i)
+            if m:
+                # Replace the whole match with only the inline URL text.
+                message = ''.join((
+                    message[:m.start()],
+                    m.group(1),
+                    message[m.end():]
+                ))
 
-                if message[i + len(d):i + 2 * len(d)] == d:
-                    # The same delimiter can't be right afterwards, if
-                    # this were the case we would match empty strings
-                    # like `` which we don't want to.
-                    continue
+                delim_size = m.end() - m.start() - len(m.group(1))
+                for ent in result:
+                    # If the end is after our start, it is affected
+                    if ent.offset + ent.length > m.start():
+                        ent.length -= delim_size
 
-                # Get rid of the delimiter by slicing it away
-                message = message[:i] + message[i + len(d):]
-                if m == MessageEntityPre:
-                    # Special case, also has 'lang'
-                    current = m(i, None, '')
-                else:
-                    current = m(i, None)
+                result.append(MessageEntityTextUrl(
+                    offset=m.start(), length=len(m.group(1)),
+                    url=del_surrogate(m.group(2))
+                ))
+                i += len(m.group(1))
+                continue
 
-                end_delimiter = d  # We expect the same delimiter.
-                break
-
-        elif message[i:i + len(end_delimiter)] == end_delimiter:
-            message = message[:i] + message[i + len(end_delimiter):]
-            current.length = i - current.offset
-            result.append(current)
-            current, end_delimiter = None, None
-            # Don't increment i here as we matched a delimiter,
-            # and there may be a new one right after. This is
-            # different than when encountering the first delimiter,
-            # as we already know there won't be the same right after.
-            continue
-
-        # Next iteration
         i += 1
 
-    # We may have found some a delimiter but not its ending pair.
-    # If this is the case, we want to insert the delimiter character back.
-    if current is not None:
-        message = (
-            message[:current.offset]
-            + end_delimiter
-            + message[current.offset:]
-        )
-
     message = strip_text(message, result)
     return del_surrogate(message), result
 
@@ -146,55 +151,43 @@ def unparse(text, entities, delimiters=None, url_fmt=None):
             return text
         delimiters = DEFAULT_DELIMITERS
 
-    if url_fmt is None:
-        url_fmt = DEFAULT_URL_FORMAT
+    if url_fmt is not None:
+        warnings.warn('url_fmt is deprecated')  # since it complicates everything *a lot*
 
     if isinstance(entities, TLObject):
         entities = (entities,)
-    else:
-        entities = tuple(sorted(entities, key=lambda e: e.offset, reverse=True))
 
     text = add_surrogate(text)
     delimiters = {v: k for k, v in delimiters.items()}
-    for entity in entities:
+    insert_at = []
+    for i, entity in enumerate(entities):
         s = entity.offset
         e = entity.offset + entity.length
         delimiter = delimiters.get(type(entity), None)
         if delimiter:
-            text = text[:s] + delimiter + text[s:e] + delimiter + text[e:]
-        elif url_fmt:
+            insert_at.append((s, i, delimiter))
+            insert_at.append((e, -i, delimiter))
+        else:
             url = None
             if isinstance(entity, MessageEntityTextUrl):
                 url = entity.url
             elif isinstance(entity, MessageEntityMentionName):
                 url = 'tg://user?id={}'.format(entity.user_id)
             if url:
-                # It's possible that entities are malformed and end up in the
-                # middle of some character, like emoji, by using malformed
-                # clients or bots. Try decoding the current one to check if
-                # this is the case, and if it is, advance the entity.
-                while e <= len(text):
-                    try:
-                        del_surrogate(text[s:e])
-                        break
-                    except UnicodeDecodeError:
-                        e += 1
-                else:
-                    # Out of bounds, no luck going forward
-                    while e > s:
-                        try:
-                            del_surrogate(text[s:e])
-                            break
-                        except UnicodeDecodeError:
-                            e -= 1
-                    else:
-                        # No luck going backwards either, ignore entity
-                        continue
+                insert_at.append((s, i, '['))
+                insert_at.append((e, -i, ']({})'.format(url)))
 
-                text = (
-                    text[:s] +
-                    add_surrogate(url_fmt.format(text[s:e], url)) +
-                    text[e:]
-                )
+    insert_at.sort(key=lambda t: (t[0], t[1]))
+    while insert_at:
+        at, _, what = insert_at.pop()
+
+        # If we are in the middle of a surrogate nudge the position by -1.
+        # Otherwise we would end up with malformed text and fail to encode.
+        # For example of bad input: "Hi \ud83d\ude1c"
+        # https://en.wikipedia.org/wiki/UTF-16#U+010000_to_U+10FFFF
+        while within_surrogate(text, at):
+            at += 1
+
+        text = text[:at] + what + text[at:]
 
     return del_surrogate(text)

telethon/extensions/messagepacker.py

@@ -22,11 +22,10 @@ class MessagePacker:
     point where outgoing requests are put, and where ready-messages are get.
     """
 
-    def __init__(self, state, loop, loggers):
+    def __init__(self, state, loggers):
         self._state = state
-        self._loop = loop
         self._deque = collections.deque()
-        self._ready = asyncio.Event(loop=loop)
+        self._ready = asyncio.Event()
         self._log = loggers[__name__]
 
     def append(self, state):

telethon/functions.py

@@ -0,0 +1 @@
+from .tl.functions import *