From ef794bf75dd00b73cf12c85ec3a0880088dcfa82 Mon Sep 17 00:00:00 2001 From: Andrei Fokau Date: Sat, 28 Oct 2017 12:21:07 +0200 Subject: [PATCH 01/44] Fix importing dependencies during installing (#384) --- setup.py | 19 ++++--------------- telethon/__init__.py | 5 ++++- telethon/telegram_bare_client.py | 4 ++-- telethon/version.py | 3 +++ 4 files changed, 13 insertions(+), 18 deletions(-) create mode 100644 telethon/version.py diff --git a/setup.py b/setup.py index 2058924f..3f8ee7a6 100755 --- a/setup.py +++ b/setup.py @@ -15,16 +15,11 @@ Extra supported commands are: from codecs import open from sys import argv import os +import re # Always prefer setuptools over distutils from setuptools import find_packages, setup -try: - from telethon import TelegramClient -except Exception as e: - print('Failed to import TelegramClient due to', e) - TelegramClient = None - class TempWorkDir: """Switches the working directory to be the one on which this file lives, @@ -94,21 +89,15 @@ def main(): fetch_errors(ERRORS_JSON) else: - if not TelegramClient: - gen_tl() - from telethon import TelegramClient as TgClient - version = TgClient.__version__ - else: - version = TelegramClient.__version__ - # Get the long description from the README file with open('README.rst', encoding='utf-8') as f: long_description = f.read() + with open('telethon/version.py', encoding='utf-8') as f: + version = re.search(r"^__version__\s+=\s+'(.*)'$", + f.read(), flags=re.MULTILINE).group(1) setup( name='Telethon', - - # Versions should comply with PEP440. version=version, description="Full-featured Telegram client library for Python 3", long_description=long_description, diff --git a/telethon/__init__.py b/telethon/__init__.py index c8593168..1210fa90 100644 --- a/telethon/__init__.py +++ b/telethon/__init__.py @@ -1,4 +1,7 @@ from .telegram_bare_client import TelegramBareClient from .telegram_client import TelegramClient from .network import ConnectionMode -from . import tl +from . import tl, version + + +__version__ = version.__version__ diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index a589fa9e..7ebf5ec1 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -9,7 +9,7 @@ from signal import signal, SIGINT, SIGTERM, SIGABRT from threading import Lock from time import sleep -from . import helpers as utils +from . import helpers as utils, version from .crypto import rsa, CdnDecrypter from .errors import ( RPCError, BrokenAuthKeyError, ServerError, @@ -60,7 +60,7 @@ class TelegramBareClient: """ # Current TelegramClient version - __version__ = '0.15.3' + __version__ = version.__version__ # TODO Make this thread-safe, all connections share the same DC _config = None # Server configuration (with .dc_options) diff --git a/telethon/version.py b/telethon/version.py new file mode 100644 index 00000000..2bde1b96 --- /dev/null +++ b/telethon/version.py @@ -0,0 +1,3 @@ +# Versions should comply with PEP440. +# This line is parsed in setup.py: +__version__ = '0.15.3' From 5adec2e1abe656f694226894209a8d904c1355de Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 28 Oct 2017 19:06:41 +0200 Subject: [PATCH 02/44] Initial attempt at parsing Markdown-like syntax --- telethon/extensions/markdown.py | 107 ++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 telethon/extensions/markdown.py diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py new file mode 100644 index 00000000..d3581cf9 --- /dev/null +++ b/telethon/extensions/markdown.py @@ -0,0 +1,107 @@ +""" +Simple markdown parser which does not support nesting. Intended primarily +for use within the library, which attempts to handle emojies correctly, +since they seem to count as two characters and it's a bit strange. +""" +import re +from enum import Enum + +from ..tl.types import ( + MessageEntityBold, MessageEntityItalic, MessageEntityCode, MessageEntityPre +) + + +class Mode(Enum): + """Different modes supported by Telegram's Markdown""" + NONE = 0 + BOLD = 1 + ITALIC = 2 + CODE = 3 + PRE = 4 + + +EMOJI_PATTERN = re.compile( + '[' + '\U0001F600-\U0001F64F' # emoticons + '\U0001F300-\U0001F5FF' # symbols & pictographs + '\U0001F680-\U0001F6FF' # transport & map symbols + '\U0001F1E0-\U0001F1FF' # flags (iOS) + ']+', flags=re.UNICODE +) + + +def is_emoji(char): + """Returns True if 'char' looks like an emoji""" + return bool(EMOJI_PATTERN.match(char)) + + +def emojiness(char): + """ + Returns the "emojiness" of an emoji, or how many characters it counts as. + 1 if it's not an emoji, 2 usual, 3 "special" (seem to count more). + """ + if not is_emoji(char): + return 1 + if ord(char) < ord('🤐'): + return 2 + else: + return 3 + + +def parse(message, delimiters=None): + """ + Parses the given message and returns the stripped message and a list + of tuples containing (start, end, mode) using the specified delimiters + dictionary (or default if None). + """ + if not delimiters: + if delimiters is not None: + return message, [] + + delimiters = { + '**': Mode.BOLD, + '__': Mode.ITALIC, + '`': Mode.CODE, + '```': Mode.PRE + } + + result = [] + current = Mode.NONE + offset = 0 + i = 0 + while i < len(message): + for d, m in delimiters.items(): + if message[i:i + len(d)] == d and current in (Mode.NONE, m): + if message[i + len(d):i + 2 * len(d)] == d: + continue # ignore two consecutive delimiters + + message = message[:i] + message[i + len(d):] + if current == Mode.NONE: + result.append(offset) + current = m + else: + result[-1] = (result[-1], offset, current) + current = Mode.NONE + break + + offset += emojiness(message[i]) + i += 1 + if result and not isinstance(result[-1], tuple): + result.pop() + return message, result + + +def parse_tg(message, delimiters=None): + """Similar to parse(), but returns a list of MessageEntity's""" + message, tuples = parse(message, delimiters=delimiters) + result = [] + for start, end, mode in tuples: + if mode == Mode.BOLD: + result.append(MessageEntityBold(start, end - start)) + elif mode == Mode.ITALIC: + result.append(MessageEntityItalic(start, end - start)) + elif mode == Mode.CODE: + result.append(MessageEntityCode(start, end - start)) + elif mode == Mode.PRE: + result.append(MessageEntityPre(start, end - start, '')) + return message, result From 9600a9ea0bfd522becd73af38b05aefac65e2f82 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 28 Oct 2017 19:17:18 +0200 Subject: [PATCH 03/44] Fix markdown parsing failing if delimiter was last character --- telethon/extensions/markdown.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index d3581cf9..2e5a899c 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -84,8 +84,10 @@ def parse(message, delimiters=None): current = Mode.NONE break - offset += emojiness(message[i]) - i += 1 + if i < len(message): + offset += emojiness(message[i]) + i += 1 + if result and not isinstance(result[-1], tuple): result.pop() return message, result From 368269cb11fc3b5944bb090f066ef66d4bd7ffaf Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 29 Oct 2017 16:33:10 +0100 Subject: [PATCH 04/44] Add ability to parse inline URLs --- telethon/extensions/markdown.py | 59 +++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 2e5a899c..90ab9d99 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -7,7 +7,8 @@ import re from enum import Enum from ..tl.types import ( - MessageEntityBold, MessageEntityItalic, MessageEntityCode, MessageEntityPre + MessageEntityBold, MessageEntityItalic, MessageEntityCode, + MessageEntityPre, MessageEntityTextUrl ) @@ -18,6 +19,7 @@ class Mode(Enum): ITALIC = 2 CODE = 3 PRE = 4 + URL = 5 EMOJI_PATTERN = re.compile( @@ -48,12 +50,19 @@ def emojiness(char): return 3 -def parse(message, delimiters=None): +def parse(message, delimiters=None, url_re=r'\[(.+?)\]\((.+?)\)'): """ Parses the given message and returns the stripped message and a list of tuples containing (start, end, mode) using the specified delimiters dictionary (or default if None). + + The url_re(gex) must contain two matching groups: the text to be + clickable and the URL itself. """ + if url_re: + if isinstance(url_re, str): + url_re = re.compile(url_re) + if not delimiters: if delimiters is not None: return message, [] @@ -70,19 +79,35 @@ def parse(message, delimiters=None): offset = 0 i = 0 while i < len(message): - for d, m in delimiters.items(): - if message[i:i + len(d)] == d and current in (Mode.NONE, m): - if message[i + len(d):i + 2 * len(d)] == d: - continue # ignore two consecutive delimiters + if current == Mode.NONE: + url_match = url_re.match(message, pos=i) + if url_match: + message = ''.join(( + message[:url_match.start()], + url_match.group(1), + message[url_match.end():] + )) + emoji_len = sum(emojiness(c) for c in url_match.group(1)) + result.append(( + offset, + i + emoji_len, + (Mode.URL, url_match.group(2)) + )) + i += len(url_match.group(1)) + else: + for d, m in delimiters.items(): + if message[i:i + len(d)] == d and current in (Mode.NONE, m): + if message[i + len(d):i + 2 * len(d)] == d: + continue # ignore two consecutive delimiters - message = message[:i] + message[i + len(d):] - if current == Mode.NONE: - result.append(offset) - current = m - else: - result[-1] = (result[-1], offset, current) - current = Mode.NONE - break + message = message[:i] + message[i + len(d):] + if current == Mode.NONE: + result.append(offset) + current = m + else: + result[-1] = (result[-1], offset, current) + current = Mode.NONE + break if i < len(message): offset += emojiness(message[i]) @@ -98,6 +123,10 @@ def parse_tg(message, delimiters=None): message, tuples = parse(message, delimiters=delimiters) result = [] for start, end, mode in tuples: + extra = None + if isinstance(mode, tuple): + mode, extra = mode + if mode == Mode.BOLD: result.append(MessageEntityBold(start, end - start)) elif mode == Mode.ITALIC: @@ -106,4 +135,6 @@ def parse_tg(message, delimiters=None): result.append(MessageEntityCode(start, end - start)) elif mode == Mode.PRE: result.append(MessageEntityPre(start, end - start, '')) + elif mode == Mode.URL: + result.append(MessageEntityTextUrl(start, end - start, extra)) return message, result From f5fafc6a27bd782a1c6e360c49bdb1183db84e98 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 29 Oct 2017 16:41:30 +0100 Subject: [PATCH 05/44] Enhance emoji detection --- telethon/extensions/markdown.py | 29 ++++++--- telethon_generator/emoji_ranges.py | 101 +++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 9 deletions(-) create mode 100644 telethon_generator/emoji_ranges.py diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 90ab9d99..fa33aace 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -22,19 +22,30 @@ class Mode(Enum): URL = 5 -EMOJI_PATTERN = re.compile( - '[' - '\U0001F600-\U0001F64F' # emoticons - '\U0001F300-\U0001F5FF' # symbols & pictographs - '\U0001F680-\U0001F6FF' # transport & map symbols - '\U0001F1E0-\U0001F1FF' # flags (iOS) - ']+', flags=re.UNICODE +# using telethon_generator/emoji_ranges.py +EMOJI_RANGES = ( + (8596, 8601), (8617, 8618), (8986, 8987), (9193, 9203), (9208, 9210), + (9642, 9643), (9723, 9726), (9728, 9733), (9735, 9746), (9748, 9751), + (9754, 9884), (9886, 9905), (9907, 9953), (9956, 9983), (9985, 9988), + (9992, 10002), (10035, 10036), (10067, 10069), (10083, 10087), + (10133, 10135), (10548, 10549), (11013, 11015), (11035, 11036), + (126976, 127166), (127169, 127183), (127185, 127231), (127245, 127247), + (127340, 127345), (127358, 127359), (127377, 127386), (127405, 127487), + (127489, 127503), (127538, 127546), (127548, 127551), (127561, 128419), + (128421, 128591), (128640, 128767), (128884, 128895), (128981, 129023), + (129036, 129039), (129096, 129103), (129114, 129119), (129160, 129167), + (129198, 129338), (129340, 129342), (129344, 129349), (129351, 129355), + (129357, 129471), (129473, 131069) ) def is_emoji(char): """Returns True if 'char' looks like an emoji""" - return bool(EMOJI_PATTERN.match(char)) + char = ord(char) + for start, end in EMOJI_RANGES: + if start <= char <= end: + return True + return False def emojiness(char): @@ -44,7 +55,7 @@ def emojiness(char): """ if not is_emoji(char): return 1 - if ord(char) < ord('🤐'): + if ord(char) < 129296: return 2 else: return 3 diff --git a/telethon_generator/emoji_ranges.py b/telethon_generator/emoji_ranges.py new file mode 100644 index 00000000..90597cf6 --- /dev/null +++ b/telethon_generator/emoji_ranges.py @@ -0,0 +1,101 @@ +""" +Simple module to allow fetching unicode.org emoji lists and printing a +Python-like tuple out of them. + +May not be accurate 100%, and is definitely not as efficient as it could be, +but it should only be ran whenever the Unicode consortium decides to add +new emojies to the list. +""" +import os +import sys +import re +import urllib.error +import urllib.request + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + + +def get(url, enc='utf-8'): + try: + with urllib.request.urlopen(url) as f: + return f.read().decode(enc, errors='replace') + except urllib.error.HTTPError as e: + eprint('Caught', e, 'for', url, '; returning empty') + return '' + + +PREFIX_URL = 'http://unicode.org/Public/emoji/' +SUFFIX_URL = '/emoji-data.txt', '/emoji-sequences.txt' +VERSION_RE = re.compile(r'>(\d+.\d+)/<') +OUTPUT_TXT = 'emojies.txt' +CODEPOINT_RE = re.compile(r'([\da-fA-F]{3,}(?:[\s.]+[\da-fA-F]{3,}))') +EMOJI_START = 0x20e3 # emoji data has many more ranges, falling outside this +EMOJI_END = 200000 # from some tests those outside the range aren't emojies + + +versions = VERSION_RE.findall(get(PREFIX_URL)) +lines = [] +if not os.path.isfile(OUTPUT_TXT): + with open(OUTPUT_TXT, 'w') as f: + for version in versions: + for s in SUFFIX_URL: + url = PREFIX_URL + version + s + for line in get(url).split('\n'): + line = line.strip() + if not line or line.startswith('#'): + continue + m = CODEPOINT_RE.search(line) + if m and m.start() == 0: + f.write(m.group(1) + '\n') + + +points = set() +with open(OUTPUT_TXT) as f: + for line in f: + line = line.strip() + if ' ' in line: + for p in line.split(): + i = int(p, 16) + if i > 255: + points.add(i) + elif '.' in line: + s, e = line.split('..') + for i in range(int(s, 16), int(e, 16) + 1): + if i > 255: + points.add(i) + else: + i = int(line, 16) + if i > 255: + points.add(int(line, 16)) + + +ranges = [] +points = tuple(sorted(points)) +start = points[0] +last = start +for point in points: + if point - last > 1: + if start == last or not (EMOJI_START < start < EMOJI_END): + eprint( + 'Dropping', last - start + 1, + 'character(s) from', hex(start), ':', chr(start) + ) + else: + ranges.append((start, last)) + start = point + + last = point + + +if start == last or not (EMOJI_START < start < EMOJI_END): + eprint( + 'Dropping', last - start + 1, + 'character(s) from', hex(start), ':', chr(start) + ) +else: + ranges.append((start, last)) + + +print('EMOJI_RANGES = ({})'.format(', '.join(repr(r) for r in ranges))) From bcaa8007a3aee7485c31eb4eafb3161b9ac0f748 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 29 Oct 2017 16:43:30 +0100 Subject: [PATCH 06/44] Fix inline URL matching swallowing all parse entities --- telethon/extensions/markdown.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index fa33aace..13246f1e 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -90,6 +90,7 @@ def parse(message, delimiters=None, url_re=r'\[(.+?)\]\((.+?)\)'): offset = 0 i = 0 while i < len(message): + url_match = None if current == Mode.NONE: url_match = url_re.match(message, pos=i) if url_match: @@ -105,7 +106,7 @@ def parse(message, delimiters=None, url_re=r'\[(.+?)\]\((.+?)\)'): (Mode.URL, url_match.group(2)) )) i += len(url_match.group(1)) - else: + if not url_match: for d, m in delimiters.items(): if message[i:i + len(d)] == d and current in (Mode.NONE, m): if message[i + len(d):i + 2 * len(d)] == d: From d47a9f83d038b9462831a8bef641cfd6ae11bb3b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 29 Oct 2017 17:07:37 +0100 Subject: [PATCH 07/44] Fix some special cases which are not treated as emojis (offset 1) --- telethon/extensions/markdown.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 13246f1e..99c7a25e 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -22,6 +22,37 @@ class Mode(Enum): URL = 5 +# TODO Special cases, these aren't count as emojies. Alternatives? +# These were determined by generating all emojies with EMOJI_RANGES, +# sending the message through an official application, and cherry-picking +# which ones weren't rendered as emojies (from the beginning one). I am +# not responsible for dropping those characters that did not render with +# my font. +NOT_EMOJIES = { + 9733, 9735, 9736, 9737, 9738, 9739, 9740, 9741, 9743, 9744, 9746, 9750, + 9751, 9754, 9755, 9756, 9758, 9759, 9761, 9764, 9765, 9767, 9768, 9769, + 9771, 9772, 9773, 9776, 9777, 9778, 9779, 9780, 9781, 9782, 9783, 9787, + 9788, 9789, 9790, 9791, 9792, 9793, 9794, 9795, 9796, 9797, 9798, 9799, + 9812, 9813, 9814, 9815, 9816, 9817, 9818, 9819, 9820, 9821, 9822, 9823, + 9825, 9826, 9828, 9831, 9833, 9834, 9835, 9836, 9837, 9838, 9839, 9840, + 9841, 9842, 9843, 9844, 9845, 9846, 9847, 9848, 9849, 9850, 9852, 9853, + 9854, 9856, 9857, 9858, 9859, 9860, 9861, 9862, 9863, 9864, 9865, 9866, + 9867, 9868, 9869, 9870, 9871, 9872, 9873, 9877, 9880, 9882, 9886, 9887, + 9890, 9891, 9892, 9893, 9894, 9895, 9896, 9897, 9900, 9901, 9902, 9903, + 9907, 9908, 9909, 9910, 9911, 9912, 9920, 9921, 9922, 9923, 9985, 9987, + 9988, 9998, 10000, 10001, 10085, 10086, 10087, 127027, 127028, 127029, + 127030, 127031, 127032, 127033, 127034, 127035, 127036, 127037, 127038, + 127039, 127040, 127041, 127042, 127043, 127044, 127045, 127046, 127047, + 127048, 127049, 127050, 127051, 127052, 127053, 127054, 127055, 127056, + 127057, 127058, 127059, 127060, 127061, 127062, 127063, 127064, 127065, + 127066, 127067, 127068, 127069, 127070, 127071, 127072, 127073, 127074, + 127075, 127076, 127077, 127078, 127079, 127080, 127081, 127082, 127083, + 127084, 127085, 127086, 127087, 127088, 127089, 127090, 127091, 127092, + 127093, 127094, 127095, 127096, 127097, 127098, 127099, 127100, 127101, + 127102, 127103, 127104, 127105, 127106, 127107, 127108, 127109, 127110, + 127111, 127112, 127113, 127114, 127115, 127116, 127117, 127118, 127119, + 127120, 127121, 127122, 127123 +} # using telethon_generator/emoji_ranges.py EMOJI_RANGES = ( (8596, 8601), (8617, 8618), (8986, 8987), (9193, 9203), (9208, 9210), @@ -44,7 +75,7 @@ def is_emoji(char): char = ord(char) for start, end in EMOJI_RANGES: if start <= char <= end: - return True + return char not in NOT_EMOJIES return False From 2609bd9bd17e4db4e1c6423dbf09d7bcce82bf5d Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 29 Oct 2017 18:21:21 +0100 Subject: [PATCH 08/44] Use constants and allow empty URL regex when parsing markdown --- telethon/extensions/markdown.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 99c7a25e..078736a2 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -70,6 +70,16 @@ EMOJI_RANGES = ( ) +DEFAULT_DELIMITERS = { + '**': Mode.BOLD, + '__': Mode.ITALIC, + '`': Mode.CODE, + '```': Mode.PRE +} + +DEFAULT_URL_RE = re.compile(r'\[(.+?)\]\((.+?)\)') + + def is_emoji(char): """Returns True if 'char' looks like an emoji""" char = ord(char) @@ -92,7 +102,7 @@ def emojiness(char): return 3 -def parse(message, delimiters=None, url_re=r'\[(.+?)\]\((.+?)\)'): +def parse(message, delimiters=None, url_re=None): """ Parses the given message and returns the stripped message and a list of tuples containing (start, end, mode) using the specified delimiters @@ -101,20 +111,16 @@ def parse(message, delimiters=None, url_re=r'\[(.+?)\]\((.+?)\)'): The url_re(gex) must contain two matching groups: the text to be clickable and the URL itself. """ - if url_re: + if url_re is None: + url_re = DEFAULT_URL_RE + elif url_re: if isinstance(url_re, str): url_re = re.compile(url_re) if not delimiters: if delimiters is not None: return message, [] - - delimiters = { - '**': Mode.BOLD, - '__': Mode.ITALIC, - '`': Mode.CODE, - '```': Mode.PRE - } + delimiters = DEFAULT_DELIMITERS result = [] current = Mode.NONE @@ -122,7 +128,7 @@ def parse(message, delimiters=None, url_re=r'\[(.+?)\]\((.+?)\)'): i = 0 while i < len(message): url_match = None - if current == Mode.NONE: + if url_re and current == Mode.NONE: url_match = url_re.match(message, pos=i) if url_match: message = ''.join(( From 6567f4b5674091f8282f9a642b8a1fd2db8738c9 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 29 Oct 2017 20:10:29 +0100 Subject: [PATCH 09/44] Clean .download_contact and a wrong indent level --- telethon/telegram_client.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index c64051bf..5a8fb5a6 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -51,6 +51,7 @@ from .tl.types import ( PeerUser, InputPeerUser, InputPeerChat, InputPeerChannel) from .tl.types.messages import DialogsSlice + class TelegramClient(TelegramBareClient): """Full featured TelegramClient meant to extend the basic functionality - @@ -825,13 +826,9 @@ class TelegramClient(TelegramBareClient): last_name = (last_name or '').replace(';', '') f.write('BEGIN:VCARD\n') f.write('VERSION:4.0\n') - f.write('N:{};{};;;\n'.format( - first_name, last_name) - ) + f.write('N:{};{};;;\n'.format(first_name, last_name)) f.write('FN:{} {}\n'.format(first_name, last_name)) - f.write('TEL;TYPE=cell;VALUE=uri:tel:+{}\n'.format( - phone_number - )) + f.write('TEL;TYPE=cell;VALUE=uri:tel:+{}\n'.format(phone_number)) f.write('END:VCARD\n') finally: # Only close the stream if we opened it @@ -1042,4 +1039,4 @@ class TelegramClient(TelegramBareClient): 'Make sure you have encountered this peer before.'.format(peer) ) - # endregion + # endregion From 05626c827477b21a6c200da613965e20fbacfaa0 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 29 Oct 2017 20:13:00 +0100 Subject: [PATCH 10/44] Implement missing .to_dict() and .stringify() on message/container --- telethon/tl/message_container.py | 19 +++++++++++++++---- telethon/tl/tl_message.py | 14 ++++++++++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/telethon/tl/message_container.py b/telethon/tl/message_container.py index 12d617cc..58fb8021 100644 --- a/telethon/tl/message_container.py +++ b/telethon/tl/message_container.py @@ -11,15 +11,20 @@ class MessageContainer(TLObject): self.content_related = False self.messages = messages + def to_dict(self, recursive=True): + return { + 'content_related': self.content_related, + 'messages': + ([] if self.messages is None else [ + None if x is None else x.to_dict() for x in self.messages + ]) if recursive else self.messages, + } + def __bytes__(self): return struct.pack( ' Date: Mon, 30 Oct 2017 10:33:45 +0100 Subject: [PATCH 11/44] Fix InputPeer* with None hash, drop them off database (closes #354) --- telethon/tl/entity_database.py | 20 +++++++++++++++++--- telethon/utils.py | 8 ++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/telethon/tl/entity_database.py b/telethon/tl/entity_database.py index b0fc70fb..0c92c75f 100644 --- a/telethon/tl/entity_database.py +++ b/telethon/tl/entity_database.py @@ -24,7 +24,12 @@ class EntityDatabase: self._entities = {} # marked_id: user|chat|channel if input_list: - self._input_entities = {k: v for k, v in input_list} + # TODO For compatibility reasons some sessions were saved with + # 'access_hash': null in the JSON session file. Drop these, as + # it means we don't have access to such InputPeers. Issue #354. + self._input_entities = { + k: v for k, v in input_list if v is not None + } else: self._input_entities = {} # marked_id: hash @@ -69,8 +74,17 @@ class EntityDatabase: try: p = utils.get_input_peer(e, allow_self=False) - new_input[utils.get_peer_id(p, add_mark=True)] = \ - getattr(p, 'access_hash', 0) # chats won't have hash + marked_id = utils.get_peer_id(p, add_mark=True) + + if isinstance(p, InputPeerChat): + # Chats don't have a hash + new_input[marked_id] = 0 + elif p.access_hash: + # Some users and channels seem to be returned without + # an 'access_hash', meaning Telegram doesn't want you + # to access them. This is the reason behind ensuring + # that the 'access_hash' is non-zero. See issue #354. + new_input[marked_id] = p.access_hash if self.enabled_full: if isinstance(e, (User, Chat, Channel)): diff --git a/telethon/utils.py b/telethon/utils.py index afb24b16..a05a4990 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -84,13 +84,13 @@ def get_input_peer(entity, allow_self=True): if entity.is_self and allow_self: return InputPeerSelf() else: - return InputPeerUser(entity.id, entity.access_hash) + return InputPeerUser(entity.id, entity.access_hash or 0) if isinstance(entity, (Chat, ChatEmpty, ChatForbidden)): return InputPeerChat(entity.id) if isinstance(entity, (Channel, ChannelForbidden)): - return InputPeerChannel(entity.id, entity.access_hash) + return InputPeerChannel(entity.id, entity.access_hash or 0) # Less common cases if isinstance(entity, UserEmpty): @@ -120,7 +120,7 @@ def get_input_channel(entity): return entity if isinstance(entity, (Channel, ChannelForbidden)): - return InputChannel(entity.id, entity.access_hash) + return InputChannel(entity.id, entity.access_hash or 0) if isinstance(entity, InputPeerChannel): return InputChannel(entity.channel_id, entity.access_hash) @@ -140,7 +140,7 @@ def get_input_user(entity): if entity.is_self: return InputUserSelf() else: - return InputUser(entity.id, entity.access_hash) + return InputUser(entity.id, entity.access_hash or 0) if isinstance(entity, InputPeerSelf): return InputUserSelf() From 0a14aa1bc6e95b47cb355c6a8c709645be04ab05 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 30 Oct 2017 10:56:39 +0100 Subject: [PATCH 12/44] Remove additional check when calculating emojies length This special check treated some emojies as 3 characters long but this shouldn't have actually been done, likely due to the old regex matching more things as emoji than it should (which would have count as 2 too, making up for 1+3 from the new is_emoji()). --- telethon/extensions/markdown.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 078736a2..432b1452 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -91,15 +91,10 @@ def is_emoji(char): def emojiness(char): """ - Returns the "emojiness" of an emoji, or how many characters it counts as. - 1 if it's not an emoji, 2 usual, 3 "special" (seem to count more). + Returns 2 if the character is an emoji, or 1 otherwise. + This seems to be the length Telegram uses for offsets and lengths. """ - if not is_emoji(char): - return 1 - if ord(char) < 129296: - return 2 - else: - return 3 + return 2 if is_emoji(char) else 1 def parse(message, delimiters=None, url_re=None): From 82cac4836cc79a5cbb92613b15372ea0047a811b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 30 Oct 2017 11:15:53 +0100 Subject: [PATCH 13/44] Fix markdown URL parsing using character index instead offset --- telethon/extensions/markdown.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 432b1452..574fe025 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -134,7 +134,7 @@ def parse(message, delimiters=None, url_re=None): emoji_len = sum(emojiness(c) for c in url_match.group(1)) result.append(( offset, - i + emoji_len, + offset + emoji_len, (Mode.URL, url_match.group(2)) )) i += len(url_match.group(1)) From 7e204632e26a109152a58b784e789aaafc70e5fc Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 30 Oct 2017 11:17:22 +0100 Subject: [PATCH 14/44] Add parse_mode parameter to TelegramClient.send_message() --- telethon/telegram_client.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index c64051bf..da248323 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -50,6 +50,8 @@ from .tl.types import ( UpdateNewChannelMessage, UpdateNewMessage, UpdateShortSentMessage, PeerUser, InputPeerUser, InputPeerChat, InputPeerChannel) from .tl.types.messages import DialogsSlice +from .extensions import markdown + class TelegramClient(TelegramBareClient): """Full featured TelegramClient meant to extend the basic functionality - @@ -347,21 +349,39 @@ class TelegramClient(TelegramBareClient): entity, message, reply_to=None, + parse_mode=None, link_preview=True): """ Sends the given message to the specified entity (user/chat/channel). - :param str | int | User | Chat | Channel entity: To who will it be sent. - :param str message: The message to be sent. - :param int | Message reply_to: Whether to reply to a message or not. - :param link_preview: Should the link preview be shown? + :param str | int | User | Chat | Channel entity: + To who will it be sent. + :param str message: + The message to be sent. + :param int | Message reply_to: + Whether to reply to a message or not. + :param str parse_mode: + Can be 'md' or 'markdown' for markdown-like parsing, in a similar + fashion how official clients work. + :param link_preview: + Should the link preview be shown? + :return Message: the sent message """ entity = self.get_input_entity(entity) + if parse_mode: + parse_mode = parse_mode.lower() + if parse_mode in {'md', 'markdown'}: + message, msg_entities = markdown.parse_tg(message) + else: + raise ValueError('Unknown parsing mode', parse_mode) + else: + msg_entities = [] + request = SendMessageRequest( peer=entity, message=message, - entities=[], + entities=msg_entities, no_webpage=not link_preview, reply_to_msg_id=self._get_reply_to(reply_to) ) From 3d6c8915e3250d3ec1b26faa73cc08ad1f811081 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 31 Oct 2017 12:48:55 +0100 Subject: [PATCH 15/44] Allow >100 limits when getting message history (implements #290) --- telethon/telegram_client.py | 110 ++++++++++++++++++++++++++---------- 1 file changed, 80 insertions(+), 30 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 9c911ddd..994a59cc 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -1,4 +1,5 @@ import os +import time from datetime import datetime, timedelta from mimetypes import guess_type @@ -48,7 +49,8 @@ from .tl.types import ( Message, MessageMediaContact, MessageMediaDocument, MessageMediaPhoto, InputUserSelf, UserProfilePhoto, ChatPhoto, UpdateMessageID, UpdateNewChannelMessage, UpdateNewMessage, UpdateShortSentMessage, - PeerUser, InputPeerUser, InputPeerChat, InputPeerChannel) + PeerUser, InputPeerUser, InputPeerChat, InputPeerChannel, MessageEmpty +) from .tl.types.messages import DialogsSlice from .extensions import markdown @@ -458,43 +460,91 @@ class TelegramClient(TelegramBareClient): """ Gets the message history for the specified entity - :param entity: The entity from whom to retrieve the message history - :param limit: Number of messages to be retrieved - :param offset_date: Offset date (messages *previous* to this date will be retrieved) - :param offset_id: Offset message ID (only messages *previous* to the given ID will be retrieved) - :param max_id: All the messages with a higher (newer) ID or equal to this will be excluded - :param min_id: All the messages with a lower (older) ID or equal to this will be excluded - :param add_offset: Additional message offset (all of the specified offsets + this offset = older messages) + :param entity: + The entity from whom to retrieve the message history. + :param limit: + Number of messages to be retrieved. Due to limitations with the API + retrieving more than 3000 messages will take longer than half a + minute (or even more based on previous calls). The limit may also + be None, which would eventually return the whole history. + :param offset_date: + Offset date (messages *previous* to this date will be retrieved). + :param offset_id: + Offset message ID (only messages *previous* to the given ID will + be retrieved). + :param max_id: + All the messages with a higher (newer) ID or equal to this will + be excluded + :param min_id: + All the messages with a lower (older) ID or equal to this will + be excluded. + :param add_offset: + Additional message offset + (all of the specified offsets + this offset = older messages). :return: A tuple containing total message count and two more lists ([messages], [senders]). Note that the sender can be null if it was not found! """ - result = self(GetHistoryRequest( - peer=self.get_input_entity(entity), - limit=limit, - offset_date=offset_date, - offset_id=offset_id, - max_id=max_id, - min_id=min_id, - add_offset=add_offset - )) + limit = float('inf') if limit is None else int(limit) + total_messages = 0 + messages = [] + entities = {} + while len(messages) < limit: + # Telegram has a hard limit of 100 + real_limit = min(limit - len(messages), 100) + result = self(GetHistoryRequest( + peer=self.get_input_entity(entity), + limit=real_limit, + offset_date=offset_date, + offset_id=offset_id, + max_id=max_id, + min_id=min_id, + add_offset=add_offset + )) + messages.extend( + m for m in result.messages if not isinstance(m, MessageEmpty) + ) + total_messages = getattr(result, 'count', len(result.messages)) - # The result may be a messages slice (not all messages were retrieved) - # or simply a messages TLObject. In the later case, no "count" - # attribute is specified, so the total messages count is simply - # the count of retrieved messages - total_messages = getattr(result, 'count', len(result.messages)) + # TODO We can potentially use self.session.database, but since + # it might be disabled, use a local dictionary. + for u in result.users: + entities[utils.get_peer_id(u, add_mark=True)] = u + for c in result.chats: + entities[utils.get_peer_id(c, add_mark=True)] = c - # Iterate over all the messages and find the sender User - entities = [ - utils.find_user_or_chat(m.from_id, result.users, result.chats) - if m.from_id is not None else - utils.find_user_or_chat(m.to_id, result.users, result.chats) + if len(result.messages) < real_limit: + break - for m in result.messages - ] + offset_id = result.messages[-1].id + offset_date = result.messages[-1].date - return total_messages, result.messages, entities + # Telegram limit seems to be 3000 messages within 30 seconds in + # batches of 100 messages each request (since the FloodWait was + # of 30 seconds). If the limit is greater than that, we will + # sleep 1s between each request. + if limit > 3000: + time.sleep(1) + + # In a new list with the same length as the messages append + # their senders, so people can zip(messages, senders). + senders = [] + for m in messages: + if m.from_id: + who = entities[utils.get_peer_id(m.from_id, add_mark=True)] + elif getattr(m, 'fwd_from', None): + # .from_id is optional, so this is the sanest fallback. + who = entities[utils.get_peer_id( + m.fwd_from.from_id or m.fwd_from.channel_id, + add_mark=True + )] + else: + # If there's not even a FwdHeader, fallback to the sender + # being where the message was sent. + who = entities[utils.get_peer_id(m.to_id, add_mark=True)] + senders.append(who) + + return total_messages, messages, senders def send_read_acknowledge(self, entity, messages=None, max_id=None): """ From 9a12738f0ed60f814c50bbc856032a20876f8d3b Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 31 Oct 2017 13:52:43 +0100 Subject: [PATCH 16/44] Fix .get_message_history not working with limit=0 --- telethon/telegram_client.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 994a59cc..0452b72b 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -485,7 +485,16 @@ class TelegramClient(TelegramBareClient): :return: A tuple containing total message count and two more lists ([messages], [senders]). Note that the sender can be null if it was not found! """ + entity = self.get_input_entity(entity) limit = float('inf') if limit is None else int(limit) + if limit == 0: + # No messages, but we still need to know the total message count + result = self(GetHistoryRequest( + peer=self.get_input_entity(entity), limit=1, + offset_date=None, offset_id=0, max_id=0, min_id=0, add_offset=0 + )) + return getattr(result, 'count', len(result.messages)), [], [] + total_messages = 0 messages = [] entities = {} @@ -493,7 +502,7 @@ class TelegramClient(TelegramBareClient): # Telegram has a hard limit of 100 real_limit = min(limit - len(messages), 100) result = self(GetHistoryRequest( - peer=self.get_input_entity(entity), + peer=entity, limit=real_limit, offset_date=offset_date, offset_id=offset_id, From 0bfd8ff0320a260c2c937f8c5ce5e923bef20d43 Mon Sep 17 00:00:00 2001 From: Dan Date: Fri, 3 Nov 2017 12:59:17 +0100 Subject: [PATCH 17/44] Add much faster integer factorization (#403 related to #199) --- telethon/crypto/factorization.py | 94 ++++++++++++-------------------- 1 file changed, 34 insertions(+), 60 deletions(-) diff --git a/telethon/crypto/factorization.py b/telethon/crypto/factorization.py index 69c8dcc9..359887d3 100644 --- a/telethon/crypto/factorization.py +++ b/telethon/crypto/factorization.py @@ -1,71 +1,45 @@ from random import randint -try: - import sympy.ntheory -except ImportError: - sympy = None class Factorization: - @staticmethod - def find_small_multiplier_lopatin(what): - """Finds the small multiplier by using Lopatin's method""" - g = 0 - for i in range(3): - q = (randint(0, 127) & 15) + 17 - x = randint(0, 1000000000) + 1 - y = x - lim = 1 << (i + 18) - for j in range(1, lim): - a, b, c = x, x, q - while b != 0: - if (b & 1) != 0: - c += a - if c >= what: - c -= what - a += a - if a >= what: - a -= what - b >>= 1 + @classmethod + def factorize(cls, pq): + if pq % 2 == 0: + return 2, pq // 2 - x = c - z = y - x if x < y else x - y - g = Factorization.gcd(z, what) - if g != 1: + y, c, m = randint(1, pq - 1), randint(1, pq - 1), randint(1, pq - 1) + g = r = q = 1 + x = ys = 0 + + while g == 1: + x = y + for i in range(r): + y = (pow(y, 2, pq) + c) % pq + + k = 0 + while k < r and g == 1: + ys = y + for i in range(min(m, r - k)): + y = (pow(y, 2, pq) + c) % pq + q = q * (abs(x - y)) % pq + + g = cls.gcd(q, pq) + k += m + + r *= 2 + + if g == pq: + while True: + ys = (pow(ys, 2, pq) + c) % pq + g = cls.gcd(abs(x - ys), pq) + if g > 1: break - if (j & (j - 1)) == 0: - y = x - - if g > 1: - break - - p = what // g - return min(p, g) + return g, pq // g @staticmethod def gcd(a, b): - """Calculates the greatest common divisor""" - while a != 0 and b != 0: - while b & 1 == 0: - b >>= 1 + while b: + a, b = b, a % b - while a & 1 == 0: - a >>= 1 - - if a > b: - a -= b - else: - b -= a - - return a if b == 0 else b - - @staticmethod - def factorize(pq): - """Factorizes the given number and returns both - the divisor and the number divided by the divisor - """ - if sympy: - return tuple(sympy.ntheory.factorint(pq).keys()) - else: - divisor = Factorization.find_small_multiplier_lopatin(pq) - return divisor, pq // divisor + return a From 1741608f2867182cd42f0fed9ecf8e89a0ef9602 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 4 Nov 2017 12:34:44 +0100 Subject: [PATCH 18/44] Use larger batches for .get_dialogs(limit=None) --- telethon/telegram_client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 0452b72b..c638c2b1 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -279,22 +279,21 @@ class TelegramClient(TelegramBareClient): The peer to be used as an offset. :return: A tuple of lists ([dialogs], [entities]). """ - if limit is None: - limit = float('inf') + limit = float('inf') if limit is None else int(limit) + if limit == 0: + return [], [] dialogs = {} # Use peer id as identifier to avoid dupes messages = {} # Used later for sorting TODO also return these? entities = {} while len(dialogs) < limit: - need = limit - len(dialogs) + real_limit = min(limit - len(dialogs), 100) r = self(GetDialogsRequest( offset_date=offset_date, offset_id=offset_id, offset_peer=offset_peer, - limit=need if need < float('inf') else 0 + limit=real_limit )) - if not r.dialogs: - break for d in r.dialogs: dialogs[utils.get_peer_id(d.peer, True)] = d @@ -307,8 +306,9 @@ class TelegramClient(TelegramBareClient): for c in r.chats: entities[c.id] = c - if not isinstance(r, DialogsSlice): - # Don't enter next iteration if we already got all + if len(r.dialogs) < real_limit or not isinstance(r, DialogsSlice): + # Less than we requested means we reached the end, or + # we didn't get a DialogsSlice which means we got all. break offset_date = r.messages[-1].date From 1e35c1cfed96d27486c8c6f65a570f86d3b5bb0e Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 4 Nov 2017 13:40:43 +0100 Subject: [PATCH 19/44] Update to layer 72 --- telethon_generator/scheme.tl | 41 +++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/telethon_generator/scheme.tl b/telethon_generator/scheme.tl index 5e949239..ae6544a1 100644 --- a/telethon_generator/scheme.tl +++ b/telethon_generator/scheme.tl @@ -161,12 +161,13 @@ inputMediaGeoPoint#f9c44144 geo_point:InputGeoPoint = InputMedia; inputMediaContact#a6e45987 phone_number:string first_name:string last_name:string = InputMedia; inputMediaUploadedDocument#e39621fd flags:# file:InputFile thumb:flags.2?InputFile mime_type:string attributes:Vector caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; inputMediaDocument#5acb668e flags:# id:InputDocument caption:string ttl_seconds:flags.0?int = InputMedia; -inputMediaVenue#2827a81a geo_point:InputGeoPoint title:string address:string provider:string venue_id:string = InputMedia; +inputMediaVenue#c13d1c11 geo_point:InputGeoPoint title:string address:string provider:string venue_id:string venue_type:string = InputMedia; inputMediaGifExternal#4843b0fd url:string q:string = InputMedia; inputMediaPhotoExternal#922aec1 flags:# url:string caption:string ttl_seconds:flags.0?int = InputMedia; inputMediaDocumentExternal#b6f74335 flags:# url:string caption:string ttl_seconds:flags.0?int = InputMedia; inputMediaGame#d33f43f3 id:InputGame = InputMedia; inputMediaInvoice#92153685 flags:# title:string description:string photo:flags.0?InputWebDocument invoice:Invoice payload:bytes provider:string start_param:string = InputMedia; +inputMediaGeoLive#7b1a118f geo_point:InputGeoPoint period:int = InputMedia; inputChatPhotoEmpty#1ca48f57 = InputChatPhoto; inputChatUploadedPhoto#927c55b4 file:InputFile = InputChatPhoto; @@ -222,7 +223,7 @@ channel#cb44b1c flags:# creator:flags.0?true left:flags.2?true editor:flags.3?tr channelForbidden#289da732 flags:# broadcast:flags.5?true megagroup:flags.8?true id:int access_hash:long title:string until_date:flags.16?int = Chat; chatFull#2e02a614 id:int participants:ChatParticipants chat_photo:Photo notify_settings:PeerNotifySettings exported_invite:ExportedChatInvite bot_info:Vector = ChatFull; -channelFull#17f45fcf flags:# can_view_participants:flags.3?true can_set_username:flags.6?true can_set_stickers:flags.7?true id:int about:string participants_count:flags.0?int admins_count:flags.1?int kicked_count:flags.2?int banned_count:flags.2?int read_inbox_max_id:int read_outbox_max_id:int unread_count:int chat_photo:Photo notify_settings:PeerNotifySettings exported_invite:ExportedChatInvite bot_info:Vector migrated_from_chat_id:flags.4?int migrated_from_max_id:flags.4?int pinned_msg_id:flags.5?int stickerset:flags.8?StickerSet = ChatFull; +channelFull#76af5481 flags:# can_view_participants:flags.3?true can_set_username:flags.6?true can_set_stickers:flags.7?true hidden_prehistory:flags.10?true id:int about:string participants_count:flags.0?int admins_count:flags.1?int kicked_count:flags.2?int banned_count:flags.2?int read_inbox_max_id:int read_outbox_max_id:int unread_count:int chat_photo:Photo notify_settings:PeerNotifySettings exported_invite:ExportedChatInvite bot_info:Vector migrated_from_chat_id:flags.4?int migrated_from_max_id:flags.4?int pinned_msg_id:flags.5?int stickerset:flags.8?StickerSet available_min_id:flags.9?int = ChatFull; chatParticipant#c8d7493e user_id:int inviter_id:int date:int = ChatParticipant; chatParticipantCreator#da13538a user_id:int = ChatParticipant; @@ -245,9 +246,10 @@ messageMediaContact#5e7d2f39 phone_number:string first_name:string last_name:str messageMediaUnsupported#9f84f49e = MessageMedia; messageMediaDocument#7c4414d3 flags:# document:flags.0?Document caption:flags.1?string ttl_seconds:flags.2?int = MessageMedia; messageMediaWebPage#a32dd600 webpage:WebPage = MessageMedia; -messageMediaVenue#7912b71f geo:GeoPoint title:string address:string provider:string venue_id:string = MessageMedia; +messageMediaVenue#2ec0533f geo:GeoPoint title:string address:string provider:string venue_id:string venue_type:string = MessageMedia; messageMediaGame#fdb19008 game:Game = MessageMedia; messageMediaInvoice#84551347 flags:# shipping_address_requested:flags.1?true test:flags.3?true title:string description:string photo:flags.0?WebDocument receipt_msg_id:flags.2?int currency:string total_amount:long start_param:string = MessageMedia; +messageMediaGeoLive#7c3c2609 geo:GeoPoint period:int = MessageMedia; messageActionEmpty#b6aef7b0 = MessageAction; messageActionChatCreate#a6638b9a title:string users:Vector = MessageAction; @@ -267,6 +269,7 @@ messageActionPaymentSentMe#8f31b327 flags:# currency:string total_amount:long pa messageActionPaymentSent#40699cd0 currency:string total_amount:long = MessageAction; messageActionPhoneCall#80e11a7f flags:# call_id:long reason:flags.0?PhoneCallDiscardReason duration:flags.1?int = MessageAction; messageActionScreenshotTaken#4792929b = MessageAction; +messageActionCustomAction#fae69f56 message:string = MessageAction; dialog#e4def5db flags:# pinned:flags.2?true peer:Peer top_message:int read_inbox_max_id:int read_outbox_max_id:int unread_count:int unread_mentions_count:int notify_settings:PeerNotifySettings pts:flags.0?int draft:flags.1?DraftMessage = Dialog; @@ -363,6 +366,8 @@ inputMessagesFilterPhoneCalls#80c99768 flags:# missed:flags.0?true = MessagesFil inputMessagesFilterRoundVoice#7a7c17a4 = MessagesFilter; inputMessagesFilterRoundVideo#b549da53 = MessagesFilter; inputMessagesFilterMyMentions#c1f8e69a = MessagesFilter; +inputMessagesFilterContacts#e062db83 = MessagesFilter; +inputMessagesFilterGeo#e7026d0d = MessagesFilter; updateNewMessage#1f2b0afd message:Message pts:int pts_count:int = Update; updateMessageID#4e90bfd6 id:int random_id:long = Update; @@ -429,6 +434,7 @@ updateLangPack#56022f4d difference:LangPackDifference = Update; updateFavedStickers#e511996d = Update; updateChannelReadMessagesContents#89893b45 channel_id:int messages:Vector = Update; updateContactsReset#7084a7be = Update; +updateChannelAvailableMessages#70db6837 channel_id:int available_min_id:int = Update; updates.state#a56c2a3e pts:int qts:int date:int seq:int unread_count:int = updates.State; @@ -455,7 +461,7 @@ upload.fileCdnRedirect#ea52fe5a dc_id:int file_token:bytes encryption_key:bytes dcOption#5d8c6cc flags:# ipv6:flags.0?true media_only:flags.1?true tcpo_only:flags.2?true cdn:flags.3?true static:flags.4?true id:int ip_address:string port:int = DcOption; -config#8df376a4 flags:# phonecalls_enabled:flags.1?true date:int expires:int test_mode:Bool this_dc:int dc_options:Vector chat_size_max:int megagroup_size_max:int forwarded_count_max:int online_update_period_ms:int offline_blur_timeout_ms:int offline_idle_timeout_ms:int online_cloud_timeout_ms:int notify_cloud_delay_ms:int notify_default_delay_ms:int chat_big_size:int push_chat_period_ms:int push_chat_limit:int saved_gifs_limit:int edit_time_limit:int rating_e_decay:int stickers_recent_limit:int stickers_faved_limit:int tmp_sessions:flags.0?int pinned_dialogs_count_max:int call_receive_timeout_ms:int call_ring_timeout_ms:int call_connect_timeout_ms:int call_packet_timeout_ms:int me_url_prefix:string suggested_lang_code:flags.2?string lang_pack_version:flags.2?int disabled_features:Vector = Config; +config#9c840964 flags:# phonecalls_enabled:flags.1?true date:int expires:int test_mode:Bool this_dc:int dc_options:Vector chat_size_max:int megagroup_size_max:int forwarded_count_max:int online_update_period_ms:int offline_blur_timeout_ms:int offline_idle_timeout_ms:int online_cloud_timeout_ms:int notify_cloud_delay_ms:int notify_default_delay_ms:int chat_big_size:int push_chat_period_ms:int push_chat_limit:int saved_gifs_limit:int edit_time_limit:int rating_e_decay:int stickers_recent_limit:int stickers_faved_limit:int channels_read_media_period:int tmp_sessions:flags.0?int pinned_dialogs_count_max:int call_receive_timeout_ms:int call_ring_timeout_ms:int call_connect_timeout_ms:int call_packet_timeout_ms:int me_url_prefix:string suggested_lang_code:flags.2?string lang_pack_version:flags.2?int disabled_features:Vector = Config; nearestDc#8e1a1775 country:string this_dc:int nearest_dc:int = NearestDc; @@ -665,6 +671,7 @@ channelParticipantsBanned#1427a5e1 q:string = ChannelParticipantsFilter; channelParticipantsSearch#656ac4b q:string = ChannelParticipantsFilter; channels.channelParticipants#f56ee2a8 count:int participants:Vector users:Vector = channels.ChannelParticipants; +channels.channelParticipantsNotModified#f0173fe9 = channels.ChannelParticipants; channels.channelParticipant#d0d9b163 participant:ChannelParticipant users:Vector = channels.ChannelParticipant; @@ -680,7 +687,7 @@ messages.savedGifs#2e0709a5 hash:int gifs:Vector = messages.SavedGifs; inputBotInlineMessageMediaAuto#292fed13 flags:# caption:string reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; inputBotInlineMessageText#3dcd7a87 flags:# no_webpage:flags.0?true message:string entities:flags.1?Vector reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; -inputBotInlineMessageMediaGeo#f4a59de1 flags:# geo_point:InputGeoPoint reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; +inputBotInlineMessageMediaGeo#c1b15d65 flags:# geo_point:InputGeoPoint period:int reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; inputBotInlineMessageMediaVenue#aaafadc8 flags:# geo_point:InputGeoPoint title:string address:string provider:string venue_id:string reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; inputBotInlineMessageMediaContact#2daf01a7 flags:# phone_number:string first_name:string last_name:string reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; inputBotInlineMessageGame#4b425864 flags:# reply_markup:flags.2?ReplyMarkup = InputBotInlineMessage; @@ -692,14 +699,14 @@ inputBotInlineResultGame#4fa417f2 id:string short_name:string send_message:Input botInlineMessageMediaAuto#a74b15b flags:# caption:string reply_markup:flags.2?ReplyMarkup = BotInlineMessage; botInlineMessageText#8c7f65e2 flags:# no_webpage:flags.0?true message:string entities:flags.1?Vector reply_markup:flags.2?ReplyMarkup = BotInlineMessage; -botInlineMessageMediaGeo#3a8fd8b8 flags:# geo:GeoPoint reply_markup:flags.2?ReplyMarkup = BotInlineMessage; +botInlineMessageMediaGeo#b722de65 flags:# geo:GeoPoint period:int reply_markup:flags.2?ReplyMarkup = BotInlineMessage; botInlineMessageMediaVenue#4366232e flags:# geo:GeoPoint title:string address:string provider:string venue_id:string reply_markup:flags.2?ReplyMarkup = BotInlineMessage; botInlineMessageMediaContact#35edb4d4 flags:# phone_number:string first_name:string last_name:string reply_markup:flags.2?ReplyMarkup = BotInlineMessage; botInlineResult#9bebaeb9 flags:# id:string type:string title:flags.1?string description:flags.2?string url:flags.3?string thumb_url:flags.4?string content_url:flags.5?string content_type:flags.5?string w:flags.6?int h:flags.6?int duration:flags.7?int send_message:BotInlineMessage = BotInlineResult; botInlineMediaResult#17db940b flags:# id:string type:string photo:flags.0?Photo document:flags.1?Document title:flags.2?string description:flags.3?string send_message:BotInlineMessage = BotInlineResult; -messages.botResults#ccd3563d flags:# gallery:flags.0?true query_id:long next_offset:flags.1?string switch_pm:flags.2?InlineBotSwitchPM results:Vector cache_time:int = messages.BotResults; +messages.botResults#947ca848 flags:# gallery:flags.0?true query_id:long next_offset:flags.1?string switch_pm:flags.2?InlineBotSwitchPM results:Vector cache_time:int users:Vector = messages.BotResults; exportedMessageLink#1f486803 link:string = ExportedMessageLink; @@ -903,6 +910,7 @@ channelAdminLogEventActionParticipantInvite#e31c34d8 participant:ChannelParticip channelAdminLogEventActionParticipantToggleBan#e6d83d7e prev_participant:ChannelParticipant new_participant:ChannelParticipant = ChannelAdminLogEventAction; channelAdminLogEventActionParticipantToggleAdmin#d5676710 prev_participant:ChannelParticipant new_participant:ChannelParticipant = ChannelAdminLogEventAction; channelAdminLogEventActionChangeStickerSet#b1c3caa7 prev_stickerset:InputStickerSet new_stickerset:InputStickerSet = ChannelAdminLogEventAction; +channelAdminLogEventActionTogglePreHistoryHidden#5f5c95f1 new_value:Bool = ChannelAdminLogEventAction; channelAdminLogEvent#3b5a3e40 id:long date:int user_id:int action:ChannelAdminLogEventAction = ChannelAdminLogEvent; @@ -917,6 +925,14 @@ cdnFileHash#77eec38f offset:int limit:int hash:bytes = CdnFileHash; messages.favedStickersNotModified#9e8fa6d3 = messages.FavedStickers; messages.favedStickers#f37f2f16 hash:int packs:Vector stickers:Vector = messages.FavedStickers; +help.recentMeUrls#e0310d7 urls:Vector chats:Vector users:Vector = help.RecentMeUrls; + +recentMeUrlUser#8dbc3336 url:string user_id:int = RecentMeUrl; +recentMeUrlChat#a01b22f9 url:string chat_id:int = RecentMeUrl; +recentMeUrlStickerSet#bc0a57dc url:string set:StickerSetCovered = RecentMeUrl; +recentMeUrlChatInvite#eb49081d url:string chat_invite:ChatInvite = RecentMeUrl; +recentMeUrlUnknown#46e1d13d url:string = RecentMeUrl; + ---functions--- invokeAfterMsg#cb9f372d {X:Type} msg_id:long query:!X = X; @@ -1048,7 +1064,7 @@ messages.getInlineBotResults#514e999d flags:# bot:InputUser peer:InputPeer geo_p messages.setInlineBotResults#eb5ea206 flags:# gallery:flags.0?true private:flags.1?true query_id:long results:Vector cache_time:int next_offset:flags.2?string switch_pm:flags.3?InlineBotSwitchPM = Bool; messages.sendInlineBotResult#b16e06fe flags:# silent:flags.5?true background:flags.6?true clear_draft:flags.7?true peer:InputPeer reply_to_msg_id:flags.0?int random_id:long query_id:long id:string = Updates; messages.getMessageEditData#fda68d36 peer:InputPeer id:int = messages.MessageEditData; -messages.editMessage#ce91e4ca flags:# no_webpage:flags.1?true peer:InputPeer id:int message:flags.11?string reply_markup:flags.2?ReplyMarkup entities:flags.3?Vector = Updates; +messages.editMessage#5d1b8dd flags:# no_webpage:flags.1?true stop_geo_live:flags.12?true peer:InputPeer id:int message:flags.11?string reply_markup:flags.2?ReplyMarkup entities:flags.3?Vector geo_point:flags.13?InputGeoPoint = Updates; messages.editInlineBotMessage#130c2c85 flags:# no_webpage:flags.1?true id:InputBotInlineMessageID message:flags.11?string reply_markup:flags.2?ReplyMarkup entities:flags.3?Vector = Bool; messages.getBotCallbackAnswer#810a9fec flags:# game:flags.1?true peer:InputPeer msg_id:int data:flags.0?bytes = messages.BotCallbackAnswer; messages.setBotCallbackAnswer#d58f130a flags:# alert:flags.1?true query_id:long message:flags.0?string url:flags.2?string cache_time:int = Bool; @@ -1080,6 +1096,8 @@ messages.sendScreenshotNotification#c97df020 peer:InputPeer reply_to_msg_id:int messages.getFavedStickers#21ce0b0e hash:int = messages.FavedStickers; messages.faveSticker#b9ffc55b id:InputDocument unfave:Bool = Bool; messages.getUnreadMentions#46578472 peer:InputPeer offset_id:int add_offset:int limit:int max_id:int min_id:int = messages.Messages; +messages.getRecentLocations#249431e2 peer:InputPeer limit:int = messages.Messages; +messages.readMentions#f0189d3 peer:InputPeer = messages.AffectedHistory; updates.getState#edd4882a = updates.State; updates.getDifference#25939651 flags:# pts:int pts_total_limit:flags.0?int date:int qts:int = updates.Difference; @@ -1108,13 +1126,14 @@ help.getAppChangelog#9010ef6f prev_app_version:string = Updates; help.getTermsOfService#350170f3 = help.TermsOfService; help.setBotUpdatesStatus#ec22cfcd pending_updates_count:int message:string = Bool; help.getCdnConfig#52029342 = CdnConfig; +help.getRecentMeUrls#3dc0f114 referer:string = help.RecentMeUrls; channels.readHistory#cc104937 channel:InputChannel max_id:int = Bool; channels.deleteMessages#84c1fd4e channel:InputChannel id:Vector = messages.AffectedMessages; channels.deleteUserHistory#d10dd71b channel:InputChannel user_id:InputUser = messages.AffectedHistory; channels.reportSpam#fe087810 channel:InputChannel user_id:InputUser id:Vector = Bool; channels.getMessages#93d7b347 channel:InputChannel id:Vector = messages.Messages; -channels.getParticipants#24d98f92 channel:InputChannel filter:ChannelParticipantsFilter offset:int limit:int = channels.ChannelParticipants; +channels.getParticipants#123e05e9 channel:InputChannel filter:ChannelParticipantsFilter offset:int limit:int hash:int = channels.ChannelParticipants; channels.getParticipant#546dd7a6 channel:InputChannel user_id:InputUser = channels.ChannelParticipant; channels.getChannels#a7f6bbb id:Vector = messages.Chats; channels.getFullChannel#8736a09 channel:InputChannel = messages.ChatFull; @@ -1139,6 +1158,8 @@ channels.editBanned#bfd915cd channel:InputChannel user_id:InputUser banned_right channels.getAdminLog#33ddf480 flags:# channel:InputChannel q:string events_filter:flags.0?ChannelAdminLogEventsFilter admins:flags.1?Vector max_id:long min_id:long limit:int = channels.AdminLogResults; channels.setStickers#ea8ca4f9 channel:InputChannel stickerset:InputStickerSet = Bool; channels.readMessageContents#eab5dc38 channel:InputChannel id:Vector = Bool; +channels.deleteHistory#af369d42 channel:InputChannel max_id:int = Bool; +channels.togglePreHistoryHidden#eabbb94c channel:InputChannel enabled:Bool = Updates; bots.sendCustomRequest#aa2769ed custom_method:string params:DataJSON = DataJSON; bots.answerWebhookJSONQuery#e6213f4d query_id:long data:DataJSON = Bool; @@ -1169,4 +1190,4 @@ langpack.getStrings#2e1ee318 lang_code:string keys:Vector = Vector; -// LAYER 71 +// LAYER 72 From c8a0953f8e6c22ce17101e2590b1941d8754158c Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 4 Nov 2017 13:40:56 +0100 Subject: [PATCH 20/44] Update to v0.15.4 --- telethon/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/version.py b/telethon/version.py index 2bde1b96..bafdcc72 100644 --- a/telethon/version.py +++ b/telethon/version.py @@ -1,3 +1,3 @@ # Versions should comply with PEP440. # This line is parsed in setup.py: -__version__ = '0.15.3' +__version__ = '0.15.4' From f381b267903cad2b7e208a5006a7cd10a02b6eb1 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 4 Nov 2017 20:46:02 +0100 Subject: [PATCH 21/44] Add optional force_sms parameter to .send_code_request() --- telethon/telegram_client.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index c638c2b1..59bf4a9a 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -23,7 +23,7 @@ from .tl.functions.account import ( ) from .tl.functions.auth import ( CheckPasswordRequest, LogOutRequest, SendCodeRequest, SignInRequest, - SignUpRequest, ImportBotAuthorizationRequest + SignUpRequest, ResendCodeRequest, ImportBotAuthorizationRequest ) from .tl.functions.contacts import ( GetContactsRequest, ResolveUsernameRequest @@ -129,16 +129,29 @@ class TelegramClient(TelegramBareClient): # region Authorization requests - def send_code_request(self, phone): + def send_code_request(self, phone, force_sms=False): """Sends a code request to the specified phone number. - :param str | int phone: The phone to which the code will be sent. - :return auth.SentCode: Information about the result of the request. + :param str | int phone: + The phone to which the code will be sent. + :param bool force_sms: + Whether to force sending as SMS. You should call it at least + once before without this set to True first. + :return auth.SentCode: + Information about the result of the request. """ phone = EntityDatabase.parse_phone(phone) or self._phone - result = self(SendCodeRequest(phone, self.api_id, self.api_hash)) + if force_sms: + if not self._phone_code_hash: + raise ValueError( + 'You must call this method without force_sms at least once.' + ) + result = self(ResendCodeRequest(phone, self._phone_code_hash)) + else: + result = self(SendCodeRequest(phone, self.api_id, self.api_hash)) + self._phone_code_hash = result.phone_code_hash + self._phone = phone - self._phone_code_hash = result.phone_code_hash return result def sign_in(self, phone=None, code=None, From 49eb2812516cd660465c26143a3053ea379f417f Mon Sep 17 00:00:00 2001 From: Viktor Oreshkin Date: Mon, 6 Nov 2017 03:17:22 +0400 Subject: [PATCH 22/44] Proper offset calculation for markdown (#407) Dan suca If Dan shared it with Traitor I'll not have to spend my time on this Not a, sorry for not letting you sleep k thx bye Will this stay in history? --- telethon/extensions/markdown.py | 75 +++------------------------------ 1 file changed, 6 insertions(+), 69 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 574fe025..d4f7ce22 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -11,6 +11,8 @@ from ..tl.types import ( MessageEntityPre, MessageEntityTextUrl ) +def tg_string_len(s): + return len(s.encode('utf-16le')) // 2 class Mode(Enum): """Different modes supported by Telegram's Markdown""" @@ -22,54 +24,6 @@ class Mode(Enum): URL = 5 -# TODO Special cases, these aren't count as emojies. Alternatives? -# These were determined by generating all emojies with EMOJI_RANGES, -# sending the message through an official application, and cherry-picking -# which ones weren't rendered as emojies (from the beginning one). I am -# not responsible for dropping those characters that did not render with -# my font. -NOT_EMOJIES = { - 9733, 9735, 9736, 9737, 9738, 9739, 9740, 9741, 9743, 9744, 9746, 9750, - 9751, 9754, 9755, 9756, 9758, 9759, 9761, 9764, 9765, 9767, 9768, 9769, - 9771, 9772, 9773, 9776, 9777, 9778, 9779, 9780, 9781, 9782, 9783, 9787, - 9788, 9789, 9790, 9791, 9792, 9793, 9794, 9795, 9796, 9797, 9798, 9799, - 9812, 9813, 9814, 9815, 9816, 9817, 9818, 9819, 9820, 9821, 9822, 9823, - 9825, 9826, 9828, 9831, 9833, 9834, 9835, 9836, 9837, 9838, 9839, 9840, - 9841, 9842, 9843, 9844, 9845, 9846, 9847, 9848, 9849, 9850, 9852, 9853, - 9854, 9856, 9857, 9858, 9859, 9860, 9861, 9862, 9863, 9864, 9865, 9866, - 9867, 9868, 9869, 9870, 9871, 9872, 9873, 9877, 9880, 9882, 9886, 9887, - 9890, 9891, 9892, 9893, 9894, 9895, 9896, 9897, 9900, 9901, 9902, 9903, - 9907, 9908, 9909, 9910, 9911, 9912, 9920, 9921, 9922, 9923, 9985, 9987, - 9988, 9998, 10000, 10001, 10085, 10086, 10087, 127027, 127028, 127029, - 127030, 127031, 127032, 127033, 127034, 127035, 127036, 127037, 127038, - 127039, 127040, 127041, 127042, 127043, 127044, 127045, 127046, 127047, - 127048, 127049, 127050, 127051, 127052, 127053, 127054, 127055, 127056, - 127057, 127058, 127059, 127060, 127061, 127062, 127063, 127064, 127065, - 127066, 127067, 127068, 127069, 127070, 127071, 127072, 127073, 127074, - 127075, 127076, 127077, 127078, 127079, 127080, 127081, 127082, 127083, - 127084, 127085, 127086, 127087, 127088, 127089, 127090, 127091, 127092, - 127093, 127094, 127095, 127096, 127097, 127098, 127099, 127100, 127101, - 127102, 127103, 127104, 127105, 127106, 127107, 127108, 127109, 127110, - 127111, 127112, 127113, 127114, 127115, 127116, 127117, 127118, 127119, - 127120, 127121, 127122, 127123 -} -# using telethon_generator/emoji_ranges.py -EMOJI_RANGES = ( - (8596, 8601), (8617, 8618), (8986, 8987), (9193, 9203), (9208, 9210), - (9642, 9643), (9723, 9726), (9728, 9733), (9735, 9746), (9748, 9751), - (9754, 9884), (9886, 9905), (9907, 9953), (9956, 9983), (9985, 9988), - (9992, 10002), (10035, 10036), (10067, 10069), (10083, 10087), - (10133, 10135), (10548, 10549), (11013, 11015), (11035, 11036), - (126976, 127166), (127169, 127183), (127185, 127231), (127245, 127247), - (127340, 127345), (127358, 127359), (127377, 127386), (127405, 127487), - (127489, 127503), (127538, 127546), (127548, 127551), (127561, 128419), - (128421, 128591), (128640, 128767), (128884, 128895), (128981, 129023), - (129036, 129039), (129096, 129103), (129114, 129119), (129160, 129167), - (129198, 129338), (129340, 129342), (129344, 129349), (129351, 129355), - (129357, 129471), (129473, 131069) -) - - DEFAULT_DELIMITERS = { '**': Mode.BOLD, '__': Mode.ITALIC, @@ -79,24 +33,6 @@ DEFAULT_DELIMITERS = { DEFAULT_URL_RE = re.compile(r'\[(.+?)\]\((.+?)\)') - -def is_emoji(char): - """Returns True if 'char' looks like an emoji""" - char = ord(char) - for start, end in EMOJI_RANGES: - if start <= char <= end: - return char not in NOT_EMOJIES - return False - - -def emojiness(char): - """ - Returns 2 if the character is an emoji, or 1 otherwise. - This seems to be the length Telegram uses for offsets and lengths. - """ - return 2 if is_emoji(char) else 1 - - def parse(message, delimiters=None, url_re=None): """ Parses the given message and returns the stripped message and a list @@ -131,10 +67,10 @@ def parse(message, delimiters=None, url_re=None): url_match.group(1), message[url_match.end():] )) - emoji_len = sum(emojiness(c) for c in url_match.group(1)) + result.append(( offset, - offset + emoji_len, + offset + tg_string_len(url_match.group(1)), (Mode.URL, url_match.group(2)) )) i += len(url_match.group(1)) @@ -154,11 +90,12 @@ def parse(message, delimiters=None, url_re=None): break if i < len(message): - offset += emojiness(message[i]) + offset += tg_string_len(message[i]) i += 1 if result and not isinstance(result[-1], tuple): result.pop() + return message, result From e8248b4b8be0097f106eb51660e693b734741ba4 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 6 Nov 2017 09:36:54 +0100 Subject: [PATCH 23/44] Remove now unused Emoji ranges generator --- telethon_generator/emoji_ranges.py | 101 ----------------------------- 1 file changed, 101 deletions(-) delete mode 100644 telethon_generator/emoji_ranges.py diff --git a/telethon_generator/emoji_ranges.py b/telethon_generator/emoji_ranges.py deleted file mode 100644 index 90597cf6..00000000 --- a/telethon_generator/emoji_ranges.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -Simple module to allow fetching unicode.org emoji lists and printing a -Python-like tuple out of them. - -May not be accurate 100%, and is definitely not as efficient as it could be, -but it should only be ran whenever the Unicode consortium decides to add -new emojies to the list. -""" -import os -import sys -import re -import urllib.error -import urllib.request - - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -def get(url, enc='utf-8'): - try: - with urllib.request.urlopen(url) as f: - return f.read().decode(enc, errors='replace') - except urllib.error.HTTPError as e: - eprint('Caught', e, 'for', url, '; returning empty') - return '' - - -PREFIX_URL = 'http://unicode.org/Public/emoji/' -SUFFIX_URL = '/emoji-data.txt', '/emoji-sequences.txt' -VERSION_RE = re.compile(r'>(\d+.\d+)/<') -OUTPUT_TXT = 'emojies.txt' -CODEPOINT_RE = re.compile(r'([\da-fA-F]{3,}(?:[\s.]+[\da-fA-F]{3,}))') -EMOJI_START = 0x20e3 # emoji data has many more ranges, falling outside this -EMOJI_END = 200000 # from some tests those outside the range aren't emojies - - -versions = VERSION_RE.findall(get(PREFIX_URL)) -lines = [] -if not os.path.isfile(OUTPUT_TXT): - with open(OUTPUT_TXT, 'w') as f: - for version in versions: - for s in SUFFIX_URL: - url = PREFIX_URL + version + s - for line in get(url).split('\n'): - line = line.strip() - if not line or line.startswith('#'): - continue - m = CODEPOINT_RE.search(line) - if m and m.start() == 0: - f.write(m.group(1) + '\n') - - -points = set() -with open(OUTPUT_TXT) as f: - for line in f: - line = line.strip() - if ' ' in line: - for p in line.split(): - i = int(p, 16) - if i > 255: - points.add(i) - elif '.' in line: - s, e = line.split('..') - for i in range(int(s, 16), int(e, 16) + 1): - if i > 255: - points.add(i) - else: - i = int(line, 16) - if i > 255: - points.add(int(line, 16)) - - -ranges = [] -points = tuple(sorted(points)) -start = points[0] -last = start -for point in points: - if point - last > 1: - if start == last or not (EMOJI_START < start < EMOJI_END): - eprint( - 'Dropping', last - start + 1, - 'character(s) from', hex(start), ':', chr(start) - ) - else: - ranges.append((start, last)) - start = point - - last = point - - -if start == last or not (EMOJI_START < start < EMOJI_END): - eprint( - 'Dropping', last - start + 1, - 'character(s) from', hex(start), ':', chr(start) - ) -else: - ranges.append((start, last)) - - -print('EMOJI_RANGES = ({})'.format(', '.join(repr(r) for r in ranges))) From 4f8042921571779ba6f163a33ffd62272a2230fa Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 6 Nov 2017 10:29:32 +0100 Subject: [PATCH 24/44] Work on byte level when parsing markdown Reasoning: instead encoding every character one by one as we encounter them to use half their length as the correct offset, we can simply encode the whole string at once as utf-16le and work with that directly. --- telethon/extensions/markdown.py | 37 +++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index d4f7ce22..f78fcb2f 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -11,8 +11,6 @@ from ..tl.types import ( MessageEntityPre, MessageEntityTextUrl ) -def tg_string_len(s): - return len(s.encode('utf-16le')) // 2 class Mode(Enum): """Different modes supported by Telegram's Markdown""" @@ -31,7 +29,10 @@ DEFAULT_DELIMITERS = { '```': Mode.PRE } -DEFAULT_URL_RE = re.compile(r'\[(.+?)\]\((.+?)\)') +# Regex used to match utf-16le encoded r'\[(.+?)\]\((.+?)\)', +# reason why there's '\0' after every match-literal character. +DEFAULT_URL_RE = re.compile(b'\\[\0(.+)\\]\0\\(\0(.+?)\\)\0') + def parse(message, delimiters=None, url_re=None): """ @@ -40,40 +41,45 @@ def parse(message, delimiters=None, url_re=None): dictionary (or default if None). The url_re(gex) must contain two matching groups: the text to be - clickable and the URL itself. + clickable and the URL itself, and be utf-16le encoded. """ + # Work on byte level with the utf-16le encoding to get the offsets right. + # The offset will just be half the index we're at. if url_re is None: url_re = DEFAULT_URL_RE elif url_re: if isinstance(url_re, str): - url_re = re.compile(url_re) + url_re = re.compile(url_re.encode('utf-16le')) if not delimiters: if delimiters is not None: return message, [] delimiters = DEFAULT_DELIMITERS + delimiters = {k.encode('utf-16le'): v for k, v in delimiters.items()} + + i = 0 result = [] current = Mode.NONE - offset = 0 - i = 0 + message = message.encode('utf-16le') while i < len(message): url_match = None if url_re and current == Mode.NONE: url_match = url_re.match(message, pos=i) if url_match: - message = ''.join(( + message = b''.join(( message[:url_match.start()], url_match.group(1), message[url_match.end():] )) result.append(( - offset, - offset + tg_string_len(url_match.group(1)), - (Mode.URL, url_match.group(2)) + i // 2, + (i + len(url_match.group(1))) // 2, + (Mode.URL, url_match.group(2).decode('utf-16le')) )) i += len(url_match.group(1)) + if not url_match: for d, m in delimiters.items(): if message[i:i + len(d)] == d and current in (Mode.NONE, m): @@ -82,21 +88,20 @@ def parse(message, delimiters=None, url_re=None): message = message[:i] + message[i + len(d):] if current == Mode.NONE: - result.append(offset) + result.append(i // 2) current = m else: - result[-1] = (result[-1], offset, current) + result[-1] = (result[-1], i // 2, current) current = Mode.NONE break if i < len(message): - offset += tg_string_len(message[i]) - i += 1 + i += 2 if result and not isinstance(result[-1], tuple): result.pop() - return message, result + return message.decode('utf-16le'), result def parse_tg(message, delimiters=None): From 07ece83aba9ff57acf7b40e5611ab4f54686805f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 6 Nov 2017 10:37:22 +0100 Subject: [PATCH 25/44] Fix overlapping markdown entities being skipped --- telethon/extensions/markdown.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index f78fcb2f..2451505e 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -78,7 +78,10 @@ def parse(message, delimiters=None, url_re=None): (i + len(url_match.group(1))) // 2, (Mode.URL, url_match.group(2).decode('utf-16le')) )) - i += len(url_match.group(1)) + # We matched the delimiter which is now gone, and we'll add + # +2 before next iteration which will make us skip a character. + # Go back by one utf-16 encoded character (-2) to avoid it. + i += len(url_match.group(1)) - 2 if not url_match: for d, m in delimiters.items(): @@ -90,9 +93,12 @@ def parse(message, delimiters=None, url_re=None): if current == Mode.NONE: result.append(i // 2) current = m + # No need to i -= 2 here because it's been already + # checked that next character won't be a delimiter. else: result[-1] = (result[-1], i // 2, current) current = Mode.NONE + i -= 2 # Delimiter matched and gone, go back 1 char break if i < len(message): From 3a2c3a9497a6606a2fc634f49eab12e6286ae450 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 6 Nov 2017 11:22:58 +0100 Subject: [PATCH 26/44] Fix URL regex for markdown was greedy (fix-up) --- telethon/extensions/markdown.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 2451505e..9641caa0 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -31,7 +31,7 @@ DEFAULT_DELIMITERS = { # Regex used to match utf-16le encoded r'\[(.+?)\]\((.+?)\)', # reason why there's '\0' after every match-literal character. -DEFAULT_URL_RE = re.compile(b'\\[\0(.+)\\]\0\\(\0(.+?)\\)\0') +DEFAULT_URL_RE = re.compile(b'\\[\0(.+?)\\]\0\\(\0(.+?)\\)\0') def parse(message, delimiters=None, url_re=None): From 83af705cc8fba30bf3f05d8a4c3ba243edc475ca Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 6 Nov 2017 11:32:40 +0100 Subject: [PATCH 27/44] Add more comments to the markdown parser --- telethon/extensions/markdown.py | 36 ++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 9641caa0..ef9c118c 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -43,13 +43,11 @@ def parse(message, delimiters=None, url_re=None): The url_re(gex) must contain two matching groups: the text to be clickable and the URL itself, and be utf-16le encoded. """ - # Work on byte level with the utf-16le encoding to get the offsets right. - # The offset will just be half the index we're at. if url_re is None: url_re = DEFAULT_URL_RE elif url_re: - if isinstance(url_re, str): - url_re = re.compile(url_re.encode('utf-16le')) + if isinstance(url_re, bytes): + url_re = re.compile(url_re) if not delimiters: if delimiters is not None: @@ -58,15 +56,22 @@ def parse(message, delimiters=None, url_re=None): delimiters = {k.encode('utf-16le'): v for k, v in delimiters.items()} + # Cannot use a for loop because we need to skip some indices i = 0 result = [] current = Mode.NONE + + # Work on byte level with the utf-16le encoding to get the offsets right. + # The offset will just be half the index we're at. message = message.encode('utf-16le') while i < len(message): url_match = None if url_re and current == Mode.NONE: + # If we're not inside a previous match since Telegram doesn't allow + # nested message entities, try matching the URL from the i'th pos. url_match = url_re.match(message, pos=i) if url_match: + # Replace the whole match with only the inline URL text. message = b''.join(( message[:url_match.start()], url_match.group(1), @@ -85,10 +90,20 @@ def parse(message, delimiters=None, url_re=None): if not url_match: for d, m in delimiters.items(): - if message[i:i + len(d)] == d and current in (Mode.NONE, m): - if message[i + len(d):i + 2 * len(d)] == d: - continue # ignore two consecutive delimiters + # Slice the string at the current i'th position to see if + # it matches the current delimiter d. + if message[i:i + len(d)] == d: + if current != Mode.NONE and current != m: + # We were inside another delimiter/mode, ignore this. + continue + if message[i + len(d):i + 2 * len(d)] == d: + # The same delimiter can't be right afterwards, if + # this were the case we would match empty strings + # like `` which we don't want to. + continue + + # Get rid of the delimiter by slicing it away message = message[:i] + message[i + len(d):] if current == Mode.NONE: result.append(i // 2) @@ -101,10 +116,13 @@ def parse(message, delimiters=None, url_re=None): i -= 2 # Delimiter matched and gone, go back 1 char break - if i < len(message): - i += 2 + # Next iteration, utf-16 encoded characters need 2 bytes. + i += 2 if result and not isinstance(result[-1], tuple): + # We may have found some a delimiter but not its ending pair. If + # that's the case we want to get rid of it before returning. + # TODO Should probably insert such delimiter back in the string. result.pop() return message.decode('utf-16le'), result From f65322af1811c763031246bf0e74f5b06ec78fe1 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 7 Nov 2017 10:15:55 +0100 Subject: [PATCH 28/44] Fix entity database not using the phone on {phone: id} Closes #412 --- telethon/tl/entity_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/tl/entity_database.py b/telethon/tl/entity_database.py index 0c92c75f..2273627b 100644 --- a/telethon/tl/entity_database.py +++ b/telethon/tl/entity_database.py @@ -132,7 +132,7 @@ class EntityDatabase: phone = getattr(entity, 'phone', None) if phone: - self._username_id[phone] = marked_id + self._phone_id[phone] = marked_id def _parse_key(self, key): """Parses the given string, integer or TLObject key into a From cad1e883a6baa989661a5643cf88a2cb86e3a3bf Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 10 Nov 2017 09:32:40 +0100 Subject: [PATCH 29/44] Don't save full entities unless they have access_hash --- telethon/tl/entity_database.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/telethon/tl/entity_database.py b/telethon/tl/entity_database.py index 2273627b..2b7e0501 100644 --- a/telethon/tl/entity_database.py +++ b/telethon/tl/entity_database.py @@ -76,17 +76,20 @@ class EntityDatabase: p = utils.get_input_peer(e, allow_self=False) marked_id = utils.get_peer_id(p, add_mark=True) + has_hash = False if isinstance(p, InputPeerChat): # Chats don't have a hash new_input[marked_id] = 0 + has_hash = True elif p.access_hash: # Some users and channels seem to be returned without # an 'access_hash', meaning Telegram doesn't want you # to access them. This is the reason behind ensuring # that the 'access_hash' is non-zero. See issue #354. new_input[marked_id] = p.access_hash + has_hash = True - if self.enabled_full: + if self.enabled_full and has_hash: if isinstance(e, (User, Chat, Channel)): new.append(e) except ValueError: From 7d75eebdabbc380d380ebee796752cbe6fdf6f0f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 10 Nov 2017 11:01:02 +0100 Subject: [PATCH 30/44] Make markdown parser use only Telegram's MessageEntity's --- telethon/extensions/markdown.py | 80 ++++++++++----------------------- telethon/telegram_client.py | 2 +- 2 files changed, 24 insertions(+), 58 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index ef9c118c..c650fdfc 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -4,29 +4,17 @@ for use within the library, which attempts to handle emojies correctly, since they seem to count as two characters and it's a bit strange. """ import re -from enum import Enum - from ..tl.types import ( MessageEntityBold, MessageEntityItalic, MessageEntityCode, MessageEntityPre, MessageEntityTextUrl ) -class Mode(Enum): - """Different modes supported by Telegram's Markdown""" - NONE = 0 - BOLD = 1 - ITALIC = 2 - CODE = 3 - PRE = 4 - URL = 5 - - DEFAULT_DELIMITERS = { - '**': Mode.BOLD, - '__': Mode.ITALIC, - '`': Mode.CODE, - '```': Mode.PRE + '**': MessageEntityBold, + '__': MessageEntityItalic, + '`': MessageEntityCode, + '```': MessageEntityPre } # Regex used to match utf-16le encoded r'\[(.+?)\]\((.+?)\)', @@ -37,8 +25,8 @@ DEFAULT_URL_RE = re.compile(b'\\[\0(.+?)\\]\0\\(\0(.+?)\\)\0') def parse(message, delimiters=None, url_re=None): """ Parses the given message and returns the stripped message and a list - of tuples containing (start, end, mode) using the specified delimiters - dictionary (or default if None). + of MessageEntity* using the specified delimiters dictionary (or default + if None). The dictionary should be a mapping {delimiter: entity class}. The url_re(gex) must contain two matching groups: the text to be clickable and the URL itself, and be utf-16le encoded. @@ -59,14 +47,14 @@ def parse(message, delimiters=None, url_re=None): # Cannot use a for loop because we need to skip some indices i = 0 result = [] - current = Mode.NONE + current = None # Work on byte level with the utf-16le encoding to get the offsets right. # The offset will just be half the index we're at. message = message.encode('utf-16le') while i < len(message): url_match = None - if url_re and current == Mode.NONE: + if url_re and current is None: # If we're not inside a previous match since Telegram doesn't allow # nested message entities, try matching the URL from the i'th pos. url_match = url_re.match(message, pos=i) @@ -78,10 +66,9 @@ def parse(message, delimiters=None, url_re=None): message[url_match.end():] )) - result.append(( - i // 2, - (i + len(url_match.group(1))) // 2, - (Mode.URL, url_match.group(2).decode('utf-16le')) + result.append(MessageEntityTextUrl( + offset=i // 2, length=len(url_match.group(1)) // 2, + url=url_match.group(2).decode('utf-16le') )) # We matched the delimiter which is now gone, and we'll add # +2 before next iteration which will make us skip a character. @@ -93,7 +80,7 @@ def parse(message, delimiters=None, url_re=None): # Slice the string at the current i'th position to see if # it matches the current delimiter d. if message[i:i + len(d)] == d: - if current != Mode.NONE and current != m: + if current is not None and not isinstance(current, m): # We were inside another delimiter/mode, ignore this. continue @@ -105,46 +92,25 @@ def parse(message, delimiters=None, url_re=None): # Get rid of the delimiter by slicing it away message = message[:i] + message[i + len(d):] - if current == Mode.NONE: - result.append(i // 2) - current = m + if current is None: + if m == MessageEntityPre: + # Special case, also has 'lang' + current = MessageEntityPre(i // 2, None, '') + else: + current = m(i // 2, None) # No need to i -= 2 here because it's been already # checked that next character won't be a delimiter. else: - result[-1] = (result[-1], i // 2, current) - current = Mode.NONE + current.length = (i // 2) - current.offset + result.append(current) + current = None i -= 2 # Delimiter matched and gone, go back 1 char break # Next iteration, utf-16 encoded characters need 2 bytes. i += 2 - if result and not isinstance(result[-1], tuple): - # We may have found some a delimiter but not its ending pair. If - # that's the case we want to get rid of it before returning. - # TODO Should probably insert such delimiter back in the string. - result.pop() + # We may have found some a delimiter but not its ending pair. + # TODO Should probably insert such delimiter back in the string. return message.decode('utf-16le'), result - - -def parse_tg(message, delimiters=None): - """Similar to parse(), but returns a list of MessageEntity's""" - message, tuples = parse(message, delimiters=delimiters) - result = [] - for start, end, mode in tuples: - extra = None - if isinstance(mode, tuple): - mode, extra = mode - - if mode == Mode.BOLD: - result.append(MessageEntityBold(start, end - start)) - elif mode == Mode.ITALIC: - result.append(MessageEntityItalic(start, end - start)) - elif mode == Mode.CODE: - result.append(MessageEntityCode(start, end - start)) - elif mode == Mode.PRE: - result.append(MessageEntityPre(start, end - start, '')) - elif mode == Mode.URL: - result.append(MessageEntityTextUrl(start, end - start, extra)) - return message, result diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index 59bf4a9a..a90214b0 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -388,7 +388,7 @@ class TelegramClient(TelegramBareClient): if parse_mode: parse_mode = parse_mode.lower() if parse_mode in {'md', 'markdown'}: - message, msg_entities = markdown.parse_tg(message) + message, msg_entities = markdown.parse(message) else: raise ValueError('Unknown parsing mode', parse_mode) else: From cb3f20db654ca5789ee5c697b36917464cf3c9a3 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 10 Nov 2017 11:41:49 +0100 Subject: [PATCH 31/44] Clean up markdown parsing since tuples aren't used anymore --- telethon/extensions/markdown.py | 68 +++++++++++++++++---------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index c650fdfc..566a1d45 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -48,12 +48,12 @@ def parse(message, delimiters=None, url_re=None): i = 0 result = [] current = None + end_delimiter = None # Work on byte level with the utf-16le encoding to get the offsets right. # The offset will just be half the index we're at. message = message.encode('utf-16le') while i < len(message): - url_match = None if url_re and current is None: # If we're not inside a previous match since Telegram doesn't allow # nested message entities, try matching the URL from the i'th pos. @@ -70,42 +70,46 @@ def parse(message, delimiters=None, url_re=None): offset=i // 2, length=len(url_match.group(1)) // 2, url=url_match.group(2).decode('utf-16le') )) - # We matched the delimiter which is now gone, and we'll add - # +2 before next iteration which will make us skip a character. - # Go back by one utf-16 encoded character (-2) to avoid it. - i += len(url_match.group(1)) - 2 + i += len(url_match.group(1)) + # Next loop iteration, don't check delimiters, since + # a new inline URL might be right after this one. + continue - if not url_match: + if end_delimiter is None: + # We're not expecting any delimiter, so check them all for d, m in delimiters.items(): # Slice the string at the current i'th position to see if - # it matches the current delimiter d. - if message[i:i + len(d)] == d: - if current is not None and not isinstance(current, m): - # We were inside another delimiter/mode, ignore this. - continue + # it matches the current delimiter d, otherwise skip it. + if message[i:i + len(d)] != d: + continue - if message[i + len(d):i + 2 * len(d)] == d: - # The same delimiter can't be right afterwards, if - # this were the case we would match empty strings - # like `` which we don't want to. - continue + if message[i + len(d):i + 2 * len(d)] == d: + # The same delimiter can't be right afterwards, if + # this were the case we would match empty strings + # like `` which we don't want to. + continue - # Get rid of the delimiter by slicing it away - message = message[:i] + message[i + len(d):] - if current is None: - if m == MessageEntityPre: - # Special case, also has 'lang' - current = MessageEntityPre(i // 2, None, '') - else: - current = m(i // 2, None) - # No need to i -= 2 here because it's been already - # checked that next character won't be a delimiter. - else: - current.length = (i // 2) - current.offset - result.append(current) - current = None - i -= 2 # Delimiter matched and gone, go back 1 char - break + # Get rid of the delimiter by slicing it away + message = message[:i] + message[i + len(d):] + if m == MessageEntityPre: + # Special case, also has 'lang' + current = m(i // 2, None, '') + else: + current = m(i // 2, None) + + end_delimiter = d # We expect the same delimiter. + break + + elif message[i:i + len(end_delimiter)] == end_delimiter: + message = message[:i] + message[i + len(end_delimiter):] + current.length = (i // 2) - current.offset + result.append(current) + current, end_delimiter = None, None + # Don't increment i here as we matched a delimiter, + # and there may be a new one right after. This is + # different than when encountering the first delimiter, + # as we already know there won't be the same right after. + continue # Next iteration, utf-16 encoded characters need 2 bytes. i += 2 From c4e07cff57d22a5f20af5420da6752236641aab9 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 10 Nov 2017 11:44:27 +0100 Subject: [PATCH 32/44] Fix unfinished markdown delimiters being stripped away --- telethon/extensions/markdown.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 566a1d45..3cdf95f7 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -115,6 +115,9 @@ def parse(message, delimiters=None, url_re=None): i += 2 # We may have found some a delimiter but not its ending pair. - # TODO Should probably insert such delimiter back in the string. + # If this is the case, we want to insert the delimiter character back. + if current is not None: + message = \ + message[:current.offset] + end_delimiter + message[current.offset:] return message.decode('utf-16le'), result From 81baced12b8d6ee0e69215f3b0357bbd35f84fc1 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 10 Nov 2017 13:27:51 +0100 Subject: [PATCH 33/44] Support t.me/ links when resolving usernames/joinchat links Closes #419 --- telethon/telegram_client.py | 23 +++++++++++++++++------ telethon/tl/entity_database.py | 21 ++++++++++++++++++++- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index a90214b0..b1eec2b2 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -30,8 +30,9 @@ from .tl.functions.contacts import ( ) from .tl.functions.messages import ( GetDialogsRequest, GetHistoryRequest, ReadHistoryRequest, SendMediaRequest, - SendMessageRequest, GetChatsRequest, - GetAllDraftsRequest) + SendMessageRequest, GetChatsRequest, GetAllDraftsRequest, + CheckChatInviteRequest +) from .tl.functions import channels from .tl.functions import messages @@ -49,13 +50,13 @@ from .tl.types import ( Message, MessageMediaContact, MessageMediaDocument, MessageMediaPhoto, InputUserSelf, UserProfilePhoto, ChatPhoto, UpdateMessageID, UpdateNewChannelMessage, UpdateNewMessage, UpdateShortSentMessage, - PeerUser, InputPeerUser, InputPeerChat, InputPeerChannel, MessageEmpty + PeerUser, InputPeerUser, InputPeerChat, InputPeerChannel, MessageEmpty, + ChatInvite, ChatInviteAlready ) from .tl.types.messages import DialogsSlice from .extensions import markdown - class TelegramClient(TelegramBareClient): """Full featured TelegramClient meant to extend the basic functionality - @@ -1053,8 +1054,18 @@ class TelegramClient(TelegramBareClient): entity = phone self(GetContactsRequest(0)) else: - entity = string.strip('@').lower() - self(ResolveUsernameRequest(entity)) + entity, is_join_chat = EntityDatabase.parse_username(string) + if is_join_chat: + invite = self(CheckChatInviteRequest(entity)) + if isinstance(invite, ChatInvite): + # If it's an invite to a chat, the user must join before + # for the link to be resolved and work, otherwise raise. + if invite.channel: + return invite.channel + elif isinstance(invite, ChatInviteAlready): + return invite.chat + else: + self(ResolveUsernameRequest(entity)) # MtProtoSender will call .process_entities on the requests made try: return self.session.entities[entity] diff --git a/telethon/tl/entity_database.py b/telethon/tl/entity_database.py index 2b7e0501..9002ebd8 100644 --- a/telethon/tl/entity_database.py +++ b/telethon/tl/entity_database.py @@ -9,6 +9,11 @@ from ..tl.types import ( from .. import utils # Keep this line the last to maybe fix #357 +USERNAME_RE = re.compile( + r'@|(?:https?://)?(?:telegram\.(?:me|dog)|t\.me)/(joinchat/)?' +) + + class EntityDatabase: def __init__(self, input_list=None, enabled=True, enabled_full=True): """Creates a new entity database with an initial load of "Input" @@ -153,7 +158,8 @@ class EntityDatabase: if phone: return self._phone_id[phone] else: - return self._username_id[key.lstrip('@').lower()] + username, _ = EntityDatabase.parse_username(key) + return self._username_id[username.lower()] except KeyError as e: raise ValueError() from e @@ -206,6 +212,19 @@ class EntityDatabase: if phone.isdigit(): return phone + @staticmethod + def parse_username(username): + """Parses the given username or channel access hash, given + a string, username or URL. Returns a tuple consisting of + both the stripped username and whether it is a joinchat/ hash. + """ + username = username.strip() + m = USERNAME_RE.match(username) + if m: + return username[m.end():], bool(m.group(1)) + else: + return username, False + def get_input_entity(self, peer): try: i = utils.get_peer_id(peer, add_mark=True) From 5a57a8a498b1461d8aa041cc9adb7ec71eb88a33 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 11 Nov 2017 19:35:57 +0100 Subject: [PATCH 34/44] Fix message history failing if sender fwd from channel Closes #424 --- telethon/telegram_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/telethon/telegram_client.py b/telethon/telegram_client.py index b1eec2b2..5a5d99ec 100644 --- a/telethon/telegram_client.py +++ b/telethon/telegram_client.py @@ -51,7 +51,7 @@ from .tl.types import ( InputUserSelf, UserProfilePhoto, ChatPhoto, UpdateMessageID, UpdateNewChannelMessage, UpdateNewMessage, UpdateShortSentMessage, PeerUser, InputPeerUser, InputPeerChat, InputPeerChannel, MessageEmpty, - ChatInvite, ChatInviteAlready + ChatInvite, ChatInviteAlready, PeerChannel ) from .tl.types.messages import DialogsSlice from .extensions import markdown @@ -558,7 +558,7 @@ class TelegramClient(TelegramBareClient): elif getattr(m, 'fwd_from', None): # .from_id is optional, so this is the sanest fallback. who = entities[utils.get_peer_id( - m.fwd_from.from_id or m.fwd_from.channel_id, + m.fwd_from.from_id or PeerChannel(m.fwd_from.channel_id), add_mark=True )] else: From 99512875a223d564c4de112f10f034f33cf78954 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 12 Nov 2017 16:25:56 +0100 Subject: [PATCH 35/44] Reconnect if invoking failed (#270) --- telethon/telegram_bare_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 7ebf5ec1..161e6b30 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -428,7 +428,10 @@ class TelegramBareClient: result = self._invoke( sender, call_receive, update_state, *requests ) - if result is not None: + if result is None: + sleep(1) + self._reconnect() + else: return result raise ValueError('Number of retries reached 0.') From 84d48ef7bf8fef4834f942372639f6826e56601d Mon Sep 17 00:00:00 2001 From: Andrey Egorov Date: Sun, 12 Nov 2017 18:51:32 +0300 Subject: [PATCH 36/44] Safer check to determine whether sockets are connected (#427) --- telethon/extensions/tcp_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index 5255513a..3f803e63 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -59,7 +59,7 @@ class TcpClient: raise def _get_connected(self): - return self._socket is not None + return self._socket is not None and self._socket.fileno() > 0 connected = property(fget=_get_connected) From 08abef78d6f5877882a98ab09e70ce92265ccf03 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 12 Nov 2017 18:03:42 +0100 Subject: [PATCH 37/44] Add missing InputUserSelf case to .get_input_peer() --- telethon/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/telethon/utils.py b/telethon/utils.py index a05a4990..5d5bb953 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -99,6 +99,9 @@ def get_input_peer(entity, allow_self=True): if isinstance(entity, InputUser): return InputPeerUser(entity.user_id, entity.access_hash) + if isinstance(entity, InputUserSelf): + return InputPeerSelf() + if isinstance(entity, UserFull): return get_input_peer(entity.user) From f3e2887452cae38234467a269a39c532c13a217f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sun, 12 Nov 2017 18:15:32 +0100 Subject: [PATCH 38/44] Add missing ChannelFull case to .get_peer_id() --- telethon/utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/telethon/utils.py b/telethon/utils.py index 5d5bb953..3259c8e2 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -19,7 +19,7 @@ from .tl.types import ( DocumentEmpty, InputDocumentEmpty, Message, GeoPoint, InputGeoPoint, GeoPointEmpty, InputGeoPointEmpty, Photo, InputPhoto, PhotoEmpty, InputPhotoEmpty, FileLocation, ChatPhotoEmpty, UserProfilePhotoEmpty, - FileLocationUnavailable, InputMediaUploadedDocument, + FileLocationUnavailable, InputMediaUploadedDocument, ChannelFull, InputMediaUploadedPhoto, DocumentAttributeFilename, photos ) @@ -325,8 +325,13 @@ def get_peer_id(peer, add_mark=False): return peer.user_id elif isinstance(peer, (PeerChat, InputPeerChat)): return -peer.chat_id if add_mark else peer.chat_id - elif isinstance(peer, (PeerChannel, InputPeerChannel)): - i = peer.channel_id + elif isinstance(peer, (PeerChannel, InputPeerChannel, ChannelFull)): + if isinstance(peer, ChannelFull): + # Special case: .get_input_peer can't return InputChannel from + # ChannelFull since it doesn't have an .access_hash attribute. + i = peer.id + else: + i = peer.channel_id if add_mark: # Concat -100 through math tricks, .to_supergroup() on Madeline # IDs will be strictly positive -> log works From 07cb001854f614def335b670862e75ef4e29f59e Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 13 Nov 2017 10:31:32 +0100 Subject: [PATCH 39/44] Attempt at cleaning up reconnection logic --- telethon/telegram_bare_client.py | 57 ++++++++++++++------------------ 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 161e6b30..48c58c1d 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -254,16 +254,13 @@ class TelegramBareClient: connects to the new data center. """ if new_dc is None: - # Assume we are disconnected due to some error, so connect again - with self._reconnect_lock: - # Another thread may have connected again, so check that first - if self.is_connected(): - return True + if self.is_connected(): + return True - try: - return self.connect() - except ConnectionResetError: - return False + try: + return self.connect() + except ConnectionResetError: + return False else: # Since we're reconnecting possibly due to a UserMigrateError, # we need to first know the Data Centers we can connect to. Do @@ -430,7 +427,16 @@ class TelegramBareClient: ) if result is None: sleep(1) - self._reconnect() + self._logger.debug('RPC failed. Attempting reconnection.') + # The ReadThread has priority when attempting reconnection, + # since this thread is constantly running while __call__ is + # only done sometimes. Here try connecting only once/retry. + if sender == self._sender: + if not self._reconnect_lock.locked(): + with self._reconnect_lock: + self._reconnect() + else: + sender.connect() else: return result @@ -494,21 +500,12 @@ class TelegramBareClient: pass # We will just retry except ConnectionResetError: - if not self._user_connected or self._reconnect_lock.locked(): - # Only attempt reconnecting if the user called connect and not - # reconnecting already. - raise - - self._logger.debug('Server disconnected us. Reconnecting and ' - 'resending request...') - - if sender != self._sender: - # TODO Try reconnecting forever too? - sender.connect() + if self._user_connected: + # Server disconnected us, __call__ will try reconnecting. + return None else: - while self._user_connected and not self._reconnect(): - sleep(0.1) # Retry forever until we can send the request - return None + # User never called .connect(), so raise this error. + raise if init_connection: # We initialized the connection successfully, even if @@ -828,8 +825,9 @@ class TelegramBareClient: pass except ConnectionResetError: self._logger.debug('Server disconnected us. Reconnecting...') - while self._user_connected and not self._reconnect(): - sleep(0.1) # Retry forever, this is instant messaging + with self._reconnect_lock: + while self._user_connected and not self._reconnect(): + sleep(0.1) # Retry forever, this is instant messaging # By using this approach, another thread will be # created and started upon connection to constantly read @@ -864,12 +862,7 @@ class TelegramBareClient: self.disconnect() break except ImportError: - "Not using PySocks, so it can't be a socket error" - - # If something strange happens we don't want to enter an - # infinite loop where all we do is raise an exception, so - # add a little sleep to avoid the CPU usage going mad. - sleep(0.1) + "Not using PySocks, so it can't be a proxy error" self._recv_thread = None From 4ac88a150570179ad65d13f4ff98f1e076b8b538 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 13 Nov 2017 10:58:10 +0100 Subject: [PATCH 40/44] Use ._logger.exception when .connect fails (#373) --- telethon/telegram_bare_client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/telethon/telegram_bare_client.py b/telethon/telegram_bare_client.py index 48c58c1d..67ba0c88 100644 --- a/telethon/telegram_bare_client.py +++ b/telethon/telegram_bare_client.py @@ -204,12 +204,10 @@ class TelegramBareClient: self.disconnect() return self.connect(_sync_updates=_sync_updates) - except (RPCError, ConnectionError) as error: + except (RPCError, ConnectionError): # Probably errors from the previous session, ignore them self.disconnect() - self._logger.debug( - 'Could not stabilise initial connection: {}'.format(error) - ) + self._logger.exception('Could not stabilise initial connection.') return False def is_connected(self): From bfc408b00add6e0a807a514b2681c2b8120fcf95 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Mon, 13 Nov 2017 10:59:43 +0100 Subject: [PATCH 41/44] Use NullHandler as default for the library --- telethon/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/telethon/__init__.py b/telethon/__init__.py index 1210fa90..2f984bf1 100644 --- a/telethon/__init__.py +++ b/telethon/__init__.py @@ -1,3 +1,4 @@ +import logging from .telegram_bare_client import TelegramBareClient from .telegram_client import TelegramClient from .network import ConnectionMode @@ -5,3 +6,4 @@ from . import tl, version __version__ = version.__version__ +logging.getLogger(__name__).addHandler(logging.NullHandler()) From d59b17c6fc9a12a743d4694062dde02cae1333ec Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 14 Nov 2017 09:48:40 +0100 Subject: [PATCH 42/44] Clear up confusing error and trailing brace (closes #429) --- telethon_examples/interactive_telegram_client.py | 2 +- telethon_generator/error_descriptions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/telethon_examples/interactive_telegram_client.py b/telethon_examples/interactive_telegram_client.py index ee179a42..52c2c356 100644 --- a/telethon_examples/interactive_telegram_client.py +++ b/telethon_examples/interactive_telegram_client.py @@ -354,7 +354,7 @@ class InteractiveTelegramClient(TelegramClient): update.message, get_display_name(who) )) else: - sprint('<< {} sent "{}"]'.format( + sprint('<< {} sent "{}"'.format( get_display_name(who), update.message )) diff --git a/telethon_generator/error_descriptions b/telethon_generator/error_descriptions index 500504d7..65894ba1 100644 --- a/telethon_generator/error_descriptions +++ b/telethon_generator/error_descriptions @@ -45,7 +45,7 @@ PHONE_NUMBER_OCCUPIED=The phone number is already in use PHONE_NUMBER_UNOCCUPIED=The phone number is not yet being used PHOTO_INVALID_DIMENSIONS=The photo dimensions are invalid TYPE_CONSTRUCTOR_INVALID=The type constructor is invalid -USERNAME_INVALID=Unacceptable username. Must match r"[a-zA-Z][\w\d]{3,30}[a-zA-Z\d]" +USERNAME_INVALID=Nobody is using this username, or the username is unacceptable. If the latter, it must match r"[a-zA-Z][\w\d]{3,30}[a-zA-Z\d]" USERNAME_NOT_MODIFIED=The username is not different from the current username USERNAME_NOT_OCCUPIED=The username is not in use by anyone else yet USERNAME_OCCUPIED=The username is already taken From 48e96ca15fdd2a2d38d0aadc39cb6e9f6e711016 Mon Sep 17 00:00:00 2001 From: Lonami Date: Tue, 14 Nov 2017 12:01:33 +0100 Subject: [PATCH 43/44] Fix ._get_connected failing when .fileno() == 0 ( #427) --- telethon/extensions/tcp_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/telethon/extensions/tcp_client.py b/telethon/extensions/tcp_client.py index 3f803e63..164429f3 100644 --- a/telethon/extensions/tcp_client.py +++ b/telethon/extensions/tcp_client.py @@ -59,7 +59,7 @@ class TcpClient: raise def _get_connected(self): - return self._socket is not None and self._socket.fileno() > 0 + return self._socket is not None and self._socket.fileno() >= 0 connected = property(fget=_get_connected) From a1c669333e22b501beced8d1be56406e374d0814 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Wed, 15 Nov 2017 12:22:18 +0100 Subject: [PATCH 44/44] Update scheme to layer 73 --- telethon_generator/scheme.tl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/telethon_generator/scheme.tl b/telethon_generator/scheme.tl index ae6544a1..2ecb31b4 100644 --- a/telethon_generator/scheme.tl +++ b/telethon_generator/scheme.tl @@ -159,7 +159,7 @@ inputMediaUploadedPhoto#2f37e231 flags:# file:InputFile caption:string stickers: inputMediaPhoto#81fa373a flags:# id:InputPhoto caption:string ttl_seconds:flags.0?int = InputMedia; inputMediaGeoPoint#f9c44144 geo_point:InputGeoPoint = InputMedia; inputMediaContact#a6e45987 phone_number:string first_name:string last_name:string = InputMedia; -inputMediaUploadedDocument#e39621fd flags:# file:InputFile thumb:flags.2?InputFile mime_type:string attributes:Vector caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; +inputMediaUploadedDocument#e39621fd flags:# nosound_video:flags.3?true file:InputFile thumb:flags.2?InputFile mime_type:string attributes:Vector caption:string stickers:flags.0?Vector ttl_seconds:flags.1?int = InputMedia; inputMediaDocument#5acb668e flags:# id:InputDocument caption:string ttl_seconds:flags.0?int = InputMedia; inputMediaVenue#c13d1c11 geo_point:InputGeoPoint title:string address:string provider:string venue_id:string venue_type:string = InputMedia; inputMediaGifExternal#4843b0fd url:string q:string = InputMedia; @@ -169,6 +169,8 @@ inputMediaGame#d33f43f3 id:InputGame = InputMedia; inputMediaInvoice#92153685 flags:# title:string description:string photo:flags.0?InputWebDocument invoice:Invoice payload:bytes provider:string start_param:string = InputMedia; inputMediaGeoLive#7b1a118f geo_point:InputGeoPoint period:int = InputMedia; +inputSingleMedia#5eaa7809 media:InputMedia random_id:long = InputSingleMedia; + inputChatPhotoEmpty#1ca48f57 = InputChatPhoto; inputChatUploadedPhoto#927c55b4 file:InputFile = InputChatPhoto; inputChatPhoto#8953ad37 id:InputPhoto = InputChatPhoto; @@ -219,7 +221,7 @@ userStatusLastMonth#77ebc742 = UserStatus; chatEmpty#9ba2d800 id:int = Chat; chat#d91cdd54 flags:# creator:flags.0?true kicked:flags.1?true left:flags.2?true admins_enabled:flags.3?true admin:flags.4?true deactivated:flags.5?true id:int title:string photo:ChatPhoto participants_count:int date:int version:int migrated_to:flags.6?InputChannel = Chat; chatForbidden#7328bdb id:int title:string = Chat; -channel#cb44b1c flags:# creator:flags.0?true left:flags.2?true editor:flags.3?true broadcast:flags.5?true verified:flags.7?true megagroup:flags.8?true restricted:flags.9?true democracy:flags.10?true signatures:flags.11?true min:flags.12?true id:int access_hash:flags.13?long title:string username:flags.6?string photo:ChatPhoto date:int version:int restriction_reason:flags.9?string admin_rights:flags.14?ChannelAdminRights banned_rights:flags.15?ChannelBannedRights = Chat; +channel#450b7115 flags:# creator:flags.0?true left:flags.2?true editor:flags.3?true broadcast:flags.5?true verified:flags.7?true megagroup:flags.8?true restricted:flags.9?true democracy:flags.10?true signatures:flags.11?true min:flags.12?true id:int access_hash:flags.13?long title:string username:flags.6?string photo:ChatPhoto date:int version:int restriction_reason:flags.9?string admin_rights:flags.14?ChannelAdminRights banned_rights:flags.15?ChannelBannedRights participants_count:flags.17?int = Chat; channelForbidden#289da732 flags:# broadcast:flags.5?true megagroup:flags.8?true id:int access_hash:long title:string until_date:flags.16?int = Chat; chatFull#2e02a614 id:int participants:ChatParticipants chat_photo:Photo notify_settings:PeerNotifySettings exported_invite:ExportedChatInvite bot_info:Vector = ChatFull; @@ -236,7 +238,7 @@ chatPhotoEmpty#37c1011c = ChatPhoto; chatPhoto#6153276a photo_small:FileLocation photo_big:FileLocation = ChatPhoto; messageEmpty#83e5de54 id:int = Message; -message#90dddc11 flags:# out:flags.1?true mentioned:flags.4?true media_unread:flags.5?true silent:flags.13?true post:flags.14?true id:int from_id:flags.8?int to_id:Peer fwd_from:flags.2?MessageFwdHeader via_bot_id:flags.11?int reply_to_msg_id:flags.3?int date:int message:string media:flags.9?MessageMedia reply_markup:flags.6?ReplyMarkup entities:flags.7?Vector views:flags.10?int edit_date:flags.15?int post_author:flags.16?string = Message; +message#44f9b43d flags:# out:flags.1?true mentioned:flags.4?true media_unread:flags.5?true silent:flags.13?true post:flags.14?true id:int from_id:flags.8?int to_id:Peer fwd_from:flags.2?MessageFwdHeader via_bot_id:flags.11?int reply_to_msg_id:flags.3?int date:int message:string media:flags.9?MessageMedia reply_markup:flags.6?ReplyMarkup entities:flags.7?Vector views:flags.10?int edit_date:flags.15?int post_author:flags.16?string grouped_id:flags.17?long = Message; messageService#9e19a1f6 flags:# out:flags.1?true mentioned:flags.4?true media_unread:flags.5?true silent:flags.13?true post:flags.14?true id:int from_id:flags.8?int to_id:Peer reply_to_msg_id:flags.3?int date:int action:MessageAction = Message; messageMediaEmpty#3ded6320 = MessageMedia; @@ -710,7 +712,7 @@ messages.botResults#947ca848 flags:# gallery:flags.0?true query_id:long next_off exportedMessageLink#1f486803 link:string = ExportedMessageLink; -messageFwdHeader#fadff4ac flags:# from_id:flags.0?int date:int channel_id:flags.1?int channel_post:flags.2?int post_author:flags.3?string = MessageFwdHeader; +messageFwdHeader#559ebe6d flags:# from_id:flags.0?int date:int channel_id:flags.1?int channel_post:flags.2?int post_author:flags.3?string saved_from_peer:flags.4?Peer saved_from_msg_id:flags.4?int = MessageFwdHeader; auth.codeTypeSms#72a3158c = auth.CodeType; auth.codeTypeCall#741cd3e3 = auth.CodeType; @@ -1017,7 +1019,7 @@ messages.receivedMessages#5a954c0 max_id:int = Vector; messages.setTyping#a3825e50 peer:InputPeer action:SendMessageAction = Bool; messages.sendMessage#fa88427a flags:# no_webpage:flags.1?true silent:flags.5?true background:flags.6?true clear_draft:flags.7?true peer:InputPeer reply_to_msg_id:flags.0?int message:string random_id:long reply_markup:flags.2?ReplyMarkup entities:flags.3?Vector = Updates; messages.sendMedia#c8f16791 flags:# silent:flags.5?true background:flags.6?true clear_draft:flags.7?true peer:InputPeer reply_to_msg_id:flags.0?int media:InputMedia random_id:long reply_markup:flags.2?ReplyMarkup = Updates; -messages.forwardMessages#708e0195 flags:# silent:flags.5?true background:flags.6?true with_my_score:flags.8?true from_peer:InputPeer id:Vector random_id:Vector to_peer:InputPeer = Updates; +messages.forwardMessages#708e0195 flags:# silent:flags.5?true background:flags.6?true with_my_score:flags.8?true grouped:flags.9?true from_peer:InputPeer id:Vector random_id:Vector to_peer:InputPeer = Updates; messages.reportSpam#cf1592db peer:InputPeer = Bool; messages.hideReportSpam#a8f1709b peer:InputPeer = Bool; messages.getPeerSettings#3672e09c peer:InputPeer = PeerSettings; @@ -1098,6 +1100,7 @@ messages.faveSticker#b9ffc55b id:InputDocument unfave:Bool = Bool; messages.getUnreadMentions#46578472 peer:InputPeer offset_id:int add_offset:int limit:int max_id:int min_id:int = messages.Messages; messages.getRecentLocations#249431e2 peer:InputPeer limit:int = messages.Messages; messages.readMentions#f0189d3 peer:InputPeer = messages.AffectedHistory; +messages.sendMultiMedia#2095512f flags:# silent:flags.5?true background:flags.6?true clear_draft:flags.7?true peer:InputPeer reply_to_msg_id:flags.0?int multi_media:Vector = Updates; updates.getState#edd4882a = updates.State; updates.getDifference#25939651 flags:# pts:int pts_total_limit:flags.0?int date:int qts:int = updates.Difference; @@ -1190,4 +1193,4 @@ langpack.getStrings#2e1ee318 lang_code:string keys:Vector = Vector; -// LAYER 72 +// LAYER 73