From 49eb2812516cd660465c26143a3053ea379f417f Mon Sep 17 00:00:00 2001 From: Viktor Oreshkin Date: Mon, 6 Nov 2017 03:17:22 +0400 Subject: [PATCH] Proper offset calculation for markdown (#407) Dan suca If Dan shared it with Traitor I'll not have to spend my time on this Not a, sorry for not letting you sleep k thx bye Will this stay in history? --- telethon/extensions/markdown.py | 75 +++------------------------------ 1 file changed, 6 insertions(+), 69 deletions(-) diff --git a/telethon/extensions/markdown.py b/telethon/extensions/markdown.py index 574fe025..d4f7ce22 100644 --- a/telethon/extensions/markdown.py +++ b/telethon/extensions/markdown.py @@ -11,6 +11,8 @@ from ..tl.types import ( MessageEntityPre, MessageEntityTextUrl ) +def tg_string_len(s): + return len(s.encode('utf-16le')) // 2 class Mode(Enum): """Different modes supported by Telegram's Markdown""" @@ -22,54 +24,6 @@ class Mode(Enum): URL = 5 -# TODO Special cases, these aren't count as emojies. Alternatives? -# These were determined by generating all emojies with EMOJI_RANGES, -# sending the message through an official application, and cherry-picking -# which ones weren't rendered as emojies (from the beginning one). I am -# not responsible for dropping those characters that did not render with -# my font. -NOT_EMOJIES = { - 9733, 9735, 9736, 9737, 9738, 9739, 9740, 9741, 9743, 9744, 9746, 9750, - 9751, 9754, 9755, 9756, 9758, 9759, 9761, 9764, 9765, 9767, 9768, 9769, - 9771, 9772, 9773, 9776, 9777, 9778, 9779, 9780, 9781, 9782, 9783, 9787, - 9788, 9789, 9790, 9791, 9792, 9793, 9794, 9795, 9796, 9797, 9798, 9799, - 9812, 9813, 9814, 9815, 9816, 9817, 9818, 9819, 9820, 9821, 9822, 9823, - 9825, 9826, 9828, 9831, 9833, 9834, 9835, 9836, 9837, 9838, 9839, 9840, - 9841, 9842, 9843, 9844, 9845, 9846, 9847, 9848, 9849, 9850, 9852, 9853, - 9854, 9856, 9857, 9858, 9859, 9860, 9861, 9862, 9863, 9864, 9865, 9866, - 9867, 9868, 9869, 9870, 9871, 9872, 9873, 9877, 9880, 9882, 9886, 9887, - 9890, 9891, 9892, 9893, 9894, 9895, 9896, 9897, 9900, 9901, 9902, 9903, - 9907, 9908, 9909, 9910, 9911, 9912, 9920, 9921, 9922, 9923, 9985, 9987, - 9988, 9998, 10000, 10001, 10085, 10086, 10087, 127027, 127028, 127029, - 127030, 127031, 127032, 127033, 127034, 127035, 127036, 127037, 127038, - 127039, 127040, 127041, 127042, 127043, 127044, 127045, 127046, 127047, - 127048, 127049, 127050, 127051, 127052, 127053, 127054, 127055, 127056, - 127057, 127058, 127059, 127060, 127061, 127062, 127063, 127064, 127065, - 127066, 127067, 127068, 127069, 127070, 127071, 127072, 127073, 127074, - 127075, 127076, 127077, 127078, 127079, 127080, 127081, 127082, 127083, - 127084, 127085, 127086, 127087, 127088, 127089, 127090, 127091, 127092, - 127093, 127094, 127095, 127096, 127097, 127098, 127099, 127100, 127101, - 127102, 127103, 127104, 127105, 127106, 127107, 127108, 127109, 127110, - 127111, 127112, 127113, 127114, 127115, 127116, 127117, 127118, 127119, - 127120, 127121, 127122, 127123 -} -# using telethon_generator/emoji_ranges.py -EMOJI_RANGES = ( - (8596, 8601), (8617, 8618), (8986, 8987), (9193, 9203), (9208, 9210), - (9642, 9643), (9723, 9726), (9728, 9733), (9735, 9746), (9748, 9751), - (9754, 9884), (9886, 9905), (9907, 9953), (9956, 9983), (9985, 9988), - (9992, 10002), (10035, 10036), (10067, 10069), (10083, 10087), - (10133, 10135), (10548, 10549), (11013, 11015), (11035, 11036), - (126976, 127166), (127169, 127183), (127185, 127231), (127245, 127247), - (127340, 127345), (127358, 127359), (127377, 127386), (127405, 127487), - (127489, 127503), (127538, 127546), (127548, 127551), (127561, 128419), - (128421, 128591), (128640, 128767), (128884, 128895), (128981, 129023), - (129036, 129039), (129096, 129103), (129114, 129119), (129160, 129167), - (129198, 129338), (129340, 129342), (129344, 129349), (129351, 129355), - (129357, 129471), (129473, 131069) -) - - DEFAULT_DELIMITERS = { '**': Mode.BOLD, '__': Mode.ITALIC, @@ -79,24 +33,6 @@ DEFAULT_DELIMITERS = { DEFAULT_URL_RE = re.compile(r'\[(.+?)\]\((.+?)\)') - -def is_emoji(char): - """Returns True if 'char' looks like an emoji""" - char = ord(char) - for start, end in EMOJI_RANGES: - if start <= char <= end: - return char not in NOT_EMOJIES - return False - - -def emojiness(char): - """ - Returns 2 if the character is an emoji, or 1 otherwise. - This seems to be the length Telegram uses for offsets and lengths. - """ - return 2 if is_emoji(char) else 1 - - def parse(message, delimiters=None, url_re=None): """ Parses the given message and returns the stripped message and a list @@ -131,10 +67,10 @@ def parse(message, delimiters=None, url_re=None): url_match.group(1), message[url_match.end():] )) - emoji_len = sum(emojiness(c) for c in url_match.group(1)) + result.append(( offset, - offset + emoji_len, + offset + tg_string_len(url_match.group(1)), (Mode.URL, url_match.group(2)) )) i += len(url_match.group(1)) @@ -154,11 +90,12 @@ def parse(message, delimiters=None, url_re=None): break if i < len(message): - offset += emojiness(message[i]) + offset += tg_string_len(message[i]) i += 1 if result and not isinstance(result[-1], tuple): result.pop() + return message, result