From eb22bce2d9a75c8e0638d555ec15d9e68fb43fe0 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 10 May 2018 16:16:23 +0200 Subject: [PATCH 1/5] Add missing connect abstractmethod --- telethon/network/connection/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/telethon/network/connection/common.py b/telethon/network/connection/common.py index 504a0c12..e4f5bf5c 100644 --- a/telethon/network/connection/common.py +++ b/telethon/network/connection/common.py @@ -22,6 +22,10 @@ class Connection(abc.ABC): self._proxy = proxy self._timeout = timeout + @abc.abstractmethod + def connect(self, ip, port): + raise NotImplementedError + @abc.abstractmethod def get_timeout(self): """Returns the timeout used by the connection.""" From ee51aa70734af5e19846e199614f66e6f532c8f4 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Thu, 10 May 2018 16:45:55 +0200 Subject: [PATCH 2/5] Fix lost requests on disconnect need_confirmation.clear The pending acks shouldn't be cleared, in case of a reconnection these would be pretty common. E.g. disconnect(), connect(), invoke, repeat. --- telethon/network/mtproto_sender.py | 1 - 1 file changed, 1 deletion(-) diff --git a/telethon/network/mtproto_sender.py b/telethon/network/mtproto_sender.py index 3c0a4ce8..e282abea 100644 --- a/telethon/network/mtproto_sender.py +++ b/telethon/network/mtproto_sender.py @@ -79,7 +79,6 @@ class MtProtoSender: """Disconnects from the server.""" __log__.info('Disconnecting MtProtoSender...') self.connection.close() - self._need_confirmation.clear() self._clear_all_pending() # region Send and receive From e3c6676795a90e13ea7c071aff4e68a8a844507f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Fri, 11 May 2018 10:02:48 +0200 Subject: [PATCH 3/5] Fix short special usernames not being valid (like vote) --- telethon/utils.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/telethon/utils.py b/telethon/utils.py index 009cea15..a1a3c897 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -34,7 +34,16 @@ USERNAME_RE = re.compile( r'@|(?:https?://)?(?:www\.)?(?:telegram\.(?:me|dog)|t\.me)/(joinchat/)?' ) -VALID_USERNAME_RE = re.compile(r'^[a-zA-Z][\w\d]{3,30}[a-zA-Z\d]$') +# The only shorter-than-five-characters usernames are those used for some +# special, very well known bots. This list may be incomplete though: +# "[...] @gif, @vid, @pic, @bing, @wiki, @imdb and @bold [...]" +# +# See https://telegram.org/blog/inline-bots#how-does-it-work +VALID_USERNAME_RE = re.compile( + r'^([a-z][\w\d]{3,30}[a-z\d]' + r'|gif|vid|pic|bing|wiki|imdb|bold|vote|like|coub)$', + re.IGNORECASE +) def get_display_name(entity): From 54405935204456c03ee6e5b853d4ad5b8fab19b9 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 12 May 2018 15:51:37 +0200 Subject: [PATCH 4/5] Update to layer 78 --- telethon_generator/data/telegram_api.tl | 20 ++++++++++++-------- telethon_generator/parsers/tlobject.py | 7 +++++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/telethon_generator/data/telegram_api.tl b/telethon_generator/data/telegram_api.tl index 711b2d38..4780b8a5 100644 --- a/telethon_generator/data/telegram_api.tl +++ b/telethon_generator/data/telegram_api.tl @@ -14,8 +14,13 @@ error#c4b9f9bb code:int text:string = Error; -ipPort ipv4:int port:int = IpPort; -help.configSimple#d997c3c5 date:int expires:int dc_id:int ip_port_list:Vector = help.ConfigSimple; +//ipPort ipv4:int port:int = IpPort; +//help.configSimple#d997c3c5 date:int expires:int dc_id:int ip_port_list:Vector = help.ConfigSimple; + +ipPort#d433ad73 ipv4:int port:int = IpPort; +ipPortSecret#37982646 ipv4:int port:int secret:bytes = IpPort; +accessPointRule#4679b65f phone_prefix_rules:string dc_id:int ips:vector = AccessPointRule; +help.configSimple#5a592a6c date:int expires:int rules:vector = help.ConfigSimple; ---functions--- @@ -104,7 +109,7 @@ userStatusLastMonth#77ebc742 = UserStatus; chatEmpty#9ba2d800 id:int = Chat; chat#d91cdd54 flags:# creator:flags.0?true kicked:flags.1?true left:flags.2?true admins_enabled:flags.3?true admin:flags.4?true deactivated:flags.5?true id:int title:string photo:ChatPhoto participants_count:int date:int version:int migrated_to:flags.6?InputChannel = Chat; chatForbidden#7328bdb id:int title:string = Chat; -channel#450b7115 flags:# creator:flags.0?true left:flags.2?true editor:flags.3?true broadcast:flags.5?true verified:flags.7?true megagroup:flags.8?true restricted:flags.9?true democracy:flags.10?true signatures:flags.11?true min:flags.12?true id:int access_hash:flags.13?long title:string username:flags.6?string photo:ChatPhoto date:int version:int restriction_reason:flags.9?string admin_rights:flags.14?ChannelAdminRights banned_rights:flags.15?ChannelBannedRights participants_count:flags.17?int = Chat; +channel#c88974ac flags:# creator:flags.0?true left:flags.2?true editor:flags.3?true broadcast:flags.5?true verified:flags.7?true megagroup:flags.8?true restricted:flags.9?true democracy:flags.10?true signatures:flags.11?true min:flags.12?true id:int access_hash:flags.13?long title:string username:flags.6?string photo:ChatPhoto date:int version:int restriction_reason:flags.9?string admin_rights:flags.14?ChannelAdminRights banned_rights:flags.15?ChannelBannedRights participants_count:flags.17?int = Chat; channelForbidden#289da732 flags:# broadcast:flags.5?true megagroup:flags.8?true id:int access_hash:long title:string until_date:flags.16?int = Chat; chatFull#2e02a614 id:int participants:ChatParticipants chat_photo:Photo notify_settings:PeerNotifySettings exported_invite:ExportedChatInvite bot_info:Vector = ChatFull; @@ -190,7 +195,6 @@ inputPeerNotifySettings#38935eb2 flags:# show_previews:flags.0?true silent:flags peerNotifyEventsEmpty#add53cb3 = PeerNotifyEvents; peerNotifyEventsAll#6d1ded88 = PeerNotifyEvents; -peerNotifySettingsEmpty#70a68512 = PeerNotifySettings; peerNotifySettings#9acda4c0 flags:# show_previews:flags.0?true silent:flags.1?true mute_until:int sound:string = PeerNotifySettings; peerSettings#818426cd flags:# report_spam:flags.0?true = PeerSettings; @@ -345,9 +349,9 @@ photos.photo#20212ca8 photo:Photo users:Vector = photos.Photo; upload.file#96a18d5 type:storage.FileType mtime:int bytes:bytes = upload.File; upload.fileCdnRedirect#f18cda44 dc_id:int file_token:bytes encryption_key:bytes encryption_iv:bytes file_hashes:Vector = upload.File; -dcOption#5d8c6cc flags:# ipv6:flags.0?true media_only:flags.1?true tcpo_only:flags.2?true cdn:flags.3?true static:flags.4?true id:int ip_address:string port:int = DcOption; +dcOption#18b7a10d flags:# ipv6:flags.0?true media_only:flags.1?true tcpo_only:flags.2?true cdn:flags.3?true static:flags.4?true id:int ip_address:string port:int secret:flags.10?bytes = DcOption; -config#86b5778e flags:# phonecalls_enabled:flags.1?true default_p2p_contacts:flags.3?true preload_featured_stickers:flags.4?true ignore_phone_entities:flags.5?true revoke_pm_inbox:flags.6?true date:int expires:int test_mode:Bool this_dc:int dc_options:Vector chat_size_max:int megagroup_size_max:int forwarded_count_max:int online_update_period_ms:int offline_blur_timeout_ms:int offline_idle_timeout_ms:int online_cloud_timeout_ms:int notify_cloud_delay_ms:int notify_default_delay_ms:int push_chat_period_ms:int push_chat_limit:int saved_gifs_limit:int edit_time_limit:int revoke_time_limit:int revoke_pm_time_limit:int rating_e_decay:int stickers_recent_limit:int stickers_faved_limit:int channels_read_media_period:int tmp_sessions:flags.0?int pinned_dialogs_count_max:int call_receive_timeout_ms:int call_ring_timeout_ms:int call_connect_timeout_ms:int call_packet_timeout_ms:int me_url_prefix:string suggested_lang_code:flags.2?string lang_pack_version:flags.2?int = Config; +config#eb7bb160 flags:# phonecalls_enabled:flags.1?true default_p2p_contacts:flags.3?true preload_featured_stickers:flags.4?true ignore_phone_entities:flags.5?true revoke_pm_inbox:flags.6?true blocked_mode:flags.8?true date:int expires:int test_mode:Bool this_dc:int dc_options:Vector chat_size_max:int megagroup_size_max:int forwarded_count_max:int online_update_period_ms:int offline_blur_timeout_ms:int offline_idle_timeout_ms:int online_cloud_timeout_ms:int notify_cloud_delay_ms:int notify_default_delay_ms:int push_chat_period_ms:int push_chat_limit:int saved_gifs_limit:int edit_time_limit:int revoke_time_limit:int revoke_pm_time_limit:int rating_e_decay:int stickers_recent_limit:int stickers_faved_limit:int channels_read_media_period:int tmp_sessions:flags.0?int pinned_dialogs_count_max:int call_receive_timeout_ms:int call_ring_timeout_ms:int call_connect_timeout_ms:int call_packet_timeout_ms:int me_url_prefix:string autoupdate_url_prefix:flags.7?string suggested_lang_code:flags.2?string lang_pack_version:flags.2?int = Config; nearestDc#8e1a1775 country:string this_dc:int nearest_dc:int = NearestDc; @@ -780,7 +784,7 @@ langPackDifference#f385c1f6 lang_code:string from_version:int version:int string langPackLanguage#117698f1 name:string native_name:string lang_code:string = LangPackLanguage; -channelAdminRights#5d7ceba5 flags:# change_info:flags.0?true post_messages:flags.1?true edit_messages:flags.2?true delete_messages:flags.3?true ban_users:flags.4?true invite_users:flags.5?true invite_link:flags.6?true pin_messages:flags.7?true add_admins:flags.9?true = ChannelAdminRights; +channelAdminRights#5d7ceba5 flags:# change_info:flags.0?true post_messages:flags.1?true edit_messages:flags.2?true delete_messages:flags.3?true ban_users:flags.4?true invite_users:flags.5?true invite_link:flags.6?true pin_messages:flags.7?true add_admins:flags.9?true manage_call:flags.10?true = ChannelAdminRights; channelBannedRights#58cf4249 flags:# view_messages:flags.0?true send_messages:flags.1?true send_media:flags.2?true send_stickers:flags.3?true send_gifs:flags.4?true send_games:flags.5?true send_inline:flags.6?true embed_links:flags.7?true until_date:int = ChannelBannedRights; @@ -1104,4 +1108,4 @@ langpack.getStrings#2e1ee318 lang_code:string keys:Vector = Vector; -// LAYER 76 +// LAYER 78 diff --git a/telethon_generator/parsers/tlobject.py b/telethon_generator/parsers/tlobject.py index a5e5945a..d2a3684e 100644 --- a/telethon_generator/parsers/tlobject.py +++ b/telethon_generator/parsers/tlobject.py @@ -38,8 +38,11 @@ class TLObject: self.id = self.infer_id() else: self.id = int(object_id, base=16) - assert self.id == self.infer_id(),\ - 'Invalid inferred ID for ' + repr(self) + # As of layer 78 ipPortSecret won't match, Telegram may still be + # developing this layer and more changes shall be to expect. + # + # assert self.id == self.infer_id(),\ + # 'Invalid inferred ID for ' + repr(self) self.class_name = snake_to_camel_case( self.name, suffix='Request' if self.is_function else '') From 32b7e9e27aa1d2587ff2990bc4d835a30b1bc31f Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Sat, 12 May 2018 16:12:42 +0200 Subject: [PATCH 5/5] Whitelist generator's mismatching ID and ya username --- setup.py | 8 +++---- telethon/utils.py | 2 +- telethon_generator/generator.py | 30 -------------------------- telethon_generator/parsers/tlobject.py | 29 +++++++++++++++++-------- 4 files changed, 25 insertions(+), 44 deletions(-) delete mode 100644 telethon_generator/generator.py diff --git a/setup.py b/setup.py index f0b958c2..4769dbf7 100755 --- a/setup.py +++ b/setup.py @@ -57,11 +57,11 @@ def generate(which): from telethon_generator.generators import\ generate_errors, generate_tlobjects, generate_docs, clean_tlobjects - tlobjects = list(itertools.chain( - parse_tl(TLOBJECT_IN_CORE_TL), parse_tl(TLOBJECT_IN_TL))) - - errors = list(parse_errors(ERRORS_IN_JSON, ERRORS_IN_DESC)) layer = find_layer(TLOBJECT_IN_TL) + errors = list(parse_errors(ERRORS_IN_JSON, ERRORS_IN_DESC)) + tlobjects = list(itertools.chain( + parse_tl(TLOBJECT_IN_CORE_TL, layer=layer), + parse_tl(TLOBJECT_IN_TL, layer=layer))) if not which: which.extend(('tl', 'errors')) diff --git a/telethon/utils.py b/telethon/utils.py index a1a3c897..0ba64177 100644 --- a/telethon/utils.py +++ b/telethon/utils.py @@ -41,7 +41,7 @@ USERNAME_RE = re.compile( # See https://telegram.org/blog/inline-bots#how-does-it-work VALID_USERNAME_RE = re.compile( r'^([a-z][\w\d]{3,30}[a-z\d]' - r'|gif|vid|pic|bing|wiki|imdb|bold|vote|like|coub)$', + r'|gif|vid|pic|bing|wiki|imdb|bold|vote|like|coub|ya)$', re.IGNORECASE ) diff --git a/telethon_generator/generator.py b/telethon_generator/generator.py deleted file mode 100644 index f1dad375..00000000 --- a/telethon_generator/generator.py +++ /dev/null @@ -1,30 +0,0 @@ -from telethon_generator.parsers import parse_errors, parse_tl, find_layer -from telethon_generator.generators import\ - generate_errors, generate_tlobjects, generate_docs -import itertools - - -ERRORS_INPUT_JSON = 'data/errors.json' -ERRORS_INPUT_DESC = 'data/error_descriptions' -ERRORS_OUTPUT = '../telethon/errors/rpc_error_list.py' - -TLOBJECT_INPUT_CORE_TL = 'data/mtproto_api.tl' -TLOBJECT_INPUT_TL = 'data/telegram_api.tl' -TLOBJECT_OUTPUT = '../telethon/tl' - -DOCS_INPUT_RES = 'data/html' -DOCS_OUTPUT = '../docs' - - -if __name__ == '__main__': - tlobjects = list(itertools.chain( - parse_tl(TLOBJECT_INPUT_CORE_TL), parse_tl(TLOBJECT_INPUT_TL))) - - errors = list(parse_errors(ERRORS_INPUT_JSON, ERRORS_INPUT_DESC)) - layer = find_layer(TLOBJECT_INPUT_TL) - - generate_tlobjects(tlobjects, layer, TLOBJECT_OUTPUT) - with open(ERRORS_OUTPUT, 'w', encoding='utf-8') as file: - generate_errors(errors, file) - - generate_docs(tlobjects, errors, layer, DOCS_INPUT_RES, DOCS_OUTPUT) diff --git a/telethon_generator/parsers/tlobject.py b/telethon_generator/parsers/tlobject.py index d2a3684e..07bf6145 100644 --- a/telethon_generator/parsers/tlobject.py +++ b/telethon_generator/parsers/tlobject.py @@ -10,9 +10,17 @@ CORE_TYPES = ( 0x1cb5c415, # vector#1cb5c415 {t:Type} # [ t ] = Vector t; ) +# https://github.com/telegramdesktop/tdesktop/blob/4bf66cb6e93f3965b40084771b595e93d0b11bcd/Telegram/SourceFiles/codegen/scheme/codegen_scheme.py#L57-L62 +WHITELISTED_MISMATCHING_IDS = { + # 0 represents any layer + 0: {'ipPortSecret', 'accessPointRule', 'help.configSimple'}, + 77: {'channel'}, + 78: {'channel'} +} + class TLObject: - def __init__(self, fullname, object_id, args, result, is_function): + def __init__(self, fullname, object_id, args, result, is_function, layer): """ Initializes a new TLObject, given its properties. @@ -22,6 +30,7 @@ class TLObject: :param args: The arguments, if any, of the TL object :param result: The result type of the TL object :param is_function: Is the object a function or a type? + :param layer: The layer this TLObject belongs to. """ # The name can or not have a namespace self.fullname = fullname @@ -38,11 +47,12 @@ class TLObject: self.id = self.infer_id() else: self.id = int(object_id, base=16) - # As of layer 78 ipPortSecret won't match, Telegram may still be - # developing this layer and more changes shall be to expect. - # - # assert self.id == self.infer_id(),\ - # 'Invalid inferred ID for ' + repr(self) + whitelist = WHITELISTED_MISMATCHING_IDS[0] |\ + WHITELISTED_MISMATCHING_IDS.get(layer, set()) + + if self.fullname not in whitelist: + assert self.id == self.infer_id(),\ + 'Invalid inferred ID for ' + repr(self) self.class_name = snake_to_camel_case( self.name, suffix='Request' if self.is_function else '') @@ -208,7 +218,7 @@ class TLArg: return str(self).replace(':date', ':int').replace('?date', '?int') -def _from_line(line, is_function): +def _from_line(line, is_function, layer): match = re.match( r'^([\w.]+)' # 'name' r'(?:#([0-9a-fA-F]+))?' # '#optionalcode' @@ -234,12 +244,13 @@ def _from_line(line, is_function): object_id=match.group(2), result=match.group(3), is_function=is_function, + layer=layer, args=[TLArg(name, arg_type, brace != '') for brace, name, arg_type in args_match] ) -def parse_tl(file_path, ignore_core=False): +def parse_tl(file_path, layer, ignore_core=False): """This method yields TLObjects from a given .tl file.""" with open(file_path, encoding='utf-8') as file: is_function = False @@ -259,7 +270,7 @@ def parse_tl(file_path, ignore_core=False): continue try: - result = _from_line(line, is_function) + result = _from_line(line, is_function, layer=layer) if not ignore_core or result.id not in CORE_TYPES: yield result except ValueError as e: