From b66c1e6084dea7e0d8ae7b893ce8dfb29fa7e406 Mon Sep 17 00:00:00 2001 From: Lonami Exo Date: Tue, 25 Dec 2018 16:50:11 +0100 Subject: [PATCH] Revert "Get rid of now broken cache" This reverts commit f73ae42a036b442661933d006267fc674fd12570. --- telethon/client/messages.py | 1 + telethon/client/uploads.py | 76 ++++++++++++++++++++++++++--- telethon/sessions/abstract.py | 2 - telethon/tl/custom/inlinebuilder.py | 9 ++-- 4 files changed, 75 insertions(+), 13 deletions(-) diff --git a/telethon/client/messages.py b/telethon/client/messages.py index 9d8f16e7..ad5a4a3a 100644 --- a/telethon/client/messages.py +++ b/telethon/client/messages.py @@ -690,6 +690,7 @@ class MessageMethods(UploadMethods, ButtonMethods, MessageParseMethods): reply_markup=self.build_reply_markup(buttons) ) msg = self._get_response_message(request, await self(request), entity) + await self._cache_media(msg, file, file_handle) return msg async def delete_messages(self, entity, message_ids, *, revoke=True): diff --git a/telethon/client/uploads.py b/telethon/client/uploads.py index a8b30a52..549ca061 100644 --- a/telethon/client/uploads.py +++ b/telethon/client/uploads.py @@ -22,7 +22,7 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): async def send_file( self, entity, file, *, caption=None, force_document=False, progress_callback=None, reply_to=None, attributes=None, - thumb=None, parse_mode=(), + thumb=None, allow_cache=True, parse_mode=(), voice_note=False, video_note=False, buttons=None, silent=None, **kwargs): """ @@ -74,6 +74,12 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): Successful thumbnails were files below 20kb and 200x200px. Width/height and dimensions/size ratios may be important. + allow_cache (`bool`, optional): + Whether to allow using the cached version stored in the + database or not. Defaults to ``True`` to avoid re-uploads. + Must be ``False`` if you wish to use different attributes + or thumb than those that were used when the file was cached. + parse_mode (`object`, optional): See the `TelegramClient.parse_mode` property for allowed values. Markdown parsing will be used by default. @@ -81,10 +87,16 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): voice_note (`bool`, optional): If ``True`` the audio will be sent as a voice note. + Set `allow_cache` to ``False`` if you sent the same file + without this setting before for it to work. + video_note (`bool`, optional): If ``True`` the video will be sent as a video note, also known as a round video message. + Set `allow_cache` to ``False`` if you sent the same file + without this setting before for it to work. + buttons (`list`, `custom.Button `, :tl:`KeyboardButton`): The matrix (list of lists), row list or button to be shown after sending the message. This parameter will only work if @@ -133,7 +145,7 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): for x in documents: result.append(await self.send_file( - entity, x, + entity, x, allow_cache=allow_cache, caption=caption, force_document=force_document, progress_callback=progress_callback, reply_to=reply_to, attributes=attributes, thumb=thumb, voice_note=voice_note, @@ -157,7 +169,7 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): file_handle, media = await self._file_to_media( file, force_document=force_document, progress_callback=progress_callback, - attributes=attributes, thumb=thumb, + attributes=attributes, allow_cache=allow_cache, thumb=thumb, voice_note=voice_note, video_note=video_note ) @@ -167,6 +179,7 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): entities=msg_entities, reply_markup=markup, silent=silent ) msg = self._get_response_message(request, await self(request), entity) + await self._cache_media(msg, file, file_handle, force_document=force_document) return msg @@ -174,6 +187,15 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): progress_callback=None, reply_to=None, parse_mode=(), silent=None): """Specialized version of .send_file for albums""" + # We don't care if the user wants to avoid cache, we will use it + # anyway. Why? The cached version will be exactly the same thing + # we need to produce right now to send albums (uploadMedia), and + # cache only makes a difference for documents where the user may + # want the attributes used on them to change. + # + # In theory documents can be sent inside the albums but they appear + # as different messages (not inside the album), and the logic to set + # the attributes/avoid cache is already written in .send_file(). entity = await self.get_input_entity(entity) if not utils.is_list_like(caption): caption = (caption,) @@ -184,15 +206,17 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): reply_to = utils.get_message_id(reply_to) + # Need to upload the media first, but only if they're not cached yet media = [] for file in files: # fh will either be InputPhoto or a modified InputFile - fh = await self.upload_file(file) + fh = await self.upload_file(file, use_cache=types.InputPhoto) if not isinstance(fh, types.InputPhoto): r = await self(functions.messages.UploadMediaRequest( entity, media=types.InputMediaUploadedPhoto(fh) )) input_photo = utils.get_input_photo(r.photo) + self.session.cache_file(fh.md5, fh.size, input_photo) fh = input_photo if captions: @@ -216,7 +240,7 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): ] async def upload_file( - self, file, *, part_size_kb=None, file_name=None, + self, file, *, part_size_kb=None, file_name=None, use_cache=None, progress_callback=None): """ Uploads the specified file and returns a handle (an instance of @@ -243,6 +267,13 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): If not specified, the name will be taken from the ``file`` and if this is not a ``str``, it will be ``"unnamed"``. + use_cache (`type`, optional): + The type of cache to use (currently either :tl:`InputDocument` + or :tl:`InputPhoto`). If present and the file is small enough + to need the MD5, it will be checked against the database, + and if a match is found, the upload won't be made. Instead, + an instance of type ``use_cache`` will be returned. + progress_callback (`callable`, optional): A callback function accepting two parameters: ``(sent bytes, total)``. @@ -291,10 +322,20 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): is_large = file_size > 10 * 1024 * 1024 hash_md5 = hashlib.md5() if not is_large: + # Calculate the MD5 hash before anything else. + # As this needs to be done always for small files, + # might as well do it before anything else and + # check the cache. if isinstance(file, str): with open(file, 'rb') as stream: file = stream.read() hash_md5.update(file) + if use_cache: + cached = self.session.get_file( + hash_md5.digest(), file_size, cls=use_cache + ) + if cached: + return cached part_count = (file_size + part_size - 1) // part_size __log__.info('Uploading file of %d bytes in %d chunks of %d', @@ -337,7 +378,7 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): async def _file_to_media( self, file, force_document=False, progress_callback=None, attributes=None, thumb=None, - voice_note=False, video_note=False): + allow_cache=True, voice_note=False, video_note=False): if not file: return None, None @@ -356,9 +397,11 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): media = None file_handle = None as_image = utils.is_image(file) and not force_document + use_cache = types.InputPhoto if as_image else types.InputDocument if not isinstance(file, str) or os.path.isfile(file): file_handle = await self.upload_file( - file, progress_callback=progress_callback + file, progress_callback=progress_callback, + use_cache=use_cache if allow_cache else None ) elif re.match('https?://', file): if as_image: @@ -379,6 +422,12 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): 'Failed to convert {} to media. Not an existing file, ' 'an HTTP URL or a valid bot-API-like file ID'.format(file) ) + elif isinstance(file_handle, use_cache): + # File was cached, so an instance of use_cache was returned + if as_image: + media = types.InputMediaPhoto(file_handle) + else: + media = types.InputMediaDocument(file_handle) elif as_image: media = types.InputMediaUploadedPhoto(file_handle) else: @@ -402,4 +451,17 @@ class UploadMethods(ButtonMethods, MessageParseMethods, UserMethods): ) return file_handle, media + async def _cache_media(self, msg, file, file_handle, + force_document=False): + if file and msg and isinstance(file_handle, + custom.InputSizedFile): + # There was a response message and we didn't use cached + # version, so cache whatever we just sent to the database. + md5, size = file_handle.md5, file_handle.size + if utils.is_image(file) and not force_document: + to_cache = utils.get_input_photo(msg.media.photo) + else: + to_cache = utils.get_input_document(msg.media.document) + self.session.cache_file(md5, size, to_cache) + # endregion diff --git a/telethon/sessions/abstract.py b/telethon/sessions/abstract.py index c5fc889d..b6f86a4d 100644 --- a/telethon/sessions/abstract.py +++ b/telethon/sessions/abstract.py @@ -120,8 +120,6 @@ class Session(ABC): """ raise NotImplementedError - # TODO get rid of now useless cache - @abstractmethod def cache_file(self, md5_digest, file_size, instance): """ diff --git a/telethon/tl/custom/inlinebuilder.py b/telethon/tl/custom/inlinebuilder.py index a87f32a6..a03338bb 100644 --- a/telethon/tl/custom/inlinebuilder.py +++ b/telethon/tl/custom/inlinebuilder.py @@ -113,7 +113,7 @@ class InlineBuilder: Same as ``file`` for `client.send_file `. """ - fh = await self._client.upload_file(file) + fh = await self._client.upload_file(file, use_cache=types.InputPhoto) if not isinstance(fh, types.InputPhoto): r = await self._client(functions.messages.UploadMediaRequest( types.InputPeerSelf(), media=types.InputMediaUploadedPhoto(fh) @@ -140,14 +140,14 @@ class InlineBuilder: async def document( self, file, title=None, *, description=None, type=None, mime_type=None, attributes=None, force_document=False, - voice_note=False, video_note=False, id=None, + voice_note=False, video_note=False, use_cache=True, id=None, text=None, parse_mode=(), link_preview=True, geo=None, period=60, contact=None, game=False, buttons=None ): """ Creates a new inline result of document type. - `mime_type`, `attributes`, `force_document`, + `use_cache`, `mime_type`, `attributes`, `force_document`, `voice_note` and `video_note` are described in `client.send_file `. @@ -174,7 +174,8 @@ class InlineBuilder: else: type = 'document' - fh = await self._client.upload_file(file) + use_cache = types.InputDocument if use_cache else None + fh = await self._client.upload_file(file, use_cache=use_cache) if not isinstance(fh, types.InputDocument): attributes, mime_type = utils.get_attributes( file,