diff --git a/apps/tgbot/tgbot/config.py b/apps/tgbot/tgbot/config.py index 76f0314..60cc038 100644 --- a/apps/tgbot/tgbot/config.py +++ b/apps/tgbot/tgbot/config.py @@ -9,3 +9,7 @@ INFLUX_PORT = os.getenv("INFLUX_PORT", None) INFLUX_USERNAME = os.getenv("INFLUX_USERNAME", None) INFLUX_PASSWORD = os.getenv("INFLUX_PASSWORD", None) INFLUX_DB = os.getenv("INFLUX_DB", None) + +# Notifications +NOTIFICATION_BOT_TOKEN = os.getenv("NOTIFICATION_BOT_TOKEN") +NOTIFICATION_USERS = os.getenv("NOTIFICATION_USERS", "").split(",") diff --git a/apps/tgbot/tgbot/handlers/helpers.py b/apps/tgbot/tgbot/handlers/helpers.py index 1f141e4..201e0d3 100644 --- a/apps/tgbot/tgbot/handlers/helpers.py +++ b/apps/tgbot/tgbot/handlers/helpers.py @@ -1,10 +1,13 @@ -from httpx import AsyncClient, Timeout, Response, ConnectError +from httpx import AsyncClient, Timeout, Response, ConnectError, ReadTimeout from typing import List from core.coretypes import APINode from ipaddress import ip_address from contextlib import suppress from loguru import logger +from aiogram.bot import Bot from tgbot.handlers.metrics import push_api_request_status +from tgbot.config import NOTIFICATION_BOT_TOKEN, NOTIFICATION_USERS +from traceback import format_exc def check_int(value) -> bool: @@ -45,13 +48,20 @@ async def send_api_requests(endpoint: str, data: dict, nodes: List[APINode]): result = await client.get( f"{node.address}/{endpoint}", params=data ) - except ConnectError as e: - logger.error(f"Node {node.address} got ConnectionError. Full exception: {e}") - # TODO: Report problems to admins - # We yield 500 response when backend is offline + except Exception as e: + logger.error(f"Node {node.address} got Error. Full exception: {e}") + # We yield 500 response when get error result = Response(500) + await send_message_to_admins(f"Node {node.address} got error {e}. Full exception: ```{format_exc()}```") await push_api_request_status( result.status_code, endpoint ) yield result + + +async def send_message_to_admins(message: str): + bot = Bot(token=NOTIFICATION_BOT_TOKEN) + for user in NOTIFICATION_USERS: + logger.info(f"Sended notification to {user}") + await bot.send_message(user, message, parse_mode='Markdown')