diff --git a/discord/__main__.py b/discord/__main__.py index a102f2f96c48..579978937e5b 100644 --- a/discord/__main__.py +++ b/discord/__main__.py @@ -24,16 +24,18 @@ from __future__ import annotations -from typing import Optional, Tuple, Dict - import argparse +import importlib.metadata +import platform import sys from pathlib import Path +from typing import Dict, Optional, Tuple -import discord -import importlib.metadata import aiohttp -import platform +import curl_cffi +import discord_protos + +import discord def show_version() -> None: @@ -47,6 +49,8 @@ def show_version() -> None: if version: entries.append(f' - discord.py-self metadata: v{version}') + entries.append(f'- discord-protos v{discord_protos.__version__}') + entries.append(f'- curl_cffi v{curl_cffi.__version__} (curl v{curl_cffi.__curl_version__})') entries.append(f'- aiohttp v{aiohttp.__version__}') uname = platform.uname() entries.append('- system info: {0.system} {0.release} {0.version}'.format(uname)) diff --git a/discord/client.py b/discord/client.py index 4a54ee2c0dd8..2c36084c34eb 100644 --- a/discord/client.py +++ b/discord/client.py @@ -155,6 +155,10 @@ class Client: A number of options can be passed to the :class:`Client`. + .. versionchanged:: 2.1 + + Removed the ``http_trace`` parameter. + Parameters ----------- max_messages: Optional[:class:`int`] @@ -215,12 +219,6 @@ class Client: Whether to keep presences up-to-date across clients. The default behavior is ``True`` (what the client does). - .. versionadded:: 2.0 - http_trace: :class:`aiohttp.TraceConfig` - The trace configuration to use for tracking HTTP requests the library does using ``aiohttp``. - This allows you to check requests the library is using. For more information, check the - `aiohttp documentation `_. - .. versionadded:: 2.0 captcha_handler: Optional[Callable[[:class:`.CaptchaRequired`, :class:`.Client`], Awaitable[:class:`str`]] A function that solves captcha challenges. @@ -254,7 +252,6 @@ def __init__(self, **options: Any) -> None: proxy: Optional[str] = options.pop('proxy', None) proxy_auth: Optional[aiohttp.BasicAuth] = options.pop('proxy_auth', None) unsync_clock: bool = options.pop('assume_unsync_clock', True) - http_trace: Optional[aiohttp.TraceConfig] = options.pop('http_trace', None) max_ratelimit_timeout: Optional[float] = options.pop('max_ratelimit_timeout', None) self.captcha_handler: Optional[Callable[[CaptchaRequired, Client], Awaitable[str]]] = options.pop( 'captcha_handler', None @@ -263,7 +260,6 @@ def __init__(self, **options: Any) -> None: proxy=proxy, proxy_auth=proxy_auth, unsync_clock=unsync_clock, - http_trace=http_trace, captcha=self.handle_captcha, max_ratelimit_timeout=max_ratelimit_timeout, locale=lambda: self._connection.locale, diff --git a/discord/errors.py b/discord/errors.py index 667edbf48693..713525e18deb 100644 --- a/discord/errors.py +++ b/discord/errors.py @@ -29,7 +29,8 @@ from .utils import _get_as_snowflake if TYPE_CHECKING: - from aiohttp import ClientResponse, ClientWebSocketResponse + from aiohttp import ClientResponse + from curl_cffi.requests import Response as CurlResponse, WebSocket from requests import Response from typing_extensions import TypeGuard @@ -41,7 +42,7 @@ FormErrorWrapper as FormErrorWrapperPayload, ) - _ResponseType = Union[ClientResponse, Response] + _ResponseType = Union[ClientResponse, CurlResponse, Response] __all__ = ( 'DiscordException', @@ -116,10 +117,10 @@ class HTTPException(DiscordException): Attributes ------------ - response: :class:`aiohttp.ClientResponse` - The response of the failed HTTP request. This is an - instance of :class:`aiohttp.ClientResponse`. In some cases - this could also be a :class:`requests.Response`. + response: Union[:class:`curl_cffi.requests.Response`, :class:`aiohttp.ClientResponse`] + The response of the failed HTTP request. This is an instance of + :class:`curl_cffi.requests.Response` or :class:`aiohttp.ClientResponse`. + In some cases this could also be a :class:`requests.Response`. text: :class:`str` The text of the error. Could be an empty string. status: :class:`int` @@ -300,10 +301,9 @@ class ConnectionClosed(ClientException): __slots__ = ('code', 'reason') - def __init__(self, socket: ClientWebSocketResponse, *, code: Optional[int] = None): + def __init__(self, code: Optional[int] = None, reason: Optional[str] = None): # This exception is just the same exception except # reconfigured to subclass ClientException for users - self.code: int = code or socket.close_code or -1 - # aiohttp doesn't seem to consistently provide close reason - self.reason: str = '' - super().__init__(f'WebSocket closed with {self.code}') + self.code: int = code or -1 + self.reason: str = reason or 'unknown' + super().__init__(f'WebSocket closed with {self.code} (reason: {self.reason!r})') diff --git a/discord/gateway.py b/discord/gateway.py index cc3872d7d877..0af7f5091835 100644 --- a/discord/gateway.py +++ b/discord/gateway.py @@ -34,12 +34,14 @@ from typing import Any, Callable, Coroutine, Dict, List, TYPE_CHECKING, NamedTuple, Optional, TypeVar -import aiohttp +from curl_cffi import CurlError +from curl_cffi.requests import WebSocket +from curl_cffi.const import CurlWsFlag import yarl from . import utils from .activity import BaseActivity, Spotify -from .enums import SpeakingState +from .enums import SpeakingState, Status from .errors import ConnectionClosed from .flags import Capabilities @@ -58,7 +60,6 @@ from .activity import ActivityTypes from .client import Client - from .enums import Status from .state import ConnectionState from .types.snowflake import Snowflake from .voice_client import VoiceClient @@ -73,9 +74,25 @@ def __init__(self, *, resume: bool = True): class WebSocketClosure(Exception): - """An exception to make up for the fact that aiohttp doesn't signal closure.""" + """An exception to make up for the fact that curl doesn't signal closure. - pass + Attributes + ----------- + code: :class:`int` + The close code of the websocket. + reason: :class:`str` + The reason provided for the closure. + """ + + __slots__ = ('code', 'reason') + + CLOSE_CODE = struct.Struct("!H") + + def __init__(self, msg: bytes): + # HACK: Unpack code and reason from raw message + self.code: int = self.CLOSE_CODE.unpack(msg[:2])[0] + self.reason: str = msg[2:].decode('utf-8') + super().__init__(f'WebSocket closed with {self.code} (reason: {self.reason!r})') class EventListener(NamedTuple): @@ -256,7 +273,7 @@ class DiscordWebSocket: RECONNECT Receive only. Tells the client to reconnect to a new gateway. REQUEST_MEMBERS - Send only. Asks for the guild members. + Send only. Asks for the guild members. Responds with GUILD_MEMBERS_CHUNK. INVALIDATE_SESSION Receive only. Tells the client to optionally invalidate the session and IDENTIFY again. @@ -265,14 +282,14 @@ class DiscordWebSocket: HEARTBEAT_ACK Receive only. Confirms receiving of a heartbeat. Not having it implies a connection issue. - GUILD_SYNC - Send only. Requests a guild sync. This is unfortunately no longer functional. CALL_CONNECT - Send only. Maybe used for calling? Probably just tracking. + Send only. Requests an existing call on a channel. Might respond with CALL_CREATE. GUILD_SUBSCRIBE Send only. Subscribes you to guilds/guild members. Might respond with GUILD_MEMBER_LIST_UPDATE. REQUEST_COMMANDS Send only. Requests application commands from a guild. Responds with GUILD_APPLICATION_COMMANDS_UPDATE. + SEARCH_RECENT_MEMBERS + Send only. Searches for recent members in a guild. Responds with GUILD_MEMBERS_CHUNK. gateway The gateway we are currently connected to. token @@ -314,8 +331,8 @@ class DiscordWebSocket: SEARCH_RECENT_MEMBERS = 35 # fmt: on - def __init__(self, socket: aiohttp.ClientWebSocketResponse, *, loop: asyncio.AbstractEventLoop) -> None: - self.socket: aiohttp.ClientWebSocketResponse = socket + def __init__(self, socket: WebSocket, *, loop: asyncio.AbstractEventLoop) -> None: + self.socket: WebSocket = socket self.loop: asyncio.AbstractEventLoop = loop # An empty dispatcher to prevent crashes @@ -333,10 +350,11 @@ def __init__(self, socket: aiohttp.ClientWebSocketResponse, *, loop: asyncio.Abs self._buffer: bytearray = bytearray() self._close_code: Optional[int] = None self._rate_limiter: GatewayRatelimiter = GatewayRatelimiter() + self._send_lock: asyncio.Lock = asyncio.Lock() @property def open(self) -> bool: - return not self.socket.closed + return self.socket.curl._curl is not None @property def capabilities(self) -> Capabilities: @@ -393,7 +411,7 @@ async def from_client( ws.sequence = sequence ws._max_heartbeat_timeout = client._connection.heartbeat_timeout ws._user_agent = client.http.user_agent - ws._super_properties = client.http.super_properties + ws._super_properties = client.http.headers.super_properties ws._zlib_enabled = zlib if client._enable_debug_events: @@ -631,8 +649,7 @@ def latency(self) -> float: heartbeat = self._keep_alive return float('inf') if heartbeat is None else heartbeat.latency - def _can_handle_close(self) -> bool: - code = self._close_code or self.socket.close_code + def _can_handle_close(self, code: Optional[int]) -> bool: return code not in (1000, 4004, 4010, 4011, 4012, 4013, 4014) async def poll_event(self) -> None: @@ -644,58 +661,70 @@ async def poll_event(self) -> None: The websocket connection was terminated for unhandled reasons. """ try: - msg = await self.socket.receive(timeout=self._max_heartbeat_timeout) - if msg.type is aiohttp.WSMsgType.TEXT: - await self.received_message(msg.data) - elif msg.type is aiohttp.WSMsgType.BINARY: - await self.received_message(msg.data) - elif msg.type is aiohttp.WSMsgType.ERROR: - _log.debug('Received %s.', msg) - raise msg.data - elif msg.type in (aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.CLOSING, aiohttp.WSMsgType.CLOSE): - _log.debug('Received %s.', msg) - raise WebSocketClosure - except (asyncio.TimeoutError, WebSocketClosure) as e: + msg, flags = await asyncio.wait_for(self.socket.arecv(), timeout=self._max_heartbeat_timeout) + if (flags & CurlWsFlag.TEXT) or (flags & CurlWsFlag.BINARY): + await self.received_message(msg) + elif flags & CurlWsFlag.CLOSE: + err = WebSocketClosure(msg) + _log.info(f'Got close {err.code} reason {err.reason}') + raise WebSocketClosure(msg) + except (asyncio.TimeoutError, CurlError, WebSocketClosure) as e: + if isinstance(e, CurlError) and e.code == 52: + _log.debug('Gateway received CURLE_GOT_NOTHING, ignoring...') + return + + _log.info(f'Got poll exception {e}') # Ensure the keep alive handler is closed if self._keep_alive: self._keep_alive.stop() self._keep_alive = None - if isinstance(e, asyncio.TimeoutError): + if isinstance(e, asyncio.TimeoutError): # is this also CancelledError?? _log.debug('Timed out receiving packet. Attempting a reconnect.') raise ReconnectWebSocket from None - code = self._close_code or self.socket.close_code - if self._can_handle_close(): + code = self._close_code or getattr(e, 'code', None) + reason = getattr(e, 'reason', None) + if isinstance(e, CurlError): + _log.debug('Received error %s', e) + reason = str(e) + + _log.info(f'Got code {code} and reason {reason}') + + if self._can_handle_close(code or None): _log.debug('Websocket closed with %s, attempting a reconnect.', code) raise ReconnectWebSocket from None else: _log.debug('Websocket closed with %s, cannot reconnect.', code) - raise ConnectionClosed(self.socket, code=code) from None + raise ConnectionClosed(code, reason) from None + + async def _sendstr(self, data: str, /) -> None: + async with self._send_lock: + await self.socket.asend(data.encode('utf-8')) async def debug_send(self, data: str, /) -> None: await self._rate_limiter.block() self._dispatch('socket_raw_send', data) - await self.socket.send_str(data) + await self._sendstr(data) async def send(self, data: str, /) -> None: await self._rate_limiter.block() - await self.socket.send_str(data) + await self._sendstr(data) async def send_as_json(self, data: Any) -> None: try: await self.send(utils._to_json(data)) except RuntimeError as exc: - if not self._can_handle_close(): - raise ConnectionClosed(self.socket) from exc + if not self._can_handle_close(self._close_code): + raise ConnectionClosed(self._close_code) from exc async def send_heartbeat(self, data: Any) -> None: # This bypasses the rate limit handling code since it has a higher priority try: - await self.socket.send_str(utils._to_json(data)) + await self._sendstr(utils._to_json(data)) except RuntimeError as exc: - if not self._can_handle_close(): - raise ConnectionClosed(self.socket) from exc + if not self._can_handle_close(self._close_code): + raise ConnectionClosed(self._close_code) from exc async def change_presence( self, @@ -872,13 +901,21 @@ async def search_recent_members( await self.send_as_json(payload) - async def close(self, code: int = 4000) -> None: + async def close(self, code: int = 4000, reason: bytes = b'') -> None: + _log.info(f'Closing websocket with code {code}') if self._keep_alive: self._keep_alive.stop() self._keep_alive = None self._close_code = code - await self.socket.close(code=code) + socket = self.socket + + # HACK: The close implementation in curl-cffi is currently broken so we do it ourselves + data = struct.pack('!H', code) + reason + await socket.asend(data, CurlWsFlag.CLOSE) + socket.keep_running = False + await self.loop.run_in_executor(None, socket.curl.close) # TODO: Do I need an executor here? + _log.info('Finished closing websocket') DVWS = TypeVar('DVWS', bound='DiscordVoiceWebSocket') @@ -938,25 +975,30 @@ class DiscordVoiceWebSocket: def __init__( self, - socket: aiohttp.ClientWebSocketResponse, + socket: WebSocket, loop: asyncio.AbstractEventLoop, *, hook: Optional[Callable[..., Coroutine[Any, Any, Any]]] = None, ) -> None: - self.ws: aiohttp.ClientWebSocketResponse = socket + self.ws: WebSocket = socket self.loop: asyncio.AbstractEventLoop = loop self._keep_alive: Optional[VoiceKeepAliveHandler] = None self._close_code: Optional[int] = None self.secret_key: Optional[str] = None + self._send_lock: asyncio.Lock = asyncio.Lock() if hook: self._hook = hook async def _hook(self, *args: Any) -> None: pass + async def _sendstr(self, data: str, /) -> None: + async with self._send_lock: + await self.ws.asend(data.encode('utf-8')) + async def send_as_json(self, data: Any) -> None: _log.debug('Voice gateway sending: %s.', data) - await self.ws.send_str(utils._to_json(data)) + await self._sendstr(utils._to_json(data)) send_heartbeat = send_as_json @@ -992,7 +1034,8 @@ async def from_client( """Creates a voice websocket for the :class:`VoiceClient`.""" gateway = 'wss://' + client.endpoint + '/?v=4' http = client._state.http - socket = await http.ws_connect(gateway, compress=15) + # TODO: is not supported by curl + socket = await http.ws_connect(gateway) ws = cls(socket, loop=client.loop, hook=hook) ws.gateway = gateway ws._connection = client @@ -1122,19 +1165,24 @@ async def load_secret_key(self, data: Dict[str, Any]) -> None: async def poll_event(self) -> None: # This exception is handled up the chain - msg = await asyncio.wait_for(self.ws.receive(), timeout=30.0) - if msg.type is aiohttp.WSMsgType.TEXT: - await self.received_message(utils._from_json(msg.data)) - elif msg.type is aiohttp.WSMsgType.ERROR: - _log.debug('Voice received %s.', msg) - raise ConnectionClosed(self.ws) from msg.data - elif msg.type in (aiohttp.WSMsgType.CLOSED, aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSING): + msg, flags = await asyncio.wait_for(self.ws.arecv(), timeout=self._max_heartbeat_timeout) + if flags & CurlWsFlag.TEXT: + await self.received_message(utils._from_json(msg)) + elif flags & CurlWsFlag.CLOSE: _log.debug('Voice received %s.', msg) - raise ConnectionClosed(self.ws, code=self._close_code) + # TODO: hack + data = WebSocketClosure(msg) + raise ConnectionClosed(data.code, data.reason) - async def close(self, code: int = 1000) -> None: - if self._keep_alive is not None: + async def close(self, code: int = 1000, reason: bytes = b'') -> None: + if self._keep_alive: self._keep_alive.stop() self._close_code = code - await self.ws.close(code=code) + socket = self.ws + + # HACK: The close implementation in curl-cffi is currently broken so we do it ourselves + data = struct.pack('!H', code) + reason + await socket.asend(data, CurlWsFlag.CLOSE) + socket.keep_running = False + await self.loop.run_in_executor(None, socket.curl.close) # TODO: Do I need an executor here? diff --git a/discord/http.py b/discord/http.py index ac6dabd3b1e4..402ac0e86108 100644 --- a/discord/http.py +++ b/discord/http.py @@ -25,11 +25,16 @@ from __future__ import annotations import asyncio +import datetime import logging -from random import choice, choices import ssl import string +import warnings +from collections import deque +from http import HTTPStatus +from random import choice, choices from typing import ( + TYPE_CHECKING, Any, Callable, ClassVar, @@ -41,54 +46,53 @@ Mapping, NamedTuple, Optional, - overload, Sequence, - TYPE_CHECKING, Type, TypeVar, Union, + overload, ) from urllib.parse import quote as _uriquote -from collections import deque -import datetime import aiohttp +from curl_cffi import requests -from .enums import NetworkConnectionType, RelationshipAction, InviteType +from . import utils +from .enums import InviteType, NetworkConnectionType, RelationshipAction from .errors import ( - HTTPException, - RateLimited, - Forbidden, - NotFound, - LoginFailure, + CaptchaRequired, DiscordServerError, + Forbidden, GatewayNotFound, - CaptchaRequired, + HTTPException, + LoginFailure, + NotFound, + RateLimited, ) -from .file import _FileBase, File -from .tracking import ContextProperties -from . import utils +from .file import File, _FileBase from .mentions import AllowedMentions +from .tracking import ContextProperties from .utils import MISSING if TYPE_CHECKING: + from types import TracebackType + from typing_extensions import Self - from .channel import TextChannel, DMChannel, GroupChannel, PartialMessageable, VoiceChannel, ForumChannel - from .threads import Thread + from .channel import DMChannel, ForumChannel, GroupChannel, PartialMessageable, TextChannel, VoiceChannel + from .embeds import Embed + from .enums import ChannelType, InteractionType + from .flags import MessageFlags from .mentions import AllowedMentions from .message import Attachment, Message - from .flags import MessageFlags - from .enums import ChannelType, InteractionType - from .embeds import Embed - + from .threads import Thread from .types import ( application, audit_log, automod, billing, - command, channel, + command, directory, emoji, entitlements, @@ -106,23 +110,21 @@ profile, promotions, read_state, - template, role, - user, - webhook, - widget, - team, - threads, scheduled_event, + sticker, store, subscriptions, - sticker, + team, + template, + threads, + user, + webhook, welcome_screen, + widget, ) from .types.snowflake import Snowflake, SnowflakeList - from types import TracebackType - T = TypeVar('T') BE = TypeVar('BE', bound=BaseException) Response = Coroutine[Any, Any, T] @@ -151,8 +153,30 @@ _log = logging.getLogger(__name__) -async def json_or_text(response: aiohttp.ClientResponse) -> Union[Dict[str, Any], str]: - text = await response.text(encoding='utf-8') +def _gen_accept_encoding_header(): + return 'gzip, deflate, br' if aiohttp.http_parser.HAS_BROTLI else 'gzip, deflate' # type: ignore + + +# For some reason, the Discord voice websocket expects this header to be +# completely lowercase while aiohttp respects spec and does it as case-insensitive +aiohttp.hdrs.WEBSOCKET = 'websocket' # type: ignore +try: + # Support brotli if installed + aiohttp.client_reqrep.ClientRequest.DEFAULT_HEADERS[aiohttp.hdrs.ACCEPT_ENCODING] = _gen_accept_encoding_header() # type: ignore +except Exception: + # aiohttp does it for us on newer versions anyway + pass + +# HACK: Ignore event loop warnings from curl_cffi +warnings.filterwarnings('ignore', module='curl_cffi') + + +async def json_or_text(response: Union[aiohttp.ClientResponse, requests.Response]) -> Union[Dict[str, Any], str]: + if isinstance(response, aiohttp.ClientResponse): + text = await response.text(encoding='utf-8') + else: + text = await response.atext() + try: if response.headers['content-type'] == 'application/json': return utils._from_json(text) @@ -184,9 +208,6 @@ async def _gen_session(session: Optional[aiohttp.ClientSession]) -> aiohttp.Clie ctx.minimum_version = ssl.TLSVersion.TLSv1_2 ctx.maximum_version = ssl.TLSVersion.TLSv1_3 ctx.set_ciphers(':'.join(CIPHERS)) - ctx.options |= ssl.OP_NO_SSLv2 - ctx.options |= ssl.OP_NO_SSLv3 - ctx.options |= ssl.OP_NO_COMPRESSION ctx.set_ecdh_curve('prime256v1') if connector is not None: @@ -354,16 +375,14 @@ def handle_message_parameters( return MultipartParameters(payload=payload, multipart=multipart, files=to_upload) -def _gen_accept_encoding_header(): - return 'gzip, deflate, br' if aiohttp.http_parser.HAS_BROTLI else 'gzip, deflate' # type: ignore - - class Route: BASE: ClassVar[str] = f'https://discord.com/api/v{INTERNAL_API_VERSION}' - def __init__(self, method: str, path: str, *, metadata: Optional[str] = None, **parameters: Any) -> None: + def __init__( + self, method: requests.session.HttpMethod, path: str, *, metadata: Optional[str] = None, **parameters: Any + ) -> None: self.path: str = path - self.method: str = method + self.method: requests.session.HttpMethod = method # Metadata is a special string used to differentiate between known sub rate limits # Since these can't be handled generically, this is the next best way to do so. self.metadata: Optional[str] = metadata @@ -446,7 +465,7 @@ def reset(self): self.reset_after = 0.0 self.dirty = False - def update(self, response: aiohttp.ClientResponse, *, use_clock: bool = False) -> None: + def update(self, response: Union[aiohttp.ClientResponse, requests.Response], *, use_clock: bool = False) -> None: headers = response.headers self.limit = int(headers.get('X-Ratelimit-Limit', 1)) @@ -551,17 +570,6 @@ async def __aexit__(self, type: Type[BE], value: BE, traceback: TracebackType) - self._wake(tokens, exception=exception) -# For some reason, the Discord voice websocket expects this header to be -# completely lowercase while aiohttp respects spec and does it as case-insensitive -aiohttp.hdrs.WEBSOCKET = 'websocket' # type: ignore -try: - # Support brotli if installed - aiohttp.client_reqrep.ClientRequest.DEFAULT_HEADERS[aiohttp.hdrs.ACCEPT_ENCODING] = _gen_accept_encoding_header() # type: ignore -except Exception: - # aiohttp does it for us on newer versions anyway - pass - - class _FakeResponse: def __init__(self, reason: str, status: int) -> None: self.reason = reason @@ -578,13 +586,14 @@ def __init__( proxy: Optional[str] = None, proxy_auth: Optional[aiohttp.BasicAuth] = None, unsync_clock: bool = True, - http_trace: Optional[aiohttp.TraceConfig] = None, captcha: Optional[Callable[[CaptchaRequired], Coroutine[Any, Any, str]]] = None, max_ratelimit_timeout: Optional[float] = None, locale: Callable[[], str] = lambda: 'en-US', + extra_headers: Optional[Mapping[str, str]] = None, ) -> None: self.connector: aiohttp.BaseConnector = connector or MISSING - self.__session: aiohttp.ClientSession = MISSING + self.__asession: aiohttp.ClientSession = MISSING + self.__session: requests.AsyncSession = MISSING # Route key -> Bucket hash self._bucket_hashes: Dict[str, str] = {} # Bucket Hash + Major Parameters -> Rate limit @@ -599,27 +608,28 @@ def __init__( self.ack_token: Optional[str] = None self.proxy: Optional[str] = proxy self.proxy_auth: Optional[aiohttp.BasicAuth] = proxy_auth - self.http_trace: Optional[aiohttp.TraceConfig] = http_trace self.use_clock: bool = not unsync_clock self.captcha_handler: Optional[Callable[[CaptchaRequired], Coroutine[Any, Any, str]]] = captcha self.max_ratelimit_timeout: Optional[float] = max(30.0, max_ratelimit_timeout) if max_ratelimit_timeout else None self.get_locale: Callable[[], str] = locale + self.extra_headers: Mapping[str, str] = extra_headers or {} - self.super_properties: Dict[str, Any] = {} - self.encoded_super_properties: str = MISSING + self.headers: utils.Headers = MISSING self._started: bool = False def __del__(self) -> None: - session = self.__session - if session: + asession = self.__asession + if asession and asession.connector: try: - session.connector._close() # type: ignore # Handled below - except AttributeError: + asession.connector._close() + except Exception: pass def clear(self) -> None: - if self.__session and self.__session.closed: + if self.__session and self.__session._closed: self.__session = MISSING + if self.__asession and self.__asession.closed: + self.__asession = MISSING async def startup(self) -> None: if self._started: @@ -630,44 +640,53 @@ async def startup(self) -> None: if self.connector is MISSING or self.connector.closed: self.connector = aiohttp.TCPConnector(limit=0) - self.__session = session = await _gen_session( - aiohttp.ClientSession( - connector=self.connector, trace_configs=None if self.http_trace is None else [self.http_trace] - ) + self.__asession = session = await _gen_session(aiohttp.ClientSession(connector=self.connector)) + self.headers = headers = await utils.Headers.default(session, self.proxy, self.proxy_auth) + _log.info( + 'Found user agent "%s", build number %s.', + headers.user_agent, + headers.super_properties.get('client_build_number'), ) - self.super_properties, self.encoded_super_properties = sp, _ = await utils._get_info(session) - _log.info('Found user agent %s, build number %s.', sp.get('browser_user_agent'), sp.get('client_build_number')) + try: + impersonate = requests.impersonate.DEFAULT_CHROME + except AttributeError: + # Legacy version + impersonate = f'chrome{self.browser_version}' + if not impersonate in requests.BrowserType: + chromes = [b.value for b in requests.BrowserType if b.value.startswith('chrome')] + impersonate = max(chromes, key=lambda c: int(c[6:].split('_')[0])) + + _log.info('Found TLS fingerprint target "%s".', impersonate) + self.__session = requests.AsyncSession(impersonate=impersonate) # type: ignore # strings do indeed work here self._started = True - async def ws_connect(self, url: str, *, compress: int = 0) -> aiohttp.ClientWebSocketResponse: - kwargs: Dict[str, Any] = { - 'proxy_auth': self.proxy_auth, - 'proxy': self.proxy, - 'max_msg_size': 0, - 'timeout': 30.0, - 'autoclose': False, - 'headers': { - 'Accept-Language': 'en-US', - 'Cache-Control': 'no-cache', - 'Connection': 'Upgrade', - 'Origin': 'https://discord.com', - 'Pragma': 'no-cache', - 'Sec-WebSocket-Extensions': 'permessage-deflate; client_max_window_bits', - 'User-Agent': self.user_agent, - }, - 'compress': compress, + async def ws_connect(self, url: str, **kwargs) -> requests.WebSocket: + await self.startup() + + headers: Dict[str, Any] = { + 'Cache-Control': 'no-cache', + 'Origin': 'https://discord.com', + 'Pragma': 'no-cache', + 'Sec-WebSocket-Extensions': 'permessage-deflate; client_max_window_bits', + 'User-Agent': self.user_agent, } - return await self.__session.ws_connect(url, **kwargs) + if self.proxy is not None: + kwargs['proxies'] = {'all': self.proxy} + if self.proxy_auth is not None: + headers['Proxy-Authorization'] = self.proxy_auth.encode() + + session = self.__session + return await session.ws_connect(url, headers=headers, impersonate=session.impersonate, timeout=30.0, **kwargs) @property - def browser_version(self) -> str: - return self.super_properties['browser_version'] + def browser_version(self) -> int: + return self.headers.major_version @property def user_agent(self) -> str: - return self.super_properties['browser_user_agent'] + return self.headers.user_agent def _try_clear_expired_ratelimits(self) -> None: if len(self._buckets) < 256: @@ -712,25 +731,20 @@ async def request( ratelimit = self.get_ratelimit(key) # Header creation + # NOTE: Many browser-specific headers are missing here because curl_cffi fills them in headers = { - 'Accept-Language': 'en-US', + **self.headers.client_hints, 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive', 'Origin': 'https://discord.com', 'Pragma': 'no-cache', 'Referer': 'https://discord.com/channels/@me', - 'Sec-CH-UA': '"Google Chrome";v="{0}", "Chromium";v="{0}", ";Not-A.Brand";v="24"'.format( - self.browser_version.split('.')[0] - ), - 'Sec-CH-UA-Mobile': '?0', - 'Sec-CH-UA-Platform': '"Windows"', 'Sec-Fetch-Dest': 'empty', 'Sec-Fetch-Mode': 'cors', 'Sec-Fetch-Site': 'same-origin', - 'User-Agent': self.user_agent, + 'User-Agent': self.headers.user_agent, 'X-Discord-Locale': self.get_locale(), 'X-Debug-Options': 'bugReporterEnabled', - 'X-Super-Properties': self.encoded_super_properties, + 'X-Super-Properties': self.headers.encoded_super_properties, } # This header isn't really necessary @@ -765,18 +779,22 @@ async def request( if kwargs.pop('super_properties_to_track', False): headers['X-Track'] = headers.pop('X-Super-Properties') + extra_headers = kwargs.pop('headers', None) + if extra_headers: + headers.update(extra_headers) + headers.update(self.extra_headers) kwargs['headers'] = headers # Proxy support if self.proxy is not None: - kwargs['proxy'] = self.proxy + kwargs['proxies'] = {'all': self.proxy} if self.proxy_auth is not None: - kwargs['proxy_auth'] = self.proxy_auth + headers['Proxy-Authorization'] = self.proxy_auth.encode() if not self._global_over.is_set(): await self._global_over.wait() - response: Optional[aiohttp.ClientResponse] = None + response: Optional[requests.Response] = None data: Optional[Union[Dict[str, Any], str]] = None failed = 0 # Number of 500'd requests async with ratelimit: @@ -790,148 +808,159 @@ async def request( form_data = aiohttp.FormData(quote_fields=False) for params in form: form_data.add_field(**params) - kwargs['data'] = form_data + kwargs['data'] = form_data() if failed: headers['X-Failed-Requests'] = str(failed) try: - async with self.__session.request(method, url, **kwargs) as response: - _log.debug('%s %s with %s has returned %s.', method, url, kwargs.get('data'), response.status) - data = await json_or_text(response) - - # Update and use rate limit information if the bucket header is present - discord_hash = response.headers.get('X-Ratelimit-Bucket') - # I am unsure if X-Ratelimit-Bucket is always available - # However, X-Ratelimit-Remaining has been a consistent cornerstone that worked - has_ratelimit_headers = 'X-Ratelimit-Remaining' in response.headers - if discord_hash is not None: - # If the hash Discord has provided is somehow different from our current hash something changed - if bucket_hash != discord_hash: - if bucket_hash is not None: - # If the previous hash was an actual Discord hash then this means the - # hash has changed sporadically. - # This can be due to two reasons - # 1. It's a sub-ratelimit which is hard to handle - # 2. The rate limit information genuinely changed - # There is no good way to discern these, Discord doesn't provide a way to do so. - # At best, there will be some form of logging to help catch it. - # Alternating sub-ratelimits means that the requests oscillate between - # different underlying rate limits -- this can lead to unexpected 429s - # It is unavoidable. - fmt = 'A route (%s) has changed hashes: %s -> %s.' - _log.debug(fmt, route_key, bucket_hash, discord_hash) - - self._bucket_hashes[route_key] = discord_hash - recalculated_key = discord_hash + route.major_parameters - self._buckets[recalculated_key] = ratelimit - self._buckets.pop(key, None) - elif route_key not in self._bucket_hashes: - fmt = '%s has found its initial rate limit bucket hash (%s).' - _log.debug(fmt, route_key, discord_hash) - self._bucket_hashes[route_key] = discord_hash - self._buckets[discord_hash + route.major_parameters] = ratelimit - - if has_ratelimit_headers: - if response.status != 429: - ratelimit.update(response, use_clock=self.use_clock) - if ratelimit.remaining == 0: - _log.debug( - 'A rate limit bucket (%s) has been exhausted. Pre-emptively rate limiting...', - discord_hash or route_key, - ) - - # 202s must be retried - if response.status == 202 and isinstance(data, dict) and data['code'] == 110000: - # We update the `attempts` query parameter - params = kwargs.get('params') - if not params: - kwargs['params'] = {'attempts': 1} - else: - params['attempts'] = (params.get('attempts') or 0) + 1 - - # Sometimes retry_after is 0, but that's undesirable - retry_after: float = data['retry_after'] or 5 - _log.debug('%s %s received a 202. Retrying in %s seconds...', method, url, retry_after) - await asyncio.sleep(retry_after) - continue - - # Request was successful so just return the text/json - if 300 > response.status >= 200: - _log.debug('%s %s has received %s.', method, url, data) - return data - - # Rate limited - if response.status == 429: - if not response.headers.get('Via') or isinstance(data, str): - # Banned by Cloudflare more than likely. - raise HTTPException(response, data) - - if ratelimit.remaining > 0: - # According to night - # https://github.com/discord/discord-api-docs/issues/2190#issuecomment-816363129 - # Remaining > 0 and 429 means that a sub ratelimit was hit. - # It is unclear what should happen in these cases other than just using the retry_after - # value in the body. + response = await self.__session.request(method, url, **kwargs, stream=True) + response.status = response.status_code # type: ignore + response.reason = HTTPStatus(response.status_code).phrase + _log.debug('%s %s with %s has returned %s.', method, url, kwargs.get('data'), response.status_code) + data = await json_or_text(response) + + # Update and use rate limit information if the bucket header is present + discord_hash = response.headers.get('X-Ratelimit-Bucket') + # I am unsure if X-Ratelimit-Bucket is always available + # However, X-Ratelimit-Remaining has been a consistent cornerstone that worked + has_ratelimit_headers = 'X-Ratelimit-Remaining' in response.headers + if discord_hash is not None: + # If the hash Discord has provided is somehow different from our current hash something changed + if bucket_hash != discord_hash: + if bucket_hash is not None: + # If the previous hash was an actual Discord hash then this means the + # hash has changed sporadically. + # This can be due to two reasons + # 1. It's a sub-ratelimit which is hard to handle + # 2. The rate limit information genuinely changed + # There is no good way to discern these, Discord doesn't provide a way to do so. + # At best, there will be some form of logging to help catch it. + # Alternating sub-ratelimits means that the requests oscillate between + # different underlying rate limits -- this can lead to unexpected 429s + # It is unavoidable. + fmt = 'A route (%s) has changed hashes: %s -> %s.' + _log.debug(fmt, route_key, bucket_hash, discord_hash) + + self._bucket_hashes[route_key] = discord_hash + recalculated_key = discord_hash + route.major_parameters + self._buckets[recalculated_key] = ratelimit + self._buckets.pop(key, None) + elif route_key not in self._bucket_hashes: + fmt = '%s has found its initial rate limit bucket hash (%s).' + _log.debug(fmt, route_key, discord_hash) + self._bucket_hashes[route_key] = discord_hash + self._buckets[discord_hash + route.major_parameters] = ratelimit + + if has_ratelimit_headers: + if response.status_code != 429: + ratelimit.update(response, use_clock=self.use_clock) + if ratelimit.remaining == 0: _log.debug( - '%s %s received a 429 despite having %s remaining requests. This is a sub-ratelimit.', - method, - url, - ratelimit.remaining, + 'A rate limit bucket (%s) has been exhausted. Pre-emptively rate limiting...', + discord_hash or route_key, ) - retry_after: float = data['retry_after'] - if self.max_ratelimit_timeout and retry_after > self.max_ratelimit_timeout: - _log.warning( - 'We are being rate limited. %s %s responded with 429. Timeout of %.2f was too long, erroring instead.', - method, - url, - retry_after, - ) - raise RateLimited(retry_after) + # 202s must be retried + if response.status_code == 202 and isinstance(data, dict) and data['code'] == 110000: + # We update the `attempts` query parameter + params = kwargs.get('params') + if not params: + kwargs['params'] = {'attempts': 1} + else: + params['attempts'] = (params.get('attempts') or 0) + 1 + + # Sometimes retry_after is 0, but that's undesirable + retry_after: float = data['retry_after'] or 5 + _log.debug('%s %s received a 202. Retrying in %s seconds...', method, url, retry_after) + await asyncio.sleep(retry_after) + continue + + # Request was successful so just return the text/json + if 300 > response.status_code >= 200: + _log.debug('%s %s has received %s.', method, url, data) + return data - fmt = 'We are being rate limited. %s %s responded with 429. Retrying in %.2f seconds.' - _log.warning(fmt, method, url, retry_after) + # Rate limited + if response.status_code == 429: + if not response.headers.get('Via') or isinstance(data, str): + # Banned by Cloudflare more than likely. + raise HTTPException(response, data) + if ratelimit.remaining > 0: + # According to night + # https://github.com/discord/discord-api-docs/issues/2190#issuecomment-816363129 + # Remaining > 0 and 429 means that a sub ratelimit was hit. + # It is unclear what should happen in these cases other than just using the retry_after + # value in the body. _log.debug( - 'Rate limit is being handled by bucket hash %s with %r major parameters.', - bucket_hash, - route.major_parameters, + '%s %s received a 429 despite having %s remaining requests. This is a sub-ratelimit.', + method, + url, + ratelimit.remaining, ) - # Check if it's a global rate limit - is_global = data.get('global', False) - if is_global: - _log.warning('Global rate limit has been hit. Retrying in %.2f seconds.', retry_after) - self._global_over.clear() - - await asyncio.sleep(retry_after) - _log.debug('Done sleeping for the rate limit. Retrying...') - - # Release the global lock now that the rate limit passed - if is_global: - self._global_over.set() - _log.debug('Global rate limit is now over.') - - continue - - # Unconditional retry - if response.status in {500, 502, 504, 507, 522, 523, 524}: - failed += 1 - await asyncio.sleep(1 + tries * 2) - continue - - # Usual error cases - if response.status == 403: - raise Forbidden(response, data) - elif response.status == 404: - raise NotFound(response, data) - elif response.status >= 500: - raise DiscordServerError(response, data) - else: - if isinstance(data, dict) and 'captcha_key' in data: - raise CaptchaRequired(response, data) # type: ignore - raise HTTPException(response, data) + retry_after: float = data['retry_after'] + if self.max_ratelimit_timeout and retry_after > self.max_ratelimit_timeout: + _log.warning( + 'We are being rate limited. %s %s responded with 429. Timeout of %.2f was too long, erroring instead.', + method, + url, + retry_after, + ) + raise RateLimited(retry_after) + + fmt = 'We are being rate limited. %s %s responded with 429. Retrying in %.2f seconds.' + _log.warning(fmt, method, url, retry_after) + + _log.debug( + 'Rate limit is being handled by bucket hash %s with %r major parameters.', + bucket_hash, + route.major_parameters, + ) + + # Check if it's a global rate limit + is_global = data.get('global', False) + if is_global: + _log.warning('Global rate limit has been hit. Retrying in %.2f seconds.', retry_after) + self._global_over.clear() + + await asyncio.sleep(retry_after) + _log.debug('Done sleeping for the rate limit. Retrying...') + + # Release the global lock now that the rate limit passed + if is_global: + self._global_over.set() + _log.debug('Global rate limit is now over.') + + continue + + # Unconditional retry + if response.status_code in {502, 504, 507, 522, 523, 524}: + failed += 1 + await asyncio.sleep(1 + tries * 2) + continue + + # Usual error cases + if response.status_code == 403: + raise Forbidden(response, data) + elif response.status_code == 404: + raise NotFound(response, data) + elif response.status_code >= 500: + raise DiscordServerError(response, data) + else: + if isinstance(data, dict) and 'captcha_key' in data: + raise CaptchaRequired(response, data) # type: ignore + raise HTTPException(response, data) + + # libcurl errors + except requests.RequestsError as e: + # Outdated library might be missing the code + if getattr(e, 'code', None) in (23, 28, 35): + failed += 1 + await asyncio.sleep(1 + tries * 2) + continue + raise # This is handling exceptions from the request except OSError as e: @@ -956,15 +985,18 @@ async def request( if response is not None: # We've run out of retries, raise - if response.status >= 500: + if response.status_code >= 500: raise DiscordServerError(response, data) raise HTTPException(response, data) raise RuntimeError('Unreachable code in HTTP handling') + # TODO: All the below could be rewritten to use curl_cffi, but I'm not sure + # about the performance and we aren't concerned about fingerprinting here + async def get_from_cdn(self, url: str) -> bytes: - async with self.__session.get(url) as resp: + async with self.__asession.get(url) as resp: if resp.status == 200: return await resp.read() elif resp.status == 404: @@ -974,8 +1006,6 @@ async def get_from_cdn(self, url: str) -> bytes: else: raise HTTPException(resp, 'failed to get asset') - raise RuntimeError('Unreachable code in HTTP handling') - async def upload_to_cloud(self, url: str, file: Union[File, str], hash: Optional[str] = None) -> Any: response: Optional[aiohttp.ClientResponse] = None data: Optional[Union[Dict[str, Any], str]] = None @@ -991,7 +1021,7 @@ async def upload_to_cloud(self, url: str, file: Union[File, str], hash: Optional file.reset(seek=tries) try: - async with self.__session.put(url, data=getattr(file, 'fp', file), headers=headers) as response: + async with self.__asession.put(url, data=getattr(file, 'fp', file), headers=headers) as response: _log.debug('PUT %s with %s has returned %s.', url, file, response.status) data = await json_or_text(response) @@ -1001,7 +1031,7 @@ async def upload_to_cloud(self, url: str, file: Union[File, str], hash: Optional return data # Unconditional retry - if response.status in {500, 502, 504}: + if response.status in {500, 502, 504, 507, 522, 523, 524}: await asyncio.sleep(1 + tries * 2) continue @@ -1029,7 +1059,7 @@ async def upload_to_cloud(self, url: str, file: Union[File, str], hash: Optional raise HTTPException(response, data) async def get_preferred_voice_regions(self) -> List[dict]: - async with self.__session.get('https://latency.discord.media/rtc') as resp: + async with self.__asession.get('https://latency.discord.media/rtc') as resp: if resp.status == 200: return await resp.json() elif resp.status == 404: @@ -1042,6 +1072,8 @@ async def get_preferred_voice_regions(self) -> List[dict]: # State management async def close(self) -> None: + if self.__asession: + await self.__asession.close() if self.__session: await self.__session.close() @@ -1454,27 +1486,6 @@ def unban(self, user_id: Snowflake, guild_id: Snowflake, *, reason: Optional[str Route('DELETE', '/guilds/{guild_id}/bans/{user_id}', guild_id=guild_id, user_id=user_id), reason=reason ) - def guild_voice_state( - self, - user_id: Snowflake, - guild_id: Snowflake, - *, - mute: Optional[bool] = None, - deafen: Optional[bool] = None, - reason: Optional[str] = None, - ) -> Response[member.Member]: - payload = {} - if mute is not None: - payload['mute'] = mute - if deafen is not None: - payload['deaf'] = deafen - - return self.request( - Route('PATCH', '/guilds/{guild_id}/members/{user_id}', guild_id=guild_id, user_id=user_id), - json=payload, - reason=reason, - ) - def edit_my_voice_state(self, guild_id: Snowflake, payload: Dict[str, Any]) -> Response[None]: return self.request(Route('PATCH', '/guilds/{guild_id}/voice-states/@me', guild_id=guild_id), json=payload) diff --git a/discord/utils.py b/discord/utils.py index 5696546bfd42..11619a50be3e 100644 --- a/discord/utils.py +++ b/discord/utils.py @@ -146,7 +146,7 @@ def __get__(self, instance, owner): if TYPE_CHECKING: - from aiohttp import ClientSession + from aiohttp import BasicAuth, ClientSession from functools import cached_property as cached_property from typing_extensions import ParamSpec, Self, TypeGuard @@ -1436,85 +1436,6 @@ def destroy(self) -> None: self._timer.cancel() -FALLBACK_BUILD_NUMBER = 9999 # Used in marketing and dev portal :) -FALLBACK_BROWSER_VERSION = '120.0.0.1' -_SENTRY_ASSET_REGEX = re.compile(r'assets/(sentry\.\w+)\.js') -_BUILD_NUMBER_REGEX = re.compile(r'buildNumber\D+(\d+)"') - - -async def _get_info(session: ClientSession) -> Tuple[Dict[str, Any], str]: - try: - async with session.post('https://cordapi.dolfi.es/api/v2/properties/web', timeout=5) as resp: - json = await resp.json() - return json['properties'], json['encoded'] - except Exception: - _log.info('Info API temporarily down. Falling back to manual retrieval...') - - try: - bn = await _get_build_number(session) - except Exception: - _log.critical('Could not retrieve client build number. Falling back to hardcoded value...') - bn = FALLBACK_BUILD_NUMBER - - try: - bv = await _get_browser_version(session) - except Exception: - _log.critical('Could not retrieve browser version. Falling back to hardcoded value...') - bv = FALLBACK_BROWSER_VERSION - - properties = { - 'os': 'Windows', - 'browser': 'Chrome', - 'device': '', - 'browser_user_agent': _get_user_agent(bv), - 'browser_version': bv, - 'os_version': '10', - 'referrer': '', - 'referring_domain': '', - 'referrer_current': '', - 'referring_domain_current': '', - 'release_channel': 'stable', - 'system_locale': 'en-US', - 'client_build_number': bn, - 'client_event_source': None, - 'design_id': 0, - } - return properties, b64encode(_to_json(properties).encode()).decode('utf-8') - - -async def _get_build_number(session: ClientSession) -> int: - """Fetches client build number""" - async with session.get('https://discord.com/login') as resp: - app = await resp.text() - match = _SENTRY_ASSET_REGEX.search(app) - if match is None: - raise RuntimeError('Could not find sentry asset file') - sentry = match.group(1) - - async with session.get(f'https://discord.com/assets/{sentry}.js') as resp: - build = await resp.text() - match = _BUILD_NUMBER_REGEX.search(build) - if match is None: - raise RuntimeError('Could not find build number') - return int(match.group(1)) - - -async def _get_browser_version(session: ClientSession) -> str: - """Fetches the latest Windows 10/Chrome major browser version.""" - async with session.get( - 'https://versionhistory.googleapis.com/v1/chrome/platforms/win/channels/stable/versions' - ) as response: - data = await response.json() - major = data['versions'][0]['version'].split('.')[0] - return f'{major}.0.0.0' - - -def _get_user_agent(version: str) -> str: - """Fetches the latest Windows 10/Chrome user-agent.""" - # Because of [user agent reduction](https://www.chromium.org/updates/ua-reduction/), we just need the major version now :) - return f'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{version} Safari/537.36' - - def is_docker() -> bool: path = '/proc/self/cgroup' return os.path.exists('/.dockerenv') or (os.path.isfile(path) and any('docker' in line for line in open(path))) @@ -1700,3 +1621,220 @@ def murmurhash32(key: Union[bytes, bytearray, memoryview, str], seed: int = 0, * return unsigned_val else: return -((unsigned_val ^ 0xFFFFFFFF) + 1) + + +_SENTRY_ASSET_REGEX = re.compile(r'assets/(sentry\.\w+)\.js') +_BUILD_NUMBER_REGEX = re.compile(r'buildNumber\D+(\d+)"') + + +class Headers: + """A class to provide standard headers for HTTP requests. + + For now, this is NOT user-customizable and always emulates Chrome on Windows. + """ + + FALLBACK_BUILD_NUMBER = 9999 # Used in marketing and dev portal :) + FALLBACK_BROWSER_VERSION = 131 + + def __init__( + self, + platform: Literal['Windows', 'macOS', 'Linux', 'Android', 'iOS'], + major_version: int, + super_properties: Dict[str, Any], + encoded_super_properties: str, + ) -> None: + self.platform = platform + self.major_version = major_version + self.super_properties = super_properties + self.encoded_super_properties = encoded_super_properties + + @classmethod + async def default( + cls: type[Self], session: ClientSession, proxy: Optional[str] = None, proxy_auth: Optional[BasicAuth] = None + ) -> Self: + """Creates a new :class:`Headers` instance using the default fetching mechanisms.""" + try: + properties, encoded = await asyncio.wait_for( + cls.get_api_properties(session, 'info', proxy=proxy, proxy_auth=proxy_auth), timeout=3 + ) + except Exception: + _log.info('Info API temporarily down. Falling back to manual retrieval...') + else: + return cls( + platform='Windows', + major_version=int(properties['browser_version'].split('.')[0]), + super_properties=properties, + encoded_super_properties=encoded, + ) + + try: + bn = await cls._get_build_number(session, proxy=proxy, proxy_auth=proxy_auth) + except Exception: + _log.critical('Could not retrieve client build number. Falling back to hardcoded value...') + bn = cls.FALLBACK_BUILD_NUMBER + + try: + bv = await cls._get_browser_version(session, proxy=proxy, proxy_auth=proxy_auth) + except Exception: + _log.critical('Could not retrieve browser version. Falling back to hardcoded value...') + bv = cls.FALLBACK_BROWSER_VERSION + + properties = { + 'os': 'Windows', + 'browser': 'Chrome', + 'device': '', + 'browser_user_agent': cls._get_user_agent(bv), + 'browser_version': f'{bv}.0.0.0', + 'os_version': '10', + 'referrer': '', + 'referring_domain': '', + 'referrer_current': '', + 'referring_domain_current': '', + 'release_channel': 'stable', + 'system_locale': 'en-US', + 'client_build_number': bn, + 'client_event_source': None, + 'has_client_mods': False, + } + + return cls( + platform='Windows', + major_version=bv, + super_properties=properties, + encoded_super_properties=b64encode(_to_json(properties).encode()).decode('utf-8'), + ) + + @cached_property + def user_agent(self) -> str: + return self.super_properties['browser_user_agent'] + + @cached_property + def client_hints(self) -> Dict[str, str]: + return { + 'Sec-CH-UA': ', '.join([f'"{brand}";v="{version}"' for brand, version in self.generate_brand_version_list()]), + 'Sec-CH-UA-Mobile': '?1' if self.platform in ('Android', 'iOS') else '?0', + 'Sec-CH-UA-Platform': f'"{self.platform}"', + } + + @staticmethod + async def get_api_properties( + session: ClientSession, type: str, *, proxy: Optional[str] = None, proxy_auth: Optional[BasicAuth] = None + ) -> Tuple[Dict[str, Any], str]: + """Fetches client properties from the API.""" + async with session.get( + f'https://cordapi.dolfi.es/api/v2/properties/{type}', proxy=proxy, proxy_auth=proxy_auth + ) as resp: + json = await resp.json() + return json['properties'], json['encoded'] + + @staticmethod + async def _get_build_number( + session: ClientSession, *, proxy: Optional[str] = None, proxy_auth: Optional[BasicAuth] = None + ) -> int: + """Fetches client build number.""" + async with session.get('https://discord.com/login', proxy=proxy, proxy_auth=proxy_auth) as resp: + app = await resp.text() + match = _SENTRY_ASSET_REGEX.search(app) + if match is None: + raise RuntimeError('Could not find sentry asset file') + sentry = match.group(1) + + async with session.get(f'https://static.discord.com/assets/{sentry}.js', proxy=proxy, proxy_auth=proxy_auth) as resp: + build = await resp.text() + match = _BUILD_NUMBER_REGEX.search(build) + if match is None: + raise RuntimeError('Could not find build number') + return int(match.group(1)) + + @staticmethod + async def _get_browser_version( + session: ClientSession, proxy: Optional[str] = None, proxy_auth: Optional[BasicAuth] = None + ) -> int: + """Fetches the latest Windows 10/Chrome major browser version.""" + async with session.get( + 'https://versionhistory.googleapis.com/v1/chrome/platforms/win/channels/stable/versions', + proxy=proxy, + proxy_auth=proxy_auth, + ) as response: + data = await response.json() + return int(data['versions'][0]['version'].split('.')[0]) + + @staticmethod + def _get_user_agent(version: int, brand: Optional[str] = None) -> str: + """Fetches the latest Windows/Chrome user-agent.""" + # Because of [user agent reduction](https://www.chromium.org/updates/ua-reduction/), we just need the major version now :) + ret = f'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{version}.0.0.0 Safari/537.36' + if brand: + # e.g. Edg/v.0.0.0 for Microsoft Edge + ret += f' {brand}/{version}.0.0.0' + return ret + + # These are all adapted from Chromium source code (https://github.com/chromium/chromium/blob/master/components/embedder_support/user_agent_utils.cc) + + def generate_brand_version_list(self, brand: Optional[str] = "Google Chrome") -> List[Tuple[str, str]]: + """Generates a list of brand and version pairs for the user-agent.""" + version = self.major_version + greasey_bv = self._get_greased_user_agent_brand_version(version) + chromium_bv = ("Chromium", version) + brand_version_list = [greasey_bv, chromium_bv] + if brand: + brand_version_list.append((brand, version)) + + order = self._get_random_order(version, len(brand_version_list)) + shuffled_brand_version_list: List[Any] = [None] * len(brand_version_list) + for i, idx in enumerate(order): + shuffled_brand_version_list[idx] = brand_version_list[i] + return shuffled_brand_version_list + + @staticmethod + def _get_random_order(seed: int, size: int) -> List[int]: + random.seed(seed) + if size == 2: + return [seed % size, (seed + 1) % size] + elif size == 3: + orders = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]] + return orders[seed % len(orders)] + else: + orders = [ + [0, 1, 2, 3], + [0, 1, 3, 2], + [0, 2, 1, 3], + [0, 2, 3, 1], + [0, 3, 1, 2], + [0, 3, 2, 1], + [1, 0, 2, 3], + [1, 0, 3, 2], + [1, 2, 0, 3], + [1, 2, 3, 0], + [1, 3, 0, 2], + [1, 3, 2, 0], + [2, 0, 1, 3], + [2, 0, 3, 1], + [2, 1, 0, 3], + [2, 1, 3, 0], + [2, 3, 0, 1], + [2, 3, 1, 0], + [3, 0, 1, 2], + [3, 0, 2, 1], + [3, 1, 0, 2], + [3, 1, 2, 0], + [3, 2, 0, 1], + [3, 2, 1, 0], + ] + return orders[seed % len(orders)] + + @staticmethod + def _get_greased_user_agent_brand_version(seed: int) -> Tuple[str, str]: + greasey_chars = [" ", "(", ":", "-", ".", "/", ")", ";", "=", "?", "_"] + greased_versions = ["8", "99", "24"] + greasey_brand = ( + f"Not{greasey_chars[seed % len(greasey_chars)]}A{greasey_chars[(seed + 1) % len(greasey_chars)]}Brand" + ) + greasey_version = greased_versions[seed % len(greased_versions)] + + version_parts = greasey_version.split('.') + if len(version_parts) > 1: + greasey_major_version = version_parts[0] + else: + greasey_major_version = greasey_version + return (greasey_brand, greasey_major_version) diff --git a/docs/conf.py b/docs/conf.py index 327a2fcba420..7ffd53bba76d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,7 +59,8 @@ intersphinx_mapping = { 'py': ('https://docs.python.org/3', None), 'aio': ('https://docs.aiohttp.org/en/stable/', None), - 'req': ('https://requests.readthedocs.io/en/latest/', None) + 'req': ('https://requests.readthedocs.io/en/latest/', None), + 'curl_cffi': ('https://curl-cffi.readthedocs.io/en/latest/', None), } rst_prolog = """ diff --git a/requirements.txt b/requirements.txt index f38243d66484..e7d0429443df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ aiohttp>=3.7.4,<4 +curl_cffi>=0.6.0b7,<1 tzlocal>=4.0.0,<6 discord_protos<1.0.0