From 9b298d46ebf4330d1d704c3ed31879fbd89e707f Mon Sep 17 00:00:00 2001 From: SaraVieira Date: Tue, 10 Sep 2024 21:00:20 +0100 Subject: [PATCH] use env & reuse request --- backend/config/__init__.py | 3 + backend/endpoints/heartbeat.py | 2 + backend/endpoints/responses/heartbeat.py | 1 + .../endpoints/responses/retroachievements.py | 13 ++-- backend/endpoints/retroachievements.py | 75 ++++--------------- backend/endpoints/sockets/scan.py | 8 -- backend/endpoints/user.py | 2 +- backend/exceptions/endpoint_exceptions.py | 3 +- backend/handler/metadata/ra_handler.py | 27 +++---- backend/handler/scan_handler.py | 6 -- backend/main.py | 4 +- env.template | 5 ++ examples/docker-compose.example.yml | 2 + .../models/MetadataSourcesDict.ts | 1 + frontend/src/views/GameDetails.vue | 2 +- frontend/src/views/Scan.vue | 8 +- 16 files changed, 54 insertions(+), 108 deletions(-) diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 1ac80f1e6..999e3e7f8 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -49,6 +49,9 @@ IGDB_CLIENT_SECRET: Final = os.environ.get( # STEAMGRIDDB STEAMGRIDDB_API_KEY: Final = os.environ.get("STEAMGRIDDB_API_KEY", "") +# STEAMGRIDDB +RETROACHIEVEMENTS_USERNAME: Final = os.environ.get("RETROACHIEVEMENTS_USERNAME", "") +RETROACHIEVEMENTS_API_KEY: Final = os.environ.get("RETROACHIEVEMENTS_API_KEY", "") # MOBYGAMES MOBYGAMES_API_KEY: Final = os.environ.get("MOBYGAMES_API_KEY", "") diff --git a/backend/endpoints/heartbeat.py b/backend/endpoints/heartbeat.py index e6e81f805..839ed8f1c 100644 --- a/backend/endpoints/heartbeat.py +++ b/backend/endpoints/heartbeat.py @@ -13,6 +13,7 @@ from handler.database import db_user_handler from handler.filesystem import fs_platform_handler from handler.metadata.igdb_handler import IGDB_API_ENABLED from handler.metadata.moby_handler import MOBY_API_ENABLED +from handler.metadata.ra_handler import RETROACHIEVEMENTS_API_ENABLED from handler.metadata.sgdb_handler import STEAMGRIDDB_API_ENABLED from utils import get_version from utils.router import APIRouter @@ -36,6 +37,7 @@ def heartbeat() -> HeartbeatResponse: "IGDB_API_ENABLED": IGDB_API_ENABLED, "MOBY_API_ENABLED": MOBY_API_ENABLED, "STEAMGRIDDB_ENABLED": STEAMGRIDDB_API_ENABLED, + "RETROACHIEVEMENTS_ENABLED": RETROACHIEVEMENTS_API_ENABLED, }, "FS_PLATFORMS": fs_platform_handler.get_platforms(), "WATCHER": { diff --git a/backend/endpoints/responses/heartbeat.py b/backend/endpoints/responses/heartbeat.py index 6a995dc16..f8a5c5c11 100644 --- a/backend/endpoints/responses/heartbeat.py +++ b/backend/endpoints/responses/heartbeat.py @@ -20,6 +20,7 @@ class MetadataSourcesDict(TypedDict): IGDB_API_ENABLED: bool MOBY_API_ENABLED: bool STEAMGRIDDB_ENABLED: bool + RETROACHIEVEMENTS_ENABLED: bool class EmulationDict(TypedDict): diff --git a/backend/endpoints/responses/retroachievements.py b/backend/endpoints/responses/retroachievements.py index 9e0f2e14f..d07f172d9 100644 --- a/backend/endpoints/responses/retroachievements.py +++ b/backend/endpoints/responses/retroachievements.py @@ -1,8 +1,10 @@ from __future__ import annotations -from fastapi import HTTPException, Query, Request, UploadFile, status + from typing import Any -from pydantic import BaseModel, Field +from fastapi import Request +from pydantic import BaseModel + class Achievements(BaseModel): ID: int @@ -22,8 +24,6 @@ class Achievements(BaseModel): type: Any - - class RetroAchievementsGameSchema(BaseModel): ID: int Title: str @@ -57,8 +57,9 @@ class RetroAchievementsGameSchema(BaseModel): HighestAwardDate: str | None = None @classmethod - def from_orm_with_request(cls, db_rom: RetroAchievementsGameSchema, request: Request) -> RetroAchievementsGameSchema: + def from_orm_with_request( + cls, db_rom: RetroAchievementsGameSchema, request: Request + ) -> RetroAchievementsGameSchema: rom = cls.model_validate(db_rom) return rom - diff --git a/backend/endpoints/retroachievements.py b/backend/endpoints/retroachievements.py index 7d27e4763..75b3ec18c 100644 --- a/backend/endpoints/retroachievements.py +++ b/backend/endpoints/retroachievements.py @@ -1,68 +1,18 @@ -from anyio import Path +import yarl from decorators.auth import protected_route -from fastapi import Request -from utils.router import APIRouter -from handler.database import db_rom_handler from endpoints.responses.retroachievements import RetroAchievementsGameSchema from exceptions.endpoint_exceptions import RomNotFoundInRetroAchievementsException -import asyncio -import http - -import httpx -import yarl -from fastapi import HTTPException, status -from logger.logger import log -from utils.context import ctx_httpx_client +from fastapi import Request +from handler.metadata.ra_handler import RetroAchievementsHandler +from utils.router import APIRouter router = APIRouter() -async def _request(url: str, timeout: int = 120) -> dict: - httpx_client = ctx_httpx_client.get() - authorized_url = yarl.URL(url) - try: - res = await httpx_client.get(str(authorized_url), timeout=timeout) - res.raise_for_status() - return res.json() - except httpx.NetworkError as exc: - log.critical( - "Connection error: can't connect to RetroAchievements", exc_info=True - ) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Can't connect to RetroAchievements, check your internet connection", - ) from exc - except httpx.HTTPStatusError as err: - if err.response.status_code == http.HTTPStatus.TOO_MANY_REQUESTS: - # Retry after 2 seconds if rate limit hit - await asyncio.sleep(2) - else: - # Log the error and return an empty dict if the request fails with a different code - log.error(err) - return {} - except httpx.TimeoutException: - # Retry the request once if it times out - pass - - try: - res = await httpx_client.get(url, timeout=timeout) - res.raise_for_status() - except (httpx.HTTPStatusError, httpx.TimeoutException) as err: - if ( - isinstance(err, httpx.HTTPStatusError) - and err.response.status_code == http.HTTPStatus.UNAUTHORIZED - ): - # Sometimes Mobygames returns 401 even with a valid API key - return {} - - # Log the error and return an empty dict if the request fails with a different code - log.error(err) - return {} - - return res.json() - @protected_route(router.get, "/retroachievements/{id}", ["roms.read"]) -async def get_rom_retroachievements(request: Request, id: int) -> RetroAchievementsGameSchema: +async def get_rom_retroachievements( + request: Request, id: int +) -> RetroAchievementsGameSchema: """Get rom endpoint Args: @@ -73,7 +23,9 @@ async def get_rom_retroachievements(request: Request, id: int) -> RetroAchieveme RetroAchievementsGameSchema: User and Game info from retro achivements """ - url = yarl.URL("https://retroachievements.org/API/API_GetGameInfoAndUserProgress.php").with_query( + url = yarl.URL( + "https://retroachievements.org/API/API_GetGameInfoAndUserProgress.php" + ).with_query( g=[id], a=["1"], u=[request.user.ra_username], @@ -81,10 +33,11 @@ async def get_rom_retroachievements(request: Request, id: int) -> RetroAchieveme y=[request.user.ra_api_key], ) - game_with_details = await _request(str(url)) + game_with_details = await RetroAchievementsHandler._request( + RetroAchievementsHandler, str(url) + ) if not game_with_details: raise RomNotFoundInRetroAchievementsException(id) - return RetroAchievementsGameSchema.from_orm_with_request(game_with_details, request) - + return RetroAchievementsGameSchema.model_validate(game_with_details) diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index d08bccdab..e44ae7c06 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -106,7 +106,6 @@ async def scan_platforms( scan_type: ScanType = ScanType.QUICK, roms_ids: list[str] | None = None, metadata_sources: list[str] | None = None, - retroAchievements_info: dict | None = None, ): """Scan all the listed platforms and fetch metadata from different sources @@ -164,7 +163,6 @@ async def scan_platforms( roms_ids=roms_ids, metadata_sources=metadata_sources, socket_manager=sm, - retroAchievements_info=retroAchievements_info, ) # Same protection for platforms @@ -190,7 +188,6 @@ async def _identify_platform( roms_ids: list[str], metadata_sources: list[str], socket_manager: socketio.AsyncRedisManager, - retroAchievements_info: dict | None = None, ) -> ScanStats: # Stop the scan if the flag is set if redis_client.get(STOP_SCAN_FLAG): @@ -264,7 +261,6 @@ async def _identify_platform( roms_ids=roms_ids, metadata_sources=metadata_sources, socket_manager=socket_manager, - retroAchievements_info=retroAchievements_info, ) # Only purge entries if there are some file remaining in the library @@ -313,7 +309,6 @@ async def _identify_rom( roms_ids: list[str], metadata_sources: list[str], socket_manager: socketio.AsyncRedisManager, - retroAchievements_info: dict | None = None, ) -> ScanStats: scan_stats = ScanStats() @@ -338,7 +333,6 @@ async def _identify_rom( scan_type=scan_type, rom=rom, metadata_sources=metadata_sources, - retroAchievements_info=retroAchievements_info, ) scan_stats.scanned_roms += 1 @@ -401,7 +395,6 @@ async def scan_handler(_sid: str, options: dict): scan_type = ScanType[options.get("type", "quick").upper()] roms_ids = options.get("roms_ids", []) metadata_sources = options.get("apis", []) - retroAchievements_info = options.get("retroAchievementsInfo", []) # Uncomment this to run scan in the current process # await scan_platforms( @@ -417,7 +410,6 @@ async def scan_handler(_sid: str, options: dict): scan_type, roms_ids, metadata_sources, - retroAchievements_info, job_timeout=SCAN_TIMEOUT, # Timeout (default of 4 hours) ) diff --git a/backend/endpoints/user.py b/backend/endpoints/user.py index c7cf1fef0..6a6154f2d 100644 --- a/backend/endpoints/user.py +++ b/backend/endpoints/user.py @@ -186,7 +186,7 @@ async def update_user( @protected_route(router.put, "/users/{id}/settings", ["me.write"]) -async def update_user( +async def update_user_settings( request: Request, id: int, form_data: Annotated[UserForm, Depends()] ) -> UserSchema: """Update user settings endpoint diff --git a/backend/exceptions/endpoint_exceptions.py b/backend/exceptions/endpoint_exceptions.py index 7128000cf..dba54200e 100644 --- a/backend/exceptions/endpoint_exceptions.py +++ b/backend/exceptions/endpoint_exceptions.py @@ -58,6 +58,7 @@ class CollectionAlreadyExistsException(Exception): def __repr__(self) -> str: return self.message + class RomNotFoundInRetroAchievementsException(Exception): def __init__(self, id): self.message = f"Rom with id '{id}' does not exist on RetroAchievements" @@ -66,4 +67,4 @@ class RomNotFoundInRetroAchievementsException(Exception): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=self.message) def __repr__(self) -> str: - return self.message \ No newline at end of file + return self.message diff --git a/backend/handler/metadata/ra_handler.py b/backend/handler/metadata/ra_handler.py index 3abcd5c29..15c8c2ee8 100644 --- a/backend/handler/metadata/ra_handler.py +++ b/backend/handler/metadata/ra_handler.py @@ -1,15 +1,21 @@ import asyncio import http -from typing import NotRequired, TypedDict +from typing import Final, NotRequired, TypedDict import httpx import yarl +from config import RETROACHIEVEMENTS_API_KEY, RETROACHIEVEMENTS_USERNAME from fastapi import HTTPException, status from logger.logger import log from utils.context import ctx_httpx_client from .base_hander import MetadataHandler +# Used to display the Mobygames API status in the frontend +RETROACHIEVEMENTS_API_ENABLED: Final = bool(RETROACHIEVEMENTS_API_KEY) and bool( + RETROACHIEVEMENTS_USERNAME +) + class RAGamesPlatform(TypedDict): slug: str @@ -70,9 +76,7 @@ class RetroAchievementsHandler(MetadataHandler): return res.json() - async def _search_rom( - self, md5_hash: str, platform_ra_id: int, retroAchievements_info: dict - ) -> dict | None: + async def _search_rom(self, md5_hash: str, platform_ra_id: int) -> dict | None: if not platform_ra_id: return None @@ -81,8 +85,8 @@ class RetroAchievementsHandler(MetadataHandler): i=[platform_ra_id], h=["1"], f=["1"], - z=[retroAchievements_info.get("username")], - y=[retroAchievements_info.get("api_key")], + z=[RETROACHIEVEMENTS_USERNAME], + y=[RETROACHIEVEMENTS_API_KEY], ) roms = await self._request(str(url)) @@ -104,20 +108,13 @@ class RetroAchievementsHandler(MetadataHandler): name=platform["name"], ) - async def get_rom( - self, md5_hash: str, retroAchievements_info: dict, platform_ra_id: int - ) -> RAGameRom: - - if not retroAchievements_info.get("api_key") or not retroAchievements_info.get( - "username" - ): - return RAGameRom(ra_id=None) + async def get_rom(self, md5_hash: str, platform_ra_id: int) -> RAGameRom: if not platform_ra_id: return RAGameRom(ra_id=None) fallback_rom = RAGameRom(ra_id=None) - res = await self._search_rom(md5_hash, platform_ra_id, retroAchievements_info) + res = await self._search_rom(md5_hash, platform_ra_id) if not res: return fallback_rom diff --git a/backend/handler/scan_handler.py b/backend/handler/scan_handler.py index 787bb393f..c488fe80e 100644 --- a/backend/handler/scan_handler.py +++ b/backend/handler/scan_handler.py @@ -50,7 +50,6 @@ async def _get_main_platform_igdb_id(platform: Platform): async def scan_platform( fs_slug: str, fs_platforms: list[str], - retroAchievements_info: dict | None = None, metadata_sources: list[str] | None = None, ) -> Platform: """Get platform details @@ -172,7 +171,6 @@ async def scan_rom( scan_type: ScanType, rom: Rom | None = None, metadata_sources: list[str] | None = None, - retroAchievements_info: dict | None = None, ) -> Rom: if not metadata_sources: metadata_sources = ["igdb", "moby", "retro_achievements"] @@ -193,9 +191,6 @@ async def scan_rom( "name": fs_rom["file_name"], "url_cover": "", "url_screenshots": [], - "crc_hash": rom.crc_hash if rom else None, - "md5_hash": rom.md5_hash if rom else None, - "sha1_hash": rom.sha1_hash if rom else None, } # Update properties from existing rom if not a complete rescan @@ -301,7 +296,6 @@ async def scan_rom( ): return await meta_ra_handler.get_rom( rom_attrs["md5_hash"], - retroAchievements_info=retroAchievements_info or {}, platform_ra_id=platform.ra_id, ) diff --git a/backend/main.py b/backend/main.py index edd5ce132..e15d097e1 100644 --- a/backend/main.py +++ b/backend/main.py @@ -19,9 +19,9 @@ from endpoints import ( feeds, firmware, heartbeat, - user, platform, raw, + retroachievements, rom, saves, screenshots, @@ -29,7 +29,7 @@ from endpoints import ( states, stats, tasks, - retroachievements + user, ) from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware diff --git a/env.template b/env.template index 138da2fcf..b6dcd7f7c 100644 --- a/env.template +++ b/env.template @@ -14,6 +14,11 @@ MOBYGAMES_API_KEY= # SteamGridDB STEAMGRIDDB_API_KEY= +# RetroAchievements +RETROACHIEVEMENTS_API_KEY= +RETROACHIEVEMENTS_USERNAME= + + # Database config DB_HOST=127.0.0.1 DB_PORT=3306 diff --git a/examples/docker-compose.example.yml b/examples/docker-compose.example.yml index f8fb7c448..108689561 100644 --- a/examples/docker-compose.example.yml +++ b/examples/docker-compose.example.yml @@ -20,6 +20,8 @@ services: - IGDB_CLIENT_SECRET= # https://api-docs.igdb.com/#account-creation - MOBYGAMES_API_KEY= # https://www.mobygames.com/info/api/ - STEAMGRIDDB_API_KEY # https://github.com/rommapp/romm/wiki/Generate-API-Keys#steamgriddb + - RETROACHIEVEMENTS_API_KEY # https://api-docs.retroachievements.org/#api-access + - RETROACHIEVEMENTS_USERNAME # https://api-docs.retroachievements.org/#api-access volumes: - romm_resources:/romm/resources # Resources fetched from IGDB (covers, screenshots, etc.) - romm_redis_data:/redis-data # Cached data for background tasks diff --git a/frontend/src/__generated__/models/MetadataSourcesDict.ts b/frontend/src/__generated__/models/MetadataSourcesDict.ts index 69b957236..d189b6493 100644 --- a/frontend/src/__generated__/models/MetadataSourcesDict.ts +++ b/frontend/src/__generated__/models/MetadataSourcesDict.ts @@ -7,5 +7,6 @@ export type MetadataSourcesDict = { IGDB_API_ENABLED: boolean; MOBY_API_ENABLED: boolean; STEAMGRIDDB_ENABLED: boolean; + RETROACHIEVEMENTS_ENABLED: boolean; }; diff --git a/frontend/src/views/GameDetails.vue b/frontend/src/views/GameDetails.vue index d0f8129a1..e99a3bf03 100644 --- a/frontend/src/views/GameDetails.vue +++ b/frontend/src/views/GameDetails.vue @@ -94,7 +94,7 @@ watch( () => route.fullPath, async () => { await fetchDetails(); - } + }, ); diff --git a/frontend/src/views/Scan.vue b/frontend/src/views/Scan.vue index 8be006840..7d6c874ff 100644 --- a/frontend/src/views/Scan.vue +++ b/frontend/src/views/Scan.vue @@ -24,7 +24,7 @@ const retroAchievements = computed(() => ({ name: "RetroAchievements", value: "retro_achievements", logo_path: "/assets/scrappers/ra.webp", - disabled: !auth.user?.ra_api_key || !auth.user.ra_username, + disabled: !heartbeat.value.METADATA_SOURCES?.RETROACHIEVEMENTS_ENABLED, })); // Use a computed property to reactively update metadataOptions based on heartbeat const metadataOptions = computed(() => [ @@ -89,12 +89,6 @@ async function scan() { socket.emit("scan", { platforms: platformsToScan.value.map((p) => p.id), type: scanType.value, - retroAchievementsInfo: retroAchievements.value.disabled - ? {} - : { - api_key: auth.user?.ra_api_key, - username: auth.user?.ra_username, - }, apis: [ ...metadataSources.value.map((s) => s.value), retroAchievements.value.value,