From cef8f54fc40d431b2a573c1987ffe24df4d71b99 Mon Sep 17 00:00:00 2001 From: 0x000011b <0x000011b@waifu.club> Date: Fri, 23 Dec 2022 16:45:09 -0300 Subject: [PATCH] fix: ignore invalid CAI JSON dumps --- waifu/datasets/characterai.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/waifu/datasets/characterai.py b/waifu/datasets/characterai.py index ee76c48..b6d4048 100644 --- a/waifu/datasets/characterai.py +++ b/waifu/datasets/characterai.py @@ -1,4 +1,5 @@ import json +import logging import os import typing as t from dataclasses import dataclass @@ -6,6 +7,8 @@ from dataclasses import dataclass from waifu.datasets import BaseDataset from waifu.utils.dataset import get_data_path +logger = logging.getLogger(__name__) + @dataclass(frozen=True) class CaiBotInfo: @@ -96,7 +99,10 @@ def _available_json_data() -> t.Generator[dict[str, t.Any], None, None]: folder_path = os.path.join(dataset_path, folder) for json_file_path in _enumerate_json_files(folder_path): with open(json_file_path, "r", encoding="utf-8") as json_file: - yield json.load(json_file) + try: + yield json.load(json_file) + except json.decoder.JSONDecodeError as ex: + logger.error("Failed to parse %s: %s", json_file_path, ex) def _bot_info_from_dict(info_dict: dict[str, t.Any]) -> CaiBotInfo: