diff --git a/logic.py b/logic.py index 592ed47..c079c2c 100644 --- a/logic.py +++ b/logic.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- +import json import threading import traceback -from datetime import datetime +from datetime import datetime, timedelta import requests from flask import jsonify, render_template @@ -16,6 +17,8 @@ from .setup import P logger = P.logger package_name = P.package_name _fetch_lock = threading.Lock() +INDIEGALA_SEEN_SETTING_KEY = "indiegala_seen_games" +INDIEGALA_MAX_DISPLAY_DAYS = 14 SOURCE_LABELS = { "epic": "Epic", @@ -54,6 +57,67 @@ def _split_source_payload(results): return grouped +def _parse_dt(value): + if not value: + return None + if isinstance(value, datetime): + return value + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")).replace(tzinfo=None) + except Exception: + return None + + +def _load_indiegala_seen(): + try: + data = json.loads(ModelSetting.get(INDIEGALA_SEEN_SETTING_KEY) or "{}") + return data if isinstance(data, dict) else {} + except Exception: + return {} + + +def _save_indiegala_seen(seen): + cutoff = datetime.now() - timedelta(days=180) + compacted = { + key: value for key, value in (seen or {}).items() + if (_parse_dt(value) or cutoff) >= cutoff + } + ModelSetting.set(INDIEGALA_SEEN_SETTING_KEY, json.dumps(compacted, ensure_ascii=False)) + + +def _filter_indiegala_items(items): + now = datetime.now() + cutoff = now - timedelta(days=INDIEGALA_MAX_DISPLAY_DAYS) + seen = _load_indiegala_seen() + existing = { + row.external_id: row + for row in F.db.session.query(ModelFreeGameItem).filter_by(platform="indiegala").all() + } + for external_id, row in existing.items(): + if external_id and external_id not in seen: + first_seen = _parse_dt(row.created_time) or _parse_dt(row.updated_time) or now + seen[external_id] = first_seen.isoformat() + + filtered = [] + for item in items or []: + external_id = str(item.get("external_id") or "") + if not external_id: + continue + first_seen = _parse_dt(seen.get(external_id)) + existing_row = existing.get(external_id) + if first_seen: + if first_seen < cutoff: + continue + if existing_row is None: + continue + else: + seen[external_id] = now.isoformat() + filtered.append(item) + + _save_indiegala_seen(seen) + return filtered + + def _discord_send(webhook_url, games): lines = [] for game in games[:10]: @@ -117,6 +181,7 @@ class Logic(PluginModuleBase): "last_fetch_started": "", "last_fetch_finished": "", "new_flags_initialized": "False", + INDIEGALA_SEEN_SETTING_KEY: "{}", } def __init__(self, PM): @@ -195,6 +260,7 @@ class Logic(PluginModuleBase): enabled_sources = set(_enabled_sources()) results = scraper.fetch_all() grouped = _split_source_payload(results) + grouped["indiegala"] = _filter_indiegala_items(grouped.get("indiegala", [])) fresh_free_games = [] for legacy_source in ["humble", "fanatical", "gmg", "directgames"]: diff --git a/scraper.py b/scraper.py index e216361..ab5ec8f 100644 --- a/scraper.py +++ b/scraper.py @@ -743,7 +743,9 @@ def fetch_stove_free(): return "https://store.onstove.com/ko/games/" + str(game_id) def _build_game(data): - game_id = _pick(data, ["productNo", "productId", "product_id"]) + if isinstance(data.get("product"), dict): + data = data["product"] + game_id = _pick(data, ["productNo", "productId", "product_id", "product_no", "game_no"]) if game_id in (None, ""): return None game_id = str(game_id).strip() @@ -754,12 +756,13 @@ def fetch_stove_free(): if _DEMO_TITLE_RE.search(title): return None - current_price = _to_float(_pick(data, ["salePrice", "discountPrice", "currentPrice", "finalPrice", "price", "sellingPrice", "sale_price"])) - original_price = _to_float(_pick(data, ["originPrice", "originalPrice", "listPrice", "basePrice", "priceBeforeDiscount", "normalPrice", "origin_price"])) + amount = data.get("amount") if isinstance(data.get("amount"), dict) else {} + current_price = _to_float(_pick(data, ["salePrice", "discountPrice", "currentPrice", "finalPrice", "price", "sellingPrice", "sale_price"]) or amount.get("sales_price")) + original_price = _to_float(_pick(data, ["originPrice", "originalPrice", "listPrice", "basePrice", "priceBeforeDiscount", "normalPrice", "origin_price"]) or amount.get("original_price")) if original_price <= 0.0 or current_price != 0.0: return None - image_url = _pick(data, ["imageUrl", "image_url", "thumbnailUrl", "thumbnail_image_url", "verticalImageUrl", "horizontalImageUrl", "coverImageUrl"]) or "" + image_url = _pick(data, ["imageUrl", "image_url", "thumbnailUrl", "thumbnail_image_url", "verticalImageUrl", "horizontalImageUrl", "coverImageUrl", "title_image_square", "title_image_rectangle"]) or "" return { "external_id": "stove_" + game_id, "platform": "stove", @@ -768,7 +771,7 @@ def fetch_stove_free(): "store_url": _store_url(game_id, data), "original_price": original_price, "current_price": 0.0, - "discount_pct": 100, + "discount_pct": int(_to_float(amount.get("discount_rate")) or 100), "is_free_period": True, "free_start": None, "free_end": None, @@ -853,6 +856,23 @@ def fetch_stove_free(): games.append(game) return games + def _resolve_nuxt_refs(data, value, stack=None): + if stack is None: + stack = set() + if isinstance(value, int) and not isinstance(value, bool): + if value < 0 or value >= len(data) or value in stack: + return None + target = data[value] + if isinstance(target, (dict, list)): + stack.add(value) + return _resolve_nuxt_refs(data, target, stack) + return target + if isinstance(value, dict): + return {key: _resolve_nuxt_refs(data, item, set(stack)) for key, item in value.items()} + if isinstance(value, list): + return [_resolve_nuxt_refs(data, item, set(stack)) for item in value] + return value + def _extract_json_from_html(html): payloads = [] for match in re.finditer(r']+type=["\']application/json["\'][^>]*>(.*?)', html, re.I | re.S): @@ -860,7 +880,15 @@ def fetch_stove_free(): if not raw: continue try: - payloads.append(json.loads(raw)) + payload = json.loads(raw) + if 'id="__NUXT_DATA__"' in match.group(0) and isinstance(payload, list): + payloads.append([ + _resolve_nuxt_refs(payload, item) + for item in payload + if isinstance(item, dict) + ]) + else: + payloads.append(payload) except Exception: pass for pattern in [r"__NUXT__\s*=\s*(\{.*?\})\s*", r"window\.__STORE__\s*=\s*(\{.*?\})\s*;"]: @@ -881,14 +909,11 @@ def fetch_stove_free(): api_headers = dict(_HEADERS, Referer="https://store.onstove.com/", Accept="application/json") html_headers = dict(_HTML_HEADERS, Referer="https://store.onstove.com/") - api_urls = [ - "https://store.onstove.com/api/v2/product/list?product_type=GAME&price_type=FREE&page=1&size=20", - "https://api.onstove.com/store/v2/product/list?product_type=GAME&price_type=FREE", - "https://store.onstove.com/api/store/v2/event/freegame", - ] + api_urls = [] page_urls = [ + "https://store.onstove.com/ko/store/Discount_Mall", + "https://store.onstove.com/ko/store/stoveindie", "https://store.onstove.com/ko/games?priceFilter=FREE", - "https://store.onstove.com/ko/promotions", ] results = [] seen = set()