Fix IndieGala and STOVE free game filtering

logic.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
+import json
 import threading
 import traceback
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import requests
 from flask import jsonify, render_template
 from plugin import PluginModuleBase
-from framework import F, Job, scheduler
+from framework import F
 
 from .model import ModelFetchLog, ModelFreeGameItem, ModelSetting
 from . import scraper
@@ -15,8 +16,9 @@ from .setup import P
 
 logger = P.logger
 package_name = P.package_name
-job_id = f"{package_name}_fetch"
 _fetch_lock = threading.Lock()
+INDIEGALA_SEEN_SETTING_KEY = "indiegala_seen_games"
+INDIEGALA_MAX_DISPLAY_DAYS = 14
 
 SOURCE_LABELS = {
     "epic": "Epic",
@@ -55,14 +57,76 @@ def _split_source_payload(results):
     return grouped
 
 
+def _parse_dt(value):
+    if not value:
+        return None
+    if isinstance(value, datetime):
+        return value
+    try:
+        return datetime.fromisoformat(str(value).replace("Z", "+00:00")).replace(tzinfo=None)
+    except Exception:
+        return None
+
+
+def _load_indiegala_seen():
+    try:
+        data = json.loads(ModelSetting.get(INDIEGALA_SEEN_SETTING_KEY) or "{}")
+        return data if isinstance(data, dict) else {}
+    except Exception:
+        return {}
+
+
+def _save_indiegala_seen(seen):
+    cutoff = datetime.now() - timedelta(days=180)
+    compacted = {
+        key: value for key, value in (seen or {}).items()
+        if (_parse_dt(value) or cutoff) >= cutoff
+    }
+    ModelSetting.set(INDIEGALA_SEEN_SETTING_KEY, json.dumps(compacted, ensure_ascii=False))
+
+
+def _filter_indiegala_items(items):
+    now = datetime.now()
+    cutoff = now - timedelta(days=INDIEGALA_MAX_DISPLAY_DAYS)
+    seen = _load_indiegala_seen()
+    existing = {
+        row.external_id: row
+        for row in F.db.session.query(ModelFreeGameItem).filter_by(platform="indiegala").all()
+    }
+    for external_id, row in existing.items():
+        if external_id and external_id not in seen:
+            first_seen = _parse_dt(row.created_time) or _parse_dt(row.updated_time) or now
+            seen[external_id] = first_seen.isoformat()
+
+    filtered = []
+    for item in items or []:
+        external_id = str(item.get("external_id") or "")
+        if not external_id:
+            continue
+        first_seen = _parse_dt(seen.get(external_id))
+        existing_row = existing.get(external_id)
+        if first_seen:
+            if first_seen < cutoff:
+                continue
+            if existing_row is None:
+                continue
+        else:
+            seen[external_id] = now.isoformat()
+        filtered.append(item)
+
+    _save_indiegala_seen(seen)
+    return filtered
+
+
 def _discord_send(webhook_url, games):
     lines = []
     for game in games[:10]:
+        prefix = "🆕 NEW " if game.get("is_new") else ""
         title = game.get("title") or "Unknown"
         platform = SOURCE_LABELS.get(game.get("platform"), game.get("platform"))
         store_url = game.get("store_url") or ""
         score = int(game.get("metacritic_score") or 0)
-        line = f"**{title}** ({platform})"
+        line = f"{prefix}**{title}** ({platform})"
         if score > 0:
             line += f" · MC {score}"
         if store_url:
@@ -75,11 +139,12 @@ def _discord_send(webhook_url, games):
 def _telegram_send(bot_token, chat_id, games):
     lines = []
     for game in games[:10]:
+        prefix = "🆕 NEW " if game.get("is_new") else ""
         title = game.get("title") or "Unknown"
         platform = SOURCE_LABELS.get(game.get("platform"), game.get("platform"))
         store_url = game.get("store_url") or ""
         score = int(game.get("metacritic_score") or 0)
-        line = f"<b>{title}</b> ({platform})"
+        line = f"{prefix}<b>{title}</b> ({platform})"
         if score > 0:
             line += f" · MC {score}"
         if store_url:
@@ -99,6 +164,8 @@ def _telegram_send(bot_token, chat_id, games):
 
 class Logic(PluginModuleBase):
     db_default = {
+        "main_auto_start": "False",
+        "main_interval": "0 */2 * * *",
         "auto_start": "False",
         "auto_interval": "0 */2 * * *",
         "notify_discord_webhook": "",
@@ -113,21 +180,32 @@ class Logic(PluginModuleBase):
         "source_cheapshark_enabled": "True",
         "last_fetch_started": "",
         "last_fetch_finished": "",
+        "new_flags_initialized": "False",
+        INDIEGALA_SEEN_SETTING_KEY: "{}",
     }
 
     def __init__(self, PM):
-        super().__init__(PM, name="main", first_menu="setting")
+        super().__init__(PM, name="main", first_menu="setting", scheduler_desc="FreeGame fetch")
 
     def plugin_load(self):
+        self._migrate_scheduler_settings()
         ModelFreeGameItem.ensure_schema()
-        if _truthy(ModelSetting.get("auto_start")):
-            self.scheduler_start()
+        if not _truthy(ModelSetting.get("new_flags_initialized")):
+            ModelFreeGameItem.reset_existing_new_flags()
+            ModelSetting.set("new_flags_initialized", "True")
+
+    def _migrate_scheduler_settings(self):
+        legacy_interval = str(ModelSetting.get("auto_interval") or "").strip()
+        if legacy_interval and str(ModelSetting.get("main_interval") or "").strip() in ["", "0 */2 * * *"]:
+            ModelSetting.set("main_interval", legacy_interval)
+        if _truthy(ModelSetting.get("auto_start")) and not _truthy(ModelSetting.get("main_auto_start")):
+            ModelSetting.set("main_auto_start", "True")
 
     def process_menu(self, sub, req):
         arg = ModelSetting.to_dict()
         arg["package_name"] = package_name
-        arg["scheduler"] = str(F.scheduler.is_include(job_id))
-        arg["is_running"] = str(F.scheduler.is_running(job_id))
+        arg["scheduler"] = str(F.scheduler.is_include(self.get_scheduler_name()))
+        arg["is_running"] = str(F.scheduler.is_running(self.get_scheduler_name()))
         arg["source_labels"] = SOURCE_LABELS
         arg["platform_counts"] = ModelFreeGameItem.get_platform_counts()
         if sub == "list":
@@ -140,21 +218,17 @@ class Logic(PluginModuleBase):
         try:
             if sub == "setting_save":
                 ret, _ = ModelSetting.setting_save(req)
-                if F.scheduler.is_include(job_id):
-                    Logic.scheduler_stop()
-                    Logic.scheduler_start()
-                elif _truthy(ModelSetting.get("auto_start")):
-                    Logic.scheduler_start()
+                self.setting_save_after(None)
                 ret["ret"] = "success"
                 return jsonify(ret)
             if sub == "scheduler_toggle":
                 if req.form["scheduler"] == "true":
-                    Logic.scheduler_start()
+                    self.P.logic.scheduler_start(self.name)
                 else:
-                    Logic.scheduler_stop()
+                    self.P.logic.scheduler_stop(self.name)
                 return jsonify({"ret": "success"})
             if sub == "execute_once":
-                threading.Thread(target=Logic.scheduler_function, daemon=True).start()
+                threading.Thread(target=self.scheduler_function, daemon=True).start()
                 return jsonify({"ret": "success"})
             if sub == "web_list":
                 ModelFreeGameItem.ensure_schema()
@@ -167,29 +241,14 @@ class Logic(PluginModuleBase):
             logger.error(traceback.format_exc())
             return jsonify({"ret": "error", "log": str(e)})
 
-    @staticmethod
-    def scheduler_start():
-        try:
-            interval = ModelSetting.get("auto_interval") or "0 */2 * * *"
-            if F.scheduler.is_include(job_id):
-                scheduler.remove_job(job_id)
-            job = Job(package_name, job_id, interval, Logic.scheduler_function, "FreeGame fetch", True)
-            scheduler.add_job_instance(job)
-            logger.info("FreeGame scheduler registered: id=%s interval=%s", job_id, interval)
-        except Exception as e:
-            logger.error("Exception:%s", e)
-            logger.error(traceback.format_exc())
+    def setting_save_after(self, change_list):
+        if F.scheduler.is_include(self.get_scheduler_name()):
+            self.P.logic.scheduler_stop(self.name)
+            self.P.logic.scheduler_start(self.name)
+        elif _truthy(ModelSetting.get("main_auto_start")):
+            self.P.logic.scheduler_start(self.name)
 
-    @staticmethod
-    def scheduler_stop():
-        try:
-            scheduler.remove_job(job_id)
-        except Exception as e:
-            logger.error("Exception:%s", e)
-            logger.error(traceback.format_exc())
-
-    @staticmethod
-    def scheduler_function():
+    def scheduler_function(self):
         if not _fetch_lock.acquire(blocking=False):
             logger.info("FreeGame fetch skipped: already running")
             return
@@ -201,6 +260,7 @@ class Logic(PluginModuleBase):
                 enabled_sources = set(_enabled_sources())
                 results = scraper.fetch_all()
                 grouped = _split_source_payload(results)
+                grouped["indiegala"] = _filter_indiegala_items(grouped.get("indiegala", []))
                 fresh_free_games = []
 
                 for legacy_source in ["humble", "fanatical", "gmg", "directgames"]:
@@ -209,6 +269,14 @@ class Logic(PluginModuleBase):
                 for source, items in grouped.items():
                     if source not in enabled_sources:
                         continue
+                    existing_ids = {
+                        row[0]
+                        for row in F.db.session.query(ModelFreeGameItem.external_id)
+                        .filter_by(platform=source)
+                        .all()
+                    }
+                    for item in items:
+                        item["is_new"] = str(item.get("external_id") or "") not in existing_ids
                     ModelFreeGameItem.replace_source_items(source, items)
                     ModelFetchLog(source, "ok", "", len(items)).save()
                     logger.info("FreeGame source=%s saved=%d", source, len(items))
@@ -221,7 +289,7 @@ class Logic(PluginModuleBase):
                 logger.info("FreeGame fetch completed: free_candidates=%d enabled_sources=%d", len(fresh_free_games), len(enabled_sources))
                 ModelFetchLog("all", "ok", f"free_candidates={len(fresh_free_games)} enabled_sources={len(enabled_sources)}", len(fresh_free_games)).save()
                 ModelSetting.set("last_fetch_finished", datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
-                Logic._notify(fresh_free_games)
+                self._notify(fresh_free_games)
         except Exception as e:
             logger.error("Exception:%s", e)
             logger.error(traceback.format_exc())
@@ -229,8 +297,7 @@ class Logic(PluginModuleBase):
         finally:
             _fetch_lock.release()
 
-    @staticmethod
-    def _notify(games):
+    def _notify(self, games):
         if _truthy(ModelSetting.get("notify_enabled")) is False:
             return
         targets = list(games or [])

model.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import json
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from sqlalchemy import desc
 from sqlalchemy import text
@@ -49,6 +49,7 @@ class ModelFreeGameItem(ModelBase):
             return []
 
     def as_dict(self):
+        is_new = bool(self.created_time and self.created_time >= datetime.now() - timedelta(hours=48))
         return {
             "id": self.id,
             "external_id": self.external_id,
@@ -66,6 +67,8 @@ class ModelFreeGameItem(ModelBase):
             "metacritic_score": self.metacritic_score,
             "metacritic_url": self.metacritic_url,
             "genres": self.genres,
+            "is_new": is_new,
+            "created_time": self.created_time.strftime("%Y-%m-%d %H:%M:%S") if self.created_time else "",
             "updated_time": self.updated_time.strftime("%Y-%m-%d %H:%M:%S") if self.updated_time else "",
         }
 
@@ -114,6 +117,25 @@ class ModelFreeGameItem(ModelBase):
             except Exception:
                 P.logger.exception("ff_freegame schema migration failed")
 
+    @classmethod
+    def reset_existing_new_flags(cls):
+        with F.app.app_context():
+            try:
+                cutoff = datetime.now() - timedelta(hours=49)
+                updated = (
+                    F.db.session.query(cls)
+                    .filter((cls.created_time == None) | (cls.created_time > cutoff))
+                    .update({cls.created_time: cutoff}, synchronize_session=False)
+                )
+                F.db.session.commit()
+                if updated:
+                    P.logger.info("ff_freegame reset existing NEW flags: %d", updated)
+                return updated
+            except Exception:
+                F.db.session.rollback()
+                P.logger.exception("ff_freegame reset existing NEW flags failed")
+                return 0
+
     @classmethod
     def delete_not_in_sources(cls, sources):
         with F.app.app_context():
@@ -127,8 +149,16 @@ class ModelFreeGameItem(ModelBase):
     @classmethod
     def replace_source_items(cls, source_name, items):
         with F.app.app_context():
-            F.db.session.query(cls).filter_by(platform=source_name).delete(synchronize_session=False)
-            F.db.session.commit()
+            incoming_ids = {
+                str(item.get("external_id") or "")
+                for item in items
+                if str(item.get("external_id") or "")
+            }
+            query = F.db.session.query(cls).filter_by(platform=source_name)
+            if incoming_ids:
+                query.filter(~cls.external_id.in_(incoming_ids)).delete(synchronize_session=False)
+            else:
+                query.delete(synchronize_session=False)
             for item in items:
                 cls.upsert(item)
             F.db.session.commit()

scraper.py

@@ -743,7 +743,9 @@ def fetch_stove_free():
         return "https://store.onstove.com/ko/games/" + str(game_id)
 
     def _build_game(data):
-        game_id = _pick(data, ["productNo", "productId", "product_id"])
+        if isinstance(data.get("product"), dict):
+            data = data["product"]
+        game_id = _pick(data, ["productNo", "productId", "product_id", "product_no", "game_no"])
         if game_id in (None, ""):
             return None
         game_id = str(game_id).strip()
@@ -754,12 +756,13 @@ def fetch_stove_free():
         if _DEMO_TITLE_RE.search(title):
             return None
 
-        current_price = _to_float(_pick(data, ["salePrice", "discountPrice", "currentPrice", "finalPrice", "price", "sellingPrice", "sale_price"]))
-        original_price = _to_float(_pick(data, ["originPrice", "originalPrice", "listPrice", "basePrice", "priceBeforeDiscount", "normalPrice", "origin_price"]))
+        amount = data.get("amount") if isinstance(data.get("amount"), dict) else {}
+        current_price = _to_float(_pick(data, ["salePrice", "discountPrice", "currentPrice", "finalPrice", "price", "sellingPrice", "sale_price"]) or amount.get("sales_price"))
+        original_price = _to_float(_pick(data, ["originPrice", "originalPrice", "listPrice", "basePrice", "priceBeforeDiscount", "normalPrice", "origin_price"]) or amount.get("original_price"))
         if original_price <= 0.0 or current_price != 0.0:
             return None
 
-        image_url = _pick(data, ["imageUrl", "image_url", "thumbnailUrl", "thumbnail_image_url", "verticalImageUrl", "horizontalImageUrl", "coverImageUrl"]) or ""
+        image_url = _pick(data, ["imageUrl", "image_url", "thumbnailUrl", "thumbnail_image_url", "verticalImageUrl", "horizontalImageUrl", "coverImageUrl", "title_image_square", "title_image_rectangle"]) or ""
         return {
             "external_id": "stove_" + game_id,
             "platform": "stove",
@@ -768,7 +771,7 @@ def fetch_stove_free():
             "store_url": _store_url(game_id, data),
             "original_price": original_price,
             "current_price": 0.0,
-            "discount_pct": 100,
+            "discount_pct": int(_to_float(amount.get("discount_rate")) or 100),
             "is_free_period": True,
             "free_start": None,
             "free_end": None,
@@ -853,6 +856,23 @@ def fetch_stove_free():
             games.append(game)
         return games
 
+    def _resolve_nuxt_refs(data, value, stack=None):
+        if stack is None:
+            stack = set()
+        if isinstance(value, int) and not isinstance(value, bool):
+            if value < 0 or value >= len(data) or value in stack:
+                return None
+            target = data[value]
+            if isinstance(target, (dict, list)):
+                stack.add(value)
+                return _resolve_nuxt_refs(data, target, stack)
+            return target
+        if isinstance(value, dict):
+            return {key: _resolve_nuxt_refs(data, item, set(stack)) for key, item in value.items()}
+        if isinstance(value, list):
+            return [_resolve_nuxt_refs(data, item, set(stack)) for item in value]
+        return value
+
     def _extract_json_from_html(html):
         payloads = []
         for match in re.finditer(r'<script[^>]+type=["\']application/json["\'][^>]*>(.*?)</script>', html, re.I | re.S):
@@ -860,7 +880,15 @@ def fetch_stove_free():
             if not raw:
                 continue
             try:
-                payloads.append(json.loads(raw))
+                payload = json.loads(raw)
+                if 'id="__NUXT_DATA__"' in match.group(0) and isinstance(payload, list):
+                    payloads.append([
+                        _resolve_nuxt_refs(payload, item)
+                        for item in payload
+                        if isinstance(item, dict)
+                    ])
+                else:
+                    payloads.append(payload)
             except Exception:
                 pass
         for pattern in [r"__NUXT__\s*=\s*(\{.*?\})\s*</script>", r"window\.__STORE__\s*=\s*(\{.*?\})\s*;"]:
@@ -881,14 +909,11 @@ def fetch_stove_free():
 
     api_headers = dict(_HEADERS, Referer="https://store.onstove.com/", Accept="application/json")
     html_headers = dict(_HTML_HEADERS, Referer="https://store.onstove.com/")
-    api_urls = [
-        "https://store.onstove.com/api/v2/product/list?product_type=GAME&price_type=FREE&page=1&size=20",
-        "https://api.onstove.com/store/v2/product/list?product_type=GAME&price_type=FREE",
-        "https://store.onstove.com/api/store/v2/event/freegame",
-    ]
+    api_urls = []
     page_urls = [
+        "https://store.onstove.com/ko/store/Discount_Mall",
+        "https://store.onstove.com/ko/store/stoveindie",
         "https://store.onstove.com/ko/games?priceFilter=FREE",
-        "https://store.onstove.com/ko/promotions",
     ]
     results = []
     seen = set()

templates/ff_freegame_main_list.html

@@ -83,6 +83,20 @@
     background: #16a34a;
     color: #fff;
   }
+  .ffg-badge-new {
+    top: auto;
+    left: 10px;
+    bottom: 10px;
+    display: inline-flex;
+    align-items: center;
+    justify-content: center;
+    min-width: 42px;
+    min-height: 24px;
+    padding: 5px 9px;
+    background: #ef4444;
+    color: #fff;
+    box-shadow: 0 0 0 2px rgba(0, 0, 0, .18);
+  }
   .ffg-platform-epic { background: #2f313a; }
   .ffg-platform-steam { background: #1b4b91; color: #fff; }
   .ffg-platform-gog { background: #4a2876; }
@@ -249,6 +263,9 @@ function gameCard(item) {
   const mcHtml = mcScore > 0
     ? `<span class="ffg-mc" title="Metacritic score">MC ${Math.round(mcScore)}</span>`
     : "";
+  const newHtml = item.is_new
+    ? `<span class="ffg-badge ffg-badge-new">NEW</span>`
+    : "";
 
   return `
     <article class="ffg-card" data-url="${escapeHtml(item.store_url || "")}">
@@ -256,6 +273,7 @@ function gameCard(item) {
         ${imageHtml}
         <span class="ffg-badge ffg-badge-platform ${platformClass(item.platform)}">${platformLabel(item.platform)}</span>
         <span class="ffg-badge ffg-badge-free">FREE</span>
+        ${newHtml}
       </div>
       <div class="ffg-body">
         <div class="ffg-title" title="${escapeHtml(item.title)}">${escapeHtml(item.title)}</div>

templates/ff_freegame_main_setting.html

@@ -3,8 +3,8 @@
 <div>
   {{ macros.m_button_group([['globalSettingSaveBtn', '설정 저장']]) }}
   <form id="setting">
-    {{ macros.setting_input_text('auto_interval', '수집 주기', value=arg.get('auto_interval', '0 */2 * * *'), desc=['기본값: 2시간마다', '크론 표현식 사용']) }}
-    {{ macros.setting_checkbox('auto_start', '시작 시 자동 스케줄 등록', value=arg.get('auto_start', 'False')) }}
+    {{ macros.setting_input_text('main_interval', '수집 주기', value=arg.get('main_interval', '0 */2 * * *'), desc=['기본값: 2시간마다', '크론 표현식 사용']) }}
+    {{ macros.setting_checkbox('main_auto_start', '시작 시 자동 스케줄 등록', value=arg.get('main_auto_start', 'False')) }}
     {{ macros.global_setting_scheduler_button(arg['scheduler'], arg['is_running']) }}
     {{ macros.setting_buttons([['ff_freegame_execute_btn', '1회 실행']], left='수동 실행') }}
     {{ macros.info_text('last_fetch_started', '최근 수집 시작', value=arg.get('last_fetch_started', '')) }}
@@ -15,7 +15,7 @@
     {{ macros.setting_input_text('notify_telegram_bot_token', 'Telegram Bot Token', value=arg.get('notify_telegram_bot_token', '')) }}
     {{ macros.setting_input_text('notify_telegram_chat_id', 'Telegram Chat ID', value=arg.get('notify_telegram_chat_id', '')) }}
     {{ macros.m_hr() }}
-    {{ macros.info_text('source_help', '활성 소스', value='체크된 무료 소스만 저장/표시됩니다.') }}
+    {{ macros.info_text('source_help', '활성 소스', value='체크한 무료 소스만 저장/표시합니다.') }}
     {{ macros.setting_checkbox('source_epic_enabled', 'Epic', value=arg.get('source_epic_enabled', 'True')) }}
     {{ macros.setting_checkbox('source_steam_enabled', 'Steam', value=arg.get('source_steam_enabled', 'True')) }}
     {{ macros.setting_checkbox('source_gog_enabled', 'GOG', value=arg.get('source_gog_enabled', 'True')) }}
@@ -30,9 +30,9 @@ const package_name = "{{ arg['package_name'] }}";
 
 $("body").on("change", "#globalSchedulerSwitchBtn", function () {
   $.ajax({
-    url: `/${package_name}/ajax/scheduler_toggle`,
+    url: `/${package_name}/ajax/scheduler`,
     type: "POST",
-    data: { scheduler: $(this).prop("checked") },
+    data: { sub: "main", scheduler: $(this).prop("checked") },
     dataType: "json",
   });
 });
@@ -40,8 +40,9 @@ $("body").on("change", "#globalSchedulerSwitchBtn", function () {
 $("body").on("click", "#ff_freegame_execute_btn", function (e) {
   e.preventDefault();
   $.ajax({
-    url: `/${package_name}/ajax/execute_once`,
+    url: `/${package_name}/ajax/one_execute`,
     type: "POST",
+    data: { sub: "main" },
     dataType: "json",
     success: function (ret) {
       if (ret.ret === "success") {