Store every date in the database as an Instant

This commit is contained in:
AntsyLich
2026-10-10 01:13:08 +06:00
parent 8972882a34
commit 44c1e1915d
68 changed files with 599 additions and 284 deletions
@@ -1,4 +1,5 @@
import kotlin.Boolean;
import kotlin.time.Instant;
import kotlinx.serialization.json.JsonObject;
CREATE TABLE chapter(
@@ -8,13 +9,13 @@ CREATE TABLE chapter(
remote_name TEXT NOT NULL,
remote_scanlator TEXT,
remote_chapter_number REAL NOT NULL,
remote_date_upload INTEGER NOT NULL,
remote_date_upload INTEGER AS Instant,
remote_order INTEGER NOT NULL,
remote_memo BLOB AS JsonObject NOT NULL,
user_read INTEGER AS Boolean NOT NULL,
user_bookmark INTEGER AS Boolean NOT NULL,
user_last_page_read INTEGER NOT NULL,
state_date_fetch INTEGER NOT NULL,
state_date_fetch INTEGER AS Instant NOT NULL,
UNIQUE(manga_id, remote_url),
FOREIGN KEY(manga_id) REFERENCES manga (id)
ON DELETE CASCADE
@@ -1,10 +1,10 @@
import java.util.Date;
import kotlin.time.Instant;
CREATE TABLE history(
id INTEGER NOT NULL PRIMARY KEY,
manga_id INTEGER NOT NULL,
chapter_id INTEGER NOT NULL UNIQUE,
read_at INTEGER AS Date,
read_at INTEGER AS Instant,
read_duration INTEGER NOT NULL,
FOREIGN KEY(manga_id) REFERENCES manga (id)
ON DELETE CASCADE,
@@ -22,12 +22,12 @@ WHERE manga_id = :mangaId;
resetHistoryById:
UPDATE history
SET read_at = 0
SET read_at = NULL
WHERE id = :id;
resetHistoryByMangaId:
UPDATE history
SET read_at = 0
SET read_at = NULL
WHERE manga_id = :mangaId;
removeAllHistory:
@@ -35,7 +35,7 @@ DELETE FROM history;
removeResettedHistory:
DELETE FROM history
WHERE read_at = 0;
WHERE read_at IS NULL;
upsert:
INSERT INTO history(manga_id, chapter_id, read_at, read_duration)
@@ -2,6 +2,7 @@ import eu.kanade.tachiyomi.source.model.UpdateStrategy;
import kotlin.Boolean;
import kotlin.String;
import kotlin.collections.List;
import kotlin.time.Instant;
import kotlinx.serialization.json.JsonObject;
CREATE TABLE manga(
@@ -17,14 +18,14 @@ CREATE TABLE manga(
remote_cover TEXT,
remote_update_strategy INTEGER AS UpdateStrategy NOT NULL,
remote_memo BLOB AS JsonObject NOT NULL,
user_favorite_at INTEGER,
user_favorite_at INTEGER AS Instant,
user_notes TEXT NOT NULL,
user_reader_flags INTEGER NOT NULL,
user_chapter_flags INTEGER NOT NULL,
state_chapter_last_update INTEGER,
state_chapter_next_update INTEGER,
state_chapter_last_update INTEGER AS Instant,
state_chapter_next_update INTEGER AS Instant,
state_chapter_fetch_interval INTEGER NOT NULL,
state_cover_last_modified INTEGER NOT NULL,
state_cover_last_modified INTEGER AS Instant,
state_initialized INTEGER AS Boolean NOT NULL,
UNIQUE(source_id, remote_url)
);
@@ -1,4 +1,5 @@
import kotlin.Boolean;
import kotlin.time.Instant;
CREATE TABLE manga_track(
id INTEGER NOT NULL PRIMARY KEY,
@@ -12,8 +13,8 @@ CREATE TABLE manga_track(
total_chapters INTEGER NOT NULL,
status INTEGER NOT NULL,
score REAL NOT NULL,
start_date INTEGER NOT NULL,
finish_date INTEGER NOT NULL,
start_date INTEGER AS Instant,
finish_date INTEGER AS Instant,
private INTEGER AS Boolean NOT NULL DEFAULT 0,
UNIQUE (manga_id, tracker_id),
FOREIGN KEY(manga_id) REFERENCES manga (id)
@@ -0,0 +1,271 @@
import eu.kanade.tachiyomi.source.model.UpdateStrategy;
import kotlin.Boolean;
import kotlin.String;
import kotlin.collections.List;
import kotlin.time.Instant;
import kotlinx.serialization.json.JsonObject;
-- A time that isn't known was stored as 0, which reads as the start of 1970 once it is an instant, so
-- it is NULL instead. SQLite cannot drop NOT NULL from a column, so the tables holding one are rebuilt.
-- Every view reads `manga` or `chapter`, so they are dropped here and recreated against the new tables below
DROP VIEW historyView;
DROP VIEW libraryView;
DROP VIEW updatesView;
-- Each table is renamed aside and rebuilt under its own name, as in migration 17. Renaming a table also
-- repoints the foreign keys of the tables that reference it, which would leave them on the copy that is
-- dropped; the legacy behavior leaves them naming the table that takes its place.
PRAGMA legacy_alter_table = 1;
-- ---------------------------------------------------------------------------------------------------------
-- table - manga
-- ---------------------------------------------------------------------------------------------------------
ALTER TABLE manga RENAME TO manga_temp;
CREATE TABLE manga(
id INTEGER NOT NULL PRIMARY KEY,
source_id INTEGER NOT NULL,
remote_url TEXT NOT NULL,
remote_title TEXT NOT NULL,
remote_author TEXT,
remote_artist TEXT,
remote_description TEXT,
remote_genre TEXT AS List<String>,
remote_status INTEGER NOT NULL,
remote_cover TEXT,
remote_update_strategy INTEGER AS UpdateStrategy NOT NULL,
remote_memo BLOB AS JsonObject NOT NULL,
user_favorite_at INTEGER AS Instant,
user_notes TEXT NOT NULL,
user_reader_flags INTEGER NOT NULL,
user_chapter_flags INTEGER NOT NULL,
state_chapter_last_update INTEGER AS Instant,
state_chapter_next_update INTEGER AS Instant,
state_chapter_fetch_interval INTEGER NOT NULL,
state_cover_last_modified INTEGER AS Instant,
state_initialized INTEGER AS Boolean NOT NULL,
UNIQUE(source_id, remote_url)
);
-- `user_favorite_at` keeps its 0: NULL already means the entry isn't in the library, and 0 is an entry
-- added before the time was recorded
INSERT INTO manga(
id, source_id, remote_url, remote_title, remote_author, remote_artist, remote_description, remote_genre,
remote_status, remote_cover, remote_update_strategy, remote_memo, user_favorite_at, user_notes,
user_reader_flags, user_chapter_flags, state_chapter_last_update, state_chapter_next_update,
state_chapter_fetch_interval, state_cover_last_modified, state_initialized
)
SELECT
id, source_id, remote_url, remote_title, remote_author, remote_artist, remote_description, remote_genre,
remote_status, remote_cover, remote_update_strategy, remote_memo, user_favorite_at, user_notes,
user_reader_flags, user_chapter_flags, nullif(state_chapter_last_update, 0), nullif(state_chapter_next_update, 0),
state_chapter_fetch_interval, nullif(state_cover_last_modified, 0), state_initialized
FROM manga_temp;
DROP TABLE manga_temp;
CREATE INDEX manga_user_favorite_at_index ON manga(user_favorite_at) WHERE user_favorite_at IS NOT NULL;
-- ---------------------------------------------------------------------------------------------------------
-- table - chapter
-- ---------------------------------------------------------------------------------------------------------
ALTER TABLE chapter RENAME TO chapter_temp;
CREATE TABLE chapter(
id INTEGER NOT NULL PRIMARY KEY,
manga_id INTEGER NOT NULL,
remote_url TEXT NOT NULL,
remote_name TEXT NOT NULL,
remote_scanlator TEXT,
remote_chapter_number REAL NOT NULL,
remote_date_upload INTEGER AS Instant,
remote_order INTEGER NOT NULL,
remote_memo BLOB AS JsonObject NOT NULL,
user_read INTEGER AS Boolean NOT NULL,
user_bookmark INTEGER AS Boolean NOT NULL,
user_last_page_read INTEGER NOT NULL,
state_date_fetch INTEGER AS Instant NOT NULL,
UNIQUE(manga_id, remote_url),
FOREIGN KEY(manga_id) REFERENCES manga (id)
ON DELETE CASCADE
);
INSERT INTO chapter(
id, manga_id, remote_url, remote_name, remote_scanlator, remote_chapter_number, remote_date_upload,
remote_order, remote_memo, user_read, user_bookmark, user_last_page_read, state_date_fetch
)
SELECT
id, manga_id, remote_url, remote_name, remote_scanlator, remote_chapter_number, nullif(remote_date_upload, 0),
remote_order, remote_memo, user_read, user_bookmark, user_last_page_read, state_date_fetch
FROM chapter_temp;
DROP TABLE chapter_temp;
CREATE INDEX chapter_state_date_fetch_index ON chapter(state_date_fetch);
CREATE INDEX chapter_unread_state_date_fetch_index ON chapter(state_date_fetch) WHERE user_read = 0;
-- ---------------------------------------------------------------------------------------------------------
-- table - manga_track
-- ---------------------------------------------------------------------------------------------------------
ALTER TABLE manga_track RENAME TO manga_track_temp;
CREATE TABLE manga_track(
id INTEGER NOT NULL PRIMARY KEY,
manga_id INTEGER NOT NULL,
tracker_id INTEGER NOT NULL,
remote_id INTEGER NOT NULL,
remote_url TEXT NOT NULL,
library_id INTEGER,
title TEXT NOT NULL,
last_chapter_read REAL NOT NULL,
total_chapters INTEGER NOT NULL,
status INTEGER NOT NULL,
score REAL NOT NULL,
start_date INTEGER AS Instant,
finish_date INTEGER AS Instant,
private INTEGER AS Boolean NOT NULL DEFAULT 0,
UNIQUE (manga_id, tracker_id),
FOREIGN KEY(manga_id) REFERENCES manga (id)
ON DELETE CASCADE
);
INSERT INTO manga_track(
id, manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read, total_chapters,
status, score, start_date, finish_date, private
)
SELECT
id, manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read, total_chapters,
status, score, nullif(start_date, 0), nullif(finish_date, 0), private
FROM manga_track_temp;
DROP TABLE manga_track_temp;
CREATE INDEX manga_track_tracker_id_remote_id_index ON manga_track(tracker_id, remote_id);
PRAGMA legacy_alter_table = 0;
-- ---------------------------------------------------------------------------------------------------------
-- table - history
-- ---------------------------------------------------------------------------------------------------------
-- 0 marked history the user removed, which NULL marks now
UPDATE history
SET read_at = NULL
WHERE read_at = 0;
-- ---------------------------------------------------------------------------------------------------------
-- views
-- ---------------------------------------------------------------------------------------------------------
CREATE VIEW historyView AS
SELECT
history.id AS id,
manga.id AS mangaId,
chapter.id AS chapterId,
manga.remote_title AS title,
manga.remote_cover AS thumbnailUrl,
manga.source_id AS sourceId,
manga.user_favorite_at IS NOT NULL AS favorite,
manga.state_cover_last_modified AS coverLastModified,
chapter.remote_chapter_number AS chapterNumber,
history.read_at AS readAt,
history.read_duration AS readDuration,
max_read.read_at AS maxReadAt,
max_read.chapter_id AS maxReadAtChapterId
FROM manga
JOIN chapter
ON manga.id = chapter.manga_id
JOIN history
ON chapter.id = history.chapter_id
JOIN (
SELECT manga_id, chapter_id, MAX(read_at) AS read_at
FROM history
GROUP BY manga_id
) AS max_read
ON chapter.manga_id = max_read.manga_id;
-- An entry without chapters or reading history has no latest upload, fetch or read time, which is NULL
-- rather than 0 now
CREATE VIEW libraryView AS
SELECT
M.*,
coalesce(C.total, 0) AS totalCount,
coalesce(C.readCount, 0) AS readCount,
C.latestUpload AS latestUpload,
C.fetchedAt AS chapterFetchedAt,
-- Joining history to every chapter is the slowest part of the query, and only entries with excluded
-- scanlators need the chapters to leave theirs out
CASE WHEN EXISTS (SELECT 0 FROM excluded_scanlator WHERE excluded_scanlator.manga_id = M.id)
THEN (
SELECT max(history.read_at)
FROM history
JOIN chapter
ON chapter.id = history.chapter_id
WHERE history.manga_id = M.id
AND NOT EXISTS (
SELECT 0 FROM excluded_scanlator
WHERE excluded_scanlator.manga_id = M.id
AND excluded_scanlator.scanlator = chapter.remote_scanlator
)
)
ELSE (SELECT max(history.read_at) FROM history WHERE history.manga_id = M.id)
END AS lastRead,
coalesce(C.bookmarkCount, 0) AS bookmarkCount,
coalesce(MC.categories, '0') AS categories
FROM manga M
LEFT JOIN (
SELECT
chapter.manga_id,
count(*) AS total,
sum(user_read) AS readCount,
max(chapter.remote_date_upload) AS latestUpload,
max(chapter.state_date_fetch) AS fetchedAt,
sum(chapter.user_bookmark) AS bookmarkCount
FROM chapter
LEFT JOIN excluded_scanlator
ON chapter.manga_id = excluded_scanlator.manga_id
AND chapter.remote_scanlator = excluded_scanlator.scanlator
WHERE excluded_scanlator.scanlator IS NULL
AND chapter.manga_id IN (SELECT id FROM manga WHERE user_favorite_at IS NOT NULL)
GROUP BY chapter.manga_id
) AS C
ON M.id = C.manga_id
LEFT JOIN (
SELECT manga_id, group_concat(category_id) AS categories
FROM manga_category
WHERE manga_id IN (SELECT id FROM manga WHERE user_favorite_at IS NOT NULL)
GROUP BY manga_id
) AS MC
ON MC.manga_id = M.id
WHERE M.user_favorite_at IS NOT NULL;
CREATE VIEW updatesView AS
SELECT
manga.id AS mangaId,
manga.remote_title AS mangaTitle,
chapter.id AS chapterId,
chapter.remote_name AS chapterName,
chapter.remote_scanlator AS scanlator,
chapter.remote_url AS chapterUrl,
chapter.user_read AS read,
chapter.user_bookmark AS bookmark,
chapter.user_last_page_read AS lastPageRead,
manga.source_id AS sourceId,
manga.user_favorite_at IS NOT NULL AS favorite,
manga.remote_cover AS thumbnailUrl,
manga.state_cover_last_modified AS coverLastModified,
chapter.remote_date_upload AS dateUpload,
chapter.state_date_fetch AS dateFetch,
excluded_scanlator.scanlator AS excludedScanlator
FROM manga JOIN chapter
ON manga.id = chapter.manga_id
LEFT JOIN excluded_scanlator
ON manga.id = excluded_scanlator.manga_id
AND chapter.remote_scanlator = excluded_scanlator.scanlator
WHERE user_favorite_at IS NOT NULL
AND state_date_fetch > user_favorite_at
ORDER BY state_date_fetch DESC;
@@ -39,7 +39,7 @@ chapterNumber,
readAt,
readDuration
FROM historyView
WHERE historyView.readAt > 0
WHERE historyView.readAt IS NOT NULL
AND maxReadAtChapterId = historyView.chapterId
AND lower(historyView.title) LIKE ('%' || :query || '%')
ORDER BY readAt DESC;
@@ -62,6 +62,6 @@ JOIN manga
ON manga.id = history.manga_id
JOIN chapter
ON chapter.id = history.chapter_id
WHERE history.read_at > 0
WHERE history.read_at IS NOT NULL
ORDER BY history.read_at DESC
LIMIT 1;
@@ -3,28 +3,25 @@ SELECT
M.*,
coalesce(C.total, 0) AS totalCount,
coalesce(C.readCount, 0) AS readCount,
coalesce(C.latestUpload, 0) AS latestUpload,
coalesce(C.fetchedAt, 0) AS chapterFetchedAt,
C.latestUpload AS latestUpload,
C.fetchedAt AS chapterFetchedAt,
-- Joining history to every chapter is the slowest part of the query, and only entries with excluded
-- scanlators need the chapters to leave theirs out
coalesce(
CASE WHEN EXISTS (SELECT 0 FROM excluded_scanlator WHERE excluded_scanlator.manga_id = M.id)
THEN (
SELECT max(history.read_at)
FROM history
JOIN chapter
ON chapter.id = history.chapter_id
WHERE history.manga_id = M.id
AND NOT EXISTS (
SELECT 0 FROM excluded_scanlator
WHERE excluded_scanlator.manga_id = M.id
AND excluded_scanlator.scanlator = chapter.remote_scanlator
)
CASE WHEN EXISTS (SELECT 0 FROM excluded_scanlator WHERE excluded_scanlator.manga_id = M.id)
THEN (
SELECT max(history.read_at)
FROM history
JOIN chapter
ON chapter.id = history.chapter_id
WHERE history.manga_id = M.id
AND NOT EXISTS (
SELECT 0 FROM excluded_scanlator
WHERE excluded_scanlator.manga_id = M.id
AND excluded_scanlator.scanlator = chapter.remote_scanlator
)
ELSE (SELECT max(history.read_at) FROM history WHERE history.manga_id = M.id)
END,
0
) AS lastRead,
)
ELSE (SELECT max(history.read_at) FROM history WHERE history.manga_id = M.id)
END AS lastRead,
coalesce(C.bookmarkCount, 0) AS bookmarkCount,
coalesce(MC.categories, '0') AS categories
FROM manga M
@@ -33,8 +30,8 @@ LEFT JOIN (
chapter.manga_id,
count(*) AS total,
sum(user_read) AS readCount,
coalesce(max(chapter.remote_date_upload), 0) AS latestUpload,
coalesce(max(chapter.state_date_fetch), 0) AS fetchedAt,
max(chapter.remote_date_upload) AS latestUpload,
max(chapter.state_date_fetch) AS fetchedAt,
sum(chapter.user_bookmark) AS bookmarkCount
FROM chapter
LEFT JOIN excluded_scanlator