import eu.kanade.tachiyomi.source.model.UpdateStrategy; import kotlin.Boolean; import kotlin.String; import kotlin.collections.List; import kotlinx.serialization.json.JsonObject; -- --------------------------------------------------------------------------------------------------------- -- table - manga -- --------------------------------------------------------------------------------------------------------- -- Every view reads `mangas`, so they are dropped here and recreated against the new table below DROP VIEW historyView; DROP VIEW libraryView; DROP VIEW updatesView; -- Entries sharing a source and url are merged into one, since the new table won't hold both. The copies are -- ranked with the ones in the library first, since that is the entry the user has, then by age. The first -- one stays, takes each detail it is missing from the first copy that has it, and takes over everything -- attached to the others. CREATE TEMP TABLE duplicate_mangas AS SELECT _id, first_value(_id) OVER (PARTITION BY source, url ORDER BY favorite_at IS NOT NULL DESC, _id ASC) AS keep_id, row_number() OVER (PARTITION BY source, url ORDER BY favorite_at IS NOT NULL DESC, _id ASC) AS precedence, favorite_at IS NOT NULL AS favorite FROM mangas WHERE (source, url) IN ( SELECT source, url FROM mangas GROUP BY source, url HAVING count(*) > 1 ); UPDATE mangas SET favorite_at = merged.favorite_at, initialized = merged.initialized, artist = merged.artist, author = merged.author, description = merged.description, genre = merged.genre, thumbnail_url = merged.thumbnail_url, status = merged.status, viewer = merged.viewer, chapter_flags = merged.chapter_flags, notes = merged.notes, memo = merged.memo, last_update = merged.last_update, cover_last_modified = merged.cover_last_modified FROM ( SELECT DISTINCT D.keep_id, -- The earliest real date if any copy has one, else 0 if any copy is in the library, else NULL coalesce( min(nullif(M.favorite_at, 0)) OVER (PARTITION BY D.keep_id), min(M.favorite_at) OVER (PARTITION BY D.keep_id) ) AS favorite_at, max(M.initialized) OVER (PARTITION BY D.keep_id) AS initialized, first_value(M.artist) OVER (PARTITION BY D.keep_id ORDER BY M.artist IS NULL, D.precedence) AS artist, first_value(M.author) OVER (PARTITION BY D.keep_id ORDER BY M.author IS NULL, D.precedence) AS author, first_value(M.description) OVER (PARTITION BY D.keep_id ORDER BY M.description IS NULL, D.precedence) AS description, first_value(M.genre) OVER (PARTITION BY D.keep_id ORDER BY M.genre IS NULL, D.precedence) AS genre, first_value(M.thumbnail_url) OVER (PARTITION BY D.keep_id ORDER BY M.thumbnail_url IS NULL, D.precedence) AS thumbnail_url, first_value(M.status) OVER (PARTITION BY D.keep_id ORDER BY M.status = 0, D.precedence) AS status, first_value(M.viewer) OVER (PARTITION BY D.keep_id ORDER BY M.viewer = 0, D.precedence) AS viewer, first_value(M.chapter_flags) OVER (PARTITION BY D.keep_id ORDER BY M.chapter_flags = 0, D.precedence) AS chapter_flags, first_value(M.notes) OVER (PARTITION BY D.keep_id ORDER BY M.notes = '', D.precedence) AS notes, first_value(M.memo) OVER (PARTITION BY D.keep_id ORDER BY M.memo = '{}', D.precedence) AS memo, max(M.last_update) OVER (PARTITION BY D.keep_id) AS last_update, max(M.cover_last_modified) OVER (PARTITION BY D.keep_id) AS cover_last_modified FROM duplicate_mangas D JOIN mangas M ON M._id = D._id ) AS merged WHERE mangas._id = merged.keep_id; -- Every child table is rebuilt further down and resolves the surviving entry there, so nothing -- else has to be rewritten here first. DELETE FROM mangas WHERE _id IN (SELECT _id FROM duplicate_mangas WHERE _id != keep_id); CREATE TABLE manga( id INTEGER NOT NULL PRIMARY KEY, source_id INTEGER NOT NULL, remote_url TEXT NOT NULL, remote_title TEXT NOT NULL, remote_author TEXT, remote_artist TEXT, remote_description TEXT, remote_genre TEXT AS List, remote_status INTEGER NOT NULL, remote_cover TEXT, remote_update_strategy INTEGER AS UpdateStrategy NOT NULL, remote_memo BLOB AS JsonObject NOT NULL, user_favorite_at INTEGER, user_notes TEXT NOT NULL, user_reader_flags INTEGER NOT NULL, user_chapter_flags INTEGER NOT NULL, state_chapter_last_update INTEGER, state_chapter_next_update INTEGER, state_chapter_fetch_interval INTEGER NOT NULL, state_cover_last_modified INTEGER NOT NULL, state_initialized INTEGER AS Boolean NOT NULL, UNIQUE(source_id, remote_url) ); INSERT INTO manga( id, source_id, remote_url, remote_title, remote_author, remote_artist, remote_description, remote_genre, remote_status, remote_cover, remote_update_strategy, remote_memo, user_favorite_at, user_notes, user_reader_flags, user_chapter_flags, state_chapter_last_update, state_chapter_next_update, state_chapter_fetch_interval, state_cover_last_modified, state_initialized ) SELECT _id, source, url, title, author, artist, description, genre, status, thumbnail_url, update_strategy, memo, favorite_at, notes, viewer, chapter_flags, last_update, next_update, calculate_interval, cover_last_modified, initialized FROM mangas; DROP TABLE mangas; CREATE INDEX manga_user_favorite_at_index ON manga(user_favorite_at) WHERE user_favorite_at IS NOT NULL; -- --------------------------------------------------------------------------------------------------------- -- table - chapter -- --------------------------------------------------------------------------------------------------------- -- The entries that merged in the previous group brought their chapters with them UPDATE chapters SET manga_id = (SELECT keep_id FROM duplicate_mangas WHERE _id = chapters.manga_id) WHERE manga_id IN (SELECT _id FROM duplicate_mangas WHERE _id != keep_id); -- Moving the chapters over brings rows for the same chapter together, and duplicates like those are -- what issue #647 is about in the first place. The one with reading history is kept, the oldest -- otherwise, and the read state and history of the others are merged into it. CREATE TEMP TABLE duplicate_chapters AS SELECT _id, first_value(_id) OVER (PARTITION BY manga_id, url ORDER BY has_history DESC, _id ASC) AS keep_id FROM ( SELECT _id, manga_id, url, EXISTS(SELECT 1 FROM history WHERE history.chapter_id = chapters._id) AS has_history FROM chapters WHERE (manga_id, url) IN ( SELECT manga_id, url FROM chapters GROUP BY manga_id, url HAVING count(*) > 1 ) ); UPDATE chapters SET read = merged.read, bookmark = merged.bookmark, last_page_read = merged.last_page_read, date_fetch = merged.date_fetch, date_upload = merged.date_upload FROM ( SELECT D.keep_id, max(C.read) AS read, max(C.bookmark) AS bookmark, max(C.last_page_read) AS last_page_read, -- 0 means the fetch date was never recorded rather than "the beginning of time", so it only -- wins when no copy recorded one. Updates only lists chapters fetched after the entry joined -- the library, so letting a 0 through would hide the merged chapter from it. coalesce(min(nullif(C.date_fetch, 0)), 0) AS date_fetch, max(C.date_upload) AS date_upload FROM duplicate_chapters D JOIN chapters C ON C._id = D._id GROUP BY D.keep_id ) AS merged WHERE chapters._id = merged.keep_id; -- The kept chapter has a history entry whenever any of its duplicates has one UPDATE history SET last_read = merged.last_read, time_read = merged.time_read FROM ( SELECT D.keep_id, max(H.last_read) AS last_read, -- Each duplicate accumulated its own reading time, so the durations add up; only the -- last-read timestamp is a point in time and takes the latest. sum(H.time_read) AS time_read FROM duplicate_chapters D JOIN history H ON H.chapter_id = D._id GROUP BY D.keep_id ) AS merged WHERE history.chapter_id = merged.keep_id; -- Foreign keys are off during migrations, so the history of a discarded chapter goes explicitly DELETE FROM history WHERE chapter_id IN (SELECT _id FROM duplicate_chapters WHERE _id != keep_id); DELETE FROM chapters WHERE _id IN (SELECT _id FROM duplicate_chapters WHERE _id != keep_id); DROP TABLE duplicate_chapters; CREATE TABLE chapter( id INTEGER NOT NULL PRIMARY KEY, manga_id INTEGER NOT NULL, remote_url TEXT NOT NULL, remote_name TEXT NOT NULL, remote_scanlator TEXT, remote_chapter_number REAL NOT NULL, remote_date_upload INTEGER NOT NULL, remote_order INTEGER NOT NULL, remote_memo BLOB NOT NULL, user_read INTEGER NOT NULL, user_bookmark INTEGER NOT NULL, user_last_page_read INTEGER NOT NULL, state_date_fetch INTEGER NOT NULL, UNIQUE(manga_id, remote_url), FOREIGN KEY(manga_id) REFERENCES manga (id) ON DELETE CASCADE ); INSERT INTO chapter( id, manga_id, remote_url, remote_name, remote_scanlator, remote_chapter_number, remote_date_upload, remote_order, remote_memo, user_read, user_bookmark, user_last_page_read, state_date_fetch ) SELECT _id, manga_id, url, name, scanlator, chapter_number, date_upload, source_order, memo, read, bookmark, last_page_read, date_fetch FROM chapters; DROP TABLE chapters; CREATE INDEX chapter_state_date_fetch_index ON chapter(state_date_fetch); CREATE INDEX chapter_unread_state_date_fetch_index ON chapter(state_date_fetch) WHERE user_read = 0; -- --------------------------------------------------------------------------------------------------------- -- table - category -- --------------------------------------------------------------------------------------------------------- CREATE TABLE category( id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL, `order` INTEGER NOT NULL UNIQUE, flags INTEGER NOT NULL ); -- `sort` was never unique, and reordering wrote colliding values often enough that it cannot simply be -- copied under the new constraint. The rows keep the order they were displayed in and are renumbered from -- 0, with ties broken by id. The system category is pinned to -1 rather than renumbered, because it is -- identified by id 0 everywhere else and sorts ahead of the categories the user made. INSERT INTO category(id, name, `order`, flags) SELECT _id, name, CASE WHEN _id <= 0 THEN -1 ELSE row_number() OVER (PARTITION BY (_id <= 0) ORDER BY sort, _id) - 1 END, flags FROM categories; -- Dropping the table takes `system_category_delete_trigger` with it DROP TABLE categories; CREATE TRIGGER IF NOT EXISTS system_category_delete_trigger BEFORE DELETE ON category BEGIN SELECT CASE WHEN old.id <= 0 THEN RAISE(ABORT, "System category can't be deleted") END; END; -- --------------------------------------------------------------------------------------------------------- -- table - manga_category -- --------------------------------------------------------------------------------------------------------- -- The surrogate key is gone: a manga belongs to a category once, and that pair is the identity of the row CREATE TABLE manga_category( manga_id INTEGER NOT NULL, category_id INTEGER NOT NULL, UNIQUE(manga_id, category_id), FOREIGN KEY(category_id) REFERENCES category (id) ON DELETE CASCADE, FOREIGN KEY(manga_id) REFERENCES manga (id) ON DELETE CASCADE ); -- Entries merged in the manga group brought their category rows with them. When the entry is in the library, -- only the copies in it say where it belongs; a copy outside it can still hold the categories it had before -- it was removed. The same pair can appear twice by now, and membership carries nothing else, so the copies -- simply collapse. INSERT INTO manga_category(manga_id, category_id) SELECT DISTINCT coalesce(D.keep_id, MC.manga_id), MC.category_id FROM mangas_categories MC LEFT JOIN duplicate_mangas D ON D._id = MC.manga_id LEFT JOIN manga K ON K.id = D.keep_id WHERE D._id IS NULL OR D.favorite OR K.user_favorite_at IS NULL; DROP TABLE mangas_categories; CREATE INDEX manga_category_category_id_index ON manga_category(category_id); -- --------------------------------------------------------------------------------------------------------- -- table - history -- --------------------------------------------------------------------------------------------------------- -- `last_read` is when the chapter was last read and `time_read` is how long it has been read for, so -- both are renamed to say so. The chapter it belongs to is already unique, which leaves -- `history_history_chapter_id_index` a prefix of that constraint. -- The table keeps its name, so the original is renamed aside and the new one built under -- the real name; renaming a table into place makes SQLite rewrite its stored definition, -- leaving it spelled differently from the same table on a fresh install. The views were -- dropped at the top of this migration, so nothing refers to the name being moved. ALTER TABLE history RENAME TO history_temp; CREATE TABLE history( id INTEGER NOT NULL PRIMARY KEY, manga_id INTEGER NOT NULL, chapter_id INTEGER NOT NULL UNIQUE, read_at INTEGER, read_duration INTEGER NOT NULL, FOREIGN KEY(manga_id) REFERENCES manga (id) ON DELETE CASCADE, FOREIGN KEY(chapter_id) REFERENCES chapter (id) ON DELETE CASCADE ); -- Reaching the entry a row belongs to meant going through its chapter; it names the entry -- itself now, taken from that same chapter. A row whose chapter is already gone cannot name -- one and is dropped, which is what the foreign key would do to it once it is enforced again. INSERT INTO history(id, manga_id, chapter_id, read_at, read_duration) SELECT H._id, C.manga_id, H.chapter_id, H.last_read, H.time_read FROM history_temp H JOIN chapter C ON C.id = H.chapter_id; DROP TABLE history_temp; CREATE INDEX history_manga_id_index ON history(manga_id); CREATE INDEX history_read_at_index ON history(read_at); -- --------------------------------------------------------------------------------------------------------- -- table - excluded_scanlator -- --------------------------------------------------------------------------------------------------------- CREATE TABLE excluded_scanlator( manga_id INTEGER NOT NULL, scanlator TEXT NOT NULL, UNIQUE(manga_id, scanlator), FOREIGN KEY(manga_id) REFERENCES manga (id) ON DELETE CASCADE ); -- Excluding a scanlator twice excludes it once, and merged entries brought their exclusions along, so -- the copies collapse. `excluded_scanlators_manga_id_index` is a prefix of the new constraint. INSERT INTO excluded_scanlator(manga_id, scanlator) SELECT DISTINCT coalesce(D.keep_id, ES.manga_id), ES.scanlator FROM excluded_scanlators ES LEFT JOIN duplicate_mangas D ON D._id = ES.manga_id; DROP TABLE excluded_scanlators; -- --------------------------------------------------------------------------------------------------------- -- table - manga_track -- --------------------------------------------------------------------------------------------------------- -- `manga_sync` is what tracking was called when it was still called syncing; `sync_id` is the tracker. -- An entry that merged can leave two rows for the same tracker on the survivor, which the unique constraint -- rejects, so the one to keep is chosen first: one on a copy in the library, then the one furthest read, then -- the one on the copy ranked first, then the oldest. A track is taken whole rather than merged field by field, -- because two rows on the same tracker can point at different remote entries. CREATE TEMP TABLE duplicate_tracks AS SELECT T._id, first_value(T._id) OVER ( PARTITION BY D.keep_id, T.sync_id ORDER BY D.favorite DESC, T.last_chapter_read DESC, D.precedence ASC, T._id ASC ) AS keep_track_id FROM manga_sync T JOIN duplicate_mangas D ON D._id = T.manga_id; DELETE FROM manga_sync WHERE _id IN (SELECT _id FROM duplicate_tracks WHERE _id != keep_track_id); DROP TABLE duplicate_tracks; CREATE TABLE manga_track( id INTEGER NOT NULL PRIMARY KEY, manga_id INTEGER NOT NULL, tracker_id INTEGER NOT NULL, remote_id INTEGER NOT NULL, remote_url TEXT NOT NULL, library_id INTEGER, title TEXT NOT NULL, last_chapter_read REAL NOT NULL, total_chapters INTEGER NOT NULL, status INTEGER NOT NULL, score REAL NOT NULL, start_date INTEGER NOT NULL, finish_date INTEGER NOT NULL, private INTEGER NOT NULL DEFAULT 0, UNIQUE (manga_id, tracker_id), FOREIGN KEY(manga_id) REFERENCES manga (id) ON DELETE CASCADE ); INSERT INTO manga_track( id, manga_id, tracker_id, remote_id, remote_url, library_id, title, last_chapter_read, total_chapters, status, score, start_date, finish_date, private ) SELECT T._id, coalesce(D.keep_id, T.manga_id), T.sync_id, T.remote_id, T.remote_url, T.library_id, T.title, T.last_chapter_read, T.total_chapters, T.status, T.score, T.start_date, T.finish_date, T.private FROM manga_sync T LEFT JOIN duplicate_mangas D ON D._id = T.manga_id; DROP TABLE duplicate_mangas; -- `idx_manga_sync_manga_id` is a prefix of the unique constraint DROP TABLE manga_sync; CREATE INDEX manga_track_tracker_id_remote_id_index ON manga_track(tracker_id, remote_id); -- --------------------------------------------------------------------------------------------------------- -- table - source -- --------------------------------------------------------------------------------------------------------- -- The last of the plural table names, and `lang` spelled out. Nothing references this table, so it is a -- straight copy; name comes before language to read the way the row does. CREATE TABLE source( id INTEGER NOT NULL PRIMARY KEY, name TEXT NOT NULL, language TEXT NOT NULL ); INSERT INTO source(id, name, language) SELECT _id, name, lang FROM sources; DROP TABLE sources; -- The views come back reading the new table CREATE VIEW historyView AS SELECT history.id AS id, manga.id AS mangaId, chapter.id AS chapterId, manga.remote_title AS title, manga.remote_cover AS thumbnailUrl, manga.source_id AS sourceId, manga.user_favorite_at IS NOT NULL AS favorite, manga.state_cover_last_modified AS coverLastModified, chapter.remote_chapter_number AS chapterNumber, history.read_at AS readAt, history.read_duration AS readDuration, max_read.read_at AS maxReadAt, max_read.chapter_id AS maxReadAtChapterId FROM manga JOIN chapter ON manga.id = chapter.manga_id JOIN history ON chapter.id = history.chapter_id JOIN ( SELECT manga_id, chapter_id, MAX(read_at) AS read_at FROM history GROUP BY manga_id ) AS max_read ON chapter.manga_id = max_read.manga_id; CREATE VIEW libraryView AS SELECT M.*, coalesce(C.total, 0) AS totalCount, coalesce(C.readCount, 0) AS readCount, coalesce(C.latestUpload, 0) AS latestUpload, coalesce(C.fetchedAt, 0) AS chapterFetchedAt, coalesce(C.lastRead, 0) AS lastRead, coalesce(C.bookmarkCount, 0) AS bookmarkCount, coalesce(MC.categories, '0') AS categories FROM manga M LEFT JOIN ( SELECT chapter.manga_id, count(*) AS total, sum(user_read) AS readCount, coalesce(max(chapter.remote_date_upload), 0) AS latestUpload, coalesce(max(history.read_at), 0) AS lastRead, coalesce(max(chapter.state_date_fetch), 0) AS fetchedAt, sum(chapter.user_bookmark) AS bookmarkCount FROM chapter LEFT JOIN excluded_scanlator ON chapter.manga_id = excluded_scanlator.manga_id AND chapter.remote_scanlator = excluded_scanlator.scanlator LEFT JOIN history ON chapter.id = history.chapter_id WHERE excluded_scanlator.scanlator IS NULL AND chapter.manga_id IN (SELECT id FROM manga WHERE user_favorite_at IS NOT NULL) GROUP BY chapter.manga_id ) AS C ON M.id = C.manga_id LEFT JOIN ( SELECT manga_id, group_concat(category_id) AS categories FROM manga_category WHERE manga_id IN (SELECT id FROM manga WHERE user_favorite_at IS NOT NULL) GROUP BY manga_id ) AS MC ON MC.manga_id = M.id WHERE M.user_favorite_at IS NOT NULL; CREATE VIEW updatesView AS SELECT manga.id AS mangaId, manga.remote_title AS mangaTitle, chapter.id AS chapterId, chapter.remote_name AS chapterName, chapter.remote_scanlator AS scanlator, chapter.remote_url AS chapterUrl, chapter.user_read AS read, chapter.user_bookmark AS bookmark, chapter.user_last_page_read AS lastPageRead, manga.source_id AS sourceId, manga.user_favorite_at IS NOT NULL AS favorite, manga.remote_cover AS thumbnailUrl, manga.state_cover_last_modified AS coverLastModified, chapter.remote_date_upload AS dateUpload, chapter.state_date_fetch AS dateFetch, excluded_scanlator.scanlator AS excludedScanlator FROM manga JOIN chapter ON manga.id = chapter.manga_id LEFT JOIN excluded_scanlator ON manga.id = excluded_scanlator.manga_id AND chapter.remote_scanlator = excluded_scanlator.scanlator WHERE user_favorite_at IS NOT NULL AND state_date_fetch > user_favorite_at ORDER BY state_date_fetch DESC;