Count early, late and backdated releases when predicting next update

This commit is contained in:
AntsyLich
2026-10-11 20:03:56 +06:00
parent fc20410ceb
commit ba44815254
6 changed files with 558 additions and 217 deletions
@@ -226,11 +226,18 @@ class SyncChaptersWithSource(
updated = updatedChapters, updated = updatedChapters,
) )
val excludedScanlators = getExcludedScanlators.await(manga.id).toHashSet()
// A first fetch lists the back catalogue, and chapters coming back under a deleted number were out already
val foundRelease = dbChapters.isNotEmpty() &&
toAdd.any { chapter ->
chapter.scanlator !in excludedScanlators &&
!(chapter.isRecognizedNumber && chapter.chapterNumber in deletedChapterNumbers)
}
// Set this manga as updated since chapters were changed // Set this manga as updated since chapters were changed
// Note that last_update actually represents last time the chapter list changed at all // Note that last_update actually represents last time the chapter list changed at all
updateManga.awaitUpdateFetchIntervalAndLastUpdate(manga, timeZone, now, fetchWindow) updateManga.awaitUpdateFetchIntervalAndLastUpdate(manga, timeZone, now, fetchWindow, foundRelease)
val excludedScanlators = getExcludedScanlators.await(manga.id).toHashSet()
return added.filterNot { it.url in changedOrDuplicateReadUrls || it.scanlator in excludedScanlators } return added.filterNot { it.url in changedOrDuplicateReadUrls || it.scanlator in excludedScanlators }
} }
@@ -44,9 +44,10 @@ class UpdateManga(
timeZone: TimeZone, timeZone: TimeZone,
dateTime: LocalDateTime, dateTime: LocalDateTime,
window: ClosedRange<Instant>?, window: ClosedRange<Instant>?,
foundRelease: Boolean,
): Boolean { ): Boolean {
return mangaRepository.update( return mangaRepository.update(
fetchInterval.withFetchInterval(manga, dateTime, timeZone, window).apply { fetchInterval.withFetchInterval(manga, dateTime, timeZone, window, foundRelease).apply {
lastUpdate = Clock.System.now() lastUpdate = Clock.System.now()
}, },
) )
@@ -1,21 +1,14 @@
package tachiyomi.domain.manga.interactor package tachiyomi.domain.manga.interactor
import dev.zacsweers.metro.Inject import dev.zacsweers.metro.Inject
import kotlinx.datetime.DatePeriod
import kotlinx.datetime.DayOfWeek
import kotlinx.datetime.LocalDate import kotlinx.datetime.LocalDate
import kotlinx.datetime.LocalDateTime import kotlinx.datetime.LocalDateTime
import kotlinx.datetime.TimeZone import kotlinx.datetime.TimeZone
import kotlinx.datetime.atStartOfDayIn import kotlinx.datetime.atStartOfDayIn
import kotlinx.datetime.daysUntil
import kotlinx.datetime.minus
import kotlinx.datetime.plus
import kotlinx.datetime.toLocalDateTime import kotlinx.datetime.toLocalDateTime
import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId
import tachiyomi.domain.chapter.model.Chapter
import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.manga.model.Manga
import tachiyomi.domain.manga.model.MangaUpdate import tachiyomi.domain.manga.model.MangaUpdate
import kotlin.math.absoluteValue
import kotlin.time.Clock import kotlin.time.Clock
import kotlin.time.Duration.Companion.days import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.milliseconds
@@ -26,28 +19,45 @@ class FetchInterval(
private val getChaptersByMangaId: GetChaptersByMangaId, private val getChaptersByMangaId: GetChaptersByMangaId,
) { ) {
/**
* Works out the entry's fetch interval and the next day to check it for new chapters.
*
* @param foundRelease whether the check that led here found new chapters.
*/
suspend fun withFetchInterval( suspend fun withFetchInterval(
manga: Manga, manga: Manga,
dateTime: LocalDateTime, dateTime: LocalDateTime,
timeZone: TimeZone, timeZone: TimeZone,
window: ClosedRange<Instant>?, window: ClosedRange<Instant>?,
foundRelease: Boolean = false,
): MangaUpdate { ): MangaUpdate {
val today = dateTime.date
val currentWindow = window ?: getWindow(Clock.System.now().toLocalDateTime(timeZone).date, timeZone) val currentWindow = window ?: getWindow(Clock.System.now().toLocalDateTime(timeZone).date, timeZone)
val nextUpdate: Instant val schedule = if (manga.fetchInterval < 0) {
val interval: Int // The check that found a release is the entry's last update
if (manga.fetchInterval < 0) { val lastUpdate = if (foundRelease) today else manga.lastUpdate?.toLocalDateTime(timeZone)?.date
interval = manga.fetchInterval ReleaseSchedule.fixed(-manga.fetchInterval, lastUpdate ?: today)
nextUpdate = calculateCustomNextUpdate(manga, interval.absoluteValue, dateTime, timeZone, currentWindow)
} else { } else {
val chapters = getChaptersByMangaId.await(manga.id, applyScanlatorFilter = true) val chapters = getChaptersByMangaId.await(manga.id, applyScanlatorFilter = true)
val schedule = estimateSchedule(chapters, timeZone, dateTime.date) ReleaseSchedule.estimate(chapters, timeZone, today)
interval = schedule.interval }
nextUpdate = calculateNextUpdate(manga, schedule, dateTime.date, timeZone, currentWindow)
val expected = manga.nextUpdate
val expectedDay = expected?.toLocalDateTime(timeZone)?.date
// A release on or after the expected day means the next one is counted from it instead
val isAwaitingRelease = expected != null &&
expectedDay != null &&
expected in currentWindow.start..currentWindow.endInclusive + 1.milliseconds &&
(schedule.lastRelease == null || schedule.lastRelease < expectedDay)
val nextUpdate = when {
foundRelease -> schedule.nextCheckAfterRelease(today).atStartOfDayIn(timeZone)
isAwaitingRelease -> expected
else -> schedule.nextCheck(today).atStartOfDayIn(timeZone)
} }
return MangaUpdate(manga.id) { return MangaUpdate(manga.id) {
this.nextUpdate = nextUpdate this.nextUpdate = nextUpdate
fetchInterval = interval fetchInterval = if (manga.fetchInterval < 0) manga.fetchInterval else schedule.interval
} }
} }
@@ -56,189 +66,9 @@ class FetchInterval(
return (today - GRACE_PERIOD.days)..(today + GRACE_PERIOD.days) return (today - GRACE_PERIOD.days)..(today + GRACE_PERIOD.days)
} }
internal fun calculateInterval(chapters: List<Chapter>, zone: TimeZone): Int {
return estimateSchedule(chapters, zone, Clock.System.now().toLocalDateTime(zone).date).interval
}
internal fun estimateSchedule(chapters: List<Chapter>, zone: TimeZone, today: LocalDate): ReleaseSchedule {
val chapterWindow = if (chapters.size <= 8) 3 else 10
// Sources that date chapters ahead of their release would otherwise pull the schedule into the future
val latestPlausibleDay = today.plus(DatePeriod(days = 1))
fun releaseDays(dates: Sequence<Instant>): List<LocalDate> {
return dates
.map { it.toLocalDateTime(zone).date }
.filter { it <= latestPlausibleDay }
.distinct()
.sortedDescending()
.take(chapterWindow)
.toList()
}
val uploadDays = releaseDays(chapters.asSequence().mapNotNull { it.dateUpload })
val days = uploadDays.takeIf { it.size >= 3 }
?: releaseDays(chapters.asSequence().map { it.dateFetch }).takeIf { it.size >= 3 }
?: return ReleaseSchedule(
interval = DEFAULT_INTERVAL,
lastRelease = uploadDays.firstOrNull(),
pattern = ReleasePattern.EveryDays(DEFAULT_INTERVAL),
)
val gaps = days.zipWithNext { newer, older -> older.daysUntil(newer) }.sorted()
val medianGap = gaps[(gaps.size - 1) / 2]
val interval = medianGap.coerceIn(1, MAX_INTERVAL)
return ReleaseSchedule(
interval = interval,
lastRelease = days.first(),
pattern = detectPattern(days, medianGap) ?: ReleasePattern.EveryDays(interval),
)
}
/**
* Finds a calendar pattern a plain day interval can't follow: a few fixed weekdays, or a day of the
* month. Both need a few releases to be told apart from chance.
*/
private fun detectPattern(days: List<LocalDate>, medianGap: Int): ReleasePattern? {
if (days.size < MIN_RELEASES_FOR_PATTERN) return null
if (medianGap in MONTHLY_GAPS) {
val daysOfMonth = days.map { it.day }
return when {
daysOfMonth.max() - daysOfMonth.min() <= DAY_OF_MONTH_TOLERANCE -> {
ReleasePattern.DayOfMonth(daysOfMonth.sorted()[(daysOfMonth.size - 1) / 2])
}
// The last day of the month moves between the 28th and the 31st
daysOfMonth.all { it >= 28 } -> ReleasePattern.DayOfMonth(31)
else -> null
}
}
val byFrequency = days.groupingBy { it.dayOfWeek }.eachCount().entries.sortedByDescending { it.value }
val weekdays = mutableSetOf<DayOfWeek>()
var covered = 0
for ((dayOfWeek, count) in byFrequency) {
if (covered >= days.size * WEEKDAY_COVERAGE) break
weekdays += dayOfWeek
covered += count
}
// One weekday is a weekly interval, and an irregular schedule spreads over most of the week
if (weekdays.size !in 2..MAX_WEEKDAYS || covered < days.size * WEEKDAY_COVERAGE) return null
// Releasing on several weekdays means more than one release a week
if (medianGap > 7 / weekdays.size + 1) return null
return ReleasePattern.Weekdays(weekdays)
}
private fun calculateNextUpdate(
manga: Manga,
schedule: ReleaseSchedule,
today: LocalDate,
timeZone: TimeZone,
window: ClosedRange<Instant>,
): Instant {
val lastRelease = listOfNotNull(schedule.lastRelease, manga.lastUpdate?.toLocalDateTime(timeZone)?.date)
.maxOrNull()
// A release on or after the expected day means the next one should be counted from it instead
val nextUpdate = manga.nextUpdate
if (
nextUpdate != null &&
nextUpdate in window.start..window.endInclusive + 1.milliseconds &&
(lastRelease == null || lastRelease < nextUpdate.toLocalDateTime(timeZone).date)
) {
return nextUpdate
}
val anchor = schedule.lastRelease ?: lastRelease ?: today
val daysSinceRelease = anchor.daysUntil(today)
val expected = if (daysSinceRelease > schedule.interval * HIATUS_CYCLES) {
// Past a few missed releases the schedule no longer holds, so checks spread out until it resumes
val interval = backOff(schedule.interval, daysSinceRelease)
anchor.plus(DatePeriod(days = ceilDiv(daysSinceRelease, interval) * interval))
} else {
// Today counts, so a release expected today keeps being checked for until the window passes it
generateSequence(schedule.pattern.nextAfter(anchor)) { schedule.pattern.nextAfter(it) }
.first { it >= today }
}
return expected.atStartOfDayIn(timeZone)
}
private fun calculateCustomNextUpdate(
manga: Manga,
interval: Int,
dateTime: LocalDateTime,
timeZone: TimeZone,
window: ClosedRange<Instant>,
): Instant {
if (manga.nextUpdate != null && manga.nextUpdate in window.start..window.endInclusive + 1.milliseconds) {
return manga.nextUpdate
}
val latestDate = (manga.lastUpdate ?: Clock.System.now()).toLocalDateTime(timeZone).date
val cycle = latestDate.daysUntil(dateTime.date).floorDiv(interval)
return latestDate.plus(DatePeriod(days = (cycle + 1) * interval)).atStartOfDayIn(timeZone)
}
private fun backOff(interval: Int, daysSinceRelease: Int): Int {
var backedOff = interval
while (backedOff < MAX_INTERVAL && daysSinceRelease > backedOff * HIATUS_CYCLES) {
backedOff *= 2
}
return backedOff.coerceAtMost(MAX_INTERVAL)
}
private fun ceilDiv(a: Int, b: Int) = -Math.floorDiv(-a, b)
companion object { companion object {
const val MAX_INTERVAL = 28 const val MAX_INTERVAL = 28
private const val DEFAULT_INTERVAL = 7
private const val GRACE_PERIOD = 1L private const val GRACE_PERIOD = 1L
private const val HIATUS_CYCLES = 4
private const val MIN_RELEASES_FOR_PATTERN = 4
private const val MAX_WEEKDAYS = 3
private const val WEEKDAY_COVERAGE = 0.8
private const val DAY_OF_MONTH_TOLERANCE = 3
private val MONTHLY_GAPS = 26..35
}
}
internal data class ReleaseSchedule(
/** Days between releases, which is what the fetch interval shows. */
val interval: Int,
/** The most recent release day the next ones are counted from, if any chapter has a date. */
val lastRelease: LocalDate?,
val pattern: ReleasePattern,
)
internal sealed interface ReleasePattern {
/** The first expected release day after [date]. */
fun nextAfter(date: LocalDate): LocalDate
data class EveryDays(val days: Int) : ReleasePattern {
override fun nextAfter(date: LocalDate) = date.plus(DatePeriod(days = days))
}
data class Weekdays(val days: Set<DayOfWeek>) : ReleasePattern {
override fun nextAfter(date: LocalDate): LocalDate {
return generateSequence(date.plus(DatePeriod(days = 1))) { it.plus(DatePeriod(days = 1)) }
.first { it.dayOfWeek in days }
}
}
/** Releases on [day] of every month, or its last day in months too short for it. */
data class DayOfMonth(val day: Int) : ReleasePattern {
override fun nextAfter(date: LocalDate): LocalDate {
val sameMonth = date.inMonthClamped(day)
if (sameMonth > date) return sameMonth
return date.plus(DatePeriod(months = 1)).inMonthClamped(day)
}
private fun LocalDate.inMonthClamped(day: Int): LocalDate {
val firstOfMonth = LocalDate(year, month, 1)
val lastDay = firstOfMonth.plus(DatePeriod(months = 1)).minus(DatePeriod(days = 1)).day
return LocalDate(year, month, day.coerceAtMost(lastDay))
}
} }
} }
@@ -0,0 +1,212 @@
package tachiyomi.domain.manga.interactor
import kotlinx.datetime.DatePeriod
import kotlinx.datetime.DayOfWeek
import kotlinx.datetime.LocalDate
import kotlinx.datetime.daysUntil
import kotlinx.datetime.isoDayNumber
import kotlinx.datetime.minus
import kotlinx.datetime.plus
import kotlin.math.abs
/** The days an entry is expected to release on. */
internal sealed interface ReleasePattern {
/** The first expected release day after [date]. */
fun nextAfter(date: LocalDate): LocalDate
/** The last expected release day before [date]. */
fun previousBefore(date: LocalDate): LocalDate
/** The expected release day closest to [date], which a release on [date] stands in for. */
fun nearest(date: LocalDate): LocalDate {
val before = previousBefore(date.plus(DatePeriod(days = 1)))
val after = nextAfter(date.minus(DatePeriod(days = 1)))
// A release halfway between is taken as late, so the next check comes sooner
return if (before.daysUntil(date) <= date.daysUntil(after)) before else after
}
/** Releases every [days] days, in step with [onSchedule]. */
data class EveryDays(val days: Int, val onSchedule: LocalDate) : ReleasePattern {
override fun nextAfter(date: LocalDate): LocalDate {
return date.plus(DatePeriod(days = days - onSchedule.daysUntil(date).mod(days)))
}
override fun previousBefore(date: LocalDate): LocalDate {
return date.minus(DatePeriod(days = (onSchedule.daysUntil(date) - 1).mod(days) + 1))
}
}
data class Weekdays(val days: Set<DayOfWeek>) : ReleasePattern {
override fun nextAfter(date: LocalDate): LocalDate {
return generateSequence(date.plus(DatePeriod(days = 1))) { it.plus(DatePeriod(days = 1)) }
.first { it.dayOfWeek in days }
}
override fun previousBefore(date: LocalDate): LocalDate {
return generateSequence(date.minus(DatePeriod(days = 1))) { it.minus(DatePeriod(days = 1)) }
.first { it.dayOfWeek in days }
}
// Release days are a day or two apart, so a release between them is more likely an extra than an early one
override fun nearest(date: LocalDate) = date
}
/** Releases for [length] days in a row, every [every] days in step with [onSchedule]. */
data class Batches(val every: Int, val onSchedule: LocalDate, val length: Int) : ReleasePattern {
override fun nextAfter(date: LocalDate): LocalDate {
val offset = onSchedule.daysUntil(date).mod(every)
return date.plus(DatePeriod(days = if (offset + 1 < length) 1 else every - offset))
}
override fun previousBefore(date: LocalDate): LocalDate {
val dayBefore = date.minus(DatePeriod(days = 1))
val offset = onSchedule.daysUntil(dayBefore).mod(every)
return dayBefore.minus(DatePeriod(days = (offset - length + 1).coerceAtLeast(0)))
}
}
/** A pattern that repeats every month. */
sealed interface Monthly : ReleasePattern {
/** The release days in the month starting on [firstOfMonth]. */
fun daysIn(firstOfMonth: LocalDate): List<LocalDate>
override fun nextAfter(date: LocalDate): LocalDate {
return generateSequence(date.firstOfMonth()) { it.plus(DatePeriod(months = 1)) }
.flatMap { daysIn(it).sorted() }
.first { it > date }
}
override fun previousBefore(date: LocalDate): LocalDate {
return generateSequence(date.firstOfMonth()) { it.minus(DatePeriod(months = 1)) }
.flatMap { daysIn(it).sortedDescending() }
.first { it < date }
}
}
/** Releases on [days] of every month, or a month's last day for those past it. */
data class DaysOfMonth(val days: Set<Int>) : Monthly {
override fun daysIn(firstOfMonth: LocalDate): List<LocalDate> {
val lastDay = firstOfMonth.lastOfMonth().day
return days.map { firstOfMonth.plus(DatePeriod(days = it.coerceAtMost(lastDay) - 1)) }.distinct()
}
}
/** Releases on [dayOfWeek] of the [week]th week of every month, or its last week for [LAST_WEEK]. */
data class WeekdayOfMonth(val week: Int, val dayOfWeek: DayOfWeek) : Monthly {
override fun daysIn(firstOfMonth: LocalDate): List<LocalDate> {
if (week == LAST_WEEK) {
val lastOfMonth = firstOfMonth.lastOfMonth()
val daysBack = (lastOfMonth.dayOfWeek.isoDayNumber - dayOfWeek.isoDayNumber).mod(7)
return listOf(lastOfMonth.minus(DatePeriod(days = daysBack)))
}
val daysAhead = (dayOfWeek.isoDayNumber - firstOfMonth.dayOfWeek.isoDayNumber).mod(7)
return listOf(firstOfMonth.plus(DatePeriod(days = daysAhead + (week - 1) * 7)))
}
}
companion object {
const val LAST_WEEK = -1
private const val MIN_RELEASES = 4
private const val MAX_WEEKDAYS = 3
private const val COVERAGE = 0.8
private const val DAY_OF_MONTH_TOLERANCE = 3
private val MONTHLY_GAPS = 26..35
private val TWICE_MONTHLY_GAPS = 12..18
private const val BATCH_HISTORY = 40
private const val BATCH_GAP = 2
private const val BATCHES = 6
private const val MIN_BATCHES = 3
/**
* Finds a calendar pattern a plain day interval can't follow in [days], newest first: a few fixed weekdays,
* days or a weekday of the month, or releases in batches, which can take longer than [days] to show up in
* [history]. They all need a few releases to be told apart from chance.
*/
fun detect(days: List<LocalDate>, history: List<LocalDate>, medianGap: Int): ReleasePattern? {
if (days.size < MIN_RELEASES) return null
// Batches come first, since their days can look like a few weekdays or a monthly schedule
return batches(history) ?: when (medianGap) {
in MONTHLY_GAPS -> monthly(days)
in TWICE_MONTHLY_GAPS -> twiceMonthly(days)
else -> weekdays(days, medianGap)
}
}
private fun monthly(days: List<LocalDate>): ReleasePattern? {
val daysOfMonth = days.map { it.day }
return when {
daysOfMonth.max() - daysOfMonth.min() <= DAY_OF_MONTH_TOLERANCE -> DaysOfMonth(
setOf(daysOfMonth.median()),
)
// The last day of the month moves between the 28th and the 31st
daysOfMonth.all { it >= 28 } -> DaysOfMonth(setOf(31))
else -> weekdayOfMonth(days)
}
}
private fun weekdayOfMonth(days: List<LocalDate>): ReleasePattern? {
val dayOfWeek = days.mostCommon { it.dayOfWeek } ?: return null
// A fifth week isn't in every month, so those days only count as the last week
val week = days.mostCommon { (it.day - 1) / 7 + 1 }?.takeIf { it < 5 }
if (week != null) return WeekdayOfMonth(week, dayOfWeek)
val isLastWeek = days.count { it.day + 7 > it.lastOfMonth().day } >= days.size * COVERAGE
return if (isLastWeek) WeekdayOfMonth(LAST_WEEK, dayOfWeek) else null
}
private fun twiceMonthly(days: List<LocalDate>): ReleasePattern? {
val daysOfMonth = days.map { it.day }.sorted()
val split = (1..<daysOfMonth.size).maxBy { daysOfMonth[it] - daysOfMonth[it - 1] }
val halves = listOf(daysOfMonth.subList(0, split), daysOfMonth.subList(split, daysOfMonth.size))
if (halves.any { it.size < 2 || it.last() - it.first() > DAY_OF_MONTH_TOLERANCE }) return null
return DaysOfMonth(halves.map { it.median() }.toSet())
}
private fun weekdays(days: List<LocalDate>, medianGap: Int): ReleasePattern? {
val byFrequency = days.groupingBy { it.dayOfWeek }.eachCount().entries.sortedByDescending { it.value }
val weekdays = mutableSetOf<DayOfWeek>()
var covered = 0
for ((dayOfWeek, count) in byFrequency) {
if (covered >= days.size * COVERAGE) break
weekdays += dayOfWeek
covered += count
}
// One weekday is a weekly interval, and an irregular schedule spreads over most of the week
if (weekdays.size !in 2..MAX_WEEKDAYS || covered < days.size * COVERAGE) return null
// Releasing on several weekdays means more than one release a week
if (medianGap > 7 / weekdays.size + 1) return null
return Weekdays(weekdays)
}
/** Runs of releases a day or two apart, starting at a steady interval and mostly longer than a day. */
private fun batches(history: List<LocalDate>): ReleasePattern? {
val runs = mutableListOf<MutableList<LocalDate>>()
for (day in history.take(BATCH_HISTORY).asReversed()) {
val run = runs.lastOrNull()
if (run != null && run.last().daysUntil(day) <= BATCH_GAP) run += day else runs += mutableListOf(day)
}
val recent = runs.takeLast(BATCHES)
if (recent.size < MIN_BATCHES || recent.count { it.size > 1 } * 2 < recent.size) return null
val gaps = recent.zipWithNext { older, newer -> older.first().daysUntil(newer.first()) }
val every = gaps.median()
val length = recent.map { it.first().daysUntil(it.last()) + 1 }.median()
if (every < length * 2 || gaps.any { abs(it - every) > every / 4 }) return null
return Batches(every, recent.last().first(), length)
}
private fun List<Int>.median() = sorted()[(size - 1) / 2]
/** The most common key of [selector], if at least [COVERAGE] of the days share it. */
private fun <T> List<LocalDate>.mostCommon(selector: (LocalDate) -> T): T? {
val (key, count) = groupingBy(selector).eachCount().maxBy { it.value }
return key.takeIf { count >= size * COVERAGE }
}
}
}
private fun LocalDate.firstOfMonth() = LocalDate(year, month, 1)
private fun LocalDate.lastOfMonth() = firstOfMonth().plus(DatePeriod(months = 1)).minus(DatePeriod(days = 1))
@@ -0,0 +1,151 @@
package tachiyomi.domain.manga.interactor
import kotlinx.datetime.DatePeriod
import kotlinx.datetime.LocalDate
import kotlinx.datetime.TimeZone
import kotlinx.datetime.daysUntil
import kotlinx.datetime.plus
import kotlinx.datetime.toLocalDateTime
import tachiyomi.domain.chapter.model.Chapter
import kotlin.time.Instant
/** How often an entry releases, and so when to check it for the next release. */
internal data class ReleaseSchedule(
/** Days between releases, which is what the fetch interval shows. */
val interval: Int,
/** The most recent release day, which the next ones are counted from. */
val lastRelease: LocalDate?,
val pattern: ReleasePattern,
/** Days past the expected day a release can still come, from how late recent ones were. */
val lateness: Int = 0,
/** Whether checks spread out once releases stop coming, which a schedule set by the user doesn't. */
val backsOff: Boolean = true,
/** Lengths in days of earlier breaks in releases, which a break like a season's end is expected to last again. */
val pastBreaks: List<Int> = emptyList(),
) {
/** The next day to check for a release, as of [today]. Today counts, so a due release keeps being checked for. */
fun nextCheck(today: LocalDate): LocalDate {
val anchor = lastRelease ?: today
val daysSinceRelease = anchor.daysUntil(today)
val isOnBreak = backsOff && daysSinceRelease > interval * HIATUS_CYCLES
// Around when earlier breaks ended, this one likely ends too, so the schedule is followed again
if (isOnBreak && pastBreaks.none { daysSinceRelease in it - interval..it + interval }) {
// Past a few missed releases the schedule no longer holds, so checks spread out until it resumes
val backedOff = backOff(daysSinceRelease)
val next = anchor.plus(DatePeriod(days = ceilDiv(daysSinceRelease, backedOff) * backedOff))
val breakEnd = pastBreaks.map {
anchor.plus(DatePeriod(days = it - interval))
}.filter { it > today }.minOrNull()
return minOf(next, breakEnd ?: next)
}
val due = pattern.nextAfter(pattern.nearest(anchor))
if (today in due..due.plus(DatePeriod(days = lateness))) return today
return generateSequence(due, pattern::nextAfter).first { it >= today }.cappedFrom(today)
}
/**
* The next day to check after a check [today] found a release.
*
* The release is dated by its newest chapter, since a check can turn up one that came out on an earlier day. It
* stands in for the expected day nearest to it, so an early, late or extra release leaves the schedule in place.
*/
fun nextCheckAfterRelease(today: LocalDate): LocalDate {
val release = lastRelease?.coerceAtMost(today) ?: today
// What was found is out already, so checking again today would only find it again
val next = pattern.nextAfter(pattern.nearest(release))
return generateSequence(next, pattern::nextAfter).first { it > today }.cappedFrom(today)
}
// Months and batches can be further apart than a check should ever wait
private fun LocalDate.cappedFrom(today: LocalDate): LocalDate {
return coerceAtMost(today.plus(DatePeriod(days = FetchInterval.MAX_INTERVAL)))
}
private fun backOff(daysSinceRelease: Int): Int {
var backedOff = interval
while (backedOff < FetchInterval.MAX_INTERVAL && daysSinceRelease > backedOff * HIATUS_CYCLES) {
backedOff *= 2
}
return backedOff.coerceAtMost(FetchInterval.MAX_INTERVAL)
}
private fun ceilDiv(a: Int, b: Int) = -Math.floorDiv(-a, b)
companion object {
private const val DEFAULT_INTERVAL = 7
private const val HIATUS_CYCLES = 4
private const val MIN_RELEASES = 3
private const val FEW_CHAPTERS = 8
private const val FEW_CHAPTERS_RELEASES = 3
private const val RECENT_RELEASES = 10
private const val PHASE_RELEASES = 5
/** A schedule the user set to every [interval] days, counted from [lastRelease]. */
fun fixed(interval: Int, lastRelease: LocalDate): ReleaseSchedule {
val pattern = ReleasePattern.EveryDays(interval, onSchedule = lastRelease)
return ReleaseSchedule(interval, lastRelease, pattern, backsOff = false)
}
/** Estimates the schedule from the recent release days of [chapters], by upload date or else fetch date. */
fun estimate(chapters: List<Chapter>, zone: TimeZone, today: LocalDate): ReleaseSchedule {
val recentReleases = if (chapters.size <= FEW_CHAPTERS) FEW_CHAPTERS_RELEASES else RECENT_RELEASES
// Sources that date chapters ahead of their release would otherwise pull the schedule into the future
val latestPlausibleDay = today.plus(DatePeriod(days = 1))
fun releaseDays(dates: Sequence<Instant>): List<LocalDate> {
return dates
.map { it.toLocalDateTime(zone).date }
.filter { it <= latestPlausibleDay }
.distinct()
.sortedDescending()
.toList()
}
val uploadDays = releaseDays(chapters.asSequence().mapNotNull { it.dateUpload })
val fetchDays = releaseDays(chapters.asSequence().map { it.dateFetch })
val history = uploadDays.takeIf { it.size >= MIN_RELEASES }
?: fetchDays.takeIf { it.size >= MIN_RELEASES }
?: run {
val lastRelease = uploadDays.firstOrNull() ?: fetchDays.firstOrNull()
val pattern = ReleasePattern.EveryDays(DEFAULT_INTERVAL, onSchedule = lastRelease ?: today)
return ReleaseSchedule(DEFAULT_INTERVAL, lastRelease, pattern)
}
val days = history.take(recentReleases)
val gaps = days.zipWithNext { newer, older -> older.daysUntil(newer) }.sorted()
val medianGap = gaps[(gaps.size - 1) / 2]
val detected = ReleasePattern.detect(days, history, medianGap)
// Within a batch releases are a day apart, but it's the batches that come every so often
val interval = ((detected as? ReleasePattern.Batches)?.every ?: medianGap)
.coerceIn(1, FetchInterval.MAX_INTERVAL)
return ReleaseSchedule(
interval = interval,
lastRelease = days.first(),
pattern = detected
?: ReleasePattern.EveryDays(
interval,
onSchedule = medianPhase(days.take(PHASE_RELEASES), interval),
),
lateness = (gaps.last() - interval).coerceIn(0, interval / 2),
pastBreaks = history
.zipWithNext { newer, older -> older.daysUntil(newer) }
.filter { it > interval * HIATUS_CYCLES },
)
}
/**
* The release in [days], newest first, that the others land closest to in step with [interval], so one early
* or late release doesn't shift the schedule. Ties go to the newest, which follows a schedule that moved.
*/
private fun medianPhase(days: List<LocalDate>, interval: Int): LocalDate {
fun offset(
from: LocalDate,
to: LocalDate,
) = from.daysUntil(to).mod(interval).let { minOf(it, interval - it) }
val phase = days.minBy { candidate -> days.sumOf { offset(candidate, it) } }
val onPhase = days.count { offset(phase, it) == 0 }
return if (onPhase * 5 >= days.size * 3) phase else days.first()
}
}
}
@@ -20,6 +20,7 @@ import org.junit.jupiter.api.parallel.ExecutionMode
import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId import tachiyomi.domain.chapter.interactor.GetChaptersByMangaId
import tachiyomi.domain.chapter.model.Chapter import tachiyomi.domain.chapter.model.Chapter
import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.manga.model.Manga
import kotlin.time.Clock
import kotlin.time.Duration import kotlin.time.Duration
import kotlin.time.Duration.Companion.days import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.hours import kotlin.time.Duration.Companion.hours
@@ -34,19 +35,17 @@ class FetchIntervalTest {
dateUpload = testTime.toInstant(testTimeZone), dateUpload = testTime.toInstant(testTimeZone),
) )
private val fetchInterval = FetchInterval(mockk())
@Test @Test
fun `returns default interval of 7 days when not enough distinct days`() { fun `returns default interval of 7 days when not enough distinct days`() {
val chaptersWithUploadDate = (1..50).map { val chaptersWithUploadDate = (1..50).map {
chapterWithTime(chapter, 1.days) chapterWithTime(chapter, 1.days)
} }
fetchInterval.calculateInterval(chaptersWithUploadDate, testTimeZone) shouldBe 7 interval(chaptersWithUploadDate) shouldBe 7
val chaptersWithoutUploadDate = chaptersWithUploadDate.map { val chaptersWithoutUploadDate = chaptersWithUploadDate.map {
it.copy(dateUpload = null) it.copy(dateUpload = null)
} }
fetchInterval.calculateInterval(chaptersWithoutUploadDate, testTimeZone) shouldBe 7 interval(chaptersWithoutUploadDate) shouldBe 7
} }
@Test @Test
@@ -60,7 +59,7 @@ class FetchIntervalTest {
val chapters = oldChapters + newChapters val chapters = oldChapters + newChapters
fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 1 interval(chapters) shouldBe 1
} }
@Test @Test
@@ -75,7 +74,7 @@ class FetchIntervalTest {
val chapters = oldChapters + newChapters val chapters = oldChapters + newChapters
fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 7 interval(chapters) shouldBe 7
} }
@Test @Test
@@ -83,7 +82,7 @@ class FetchIntervalTest {
val chapters = (1..10).map { val chapters = (1..10).map {
chapterWithTime(chapter, 10.hours) chapterWithTime(chapter, 10.hours)
} }
fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 7 interval(chapters) shouldBe 7
} }
@Test @Test
@@ -93,7 +92,7 @@ class FetchIntervalTest {
} + (1..5).map { } + (1..5).map {
chapterWithTime(chapter, 2.days) chapterWithTime(chapter, 2.days)
} }
fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 7 interval(chapters) shouldBe 7
} }
@Test @Test
@@ -101,7 +100,7 @@ class FetchIntervalTest {
val chapters = (1..20).map { val chapters = (1..20).map {
chapterWithTime(chapter, it.days) chapterWithTime(chapter, it.days)
} }
fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 1 interval(chapters) shouldBe 1
} }
@Test @Test
@@ -109,7 +108,7 @@ class FetchIntervalTest {
val chapters = (1..20).map { val chapters = (1..20).map {
chapterWithTime(chapter, (15 * it).hours) chapterWithTime(chapter, (15 * it).hours)
} }
fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 1 interval(chapters) shouldBe 1
} }
@Test @Test
@@ -117,7 +116,7 @@ class FetchIntervalTest {
val chapters = (1..20).map { val chapters = (1..20).map {
chapterWithTime(chapter, (2 * it).days) chapterWithTime(chapter, (2 * it).days)
} }
fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 2 interval(chapters) shouldBe 2
} }
@Test @Test
@@ -125,12 +124,12 @@ class FetchIntervalTest {
val chaptersWithUploadDate = (1..5).map { val chaptersWithUploadDate = (1..5).map {
chapterWithTime(chapter, (25 * it).hours) chapterWithTime(chapter, (25 * it).hours)
} }
fetchInterval.calculateInterval(chaptersWithUploadDate, testTimeZone) shouldBe 1 interval(chaptersWithUploadDate) shouldBe 1
val chaptersWithoutUploadDate = chaptersWithUploadDate.map { val chaptersWithoutUploadDate = chaptersWithUploadDate.map {
it.copy(dateUpload = null) it.copy(dateUpload = null)
} }
fetchInterval.calculateInterval(chaptersWithoutUploadDate, testTimeZone) shouldBe 1 interval(chaptersWithoutUploadDate) shouldBe 1
} }
@Test @Test
@@ -138,7 +137,7 @@ class FetchIntervalTest {
val chapters = (1..20).map { val chapters = (1..20).map {
chapterWithTime(chapter, (43 * it).hours) chapterWithTime(chapter, (43 * it).hours)
} }
fetchInterval.calculateInterval(chapters, testTimeZone) shouldBe 2 interval(chapters) shouldBe 2
} }
@Test @Test
@@ -149,7 +148,7 @@ class FetchIntervalTest {
.toList() .toList()
val chapters = chaptersOn(mondaysAndThursdays) val chapters = chaptersOn(mondaysAndThursdays)
fetchInterval.estimateSchedule(chapters, testTimeZone, LocalDate.parse("2020-01-21")).pattern shouldBe ReleaseSchedule.estimate(chapters, testTimeZone, LocalDate.parse("2020-01-21")).pattern shouldBe
ReleasePattern.Weekdays(setOf(DayOfWeek.MONDAY, DayOfWeek.THURSDAY)) ReleasePattern.Weekdays(setOf(DayOfWeek.MONDAY, DayOfWeek.THURSDAY))
nextUpdate(today = "2020-01-21", chapters) shouldBe LocalDate.parse("2020-01-23") nextUpdate(today = "2020-01-21", chapters) shouldBe LocalDate.parse("2020-01-23")
} }
@@ -174,6 +173,62 @@ class FetchIntervalTest {
nextUpdate(today = "2020-02-05", chapters) shouldBe LocalDate.parse("2020-02-29") nextUpdate(today = "2020-02-05", chapters) shouldBe LocalDate.parse("2020-02-29")
} }
@Test
fun `follows releases on two days of the month`() = runTest {
val days =
listOf("2019-10-01", "2019-10-15", "2019-11-01", "2019-11-16", "2019-12-01", "2019-12-15", "2020-01-01")
val chapters = chaptersOn((days + "2020-01-15").map(LocalDate::parse), perDay = 2)
ReleaseSchedule.estimate(chapters, testTimeZone, LocalDate.parse("2020-01-20")).pattern shouldBe
ReleasePattern.DaysOfMonth(setOf(1, 15))
nextUpdate(today = "2020-01-20", chapters) shouldBe LocalDate.parse("2020-02-01")
}
@Test
fun `follows releases on a weekday of the month`() = runTest {
val firstSaturdays = listOf("2019-09-07", "2019-10-05", "2019-11-02", "2019-12-07", "2020-01-04")
val chapters = chaptersOn(firstSaturdays.map(LocalDate::parse), perDay = 2)
nextUpdate(today = "2020-01-10", chapters) shouldBe LocalDate.parse("2020-02-01")
}
@Test
fun `follows releases on a weekday of the last week of the month`() = runTest {
val lastFridays = listOf("2019-09-27", "2019-10-25", "2019-11-29", "2019-12-27", "2020-01-31")
val chapters = chaptersOn(lastFridays.map(LocalDate::parse), perDay = 2)
nextUpdate(today = "2020-02-03", chapters) shouldBe LocalDate.parse("2020-02-28")
}
@Test
fun `follows releases in batches`() = runTest {
// Three days of releases every three weeks
val batches = every(days = 21, from = "2019-09-02", to = "2019-12-16")
.flatMap { start -> (0..2).map { start.plus(DatePeriod(days = it)) } }
val chapters = chaptersOn(batches.filter { it <= LocalDate.parse("2019-12-17") })
nextUpdate(today = "2019-12-17", chapters, foundRelease = true) shouldBe LocalDate.parse("2019-12-18")
nextUpdate(today = "2019-12-18", chaptersOn(batches), foundRelease = true) shouldBe
LocalDate.parse("2020-01-06")
}
@Test
fun `checks again around when an earlier break ended`() = runTest {
// A season of weekly releases, a 98 day break, and another season
val chapters = chaptersOn(weekly(from = "2019-01-02", to = "2019-03-27") + weekly("2019-07-03", "2019-09-25"))
nextUpdate(today = "2019-12-20", chapters) shouldBe LocalDate.parse("2019-12-25")
nextUpdate(today = "2019-12-26", chapters) shouldBe LocalDate.parse("2020-01-01")
}
@Test
fun `never waits more than 28 days between checks`() = runTest {
val lastDays = listOf("2019-09-30", "2019-10-31", "2019-11-30", "2019-12-31", "2020-01-31")
val chapters = chaptersOn(lastDays.map(LocalDate::parse), perDay = 2)
nextUpdate(today = "2020-01-31", chapters, foundRelease = true) shouldBe LocalDate.parse("2020-02-28")
}
@Test @Test
fun `keeps checking on the expected day`() = runTest { fun `keeps checking on the expected day`() = runTest {
val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25")) val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25"))
@@ -197,6 +252,71 @@ class FetchIntervalTest {
nextUpdate(today = "2020-01-01", chapters, manga) shouldBe LocalDate.parse("2020-01-08") nextUpdate(today = "2020-01-01", chapters, manga) shouldBe LocalDate.parse("2020-01-08")
} }
@Test
fun `keeps the schedule for a release uploaded the day before it was found`() = runTest {
val chapters = chaptersOn(weekly(from = "2019-10-02", to = "2019-12-25") + LocalDate.parse("2019-12-31"))
nextUpdate(today = "2020-01-01", chapters, expecting("2020-01-01"), foundRelease = true) shouldBe
LocalDate.parse("2020-01-08")
}
@Test
fun `keeps the schedule after a release shortly before the expected day`() = runTest {
val chapters = chaptersOn(weekly(from = "2019-10-02", to = "2019-12-25") + LocalDate.parse("2019-12-30"))
nextUpdate(today = "2019-12-30", chapters, expecting("2020-01-01"), foundRelease = true) shouldBe
LocalDate.parse("2020-01-08")
}
@Test
fun `keeps the schedule after a late release`() = runTest {
val chapters = chaptersOn(weekly(from = "2019-12-04", to = "2019-12-25") + LocalDate.parse("2020-01-03"))
nextUpdate(today = "2020-01-03", chapters, expecting("2020-01-01"), foundRelease = true) shouldBe
LocalDate.parse("2020-01-08")
}
@Test
fun `keeps expecting the release after an extra one well before it`() = runTest {
val chapters = chaptersOn(weekly(from = "2019-10-02", to = "2020-01-01") + LocalDate.parse("2020-01-02"))
nextUpdate(today = "2020-01-02", chapters, expecting("2020-01-08"), foundRelease = true) shouldBe
LocalDate.parse("2020-01-08")
}
@Test
fun `follows the schedule again when releases resume after a hiatus`() = runTest {
val chapters = chaptersOn(weekly(from = "2019-09-11", to = "2019-10-02") + LocalDate.parse("2020-01-01"))
nextUpdate(today = "2020-01-01", chapters, expecting("2020-01-22"), foundRelease = true) shouldBe
LocalDate.parse("2020-01-08")
}
@Test
fun `follows the schedule once releases keep coming early`() = runTest {
val history = every(days = 5, from = "2019-10-01", to = "2019-12-15")
val chapters = chaptersOn(history + every(days = 5, from = "2019-12-19", to = "2019-12-29"))
nextUpdate(today = "2019-12-29", chapters, expecting("2019-12-30"), foundRelease = true) shouldBe
LocalDate.parse("2020-01-03")
}
@Test
fun `keeps checking for an overdue release while recent ones came that late`() = runTest {
val chapters = chaptersOn(afterGaps("2019-10-18", 6, 5, 8, 6, 5, 6, 8, 6, 5))
// Due on the 18th from a median gap of 6, and gaps have run to 8
nextUpdate(today = "2019-12-20", chapters, expecting("2019-12-18")) shouldBe LocalDate.parse("2019-12-20")
}
@Test
fun `counts from an early release with a custom interval`() = runTest {
val manga = expecting("2020-01-01").copy(fetchInterval = -7)
nextUpdate(today = "2019-12-31", emptyList(), manga, foundRelease = true) shouldBe
LocalDate.parse("2020-01-07")
}
@Test @Test
fun `spreads out checks during a hiatus`() = runTest { fun `spreads out checks during a hiatus`() = runTest {
val chapters = chaptersOn(weekly(from = "2019-09-11", to = "2019-10-02")) val chapters = chaptersOn(weekly(from = "2019-09-11", to = "2019-10-02"))
@@ -215,6 +335,7 @@ class FetchIntervalTest {
today: String, today: String,
chapters: List<Chapter>, chapters: List<Chapter>,
manga: Manga = Manga.create(), manga: Manga = Manga.create(),
foundRelease: Boolean = false,
): LocalDate { ): LocalDate {
val getChaptersByMangaId = mockk<GetChaptersByMangaId>() val getChaptersByMangaId = mockk<GetChaptersByMangaId>()
coEvery { getChaptersByMangaId.await(any(), any()) } returns chapters coEvery { getChaptersByMangaId.await(any(), any()) } returns chapters
@@ -225,16 +346,35 @@ class FetchIntervalTest {
dateTime = date.atTime(10, 0), dateTime = date.atTime(10, 0),
timeZone = testTimeZone, timeZone = testTimeZone,
window = fetchInterval.getWindow(date, testTimeZone), window = fetchInterval.getWindow(date, testTimeZone),
foundRelease = foundRelease,
) )
return update.nextUpdate!!.toLocalDateTime(testTimeZone).date return update.nextUpdate!!.toLocalDateTime(testTimeZone).date
} }
private fun weekly(from: String, to: String): List<LocalDate> { private fun expecting(day: String): Manga {
return generateSequence(LocalDate.parse(from)) { it.plus(DatePeriod(days = 7)) } return Manga.create().copy(nextUpdate = LocalDate.parse(day).atStartOfDayIn(testTimeZone))
}
private fun interval(chapters: List<Chapter>): Int {
return ReleaseSchedule.estimate(
chapters,
testTimeZone,
Clock.System.now().toLocalDateTime(testTimeZone).date,
).interval
}
private fun weekly(from: String, to: String) = every(days = 7, from, to)
private fun every(days: Int, from: String, to: String): List<LocalDate> {
return generateSequence(LocalDate.parse(from)) { it.plus(DatePeriod(days = days)) }
.takeWhile { it <= LocalDate.parse(to) } .takeWhile { it <= LocalDate.parse(to) }
.toList() .toList()
} }
private fun afterGaps(from: String, vararg gaps: Int): List<LocalDate> {
return gaps.runningFold(LocalDate.parse(from)) { day, gap -> day.plus(DatePeriod(days = gap)) }
}
private fun chaptersOn(days: List<LocalDate>, perDay: Int = 1): List<Chapter> { private fun chaptersOn(days: List<LocalDate>, perDay: Int = 1): List<Chapter> {
return days.flatMap { day -> return days.flatMap { day ->
val time = day.atStartOfDayIn(testTimeZone) + 12.hours val time = day.atStartOfDayIn(testTimeZone) + 12.hours