github/tachiyomi
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Clean up fetch interval tests a bit

Browse source 
Also limit the dates we look at to most recent 10 distinct dates only. Closes #9930
This commit is contained in:
arkon 2023-09-17 12:06:17 -04:00
parent e5f83d0c6e
commit 6663abebaf
8 changed files with 167 additions and 139 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/src/main/java/eu/kanade/domain/DomainModule.kt
View file

@ -50,6 +50,7 @@ import tachiyomi.domain.history.interactor.GetTotalReadDuration
import tachiyomi.domain.history.interactor.RemoveHistory
import tachiyomi.domain.history.interactor.UpsertHistory
import tachiyomi.domain.history.repository.HistoryRepository
import tachiyomi.domain.manga.interactor.FetchInterval
import tachiyomi.domain.manga.interactor.GetDuplicateLibraryManga
import tachiyomi.domain.manga.interactor.GetFavorites
import tachiyomi.domain.manga.interactor.GetLibraryManga
@ -57,7 +58,6 @@ import tachiyomi.domain.manga.interactor.GetManga
import tachiyomi.domain.manga.interactor.GetMangaWithChapters
import tachiyomi.domain.manga.interactor.NetworkToLocalManga
import tachiyomi.domain.manga.interactor.ResetViewerFlags
import tachiyomi.domain.manga.interactor.SetFetchInterval
import tachiyomi.domain.manga.interactor.SetMangaChapterFlags
import tachiyomi.domain.manga.repository.MangaRepository
import tachiyomi.domain.release.interactor.GetApplicationRelease
@ -102,7 +102,7 @@ class DomainModule : InjektModule {
addFactory { GetNextChapters(get(), get(), get()) }
addFactory { ResetViewerFlags(get()) }
addFactory { SetMangaChapterFlags(get()) }
addFactory { SetFetchInterval(get()) }
addFactory { FetchInterval(get()) }
addFactory { SetMangaDefaultChapterFlags(get(), get(), get()) }
addFactory { SetMangaViewerFlags(get()) }
addFactory { NetworkToLocalManga(get()) }

8
app/src/main/java/eu/kanade/domain/manga/interactor/UpdateManga.kt
View file

@ -3,7 +3,7 @@ package eu.kanade.domain.manga.interactor
import eu.kanade.domain.manga.model.hasCustomCover
import eu.kanade.tachiyomi.data.cache.CoverCache
import eu.kanade.tachiyomi.source.model.SManga
import tachiyomi.domain.manga.interactor.SetFetchInterval
import tachiyomi.domain.manga.interactor.FetchInterval
import tachiyomi.domain.manga.model.Manga
import tachiyomi.domain.manga.model.MangaUpdate
import tachiyomi.domain.manga.repository.MangaRepository
@ -15,7 +15,7 @@ import java.util.Date
class UpdateManga(
private val mangaRepository: MangaRepository,
private val setFetchInterval: SetFetchInterval,
private val fetchInterval: FetchInterval,
) {
suspend fun await(mangaUpdate: MangaUpdate): Boolean {
@ -79,9 +79,9 @@ class UpdateManga(
suspend fun awaitUpdateFetchInterval(
manga: Manga,
dateTime: ZonedDateTime = ZonedDateTime.now(),
window: Pair<Long, Long> = setFetchInterval.getWindow(dateTime),
window: Pair<Long, Long> = fetchInterval.getWindow(dateTime),
): Boolean {
return setFetchInterval.toMangaUpdateOrNull(manga, dateTime, window)
return fetchInterval.toMangaUpdateOrNull(manga, dateTime, window)
?.let { mangaRepository.update(it) }
?: false
}

4
app/src/main/java/eu/kanade/presentation/manga/components/MangaDialogs.kt
View file

@ -16,7 +16,7 @@ import androidx.compose.ui.res.stringResource
import androidx.compose.ui.unit.DpSize
import androidx.compose.ui.unit.dp
import eu.kanade.tachiyomi.R
import tachiyomi.domain.manga.interactor.MAX_FETCH_INTERVAL
import tachiyomi.domain.manga.interactor.FetchInterval
import tachiyomi.presentation.core.components.WheelTextPicker
@Composable
@ -67,7 +67,7 @@ fun SetIntervalDialog(
contentAlignment = Alignment.Center,
) {
val size = DpSize(width = maxWidth / 2, height = 128.dp)
val items = (0..MAX_FETCH_INTERVAL).map {
val items = (0..FetchInterval.MAX_INTERVAL).map {
if (it == 0) {
stringResource(R.string.label_default)
} else {

8
app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupRestorer.kt
View file

@ -14,7 +14,7 @@ import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.isActive
import tachiyomi.domain.chapter.model.Chapter
import tachiyomi.domain.chapter.repository.ChapterRepository
import tachiyomi.domain.manga.interactor.SetFetchInterval
import tachiyomi.domain.manga.interactor.FetchInterval
import tachiyomi.domain.manga.model.Manga
import tachiyomi.domain.track.model.Track
import uy.kohesive.injekt.Injekt
@ -31,10 +31,10 @@ class BackupRestorer(
) {
private val updateManga: UpdateManga = Injekt.get()
private val chapterRepository: ChapterRepository = Injekt.get()
private val setFetchInterval: SetFetchInterval = Injekt.get()
private val fetchInterval: FetchInterval = Injekt.get()
private var now = ZonedDateTime.now()
private var currentFetchWindow = setFetchInterval.getWindow(now)
private var currentFetchWindow = fetchInterval.getWindow(now)
private var backupManager = BackupManager(context)
@ -103,7 +103,7 @@ class BackupRestorer(
val backupMaps = backup.backupBrokenSources.map { BackupSource(it.name, it.sourceId) } + backup.backupSources
sourceMapping = backupMaps.associate { it.sourceId to it.name }
now = ZonedDateTime.now()
currentFetchWindow = setFetchInterval.getWindow(now)
currentFetchWindow = fetchInterval.getWindow(now)
return coroutineScope {
// Restore individual manga

6
app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateJob.kt
View file

@ -59,9 +59,9 @@ import tachiyomi.domain.library.service.LibraryPreferences.Companion.MANGA_HAS_U
import tachiyomi.domain.library.service.LibraryPreferences.Companion.MANGA_NON_COMPLETED
import tachiyomi.domain.library.service.LibraryPreferences.Companion.MANGA_NON_READ
import tachiyomi.domain.library.service.LibraryPreferences.Companion.MANGA_OUTSIDE_RELEASE_PERIOD
import tachiyomi.domain.manga.interactor.FetchInterval
import tachiyomi.domain.manga.interactor.GetLibraryManga
import tachiyomi.domain.manga.interactor.GetManga
import tachiyomi.domain.manga.interactor.SetFetchInterval
import tachiyomi.domain.manga.model.Manga
import tachiyomi.domain.manga.model.toMangaUpdate
import tachiyomi.domain.source.model.SourceNotInstalledException
@ -90,7 +90,7 @@ class LibraryUpdateJob(private val context: Context, workerParams: WorkerParamet
private val getCategories: GetCategories = Injekt.get()
private val syncChaptersWithSource: SyncChaptersWithSource = Injekt.get()
private val refreshTracks: RefreshTracks = Injekt.get()
private val setFetchInterval: SetFetchInterval = Injekt.get()
private val fetchInterval: FetchInterval = Injekt.get()
private val notifier = LibraryUpdateNotifier(context)
@ -216,7 +216,7 @@ class LibraryUpdateJob(private val context: Context, workerParams: WorkerParamet
val failedUpdates = CopyOnWriteArrayList<Pair<Manga, String?>>()
val hasDownloads = AtomicBoolean(false)
val restrictions = libraryPreferences.autoUpdateMangaRestrictions().get()
val fetchWindow = setFetchInterval.getWindow(ZonedDateTime.now())
val fetchWindow = fetchInterval.getWindow(ZonedDateTime.now())
coroutineScope {
mangaToUpdate.groupBy { it.manga.source }.values

45
domain/src/main/java/tachiyomi/domain/manga/interactor/SetFetchInterval.kt → domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt
View file

@ -5,14 +5,12 @@ import tachiyomi.domain.chapter.model.Chapter
import tachiyomi.domain.manga.model.Manga
import tachiyomi.domain.manga.model.MangaUpdate
import java.time.Instant
import java.time.ZoneId
import java.time.ZonedDateTime
import java.time.temporal.ChronoUnit
import kotlin.math.absoluteValue
const val MAX_FETCH_INTERVAL = 28
private const val FETCH_INTERVAL_GRACE_PERIOD = 1
class SetFetchInterval(
class FetchInterval(
private val getChapterByMangaId: GetChapterByMangaId,
) {
@ -29,7 +27,7 @@ class SetFetchInterval(
val chapters = getChapterByMangaId.await(manga.id)
val interval = manga.fetchInterval.takeIf { it < 0 } ?: calculateInterval(
chapters,
dateTime,
dateTime.zone,
)
val nextUpdate = calculateNextUpdate(manga, interval, dateTime, currentWindow)
@ -42,33 +40,34 @@ class SetFetchInterval(
fun getWindow(dateTime: ZonedDateTime): Pair<Long, Long> {
val today = dateTime.toLocalDate().atStartOfDay(dateTime.zone)
val lowerBound = today.minusDays(FETCH_INTERVAL_GRACE_PERIOD.toLong())
val upperBound = today.plusDays(FETCH_INTERVAL_GRACE_PERIOD.toLong())
val lowerBound = today.minusDays(GRACE_PERIOD)
val upperBound = today.plusDays(GRACE_PERIOD)
return Pair(lowerBound.toEpochSecond() * 1000, upperBound.toEpochSecond() * 1000 - 1)
}
internal fun calculateInterval(chapters: List<Chapter>, zonedDateTime: ZonedDateTime): Int {
val sortedChapters = chapters
.sortedWith(
compareByDescending<Chapter> { it.dateUpload }.thenByDescending { it.dateFetch },
)
.take(50)
val uploadDates = sortedChapters
internal fun calculateInterval(chapters: List<Chapter>, zone: ZoneId): Int {
val uploadDates = chapters.asSequence()
.filter { it.dateUpload > 0L }
.sortedByDescending { it.dateUpload }
.map {
ZonedDateTime.ofInstant(Instant.ofEpochMilli(it.dateUpload), zonedDateTime.zone)
ZonedDateTime.ofInstant(Instant.ofEpochMilli(it.dateUpload), zone)
.toLocalDate()
.atStartOfDay()
}
.distinct()
val fetchDates = sortedChapters
.take(10)
.toList()
val fetchDates = chapters.asSequence()
.sortedByDescending { it.dateFetch }
.map {
ZonedDateTime.ofInstant(Instant.ofEpochMilli(it.dateFetch), zonedDateTime.zone)
ZonedDateTime.ofInstant(Instant.ofEpochMilli(it.dateFetch), zone)
.toLocalDate()
.atStartOfDay()
}
.distinct()
.take(10)
.toList()
val interval = when {
// Enough upload date from source
@ -87,7 +86,7 @@ class SetFetchInterval(
else -> 7
}
return interval.coerceIn(1, MAX_FETCH_INTERVAL)
return interval.coerceIn(1, MAX_INTERVAL)
}
private fun calculateNextUpdate(
@ -118,7 +117,7 @@ class SetFetchInterval(
}
private fun doubleInterval(delta: Int, timeSinceLatest: Int, doubleWhenOver: Int): Int {
if (delta >= MAX_FETCH_INTERVAL) return MAX_FETCH_INTERVAL
if (delta >= MAX_INTERVAL) return MAX_INTERVAL
// double delta again if missed more than 9 check in new delta
val cycle = timeSinceLatest.floorDiv(delta) + 1
@ -128,4 +127,10 @@ class SetFetchInterval(
delta
}
}
companion object {
const val MAX_INTERVAL = 28
private const val GRACE_PERIOD = 1L
}
}

127
domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt Normal file
View file

@ -0,0 +1,127 @@
package tachiyomi.domain.manga.interactor
import io.kotest.matchers.shouldBe
import io.mockk.mockk
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.parallel.Execution
import org.junit.jupiter.api.parallel.ExecutionMode
import tachiyomi.domain.chapter.model.Chapter
import java.time.ZoneOffset
import java.time.ZonedDateTime
import kotlin.time.Duration
import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.hours
import kotlin.time.DurationUnit
import kotlin.time.toDuration
import kotlin.time.toJavaDuration
@Execution(ExecutionMode.CONCURRENT)
class FetchIntervalTest {
private val testTime = ZonedDateTime.parse("2020-01-01T00:00:00Z")
private val testZoneId = ZoneOffset.UTC
private var chapter = Chapter.create().copy(
dateFetch = testTime.toEpochSecond() * 1000,
dateUpload = testTime.toEpochSecond() * 1000,
)
private val fetchInterval = FetchInterval(mockk())
@Test
fun `returns default interval of 7 days when not enough distinct days`() {
val chaptersWithUploadDate = (1..50).map {
chapterWithTime(chapter, 1.days)
}
fetchInterval.calculateInterval(chaptersWithUploadDate, testZoneId) shouldBe 7
val chaptersWithoutUploadDate = chaptersWithUploadDate.map {
it.copy(dateUpload = 0L)
}
fetchInterval.calculateInterval(chaptersWithoutUploadDate, testZoneId) shouldBe 7
}
@Test
fun `returns interval based on more recent chapters`() {
val oldChapters = (1..5).map {
chapterWithTime(chapter, (it * 7).days) // Would have interval of 7 days
}
val newChapters = (1..10).map {
chapterWithTime(chapter, oldChapters.lastUploadDate() + it.days)
}
val chapters = oldChapters + newChapters
fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 1
}
@Test
fun `returns interval of 7 days when multiple chapters in 1 day`() {
val chapters = (1..10).map {
chapterWithTime(chapter, 10.hours)
}
fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 7
}
@Test
fun `returns interval of 7 days when multiple chapters in 2 days`() {
val chapters = (1..2).map {
chapterWithTime(chapter, 1.days)
} + (1..5).map {
chapterWithTime(chapter, 2.days)
}
fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 7
}
@Test
fun `returns interval of 1 day when chapters are released every 1 day`() {
val chapters = (1..20).map {
chapterWithTime(chapter, it.days)
}
fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 1
}
@Test
fun `returns interval of 1 day when delta is less than 1 day`() {
val chapters = (1..20).map {
chapterWithTime(chapter, (15 * it).hours)
}
fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 1
}
@Test
fun `returns interval of 2 days when chapters are released every 2 days`() {
val chapters = (1..20).map {
chapterWithTime(chapter, (2 * it).days)
}
fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 2
}
@Test
fun `returns interval with floored value when interval is decimal`() {
val chaptersWithUploadDate = (1..5).map {
chapterWithTime(chapter, (25 * it).hours)
}
fetchInterval.calculateInterval(chaptersWithUploadDate, testZoneId) shouldBe 1
val chaptersWithoutUploadDate = chaptersWithUploadDate.map {
it.copy(dateUpload = 0L)
}
fetchInterval.calculateInterval(chaptersWithoutUploadDate, testZoneId) shouldBe 1
}
@Test
fun `returns interval of 1 day when chapters are released just below every 2 days`() {
val chapters = (1..20).map {
chapterWithTime(chapter, (43 * it).hours)
}
fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 1
}
private fun chapterWithTime(chapter: Chapter, duration: Duration): Chapter {
val newTime = testTime.plus(duration.toJavaDuration()).toEpochSecond() * 1000
return chapter.copy(dateFetch = newTime, dateUpload = newTime)
}
private fun List<Chapter>.lastUploadDate() =
last().dateUpload.toDuration(DurationUnit.MILLISECONDS)
}

104
domain/src/test/java/tachiyomi/domain/manga/interactor/SetFetchIntervalTest.kt
View file

@ -1,104 +0,0 @@
package tachiyomi.domain.manga.interactor
import io.kotest.matchers.shouldBe
import io.mockk.mockk
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.parallel.Execution
import org.junit.jupiter.api.parallel.ExecutionMode
import tachiyomi.domain.chapter.model.Chapter
import java.time.ZonedDateTime
import kotlin.time.Duration
import kotlin.time.Duration.Companion.hours
import kotlin.time.toJavaDuration
@Execution(ExecutionMode.CONCURRENT)
class SetFetchIntervalTest {
private val testTime = ZonedDateTime.parse("2020-01-01T00:00:00Z")
private var chapter = Chapter.create().copy(
dateFetch = testTime.toEpochSecond() * 1000,
dateUpload = testTime.toEpochSecond() * 1000,
)
private val setFetchInterval = SetFetchInterval(mockk())
@Test
fun `calculateInterval returns default of 7 days when less than 3 distinct days`() {
val chapters = (1..2).map {
chapterWithTime(chapter, 10.hours)
}
setFetchInterval.calculateInterval(chapters, testTime) shouldBe 7
}
@Test
fun `calculateInterval returns 7 when 5 chapters in 1 day`() {
val chapters = (1..5).map {
chapterWithTime(chapter, 10.hours)
}
setFetchInterval.calculateInterval(chapters, testTime) shouldBe 7
}
@Test
fun `calculateInterval returns 7 when 7 chapters in 48 hours, 2 day`() {
val chapters = (1..2).map {
chapterWithTime(chapter, 24.hours)
} + (1..5).map {
chapterWithTime(chapter, 48.hours)
}
setFetchInterval.calculateInterval(chapters, testTime) shouldBe 7
}
@Test
fun `calculateInterval returns default of 1 day when interval less than 1`() {
val chapters = (1..5).map {
chapterWithTime(chapter, (15 * it).hours)
}
setFetchInterval.calculateInterval(chapters, testTime) shouldBe 1
}
// Normal interval calculation
@Test
fun `calculateInterval returns 1 when 5 chapters in 120 hours, 5 days`() {
val chapters = (1..5).map {
chapterWithTime(chapter, (24 * it).hours)
}
setFetchInterval.calculateInterval(chapters, testTime) shouldBe 1
}
@Test
fun `calculateInterval returns 2 when 5 chapters in 240 hours, 10 days`() {
val chapters = (1..5).map {
chapterWithTime(chapter, (48 * it).hours)
}
setFetchInterval.calculateInterval(chapters, testTime) shouldBe 2
}
@Test
fun `calculateInterval returns floored value when interval is decimal`() {
val chapters = (1..5).map {
chapterWithTime(chapter, (25 * it).hours)
}
setFetchInterval.calculateInterval(chapters, testTime) shouldBe 1
}
@Test
fun `calculateInterval returns 1 when 5 chapters in 215 hours, 5 days`() {
val chapters = (1..5).map {
chapterWithTime(chapter, (43 * it).hours)
}
setFetchInterval.calculateInterval(chapters, testTime) shouldBe 1
}
@Test
fun `calculateInterval returns interval based on fetch time if upload time not available`() {
val chapters = (1..5).map {
chapterWithTime(chapter, (25 * it).hours).copy(dateUpload = 0L)
}
setFetchInterval.calculateInterval(chapters, testTime) shouldBe 1
}
private fun chapterWithTime(chapter: Chapter, duration: Duration): Chapter {
val newTime = testTime.plus(duration.toJavaDuration()).toEpochSecond() * 1000
return chapter.copy(dateFetch = newTime, dateUpload = newTime)
}
}