From 6663abebaf69241741d0702331857951834ba806 Mon Sep 17 00:00:00 2001 From: arkon Date: Sun, 17 Sep 2023 12:06:17 -0400 Subject: [PATCH] Clean up fetch interval tests a bit Also limit the dates we look at to most recent 10 distinct dates only. Closes #9930 --- .../java/eu/kanade/domain/DomainModule.kt | 4 +- .../domain/manga/interactor/UpdateManga.kt | 8 +- .../manga/components/MangaDialogs.kt | 4 +- .../tachiyomi/data/backup/BackupRestorer.kt | 8 +- .../data/library/LibraryUpdateJob.kt | 6 +- .../{SetFetchInterval.kt => FetchInterval.kt} | 45 ++++--- .../manga/interactor/FetchIntervalTest.kt | 127 ++++++++++++++++++ .../manga/interactor/SetFetchIntervalTest.kt | 104 -------------- 8 files changed, 167 insertions(+), 139 deletions(-) rename domain/src/main/java/tachiyomi/domain/manga/interactor/{SetFetchInterval.kt => FetchInterval.kt} (83%) create mode 100644 domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt delete mode 100644 domain/src/test/java/tachiyomi/domain/manga/interactor/SetFetchIntervalTest.kt diff --git a/app/src/main/java/eu/kanade/domain/DomainModule.kt b/app/src/main/java/eu/kanade/domain/DomainModule.kt index 494033c877..f5e428e8af 100644 --- a/app/src/main/java/eu/kanade/domain/DomainModule.kt +++ b/app/src/main/java/eu/kanade/domain/DomainModule.kt @@ -50,6 +50,7 @@ import tachiyomi.domain.history.interactor.GetTotalReadDuration import tachiyomi.domain.history.interactor.RemoveHistory import tachiyomi.domain.history.interactor.UpsertHistory import tachiyomi.domain.history.repository.HistoryRepository +import tachiyomi.domain.manga.interactor.FetchInterval import tachiyomi.domain.manga.interactor.GetDuplicateLibraryManga import tachiyomi.domain.manga.interactor.GetFavorites import tachiyomi.domain.manga.interactor.GetLibraryManga @@ -57,7 +58,6 @@ import tachiyomi.domain.manga.interactor.GetManga import tachiyomi.domain.manga.interactor.GetMangaWithChapters import tachiyomi.domain.manga.interactor.NetworkToLocalManga import tachiyomi.domain.manga.interactor.ResetViewerFlags -import tachiyomi.domain.manga.interactor.SetFetchInterval import tachiyomi.domain.manga.interactor.SetMangaChapterFlags import tachiyomi.domain.manga.repository.MangaRepository import tachiyomi.domain.release.interactor.GetApplicationRelease @@ -102,7 +102,7 @@ class DomainModule : InjektModule { addFactory { GetNextChapters(get(), get(), get()) } addFactory { ResetViewerFlags(get()) } addFactory { SetMangaChapterFlags(get()) } - addFactory { SetFetchInterval(get()) } + addFactory { FetchInterval(get()) } addFactory { SetMangaDefaultChapterFlags(get(), get(), get()) } addFactory { SetMangaViewerFlags(get()) } addFactory { NetworkToLocalManga(get()) } diff --git a/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateManga.kt b/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateManga.kt index 38d6083ff8..468ea23898 100644 --- a/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateManga.kt +++ b/app/src/main/java/eu/kanade/domain/manga/interactor/UpdateManga.kt @@ -3,7 +3,7 @@ package eu.kanade.domain.manga.interactor import eu.kanade.domain.manga.model.hasCustomCover import eu.kanade.tachiyomi.data.cache.CoverCache import eu.kanade.tachiyomi.source.model.SManga -import tachiyomi.domain.manga.interactor.SetFetchInterval +import tachiyomi.domain.manga.interactor.FetchInterval import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.manga.model.MangaUpdate import tachiyomi.domain.manga.repository.MangaRepository @@ -15,7 +15,7 @@ import java.util.Date class UpdateManga( private val mangaRepository: MangaRepository, - private val setFetchInterval: SetFetchInterval, + private val fetchInterval: FetchInterval, ) { suspend fun await(mangaUpdate: MangaUpdate): Boolean { @@ -79,9 +79,9 @@ class UpdateManga( suspend fun awaitUpdateFetchInterval( manga: Manga, dateTime: ZonedDateTime = ZonedDateTime.now(), - window: Pair = setFetchInterval.getWindow(dateTime), + window: Pair = fetchInterval.getWindow(dateTime), ): Boolean { - return setFetchInterval.toMangaUpdateOrNull(manga, dateTime, window) + return fetchInterval.toMangaUpdateOrNull(manga, dateTime, window) ?.let { mangaRepository.update(it) } ?: false } diff --git a/app/src/main/java/eu/kanade/presentation/manga/components/MangaDialogs.kt b/app/src/main/java/eu/kanade/presentation/manga/components/MangaDialogs.kt index 1322adb26b..94f34eec2b 100644 --- a/app/src/main/java/eu/kanade/presentation/manga/components/MangaDialogs.kt +++ b/app/src/main/java/eu/kanade/presentation/manga/components/MangaDialogs.kt @@ -16,7 +16,7 @@ import androidx.compose.ui.res.stringResource import androidx.compose.ui.unit.DpSize import androidx.compose.ui.unit.dp import eu.kanade.tachiyomi.R -import tachiyomi.domain.manga.interactor.MAX_FETCH_INTERVAL +import tachiyomi.domain.manga.interactor.FetchInterval import tachiyomi.presentation.core.components.WheelTextPicker @Composable @@ -67,7 +67,7 @@ fun SetIntervalDialog( contentAlignment = Alignment.Center, ) { val size = DpSize(width = maxWidth / 2, height = 128.dp) - val items = (0..MAX_FETCH_INTERVAL).map { + val items = (0..FetchInterval.MAX_INTERVAL).map { if (it == 0) { stringResource(R.string.label_default) } else { diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupRestorer.kt b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupRestorer.kt index 544c5c18d9..765a700c12 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupRestorer.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/backup/BackupRestorer.kt @@ -14,7 +14,7 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.isActive import tachiyomi.domain.chapter.model.Chapter import tachiyomi.domain.chapter.repository.ChapterRepository -import tachiyomi.domain.manga.interactor.SetFetchInterval +import tachiyomi.domain.manga.interactor.FetchInterval import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.track.model.Track import uy.kohesive.injekt.Injekt @@ -31,10 +31,10 @@ class BackupRestorer( ) { private val updateManga: UpdateManga = Injekt.get() private val chapterRepository: ChapterRepository = Injekt.get() - private val setFetchInterval: SetFetchInterval = Injekt.get() + private val fetchInterval: FetchInterval = Injekt.get() private var now = ZonedDateTime.now() - private var currentFetchWindow = setFetchInterval.getWindow(now) + private var currentFetchWindow = fetchInterval.getWindow(now) private var backupManager = BackupManager(context) @@ -103,7 +103,7 @@ class BackupRestorer( val backupMaps = backup.backupBrokenSources.map { BackupSource(it.name, it.sourceId) } + backup.backupSources sourceMapping = backupMaps.associate { it.sourceId to it.name } now = ZonedDateTime.now() - currentFetchWindow = setFetchInterval.getWindow(now) + currentFetchWindow = fetchInterval.getWindow(now) return coroutineScope { // Restore individual manga diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateJob.kt b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateJob.kt index f5e1b88b4a..734b5180c4 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateJob.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateJob.kt @@ -59,9 +59,9 @@ import tachiyomi.domain.library.service.LibraryPreferences.Companion.MANGA_HAS_U import tachiyomi.domain.library.service.LibraryPreferences.Companion.MANGA_NON_COMPLETED import tachiyomi.domain.library.service.LibraryPreferences.Companion.MANGA_NON_READ import tachiyomi.domain.library.service.LibraryPreferences.Companion.MANGA_OUTSIDE_RELEASE_PERIOD +import tachiyomi.domain.manga.interactor.FetchInterval import tachiyomi.domain.manga.interactor.GetLibraryManga import tachiyomi.domain.manga.interactor.GetManga -import tachiyomi.domain.manga.interactor.SetFetchInterval import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.manga.model.toMangaUpdate import tachiyomi.domain.source.model.SourceNotInstalledException @@ -90,7 +90,7 @@ class LibraryUpdateJob(private val context: Context, workerParams: WorkerParamet private val getCategories: GetCategories = Injekt.get() private val syncChaptersWithSource: SyncChaptersWithSource = Injekt.get() private val refreshTracks: RefreshTracks = Injekt.get() - private val setFetchInterval: SetFetchInterval = Injekt.get() + private val fetchInterval: FetchInterval = Injekt.get() private val notifier = LibraryUpdateNotifier(context) @@ -216,7 +216,7 @@ class LibraryUpdateJob(private val context: Context, workerParams: WorkerParamet val failedUpdates = CopyOnWriteArrayList>() val hasDownloads = AtomicBoolean(false) val restrictions = libraryPreferences.autoUpdateMangaRestrictions().get() - val fetchWindow = setFetchInterval.getWindow(ZonedDateTime.now()) + val fetchWindow = fetchInterval.getWindow(ZonedDateTime.now()) coroutineScope { mangaToUpdate.groupBy { it.manga.source }.values diff --git a/domain/src/main/java/tachiyomi/domain/manga/interactor/SetFetchInterval.kt b/domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt similarity index 83% rename from domain/src/main/java/tachiyomi/domain/manga/interactor/SetFetchInterval.kt rename to domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt index 8be4350478..740c0a1506 100644 --- a/domain/src/main/java/tachiyomi/domain/manga/interactor/SetFetchInterval.kt +++ b/domain/src/main/java/tachiyomi/domain/manga/interactor/FetchInterval.kt @@ -5,14 +5,12 @@ import tachiyomi.domain.chapter.model.Chapter import tachiyomi.domain.manga.model.Manga import tachiyomi.domain.manga.model.MangaUpdate import java.time.Instant +import java.time.ZoneId import java.time.ZonedDateTime import java.time.temporal.ChronoUnit import kotlin.math.absoluteValue -const val MAX_FETCH_INTERVAL = 28 -private const val FETCH_INTERVAL_GRACE_PERIOD = 1 - -class SetFetchInterval( +class FetchInterval( private val getChapterByMangaId: GetChapterByMangaId, ) { @@ -29,7 +27,7 @@ class SetFetchInterval( val chapters = getChapterByMangaId.await(manga.id) val interval = manga.fetchInterval.takeIf { it < 0 } ?: calculateInterval( chapters, - dateTime, + dateTime.zone, ) val nextUpdate = calculateNextUpdate(manga, interval, dateTime, currentWindow) @@ -42,33 +40,34 @@ class SetFetchInterval( fun getWindow(dateTime: ZonedDateTime): Pair { val today = dateTime.toLocalDate().atStartOfDay(dateTime.zone) - val lowerBound = today.minusDays(FETCH_INTERVAL_GRACE_PERIOD.toLong()) - val upperBound = today.plusDays(FETCH_INTERVAL_GRACE_PERIOD.toLong()) + val lowerBound = today.minusDays(GRACE_PERIOD) + val upperBound = today.plusDays(GRACE_PERIOD) return Pair(lowerBound.toEpochSecond() * 1000, upperBound.toEpochSecond() * 1000 - 1) } - internal fun calculateInterval(chapters: List, zonedDateTime: ZonedDateTime): Int { - val sortedChapters = chapters - .sortedWith( - compareByDescending { it.dateUpload }.thenByDescending { it.dateFetch }, - ) - .take(50) - - val uploadDates = sortedChapters + internal fun calculateInterval(chapters: List, zone: ZoneId): Int { + val uploadDates = chapters.asSequence() .filter { it.dateUpload > 0L } + .sortedByDescending { it.dateUpload } .map { - ZonedDateTime.ofInstant(Instant.ofEpochMilli(it.dateUpload), zonedDateTime.zone) + ZonedDateTime.ofInstant(Instant.ofEpochMilli(it.dateUpload), zone) .toLocalDate() .atStartOfDay() } .distinct() - val fetchDates = sortedChapters + .take(10) + .toList() + + val fetchDates = chapters.asSequence() + .sortedByDescending { it.dateFetch } .map { - ZonedDateTime.ofInstant(Instant.ofEpochMilli(it.dateFetch), zonedDateTime.zone) + ZonedDateTime.ofInstant(Instant.ofEpochMilli(it.dateFetch), zone) .toLocalDate() .atStartOfDay() } .distinct() + .take(10) + .toList() val interval = when { // Enough upload date from source @@ -87,7 +86,7 @@ class SetFetchInterval( else -> 7 } - return interval.coerceIn(1, MAX_FETCH_INTERVAL) + return interval.coerceIn(1, MAX_INTERVAL) } private fun calculateNextUpdate( @@ -118,7 +117,7 @@ class SetFetchInterval( } private fun doubleInterval(delta: Int, timeSinceLatest: Int, doubleWhenOver: Int): Int { - if (delta >= MAX_FETCH_INTERVAL) return MAX_FETCH_INTERVAL + if (delta >= MAX_INTERVAL) return MAX_INTERVAL // double delta again if missed more than 9 check in new delta val cycle = timeSinceLatest.floorDiv(delta) + 1 @@ -128,4 +127,10 @@ class SetFetchInterval( delta } } + + companion object { + const val MAX_INTERVAL = 28 + + private const val GRACE_PERIOD = 1L + } } diff --git a/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt b/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt new file mode 100644 index 0000000000..468d7eb2dd --- /dev/null +++ b/domain/src/test/java/tachiyomi/domain/manga/interactor/FetchIntervalTest.kt @@ -0,0 +1,127 @@ +package tachiyomi.domain.manga.interactor + +import io.kotest.matchers.shouldBe +import io.mockk.mockk +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.parallel.Execution +import org.junit.jupiter.api.parallel.ExecutionMode +import tachiyomi.domain.chapter.model.Chapter +import java.time.ZoneOffset +import java.time.ZonedDateTime +import kotlin.time.Duration +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.hours +import kotlin.time.DurationUnit +import kotlin.time.toDuration +import kotlin.time.toJavaDuration + +@Execution(ExecutionMode.CONCURRENT) +class FetchIntervalTest { + + private val testTime = ZonedDateTime.parse("2020-01-01T00:00:00Z") + private val testZoneId = ZoneOffset.UTC + private var chapter = Chapter.create().copy( + dateFetch = testTime.toEpochSecond() * 1000, + dateUpload = testTime.toEpochSecond() * 1000, + ) + + private val fetchInterval = FetchInterval(mockk()) + + @Test + fun `returns default interval of 7 days when not enough distinct days`() { + val chaptersWithUploadDate = (1..50).map { + chapterWithTime(chapter, 1.days) + } + fetchInterval.calculateInterval(chaptersWithUploadDate, testZoneId) shouldBe 7 + + val chaptersWithoutUploadDate = chaptersWithUploadDate.map { + it.copy(dateUpload = 0L) + } + fetchInterval.calculateInterval(chaptersWithoutUploadDate, testZoneId) shouldBe 7 + } + + @Test + fun `returns interval based on more recent chapters`() { + val oldChapters = (1..5).map { + chapterWithTime(chapter, (it * 7).days) // Would have interval of 7 days + } + val newChapters = (1..10).map { + chapterWithTime(chapter, oldChapters.lastUploadDate() + it.days) + } + + val chapters = oldChapters + newChapters + + fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 1 + } + + @Test + fun `returns interval of 7 days when multiple chapters in 1 day`() { + val chapters = (1..10).map { + chapterWithTime(chapter, 10.hours) + } + fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 7 + } + + @Test + fun `returns interval of 7 days when multiple chapters in 2 days`() { + val chapters = (1..2).map { + chapterWithTime(chapter, 1.days) + } + (1..5).map { + chapterWithTime(chapter, 2.days) + } + fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 7 + } + + @Test + fun `returns interval of 1 day when chapters are released every 1 day`() { + val chapters = (1..20).map { + chapterWithTime(chapter, it.days) + } + fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 1 + } + + @Test + fun `returns interval of 1 day when delta is less than 1 day`() { + val chapters = (1..20).map { + chapterWithTime(chapter, (15 * it).hours) + } + fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 1 + } + + @Test + fun `returns interval of 2 days when chapters are released every 2 days`() { + val chapters = (1..20).map { + chapterWithTime(chapter, (2 * it).days) + } + fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 2 + } + + @Test + fun `returns interval with floored value when interval is decimal`() { + val chaptersWithUploadDate = (1..5).map { + chapterWithTime(chapter, (25 * it).hours) + } + fetchInterval.calculateInterval(chaptersWithUploadDate, testZoneId) shouldBe 1 + + val chaptersWithoutUploadDate = chaptersWithUploadDate.map { + it.copy(dateUpload = 0L) + } + fetchInterval.calculateInterval(chaptersWithoutUploadDate, testZoneId) shouldBe 1 + } + + @Test + fun `returns interval of 1 day when chapters are released just below every 2 days`() { + val chapters = (1..20).map { + chapterWithTime(chapter, (43 * it).hours) + } + fetchInterval.calculateInterval(chapters, testZoneId) shouldBe 1 + } + + private fun chapterWithTime(chapter: Chapter, duration: Duration): Chapter { + val newTime = testTime.plus(duration.toJavaDuration()).toEpochSecond() * 1000 + return chapter.copy(dateFetch = newTime, dateUpload = newTime) + } + + private fun List.lastUploadDate() = + last().dateUpload.toDuration(DurationUnit.MILLISECONDS) +} diff --git a/domain/src/test/java/tachiyomi/domain/manga/interactor/SetFetchIntervalTest.kt b/domain/src/test/java/tachiyomi/domain/manga/interactor/SetFetchIntervalTest.kt deleted file mode 100644 index 8c329ca228..0000000000 --- a/domain/src/test/java/tachiyomi/domain/manga/interactor/SetFetchIntervalTest.kt +++ /dev/null @@ -1,104 +0,0 @@ -package tachiyomi.domain.manga.interactor - -import io.kotest.matchers.shouldBe -import io.mockk.mockk -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.parallel.Execution -import org.junit.jupiter.api.parallel.ExecutionMode -import tachiyomi.domain.chapter.model.Chapter -import java.time.ZonedDateTime -import kotlin.time.Duration -import kotlin.time.Duration.Companion.hours -import kotlin.time.toJavaDuration - -@Execution(ExecutionMode.CONCURRENT) -class SetFetchIntervalTest { - - private val testTime = ZonedDateTime.parse("2020-01-01T00:00:00Z") - private var chapter = Chapter.create().copy( - dateFetch = testTime.toEpochSecond() * 1000, - dateUpload = testTime.toEpochSecond() * 1000, - ) - - private val setFetchInterval = SetFetchInterval(mockk()) - - @Test - fun `calculateInterval returns default of 7 days when less than 3 distinct days`() { - val chapters = (1..2).map { - chapterWithTime(chapter, 10.hours) - } - setFetchInterval.calculateInterval(chapters, testTime) shouldBe 7 - } - - @Test - fun `calculateInterval returns 7 when 5 chapters in 1 day`() { - val chapters = (1..5).map { - chapterWithTime(chapter, 10.hours) - } - setFetchInterval.calculateInterval(chapters, testTime) shouldBe 7 - } - - @Test - fun `calculateInterval returns 7 when 7 chapters in 48 hours, 2 day`() { - val chapters = (1..2).map { - chapterWithTime(chapter, 24.hours) - } + (1..5).map { - chapterWithTime(chapter, 48.hours) - } - setFetchInterval.calculateInterval(chapters, testTime) shouldBe 7 - } - - @Test - fun `calculateInterval returns default of 1 day when interval less than 1`() { - val chapters = (1..5).map { - chapterWithTime(chapter, (15 * it).hours) - } - setFetchInterval.calculateInterval(chapters, testTime) shouldBe 1 - } - - // Normal interval calculation - @Test - fun `calculateInterval returns 1 when 5 chapters in 120 hours, 5 days`() { - val chapters = (1..5).map { - chapterWithTime(chapter, (24 * it).hours) - } - setFetchInterval.calculateInterval(chapters, testTime) shouldBe 1 - } - - @Test - fun `calculateInterval returns 2 when 5 chapters in 240 hours, 10 days`() { - val chapters = (1..5).map { - chapterWithTime(chapter, (48 * it).hours) - } - setFetchInterval.calculateInterval(chapters, testTime) shouldBe 2 - } - - @Test - fun `calculateInterval returns floored value when interval is decimal`() { - val chapters = (1..5).map { - chapterWithTime(chapter, (25 * it).hours) - } - setFetchInterval.calculateInterval(chapters, testTime) shouldBe 1 - } - - @Test - fun `calculateInterval returns 1 when 5 chapters in 215 hours, 5 days`() { - val chapters = (1..5).map { - chapterWithTime(chapter, (43 * it).hours) - } - setFetchInterval.calculateInterval(chapters, testTime) shouldBe 1 - } - - @Test - fun `calculateInterval returns interval based on fetch time if upload time not available`() { - val chapters = (1..5).map { - chapterWithTime(chapter, (25 * it).hours).copy(dateUpload = 0L) - } - setFetchInterval.calculateInterval(chapters, testTime) shouldBe 1 - } - - private fun chapterWithTime(chapter: Chapter, duration: Duration): Chapter { - val newTime = testTime.plus(duration.toJavaDuration()).toEpochSecond() * 1000 - return chapter.copy(dateFetch = newTime, dateUpload = newTime) - } -}