github/tachiyomi
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Initialize download index disk cache (#9179)

Browse source
This commit is contained in:
Ivan Iskandar 2023-03-17 09:18:11 +07:00 committed by GitHub
parent a335b4ee9e
commit 4d3e13b0d1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 163 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

241
app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt
View file

@ -1,6 +1,8 @@
package eu.kanade.tachiyomi.data.download
import android.app.Application
import android.content.Context
import android.net.Uri
import androidx.core.net.toUri
import com.hippo.unifile.UniFile
import eu.kanade.core.util.mapNotNullKeys
@ -14,6 +16,7 @@ import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.delay
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.SharingStarted
import kotlinx.coroutines.flow.debounce
@ -23,7 +26,20 @@ import kotlinx.coroutines.flow.onStart
import kotlinx.coroutines.flow.receiveAsFlow
import kotlinx.coroutines.flow.shareIn
import kotlinx.coroutines.flow.stateIn
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withTimeoutOrNull
import kotlinx.serialization.KSerializer
import kotlinx.serialization.Serializable
import kotlinx.serialization.decodeFromByteArray
import kotlinx.serialization.descriptors.PrimitiveKind
import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.encodeToByteArray
import kotlinx.serialization.encoding.Decoder
import kotlinx.serialization.encoding.Encoder
import kotlinx.serialization.protobuf.ProtoBuf
import logcat.LogPriority
import tachiyomi.core.util.lang.launchIO
import tachiyomi.core.util.lang.launchNonCancellable
@ -34,7 +50,7 @@ import tachiyomi.domain.manga.model.Manga
import tachiyomi.domain.source.service.SourceManager
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.util.concurrent.ConcurrentHashMap
import java.io.File
import kotlin.time.Duration.Companion.hours
import kotlin.time.Duration.Companion.seconds
@ -76,7 +92,11 @@ class DownloadCache(
.debounce(1000L) // Don't notify if it finishes quickly enough
.stateIn(scope, SharingStarted.WhileSubscribed(), false)
private var rootDownloadsDir = RootDirectory(getDirectoryFromPreference())
private val diskCacheFile: File
get() = File(context.cacheDir, "dl_index_cache")
private val rootDownloadsDirLock = Mutex()
private var rootDownloadsDir: RootDirectory
init {
downloadPreferences.downloadsDirectory().changes()
@ -85,6 +105,21 @@ class DownloadCache(
invalidateCache()
}
.launchIn(scope)
rootDownloadsDir = runBlocking(Dispatchers.IO) {
try {
val diskCache = diskCacheFile.inputStream().use {
ProtoBuf.decodeFromByteArray<RootDirectory>(it.readBytes())
}
lastRenew = 1 // Just so that the banner won't show up
diskCache
} catch (e: Throwable) {
diskCacheFile.delete()
null
}
} ?: RootDirectory(getDirectoryFromPreference())
notifyChanges()
}
/**
@ -158,27 +193,28 @@ class DownloadCache(
* @param mangaUniFile the directory of the manga.
* @param manga the manga of the chapter.
*/
@Synchronized
fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) {
// Retrieve the cached source directory or cache a new one
var sourceDir = rootDownloadsDir.sourceDirs[manga.source]
if (sourceDir == null) {
val source = sourceManager.get(manga.source) ?: return
val sourceUniFile = provider.findSourceDir(source) ?: return
sourceDir = SourceDirectory(sourceUniFile)
rootDownloadsDir.sourceDirs += manga.source to sourceDir
}
suspend fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) {
rootDownloadsDirLock.withLock {
// Retrieve the cached source directory or cache a new one
var sourceDir = rootDownloadsDir.sourceDirs[manga.source]
if (sourceDir == null) {
val source = sourceManager.get(manga.source) ?: return
val sourceUniFile = provider.findSourceDir(source) ?: return
sourceDir = SourceDirectory(sourceUniFile)
rootDownloadsDir.sourceDirs += manga.source to sourceDir
}
// Retrieve the cached manga directory or cache a new one
val mangaDirName = provider.getMangaDirName(manga.title)
var mangaDir = sourceDir.mangaDirs[mangaDirName]
if (mangaDir == null) {
mangaDir = MangaDirectory(mangaUniFile)
sourceDir.mangaDirs += mangaDirName to mangaDir
}
// Retrieve the cached manga directory or cache a new one
val mangaDirName = provider.getMangaDirName(manga.title)
var mangaDir = sourceDir.mangaDirs[mangaDirName]
if (mangaDir == null) {
mangaDir = MangaDirectory(mangaUniFile)
sourceDir.mangaDirs += mangaDirName to mangaDir
}
// Save the chapter directory
mangaDir.chapterDirs += chapterDirName
// Save the chapter directory
mangaDir.chapterDirs += chapterDirName
}
notifyChanges()
}
@ -189,13 +225,14 @@ class DownloadCache(
* @param chapter the chapter to remove.
* @param manga the manga of the chapter.
*/
@Synchronized
fun removeChapter(chapter: Chapter, manga: Manga) {
val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
if (it in mangaDir.chapterDirs) {
mangaDir.chapterDirs -= it
suspend fun removeChapter(chapter: Chapter, manga: Manga) {
rootDownloadsDirLock.withLock {
val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
if (it in mangaDir.chapterDirs) {
mangaDir.chapterDirs -= it
}
}
}
@ -208,14 +245,15 @@ class DownloadCache(
* @param chapters the list of chapter to remove.
* @param manga the manga of the chapter.
*/
@Synchronized
fun removeChapters(chapters: List<Chapter>, manga: Manga) {
val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
chapters.forEach { chapter ->
provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
if (it in mangaDir.chapterDirs) {
mangaDir.chapterDirs -= it
suspend fun removeChapters(chapters: List<Chapter>, manga: Manga) {
rootDownloadsDirLock.withLock {
val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
chapters.forEach { chapter ->
provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
if (it in mangaDir.chapterDirs) {
mangaDir.chapterDirs -= it
}
}
}
}
@ -228,20 +266,22 @@ class DownloadCache(
*
* @param manga the manga to remove.
*/
@Synchronized
fun removeManga(manga: Manga) {
val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
val mangaDirName = provider.getMangaDirName(manga.title)
if (sourceDir.mangaDirs.containsKey(mangaDirName)) {
sourceDir.mangaDirs -= mangaDirName
suspend fun removeManga(manga: Manga) {
rootDownloadsDirLock.withLock {
val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
val mangaDirName = provider.getMangaDirName(manga.title)
if (sourceDir.mangaDirs.containsKey(mangaDirName)) {
sourceDir.mangaDirs -= mangaDirName
}
}
notifyChanges()
}
@Synchronized
fun removeSource(source: Source) {
rootDownloadsDir.sourceDirs -= source.id
suspend fun removeSource(source: Source) {
rootDownloadsDirLock.withLock {
rootDownloadsDir.sourceDirs -= source.id
}
notifyChanges()
}
@ -287,46 +327,48 @@ class DownloadCache(
}
}
val sourceDirs = rootDownloadsDir.dir.listFiles().orEmpty()
.associate { it.name to SourceDirectory(it) }
.mapNotNullKeys { entry ->
sources.find {
provider.getSourceDirName(it).equals(entry.key, ignoreCase = true)
}?.id
}
rootDownloadsDirLock.withLock {
val sourceDirs = rootDownloadsDir.dir.listFiles().orEmpty()
.associate { it.name to SourceDirectory(it) }
.mapNotNullKeys { entry ->
sources.find {
provider.getSourceDirName(it).equals(entry.key, ignoreCase = true)
}?.id
}
rootDownloadsDir.sourceDirs = sourceDirs
rootDownloadsDir.sourceDirs = sourceDirs
sourceDirs.values
.map { sourceDir ->
async {
val mangaDirs = sourceDir.dir.listFiles().orEmpty()
.filterNot { it.name.isNullOrBlank() }
.associate { it.name!! to MangaDirectory(it) }
sourceDirs.values
.map { sourceDir ->
async {
sourceDir.mangaDirs = sourceDir.dir.listFiles().orEmpty()
.filterNot { it.name.isNullOrBlank() }
.associate { it.name!! to MangaDirectory(it) }
sourceDir.mangaDirs = ConcurrentHashMap(mangaDirs)
mangaDirs.values.forEach { mangaDir ->
val chapterDirs = mangaDir.dir.listFiles().orEmpty()
.mapNotNull {
when {
// Ignore incomplete downloads
it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null
// Folder of images
it.isDirectory -> it.name
// CBZ files
it.isFile && it.name?.endsWith(".cbz") == true -> it.name!!.substringBeforeLast(".cbz")
// Anything else is irrelevant
else -> null
sourceDir.mangaDirs.values.forEach { mangaDir ->
val chapterDirs = mangaDir.dir.listFiles().orEmpty()
.mapNotNull {
when {
// Ignore incomplete downloads
it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null
// Folder of images
it.isDirectory -> it.name
// CBZ files
it.isFile && it.name?.endsWith(".cbz") == true -> it.name!!.substringBeforeLast(
".cbz",
)
// Anything else is irrelevant
else -> null
}
}
}
.toMutableSet()
.toMutableSet()
mangaDir.chapterDirs = chapterDirs
mangaDir.chapterDirs = chapterDirs
}
}
}
}
.awaitAll()
.awaitAll()
}
_isInitializing.emit(false)
}.also {
@ -335,6 +377,7 @@ class DownloadCache(
logcat(LogPriority.ERROR, exception) { "Failed to create download cache" }
}
lastRenew = System.currentTimeMillis()
notifyChanges()
}
}
@ -351,29 +394,67 @@ class DownloadCache(
scope.launchNonCancellable {
_changes.send(Unit)
}
updateDiskCache()
}
private var updateDiskCacheJob: Job? = null
private fun updateDiskCache() {
updateDiskCacheJob?.cancel()
updateDiskCacheJob = scope.launchIO {
delay(1000)
ensureActive()
val bytes = ProtoBuf.encodeToByteArray(rootDownloadsDir)
ensureActive()
try {
diskCacheFile.writeBytes(bytes)
} catch (e: Throwable) {
logcat(
priority = LogPriority.ERROR,
throwable = e,
message = { "Failed to write disk cache file" },
)
}
}
}
}
/**
* Class to store the files under the root downloads directory.
*/
@Serializable
private class RootDirectory(
@Serializable(with = UniFileAsStringSerializer::class)
val dir: UniFile,
var sourceDirs: ConcurrentHashMap<Long, SourceDirectory> = ConcurrentHashMap(),
var sourceDirs: Map<Long, SourceDirectory> = mapOf(),
)
/**
* Class to store the files under a source directory.
*/
@Serializable
private class SourceDirectory(
@Serializable(with = UniFileAsStringSerializer::class)
val dir: UniFile,
var mangaDirs: ConcurrentHashMap<String, MangaDirectory> = ConcurrentHashMap(),
var mangaDirs: Map<String, MangaDirectory> = mapOf(),
)
/**
* Class to store the files under a manga directory.
*/
@Serializable
private class MangaDirectory(
@Serializable(with = UniFileAsStringSerializer::class)
val dir: UniFile,
var chapterDirs: MutableSet<String> = mutableSetOf(),
)
private object UniFileAsStringSerializer : KSerializer<UniFile> {
override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("UniFile", PrimitiveKind.STRING)
override fun serialize(encoder: Encoder, value: UniFile) {
return encoder.encodeString(value.uri.toString())
}
override fun deserialize(decoder: Decoder): UniFile {
return UniFile.fromUri(Injekt.get<Application>(), Uri.parse(decoder.decodeString()))
}
}

2
app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt
View file

@ -325,7 +325,7 @@ class DownloadManager(
* @param oldChapter the existing chapter with the old name.
* @param newChapter the target chapter with the new name.
*/
fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
suspend fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
val oldNames = provider.getValidChapterDirNames(oldChapter.name, oldChapter.scanlator)
val mangaDir = provider.getMangaDir(manga.title, source)

2
app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt
View file

@ -527,7 +527,7 @@ class Downloader(
* @param tmpDir the directory where the download is currently stored.
* @param dirname the real (non temporary) directory name of the download.
*/
private fun ensureSuccessfulDownload(
private suspend fun ensureSuccessfulDownload(
download: Download,
mangaDir: UniFile,
tmpDir: UniFile,