rework the way media files are fetched

app/build.gradle

@@ -46,7 +46,7 @@ ext {
 }
 
 dependencies {
-    implementation 'com.simplemobiletools:commons:3.18.11'
+    implementation 'com.simplemobiletools:commons:3.18.14'
     implementation 'com.theartofdev.edmodo:android-image-cropper:2.6.0'
     implementation 'com.android.support:multidex:1.0.3'
     implementation 'it.sephiroth.android.exif:library:1.0.1'

app/src/main/kotlin/com/simplemobiletools/gallery/activities/MainActivity.kt

@@ -551,11 +551,11 @@ class MainActivity : SimpleActivity(), DirectoryAdapter.DirOperationsListener {
     }
 
     private fun gotDirectories(newDirs: ArrayList<Directory>, isFromCache: Boolean) {
-        if (!isFromCache) {
+        /*if (!isFromCache) {
             Thread {
                 checkFolderContentChange(newDirs)
             }.start()
-        }
+        }*/
 
         val dirs = getSortedDirectories(newDirs)
         directories_refresh_layout.isRefreshing = false

app/src/main/kotlin/com/simplemobiletools/gallery/asynctasks/GetMediaAsynctask.kt

@@ -25,7 +25,7 @@ class GetMediaAsynctask(val context: Context, val mPath: String, val isPickVideo
             media.sort()
             media
         } else {
-            mediaFetcher.getFilesFrom(mPath, isPickImage, isPickVideo, false)
+            mediaFetcher.getFilesFrom(mPath, isPickImage, isPickVideo)
         }
     }
 

app/src/main/kotlin/com/simplemobiletools/gallery/helpers/MediaFetcher.kt

@@ -3,7 +3,6 @@ package com.simplemobiletools.gallery.helpers
 import android.content.Context
 import android.database.Cursor
 import android.net.Uri
-import android.os.Environment
 import android.provider.MediaStore
 import com.simplemobiletools.commons.extensions.*
 import com.simplemobiletools.commons.helpers.*
@@ -12,14 +11,16 @@ import com.simplemobiletools.gallery.models.Medium
 import java.io.File
 import java.util.LinkedHashMap
 import kotlin.collections.ArrayList
+import kotlin.collections.HashSet
 import kotlin.collections.component1
 import kotlin.collections.component2
+import kotlin.collections.set
 
 class MediaFetcher(val context: Context) {
     var shouldStop = false
 
     fun getMediaByDirectories(isPickVideo: Boolean, isPickImage: Boolean): HashMap<String, ArrayList<Medium>> {
-        val media = getFilesFrom("", isPickImage, isPickVideo, true)
+        val media = getFilesFrom("", isPickImage, isPickVideo)
         val excludedPaths = context.config.excludedFolders
         val includedPaths = context.config.includedFolders
         val showHidden = context.config.shouldShowHidden
@@ -38,182 +39,124 @@ class MediaFetcher(val context: Context) {
             directories.remove(it)
         }
 
-        //searchNewFiles(directories, showHidden)
         return directories
     }
 
-    // search for undiscovered media files in the folders, from which we already have some media files
+    fun getFilesFrom(curPath: String, isPickImage: Boolean, isPickVideo: Boolean): ArrayList<Medium> {
-    private fun searchNewFiles(directories: Map<String, ArrayList<Medium>>, showHidden: Boolean) {
+        val filterMedia = context.config.filterMedia
-        Thread {
+        if (filterMedia == 0) {
-            // try not to delay the main media file loading
+            return ArrayList()
-            Thread.sleep(3000)
+        }
-            for ((path, dirMedia) in directories) {
-                if (path.contains("/.thumbnails/", true)) {
-                    continue
-                }
 
-                // get the file parent this way, "path" is lowercased
-                val folder = File(dirMedia.first().path).parentFile
-                val files = folder.listFiles() ?: continue
-                val fileCnt = files.filter { it.isFile }.size
-                val newPaths = ArrayList<String>()
-
-                if (dirMedia.size != fileCnt) {
-                    val dirPaths = dirMedia.map { it.path }
-                    files.forEach {
-                        val filePath = it.absolutePath
-                        if ((showHidden || !it.name.startsWith(".")) && !dirPaths.contains(filePath)) {
-                            if (it.exists() && it.length() > 0 && it.isImageVideoGif() && !context.isPathInMediaStore(it.absolutePath)) {
-                                newPaths.add(it.absolutePath)
-                            }
-                        }
-                    }
-                }
-                context.scanPaths(newPaths)
-            }
-        }.start()
-    }
-
-    fun getFilesFrom(curPath: String, isPickImage: Boolean, isPickVideo: Boolean, allowRecursion: Boolean): ArrayList<Medium> {
         if (curPath.startsWith(OTG_PATH)) {
             val curMedia = ArrayList<Medium>()
-            getMediaOnOTG(curPath, curMedia, isPickImage, isPickVideo, context.config.filterMedia, allowRecursion)
+            getMediaOnOTG(curPath, curMedia, isPickImage, isPickVideo, filterMedia)
             return curMedia
         } else {
-            val projection = arrayOf(MediaStore.Images.Media._ID,
+            val projection = arrayOf(MediaStore.Images.Media.DATA)
-                    MediaStore.Images.Media.DISPLAY_NAME,
-                    MediaStore.Images.Media.DATE_TAKEN,
-                    MediaStore.Images.Media.DATE_MODIFIED,
-                    MediaStore.Images.Media.DATA,
-                    MediaStore.Images.Media.SIZE)
             val uri = MediaStore.Files.getContentUri("external")
-            val selection = getSelectionQuery(curPath)
+
-            val selectionArgs = getSelectionArgsQuery(curPath)
+            val selection = "${getSelectionQuery(curPath, filterMedia)} ${MediaStore.Images.ImageColumns.BUCKET_ID} IS NOT NULL) GROUP BY (${MediaStore.Images.ImageColumns.BUCKET_ID}"
+            val selectionArgs = getSelectionArgsQuery(curPath, filterMedia).toTypedArray()
 
             return try {
                 val cur = context.contentResolver.query(uri, projection, selection, selectionArgs, getSortingForFolder(curPath))
-                parseCursor(context, cur, isPickImage, isPickVideo, curPath, allowRecursion)
+                parseCursor(context, cur, isPickImage, isPickVideo, curPath, filterMedia)
             } catch (e: Exception) {
                 ArrayList()
             }
         }
     }
 
-    private fun getSelectionQuery(path: String): String? {
+    private fun getSelectionQuery(path: String, filterMedia: Int): String {
-        val dataQuery = "${MediaStore.Images.Media.DATA} LIKE ?"
+        val query = StringBuilder()
-        return if (path.isEmpty()) {
+        if (path.isNotEmpty()) {
-            if (isAndroidFour())
+            query.append("${MediaStore.Images.Media.DATA} LIKE ? AND ${MediaStore.Images.Media.DATA} NOT LIKE ? AND ")
-                return null
-
-            var query = "($dataQuery)"
-            if (context.hasExternalSDCard()) {
-                query += " OR ($dataQuery)"
-            }
-            query
-        } else {
-            "($dataQuery AND ${MediaStore.Images.Media.DATA} NOT LIKE ?)"
         }
+
+        query.append("(")
+        if (filterMedia and IMAGES != 0) {
+            photoExtensions.forEach {
+                query.append("${MediaStore.Images.Media.DATA} LIKE ? OR ")
+            }
+        }
+
+        if (filterMedia and VIDEOS != 0) {
+            videoExtensions.forEach {
+                query.append("${MediaStore.Images.Media.DATA} LIKE ? OR ")
+            }
+        }
+
+        if (filterMedia and GIFS != 0) {
+            query.append("${MediaStore.Images.Media.DATA} LIKE ?")
+        }
+
+        var selectionQuery = query.toString().trim().removeSuffix("OR")
+        selectionQuery += ") AND "
+        return selectionQuery
     }
 
-    private fun getSelectionArgsQuery(path: String): Array<String>? {
+    private fun getSelectionArgsQuery(path: String, filterMedia: Int): ArrayList<String> {
-        return if (path.isEmpty()) {
+        val args = ArrayList<String>()
-            if (isAndroidFour()) {
+        if (path.isNotEmpty()) {
-                return null
+            args.add("$path/%")
-            }
+            args.add("$path/%/%")
-
-            if (context.hasExternalSDCard()) {
-                arrayOf("${context.internalStoragePath}/%", "${context.sdCardPath}/%")
-            } else {
-                arrayOf("${context.internalStoragePath}/%")
-            }
-        } else {
-            arrayOf("$path/%", "$path/%/%")
         }
+
+        if (filterMedia and IMAGES != 0) {
+            photoExtensions.forEach {
+                args.add("%$it")
+            }
+        }
+
+        if (filterMedia and VIDEOS != 0) {
+            videoExtensions.forEach {
+                args.add("%$it")
+            }
+        }
+
+        if (filterMedia and GIFS != 0) {
+            args.add("%.gif")
+        }
+
+        return args
     }
 
-    private fun parseCursor(context: Context, cur: Cursor, isPickImage: Boolean, isPickVideo: Boolean, curPath: String, allowRecursion: Boolean): ArrayList<Medium> {
+    private fun parseCursor(context: Context, cur: Cursor, isPickImage: Boolean, isPickVideo: Boolean, curPath: String, filterMedia: Int): ArrayList<Medium> {
         val curMedia = ArrayList<Medium>()
         val config = context.config
-        val filterMedia = config.filterMedia
         val showHidden = config.shouldShowHidden
+        val excludedFolders = config.excludedFolders
+        val includedFolders = config.includedFolders
         val isThirdPartyIntent = config.isThirdPartyIntent
-        val doExtraCheck = config.doExtraCheck
+        val foldersToScan = HashSet<String>()
 
         cur.use {
             if (cur.moveToFirst()) {
                 do {
-                    try {
+                    val path = cur.getStringValue(MediaStore.Images.Media.DATA).trim()
-                        if (shouldStop) {
+                    val parentPath = File(path).parent.trimEnd('/')
-                            break
+                    if (!includedFolders.contains(parentPath)) {
-                        }
+                        foldersToScan.add(parentPath)
-
-                        val path = cur.getStringValue(MediaStore.Images.Media.DATA).trim()
-                        var filename = cur.getStringValue(MediaStore.Images.Media.DISPLAY_NAME)?.trim() ?: ""
-                        if (filename.isEmpty())
-                            filename = path.getFilenameFromPath()
-
-                        val isImage = filename.isImageFast()
-                        val isVideo = if (isImage) false else filename.isVideoFast()
-                        val isGif = if (isImage || isVideo) false else filename.isGif()
-
-                        if (!isImage && !isVideo && !isGif)
-                            continue
-
-                        if (isVideo && (isPickImage || filterMedia and VIDEOS == 0))
-                            continue
-
-                        if (isImage && (isPickVideo || filterMedia and IMAGES == 0))
-                            continue
-
-                        if (isGif && filterMedia and GIFS == 0)
-                            continue
-
-                        if (!showHidden && filename.startsWith('.'))
-                            continue
-
-                        var size = cur.getLongValue(MediaStore.Images.Media.SIZE)
-                        val file = File(path)
-                        if (size == 0L) {
-                            size = file.length()
-                        }
-
-                        if (size <= 0L || (doExtraCheck && !file.exists()))
-                            continue
-
-                        val dateTaken = cur.getLongValue(MediaStore.Images.Media.DATE_TAKEN)
-                        val dateModified = cur.getIntValue(MediaStore.Images.Media.DATE_MODIFIED) * 1000L
-
-                        val type = when {
-                            isImage -> TYPE_IMAGE
-                            isVideo -> TYPE_VIDEO
-                            else -> TYPE_GIF
-                        }
-
-                        val medium = Medium(filename, path, dateModified, dateTaken, size, type)
-                        curMedia.add(medium)
-                    } catch (e: Exception) {
-                        continue
                     }
                 } while (cur.moveToNext())
             }
         }
 
-        val screenshotsFolder = "${Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_PICTURES)}/Screenshots"
+        includedFolders.forEach {
-        val foldersToScan = config.includedFolders
+            if (curPath.isEmpty()) {
-        if (File(screenshotsFolder).exists()) {
+                addFolder(foldersToScan, it)
-            foldersToScan.add(screenshotsFolder)
+            } else if (curPath == it) {
-        }
+                foldersToScan.add(it)
-
-        foldersToScan.filter { it.isNotEmpty() && (curPath.isEmpty() || it == curPath) }.forEach {
-            if (it.startsWith(OTG_PATH)) {
-                getMediaOnOTG(it, curMedia, isPickImage, isPickVideo, filterMedia, allowRecursion)
-            } else {
-                getMediaInFolder(it, curMedia, isPickImage, isPickVideo, filterMedia, allowRecursion)
             }
         }
 
+        foldersToScan.filter { shouldFolderBeVisible(it, excludedFolders, includedFolders, showHidden) }.toList().forEach {
+            fetchFolderContent(it, curMedia, isPickImage, isPickVideo, filterMedia)
+        }
+
         if (isThirdPartyIntent && curPath.isNotEmpty() && curMedia.isEmpty()) {
-            getMediaInFolder(curPath, curMedia, isPickImage, isPickVideo, filterMedia, allowRecursion)
+            getMediaInFolder(curPath, curMedia, isPickImage, isPickVideo, filterMedia)
         }
 
         Medium.sorting = config.getFileSorting(curPath)
@@ -222,6 +165,24 @@ class MediaFetcher(val context: Context) {
         return curMedia
     }
 
+    private fun addFolder(curFolders: HashSet<String>, folder: String) {
+        curFolders.add(folder)
+        val files = File(folder).listFiles() ?: return
+        for (file in files) {
+            if (file.isDirectory) {
+                addFolder(curFolders, file.absolutePath)
+            }
+        }
+    }
+
+    private fun fetchFolderContent(path: String, curMedia: ArrayList<Medium>, isPickImage: Boolean, isPickVideo: Boolean, filterMedia: Int) {
+        if (path.startsWith(OTG_PATH)) {
+            getMediaOnOTG(path, curMedia, isPickImage, isPickVideo, filterMedia)
+        } else {
+            getMediaInFolder(path, curMedia, isPickImage, isPickVideo, filterMedia)
+        }
+    }
+
     private fun groupDirectories(media: ArrayList<Medium>): HashMap<String, ArrayList<Medium>> {
         val directories = LinkedHashMap<String, ArrayList<Medium>>()
         val hasOTG = context.hasOTGConnected() && context.config.OTGBasePath.isNotEmpty()
@@ -243,7 +204,9 @@ class MediaFetcher(val context: Context) {
 
     private fun shouldFolderBeVisible(path: String, excludedPaths: MutableSet<String>, includedPaths: MutableSet<String>, showHidden: Boolean): Boolean {
         val file = File(path)
-        return if (path.isThisOrParentIncluded(includedPaths)) {
+        return if (path.isEmpty()) {
+            false
+        } else if (path.isThisOrParentIncluded(includedPaths)) {
             true
         } else if (path.isThisOrParentExcluded(excludedPaths)) {
             false
@@ -258,18 +221,14 @@ class MediaFetcher(val context: Context) {
         }
     }
 
-    private fun getMediaInFolder(folder: String, curMedia: ArrayList<Medium>, isPickImage: Boolean, isPickVideo: Boolean, filterMedia: Int, allowRecursion: Boolean) {
+    private fun getMediaInFolder(folder: String, curMedia: ArrayList<Medium>, isPickImage: Boolean, isPickVideo: Boolean, filterMedia: Int) {
+        val doExtraCheck = context.config.doExtraCheck
         val files = File(folder).listFiles() ?: return
         for (file in files) {
             if (shouldStop) {
                 break
             }
 
-            if (file.isDirectory && allowRecursion) {
-                getMediaInFolder(file.absolutePath, curMedia, isPickImage, isPickVideo, filterMedia, allowRecursion)
-                continue
-            }
-
             val filename = file.name
             val isImage = filename.isImageFast()
             val isVideo = if (isImage) false else filename.isVideoFast()
@@ -288,7 +247,7 @@ class MediaFetcher(val context: Context) {
                 continue
 
             val size = file.length()
-            if (size <= 0L && !file.exists())
+            if (size <= 0L || (doExtraCheck && !file.exists()))
                 continue
 
             val dateTaken = file.lastModified()
@@ -301,25 +260,17 @@ class MediaFetcher(val context: Context) {
             }
 
             val medium = Medium(filename, file.absolutePath, dateModified, dateTaken, size, type)
-            val isAlreadyAdded = curMedia.any { it.path == file.absolutePath }
+            curMedia.add(medium)
-            if (!isAlreadyAdded) {
-                curMedia.add(medium)
-            }
         }
     }
 
-    private fun getMediaOnOTG(folder: String, curMedia: ArrayList<Medium>, isPickImage: Boolean, isPickVideo: Boolean, filterMedia: Int, allowRecursion: Boolean) {
+    private fun getMediaOnOTG(folder: String, curMedia: ArrayList<Medium>, isPickImage: Boolean, isPickVideo: Boolean, filterMedia: Int) {
         val files = context.getDocumentFile(folder)?.listFiles() ?: return
         for (file in files) {
             if (shouldStop) {
                 return
             }
 
-            if (file.isDirectory && allowRecursion) {
-                getMediaOnOTG("$folder${file.name}", curMedia, isPickImage, isPickVideo, filterMedia, allowRecursion)
-                continue
-            }
-
             val filename = file.name
             val isImage = filename.isImageFast()
             val isVideo = if (isImage) false else filename.isVideoFast()
@@ -352,10 +303,7 @@ class MediaFetcher(val context: Context) {
 
             val path = Uri.decode(file.uri.toString().replaceFirst("${context.config.OTGBasePath}%3A", OTG_PATH))
             val medium = Medium(filename, path, dateModified, dateTaken, size, type)
-            val isAlreadyAdded = curMedia.any { it.path == path }
+            curMedia.add(medium)
-            if (!isAlreadyAdded) {
-                curMedia.add(medium)
-            }
         }
     }