First Commit

This commit is contained in:
2025-12-18 16:28:50 +07:00
commit 8c3e4f491f
9974 changed files with 396488 additions and 0 deletions
@@ -0,0 +1,27 @@
import extension.testCommonDependencies
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023, 2024 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
plugins {
id("io.element.android-library")
}
android {
namespace = "io.element.android.libraries.mediaupload.api"
}
dependencies {
implementation(projects.libraries.architecture)
implementation(projects.libraries.androidutils)
implementation(projects.libraries.core)
implementation(projects.libraries.di)
api(projects.libraries.matrix.api)
api(projects.libraries.preferences.api)
implementation(libs.coroutines.core)
}
@@ -0,0 +1,16 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.api
/**
* Provides the maximum upload size allowed by the Matrix server.
*/
fun interface MaxUploadSizeProvider {
suspend fun getMaxUploadSize(): Result<Long>
}
@@ -0,0 +1,23 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.api
import io.element.android.libraries.androidutils.media.VideoCompressorHelper
import io.element.android.libraries.preferences.api.store.VideoCompressionPreset
data class MediaOptimizationConfig(
val compressImages: Boolean,
val videoCompressionPreset: VideoCompressionPreset,
)
fun VideoCompressionPreset.compressorHelper(): VideoCompressorHelper = when (this) {
VideoCompressionPreset.STANDARD -> VideoCompressorHelper(1280)
VideoCompressionPreset.HIGH -> VideoCompressorHelper(1920)
VideoCompressionPreset.LOW -> VideoCompressorHelper(640)
}
@@ -0,0 +1,13 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.api
fun interface MediaOptimizationConfigProvider {
suspend fun get(): MediaOptimizationConfig
}
@@ -0,0 +1,32 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.api
import android.net.Uri
interface MediaPreProcessor {
/**
* Given a [uri] and [mimeType], pre-processes the media before it's uploaded, resizing, transcoding, and removing sensitive info from its metadata.
* If [deleteOriginal] is `true`, the file reference by the [uri] will be automatically deleted too when this process finishes.
* @return a [Result] with the [MediaUploadInfo] containing all the info needed to begin the upload.
*/
suspend fun process(
uri: Uri,
mimeType: String,
deleteOriginal: Boolean,
mediaOptimizationConfig: MediaOptimizationConfig,
): Result<MediaUploadInfo>
/**
* Clean up any temporary files or resources used during the media processing.
*/
fun cleanUp()
data class Failure(override val cause: Throwable?) : Exception(cause)
}
@@ -0,0 +1,65 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.api
import android.net.Uri
import io.element.android.libraries.matrix.api.core.EventId
import io.element.android.libraries.matrix.api.room.JoinedRoom
import io.element.android.libraries.matrix.api.timeline.Timeline
fun interface MediaSenderFactory {
/**
* Create a [MediaSender] for the given [Timeline.Mode], in the Room Scope.
*/
fun create(
timelineMode: Timeline.Mode,
): MediaSender
}
fun interface MediaSenderRoomFactory {
/**
* Create a [MediaSender] for the given [JoinedRoom], with timeline mode Live.
*/
fun create(
room: JoinedRoom,
): MediaSender
}
interface MediaSender {
suspend fun preProcessMedia(
uri: Uri,
mimeType: String,
mediaOptimizationConfig: MediaOptimizationConfig,
): Result<MediaUploadInfo>
suspend fun sendPreProcessedMedia(
mediaUploadInfo: MediaUploadInfo,
caption: String?,
formattedCaption: String?,
inReplyToEventId: EventId?,
): Result<Unit>
suspend fun sendMedia(
uri: Uri,
mimeType: String,
caption: String? = null,
formattedCaption: String? = null,
inReplyToEventId: EventId? = null,
mediaOptimizationConfig: MediaOptimizationConfig,
): Result<Unit>
suspend fun sendVoiceMessage(
uri: Uri,
mimeType: String,
waveForm: List<Float>,
inReplyToEventId: EventId? = null,
): Result<Unit>
fun cleanUp()
}
@@ -0,0 +1,33 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.api
import io.element.android.libraries.matrix.api.media.AudioInfo
import io.element.android.libraries.matrix.api.media.FileInfo
import io.element.android.libraries.matrix.api.media.ImageInfo
import io.element.android.libraries.matrix.api.media.VideoInfo
import java.io.File
sealed interface MediaUploadInfo {
val file: File
data class Image(override val file: File, val imageInfo: ImageInfo, val thumbnailFile: File?) : MediaUploadInfo
data class Video(override val file: File, val videoInfo: VideoInfo, val thumbnailFile: File?) : MediaUploadInfo
data class Audio(override val file: File, val audioInfo: AudioInfo) : MediaUploadInfo
data class VoiceMessage(override val file: File, val audioInfo: AudioInfo, val waveform: List<Float>) : MediaUploadInfo
data class AnyFile(override val file: File, val fileInfo: FileInfo) : MediaUploadInfo
}
fun MediaUploadInfo.allFiles(): List<File> {
return listOfNotNull(
file,
(this@allFiles as? MediaUploadInfo.Image)?.thumbnailFile,
(this@allFiles as? MediaUploadInfo.Video)?.thumbnailFile,
)
}
@@ -0,0 +1,48 @@
import extension.setupDependencyInjection
import extension.testCommonDependencies
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023, 2024 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
plugins {
id("io.element.android-library")
}
android {
namespace = "io.element.android.libraries.mediaupload.impl"
testOptions {
unitTests {
isIncludeAndroidResources = true
}
}
}
setupDependencyInjection()
dependencies {
api(projects.libraries.mediaupload.api)
implementation(projects.libraries.architecture)
implementation(projects.libraries.androidutils)
implementation(projects.libraries.core)
implementation(projects.libraries.di)
implementation(projects.libraries.matrix.api)
implementation(projects.services.toolbox.api)
implementation(libs.androidx.exifinterface)
implementation(libs.androidx.media3.transformer)
implementation(libs.androidx.media3.effect)
implementation(libs.androidx.media3.common)
implementation(libs.coroutines.core)
implementation(libs.vanniktech.blurhash)
testCommonDependencies(libs)
testImplementation(projects.services.toolbox.test)
testImplementation(projects.libraries.matrix.test)
testImplementation(projects.libraries.preferences.test)
testImplementation(projects.libraries.mediaupload.test)
}
@@ -0,0 +1,373 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import android.content.Context
import android.graphics.BitmapFactory
import android.media.MediaMetadataRetriever
import android.net.Uri
import androidx.exifinterface.media.ExifInterface
import dev.zacsweers.metro.AppScope
import dev.zacsweers.metro.ContributesBinding
import io.element.android.libraries.androidutils.file.TemporaryUriDeleter
import io.element.android.libraries.androidutils.file.createTmpFile
import io.element.android.libraries.androidutils.file.getFileName
import io.element.android.libraries.androidutils.file.safeRenameTo
import io.element.android.libraries.androidutils.hash.hash
import io.element.android.libraries.androidutils.media.runAndRelease
import io.element.android.libraries.core.coroutine.CoroutineDispatchers
import io.element.android.libraries.core.data.tryOrNull
import io.element.android.libraries.core.extensions.mapFailure
import io.element.android.libraries.core.extensions.runCatchingExceptions
import io.element.android.libraries.core.mimetype.MimeTypes
import io.element.android.libraries.core.mimetype.MimeTypes.isMimeTypeAudio
import io.element.android.libraries.core.mimetype.MimeTypes.isMimeTypeImage
import io.element.android.libraries.core.mimetype.MimeTypes.isMimeTypeVideo
import io.element.android.libraries.di.annotations.ApplicationContext
import io.element.android.libraries.matrix.api.media.AudioInfo
import io.element.android.libraries.matrix.api.media.FileInfo
import io.element.android.libraries.matrix.api.media.ImageInfo
import io.element.android.libraries.matrix.api.media.VideoInfo
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfig
import io.element.android.libraries.mediaupload.api.MediaPreProcessor
import io.element.android.libraries.mediaupload.api.MediaUploadInfo
import io.element.android.libraries.preferences.api.store.VideoCompressionPreset
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.withContext
import timber.log.Timber
import java.io.File
import java.io.InputStream
import java.util.UUID
import kotlin.time.Duration
import kotlin.time.Duration.Companion.milliseconds
@ContributesBinding(AppScope::class)
class AndroidMediaPreProcessor(
@ApplicationContext private val context: Context,
private val thumbnailFactory: ThumbnailFactory,
private val imageCompressor: ImageCompressor,
private val videoCompressor: VideoCompressor,
private val coroutineDispatchers: CoroutineDispatchers,
private val temporaryUriDeleter: TemporaryUriDeleter,
) : MediaPreProcessor {
companion object {
/**
* Used for calculating `inSampleSize` for bitmaps.
*
* *Note*: Ideally, this should result in images of up to (but not included) 1280x1280 being sent. However, images with very different width and height
* values may surpass this limit. (i.e.: an image of `480x3000px` would have `inSampleSize=1` and be sent as is).
*/
private const val IMAGE_SCALE_REF_SIZE = 640
private val notCompressibleImageTypes = listOf(MimeTypes.Gif, MimeTypes.WebP, MimeTypes.Svg)
}
private val contentResolver = context.contentResolver
private val cacheDir = context.cacheDir
private val baseTmpFileDir = File(cacheDir, "uploads")
override suspend fun process(
uri: Uri,
mimeType: String,
deleteOriginal: Boolean,
mediaOptimizationConfig: MediaOptimizationConfig,
): Result<MediaUploadInfo> = withContext(coroutineDispatchers.computation) {
runCatchingExceptions {
val result = when {
// Special case for SVG, since Android can't read its metadata or create a thumbnail, it must be sent as a file
mimeType == MimeTypes.Svg -> {
processFile(uri, mimeType)
}
mimeType.isMimeTypeImage() -> {
val shouldBeCompressed = mediaOptimizationConfig.compressImages && mimeType !in notCompressibleImageTypes
processImage(uri, mimeType, shouldBeCompressed)
}
mimeType.isMimeTypeVideo() -> processVideo(uri, mimeType, mediaOptimizationConfig.videoCompressionPreset)
mimeType.isMimeTypeAudio() -> processAudio(uri, mimeType)
else -> processFile(uri, mimeType)
}
if (deleteOriginal) {
tryOrNull {
Timber.w("Deleting original uri $uri")
contentResolver.delete(uri, null, null)
}
} else {
temporaryUriDeleter.delete(uri)
}
result.postProcess(uri)
}
}.mapFailure { MediaPreProcessor.Failure(it) }
override fun cleanUp() {
Timber.d("Cleaning up temporary media files")
// Clear temporary files created in older versions of the app
cacheDir.listFiles()?.onEach { file ->
if (file.isFile) {
val nameWithoutExtension = file.nameWithoutExtension
// UUIDs are 36 characters long, so we check if we can take those 36 characters
val nameWithoutExtensionAndRandom = if (nameWithoutExtension.length > 36) {
nameWithoutExtension.substring(0, 36)
} else {
// Not a temp file
return@onEach
}
val isUUID = tryOrNull { UUID.fromString(nameWithoutExtensionAndRandom) } != null
if (isUUID && file.extension.isNotEmpty()) {
file.delete()
}
}
}
// Clear temporary files created by this pre-processor in the separate uploads directory
baseTmpFileDir.listFiles()?.onEach { it.delete() }
}
private suspend fun processFile(uri: Uri, mimeType: String): MediaUploadInfo {
Timber.d("Processing file ${uri.path.orEmpty().hash()}")
val file = copyToTmpFile(uri)
val info = FileInfo(
mimetype = mimeType,
size = file.length(),
thumbnailInfo = null,
thumbnailSource = null,
)
return MediaUploadInfo.AnyFile(file, info)
}
private fun MediaUploadInfo.postProcess(uri: Uri): MediaUploadInfo {
Timber.d("Finished processing, post-processing ${uri.path.orEmpty().hash()}")
val name = context.getFileName(uri) ?: return this
val renamedFile = File(context.cacheDir, name).also {
file.safeRenameTo(it)
}
return when (this) {
is MediaUploadInfo.AnyFile -> copy(file = renamedFile)
is MediaUploadInfo.Audio -> copy(file = renamedFile)
is MediaUploadInfo.Image -> copy(file = renamedFile)
is MediaUploadInfo.Video -> copy(file = renamedFile)
is MediaUploadInfo.VoiceMessage -> copy(file = renamedFile)
}
}
private suspend fun processImage(uri: Uri, mimeType: String, shouldBeCompressed: Boolean): MediaUploadInfo {
Timber.d("Processing image ${uri.path.orEmpty().hash()}")
suspend fun processImageWithCompression(): MediaUploadInfo {
// Read the orientation metadata from its own stream. Trying to reuse this stream for compression will fail.
val orientation = contentResolver.openInputStream(uri).use { input ->
val exifInterface = input?.let { ExifInterface(it) }
exifInterface?.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_UNDEFINED)
} ?: ExifInterface.ORIENTATION_UNDEFINED
val compressionResult = imageCompressor.compressToTmpFile(
inputStreamProvider = { contentResolver.openInputStream(uri)!! },
resizeMode = ResizeMode.Approximate(IMAGE_SCALE_REF_SIZE, IMAGE_SCALE_REF_SIZE),
mimeType = mimeType,
orientation = orientation,
).getOrThrow()
val thumbnailResult = thumbnailFactory.createImageThumbnail(
file = compressionResult.file,
mimeType = mimeType,
)
val imageInfo = compressionResult.toImageInfo(
mimeType = mimeType,
thumbnailResult = thumbnailResult
)
removeSensitiveImageMetadata(compressionResult.file)
return MediaUploadInfo.Image(
file = compressionResult.file,
imageInfo = imageInfo,
thumbnailFile = thumbnailResult?.file
)
}
suspend fun processImageWithoutCompression(): MediaUploadInfo {
val file = copyToTmpFile(uri)
val thumbnailResult = thumbnailFactory.createImageThumbnail(
file = file,
mimeType = mimeType,
)
val imageInfo = contentResolver.openInputStream(uri).use { input ->
val bitmap = BitmapFactory.decodeStream(input, null, null)!!
ImageInfo(
width = bitmap.width.toLong(),
height = bitmap.height.toLong(),
mimetype = mimeType,
size = file.length(),
thumbnailInfo = thumbnailResult?.info,
thumbnailSource = null,
blurhash = thumbnailResult?.blurhash,
)
}
removeSensitiveImageMetadata(file)
return MediaUploadInfo.Image(
file = file,
imageInfo = imageInfo,
thumbnailFile = thumbnailResult?.file
)
}
return if (shouldBeCompressed) {
processImageWithCompression()
} else {
processImageWithoutCompression()
}
}
private suspend fun processVideo(uri: Uri, mimeType: String?, videoCompressionPreset: VideoCompressionPreset): MediaUploadInfo {
Timber.d("Processing video ${uri.path.orEmpty().hash()}")
val resultFile = runCatchingExceptions {
videoCompressor.compress(uri, videoCompressionPreset)
.onEach {
if (it is VideoTranscodingEvent.Progress) {
Timber.d("Video compression progress: ${it.value}%")
} else if (it is VideoTranscodingEvent.Completed) {
Timber.d("Video compression completed: ${it.file.path}")
}
}
.filterIsInstance<VideoTranscodingEvent.Completed>()
.first()
.file
}
.onFailure {
Timber.e(it, "Failed to compress video: $uri")
}
.getOrNull()
if (resultFile != null) {
val thumbnailInfo = thumbnailFactory.createVideoThumbnail(resultFile)
val videoInfo = extractVideoMetadata(resultFile, mimeType, thumbnailInfo)
return MediaUploadInfo.Video(
file = resultFile,
videoInfo = videoInfo,
thumbnailFile = thumbnailInfo?.file
)
} else {
Timber.d("Could not transcode video ${uri.path.orEmpty().hash()}, sending original file as plain file")
// If the video could not be compressed, just use the original one, but send it as a file
return processFile(uri, MimeTypes.OctetStream)
}
}
private suspend fun processAudio(uri: Uri, mimeType: String?): MediaUploadInfo {
Timber.d("Processing audio ${uri.path.orEmpty().hash()}")
val file = copyToTmpFile(uri)
return MediaMetadataRetriever().runAndRelease {
setDataSource(context, Uri.fromFile(file))
val info = AudioInfo(
duration = extractDuration(),
size = file.length(),
mimetype = mimeType,
)
MediaUploadInfo.Audio(file, info)
}
}
private fun removeSensitiveImageMetadata(file: File) {
// Remove GPS info, user comments and subject location tags
ExifInterface(file).apply {
// See ExifInterface.TAG_GPS_INFO_IFD_POINTER
setAttribute("GPSInfoIFDPointer", null)
setAttribute(ExifInterface.TAG_USER_COMMENT, null)
setAttribute(ExifInterface.TAG_IMAGE_DESCRIPTION, null)
setAttribute(ExifInterface.TAG_GPS_VERSION_ID, null)
setAttribute(ExifInterface.TAG_GPS_ALTITUDE_REF, null)
setAttribute(ExifInterface.TAG_GPS_ALTITUDE, null)
setAttribute(ExifInterface.TAG_GPS_TIMESTAMP, null)
setAttribute(ExifInterface.TAG_GPS_DATESTAMP, null)
setAttribute(ExifInterface.TAG_GPS_SATELLITES, null)
setAttribute(ExifInterface.TAG_GPS_STATUS, null)
setAttribute(ExifInterface.TAG_GPS_MEASURE_MODE, null)
setAttribute(ExifInterface.TAG_GPS_DOP, null)
setAttribute(ExifInterface.TAG_GPS_SPEED_REF, null)
setAttribute(ExifInterface.TAG_GPS_SPEED, null)
setAttribute(ExifInterface.TAG_GPS_TRACK_REF, null)
setAttribute(ExifInterface.TAG_GPS_TRACK, null)
setAttribute(ExifInterface.TAG_GPS_IMG_DIRECTION_REF, null)
setAttribute(ExifInterface.TAG_GPS_IMG_DIRECTION, null)
setAttribute(ExifInterface.TAG_GPS_MAP_DATUM, null)
setAttribute(ExifInterface.TAG_GPS_DEST_BEARING_REF, null)
setAttribute(ExifInterface.TAG_GPS_DEST_BEARING, null)
setAttribute(ExifInterface.TAG_GPS_DEST_DISTANCE_REF, null)
setAttribute(ExifInterface.TAG_GPS_DEST_DISTANCE, null)
setAttribute(ExifInterface.TAG_GPS_PROCESSING_METHOD, null)
setAttribute(ExifInterface.TAG_GPS_AREA_INFORMATION, null)
setAttribute(ExifInterface.TAG_GPS_DIFFERENTIAL, null)
setAttribute(ExifInterface.TAG_GPS_H_POSITIONING_ERROR, null)
setAttribute(ExifInterface.TAG_GPS_LATITUDE, null)
setAttribute(ExifInterface.TAG_GPS_LATITUDE_REF, null)
setAttribute(ExifInterface.TAG_GPS_LONGITUDE, null)
setAttribute(ExifInterface.TAG_GPS_LONGITUDE_REF, null)
setAttribute(ExifInterface.TAG_GPS_DEST_LONGITUDE, null)
setAttribute(ExifInterface.TAG_GPS_DEST_LONGITUDE_REF, null)
tryOrNull { saveAttributes() }
}
}
private suspend fun createTmpFileWithInput(inputStream: InputStream): File? {
return withContext(coroutineDispatchers.io) {
tryOrNull {
if (!baseTmpFileDir.exists()) {
baseTmpFileDir.mkdirs()
}
val tmpFile = context.createTmpFile(baseTmpFileDir)
tmpFile.outputStream().use { inputStream.copyTo(it) }
tmpFile
}
}
}
private fun extractVideoMetadata(file: File, mimeType: String?, thumbnailResult: ThumbnailResult?): VideoInfo =
MediaMetadataRetriever().runAndRelease {
setDataSource(context, Uri.fromFile(file))
val rotation = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION)?.toInt() ?: 0
val rawWidth = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)?.toLong() ?: 0L
val rawHeight = extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toLong() ?: 0L
val (width, height) = if (rotation == 90 || rotation == 270) rawHeight to rawWidth else rawWidth to rawHeight
VideoInfo(
duration = extractDuration(),
width = width,
height = height,
mimetype = mimeType,
size = file.length(),
thumbnailInfo = thumbnailResult?.info,
// Will be computed by the rust sdk
thumbnailSource = null,
blurhash = thumbnailResult?.blurhash,
)
}
private suspend fun copyToTmpFile(uri: Uri): File {
return contentResolver.openInputStream(uri)?.use { createTmpFileWithInput(it) }
?: error("Could not copy the contents of $uri to a temporary file")
}
}
private fun ImageCompressionResult.toImageInfo(mimeType: String, thumbnailResult: ThumbnailResult?) = ImageInfo(
width = width.toLong(),
height = height.toLong(),
mimetype = mimeType,
size = size,
thumbnailInfo = thumbnailResult?.info,
// Will be computed by the rust sdk
thumbnailSource = null,
blurhash = thumbnailResult?.blurhash,
)
private fun MediaMetadataRetriever.extractDuration(): Duration {
val durationInMs = extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLong() ?: 0L
return durationInMs.milliseconds
}
@@ -0,0 +1,26 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import dev.zacsweers.metro.ContributesBinding
import io.element.android.libraries.di.SessionScope
import io.element.android.libraries.matrix.api.MatrixClient
import io.element.android.libraries.mediaupload.api.MaxUploadSizeProvider
/**
* Provides the maximum upload size allowed by the Matrix server.
*/
@ContributesBinding(SessionScope::class)
class DefaultMaxUploadSizeProvider(
private val matrixClient: MatrixClient,
) : MaxUploadSizeProvider {
override suspend fun getMaxUploadSize(): Result<Long> {
return matrixClient.getMaxFileUploadSize()
}
}
@@ -0,0 +1,26 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import dev.zacsweers.metro.ContributesBinding
import io.element.android.libraries.di.SessionScope
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfig
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfigProvider
import io.element.android.libraries.preferences.api.store.SessionPreferencesStore
import kotlinx.coroutines.flow.first
@ContributesBinding(SessionScope::class)
class DefaultMediaOptimizationConfigProvider(
private val sessionPreferencesStore: SessionPreferencesStore,
) : MediaOptimizationConfigProvider {
override suspend fun get(): MediaOptimizationConfig = MediaOptimizationConfig(
compressImages = sessionPreferencesStore.doesOptimizeImages().first(),
videoCompressionPreset = sessionPreferencesStore.getVideoCompressionPreset().first(),
)
}
@@ -0,0 +1,264 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import android.net.Uri
import dev.zacsweers.metro.ContributesBinding
import io.element.android.libraries.androidutils.hash.hash
import io.element.android.libraries.core.extensions.flatMap
import io.element.android.libraries.core.extensions.flatMapCatching
import io.element.android.libraries.di.RoomScope
import io.element.android.libraries.di.SessionScope
import io.element.android.libraries.matrix.api.core.EventId
import io.element.android.libraries.matrix.api.media.MediaUploadHandler
import io.element.android.libraries.matrix.api.room.CreateTimelineParams
import io.element.android.libraries.matrix.api.room.JoinedRoom
import io.element.android.libraries.matrix.api.timeline.Timeline
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfig
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfigProvider
import io.element.android.libraries.mediaupload.api.MediaPreProcessor
import io.element.android.libraries.mediaupload.api.MediaSender
import io.element.android.libraries.mediaupload.api.MediaSenderFactory
import io.element.android.libraries.mediaupload.api.MediaSenderRoomFactory
import io.element.android.libraries.mediaupload.api.MediaUploadInfo
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Job
import timber.log.Timber
import java.io.File
import java.util.concurrent.ConcurrentHashMap
@ContributesBinding(RoomScope::class)
class DefaultMediaSenderFactory(
private val preProcessor: MediaPreProcessor,
private val room: JoinedRoom,
private val mediaOptimizationConfigProvider: MediaOptimizationConfigProvider,
) : MediaSenderFactory {
override fun create(
timelineMode: Timeline.Mode,
): MediaSender {
return DefaultMediaSender(
preProcessor = preProcessor,
room = room,
timelineMode = timelineMode,
mediaOptimizationConfigProvider = mediaOptimizationConfigProvider,
)
}
}
@ContributesBinding(SessionScope::class)
class DefaultMediaSenderRoomFactory(
private val preProcessor: MediaPreProcessor,
private val mediaOptimizationConfigProvider: MediaOptimizationConfigProvider,
) : MediaSenderRoomFactory {
override fun create(
room: JoinedRoom,
): MediaSender {
return DefaultMediaSender(
preProcessor = preProcessor,
room = room,
timelineMode = Timeline.Mode.Live,
mediaOptimizationConfigProvider = mediaOptimizationConfigProvider,
)
}
}
class DefaultMediaSender(
private val preProcessor: MediaPreProcessor,
private val room: JoinedRoom,
private val timelineMode: Timeline.Mode,
private val mediaOptimizationConfigProvider: MediaOptimizationConfigProvider,
) : MediaSender {
private val ongoingUploadJobs = ConcurrentHashMap<Job.Key, MediaUploadHandler>()
val hasOngoingMediaUploads get() = ongoingUploadJobs.isNotEmpty()
override suspend fun preProcessMedia(
uri: Uri,
mimeType: String,
mediaOptimizationConfig: MediaOptimizationConfig,
): Result<MediaUploadInfo> {
Timber.d("Pre-processing media | uri: ${mediaId(uri)} | mimeType: $mimeType")
return preProcessor
.process(
uri = uri,
mimeType = mimeType,
deleteOriginal = false,
mediaOptimizationConfig = mediaOptimizationConfig,
)
}
override suspend fun sendPreProcessedMedia(
mediaUploadInfo: MediaUploadInfo,
caption: String?,
formattedCaption: String?,
inReplyToEventId: EventId?,
): Result<Unit> {
val mediaLogId = mediaId(mediaUploadInfo.file)
return getTimeline().flatMap {
Timber.d("Started sending media $mediaLogId using timeline: ${it.mode}")
it.sendMedia(
uploadInfo = mediaUploadInfo,
caption = caption,
formattedCaption = formattedCaption,
inReplyToEventId = inReplyToEventId,
)
}
.handleSendResult(mediaLogId)
}
override suspend fun sendMedia(
uri: Uri,
mimeType: String,
caption: String?,
formattedCaption: String?,
inReplyToEventId: EventId?,
mediaOptimizationConfig: MediaOptimizationConfig,
): Result<Unit> {
return preProcessor
.process(
uri = uri,
mimeType = mimeType,
deleteOriginal = false,
mediaOptimizationConfig = mediaOptimizationConfig,
)
.flatMapCatching { info ->
getTimeline().getOrThrow().sendMedia(
uploadInfo = info,
caption = caption,
formattedCaption = formattedCaption,
inReplyToEventId = inReplyToEventId,
)
}
.handleSendResult(mediaId(uri))
}
override suspend fun sendVoiceMessage(
uri: Uri,
mimeType: String,
waveForm: List<Float>,
inReplyToEventId: EventId?,
): Result<Unit> {
return preProcessor
.process(
uri = uri,
mimeType = mimeType,
deleteOriginal = true,
mediaOptimizationConfig = mediaOptimizationConfigProvider.get(),
)
.flatMapCatching { info ->
val audioInfo = (info as MediaUploadInfo.Audio).audioInfo
val newInfo = MediaUploadInfo.VoiceMessage(
file = info.file,
audioInfo = audioInfo,
waveform = waveForm,
)
getTimeline().getOrThrow().sendMedia(
uploadInfo = newInfo,
caption = null,
formattedCaption = null,
inReplyToEventId = inReplyToEventId,
)
}
.handleSendResult(mediaId(uri))
}
private fun Result<Unit>.handleSendResult(mediaId: String) = this
.onFailure { error ->
val job = ongoingUploadJobs.remove(Job)
Timber.e(error, "Sending media $mediaId failed. Removing ongoing upload job. Total: ${ongoingUploadJobs.size}")
if (error !is CancellationException) {
job?.cancel()
}
}
.onSuccess {
Timber.d("Sent media $mediaId successfully. Removing ongoing upload job. Total: ${ongoingUploadJobs.size}")
ongoingUploadJobs.remove(Job)
}
private suspend fun Timeline.sendMedia(
uploadInfo: MediaUploadInfo,
caption: String?,
formattedCaption: String?,
inReplyToEventId: EventId?,
): Result<Unit> {
val handler = when (uploadInfo) {
is MediaUploadInfo.Image -> {
sendImage(
file = uploadInfo.file,
thumbnailFile = uploadInfo.thumbnailFile,
imageInfo = uploadInfo.imageInfo,
caption = caption,
formattedCaption = formattedCaption,
inReplyToEventId = inReplyToEventId,
)
}
is MediaUploadInfo.Video -> {
sendVideo(
file = uploadInfo.file,
thumbnailFile = uploadInfo.thumbnailFile,
videoInfo = uploadInfo.videoInfo,
caption = caption,
formattedCaption = formattedCaption,
inReplyToEventId = inReplyToEventId,
)
}
is MediaUploadInfo.Audio -> {
sendAudio(
file = uploadInfo.file,
audioInfo = uploadInfo.audioInfo,
caption = caption,
formattedCaption = formattedCaption,
inReplyToEventId = inReplyToEventId,
)
}
is MediaUploadInfo.VoiceMessage -> {
sendVoiceMessage(
file = uploadInfo.file,
audioInfo = uploadInfo.audioInfo,
waveform = uploadInfo.waveform,
inReplyToEventId = inReplyToEventId,
)
}
is MediaUploadInfo.AnyFile -> {
sendFile(
file = uploadInfo.file,
fileInfo = uploadInfo.fileInfo,
caption = caption,
formattedCaption = formattedCaption,
inReplyToEventId = inReplyToEventId,
)
}
}
// We handle the cancellations here manually, so we suppress the warning
@Suppress("RunCatchingNotAllowed")
return handler
.mapCatching { uploadHandler ->
Timber.d("Added ongoing upload job, total: ${ongoingUploadJobs.size + 1}")
ongoingUploadJobs[Job] = uploadHandler
uploadHandler.await()
}
}
private suspend fun getTimeline(): Result<Timeline> {
return when (timelineMode) {
is Timeline.Mode.Thread -> {
room.createTimeline(CreateTimelineParams.Threaded(threadRootEventId = timelineMode.threadRootId))
}
else -> Result.success(room.liveTimeline)
}
}
/**
* Clean up any temporary files or resources used during the media processing.
*/
override fun cleanUp() = preProcessor.cleanUp()
}
private fun mediaId(uri: Uri?): String = uri?.path.orEmpty().hash()
private fun mediaId(file: File): String = file.path.orEmpty().hash()
@@ -0,0 +1,121 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import android.content.Context
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import androidx.exifinterface.media.ExifInterface
import dev.zacsweers.metro.Inject
import io.element.android.libraries.androidutils.bitmap.calculateInSampleSize
import io.element.android.libraries.androidutils.bitmap.resizeToMax
import io.element.android.libraries.androidutils.bitmap.rotateToExifMetadataOrientation
import io.element.android.libraries.androidutils.file.createTmpFile
import io.element.android.libraries.core.coroutine.CoroutineDispatchers
import io.element.android.libraries.core.extensions.runCatchingExceptions
import io.element.android.libraries.di.annotations.ApplicationContext
import kotlinx.coroutines.withContext
import java.io.File
import java.io.InputStream
@Inject
class ImageCompressor(
@ApplicationContext private val context: Context,
private val dispatchers: CoroutineDispatchers,
) {
/**
* Decodes the [inputStream] into a [Bitmap] and applies the needed transformations (rotation, scale) based on [resizeMode], then writes it into a
* temporary file using the passed [format], [orientation] and [desiredQuality].
* @return a [Result] containing the resulting [ImageCompressionResult] with the temporary [File] and some metadata.
*/
suspend fun compressToTmpFile(
inputStreamProvider: () -> InputStream,
resizeMode: ResizeMode,
mimeType: String,
orientation: Int = ExifInterface.ORIENTATION_UNDEFINED,
desiredQuality: Int = 78,
): Result<ImageCompressionResult> = withContext(dispatchers.io) {
runCatchingExceptions {
val format = mimeTypeToCompressFormat(mimeType)
val extension = mimeTypeToCompressFileExtension(mimeType)
val compressedBitmap = compressToBitmap(inputStreamProvider, resizeMode, orientation).getOrThrow()
// Encode bitmap to the destination temporary file
val tmpFile = context.createTmpFile(extension = extension)
tmpFile.outputStream().use {
compressedBitmap.compress(format, desiredQuality, it)
}
ImageCompressionResult(
file = tmpFile,
width = compressedBitmap.width,
height = compressedBitmap.height,
size = tmpFile.length()
)
}
}
/**
* Decodes the inputStream from [inputStreamProvider] into a [Bitmap] and applies the needed transformations (rotation, scale)
* based on [resizeMode] and [orientation].
* @return a [Result] containing the resulting [Bitmap].
*/
fun compressToBitmap(
inputStreamProvider: () -> InputStream,
resizeMode: ResizeMode,
orientation: Int,
): Result<Bitmap> = runCatchingExceptions {
val options = BitmapFactory.Options()
// Decode bounds
inputStreamProvider().use { input ->
calculateDecodingScale(input, resizeMode, options)
}
// Decode the actual bitmap
inputStreamProvider().use { input ->
// Now read the actual image and rotate it to match its metadata
options.inJustDecodeBounds = false
val decodedBitmap = BitmapFactory.decodeStream(input, null, options)
?: error("Decoding Bitmap from InputStream failed")
val rotatedBitmap = decodedBitmap.rotateToExifMetadataOrientation(orientation)
if (resizeMode is ResizeMode.Strict) {
rotatedBitmap.resizeToMax(resizeMode.maxWidth, resizeMode.maxHeight)
} else {
rotatedBitmap
}
}
}
private fun calculateDecodingScale(
inputStream: InputStream,
resizeMode: ResizeMode,
options: BitmapFactory.Options
) {
val (width, height) = when (resizeMode) {
is ResizeMode.Approximate -> resizeMode.desiredWidth to resizeMode.desiredHeight
is ResizeMode.Strict -> resizeMode.maxWidth / 2 to resizeMode.maxHeight / 2
is ResizeMode.None -> return
}
// Read bounds only
options.inJustDecodeBounds = true
BitmapFactory.decodeStream(inputStream, null, options)
// Set sample size based on the outWidth and outHeight
options.inSampleSize = options.calculateInSampleSize(width, height)
}
}
data class ImageCompressionResult(
val file: File,
val width: Int,
val height: Int,
val size: Long,
)
sealed interface ResizeMode {
data object None : ResizeMode
data class Approximate(val desiredWidth: Int, val desiredHeight: Int) : ResizeMode
data class Strict(val maxWidth: Int, val maxHeight: Int) : ResizeMode
}
@@ -0,0 +1,27 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2024, 2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import android.graphics.Bitmap
import io.element.android.libraries.core.mimetype.MimeTypes
fun mimeTypeToCompressFormat(mimeType: String) = when (mimeType) {
MimeTypes.Png -> Bitmap.CompressFormat.PNG
else -> Bitmap.CompressFormat.JPEG
}
fun mimeTypeToCompressFileExtension(mimeType: String) = when (mimeType) {
MimeTypes.Png -> "png"
else -> "jpeg"
}
fun mimeTypeToThumbnailMimeType(mimeType: String) = when (mimeType) {
MimeTypes.Png -> MimeTypes.Png
else -> MimeTypes.Jpeg
}
@@ -0,0 +1,144 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import android.annotation.SuppressLint
import android.content.Context
import android.graphics.Bitmap
import android.media.MediaMetadataRetriever
import android.media.MediaMetadataRetriever.OPTION_CLOSEST_SYNC
import android.media.ThumbnailUtils
import android.os.Build
import android.os.CancellationSignal
import android.provider.MediaStore
import android.util.Size
import androidx.core.net.toUri
import com.vanniktech.blurhash.BlurHash
import dev.zacsweers.metro.Inject
import io.element.android.libraries.androidutils.bitmap.resizeToMax
import io.element.android.libraries.androidutils.file.createTmpFile
import io.element.android.libraries.androidutils.media.runAndRelease
import io.element.android.libraries.core.mimetype.MimeTypes
import io.element.android.libraries.di.annotations.ApplicationContext
import io.element.android.libraries.matrix.api.media.ThumbnailInfo
import io.element.android.services.toolbox.api.sdk.BuildVersionSdkIntProvider
import kotlinx.coroutines.suspendCancellableCoroutine
import timber.log.Timber
import java.io.File
import java.io.IOException
import kotlin.coroutines.resume
/**
* Max width of thumbnail images.
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails).
*/
private const val THUMB_MAX_WIDTH = 800
/**
* Max height of thumbnail images.
* See [the Matrix spec](https://spec.matrix.org/latest/client-server-api/?ref=blog.gitter.im#thumbnails).
*/
private const val THUMB_MAX_HEIGHT = 600
/**
* Frame of the video to be used for generating a thumbnail.
*/
private const val VIDEO_THUMB_FRAME = 0L
@Inject
class ThumbnailFactory(
@ApplicationContext private val context: Context,
private val sdkIntProvider: BuildVersionSdkIntProvider
) {
@SuppressLint("NewApi")
suspend fun createImageThumbnail(
file: File,
mimeType: String,
): ThumbnailResult? {
return createThumbnail(mimeType = mimeType) { cancellationSignal ->
try {
// This API works correctly with GIF
if (sdkIntProvider.isAtLeast(Build.VERSION_CODES.Q)) {
try {
ThumbnailUtils.createImageThumbnail(
file,
Size(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT),
cancellationSignal
)
} catch (ioException: IOException) {
Timber.w(ioException, "Failed to create thumbnail for $file")
null
}
} else {
@Suppress("DEPRECATION")
ThumbnailUtils.createImageThumbnail(
file.path,
MediaStore.Images.Thumbnails.MINI_KIND,
)
}
} catch (throwable: Throwable) {
Timber.w(throwable, "Failed to create thumbnail for $file")
null
}
}
}
suspend fun createVideoThumbnail(file: File): ThumbnailResult? {
return createThumbnail(mimeType = MimeTypes.Jpeg) {
MediaMetadataRetriever().runAndRelease {
setDataSource(context, file.toUri())
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) {
getScaledFrameAtTime(VIDEO_THUMB_FRAME, OPTION_CLOSEST_SYNC, THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT)
} else {
getFrameAtTime(VIDEO_THUMB_FRAME)?.resizeToMax(THUMB_MAX_WIDTH, THUMB_MAX_HEIGHT)
}
}
}
}
private suspend fun createThumbnail(
mimeType: String,
bitmapFactory: (CancellationSignal) -> Bitmap?,
): ThumbnailResult? = suspendCancellableCoroutine { continuation ->
val cancellationSignal = CancellationSignal()
continuation.invokeOnCancellation {
cancellationSignal.cancel()
}
val bitmapThumbnail: Bitmap? = bitmapFactory(cancellationSignal)
if (bitmapThumbnail == null) {
continuation.resume(null)
return@suspendCancellableCoroutine
}
val format = mimeTypeToCompressFormat(mimeType)
val extension = mimeTypeToCompressFileExtension(mimeType)
val thumbnailFile = context.createTmpFile(extension = extension)
thumbnailFile.outputStream().use { outputStream ->
bitmapThumbnail.compress(format, 78, outputStream)
}
val blurhash = BlurHash.encode(bitmapThumbnail, 3, 3)
val thumbnailResult = ThumbnailResult(
file = thumbnailFile,
info = ThumbnailInfo(
width = bitmapThumbnail.width.toLong(),
height = bitmapThumbnail.height.toLong(),
mimetype = mimeTypeToThumbnailMimeType(mimeType),
size = thumbnailFile.length()
),
blurhash = blurhash
)
bitmapThumbnail.recycle()
continuation.resume(thumbnailResult)
}
}
data class ThumbnailResult(
val file: File,
val info: ThumbnailInfo,
val blurhash: String,
)
@@ -0,0 +1,196 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import android.content.Context
import android.media.MediaCodecInfo
import android.media.MediaMetadataRetriever
import android.net.Uri
import android.util.Size
import androidx.annotation.OptIn
import androidx.media3.common.MediaItem
import androidx.media3.common.MimeTypes
import androidx.media3.common.util.UnstableApi
import androidx.media3.effect.Presentation
import androidx.media3.transformer.Composition
import androidx.media3.transformer.DefaultEncoderFactory
import androidx.media3.transformer.EditedMediaItem
import androidx.media3.transformer.Effects
import androidx.media3.transformer.ExportException
import androidx.media3.transformer.ExportResult
import androidx.media3.transformer.ProgressHolder
import androidx.media3.transformer.TransformationRequest
import androidx.media3.transformer.Transformer
import androidx.media3.transformer.VideoEncoderSettings
import dev.zacsweers.metro.Inject
import io.element.android.libraries.androidutils.file.createTmpFile
import io.element.android.libraries.androidutils.file.safeDelete
import io.element.android.libraries.core.extensions.runCatchingExceptions
import io.element.android.libraries.di.annotations.ApplicationContext
import io.element.android.libraries.preferences.api.store.VideoCompressionPreset
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.channels.awaitClose
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.callbackFlow
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import timber.log.Timber
import java.io.File
@Inject
class VideoCompressor(
@ApplicationContext private val context: Context,
) {
@OptIn(UnstableApi::class)
fun compress(uri: Uri, videoCompressionPreset: VideoCompressionPreset): Flow<VideoTranscodingEvent> = callbackFlow {
val metadata = getVideoMetadata(uri)
val videoCompressorConfig = VideoCompressorConfigFactory.create(
metadata = metadata,
preset = videoCompressionPreset,
)
val tmpFile = context.createTmpFile(extension = "mp4")
val width = metadata?.width ?: Int.MAX_VALUE
val height = metadata?.height ?: Int.MAX_VALUE
val videoResizeEffect = run {
val outputSize = videoCompressorConfig.videoCompressorHelper.getOutputSize(Size(width, height))
if (metadata?.rotation == 90 || metadata?.rotation == 270) {
// If the video is rotated, we need to swap width and height
Presentation.createForWidthAndHeight(
outputSize.height,
outputSize.width,
Presentation.LAYOUT_SCALE_TO_FIT,
)
} else {
// Otherwise, we can use the original width and height
Presentation.createForWidthAndHeight(
outputSize.width,
outputSize.height,
Presentation.LAYOUT_SCALE_TO_FIT,
)
}
}
// If we are resizing, we also want to reduce set frame rate to the default value (30fps)
val newFrameRate = videoCompressorConfig.newFrameRate
// If we need to resize the video, we also want to recalculate the bitrate
val newBitrate = videoCompressorConfig.newBitRate
val inputMediaItem = MediaItem.fromUri(uri)
val outputMediaItem = EditedMediaItem.Builder(inputMediaItem)
.setFrameRate(newFrameRate)
.setEffects(Effects(emptyList(), listOf(videoResizeEffect)))
.build()
val encoderFactory = DefaultEncoderFactory.Builder(context)
.setRequestedVideoEncoderSettings(
VideoEncoderSettings.Builder()
// Use VBR which is generally better for quality and compatibility, although slightly worse for file size
.setBitrateMode(MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR)
.setBitrate(newBitrate)
.build()
)
.build()
val videoTransformer = Transformer.Builder(context)
.setVideoMimeType(MimeTypes.VIDEO_H264)
.setAudioMimeType(MimeTypes.AUDIO_AAC)
.setPortraitEncodingEnabled(false)
.setEncoderFactory(encoderFactory)
.addListener(object : Transformer.Listener {
override fun onCompleted(composition: Composition, exportResult: ExportResult) {
trySend(VideoTranscodingEvent.Completed(tmpFile))
close()
}
override fun onError(composition: Composition, exportResult: ExportResult, exportException: ExportException) {
Timber.e(exportException, "Video transcoding failed")
tmpFile.safeDelete()
close(exportException)
}
override fun onFallbackApplied(
composition: Composition,
originalTransformationRequest: TransformationRequest,
fallbackTransformationRequest: TransformationRequest
) = Unit
})
.build()
val progressJob = launch(Dispatchers.Main) {
val progressHolder = ProgressHolder()
while (isActive) {
val state = videoTransformer.getProgress(progressHolder)
if (state != Transformer.PROGRESS_STATE_NOT_STARTED) {
channel.send(VideoTranscodingEvent.Progress(progressHolder.progress.toFloat()))
}
delay(500)
}
}
withContext(Dispatchers.Main) {
videoTransformer.start(outputMediaItem, tmpFile.path)
}
awaitClose {
progressJob.cancel()
}
}
private fun getVideoMetadata(uri: Uri): VideoFileMetadata? {
return runCatchingExceptions {
MediaMetadataRetriever().use {
it.setDataSource(context, uri)
val width = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)?.toIntOrNull() ?: -1
val height = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)?.toIntOrNull() ?: -1
val bitrate = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_BITRATE)?.toLongOrNull() ?: -1
val frameRate = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE)?.toIntOrNull() ?: -1
val rotation = it.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION)?.toIntOrNull() ?: 0
val (actualWidth, actualHeight) = if (width == -1 || height == -1) {
// Try getting the first frame instead
val bitmap = it.getFrameAtTime(0) ?: return null
bitmap.width to bitmap.height
} else {
width to height
}
VideoFileMetadata(
width = actualWidth,
height = actualHeight,
bitrate = bitrate,
frameRate = frameRate,
rotation = rotation,
)
}
}.onFailure {
Timber.e(it, "Failed to get video dimensions")
}.getOrNull()
}
}
internal data class VideoFileMetadata(
val width: Int,
val height: Int,
val bitrate: Long,
val frameRate: Int,
val rotation: Int,
)
sealed interface VideoTranscodingEvent {
data class Progress(val value: Float) : VideoTranscodingEvent
data class Completed(val file: File) : VideoTranscodingEvent
}
@@ -0,0 +1,52 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import android.util.Size
import androidx.annotation.OptIn
import androidx.media3.common.util.UnstableApi
import io.element.android.libraries.androidutils.media.VideoCompressorHelper
import io.element.android.libraries.mediaupload.api.compressorHelper
import io.element.android.libraries.preferences.api.store.VideoCompressionPreset
import kotlin.math.min
@OptIn(UnstableApi::class)
internal object VideoCompressorConfigFactory {
private const val DEFAULT_FRAME_RATE = 30
fun create(
metadata: VideoFileMetadata?,
preset: VideoCompressionPreset,
): VideoCompressorConfig {
val width = metadata?.width?.takeIf { it >= 0 } ?: Int.MAX_VALUE
val height = metadata?.height?.takeIf { it >= 0 } ?: Int.MAX_VALUE
val originalFrameRate = metadata?.frameRate?.takeIf { it >= 0 } ?: DEFAULT_FRAME_RATE
val resizer = preset.compressorHelper()
// If we are resizing, we also want to reduce the frame rate to the default value (30fps)
val newFrameRate = min(originalFrameRate, DEFAULT_FRAME_RATE)
// If we need to resize the video, we also want to recalculate the bitrate
val newBitrate = resizer.calculateOptimalBitrate(Size(width, height), newFrameRate)
return VideoCompressorConfig(
videoCompressorHelper = resizer,
newBitRate = newBitrate.toInt(),
newFrameRate = newFrameRate,
)
}
}
@OptIn(UnstableApi::class)
internal data class VideoCompressorConfig(
val videoCompressorHelper: VideoCompressorHelper,
val newBitRate: Int,
val newFrameRate: Int,
)
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:6831610b21668c49e31732f9005177e959277233d3cab758910e061294f91d79
size 687979
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:77276f9b174f8823eaf787ab0a659199ef5d30c0361ec8b9b4f0890adb1907a1
size 9986336
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:a980f7b74cb9edc323919db8652798da4b3dcf865fc7b6a1eb1110096b7bfb4f
size 1856786
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0244590f2b4bcb62352b574e78bea940e8d89cfa69823b5208ef4c43e0abcb44
size 52079
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0ba904eae8773b70c75333db4de2f3ac45a8ad4ddba1b242f0b3cfc199391dd8
size 13
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fb58436524db95bd0c10b2c3023c2eb7b87404a2eab8987939f051647eb859d3
size 1673712
@@ -0,0 +1,443 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import android.content.Context
import android.net.Uri
import android.os.Build
import androidx.core.net.toUri
import androidx.test.platform.app.InstrumentationRegistry
import com.google.common.truth.Truth.assertThat
import io.element.android.libraries.androidutils.file.TemporaryUriDeleter
import io.element.android.libraries.core.mimetype.MimeTypes
import io.element.android.libraries.matrix.api.media.AudioInfo
import io.element.android.libraries.matrix.api.media.FileInfo
import io.element.android.libraries.matrix.api.media.ImageInfo
import io.element.android.libraries.matrix.api.media.ThumbnailInfo
import io.element.android.libraries.matrix.api.media.VideoInfo
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfig
import io.element.android.libraries.mediaupload.api.MediaPreProcessor
import io.element.android.libraries.mediaupload.api.MediaUploadInfo
import io.element.android.libraries.preferences.api.store.VideoCompressionPreset
import io.element.android.services.toolbox.test.sdk.FakeBuildVersionSdkIntProvider
import io.element.android.tests.testutils.fake.FakeTemporaryUriDeleter
import io.element.android.tests.testutils.lambda.lambdaRecorder
import io.element.android.tests.testutils.testCoroutineDispatchers
import kotlinx.coroutines.test.TestScope
import kotlinx.coroutines.test.runTest
import org.junit.Ignore
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
import java.io.File
import java.io.FileNotFoundException
import java.io.IOException
import kotlin.time.Duration
@RunWith(RobolectricTestRunner::class)
class AndroidMediaPreProcessorTest {
private suspend fun TestScope.process(
asset: Asset,
mediaOptimizationConfig: MediaOptimizationConfig,
sdkIntVersion: Int = Build.VERSION_CODES.P,
deleteOriginal: Boolean = false,
): MediaUploadInfo {
val context = InstrumentationRegistry.getInstrumentation().context
val deleteCallback = lambdaRecorder<Uri?, Unit> {}
val sut = createAndroidMediaPreProcessor(
context = context,
sdkIntVersion = sdkIntVersion,
temporaryUriDeleter = FakeTemporaryUriDeleter(deleteCallback),
)
val file = getFileFromAssets(context, asset.filename)
val result = sut.process(
uri = file.toUri(),
mimeType = asset.mimeType,
deleteOriginal = deleteOriginal,
mediaOptimizationConfig = mediaOptimizationConfig,
)
val data = result.getOrThrow()
assertThat(data.file.path).endsWith(asset.filename)
deleteCallback.assertions().isCalledExactly(if (deleteOriginal) 0 else 1)
return data
}
@Test
@Ignore("Ignore now that min API for enterprise is 33")
fun `test processing png`() = runTest {
val mediaUploadInfo = process(
asset = assetImagePng,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
)
val info = mediaUploadInfo as MediaUploadInfo.Image
assertThat(info.thumbnailFile).isNotNull()
assertThat(info.imageInfo).isEqualTo(
ImageInfo(
height = assetImagePng.height,
width = assetImagePng.width,
mimetype = assetImagePng.mimeType,
size = 2_026_433,
ThumbnailInfo(height = 25, width = 25, mimetype = MimeTypes.Png, size = 91),
thumbnailSource = null,
blurhash = "K00000fQfQfQfQfQfQfQfQ"
)
)
}
@Test
fun `test processing png api Q`() = runTest {
val mediaUploadInfo = process(
asset = assetImagePng,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
sdkIntVersion = Build.VERSION_CODES.Q,
)
val info = mediaUploadInfo as MediaUploadInfo.Image
assertThat(info.thumbnailFile).isNull()
assertThat(info.imageInfo).isEqualTo(
ImageInfo(
height = assetImagePng.height,
width = assetImagePng.width,
mimetype = assetImagePng.mimeType,
size = 2_026_433,
thumbnailInfo = null,
thumbnailSource = null,
blurhash = null,
)
)
}
@Test
@Ignore("Ignore now that min API for enterprise is 33")
fun `test processing png no compression`() = runTest {
val mediaUploadInfo = process(
asset = assetImagePng,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = false,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
)
val info = mediaUploadInfo as MediaUploadInfo.Image
assertThat(info.thumbnailFile).isNotNull()
assertThat(info.imageInfo).isEqualTo(
ImageInfo(
height = assetImagePng.height,
width = assetImagePng.width,
mimetype = assetImagePng.mimeType,
size = assetImagePng.size,
thumbnailInfo = ThumbnailInfo(height = 25, width = 25, mimetype = MimeTypes.Png, size = 91),
thumbnailSource = null,
blurhash = "K00000fQfQfQfQfQfQfQfQ",
)
)
}
@Test
@Ignore("Ignore now that min API for enterprise is 33")
fun `test processing png and delete`() = runTest {
val mediaUploadInfo = process(
asset = assetImagePng,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = false,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
deleteOriginal = true,
)
val info = mediaUploadInfo as MediaUploadInfo.Image
assertThat(info.thumbnailFile).isNotNull()
assertThat(info.imageInfo).isEqualTo(
ImageInfo(
height = assetImagePng.height,
width = assetImagePng.width,
mimetype = assetImagePng.mimeType,
size = assetImagePng.size,
thumbnailInfo = ThumbnailInfo(height = 25, width = 25, mimetype = MimeTypes.Png, size = 91),
thumbnailSource = null,
blurhash = "K00000fQfQfQfQfQfQfQfQ",
)
)
// Does not work
// assertThat(file.exists()).isFalse()
}
@Test
@Ignore("Ignore now that min API for enterprise is 33")
fun `test processing jpeg`() = runTest {
val mediaUploadInfo = process(
asset = assetImageJpeg,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
)
val info = mediaUploadInfo as MediaUploadInfo.Image
assertThat(info.thumbnailFile).isNotNull()
assertThat(info.imageInfo).isEqualTo(
ImageInfo(
height = 979,
width = 3006,
mimetype = MimeTypes.Jpeg,
size = 84_845,
ThumbnailInfo(height = 244, width = 751, mimetype = MimeTypes.Jpeg, size = 7_178),
thumbnailSource = null,
blurhash = "K07gBzX=j_D4xZjoaSe,s:"
)
)
}
@Test
fun `test processing jpeg api Q`() = runTest {
val mediaUploadInfo = process(
asset = assetImageJpeg,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
sdkIntVersion = Build.VERSION_CODES.Q,
)
val info = mediaUploadInfo as MediaUploadInfo.Image
assertThat(info.thumbnailFile).isNull()
assertThat(info.imageInfo).isEqualTo(
ImageInfo(
height = 979,
width = 3_006,
mimetype = MimeTypes.Jpeg,
size = 84_845,
thumbnailInfo = null,
thumbnailSource = null,
blurhash = null,
)
)
}
@Test
@Ignore("Ignore now that min API for enterprise is 33")
fun `test processing jpeg no compression`() = runTest {
val mediaUploadInfo = process(
asset = assetImageJpeg,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = false,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
)
val info = mediaUploadInfo as MediaUploadInfo.Image
assertThat(info.thumbnailFile).isNotNull()
assertThat(info.imageInfo).isEqualTo(
ImageInfo(
height = assetImageJpeg.height,
width = assetImageJpeg.width,
mimetype = assetImageJpeg.mimeType,
size = assetImageJpeg.size,
thumbnailInfo = ThumbnailInfo(height = 6, width = 6, mimetype = MimeTypes.Jpeg, size = 631),
thumbnailSource = null,
blurhash = "K00000fQfQfQfQfQfQfQfQ",
)
)
}
@Test
@Ignore("Ignore now that min API for enterprise is 33")
fun `test processing jpeg and delete`() = runTest {
val mediaUploadInfo = process(
asset = assetImageJpeg,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = false,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
deleteOriginal = true,
)
val info = mediaUploadInfo as MediaUploadInfo.Image
assertThat(info.thumbnailFile).isNotNull()
assertThat(info.imageInfo).isEqualTo(
ImageInfo(
height = assetImageJpeg.height,
width = assetImageJpeg.width,
mimetype = assetImageJpeg.mimeType,
size = assetImageJpeg.size,
thumbnailInfo = ThumbnailInfo(height = 6, width = 6, mimetype = MimeTypes.Jpeg, size = 631),
thumbnailSource = null,
blurhash = "K00000fQfQfQfQfQfQfQfQ",
)
)
// Does not work
// assertThat(file.exists()).isFalse()
}
@Test
@Ignore("Ignore now that min API for enterprise is 33")
fun `test processing gif`() = runTest {
val mediaUploadInfo = process(
asset = assetAnimatedGif,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
)
val info = mediaUploadInfo as MediaUploadInfo.Image
assertThat(info.thumbnailFile).isNotNull()
assertThat(info.imageInfo).isEqualTo(
ImageInfo(
height = assetAnimatedGif.height,
width = assetAnimatedGif.width,
mimetype = assetAnimatedGif.mimeType,
size = assetAnimatedGif.size,
thumbnailInfo = ThumbnailInfo(height = 50, width = 50, mimetype = MimeTypes.Jpeg, size = 691),
thumbnailSource = null,
blurhash = "K00000fQfQfQfQfQfQfQfQ",
)
)
}
@Test
fun `test processing file`() = runTest {
val mediaUploadInfo = process(
asset = assetText,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
)
val info = mediaUploadInfo as MediaUploadInfo.AnyFile
assertThat(info.fileInfo).isEqualTo(
FileInfo(
mimetype = assetText.mimeType,
size = assetText.size,
thumbnailInfo = null,
thumbnailSource = null,
)
)
}
@Ignore("Compressing video is not working with Robolectric")
@Test
fun `test processing video`() = runTest {
val mediaUploadInfo = process(
asset = assetVideo,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
)
val info = mediaUploadInfo as MediaUploadInfo.Video
assertThat(info.thumbnailFile).isNotNull()
assertThat(info.videoInfo).isEqualTo(
VideoInfo(
// Not available with Robolectric?
duration = Duration.ZERO,
height = 1_178,
width = 1_818,
mimetype = MimeTypes.Mp4,
size = 114_867,
thumbnailInfo = null,
thumbnailSource = null,
blurhash = null,
)
)
}
@Ignore("Compressing video is not working with Robolectric")
@Test
fun `test processing video no compression`() = runTest {
val mediaUploadInfo = process(
asset = assetVideo,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.HIGH,
),
)
val info = mediaUploadInfo as MediaUploadInfo.Video
// Computing thumbnailFile is failing with Robolectric
assertThat(info.thumbnailFile).isNull()
assertThat(info.videoInfo).isEqualTo(
VideoInfo(
// Not available with Robolectric?
duration = Duration.ZERO,
// Not available with Robolectric?
height = 0,
// Not available with Robolectric?
width = 0,
mimetype = MimeTypes.Mp4,
size = 1_673_712,
thumbnailInfo = null,
thumbnailSource = null,
blurhash = null,
)
)
}
@Test
fun `test processing audio`() = runTest {
val mediaUploadInfo = process(
asset = assetAudio,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
)
val info = mediaUploadInfo as MediaUploadInfo.Audio
assertThat(info.audioInfo).isEqualTo(
AudioInfo(
// Not available with Robolectric?
duration = Duration.ZERO,
size = 52_079,
mimetype = MimeTypes.Mp3,
)
)
}
@Test
fun `test file which does not exist`() = runTest {
val context = InstrumentationRegistry.getInstrumentation().context
val sut = createAndroidMediaPreProcessor(context)
val file = File(context.cacheDir, "not found.txt")
val result = sut.process(
uri = file.toUri(),
mimeType = MimeTypes.PlainText,
deleteOriginal = false,
mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
),
)
assertThat(result.isFailure).isTrue()
val failure = result.exceptionOrNull()
assertThat(failure).isInstanceOf(MediaPreProcessor.Failure::class.java)
assertThat(failure?.cause).isInstanceOf(FileNotFoundException::class.java)
}
private fun TestScope.createAndroidMediaPreProcessor(
context: Context,
sdkIntVersion: Int = Build.VERSION_CODES.P,
temporaryUriDeleter: TemporaryUriDeleter = FakeTemporaryUriDeleter(),
) = AndroidMediaPreProcessor(
context = context,
thumbnailFactory = ThumbnailFactory(context, FakeBuildVersionSdkIntProvider(sdkIntVersion)),
imageCompressor = ImageCompressor(context, testCoroutineDispatchers()),
videoCompressor = VideoCompressor(context),
coroutineDispatchers = testCoroutineDispatchers(),
temporaryUriDeleter = temporaryUriDeleter,
)
@Throws(IOException::class)
private fun getFileFromAssets(context: Context, fileName: String): File = File(context.cacheDir, fileName)
.also {
if (!it.exists()) {
it.outputStream().use { cache ->
context.assets.open(fileName).use { inputStream ->
inputStream.copyTo(cache)
}
}
}
}
}
@@ -0,0 +1,85 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2024, 2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import io.element.android.libraries.core.mimetype.MimeTypes
data class Asset(
val filename: String,
val mimeType: String,
val size: Long,
val width: Long?,
val height: Long?,
)
/**
* "image.png" is a 1_818 x 1_178 PNG image with a size of 1_856_786 bytes.
*/
val assetImagePng = Asset(
filename = "image.png",
mimeType = MimeTypes.Png,
size = 1_856_786,
width = 1_818,
height = 1_178,
)
/**
* "image.jpeg" is a 12_024 x 3_916, JPEG image with a size of 9_986_336 bytes.
*/
val assetImageJpeg = Asset(
filename = "image.jpeg",
mimeType = MimeTypes.Jpeg,
size = 9_986_336,
width = 12_024,
height = 3_916,
)
/**
* "video.mp4" is a 1_280 x 720, MP4 video with a size of 1_673_712 bytes.
*/
val assetVideo = Asset(
filename = "video.mp4",
mimeType = MimeTypes.Mp4,
size = 1_673_712,
width = 1_280,
height = 720,
)
/**
* "sample3s.mp3" is a 3 seconds MP3 audio file with a size of 52_079 bytes.
*/
val assetAudio = Asset(
filename = "sample3s.mp3",
mimeType = MimeTypes.Mp3,
size = 52_079,
width = null,
height = null,
)
/**
* "text.txt" is a 13 bytes text file.
*/
val assetText = Asset(
filename = "text.txt",
mimeType = MimeTypes.PlainText,
size = 13,
width = null,
height = null,
)
/**
* "animated_gif.gif" is a 800 x 600, GIF image with a size of 687_979 bytes.
*/
val assetAnimatedGif = Asset(
filename = "animated_gif.gif",
mimeType = MimeTypes.Gif,
size = 687_979,
width = 800,
height = 600,
)
@@ -0,0 +1,171 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import android.net.Uri
import com.google.common.truth.Truth.assertThat
import io.element.android.libraries.core.mimetype.MimeTypes
import io.element.android.libraries.matrix.api.core.EventId
import io.element.android.libraries.matrix.api.media.FileInfo
import io.element.android.libraries.matrix.api.media.ImageInfo
import io.element.android.libraries.matrix.api.room.JoinedRoom
import io.element.android.libraries.matrix.api.timeline.Timeline
import io.element.android.libraries.matrix.test.media.FakeMediaUploadHandler
import io.element.android.libraries.matrix.test.room.FakeJoinedRoom
import io.element.android.libraries.matrix.test.timeline.FakeTimeline
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfig
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfigProvider
import io.element.android.libraries.mediaupload.api.MediaPreProcessor
import io.element.android.libraries.mediaupload.test.FakeMediaPreProcessor
import io.element.android.libraries.preferences.api.store.VideoCompressionPreset
import io.element.android.tests.testutils.lambda.lambdaRecorder
import kotlinx.coroutines.ExperimentalCoroutinesApi
import kotlinx.coroutines.launch
import kotlinx.coroutines.test.StandardTestDispatcher
import kotlinx.coroutines.test.advanceTimeBy
import kotlinx.coroutines.test.runTest
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
import java.io.File
@RunWith(RobolectricTestRunner::class)
class DefaultMediaSenderTest {
private val mediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
)
@Test
fun `given an attachment when sending it the preprocessor always runs`() = runTest {
val preProcessor = FakeMediaPreProcessor()
val sender = createDefaultMediaSender(
preProcessor = preProcessor,
room = FakeJoinedRoom(
liveTimeline = FakeTimeline().apply {
sendFileLambda = lambdaRecorder<
File,
FileInfo,
String?,
String?,
EventId?,
Result<FakeMediaUploadHandler>,
> { _, _, _, _, _ ->
Result.success(FakeMediaUploadHandler())
}
},
)
)
val uri = Uri.parse("content://image.jpg")
sender.sendMedia(uri = uri, mimeType = MimeTypes.Jpeg, mediaOptimizationConfig = mediaOptimizationConfig)
assertThat(preProcessor.processCallCount).isEqualTo(1)
}
@Test
fun `given an attachment when sending it the Room will call sendMedia`() = runTest {
val sendImageResult =
lambdaRecorder { _: File, _: File?, _: ImageInfo, _: String?, _: String?, _: EventId? ->
Result.success(FakeMediaUploadHandler())
}
val room = FakeJoinedRoom(
liveTimeline = FakeTimeline().apply {
sendImageLambda = sendImageResult
},
)
val sender = createDefaultMediaSender(room = room)
val uri = Uri.parse("content://image.jpg")
sender.sendMedia(uri = uri, mimeType = MimeTypes.Jpeg, mediaOptimizationConfig = mediaOptimizationConfig)
}
@Test
fun `given a failure in the preprocessor when sending the whole process fails`() = runTest {
val preProcessor = FakeMediaPreProcessor().apply {
givenResult(Result.failure(Exception()))
}
val sender = createDefaultMediaSender(preProcessor)
val uri = Uri.parse("content://image.jpg")
val result = sender.sendMedia(uri = uri, mimeType = MimeTypes.Jpeg, mediaOptimizationConfig = mediaOptimizationConfig)
assertThat(result.exceptionOrNull()).isNotNull()
}
@Test
fun `given a failure in the media upload when sending the whole process fails`() = runTest {
val preProcessor = FakeMediaPreProcessor().apply {
givenImageResult()
}
val sendImageResult =
lambdaRecorder { _: File, _: File?, _: ImageInfo, _: String?, _: String?, _: EventId? ->
Result.failure<FakeMediaUploadHandler>(Exception())
}
val room = FakeJoinedRoom(
liveTimeline = FakeTimeline().apply {
sendImageLambda = sendImageResult
},
)
val sender = createDefaultMediaSender(
preProcessor = preProcessor,
room = room,
)
val uri = Uri.parse("content://image.jpg")
val result = sender.sendMedia(uri = uri, mimeType = MimeTypes.Jpeg, mediaOptimizationConfig = mediaOptimizationConfig)
assertThat(result.exceptionOrNull()).isNotNull()
}
@OptIn(ExperimentalCoroutinesApi::class)
@Test
fun `given a cancellation in the media upload when sending the job is cancelled`() = runTest(StandardTestDispatcher()) {
val sendFileResult =
lambdaRecorder<File, FileInfo, String?, String?, EventId?, Result<FakeMediaUploadHandler>> { _, _, _, _, _ ->
Result.success(FakeMediaUploadHandler())
}
val room = FakeJoinedRoom(
liveTimeline = FakeTimeline().apply {
sendFileLambda = sendFileResult
},
)
val sender = createDefaultMediaSender(room = room)
val sendJob = launch {
val uri = Uri.parse("content://image.jpg")
sender.sendMedia(uri = uri, mimeType = MimeTypes.Jpeg, mediaOptimizationConfig = mediaOptimizationConfig)
}
// Wait until several internal tasks run and the file is being uploaded
advanceTimeBy(3L)
// Assert the file is being uploaded
assertThat(sender.hasOngoingMediaUploads).isTrue()
// Cancel the coroutine
sendJob.cancel()
// Wait for the coroutine cleanup to happen
advanceTimeBy(1L)
// Assert the file is not being uploaded anymore
assertThat(sender.hasOngoingMediaUploads).isFalse()
sendFileResult.assertions().isCalledOnce()
}
private fun createDefaultMediaSender(
preProcessor: MediaPreProcessor = FakeMediaPreProcessor(),
room: JoinedRoom = FakeJoinedRoom(),
mediaOptimizationConfigProvider: MediaOptimizationConfigProvider = MediaOptimizationConfigProvider { mediaOptimizationConfig },
) = DefaultMediaSender(
preProcessor = preProcessor,
room = room,
timelineMode = Timeline.Mode.Live,
mediaOptimizationConfigProvider = mediaOptimizationConfigProvider,
)
}
@@ -0,0 +1,110 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.impl
import androidx.media3.transformer.VideoEncoderSettings
import com.google.common.truth.Truth.assertThat
import io.element.android.libraries.preferences.api.store.VideoCompressionPreset
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
@Suppress("NOTHING_TO_INLINE")
@RunWith(RobolectricTestRunner::class)
class VideoCompressorConfigFactoryTest {
@Test
fun `if we don't have metadata the video will be resized`() {
// Given
val metadata = null
val preset = VideoCompressionPreset.STANDARD
// When
val videoCompressorConfig = VideoCompressorConfigFactory.create(
metadata = metadata,
preset = preset,
)
// Then
assertThat(videoCompressorConfig.videoCompressorHelper).isNotNull()
assertThat(videoCompressorConfig.newFrameRate).isEqualTo(30)
assertThat(videoCompressorConfig.newBitRate).isNotEqualTo(VideoEncoderSettings.NO_VALUE)
}
@Test
fun `if the video should be compressed and is larger than 720p it will be resized`() {
// Given
val metadata = VideoFileMetadata(width = 1920, height = 1080, bitrate = 1_000_000, frameRate = 50, rotation = 0)
val preset = VideoCompressionPreset.STANDARD
// When
val videoCompressorConfig = VideoCompressorConfigFactory.create(
metadata = metadata,
preset = preset,
)
// Then
assertIsResized(videoCompressorConfig, metadata.width)
}
@Test
fun `if the video should be compressed and is smaller or equal to 720p it will not be resized`() {
// Given
val metadata = VideoFileMetadata(width = 1280, height = 720, bitrate = 1_000_000, frameRate = 50, rotation = 0)
val preset = VideoCompressionPreset.STANDARD
// When
val videoCompressorConfig = VideoCompressorConfigFactory.create(
metadata = metadata,
preset = preset,
)
// Then
assertIsNotResized(videoCompressorConfig, 1280)
}
@Test
fun `if the video should not be compressed and is larger than 1080p it will be resized`() {
// Given
val metadata = VideoFileMetadata(width = 2560, height = 1440, bitrate = 1_000_000, frameRate = 50, rotation = 0)
val preset = VideoCompressionPreset.HIGH
// When
val videoCompressorConfig = VideoCompressorConfigFactory.create(
metadata = metadata,
preset = preset,
)
// Then
assertIsResized(videoCompressorConfig, metadata.width)
}
@Test
fun `if the video should not be compressed and is smaller or equal than 1080p it will not be resized`() {
// Given
val metadata = VideoFileMetadata(width = 1920, height = 1080, bitrate = 1_000_000, frameRate = 50, rotation = 0)
val preset = VideoCompressionPreset.HIGH
// When
val videoCompressorConfig = VideoCompressorConfigFactory.create(
metadata = metadata,
preset = preset,
)
// Then
assertIsNotResized(videoCompressorConfig, 1920)
}
private inline fun assertIsResized(videoCompressorConfig: VideoCompressorConfig, referenceSize: Int) {
assertThat(videoCompressorConfig.videoCompressorHelper.maxSize).isNotEqualTo(referenceSize)
}
private inline fun assertIsNotResized(videoCompressorConfig: VideoCompressorConfig, referenceSize: Int) {
assertThat(videoCompressorConfig.videoCompressorHelper.maxSize).isEqualTo(referenceSize)
}
}
@@ -0,0 +1,21 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
plugins {
id("io.element.android-library")
}
android {
namespace = "io.element.android.libraries.mediaupload.test"
}
dependencies {
api(projects.libraries.mediaupload.api)
implementation(projects.libraries.core)
implementation(projects.tests.testutils)
}
@@ -0,0 +1,22 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.test
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfig
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfigProvider
import io.element.android.libraries.preferences.api.store.VideoCompressionPreset
class FakeMediaOptimizationConfigProvider(
val config: MediaOptimizationConfig = MediaOptimizationConfig(
compressImages = true,
videoCompressionPreset = VideoCompressionPreset.STANDARD,
)
) : MediaOptimizationConfigProvider {
override suspend fun get(): MediaOptimizationConfig = config
}
@@ -0,0 +1,120 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
* Copyright 2023-2025 New Vector Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.test
import android.net.Uri
import io.element.android.libraries.core.mimetype.MimeTypes
import io.element.android.libraries.matrix.api.media.AudioInfo
import io.element.android.libraries.matrix.api.media.FileInfo
import io.element.android.libraries.matrix.api.media.ImageInfo
import io.element.android.libraries.matrix.api.media.VideoInfo
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfig
import io.element.android.libraries.mediaupload.api.MediaPreProcessor
import io.element.android.libraries.mediaupload.api.MediaUploadInfo
import io.element.android.tests.testutils.simulateLongTask
import kotlinx.coroutines.CompletableDeferred
import java.io.File
import kotlin.time.Duration.Companion.seconds
class FakeMediaPreProcessor(
private val processLatch: CompletableDeferred<Unit>? = null,
) : MediaPreProcessor {
var processCallCount = 0
private set
var cleanUpCallCount = 0
private set
private var result: Result<MediaUploadInfo> = Result.success(
MediaUploadInfo.AnyFile(
File("test"),
FileInfo(
mimetype = MimeTypes.Any,
size = 999L,
thumbnailInfo = null,
thumbnailSource = null,
)
)
)
override suspend fun process(
uri: Uri,
mimeType: String,
deleteOriginal: Boolean,
mediaOptimizationConfig: MediaOptimizationConfig,
): Result<MediaUploadInfo> = simulateLongTask {
processLatch?.await()
processCallCount++
result
}
fun givenResult(value: Result<MediaUploadInfo>) {
this.result = value
}
fun givenAudioResult() {
givenResult(
Result.success(
MediaUploadInfo.Audio(
file = File("audio.ogg"),
audioInfo = AudioInfo(
duration = 1000.seconds,
size = 1000,
mimetype = MimeTypes.Ogg,
),
)
)
)
}
fun givenImageResult() {
givenResult(
Result.success(
MediaUploadInfo.Image(
file = File("image.jpg"),
imageInfo = ImageInfo(
height = 100,
width = 100,
mimetype = MimeTypes.Jpeg,
size = 1000,
thumbnailInfo = null,
thumbnailSource = null,
blurhash = null,
),
thumbnailFile = null,
)
)
)
}
fun givenVideoResult() {
givenResult(
Result.success(
MediaUploadInfo.Video(
file = File("image.jpg"),
videoInfo = VideoInfo(
duration = 1000.seconds,
height = 100,
width = 100,
mimetype = MimeTypes.Mp4,
size = 1000,
thumbnailInfo = null,
thumbnailSource = null,
blurhash = null,
),
thumbnailFile = null,
)
)
)
}
override fun cleanUp() {
cleanUpCallCount += 1
}
}
@@ -0,0 +1,64 @@
/*
* Copyright (c) 2025 Element Creations Ltd.
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial.
* Please see LICENSE files in the repository root for full details.
*/
package io.element.android.libraries.mediaupload.test
import android.net.Uri
import io.element.android.libraries.matrix.api.core.EventId
import io.element.android.libraries.mediaupload.api.MediaOptimizationConfig
import io.element.android.libraries.mediaupload.api.MediaSender
import io.element.android.libraries.mediaupload.api.MediaUploadInfo
import io.element.android.tests.testutils.lambda.lambdaError
class FakeMediaSender(
private val preProcessMediaResult: () -> Result<MediaUploadInfo> = { lambdaError() },
private val sendPreProcessedMediaResult: () -> Result<Unit> = { lambdaError() },
private val sendMediaResult: () -> Result<Unit> = { lambdaError() },
private val sendVoiceMessageResult: () -> Result<Unit> = { lambdaError() },
private val cleanUpResult: () -> Unit = { lambdaError() },
) : MediaSender {
override suspend fun preProcessMedia(
uri: Uri,
mimeType: String,
mediaOptimizationConfig: MediaOptimizationConfig,
): Result<MediaUploadInfo> {
return preProcessMediaResult()
}
override suspend fun sendPreProcessedMedia(
mediaUploadInfo: MediaUploadInfo,
caption: String?,
formattedCaption: String?,
inReplyToEventId: EventId?,
): Result<Unit> {
return sendPreProcessedMediaResult()
}
override suspend fun sendMedia(
uri: Uri,
mimeType: String,
caption: String?,
formattedCaption: String?,
inReplyToEventId: EventId?,
mediaOptimizationConfig: MediaOptimizationConfig,
): Result<Unit> {
return sendMediaResult()
}
override suspend fun sendVoiceMessage(
uri: Uri,
mimeType: String,
waveForm: List<Float>,
inReplyToEventId: EventId?,
): Result<Unit> {
return sendVoiceMessageResult()
}
override fun cleanUp() {
cleanUpResult()
}
}