diff --git a/CHANGELOG.md b/CHANGELOG.md index 92eb637fe..0c39fd782 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Fixed +- Fix cause of unnecessarily large database when using Emby or Jellyfin libraries + - This was caused by orphaned actor and actor artwork records, which will be cleaned up hourly + - When you see logs like `No orphaned actors to delete` and `No orphaned artwork to delete`, you can then reclaim disk space by: + - Stopping ErsatzTV Legacy + - Running `sqlite3 ersatztv.sqlite3 'PRAGMA wal_checkpoint(TRUNCATE); VACUUM;'` in your config folder +- Fix Next Engine playout sync when channel is configured to use channel logo as watermark ## [26.5.0] - 2026-05-08 ### Added diff --git a/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtwork.cs b/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtwork.cs index c6d625e0c..d99b17070 100644 --- a/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtwork.cs +++ b/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtwork.cs @@ -2,4 +2,4 @@ namespace ErsatzTV.Application.Maintenance; -public record DeleteOrphanedArtwork : IRequest>, IBackgroundServiceRequest; +public record DeleteOrphanedArtwork(int? MaxToDelete) : IRequest>, IBackgroundServiceRequest; diff --git a/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs b/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs index 3bffbab70..02cc4f6a7 100644 --- a/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs +++ b/ErsatzTV.Application/Maintenance/Commands/DeleteOrphanedArtworkHandler.cs @@ -23,8 +23,10 @@ public class DeleteOrphanedArtworkHandler( { try { - await CleanUpDatabase(); - await CleanUpFileSystem(cancellationToken); + await CleanUpDatabase(request, cancellationToken); + + // temporarily disabled since this is now scheduled + //await CleanUpFileSystem(cancellationToken); return Unit.Default; } @@ -34,13 +36,31 @@ public class DeleteOrphanedArtworkHandler( } } - private async Task CleanUpDatabase() + private async Task CleanUpDatabase(DeleteOrphanedArtwork request, CancellationToken cancellationToken) { - List ids = await artworkRepository.GetOrphanedArtworkIds(); - if (ids.Count > 0) + // delete actors that no longer reference any metadata; otherwise the artwork + // they point to is shielded from the orphaned artwork cleanup below + int deletedActors = await artworkRepository.DeleteOrphanedActors(request.MaxToDelete, cancellationToken); + if (deletedActors > 0) + { + logger.LogInformation("Deleted {Count} orphaned actors", deletedActors); + } + else { - await artworkRepository.Delete(ids); + logger.LogInformation("No orphaned actors to delete"); } + + int deletedArtwork = await artworkRepository.DeleteOrphanedArtwork(request.MaxToDelete, cancellationToken); + if (deletedArtwork > 0) + { + logger.LogInformation("Deleted {Count} orphaned artwork", deletedArtwork); + } + else + { + logger.LogInformation("No orphaned artwork to delete"); + } + + logger.LogInformation("Done cleaning!"); } private async Task CleanUpFileSystem(CancellationToken cancellationToken) diff --git a/ErsatzTV.Application/Playouts/Commands/SyncNextPlayoutHandler.cs b/ErsatzTV.Application/Playouts/Commands/SyncNextPlayoutHandler.cs index d32d379ed..a78a706b7 100644 --- a/ErsatzTV.Application/Playouts/Commands/SyncNextPlayoutHandler.cs +++ b/ErsatzTV.Application/Playouts/Commands/SyncNextPlayoutHandler.cs @@ -270,6 +270,7 @@ public partial class SyncNextPlayoutHandler( maybeChannel = await dbContext.Channels .AsNoTracking() .Include(c => c.Watermark) + .Include(c => c.Artwork) .SingleOrDefaultAsync(c => c.Number == channelNumber, cancellationToken); foreach (Channel channel in maybeChannel) { diff --git a/ErsatzTV.Core/Interfaces/Repositories/IArtworkRepository.cs b/ErsatzTV.Core/Interfaces/Repositories/IArtworkRepository.cs index c696ca774..664e58a4d 100644 --- a/ErsatzTV.Core/Interfaces/Repositories/IArtworkRepository.cs +++ b/ErsatzTV.Core/Interfaces/Repositories/IArtworkRepository.cs @@ -2,6 +2,6 @@ public interface IArtworkRepository { - Task> GetOrphanedArtworkIds(); - Task Delete(List artworkIds); + Task DeleteOrphanedActors(int? max, CancellationToken cancellationToken); + Task DeleteOrphanedArtwork(int? max, CancellationToken cancellationToken); } diff --git a/ErsatzTV.Infrastructure/Data/Repositories/ArtworkRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/ArtworkRepository.cs index fe35174aa..69982573c 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/ArtworkRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/ArtworkRepository.cs @@ -1,34 +1,98 @@ using Dapper; using ErsatzTV.Core.Interfaces.Repositories; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; namespace ErsatzTV.Infrastructure.Data.Repositories; -public class ArtworkRepository(IDbContextFactory dbContextFactory) : IArtworkRepository +public class ArtworkRepository(IDbContextFactory dbContextFactory, ILogger logger) : IArtworkRepository { - public async Task> GetOrphanedArtworkIds() + public async Task DeleteOrphanedActors(int? max, CancellationToken cancellationToken) { - await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(); - return await dbContext.Artwork - .TagWithCallSite() - .Where(a => a.IsMetadataOrphan == true) - .Where(a => !dbContext.Actors.Any(actor => actor.ArtworkId == a.Id)) - .Select(a => a.Id) - .ToListAsync(); + await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken); + + const string ORPHANED_ACTOR_QUERY = + """ + SELECT Id FROM Actor + WHERE ArtistMetadataId IS NULL + AND EpisodeMetadataId IS NULL + AND ImageMetadataId IS NULL + AND MovieMetadataId IS NULL + AND MusicVideoMetadataId IS NULL + AND OtherVideoMetadataId IS NULL + AND RemoteStreamMetadataId IS NULL + AND SeasonMetadataId IS NULL + AND ShowMetadataId IS NULL + AND SongMetadataId IS NULL + LIMIT 5000 + """; + + var totalDeleted = 0; + while (true) + { + List ids = (await dbContext.Connection.QueryAsync(ORPHANED_ACTOR_QUERY)).ToList(); + if (ids.Count == 0 || totalDeleted >= max) + { + break; + } + + if (totalDeleted > 0) + { + logger.LogDebug("Deleted {Count} orphaned actors; still have more to delete...", totalDeleted); + } + + foreach (List chunk in Chunk(ids, 100)) + { + await dbContext.Connection.ExecuteAsync( + new CommandDefinition("DELETE FROM Actor WHERE Id IN @Ids", + parameters: new { Ids = chunk }, + cancellationToken: cancellationToken)); + } + + totalDeleted += ids.Count; + } + + return totalDeleted; } - public async Task Delete(List artworkIds) + public async Task DeleteOrphanedArtwork(int? max, CancellationToken cancellationToken) { - await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(); - IEnumerable> chunks = Chunk(artworkIds, 100); - foreach (List chunk in chunks) + await using TvContext dbContext = await dbContextFactory.CreateDbContextAsync(cancellationToken); + + var totalDeleted = 0; + while (true) { - await dbContext.Connection.ExecuteAsync( - "DELETE FROM Artwork WHERE Id IN @Ids", - new { Ids = chunk }); + IQueryable query = dbContext.Artwork + .TagWithCallSite() + .Where(a => a.IsMetadataOrphan == true) + .Where(a => !dbContext.Actors.Any(actor => actor.ArtworkId == a.Id)) + .OrderBy(a => a.Id) + .Take(5000) + .Select(a => a.Id); + + List ids = await query.ToListAsync(cancellationToken); + if (ids.Count == 0 || totalDeleted >= max) + { + break; + } + + if (totalDeleted > 0) + { + logger.LogDebug("Deleted {Count} orphaned artwork; still have more to delete...", totalDeleted); + } + + foreach (List chunk in Chunk(ids, 100)) + { + await dbContext.Connection.ExecuteAsync( + new CommandDefinition("DELETE FROM Artwork WHERE Id IN @Ids", + parameters: new { Ids = chunk }, + cancellationToken: cancellationToken)); + } + + totalDeleted += ids.Count; } - return Unit.Default; + return totalDeleted; } private static IEnumerable> Chunk(IEnumerable collection, int size) diff --git a/ErsatzTV.Infrastructure/Data/Repositories/EmbyMovieRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/EmbyMovieRepository.cs index 7195fe4ad..bd3daf3ba 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/EmbyMovieRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/EmbyMovieRepository.cs @@ -316,6 +316,7 @@ public class EmbyMovieRepository : IEmbyMovieRepository .ToList()) { metadata.Actors.Remove(actor); + dbContext.Actors.Remove(actor); } foreach (Actor actor in incomingMetadata.Actors diff --git a/ErsatzTV.Infrastructure/Data/Repositories/EmbyTelevisionRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/EmbyTelevisionRepository.cs index ac8408a06..5dad6cb8d 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/EmbyTelevisionRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/EmbyTelevisionRepository.cs @@ -590,6 +590,7 @@ public class EmbyTelevisionRepository( .ToList()) { metadata.Actors.Remove(actor); + dbContext.Actors.Remove(actor); } foreach (Actor actor in incomingMetadata.Actors diff --git a/ErsatzTV.Infrastructure/Data/Repositories/JellyfinMovieRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/JellyfinMovieRepository.cs index 6150b048e..c5ccee000 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/JellyfinMovieRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/JellyfinMovieRepository.cs @@ -278,6 +278,7 @@ public class JellyfinMovieRepository : IJellyfinMovieRepository .ToList()) { metadata.Actors.Remove(actor); + dbContext.Actors.Remove(actor); } foreach (Actor actor in incomingMetadata.Actors diff --git a/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs b/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs index 52d415689..5813fd6b1 100644 --- a/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs +++ b/ErsatzTV.Infrastructure/Data/Repositories/JellyfinTelevisionRepository.cs @@ -626,6 +626,7 @@ public class JellyfinTelevisionRepository : IJellyfinTelevisionRepository .ToList()) { metadata.Actors.Remove(actor); + dbContext.Actors.Remove(actor); } foreach (Actor actor in incomingMetadata.Actors diff --git a/ErsatzTV/Controllers/Api/MaintenanceController.cs b/ErsatzTV/Controllers/Api/MaintenanceController.cs index b6009e32a..ab670c9a1 100644 --- a/ErsatzTV/Controllers/Api/MaintenanceController.cs +++ b/ErsatzTV/Controllers/Api/MaintenanceController.cs @@ -42,9 +42,9 @@ public class MaintenanceController(IMediator mediator, ChannelWriter CleanArtwork(CancellationToken cancellationToken) + public async Task CleanArtwork(CancellationToken cancellationToken, [FromQuery] int limit = 100_000) { - await workerChannel.WriteAsync(new DeleteOrphanedArtwork(), cancellationToken); + await workerChannel.WriteAsync(new DeleteOrphanedArtwork(limit), cancellationToken); return new OkResult(); } } diff --git a/ErsatzTV/Services/SchedulerService.cs b/ErsatzTV/Services/SchedulerService.cs index 42e792c36..7e262e5ad 100644 --- a/ErsatzTV/Services/SchedulerService.cs +++ b/ErsatzTV/Services/SchedulerService.cs @@ -119,6 +119,7 @@ public class SchedulerService : BackgroundService try { await DeleteOrphanedSubtitles(cancellationToken); + await DeleteOrphanedArtwork(cancellationToken); await RefreshMpegTsScripts(cancellationToken); await RefreshChannelGuideChannelList(cancellationToken); await BuildPlayouts(cancellationToken); @@ -365,6 +366,9 @@ public class SchedulerService : BackgroundService private ValueTask DeleteOrphanedSubtitles(CancellationToken cancellationToken) => _workerChannel.WriteAsync(new DeleteOrphanedSubtitles(), cancellationToken); + private ValueTask DeleteOrphanedArtwork(CancellationToken cancellationToken) => + _workerChannel.WriteAsync(new DeleteOrphanedArtwork(100_000), cancellationToken); + private ValueTask ReleaseMemory(CancellationToken cancellationToken) => _workerChannel.WriteAsync(new ReleaseMemory(false), cancellationToken);