using Bugsnag; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Extensions; using ErsatzTV.Core.FFmpeg; using ErsatzTV.Core.Interfaces.FFmpeg; using ErsatzTV.Core.Interfaces.Images; using ErsatzTV.Core.Interfaces.Metadata; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Interfaces.Search; using MediatR; using Microsoft.Extensions.Logging; namespace ErsatzTV.Core.Metadata; public class SongFolderScanner : LocalFolderScanner, ISongFolderScanner { private readonly ILocalFileSystem _localFileSystem; private readonly ILocalMetadataProvider _localMetadataProvider; private readonly IMediator _mediator; private readonly ISearchIndex _searchIndex; private readonly ISearchRepository _searchRepository; private readonly ISongRepository _songRepository; private readonly ILibraryRepository _libraryRepository; private readonly IClient _client; private readonly ILogger _logger; public SongFolderScanner( ILocalFileSystem localFileSystem, ILocalStatisticsProvider localStatisticsProvider, ILocalMetadataProvider localMetadataProvider, IMetadataRepository metadataRepository, IImageCache imageCache, IMediator mediator, ISearchIndex searchIndex, ISearchRepository searchRepository, ISongRepository songRepository, ILibraryRepository libraryRepository, IMediaItemRepository mediaItemRepository, IFFmpegProcessServiceFactory ffmpegProcessServiceFactory, ITempFilePool tempFilePool, IClient client, ILogger logger) : base( localFileSystem, localStatisticsProvider, metadataRepository, mediaItemRepository, imageCache, ffmpegProcessServiceFactory, tempFilePool, client, logger) { _localFileSystem = localFileSystem; _localMetadataProvider = localMetadataProvider; _mediator = mediator; _searchIndex = searchIndex; _searchRepository = searchRepository; _songRepository = songRepository; _libraryRepository = libraryRepository; _client = client; _logger = logger; } public async Task> ScanFolder( LibraryPath libraryPath, string ffprobePath, string ffmpegPath, decimal progressMin, decimal progressMax) { decimal progressSpread = progressMax - progressMin; var foldersCompleted = 0; var folderQueue = new Queue(); if (ShouldIncludeFolder(libraryPath.Path)) { folderQueue.Enqueue(libraryPath.Path); } foreach (string folder in _localFileSystem.ListSubdirectories(libraryPath.Path) .Filter(ShouldIncludeFolder) .OrderBy(identity)) { folderQueue.Enqueue(folder); } while (folderQueue.Count > 0) { decimal percentCompletion = (decimal)foldersCompleted / (foldersCompleted + folderQueue.Count); await _mediator.Publish( new LibraryScanProgress(libraryPath.LibraryId, progressMin + percentCompletion * progressSpread)); string songFolder = folderQueue.Dequeue(); foldersCompleted++; var filesForEtag = _localFileSystem.ListFiles(songFolder).ToList(); var allFiles = filesForEtag .Filter(f => AudioFileExtensions.Contains(Path.GetExtension(f))) .Filter(f => !Path.GetFileName(f).StartsWith("._")) .ToList(); foreach (string subdirectory in _localFileSystem.ListSubdirectories(songFolder) .Filter(ShouldIncludeFolder) .OrderBy(identity)) { folderQueue.Enqueue(subdirectory); } string etag = FolderEtag.Calculate(songFolder, _localFileSystem); Option knownFolder = libraryPath.LibraryFolders .Filter(f => f.Path == songFolder) .HeadOrNone(); // skip folder if etag matches if (!allFiles.Any() || await knownFolder.Map(f => f.Etag ?? string.Empty).IfNoneAsync(string.Empty) == etag) { continue; } _logger.LogDebug( "UPDATE: Etag has changed for folder {Folder}", songFolder); foreach (string file in allFiles.OrderBy(identity)) { Either> maybeSong = await _songRepository .GetOrAdd(libraryPath, file) .BindT(video => UpdateStatistics(video, ffmpegPath, ffprobePath)) .BindT(video => UpdateMetadata(video, ffprobePath)) .BindT(video => UpdateThumbnail(video, ffmpegPath)) .BindT(FlagNormal); await maybeSong.Match( async result => { if (result.IsAdded) { await _searchIndex.AddItems(_searchRepository, new List { result.Item }); } else if (result.IsUpdated) { await _searchIndex.UpdateItems(_searchRepository, new List { result.Item }); } await _libraryRepository.SetEtag(libraryPath, knownFolder, songFolder, etag); }, error => { _logger.LogWarning("Error processing song at {Path}: {Error}", file, error.Value); return Task.CompletedTask; }); } } foreach (string path in await _songRepository.FindSongPaths(libraryPath)) { if (!_localFileSystem.FileExists(path)) { _logger.LogInformation("Flagging missing song at {Path}", path); List songIds = await FlagFileNotFound(libraryPath, path); await _searchIndex.RebuildItems(_searchRepository, songIds); } else if (Path.GetFileName(path).StartsWith("._")) { _logger.LogInformation("Removing dot underscore file at {Path}", path); List songIds = await _songRepository.DeleteByPath(libraryPath, path); await _searchIndex.RemoveItems(songIds); } } await _libraryRepository.CleanEtagsForLibraryPath(libraryPath); _searchIndex.Commit(); return Unit.Default; } private async Task>> UpdateMetadata( MediaItemScanResult result, string ffprobePath) { try { Song song = result.Item; string path = song.GetHeadVersion().MediaFiles.Head().Path; bool shouldUpdate = Optional(song.SongMetadata).Flatten().HeadOrNone().Match( m => m.MetadataKind == MetadataKind.Fallback || m.DateUpdated != _localFileSystem.GetLastWriteTime(path), true); if (shouldUpdate) { song.SongMetadata ??= new List(); _logger.LogDebug("Refreshing {Attribute} for {Path}", "Metadata", path); if (await _localMetadataProvider.RefreshTagMetadata(song, ffprobePath)) { result.IsUpdated = true; } } return result; } catch (Exception ex) { _client.Notify(ex); return BaseError.New(ex.ToString()); } } private async Task>> UpdateThumbnail( MediaItemScanResult result, string ffmpegPath) { try { // reload the song from the database at this point if (result.IsAdded) { LibraryPath libraryPath = result.Item.LibraryPath; string path = result.Item.GetHeadVersion().MediaFiles.Head().Path; foreach (MediaItemScanResult s in (await _songRepository.GetOrAdd(libraryPath, path)) .RightToSeq()) { result.Item = s.Item; } } Song song = result.Item; await LocateThumbnail(song).Match( async thumbnailFile => { SongMetadata metadata = song.SongMetadata.Head(); await RefreshArtwork(thumbnailFile, metadata, ArtworkKind.Thumbnail, ffmpegPath, None); }, () => ExtractEmbeddedArtwork(song, ffmpegPath)); return result; } catch (Exception ex) { _client.Notify(ex); return BaseError.New(ex.ToString()); } } private Option LocateThumbnail(Song song) { string path = song.MediaVersions.Head().MediaFiles.Head().Path; Option parent = Optional(Directory.GetParent(path)); return parent.Map( di => { string coverPath = Path.Combine(di.FullName, "cover.jpg"); return ImageFileExtensions .Map(ext => Path.ChangeExtension(coverPath, ext)) .Filter(f => _localFileSystem.FileExists(f)) .HeadOrNone(); }).Flatten(); } private async Task ExtractEmbeddedArtwork(Song song, string ffmpegPath) { Option maybeArtworkStream = Optional(song.GetHeadVersion().Streams.Find(ms => ms.AttachedPic)); foreach (MediaStream artworkStream in maybeArtworkStream) { await RefreshArtwork( song.GetHeadVersion().MediaFiles.Head().Path, song.SongMetadata.Head(), ArtworkKind.Thumbnail, ffmpegPath, artworkStream.Index); } } }