use embedded song cover art (#514)

CHANGELOG.md

@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 - Add song genres to search index
+- Use embedded song cover art when sidecar cover art is unavailable
 
 ### Changed
 - Randomly place song cover art on left or right side of screen

ErsatzTV.Core/FFmpeg/FFmpegProcessBuilder.cs

@@ -190,7 +190,14 @@ namespace ErsatzTV.Core.FFmpeg
         public FFmpegProcessBuilder WithInput(string input)
         {
             _arguments.Add("-i");
-            _arguments.Add($"{input}");
+            _arguments.Add(input);
+            return this;
+        }
+
+        public FFmpegProcessBuilder WithMap(string map)
+        {
+            _arguments.Add("-map");
+            _arguments.Add(map);
             return this;
         }
 

ErsatzTV.Core/FFmpeg/FFmpegProcessService.cs

@@ -257,6 +257,17 @@ namespace ErsatzTV.Core.FFmpeg
                 .Build();
         }
 
+        public Process ExtractAttachedPicAsPng(string ffmpegPath, string inputFile, int streamIndex, string outputFile)
+        {
+            return new FFmpegProcessBuilder(ffmpegPath, false, _logger)
+                .WithThreads(1)
+                .WithQuiet()
+                .WithInput(inputFile)
+                .WithMap($"0:{streamIndex}")
+                .WithOutputFormat("apng", outputFile)
+                .Build();
+        }
+
         public async Task<Either<BaseError, string>> GenerateSongImage(
             string ffmpegPath,
             Option<string> subtitleFile,

ErsatzTV.Core/Interfaces/FFmpeg/IFFmpegProcessService.cs

@@ -40,6 +40,8 @@ namespace ErsatzTV.Core.Interfaces.FFmpeg
 
         Process ConvertToPng(string ffmpegPath, string inputFile, string outputFile);
 
+        Process ExtractAttachedPicAsPng(string ffmpegPath, string inputFile, int streamIndex, string outputFile);
+
         Task<Either<BaseError, string>> GenerateSongImage(
             string ffmpegPath,
             Option<string> subtitleFile,

ErsatzTV.Core/Metadata/LocalFolderScanner.cs

@@ -116,7 +116,12 @@ namespace ErsatzTV.Core.Metadata
             }
         }
 
-        protected async Task<bool> RefreshArtwork(string artworkFile, Domain.Metadata metadata, ArtworkKind artworkKind, Option<string> ffmpegPath)
+        protected async Task<bool> RefreshArtwork(
+            string artworkFile,
+            Domain.Metadata metadata,
+            ArtworkKind artworkKind,
+            Option<string> ffmpegPath,
+            Option<int> attachedPicIndex)
         {
             DateTime lastWriteTime = _localFileSystem.GetLastWriteTime(artworkFile);
 
@@ -134,15 +139,34 @@ namespace ErsatzTV.Core.Metadata
                 {
                     _logger.LogDebug("Refreshing {Attribute} from {Path}", artworkKind, artworkFile);
 
-                    // if ffmpeg path is passed, we want to convert to png
+                    // if ffmpeg path is passed, we need pre-processing
                     foreach (string path in ffmpegPath)
                     {
-                        string tempName = _tempFilePool.GetNextTempFile(TempFileCategory.CoverArt);
+                        artworkFile = await attachedPicIndex.Match(
-                        using Process process = _ffmpegProcessService.ConvertToPng(path, artworkFile, tempName);
+                            async picIndex =>
-                        process.Start();
+                            {
-                        await process.WaitForExitAsync();
+                                // extract attached pic (and convert to png)
-
+                                string tempName = _tempFilePool.GetNextTempFile(TempFileCategory.CoverArt);
-                        artworkFile = tempName;
+                                using Process process = _ffmpegProcessService.ExtractAttachedPicAsPng(
+                                    path,
+                                    artworkFile,
+                                    picIndex,
+                                    tempName);
+                                process.Start();
+                                await process.WaitForExitAsync();
+
+                                return tempName;
+                            },
+                            async () =>
+                            {
+                                // no attached pic index means convert to png
+                                string tempName = _tempFilePool.GetNextTempFile(TempFileCategory.CoverArt);
+                                using Process process = _ffmpegProcessService.ConvertToPng(path, artworkFile, tempName);
+                                process.Start();
+                                await process.WaitForExitAsync();
+
+                                return tempName;
+                            });
                     }
 
                     Either<BaseError, string> maybeCacheName =

ErsatzTV.Core/Metadata/MediaItemScanResult.cs

@@ -6,7 +6,7 @@ namespace ErsatzTV.Core.Metadata
     {
         public MediaItemScanResult(T item) => Item = item;
 
-        public T Item { get; }
+        public T Item { get; set; }
 
         public bool IsAdded { get; set; }
         public bool IsUpdated { get; set; }

ErsatzTV.Core/Metadata/MovieFolderScanner.cs

@@ -239,7 +239,7 @@ namespace ErsatzTV.Core.Metadata
                     async posterFile =>
                     {
                         MovieMetadata metadata = movie.MovieMetadata.Head();
-                        await RefreshArtwork(posterFile, metadata, artworkKind, None);
+                        await RefreshArtwork(posterFile, metadata, artworkKind, None, None);
                     });
 
                 return result;

ErsatzTV.Core/Metadata/MusicVideoFolderScanner.cs

@@ -222,7 +222,7 @@ namespace ErsatzTV.Core.Metadata
                     async artworkFile =>
                     {
                         ArtistMetadata metadata = artist.ArtistMetadata.Head();
-                        await RefreshArtwork(artworkFile, metadata, artworkKind, None);
+                        await RefreshArtwork(artworkFile, metadata, artworkKind, None, None);
                     });
 
                 return result;
@@ -385,7 +385,7 @@ namespace ErsatzTV.Core.Metadata
                     async thumbnailFile =>
                     {
                         MusicVideoMetadata metadata = musicVideo.MusicVideoMetadata.Head();
-                        await RefreshArtwork(thumbnailFile, metadata, ArtworkKind.Thumbnail, None);
+                        await RefreshArtwork(thumbnailFile, metadata, ArtworkKind.Thumbnail, None, None);
                     });
 
                 return result;

ErsatzTV.Core/Metadata/SongFolderScanner.cs

@@ -130,7 +130,7 @@ namespace ErsatzTV.Core.Metadata
                         .GetOrAdd(libraryPath, file)
                         .BindT(video => UpdateStatistics(video, ffprobePath))
                         .BindT(video => UpdateMetadata(video, ffprobePath))
-                        .BindT(video => UpdateThumbnail(video, ffprobePath, ffmpegPath));
+                        .BindT(video => UpdateThumbnail(video, ffmpegPath));
 
                     await maybeSong.Match(
                         async result =>
@@ -208,19 +208,31 @@ namespace ErsatzTV.Core.Metadata
 
         private async Task<Either<BaseError, MediaItemScanResult<Song>>> UpdateThumbnail(
             MediaItemScanResult<Song> result,
-            string ffprobePath,
             string ffmpegPath)
         {
             try
             {
+                // reload the song from the database at this point
+                if (result.IsAdded)
+                {
+                    LibraryPath libraryPath = result.Item.LibraryPath;
+                    string path = result.Item.GetHeadVersion().MediaFiles.Head().Path;
+                    foreach (MediaItemScanResult<Song> s in (await _songRepository.GetOrAdd(libraryPath, path))
+                        .RightToSeq())
+                    {
+                        result.Item = s.Item;
+                    }
+                }
+
                 Song song = result.Item;
+
                 await LocateThumbnail(song).Match(
                     async thumbnailFile =>
                     {
                         SongMetadata metadata = song.SongMetadata.Head();
-                        await RefreshArtwork(thumbnailFile, metadata, ArtworkKind.Thumbnail, ffmpegPath);
+                        await RefreshArtwork(thumbnailFile, metadata, ArtworkKind.Thumbnail, ffmpegPath, None);
                     },
-                    () => Task.CompletedTask); // TODO: check for embedded artwork
+                    () => ExtractEmbeddedArtwork(song, ffmpegPath));
         
                 return result;
             }
@@ -245,5 +257,19 @@ namespace ErsatzTV.Core.Metadata
                         .HeadOrNone();
                 }).Flatten();
         }
+
+        private async Task ExtractEmbeddedArtwork(Song song, string ffmpegPath)
+        {
+            Option<MediaStream> maybeArtworkStream = Optional(song.GetHeadVersion().Streams.Find(ms => ms.AttachedPic));
+            foreach (MediaStream artworkStream in maybeArtworkStream)
+            {
+                await RefreshArtwork(
+                    song.GetHeadVersion().MediaFiles.Head().Path,
+                    song.SongMetadata.Head(),
+                    ArtworkKind.Thumbnail,
+                    ffmpegPath,
+                    artworkStream.Index);
+            }
+        }
     }
 }

ErsatzTV.Core/Metadata/TelevisionFolderScanner.cs

@@ -367,7 +367,7 @@ namespace ErsatzTV.Core.Metadata
                     async artworkFile =>
                     {
                         ShowMetadata metadata = show.ShowMetadata.Head();
-                        await RefreshArtwork(artworkFile, metadata, artworkKind, None);
+                        await RefreshArtwork(artworkFile, metadata, artworkKind, None, None);
                     });
 
                 return result;
@@ -386,7 +386,7 @@ namespace ErsatzTV.Core.Metadata
                     async posterFile =>
                     {
                         SeasonMetadata metadata = season.SeasonMetadata.Head();
-                        await RefreshArtwork(posterFile, metadata, ArtworkKind.Poster, None);
+                        await RefreshArtwork(posterFile, metadata, ArtworkKind.Poster, None, None);
                     });
 
                 return season;
@@ -406,7 +406,7 @@ namespace ErsatzTV.Core.Metadata
                     {
                         foreach (EpisodeMetadata metadata in episode.EpisodeMetadata)
                         {
-                            await RefreshArtwork(posterFile, metadata, ArtworkKind.Thumbnail, None);
+                            await RefreshArtwork(posterFile, metadata, ArtworkKind.Thumbnail, None, None);
                         }
                     });
 

ErsatzTV.Infrastructure/Data/Repositories/MetadataRepository.cs

@@ -114,7 +114,7 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
             MediaVersion incoming,
             bool updateVersion = true)
         {
-            await using TvContext dbContext = _dbContextFactory.CreateDbContext();
+            await using TvContext dbContext = await _dbContextFactory.CreateDbContextAsync();
             Option<MediaVersion> maybeVersion = await dbContext.MediaVersions
                 .Include(v => v.Streams)
                 .Include(v => v.Chapters)
@@ -161,6 +161,9 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
                         existingStream.Title = incomingStream.Title;
                         existingStream.Default = incomingStream.Default;
                         existingStream.Forced = incomingStream.Forced;
+                        existingStream.AttachedPic = incomingStream.AttachedPic;
+                        existingStream.PixelFormat = incomingStream.PixelFormat;
+                        existingStream.BitsPerRawSample = incomingStream.BitsPerRawSample;
                     }
                     
                     var chaptersToAdd = incoming.Chapters