add `language_tag` and `seconds` fields to search index (#1479)

* add `language_tag` and `seconds` fields to search index

* simplify

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Log playout item title and path when starting a stream
   - This will help with media server libraries where the URL passed to ffmpeg doesn't indicate which file is streaming
 - Add QSV Capabilities to Troubleshooting page
+- Add `language_tag` and `seconds` fields to search index
 
 ### Fixed
 - Fix playout bug that caused some schedule items with fixed start times to be pushed to the next day

ErsatzTV.Infrastructure/Search/CustomMultiFieldQueryParser.cs

@@ -14,6 +14,7 @@ public class CustomMultiFieldQueryParser : MultiFieldQueryParser
     private static readonly List<string> NumericFields = new()
     {
         LuceneSearchIndex.MinutesField,
+        LuceneSearchIndex.SecondsField,
         LuceneSearchIndex.HeightField,
         LuceneSearchIndex.WidthField,
         LuceneSearchIndex.SeasonNumberField,

ErsatzTV.Infrastructure/Search/ElasticSearchIndex.cs

@@ -46,7 +46,7 @@ public class ElasticSearchIndex : ISearchIndex
         return exists.IsValidResponse;
     }
 
-    public int Version => 36;
+    public int Version => 37;
 
     public async Task<bool> Initialize(
         ILocalFileSystem localFileSystem,
@@ -292,6 +292,7 @@ public class ElasticSearchIndex : ISearchIndex
                     State = movie.State.ToString(),
                     MetadataKind = metadata.MetadataKind.ToString(),
                     Language = await GetLanguages(searchRepository, movie.MediaVersions),
+                    LanguageTag = GetLanguageTags(movie.MediaVersions),
                     ContentRating = GetContentRatings(metadata.ContentRating),
                     ReleaseDate = GetReleaseDate(metadata.ReleaseDate),
                     AddedDate = GetAddedDate(metadata.DateAdded),
@@ -342,6 +343,7 @@ public class ElasticSearchIndex : ISearchIndex
                     State = show.State.ToString(),
                     MetadataKind = metadata.MetadataKind.ToString(),
                     Language = await GetLanguages(searchRepository, await searchRepository.GetLanguagesForShow(show)),
+                    LanguageTag = await searchRepository.GetLanguagesForShow(show),
                     ContentRating = GetContentRatings(metadata.ContentRating),
                     ReleaseDate = GetReleaseDate(metadata.ReleaseDate),
                     AddedDate = GetAddedDate(metadata.DateAdded),
@@ -400,6 +402,7 @@ public class ElasticSearchIndex : ISearchIndex
                     Language = await GetLanguages(
                         searchRepository,
                         await searchRepository.GetLanguagesForSeason(season)),
+                    LanguageTag = await searchRepository.GetLanguagesForSeason(season),
                     ContentRating = GetContentRatings(showMetadata.ContentRating),
                     ReleaseDate = GetReleaseDate(metadata.ReleaseDate),
                     AddedDate = GetAddedDate(metadata.DateAdded),
@@ -444,6 +447,7 @@ public class ElasticSearchIndex : ISearchIndex
                     Language = await GetLanguages(
                         searchRepository,
                         await searchRepository.GetLanguagesForArtist(artist)),
+                    LanguageTag = await searchRepository.GetLanguagesForArtist(artist),
                     AddedDate = GetAddedDate(metadata.DateAdded),
                     Genre = metadata.Genres.Map(g => g.Name).ToList(),
                     Style = metadata.Styles.Map(t => t.Name).ToList(),
@@ -484,6 +488,7 @@ public class ElasticSearchIndex : ISearchIndex
                     State = musicVideo.State.ToString(),
                     MetadataKind = metadata.MetadataKind.ToString(),
                     Language = await GetLanguages(searchRepository, musicVideo.MediaVersions),
+                    LanguageTag = GetLanguageTags(musicVideo.MediaVersions),
                     ReleaseDate = GetReleaseDate(metadata.ReleaseDate),
                     AddedDate = GetAddedDate(metadata.DateAdded),
                     Album = metadata.Album ?? string.Empty,
@@ -562,6 +567,7 @@ public class ElasticSearchIndex : ISearchIndex
                     SeasonNumber = episode.Season?.SeasonNumber ?? 0,
                     EpisodeNumber = metadata.EpisodeNumber,
                     Language = await GetLanguages(searchRepository, episode.MediaVersions),
+                    LanguageTag = GetLanguageTags(episode.MediaVersions),
                     ReleaseDate = GetReleaseDate(metadata.ReleaseDate),
                     AddedDate = GetAddedDate(metadata.DateAdded),
                     Plot = metadata.Plot ?? string.Empty,
@@ -619,6 +625,7 @@ public class ElasticSearchIndex : ISearchIndex
                     State = otherVideo.State.ToString(),
                     MetadataKind = metadata.MetadataKind.ToString(),
                     Language = await GetLanguages(searchRepository, otherVideo.MediaVersions),
+                    LanguageTag = GetLanguageTags(otherVideo.MediaVersions),
                     ContentRating = GetContentRatings(metadata.ContentRating),
                     ReleaseDate = GetReleaseDate(metadata.ReleaseDate),
                     AddedDate = GetAddedDate(metadata.DateAdded),
@@ -667,6 +674,7 @@ public class ElasticSearchIndex : ISearchIndex
                     State = song.State.ToString(),
                     MetadataKind = metadata.MetadataKind.ToString(),
                     Language = await GetLanguages(searchRepository, song.MediaVersions),
+                    LanguageTag = GetLanguageTags(song.MediaVersions),
                     AddedDate = GetAddedDate(metadata.DateAdded),
                     Album = metadata.Album ?? string.Empty,
                     Artist = !string.IsNullOrWhiteSpace(metadata.Artist) ? new List<string> { metadata.Artist } : null,
@@ -733,7 +741,7 @@ public class ElasticSearchIndex : ISearchIndex
 
         return result;
     }
-
+    
     private async Task<List<string>> GetLanguages(ISearchRepository searchRepository, List<string> mediaCodes)
     {
         var englishNames = new System.Collections.Generic.HashSet<string>();
@@ -749,12 +757,21 @@ public class ElasticSearchIndex : ISearchIndex
 
         return englishNames.ToList();
     }
+    
+    private static List<string> GetLanguageTags(IEnumerable<MediaVersion> mediaVersions) =>
+        mediaVersions
+            .Map(mv => mv.Streams.Filter(ms => ms.MediaStreamKind == MediaStreamKind.Audio).Map(ms => ms.Language))
+            .Flatten()
+            .Filter(s => !string.IsNullOrWhiteSpace(s))
+            .Distinct()
+            .ToList();
 
     private static void AddStatistics(ElasticSearchItem doc, IEnumerable<MediaVersion> mediaVersions)
     {
         foreach (MediaVersion version in mediaVersions.HeadOrNone())
         {
             doc.Minutes = (int)Math.Ceiling(version.Duration.TotalMinutes);
+            doc.Seconds = (int)Math.Ceiling(version.Duration.TotalSeconds);
 
             foreach (MediaStream videoStream in version.Streams
                          .Filter(s => s.MediaStreamKind == MediaStreamKind.Video)

ErsatzTV.Infrastructure/Search/LuceneSearchIndex.cs

@@ -45,6 +45,7 @@ public sealed class LuceneSearchIndex : ISearchIndex
     internal const string JumpLetterField = "jump_letter";
     internal const string StudioField = "studio";
     internal const string LanguageField = "language";
+    internal const string LanguageTagField = "language_tag";
     internal const string StyleField = "style";
     internal const string MoodField = "mood";
     internal const string ActorField = "actor";
@@ -64,6 +65,7 @@ public sealed class LuceneSearchIndex : ISearchIndex
     internal const string VideoDynamicRange = "video_dynamic_range";
 
     internal const string MinutesField = "minutes";
+    internal const string SecondsField = "seconds";
     internal const string HeightField = "height";
     internal const string WidthField = "width";
     internal const string SeasonNumberField = "season_number";
@@ -106,7 +108,7 @@ public sealed class LuceneSearchIndex : ISearchIndex
         return Task.FromResult(directoryExists && fileExists);
     }
 
-    public int Version => 36;
+    public int Version => 37;
 
     public async Task<bool> Initialize(
         ILocalFileSystem localFileSystem,
@@ -495,23 +497,28 @@ public sealed class LuceneSearchIndex : ISearchIndex
         }
     }
 
-    private async Task AddLanguages(ISearchRepository searchRepository, Document doc, List<MediaVersion> mediaVersions)
+    private async Task AddLanguages(
+        ISearchRepository searchRepository,
+        Document doc,
+        IEnumerable<MediaVersion> mediaVersions)
     {
-        Option<MediaVersion> maybeVersion = mediaVersions.HeadOrNone();
+        var mediaCodes = mediaVersions
-        if (maybeVersion.IsSome)
+            .Map(mv => mv.Streams.Filter(ms => ms.MediaStreamKind == MediaStreamKind.Audio).Map(ms => ms.Language))
-        {
+            .Flatten()
-            MediaVersion version = maybeVersion.ValueUnsafe();
+            .Filter(c => !string.IsNullOrWhiteSpace(c))
-            var mediaCodes = version.Streams
+            .Distinct()
-                .Filter(ms => ms.MediaStreamKind == MediaStreamKind.Audio)
+            .ToList();
-                .Map(ms => ms.Language).Distinct()
+
-                .ToList();
+        await AddLanguages(searchRepository, doc, mediaCodes);
-
-            await AddLanguages(searchRepository, doc, mediaCodes);
-        }
     }
 
     private async Task AddLanguages(ISearchRepository searchRepository, Document doc, List<string> mediaCodes)
     {
+        foreach (string code in mediaCodes.Where(c => !string.IsNullOrWhiteSpace(c)).Distinct())
+        {
+            doc.Add(new TextField(LanguageTagField, code, Field.Store.NO));
+        }
+        
         var englishNames = new System.Collections.Generic.HashSet<string>();
         foreach (string code in await searchRepository.GetAllLanguageCodes(mediaCodes))
         {
@@ -1240,6 +1247,7 @@ public sealed class LuceneSearchIndex : ISearchIndex
         foreach (MediaVersion version in mediaVersions)
         {
             doc.Add(new Int32Field(MinutesField, (int)Math.Ceiling(version.Duration.TotalMinutes), Field.Store.NO));
+            doc.Add(new Int32Field(SecondsField, (int)Math.Ceiling(version.Duration.TotalSeconds), Field.Store.NO));
 
             if (version.Streams.Any(s => s.MediaStreamKind == MediaStreamKind.Video))
             {

ErsatzTV.Infrastructure/Search/Models/ElasticSearchItem.cs

@@ -27,9 +27,15 @@ public class ElasticSearchItem : MinimalElasticSearchItem
 
     [JsonPropertyName(LuceneSearchIndex.LanguageField)]
     public List<string> Language { get; set; }
+    
+    [JsonPropertyName(LuceneSearchIndex.LanguageTagField)]
+    public List<string> LanguageTag { get; set; }
 
     [JsonPropertyName(LuceneSearchIndex.MinutesField)]
     public int Minutes { get; set; }
+    
+    [JsonPropertyName(LuceneSearchIndex.SecondsField)]
+    public int Seconds { get; set; }
 
     [JsonPropertyName(LuceneSearchIndex.HeightField)]
     public int Height { get; set; }