fix languages that have multiple codes (#270)

CHANGELOG.md

@@ -4,10 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
+### Added
+- Add warning during playout rebuild when schedule has been emptied
+
 ### Fixed
 - Fix ui crash adding a channel without a watermark
 - Clear playout detail table when playout is deleted
 - Fix blazor error font color
+- Fix some audio stream languages missing from UI and search index
+- Fix audio stream selection for languages with multiple codes
 
 ## [0.0.46-prealpha] - 2021-06-14
 ### Added

ErsatzTV.Application/Artists/Queries/GetArtistByIdHandler.cs

@@ -28,8 +28,9 @@ namespace ErsatzTV.Application.Artists.Queries
             return await maybeArtist.Match<Task<Option<ArtistViewModel>>>(
                 async artist =>
                 {
-                    List<string> languages = await _searchRepository.GetLanguagesForArtist(artist);
-                    return ProjectToViewModel(artist, languages);
+                    List<string> mediaCodes = await _searchRepository.GetLanguagesForArtist(artist);
+                    List<string> languageCodes = await _searchRepository.GetAllLanguageCodes(mediaCodes);
+                    return ProjectToViewModel(artist, languageCodes);
                 },
                 () => Task.FromResult(Option<ArtistViewModel>.None));
         }

ErsatzTV.Application/MediaItems/Queries/GetAllLanguageCodesHandler.cs

@@ -1,35 +1,53 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using ErsatzTV.Core.Interfaces.Repositories;
+using ErsatzTV.Infrastructure.Data;
+using ErsatzTV.Infrastructure.Extensions;
 using LanguageExt;
 using MediatR;
+using Microsoft.EntityFrameworkCore;
 
 namespace ErsatzTV.Application.MediaItems.Queries
 {
     public class GetAllLanguageCodesHandler : IRequestHandler<GetAllLanguageCodes, List<CultureInfo>>
     {
+        private readonly IDbContextFactory<TvContext> _dbContextFactory;
         private readonly IMediaItemRepository _mediaItemRepository;
 
-        public GetAllLanguageCodesHandler(IMediaItemRepository mediaItemRepository) =>
+        public GetAllLanguageCodesHandler(
+            IDbContextFactory<TvContext> dbContextFactory,
+            IMediaItemRepository mediaItemRepository)
+        {
+            _dbContextFactory = dbContextFactory;
             _mediaItemRepository = mediaItemRepository;
+        }
 
         public async Task<List<CultureInfo>> Handle(GetAllLanguageCodes request, CancellationToken cancellationToken)
         {
-            var result = new List<CultureInfo>();
+            await using TvContext dbContext = _dbContextFactory.CreateDbContext();
+            
+            var result = new System.Collections.Generic.HashSet<CultureInfo>();
 
             CultureInfo[] allCultures = CultureInfo.GetCultures(CultureTypes.NeutralCultures);
-            List<string> allLanguageCodes = await _mediaItemRepository.GetAllLanguageCodes();
-            foreach (string code in allLanguageCodes)
+            List<string> mediaCodes = await _mediaItemRepository.GetAllLanguageCodes();
+            foreach (string mediaCode in mediaCodes)
             {
-                Option<CultureInfo> maybeCulture = allCultures.Find(
-                    ci => string.Equals(code, ci.ThreeLetterISOLanguageName, StringComparison.OrdinalIgnoreCase));
-                await maybeCulture.IfSomeAsync(cultureInfo => result.Add(cultureInfo));
+                foreach (string code in await dbContext.LanguageCodes.GetAllLanguageCodes(mediaCode))
+                {
+                    Option<CultureInfo> maybeCulture = allCultures.Find(
+                        c => string.Equals(code, c.ThreeLetterISOLanguageName, StringComparison.OrdinalIgnoreCase));
+                    foreach (CultureInfo culture in maybeCulture)
+                    {
+                        result.Add(culture);
+                    }
+                }
             }
 
-            return result;
+            return result.ToList();
         }
     }
 }

ErsatzTV.Application/Movies/Mapper.cs

@@ -15,6 +15,7 @@ namespace ErsatzTV.Application.Movies
     {
         internal static MovieViewModel ProjectToViewModel(
             Movie movie,
+            List<string> languageCodes,
             Option<JellyfinMediaSource> maybeJellyfin,
             Option<EmbyMediaSource> maybeEmby)
         {
@@ -28,7 +29,7 @@ namespace ErsatzTV.Application.Movies
                 metadata.Studios.Map(s => s.Name).ToList(),
                 (metadata.ContentRating ?? string.Empty).Split("/").Map(s => s.Trim())
                 .Where(x => !string.IsNullOrWhiteSpace(x)).ToList(),
-                LanguagesForMovie(movie),
+                LanguagesForMovie(languageCodes),
                 metadata.Actors.OrderBy(a => a.Order).ThenBy(a => a.Id)
                     .Map(a => MediaCards.Mapper.ProjectToViewModel(a, maybeJellyfin, maybeEmby))
                     .ToList(),
@@ -40,13 +41,11 @@ namespace ErsatzTV.Application.Movies
             };
         }
 
-        private static List<CultureInfo> LanguagesForMovie(Movie movie)
+        private static List<CultureInfo> LanguagesForMovie(List<string> languageCodes)
         {
             CultureInfo[] allCultures = CultureInfo.GetCultures(CultureTypes.NeutralCultures);
 
-            return movie.MediaVersions
-                .Map(mv => mv.Streams.Filter(s => s.MediaStreamKind == MediaStreamKind.Audio).Map(s => s.Language))
-                .Flatten()
+            return languageCodes
                 .Distinct()
                 .Map(
                     lang => allCultures.Filter(

ErsatzTV.Application/Movies/Queries/GetMovieByIdHandler.cs

@@ -1,9 +1,14 @@
-using System.Threading;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Repositories;
+using ErsatzTV.Infrastructure.Data;
+using ErsatzTV.Infrastructure.Extensions;
 using LanguageExt;
 using MediatR;
+using Microsoft.EntityFrameworkCore;
 using static ErsatzTV.Application.Movies.Mapper;
 
 namespace ErsatzTV.Application.Movies.Queries
@@ -11,10 +16,15 @@ namespace ErsatzTV.Application.Movies.Queries
     public class GetMovieByIdHandler : IRequestHandler<GetMovieById, Option<MovieViewModel>>
     {
         private readonly IMediaSourceRepository _mediaSourceRepository;
+        private readonly IDbContextFactory<TvContext> _dbContextFactory;
         private readonly IMovieRepository _movieRepository;
 
-        public GetMovieByIdHandler(IMovieRepository movieRepository, IMediaSourceRepository mediaSourceRepository)
+        public GetMovieByIdHandler(
+            IDbContextFactory<TvContext> dbContextFactory,
+            IMovieRepository movieRepository,
+            IMediaSourceRepository mediaSourceRepository)
         {
+            _dbContextFactory = dbContextFactory;
             _movieRepository = movieRepository;
             _mediaSourceRepository = mediaSourceRepository;
         }
@@ -23,6 +33,8 @@ namespace ErsatzTV.Application.Movies.Queries
             GetMovieById request,
             CancellationToken cancellationToken)
         {
+            await using TvContext dbContext = _dbContextFactory.CreateDbContext();
+            
             Option<JellyfinMediaSource> maybeJellyfin = await _mediaSourceRepository.GetAllJellyfin()
                 .Map(list => list.HeadOrNone());
 
@@ -30,7 +42,20 @@ namespace ErsatzTV.Application.Movies.Queries
                 .Map(list => list.HeadOrNone());
 
             Option<Movie> movie = await _movieRepository.GetMovie(request.Id);
-            return movie.Map(m => ProjectToViewModel(m, maybeJellyfin, maybeEmby));
+
+            Option<MediaVersion> maybeVersion = movie.Map(m => m.MediaVersions.HeadOrNone()).Flatten();
+            var languageCodes = new List<string>();
+            foreach (MediaVersion version in maybeVersion)
+            {
+                var mediaCodes = version.Streams
+                    .Filter(ms => ms.MediaStreamKind == MediaStreamKind.Audio)
+                    .Map(ms => ms.Language)
+                    .ToList();
+
+                languageCodes.AddRange(await dbContext.LanguageCodes.GetAllLanguageCodes(mediaCodes));
+            }
+
+            return movie.Map(m => ProjectToViewModel(m, languageCodes, maybeJellyfin, maybeEmby));
         }
     }
 }

ErsatzTV.Application/Television/Queries/GetTelevisionShowByIdHandler.cs

@@ -39,8 +39,9 @@ namespace ErsatzTV.Application.Television.Queries
                     Option<EmbyMediaSource> maybeEmby = await _mediaSourceRepository.GetAllEmby()
                         .Map(list => list.HeadOrNone());
 
-                    List<string> languages = await _searchRepository.GetLanguagesForShow(show);
-                    return ProjectToViewModel(show, languages, maybeJellyfin, maybeEmby);
+                    List<string> mediaCodes = await _searchRepository.GetLanguagesForShow(show);
+                    List<string> languageCodes = await _searchRepository.GetAllLanguageCodes(mediaCodes);
+                    return ProjectToViewModel(show, languageCodes, maybeJellyfin, maybeEmby);
                 },
                 () => Task.FromResult(Option<TelevisionShowViewModel>.None));
         }

ErsatzTV.Core/Domain/LanguageCode.cs

@@ -0,0 +1,12 @@
+namespace ErsatzTV.Core.Domain
+{
+    public class LanguageCode
+    {
+        public int Id { get; set; }
+        public string ThreeCode1 { get; set; }
+        public string ThreeCode2 { get; set; }
+        public string TwoCode { get; set; }
+        public string EnglishName { get; set; }
+        public string FrenchName { get; set; }
+    }
+}

ErsatzTV.Core/FFmpeg/FFmpegStreamSelector.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using ErsatzTV.Core.Domain;
@@ -13,12 +14,15 @@ namespace ErsatzTV.Core.FFmpeg
     public class FFmpegStreamSelector : IFFmpegStreamSelector
     {
         private readonly IConfigElementRepository _configElementRepository;
+        private readonly ISearchRepository _searchRepository;
         private readonly ILogger<FFmpegStreamSelector> _logger;
 
         public FFmpegStreamSelector(
+            ISearchRepository searchRepository,
             ILogger<FFmpegStreamSelector> logger,
             IConfigElementRepository configElementRepository)
         {
+            _searchRepository = searchRepository;
             _logger = logger;
             _configElementRepository = configElementRepository;
         }
@@ -54,24 +58,31 @@ namespace ErsatzTV.Core.FFmpeg
                     });
             }
 
+            List<string> allCodes = await _searchRepository.GetAllLanguageCodes(new List<string> { language });
+            if (allCodes.Count > 1)
+            {
+                _logger.LogDebug("Preferred language has multiple codes {Codes}", allCodes);
+            }
+
             var correctLanguage = audioStreams.Filter(
-                s => string.Equals(
-                    s.Language,
-                    language,
-                    StringComparison.InvariantCultureIgnoreCase)).ToList();
+                s => allCodes.Any(
+                    c => string.Equals(
+                        s.Language,
+                        c,
+                        StringComparison.InvariantCultureIgnoreCase))).ToList();
             if (correctLanguage.Any())
             {
                 _logger.LogDebug(
-                    "Found {Count} audio streams with preferred language code {Code}; selecting stream with most channels",
+                    "Found {Count} audio streams with preferred language code(s) {Code}; selecting stream with most channels",
                     correctLanguage.Count,
-                    language);
+                    allCodes);
 
                 return correctLanguage.OrderByDescending(s => s.Channels).Head();
             }
 
             _logger.LogDebug(
-                "Unable to find audio stream with preferred language code {Code}; selecting stream with most channels",
-                language);
+                "Unable to find audio stream with preferred language code(s) {Code}; selecting stream with most channels",
+                allCodes);
 
             return audioStreams.OrderByDescending(s => s.Channels).Head();
         }

ErsatzTV.Core/Interfaces/Repositories/ISearchRepository.cs

@@ -7,9 +7,10 @@ namespace ErsatzTV.Core.Interfaces.Repositories
 {
     public interface ISearchRepository
     {
-        public Task<List<int>> GetItemIdsToIndex();
-        public Task<Option<MediaItem>> GetItemToIndex(int id);
-        public Task<List<string>> GetLanguagesForShow(Show show);
-        public Task<List<string>> GetLanguagesForArtist(Artist artist);
+        Task<List<int>> GetItemIdsToIndex();
+        Task<Option<MediaItem>> GetItemToIndex(int id);
+        Task<List<string>> GetLanguagesForShow(Show show);
+        Task<List<string>> GetLanguagesForArtist(Artist artist);
+        Task<List<string>> GetAllLanguageCodes(List<string> mediaCodes);
     }
 }

ErsatzTV.Core/Scheduling/PlayoutBuilder.cs

@@ -52,6 +52,15 @@ namespace ErsatzTV.Core.Scheduling
                 .Distinct()
                 .ToList();
 
+            if (!collectionKeys.Any())
+            {
+                _logger.LogWarning(
+                    "Playout {Playout} schedule {Schedule} has no items",
+                    playout.Channel.Name,
+                    playout.ProgramSchedule.Name);
+                return playout;
+            }
+
             IEnumerable<Tuple<CollectionKey, List<MediaItem>>> tuples = await collectionKeys.Map(
                 async collectionKey =>
                 {

ErsatzTV.Infrastructure/Data/Configurations/LanguageCodeConfiguration.cs

@@ -0,0 +1,11 @@
+using ErsatzTV.Core.Domain;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace ErsatzTV.Infrastructure.Data.Configurations
+{
+    public class LanguageCodeConfiguration : IEntityTypeConfiguration<LanguageCode>
+    {
+        public void Configure(EntityTypeBuilder<LanguageCode> builder) => builder.ToTable("LanguageCode");
+    }
+}

ErsatzTV.Infrastructure/Data/DbInitializer.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
 using ErsatzTV.Core.Domain;
@@ -12,6 +14,43 @@ namespace ErsatzTV.Infrastructure.Data
     {
         public static async Task<Unit> Initialize(TvContext context, CancellationToken cancellationToken)
         {
+            if (!context.LanguageCodes.Any())
+            {
+                var assembly = Assembly.GetEntryAssembly();
+                if (assembly != null)
+                {
+                    await using Stream resource =
+                        assembly.GetManifestResourceStream("ErsatzTV.Resources.ISO-639-2_utf-8.txt");
+                    if (resource != null)
+                    {
+                        using var reader = new StreamReader(resource);
+                        while (!reader.EndOfStream)
+                        {
+                            string line = await reader.ReadLineAsync();
+                            if (line != null)
+                            {
+                                string[] split = line.Split("|");
+                                if (split.Length == 5)
+                                {
+                                    var languageCode = new LanguageCode
+                                    {
+                                        ThreeCode1 = split[0],
+                                        ThreeCode2 = split[1],
+                                        TwoCode = split[2],
+                                        EnglishName = split[3],
+                                        FrenchName = split[4]
+                                    };
+
+                                    await context.LanguageCodes.AddAsync(languageCode, cancellationToken);
+                                }
+                            }
+                        }
+                    }
+                }
+
+                await context.SaveChangesAsync(cancellationToken);
+            }
+
             if (context.Resolutions.Any())
             {
                 return Unit.Default;

ErsatzTV.Infrastructure/Data/Repositories/SearchRepository.cs

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
 using Dapper;
 using ErsatzTV.Core.Domain;
 using ErsatzTV.Core.Interfaces.Repositories;
+using ErsatzTV.Infrastructure.Extensions;
 using LanguageExt;
 using Microsoft.EntityFrameworkCore;
 using static LanguageExt.Prelude;
@@ -110,5 +111,11 @@ namespace ErsatzTV.Infrastructure.Data.Repositories
                     INNER JOIN Artist A on MV.ArtistId = A.Id
                     WHERE MediaStreamKind = 2 AND A.Id = @ArtistId",
                 new { ArtistId = artist.Id }).Map(result => result.ToList());
+        
+        public async Task<List<string>> GetAllLanguageCodes(List<string> mediaCodes)
+        {
+            await using TvContext dbContext = _dbContextFactory.CreateDbContext();
+            return await dbContext.LanguageCodes.GetAllLanguageCodes(mediaCodes);
+        }
     }
 }

ErsatzTV.Infrastructure/Data/TvContext.cs

@@ -65,6 +65,7 @@ namespace ErsatzTV.Infrastructure.Data
         public DbSet<PlayoutProgramScheduleAnchor> PlayoutProgramScheduleItemAnchors { get; set; }
         public DbSet<FFmpegProfile> FFmpegProfiles { get; set; }
         public DbSet<Resolution> Resolutions { get; set; }
+        public DbSet<LanguageCode> LanguageCodes { get; set; }
 
         protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) =>
             optionsBuilder.UseLoggerFactory(_loggerFactory);

ErsatzTV.Infrastructure/Extensions/LanguageCodeQueryableExtensions.cs

@@ -0,0 +1,78 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using ErsatzTV.Core.Domain;
+using Microsoft.EntityFrameworkCore;
+
+namespace ErsatzTV.Infrastructure.Extensions
+{
+    public static class LanguageCodeQueryableExtensions
+    {
+        public static async Task<List<string>> GetAllLanguageCodes(
+            this IQueryable<LanguageCode> languageCodes,
+            string mediaCode)
+        {
+            if (string.IsNullOrWhiteSpace(mediaCode))
+            {
+                return new List<string>();
+            }
+
+            string code = mediaCode.ToLowerInvariant();
+
+            List<LanguageCode> maybeLanguages = await languageCodes
+                .Filter(lc => lc.ThreeCode1 == code || lc.ThreeCode2 == code)
+                .ToListAsync();
+
+            var result = new HashSet<string>();
+            foreach (LanguageCode language in maybeLanguages)
+            {
+                if (!string.IsNullOrWhiteSpace(language.ThreeCode1))
+                {
+                    result.Add(language.ThreeCode1);
+                }
+
+                if (!string.IsNullOrWhiteSpace(language.ThreeCode2))
+                {
+                    result.Add(language.ThreeCode2);
+                }
+            }
+
+            return result.ToList();
+        }
+
+        public static async Task<List<string>> GetAllLanguageCodes(
+            this IQueryable<LanguageCode> languageCodes,
+            List<string> mediaCodes)
+        {
+            var validCodes = mediaCodes
+                .Filter(c => !string.IsNullOrWhiteSpace(c))
+                .Map(c => c.ToLowerInvariant())
+                .ToList();
+
+            if (validCodes.Count == 0)
+            {
+                return new List<string>();
+            }
+
+            List<LanguageCode> maybeLanguages = await languageCodes
+                .Filter(lc => validCodes.Contains(lc.ThreeCode1) || validCodes.Contains(lc.ThreeCode2))
+                .ToListAsync();
+
+            var result = new HashSet<string>();
+            foreach (LanguageCode language in maybeLanguages)
+            {
+                if (!string.IsNullOrWhiteSpace(language.ThreeCode1))
+                {
+                    result.Add(language.ThreeCode1);
+                }
+
+                if (!string.IsNullOrWhiteSpace(language.ThreeCode2))
+                {
+                    result.Add(language.ThreeCode2);
+                }
+            }
+
+            return result.ToList();
+        }
+    }
+}

ErsatzTV.Infrastructure/Migrations/20210615014552_Add_LanguageCode.cs

@@ -0,0 +1,33 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace ErsatzTV.Infrastructure.Migrations
+{
+    public partial class Add_LanguageCode : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "LanguageCode",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "INTEGER", nullable: false)
+                        .Annotation("Sqlite:Autoincrement", true),
+                    ThreeCode1 = table.Column<string>(type: "TEXT", nullable: true),
+                    ThreeCode2 = table.Column<string>(type: "TEXT", nullable: true),
+                    TwoCode = table.Column<string>(type: "TEXT", nullable: true),
+                    EnglishName = table.Column<string>(type: "TEXT", nullable: true),
+                    FrenchName = table.Column<string>(type: "TEXT", nullable: true)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_LanguageCode", x => x.Id);
+                });
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "LanguageCode");
+        }
+    }
+}

ErsatzTV.Infrastructure/Migrations/TvContextModelSnapshot.cs

@@ -583,6 +583,32 @@ namespace ErsatzTV.Infrastructure.Migrations
                     b.ToTable("JellyfinPathReplacement");
                 });
 
+            modelBuilder.Entity("ErsatzTV.Core.Domain.LanguageCode", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER");
+
+                    b.Property<string>("EnglishName")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("FrenchName")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("ThreeCode1")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("ThreeCode2")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("TwoCode")
+                        .HasColumnType("TEXT");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("LanguageCode");
+                });
+
             modelBuilder.Entity("ErsatzTV.Core.Domain.Library", b =>
                 {
                     b.Property<int>("Id")

ErsatzTV.Infrastructure/Search/SearchIndex.cs

@@ -72,7 +72,7 @@ namespace ErsatzTV.Infrastructure.Search
             _initialized = false;
         }
 
-        public int Version => 13;
+        public int Version => 14;
 
         public Task<bool> Initialize(ILocalFileSystem localFileSystem)
         {
@@ -91,41 +91,6 @@ namespace ErsatzTV.Infrastructure.Search
             return Task.FromResult(_initialized);
         }
 
-        public async Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds)
-        {
-            _writer.DeleteAll();
-
-            foreach (int id in itemIds)
-            {
-                Option<MediaItem> maybeMediaItem = await searchRepository.GetItemToIndex(id);
-                if (maybeMediaItem.IsSome)
-                {
-                    MediaItem mediaItem = maybeMediaItem.ValueUnsafe();
-                    switch (mediaItem)
-                    {
-                        case Movie movie:
-                            UpdateMovie(movie);
-                            break;
-                        case Show show:
-                            await UpdateShow(searchRepository, show);
-                            break;
-                        case Artist artist:
-                            await UpdateArtist(searchRepository, artist);
-                            break;
-                        case MusicVideo musicVideo:
-                            UpdateMusicVideo(musicVideo);
-                            break;
-                        case Episode episode:
-                            UpdateEpisode(episode);
-                            break;
-                    }
-                }
-            }
-
-            _writer.Commit();
-            return Unit.Default;
-        }
-
         public Task<Unit> AddItems(ISearchRepository searchRepository, List<MediaItem> items) =>
             UpdateItems(searchRepository, items);
 
@@ -136,7 +101,7 @@ namespace ErsatzTV.Infrastructure.Search
                 switch (item)
                 {
                     case Movie movie:
-                        UpdateMovie(movie);
+                        await UpdateMovie(searchRepository, movie);
                         break;
                     case Show show:
                         await UpdateShow(searchRepository, show);
@@ -145,10 +110,10 @@ namespace ErsatzTV.Infrastructure.Search
                         await UpdateArtist(searchRepository, artist);
                         break;
                     case MusicVideo musicVideo:
-                        UpdateMusicVideo(musicVideo);
+                        await UpdateMusicVideo(searchRepository, musicVideo);
                         break;
                     case Episode episode:
-                        UpdateEpisode(episode);
+                        await UpdateEpisode(searchRepository, episode);
                         break;
                 }
             }
@@ -217,6 +182,41 @@ namespace ErsatzTV.Infrastructure.Search
             _directory?.Dispose();
         }
 
+        public async Task<Unit> Rebuild(ISearchRepository searchRepository, List<int> itemIds)
+        {
+            _writer.DeleteAll();
+
+            foreach (int id in itemIds)
+            {
+                Option<MediaItem> maybeMediaItem = await searchRepository.GetItemToIndex(id);
+                if (maybeMediaItem.IsSome)
+                {
+                    MediaItem mediaItem = maybeMediaItem.ValueUnsafe();
+                    switch (mediaItem)
+                    {
+                        case Movie movie:
+                            await UpdateMovie(searchRepository, movie);
+                            break;
+                        case Show show:
+                            await UpdateShow(searchRepository, show);
+                            break;
+                        case Artist artist:
+                            await UpdateArtist(searchRepository, artist);
+                            break;
+                        case MusicVideo musicVideo:
+                            await UpdateMusicVideo(searchRepository, musicVideo);
+                            break;
+                        case Episode episode:
+                            await UpdateEpisode(searchRepository, episode);
+                            break;
+                    }
+                }
+            }
+
+            _writer.Commit();
+            return Unit.Default;
+        }
+
         private static Option<SearchPageMap> GetSearchPageMap(
             IndexSearcher searcher,
             Query query,
@@ -263,7 +263,7 @@ namespace ErsatzTV.Infrastructure.Search
             return new SearchPageMap(map);
         }
 
-        private void UpdateMovie(Movie movie)
+        private async Task UpdateMovie(ISearchRepository searchRepository, Movie movie)
         {
             Option<MovieMetadata> maybeMetadata = movie.MovieMetadata.HeadOrNone();
             if (maybeMetadata.IsSome)
@@ -284,7 +284,7 @@ namespace ErsatzTV.Infrastructure.Search
                         new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
                     };
 
-                    AddLanguages(doc, movie.MediaVersions);
+                    await AddLanguages(searchRepository, doc, movie.MediaVersions);
 
                     if (!string.IsNullOrWhiteSpace(metadata.ContentRating))
                     {
@@ -349,25 +349,38 @@ namespace ErsatzTV.Infrastructure.Search
             }
         }
 
-        private void AddLanguages(Document doc, List<MediaVersion> mediaVersions)
+        private async Task AddLanguages(ISearchRepository searchRepository, Document doc, List<MediaVersion> mediaVersions)
         {
             Option<MediaVersion> maybeVersion = mediaVersions.HeadOrNone();
             if (maybeVersion.IsSome)
             {
                 MediaVersion version = maybeVersion.ValueUnsafe();
-                foreach (CultureInfo cultureInfo in version.Streams
+                var mediaCodes = version.Streams
                     .Filter(ms => ms.MediaStreamKind == MediaStreamKind.Audio)
                     .Map(ms => ms.Language).Distinct()
-                    .Filter(s => !string.IsNullOrWhiteSpace(s))
-                    .Map(
-                        l => _cultureInfos.Filter(
-                            c => string.Equals(c.ThreeLetterISOLanguageName, l, StringComparison.OrdinalIgnoreCase)))
-                    .Sequence()
-                    .Flatten())
+                    .ToList();
+
+                await AddLanguages(searchRepository, doc, mediaCodes);
+            }
+        }
+
+        private async Task AddLanguages(ISearchRepository searchRepository, Document doc, List<string> mediaCodes)
+        {
+            var englishNames = new System.Collections.Generic.HashSet<string>();
+            foreach (string code in await searchRepository.GetAllLanguageCodes(mediaCodes))
+            {
+                Option<CultureInfo> maybeCultureInfo = _cultureInfos.Find(
+                    ci => string.Equals(ci.ThreeLetterISOLanguageName, code, StringComparison.OrdinalIgnoreCase));
+                foreach (CultureInfo cultureInfo in maybeCultureInfo)
                 {
-                    doc.Add(new TextField(LanguageField, cultureInfo.EnglishName, Field.Store.NO));
+                    englishNames.Add(cultureInfo.EnglishName);
                 }
             }
+
+            foreach (string englishName in englishNames)
+            {
+                doc.Add(new TextField(LanguageField, englishName, Field.Store.NO));
+            }
         }
 
         private async Task UpdateShow(ISearchRepository searchRepository, Show show)
@@ -392,20 +405,7 @@ namespace ErsatzTV.Infrastructure.Search
                     };
 
                     List<string> languages = await searchRepository.GetLanguagesForShow(show);
-                    foreach (CultureInfo cultureInfo in languages
-                        .Distinct()
-                        .Filter(s => !string.IsNullOrWhiteSpace(s))
-                        .Map(
-                            l => _cultureInfos.Filter(
-                                c => string.Equals(
-                                    c.ThreeLetterISOLanguageName,
-                                    l,
-                                    StringComparison.OrdinalIgnoreCase)))
-                        .Sequence()
-                        .Flatten())
-                    {
-                        doc.Add(new TextField(LanguageField, cultureInfo.EnglishName, Field.Store.NO));
-                    }
+                    await AddLanguages(searchRepository, doc, languages);
 
                     if (!string.IsNullOrWhiteSpace(metadata.ContentRating))
                     {
@@ -482,20 +482,7 @@ namespace ErsatzTV.Infrastructure.Search
                     };
 
                     List<string> languages = await searchRepository.GetLanguagesForArtist(artist);
-                    foreach (CultureInfo cultureInfo in languages
-                        .Distinct()
-                        .Filter(s => !string.IsNullOrWhiteSpace(s))
-                        .Map(
-                            l => _cultureInfos.Filter(
-                                c => string.Equals(
-                                    c.ThreeLetterISOLanguageName,
-                                    l,
-                                    StringComparison.OrdinalIgnoreCase)))
-                        .Sequence()
-                        .Flatten())
-                    {
-                        doc.Add(new TextField(LanguageField, cultureInfo.EnglishName, Field.Store.NO));
-                    }
+                    await AddLanguages(searchRepository, doc, languages);
 
                     foreach (Genre genre in metadata.Genres)
                     {
@@ -522,7 +509,7 @@ namespace ErsatzTV.Infrastructure.Search
             }
         }
 
-        private void UpdateMusicVideo(MusicVideo musicVideo)
+        private async Task UpdateMusicVideo(ISearchRepository searchRepository, MusicVideo musicVideo)
         {
             Option<MusicVideoMetadata> maybeMetadata = musicVideo.MusicVideoMetadata.HeadOrNone();
             if (maybeMetadata.IsSome)
@@ -543,7 +530,7 @@ namespace ErsatzTV.Infrastructure.Search
                         new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES)
                     };
 
-                    AddLanguages(doc, musicVideo.MediaVersions);
+                    await AddLanguages(searchRepository, doc, musicVideo.MediaVersions);
 
                     if (metadata.ReleaseDate.HasValue)
                     {
@@ -584,7 +571,7 @@ namespace ErsatzTV.Infrastructure.Search
             }
         }
 
-        private void UpdateEpisode(Episode episode)
+        private async Task UpdateEpisode(ISearchRepository searchRepository, Episode episode)
         {
             foreach (EpisodeMetadata metadata in episode.EpisodeMetadata)
             {
@@ -611,7 +598,7 @@ namespace ErsatzTV.Infrastructure.Search
                     doc.Add(new StringField(TitleAndYearField, GetTitleAndYear(metadata), Field.Store.NO));
                     doc.Add(new StringField(JumpLetterField, GetJumpLetter(metadata), Field.Store.YES));
 
-                    AddLanguages(doc, episode.MediaVersions);
+                    await AddLanguages(searchRepository, doc, episode.MediaVersions);
 
                     if (metadata.ReleaseDate.HasValue)
                     {

ErsatzTV/ErsatzTV.csproj

@@ -53,6 +53,7 @@
       <EmbeddedResource Include="Resources\background.png" />
       <EmbeddedResource Include="Resources\ErsatzTV.png" />
       <EmbeddedResource Include="Resources\Roboto-Regular.ttf" />
+      <EmbeddedResource Include="Resources\ISO-639-2_utf-8.txt" />
     </ItemGroup>
 
 </Project>

ErsatzTV/Resources/ISO-639-2_utf-8.txt

@@ -0,0 +1,487 @@
+aar||aa|Afar|afar
+abk||ab|Abkhazian|abkhaze
+ace|||Achinese|aceh
+ach|||Acoli|acoli
+ada|||Adangme|adangme
+ady|||Adyghe; Adygei|adyghé
+afa|||Afro-Asiatic languages|afro-asiatiques, langues
+afh|||Afrihili|afrihili
+afr||af|Afrikaans|afrikaans
+ain|||Ainu|aïnou
+aka||ak|Akan|akan
+akk|||Akkadian|akkadien
+alb|sqi|sq|Albanian|albanais
+ale|||Aleut|aléoute
+alg|||Algonquian languages|algonquines, langues
+alt|||Southern Altai|altai du Sud
+amh||am|Amharic|amharique
+ang|||English, Old (ca.450-1100)|anglo-saxon (ca.450-1100)
+anp|||Angika|angika
+apa|||Apache languages|apaches, langues
+ara||ar|Arabic|arabe
+arc|||Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)|araméen d'empire (700-300 BCE)
+arg||an|Aragonese|aragonais
+arm|hye|hy|Armenian|arménien
+arn|||Mapudungun; Mapuche|mapudungun; mapuche; mapuce
+arp|||Arapaho|arapaho
+art|||Artificial languages|artificielles, langues
+arw|||Arawak|arawak
+asm||as|Assamese|assamais
+ast|||Asturian; Bable; Leonese; Asturleonese|asturien; bable; léonais; asturoléonais
+ath|||Athapascan languages|athapascanes, langues
+aus|||Australian languages|australiennes, langues
+ava||av|Avaric|avar
+ave||ae|Avestan|avestique
+awa|||Awadhi|awadhi
+aym||ay|Aymara|aymara
+aze||az|Azerbaijani|azéri
+bad|||Banda languages|banda, langues
+bai|||Bamileke languages|bamiléké, langues
+bak||ba|Bashkir|bachkir
+bal|||Baluchi|baloutchi
+bam||bm|Bambara|bambara
+ban|||Balinese|balinais
+baq|eus|eu|Basque|basque
+bas|||Basa|basa
+bat|||Baltic languages|baltes, langues
+bej|||Beja; Bedawiyet|bedja
+bel||be|Belarusian|biélorusse
+bem|||Bemba|bemba
+ben||bn|Bengali|bengali
+ber|||Berber languages|berbères, langues
+bho|||Bhojpuri|bhojpuri
+bih||bh|Bihari languages|langues biharis
+bik|||Bikol|bikol
+bin|||Bini; Edo|bini; edo
+bis||bi|Bislama|bichlamar
+bla|||Siksika|blackfoot
+bnt|||Bantu languages|bantou, langues
+bos||bs|Bosnian|bosniaque
+bra|||Braj|braj
+bre||br|Breton|breton
+btk|||Batak languages|batak, langues
+bua|||Buriat|bouriate
+bug|||Buginese|bugi
+bul||bg|Bulgarian|bulgare
+bur|mya|my|Burmese|birman
+byn|||Blin; Bilin|blin; bilen
+cad|||Caddo|caddo
+cai|||Central American Indian languages|amérindiennes de L'Amérique centrale, langues
+car|||Galibi Carib|karib; galibi; carib
+cat||ca|Catalan; Valencian|catalan; valencien
+cau|||Caucasian languages|caucasiennes, langues
+ceb|||Cebuano|cebuano
+cel|||Celtic languages|celtiques, langues; celtes, langues
+cha||ch|Chamorro|chamorro
+chb|||Chibcha|chibcha
+che||ce|Chechen|tchétchène
+chg|||Chagatai|djaghataï
+chi|zho|zh|Chinese|chinois
+chk|||Chuukese|chuuk
+chm|||Mari|mari
+chn|||Chinook jargon|chinook, jargon
+cho|||Choctaw|choctaw
+chp|||Chipewyan; Dene Suline|chipewyan
+chr|||Cherokee|cherokee
+chu||cu|Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic|slavon d'église; vieux slave; slavon liturgique; vieux bulgare
+chv||cv|Chuvash|tchouvache
+chy|||Cheyenne|cheyenne
+cmc|||Chamic languages|chames, langues
+cnr|||Montenegrin|monténégrin
+cop|||Coptic|copte
+cor||kw|Cornish|cornique
+cos||co|Corsican|corse
+cpe|||Creoles and pidgins, English based|créoles et pidgins basés sur l'anglais
+cpf|||Creoles and pidgins, French-based|créoles et pidgins basés sur le français
+cpp|||Creoles and pidgins, Portuguese-based|créoles et pidgins basés sur le portugais
+cre||cr|Cree|cree
+crh|||Crimean Tatar; Crimean Turkish|tatar de Crimé
+crp|||Creoles and pidgins|créoles et pidgins
+csb|||Kashubian|kachoube
+cus|||Cushitic languages|couchitiques, langues
+cze|ces|cs|Czech|tchèque
+dak|||Dakota|dakota
+dan||da|Danish|danois
+dar|||Dargwa|dargwa
+day|||Land Dayak languages|dayak, langues
+del|||Delaware|delaware
+den|||Slave (Athapascan)|esclave (athapascan)
+dgr|||Dogrib|dogrib
+din|||Dinka|dinka
+div||dv|Divehi; Dhivehi; Maldivian|maldivien
+doi|||Dogri|dogri
+dra|||Dravidian languages|dravidiennes, langues
+dsb|||Lower Sorbian|bas-sorabe
+dua|||Duala|douala
+dum|||Dutch, Middle (ca.1050-1350)|néerlandais moyen (ca. 1050-1350)
+dut|nld|nl|Dutch; Flemish|néerlandais; flamand
+dyu|||Dyula|dioula
+dzo||dz|Dzongkha|dzongkha
+efi|||Efik|efik
+egy|||Egyptian (Ancient)|égyptien
+eka|||Ekajuk|ekajuk
+elx|||Elamite|élamite
+eng||en|English|anglais
+enm|||English, Middle (1100-1500)|anglais moyen (1100-1500)
+epo||eo|Esperanto|espéranto
+est||et|Estonian|estonien
+ewe||ee|Ewe|éwé
+ewo|||Ewondo|éwondo
+fan|||Fang|fang
+fao||fo|Faroese|féroïen
+fat|||Fanti|fanti
+fij||fj|Fijian|fidjien
+fil|||Filipino; Pilipino|filipino; pilipino
+fin||fi|Finnish|finnois
+fiu|||Finno-Ugrian languages|finno-ougriennes, langues
+fon|||Fon|fon
+fre|fra|fr|French|français
+frm|||French, Middle (ca.1400-1600)|français moyen (1400-1600)
+fro|||French, Old (842-ca.1400)|français ancien (842-ca.1400)
+frr|||Northern Frisian|frison septentrional
+frs|||Eastern Frisian|frison oriental
+fry||fy|Western Frisian|frison occidental
+ful||ff|Fulah|peul
+fur|||Friulian|frioulan
+gaa|||Ga|ga
+gay|||Gayo|gayo
+gba|||Gbaya|gbaya
+gem|||Germanic languages|germaniques, langues
+geo|kat|ka|Georgian|géorgien
+ger|deu|de|German|allemand
+gez|||Geez|guèze
+gil|||Gilbertese|kiribati
+gla||gd|Gaelic; Scottish Gaelic|gaélique; gaélique écossais
+gle||ga|Irish|irlandais
+glg||gl|Galician|galicien
+glv||gv|Manx|manx; mannois
+gmh|||German, Middle High (ca.1050-1500)|allemand, moyen haut (ca. 1050-1500)
+goh|||German, Old High (ca.750-1050)|allemand, vieux haut (ca. 750-1050)
+gon|||Gondi|gond
+gor|||Gorontalo|gorontalo
+got|||Gothic|gothique
+grb|||Grebo|grebo
+grc|||Greek, Ancient (to 1453)|grec ancien (jusqu'à 1453)
+gre|ell|el|Greek, Modern (1453-)|grec moderne (après 1453)
+grn||gn|Guarani|guarani
+gsw|||Swiss German; Alemannic; Alsatian|suisse alémanique; alémanique; alsacien
+guj||gu|Gujarati|goudjrati
+gwi|||Gwich'in|gwich'in
+hai|||Haida|haida
+hat||ht|Haitian; Haitian Creole|haïtien; créole haïtien
+hau||ha|Hausa|haoussa
+haw|||Hawaiian|hawaïen
+heb||he|Hebrew|hébreu
+her||hz|Herero|herero
+hil|||Hiligaynon|hiligaynon
+him|||Himachali languages; Western Pahari languages|langues himachalis; langues paharis occidentales
+hin||hi|Hindi|hindi
+hit|||Hittite|hittite
+hmn|||Hmong; Mong|hmong
+hmo||ho|Hiri Motu|hiri motu
+hrv||hr|Croatian|croate
+hsb|||Upper Sorbian|haut-sorabe
+hun||hu|Hungarian|hongrois
+hup|||Hupa|hupa
+iba|||Iban|iban
+ibo||ig|Igbo|igbo
+ice|isl|is|Icelandic|islandais
+ido||io|Ido|ido
+iii||ii|Sichuan Yi; Nuosu|yi de Sichuan
+ijo|||Ijo languages|ijo, langues
+iku||iu|Inuktitut|inuktitut
+ile||ie|Interlingue; Occidental|interlingue
+ilo|||Iloko|ilocano
+ina||ia|Interlingua (International Auxiliary Language Association)|interlingua (langue auxiliaire internationale)
+inc|||Indic languages|indo-aryennes, langues
+ind||id|Indonesian|indonésien
+ine|||Indo-European languages|indo-européennes, langues
+inh|||Ingush|ingouche
+ipk||ik|Inupiaq|inupiaq
+ira|||Iranian languages|iraniennes, langues
+iro|||Iroquoian languages|iroquoises, langues
+ita||it|Italian|italien
+jav||jv|Javanese|javanais
+jbo|||Lojban|lojban
+jpn||ja|Japanese|japonais
+jpr|||Judeo-Persian|judéo-persan
+jrb|||Judeo-Arabic|judéo-arabe
+kaa|||Kara-Kalpak|karakalpak
+kab|||Kabyle|kabyle
+kac|||Kachin; Jingpho|kachin; jingpho
+kal||kl|Kalaallisut; Greenlandic|groenlandais
+kam|||Kamba|kamba
+kan||kn|Kannada|kannada
+kar|||Karen languages|karen, langues
+kas||ks|Kashmiri|kashmiri
+kau||kr|Kanuri|kanouri
+kaw|||Kawi|kawi
+kaz||kk|Kazakh|kazakh
+kbd|||Kabardian|kabardien
+kha|||Khasi|khasi
+khi|||Khoisan languages|khoïsan, langues
+khm||km|Central Khmer|khmer central
+kho|||Khotanese; Sakan|khotanais; sakan
+kik||ki|Kikuyu; Gikuyu|kikuyu
+kin||rw|Kinyarwanda|rwanda
+kir||ky|Kirghiz; Kyrgyz|kirghiz
+kmb|||Kimbundu|kimbundu
+kok|||Konkani|konkani
+kom||kv|Komi|kom
+kon||kg|Kongo|kongo
+kor||ko|Korean|coréen
+kos|||Kosraean|kosrae
+kpe|||Kpelle|kpellé
+krc|||Karachay-Balkar|karatchai balkar
+krl|||Karelian|carélien
+kro|||Kru languages|krou, langues
+kru|||Kurukh|kurukh
+kua||kj|Kuanyama; Kwanyama|kuanyama; kwanyama
+kum|||Kumyk|koumyk
+kur||ku|Kurdish|kurde
+kut|||Kutenai|kutenai
+lad|||Ladino|judéo-espagnol
+lah|||Lahnda|lahnda
+lam|||Lamba|lamba
+lao||lo|Lao|lao
+lat||la|Latin|latin
+lav||lv|Latvian|letton
+lez|||Lezghian|lezghien
+lim||li|Limburgan; Limburger; Limburgish|limbourgeois
+lin||ln|Lingala|lingala
+lit||lt|Lithuanian|lituanien
+lol|||Mongo|mongo
+loz|||Lozi|lozi
+ltz||lb|Luxembourgish; Letzeburgesch|luxembourgeois
+lua|||Luba-Lulua|luba-lulua
+lub||lu|Luba-Katanga|luba-katanga
+lug||lg|Ganda|ganda
+lui|||Luiseno|luiseno
+lun|||Lunda|lunda
+luo|||Luo (Kenya and Tanzania)|luo (Kenya et Tanzanie)
+lus|||Lushai|lushai
+mac|mkd|mk|Macedonian|macédonien
+mad|||Madurese|madourais
+mag|||Magahi|magahi
+mah||mh|Marshallese|marshall
+mai|||Maithili|maithili
+mak|||Makasar|makassar
+mal||ml|Malayalam|malayalam
+man|||Mandingo|mandingue
+mao|mri|mi|Maori|maori
+map|||Austronesian languages|austronésiennes, langues
+mar||mr|Marathi|marathe
+mas|||Masai|massaï
+may|msa|ms|Malay|malais
+mdf|||Moksha|moksa
+mdr|||Mandar|mandar
+men|||Mende|mendé
+mga|||Irish, Middle (900-1200)|irlandais moyen (900-1200)
+mic|||Mi'kmaq; Micmac|mi'kmaq; micmac
+min|||Minangkabau|minangkabau
+mis|||Uncoded languages|langues non codées
+mkh|||Mon-Khmer languages|môn-khmer, langues
+mlg||mg|Malagasy|malgache
+mlt||mt|Maltese|maltais
+mnc|||Manchu|mandchou
+mni|||Manipuri|manipuri
+mno|||Manobo languages|manobo, langues
+moh|||Mohawk|mohawk
+mon||mn|Mongolian|mongol
+mos|||Mossi|moré
+mul|||Multiple languages|multilingue
+mun|||Munda languages|mounda, langues
+mus|||Creek|muskogee
+mwl|||Mirandese|mirandais
+mwr|||Marwari|marvari
+myn|||Mayan languages|maya, langues
+myv|||Erzya|erza
+nah|||Nahuatl languages|nahuatl, langues
+nai|||North American Indian languages|nord-amérindiennes, langues
+nap|||Neapolitan|napolitain
+nau||na|Nauru|nauruan
+nav||nv|Navajo; Navaho|navaho
+nbl||nr|Ndebele, South; South Ndebele|ndébélé du Sud
+nde||nd|Ndebele, North; North Ndebele|ndébélé du Nord
+ndo||ng|Ndonga|ndonga
+nds|||Low German; Low Saxon; German, Low; Saxon, Low|bas allemand; bas saxon; allemand, bas; saxon, bas
+nep||ne|Nepali|népalais
+new|||Nepal Bhasa; Newari|nepal bhasa; newari
+nia|||Nias|nias
+nic|||Niger-Kordofanian languages|nigéro-kordofaniennes, langues
+niu|||Niuean|niué
+nno||nn|Norwegian Nynorsk; Nynorsk, Norwegian|norvégien nynorsk; nynorsk, norvégien
+nob||nb|Bokmål, Norwegian; Norwegian Bokmål|norvégien bokmål
+nog|||Nogai|nogaï; nogay
+non|||Norse, Old|norrois, vieux
+nor||no|Norwegian|norvégien
+nqo|||N'Ko|n'ko
+nso|||Pedi; Sepedi; Northern Sotho|pedi; sepedi; sotho du Nord
+nub|||Nubian languages|nubiennes, langues
+nwc|||Classical Newari; Old Newari; Classical Nepal Bhasa|newari classique
+nya||ny|Chichewa; Chewa; Nyanja|chichewa; chewa; nyanja
+nym|||Nyamwezi|nyamwezi
+nyn|||Nyankole|nyankolé
+nyo|||Nyoro|nyoro
+nzi|||Nzima|nzema
+oci||oc|Occitan (post 1500)|occitan (après 1500)
+oji||oj|Ojibwa|ojibwa
+ori||or|Oriya|oriya
+orm||om|Oromo|galla
+osa|||Osage|osage
+oss||os|Ossetian; Ossetic|ossète
+ota|||Turkish, Ottoman (1500-1928)|turc ottoman (1500-1928)
+oto|||Otomian languages|otomi, langues
+paa|||Papuan languages|papoues, langues
+pag|||Pangasinan|pangasinan
+pal|||Pahlavi|pahlavi
+pam|||Pampanga; Kapampangan|pampangan
+pan||pa|Panjabi; Punjabi|pendjabi
+pap|||Papiamento|papiamento
+pau|||Palauan|palau
+peo|||Persian, Old (ca.600-400 B.C.)|perse, vieux (ca. 600-400 av. J.-C.)
+per|fas|fa|Persian|persan
+phi|||Philippine languages|philippines, langues
+phn|||Phoenician|phénicien
+pli||pi|Pali|pali
+pol||pl|Polish|polonais
+pon|||Pohnpeian|pohnpei
+por||pt|Portuguese|portugais
+pra|||Prakrit languages|prâkrit, langues
+pro|||Provençal, Old (to 1500); Occitan, Old (to 1500)|provençal ancien (jusqu'à 1500); occitan ancien (jusqu'à 1500)
+pus||ps|Pushto; Pashto|pachto
+qaa-qtz|||Reserved for local use|réservée à l'usage local
+que||qu|Quechua|quechua
+raj|||Rajasthani|rajasthani
+rap|||Rapanui|rapanui
+rar|||Rarotongan; Cook Islands Maori|rarotonga; maori des îles Cook
+roa|||Romance languages|romanes, langues
+roh||rm|Romansh|romanche
+rom|||Romany|tsigane
+rum|ron|ro|Romanian; Moldavian; Moldovan|roumain; moldave
+run||rn|Rundi|rundi
+rup|||Aromanian; Arumanian; Macedo-Romanian|aroumain; macédo-roumain
+rus||ru|Russian|russe
+sad|||Sandawe|sandawe
+sag||sg|Sango|sango
+sah|||Yakut|iakoute
+sai|||South American Indian languages|sud-amérindiennes, langues
+sal|||Salishan languages|salishennes, langues
+sam|||Samaritan Aramaic|samaritain
+san||sa|Sanskrit|sanskrit
+sas|||Sasak|sasak
+sat|||Santali|santal
+scn|||Sicilian|sicilien
+sco|||Scots|écossais
+sel|||Selkup|selkoupe
+sem|||Semitic languages|sémitiques, langues
+sga|||Irish, Old (to 900)|irlandais ancien (jusqu'à 900)
+sgn|||Sign Languages|langues des signes
+shn|||Shan|chan
+sid|||Sidamo|sidamo
+sin||si|Sinhala; Sinhalese|singhalais
+sio|||Siouan languages|sioux, langues
+sit|||Sino-Tibetan languages|sino-tibétaines, langues
+sla|||Slavic languages|slaves, langues
+slo|slk|sk|Slovak|slovaque
+slv||sl|Slovenian|slovène
+sma|||Southern Sami|sami du Sud
+sme||se|Northern Sami|sami du Nord
+smi|||Sami languages|sames, langues
+smj|||Lule Sami|sami de Lule
+smn|||Inari Sami|sami d'Inari
+smo||sm|Samoan|samoan
+sms|||Skolt Sami|sami skolt
+sna||sn|Shona|shona
+snd||sd|Sindhi|sindhi
+snk|||Soninke|soninké
+sog|||Sogdian|sogdien
+som||so|Somali|somali
+son|||Songhai languages|songhai, langues
+sot||st|Sotho, Southern|sotho du Sud
+spa||es|Spanish; Castilian|espagnol; castillan
+srd||sc|Sardinian|sarde
+srn|||Sranan Tongo|sranan tongo
+srp||sr|Serbian|serbe
+srr|||Serer|sérère
+ssa|||Nilo-Saharan languages|nilo-sahariennes, langues
+ssw||ss|Swati|swati
+suk|||Sukuma|sukuma
+sun||su|Sundanese|soundanais
+sus|||Susu|soussou
+sux|||Sumerian|sumérien
+swa||sw|Swahili|swahili
+swe||sv|Swedish|suédois
+syc|||Classical Syriac|syriaque classique
+syr|||Syriac|syriaque
+tah||ty|Tahitian|tahitien
+tai|||Tai languages|tai, langues
+tam||ta|Tamil|tamoul
+tat||tt|Tatar|tatar
+tel||te|Telugu|télougou
+tem|||Timne|temne
+ter|||Tereno|tereno
+tet|||Tetum|tetum
+tgk||tg|Tajik|tadjik
+tgl||tl|Tagalog|tagalog
+tha||th|Thai|thaï
+tib|bod|bo|Tibetan|tibétain
+tig|||Tigre|tigré
+tir||ti|Tigrinya|tigrigna
+tiv|||Tiv|tiv
+tkl|||Tokelau|tokelau
+tlh|||Klingon; tlhIngan-Hol|klingon
+tli|||Tlingit|tlingit
+tmh|||Tamashek|tamacheq
+tog|||Tonga (Nyasa)|tonga (Nyasa)
+ton||to|Tonga (Tonga Islands)|tongan (Îles Tonga)
+tpi|||Tok Pisin|tok pisin
+tsi|||Tsimshian|tsimshian
+tsn||tn|Tswana|tswana
+tso||ts|Tsonga|tsonga
+tuk||tk|Turkmen|turkmène
+tum|||Tumbuka|tumbuka
+tup|||Tupi languages|tupi, langues
+tur||tr|Turkish|turc
+tut|||Altaic languages|altaïques, langues
+tvl|||Tuvalu|tuvalu
+twi||tw|Twi|twi
+tyv|||Tuvinian|touva
+udm|||Udmurt|oudmourte
+uga|||Ugaritic|ougaritique
+uig||ug|Uighur; Uyghur|ouïgour
+ukr||uk|Ukrainian|ukrainien
+umb|||Umbundu|umbundu
+und|||Undetermined|indéterminée
+urd||ur|Urdu|ourdou
+uzb||uz|Uzbek|ouszbek
+vai|||Vai|vaï
+ven||ve|Venda|venda
+vie||vi|Vietnamese|vietnamien
+vol||vo|Volapük|volapük
+vot|||Votic|vote
+wak|||Wakashan languages|wakashanes, langues
+wal|||Wolaitta; Wolaytta|wolaitta; wolaytta
+war|||Waray|waray
+was|||Washo|washo
+wel|cym|cy|Welsh|gallois
+wen|||Sorbian languages|sorabes, langues
+wln||wa|Walloon|wallon
+wol||wo|Wolof|wolof
+xal|||Kalmyk; Oirat|kalmouk; oïrat
+xho||xh|Xhosa|xhosa
+yao|||Yao|yao
+yap|||Yapese|yapois
+yid||yi|Yiddish|yiddish
+yor||yo|Yoruba|yoruba
+ypk|||Yupik languages|yupik, langues
+zap|||Zapotec|zapotèque
+zbl|||Blissymbols; Blissymbolics; Bliss|symboles Bliss; Bliss
+zen|||Zenaga|zenaga
+zgh|||Standard Moroccan Tamazight|amazighe standard marocain
+zha||za|Zhuang; Chuang|zhuang; chuang
+znd|||Zande languages|zandé, langues
+zul||zu|Zulu|zoulou
+zun|||Zuni|zuni
+zxx|||No linguistic content; Not applicable|pas de contenu linguistique; non applicable
+zza|||Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki|zaza; dimili; dimli; kirdki; kirmanjki; zazaki