mirror of https://github.com/ErsatzTV/ErsatzTV.git
Browse Source
* support external subtitles in local movie libraries * code cleanup * simplify subtitle updating * skip external subtitles from media servers * fix plex subtitlespull/746/head
Jason Dove
4 years ago
committed by
GitHub
47 changed files with 12875 additions and 448 deletions
@ -1,45 +1,15 @@
@@ -1,45 +1,15 @@
|
||||
using System.Globalization; |
||||
using ErsatzTV.Core.Interfaces.Repositories; |
||||
using ErsatzTV.Infrastructure.Data; |
||||
using ErsatzTV.Infrastructure.Extensions; |
||||
using Microsoft.EntityFrameworkCore; |
||||
|
||||
namespace ErsatzTV.Application.MediaItems; |
||||
|
||||
public class GetAllLanguageCodesHandler : IRequestHandler<GetAllLanguageCodes, List<CultureInfo>> |
||||
{ |
||||
private readonly IDbContextFactory<TvContext> _dbContextFactory; |
||||
private readonly IMediaItemRepository _mediaItemRepository; |
||||
|
||||
public GetAllLanguageCodesHandler( |
||||
IDbContextFactory<TvContext> dbContextFactory, |
||||
IMediaItemRepository mediaItemRepository) |
||||
{ |
||||
_dbContextFactory = dbContextFactory; |
||||
public GetAllLanguageCodesHandler(IMediaItemRepository mediaItemRepository) => |
||||
_mediaItemRepository = mediaItemRepository; |
||||
} |
||||
|
||||
public async Task<List<CultureInfo>> Handle(GetAllLanguageCodes request, CancellationToken cancellationToken) |
||||
{ |
||||
await using TvContext dbContext = _dbContextFactory.CreateDbContext(); |
||||
|
||||
var result = new System.Collections.Generic.HashSet<CultureInfo>(); |
||||
|
||||
CultureInfo[] allCultures = CultureInfo.GetCultures(CultureTypes.NeutralCultures); |
||||
List<string> mediaCodes = await _mediaItemRepository.GetAllLanguageCodes(); |
||||
foreach (string mediaCode in mediaCodes) |
||||
{ |
||||
foreach (string code in await dbContext.LanguageCodes.GetAllLanguageCodes(mediaCode)) |
||||
{ |
||||
Option<CultureInfo> maybeCulture = allCultures.Find( |
||||
c => string.Equals(code, c.ThreeLetterISOLanguageName, StringComparison.OrdinalIgnoreCase)); |
||||
foreach (CultureInfo culture in maybeCulture) |
||||
{ |
||||
result.Add(culture); |
||||
} |
||||
} |
||||
} |
||||
|
||||
return result.ToList(); |
||||
} |
||||
public async Task<List<CultureInfo>> Handle(GetAllLanguageCodes request, CancellationToken cancellationToken) => |
||||
await _mediaItemRepository.GetAllLanguageCodeCultures(); |
||||
} |
||||
|
||||
@ -0,0 +1,108 @@
@@ -0,0 +1,108 @@
|
||||
using System.Globalization; |
||||
using ErsatzTV.Core.Domain; |
||||
using ErsatzTV.Core.Interfaces.Repositories; |
||||
using ErsatzTV.Core.Metadata; |
||||
using ErsatzTV.Core.Tests.Fakes; |
||||
using FluentAssertions; |
||||
using Microsoft.Extensions.Logging; |
||||
using Moq; |
||||
using NUnit.Framework; |
||||
|
||||
namespace ErsatzTV.Core.Tests.Metadata; |
||||
|
||||
[TestFixture] |
||||
public class LocalSubtitlesProviderTests |
||||
{ |
||||
// test cases are from plex's example folder layout here
|
||||
// https://support.plex.tv/articles/200471133-adding-local-subtitles-to-your-media/
|
||||
// /Movies
|
||||
// /Avatar (2009)
|
||||
// Avatar (2009).mkv
|
||||
// Avatar (2009).eng.srt
|
||||
// Avatar (2009).en.forced.ass
|
||||
// Avatar (2009).en.sdh.srt
|
||||
// Avatar (2009).de.srt
|
||||
// Avatar (2009).de.sdh.forced.srt
|
||||
|
||||
[Test] |
||||
public void Should_Find_All_Languages_Codecs_And_Flags_With_Full_Paths() |
||||
{ |
||||
// normally this will have a full list from the database, but we just need these two for testing
|
||||
var cultures = new List<CultureInfo> |
||||
{ |
||||
CultureInfo.GetCultureInfo("en-US"), |
||||
CultureInfo.GetCultureInfo("de-DE") |
||||
}; |
||||
|
||||
var fakeFiles = new List<FakeFileEntry> |
||||
{ |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).mkv"), |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).eng.srt"), |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).en.forced.ass"), |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).en.sdh.srt"), |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).de.srt"), |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).de.sdh.forced.srt") |
||||
}; |
||||
|
||||
var provider = new LocalSubtitlesProvider( |
||||
new Mock<IMediaItemRepository>().Object, |
||||
new Mock<IMetadataRepository>().Object, |
||||
new FakeLocalFileSystem(fakeFiles), |
||||
new Mock<ILogger<LocalSubtitlesProvider>>().Object); |
||||
|
||||
List<Subtitle> result = provider.LocateExternalSubtitles( |
||||
cultures, |
||||
@"/Movies/Avatar (2009)/Avatar (2009).mkv", |
||||
true); |
||||
|
||||
result.Count.Should().Be(5); |
||||
result.Count(s => s.Language == "eng").Should().Be(3); |
||||
result.Count(s => s.Language == "deu").Should().Be(2); |
||||
result.Count(s => s.Forced).Should().Be(2); |
||||
result.Count(s => s.SDH).Should().Be(2); |
||||
result.Count(s => s.Codec == "subrip").Should().Be(4); |
||||
result.Count(s => s.Codec == "ass").Should().Be(1); |
||||
result.All(s => s.Path.Contains(@"/Movies/Avatar (2009)/")).Should().BeTrue(); |
||||
} |
||||
|
||||
[Test] |
||||
public void Should_Find_All_Languages_Codecs_And_Flags_With_File_Names() |
||||
{ |
||||
// normally this will have a full list from the database, but we just need these two for testing
|
||||
var cultures = new List<CultureInfo> |
||||
{ |
||||
CultureInfo.GetCultureInfo("en-US"), |
||||
CultureInfo.GetCultureInfo("de-DE") |
||||
}; |
||||
|
||||
var fakeFiles = new List<FakeFileEntry> |
||||
{ |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).mkv"), |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).eng.srt"), |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).en.forced.ass"), |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).en.sdh.srt"), |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).de.srt"), |
||||
new(@"/Movies/Avatar (2009)/Avatar (2009).de.sdh.forced.srt") |
||||
}; |
||||
|
||||
var provider = new LocalSubtitlesProvider( |
||||
new Mock<IMediaItemRepository>().Object, |
||||
new Mock<IMetadataRepository>().Object, |
||||
new FakeLocalFileSystem(fakeFiles), |
||||
new Mock<ILogger<LocalSubtitlesProvider>>().Object); |
||||
|
||||
List<Subtitle> result = provider.LocateExternalSubtitles( |
||||
cultures, |
||||
@"/Movies/Avatar (2009)/Avatar (2009).mkv", |
||||
false); |
||||
|
||||
result.Count.Should().Be(5); |
||||
result.Count(s => s.Language == "eng").Should().Be(3); |
||||
result.Count(s => s.Language == "deu").Should().Be(2); |
||||
result.Count(s => s.Forced).Should().Be(2); |
||||
result.Count(s => s.SDH).Should().Be(2); |
||||
result.Count(s => s.Codec == "subrip").Should().Be(4); |
||||
result.Count(s => s.Codec == "ass").Should().Be(1); |
||||
result.Count(s => s.Path.Contains(@"/Movies/Avatar (2009)/")).Should().Be(0); |
||||
} |
||||
} |
||||
@ -0,0 +1,8 @@
@@ -0,0 +1,8 @@
|
||||
using ErsatzTV.Core.Domain; |
||||
|
||||
namespace ErsatzTV.Core.Interfaces.Metadata; |
||||
|
||||
public interface ILocalSubtitlesProvider |
||||
{ |
||||
Task<bool> UpdateSubtitles(MediaItem mediaItem, Option<string> localPath, bool saveFullPath); |
||||
} |
||||
@ -0,0 +1,142 @@
@@ -0,0 +1,142 @@
|
||||
using System.Globalization; |
||||
using ErsatzTV.Core.Domain; |
||||
using ErsatzTV.Core.Extensions; |
||||
using ErsatzTV.Core.Interfaces.Metadata; |
||||
using ErsatzTV.Core.Interfaces.Repositories; |
||||
using Microsoft.Extensions.Logging; |
||||
|
||||
namespace ErsatzTV.Core.Metadata; |
||||
|
||||
public class LocalSubtitlesProvider : ILocalSubtitlesProvider |
||||
{ |
||||
private readonly List<CultureInfo> _languageCodes = new(); |
||||
private readonly ILocalFileSystem _localFileSystem; |
||||
private readonly ILogger<LocalSubtitlesProvider> _logger; |
||||
private readonly IMediaItemRepository _mediaItemRepository; |
||||
private readonly IMetadataRepository _metadataRepository; |
||||
|
||||
private readonly SemaphoreSlim _slim = new(1, 1); |
||||
|
||||
public LocalSubtitlesProvider( |
||||
IMediaItemRepository mediaItemRepository, |
||||
IMetadataRepository metadataRepository, |
||||
ILocalFileSystem localFileSystem, |
||||
ILogger<LocalSubtitlesProvider> logger) |
||||
{ |
||||
_mediaItemRepository = mediaItemRepository; |
||||
_metadataRepository = metadataRepository; |
||||
_localFileSystem = localFileSystem; |
||||
_logger = logger; |
||||
} |
||||
|
||||
public async Task<bool> UpdateSubtitles(MediaItem mediaItem, Option<string> localPath, bool saveFullPath) |
||||
{ |
||||
if (_languageCodes.Count == 0) |
||||
{ |
||||
await _slim.WaitAsync(); |
||||
try |
||||
{ |
||||
_languageCodes.AddRange(await _mediaItemRepository.GetAllLanguageCodeCultures()); |
||||
} |
||||
finally |
||||
{ |
||||
_slim.Release(); |
||||
} |
||||
} |
||||
|
||||
if (_languageCodes.Count == 0) |
||||
{ |
||||
_logger.LogError("Failed to update subtitles; unable to load languages from database"); |
||||
return false; |
||||
} |
||||
|
||||
Option<Domain.Metadata> maybeMetadata = mediaItem switch |
||||
{ |
||||
Episode e => e.EpisodeMetadata.OfType<Domain.Metadata>().HeadOrNone(), |
||||
Movie m => m.MovieMetadata.OfType<Domain.Metadata>().HeadOrNone(), |
||||
MusicVideo mv => mv.MusicVideoMetadata.OfType<Domain.Metadata>().HeadOrNone(), |
||||
OtherVideo ov => ov.OtherVideoMetadata.OfType<Domain.Metadata>().HeadOrNone(), |
||||
_ => None |
||||
}; |
||||
|
||||
foreach (Domain.Metadata metadata in maybeMetadata) |
||||
{ |
||||
MediaVersion version = mediaItem.GetHeadVersion(); |
||||
var subtitleStreams = version.Streams |
||||
.Filter(s => s.MediaStreamKind == MediaStreamKind.Subtitle) |
||||
.ToList(); |
||||
|
||||
var subtitles = subtitleStreams.Map(Subtitle.FromMediaStream).ToList(); |
||||
string mediaItemPath = await localPath.IfNoneAsync(() => mediaItem.GetHeadVersion().MediaFiles.Head().Path); |
||||
subtitles.AddRange(LocateExternalSubtitles(_languageCodes, mediaItemPath, saveFullPath)); |
||||
await _metadataRepository.UpdateSubtitles(metadata, subtitles); |
||||
} |
||||
|
||||
return false; |
||||
} |
||||
|
||||
public List<Subtitle> LocateExternalSubtitles( |
||||
List<CultureInfo> languageCodes, |
||||
string mediaItemPath, |
||||
bool saveFullPath) |
||||
{ |
||||
var result = new List<Subtitle>(); |
||||
|
||||
string folder = Path.GetDirectoryName(mediaItemPath); |
||||
string withoutExtension = Path.GetFileNameWithoutExtension(mediaItemPath); |
||||
foreach (string file in _localFileSystem.ListFiles(folder, $"{withoutExtension}*")) |
||||
{ |
||||
string fileName = Path.GetFileName(file); |
||||
if (!fileName.StartsWith(withoutExtension)) |
||||
{ |
||||
continue; |
||||
} |
||||
|
||||
string extension = Path.GetExtension(file); |
||||
string codec = extension switch |
||||
{ |
||||
".ssa" or ".ass" => "ass", |
||||
".srt" => "subrip", |
||||
".vtt" => "webvtt", |
||||
_ => string.Empty |
||||
}; |
||||
|
||||
if (string.IsNullOrWhiteSpace(codec)) |
||||
{ |
||||
continue; |
||||
} |
||||
|
||||
bool forced = file.Contains(".forced."); |
||||
bool sdh = file.Contains(".sdh.") || file.Contains(".cc."); |
||||
|
||||
string language = fileName |
||||
.Replace($"{withoutExtension}.", string.Empty)[..3] |
||||
.Replace(".", string.Empty); |
||||
|
||||
Option<CultureInfo> maybeCulture = languageCodes.Find( |
||||
ci => ci.TwoLetterISOLanguageName == language || ci.ThreeLetterISOLanguageName == language); |
||||
|
||||
foreach (CultureInfo culture in maybeCulture) |
||||
{ |
||||
_logger.LogDebug("Located {Attribute} at {Path}", "External Subtitles", file); |
||||
|
||||
result.Add( |
||||
new Subtitle |
||||
{ |
||||
SubtitleKind = SubtitleKind.Sidecar, |
||||
Codec = codec, |
||||
Default = false, |
||||
Forced = forced, |
||||
SDH = sdh, |
||||
Language = culture.ThreeLetterISOLanguageName, |
||||
Path = saveFullPath ? file : Path.GetFileName(file), |
||||
DateAdded = DateTime.UtcNow, |
||||
DateUpdated = _localFileSystem.GetLastWriteTime(file) |
||||
}); |
||||
} |
||||
} |
||||
|
||||
|
||||
return result; |
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,26 @@
@@ -0,0 +1,26 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations; |
||||
|
||||
#nullable disable |
||||
|
||||
namespace ErsatzTV.Infrastructure.Migrations |
||||
{ |
||||
public partial class Add_SubtitleSDH : Migration |
||||
{ |
||||
protected override void Up(MigrationBuilder migrationBuilder) |
||||
{ |
||||
migrationBuilder.AddColumn<bool>( |
||||
name: "SDH", |
||||
table: "Subtitle", |
||||
type: "INTEGER", |
||||
nullable: false, |
||||
defaultValue: false); |
||||
} |
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder) |
||||
{ |
||||
migrationBuilder.DropColumn( |
||||
name: "SDH", |
||||
table: "Subtitle"); |
||||
} |
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,25 @@
@@ -0,0 +1,25 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations; |
||||
|
||||
#nullable disable |
||||
|
||||
namespace ErsatzTV.Infrastructure.Migrations |
||||
{ |
||||
public partial class Add_SubtitleTitle : Migration |
||||
{ |
||||
protected override void Up(MigrationBuilder migrationBuilder) |
||||
{ |
||||
migrationBuilder.AddColumn<string>( |
||||
name: "Title", |
||||
table: "Subtitle", |
||||
type: "TEXT", |
||||
nullable: true); |
||||
} |
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder) |
||||
{ |
||||
migrationBuilder.DropColumn( |
||||
name: "Title", |
||||
table: "Subtitle"); |
||||
} |
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,23 @@
@@ -0,0 +1,23 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations; |
||||
|
||||
#nullable disable |
||||
|
||||
namespace ErsatzTV.Infrastructure.Migrations |
||||
{ |
||||
public partial class Reset_AllStatistics20220420 : Migration |
||||
{ |
||||
protected override void Up(MigrationBuilder migrationBuilder) |
||||
{ |
||||
// reset all local libraries
|
||||
|
||||
migrationBuilder.Sql("UPDATE MediaVersion SET DateUpdated = '0001-01-01 00:00:00' WHERE SongId IS NULL"); |
||||
migrationBuilder.Sql("UPDATE LibraryFolder SET Etag = NULL WHERE Id IN (SELECT LF.Id FROM LibraryFolder LF INNER JOIN LibraryPath LP on LP.Id = LF.LibraryPathId INNER JOIN Library L on L.Id = LP.LibraryId WHERE MediaKind != 5)"); |
||||
migrationBuilder.Sql("UPDATE LibraryPath SET LastScan = '0001-01-01 00:00:00' WHERE Id IN (SELECT LP.Id FROM LibraryPath LP INNER JOIN Library L on L.Id = LP.LibraryId WHERE MediaKind != 5)"); |
||||
migrationBuilder.Sql("UPDATE Library SET LastScan = '0001-01-01 00:00:00' WHERE MediaKind != 5"); |
||||
} |
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder) |
||||
{ |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue