diff --git a/KamihamaWeb/Controllers/ResourceController.cs b/KamihamaWeb/Controllers/ResourceController.cs index 7c62084..1ea5a17 100644 --- a/KamihamaWeb/Controllers/ResourceController.cs +++ b/KamihamaWeb/Controllers/ResourceController.cs @@ -8,6 +8,7 @@ using System.Text; using System.Threading.Tasks; using KamihamaWeb.Interfaces; using KamihamaWeb.Models; +using KamihamaWeb.Services; using Marvin.Cache.Headers; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; @@ -90,7 +91,7 @@ namespace KamihamaWeb.Controllers { var md5 = qs.Value.Substring(1); - if (_masterService.EnglishMasterAssets.ContainsKey(url)) + if (!url.StartsWith("scenario/json/general/") && _masterService.EnglishMasterAssets.ContainsKey(url)) { if (_masterService.EnglishMasterAssets[url].Md5 == md5) { @@ -120,17 +121,17 @@ namespace KamihamaWeb.Controllers } var asset = await _diskCache.Get(url, md5); - if (asset.Item1 == 404) + if (asset.Result == DiskCacheResultType.NotFound) { return new APIResult(404, "asset not found"); } - else if (asset.Item1 == 500) + else if (asset.Result == DiskCacheResultType.Failed) { return new APIResult(503, "internal error fetching asset"); } else { - return new FileStreamResult(asset.Item2, "binary/octet-stream"); + return new FileStreamResult(asset.Data, "binary/octet-stream"); } } else diff --git a/KamihamaWeb/Interfaces/IDiskCacheService.cs b/KamihamaWeb/Interfaces/IDiskCacheService.cs index d94c6a3..113fff8 100644 --- a/KamihamaWeb/Interfaces/IDiskCacheService.cs +++ b/KamihamaWeb/Interfaces/IDiskCacheService.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Threading.Tasks; +using KamihamaWeb.Services; namespace KamihamaWeb.Interfaces { @@ -11,6 +12,7 @@ namespace KamihamaWeb.Interfaces public interface IDiskCacheSingleton: IDiskCacheService { - public Task> Get(string cacheItem, string versionMd5); + public Task Get(string cacheItem, string versionMd5, bool forceOrigin = false); + public Task Store(string filepath, byte[] storeContents, DiskCacheService.StoreType type); } } \ No newline at end of file diff --git a/KamihamaWeb/Interfaces/IMasterListBuilder.cs b/KamihamaWeb/Interfaces/IMasterListBuilder.cs new file mode 100644 index 0000000..aa2bde3 --- /dev/null +++ b/KamihamaWeb/Interfaces/IMasterListBuilder.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using KamihamaWeb.Models; + +namespace KamihamaWeb.Interfaces +{ + public interface IMasterListBuilder + { + Task> GenerateEnglishAssetList(); + + public GamedataAsset GetFileInformation(string file); + + public Task BuildScenarioGeneralJson(GamedataAsset generalAsset, + Dictionary englishAssets); + } +} \ No newline at end of file diff --git a/KamihamaWeb/Interfaces/IRestSharpClient.cs b/KamihamaWeb/Interfaces/IRestSharpClient.cs index a2796fc..da30fc3 100644 --- a/KamihamaWeb/Interfaces/IRestSharpClient.cs +++ b/KamihamaWeb/Interfaces/IRestSharpClient.cs @@ -2,6 +2,7 @@ using System.IO; using System.Threading.Tasks; using KamihamaWeb.Models; +using KamihamaWeb.Services; namespace KamihamaWeb.Interfaces { @@ -13,7 +14,7 @@ namespace KamihamaWeb.Interfaces public interface IRestSharpTransient : IRestSharpClient { Task GetMasterJson(string masterJsonEndpoint); - Task> FetchAsset(string item); + Task FetchAsset(string item); Task GetAdditionalJson(string item); } } \ No newline at end of file diff --git a/KamihamaWeb/Models/GamedataJsonModel.cs b/KamihamaWeb/Models/GamedataJsonModel.cs index 2a431df..0e15b05 100644 --- a/KamihamaWeb/Models/GamedataJsonModel.cs +++ b/KamihamaWeb/Models/GamedataJsonModel.cs @@ -51,7 +51,8 @@ namespace KamihamaWeb.Models public enum AssetSourceType { Local, - Remote + Remote, + GeneralScript } } \ No newline at end of file diff --git a/KamihamaWeb/Models/ScenarioGeneral.cs b/KamihamaWeb/Models/ScenarioGeneral.cs new file mode 100644 index 0000000..5c57bfa --- /dev/null +++ b/KamihamaWeb/Models/ScenarioGeneral.cs @@ -0,0 +1,17 @@ +using System.Collections; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace KamihamaWeb.Models +{ + public class ScenarioGeneral + { + + [JsonProperty("story")] + public Dictionary story { get; set; } + + [JsonProperty("version")] + public int version { get; set; } + + } +} \ No newline at end of file diff --git a/KamihamaWeb/Services/DiskCacheService.cs b/KamihamaWeb/Services/DiskCacheService.cs index a53fe57..adb42a0 100644 --- a/KamihamaWeb/Services/DiskCacheService.cs +++ b/KamihamaWeb/Services/DiskCacheService.cs @@ -26,18 +26,42 @@ namespace KamihamaWeb.Services } CacheDirectory = config["MagiRecoServer:CacheDirectory"]; + ScenarioCacheDirectory = $"{config["MagiRecoServer:CacheDirectory"]}/scenario/json/general/"; Directory.CreateDirectory(CacheDirectory); + Directory.CreateDirectory(ScenarioCacheDirectory); } public Guid Guid { get; } private IRestSharpTransient Rest { get; set; } private string CacheDirectory { get; set; } = ""; + private string ScenarioCacheDirectory { get; set; } = ""; - public async Task> Get(string cacheItem, string versionMd5) + public async Task Get(string cacheItem, string versionMd5, bool forceOrigin = false) { var filename = CryptUtil.CalculateSha256(cacheItem + "?" + versionMd5); var filePath = Path.Combine(CacheDirectory, filename); + + if (!forceOrigin && cacheItem.StartsWith("scenario/json/general")) + { + var generalJson = Path.Combine(CacheDirectory, cacheItem + versionMd5); + if (File.Exists(generalJson)) + { + return new DiskCacheItem() + { + Data = File.Open(generalJson, FileMode.Open, FileAccess.Read, FileShare.Read), + Result = DiskCacheResultType.Success + }; + } + else + { + Log.Warning($"Cache item {generalJson} not found!"); + return new DiskCacheItem() + { + Result = DiskCacheResultType.Failed + }; + } + } if (File.Exists(filePath)) { Log.Debug($"Loading {cacheItem} from disk ({filePath})."); @@ -56,7 +80,10 @@ namespace KamihamaWeb.Services } else { - return new Tuple(0, stream); + return new DiskCacheItem() + { + Data = stream + }; } } catch (IOException) // File in use, wait @@ -66,14 +93,39 @@ namespace KamihamaWeb.Services } } Log.Warning($"Max loops exceeded in DiskCacheService.Get() for {cacheItem}."); - return new Tuple(500, null); + return new DiskCacheItem() + { + Result = DiskCacheResultType.Failed + }; } Log.Information($"Fetching {cacheItem}."); return await FastFetch(cacheItem, filePath, versionMd5); } - private async Task> FastFetch(string item, string path, string md5) + public async Task Store(string filepath, byte[] storeContents, StoreType type) + { + string storePath; + switch (type) + { + case StoreType.ScenarioGeneral: + var md5 = CryptUtil.CalculateMd5Bytes(storeContents); + storePath = Path.Combine(CacheDirectory, filepath + md5); + await File.WriteAllBytesAsync(storePath, storeContents); + break; + default: + throw new Exception("Invalid StoreType."); + } + + return storePath; + } + + public enum StoreType + { + ScenarioGeneral + } + + private async Task FastFetch(string item, string path, string md5) { var maxLoops = 5; var deleteFlag = false; @@ -89,19 +141,23 @@ namespace KamihamaWeb.Services if (file.Length > 0) { - return new Tuple(0, file); + return new DiskCacheItem() + { + Data = file, + Result = DiskCacheResultType.Success + }; } var stream = await Rest.FetchAsset(item + $"?{md5}"); - if (stream.Item2 == null) + if (stream.Data == null) { deleteFlag = true; return stream; } - ((MemoryStream) stream.Item2).WriteTo(file); - stream.Item2.Seek(0, SeekOrigin.Begin); + ((MemoryStream) stream.Data).WriteTo(file); + stream.Data.Seek(0, SeekOrigin.Begin); return stream; } catch (Exception) @@ -126,9 +182,25 @@ namespace KamihamaWeb.Services } Log.Warning($"Max loops exceeded in DiskCacheService.FastFetch() for {item}."); File.Delete(path); - return new Tuple(500, null); + return new DiskCacheItem() + { + Result = DiskCacheResultType.Failed + }; } } + + public class DiskCacheItem + { + public Stream Data { get; set; } = null; + public DiskCacheResultType Result { get; set; } = DiskCacheResultType.Success; + } + + public enum DiskCacheResultType + { + Success = 0, + Failed = 500, + NotFound = 404 + } } \ No newline at end of file diff --git a/KamihamaWeb/Util/MasterListBuilder.cs b/KamihamaWeb/Services/MasterListBuilder.cs similarity index 60% rename from KamihamaWeb/Util/MasterListBuilder.cs rename to KamihamaWeb/Services/MasterListBuilder.cs index 1400f35..958d4a1 100644 --- a/KamihamaWeb/Util/MasterListBuilder.cs +++ b/KamihamaWeb/Services/MasterListBuilder.cs @@ -2,27 +2,41 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; +using KamihamaWeb.Interfaces; using KamihamaWeb.Models; +using KamihamaWeb.Util; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.Caching.StackExchangeRedis; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using Serilog; +using StackExchange.Redis; -namespace KamihamaWeb.Util +namespace KamihamaWeb.Services { - public class MasterListBuilder + public class MasterListBuilder: IMasterListBuilder { - private Regex multiPartRegex = new Regex(@"\.a[a-z]{2}"); // A bit crude - public MasterListBuilder() + private readonly Regex _multiPartRegex = new Regex(@"\.a[a-z]{2}"); // A bit crude + + private IDiskCacheSingleton _disk; + private IDatabase _cache; + + public MasterListBuilder(IDiskCacheSingleton disk, IDistributedCache cache) { BasePathLength = @"MagiRecoStatic/magica/resource/download/asset/master/resource/".Length; + + _cache = ((RedisCache)cache).GetConnection().GetDatabase(); + _disk = disk; } private int BasePathLength { get; } + private string StaticDirectory { get; set; } = "MagiRecoStatic/"; + public async Task> GenerateEnglishAssetList() { if (File.Exists("en_cache.json")) @@ -48,9 +62,9 @@ namespace KamihamaWeb.Util - if (Directory.Exists("MagiRecoStatic/")) + if (Directory.Exists(StaticDirectory)) { - List files = Directory.GetFiles("MagiRecoStatic/magica/resource/download/asset/master", + List files = Directory.GetFiles(Path.Combine(StaticDirectory, "magica/resource/download/asset/master"), "*.*", SearchOption.AllDirectories).ToList(); @@ -93,7 +107,7 @@ namespace KamihamaWeb.Util // Remove multi-part files foreach (var asset in englishAssets) { - if (multiPartRegex.IsMatch(asset.Value.Path.Substring(asset.Value.Path.Length - 4))) + if (_multiPartRegex.IsMatch(asset.Value.Path.Substring(asset.Value.Path.Length - 4))) { Log.Debug($"Removing duplicate asset {asset.Key}"); englishAssets.Remove(asset.Value.Path); @@ -101,7 +115,7 @@ namespace KamihamaWeb.Util else if (asset.Key.StartsWith("image_native/mini/") || asset.Key.StartsWith("image_native/live2d/") //|| asset.Key.StartsWith("image_native/scene/gacha") - || asset.Key.StartsWith("scenario/json/general/") + //|| asset.Key.StartsWith("scenario/json/general/") || asset.Key.StartsWith("scenario/json/oneShot/") || asset.Key.StartsWith("image_native/scene/event/") || asset.Key.StartsWith("image_native/scene/emotion/") @@ -163,5 +177,76 @@ namespace KamihamaWeb.Util return asset; } + + public async Task BuildScenarioGeneralJson(GamedataAsset generalAsset, Dictionary englishAssets) + { + Log.Information($"Building scenario JSON for {generalAsset.Path}."); + if (!englishAssets.ContainsKey(generalAsset.Path)) // No english to replace with (yet) + { + Log.Warning($"No english asset for {generalAsset.Path}! This should be caught earlier!"); + return generalAsset; + } + + // Fetch JP asset + DiskCacheItem jpAsset = await _disk.Get(generalAsset.Path, generalAsset.Md5, true); + + if (jpAsset.Result != DiskCacheResultType.Success) + { + Log.Warning("An error has occurred fetching a general scenario asset."); + return generalAsset; + } + + // Merge assets + var enPath = Path.Combine(StaticDirectory, "magica/resource/download/asset/master/resource", + generalAsset.Path); + + if (!File.Exists(enPath)) + { + Log.Warning($"File {enPath} does not exist!"); + return generalAsset; + } + var mergedAsset = MergeScenarioGeneral( + Encoding.UTF8.GetString( + CryptUtil.ReadFully(jpAsset.Data) + ), + await File.ReadAllTextAsync(enPath) + ); + + var storeFilePath = await _disk.Store(generalAsset.Path, Encoding.UTF8.GetBytes(mergedAsset), + DiskCacheService.StoreType.ScenarioGeneral); + + generalAsset.AssetSource = AssetSourceType.GeneralScript; + generalAsset.Md5 = CryptUtil.CalculateMd5File(storeFilePath); + generalAsset.FileList[0].Size = new FileInfo(storeFilePath).Length; + return generalAsset; + } + + private string MergeScenarioGeneral(string jp, string en) + { + if (jp == en) + { + return jp; + } + + var jp_json = JsonConvert.DeserializeObject(jp); + var en_json = JsonConvert.DeserializeObject(en); + + foreach (var item in jp_json.story) + { + if (!en_json.story.ContainsKey(item.Key)) + { + Log.Information($"Adding key {item.Key}."); + en_json.story[item.Key] = item.Value; + } + } + + /*jp_json.Merge(en_json, new JsonMergeSettings() + { + MergeArrayHandling = MergeArrayHandling.Union, + MergeNullValueHandling = MergeNullValueHandling.Ignore, + });*/ + + return JsonConvert.SerializeObject(en_json); + } } } \ No newline at end of file diff --git a/KamihamaWeb/Services/MasterService.cs b/KamihamaWeb/Services/MasterService.cs index 8234fbd..b38d24e 100644 --- a/KamihamaWeb/Services/MasterService.cs +++ b/KamihamaWeb/Services/MasterService.cs @@ -21,12 +21,14 @@ namespace KamihamaWeb.Services private IDatabase _cache { get; set; } private IConfiguration _config { get; set; } private IRestSharpTransient _rest { get; set; } + private IMasterListBuilder _builder { get; set; } public MasterService( IDistributedCache cache, IConfiguration config, - IRestSharpTransient rest - ) : this(Guid.NewGuid(), cache, config, rest) + IRestSharpTransient rest, + IMasterListBuilder builder + ) : this(Guid.NewGuid(), cache, config, rest, builder) { } @@ -34,13 +36,15 @@ namespace KamihamaWeb.Services Guid guid, IDistributedCache cache, IConfiguration config, - IRestSharpTransient rest + IRestSharpTransient rest, + IMasterListBuilder builder ) { Guid = guid; _config = config; _cache = ((RedisCache) cache).GetConnection().GetDatabase(); _rest = rest; + _builder = builder; Task.Run(Initialize); } public Guid Guid { get; set; } @@ -79,9 +83,13 @@ namespace KamihamaWeb.Services Log.Information("Configuring master list..."); + var postProcessingGeneralScenario = new Dictionary(); + long counterReplace = 0; long counterSkip = 0; long counterNew = 0; + long counterPost = 0; + foreach (var assetType in workGamedataAssets) { var readyAssets = new Dictionary(); @@ -90,7 +98,12 @@ namespace KamihamaWeb.Services // Replace with english assets as needed if (EnglishMasterAssets.ContainsKey(asset.Path)) { - if (EnglishMasterAssets[asset.Path].Md5 != asset.Md5) + if (asset.Path.StartsWith("scenario/json/general/")) + { + postProcessingGeneralScenario.Add(asset.Path, asset); + counterPost++; + } + else if (EnglishMasterAssets[asset.Path].Md5 != asset.Md5) { //Log.Debug($"Replacing Japanese asset with English asset for {asset.Path}."); readyAssets.Add(asset.Path, EnglishMasterAssets[asset.Path]); @@ -112,7 +125,7 @@ namespace KamihamaWeb.Services } GamedataAssets.Add(assetType.Key, readyAssets); } - Log.Information($"Finished setting up. {counterReplace} replaced assets, {counterSkip} duplicate assets, {counterNew} new assets."); + Log.Information($"Finished setting up. {counterReplace} replaced assets, {counterSkip} duplicate assets, {counterNew} new assets, {counterPost} assets for post processing."); // Add scripts foreach (var asset in EnglishMasterAssets) @@ -128,6 +141,14 @@ namespace KamihamaWeb.Services }); } } + + // Post processing + foreach (var asset in postProcessingGeneralScenario) + { + var builtJson = await _builder.BuildScenarioGeneralJson(asset.Value, EnglishMasterAssets); + + GamedataAssets["asset_main"].Add(builtJson.Path, builtJson); + } IsReady = true; return true; } @@ -155,8 +176,8 @@ namespace KamihamaWeb.Services await Task.Delay(delay); delay *= 2; } - var builder= new MasterListBuilder(); - var lists = await builder.GenerateEnglishAssetList(); + + var lists = await _builder.GenerateEnglishAssetList(); if (lists != null) { @@ -189,11 +210,6 @@ namespace KamihamaWeb.Services } } - //public async Task ProvideFile(string filePath) - //{ - -// } - public async Task ProvideJson(string which) { if (which == "asset_config") diff --git a/KamihamaWeb/Services/RestSharpClient.cs b/KamihamaWeb/Services/RestSharpClient.cs index d24c46c..ce39754 100644 --- a/KamihamaWeb/Services/RestSharpClient.cs +++ b/KamihamaWeb/Services/RestSharpClient.cs @@ -72,7 +72,7 @@ namespace KamihamaWeb.Services return result.Content; } - public async Task> FetchAsset(string item) + public async Task FetchAsset(string item) { var request = new RestRequest("resource/" + item, Method.GET); @@ -82,17 +82,27 @@ namespace KamihamaWeb.Services if (response.StatusCode == HttpStatusCode.NotFound || response.ContentType == "text/html") { - return new Tuple(404, null); + return new DiskCacheItem() + { + Result = DiskCacheResultType.NotFound + }; } var stream = new MemoryStream(response.RawBytes); - return new Tuple(0, stream); + return new DiskCacheItem() + { + Result = DiskCacheResultType.Success, + Data = stream + }; } catch (WebException ex) { Log.Warning($"Web exception thrown: Status code {ex.Status}, {ex.ToString()}"); } - return new Tuple(500, null); + return new DiskCacheItem() + { + Result = DiskCacheResultType.Failed + }; } } } \ No newline at end of file diff --git a/KamihamaWeb/Startup.cs b/KamihamaWeb/Startup.cs index d96a7c5..53a160c 100644 --- a/KamihamaWeb/Startup.cs +++ b/KamihamaWeb/Startup.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Threading.Tasks; using KamihamaWeb.Interfaces; using KamihamaWeb.Services; +using KamihamaWeb.Util; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.HttpsPolicy; @@ -46,6 +47,7 @@ namespace KamihamaWeb services.AddTransient(); services.AddSingleton(); services.AddSingleton(); + services.AddTransient(); } // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. diff --git a/KamihamaWeb/Util/CryptUtil.cs b/KamihamaWeb/Util/CryptUtil.cs index b8df560..f887e5e 100644 --- a/KamihamaWeb/Util/CryptUtil.cs +++ b/KamihamaWeb/Util/CryptUtil.cs @@ -32,5 +32,26 @@ namespace KamihamaWeb.Util return sb.ToString(); } + + public static byte[] ReadFully(Stream input) + { + using MemoryStream ms = new MemoryStream(); + input.CopyTo(ms); + return ms.ToArray(); + } + + public static object CalculateMd5Bytes(byte[] storeContents) + { + using var md5 = MD5.Create(); + using var stream = new MemoryStream(storeContents); + + StringBuilder sb = new StringBuilder(); + foreach (byte b in md5.ComputeHash(stream)) + { + sb.Append(b.ToString("x2")); + } + + return sb.ToString(); + } } } \ No newline at end of file