Try to fix missing generals

KamihamaWeb/Controllers/ResourceController.cs

@@ -8,6 +8,7 @@ using System.Text;
 using System.Threading.Tasks;
 using KamihamaWeb.Interfaces;
 using KamihamaWeb.Models;
+using KamihamaWeb.Services;
 using Marvin.Cache.Headers;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
@@ -90,7 +91,7 @@ namespace KamihamaWeb.Controllers
             {
                 var md5 = qs.Value.Substring(1);
 
-                if (_masterService.EnglishMasterAssets.ContainsKey(url))
+                if (!url.StartsWith("scenario/json/general/") && _masterService.EnglishMasterAssets.ContainsKey(url))
                 {
                     if (_masterService.EnglishMasterAssets[url].Md5 == md5)
                     {
@@ -120,17 +121,17 @@ namespace KamihamaWeb.Controllers
                 }
                 var asset = await _diskCache.Get(url, md5);
 
-                if (asset.Item1 == 404)
+                if (asset.Result == DiskCacheResultType.NotFound)
                 {
                     return new APIResult(404, "asset not found");
                 }
-                else if (asset.Item1 == 500)
+                else if (asset.Result == DiskCacheResultType.Failed)
                 {
                     return new APIResult(503, "internal error fetching asset");
                 }
                 else
                 {
-                    return new FileStreamResult(asset.Item2, "binary/octet-stream");
+                    return new FileStreamResult(asset.Data, "binary/octet-stream");
                 }
             }
             else

KamihamaWeb/Interfaces/IDiskCacheService.cs

@@ -1,6 +1,7 @@
 using System;
 using System.IO;
 using System.Threading.Tasks;
+using KamihamaWeb.Services;
 
 namespace KamihamaWeb.Interfaces
 {
@@ -11,6 +12,7 @@ namespace KamihamaWeb.Interfaces
 
     public interface IDiskCacheSingleton: IDiskCacheService
     {
-        public Task<Tuple<int, Stream>> Get(string cacheItem, string versionMd5);
+        public Task<DiskCacheItem> Get(string cacheItem, string versionMd5, bool forceOrigin = false);
+        public Task<string> Store(string filepath, byte[] storeContents, DiskCacheService.StoreType type);
     }
 }

KamihamaWeb/Interfaces/IMasterListBuilder.cs

@@ -0,0 +1,16 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using KamihamaWeb.Models;
+
+namespace KamihamaWeb.Interfaces
+{
+    public interface IMasterListBuilder
+    {
+        Task<Dictionary<string, GamedataAsset>> GenerateEnglishAssetList();
+
+        public GamedataAsset GetFileInformation(string file);
+
+        public Task<GamedataAsset> BuildScenarioGeneralJson(GamedataAsset generalAsset,
+            Dictionary<string, GamedataAsset> englishAssets);
+    }
+}

KamihamaWeb/Interfaces/IRestSharpClient.cs

@@ -2,6 +2,7 @@
 using System.IO;
 using System.Threading.Tasks;
 using KamihamaWeb.Models;
+using KamihamaWeb.Services;
 
 namespace KamihamaWeb.Interfaces
 {
@@ -13,7 +14,7 @@ namespace KamihamaWeb.Interfaces
     public interface IRestSharpTransient : IRestSharpClient
     {
         Task<T> GetMasterJson<T>(string masterJsonEndpoint);
-        Task<Tuple<int, Stream>> FetchAsset(string item);
+        Task<DiskCacheItem> FetchAsset(string item);
         Task<string> GetAdditionalJson(string item);
     }
 }

KamihamaWeb/Models/GamedataJsonModel.cs

@@ -51,7 +51,8 @@ namespace KamihamaWeb.Models
     public enum AssetSourceType
     {
         Local,
-        Remote
+        Remote,
+        GeneralScript
     }
 
 }

KamihamaWeb/Models/ScenarioGeneral.cs

@@ -0,0 +1,17 @@
+using System.Collections;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace KamihamaWeb.Models
+{
+    public class ScenarioGeneral
+    {
+
+            [JsonProperty("story")]
+            public Dictionary<string, dynamic> story { get; set; }
+
+            [JsonProperty("version")]
+            public int version { get; set; }
+        
+    }
+}

KamihamaWeb/Services/DiskCacheService.cs

@@ -26,18 +26,42 @@ namespace KamihamaWeb.Services
             }
 
             CacheDirectory = config["MagiRecoServer:CacheDirectory"];
+            ScenarioCacheDirectory = $"{config["MagiRecoServer:CacheDirectory"]}/scenario/json/general/";
             Directory.CreateDirectory(CacheDirectory);
+            Directory.CreateDirectory(ScenarioCacheDirectory);
         }
 
         public Guid Guid { get; }
         private IRestSharpTransient Rest { get; set; }
         private string CacheDirectory { get; set; } = "";
+        private string ScenarioCacheDirectory { get; set; } = "";
 
-        public async Task<Tuple<int, Stream>> Get(string cacheItem, string versionMd5)
+        public async Task<DiskCacheItem> Get(string cacheItem, string versionMd5, bool forceOrigin = false)
         {
             var filename = CryptUtil.CalculateSha256(cacheItem + "?" + versionMd5);
 
             var filePath = Path.Combine(CacheDirectory, filename);
+
+            if (!forceOrigin && cacheItem.StartsWith("scenario/json/general"))
+            {
+                var generalJson = Path.Combine(CacheDirectory, cacheItem + versionMd5);
+                if (File.Exists(generalJson))
+                {
+                    return new DiskCacheItem()
+                    {
+                        Data = File.Open(generalJson, FileMode.Open, FileAccess.Read, FileShare.Read),
+                        Result = DiskCacheResultType.Success
+                    };
+                }
+                else
+                {
+                    Log.Warning($"Cache item {generalJson} not found!");
+                    return new DiskCacheItem()
+                    {
+                        Result = DiskCacheResultType.Failed
+                    };
+                }
+            }
             if (File.Exists(filePath))
             {
                 Log.Debug($"Loading {cacheItem} from disk ({filePath}).");
@@ -56,7 +80,10 @@ namespace KamihamaWeb.Services
                         }
                         else
                         {
-                            return new Tuple<int, Stream>(0, stream);
+                            return new DiskCacheItem()
+                            {
+                                Data = stream
+                            };
                         }
                     }
                     catch (IOException) // File in use, wait
@@ -66,14 +93,39 @@ namespace KamihamaWeb.Services
                     }
                 }
                 Log.Warning($"Max loops exceeded in DiskCacheService.Get() for {cacheItem}.");
-                return new Tuple<int, Stream>(500, null);
+                return new DiskCacheItem()
+                {
+                    Result = DiskCacheResultType.Failed
+                };
             }
             Log.Information($"Fetching {cacheItem}.");
             
             return await FastFetch(cacheItem, filePath, versionMd5);
         }
 
-        private async Task<Tuple<int, Stream>> FastFetch(string item, string path, string md5)
+        public async Task<string> Store(string filepath, byte[] storeContents, StoreType type)
+        {
+            string storePath;
+            switch (type)
+            {
+                case StoreType.ScenarioGeneral:
+                    var md5 = CryptUtil.CalculateMd5Bytes(storeContents);
+                    storePath = Path.Combine(CacheDirectory, filepath + md5);
+                    await File.WriteAllBytesAsync(storePath, storeContents);
+                    break;
+                default:
+                    throw new Exception("Invalid StoreType.");
+            }
+
+            return storePath;
+        }
+
+        public enum StoreType
+        {
+            ScenarioGeneral
+        }
+
+        private async Task<DiskCacheItem> FastFetch(string item, string path, string md5)
         {
             var maxLoops = 5;
             var deleteFlag = false;
@@ -89,19 +141,23 @@ namespace KamihamaWeb.Services
 
                         if (file.Length > 0)
                         {
-                            return new Tuple<int, Stream>(0, file);
+                            return new DiskCacheItem()
+                            {
+                                Data = file,
+                                Result = DiskCacheResultType.Success
+                            };
                         }
 
                         var stream = await Rest.FetchAsset(item + $"?{md5}");
 
-                        if (stream.Item2 == null)
+                        if (stream.Data == null)
                         {
                             deleteFlag = true;
                             return stream;
                         }
 
-                        ((MemoryStream) stream.Item2).WriteTo(file);
+                        ((MemoryStream) stream.Data).WriteTo(file);
-                        stream.Item2.Seek(0, SeekOrigin.Begin);
+                        stream.Data.Seek(0, SeekOrigin.Begin);
                         return stream;
                     }
                     catch (Exception)
@@ -126,9 +182,25 @@ namespace KamihamaWeb.Services
             }
             Log.Warning($"Max loops exceeded in DiskCacheService.FastFetch() for {item}.");
             File.Delete(path);
-            return new Tuple<int, Stream>(500, null);
+            return new DiskCacheItem()
+            {
+                Result = DiskCacheResultType.Failed
+            };
         }
 
 
     }
+
+    public class DiskCacheItem
+    {
+        public Stream Data { get; set; } = null;
+        public DiskCacheResultType Result { get; set; } = DiskCacheResultType.Success;
+    }
+
+    public enum DiskCacheResultType
+    {
+        Success = 0,
+        Failed = 500,
+        NotFound = 404
+    }
 }

KamihamaWeb/Services/MasterListBuilder.cs

@@ -2,27 +2,41 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Security.Cryptography;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using KamihamaWeb.Interfaces;
 using KamihamaWeb.Models;
+using KamihamaWeb.Util;
+using Microsoft.Extensions.Caching.Distributed;
+using Microsoft.Extensions.Caching.StackExchangeRedis;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using Serilog;
+using StackExchange.Redis;
 
-namespace KamihamaWeb.Util
+namespace KamihamaWeb.Services
 {
-    public class MasterListBuilder
+    public class MasterListBuilder: IMasterListBuilder
     {
-        private Regex multiPartRegex = new Regex(@"\.a[a-z]{2}"); // A bit crude
+        private readonly Regex _multiPartRegex = new Regex(@"\.a[a-z]{2}"); // A bit crude
-        public MasterListBuilder()
+
+        private IDiskCacheSingleton _disk;
+        private IDatabase _cache;
+
+        public MasterListBuilder(IDiskCacheSingleton disk, IDistributedCache cache)
         {
             BasePathLength =
                 @"MagiRecoStatic/magica/resource/download/asset/master/resource/".Length;
+
+            _cache = ((RedisCache)cache).GetConnection().GetDatabase();
+            _disk = disk;
         }
 
         private int BasePathLength { get; }
 
+        private string StaticDirectory { get; set; } = "MagiRecoStatic/";
+
         public async Task<Dictionary<string, GamedataAsset>> GenerateEnglishAssetList()
         {
             if (File.Exists("en_cache.json"))
@@ -48,9 +62,9 @@ namespace KamihamaWeb.Util
 
 
 
-            if (Directory.Exists("MagiRecoStatic/"))
+            if (Directory.Exists(StaticDirectory))
             {
-                List<string> files = Directory.GetFiles("MagiRecoStatic/magica/resource/download/asset/master",
+                List<string> files = Directory.GetFiles(Path.Combine(StaticDirectory, "magica/resource/download/asset/master"),
                     "*.*",
                     SearchOption.AllDirectories).ToList();
 
@@ -93,7 +107,7 @@ namespace KamihamaWeb.Util
                 // Remove multi-part files
                 foreach (var asset in englishAssets)
                 {
-                    if (multiPartRegex.IsMatch(asset.Value.Path.Substring(asset.Value.Path.Length - 4)))
+                    if (_multiPartRegex.IsMatch(asset.Value.Path.Substring(asset.Value.Path.Length - 4)))
                     {
                         Log.Debug($"Removing duplicate asset {asset.Key}");
                         englishAssets.Remove(asset.Value.Path);
@@ -101,7 +115,7 @@ namespace KamihamaWeb.Util
                     else if (asset.Key.StartsWith("image_native/mini/") 
                              || asset.Key.StartsWith("image_native/live2d/")
                              //|| asset.Key.StartsWith("image_native/scene/gacha")
-                             || asset.Key.StartsWith("scenario/json/general/")
+                             //|| asset.Key.StartsWith("scenario/json/general/")
                              || asset.Key.StartsWith("scenario/json/oneShot/")
                              || asset.Key.StartsWith("image_native/scene/event/")
                              || asset.Key.StartsWith("image_native/scene/emotion/")
@@ -163,5 +177,76 @@ namespace KamihamaWeb.Util
 
             return asset;
         }
+
+        public async Task<GamedataAsset> BuildScenarioGeneralJson(GamedataAsset generalAsset, Dictionary<string, GamedataAsset> englishAssets)
+        {
+            Log.Information($"Building scenario JSON for {generalAsset.Path}.");
+            if (!englishAssets.ContainsKey(generalAsset.Path)) // No english to replace with (yet)
+            {
+                Log.Warning($"No english asset for {generalAsset.Path}! This should be caught earlier!");
+                return generalAsset;
+            }
+
+            // Fetch JP asset
+            DiskCacheItem jpAsset = await _disk.Get(generalAsset.Path, generalAsset.Md5, true);
+
+            if (jpAsset.Result != DiskCacheResultType.Success)
+            {
+                Log.Warning("An error has occurred fetching a general scenario asset.");
+                return generalAsset;
+            }
+
+            // Merge assets
+            var enPath = Path.Combine(StaticDirectory, "magica/resource/download/asset/master/resource",
+                generalAsset.Path);
+
+            if (!File.Exists(enPath))
+            {
+                Log.Warning($"File {enPath} does not exist!");
+                return generalAsset;
+            }
+            var mergedAsset = MergeScenarioGeneral(
+                Encoding.UTF8.GetString(
+                    CryptUtil.ReadFully(jpAsset.Data)
+                    ),
+                await File.ReadAllTextAsync(enPath)
+                );
+
+            var storeFilePath = await _disk.Store(generalAsset.Path, Encoding.UTF8.GetBytes(mergedAsset),
+                DiskCacheService.StoreType.ScenarioGeneral);
+
+            generalAsset.AssetSource = AssetSourceType.GeneralScript;
+            generalAsset.Md5 = CryptUtil.CalculateMd5File(storeFilePath);
+            generalAsset.FileList[0].Size = new FileInfo(storeFilePath).Length;
+            return generalAsset;
+        }
+
+        private string MergeScenarioGeneral(string jp, string en)
+        {
+            if (jp == en)
+            {
+                return jp;
+            }
+
+            var jp_json = JsonConvert.DeserializeObject<ScenarioGeneral>(jp);
+            var en_json = JsonConvert.DeserializeObject<ScenarioGeneral>(en);
+
+            foreach (var item in jp_json.story)
+            {
+                if (!en_json.story.ContainsKey(item.Key))
+                {
+                    Log.Information($"Adding key {item.Key}.");
+                    en_json.story[item.Key] = item.Value;
+                }
+            }
+
+            /*jp_json.Merge(en_json, new JsonMergeSettings()
+            {
+                MergeArrayHandling = MergeArrayHandling.Union,
+                MergeNullValueHandling = MergeNullValueHandling.Ignore,
+            });*/
+
+            return JsonConvert.SerializeObject(en_json);
+        }
     }
 }

KamihamaWeb/Services/MasterService.cs

@@ -21,12 +21,14 @@ namespace KamihamaWeb.Services
         private IDatabase _cache { get; set; }
         private IConfiguration _config { get; set; }
         private IRestSharpTransient _rest { get; set; }
+        private IMasterListBuilder _builder { get; set; }
 
         public MasterService(
             IDistributedCache cache, 
             IConfiguration config,
-            IRestSharpTransient rest
+            IRestSharpTransient rest,
-            ) : this(Guid.NewGuid(), cache, config, rest)
+            IMasterListBuilder builder
+            ) : this(Guid.NewGuid(), cache, config, rest, builder)
         {
         }
 
@@ -34,13 +36,15 @@ namespace KamihamaWeb.Services
             Guid guid, 
             IDistributedCache cache, 
             IConfiguration config,
-            IRestSharpTransient rest
+            IRestSharpTransient rest,
+            IMasterListBuilder builder
             )
         {
             Guid = guid;
             _config = config;
             _cache = ((RedisCache) cache).GetConnection().GetDatabase();
             _rest = rest;
+            _builder = builder;
             Task.Run(Initialize);
         }
         public Guid Guid { get; set; }
@@ -79,9 +83,13 @@ namespace KamihamaWeb.Services
 
             Log.Information("Configuring master list...");
 
+            var postProcessingGeneralScenario = new Dictionary<string, GamedataAsset>();
+
             long counterReplace = 0;
             long counterSkip = 0;
             long counterNew = 0;
+            long counterPost = 0;
+
             foreach (var assetType in workGamedataAssets)
             {
                 var readyAssets = new Dictionary<string, GamedataAsset>();
@@ -90,7 +98,12 @@ namespace KamihamaWeb.Services
                     // Replace with english assets as needed
                     if (EnglishMasterAssets.ContainsKey(asset.Path))
                     {
-                        if (EnglishMasterAssets[asset.Path].Md5 != asset.Md5)
+                        if (asset.Path.StartsWith("scenario/json/general/"))
+                        {
+                            postProcessingGeneralScenario.Add(asset.Path, asset);
+                            counterPost++;
+                        }
+                        else if (EnglishMasterAssets[asset.Path].Md5 != asset.Md5)
                         {
                             //Log.Debug($"Replacing Japanese asset with English asset for {asset.Path}.");
                             readyAssets.Add(asset.Path, EnglishMasterAssets[asset.Path]);
@@ -112,7 +125,7 @@ namespace KamihamaWeb.Services
                 }
                 GamedataAssets.Add(assetType.Key, readyAssets);
             }
-            Log.Information($"Finished setting up. {counterReplace} replaced assets, {counterSkip} duplicate assets, {counterNew} new assets.");
+            Log.Information($"Finished setting up. {counterReplace} replaced assets, {counterSkip} duplicate assets, {counterNew} new assets, {counterPost} assets for post processing.");
 
             // Add scripts
             foreach (var asset in EnglishMasterAssets)
@@ -128,6 +141,14 @@ namespace KamihamaWeb.Services
                     });
                 }
             }
+
+            // Post processing
+            foreach (var asset in postProcessingGeneralScenario)
+            {
+                var builtJson = await _builder.BuildScenarioGeneralJson(asset.Value, EnglishMasterAssets);
+
+                GamedataAssets["asset_main"].Add(builtJson.Path, builtJson);
+            }
             IsReady = true;
             return true;
         }
@@ -155,8 +176,8 @@ namespace KamihamaWeb.Services
                 await Task.Delay(delay);
                 delay *= 2;
             }
-            var builder= new MasterListBuilder();
+
-            var lists = await builder.GenerateEnglishAssetList();
+            var lists = await _builder.GenerateEnglishAssetList();
 
             if (lists != null)
             {
@@ -189,11 +210,6 @@ namespace KamihamaWeb.Services
             }
         }
 
-        //public async Task<FileStream> ProvideFile(string filePath)
-        //{
-
-//        }
-
         public async Task<string> ProvideJson(string which)
         {
             if (which == "asset_config")

KamihamaWeb/Services/RestSharpClient.cs

@@ -72,7 +72,7 @@ namespace KamihamaWeb.Services
             return result.Content;
         }
 
-        public async Task<Tuple<int, Stream>> FetchAsset(string item)
+        public async Task<DiskCacheItem> FetchAsset(string item)
         {
             var request = new RestRequest("resource/" + item, Method.GET);
             
@@ -82,17 +82,27 @@ namespace KamihamaWeb.Services
 
                 if (response.StatusCode == HttpStatusCode.NotFound || response.ContentType == "text/html")
                 {
-                    return new Tuple<int, Stream>(404, null);
+                    return new DiskCacheItem()
+                    {
+                        Result = DiskCacheResultType.NotFound
+                    };
                 }
 
                 var stream = new MemoryStream(response.RawBytes);
-                return new Tuple<int, Stream>(0, stream);
+                return new DiskCacheItem()
+                {
+                    Result = DiskCacheResultType.Success,
+                    Data = stream
+                };
             }
             catch (WebException ex)
             {
                 Log.Warning($"Web exception thrown: Status code {ex.Status}, {ex.ToString()}");
             }
-            return new Tuple<int, Stream>(500, null);
+            return new DiskCacheItem()
+            {
+                Result = DiskCacheResultType.Failed
+            };
         }
     }
 }

KamihamaWeb/Startup.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Threading.Tasks;
 using KamihamaWeb.Interfaces;
 using KamihamaWeb.Services;
+using KamihamaWeb.Util;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.HttpsPolicy;
@@ -46,6 +47,7 @@ namespace KamihamaWeb
             services.AddTransient<IRestSharpTransient, RestSharpClient>();
             services.AddSingleton<IMasterSingleton, MasterService>();
             services.AddSingleton<IDiskCacheSingleton, DiskCacheService>();
+            services.AddTransient<IMasterListBuilder, MasterListBuilder>();
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.

KamihamaWeb/Util/CryptUtil.cs

@@ -32,5 +32,26 @@ namespace KamihamaWeb.Util
 
             return sb.ToString();
         }
+
+        public static byte[] ReadFully(Stream input)
+        {
+            using MemoryStream ms = new MemoryStream();
+            input.CopyTo(ms);
+            return ms.ToArray();
+        }
+
+        public static object CalculateMd5Bytes(byte[] storeContents)
+        {
+            using var md5 = MD5.Create();
+            using var stream = new MemoryStream(storeContents);
+
+            StringBuilder sb = new StringBuilder();
+            foreach (byte b in md5.ComputeHash(stream))
+            {
+                sb.Append(b.ToString("x2"));
+            }
+
+            return sb.ToString();
+        }
     }
 }