kamihama
/
kamihama-server
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Try to fix missing generals

This commit is contained in:
Neo 2020-10-24 03:42:09 +01:00
parent 1ad0087a03
commit ceea825fa2
12 changed files with 285 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
KamihamaWeb/Controllers/ResourceController.cs
View File

@ -8,6 +8,7 @@ using System.Text;
using System.Threading.Tasks;
using KamihamaWeb.Interfaces;
using KamihamaWeb.Models;
using KamihamaWeb.Services;
using Marvin.Cache.Headers;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
@ -90,7 +91,7 @@ namespace KamihamaWeb.Controllers
{
var md5 = qs.Value.Substring(1);
if (_masterService.EnglishMasterAssets.ContainsKey(url))
if (!url.StartsWith("scenario/json/general/") && _masterService.EnglishMasterAssets.ContainsKey(url))
{
if (_masterService.EnglishMasterAssets[url].Md5 == md5)
{
@ -120,17 +121,17 @@ namespace KamihamaWeb.Controllers
}
var asset = await _diskCache.Get(url, md5);
if (asset.Item1 == 404)
if (asset.Result == DiskCacheResultType.NotFound)
{
return new APIResult(404, "asset not found");
}
else if (asset.Item1 == 500)
else if (asset.Result == DiskCacheResultType.Failed)
{
return new APIResult(503, "internal error fetching asset");
}
else
{
return new FileStreamResult(asset.Item2, "binary/octet-stream");
return new FileStreamResult(asset.Data, "binary/octet-stream");
}
}
else

4
KamihamaWeb/Interfaces/IDiskCacheService.cs
View File

@ -1,6 +1,7 @@
using System;
using System.IO;
using System.Threading.Tasks;
using KamihamaWeb.Services;
namespace KamihamaWeb.Interfaces
{
@ -11,6 +12,7 @@ namespace KamihamaWeb.Interfaces
public interface IDiskCacheSingleton: IDiskCacheService
{
public Task<Tuple<int, Stream>> Get(string cacheItem, string versionMd5);
public Task<DiskCacheItem> Get(string cacheItem, string versionMd5, bool forceOrigin = false);
public Task<string> Store(string filepath, byte[] storeContents, DiskCacheService.StoreType type);
}
}

16
KamihamaWeb/Interfaces/IMasterListBuilder.cs Normal file
View File

@ -0,0 +1,16 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using KamihamaWeb.Models;
namespace KamihamaWeb.Interfaces
{
public interface IMasterListBuilder
{
Task<Dictionary<string, GamedataAsset>> GenerateEnglishAssetList();
public GamedataAsset GetFileInformation(string file);
public Task<GamedataAsset> BuildScenarioGeneralJson(GamedataAsset generalAsset,
Dictionary<string, GamedataAsset> englishAssets);
}
}

3
KamihamaWeb/Interfaces/IRestSharpClient.cs
View File

@ -2,6 +2,7 @@
using System.IO;
using System.Threading.Tasks;
using KamihamaWeb.Models;
using KamihamaWeb.Services;
namespace KamihamaWeb.Interfaces
{
@ -13,7 +14,7 @@ namespace KamihamaWeb.Interfaces
public interface IRestSharpTransient : IRestSharpClient
{
Task<T> GetMasterJson<T>(string masterJsonEndpoint);
Task<Tuple<int, Stream>> FetchAsset(string item);
Task<DiskCacheItem> FetchAsset(string item);
Task<string> GetAdditionalJson(string item);
}
}

3
KamihamaWeb/Models/GamedataJsonModel.cs
View File

@ -51,7 +51,8 @@ namespace KamihamaWeb.Models
public enum AssetSourceType
{
Local,
Remote
Remote,
GeneralScript
}
}

17
KamihamaWeb/Models/ScenarioGeneral.cs Normal file
View File

@ -0,0 +1,17 @@
using System.Collections;
using System.Collections.Generic;
using Newtonsoft.Json;
namespace KamihamaWeb.Models
{
public class ScenarioGeneral
{
[JsonProperty("story")]
public Dictionary<string, dynamic> story { get; set; }
[JsonProperty("version")]
public int version { get; set; }
}
}

90
KamihamaWeb/Services/DiskCacheService.cs
View File

@ -26,18 +26,42 @@ namespace KamihamaWeb.Services
}
CacheDirectory = config["MagiRecoServer:CacheDirectory"];
ScenarioCacheDirectory = $"{config["MagiRecoServer:CacheDirectory"]}/scenario/json/general/";
Directory.CreateDirectory(CacheDirectory);
Directory.CreateDirectory(ScenarioCacheDirectory);
}
public Guid Guid { get; }
private IRestSharpTransient Rest { get; set; }
private string CacheDirectory { get; set; } = "";
private string ScenarioCacheDirectory { get; set; } = "";
public async Task<Tuple<int, Stream>> Get(string cacheItem, string versionMd5)
public async Task<DiskCacheItem> Get(string cacheItem, string versionMd5, bool forceOrigin = false)
{
var filename = CryptUtil.CalculateSha256(cacheItem + "?" + versionMd5);
var filePath = Path.Combine(CacheDirectory, filename);
if (!forceOrigin && cacheItem.StartsWith("scenario/json/general"))
{
var generalJson = Path.Combine(CacheDirectory, cacheItem + versionMd5);
if (File.Exists(generalJson))
{
return new DiskCacheItem()
{
Data = File.Open(generalJson, FileMode.Open, FileAccess.Read, FileShare.Read),
Result = DiskCacheResultType.Success
};
}
else
{
Log.Warning($"Cache item {generalJson} not found!");
return new DiskCacheItem()
{
Result = DiskCacheResultType.Failed
};
}
}
if (File.Exists(filePath))
{
Log.Debug($"Loading {cacheItem} from disk ({filePath}).");
@ -56,7 +80,10 @@ namespace KamihamaWeb.Services
}
else
{
return new Tuple<int, Stream>(0, stream);
return new DiskCacheItem()
{
Data = stream
};
}
}
catch (IOException) // File in use, wait
@ -66,14 +93,39 @@ namespace KamihamaWeb.Services
}
}
Log.Warning($"Max loops exceeded in DiskCacheService.Get() for {cacheItem}.");
return new Tuple<int, Stream>(500, null);
return new DiskCacheItem()
{
Result = DiskCacheResultType.Failed
};
}
Log.Information($"Fetching {cacheItem}.");
return await FastFetch(cacheItem, filePath, versionMd5);
}
private async Task<Tuple<int, Stream>> FastFetch(string item, string path, string md5)
public async Task<string> Store(string filepath, byte[] storeContents, StoreType type)
{
string storePath;
switch (type)
{
case StoreType.ScenarioGeneral:
var md5 = CryptUtil.CalculateMd5Bytes(storeContents);
storePath = Path.Combine(CacheDirectory, filepath + md5);
await File.WriteAllBytesAsync(storePath, storeContents);
break;
default:
throw new Exception("Invalid StoreType.");
}
return storePath;
}
public enum StoreType
{
ScenarioGeneral
}
private async Task<DiskCacheItem> FastFetch(string item, string path, string md5)
{
var maxLoops = 5;
var deleteFlag = false;
@ -89,19 +141,23 @@ namespace KamihamaWeb.Services
if (file.Length > 0)
{
return new Tuple<int, Stream>(0, file);
return new DiskCacheItem()
{
Data = file,
Result = DiskCacheResultType.Success
};
}
var stream = await Rest.FetchAsset(item + $"?{md5}");
if (stream.Item2 == null)
if (stream.Data == null)
{
deleteFlag = true;
return stream;
}
((MemoryStream) stream.Item2).WriteTo(file);
stream.Item2.Seek(0, SeekOrigin.Begin);
((MemoryStream) stream.Data).WriteTo(file);
stream.Data.Seek(0, SeekOrigin.Begin);
return stream;
}
catch (Exception)
@ -126,9 +182,25 @@ namespace KamihamaWeb.Services
}
Log.Warning($"Max loops exceeded in DiskCacheService.FastFetch() for {item}.");
File.Delete(path);
return new Tuple<int, Stream>(500, null);
return new DiskCacheItem()
{
Result = DiskCacheResultType.Failed
};
}
}
public class DiskCacheItem
{
public Stream Data { get; set; } = null;
public DiskCacheResultType Result { get; set; } = DiskCacheResultType.Success;
}
public enum DiskCacheResultType
{
Success = 0,
Failed = 500,
NotFound = 404
}
}

103
KamihamaWeb/Util/MasterListBuilder.cs → KamihamaWeb/Services/MasterListBuilder.cs
View File

@ -2,27 +2,41 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using KamihamaWeb.Interfaces;
using KamihamaWeb.Models;
using KamihamaWeb.Util;
using Microsoft.Extensions.Caching.Distributed;
using Microsoft.Extensions.Caching.StackExchangeRedis;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Serilog;
using StackExchange.Redis;
namespace KamihamaWeb.Util
namespace KamihamaWeb.Services
{
public class MasterListBuilder
public class MasterListBuilder: IMasterListBuilder
{
private Regex multiPartRegex = new Regex(@"\.a[a-z]{2}"); // A bit crude
public MasterListBuilder()
private readonly Regex _multiPartRegex = new Regex(@"\.a[a-z]{2}"); // A bit crude
private IDiskCacheSingleton _disk;
private IDatabase _cache;
public MasterListBuilder(IDiskCacheSingleton disk, IDistributedCache cache)
{
BasePathLength =
@"MagiRecoStatic/magica/resource/download/asset/master/resource/".Length;
_cache = ((RedisCache)cache).GetConnection().GetDatabase();
_disk = disk;
}
private int BasePathLength { get; }
private string StaticDirectory { get; set; } = "MagiRecoStatic/";
public async Task<Dictionary<string, GamedataAsset>> GenerateEnglishAssetList()
{
if (File.Exists("en_cache.json"))
@ -48,9 +62,9 @@ namespace KamihamaWeb.Util
if (Directory.Exists("MagiRecoStatic/"))
if (Directory.Exists(StaticDirectory))
{
List<string> files = Directory.GetFiles("MagiRecoStatic/magica/resource/download/asset/master",
List<string> files = Directory.GetFiles(Path.Combine(StaticDirectory, "magica/resource/download/asset/master"),
"*.*",
SearchOption.AllDirectories).ToList();
@ -93,7 +107,7 @@ namespace KamihamaWeb.Util
// Remove multi-part files
foreach (var asset in englishAssets)
{
if (multiPartRegex.IsMatch(asset.Value.Path.Substring(asset.Value.Path.Length - 4)))
if (_multiPartRegex.IsMatch(asset.Value.Path.Substring(asset.Value.Path.Length - 4)))
{
Log.Debug($"Removing duplicate asset {asset.Key}");
englishAssets.Remove(asset.Value.Path);
@ -101,7 +115,7 @@ namespace KamihamaWeb.Util
else if (asset.Key.StartsWith("image_native/mini/")
|| asset.Key.StartsWith("image_native/live2d/")
//|| asset.Key.StartsWith("image_native/scene/gacha")
|| asset.Key.StartsWith("scenario/json/general/")
//|| asset.Key.StartsWith("scenario/json/general/")
|| asset.Key.StartsWith("scenario/json/oneShot/")
|| asset.Key.StartsWith("image_native/scene/event/")
|| asset.Key.StartsWith("image_native/scene/emotion/")
@ -163,5 +177,76 @@ namespace KamihamaWeb.Util
return asset;
}
public async Task<GamedataAsset> BuildScenarioGeneralJson(GamedataAsset generalAsset, Dictionary<string, GamedataAsset> englishAssets)
{
Log.Information($"Building scenario JSON for {generalAsset.Path}.");
if (!englishAssets.ContainsKey(generalAsset.Path)) // No english to replace with (yet)
{
Log.Warning($"No english asset for {generalAsset.Path}! This should be caught earlier!");
return generalAsset;
}
// Fetch JP asset
DiskCacheItem jpAsset = await _disk.Get(generalAsset.Path, generalAsset.Md5, true);
if (jpAsset.Result != DiskCacheResultType.Success)
{
Log.Warning("An error has occurred fetching a general scenario asset.");
return generalAsset;
}
// Merge assets
var enPath = Path.Combine(StaticDirectory, "magica/resource/download/asset/master/resource",
generalAsset.Path);
if (!File.Exists(enPath))
{
Log.Warning($"File {enPath} does not exist!");
return generalAsset;
}
var mergedAsset = MergeScenarioGeneral(
Encoding.UTF8.GetString(
CryptUtil.ReadFully(jpAsset.Data)
),
await File.ReadAllTextAsync(enPath)
);
var storeFilePath = await _disk.Store(generalAsset.Path, Encoding.UTF8.GetBytes(mergedAsset),
DiskCacheService.StoreType.ScenarioGeneral);
generalAsset.AssetSource = AssetSourceType.GeneralScript;
generalAsset.Md5 = CryptUtil.CalculateMd5File(storeFilePath);
generalAsset.FileList[0].Size = new FileInfo(storeFilePath).Length;
return generalAsset;
}
private string MergeScenarioGeneral(string jp, string en)
{
if (jp == en)
{
return jp;
}
var jp_json = JsonConvert.DeserializeObject<ScenarioGeneral>(jp);
var en_json = JsonConvert.DeserializeObject<ScenarioGeneral>(en);
foreach (var item in jp_json.story)
{
if (!en_json.story.ContainsKey(item.Key))
{
Log.Information($"Adding key {item.Key}.");
en_json.story[item.Key] = item.Value;
}
}
/*jp_json.Merge(en_json, new JsonMergeSettings()
{
MergeArrayHandling = MergeArrayHandling.Union,
MergeNullValueHandling = MergeNullValueHandling.Ignore,
});*/
return JsonConvert.SerializeObject(en_json);
}
}
}

40
KamihamaWeb/Services/MasterService.cs
View File

@ -21,12 +21,14 @@ namespace KamihamaWeb.Services
private IDatabase _cache { get; set; }
private IConfiguration _config { get; set; }
private IRestSharpTransient _rest { get; set; }
private IMasterListBuilder _builder { get; set; }
public MasterService(
IDistributedCache cache,
IConfiguration config,
IRestSharpTransient rest
) : this(Guid.NewGuid(), cache, config, rest)
IRestSharpTransient rest,
IMasterListBuilder builder
) : this(Guid.NewGuid(), cache, config, rest, builder)
{
}
@ -34,13 +36,15 @@ namespace KamihamaWeb.Services
Guid guid,
IDistributedCache cache,
IConfiguration config,
IRestSharpTransient rest
IRestSharpTransient rest,
IMasterListBuilder builder
)
{
Guid = guid;
_config = config;
_cache = ((RedisCache) cache).GetConnection().GetDatabase();
_rest = rest;
_builder = builder;
Task.Run(Initialize);
}
public Guid Guid { get; set; }
@ -79,9 +83,13 @@ namespace KamihamaWeb.Services
Log.Information("Configuring master list...");
var postProcessingGeneralScenario = new Dictionary<string, GamedataAsset>();
long counterReplace = 0;
long counterSkip = 0;
long counterNew = 0;
long counterPost = 0;
foreach (var assetType in workGamedataAssets)
{
var readyAssets = new Dictionary<string, GamedataAsset>();
@ -90,7 +98,12 @@ namespace KamihamaWeb.Services
// Replace with english assets as needed
if (EnglishMasterAssets.ContainsKey(asset.Path))
{
if (EnglishMasterAssets[asset.Path].Md5 != asset.Md5)
if (asset.Path.StartsWith("scenario/json/general/"))
{
postProcessingGeneralScenario.Add(asset.Path, asset);
counterPost++;
}
else if (EnglishMasterAssets[asset.Path].Md5 != asset.Md5)
{
//Log.Debug($"Replacing Japanese asset with English asset for {asset.Path}.");
readyAssets.Add(asset.Path, EnglishMasterAssets[asset.Path]);
@ -112,7 +125,7 @@ namespace KamihamaWeb.Services
}
GamedataAssets.Add(assetType.Key, readyAssets);
}
Log.Information($"Finished setting up. {counterReplace} replaced assets, {counterSkip} duplicate assets, {counterNew} new assets.");
Log.Information($"Finished setting up. {counterReplace} replaced assets, {counterSkip} duplicate assets, {counterNew} new assets, {counterPost} assets for post processing.");
// Add scripts
foreach (var asset in EnglishMasterAssets)
@ -128,6 +141,14 @@ namespace KamihamaWeb.Services
});
}
}
// Post processing
foreach (var asset in postProcessingGeneralScenario)
{
var builtJson = await _builder.BuildScenarioGeneralJson(asset.Value, EnglishMasterAssets);
GamedataAssets["asset_main"].Add(builtJson.Path, builtJson);
}
IsReady = true;
return true;
}
@ -155,8 +176,8 @@ namespace KamihamaWeb.Services
await Task.Delay(delay);
delay *= 2;
}
var builder= new MasterListBuilder();
var lists = await builder.GenerateEnglishAssetList();
var lists = await _builder.GenerateEnglishAssetList();
if (lists != null)
{
@ -189,11 +210,6 @@ namespace KamihamaWeb.Services
}
}
//public async Task<FileStream> ProvideFile(string filePath)
//{
// }
public async Task<string> ProvideJson(string which)
{
if (which == "asset_config")

18
KamihamaWeb/Services/RestSharpClient.cs
View File

@ -72,7 +72,7 @@ namespace KamihamaWeb.Services
return result.Content;
}
public async Task<Tuple<int, Stream>> FetchAsset(string item)
public async Task<DiskCacheItem> FetchAsset(string item)
{
var request = new RestRequest("resource/" + item, Method.GET);
@ -82,17 +82,27 @@ namespace KamihamaWeb.Services
if (response.StatusCode == HttpStatusCode.NotFound || response.ContentType == "text/html")
{
return new Tuple<int, Stream>(404, null);
return new DiskCacheItem()
{
Result = DiskCacheResultType.NotFound
};
}
var stream = new MemoryStream(response.RawBytes);
return new Tuple<int, Stream>(0, stream);
return new DiskCacheItem()
{
Result = DiskCacheResultType.Success,
Data = stream
};
}
catch (WebException ex)
{
Log.Warning($"Web exception thrown: Status code {ex.Status}, {ex.ToString()}");
}
return new Tuple<int, Stream>(500, null);
return new DiskCacheItem()
{
Result = DiskCacheResultType.Failed
};
}
}
}

2
KamihamaWeb/Startup.cs
View File

@ -4,6 +4,7 @@ using System.Linq;
using System.Threading.Tasks;
using KamihamaWeb.Interfaces;
using KamihamaWeb.Services;
using KamihamaWeb.Util;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.HttpsPolicy;
@ -46,6 +47,7 @@ namespace KamihamaWeb
services.AddTransient<IRestSharpTransient, RestSharpClient>();
services.AddSingleton<IMasterSingleton, MasterService>();
services.AddSingleton<IDiskCacheSingleton, DiskCacheService>();
services.AddTransient<IMasterListBuilder, MasterListBuilder>();
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.

21
KamihamaWeb/Util/CryptUtil.cs
View File

@ -32,5 +32,26 @@ namespace KamihamaWeb.Util
return sb.ToString();
}
public static byte[] ReadFully(Stream input)
{
using MemoryStream ms = new MemoryStream();
input.CopyTo(ms);
return ms.ToArray();
}
public static object CalculateMd5Bytes(byte[] storeContents)
{
using var md5 = MD5.Create();
using var stream = new MemoryStream(storeContents);
StringBuilder sb = new StringBuilder();
foreach (byte b in md5.ComputeHash(stream))
{
sb.Append(b.ToString("x2"));
}
return sb.ToString();
}
}
}