gaoxing/UNIVPLMDataIntegration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复service_Mid.cs.vm修复中间件跨服务器异库因主库权限无法读取问题；

Browse Source 
新增缓存批量设置和获取的扩展（只对雪花Id有效），适用实体大数据量碎片化存储缓存操作，自动分批处理
This commit is contained in:
lqc 2025-03-10 01:17:32 +08:00
parent a3659cbaf2
commit f0ab2f2bd9
2 changed files with 303 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

309
Admin.NET/Admin.NET.Core/Service/Cache/SysCacheService.cs
View File

@ -4,6 +4,9 @@
//
// 不得利用本项目从事危害国家安全、扰乱社会秩序、侵犯他人合法权益等法律法规禁止的活动！任何基于本项目二次开发而产生的一切法律纠纷和责任，我们不承担任何责任！
using NewLife;
using Newtonsoft.Json;
namespace Admin.NET.Core.Service;
/// <summary>
@ -143,7 +146,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
private static string KeySingle(object t)
{
return t.GetType().IsClass && !t.GetType().IsPrimitive ? JSON.Serialize(t) : t.ToString();
return t.GetType().IsClass && !t.GetType().IsPrimitive ? JsonConvert.SerializeObject(t) : t.ToString();
}
/// <summary>
@ -152,7 +155,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
/// <param name="key"></param>
/// <returns></returns>
[NonAction]
public static TimeSpan GetExpire(string key)
public TimeSpan GetExpire(string key)
{
return _cacheProvider.Cache.GetExpire(key);
}
@ -168,7 +171,250 @@ public class SysCacheService : IDynamicApiController, ISingleton
{
return _cacheProvider.Cache.Get<T>($"{_cacheOptions.Prefix}{key}");
}
/// <summary>
/// 批量获取缓存值（普通键值结构）🔖
/// </summary>
/// <typeparam name="T">值类型</typeparam>
/// <param name="keys">缓存键集合</param>
/// <returns>与键顺序对应的值列表</returns>
[NonAction]
public List<T> GetBatch<T>(IEnumerable<string> keys)
{
var prefixedKeys = keys.Select(k => $"{_cacheOptions.Prefix}{k}");
return prefixedKeys.Select(k => _cacheProvider.Cache.Get<T>(k)).ToList();
}
/// <summary>
/// 批量设置缓存项（兼容现有键规则）❄️，方法只对雪花Id有效
/// </summary>
/// <typeparam name="T">实体类型（需包含Id属性）</typeparam>
/// <param name="items">待缓存数据集合</param>
/// <param name="expire">统一过期时间</param>
/// <param name="batchSize">批次大小（默认500）</param>
[NonAction]
public void SetList<T>(IEnumerable<T> items, TimeSpan? expire = null, int batchSize = 500) where T : class
{
if (items == null) return;
var itemList = items.ToList();
if (itemList.Count == 0) return;
// 获取雪花ID属性
var idProperty = typeof(T).GetProperty("Id")
?? throw new ArgumentException("实体必须包含Id属性");
// 分批次处理
foreach (var batch in itemList.Batch(batchSize))
{
var dic = batch.ToDictionary(
item => $"{_cacheOptions.Prefix}{idProperty.GetValue(item)}",
item => item
);
if (_cacheProvider.Cache is Redis redis)
{
// Redis管道批量设置
redis.StartPipeline();
try
{
foreach (var kv in dic)
{
redis.Set(kv.Key, kv.Value, expire ?? TimeSpan.Zero);
}
}
finally
{
redis.StopPipeline(true);
}
}
else
{
// 通用缓存实现
foreach (var kv in dic)
{
_cacheProvider.Cache.Set(kv.Key, kv.Value, expire ?? TimeSpan.Zero);
}
}
}
}
/// <summary>
/// 异步批量获取（当前为同步实现，未来可升级）
/// <typeparam name="T">实体类型</typeparam>
/// <param name="ids">雪花ID集合</param>
/// <param name="loadFromDb">数据加载方法</param>
/// <param name="cacheNull">是否缓存空值（防穿透）</param>
/// <param name="nullExpire">空值缓存时间（默认永久）</param>
/// </summary>
[NonAction]
public async Task<List<T>> GetListAsync<T>(
IEnumerable<long> ids,
Func<List<long>, Task<List<T>>> loadFromDb, // 改为异步委托
bool cacheNull = true,
TimeSpan? nullExpire = null
) where T : class
{
var idList = ids.Distinct().ToList();
if (idList.Count == 0) return new List<T>();
// 1. 批量获取缓存（保持同步，假设缓存操作快速）
var cachedItems = GetFromCache<T>(idList);
// 2. 识别未命中ID
var missedIds = new List<long>();
var resultDict = new Dictionary<long, T>();
for (int i = 0; i < idList.Count; i++)
{
if (cachedItems[i] != null)
{
resultDict[idList[i]] = cachedItems[i];
}
else
{
missedIds.Add(idList[i]);
}
}
// 3. 异步加载缺失数据
if (missedIds.Count > 0)
{
var dbItems = await loadFromDb(missedIds).ConfigureAwait(false); // 异步等待
var dbDict = dbItems.ToDictionary(GetId);
// 4. 缓存回填
var toCache = new List<T>();
foreach (var id in missedIds)
{
if (dbDict.TryGetValue(id, out var item))
{
resultDict[id] = item;
toCache.Add(item);
}
//else if (cacheNull)
//{
// // 使用 default(T) 作为空值标记
// toCache.Add(default(T));
//}
}
if (toCache.Count > 0) SetList(toCache, cacheNull ? nullExpire : null); // 保持同步缓存写入
}
// 5. 按原始顺序返回
return idList.Select(id => resultDict.TryGetValue(id, out var item)
? (item ==null ? null : item)
: null).ToList();
}
/// <summary>
/// 批量获取（自动加载缺失数据+缓存回填）🔁
/// </summary>
/// <typeparam name="T">实体类型</typeparam>
/// <param name="ids">雪花ID集合</param>
/// <param name="loadFromDb">数据加载方法</param>
/// <param name="cacheNull">是否缓存空值（防穿透）</param>
/// <param name="nullExpire">空值缓存时间（默认永久）</param>
[NonAction]
public List<T> GetList<T>(
IEnumerable<long> ids,
Func<List<long>, List<T>> loadFromDb,
bool cacheNull = true,
TimeSpan? nullExpire = null
) where T : class
{
var idList = ids.Distinct().ToList();
if (idList.Count == 0) return new List<T>();
// 1. 批量获取缓存
var cachedItems = GetFromCache<T>(idList);
// 2. 识别未命中ID
var missedIds = new List<long>();
var resultDict = new Dictionary<long, T>();
for (int i = 0; i < idList.Count; i++)
{
if (cachedItems[i] != null)
{
resultDict[idList[i]] = cachedItems[i];
}
else
{
missedIds.Add(idList[i]);
}
}
// 3. 加载缺失数据
if (missedIds.Count > 0)
{
var dbItems = loadFromDb(missedIds);
var dbDict = dbItems.ToDictionary(GetId);
// 4. 缓存回填
var toCache = new List<T>();
foreach (var id in missedIds)
{
if (dbDict.TryGetValue(id, out var item))
{
resultDict[id] = item;
toCache.Add(item);
}
//else if (cacheNull)
//{
// // 缓存空值标记
// toCache.Add(default(T));
//}
}
//SetList(toCache, cacheNull ? (nullExpire ?? TimeSpan.FromMinutes(5)) : null);
// 将默认过期时间改为null（一直存储）
if(toCache.Count>0) SetList(toCache, cacheNull ? nullExpire : null);
}
// 5. 按原始顺序返回
return idList.Select(id => resultDict.TryGetValue(id, out var item)
? (item ==null ? null : item)
: null).ToList();
}
private long GetId<T>(T item)
{
var prop = typeof(T).GetProperty("Id");
return (long)prop.GetValue(item);
}
/// <summary>
/// 基础方法：仅从缓存获取数据
/// </summary>
[NonAction]
public List<T> GetFromCache<T>(List<long> ids) where T : class
{
if (ids == null || ids.Count == 0)
return new List<T>();
var keys = ids.Select(id => $"{_cacheOptions.Prefix}{id}").ToList();
if (_cacheProvider.Cache is FullRedis redis)
{
var result = redis.GetAll<T>(keys);
return keys.Select(k => result.TryGetValue(k, out var val) ? val : null).ToList();
}
return keys.Select(k => _cacheProvider.Cache.Get<T>(k)).ToList();
}
// <summary>
/// 批量获取哈希缓存字段值（哈希结构）🔖
/// </summary>
/// <typeparam name="T">值类型</typeparam>
/// <param name="key">哈希键名</param>
/// <param name="fields">要获取的字段集合</param>
/// <returns>与字段顺序对应的值列表</returns>
[NonAction]
public List<T> HashGetBatch<T>(string key, IEnumerable<string> fields)
{
var hash = GetHashMap<T>($"{_cacheOptions.Prefix}{key}");
return fields.Select(f => hash.TryGetValue(f, out T val) ? val : default).ToList();
}
/// <summary>
/// 删除缓存 🔖
/// </summary>
@ -272,7 +518,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
/// <param name="key"></param>
/// <returns></returns>
[NonAction]
public static IDictionary<String, T> GetHashMap<T>(string key)
public IDictionary<String, T> GetHashMap<T>(string key)
{
return _cacheProvider.Cache.GetDictionary<T>(key);
}
@ -285,7 +531,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
/// <param name="dic"></param>
/// <returns></returns>
[NonAction]
public static bool HashSet<T>(string key, Dictionary<string, T> dic)
public bool HashSet<T>(string key, Dictionary<string, T> dic)
{
var hash = GetHashMap<T>(key);
foreach (var v in dic)
@ -303,7 +549,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
/// <param name="hashKey"></param>
/// <param name="value"></param>
[NonAction]
public static void HashAdd<T>(string key, string hashKey, T value)
public void HashAdd<T>(string key, string hashKey, T value)
{
var hash = GetHashMap<T>(key);
hash.Add(hashKey, value);
@ -317,7 +563,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
/// <param name="hashKey"></param>
/// <param name="value"></param>
[NonAction]
public static void HashAddOrUpdate<T>(string key, string hashKey, T value)
public void HashAddOrUpdate<T>(string key, string hashKey, T value)
{
var hash = GetHashMap<T>(key);
if (hash.ContainsKey(hashKey))
@ -334,7 +580,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
/// <param name="fields"></param>
/// <returns></returns>
[NonAction]
public static List<T> HashGet<T>(string key, params string[] fields)
public List<T> HashGet<T>(string key, params string[] fields)
{
var hash = GetHashMap<T>(key);
return hash.Where(t => fields.Any(c => t.Key == c)).Select(t => t.Value).ToList();
@ -348,12 +594,50 @@ public class SysCacheService : IDynamicApiController, ISingleton
/// <param name="field"></param>
/// <returns></returns>
[NonAction]
public static T HashGetOne<T>(string key, string field)
public T HashGetOne<T>(string key, string field)
{
var hash = GetHashMap<T>(key);
return hash.TryGetValue(field, out T value) ? value : default;
}
// 新增方法：获取哈希表所有键
public List<string> HashGetAllKeys(string key)
{
var hash = GetHashMap<string>(key); // 假设值为任意类型
return hash.Keys.ToList();
}
// 增强的哈希设置方法（带过期时间）
public bool HashSet<T>(string key, Dictionary<string, T> items, TimeSpan? expiry = null)
{
var hash = GetHashMap<T>(key);
foreach (var item in items)
{
hash[item.Key] = item.Value;
}
if (expiry.HasValue)
{
_cacheProvider.Cache.SetExpire(key, expiry.Value);
}
return true;
}
// 异步批量设置哈希，目前没有，先保留扩展
public async Task<bool> HashSetAsync<T>(string key, Dictionary<string, T> items, TimeSpan? expiry = null)
{
var hash = GetHashMap<T>(key);
foreach (var item in items)
{
hash[item.Key] = item.Value;
}
if (expiry.HasValue)
{
_cacheProvider.Cache.SetExpire(key, expiry.Value);
}
return true;
}
// 异步设置过期时间
/// <summary>
/// 根据KEY获取所有HASH
/// </summary>
@ -361,7 +645,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
/// <param name="key"></param>
/// <returns></returns>
[NonAction]
public static IDictionary<string, T> HashGetAll<T>(string key)
public IDictionary<string, T> HashGetAll<T>(string key)
{
var hash = GetHashMap<T>(key);
return hash;
@ -375,7 +659,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
/// <param name="fields"></param>
/// <returns></returns>
[NonAction]
public static int HashDel<T>(string key, params string[] fields)
public int HashDel<T>(string key, params string[] fields)
{
var hash = GetHashMap<T>(key);
fields.ToList().ForEach(t => hash.Remove(t));
@ -410,4 +694,9 @@ public class SysCacheService : IDynamicApiController, ISingleton
// var hash = GetHashMap<T>(key);
// return hash.Search(pattern, count).ToList();
//}
}
public class CacheItem<T>
{
public T Value { get; set; }
public bool IsNull { get; set; }
}

8
Admin.NET/Admin.NET.Web.Entry/wwwroot/template/service_Mid.cs.vm
View File

@ -81,11 +81,11 @@ public partial class @(@Model.ClassName)Mid
@: var key = $"@(@column.FkEntityName)_{t.@(@column.PropertyName)}";
@: if (!sysCacheService.ExistKey(key))
@: {
@: var m = db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName));
@: var m = db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName));
@: if (m != null) sysCacheService.Set(key, m);
@: }
@: t.@(@column.PropertyName)@(@column.FkColumnName) = sysCacheService.Get<@(@column.FkEntityName)>(key)?.@(@column.FkColumnName);
@: //t.@(@column.PropertyName)@(@column.FkColumnName)=db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
@: //t.@(@column.PropertyName)@(@column.FkColumnName)=db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
@:})
}
else if(@column.EffectType == "ApiTreeSelector"){
@ -95,11 +95,11 @@ public partial class @(@Model.ClassName)Mid
@: var key = $"@(@column.FkEntityName)_{t.@(@column.PropertyName)}";
@: if (!sysCacheService.ExistKey(key))
@: {
@: var m = db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.ValueColumn) == t.@(@column.PropertyName));
@: var m = db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.ValueColumn) == t.@(@column.PropertyName));
@: if (m != null) sysCacheService.Set(key, m);
@: }
@: t.@(@column.PropertyName)@(@column.DisplayColumn) = sysCacheService.Get<@(@column.FkEntityName)>(key)?.@(@column.DisplayColumn);
@: //t.@(@column.PropertyName)@(@column.FkColumnName)=db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
@: //t.@(@column.PropertyName)@(@column.FkColumnName)=db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
@:})
}
}