Merge pull request '1.修复service_Mid.cs.vm修复中间件跨服务器异库因主库权限无法读取问题；2.新增缓存批量设置和获取的扩展（只对雪花Id有效），适用实体大数据量碎片化存储缓存操作，自动分批处理' (#292) from aq982 into v2
Reviewed-on: https://code.adminnet.top/Admin.NET/Admin.NET.Pro/pulls/292
This commit is contained in:
zuohuaijun
commit
c0d593b497
@ -5,6 +5,7 @@
|
||||
// 不得利用本项目从事危害国家安全、扰乱社会秩序、侵犯他人合法权益等法律法规禁止的活动！任何基于本项目二次开发而产生的一切法律纠纷和责任，我们不承担任何责任！
|
||||
|
||||
global using Admin.NET.Core.Service;
|
||||
global using Admin.NET.Core.Utils;
|
||||
global using Furion;
|
||||
global using Furion.ConfigurableOptions;
|
||||
global using Furion.DatabaseAccessor;
|
||||
|
||||
@ -4,6 +4,9 @@
|
||||
//
|
||||
// 不得利用本项目从事危害国家安全、扰乱社会秩序、侵犯他人合法权益等法律法规禁止的活动！任何基于本项目二次开发而产生的一切法律纠纷和责任，我们不承担任何责任！
|
||||
|
||||
using NewLife;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace Admin.NET.Core.Service;
|
||||
|
||||
/// <summary>
|
||||
@ -168,7 +171,250 @@ public class SysCacheService : IDynamicApiController, ISingleton
|
||||
{
|
||||
return _cacheProvider.Cache.Get<T>($"{_cacheOptions.Prefix}{key}");
|
||||
}
|
||||
/// <summary>
|
||||
/// 批量获取缓存值（普通键值结构）🔖
|
||||
/// </summary>
|
||||
/// <typeparam name="T">值类型</typeparam>
|
||||
/// <param name="keys">缓存键集合</param>
|
||||
/// <returns>与键顺序对应的值列表</returns>
|
||||
[NonAction]
|
||||
public List<T> GetBatch<T>(IEnumerable<string> keys)
|
||||
{
|
||||
var prefixedKeys = keys.Select(k => $"{_cacheOptions.Prefix}{k}");
|
||||
return prefixedKeys.Select(k => _cacheProvider.Cache.Get<T>(k)).ToList();
|
||||
}
|
||||
/// <summary>
|
||||
/// 批量设置缓存项（兼容现有键规则）❄️，方法只对雪花Id有效
|
||||
/// </summary>
|
||||
/// <typeparam name="T">实体类型（需包含Id属性）</typeparam>
|
||||
/// <param name="items">待缓存数据集合</param>
|
||||
/// <param name="expire">统一过期时间</param>
|
||||
/// <param name="batchSize">批次大小（默认500）</param>
|
||||
[NonAction]
|
||||
public void SetList<T>(IEnumerable<T> items, TimeSpan? expire = null, int batchSize = 500) where T : class
|
||||
{
|
||||
if (items == null) return;
|
||||
|
||||
var itemList = items.ToList();
|
||||
if (itemList.Count == 0) return;
|
||||
|
||||
// 获取雪花ID属性
|
||||
var idProperty = typeof(T).GetProperty("Id")
|
||||
?? throw new ArgumentException("实体必须包含Id属性");
|
||||
|
||||
// 分批次处理
|
||||
foreach (var batch in itemList.Batch(batchSize))
|
||||
{
|
||||
var dic = batch.ToDictionary(
|
||||
item => $"{_cacheOptions.Prefix}{idProperty.GetValue(item)}",
|
||||
item => item
|
||||
);
|
||||
|
||||
if (_cacheProvider.Cache is Redis redis)
|
||||
{
|
||||
// Redis管道批量设置
|
||||
redis.StartPipeline();
|
||||
try
|
||||
{
|
||||
foreach (var kv in dic)
|
||||
{
|
||||
redis.Set(kv.Key, kv.Value, expire ?? TimeSpan.Zero);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
redis.StopPipeline(true);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// 通用缓存实现
|
||||
foreach (var kv in dic)
|
||||
{
|
||||
_cacheProvider.Cache.Set(kv.Key, kv.Value, expire ?? TimeSpan.Zero);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 异步批量获取（当前为同步实现，未来可升级）
|
||||
/// <typeparam name="T">实体类型</typeparam>
|
||||
/// <param name="ids">雪花ID集合</param>
|
||||
/// <param name="loadFromDb">数据加载方法</param>
|
||||
/// <param name="cacheNull">是否缓存空值（防穿透）</param>
|
||||
/// <param name="nullExpire">空值缓存时间（默认永久）</param>
|
||||
/// </summary>
|
||||
[NonAction]
|
||||
public async Task<List<T>> GetListAsync<T>(
|
||||
IEnumerable<long> ids,
|
||||
Func<List<long>, Task<List<T>>> loadFromDb, // 改为异步委托
|
||||
bool cacheNull = true,
|
||||
TimeSpan? nullExpire = null
|
||||
) where T : class
|
||||
{
|
||||
var idList = ids.Distinct().ToList();
|
||||
if (idList.Count == 0) return new List<T>();
|
||||
|
||||
// 1. 批量获取缓存（保持同步，假设缓存操作快速）
|
||||
var cachedItems = GetFromCache<T>(idList);
|
||||
|
||||
// 2. 识别未命中ID
|
||||
var missedIds = new List<long>();
|
||||
var resultDict = new Dictionary<long, T>();
|
||||
|
||||
for (int i = 0; i < idList.Count; i++)
|
||||
{
|
||||
if (cachedItems[i] != null)
|
||||
{
|
||||
resultDict[idList[i]] = cachedItems[i];
|
||||
}
|
||||
else
|
||||
{
|
||||
missedIds.Add(idList[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 异步加载缺失数据
|
||||
if (missedIds.Count > 0)
|
||||
{
|
||||
var dbItems = await loadFromDb(missedIds).ConfigureAwait(false); // 异步等待
|
||||
var dbDict = dbItems.ToDictionary(GetId);
|
||||
|
||||
// 4. 缓存回填
|
||||
var toCache = new List<T>();
|
||||
foreach (var id in missedIds)
|
||||
{
|
||||
if (dbDict.TryGetValue(id, out var item))
|
||||
{
|
||||
resultDict[id] = item;
|
||||
toCache.Add(item);
|
||||
}
|
||||
//else if (cacheNull)
|
||||
//{
|
||||
// // 使用 default(T) 作为空值标记
|
||||
// toCache.Add(default(T));
|
||||
//}
|
||||
}
|
||||
|
||||
if (toCache.Count > 0) SetList(toCache, cacheNull ? nullExpire : null); // 保持同步缓存写入
|
||||
}
|
||||
|
||||
// 5. 按原始顺序返回
|
||||
return idList.Select(id => resultDict.TryGetValue(id, out var item)
|
||||
? (item ==null ? null : item)
|
||||
: null).ToList();
|
||||
}
|
||||
/// <summary>
|
||||
/// 批量获取（自动加载缺失数据+缓存回填）🔁
|
||||
/// </summary>
|
||||
/// <typeparam name="T">实体类型</typeparam>
|
||||
/// <param name="ids">雪花ID集合</param>
|
||||
/// <param name="loadFromDb">数据加载方法</param>
|
||||
/// <param name="cacheNull">是否缓存空值（防穿透）</param>
|
||||
/// <param name="nullExpire">空值缓存时间（默认永久）</param>
|
||||
[NonAction]
|
||||
public List<T> GetList<T>(
|
||||
IEnumerable<long> ids,
|
||||
Func<List<long>, List<T>> loadFromDb,
|
||||
bool cacheNull = true,
|
||||
TimeSpan? nullExpire = null
|
||||
) where T : class
|
||||
{
|
||||
var idList = ids.Distinct().ToList();
|
||||
if (idList.Count == 0) return new List<T>();
|
||||
|
||||
// 1. 批量获取缓存
|
||||
var cachedItems = GetFromCache<T>(idList);
|
||||
|
||||
// 2. 识别未命中ID
|
||||
var missedIds = new List<long>();
|
||||
var resultDict = new Dictionary<long, T>();
|
||||
|
||||
for (int i = 0; i < idList.Count; i++)
|
||||
{
|
||||
if (cachedItems[i] != null)
|
||||
{
|
||||
resultDict[idList[i]] = cachedItems[i];
|
||||
}
|
||||
else
|
||||
{
|
||||
missedIds.Add(idList[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 加载缺失数据
|
||||
if (missedIds.Count > 0)
|
||||
{
|
||||
var dbItems = loadFromDb(missedIds);
|
||||
var dbDict = dbItems.ToDictionary(GetId);
|
||||
|
||||
// 4. 缓存回填
|
||||
var toCache = new List<T>();
|
||||
foreach (var id in missedIds)
|
||||
{
|
||||
if (dbDict.TryGetValue(id, out var item))
|
||||
{
|
||||
resultDict[id] = item;
|
||||
toCache.Add(item);
|
||||
}
|
||||
//else if (cacheNull)
|
||||
//{
|
||||
// // 缓存空值标记
|
||||
// toCache.Add(default(T));
|
||||
//}
|
||||
}
|
||||
|
||||
//SetList(toCache, cacheNull ? (nullExpire ?? TimeSpan.FromMinutes(5)) : null);
|
||||
// 将默认过期时间改为null（一直存储）
|
||||
|
||||
if(toCache.Count>0) SetList(toCache, cacheNull ? nullExpire : null);
|
||||
}
|
||||
|
||||
// 5. 按原始顺序返回
|
||||
return idList.Select(id => resultDict.TryGetValue(id, out var item)
|
||||
? (item ==null ? null : item)
|
||||
: null).ToList();
|
||||
}
|
||||
|
||||
private long GetId<T>(T item)
|
||||
{
|
||||
var prop = typeof(T).GetProperty("Id");
|
||||
return (long)prop.GetValue(item);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 基础方法：仅从缓存获取数据
|
||||
/// </summary>
|
||||
[NonAction]
|
||||
public List<T> GetFromCache<T>(List<long> ids) where T : class
|
||||
{
|
||||
if (ids == null || ids.Count == 0)
|
||||
return new List<T>();
|
||||
|
||||
var keys = ids.Select(id => $"{_cacheOptions.Prefix}{id}").ToList();
|
||||
|
||||
if (_cacheProvider.Cache is FullRedis redis)
|
||||
{
|
||||
var result = redis.GetAll<T>(keys);
|
||||
return keys.Select(k => result.TryGetValue(k, out var val) ? val : null).ToList();
|
||||
}
|
||||
|
||||
return keys.Select(k => _cacheProvider.Cache.Get<T>(k)).ToList();
|
||||
}
|
||||
// <summary>
|
||||
/// 批量获取哈希缓存字段值（哈希结构）🔖
|
||||
/// </summary>
|
||||
/// <typeparam name="T">值类型</typeparam>
|
||||
/// <param name="key">哈希键名</param>
|
||||
/// <param name="fields">要获取的字段集合</param>
|
||||
/// <returns>与字段顺序对应的值列表</returns>
|
||||
[NonAction]
|
||||
public List<T> HashGetBatch<T>(string key, IEnumerable<string> fields)
|
||||
{
|
||||
var hash = GetHashMap<T>($"{_cacheOptions.Prefix}{key}");
|
||||
return fields.Select(f => hash.TryGetValue(f, out T val) ? val : default).ToList();
|
||||
}
|
||||
/// <summary>
|
||||
/// 删除缓存 🔖
|
||||
/// </summary>
|
||||
@ -259,7 +505,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
|
||||
/// <param name="expire">过期时间，单位秒</param>
|
||||
/// <returns></returns>
|
||||
[NonAction]
|
||||
public T GetOrAdd<T>(string key, Func<string, T> callback, int expire = -1)
|
||||
public T GetOrAdd<T>(string key, Func<string, T> callback, int expire = -1)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(key)) return default;
|
||||
return _cacheProvider.Cache.GetOrAdd($"{_cacheOptions.Prefix}{key}", callback, expire);
|
||||
@ -353,7 +599,45 @@ public class SysCacheService : IDynamicApiController, ISingleton
|
||||
var hash = GetHashMap<T>(key);
|
||||
return hash.TryGetValue(field, out T value) ? value : default;
|
||||
}
|
||||
// 新增方法：获取哈希表所有键
|
||||
public static List<string> HashGetAllKeys(string key)
|
||||
{
|
||||
var hash = GetHashMap<string>(key); // 假设值为任意类型
|
||||
return hash.Keys.ToList();
|
||||
}
|
||||
|
||||
// 增强的哈希设置方法（带过期时间）
|
||||
public static bool HashSet<T>(string key, Dictionary<string, T> items, TimeSpan? expiry = null)
|
||||
{
|
||||
var hash = GetHashMap<T>(key);
|
||||
foreach (var item in items)
|
||||
{
|
||||
hash[item.Key] = item.Value;
|
||||
}
|
||||
|
||||
if (expiry.HasValue)
|
||||
{
|
||||
_cacheProvider.Cache.SetExpire(key, expiry.Value);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
// 异步批量设置哈希，目前没有，先保留扩展
|
||||
public static async Task<bool> HashSetAsync<T>(string key, Dictionary<string, T> items, TimeSpan? expiry = null)
|
||||
{
|
||||
var hash = GetHashMap<T>(key);
|
||||
foreach (var item in items)
|
||||
{
|
||||
hash[item.Key] = item.Value;
|
||||
}
|
||||
if (expiry.HasValue)
|
||||
{
|
||||
_cacheProvider.Cache.SetExpire(key, expiry.Value);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// 异步设置过期时间
|
||||
|
||||
/// <summary>
|
||||
/// 根据KEY获取所有HASH
|
||||
/// </summary>
|
||||
@ -410,4 +694,9 @@ public class SysCacheService : IDynamicApiController, ISingleton
|
||||
// var hash = GetHashMap<T>(key);
|
||||
// return hash.Search(pattern, count).ToList();
|
||||
//}
|
||||
}
|
||||
public class CacheItem<T>
|
||||
{
|
||||
public T Value { get; set; }
|
||||
public bool IsNull { get; set; }
|
||||
}
|
||||
28
Admin.NET/Admin.NET.Core/Utils/EnumerableExtensions.cs
Normal file
28
Admin.NET/Admin.NET.Core/Utils/EnumerableExtensions.cs
Normal file
@ -0,0 +1,28 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Admin.NET.Core.Utils;
|
||||
public static class EnumerableExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// 将集合分批次处理
|
||||
/// </summary>
|
||||
public static IEnumerable<IEnumerable<T>> Batch<T>(this IEnumerable<T> source, int batchSize)
|
||||
{
|
||||
var batch = new List<T>(batchSize);
|
||||
foreach (var item in source)
|
||||
{
|
||||
batch.Add(item);
|
||||
if (batch.Count == batchSize)
|
||||
{
|
||||
yield return batch;
|
||||
batch = new List<T>(batchSize);
|
||||
}
|
||||
}
|
||||
if (batch.Count > 0)
|
||||
yield return batch;
|
||||
}
|
||||
}
|
||||
@ -81,11 +81,11 @@ public partial class @(@Model.ClassName)Mid
|
||||
@: var key = $"@(@column.FkEntityName)_{t.@(@column.PropertyName)}";
|
||||
@: if (!sysCacheService.ExistKey(key))
|
||||
@: {
|
||||
@: var m = db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName));
|
||||
@: var m = db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName));
|
||||
@: if (m != null) sysCacheService.Set(key, m);
|
||||
@: }
|
||||
@: t.@(@column.PropertyName)@(@column.FkColumnName) = sysCacheService.Get<@(@column.FkEntityName)>(key)?.@(@column.FkColumnName);
|
||||
@: //t.@(@column.PropertyName)@(@column.FkColumnName)=db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
|
||||
@: //t.@(@column.PropertyName)@(@column.FkColumnName)=db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
|
||||
@:})
|
||||
}
|
||||
else if(@column.EffectType == "ApiTreeSelector"){
|
||||
@ -95,11 +95,11 @@ public partial class @(@Model.ClassName)Mid
|
||||
@: var key = $"@(@column.FkEntityName)_{t.@(@column.PropertyName)}";
|
||||
@: if (!sysCacheService.ExistKey(key))
|
||||
@: {
|
||||
@: var m = db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.ValueColumn) == t.@(@column.PropertyName));
|
||||
@: var m = db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.ValueColumn) == t.@(@column.PropertyName));
|
||||
@: if (m != null) sysCacheService.Set(key, m);
|
||||
@: }
|
||||
@: t.@(@column.PropertyName)@(@column.DisplayColumn) = sysCacheService.Get<@(@column.FkEntityName)>(key)?.@(@column.DisplayColumn);
|
||||
@: //t.@(@column.PropertyName)@(@column.FkColumnName)=db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
|
||||
@: //t.@(@column.PropertyName)@(@column.FkColumnName)=db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
|
||||
@:})
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user