Merge pull request '1.修复service_Mid.cs.vm修复中间件跨服务器异库因主库权限无法读取问题；2.新增缓存批量设置和获取的扩展（只对雪花Id有效），适用实体大数据量碎片化存储缓存操作，自动分批处理' (#292) from aq982 into v2

Reviewed-on: https://code.adminnet.top/Admin.NET/Admin.NET.Pro/pulls/292

Admin.NET/Admin.NET.Core/GlobalUsings.cs

@@ -5,6 +5,7 @@
 // 不得利用本项目从事危害国家安全、扰乱社会秩序、侵犯他人合法权益等法律法规禁止的活动！任何基于本项目二次开发而产生的一切法律纠纷和责任，我们不承担任何责任！
 
 global using Admin.NET.Core.Service;
+global using Admin.NET.Core.Utils;
 global using Furion;
 global using Furion.ConfigurableOptions;
 global using Furion.DatabaseAccessor;

Admin.NET/Admin.NET.Core/Service/Cache/SysCacheService.cs

@@ -4,6 +4,9 @@
 //
 // 不得利用本项目从事危害国家安全、扰乱社会秩序、侵犯他人合法权益等法律法规禁止的活动！任何基于本项目二次开发而产生的一切法律纠纷和责任，我们不承担任何责任！
 
+using NewLife;
+using Newtonsoft.Json;
+
 namespace Admin.NET.Core.Service;
 
 /// <summary>
@@ -168,7 +171,250 @@ public class SysCacheService : IDynamicApiController, ISingleton
     {
         return _cacheProvider.Cache.Get<T>($"{_cacheOptions.Prefix}{key}");
     }
+    /// <summary>
+    /// 批量获取缓存值（普通键值结构）🔖
+    /// </summary>
+    /// <typeparam name="T">值类型</typeparam>
+    /// <param name="keys">缓存键集合</param>
+    /// <returns>与键顺序对应的值列表</returns>
+    [NonAction]
+    public List<T> GetBatch<T>(IEnumerable<string> keys)
+    {
+        var prefixedKeys = keys.Select(k => $"{_cacheOptions.Prefix}{k}");
+        return prefixedKeys.Select(k => _cacheProvider.Cache.Get<T>(k)).ToList();
+    }
+    /// <summary>
+    /// 批量设置缓存项（兼容现有键规则）❄️，方法只对雪花Id有效
+    /// </summary>
+    /// <typeparam name="T">实体类型（需包含Id属性）</typeparam>
+    /// <param name="items">待缓存数据集合</param>
+    /// <param name="expire">统一过期时间</param>
+    /// <param name="batchSize">批次大小（默认500）</param>
+    [NonAction]
+    public void SetList<T>(IEnumerable<T> items, TimeSpan? expire = null, int batchSize = 500) where T : class
+    {
+        if (items == null) return;
 
+        var itemList = items.ToList();
+        if (itemList.Count == 0) return;
+
+        // 获取雪花ID属性
+        var idProperty = typeof(T).GetProperty("Id")
+            ?? throw new ArgumentException("实体必须包含Id属性");
+
+        // 分批次处理
+        foreach (var batch in itemList.Batch(batchSize))
+        {
+            var dic = batch.ToDictionary(
+                item => $"{_cacheOptions.Prefix}{idProperty.GetValue(item)}",
+                item => item
+            );
+
+            if (_cacheProvider.Cache is Redis redis)
+            {
+                // Redis管道批量设置
+                redis.StartPipeline();
+                try
+                {
+                    foreach (var kv in dic)
+                    {
+                        redis.Set(kv.Key, kv.Value, expire ?? TimeSpan.Zero);
+                    }
+                }
+                finally
+                {
+                    redis.StopPipeline(true);
+                }
+            }
+            else
+            {
+                // 通用缓存实现
+                foreach (var kv in dic)
+                {
+                    _cacheProvider.Cache.Set(kv.Key, kv.Value, expire ?? TimeSpan.Zero);
+                }
+            }
+        }
+    }
+
+    /// <summary>
+    /// 异步批量获取（当前为同步实现，未来可升级）
+    /// <typeparam name="T">实体类型</typeparam>
+    /// <param name="ids">雪花ID集合</param>
+    /// <param name="loadFromDb">数据加载方法</param>
+    /// <param name="cacheNull">是否缓存空值（防穿透）</param>
+    /// <param name="nullExpire">空值缓存时间（默认永久）</param>
+    /// </summary>
+    [NonAction]
+    public async Task<List<T>> GetListAsync<T>(
+    IEnumerable<long> ids,
+    Func<List<long>, Task<List<T>>> loadFromDb,  // 改为异步委托
+    bool cacheNull = true,
+    TimeSpan? nullExpire = null
+) where T : class
+    {
+        var idList = ids.Distinct().ToList();
+        if (idList.Count == 0) return new List<T>();
+
+        // 1. 批量获取缓存（保持同步，假设缓存操作快速）
+        var cachedItems = GetFromCache<T>(idList);
+
+        // 2. 识别未命中ID
+        var missedIds = new List<long>();
+        var resultDict = new Dictionary<long, T>();
+
+        for (int i = 0; i < idList.Count; i++)
+        {
+            if (cachedItems[i] != null)
+            {
+                resultDict[idList[i]] = cachedItems[i];
+            }
+            else
+            {
+                missedIds.Add(idList[i]);
+            }
+        }
+
+        // 3. 异步加载缺失数据
+        if (missedIds.Count > 0)
+        {
+            var dbItems = await loadFromDb(missedIds).ConfigureAwait(false);  // 异步等待
+            var dbDict = dbItems.ToDictionary(GetId);
+
+            // 4. 缓存回填
+            var toCache = new List<T>();
+            foreach (var id in missedIds)
+            {
+                if (dbDict.TryGetValue(id, out var item))
+                {
+                    resultDict[id] = item;
+                    toCache.Add(item);
+                }
+                //else if (cacheNull)
+                //{
+                //    // 使用 default(T) 作为空值标记
+                //    toCache.Add(default(T));
+                //}
+            }
+
+            if (toCache.Count > 0) SetList(toCache, cacheNull ? nullExpire : null);  // 保持同步缓存写入
+        }
+
+        // 5. 按原始顺序返回
+        return idList.Select(id => resultDict.TryGetValue(id, out var item)
+            ? (item ==null ? null : item)
+            : null).ToList();
+    }
+    /// <summary>
+    /// 批量获取（自动加载缺失数据+缓存回填）🔁
+    /// </summary>
+    /// <typeparam name="T">实体类型</typeparam>
+    /// <param name="ids">雪花ID集合</param>
+    /// <param name="loadFromDb">数据加载方法</param>
+    /// <param name="cacheNull">是否缓存空值（防穿透）</param>
+    /// <param name="nullExpire">空值缓存时间（默认永久）</param>
+    [NonAction]
+    public List<T> GetList<T>(
+        IEnumerable<long> ids,
+        Func<List<long>, List<T>> loadFromDb,
+        bool cacheNull = true,
+        TimeSpan? nullExpire = null
+    ) where T : class
+    {
+        var idList = ids.Distinct().ToList();
+        if (idList.Count == 0) return new List<T>();
+
+        // 1. 批量获取缓存
+        var cachedItems = GetFromCache<T>(idList);
+
+        // 2. 识别未命中ID
+        var missedIds = new List<long>();
+        var resultDict = new Dictionary<long, T>();
+
+        for (int i = 0; i < idList.Count; i++)
+        {
+            if (cachedItems[i] != null)
+            {
+                resultDict[idList[i]] = cachedItems[i];
+            }
+            else
+            {
+                missedIds.Add(idList[i]);
+            }
+        }
+
+        // 3. 加载缺失数据
+        if (missedIds.Count > 0)
+        {
+            var dbItems = loadFromDb(missedIds);
+            var dbDict = dbItems.ToDictionary(GetId);
+
+            // 4. 缓存回填
+            var toCache = new List<T>();
+            foreach (var id in missedIds)
+            {
+                if (dbDict.TryGetValue(id, out var item))
+                {
+                    resultDict[id] = item;
+                    toCache.Add(item);
+                }
+                //else if (cacheNull)
+                //{
+                //    // 缓存空值标记
+                //    toCache.Add(default(T));
+                //}
+            }
+
+            //SetList(toCache, cacheNull ? (nullExpire ?? TimeSpan.FromMinutes(5)) : null);
+            // 将默认过期时间改为null（一直存储）
+            
+            if(toCache.Count>0) SetList(toCache, cacheNull ? nullExpire : null);
+        }
+
+        // 5. 按原始顺序返回
+        return idList.Select(id => resultDict.TryGetValue(id, out var item)
+            ? (item ==null ? null : item)
+            : null).ToList();
+    }
+
+    private long GetId<T>(T item)
+    {
+        var prop = typeof(T).GetProperty("Id");
+        return (long)prop.GetValue(item);
+    }
+
+    /// <summary>
+    /// 基础方法：仅从缓存获取数据
+    /// </summary>
+    [NonAction]
+    public List<T> GetFromCache<T>(List<long> ids) where T : class
+    {
+        if (ids == null || ids.Count == 0)
+            return new List<T>();
+
+        var keys = ids.Select(id => $"{_cacheOptions.Prefix}{id}").ToList();
+
+        if (_cacheProvider.Cache is FullRedis redis)
+        {
+            var result = redis.GetAll<T>(keys);
+            return keys.Select(k => result.TryGetValue(k, out var val) ? val : null).ToList();
+        }
+
+        return keys.Select(k => _cacheProvider.Cache.Get<T>(k)).ToList();
+    }
+    // <summary>
+    /// 批量获取哈希缓存字段值（哈希结构）🔖
+    /// </summary>
+    /// <typeparam name="T">值类型</typeparam>
+    /// <param name="key">哈希键名</param>
+    /// <param name="fields">要获取的字段集合</param>
+    /// <returns>与字段顺序对应的值列表</returns>
+    [NonAction]
+    public List<T> HashGetBatch<T>(string key, IEnumerable<string> fields)
+    {
+        var hash = GetHashMap<T>($"{_cacheOptions.Prefix}{key}");
+        return fields.Select(f => hash.TryGetValue(f, out T val) ? val : default).ToList();
+    }
     /// <summary>
     /// 删除缓存 🔖
     /// </summary>
@@ -259,7 +505,7 @@ public class SysCacheService : IDynamicApiController, ISingleton
     /// <param name="expire">过期时间，单位秒</param>
     /// <returns></returns>
     [NonAction]
-    public T GetOrAdd<T>(string key, Func<string, T> callback, int expire = -1)
+    public  T GetOrAdd<T>(string key, Func<string, T> callback, int expire = -1)
     {
         if (string.IsNullOrWhiteSpace(key)) return default;
         return _cacheProvider.Cache.GetOrAdd($"{_cacheOptions.Prefix}{key}", callback, expire);
@@ -353,7 +599,45 @@ public class SysCacheService : IDynamicApiController, ISingleton
         var hash = GetHashMap<T>(key);
         return hash.TryGetValue(field, out T value) ? value : default;
     }
+    // 新增方法：获取哈希表所有键
+    public static List<string> HashGetAllKeys(string key)
+    {
+        var hash = GetHashMap<string>(key); // 假设值为任意类型
+        return hash.Keys.ToList();
+    }
 
+    // 增强的哈希设置方法（带过期时间）
+    public static bool HashSet<T>(string key, Dictionary<string, T> items, TimeSpan? expiry = null)
+    {
+        var hash = GetHashMap<T>(key);
+        foreach (var item in items)
+        {
+            hash[item.Key] = item.Value;
+        }
+
+        if (expiry.HasValue)
+        {
+            _cacheProvider.Cache.SetExpire(key, expiry.Value);
+        }
+        return true;
+    }
+    // 异步批量设置哈希，目前没有，先保留扩展
+    public static async Task<bool> HashSetAsync<T>(string key, Dictionary<string, T> items, TimeSpan? expiry = null)
+    {
+        var hash = GetHashMap<T>(key);
+        foreach (var item in items)
+        {
+            hash[item.Key] = item.Value;
+        }
+        if (expiry.HasValue)
+        {
+            _cacheProvider.Cache.SetExpire(key, expiry.Value);
+        }
+        return true;
+    }
+
+    // 异步设置过期时间
+   
     /// <summary>
     /// 根据KEY获取所有HASH
     /// </summary>
@@ -410,4 +694,9 @@ public class SysCacheService : IDynamicApiController, ISingleton
     //    var hash = GetHashMap<T>(key);
     //    return hash.Search(pattern, count).ToList();
     //}
+}
+public class CacheItem<T>
+{
+    public T Value { get; set; }
+    public bool IsNull { get; set; }
 }

Admin.NET/Admin.NET.Core/Utils/EnumerableExtensions.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Admin.NET.Core.Utils;
+public static class EnumerableExtensions
+{
+    /// <summary>
+    /// 将集合分批次处理
+    /// </summary>
+    public static IEnumerable<IEnumerable<T>> Batch<T>(this IEnumerable<T> source, int batchSize)
+    {
+        var batch = new List<T>(batchSize);
+        foreach (var item in source)
+        {
+            batch.Add(item);
+            if (batch.Count == batchSize)
+            {
+                yield return batch;
+                batch = new List<T>(batchSize);
+            }
+        }
+        if (batch.Count > 0)
+            yield return batch;
+    }
+}

Admin.NET/Admin.NET.Web.Entry/wwwroot/template/service_Mid.cs.vm

@@ -81,11 +81,11 @@ public partial class @(@Model.ClassName)Mid
             @:   var key = $"@(@column.FkEntityName)_{t.@(@column.PropertyName)}";
             @:   if (!sysCacheService.ExistKey(key))
             @:   {
-            @:       var m = db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName));
+            @:       var m = db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName));
             @:       if (m != null) sysCacheService.Set(key, m);
             @:   }
             @:   t.@(@column.PropertyName)@(@column.FkColumnName) = sysCacheService.Get<@(@column.FkEntityName)>(key)?.@(@column.FkColumnName);
-            @:   //t.@(@column.PropertyName)@(@column.FkColumnName)=db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
+            @:   //t.@(@column.PropertyName)@(@column.FkColumnName)=db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
             @:})
             }
             else if(@column.EffectType == "ApiTreeSelector"){ 
@@ -95,11 +95,11 @@ public partial class @(@Model.ClassName)Mid
             @:   var key = $"@(@column.FkEntityName)_{t.@(@column.PropertyName)}";
             @:   if (!sysCacheService.ExistKey(key))
             @:   {
-            @:       var m = db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.ValueColumn) == t.@(@column.PropertyName));
+            @:       var m = db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.ValueColumn) == t.@(@column.PropertyName));
             @:       if (m != null) sysCacheService.Set(key, m);
             @:   }
             @:   t.@(@column.PropertyName)@(@column.DisplayColumn) = sysCacheService.Get<@(@column.FkEntityName)>(key)?.@(@column.DisplayColumn);
-            @:   //t.@(@column.PropertyName)@(@column.FkColumnName)=db.CopyNew().GetSimpleClient<@(@column.FkEntityName)>().GetFirst(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
+            @:   //t.@(@column.PropertyName)@(@column.FkColumnName)=db.Queryable<@(@column.FkEntityName)>().FirstAsync(f => f.@(@column.FkLinkColumnName) == t.@(@column.PropertyName))).@(@column.FkColumnName);//
             @:})
             }
         }