From 22e848871436811496f03dae3cf536c8fb53cfb8 Mon Sep 17 00:00:00 2001
From: hxp <ale99527@vip.qq.com>
Date: 星期二, 19 三月 2024 19:40:00 +0800
Subject: [PATCH] 10138 内存分析（优化py自定配置表数据内存占用及加载方式）
---
PySysDB/生成IpyGameDataPY/IpyGameDataPYTemp.py | 210 +++++++++++++++++++++++++++++++----------
ServerPython/CoreServerGroup/GameServer/Script/ChConfig.py | 6 +
PySysDB/生成IpyGameDataPY/IpyGameDataPYCreater.py | 27 +++--
ServerPython/ZoneServerGroup/map1_8G/MapServer/MapServerData/Script/ChConfig.py | 7 +
4 files changed, 184 insertions(+), 66 deletions(-)
diff --git "a/PySysDB/\347\224\237\346\210\220IpyGameDataPY/IpyGameDataPYCreater.py" "b/PySysDB/\347\224\237\346\210\220IpyGameDataPY/IpyGameDataPYCreater.py"
index f585874..435127c 100644
--- "a/PySysDB/\347\224\237\346\210\220IpyGameDataPY/IpyGameDataPYCreater.py"
+++ "b/PySysDB/\347\224\237\346\210\220IpyGameDataPY/IpyGameDataPYCreater.py"
@@ -38,12 +38,16 @@
ClassFuncTemp = " def Get%s(self): return self.%s%s%s"
# 管理器初始化表数据缓存列表对象模板
-MgrTableCacheInit = " self.ipy%sCache = self.__LoadFileData(\"%s\", IPY_%s)%s"
-MgrTableLenInit = " self.ipy%sLen = len(self.ipy%sCache)%s"
-# 管理器获取表数据条数函数模板
-MgrGetCountFunc = " def Get%sCount(self): return self.ipy%sLen%s"
-# 管理器根据索引获取表数据函数模板
-MgrGetByIndexFunc = " def Get%sByIndex(self, index): return self.ipy%sCache[index]%s"
+MgrTableCacheInit = " self.__LoadFileData(\"%s\", onlyCheck)%s"
+# 管理器获取表数据条数、数据函数模板
+MgrGetCountFunc = '''
+ def Get%sCount(self):
+ self.CheckLoadData("%s")
+ return self.ipy%sLen\r
+ def Get%sByIndex(self, index):
+ self.CheckLoadData("%s")
+ return self.ipy%sCache[index]\r
+'''
Def_RN = "\r\n"
@@ -159,19 +163,18 @@
classInitInfo += ClassInitTemp % (fieldName, "0.0", lineEnd)
else:
classInitInfo += ClassInitTemp % (fieldName, "0", lineEnd)
- classFuncInfo += ClassFuncTemp % (fieldName, fieldName, noteInfo, lineEnd)
+ callAttrValue = "attrTuple[%s]" % j # fieldName
+ classFuncInfo += ClassFuncTemp % (fieldName, callAttrValue, noteInfo, lineEnd)
ipyTableDef += TableDefEnd % (Def_RN if i == len(Def_IpyTable) - 1 else (Def_RN * 2))
+ classInitInfo = ClassInitTemp % ("attrTuple", "None", "") # 优化内存，不使用内置 __dict__ 访问属性，改为使用tuple存value
classContent += ClassTemp % (tableNameCh, tableName, classInitInfo, classFuncInfo)
# 表列表、长度对象缓存
- mgrTableCacheObjInit += MgrTableCacheInit % (tableName, tableName, tableName, Def_RN)
- mgrTableCacheObjInit += MgrTableLenInit % (tableName, tableName, Def_RN)
+ mgrTableCacheObjInit += MgrTableCacheInit % (tableName, Def_RN)
# 表列表、长度获取函数
- mgrTableFunc += MgrGetCountFunc % (tableName, tableName, Def_RN)
- mgrTableFunc += MgrGetByIndexFunc % (tableName, tableName, Def_RN)
-
+ mgrTableFunc += MgrGetCountFunc % (tableName, tableName, tableName, tableName, tableName, tableName)
# 读取模板生成py代码
createPYContent = ""
diff --git "a/PySysDB/\347\224\237\346\210\220IpyGameDataPY/IpyGameDataPYTemp.py" "b/PySysDB/\347\224\237\346\210\220IpyGameDataPY/IpyGameDataPYTemp.py"
index ba8614b..4df76c0 100644
--- "a/PySysDB/\347\224\237\346\210\220IpyGameDataPY/IpyGameDataPYTemp.py"
+++ "b/PySysDB/\347\224\237\346\210\220IpyGameDataPY/IpyGameDataPYTemp.py"
@@ -50,55 +50,74 @@
class IPY_DataMgr():
def __init__(self):
+ Log("IPY_DataMgr Init...")
self.fileMD5Dict = {} # 数据表文件md5字典, 用于对比文件差异判断是否重读 {dtName:md5, ...}
self.ipyConfigEx = {} # 自定义数据缓存 {key:configData, ...}
self.ipyDataIndexMap = {} # 数据表索引查询缓存 {dtName:{args:[index, ...], ...}, ...}
self.ipyDataIndexMapEx = {} # 数据表自定义查询条件查询缓存 {dtName_(findKey,...):{(findKeyValue, ...):[index, ...], ...}, ...}
self.ipyFuncConfigDict = {} # 功能配置表缓存 {key:IPY_FuncConfig, ...}
- self.IpyDataClear()
+ self.classSizeDict = {} # 数据表类对象占用内存大小 {dtName:size, ...}
+ self.IpyDataClear(True)
return
- def IpyDataClear(self):
- Log("IPY_DataMgr Init...")
- self.ipyConfigEx = {}
+ def IpyDataClear(self, onlyCheck=False):
+ Log("IPY_DataMgr Reload... onlyCheck=%s" % onlyCheck)
+ if not onlyCheck:
+ self.ipyConfigEx = {}
#<%Ipy_Cache_Init%>
- Log("IPY_FuncConfig count=%s" % len(self.ipyFuncConfigDict))
- Log("IPY_DataMgr InitOK!")
+ Log("IPY_DataMgr ReloadOK! onlyCheck=%s" % onlyCheck)
return
- def __LoadFileData(self, tableName, Class):
+ def CheckLoadData(self, dtName):
+ if hasattr(self, "ipy%sLen" % dtName):
+ return
+ setattr(self, "ipy%sLen" % dtName, 0) # 设置该属性，通过判断是否有该属性决定是否加载
+ self.__LoadFileData(dtName)
+ return
+
+ def __LoadFileData(self, tableName, onlyCheck=False):
+ # @param onlyCheck: 是否仅检查格式，一般启动时用
curPath = "<%LoadStructPath%>" + "\\PySysDB\\tag" + tableName + ".txt"
if not os.path.isfile(curPath):
ErrLog("can not find file = %s,%s" % (tableName, curPath))
raise Exception("can not find file = %s,%s" % (tableName, curPath))
+ if not onlyCheck:
+ if not hasattr(self, "ipy%sLen" % tableName):
+ # 没有该属性的不加载
+ return
+
fileObj = open(curPath, 'rb')
content = fileObj.read()
fileObj.close()
- md5_obj = hashlib.md5()
- md5_obj.update(content)
- newMD5Code = md5_obj.hexdigest()
- if tableName in self.fileMD5Dict:
- oldMD5Code = self.fileMD5Dict[tableName]
- if newMD5Code == oldMD5Code:
- return getattr(self, "ipy%sCache" % tableName)
+ if not onlyCheck:
+ md5_obj = hashlib.md5()
+ md5_obj.update(content)
+ newMD5Code = md5_obj.hexdigest()
+ if tableName in self.fileMD5Dict:
+ oldMD5Code = self.fileMD5Dict[tableName]
+ if newMD5Code == oldMD5Code:
+ return getattr(self, "ipy%sCache" % tableName)
+
+ if tableName in self.ipyDataIndexMap:
+ self.ipyDataIndexMap.pop(tableName)
+ for dtName_Findkey in self.ipyDataIndexMapEx.keys():
+ findStr = "%s_" % tableName
+ if findStr in dtName_Findkey:
+ self.ipyDataIndexMapEx.pop(dtName_Findkey)
+ if tableName == "FuncConfig":
+ self.ipyFuncConfigDict = {}
+ self.fileMD5Dict[tableName] = newMD5Code
- if tableName in self.ipyDataIndexMap:
- self.ipyDataIndexMap.pop(tableName)
- for dtName_Findkey in self.ipyDataIndexMapEx.keys():
- findStr = "%s_" % tableName
- if findStr in dtName_Findkey:
- self.ipyDataIndexMapEx.pop(dtName_Findkey)
- if tableName == "FuncConfig":
- self.ipyFuncConfigDict = {}
- self.fileMD5Dict[tableName] = newMD5Code
-
dataIndex = 0
indexDict = {}
cacheList = []
fieldList = Def_IpyTable[tableName]
infoList = content.split('\r\n')
+ curClassSizeTotal = 0
+ dataCount = 0
+ Class = eval("IPY_%s" % tableName)
for line in xrange(len(infoList)):
if line == 0:
continue
@@ -109,10 +128,16 @@
if len(fieldList) != len(rowList):
ErrLog("field count error!, %s, line=%s, len=%s,rowCount=%s" % (tableName, line, len(fieldList), len(rowList)))
raise Exception("field count error!, %s, line=%s, len=%s,rowCount=%s" % (tableName, line,len(fieldList), len(rowList)))
-
+ if tableName == "FuncConfig":
+ classObj = self.__LoadFuncConfigData(fieldList, rowList, onlyCheck)
+ dataCount += 1
+ if onlyCheck:
+ continue
+ curClassSizeTotal += ChConfig.GetSizeof(classObj)
+ continue
try:
indexKey = []
- classObj = Class()
+ valueList = []
for j, value in enumerate(rowList):
fieldType, fieldName, isIndex = fieldList[j]
if fieldType == "char":
@@ -127,9 +152,15 @@
attrValue = float(value)
else:
attrValue = 0 if not value.isdigit() else int(value)
- setattr(classObj, "%s" % fieldName, attrValue)
+ valueList.append(attrValue)
if isIndex:
indexKey.append(attrValue)
+ dataCount += 1
+ if onlyCheck:
+ continue
+ classObj = Class()
+ setattr(classObj, "attrTuple", tuple(valueList))
+ curClassSizeTotal += ChConfig.GetSizeof(classObj)
cacheList.append(classObj)
indexKey = tuple(indexKey)
indexList = indexDict.get(indexKey, [])
@@ -139,22 +170,26 @@
except BaseException:
ErrLog("SetIpyDataError: tableName=%s,line=%s,fieldName=%s,fieldType=%s,value=%s" % (tableName, line, fieldName, fieldType, value))
raise
- if tableName == "FuncConfig":
- self.__LoadFuncConfigData(fieldList, rowList)
+ if onlyCheck:
+ Log("CheckIpydata: %s, dataCount=%s OK" % (tableName, dataCount))
+ return
if tableName != "FuncConfig":
self.ipyDataIndexMap[tableName] = indexDict
- Log("LoadIpydata: %s, content count=%s" % (tableName, len(cacheList)))
- return cacheList
+ self.classSizeDict[tableName] = curClassSizeTotal
+ classSizeTotal = sum(self.classSizeDict.values())
+ alreadyLoad = len(self.classSizeDict)
+ setattr(self, "ipy%sCache" % tableName, cacheList)
+ setattr(self, "ipy%sLen" % tableName, len(cacheList))
+ Log("LoadIpydata: %s, dataCount=%s, alreadyLoad=%s" % (tableName, dataCount, alreadyLoad), curClassSizeTotal, classSizeTotal)
+ return
- def __LoadFuncConfigData(self, fieldList, rowList):
- funcConfigObj = IPY_FuncConfig()
+ def __LoadFuncConfigData(self, fieldList, rowList, onlyCheck):
key = rowList[0]
- funcConfigObj.Key = key
+ valueList = [key]
for i, strValue in enumerate(rowList):
if i == 0:
continue
try:
- fieldName = fieldList[i][1]
strValue = strValue.lstrip().rstrip()
if strValue.isdigit():
configValue = int(strValue)
@@ -173,9 +208,15 @@
except BaseException:
ErrLog("SetIpyDataError: tableName=%s,key=%s,i=%s,value=%s" % ("FuncConfig", key, i, strValue))
raise
- setattr(funcConfigObj, fieldName, configValue)
+ if onlyCheck:
+ continue
+ valueList.append(configValue)
+ if onlyCheck:
+ return
+ funcConfigObj = IPY_FuncConfig()
+ setattr(funcConfigObj, "attrTuple", tuple(valueList))
self.ipyFuncConfigDict[key] = funcConfigObj
- return
+ return funcConfigObj
def __ToFloat(self, strValue):
try:
@@ -255,6 +296,7 @@
@return: 对应查询条件的 ipyData 数据实例，只返回单个实例
@使用说明: IpyGameDataPY.GetIpyGameData(表名, 索引1查询值, 索引2查询值, … )
'''
+ IPYData.CheckLoadData(dtName)
if dtName not in IPYData.ipyDataIndexMap:
ErrLog("Can not found ipyData dtName=%s" % (dtName))
return
@@ -272,6 +314,7 @@
@return: 对应查询条件的 ipyData 数据实例列表
@使用说明: 与 GetIpyGameData 函数相同
'''
+ IPYData.CheckLoadData(dtName)
if dtName not in IPYData.ipyDataIndexMap:
ErrLog("Can not found ipyDataList dtName=%s" % (dtName))
return
@@ -286,6 +329,7 @@
def GetIpyGameDataNotLog(dtName, *args):
'''与 GetIpyGameData 函数相同, 只是找不到数据时不会输出日志
'''
+ IPYData.CheckLoadData(dtName)
if dtName not in IPYData.ipyDataIndexMap:
#ErrLog("Can not found ipyData dtName=%s" % (dtName))
return
@@ -299,6 +343,7 @@
def GetIpyGameDataListNotLog(dtName, *args):
'''与 GetIpyGameDataList 函数相同, 只是找不到数据时不会输出日志
'''
+ IPYData.CheckLoadData(dtName)
if dtName not in IPYData.ipyDataIndexMap:
#ErrLog("Can not found ipyDataList dtName=%s" % (dtName))
return
@@ -318,6 +363,7 @@
@param isLogNone: 找不到数据时是否数据日志，默认是
@return: 找不到数据时返回 None，有数据时根据参数是否返回列表返回对应的数据实例或数据实例列表
'''
+ IPYData.CheckLoadData(dtName)
fieldList = keyDict.keys()
valueList = keyDict.values()
findFieldKey = "%s_%s" % (dtName, fieldList)
@@ -327,7 +373,7 @@
if findFieldKey not in IPYData.ipyDataIndexMapEx:
indexMapDict = {}
for index, iData in enumerate(cacheList):
- valuekey = tuple([getattr(iData, "%s" % field) for field in fieldList])
+ valuekey = tuple([getattr(iData, "Get%s" % field)() for field in fieldList])
indexList = indexMapDict.get(valuekey, [])
indexList.append(index)
indexMapDict[valuekey] = indexList
@@ -348,6 +394,7 @@
@param key: 配置key
@return: 直接返回该配置key对应的配置ipyData实例
'''
+ IPYData.CheckLoadData("FuncConfig")
if key not in IPYData.ipyFuncConfigDict:
ErrLog("Can not found ipyData FuncConfig key=%s!" % key)
return ""
@@ -359,20 +406,21 @@
@param index: 第几个配置值，支持1~5
@return: 直接返回对应的数据类型 int、str，不用再手动转int
'''
+ IPYData.CheckLoadData("FuncConfig")
if key not in IPYData.ipyFuncConfigDict:
ErrLog("Can not found ipyData FuncConfig key=%s!" % key)
return ""
cfgObj = IPYData.ipyFuncConfigDict[key]
if index == 1:
- return cfgObj.Numerical1
+ return cfgObj.attrTuple[1]
if index == 2:
- return cfgObj.Numerical2
+ return cfgObj.attrTuple[2]
if index == 3:
- return cfgObj.Numerical3
+ return cfgObj.attrTuple[3]
if index == 4:
- return cfgObj.Numerical4
+ return cfgObj.attrTuple[4]
if index == 5:
- return cfgObj.Numerical5
+ return cfgObj.attrTuple[5]
ErrLog("Can not found ipyData FuncConfig key=%s,index=%s!" % (key, index))
return ""
@@ -387,20 +435,21 @@
当然如果配置的内容本身就为python的列表、字典结构的话可使用上面的函数
不过为了统一，建议功能配置表读列表、字典时都使用该函数
'''
+ IPYData.CheckLoadData("FuncConfig")
if key not in IPYData.ipyFuncConfigDict:
ErrLog("Can not found ipyData FuncConfig key=%s!" % key)
return defaultValue
cfgObj = IPYData.ipyFuncConfigDict[key]
if index == 1:
- curConfig = cfgObj.Numerical1
+ curConfig = cfgObj.attrTuple[1]
elif index == 2:
- curConfig = cfgObj.Numerical2
+ curConfig = cfgObj.attrTuple[2]
elif index == 3:
- curConfig = cfgObj.Numerical3
+ curConfig = cfgObj.attrTuple[3]
elif index == 4:
- curConfig = cfgObj.Numerical4
+ curConfig = cfgObj.attrTuple[4]
elif index == 5:
- curConfig = cfgObj.Numerical5
+ curConfig = cfgObj.attrTuple[5]
else:
ErrLog("Can not found ipyData FuncConfig key=%s,index=%s!" % (key, index))
return defaultValue
@@ -427,6 +476,7 @@
@param conditionDict: 查询条件，{查询字段名:字段值, ...}
@return: 找不到数据返回 None ， 否则返回对应的 ipyData 数据实例
'''
+ IPYData.CheckLoadData(dtName)
if not conditionDict:
dataList = getattr(IPYData, "ipy%sCache" % dtName)
else:
@@ -436,34 +486,88 @@
low = 0
lowData = dataList[0]
- lowValue = getattr(lowData, "%s" % keyName)
+ lowValue = getattr(lowData, "Get%s" % keyName)()
if keyValue < lowValue:
return
high = len(dataList) - 1
highData = dataList[high]
- highValue = getattr(highData, "%s" % keyName)
+ highValue = getattr(highData, "Get%s" % keyName)()
if keyValue >= highValue:
return highData
near = low + int((high - low) * (keyValue - lowValue)/(highValue - lowValue))
nearData = dataList[near]
- nearValue = getattr(nearData, "%s" % keyName)
+ nearValue = getattr(nearData, "Get%s" % keyName)()
if keyValue < nearValue:
for i in xrange(near - 1, low - 1, -1):
nearData = dataList[i]
- nearValue = getattr(nearData, "%s" % keyName)
+ nearValue = getattr(nearData, "Get%s" % keyName)()
if nearValue <= keyValue:
return nearData
elif keyValue > nearValue:
for i in xrange(near + 1, high + 1):
nearData = dataList[i]
- nearValue = getattr(nearData, "%s" % keyName)
+ nearValue = getattr(nearData, "Get%s" % keyName)()
if nearValue > keyValue:
return dataList[i - 1]
return nearData
+#if __name__ == "__main__":
+# IPYDataTotalSize = ChConfig.GetSizeof(IPYData)
+# classSizeSum = sum(IPYData.classSizeDict.values())
+# Log("数据类内存: %s b = %s M" % (classSizeSum, str(classSizeSum/float(1024*1024))))
+# Log("总占用内存: %s b = %s M" % (IPYDataTotalSize, str(IPYDataTotalSize/float(1024*1024))))
+# for key in ["OpenBagItem", "RuneUnlock", "IceLodeNeedPoint", "RealmDifficulty", "TongTianLing", "TongTianLingaaa"]:
+# for i in range(1, 6):
+# v = GetFuncCfg(key, i)
+# Log("key:%s,i=%s,value=%s\t%s" % (key, i, type(v), str(v)))
+#
+# ipyData = GetIpyGameData("FlashGiftbag", 303)
+# giftbagIpyDataList = GetIpyGameDataByCondition("FlashGiftbag", {"GiftbagType":1002}, True, True)
+# for ipyData in giftbagIpyDataList:
+# print
+# print type(ipyData.GetGiftbagID()), ipyData.GetGiftbagID()
+# print type(ipyData.GetGiftbagType()), ipyData.GetGiftbagType()
+# print type(ipyData.GetOriginalRMB()), ipyData.GetOriginalRMB()
+# print type(ipyData.GetBuyCountLimit()), ipyData.GetBuyCountLimit()
+# print type(ipyData.GetGiftItemList()), ipyData.GetGiftItemList()
+# print type(ipyData.GetMainItemID()), ipyData.GetMainItemID()
+# print type(ipyData.GetNotifyKey()), ipyData.GetNotifyKey()
+#
+# collNPCIpyDataList = GetIpyGameDataListNotLog("MapRefreshNPC", 32050)
+# if collNPCIpyDataList:
+# npcIDList = []
+# for ipyData in collNPCIpyDataList:
+# print
+# print type(ipyData.GetMapID()), ipyData.GetMapID()
+# print type(ipyData.GetNPCIDList()), ipyData.GetNPCIDList()
+# print type(ipyData.GetRefreshMarkList()), ipyData.GetRefreshMarkList()
+# npcIDList += ipyData.GetNPCIDList()
+# print "npcIDList", npcIDList
+# else:
+# print "None"
+#
+# ipyData = GetIpyGameData("RuneCompound", 4415)
+# if ipyData:
+# print
+# print type(ipyData.GetTagItemID()), ipyData.GetTagItemID()
+# print type(ipyData.GetNeedItem()), ipyData.GetNeedItem()
+# print type(ipyData.GetNeedMJ()), ipyData.GetNeedMJ()
+# else:
+# print "None"
+#
+# ipyData = GetIpyGameData("DailyGiftbag", 703, 105)
+# if ipyData:
+# print
+# print type(ipyData.GetGiftbagType()), ipyData.GetGiftbagType()
+# print type(ipyData.GetGiftbagID()), ipyData.GetGiftbagID()
+# print type(ipyData.GetBuyCountLimit()), ipyData.GetBuyCountLimit()
+# print type(ipyData.GetGiftItemList()), ipyData.GetGiftItemList()
+# else:
+# print "None"
+
\ No newline at end of file
diff --git a/ServerPython/CoreServerGroup/GameServer/Script/ChConfig.py b/ServerPython/CoreServerGroup/GameServer/Script/ChConfig.py
index 25e668a..c13904f 100644
--- a/ServerPython/CoreServerGroup/GameServer/Script/ChConfig.py
+++ b/ServerPython/CoreServerGroup/GameServer/Script/ChConfig.py
@@ -58,6 +58,12 @@
##获取服务器根路径
return GameServerPath.split("CoreServerGroup")[0]
+def GetSizeof(o, isAsize=True):
+ return 0
+ #if isAsize:
+ # return asizeof.asizeof(o)
+ #return sys.getsizeof(0)
+
#------主角相关设定
#主角发型列表 [职业:发型列表]
Def_RoleHair = {
diff --git a/ServerPython/ZoneServerGroup/map1_8G/MapServer/MapServerData/Script/ChConfig.py b/ServerPython/ZoneServerGroup/map1_8G/MapServer/MapServerData/Script/ChConfig.py
index 995c3cf..64f234c 100644
--- a/ServerPython/ZoneServerGroup/map1_8G/MapServer/MapServerData/Script/ChConfig.py
+++ b/ServerPython/ZoneServerGroup/map1_8G/MapServer/MapServerData/Script/ChConfig.py
@@ -49,7 +49,12 @@
##获取服务器根路径
return MapServerPath.split("ZoneServerGroup")[0]
-
+def GetSizeof(o, isAsize=True):
+ return 0
+ #if isAsize:
+ # return asizeof.asizeof(o)
+ #return sys.getsizeof(0)
+
# CanRepeatTime字段个位数
(
Def_BuffTime_Reset, # 0 重置时间
--
Gitblit v1.8.0