10138 内存分析（优化py自定配置表数据内存占用及加载方式）

hxp

2024-03-19 22e848871436811496f03dae3cf536c8fb53cfb8

10138 内存分析（优化py自定配置表数据内存占用及加载方式）

# Conflicts:
#	ServerPython/CoreServerGroup/GameServer/Script/IpyGameDataPY.py
#	ServerPython/ZoneServerGroup/map1_8G/MapServer/MapServerData/Script/IpyGameDataPY.py

 PySysDB/Éú³ÉIpyGameDataPY/IpyGameDataPYCreater.py

@@ -38,12 +38,16 @@
ClassFuncTemp = "    def Get%s(self): return self.%s%s%s"



# ç®¡ç†å™¨åˆå§‹åŒ–表数据缓存列表对象模板

MgrTableCacheInit = "        self.ipy%sCache = self.__LoadFileData(\"%s\", IPY_%s)%s"

MgrTableLenInit = "        self.ipy%sLen = len(self.ipy%sCache)%s"

# ç®¡ç†å™¨èŽ·å–表数据条数函数模板

MgrGetCountFunc = "    def Get%sCount(self): return self.ipy%sLen%s"

# ç®¡ç†å™¨æ ¹æ®ç´¢å¼•èŽ·å–表数据函数模板

MgrGetByIndexFunc = "    def Get%sByIndex(self, index): return self.ipy%sCache[index]%s"

MgrTableCacheInit = "        self.__LoadFileData(\"%s\", onlyCheck)%s"

# ç®¡ç†å™¨èŽ·å–表数据条数、数据函数模板

MgrGetCountFunc = '''

    def Get%sCount(self):

        self.CheckLoadData("%s")

        return self.ipy%sLen\r

    def Get%sByIndex(self, index):

        self.CheckLoadData("%s")

        return self.ipy%sCache[index]\r

'''



Def_RN = "\r\n"



@@ -159,19 +163,18 @@
                classInitInfo += ClassInitTemp % (fieldName, "0.0", lineEnd)

            else:

                classInitInfo += ClassInitTemp % (fieldName, "0", lineEnd)

            classFuncInfo += ClassFuncTemp % (fieldName, fieldName, noteInfo, lineEnd)

            callAttrValue = "attrTuple[%s]" % j # fieldName

            classFuncInfo += ClassFuncTemp % (fieldName, callAttrValue, noteInfo, lineEnd)

            

        ipyTableDef += TableDefEnd % (Def_RN if i == len(Def_IpyTable) - 1 else (Def_RN * 2))

        classInitInfo = ClassInitTemp % ("attrTuple", "None", "") # ä¼˜åŒ–内存，不使用内置 __dict__ è®¿é—®å±žæ€§ï¼Œæ”¹ä¸ºä½¿ç”¨tuple存value

        classContent += ClassTemp % (tableNameCh, tableName, classInitInfo, classFuncInfo)

        

        # è¡¨åˆ—表、长度对象缓存

        mgrTableCacheObjInit += MgrTableCacheInit % (tableName, tableName, tableName, Def_RN)

        mgrTableCacheObjInit += MgrTableLenInit % (tableName, tableName, Def_RN)

        mgrTableCacheObjInit += MgrTableCacheInit % (tableName, Def_RN)

        

        # è¡¨åˆ—表、长度获取函数

        mgrTableFunc += MgrGetCountFunc % (tableName, tableName, Def_RN)

        mgrTableFunc += MgrGetByIndexFunc % (tableName, tableName, Def_RN)

        
        mgrTableFunc += MgrGetCountFunc % (tableName, tableName, tableName, tableName, tableName, tableName)

    

    # è¯»å–模板生成py代码

    createPYContent = ""


 PySysDB/Éú³ÉIpyGameDataPY/IpyGameDataPYTemp.py

@@ -50,55 +50,74 @@
class IPY_DataMgr():

    

    def __init__(self):

        Log("IPY_DataMgr Init...")

        self.fileMD5Dict = {} # æ•°æ®è¡¨æ–‡ä»¶md5字典, ç”¨äºŽå¯¹æ¯”文件差异判断是否重读 {dtName:md5, ...}

        self.ipyConfigEx = {} # è‡ªå®šä¹‰æ•°æ®ç¼“å­˜ {key:configData, ...}

        self.ipyDataIndexMap = {} # æ•°æ®è¡¨ç´¢å¼•æŸ¥è¯¢ç¼“å­˜ {dtName:{args:[index, ...], ...}, ...}

        self.ipyDataIndexMapEx = {} # æ•°æ®è¡¨è‡ªå®šä¹‰æŸ¥è¯¢æ¡ä»¶æŸ¥è¯¢ç¼“å­˜ {dtName_(findKey,...):{(findKeyValue, ...):[index, ...], ...}, ...}

        self.ipyFuncConfigDict = {} # åŠŸèƒ½é…ç½®è¡¨ç¼“å­˜ {key:IPY_FuncConfig, ...}

        self.IpyDataClear()

        self.classSizeDict = {} # æ•°æ®è¡¨ç±»å¯¹è±¡å ç”¨å†…存大小 {dtName:size, ...}

        self.IpyDataClear(True)

        return

    

    def IpyDataClear(self):

        Log("IPY_DataMgr Init...")

        self.ipyConfigEx = {}

    def IpyDataClear(self, onlyCheck=False):

        Log("IPY_DataMgr Reload... onlyCheck=%s" % onlyCheck)

        if not onlyCheck:

            self.ipyConfigEx = {}

        #<%Ipy_Cache_Init%>

        Log("IPY_FuncConfig count=%s" % len(self.ipyFuncConfigDict))

        Log("IPY_DataMgr InitOK!")

        Log("IPY_DataMgr ReloadOK! onlyCheck=%s" % onlyCheck)

        return

    

    def __LoadFileData(self, tableName, Class):

    def CheckLoadData(self, dtName):

        if hasattr(self, "ipy%sLen" % dtName):

            return

        setattr(self, "ipy%sLen" % dtName, 0) # è®¾ç½®è¯¥å±žæ€§ï¼Œé€šè¿‡åˆ¤æ–­æ˜¯å¦æœ‰è¯¥å±žæ€§å†³å®šæ˜¯å¦åŠ è½½

        self.__LoadFileData(dtName)

        return

    
    def __LoadFileData(self, tableName, onlyCheck=False):

        # @param onlyCheck: æ˜¯å¦ä»…检查格式，一般启动时用

        curPath = "<%LoadStructPath%>" + "\\PySysDB\\tag" + tableName + ".txt"

        if not os.path.isfile(curPath):

            ErrLog("can not find file = %s,%s" % (tableName, curPath))

            raise Exception("can not find file = %s,%s" % (tableName, curPath))

        

        if not onlyCheck:

            if not hasattr(self, "ipy%sLen" % tableName):

                # æ²¡æœ‰è¯¥å±žæ€§çš„不加载

                return

        
        fileObj = open(curPath, 'rb')

        content = fileObj.read()

        fileObj.close()

        

        md5_obj = hashlib.md5()

        md5_obj.update(content)

        newMD5Code = md5_obj.hexdigest()

        if tableName in self.fileMD5Dict:

            oldMD5Code = self.fileMD5Dict[tableName]

            if newMD5Code == oldMD5Code:

                return getattr(self, "ipy%sCache" % tableName)

        if not onlyCheck:

            md5_obj = hashlib.md5()

            md5_obj.update(content)

            newMD5Code = md5_obj.hexdigest()

            if tableName in self.fileMD5Dict:

                oldMD5Code = self.fileMD5Dict[tableName]

                if newMD5Code == oldMD5Code:

                    return getattr(self, "ipy%sCache" % tableName)

                
                if tableName in self.ipyDataIndexMap:

                    self.ipyDataIndexMap.pop(tableName)

                for dtName_Findkey in self.ipyDataIndexMapEx.keys():

                    findStr = "%s_" % tableName

                    if findStr in dtName_Findkey:

                        self.ipyDataIndexMapEx.pop(dtName_Findkey)

                if tableName == "FuncConfig":

                    self.ipyFuncConfigDict = {}

            self.fileMD5Dict[tableName] = newMD5Code

            

            if tableName in self.ipyDataIndexMap:

                self.ipyDataIndexMap.pop(tableName)

            for dtName_Findkey in self.ipyDataIndexMapEx.keys():

                findStr = "%s_" % tableName

                if findStr in dtName_Findkey:

                    self.ipyDataIndexMapEx.pop(dtName_Findkey)

            if tableName == "FuncConfig":

                self.ipyFuncConfigDict = {}

        self.fileMD5Dict[tableName] = newMD5Code

        
        dataIndex = 0

        indexDict = {}

        cacheList = []

        fieldList = Def_IpyTable[tableName]

        infoList = content.split('\r\n')

        curClassSizeTotal = 0

        dataCount = 0

        Class = eval("IPY_%s" % tableName)

        for line in xrange(len(infoList)):

            if line == 0:

                continue

@@ -109,10 +128,16 @@
            if len(fieldList) != len(rowList):

                ErrLog("field count error!, %s, line=%s, len=%s,rowCount=%s" % (tableName, line, len(fieldList), len(rowList)))

                raise Exception("field count error!, %s, line=%s, len=%s,rowCount=%s" % (tableName, line,len(fieldList), len(rowList)))

            
            if tableName == "FuncConfig":

                classObj = self.__LoadFuncConfigData(fieldList, rowList, onlyCheck)

                dataCount += 1

                if onlyCheck:

                    continue

                curClassSizeTotal += ChConfig.GetSizeof(classObj)

                continue

            try:

                indexKey = []

                classObj = Class()

                valueList = []

                for j, value in enumerate(rowList):

                    fieldType, fieldName, isIndex = fieldList[j]

                    if fieldType == "char":

@@ -127,9 +152,15 @@
                        attrValue = float(value)

                    else:

                        attrValue = 0 if not value.isdigit() else int(value)

                    setattr(classObj, "%s" % fieldName, attrValue)

                    valueList.append(attrValue)

                    if isIndex:

                        indexKey.append(attrValue)

                dataCount += 1

                if onlyCheck:

                    continue

                classObj = Class()

                setattr(classObj, "attrTuple", tuple(valueList))

                curClassSizeTotal += ChConfig.GetSizeof(classObj)

                cacheList.append(classObj)

                indexKey = tuple(indexKey)

                indexList = indexDict.get(indexKey, [])

@@ -139,22 +170,26 @@
            except BaseException:

                ErrLog("SetIpyDataError: tableName=%s,line=%s,fieldName=%s,fieldType=%s,value=%s" % (tableName, line, fieldName, fieldType, value))

                raise

            if tableName == "FuncConfig":

                self.__LoadFuncConfigData(fieldList, rowList)

        if onlyCheck:

            Log("CheckIpydata: %s, dataCount=%s OK" % (tableName, dataCount))

            return

        if tableName != "FuncConfig":

            self.ipyDataIndexMap[tableName] = indexDict

        Log("LoadIpydata: %s, content count=%s" % (tableName, len(cacheList)))

        return cacheList

        self.classSizeDict[tableName] = curClassSizeTotal

        classSizeTotal = sum(self.classSizeDict.values())

        alreadyLoad = len(self.classSizeDict)

        setattr(self, "ipy%sCache" % tableName, cacheList)

        setattr(self, "ipy%sLen" % tableName, len(cacheList))

        Log("LoadIpydata: %s, dataCount=%s, alreadyLoad=%s" % (tableName, dataCount, alreadyLoad), curClassSizeTotal, classSizeTotal)

        return

    

    def __LoadFuncConfigData(self, fieldList, rowList):

        funcConfigObj = IPY_FuncConfig()

    def __LoadFuncConfigData(self, fieldList, rowList, onlyCheck):

        key = rowList[0]

        funcConfigObj.Key = key

        valueList = [key]

        for i, strValue in enumerate(rowList):

            if i == 0:

                continue

            try:

                fieldName = fieldList[i][1]

                strValue = strValue.lstrip().rstrip()

                if strValue.isdigit():

                    configValue = int(strValue)

@@ -173,9 +208,15 @@
            except BaseException:

                ErrLog("SetIpyDataError: tableName=%s,key=%s,i=%s,value=%s" % ("FuncConfig", key, i, strValue))

                raise

            setattr(funcConfigObj, fieldName, configValue)

            if onlyCheck:

                continue

            valueList.append(configValue)

        if onlyCheck:

            return

        funcConfigObj = IPY_FuncConfig()

        setattr(funcConfigObj, "attrTuple", tuple(valueList))

        self.ipyFuncConfigDict[key] = funcConfigObj

        return

        return funcConfigObj

    

    def __ToFloat(self, strValue):

        try:

@@ -255,6 +296,7 @@
    @return: å¯¹åº”查询条件的 ipyData æ•°æ®å®žä¾‹ï¼Œåªè¿”回单个实例

    @使用说明: IpyGameDataPY.GetIpyGameData(表名, ç´¢å¼•1查询值, ç´¢å¼•2查询值, â€¦ )

    '''

    IPYData.CheckLoadData(dtName)

    if dtName not in IPYData.ipyDataIndexMap:

        ErrLog("Can not found ipyData dtName=%s" % (dtName))

        return

@@ -272,6 +314,7 @@
    @return: å¯¹åº”查询条件的 ipyData æ•°æ®å®žä¾‹åˆ—表

    @使用说明: ä¸Ž GetIpyGameData å‡½æ•°ç›¸åŒ

    '''

    IPYData.CheckLoadData(dtName)

    if dtName not in IPYData.ipyDataIndexMap:

        ErrLog("Can not found ipyDataList dtName=%s" % (dtName))

        return

@@ -286,6 +329,7 @@
def GetIpyGameDataNotLog(dtName, *args):

    '''与 GetIpyGameData å‡½æ•°ç›¸åŒ, åªæ˜¯æ‰¾ä¸åˆ°æ•°æ®æ—¶ä¸ä¼šè¾“出日志

    '''

    IPYData.CheckLoadData(dtName)

    if dtName not in IPYData.ipyDataIndexMap:

        #ErrLog("Can not found ipyData dtName=%s" % (dtName))

        return

@@ -299,6 +343,7 @@
def GetIpyGameDataListNotLog(dtName, *args):

    '''与 GetIpyGameDataList å‡½æ•°ç›¸åŒ, åªæ˜¯æ‰¾ä¸åˆ°æ•°æ®æ—¶ä¸ä¼šè¾“出日志

    '''

    IPYData.CheckLoadData(dtName)

    if dtName not in IPYData.ipyDataIndexMap:

        #ErrLog("Can not found ipyDataList dtName=%s" % (dtName))

        return

@@ -318,6 +363,7 @@
    @param isLogNone: æ‰¾ä¸åˆ°æ•°æ®æ—¶æ˜¯å¦æ•°æ®æ—¥å¿—，默认是

    @return: æ‰¾ä¸åˆ°æ•°æ®æ—¶è¿”回 None，有数据时根据参数是否返回列表返回对应的数据实例或数据实例列表

    '''

    IPYData.CheckLoadData(dtName)

    fieldList = keyDict.keys()

    valueList = keyDict.values()

    findFieldKey = "%s_%s" % (dtName, fieldList)

@@ -327,7 +373,7 @@
    if findFieldKey not in IPYData.ipyDataIndexMapEx:

        indexMapDict = {}

        for index, iData in enumerate(cacheList):

            valuekey = tuple([getattr(iData, "%s" % field) for field in fieldList])

            valuekey = tuple([getattr(iData, "Get%s" % field)() for field in fieldList])

            indexList = indexMapDict.get(valuekey, [])

            indexList.append(index)

            indexMapDict[valuekey] = indexList        

@@ -348,6 +394,7 @@
    @param key: é…ç½®key

    @return: ç›´æŽ¥è¿”回该配置key对应的配置ipyData实例

    '''

    IPYData.CheckLoadData("FuncConfig")

    if key not in IPYData.ipyFuncConfigDict:

        ErrLog("Can not found ipyData FuncConfig key=%s!" % key)

        return ""

@@ -359,20 +406,21 @@
    @param index: ç¬¬å‡ ä¸ªé…ç½®å€¼ï¼Œæ”¯æŒ1~5

    @return: ç›´æŽ¥è¿”回对应的数据类型 int、str，不用再手动转int

    '''

    IPYData.CheckLoadData("FuncConfig")

    if key not in IPYData.ipyFuncConfigDict:

        ErrLog("Can not found ipyData FuncConfig key=%s!" % key)

        return ""

    cfgObj = IPYData.ipyFuncConfigDict[key]

    if index == 1:

        return cfgObj.Numerical1

        return cfgObj.attrTuple[1]

    if index == 2:

        return cfgObj.Numerical2

        return cfgObj.attrTuple[2]

    if index == 3:

        return cfgObj.Numerical3

        return cfgObj.attrTuple[3]

    if index == 4:

        return cfgObj.Numerical4

        return cfgObj.attrTuple[4]

    if index == 5:

        return cfgObj.Numerical5

        return cfgObj.attrTuple[5]

    ErrLog("Can not found ipyData FuncConfig key=%s,index=%s!" % (key, index))

    return ""



@@ -387,20 +435,21 @@
    å½“然如果配置的内容本身就为python的列表、字典结构的话可使用上面的函数

    ä¸è¿‡ä¸ºäº†ç»Ÿä¸€ï¼Œå»ºè®®åŠŸèƒ½é…ç½®è¡¨è¯»åˆ—表、字典时都使用该函数

    '''

    IPYData.CheckLoadData("FuncConfig")

    if key not in IPYData.ipyFuncConfigDict:

        ErrLog("Can not found ipyData FuncConfig key=%s!" % key)

        return defaultValue

    cfgObj = IPYData.ipyFuncConfigDict[key]

    if index == 1:

        curConfig = cfgObj.Numerical1

        curConfig = cfgObj.attrTuple[1]

    elif index == 2:

        curConfig = cfgObj.Numerical2

        curConfig = cfgObj.attrTuple[2]

    elif index == 3:

        curConfig = cfgObj.Numerical3

        curConfig = cfgObj.attrTuple[3]

    elif index == 4:

        curConfig = cfgObj.Numerical4

        curConfig = cfgObj.attrTuple[4]

    elif index == 5:

        curConfig = cfgObj.Numerical5

        curConfig = cfgObj.attrTuple[5]

    else:

        ErrLog("Can not found ipyData FuncConfig key=%s,index=%s!" % (key, index))

        return defaultValue

@@ -427,6 +476,7 @@
    @param conditionDict: æŸ¥è¯¢æ¡ä»¶ï¼Œ{查询字段名:字段值, ...}

    @return: æ‰¾ä¸åˆ°æ•°æ®è¿”回 None ï¼Œ å¦åˆ™è¿”回对应的 ipyData æ•°æ®å®žä¾‹

    '''

    IPYData.CheckLoadData(dtName)

    if not conditionDict:

        dataList = getattr(IPYData, "ipy%sCache" % dtName)

    else:

@@ -436,34 +486,88 @@
    

    low = 0

    lowData = dataList[0]

    lowValue = getattr(lowData, "%s" % keyName)

    lowValue = getattr(lowData, "Get%s" % keyName)()

    if keyValue < lowValue:

        return

    

    high = len(dataList) - 1

    highData = dataList[high]

    highValue = getattr(highData, "%s" % keyName)

    highValue = getattr(highData, "Get%s" % keyName)()

    if keyValue >= highValue:

        return highData

    

    near = low + int((high - low) * (keyValue - lowValue)/(highValue - lowValue))

    

    nearData = dataList[near]

    nearValue = getattr(nearData, "%s" % keyName)

    nearValue = getattr(nearData, "Get%s" % keyName)()

    

    if keyValue < nearValue:

        for i in xrange(near - 1, low - 1, -1):

            nearData = dataList[i]

            nearValue = getattr(nearData, "%s" % keyName)

            nearValue = getattr(nearData, "Get%s" % keyName)()

            if nearValue <= keyValue:

                return nearData

            

    elif keyValue > nearValue:

        for i in xrange(near + 1, high + 1):

            nearData = dataList[i]

            nearValue = getattr(nearData, "%s" % keyName)

            nearValue = getattr(nearData, "Get%s" % keyName)()

            if nearValue > keyValue:

                return dataList[i - 1]

            

    return nearData



#if __name__ == "__main__":

#    IPYDataTotalSize = ChConfig.GetSizeof(IPYData)

#    classSizeSum = sum(IPYData.classSizeDict.values())

#    Log("数据类内存: %s b = %s M" % (classSizeSum, str(classSizeSum/float(1024*1024))))

#    Log("总占用内存: %s b = %s M" % (IPYDataTotalSize, str(IPYDataTotalSize/float(1024*1024))))

#    for key in ["OpenBagItem", "RuneUnlock", "IceLodeNeedPoint", "RealmDifficulty", "TongTianLing", "TongTianLingaaa"]:

#        for i in range(1, 6):

#            v = GetFuncCfg(key, i)

#            Log("key:%s,i=%s,value=%s\t%s" % (key, i, type(v), str(v)))

#            
#    ipyData = GetIpyGameData("FlashGiftbag", 303)

#    giftbagIpyDataList = GetIpyGameDataByCondition("FlashGiftbag", {"GiftbagType":1002}, True, True)

#    for ipyData in giftbagIpyDataList:

#        print 
#        print type(ipyData.GetGiftbagID()), ipyData.GetGiftbagID()

#        print type(ipyData.GetGiftbagType()), ipyData.GetGiftbagType()

#        print type(ipyData.GetOriginalRMB()), ipyData.GetOriginalRMB()

#        print type(ipyData.GetBuyCountLimit()), ipyData.GetBuyCountLimit()

#        print type(ipyData.GetGiftItemList()), ipyData.GetGiftItemList()

#        print type(ipyData.GetMainItemID()), ipyData.GetMainItemID()

#        print type(ipyData.GetNotifyKey()), ipyData.GetNotifyKey()

#        
#    collNPCIpyDataList = GetIpyGameDataListNotLog("MapRefreshNPC", 32050)

#    if collNPCIpyDataList:

#        npcIDList = []

#        for ipyData in collNPCIpyDataList:

#            print 
#            print type(ipyData.GetMapID()), ipyData.GetMapID()

#            print type(ipyData.GetNPCIDList()), ipyData.GetNPCIDList()

#            print type(ipyData.GetRefreshMarkList()), ipyData.GetRefreshMarkList()

#            npcIDList += ipyData.GetNPCIDList()

#        print "npcIDList", npcIDList

#    else:

#        print "None"

#        
#    ipyData = GetIpyGameData("RuneCompound", 4415)

#    if ipyData:

#        print 
#        print type(ipyData.GetTagItemID()), ipyData.GetTagItemID()

#        print type(ipyData.GetNeedItem()), ipyData.GetNeedItem()

#        print type(ipyData.GetNeedMJ()), ipyData.GetNeedMJ()

#    else:

#        print "None"

#        
#    ipyData = GetIpyGameData("DailyGiftbag", 703, 105)

#    if ipyData:

#        print 
#        print type(ipyData.GetGiftbagType()), ipyData.GetGiftbagType()

#        print type(ipyData.GetGiftbagID()), ipyData.GetGiftbagID()

#        print type(ipyData.GetBuyCountLimit()), ipyData.GetBuyCountLimit()

#        print type(ipyData.GetGiftItemList()), ipyData.GetGiftItemList()

#    else:

#        print "None"

            

 ServerPython/CoreServerGroup/GameServer/Script/ChConfig.py

@@ -58,6 +58,12 @@
    ##获取服务器根路径

    return GameServerPath.split("CoreServerGroup")[0]



def GetSizeof(o, isAsize=True):

    return 0

    #if isAsize:

    #    return asizeof.asizeof(o)

    #return sys.getsizeof(0)

    
#------主角相关设定

#主角发型列表 [职业:发型列表]

Def_RoleHair = {


 ServerPython/ZoneServerGroup/map1_8G/MapServer/MapServerData/Script/ChConfig.py

@@ -49,7 +49,12 @@
    ##获取服务器根路径

    return MapServerPath.split("ZoneServerGroup")[0]





def GetSizeof(o, isAsize=True):

    return 0

    #if isAsize:

    #    return asizeof.asizeof(o)

    #return sys.getsizeof(0)

    
# CanRepeatTime字段个位数

(

Def_BuffTime_Reset,     # 0    é‡ç½®æ—¶é—´