#!/usr/bin/python
|
# -*- coding: utf-8 -*-
|
#-------------------------------------------------------------------------------
|
|
import sys
|
import datetime
|
import traceback
|
import urllib2
|
import urllib
|
import json
|
import os
|
import logging
|
import gettext
|
import sys
|
import ConfigParser
|
|
Encoding = "utf-8"
|
|
FinalDataSign = "_FinalData"
|
|
def encode(s):
|
if isinstance(s, unicode):
|
return s.encode(Encoding)
|
elif isinstance(s, int) or isinstance(s, long):
|
return s
|
elif not isinstance(s, str):
|
return str(s)
|
return s
|
|
def toGBK(s):
|
return s.decode(Encoding).encode("gbk")
|
|
def encodePych(pyChStr):
|
## 转化py中文
|
return unicode(pyChStr, Encoding).encode(Encoding)
|
|
def setdefaultencoding(encoding=None):
|
reload(sys)
|
sys.setdefaultencoding(encoding if encoding else Encoding)
|
return
|
|
def parse_args():
|
## 解析运行参数
|
#print "sys.argv: ", len(sys.argv), sys.argv
|
argvDict = {}
|
for argvInfo in sys.argv[1:]:
|
kvList = argvInfo.split("=")
|
if not kvList:
|
continue
|
value = kvList[1] if len(kvList) > 1 else ""
|
# 中文参数超过4个字会有问题，但是用符号包起来就没问题，暂未知原因，先这样处理，参数固定用[]包起来
|
if value.startswith("[") and value.endswith("]"):
|
value = value[1:-1]
|
argvDict[kvList[0]] = value
|
#print "parse_args: ", argvDict
|
return argvDict
|
|
def toInt(inputValue, defValue=0):
|
try:
|
result = int(inputValue)
|
return result
|
except:
|
return defValue
|
|
def checkDate(startDate, endDate, checkDate):
|
## 检查对应日期是否在指定日期范围内，参数都是 datetime 类型
|
|
if not checkDate:
|
return False
|
|
if startDate and checkDate < startDate:
|
return False
|
|
if endDate and checkDate > endDate:
|
return False
|
|
return True
|
|
def strToDatetime(timeStr, strFormat="%Y-%m-%d"):
|
try:
|
return datetime.datetime.strptime(timeStr, strFormat)
|
except:
|
return
|
|
def getDiffDaysDate(diffDays, dateObj=None):
|
## 日期加减运算
|
if dateObj == None:
|
dateObj = datetime.datetime.today()
|
return dateObj + datetime.timedelta(days=diffDays)
|
|
def getCurrentTime():
|
return datetime.datetime.today()
|
|
def getCurrentDateStr():
|
curTime = datetime.datetime.today()
|
return "%d-%02d-%02d" % (curTime.year, curTime.month, curTime.day)
|
|
def getServerID(accID):
|
infoList = accID.split("@")
|
return 0 if len(infoList) < 3 else int(infoList[-1][1:])
|
|
def coinToY(coinValue, rate=100):
|
## coin转为元
|
toValue = coinValue / float(rate)
|
if str(toValue).endswith(".0"):
|
toValue = int(toValue)
|
return toValue
|
|
def getCoinRate(cfg, spid):
|
## 获取coin的转化比例，默认100
|
if cfg.has_option("ExChange", "ExChangeScale_%s" % spid):
|
return int(cfg.get("ExChange", "ExChangeScale_%s" % spid))
|
if cfg.has_option("ExChange", "ExChangeScale"):
|
return int(cfg.get("ExChange", "ExChangeScale"))
|
return 100
|
|
def roundInt(value, ndigits=2):
|
## 保留x位小数，如果刚好x.0则取整
|
value = round(value, ndigits)
|
if str(value).endswith(".0"):
|
value = int(value)
|
return value
|
|
def matchDr(drDict, matchSet={}, filterSet={}):
|
''' 检查流向数据是否是需要满足匹配条件的数据
|
@param matchSet: 匹配设定 {匹配key:[匹配值列表], ...}, 即只会匹配对应key为指定值列表的内容
|
@param filterSet: 过滤设定, {过滤key:[过滤值列表], ...}, 即会过滤掉对应key为某些值的内容
|
@return: 是否是需要统计的流向数据
|
'''
|
# 优先检查被过滤掉的
|
for filterKey, filterList in filterSet.items():
|
if filterKey not in drDict:
|
continue
|
if drDict[filterKey] in filterList:
|
return False
|
|
# 检查精确匹配的
|
for matchKey, matchList in matchSet.items():
|
if matchKey not in drDict:
|
return False
|
if drDict[matchKey] not in matchList:
|
return False
|
|
return True
|
|
def parseMixServerIDInfo(mixServerIDInfo, useAllIDList=False):
|
## 解析合服服务器ID组合信息 支持范围： 服务器ID~服务器ID 或 单服：服务器ID; 多组
|
''' 解析合服服务器ID组合信息
|
支持格式：
|
范围: 服务器ID~服务器ID
|
单服：服务器ID
|
|
组合格式：按支持的格式使用英文逗号分割，如 1~10,20,30 代表 1至10服 及 20服 及 30服
|
'''
|
serverIDList = []
|
if not mixServerIDInfo:
|
return serverIDList
|
splitList = mixServerIDInfo.replace(" ", "").split(",")
|
for idInfo in splitList:
|
if idInfo.isdigit():
|
serverIDList.append(int(idInfo))
|
elif "~" in idInfo:
|
rangeIDList = idInfo.split("~")
|
if useAllIDList:
|
serverIDList.extend(range(int(rangeIDList[0]), int(rangeIDList[1]) + 1))
|
else:
|
serverIDList.append([int(rangeIDList[0]), int(rangeIDList[1])])
|
return serverIDList
|
|
def checkIsBackupServerFolder(queryServerIDList, checkServerInfo):
|
''' 检查是否满足需要查询的区服备份
|
@param queryServerIDList: 需要查询的目标服务器组合列表，由 parseMixServerIDInfo 解析得到
|
@param checkServerInfo: 需要检查是否满足条件的区分ID信息串，可由 parseMixServerIDInfo 解析的格式
|
@return: 是否满足
|
'''
|
checkServerIDList = parseMixServerIDInfo(checkServerInfo)
|
if not checkServerIDList:
|
return False
|
|
for checkServerIDInfo in checkServerIDList:
|
if isinstance(checkServerIDInfo, int):
|
checkID = checkServerIDInfo
|
if checkID in queryServerIDList:
|
return True
|
for queryServerIDInfo in queryServerIDList:
|
if isinstance(queryServerIDInfo, int):
|
continue
|
queryIDMin, queryIDMax = queryServerIDInfo
|
if queryIDMin <= checkID <= queryIDMax:
|
return True
|
else:
|
checkIDMin, checkIDMax = checkServerIDInfo
|
for queryServerIDInfo in queryServerIDList:
|
if isinstance(queryServerIDInfo, int):
|
if checkIDMin <= queryServerIDInfo <= checkIDMax:
|
return True
|
else:
|
queryIDMin, queryIDMax = queryServerIDInfo
|
if checkIDMin <= queryIDMin <= checkIDMax or checkIDMin <= queryIDMax <= checkIDMax \
|
or queryIDMin <= checkIDMin <= queryIDMax or queryIDMin <= checkIDMin <= queryIDMax:
|
return True
|
|
return False
|
|
def queryBackupCenterDR(cfg, postData={}):
|
## 查询中心合服备份流向数据
|
# @return: 中心返回数据，None代表失败，需用None判断，因为可能实际数据就是空的列表
|
url = cfg.get("ServerInfo", "EventDataBackupToolUrl")
|
postData["queryCenterbak"] = 1
|
req = urllib2.Request(url=url, data=urllib.urlencode(postData))
|
res_data = urllib2.urlopen(req, timeout=120)
|
resMsg = res_data.read()
|
if not resMsg:
|
print "error: query backup center's eventdata no message return!<br/>"
|
return
|
|
if resMsg.startswith("error:") or "Traceback" in resMsg:
|
print resMsg
|
return
|
|
return json.loads(resMsg)
|
|
def queryBackupCenterError(errorMsg):
|
print "error: %s" % errorMsg
|
return
|
|
def queryBackupCenterOK(retData):
|
## 成功默认返回json格式数据，支持普通kv字典，或类实例转化json
|
#logging.info("retData:%s", retData)
|
print json.dumps(retData, ensure_ascii=False, default=lambda obj: obj.__dict__)
|
return
|
|
def printExceptionError():
|
## 显示报错等异常信息
|
error = traceback.format_exc()
|
error = error.replace("<", "<").replace(">", ">").replace("\r", "<br/>").replace("\n", "<br/>")
|
print "error:<br/>:", error
|
|
def loopMainServerDR(cfg, startDate, endDate, argvDict, checkDrFileNeedParseFunc, parseLineFunc, *parseArgs, **kv):
|
''' 循环遍历游戏主服流向信息，并解析满足时间范围流向
|
@param cfg: InterfaceConfig.php 配置对象
|
@param startDate: 起始日期
|
@param endDate: 结束日期
|
@param argvDict: post到中心的参数
|
@param checkDrFileNeedParseFunc: 额外检查流向文件是否需要解析的函数，传入参数(流向文件名 drFileName, *parseArgs, **kv)，返回自定义信息提供 parseLineFunc 用
|
@param parseLineFunc: 解析流向每行内容的逻辑函数，传入参数(流向名drName, 所属dateStr, checkDrFileNeedParseFunc 返回的结果信息 checkNeedParseRetInfo, 流向行内容line, *parseArgs, **kv)
|
@param *parseArgs: 自定义参数
|
|
@return: needQueryCenterbak 是否需要查询中心备份流向数据
|
'''
|
|
startDatetime = strToDatetime(startDate)
|
endDatetime = strToDatetime(endDate)
|
minDatetime = None
|
logging.info("loopMainServerDR %s" % str(argvDict))
|
# 流向路径
|
FolderPath = os.path.join(cfg.get("ServerInfo", "ServerPath"), "EventServer\EventData")
|
for parent, _, fileList in os.walk(FolderPath):
|
parentList = parent.split("\\")
|
lastStr = parentList[-1]
|
fileDatetime = strToDatetime(lastStr)
|
|
if fileDatetime and (not minDatetime or fileDatetime < minDatetime):
|
minDatetime = fileDatetime
|
|
if not checkDate(startDatetime, endDatetime, fileDatetime):
|
continue
|
|
# 指定遍历流向文件及顺序
|
if "drNameList" in kv:
|
loopFileList = ["%s_%s.txt" % (drName, lastStr) for drName in kv["drNameList"]]
|
else:
|
loopFileList = fileList
|
|
for drFileName in loopFileList:
|
fullPath = os.path.join(parent, drFileName)
|
|
if not os.path.isfile(fullPath):
|
continue
|
|
dateStr = lastStr # 流向日期
|
|
isNeed, checkNeedParseRetInfo = checkDrFileNeedParseFunc(drFileName, *parseArgs, **kv)
|
if not isNeed:
|
continue
|
|
# 以下为已经确定需要处理的流向文件逻辑，可进行open、readlines、eval等操作
|
drName = drFileName[:drFileName.rindex("_")] # 流向名
|
f = open(fullPath, "r")
|
lines = f.readlines()
|
f.close()
|
for line in lines:
|
parseLineFunc(drName, dateStr, checkNeedParseRetInfo, line, *parseArgs, **kv)
|
|
## 查询所有数据（含备份在中心服务器的本次合服前的数据）
|
needQueryCenterbak = argvDict.get("queryAllData") and (not startDatetime or not minDatetime or startDatetime <= minDatetime)
|
if needQueryCenterbak:
|
argvDict["queryCenterbak"] = 1 # 设置查询中心备份数据 与 isQueryCenterbak 对应
|
logging.info("End")
|
return needQueryCenterbak
|
|
def isQueryCenterbak(argvDict):
|
## 是否查询中心备份的
|
return argvDict.get("queryCenterbak") != None
|
|
def loopCenterbakRarDR(cfg, startDate, endDate, argvDict, checkDrFileNeedParseFunc, parseLineFunc, *parseArgs, **kv):
|
''' 循环遍历中心服务器备份流向信息，并解析满足时间范围及区服条件的流向
|
@note: Rar专用
|
@param cfg: InterfaceConfig.php 配置对象
|
@param argvDict: post到中心的参数
|
@param checkDrFileNeedParseFunc: 额外检查流向文件是否需要解析的函数，传入参数(流向文件名 drFileName, *parseArgs, **kv)，返回自定义信息提供 parseLineFunc 用
|
@param parseLineFunc: 解析流向每行内容的逻辑函数，传入参数(流向名drName, 所属dateStr, checkDrFileNeedParseFunc 返回的结果信息 checkNeedParseRetInfo, 流向行内容line, *parseArgs, **kv)
|
@param *parseArgs: 解析函数所需的参数
|
|
@return: 是否成功
|
'''
|
logging.info("loopCenterbakRarDR %s" % str(argvDict))
|
allServer = argvDict.get("allServer", 0)
|
mixServerIDInfo = argvDict.get("mixServerIDInfo", "")
|
if not allServer and not mixServerIDInfo:
|
queryBackupCenterError("can not found mixServerIDInfo!")
|
return
|
|
spID = argvDict.get("spID", "")
|
backupPath = os.path.join(cfg.get("ServerInfo", "EventDataBackupPath"), spID, "EventData")
|
isUseRar = cfg.getint("ServerInfo", "EventDataBackupUseRar") == 1 # 是否使用压缩包
|
logging.info("isUseRar=%s,spID=%s,backupPath=%s" % (isUseRar, spID, backupPath))
|
if not os.path.exists(backupPath):
|
queryBackupCenterError("can not found backup path: %s" % backupPath)
|
return
|
|
startDatetime = strToDatetime(startDate)
|
endDatetime = strToDatetime(endDate)
|
OnlyServerID = int(argvDict.get("OnlyServerID", 0))
|
serverIDList = [OnlyServerID] if OnlyServerID else parseMixServerIDInfo(mixServerIDInfo)
|
spfileFormat = "Event_%s-s" % spID
|
|
logging.info("allServer=%s,serverIDList=%s,spfileFormat=%s" % (allServer, serverIDList, spfileFormat))
|
# 备份日期文件夹
|
for backupDateFile in os.listdir(backupPath):
|
backupDateStr = backupDateFile[len("Event_"):]
|
backupDatetime = strToDatetime(backupDateStr)
|
if not checkDate(startDatetime, "", backupDatetime):
|
continue
|
|
if not isUseRar:
|
# 备份分区文件夹
|
for backupServer in os.listdir(os.path.join(backupPath, backupDateFile)):
|
if backupServer.endswith(".rar"):
|
# rar文件的不处理
|
continue
|
if not backupServer.startswith(spfileFormat):
|
continue
|
serverInfo = backupServer[len(spfileFormat):]
|
if not allServer and not checkIsBackupServerFolder(serverIDList, serverInfo):
|
continue
|
|
logging.info("loop %s" % os.path.join(backupPath, backupDateFile, backupServer))
|
for parent, _, fileList in os.walk(os.path.join(backupPath, backupDateFile, backupServer)):
|
if not fileList:
|
continue
|
|
parentList = parent.split("\\")
|
lastStr = parentList[-1]
|
fileDatetime = strToDatetime(lastStr)
|
|
if not checkDate(startDatetime, endDatetime, fileDatetime):
|
continue
|
|
# 指定遍历流向文件及顺序
|
if "drNameList" in kv:
|
loopFileList = ["%s_%s.txt" % (drName, lastStr) for drName in kv["drNameList"]]
|
else:
|
loopFileList = fileList
|
|
for drFileName in loopFileList:
|
fullPath = os.path.join(parent, drFileName)
|
if not os.path.isfile(fullPath):
|
continue
|
|
dateStr = lastStr # 流向日期
|
|
checkNeedParseRetInfo = []
|
if checkDrFileNeedParseFunc:
|
isNeed, checkNeedParseRetInfo = checkDrFileNeedParseFunc(drFileName, *parseArgs, **kv)
|
if not isNeed:
|
continue
|
|
# 以下为已经确定需要处理的流向文件逻辑，可进行open、readlines、eval等操作
|
drName = drFileName[:drFileName.rindex("_")] # 流向名
|
|
f = open(fullPath, "r")
|
lines = f.readlines()
|
f.close()
|
for line in lines:
|
parseLineFunc(drName, dateStr, checkNeedParseRetInfo, line, *parseArgs, **kv)
|
|
else:
|
from unrar import rarfile # 放在函数内，子服用不到此模块，这样就不需要安装此依赖了
|
# 备份分区文件夹
|
for backupServer in os.listdir(os.path.join(backupPath, backupDateFile)):
|
if not backupServer.endswith(".rar"):
|
# 非rar文件的不处理
|
continue
|
if not backupServer.startswith(spfileFormat):
|
continue
|
serverInfo = backupServer[len(spfileFormat):-4]
|
if not allServer and not checkIsBackupServerFolder(serverIDList, serverInfo):
|
continue
|
#logging.info("backupServer:%s" % backupServer)
|
#rarfile.namelist() # 列表，rar文件中所有子文件的path（相对于rar文件包而言的）
|
#rar解析只能遍历rar内所有文件，无法直接定位，会相对文件夹遍历慢
|
# drPath is like: server\EventServer\EventData\Group_0\Server_0\2021-01-04\OpenPackCount_2021-01-04.txt
|
drRarFile = rarfile.RarFile(os.path.join(backupPath, backupDateFile, backupServer))
|
drFileNameList = drRarFile.namelist()
|
loopFileList = []
|
if "drNameList" in kv:
|
loopDateInfo = {}
|
for drPath in drFileNameList[::-1]:
|
if "." in drPath:
|
continue
|
rspList = drPath.split("\\")
|
dateStr = rspList[-1] # 日期字符串
|
fileDatetime = strToDatetime(dateStr)
|
if not fileDatetime:
|
continue
|
if not checkDate(startDatetime, endDatetime, fileDatetime):
|
continue
|
drDatePath = drPath
|
loopDateInfo[dateStr] = drDatePath
|
|
loopDateList = loopDateInfo.keys()
|
loopDateList.sort()
|
|
# 需要遍历的日期文件夹路径列表
|
for dateStr in loopDateList:
|
drDatePath = loopDateInfo[dateStr]
|
for drName in kv["drNameList"]:
|
loopDrPath = os.path.join(drDatePath, "%s_%s.txt" % (drName, dateStr))
|
if loopDrPath in drFileNameList:
|
loopFileList.append(os.path.join(drDatePath, "%s_%s.txt" % (drName, dateStr)))
|
|
else:
|
loopFileList = drFileNameList
|
|
for drPath in loopFileList:
|
if "." not in drPath:
|
continue
|
#logging.info("loop: %s", str(drPath))
|
rspList = drPath.split("\\")
|
if len(rspList) < 2:
|
continue
|
dateStr = rspList[-2] # 日期字符串
|
fileDatetime = strToDatetime(dateStr)
|
if not checkDate(startDatetime, endDatetime, fileDatetime):
|
continue
|
|
drFileName = rspList[-1] # 流向文件名 xxxx_日期.txt
|
|
checkNeedParseRetInfo = []
|
if checkDrFileNeedParseFunc:
|
isNeed, checkNeedParseRetInfo = checkDrFileNeedParseFunc(drFileName, *parseArgs, **kv)
|
if not isNeed:
|
continue
|
|
drName = drFileName[:drFileName.rindex("_")] # 流向名
|
|
rarOpenObj = drRarFile.open(drPath)
|
for line in rarOpenObj.readlines():
|
parseLineFunc(drName, dateStr, checkNeedParseRetInfo, line, *parseArgs, **kv)
|
|
logging.info("loopEnd")
|
return True
|
|
def editTableHtml(dataList, tdList, rowNumTitle="", styleInfo={}, printCount=True):
|
''' 编辑表格html
|
@param dataList: 数据列表，元素支持 类实例 或 字典
|
@param tdList: 表格属性字段key顺序列表
|
@param rowNumTitle: 行编号标题，如果空表示不显示行编号，非空则传入行编号标题名，行编号默认为 index + 1
|
@param styleInfo: 属性字段样式信息
|
'''
|
tablHtml = ""
|
if printCount:
|
tablHtml += _(u"数据条数") + ": %s<br/>" % len(dataList)
|
tablHtml += "<table id=\"tt\" class=\"border-table\">"
|
|
# 标题行
|
tablHtml += "<tr>"
|
if rowNumTitle:
|
tablHtml += "<td align='center'>%s</td>" % (rowNumTitle)
|
for tdKey in tdList:
|
style = styleInfo.get(tdKey, {})
|
tablHtml += "<td align='center'>%s</td>" % (style.get("title", tdKey))
|
tablHtml += "</tr>"
|
|
# 内容
|
for i, dataObj in enumerate(dataList):
|
tablHtml += "<tr>"
|
if rowNumTitle:
|
tablHtml += "<td align='center'>%s</td>" % (i + 1)
|
|
for tdKey in tdList:
|
# 仅支持字典及类
|
tdValue = ""
|
if isinstance(dataObj, dict):
|
tdValue = dataObj.get(tdKey, "")
|
elif hasattr(dataObj, tdKey):
|
tdValue = getattr(dataObj, tdKey)
|
|
if isinstance(tdValue, unicode):
|
tdValue = tdValue.encode(Encoding)
|
elif not isinstance(tdValue, str):
|
tdValue = str(tdValue)
|
style = styleInfo.get(tdKey, {})
|
tablHtml += "<td align='%s'>%s</td>" % (style.get("align", "center"), tdValue)
|
|
tablHtml += "</tr>"
|
|
tablHtml += "</table>"
|
return tablHtml
|
|
def getCfgKeyNameDict(cfgName, argvDict):
|
## 根据游戏配置母表字段id及name文件名获取对应的id对应name字典，用于转化ID名称用
|
txtFileDir = "../../Common/config/%s/%s/%s.txt" % (argvDict.get("spID", ""), argvDict.get("lang", ""), cfgName)
|
if not os.path.exists(txtFileDir):
|
txtFileDir = "../../Common/config/%s/%s.txt" % (argvDict.get("spID", ""), cfgName)
|
if not os.path.exists(txtFileDir):
|
return {}
|
kvDict = {}
|
f = open(txtFileDir, "r")
|
lines = f.readlines()
|
for line in lines:
|
if not line:
|
continue
|
spList = line.split("\t")
|
if len(spList) != 2:
|
continue
|
kvDict[spList[0]] = spList[1].replace("\r", "").replace("\n", "")
|
return kvDict
|
|
def getSPConfig(argvDict, section, option, defaultValue=None):
|
'''获取sp运营项目对应专属配置
|
@param section 配置节点
|
@param option 配置项
|
'''
|
iniFileDir = "../../Common/config/%s/config.ini" % argvDict.get("spID", "")
|
if not os.path.exists(iniFileDir):
|
return defaultValue
|
cfg = ConfigParser.ConfigParser()
|
cfg.read(iniFileDir)
|
if not cfg.has_option(section, option):
|
return defaultValue
|
return cfg.get(section, option)
|
|
def transformNum(num):
|
if num < 10000:
|
fpStr = str(num)
|
elif num < 1000000:
|
fpStr = "%.2f%s" % (num / 10000.0, _(u"万"))
|
elif num < 100000000:
|
fpStr = "%.1f%s" % (num / 10000.0, _(u"万"))
|
else:
|
fpStr = "%.3f%s" % (num / 100000000.0, _(u"亿"))
|
return fpStr
|
|
def gettextInstall(languages=None):
|
## 初始化翻译 gettext
|
if not languages:
|
languages = "zh_CN"
|
if type(languages) == str:
|
languages = [languages]
|
gt = gettext.translation("eventDataPy", "../../language", languages)
|
gt.install()
|
return
|
|
def getPayOrderTypeName(payOrderType):
|
PayOrderTypeName = {1:_(u"人民币"), 2:_(u"美元"), 3:_(u"越南盾"), 4:_(u"soha币"), 5:_(u"金票点券"), 6:_(u"代币")}
|
return PayOrderTypeName.get(payOrderType, "%s[%s]" % (_(u"未知订单类型"), payOrderType))
|
|
def getPayOrderCoinRate(payOrderType):
|
rateDict = {3:1, 6:1}
|
return rateDict.get(payOrderType, 100)
|
