www.git - Gitblit

parent: bb9eccc3 | 补丁 | 提交 | ignore whitespace

hxp

2024-11-22 4722efaac352d83e7ca8ff7564f0b2a91fa970a9

10284 【后台】 优化导出中心备档流向日活数据方式；

3个文件已修改

1个文件已添加

	Server/eventdata/CommFunc.py	10 ●●●●● 补丁 \| 查看 \| 原始文档 \| blame \| 历史
	Server/eventdata/ImportBakDailyActive.py	105 ●●●●● 补丁 \| 查看 \| 原始文档 \| blame \| 历史
	db/DBOper.php	4 ●●●●● 补丁 \| 查看 \| 原始文档 \| blame \| 历史
	serverrep/ImportAccountLoginpay.php	91 ●●●●● 补丁 \| 查看 \| 原始文档 \| blame \| 历史

 Server/eventdata/CommFunc.py

@@ -336,8 +336,14 @@
    spfileFormat = "Event_%s-s" % spID
    
    logging.info("allServer=%s,serverIDList=%s,spfileFormat=%s" % (allServer, serverIDList, spfileFormat))
    logloop = "logloop" in kv
    printloop = "printloop" in kv
    # 备份日期文件夹
    for backupDateFile in os.listdir(backupPath):
        if logloop:
            logging.info("backupDateFile:%s" % backupDateFile)
        if printloop:
            print "backupDateFile:%s" % backupDateFile
        backupDateStr = backupDateFile[len("Event_"):]
        backupDatetime = strToDatetime(backupDateStr)
        if not checkDate(startDatetime, "", backupDatetime):
@@ -399,6 +405,10 @@
            from unrar import rarfile # 放在函数内，子服用不到此模块，这样就不需要安装此依赖了
            # 备份分区文件夹
            for backupServer in os.listdir(os.path.join(backupPath, backupDateFile)):
                if logloop:
                    logging.info("  %s/%s" % (backupDateFile, backupServer))
                if printloop:
                    print "  %s/%s" % (backupDateFile, backupServer)
                if not backupServer.endswith(".rar"):
                    # 非rar文件的不处理
                    continue

 Server/eventdata/ImportBakDailyActive.py

New file
@@ -0,0 +1,105 @@
#!/usr/bin/python

# -*- coding: utf-8 -*-



import CommFunc

import ConfigParser

import logging

import mylog

import json

import os

from unrar import rarfile



cfg = ConfigParser.ConfigParser()

cfg.read("../../InterfaceConfig.php")

# 需要处理的流向名及顺序列表

DRNameList = ["LogInOut"]



argvDict = {"spID": "hyyngame",

            "eventType": "ImportBakDailyActive",

            "queryCenterbak": 1,

            "allServer": 1,

            "startDate": "2024-05-01",

            }





def queryAccountLoginoutInfo(argvDict):

    logging.info("queryAccountLoginoutInfo %s" % str(argvDict))



    startDate = argvDict.get("startDate", "")

    endDate = argvDict.get("endDate", "")

    spID = argvDict.get("spID", "")

    # 查询中心备份的

    # {"日期":{"AccID":[loginTime, logoutTime, IP], ...}, ...}

    loginoutDateAccIDInfo = {}

    if not CommFunc.loopCenterbakRarDR(cfg, startDate, endDate, argvDict, checkDrFileNeedParseFunc, parseLineFunc,

                                       loginoutDateAccIDInfo, drNameList=DRNameList, logloop=1, printloop=1):

        return



    logging.info("query all data OK")

    logging.info("%s" % loginoutDateAccIDInfo)

    
    content = json.dumps(loginoutDateAccIDInfo, ensure_ascii=False, default=lambda obj: obj.__dict__)

    with open('BakDailyActive.json', 'w') as file:

        file.write(content)  # 将内容写入文件

    print "save ok"

    logging.info("save ok")

    return





def checkDrFileNeedParseFunc(drFileName, *parseArgs, **kv):

    ''' 检查流向是否需要处理

    @param drFileName: 流向文件名  xxx_日期.txt

    @param *parseArgs: 自定义参数

    @return: isNeed, checkNeedParseRetInfo

    '''

    return True, None





def parseLineFunc(drName, dateStr, checkNeedParseRetInfo, line, *parseArgs, **kv):

    ''' 解析流向行内容

    @param drName: 流向名

    @param dateStr: 对应日期字符串

    @param checkNeedParseRetInfo: checkDrFileNeedParseFunc 返回的内容

    @param line: 本行内容

    @param *parseArgs: 自定义参数

    '''



    loginoutDateAccIDInfo = parseArgs[0]

    # 统计登录

    if drName == "LogInOut":

        if "127.0.0.1" in line:

            #loginfo("    脱机登录，不处理")

            return



        drDict = eval(line)

        accID = drDict["AccID"]



        if dateStr not in loginoutDateAccIDInfo:

            loginoutDateAccIDInfo[dateStr] = {}

        accIDDict = loginoutDateAccIDInfo[dateStr]

        if accID not in accIDDict:

            accIDDict[accID] = ["", "", ""]

        loginoutList = accIDDict[accID]



        if drDict["Type"] == "login":

            loginoutList[0] = drDict["LoginTime"]

        else:

            loginoutList[1] = drDict["LogoutTime"]

        loginoutList[2] = drDict["IP"]



    return





def main():

    CommFunc.setdefaultencoding()

    mylog.InitMyLog(argvDict.get("eventType", ""))

    queryAccountLoginoutInfo(argvDict)

    return





if __name__ == "__main__":

    try:

        main()

    except:

        CommFunc.printExceptionError()

    os.system('pause')


 db/DBOper.php

@@ -286,7 +286,7 @@
        }
        $dbName = GetDBName();
        $ret = $conn->$dbName->$collectionName->remove($find);
        \Logging\LogInfo('Mongo Remove: $ret=' . $ret . ', find=' . print_r($find, true));
        \Logging\LogInfo('Mongo Remove:' . $collectionName . ' $ret=' . $ret . ', find=' . print_r($find, true));
        MongoDBPool::Close($conn);
        return true;
    }
@@ -504,4 +504,4 @@
    }

    return true;
}
}

 serverrep/ImportAccountLoginpay.php

@@ -17,7 +17,7 @@
    echo "no appID";

    exit();

}

logweb("开始导入:" . $Channel);

logweb("开始导入:" . $Channel . " opt:" . $opt);

if (array_key_exists("reset", $_GET)) {

    logweb("重置重新导出:" . $Channel);

    $AccountFirstLoginOK = \DBOper\Remove("AccountFirstLogin", array("Channel" => $Channel));

@@ -42,12 +42,29 @@
}

if (!$opt || $opt == "day") {

    logweb("daydaydaydayday:" . $Channel);

    impAccountDayActive();

    impAccountDayActive("rar");

    // impAccountDayActiveTestData();

}

if (!$opt || $opt == "rep") {

    logweb("repreprepreprep:" . $Channel);

// 重导日活、重刷报表；前提是先试用 /Server/eventdata/ImportBakDailyActive.py 运行导出中心流向备档日活数据结果保存到json文件

if ($opt == "reDayActive") {

    $AccountDayActiveOK = \DBOper\Remove("AccountDayActive", array("Channel" => $Channel));

    logweb("AccountDayActiveOK:" . $AccountDayActiveOK);

    impAccountDayActive("json");

}

if (!$opt || $opt == "rep" || $opt == "reDayActive") {

    logweb("开始导出每日报表:" . $Channel);

    if ($opt == "reDayActive") {

        $DailyReportOK = \DBOper\Remove("DailyReport", array("Channel" => $Channel));

        $AccountFirstLoginReportOK = \DBOper\Remove("AccountFirstLoginReport", array("Channel" => $Channel));

        $AccountFirstPayReportOK = \DBOper\Remove("AccountFirstPayReport", array("Channel" => $Channel));

        $eventYMDOK = \DBOper\Remove("ServerEvent", array("Key" => "FirstKeepReportYMD_" . $Channel));

        logweb("DailyReportOK:" . $DailyReportOK);

        logweb("AccountFirstLoginReportOK:" . $AccountFirstLoginReportOK);

        logweb("AccountFirstPayReportOK:" . $AccountFirstPayReportOK);

        logweb("eventYMDOK:" . $eventYMDOK);

    }

    \Report\CheckAndExportDailyReport($Channel);

    logweb("开始导出留存报表:" . $Channel);

    \Report\CheckAndExportFirstKeepReport($Channel);

}

logweb("处理完毕!");

@@ -234,7 +251,7 @@
    echo "<hr/>";

}



function impAccountDayActive()

function impAccountDayActive($bakWay = "")

{

    global $Channel;

    $startDate = $_GET["startDate"];

@@ -245,30 +262,44 @@


    $allDateAccIDInfo = array(); // 所有记录

    // 查询备份流向

    $pack_data = array();

    $pack_data["spID"] = $Channel;

    $pack_data["eventType"] = "QueryAccountLoginOut";

    $pack_data["queryCenterbak"] = 1; # 设置查询中心的

    $pack_data["allServer"] = 1; # 设置检索所有服务器流向

    $pack_data["startDate"] = $startDate;

    $pack_data["endDate"] = "";

    $centerToolUrl = 'http://' . $_SERVER['HTTP_HOST'] . "/Server/eventdata/toolcenter.php";

    logweb("centerToolUrl:" . $centerToolUrl);

    if ($bakWay == "rar") {

        $pack_data = array();

        $pack_data["spID"] = $Channel;

        $pack_data["eventType"] = "QueryAccountLoginOut";

        $pack_data["queryCenterbak"] = 1; # 设置查询中心的

        $pack_data["allServer"] = 1; # 设置检索所有服务器流向

        $pack_data["startDate"] = $startDate;

        $pack_data["endDate"] = "";

        $centerToolUrl = 'http://' . $_SERVER['HTTP_HOST'] . "/Server/eventdata/toolcenter.php";

        logweb("centerToolUrl:" . $centerToolUrl);



    $retStr = \CommFunc\DoPost($centerToolUrl, $pack_data, false, 300);

    $ret = json_decode($retStr, true);

    if (isset($ret)) {

        $allDateAccIDInfo = $ret[0];

        ksort($allDateAccIDInfo);

        foreach ($allDateAccIDInfo as $dateStr => $accIDInfo) {

            logweb("中心流向备档日活数量:" . " dateStr:" . $dateStr . " count:" . count($accIDInfo));

            // foreach ($accIDInfo as $accID => $loginoutInfo) {

            //     logweb("    accID:" . $accID. json_encode($loginoutInfo));

            // }

        $retStr = \CommFunc\DoPost($centerToolUrl, $pack_data, false, 300);

        $ret = json_decode($retStr, true);

        if (isset($ret)) {

            $allDateAccIDInfo = $ret[0];

        } else {

            logweb("query center backup error: " . $retStr);

            return;

        }

    } else {

        logweb("query center backup error: " . $retStr);

    } elseif ($bakWay == "json") {

        if (!\CommFunc\GetConfig("ServerInfo", "PlayerCenterRoot", $PlayerCenterRoot)) {

            return "";

        }

        $jsonFile = $PlayerCenterRoot . "/Server/eventdata/BakDailyActive.json";

        logweb("jsonFile:" . $jsonFile);

        if (!file_exists($jsonFile)) {

            logweb("###jsonFile is not exists:" . $jsonFile);

            return;

        }

        $allDateAccIDInfo = json_decode(file_get_contents($jsonFile), true);

    }

    if (!isset($allDateAccIDInfo)) {

        logweb("###center bak AccountDayActive error.");

        return;

    }

    ksort($allDateAccIDInfo);

    foreach ($allDateAccIDInfo as $dateStr => $accIDInfo) {

        logweb("中心流向备档日活数量:" . " dateStr:" . $dateStr . " count:" . count($accIDInfo));

    }



    logweb("开始从子服导入日活数据! startDate:" . $startDate);

@@ -326,7 +357,8 @@


    ksort($allDateAccIDInfo);

    foreach ($allDateAccIDInfo as $dateStr => $accIDInfo) {

        logweb("最终日活数量:" . " dateStr:" . $dateStr . " count:" . count($accIDInfo));

        logweb("日活游戏账号数量:" . " dateStr:" . $dateStr . " count:" . count($accIDInfo));

        $accountIDArray = array();

        $dateInsInfo = array();

        foreach ($accIDInfo as $accID => $loginoutInfo) {

            // logweb("    accID:" . $accID. json_encode($loginoutInfo));

@@ -343,6 +375,11 @@
                $ActiveYMD = substr($LogoffTime, 0, 10);

            }



            if ($accountIDArray[$AccountID]) {

                logweb("###今日日活账号已存在:AccountID:" . $$AccountID . " accID:" . $accID);

                continue;

            }

            $accountIDArray[$AccountID] = 1;

            array_push($dateInsInfo, array(

                "Channel" => $Channel,

                "AccountID" => $AccountID,

			@@ -336,8 +336,14 @@
			spfileFormat = "Event_%s-s" % spID

			logging.info("allServer=%s,serverIDList=%s,spfileFormat=%s" % (allServer, serverIDList, spfileFormat))
			logloop = "logloop" in kv
			printloop = "printloop" in kv
			# 备份日期文件夹
			for backupDateFile in os.listdir(backupPath):
			if logloop:
			logging.info("backupDateFile:%s" % backupDateFile)
			if printloop:
			print "backupDateFile:%s" % backupDateFile
			backupDateStr = backupDateFile[len("Event_"):]
			backupDatetime = strToDatetime(backupDateStr)
			if not checkDate(startDatetime, "", backupDatetime):
			@@ -399,6 +405,10 @@
			from unrar import rarfile # 放在函数内，子服用不到此模块，这样就不需要安装此依赖了
			# 备份分区文件夹
			for backupServer in os.listdir(os.path.join(backupPath, backupDateFile)):
			if logloop:
			logging.info(" %s/%s" % (backupDateFile, backupServer))
			if printloop:
			print " %s/%s" % (backupDateFile, backupServer)
			if not backupServer.endswith(".rar"):
			# 非rar文件的不处理
			continue

New file
			@@ -0,0 +1,105 @@
			#!/usr/bin/python
			# -- coding: utf-8 --

			import CommFunc
			import ConfigParser
			import logging
			import mylog
			import json
			import os
			from unrar import rarfile

			cfg = ConfigParser.ConfigParser()
			cfg.read("../../InterfaceConfig.php")
			# 需要处理的流向名及顺序列表
			DRNameList = ["LogInOut"]

			argvDict = {"spID": "hyyngame",
			"eventType": "ImportBakDailyActive",
			"queryCenterbak": 1,
			"allServer": 1,
			"startDate": "2024-05-01",
			}


			def queryAccountLoginoutInfo(argvDict):
			logging.info("queryAccountLoginoutInfo %s" % str(argvDict))

			startDate = argvDict.get("startDate", "")
			endDate = argvDict.get("endDate", "")
			spID = argvDict.get("spID", "")
			# 查询中心备份的
			# {"日期":{"AccID":[loginTime, logoutTime, IP], ...}, ...}
			loginoutDateAccIDInfo = {}
			if not CommFunc.loopCenterbakRarDR(cfg, startDate, endDate, argvDict, checkDrFileNeedParseFunc, parseLineFunc,
			loginoutDateAccIDInfo, drNameList=DRNameList, logloop=1, printloop=1):
			return

			logging.info("query all data OK")
			logging.info("%s" % loginoutDateAccIDInfo)

			content = json.dumps(loginoutDateAccIDInfo, ensure_ascii=False, default=lambda obj: obj.__dict__)
			with open('BakDailyActive.json', 'w') as file:
			file.write(content) # 将内容写入文件
			print "save ok"
			logging.info("save ok")
			return


			def checkDrFileNeedParseFunc(drFileName, parseArgs, *kv):
			''' 检查流向是否需要处理
			@param drFileName: 流向文件名 xxx_日期.txt
			@param *parseArgs: 自定义参数
			@return: isNeed, checkNeedParseRetInfo
			'''
			return True, None


			def parseLineFunc(drName, dateStr, checkNeedParseRetInfo, line, parseArgs, *kv):
			''' 解析流向行内容
			@param drName: 流向名
			@param dateStr: 对应日期字符串
			@param checkNeedParseRetInfo: checkDrFileNeedParseFunc 返回的内容
			@param line: 本行内容
			@param *parseArgs: 自定义参数
			'''

			loginoutDateAccIDInfo = parseArgs[0]
			# 统计登录
			if drName == "LogInOut":
			if "127.0.0.1" in line:
			#loginfo(" 脱机登录，不处理")
			return

			drDict = eval(line)
			accID = drDict["AccID"]

			if dateStr not in loginoutDateAccIDInfo:
			loginoutDateAccIDInfo[dateStr] = {}
			accIDDict = loginoutDateAccIDInfo[dateStr]
			if accID not in accIDDict:
			accIDDict[accID] = ["", "", ""]
			loginoutList = accIDDict[accID]

			if drDict["Type"] == "login":
			loginoutList[0] = drDict["LoginTime"]
			else:
			loginoutList[1] = drDict["LogoutTime"]
			loginoutList[2] = drDict["IP"]

			return


			def main():
			CommFunc.setdefaultencoding()
			mylog.InitMyLog(argvDict.get("eventType", ""))
			queryAccountLoginoutInfo(argvDict)
			return


			if __name__ == "__main__":
			try:
			main()
			except:
			CommFunc.printExceptionError()
			os.system('pause')

			@@ -286,7 +286,7 @@
			}
			$dbName = GetDBName();
			$ret = $conn->$dbName->$collectionName->remove($find);
			\Logging\LogInfo('Mongo Remove: $ret=' . $ret . ', find=' . print_r($find, true));
			\Logging\LogInfo('Mongo Remove:' . $collectionName . ' $ret=' . $ret . ', find=' . print_r($find, true));
			MongoDBPool::Close($conn);
			return true;
			}
			@@ -504,4 +504,4 @@
			}

			return true;
			}
			}

			@@ -17,7 +17,7 @@
			echo "no appID";
			exit();
			}
			logweb("开始导入:" . $Channel);
			logweb("开始导入:" . $Channel . " opt:" . $opt);
			if (array_key_exists("reset", $_GET)) {
			logweb("重置重新导出:" . $Channel);
			$AccountFirstLoginOK = \DBOper\Remove("AccountFirstLogin", array("Channel" => $Channel));
			@@ -42,12 +42,29 @@
			}
			if (!$opt \|\| $opt == "day") {
			logweb("daydaydaydayday:" . $Channel);
			impAccountDayActive();
			impAccountDayActive("rar");
			// impAccountDayActiveTestData();
			}
			if (!$opt \|\| $opt == "rep") {
			logweb("repreprepreprep:" . $Channel);
			// 重导日活、重刷报表；前提是先试用 /Server/eventdata/ImportBakDailyActive.py 运行导出中心流向备档日活数据结果保存到json文件
			if ($opt == "reDayActive") {
			$AccountDayActiveOK = \DBOper\Remove("AccountDayActive", array("Channel" => $Channel));
			logweb("AccountDayActiveOK:" . $AccountDayActiveOK);
			impAccountDayActive("json");
			}
			if (!$opt \|\| $opt == "rep" \|\| $opt == "reDayActive") {
			logweb("开始导出每日报表:" . $Channel);
			if ($opt == "reDayActive") {
			$DailyReportOK = \DBOper\Remove("DailyReport", array("Channel" => $Channel));
			$AccountFirstLoginReportOK = \DBOper\Remove("AccountFirstLoginReport", array("Channel" => $Channel));
			$AccountFirstPayReportOK = \DBOper\Remove("AccountFirstPayReport", array("Channel" => $Channel));
			$eventYMDOK = \DBOper\Remove("ServerEvent", array("Key" => "FirstKeepReportYMD_" . $Channel));
			logweb("DailyReportOK:" . $DailyReportOK);
			logweb("AccountFirstLoginReportOK:" . $AccountFirstLoginReportOK);
			logweb("AccountFirstPayReportOK:" . $AccountFirstPayReportOK);
			logweb("eventYMDOK:" . $eventYMDOK);
			}
			\Report\CheckAndExportDailyReport($Channel);
			logweb("开始导出留存报表:" . $Channel);
			\Report\CheckAndExportFirstKeepReport($Channel);
			}
			logweb("处理完毕!");
			@@ -234,7 +251,7 @@
			echo "<hr/>";
			}

			function impAccountDayActive()
			function impAccountDayActive($bakWay = "")
			{
			global $Channel;
			$startDate = $_GET["startDate"];
			@@ -245,30 +262,44 @@

			$allDateAccIDInfo = array(); // 所有记录
			// 查询备份流向
			$pack_data = array();
			$pack_data["spID"] = $Channel;
			$pack_data["eventType"] = "QueryAccountLoginOut";
			$pack_data["queryCenterbak"] = 1; # 设置查询中心的
			$pack_data["allServer"] = 1; # 设置检索所有服务器流向
			$pack_data["startDate"] = $startDate;
			$pack_data["endDate"] = "";
			$centerToolUrl = 'http://' . $_SERVER['HTTP_HOST'] . "/Server/eventdata/toolcenter.php";
			logweb("centerToolUrl:" . $centerToolUrl);
			if ($bakWay == "rar") {
			$pack_data = array();
			$pack_data["spID"] = $Channel;
			$pack_data["eventType"] = "QueryAccountLoginOut";
			$pack_data["queryCenterbak"] = 1; # 设置查询中心的
			$pack_data["allServer"] = 1; # 设置检索所有服务器流向
			$pack_data["startDate"] = $startDate;
			$pack_data["endDate"] = "";
			$centerToolUrl = 'http://' . $_SERVER['HTTP_HOST'] . "/Server/eventdata/toolcenter.php";
			logweb("centerToolUrl:" . $centerToolUrl);

			$retStr = \CommFunc\DoPost($centerToolUrl, $pack_data, false, 300);
			$ret = json_decode($retStr, true);
			if (isset($ret)) {
			$allDateAccIDInfo = $ret[0];
			ksort($allDateAccIDInfo);
			foreach ($allDateAccIDInfo as $dateStr => $accIDInfo) {
			logweb("中心流向备档日活数量:" . " dateStr:" . $dateStr . " count:" . count($accIDInfo));
			// foreach ($accIDInfo as $accID => $loginoutInfo) {
			// logweb(" accID:" . $accID. json_encode($loginoutInfo));
			// }
			$retStr = \CommFunc\DoPost($centerToolUrl, $pack_data, false, 300);
			$ret = json_decode($retStr, true);
			if (isset($ret)) {
			$allDateAccIDInfo = $ret[0];
			} else {
			logweb("query center backup error: " . $retStr);
			return;
			}
			} else {
			logweb("query center backup error: " . $retStr);
			} elseif ($bakWay == "json") {
			if (!\CommFunc\GetConfig("ServerInfo", "PlayerCenterRoot", $PlayerCenterRoot)) {
			return "";
			}
			$jsonFile = $PlayerCenterRoot . "/Server/eventdata/BakDailyActive.json";
			logweb("jsonFile:" . $jsonFile);
			if (!file_exists($jsonFile)) {
			logweb("###jsonFile is not exists:" . $jsonFile);
			return;
			}
			$allDateAccIDInfo = json_decode(file_get_contents($jsonFile), true);
			}
			if (!isset($allDateAccIDInfo)) {
			logweb("###center bak AccountDayActive error.");
			return;
			}
			ksort($allDateAccIDInfo);
			foreach ($allDateAccIDInfo as $dateStr => $accIDInfo) {
			logweb("中心流向备档日活数量:" . " dateStr:" . $dateStr . " count:" . count($accIDInfo));
			}

			logweb("开始从子服导入日活数据! startDate:" . $startDate);
			@@ -326,7 +357,8 @@

			ksort($allDateAccIDInfo);
			foreach ($allDateAccIDInfo as $dateStr => $accIDInfo) {
			logweb("最终日活数量:" . " dateStr:" . $dateStr . " count:" . count($accIDInfo));
			logweb("日活游戏账号数量:" . " dateStr:" . $dateStr . " count:" . count($accIDInfo));
			$accountIDArray = array();
			$dateInsInfo = array();
			foreach ($accIDInfo as $accID => $loginoutInfo) {
			// logweb(" accID:" . $accID. json_encode($loginoutInfo));
			@@ -343,6 +375,11 @@
			$ActiveYMD = substr($LogoffTime, 0, 10);
			}

			if ($accountIDArray[$AccountID]) {
			logweb("###今日日活账号已存在:AccountID:" . $$AccountID . " accID:" . $accID);
			continue;
			}
			$accountIDArray[$AccountID] = 1;
			array_push($dateInsInfo, array(
			"Channel" => $Channel,
			"AccountID" => $AccountID,