first commit

常用脚本/python/msg_log.ini

@@ -0,0 +1,18 @@
+[_msg_log_]
+level=info
+file=/path/to/default.log
+stderr=1
+syslog=0
+center=1
+sysID=019
+subsysID=13
+file_size=300
+file_counter=3
+
+[table_partition]
+file=/path/to/table_partition.log
+stderr=1
+level=info
+file_size=100
+file_counter=6
+

常用脚本/python/rising_log.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python
+#-*- encoding: gbk -*-
+#    
+
+import ConfigParser
+import logging
+import time, os
+from logging.handlers import RotatingFileHandler
+import platform
+
+#global
+log_msg_root = '_msg_log_'
+
+#log_stderr_format =  '$RESET$COLOR<%(levelname)-5s><%(name)s:%(process)d:%(threadName)s><%(filename)s:%(lineno)d>%(message)s'
+log_stderr_format =  '$COLOR$BOLD<%(levelname)-5s><%(name)s:%(process)d:%(threadName)s>$RESET$COLOR<%(filename)s:%(lineno)d>%(message)s'
+#log_stderr_format = "$COLOR%(levelname)s $RESET %(asctime)s $BOLD$COLOR%(name)s$RESET %(message)s"
+
+if platform.system() == 'Windows':
+	log_file_format = '<%(levelname)s><%(name)s:%(process)d>%(message)s'
+else:
+	log_file_format = '%(asctime)s <%(levelname)-5s><%(name)s:%(process)d:%(threadName)s><%(filename)s:%(lineno)d>%(message)s'
+
+
+# for colorful stderr output, copy from http://stackoverflow.com/questions/384076/how-can-i-make-the-python-logging-output-to-be-colored
+# by rrt
+
+BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
+RESET_SEQ = "\033[0m"
+#COLOR_SEQ = "\033[1;%dm"         # color & bold
+COLOR_SEQ = "\033[0;%dm"        # color only
+BOLD_SEQ = "\033[1m" 
+
+COLORS={
+	'WARNING':WHITE,
+	'INFO':YELLOW,
+	'DEBUG':GREEN,
+	'CRITICAL':YELLOW,
+	'ERROR':RED,
+	'RED':RED,
+	'GREEN':GREEN,
+	'YELLOW':YELLOW,
+	'BLUE':BLUE,
+	'MAGENTA':MAGENTA,
+	'CYAN':CYAN,
+	'WHITE':WHITE,}
+"""
+COLORS = {
+	'INFO': YELLOW,
+	'DEBUG': GREEN,
+	'ERROR': RED } 
+	"""
+
+class ColoredFormatter(logging.Formatter):
+	def __init__(self, *args, **kwargs):
+		logging.Formatter.__init__(self, *args, **kwargs)
+
+	def format(self, record):
+		levelname = record.levelname
+		color     = COLOR_SEQ % (30 + COLORS[levelname])
+		message   = logging.Formatter.format(self, record)
+		#print "1--" + repr(message)
+		message   = message.replace("$RESET", RESET_SEQ)\
+				.replace("$BOLD",  BOLD_SEQ)\
+				.replace("$COLOR", color)
+		#print "2--" + repr(message)
+		#for k,v in COLORS.items():
+		#	message = message.replace("$" + k,    COLOR_SEQ % (v+30))\
+		#			.replace("$BG" + k,  COLOR_SEQ % (v+40))\
+		#			.replace("$BG-" + k, COLOR_SEQ % (v+40))
+		#print "3--" + repr(message)
+		return message + RESET_SEQ
+
+
+
+#This class copy form liup's blog http://hi.baidu.com/dalier/blog/item/68419d64b98604faf63654e4.html
+#by rrt on 2010-12-3
+
+class DateRotatingFileHandler(RotatingFileHandler):
+	def __init__(self , filename , mode='a' , maxBytes=0, backupCount=0, encoding=None):
+		self.current = time.strftime("%Y%m%d" , time.localtime(time.time()))
+		self.path = os.path.dirname(filename)
+		self.filename = os.path.basename(filename)
+		newdir = os.path.join(self.path , self.current)
+		softlink = os.path.join(self.path, 'current')
+		if not os.access(newdir , os.X_OK):
+			os.mkdir(newdir)
+		try:
+			os.symlink(newdir ,softlink)
+		except:
+			os.system("rm -f " + softlink)
+			os.symlink(newdir ,softlink)
+		newfile = os.path.join(newdir , self.filename)
+		RotatingFileHandler.__init__(self, newfile , mode, maxBytes , backupCount , encoding)
+
+	def doRollover(self):
+		#print "doRollover , current=%s , filename=%s"%(self.current , self.baseFilename)
+		self.stream.close()
+		self.current = time.strftime("%Y%m%d" , time.localtime(time.time()))
+		
+		#Modified by rrt on 2010-12-4 for a log path bug: /'20101204' 
+		#repr() is not needed , time.strftime() return a string not a integer
+		#newdir = os.path.join(self.path , repr(self.current))
+		newdir = os.path.join(self.path , self.current)
+		if not os.access(newdir , os.X_OK):
+			os.mkdir(newdir)
+		self.baseFilename = os.path.join(newdir , self.filename)
+
+		if self.encoding:
+			self.stream = codecs.open(self.baseFilename, 'w', self.encoding)
+		else:
+			self.stream = open(self.baseFilename, 'w')
+
+	def shouldRollover(self, record):
+		if RotatingFileHandler.shouldRollover(self , record):
+			RotatingFileHandler.doRollover(self)
+
+		t = time.strftime("%Y%m%d" , time.localtime(time.time()))
+		if (cmp(self.current , t) < 0) :
+			return 1
+
+		return 0
+
+
+def init_log(prefix, cfg_file, type,level):
+	global log_msg_root, log_file_format, log_stderr_format
+
+	cf = ConfigParser.ConfigParser()
+	cf.read(cfg_file)
+	config = {}
+
+	# we can use
+	#	= cf.get(prefix, 'level').split('#')[0].strip()
+	# to enable "#" to comment behind a value string.
+
+	try:
+		config['level'] = cf.get(prefix, 'level')
+	except:
+		config['level'] = cf.get(log_msg_root, 'level')
+
+	try:
+		config['file'] = cf.get(prefix, 'file')
+	except:
+		path = cf.get(log_msg_root, 'file')
+		path = os.path.dirname(path)
+		config['file'] = os.path.join(path, prefix+'.log')
+		
+	try:
+		config['stderr'] = cf.getint(prefix, 'stderr')
+	except:
+		config['stderr'] = cf.getint(log_msg_root, 'stderr')
+
+	try:
+		config['syslog'] = cf.getint(prefix, 'syslog')
+	except:
+		config['syslog'] = cf.getint(log_msg_root, 'syslog')
+
+	try:
+		config['file_size'] = cf.getint(prefix, 'file_size')
+	except:
+		config['file_size'] = cf.getint(log_msg_root, 'file_size')
+
+	try:
+		config['file_cnt'] = cf.getint(prefix, 'file_counter')
+	except:
+		config['file_cnt'] = cf.getint(log_msg_root, 'file_counter')
+	
+	print config
+
+	#new a root logger
+	logger = logging.getLogger(prefix)
+	#set root logger level
+	if config['level'] == 'debug':
+		logger.setLevel(logging.DEBUG)
+	elif config['level'] == 'info':
+		logger.setLevel(logging.INFO)
+	else:
+		logger.setLevel(logging.ERROR)
+	#new a log file handle
+	log_handler = DateRotatingFileHandler(config['file'], mode="a", maxBytes = config['file_size']*1024*1024, backupCount=config['file_cnt'])
+	#set log file handle format
+	formatter = logging.Formatter(log_file_format, datefmt='%Y-%m-%d %H:%M:%S')# '%F %T') jython not work
+	log_handler.setFormatter(formatter)
+	#add to root logger
+	logger.addHandler(log_handler)
+	#new a stderr logger if need
+	if not config['stderr']:
+		return logger
+	else:
+
+		formatter = ColoredFormatter(log_stderr_format)
+		#formatter = logging.Formatter(log_stderr_format)
+		log_handler = logging.StreamHandler()
+		log_handler.setFormatter(formatter)
+		logger.addHandler(log_handler)
+		return logger
+	
+
+
+if __name__ == '__main__':
+	global logger
+
+	logger = init_log('test_log_lib', 'msg_log.ini', 0, 0)
+	logger.debug('debug rrrrrrrrrrrrrrrrrrrrrrrrrrrt')
+	logger.info('info rrrrrrrrrrrrrrrrrrrrrrrrrrrt')
+	logger.error('error rrrrrrrrrrrrrrrrrrrrrrrrrrrt')
+

常用脚本/python/rollback_log.cfg

@@ -0,0 +1,5 @@
+[conf]
+interval=43200
+day=7
+used=1024
+log_dir=/log/

常用脚本/python/rollback_log.py

@@ -0,0 +1,131 @@
+#!/usr/bin/python2
+
+from socket import *
+import time
+import sys
+import os
+import re
+import signal
+import ConfigParser
+import struct
+import subprocess
+import shutil
+from subprocess import Popen
+
+#sys.path.append('/lib/')
+import rising_log
+
+ROLLBACK_CONF = 'rollback_log.cfg'
+
+### init log ###
+global logger
+logger = rising_log.init_log('rollback_log', 'msg_log.ini', 0, 0)
+
+def _sig_INT_handle(signum, frame):
+        logger.error("SIGINT recieve, exit()")
+        sys.exit()
+		
+def dir_status(log_dir,used):
+
+	syscmd = 'du -s %s'%(log_dir)
+	
+	output = Popen(syscmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+		
+	output_err = output.stderr.readlines()
+	output_out = output.stdout.readlines()
+		
+	if len(output_err) > 0:
+		for line in output_err:
+			logger.error("%s",line)
+		logger.error("syscmd:%s failed",syscmd)
+		return 0
+			
+	if len(output_out) > 0:
+		for line in output_out:
+			if log_dir in line:
+				dir_used = int(line.split("\t")[0])/1024
+						
+	logger.info("dir used:%d, max:%d",dir_used,used)
+
+	if dir_used > used:
+		logger.info("disk used:%d to max:%d",dir_used,used)
+		return -1
+	return 0
+		
+def check_dir(data_dir):
+	if not os.path.exists(data_dir):
+		logger.error("no dir:%s,then create dir",data_dir)
+		os.makedirs(data_dir)
+
+def init_rollback_config(config_file):
+	rollback_config = {}
+	try:
+		config = ConfigParser.ConfigParser()
+		config.read(config_file)
+
+		rollback_config['interval'] = config.get('conf', 'interval')
+		rollback_config['day'] = config.get('conf', 'day')
+		rollback_config['used'] = config.get('conf', 'used')
+		rollback_config['log_dir'] = config.get('conf', 'log_dir')
+
+	except Exception,msg:
+		logger.error(str(msg).strip())
+		sys.exit(1)
+
+	return rollback_config
+
+
+def rollback_log(log_dir,day,used):
+
+	file_list = []
+	logdir_list =[]
+
+	file_list = os.listdir(log_dir)
+	
+	if len(file_list) <= 0:
+		logger.debug("no log data in dir:%s",log_dir)
+		return 0
+
+	file_list.sort(reverse=True)
+	dir_num = 0
+	for i in file_list:
+		logdir_name = os.path.join(log_dir, i)
+		if os.path.isdir(logdir_name):
+			dir_num = dir_num + 1
+			if dir_num > day:
+				logger.info("dir_num:%d, maxnum:%d",dir_num,day)
+				logger.info("delete log dir:%s",logdir_name)
+				shutil.rmtree(logdir_name)
+
+			else:
+				logdir_list.append(logdir_name)
+
+	logdir_list.sort()
+	del logdir_list[-1]
+
+	for i in logdir_list:
+		if dir_status(log_dir,used) < 0 :
+			logger.info("delete log dir:%s",i)
+			shutil.rmtree(i)
+		else:
+			break
+	
+	return 1
+
+def main():
+	signal.signal(signal.SIGINT, _sig_INT_handle)
+
+	while(1):
+		rollback_config = init_rollback_config(ROLLBACK_CONF)
+	
+		sleep_interval = int(rollback_config['interval'])
+		log_dir = rollback_config['log_dir']
+		day = int(rollback_config['day'])
+		used = int(rollback_config['used'])
+
+		rollback_log(log_dir,day,used)
+		time.sleep(sleep_interval)
+
+if __name__ == '__main__':
+        main()
+