first commit

This commit is contained in:
2021-08-29 00:02:47 +08:00
commit 01e8b33396
52 changed files with 4404 additions and 0 deletions

212
常用脚本/agent.py Executable file
View File

@@ -0,0 +1,212 @@
#!/usr/bin/python2
# -*- coding:utf-8 -*-
#=========================================
# Filename : agent.py
# Filetype : Python
# Author : Colben
# Create : 2016-07-28 13:36:04
#=========================================
import os, sys, paramiko, re, csv
from optparse import OptionParser
from multiprocessing import Process, cpu_count, Lock
from time import sleep, time
from threading import Thread
def print_process_result():
output_fmt = {}
output_fmt['ERROR'] = '\033[31;5mERROR\033[0m'
output_fmt['WRONG'] = '\033[33;1mWRONG\033[0m'
output_fmt['RIGHT'] = '\033[32mRIGHT\033[0m'
output_fmt['LOCAL'] = '\033[32mLOCAL\033[0m'
csv_rfp = open('/tmp/agent_result.csv', 'rb')
for record in csv.reader(csv_rfp):
record[0] = record[0] + '-'*(15-len(record[0]))
record[1] = output_fmt[record[1]]
print '-> '.join(record)
csv_rfp.close()
return
def write_csv(record):
process_lock.acquire()
csv_writer.writerow(record)
process_lock.release()
return
def time_out(record, timeout):
detail = record[2]
begin_time = int(time())
while int(time()) - begin_time < timeout:
if detail != record[2]: return
sleep(0.5)
write_csv(record)
os.kill(os.getpid(), 9)
return
def start_timer(record, timeout):
if 0 >= options.timeout:
return
thread_timer = Thread(target = time_out, args = (record, timeout, ))
thread_timer.setDaemon(1)
thread_timer.start()
return
def ssh_command(ip):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
record = [ip, 'ERROR', 'CONN TIMEOUT']
start_timer(record, 10)
try:
ssh.connect(ip, port=22, username='root', pkey=ssh_key)
#ssh.connect(ip, 22, 'root', 'password')
except Exception, e:
record[2] = str(e)
write_csv(record)
return
record[2], record[1] = 'TASK TIMEOUT', 'WRONG'
command = re.sub(r'IIPP', ip, options.remote_cmd)
start_timer(record, options.timeout)
stdin, stdout, stderr = ssh.exec_command(command)
err = stderr.readlines()
out = stdout.readlines()
if err:
record[2] = ' '.join([x.strip() for x in err])
else:
record[2], record[1] = ' '.join([x.strip() for x in out]), 'RIGHT'
write_csv(record)
return
def ssh_transport(ip):
record = [ip, 'ERROR', 'CONN TIMEOUT']
start_timer(record, 10)
try:
t = paramiko.Transport((ip, 22))
t.connect(username='root', pkey=ssh_key)
#t.connect(username='root', password='password')
except Exception, e:
record[2] = str(e)
write_csv(record)
return
record[2], record[1] = 'TASK TIMEOUT', 'WRONG'
remote_file = re.sub(r'IIPP',ip,os.path.join(options.dst_path, os.path.basename(options.src_file)))
start_timer(record, options.timeout)
try:
sftp = paramiko.SFTPClient.from_transport(t)
sftp.put(options.src_file, remote_file)
record[2], record[1] = '', 'RIGHT'
except Exception, e:
record[2] = str(e)
write_csv(record)
return
def local_command(ip):
command = re.sub(r'IIPP', ip, options.local_cmd)
record = [ip, 'WRONG', 'TASK TIMEOUT']
start_timer(record, options.timeout)
result = os.popen(command)
record[2], record[1] = ' '.join([x.strip() for x in result.readlines()]), 'LOCAL'
write_csv(record)
return
def check_pool_state(empty=False):
while True:
print "\033[2J\033[0;0H"
for ip, process in process_list.items():
if process.is_alive():
print 'Handling', ip, '...'
else:
#print ip, '\033[32mfinished\033[0m'
process_list.pop(ip)
if not empty and options.process > len(process_list): break
if empty and 0 == len(process_list):break
sleep(0.5)
return
def start_process_pool():
for ip in ips:
if options.remote_cmd:
sub_process = Process(target=ssh_command, args=(ip,))
elif options.src_file:
sub_process = Process(target=ssh_transport, args=(ip,))
elif options.local_cmd:
sub_process = Process(target=local_command, args=(ip,))
else:
print 'Unknown parameter ...'
sys.exit(1)
sub_process.start()
process_list[ip] = sub_process
check_pool_state()
check_pool_state(empty=True)
return
def get_hosts(hosts_file):
if '/dev/stdin' == hosts_file:
print 'Specify no host file, enter ip manually here ...'
try:
fp = open(hosts_file, 'r')
except Exception, e:
print 'Failed to read hosts_file,', e
sys.exit(1)
ips = []
for line in fp:
matchs = re.match(r'^(10\.\d{1,3}\.\d{1,3}\.\d{1,3})', line)
if matchs: ips.append(matchs.group(1))
if 0 == len(ips):
print 'No ips found, quit ...'
return ips
def initial():
global csv_writer, process_lock, ssh_key, ips, csv_fp, process_list
try:
ssh_key = paramiko.RSAKey.from_private_key_file('/root/.ssh/id_rsa')
csv_fp = open('/tmp/agent_result.csv', 'wb', 0)
csv_writer = csv.writer(csv_fp)
except Exception, e:
print e
sys.exit(1)
ips = get_hosts(options.host)
process_lock = Lock()
process_list = {}
return
def get_options():
global options
usage = '%s\n\t-R remote-cmd -H ip-file -P process-num -T timeout'%sys.argv[0]
usage += '\n\t-S local-file -D remote-dir -H ip-file -P process-num -T timeout'
usage += '\n\t-L local-cmd -H ip-file -P process-num -T timeout'
usage += '\nThe pattern "IIPP" in options "RSDL" will be replaced by the ip contained in each process.'
usage += '\nThe result file is /tmp/agent_result.csv'
parser = OptionParser(usage)
parser.add_option('-R', '--remote_cmd', action='store', help='Run a shell command in remote servers.')
parser.add_option('-S', '--src_file', action='store', help='specify the file to remote servers.')
parser.add_option('-D', '--dst_path', action='store', help='specify the path in remote servers.')
parser.add_option('-L', '--local_cmd', action='store', help='Run a shell command in localhost.')
parser.add_option('-H', '--host', action='store', default='/dev/stdin', help='Specify the file contains ip.')
parser.add_option('-P', '--process', action='store', default=cpu_count(), type='int', help='Specify the num of processes.')
parser.add_option('-T', '--timeout', action='store', default=0, type='int', help='Specify the seconds of timeout.')
options, args = parser.parse_args()
for opt in [options.src_file, options.dst_path, options.local_cmd]:
if options.remote_cmd and opt:
parser.print_help()
sys.exit(1)
for opt in [options.src_file, options.dst_path]:
if options.local_cmd and opt:
parser.print_help()
sys.exit(1)
if options.src_file and not options.dst_path or options.dst_path and not options.src_file:
parser.print_help()
sys.exit(1)
if not options.remote_cmd and not options.src_file and not options.local_cmd:
parser.print_help()
sys.exit(1)
return
def main():
get_options()
initial()
start_process_pool()
print_process_result()
return 0
if '__main__' == __name__:
main()

61
常用脚本/ipflow Executable file
View File

@@ -0,0 +1,61 @@
#!/bin/bash
type function > /dev/null 2>&1
[ 0 -ne $? ] && echo 'Run with bash.' && exit 3
[ 1 -lt $# ] && echo "Usage: $0 [seconds]" && exit 1
[[ ! "$2" =~ ^[0-9]*$ ]] && "Usage: $0 [seconds]" && exit 2
CONF='/tmp/ipflow.conf'
function deal_forward {
for ip in $(cat $CONF); do
iptables -$1 FORWARD -s $ip -j COLBEN_UPLOAD
iptables -$1 FORWARD -d $ip -j COLBEN_DOWNLOAD
done
}
function insert_forward {
cat /proc/net/arp | grep '^10' | grep -v '00:00:00:00:00:00' | awk '{print $1}' > $CONF
deal_forward I
}
function clean_forward {
[ ! -f "$CONF" ] && return 0
deal_forward D
rm -f $CONF
}
function show_result {
clear
echo 'Download:'
iptables -nvxL FORWARD | grep COLBEN_DOWNLOAD | awk "{printf(\"%-4dKB/s:%-10s \n\",\$2/1024/$1,\$9)}" | sort -t: -r
echo -e '\033[s\033[1;25HUpload:'
line_num=1
iptables -nvxL FORWARD | grep COLBEN_UPLOAD | awk "{printf(\"%-4dKB/s:%-10s\n\",\$2/1024/$1,\$8)}" | sort -t: -r | while read line; do
let "line_num=1+$line_num"
echo -e "\033[$line_num;25H$line"
done
echo -e '\033[u'
sleep $1
}
function quit {
clean_forward
iptables -X COLBEN_UPLOAD
iptables -X COLBEN_DOWNLOAD
exit 0
}
iptables -N COLBEN_UPLOAD
iptables -N COLBEN_DOWNLOAD
trap "quit" 2 3 15
while :; do
insert_forward
iptables -Z FORWARD
show_result ${1:-2}
clean_forward
sleep 1
done

232
常用脚本/oracle/oracle_mul.py Executable file
View File

@@ -0,0 +1,232 @@
#!/nmsmw/python/bin/python
# -*- encoding: gbk -*-
#=========================================
# Filename : oracle_mul.py
# Filetype : Python
# Author : Colben
# Create : 2014-12-19 10:40:30
#=========================================
import os, sys, re, cx_Oracle, signal
from multiprocessing import Pool, cpu_count
os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.ZHS16GBK'
#os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.AL32UTF8'
remote_ora = {}#<<FOLD<<
#remote_ora['gk'] = 'boya/boya@10.128.75.65:1521/orcl.ntars.com'
remote_ora['gk'] = 'ibms/ibms@10.166.64.41:1521/orcl'
remote_ora['108'] = 'boya/boya@10.166.64.41:1521/orcl'
remote_ora['anhui'] = 'boya/boya@10.152.64.33:1521/orcl.idevelopment.info'
remote_ora['beijing'] = 'boya/boya@10.134.64.33:1521/orcl.ntars.com'
remote_ora['chongqing'] = 'boya/boya@10.137.64.33:1521/orcl'
remote_ora['fujian'] = 'boya/boya@10.162.64.33:1521/orcl.ntars.com'
remote_ora['gansu'] = 'boya/boya@10.142.64.33:1521/orcl'
remote_ora['guangdong'] = 'boya/boya@10.164.64.33:1521/orcl.ntars.com'
remote_ora['guangxi'] = 'boya/boya@10.161.64.33:1521/orcl'
remote_ora['guizhou'] = 'boya/boya@10.157.64.33:1521/orcl'
remote_ora['hainan'] = 'boya/boya@10.160.64.33:1521/orcl'
remote_ora['hebei'] = 'boya/boya@10.141.64.33:1521/orcl.ntars.com'
remote_ora['heilong'] = 'boya/boya@10.146.64.33:1521/orcl.ntars.com'
remote_ora['henan'] = 'boya/boya@10.151.64.33:1521/orcl.ntars.com'
remote_ora['hubei'] = 'boya/boya@10.155.64.33:1521/orcl.ntars.com'
remote_ora['hunan'] = 'boya/boya@10.158.64.33:1521/orcl.ntars.com'
remote_ora['jiangsu'] = 'boya/boya@10.150.64.33:1521/orclXDB.ntars.com'
remote_ora['jiangxi'] = 'boya/boya@10.159.64.33:1521/orcl.info'
remote_ora['jilin'] = 'boya/boya@10.147.64.33:1521/orcl'
remote_ora['liaoning'] = 'boya/boya@10.143.64.33:1521/orcl.ntars.info'
remote_ora['neimeng'] = 'boya/boya@10.135.64.33:1521/orcl'
remote_ora['ningxia'] = 'boya3w/boya3w@10.139.64.33:1521/orcl'
remote_ora['qinghai'] = 'boya/boya@10.144.64.33:1521/orcl.ntars.com'
remote_ora['shandong'] = 'boya/boya@10.154.64.33:1521/orcl.ntars.com'
remote_ora['shanghai'] = 'boya/boya@10.163.64.33:1521/orcl.ntars.com'
remote_ora['shanxi'] = 'boya/boya@10.140.64.33:1521/orcl.ntars.com'
remote_ora['sanxi'] = 'boya/boya@10.145.64.33:1521/orcl.ntars.com'
remote_ora['sichuan'] = 'boya/boya@10.138.64.33:1521/orcl.ntars.com'
remote_ora['tianjin'] = 'boya/boya@10.148.64.33:1521/orcl'
remote_ora['xinjiang'] = 'boya/boya@10.149.64.33:1521/orcl.ntars.com'
remote_ora['xizang'] = 'boya/boya@10.136.64.33:1521/orcl.ntars.com'
remote_ora['yunnan'] = 'boya/boya@10.156.64.33:1521/orcl'
remote_ora['zhejiang'] = 'boya/boya@10.153.64.33:1521/orcl.ntars.com'
remote_rac = {}
#remote_rac['gk'] = 'boya/boya@10.128.75.67:1521/orcl.ntars.com'
remote_rac['anhui'] = 'boya/boya@10.152.64.34:1521/orcl.idevelopment.info'
remote_rac['beijing'] = 'boya/boya@10.134.64.34:1521/orcl.ntars.com'
remote_rac['chongqing'] = 'boya/boya@10.137.64.34:1521/orcl'
remote_rac['fujian'] = 'boya/boya@10.162.64.34:1521/orcl.ntars.com'
remote_rac['gansu'] = 'boya/boya@10.142.64.34:1521/orcl'
remote_rac['guangdong'] = 'boya/boya@10.164.64.34:1521/orcl.ntars.com'
remote_rac['guangxi'] = 'boya/boya@10.161.64.34:1521/orcl'
remote_rac['guizhou'] = 'boya/boya@10.157.64.34:1521/orcl'
remote_rac['hainan'] = 'boya/boya@10.160.64.34:1521/orcl'
remote_rac['hebei'] = 'boya/boya@10.141.64.34:1521/orcl.ntars.com'
remote_rac['heilong'] = 'boya/boya@10.146.64.34:1521/orcl.ntars.com'
remote_rac['henan'] = 'boya/boya@10.151.64.34:1521/orcl.ntars.com'
remote_rac['hubei'] = 'boya/boya@10.155.64.34:1521/orcl.ntars.com'
remote_rac['hunan'] = 'boya/boya@10.158.64.34:1521/orcl.ntars.com'
remote_rac['jiangsu'] = 'boya/boya@10.150.64.34:1521/orclXDB.ntars.com'
remote_rac['jiangxi'] = 'boya/boya@10.159.64.34:1521/orcl.info'
remote_rac['jilin'] = 'boya/boya@10.147.64.34:1521/orcl'
remote_rac['liaoning'] = 'boya/boya@10.143.64.34:1521/orcl.ntars.info'
remote_rac['neimeng'] = 'boya/boya@10.135.64.34:1521/orcl'
remote_rac['ningxia'] = 'boya3w/boya3w@10.139.64.34:1521/orcl'
remote_rac['qinghai'] = 'boya/boya@10.144.64.34:1521/orcl.ntars.com'
remote_rac['shandong'] = 'boya/boya@10.154.64.34:1521/orcl.ntars.com'
remote_rac['shanghai'] = 'boya/boya@10.163.64.34:1521/orcl.ntars.com'
remote_rac['shanxi'] = 'boya/boya@10.140.64.34:1521/orcl.ntars.com'
remote_rac['sanxi'] = 'boya/boya@10.145.64.34:1521/orcl.ntars.com'
remote_rac['sichuan'] = 'boya/boya@10.138.64.34:1521/orcl.ntars.com'
remote_rac['tianjin'] = 'boya/boya@10.148.64.34:1521/orcl'
remote_rac['xinjiang'] = 'boya/boya@10.149.64.34:1521/orcl.ntars.com'
remote_rac['xizang'] = 'boya/boya@10.136.64.34:1521/orcl.ntars.com'
remote_rac['yunnan'] = 'boya/boya@10.156.64.34:1521/orcl'
remote_rac['zhejiang'] = 'boya/boya@10.153.64.34:1521/orcl.ntars.com'
#>>FOLD>>
def close_dbs():#<<FOLD<<
print 'Terminating db connections ...'
for ora, db in accessed_dbs.items():
try: db.close()
except Exception, e:
print 'Close', ora ,'failed:', e
sys.exit()
#>>FOLD>>
def handle_one_db(ora):#<<FOLD<<
result = []
cursor = accessed_dbs[ora].cursor()
for sql in sql_list:
try:
cursor.execute(sql)
except Exception, e:
result.append((sql + ';', e.__class__.__name__ + ': ' + str(e).strip()))
continue
try:
result.append((sql + ';', cursor.fetchall()))
except cx_Oracle.InterfaceError:
result.append((sql + ';', 'NULL'))
cursor.close()
return (ora, result)
#>>FOLD>>
def runsqls_on_alldbs():#<<FOLD<<
process_pool = Pool(cpu_count())
pool_return = process_pool.map_async(handle_one_db, accessed_dbs.keys())
for ora, result in pool_return.get():
result_fp.write('Database: ' + ora + '\n')
for line in result:
result_fp.write('Sql: ' + line[0] + '\n')
if type(line[1]) is str:
result_fp.write(line[1] + '\n')
else:
for record in line[1]:
result_fp.write(','.join(map(str, record)))
result_fp.write('\n')
result_fp.write('\n')
process_pool.close()
process_pool.terminate()
return
#>>FOLD>>
def get_fp(filename):#<<FOLD<<
global result_fp
result_fp = sys.stdout
if '' != filename:
try:
result_fp = open(filename, 'a', 0)
except:
print 'Open', filename, 'failed, now print it stdout.'
return
#>>FOLD>>
def get_sqls(sql):#<<FOLD<<
global sql_list
sql_list = []
if '@' != sql[0]:
sql_list.append(sql)
else:
try:
sql_f = open(sql[1:].strip(), 'r')
for line in sql_f:
sql=line.strip().strip(';')
if sql:
sql_list.append(sql)
sql_f.close()
if 0 == len(sql_list):
return False
except Exception, e:
print e.__class__.__name__, ':', e
return False
return True
#>>FOLD>>
def input_sqls():#<<FOLD<<
while True:
sql = raw_input('\nSQL>> ')
if re.match(r'exit.*', sql.lower()) or re.match(r'quit.*', sql.lower()):
close_dbs()
try:
sql, filename = re.split(r';', sql, 1)
except:
print 'Wrong, missing ";" in the end.'
continue
if not get_sqls(sql.strip()):
continue
get_fp(filename.strip())
runsqls_on_alldbs()
return
#>>FOLD>>
def access_db(ora):#<<FOLD<<
if ora not in remote_ora.keys():
return (False, 'Unknown DB')
try:
db = cx_Oracle.connect(remote_ora[ora])
db.autocommit = True
return (True, db)
except Exception, e:
if ora not in remote_rac.keys():
return (False, 'Access Failed: ' + str(e))
try:
db = cx_Oracle.connect(remote_rac[ora])
db.autocommit = True
return (True, db)
except Exception, e:
return (False, 'Access Failed: ' + str(e))
#>>FOLD>>
def check_dbs():#<<FOLD<<
global accessed_dbs
accessed_dbs = {}
oras = remote_ora.keys()
if 1 != len(sys.argv):
oras = sys.argv[1:]
for ora in oras:
accessed, db = access_db(ora)
if not accessed:
print ora + ' : ' + db
continue
accessed_dbs[ora] = db
if 0 == len(accessed_dbs):
print 'Accessed no dbs, quit ...'
sys.exit()
return
#>>FOLD>>
def signal_quit(signum, frame):#<<FOLD<<
print 'Got quit signal ...'
close_dbs()
#>>FOLD>>
def main():#<<FOLD<<
check_dbs()
input_sqls()
return
#>>FOLD>>
if '__main__' == __name__:
signal.signal(2, signal_quit)
#signal.signal(3, signal_quit)
#signal.signal(15, signal_quit)
main()

View File

@@ -0,0 +1,305 @@
#!/nmsmw/python/bin/python
import os
import sys
import re
import time
import datetime
import cx_Oracle
from multiprocessing import Pool
remote_ora = {}#<<FOLD<<
#remote_ora['gk'] = 'boya/boya@10.128.75.65:1521/orcl.ntars.com'
remote_ora['gk'] = 'ibms/ibms@10.166.64.41:1521/orcl'
remote_ora['108'] = 'boya/boya@10.166.64.41:1521/orcl'
remote_ora['anhui'] = 'boya/boya@10.152.64.33:1521/orcl.idevelopment.info'
remote_ora['beijing'] = 'boya/boya@10.134.64.33:1521/orcl.ntars.com'
remote_ora['chongqing'] = 'boya/boya@10.137.64.33:1521/orcl'
remote_ora['fujian'] = 'boya/boya@10.162.64.33:1521/orcl.ntars.com'
remote_ora['gansu'] = 'boya/boya@10.142.64.33:1521/orcl'
remote_ora['guangdong'] = 'boya/boya@10.164.64.33:1521/orcl.ntars.com'
remote_ora['guangxi'] = 'boya/boya@10.161.64.33:1521/orcl'
remote_ora['guizhou'] = 'boya/boya@10.157.64.33:1521/orcl'
remote_ora['hainan'] = 'boya/boya@10.160.64.33:1521/orcl'
remote_ora['hebei'] = 'boya/boya@10.141.64.33:1521/orcl.ntars.com'
remote_ora['heilong'] = 'boya/boya@10.146.64.33:1521/orcl.ntars.com'
remote_ora['henan'] = 'boya/boya@10.151.64.33:1521/orcl.ntars.com'
remote_ora['hubei'] = 'boya/boya@10.155.64.33:1521/orcl.ntars.com'
remote_ora['hunan'] = 'boya/boya@10.158.64.33:1521/orcl.ntars.com'
remote_ora['jiangsu'] = 'boya/boya@10.150.64.33:1521/orclXDB.ntars.com'
remote_ora['jiangxi'] = 'boya/boya@10.159.64.33:1521/orcl.info'
remote_ora['jilin'] = 'boya/boya@10.147.64.33:1521/orcl'
remote_ora['liaoning'] = 'boya/boya@10.143.64.33:1521/orcl.ntars.info'
remote_ora['neimeng'] = 'boya/boya@10.135.64.33:1521/orcl'
remote_ora['ningxia'] = 'boya3w/boya3w@10.139.64.33:1521/orcl'
remote_ora['qinghai'] = 'boya/boya@10.144.64.33:1521/orcl.ntars.com'
remote_ora['shandong'] = 'boya/boya@10.154.64.33:1521/orcl.ntars.com'
#remote_ora['shandong'] = 'ntars/sddb_2008@10.154.64.33:1521/orcl.ntars.com'
remote_ora['shanghai'] = 'boya/boya@10.163.64.33:1521/orcl.ntars.com'
remote_ora['shanxi'] = 'boya/boya@10.140.64.33:1521/orcl.ntars.com'
remote_ora['sanxi'] = 'boya/boya@10.145.64.33:1521/orcl.ntars.com'
remote_ora['sichuan'] = 'boya/boya@10.138.64.33:1521/orcl.ntars.com'
remote_ora['tianjin'] = 'boya/boya@10.148.64.33:1521/orcl'
remote_ora['xinjiang'] = 'boya/boya@10.149.64.33:1521/orcl.ntars.com'
remote_ora['xizang'] = 'boya/boya@10.136.64.33:1521/orcl.ntars.com'
remote_ora['yunnan'] = 'boya/boya@10.156.64.33:1521/orcl'
remote_ora['zhejiang'] = 'boya/boya@10.153.64.33:1521/orcl.ntars.com'
remote_ora['ipv6'] = 'boya/boya@10.128.75.6:1521/orcl'
#>>FOLD>>
remote_rac = {}#<<FOLD<<
#remote_rac['gk'] = 'boya/boya@10.128.75.67:1521/orcl.ntars.com'
remote_rac['anhui'] = 'boya/boya@10.152.64.34:1521/orcl.idevelopment.info'
remote_rac['beijing'] = 'boya/boya@10.134.64.34:1521/orcl.ntars.com'
remote_rac['chongqing'] = 'boya/boya@10.137.64.34:1521/orcl'
remote_rac['fujian'] = 'boya/boya@10.162.64.34:1521/orcl.ntars.com'
remote_rac['gansu'] = 'boya/boya@10.142.64.34:1521/orcl'
remote_rac['guangdong'] = 'boya/boya@10.164.64.34:1521/orcl.ntars.com'
remote_rac['guangxi'] = 'boya/boya@10.161.64.34:1521/orcl'
remote_rac['guizhou'] = 'boya/boya@10.157.64.34:1521/orcl'
remote_rac['hainan'] = 'boya/boya@10.160.64.34:1521/orcl'
remote_rac['hebei'] = 'boya/boya@10.141.64.34:1521/orcl.ntars.com'
remote_rac['heilong'] = 'boya/boya@10.146.64.34:1521/orcl.ntars.com'
remote_rac['henan'] = 'boya/boya@10.151.64.34:1521/orcl.ntars.com'
remote_rac['hubei'] = 'boya/boya@10.155.64.34:1521/orcl.ntars.com'
remote_rac['hunan'] = 'boya/boya@10.158.64.34:1521/orcl.ntars.com'
remote_rac['jiangsu'] = 'boya/boya@10.150.64.34:1521/orclXDB.ntars.com'
remote_rac['jiangxi'] = 'boya/boya@10.159.64.34:1521/orcl.info'
remote_rac['jilin'] = 'boya/boya@10.147.64.34:1521/orcl'
remote_rac['liaoning'] = 'boya/boya@10.143.64.34:1521/orcl.ntars.info'
remote_rac['neimeng'] = 'boya/boya@10.135.64.34:1521/orcl'
remote_rac['ningxia'] = 'boya3w/boya3w@10.139.64.34:1521/orcl'
remote_rac['qinghai'] = 'boya/boya@10.144.64.34:1521/orcl.ntars.com'
remote_rac['shandong'] = 'boya/boya@10.154.64.34:1521/orcl.ntars.com'
remote_rac['shanghai'] = 'boya/boya@10.163.64.34:1521/orcl.ntars.com'
remote_rac['shanxi'] = 'boya/boya@10.140.64.34:1521/orcl.ntars.com'
remote_rac['sanxi'] = 'boya/boya@10.145.64.34:1521/orcl.ntars.com'
remote_rac['sichuan'] = 'boya/boya@10.138.64.34:1521/orcl.ntars.com'
remote_rac['tianjin'] = 'boya/boya@10.148.64.34:1521/orcl'
remote_rac['xinjiang'] = 'boya/boya@10.149.64.34:1521/orcl.ntars.com'
remote_rac['xizang'] = 'boya/boya@10.136.64.34:1521/orcl.ntars.com'
remote_rac['yunnan'] = 'boya/boya@10.156.64.34:1521/orcl'
remote_rac['zhejiang'] = 'boya/boya@10.153.64.34:1521/orcl.ntars.com'
#>>FOLD>>
mtx_tables = {}#<<FOLD<<
mtx_tables['MONITOR_FAULT'] = 'COLLECTOR_STATE_SPACE'
mtx_tables['MT_COLLECTOR_STATE'] = 'COLLECTOR_STATE_SPACE'
mtx_tables['MT_COLLECTOR_DIR'] = 'COLLECTOR_STATE_SPACE'
mtx_tables['MTX_PERF_NET'] = 'MTX_PERF_NET'
mtx_tables['MTXRPT_EVENT_M'] = 'EVENT_COUNT_SPACE'
mtx_tables['MTXRPT_DEVICE_M'] = 'EVENT_COUNT_SPACE'
mtx_tables['MTXRPT_EVENTMOBILE_M'] = 'EVENT_COUNT_SPACE'
mtx_tables['RPT_MTXMOBILE_DEVICE_M'] = 'EVENT_COUNT_SPACE'
mtx_tables['RPT_MTXMOBILE_EVENT_M'] = 'EVENT_COUNT_SPACE'
mtx_tables['RPT_MTXMOBILE_IMEI_M'] = 'EVENT_COUNT_SPACE'
mtx_tables['RPT_MTXMOBILE_INOUT_H'] = 'EVENT_COUNT_SPACE'
mtx_tables['RPT_MTXMOBILE_PROVINCE_H'] = 'EVENT_COUNT_SPACE'
mtx_tables['RPT_MTXMOBILE_PROVINCE_M'] = 'EVENT_COUNT_SPACE'
mtx_tables['MTX_EVENT_REALTIME'] = 'EVENT_COUNT_SPACE'
#>>FOLD>>
dns_tables = {}#<<FOLD<<
dns_tables['DOMAIN_AA'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_CLIENT_COUNT'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_COUNT'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_DNAME_LENGTH'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_FLUX'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_OPCODE_COUNT'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_QUERY_PACKET_LENGTH'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_IMSI_TOPN'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_IMSI_TOP100'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_RA'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_RCODE'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_RD'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_RESPONSE_PACKET_LENGTH'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_TC'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_TTL_LENGTH'] = 'DOMAIN_DATA_SPACE'
dns_tables['DOMAIN_TYPE_COUNT'] = 'DOMAIN_DATA_SPACE'
dns_tables['ERROR_PKT'] = 'DOMAIN_DATA_SPACE'
dns_tables['TOPN_CLIENT_BYTE'] = 'DOMAIN_DATA_SPACE'
dns_tables['TOPN_CLIENT_PACKET'] = 'DOMAIN_DATA_SPACE'
dns_tables['TOPN_DOMAIN_NAME'] = 'DOMAIN_DATA_SPACE'
dns_tables['TOPN_NXDOMAIN'] = 'DOMAIN_DATA_SPACE'
dns_tables['TOPN_TLD'] = 'DOMAIN_DATA_SPACE'
dns_tables['DNSHJK'] = 'DOMAIN_DATA_SPACE'
dns_tables['ABNORMAL_BASE_DNS'] = 'DOMAIN_DATA_SPACE'
dns_tables['ABNORMAL_SPECIAL_QUEST'] = 'DOMAIN_DATA_SPACE'
dns_tables['ABNORMAL_SERVER_QA'] = 'DOMAIN_DATA_SPACE'
#>>FOLD>>
dnspod_tables = {}#<<FOLD<<
dnspod_tables['DNSPOD_IP_COUNT'] = 'DOMAIN_DATA_SPACE'
dnspod_tables['DNSPOD_IP_FLUX'] = 'DOMAIN_DATA_SPACE'
dnspod_tables['DNSPOD_LOCATION_COUNT'] = 'DOMAIN_DATA_SPACE'
dnspod_tables['DNSPOD_LOCATION_DOMAIN'] = 'DOMAIN_DATA_SPACE'
dnspod_tables['DNSPOD_PCI_COUNT'] = 'DOMAIN_DATA_SPACE'
dnspod_tables['DNSPOD_TOP100_DOMAIN'] = 'DOMAIN_DATA_SPACE'
dnspod_tables['DNSPOD_TOP_DOMAIN'] = 'DOMAIN_DATA_SPACE'
#>>FOLD>>
vds_tables = {}#<<FOLD<<
vds_tables['VDSRPT_DEVICE_M'] = 'EVENT_COUNT_SPACE'
vds_tables['VDSRPT_EVENT_M'] = 'EVENT_COUNT_SPACE'
#>>FOLD>>
ntars_tables = {}#<<FOLD<<
#ntars_tables['ROUTER_STAT_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_IPSRCPACKET_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_IPDSTBYTE_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_IPDSTPACKET_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_IPCOUNT_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_CPORT_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_IPCOUNT_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_TOS_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_IPSRCBYTE_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_IPDSTBYTE_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_PEERIPDSTBYTE_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_PEERIPDSTPACKET_H'] = 'REPORT_H_SPACE'
#ntars_tables['FILTER_REPORT_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_PROTOCOL_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_PROTOCOL_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_SRCPORT_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_IPDSTPACKET_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_DSTPORT_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_IPSRCPACKET_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_REGION_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_PEERIPSRCPACKET_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_CPORT_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_SRCPORT_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_PEERIPSRCBYTE_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_TOS_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_DSTPORT_H'] = 'REPORT_H_SPACE'
#ntars_tables['ROUTER_IPSRCBYTE_H'] = 'REPORT_H_SPACE'
#ntars_tables['CUSTOMER_REGION_H'] = 'REPORT_H_SPACE'
#ntars_tables['RG_CUSTOMER_SRCPORT_H'] = 'RG_CS_SRCPORT_H_TS'
ntars_tables['ROUTER_STAT_M'] = 'REPORT_M_SPACE'
ntars_tables['ROUTER_IPDSTBYTE_M'] = 'REPORT_M_SPACE'
ntars_tables['ROUTER_IPCOUNT_M'] = 'REPORT_M_SPACE'
ntars_tables['ROUTER_CPORT_M'] = 'REPORT_M_SPACE'
ntars_tables['ROUTER_PROTOCOL_M'] = 'REPORT_M_SPACE'
ntars_tables['ROUTER_REGION_M'] = 'REPORT_M_SPACE'
ntars_tables['ROUTER_SRCPORT_M'] = 'REPORT_M_SPACE'
ntars_tables['ROUTER_DSTPORT_M'] = 'REPORT_M_SPACE'
ntars_tables['ROUTER_IPSRCBYTE_M'] = 'REPORT_M_SPACE'
#>>FOLD>>
topn_tables = {}#<<FOLD<<
topn_tables['TOPN_CLIENT_BYTE'] = 'DOMAIN_DATA_SPACE'
topn_tables['TOPN_CLIENT_PACKET'] = 'DOMAIN_DATA_SPACE'
topn_tables['TOPN_DOMAIN_NAME'] = 'DOMAIN_DATA_SPACE'
topn_tables['TOPN_NXDOMAIN'] = 'DOMAIN_DATA_SPACE'
topn_tables['TOPN_TLD'] = 'DOMAIN_DATA_SPACE'
#>>FOLD>>
extra_tables = {}
extra_tables['ERROR_PKT'] = 'DOMAIN_DATA_SPACE'
extra_tables['MTXRPT_EVENT_M'] = 'DOMAIN_DATA_SPACE'
extra_tables['DOMAIN_IMSI_TOPN'] = 'DOMAIN_DATA_SPACE'
extra_tables['ABNORMAL_BASE_DNS'] = 'DOMAIN_DATA_SPACE'
extra_tables['ABNORMAL_SERVER_QA'] = 'DOMAIN_DATA_SPACE'
extra_tables['ABNORMAL_SPECIAL_QUEST'] = 'DOMAIN_DATA_SPACE'
extra_tables['DNSPOD_PCI_COUNT'] = 'DOMAIN_DATA_SPACE'
extra_tables['DNSPOD_TOP100_DOMAIN'] = 'DOMAIN_DATA_SPACE'
extra_tables['DNSPOD_TOP_DOMAIN'] = 'DOMAIN_DATA_SPACE'
extra_tables['DNSPOD_IP_COUNT'] = 'DOMAIN_DATA_SPACE'
extra_tables['DNSPOD_LOCATION_COUNT'] = 'DOMAIN_DATA_SPACE'
extra_tables['DNSHJK'] = 'DOMAIN_DATA_SPACE'
def StringToTime(strtime):#<<FOLD<<
t_tuple = time.strptime(strtime,"%Y%m%d")
return int(time.mktime(t_tuple))
#>>FOLD>>
def gen_sqls(operate, date):#<<FOLD<<
#tables = dict(dict(dict(mtx_tables, **dns_tables), **dnspod_tables), **vds_tables)
#tables = dict(dns_tables, **dnspod_tables)
tables = mtx_tables
sqls = []
if 'add' == operate:
date1 = StringToTime(date)
for table in tables:
sqls.append('alter table %s add partition PART_%s values less than(%s) tablespace %s'%(table, date, date1, tables[table]))
elif 'drop' == operate:
for table in tables:
sqls.append('alter table %s drop partition PART_%s'%(table, date))
else:
print 'Wrong operation on table partitions, now quit.'
return sqls
#>>FOLD>>
def ora_init(ora):#<<FOLD<<
if ora in remote_ora.keys():
#print ' ------>', ora, '<------'
try:
db = cx_Oracle.connect(remote_ora[ora])
db.autocommit = True
#print 'Connect to oracle server successfully ...'
except:
if ora in remote_rac.keys():
#print 'Time out, now try to connect the rac ...'
try:
db = cx_Oracle.connect(remote_rac[ora])
db.autocommit = True
#print 'Connect to oracle server successfully ...'
except:
#print 'Error, Failed to connect oracle', ora
return
else:
#print 'Error, Failed to connect oracle', ora
return
else:
#print 'Error, Wrong oracle', ora
return
return db
#>>FOLD>>
def operate_partitions(ora):#<<FOLD<<
db = ora_init(ora)
if not db:
print '\n\033[31m %s Connect failed.\033[0m'%ora
return (ora, 'Connect failed')
cur = db.cursor()
month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
for day in range(0, month[int(sys.argv[2])%100-1]):
date = str(int(sys.argv[2])*100+1+day)
sqls = gen_sqls(sys.argv[1], date)
for sql in sqls:
try:
cur.execute(sql)
sys.stdout.write('.')
sys.stdout.flush()
except Exception, e:
matched = 0
ignore_errors = ['ORA-00942', 'ORA-02149']
for error in ignore_errors:
if error in str(e).upper():
matched = 1
sys.stdout.write('.')
sys.stdout.flush()
break
if not matched:
print "\n\033[31mRUN \"%s\" ON %s failed:%s\033[0m"%(sql, ora, str(e))
cur.close()
db.close()
print '\n\033[32m %s Complete.\033[0m'%ora
return (ora, 'Complete')
#>>FOLD>>
def main():#<<FOLD<<
if 0 == len(sys.argv[3:]):
oras = remote_ora.keys()
else:
oras = sys.argv[3:]
process_pool = Pool(8)
result = process_pool.map_async(operate_partitions, oras)
for res in result.get():
print res
#>>FOLD>>
if __name__ == '__main__':
if 2 < len(sys.argv) and sys.argv[2].isdigit():
main()
else:
print 'Usage: %s {add|drop} 201603 [pc1, pc2 ...]'%sys.argv[0]

View File

@@ -0,0 +1,187 @@
#!/nmsmw/python/bin/python
import os
import sys
import re
import time
import datetime
import cx_Oracle
from multiprocessing import Pool
remote_ora = {}#<<FOLD<<
#remote_ora['gk'] = 'boya/boya@10.128.75.65:1521/orcl.ntars.com'
#remote_ora['gk'] = 'ibms/ibms@10.166.64.41:1521/orcl'
remote_ora['108'] = 'boya/boya@10.166.64.41:1521/orcl'
remote_ora['anhui'] = 'boya/boya@10.152.64.33:1521/orcl.idevelopment.info'
remote_ora['beijing'] = 'boya/boya@10.134.64.33:1521/orcl.ntars.com'
remote_ora['chongqing'] = 'boya/boya@10.137.64.33:1521/orcl'
remote_ora['fujian'] = 'boya/boya@10.162.64.33:1521/orcl.ntars.com'
remote_ora['gansu'] = 'boya/boya@10.142.64.33:1521/orcl'
remote_ora['guangdong'] = 'boya/boya@10.164.64.33:1521/orcl.ntars.com'
remote_ora['guangxi'] = 'boya/boya@10.161.64.33:1521/orcl'
remote_ora['guizhou'] = 'boya/boya@10.157.64.33:1521/orcl'
remote_ora['hainan'] = 'boya/boya@10.160.64.33:1521/orcl'
remote_ora['hebei'] = 'boya/boya@10.141.64.33:1521/orcl.ntars.com'
remote_ora['heilong'] = 'boya/boya@10.146.64.33:1521/orcl.ntars.com'
remote_ora['henan'] = 'boya/boya@10.151.64.33:1521/orcl.ntars.com'
remote_ora['hubei'] = 'boya/boya@10.155.64.33:1521/orcl.ntars.com'
remote_ora['hunan'] = 'boya/boya@10.158.64.33:1521/orcl.ntars.com'
remote_ora['jiangsu'] = 'boya/boya@10.150.64.33:1521/orclXDB.ntars.com'
remote_ora['jiangxi'] = 'boya/boya@10.159.64.33:1521/orcl.info'
remote_ora['jilin'] = 'boya/boya@10.147.64.33:1521/orcl'
remote_ora['liaoning'] = 'boya/boya@10.143.64.33:1521/orcl.ntars.info'
remote_ora['neimeng'] = 'boya/boya@10.135.64.33:1521/orcl'
remote_ora['ningxia'] = 'boya3w/boya3w@10.139.64.33:1521/orcl'
remote_ora['qinghai'] = 'boya/boya@10.144.64.33:1521/orcl.ntars.com'
remote_ora['shandong'] = 'boya/boya@10.154.64.33:1521/orcl.ntars.com'
remote_ora['shanghai'] = 'boya/boya@10.163.64.33:1521/orcl.ntars.com'
remote_ora['shanxi'] = 'boya/boya@10.140.64.33:1521/orcl.ntars.com'
remote_ora['sanxi'] = 'boya/boya@10.145.64.33:1521/orcl.ntars.com'
remote_ora['sichuan'] = 'boya/boya@10.138.64.33:1521/orcl.ntars.com'
remote_ora['tianjin'] = 'boya/boya@10.148.64.33:1521/orcl'
remote_ora['xinjiang'] = 'boya/boya@10.149.64.33:1521/orcl.ntars.com'
remote_ora['xizang'] = 'boya/boya@10.136.64.33:1521/orcl.ntars.com'
remote_ora['yunnan'] = 'boya/boya@10.156.64.33:1521/orcl'
remote_ora['zhejiang'] = 'boya/boya@10.153.64.33:1521/orcl.ntars.com'
remote_rac = {}
#remote_rac['gk'] = 'boya/boya@10.128.75.67:1521/orcl.ntars.com'
remote_rac['anhui'] = 'boya/boya@10.152.64.34:1521/orcl.idevelopment.info'
remote_rac['beijing'] = 'boya/boya@10.134.64.34:1521/orcl.ntars.com'
remote_rac['chongqing'] = 'boya/boya@10.137.64.34:1521/orcl'
remote_rac['fujian'] = 'boya/boya@10.162.64.34:1521/orcl.ntars.com'
remote_rac['gansu'] = 'boya/boya@10.142.64.34:1521/orcl'
remote_rac['guangdong'] = 'boya/boya@10.164.64.34:1521/orcl.ntars.com'
remote_rac['guangxi'] = 'boya/boya@10.161.64.34:1521/orcl'
remote_rac['guizhou'] = 'boya/boya@10.157.64.34:1521/orcl'
remote_rac['hainan'] = 'boya/boya@10.160.64.34:1521/orcl'
remote_rac['hebei'] = 'boya/boya@10.141.64.34:1521/orcl.ntars.com'
remote_rac['heilong'] = 'boya/boya@10.146.64.34:1521/orcl.ntars.com'
remote_rac['henan'] = 'boya/boya@10.151.64.34:1521/orcl.ntars.com'
remote_rac['hubei'] = 'boya/boya@10.155.64.34:1521/orcl.ntars.com'
remote_rac['hunan'] = 'boya/boya@10.158.64.34:1521/orcl.ntars.com'
remote_rac['jiangsu'] = 'boya/boya@10.150.64.34:1521/orclXDB.ntars.com'
remote_rac['jiangxi'] = 'boya/boya@10.159.64.34:1521/orcl.info'
remote_rac['jilin'] = 'boya/boya@10.147.64.34:1521/orcl'
remote_rac['liaoning'] = 'boya/boya@10.143.64.34:1521/orcl.ntars.info'
remote_rac['neimeng'] = 'boya/boya@10.135.64.34:1521/orcl'
remote_rac['ningxia'] = 'boya3w/boya3w@10.139.64.34:1521/orcl'
remote_rac['qinghai'] = 'boya/boya@10.144.64.34:1521/orcl.ntars.com'
remote_rac['shandong'] = 'boya/boya@10.154.64.34:1521/orcl.ntars.com'
remote_rac['shanghai'] = 'boya/boya@10.163.64.34:1521/orcl.ntars.com'
remote_rac['shanxi'] = 'boya/boya@10.140.64.34:1521/orcl.ntars.com'
remote_rac['sanxi'] = 'boya/boya@10.145.64.34:1521/orcl.ntars.com'
remote_rac['sichuan'] = 'boya/boya@10.138.64.34:1521/orcl.ntars.com'
remote_rac['tianjin'] = 'boya/boya@10.148.64.34:1521/orcl'
remote_rac['xinjiang'] = 'boya/boya@10.149.64.34:1521/orcl.ntars.com'
remote_rac['xizang'] = 'boya/boya@10.136.64.34:1521/orcl.ntars.com'
remote_rac['yunnan'] = 'boya/boya@10.156.64.34:1521/orcl'
remote_rac['zhejiang'] = 'boya/boya@10.153.64.34:1521/orcl.ntars.com'
#>>FOLD>>
app_tables = {}#<<FOLD<<
app_tables['APPRPT_APPSTORE_D'] = 'APPRPT_SPACE'
app_tables['APPRPT_APP_USING_D'] = 'APPRPT_SPACE'
app_tables['APPRPT_DEVICE_BAND_D'] = 'APPRPT_SPACE'
app_tables['APPRPT_DEVICE_BAND_M'] = 'APPRPT_SPACE'
app_tables['APPRPT_DEVICE_TYPE_D'] = 'APPRPT_SPACE'
app_tables['APPRPT_DEVICE_TYPE_M'] = 'APPRPT_SPACE'
app_tables['APPRPT_DOWNLOAD_D'] = 'APPRPT_SPACE'
app_tables['APPRPT_SPECIAL_USING_D'] = 'APPRPT_SPACE'
#>>FOLD>>
def StringToTime(strtime):#<<FOLD<<
t_tuple = time.strptime(str(strtime),"%Y%m%d")
return int(time.mktime(t_tuple))
#>>FOLD>>
def gen_sqls(operate, date):#<<FOLD<<
tables = app_tables
sqls = []
if 'add' == operate:
date1 = StringToTime(date)
for table in tables:
sqls.append('alter table %s add partition PART_%s values less than(%s) tablespace %s'%(table, date/100, date1, tables[table]))
elif 'drop' == operate:
for table in tables:
sqls.append('alter table %s drop partition PART_%s'%(table, date/100))
else:
print 'Wrong operation on table partitions, now quit.'
return
return sqls
#>>FOLD>>
def ora_init(ora):#<<FOLD<<
if ora in remote_ora.keys():
#print ' ------>', ora, '<------'
try:
db = cx_Oracle.connect(remote_ora[ora])
db.autocommit = True
#print 'Connect to oracle server successfully ...'
except:
if ora in remote_rac.keys():
#print 'Time out, now try to connect the rac ...'
try:
db = cx_Oracle.connect(remote_rac[ora])
db.autocommit = True
#print 'Connect to oracle server successfully ...'
except:
#print 'Error, Failed to connect oracle', ora
return
else:
#print 'Error, Failed to connect oracle', ora
return
else:
#print 'Error, Wrong oracle', ora
return
return db
#>>FOLD>>
def operate_partitions(ora):#<<FOLD<<
db = ora_init(ora)
if not db:
print '\n\033[31m %s Connect failed.\033[0m'%ora
return (ora, 'Connect failed')
cur = db.cursor()
for month in range(0, 12):
date = (int(sys.argv[2])*100+1+month)*100+1
sqls = gen_sqls(sys.argv[1], date)
for sql in sqls:
try:
cur.execute(sql)
sys.stdout.write('.')
sys.stdout.flush()
except Exception, e:
matched = 0
ignore_errors = ['ORA-00942', 'ORA-02149']
for error in ignore_errors:
if error in str(e).upper():
matched = 1
sys.stdout.write('.')
sys.stdout.flush()
break
if not matched:
print "\n\033[31mRUN %s ON %s failed:%s\033[0m"%(sql, ora, e)
cur.close()
db.close()
print '\n\033[32m %s Complete.\033[0m'%ora
return (ora, 'Complete')
#>>FOLD>>
def main():#<<FOLD<<
if 0 == len(sys.argv[3:]):
oras = remote_ora.keys()
else:
oras = sys.argv[3:]
process_pool = Pool(8)
result = process_pool.map_async(operate_partitions, oras)
for res in result.get():
print res
#>>FOLD>>
if __name__ == '__main__':
if 2 < len(sys.argv) and sys.argv[2].isdigit():
main()
else:
print 'Usage: %s {add|drop} 2016 [pc1, pc2 ...]'%sys.argv[0]

View File

@@ -0,0 +1,188 @@
#!/nmsmw/python/bin/python
import os
import sys
import re
import time
import datetime
import cx_Oracle
from multiprocessing import Pool
remote_ora = {}#<<FOLD<<
#remote_ora['gk'] = 'boya/boya@10.128.75.65:1521/orcl.ntars.com'
#remote_ora['gk'] = 'ibms/ibms@10.166.64.41:1521/orcl'
remote_ora['108'] = 'boya/boya@10.166.64.41:1521/orcl'
remote_ora['anhui'] = 'boya/boya@10.152.64.33:1521/orcl.idevelopment.info'
remote_ora['beijing'] = 'boya/boya@10.134.64.33:1521/orcl.ntars.com'
remote_ora['chongqing'] = 'boya/boya@10.137.64.33:1521/orcl'
remote_ora['fujian'] = 'boya/boya@10.162.64.33:1521/orcl.ntars.com'
remote_ora['gansu'] = 'boya/boya@10.142.64.33:1521/orcl'
remote_ora['guangdong'] = 'boya/boya@10.164.64.33:1521/orcl.ntars.com'
remote_ora['guangxi'] = 'boya/boya@10.161.64.33:1521/orcl'
remote_ora['guizhou'] = 'boya/boya@10.157.64.33:1521/orcl'
remote_ora['hainan'] = 'boya/boya@10.160.64.33:1521/orcl'
remote_ora['hebei'] = 'boya/boya@10.141.64.33:1521/orcl.ntars.com'
remote_ora['heilong'] = 'boya/boya@10.146.64.33:1521/orcl.ntars.com'
remote_ora['henan'] = 'boya/boya@10.151.64.33:1521/orcl.ntars.com'
remote_ora['hubei'] = 'boya/boya@10.155.64.33:1521/orcl.ntars.com'
remote_ora['hunan'] = 'boya/boya@10.158.64.33:1521/orcl.ntars.com'
remote_ora['jiangsu'] = 'boya/boya@10.150.64.33:1521/orclXDB.ntars.com'
remote_ora['jiangxi'] = 'boya/boya@10.159.64.33:1521/orcl.info'
remote_ora['jilin'] = 'boya/boya@10.147.64.33:1521/orcl'
remote_ora['liaoning'] = 'boya/boya@10.143.64.33:1521/orcl.ntars.info'
remote_ora['neimeng'] = 'boya/boya@10.135.64.33:1521/orcl'
remote_ora['ningxia'] = 'boya3w/boya3w@10.139.64.33:1521/orcl'
remote_ora['qinghai'] = 'boya/boya@10.144.64.33:1521/orcl.ntars.com'
remote_ora['shandong'] = 'boya/boya@10.154.64.33:1521/orcl.ntars.com'
remote_ora['shanghai'] = 'boya/boya@10.163.64.33:1521/orcl.ntars.com'
remote_ora['shanxi'] = 'boya/boya@10.140.64.33:1521/orcl.ntars.com'
remote_ora['sanxi'] = 'boya/boya@10.145.64.33:1521/orcl.ntars.com'
remote_ora['sichuan'] = 'boya/boya@10.138.64.33:1521/orcl.ntars.com'
remote_ora['tianjin'] = 'boya/boya@10.148.64.33:1521/orcl'
remote_ora['xinjiang'] = 'boya/boya@10.149.64.33:1521/orcl.ntars.com'
remote_ora['xizang'] = 'boya/boya@10.136.64.33:1521/orcl.ntars.com'
remote_ora['yunnan'] = 'boya/boya@10.156.64.33:1521/orcl'
remote_ora['zhejiang'] = 'boya/boya@10.153.64.33:1521/orcl.ntars.com'
remote_rac = {}
#remote_rac['gk'] = 'boya/boya@10.128.75.67:1521/orcl.ntars.com'
remote_rac['anhui'] = 'boya/boya@10.152.64.34:1521/orcl.idevelopment.info'
remote_rac['beijing'] = 'boya/boya@10.134.64.34:1521/orcl.ntars.com'
remote_rac['chongqing'] = 'boya/boya@10.137.64.34:1521/orcl'
remote_rac['fujian'] = 'boya/boya@10.162.64.34:1521/orcl.ntars.com'
remote_rac['gansu'] = 'boya/boya@10.142.64.34:1521/orcl'
remote_rac['guangdong'] = 'boya/boya@10.164.64.34:1521/orcl.ntars.com'
remote_rac['guangxi'] = 'boya/boya@10.161.64.34:1521/orcl'
remote_rac['guizhou'] = 'boya/boya@10.157.64.34:1521/orcl'
remote_rac['hainan'] = 'boya/boya@10.160.64.34:1521/orcl'
remote_rac['hebei'] = 'boya/boya@10.141.64.34:1521/orcl.ntars.com'
remote_rac['heilong'] = 'boya/boya@10.146.64.34:1521/orcl.ntars.com'
remote_rac['henan'] = 'boya/boya@10.151.64.34:1521/orcl.ntars.com'
remote_rac['hubei'] = 'boya/boya@10.155.64.34:1521/orcl.ntars.com'
remote_rac['hunan'] = 'boya/boya@10.158.64.34:1521/orcl.ntars.com'
remote_rac['jiangsu'] = 'boya/boya@10.150.64.34:1521/orclXDB.ntars.com'
remote_rac['jiangxi'] = 'boya/boya@10.159.64.34:1521/orcl.info'
remote_rac['jilin'] = 'boya/boya@10.147.64.34:1521/orcl'
remote_rac['liaoning'] = 'boya/boya@10.143.64.34:1521/orcl.ntars.info'
remote_rac['neimeng'] = 'boya/boya@10.135.64.34:1521/orcl'
remote_rac['ningxia'] = 'boya3w/boya3w@10.139.64.34:1521/orcl'
remote_rac['qinghai'] = 'boya/boya@10.144.64.34:1521/orcl.ntars.com'
remote_rac['shandong'] = 'boya/boya@10.154.64.34:1521/orclXDB.ntars.com'
remote_rac['shanghai'] = 'boya/boya@10.163.64.34:1521/orcl.ntars.com'
remote_rac['shanxi'] = 'boya/boya@10.140.64.34:1521/orcl.ntars.com'
remote_rac['sanxi'] = 'boya/boya@10.145.64.34:1521/orcl.ntars.com'
remote_rac['sichuan'] = 'boya/boya@10.138.64.34:1521/orcl.ntars.com'
remote_rac['tianjin'] = 'boya/boya@10.148.64.34:1521/orcl'
remote_rac['xinjiang'] = 'boya/boya@10.149.64.34:1521/orcl.ntars.com'
remote_rac['xizang'] = 'boya/boya@10.136.64.34:1521/orcl.ntars.com'
remote_rac['yunnan'] = 'boya/boya@10.156.64.34:1521/orcl'
remote_rac['zhejiang'] = 'boya/boya@10.153.64.34:1521/orcl.ntars.com'
#>>FOLD>>
vds_tables = {}#<<FOLD<<
vds_tables['VDSRPT_DOMAIN_DOWNLOAD'] = 'VDS_REPORT_SPACE'
vds_tables['VDSRPT_PROGRAM_DOWNLOAD'] = 'VDS_REPORT_SPACE'
vds_tables['VDSRPT_SIP_DOWNLOAD'] = 'VDS_REPORT_SPACE'
vds_tables['VDSRPT_URL_DOWNLOAD'] = 'VDS_REPORT_SPACE'
vds_tables['VDSRPT_VIRUSFALIMY_DOWNLOAD'] = 'VDS_REPORT_SPACE'
vds_tables['VDSRPT_VIRUSTYPE_DOWNLOAD'] = 'VDS_REPORT_SPACE'
vds_tables['VDSRPT_VIRUSTYPE_VISIT'] = 'VDS_REPORT_SPACE'
vds_tables['VDSRPT_VIRUS_DOWNLOAD'] = 'VDS_REPORT_SPACE'
vds_tables['VDSRPT_VIRUS_NEW'] = 'VDS_REPORT_SPACE'
vds_tables['VDSRPT_VIRUS_VISIT'] = 'VDS_REPORT_SPACE'
#>>FOLD>>
def StringToTime(strtime):#<<FOLD<<
t_tuple = time.strptime(str(strtime),"%Y%m%d")
return int(time.mktime(t_tuple))
#>>FOLD>>
def gen_sqls(operate, date):#<<FOLD<<
tables = vds_tables
sqls = []
if 'add' == operate:
date1 = StringToTime(date)
for table in tables:
sqls.append('alter table %s add partition PART_%s values less than(%s) tablespace %s'%(table, date/10000, date1, tables[table]))
elif 'drop' == operate:
for table in tables:
sqls.append('alter table %s drop partition PART_%s'%(table, date/10000))
else:
print 'Wrong operation on table partitions, now quit.'
return
return sqls
#>>FOLD>>
def ora_init(ora):#<<FOLD<<
if ora in remote_ora.keys():
#print ' ------>', ora, '<------'
try:
db = cx_Oracle.connect(remote_ora[ora])
db.autocommit = True
#print 'Connect to oracle server successfully ...'
except:
if ora in remote_rac.keys():
#print 'Time out, now try to connect the rac ...'
try:
db = cx_Oracle.connect(remote_rac[ora])
db.autocommit = True
#print 'Connect to oracle server successfully ...'
except:
#print 'Error, Failed to connect oracle', ora
return
else:
#print 'Error, Failed to connect oracle', ora
return
else:
#print 'Error, Wrong oracle', ora
return
return db
#>>FOLD>>
def operate_partitions(ora):#<<FOLD<<
db = ora_init(ora)
if not db:
print '\n\033[31m %s Connect failed.\033[0m'%ora
return (ora, 'Connect failed')
cur = db.cursor()
date = (int(sys.argv[2])*100+1)*100+1
sqls = gen_sqls(sys.argv[1], date)
for sql in sqls:
try:
cur.execute(sql)
sys.stdout.write('.')
sys.stdout.flush()
except Exception, e:
matched = 0
ignore_errors = ['ORA-00942', 'ORA-02149']
for error in ignore_errors:
if error in str(e).upper():
matched = 1
sys.stdout.write('.')
sys.stdout.flush()
break
if not matched:
print "\n\033[31mRUN %s ON %s failed:%s\033[0m"%(sql, ora, e)
cur.close()
db.close()
print '\n\033[32m %s Complete.\033[0m'%ora
return (ora, 'Complete')
#>>FOLD>>
def main():#<<FOLD<<
if 0 == len(sys.argv[3:]):
oras = remote_ora.keys()
else:
oras = sys.argv[3:]
process_pool = Pool(8)
result = process_pool.map_async(operate_partitions, oras)
for res in result.get():
print res
#>>FOLD>>
if __name__ == '__main__':
if 2 < len(sys.argv) and sys.argv[2].isdigit():
main()
else:
print 'Usage: %s {add|drop} 2017 [pc1, pc2 ...]'%sys.argv[0]

View File

@@ -0,0 +1,65 @@
[day-180-2]
MONITOR_FAULT = COLLECTOR_STATE_SPACE
MT_COLLECTOR_STATE = COLLECTOR_STATE_SPACE
MT_COLLECTOR_DIR = COLLECTOR_STATE_SPACE
MTX_PERF_NET = MTX_PERF_NET
MTXRPT_EVENT_M = EVENT_COUNT_SPACE
MTXRPT_DEVICE_M = EVENT_COUNT_SPACE
MTXRPT_EVENTMOBILE_M = EVENT_COUNT_SPACE
RPT_MTXMOBILE_DEVICE_M = EVENT_COUNT_SPACE
RPT_MTXMOBILE_EVENT_M = EVENT_COUNT_SPACE
RPT_MTXMOBILE_IMEI_M = EVENT_COUNT_SPACE
RPT_MTXMOBILE_INOUT_H = EVENT_COUNT_SPACE
RPT_MTXMOBILE_PROVINCE_H = EVENT_COUNT_SPACE
RPT_MTXMOBILE_PROVINCE_M = EVENT_COUNT_SPACE
MTX_EVENT_REALTIME = EVENT_COUNT_SPACE
VDSRPT_DEVICE_M = EVENT_COUNT_SPACE
VDSRPT_EVENT_M = EVENT_COUNT_SPACE
[day-90-2]
DOMAIN_AA = DOMAIN_DATA_SPACE
DOMAIN_CLIENT_COUNT = DOMAIN_DATA_SPACE
DOMAIN_COUNT = DOMAIN_DATA_SPACE
DOMAIN_DNAME_LENGTH = DOMAIN_DATA_SPACE
DOMAIN_FLUX = DOMAIN_DATA_SPACE
DOMAIN_OPCODE_COUNT = DOMAIN_DATA_SPACE
DOMAIN_QUERY_PACKET_LENGTH = DOMAIN_DATA_SPACE
DOMAIN_IMSI_TOPN = DOMAIN_DATA_SPACE
DOMAIN_IMSI_TOP100 = DOMAIN_DATA_SPACE
DOMAIN_RA = DOMAIN_DATA_SPACE
DOMAIN_RCODE = DOMAIN_DATA_SPACE
DOMAIN_RD = DOMAIN_DATA_SPACE
DOMAIN_RESPONSE_PACKET_LENGTH = DOMAIN_DATA_SPACE
DOMAIN_TC = DOMAIN_DATA_SPACE
DOMAIN_TTL_LENGTH = DOMAIN_DATA_SPACE
DOMAIN_TYPE_COUNT = DOMAIN_DATA_SPACE
ERROR_PKT = DOMAIN_DATA_SPACE
DNSHJK = DOMAIN_DATA_SPACE
ABNORMAL_BASE_DNS = DOMAIN_DATA_SPACE
ABNORMAL_SPECIAL_QUEST = DOMAIN_DATA_SPACE
ABNORMAL_SERVER_QA = DOMAIN_DATA_SPACE
DNSPOD_IP_COUNT = DOMAIN_DATA_SPACE
DNSPOD_IP_FLUX = DOMAIN_DATA_SPACE
DNSPOD_LOCATION_COUNT = DOMAIN_DATA_SPACE
DNSPOD_LOCATION_DOMAIN = DOMAIN_DATA_SPACE
DNSPOD_PCI_COUNT = DOMAIN_DATA_SPACE
DNSPOD_TOP100_DOMAIN = DOMAIN_DATA_SPACE
DNSPOD_TOP_DOMAIN = DOMAIN_DATA_SPACE
[day-30-2]
TOPN_CLIENT_BYTE = DOMAIN_DATA_SPACE
TOPN_CLIENT_PACKET = DOMAIN_DATA_SPACE
TOPN_DOMAIN_NAME = DOMAIN_DATA_SPACE
TOPN_NXDOMAIN = DOMAIN_DATA_SPACE
TOPN_TLD = DOMAIN_DATA_SPACE
[month-1-2]
APPRPT_APPSTORE_D = APPRPT_SPACE
APPRPT_DOWNLOAD_D = APPRPT_SPACE
APPRPT_APP_USING_D = APPRPT_SPACE
APPRPT_DEVICE_BAND_D = APPRPT_SPACE
APPRPT_DEVICE_BAND_M = APPRPT_SPACE
APPRPT_DEVICE_TYPE_D = APPRPT_SPACE
APPRPT_DEVICE_TYPE_M = APPRPT_SPACE
APPRPT_SPECIAL_USING_D = APPRPT_SPACE

View File

@@ -0,0 +1,177 @@
#!/usr/bin/python2
import os
import re
import sys
import cx_Oracle
import ConfigParser
from time import sleep
HOST_CONF = 'host.conf'
TABLE_CONF = 'table_partition.cfg'
SQL = {}
SQL_HEAD = '''-- <<FOLD<<
DECLARE
A VARCHAR2(16);
B NUMBER(8);
NOW DATE;
FIRST_DAY DATE;
SQL_ VARCHAR2(256);
TIME_STAMP NUMBER(16);
TABLE_NAME VARCHAR2(32);
LOW_PART_BOUND EXCEPTION;
NO_EXIST_TABLE EXCEPTION;
NO_EXIST_PART EXCEPTION;
THE_ONLY_PART EXCEPTION;
PRAGMA EXCEPTION_INIT(LOW_PART_BOUND, -14074);
PRAGMA EXCEPTION_INIT(NO_EXIST_TABLE, -00942);
PRAGMA EXCEPTION_INIT(NO_EXIST_PART, -02149);
PRAGMA EXCEPTION_INIT(THE_ONLY_PART, -14083);
PROCEDURE IGNORE_ERROR_EXECUTE_SQL(SQL_ IN VARCHAR2) IS
BEGIN
EXECUTE IMMEDIATE SQL_;
EXCEPTION
WHEN LOW_PART_BOUND OR NO_EXIST_TABLE OR NO_EXIST_PART OR THE_ONLY_PART THEN
NULL;
END;
BEGIN
'''
# >>FOLD>>
SQL['day'] = '''-- <<FOLD<<
FOR A IN
(SELECT PARTITION_NAME
FROM USER_TAB_PARTITIONS
WHERE TABLE_NAME = '{table}' AND
PARTITION_NAME != 'PART_1' AND
PARTITION_NAME <= 'PART_' || TO_CHAR(SYSDATE-{n_keep}, 'yyyyMMdd'))
LOOP
SQL_ := 'ALTER TABLE {table} DROP PARTITION ' || A.PARTITION_NAME;
IGNORE_ERROR_EXECUTE_SQL(SQL_);
END LOOP;
B := 2;
NOW := SYSDATE;
WHILE B < (2 + {n_create})
LOOP
FIRST_DAY := TO_DATE(TO_CHAR(NOW + B, 'YYYYMMDD') || '-00:00:00', 'YYYYMMDD-HH24:MI:SS');
SELECT (FIRST_DAY - TO_DATE('19700101-08:00:00', 'YYYYMMDD-HH24:MI:SS')) * 24 * 60 * 60 INTO TIME_STAMP FROM DUAL;
SQL_ := 'ALTER TABLE {table} ADD PARTITION PART_' || TO_CHAR(FIRST_DAY, 'YYYYMMDD') || ' VALUES LESS THAN(' || TIME_STAMP || ') TABLESPACE {tablespace}';
IGNORE_ERROR_EXECUTE_SQL(SQL_);
B := B + 1;
END LOOP;
'''
# >>FOLD>>
SQL['month'] = '''-- <<FOLD<<
FOR A IN
(SELECT PARTITION_NAME
FROM USER_TAB_PARTITIONS
WHERE TABLE_NAME = '{table}' AND
PARTITION_NAME != 'PART_1' AND
PARTITION_NAME <= 'PART_' || TO_CHAR(ADD_MONTHS(SYSDATE, -{n_keep}), 'yyyyMM'))
LOOP
SQL_ := 'ALTER TABLE {table} DROP PARTITION ' || A.PARTITION_NAME;
IGNORE_ERROR_EXECUTE_SQL(SQL_);
END LOOP;
B := 2;
NOW := SYSDATE;
WHILE B < (2 + {n_create})
LOOP
FIRST_DAY := TO_DATE(TO_CHAR(ADD_MONTHS(NOW, B), 'YYYYMM') || '01-00:00:00', 'YYYYMMDD-HH24:MI:SS');
SELECT (FIRST_DAY - TO_DATE('19700101-08:00:00', 'YYYYMMDD-HH24:MI:SS')) * 24 * 60 * 60 INTO TIME_STAMP FROM DUAL;
SQL_ := 'ALTER TABLE {table} ADD PARTITION PART_' || TO_CHAR(FIRST_DAY, 'YYYYMM') || ' VALUES LESS THAN(' || TIME_STAMP || ') TABLESPACE {tablespace}';
IGNORE_ERROR_EXECUTE_SQL(SQL_);
B := B + 1;
END LOOP;
'''
# >>FOLD>>
SQL['year'] = '''-- <<FOLD<<
FOR A IN
(SELECT PARTITION_NAME
FROM USER_TAB_PARTITIONS
WHERE TABLE_NAME = '{table}' AND
PARTITION_NAME != 'PART_1' AND
PARTITION_NAME <= 'PART_' || (TO_CHAR(SYSDATE, 'yyyy') - {n_keep}))
LOOP
SQL_ := 'ALTER TABLE {table} DROP PARTITION ' || A.PARTITION_NAME;
IGNORE_ERROR_EXECUTE_SQL(SQL_);
END LOOP;
B := 2;
NOW := SYSDATE;
WHILE B < (2 + {n_create})
LOOP
FIRST_DAY := TO_DATE(TO_CHAR(ADD_MONTHS(NOW, B*12), 'YYYY') || '0101-00:00:00', 'YYYYMMDD-HH24:MI:SS');
SELECT (FIRST_DAY - TO_DATE('19700101-08:00:00', 'YYYYMMDD HH24:MI:SS')) * 24 * 60 * 60 INTO TIME_STAMP FROM DUAL;
SQL_ := 'ALTER TABLE {table} ADD PARTITION PART_' || TO_CHAR(FIRST_DAY, 'YYYY') || ' VALUES LESS THAN(' || TIME_STAMP || ') TABLESPACE {tablespace}';
IGNORE_ERROR_EXECUTE_SQL(SQL_);
B := B + 1;
END LOOP;
'''
# >>FOLD>>
SQL_FOOT = '''# <<FOLD<<
END;
'''
# >>FOLD>>
def generate_plsql(config):# <<FOLD<<
plsql = SQL_HEAD
for section in config.sections():
part_type, n_keep, n_create = section.split('-')
n_keep = int(n_keep)
n_create = int(n_create)
for table in map(lambda x: x.upper(), config.options(section)):
plsql += SQL[part_type].format(table=table, n_keep=n_keep, n_create=n_create, tablespace=config.get(section, table))
plsql += SQL_FOOT
return plsql
# >>FOLD>>
def operate_plsql(db, plsql):# <<FOLD<<
#print plsql
cur = db.cursor()
cur.execute(plsql)
#try:
# cur.execute(sql)
#except Exception, e:
# matched = 0
# ignore_errors = ['ORA-00942', 'ORA-02149']
# for error in ignore_errors:
# if error in str(e).upper():
# matched = 1
# sys.stdout.write('.')
# sys.stdout.flush()
# break
# if not matched:
# print "\n\033[31mRUN \"%s\" ON %s failed:%s\033[0m"%(sql, ora, str(e))
cur.close()
db.close()
return
# >>FOLD>>
def connect_db():# <<FOLD<<
config = ConfigParser.ConfigParser()
config.read(HOST_CONF)
user = config.get('db', 'user')
pswd = config.get('db', 'passwd')
orcl = config.get('db', 'db')
try:
db = cx_Oracle.connect(user, pswd, orcl)
db.autocommit = True
except Exception, e:
print 'Connect Oracle failed: %s'%str(e)
return False
return db
# >>FOLD>>
def main():# <<FOLD<<
global config
while 1:
config = ConfigParser.ConfigParser()
config.read(TABLE_CONF)
plsql = generate_plsql(config)
db = connect_db()
if db: operate_plsql(db, plsql)
sleep(14400)
# >>FOLD>>
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,18 @@
[_msg_log_]
level=info
file=/path/to/default.log
stderr=1
syslog=0
center=1
sysID=019
subsysID=13
file_size=300
file_counter=3
[table_partition]
file=/path/to/table_partition.log
stderr=1
level=info
file_size=100
file_counter=6

206
常用脚本/python/rising_log.py Executable file
View File

@@ -0,0 +1,206 @@
#!/usr/bin/env python
#-*- encoding: gbk -*-
#
import ConfigParser
import logging
import time, os
from logging.handlers import RotatingFileHandler
import platform
#global
log_msg_root = '_msg_log_'
#log_stderr_format = '$RESET$COLOR<%(levelname)-5s><%(name)s:%(process)d:%(threadName)s><%(filename)s:%(lineno)d>%(message)s'
log_stderr_format = '$COLOR$BOLD<%(levelname)-5s><%(name)s:%(process)d:%(threadName)s>$RESET$COLOR<%(filename)s:%(lineno)d>%(message)s'
#log_stderr_format = "$COLOR%(levelname)s $RESET %(asctime)s $BOLD$COLOR%(name)s$RESET %(message)s"
if platform.system() == 'Windows':
log_file_format = '<%(levelname)s><%(name)s:%(process)d>%(message)s'
else:
log_file_format = '%(asctime)s <%(levelname)-5s><%(name)s:%(process)d:%(threadName)s><%(filename)s:%(lineno)d>%(message)s'
# for colorful stderr output, copy from http://stackoverflow.com/questions/384076/how-can-i-make-the-python-logging-output-to-be-colored
# by rrt
BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
RESET_SEQ = "\033[0m"
#COLOR_SEQ = "\033[1;%dm" # color & bold
COLOR_SEQ = "\033[0;%dm" # color only
BOLD_SEQ = "\033[1m"
COLORS={
'WARNING':WHITE,
'INFO':YELLOW,
'DEBUG':GREEN,
'CRITICAL':YELLOW,
'ERROR':RED,
'RED':RED,
'GREEN':GREEN,
'YELLOW':YELLOW,
'BLUE':BLUE,
'MAGENTA':MAGENTA,
'CYAN':CYAN,
'WHITE':WHITE,}
"""
COLORS = {
'INFO': YELLOW,
'DEBUG': GREEN,
'ERROR': RED }
"""
class ColoredFormatter(logging.Formatter):
def __init__(self, *args, **kwargs):
logging.Formatter.__init__(self, *args, **kwargs)
def format(self, record):
levelname = record.levelname
color = COLOR_SEQ % (30 + COLORS[levelname])
message = logging.Formatter.format(self, record)
#print "1--" + repr(message)
message = message.replace("$RESET", RESET_SEQ)\
.replace("$BOLD", BOLD_SEQ)\
.replace("$COLOR", color)
#print "2--" + repr(message)
#for k,v in COLORS.items():
# message = message.replace("$" + k, COLOR_SEQ % (v+30))\
# .replace("$BG" + k, COLOR_SEQ % (v+40))\
# .replace("$BG-" + k, COLOR_SEQ % (v+40))
#print "3--" + repr(message)
return message + RESET_SEQ
#This class copy form liup's blog http://hi.baidu.com/dalier/blog/item/68419d64b98604faf63654e4.html
#by rrt on 2010-12-3
class DateRotatingFileHandler(RotatingFileHandler):
def __init__(self , filename , mode='a' , maxBytes=0, backupCount=0, encoding=None):
self.current = time.strftime("%Y%m%d" , time.localtime(time.time()))
self.path = os.path.dirname(filename)
self.filename = os.path.basename(filename)
newdir = os.path.join(self.path , self.current)
softlink = os.path.join(self.path, 'current')
if not os.access(newdir , os.X_OK):
os.mkdir(newdir)
try:
os.symlink(newdir ,softlink)
except:
os.system("rm -f " + softlink)
os.symlink(newdir ,softlink)
newfile = os.path.join(newdir , self.filename)
RotatingFileHandler.__init__(self, newfile , mode, maxBytes , backupCount , encoding)
def doRollover(self):
#print "doRollover , current=%s , filename=%s"%(self.current , self.baseFilename)
self.stream.close()
self.current = time.strftime("%Y%m%d" , time.localtime(time.time()))
#Modified by rrt on 2010-12-4 for a log path bug: /'20101204'
#repr() is not needed , time.strftime() return a string not a integer
#newdir = os.path.join(self.path , repr(self.current))
newdir = os.path.join(self.path , self.current)
if not os.access(newdir , os.X_OK):
os.mkdir(newdir)
self.baseFilename = os.path.join(newdir , self.filename)
if self.encoding:
self.stream = codecs.open(self.baseFilename, 'w', self.encoding)
else:
self.stream = open(self.baseFilename, 'w')
def shouldRollover(self, record):
if RotatingFileHandler.shouldRollover(self , record):
RotatingFileHandler.doRollover(self)
t = time.strftime("%Y%m%d" , time.localtime(time.time()))
if (cmp(self.current , t) < 0) :
return 1
return 0
def init_log(prefix, cfg_file, type,level):
global log_msg_root, log_file_format, log_stderr_format
cf = ConfigParser.ConfigParser()
cf.read(cfg_file)
config = {}
# we can use
# = cf.get(prefix, 'level').split('#')[0].strip()
# to enable "#" to comment behind a value string.
try:
config['level'] = cf.get(prefix, 'level')
except:
config['level'] = cf.get(log_msg_root, 'level')
try:
config['file'] = cf.get(prefix, 'file')
except:
path = cf.get(log_msg_root, 'file')
path = os.path.dirname(path)
config['file'] = os.path.join(path, prefix+'.log')
try:
config['stderr'] = cf.getint(prefix, 'stderr')
except:
config['stderr'] = cf.getint(log_msg_root, 'stderr')
try:
config['syslog'] = cf.getint(prefix, 'syslog')
except:
config['syslog'] = cf.getint(log_msg_root, 'syslog')
try:
config['file_size'] = cf.getint(prefix, 'file_size')
except:
config['file_size'] = cf.getint(log_msg_root, 'file_size')
try:
config['file_cnt'] = cf.getint(prefix, 'file_counter')
except:
config['file_cnt'] = cf.getint(log_msg_root, 'file_counter')
print config
#new a root logger
logger = logging.getLogger(prefix)
#set root logger level
if config['level'] == 'debug':
logger.setLevel(logging.DEBUG)
elif config['level'] == 'info':
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.ERROR)
#new a log file handle
log_handler = DateRotatingFileHandler(config['file'], mode="a", maxBytes = config['file_size']*1024*1024, backupCount=config['file_cnt'])
#set log file handle format
formatter = logging.Formatter(log_file_format, datefmt='%Y-%m-%d %H:%M:%S')# '%F %T') jython not work
log_handler.setFormatter(formatter)
#add to root logger
logger.addHandler(log_handler)
#new a stderr logger if need
if not config['stderr']:
return logger
else:
formatter = ColoredFormatter(log_stderr_format)
#formatter = logging.Formatter(log_stderr_format)
log_handler = logging.StreamHandler()
log_handler.setFormatter(formatter)
logger.addHandler(log_handler)
return logger
if __name__ == '__main__':
global logger
logger = init_log('test_log_lib', 'msg_log.ini', 0, 0)
logger.debug('debug rrrrrrrrrrrrrrrrrrrrrrrrrrrt')
logger.info('info rrrrrrrrrrrrrrrrrrrrrrrrrrrt')
logger.error('error rrrrrrrrrrrrrrrrrrrrrrrrrrrt')

View File

@@ -0,0 +1,5 @@
[conf]
interval=43200
day=7
used=1024
log_dir=/log/

View File

@@ -0,0 +1,131 @@
#!/usr/bin/python2
from socket import *
import time
import sys
import os
import re
import signal
import ConfigParser
import struct
import subprocess
import shutil
from subprocess import Popen
#sys.path.append('/lib/')
import rising_log
ROLLBACK_CONF = 'rollback_log.cfg'
### init log ###
global logger
logger = rising_log.init_log('rollback_log', 'msg_log.ini', 0, 0)
def _sig_INT_handle(signum, frame):
logger.error("SIGINT recieve, exit()")
sys.exit()
def dir_status(log_dir,used):
syscmd = 'du -s %s'%(log_dir)
output = Popen(syscmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
output_err = output.stderr.readlines()
output_out = output.stdout.readlines()
if len(output_err) > 0:
for line in output_err:
logger.error("%s",line)
logger.error("syscmd:%s failed",syscmd)
return 0
if len(output_out) > 0:
for line in output_out:
if log_dir in line:
dir_used = int(line.split("\t")[0])/1024
logger.info("dir used:%d, max:%d",dir_used,used)
if dir_used > used:
logger.info("disk used:%d to max:%d",dir_used,used)
return -1
return 0
def check_dir(data_dir):
if not os.path.exists(data_dir):
logger.error("no dir:%s,then create dir",data_dir)
os.makedirs(data_dir)
def init_rollback_config(config_file):
rollback_config = {}
try:
config = ConfigParser.ConfigParser()
config.read(config_file)
rollback_config['interval'] = config.get('conf', 'interval')
rollback_config['day'] = config.get('conf', 'day')
rollback_config['used'] = config.get('conf', 'used')
rollback_config['log_dir'] = config.get('conf', 'log_dir')
except Exception,msg:
logger.error(str(msg).strip())
sys.exit(1)
return rollback_config
def rollback_log(log_dir,day,used):
file_list = []
logdir_list =[]
file_list = os.listdir(log_dir)
if len(file_list) <= 0:
logger.debug("no log data in dir:%s",log_dir)
return 0
file_list.sort(reverse=True)
dir_num = 0
for i in file_list:
logdir_name = os.path.join(log_dir, i)
if os.path.isdir(logdir_name):
dir_num = dir_num + 1
if dir_num > day:
logger.info("dir_num:%d, maxnum:%d",dir_num,day)
logger.info("delete log dir:%s",logdir_name)
shutil.rmtree(logdir_name)
else:
logdir_list.append(logdir_name)
logdir_list.sort()
del logdir_list[-1]
for i in logdir_list:
if dir_status(log_dir,used) < 0 :
logger.info("delete log dir:%s",i)
shutil.rmtree(i)
else:
break
return 1
def main():
signal.signal(signal.SIGINT, _sig_INT_handle)
while(1):
rollback_config = init_rollback_config(ROLLBACK_CONF)
sleep_interval = int(rollback_config['interval'])
log_dir = rollback_config['log_dir']
day = int(rollback_config['day'])
used = int(rollback_config['used'])
rollback_log(log_dir,day,used)
time.sleep(sleep_interval)
if __name__ == '__main__':
main()

67
常用脚本/shell/check_net.sh Executable file
View File

@@ -0,0 +1,67 @@
#!/bin/bash
#=========================================
# Filename : check_net.sh
# Author : Colben
# Create : 2017-12-04 11:04
#=========================================
PROCS=()
function Quit {
[ -n "$1" ] && echo -e "\033[31;1mERROR: $1 !\033[0m"
exit 1
}
function StopPing {
for pid in ${!PROCS[@]}; do
kill $pid
done
Quit
}
function PingAddr {
local oldState=
local newState=
local alterTime=
while :; do
ping -w4 -c2 -q $1 > /dev/null 2>&1
newState=$?
[ "$oldState" = "$newState" ] || alterTime="$(date +'%H:%M')"
oldState=$newState
if [ '0' = "$newState" ]; then
echo "\033[32;1m connected\033[0m on $alterTime." > $1
sleep 8
else
echo "\033[31;1m lost\033[0m on $alterTime!" > $1
sleep 4
fi
done
}
# start
[ 0 -eq $# ] && Quit "No hosts found"
mkdir -p /tmp/check_net || Quit "create dir failed"
cd /tmp/check_net || Quit "change dir failed"
trap 2 3 15 "StopPing"
for addr in $@; do
PingAddr $addr &
PROCS[$!]="$addr"
done
while :; do
for pid in ${!PROCS[@]}; do
if [ ! -f /proc/$pid/stat ]; then
PingAddr ${PROCS[$pid]} &
PROC[$!]=PROC[$pid]
unset PROC[$pid]
fi
done
clear
for f in $(ls); do
echo -e "$f:$(cat $f)"
done
sleep 4
done

55
常用脚本/shell/install.sh Executable file
View File

@@ -0,0 +1,55 @@
#!/bin/bash
# usage
USAGE="Usage: $0"
# installation processes
PROCESSES='
CheckFun1
CheckFun2
InitFun1
InitFun2
'
# report error and quit
function Quit {
[ -n "$1" ] && echo "$1!" > /dev/stderr
echo -e "\033[31;1mInstallaion failed!\033[0m\n"
exit 1
}
function CheckFun1 {
echo ''
}
function CheckFun2 {
echo ''
}
function InitFun1 {
echo ''
}
function InitFun2 {
echo ''
}
# execute every function in PROCESSES
function Main {
for process in $PROCESSES; do
echo -e "\n\033[33;1m- $func ...\033[0m"
$process
done
echo -e "\033[32;1mInstall successfully!\033[0m\n"
}
# start here
# check root privilege
[ 0 -ne $UID ] && Quit "Can't run without ROOT"
# check arguments
[ 0 -ne $# ] && Quit "$USAGE"
# cd to script dir
cd $(dirname $0) || Quit
# install
Main

View File

@@ -0,0 +1,19 @@
#!/bin/bash
SERVICES=(
'rabbitmq-server'
'redis-server'
'mongod'
'mysql'
'tomcat7'
'nginx'
)
DAEMONS=(
'/a/b/c -f -g'
'd -e iiii'
'/j/k'
'llll'
'/mm/nn -a -b -c'
)

180
常用脚本/shell/rc.all Executable file
View File

@@ -0,0 +1,180 @@
#!/bin/bash
function Quit {
[ -n "$1" ] && echo -e "\n\033[31;1mERROR: $1\033[0m!\n"
exit 3
}
function Spreadout {
local i=
local service=
local daemon=
echo "System ${ROOT_DIR##*/} processes ..."
for service in ${SERVICES[@]}; do
echo "Service: \"$service\""
done
for i in ${!DAEMONS[@]}; do
daemon="${DAEMONS[i]}"
echo "Daemon: \"$daemon\""
done
exit 0
}
function ShowState {
local name=$1
local err=$2
local total=$3
local style=$4
if [ '0' = "$total" ]; then
echo -e "$name \033[32${style}mnot found\033[0m ..."
return 254
elif [ '0' = "$err" ]; then
echo -e "$name \033[32${style}mrunning\033[0m ..."
return 0
elif [ -z "$total" -o "$total" = "$err" ]; then
echo -e "$name \033[33${style}mstopped\033[0m!"
return 1
else
echo -e "$name \033[31${style}merror\033[0m!!"
return 255
fi
}
function HandleServices {
local service=
for service in ${SERVICES[@]}; do
echo "===> ${1}ing \"$service\" ..."
systemctl $1 $service
sleep 0.4
done
}
function HandleDaemons {
local i=
local daemon=
for i in ${!DAEMONS[@]}; do
daemon="${DAEMONS[i]}"
echo "===> ${1}ing \"$daemon\" ..."
$WATCH_PROC $1 $daemon
sleep 0.4
done
}
function CheckServices {
local err=0
local service=
for service in ${SERVICES[@]}; do
systemctl status $service > /dev/null 2>/dev/null
ShowState "Service: \"$service\"" $? || let "err=1+$err"
done
ShowState Services $err ${#SERVICES[@]} ';1;3'
return $?
}
function CheckDaemons {
local err=0
local daemon=
local daemon_name=
for i in ${!DAEMONS[@]}; do
daemon="${DAEMONS[i]}"
daemon_name="$(basename ${daemon%% *})"
pgrep -x ${daemon_name:0:15} > /dev/null 2>/dev/null
ShowState "Daemon: \"$daemon\"" $? || let "err=1+$err"
done
ShowState Daemons $err ${#DAEMONS[@]} ';1;3'
return $?
}
function CheckSystem {
local i=
local err=0
local ret=0
local output=
local items=2
for i in CheckServices CheckDaemons; do
output=$($i)
ret=$?
if [ 254 -eq $ret ]; then
let "items=${items}-1"
elif [ 0 -lt $ret ]; then
let "err=1+$err"
fi
if [ -z "$1" ]; then
echo -e "$output" | tail -1
else
echo -e "$output"
fi
done
ShowState "System \"${ROOT_DIR##*/}\"" $err $items ';1'
exit $?
}
# start here
type pgrep > /dev/null || exit 1
cd $(dirname $0)
. processes || Quit 'File "processes" not found'
[ -x rc.pwatch ] || Quit 'Executalbe process not found'
WATCH_PROC="$(pwd)/rc.pwatch"
cd .. && ROOT_DIR=$(pwd) || Quit 'Change dir failed'
case $* in
'')
CheckSystem
;;
status)
CheckSystem detail
;;
start)
for check in CheckServices CheckDaemons; do
$check > /dev/null 2>/dev/null
check_ret=$?
[ 254 -eq $check_ret ] && continue
if [ 0 -ne $check_ret ]; then
Handle${check#Check} start
output=$($check)
if [ 0 -ne $? ]; then
echo -e "$output"
Quit "${check#Check} start failed"
fi
fi
done
CheckSystem
;;
stop)
for check in CheckDaemons CheckServices; do
$check > /dev/null 2>/dev/null
check_ret=$?
[ 254 -eq $check_ret ] && continue
[ 1 -ne $check_ret ] && Handle${check#Check} stop
done
CheckSystem
;;
startservice)
CheckServices > /dev/null 2>/dev/null
check_ret=$?
if [ 254 -eq $check_ret ]; then
CheckSystem
elif [ 0 -ne $check_ret ]; then
HandleServices start
CheckSystem
fi
;;
stopdaemon)
CheckDaemons > /dev/null 2>/dev/null
check_ret=$?
if [ 254 -eq $check_ret ]; then
CheckSystem
elif [ 1 -ne $check_ret ]; then
HandleDaemons stop
CheckSystem
fi
;;
list)
Spreadout
;;
*)
echo "Usage: $0 [status|start[service]|stop[daemon]|list]"
exit 3
;;
esac

184
常用脚本/shell/rc.all_mul Executable file
View File

@@ -0,0 +1,184 @@
#!/bin/bash
function Quit {
[ -n "$1" ] && echo -e "\n\033[31;1mERROR: $1\033[0m!\n"
exit 3
}
function Spreadout {
local i=
local service=
local daemon=
echo "System ${ROOT_DIR##*/} processes ..."
for service in ${SERVICES[@]}; do
echo "Service: \"$service\""
done
for i in ${!DAEMONS[@]}; do
daemon="${DAEMONS[i]}"
echo "Daemon: \"$daemon\""
done
exit 0
}
function ShowState {
local name=$1
local err=$2
local total=$3
local style=$4
if [ '0' = "$total" ]; then
echo -e "$name \033[32${style}mnot found\033[0m ..."
return 254
elif [ '0' = "$err" ]; then
echo -e "$name \033[32${style}mrunning\033[0m ..."
return 0
elif [ -z "$total" -o "$total" = "$err" ]; then
echo -e "$name \033[33${style}mstopped\033[0m!"
return 1
else
echo -e "$name \033[31${style}merror\033[0m!!"
return 255
fi
}
function HandleServices {
local service=
for service in ${SERVICES[@]}; do
echo "===> ${1}ing \"$service\" ..."
systemctl $1 $service
sleep 0.4
done
}
function HandleDaemons {
local i=
local daemon=
for i in ${!DAEMONS[@]}; do
daemon="${DAEMONS[i]}"
echo "===> ${1}ing \"$daemon\" ..."
$WATCH_PROC $1 $daemon
sleep 0.4
done
}
function CheckServices {
local err=0
local service=
for service in ${SERVICES[@]}; do
systemctl status $service > /dev/null 2>/dev/null
ShowState "Service: \"$service\"" $? || let "err=1+$err"
done
ShowState Services $err ${#SERVICES[@]} ';1;3'
return $?
}
function CheckDaemons {
local err=0
local daemon=
local daemon_cmd=
local output=
local output_watch=
for i in ${!DAEMONS[@]}; do
daemon="${DAEMONS[i]}"
daemon_cmd="${daemon#$(dirname ${daemon%% *})/}"
output=$(pgrep -f "$daemon_cmd$")
output_watch=(pgrep -f "$(basename $WATCH_PROC) +start +[^ ]*\<$daemon_cmd$")
[ -n "$output" -a "$output" != "$output_watch" ]
ShowState "Daemon: \"$daemon\"" $? || let "err=1+$err"
done
ShowState Daemons $err ${#DAEMONS[@]} ';1;3'
return $?
}
function CheckSystem {
local i=
local err=0
local ret=0
local output=
local items=2
for i in CheckServices CheckDaemons; do
output=$($i)
ret=$?
if [ 254 -eq $ret ]; then
let "items=${items}-1"
elif [ 0 -lt $ret ]; then
let "err=1+$err"
fi
if [ -z "$1" ]; then
echo -e "$output" | tail -1
else
echo -e "$output"
fi
done
ShowState "System \"${ROOT_DIR##*/}\"" $err $items ';1'
exit $?
}
# start here
type pgrep > /dev/null || exit 1
cd $(dirname $0)
. processes || Quit 'File "processes" not found'
[ -x rc.pwatch ] || Quit 'Executalbe process not found'
WATCH_PROC="$(pwd)/rc.pwatch"
cd .. && ROOT_DIR=$(pwd) || Quit 'Change dir failed'
case $* in
'')
CheckSystem
;;
status)
CheckSystem detail
;;
start)
for check in CheckServices CheckDaemons; do
$check > /dev/null 2>/dev/null
check_ret=$?
[ 254 -eq $check_ret ] && continue
if [ 0 -ne $check_ret ]; then
Handle${check#Check} start
output=$($check)
if [ 0 -ne $? ]; then
echo -e "$output"
Quit "${check#Check} start failed"
fi
fi
done
CheckSystem
;;
stop)
for check in CheckDaemons CheckServices; do
$check > /dev/null 2>/dev/null
check_ret=$?
[ 254 -eq $check_ret ] && continue
[ 1 -ne $check_ret ] && Handle${check#Check} stop
done
CheckSystem
;;
startservice)
CheckServices > /dev/null 2>/dev/null
check_ret=$?
if [ 254 -eq $check_ret ]; then
CheckSystem
elif [ 0 -ne $check_ret ]; then
HandleServices start
CheckSystem
fi
;;
stopdaemon)
CheckDaemons > /dev/null 2>/dev/null
check_ret=$?
if [ 254 -eq $check_ret ]; then
CheckSystem
elif [ 1 -ne $check_ret ]; then
HandleDaemons stop
CheckSystem
fi
;;
list)
Spreadout
;;
*)
echo "Usage: $0 [status|start[service]|stop[daemon]|list]"
exit 3
;;
esac

50
常用脚本/shell/rc.pwatch Executable file
View File

@@ -0,0 +1,50 @@
#!/bin/bash
type pgrep > /dev/null || exit 1
[ 2 -gt $# ] && echo -e "\tUsage:$0 {start|stop} {cmd}\n" && exit 1
cd $(dirname $0)/../
WATCH_LOG=/var/log/watch_proc.log
WATCH_NAME=$(basename $0)
OPERATION=$1
shift
PROC_CMD="$*"
PROC_NAME=$(basename ${PROC_CMD%% *})
function StopProc {
local cpids=$(pgrep -P $1)
local cpid=
printf "%${2}s|-$1\n"
[ -f /proc/$1/stat ] && kill $1
local n=$(expr 2 + $2)
for cpid in $cpids; do
StopProc "$cpid" $n
done
}
case $OPERATION in
start)
pid=$(pgrep -f "$WATCH_NAME +start +[^ ]*\<$PROC_NAME\>")
pid=$(echo -e "$pid"|grep -v "^$$$")
[ -n "$pid" ] && echo "WATCH on \"$PROC_CMD\" is runing with PID:$pid" && exit 2
while :; do
$PROC_CMD > /dev/null 2>/dev/null &
pid=$!
while sleep 4; do [ -f /proc/$pid/stat ] || break; done
echo "$(date +'%F %T') \"$PROC_CMD\" quit exceptionally!" >> $WATCH_LOG
StopProc "$pid" 0 >> $WATCH_LOG
echo "$(date +'%F %T') \"$PROC_CMD\" restart." >> $WATCH_LOG
done > /dev/null 2>/dev/null & ;;
stop)
ppid=$(pgrep -f "$WATCH_NAME +start +[^ ]*\<$PROC_NAME\>")
[ -z "$ppid" ] && echo "No WATCH on \"$PROC_CMD\" !" && exit 1
pid=$(pgrep -lP $ppid|grep -v '^[0-9]\+ \+sleep$'|awk '{print $1}')
[ -z "$pid" ] && echo "No process: \"$PROC_CMD\"" && kill -9 $ppid && exit 0
kill -9 $ppid
echo "$(date +'%F %T') \"$PROC_CMD\" quit manually." >> $WATCH_LOG
StopProc "$pid" 0
exit 0 ;;
*)
echo -e "\tUsage:$0 start|stop cmd argv ...\n" ;;
esac

View File

@@ -0,0 +1,51 @@
#!/bin/bash
type pgrep > /dev/null || exit 1
[ 2 -gt $# ] && echo -e "\tUsage:$0 {start|stop} {cmd}\n" && exit 1
cd $(dirname $0)/../
WATCH_LOG=/var/log/watch_proc.log
WATCH_NAME=$(basename $0)
OPERATION=$1
shift
PROC_CMD="$*"
PROC_NAME=$(basename ${PROC_CMD%% *})
PROC_ARGS="${PROC_CMD#*$PROC_NAME}"
function StopProc {
local cpids=$(pgrep -P $1)
local cpid=
printf "%${2}s|-$1\n"
[ -f /proc/$1/stat ] && kill $1
local n=$(expr 2 + $2)
for cpid in $cpids; do
StopProc "$cpid" $n
done
}
case $OPERATION in
start)
pid=$(pgrep -f "$WATCH_NAME +start +[^ ]*\<$PROC_NAME\>$PROC_ARGS$")
pid=$(echo -e "$pid"|grep -v "^$$$")
[ -n "$pid" ] && echo "WATCH on \"$PROC_CMD\" is runing with PID:$pid" && exit 2
while :; do
$PROC_CMD > /dev/null 2>/dev/null &
pid=$!
while sleep 4; do [ -f /proc/$pid/stat ] || break; done
echo "$(date +'%F %T') \"$PROC_CMD\" quit exceptionally!" >> $WATCH_LOG
StopProc "$pid" 0 >> $WATCH_LOG
echo "$(date +'%F %T') \"$PROC_CMD\" restart." >> $WATCH_LOG
done > /dev/null 2>/dev/null & ;;
stop)
ppid=$(pgrep -f "$WATCH_NAME +start +[^ ]*\<$PROC_NAME\>$PROC_ARGS$")
[ -z "$ppid" ] && echo "No WATCH on \"$PROC_CMD\" !" && exit 1
pid=$(pgrep -lP $ppid|grep -v '^[0-9]\+ \+sleep$'|awk '{print $1}')
[ -z "$pid" ] && echo "No process: \"$PROC_CMD\"" && kill -9 $ppid && exit 0
kill -9 $ppid
echo "$(date +'%F %T') \"$PROC_CMD\" quit manually." >> $WATCH_LOG
StopProc "$pid" 0
exit 0 ;;
*)
echo -e "\tUsage:$0 start|stop cmd argv ...\n" ;;
esac

351
常用脚本/transfer Executable file
View File

@@ -0,0 +1,351 @@
#!/usr/bin/python2
# -*- coding:utf-8 -*-
#=========================================
# Filename : transfer.py
# Author : Colben
# Create : 2017-05-11 12:27
#=========================================
import os, sys, platform
import shutil
import re
import time
import zipfile
import BaseHTTPServer, HTMLParser
import requests, mimetypes, base64
from SocketServer import ThreadingMixIn
from threading import Thread
reload(sys)
sys.setdefaultencoding("utf-8")
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
KEY = base64.b64encode('lijc:lijc')
AUTH = ('lijc', 'lijc')
URLS = []
THREAD_RESULT = {}
def sizeof_fmt(num):# <<FOLD<<
for x in ['B', 'KB', 'MB', 'GB']:
if num < 1024.0:
return "%3.1f%s" % (num, x)
num /= 1024.0
return "%3.1f%s" % (num, 'TB')# >>FOLD>>
def modification_date(filename):
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(os.path.getmtime(filename)))
def handle_attachment(filename, user_agent):# <<FOLD<<
r, info = True, "File '%s' upload success!"
if filename.endswith('.zip'):
try:
zf = zipfile.ZipFile(filename, 'r')
zf.extractall('/home/attachment/')
except Exception, e:
r, info = False, str(e)
else:
try:
dest_file = os.path.join('/home/attachment/', os.path.basename(filename))
if os.path.exists(dest_file):
os.remove(dest_file)
os.rename(filename, dest_file)
filename = dest_file
except Exception, e:
r, info = False, str(e)
if r and user_agent != 'upload/sync':
for url in URLS:
sub_thread = Thread(target=sync_file, args=(url, filename,))
sub_thread.setDaemon(1)
sub_thread.start()
return (r, info%os.path.basename(filename))# >>FOLD>>
def sync_file(url, filename):# <<FOLD<<
THREAD_RESULT[url] = '<br>Syncing to %s ...'%url
headers = {'User-Agent': 'upload/sync'}
files = {'file': (os.path.basename(filename), open(filename, 'rb'), 'upload/file')}
resp = requests.post(url, auth=AUTH, headers=headers, files=files)
THREAD_RESULT[url] = "<br>Sync to '%s': %s"%(url, resp.headers['Sync-Result'])
return# >>FOLD>>
class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):# <<FOLD<<
parser=HTMLParser.HTMLParser()
#def do_GET(self):# <<FOLD<<
# """Serve a GET request."""
# f = self.send_head()
# if f:
# self.copyfile(f, self.wfile)
# f.close()# >>FOLD>>
def do_GET(self):# <<FOLD<<
''' Present frontpage with user authentication. '''
if self.headers.getheader('Authorization') == None:
self.do_AUTHHEAD()
self.wfile.write('no auth header received')
pass
elif self.headers.getheader('Authorization') == 'Basic '+KEY:
"""Serve a GET request."""
f = self.send_head()
if f:
self.copyfile(f, self.wfile)
f.close()
pass
else:
self.do_AUTHHEAD()
self.wfile.write(self.headers.getheader('Authorization'))
self.wfile.write('not authenticated')
pass# >>FOLD>>
def do_HEAD(self):# <<FOLD<<
"""Serve a HEAD request."""
f = self.send_head()
if f:
f.close()# >>FOLD>>
def do_AUTHHEAD(self):# <<FOLD<<
print "send header"
self.send_response(401)
self.send_header('WWW-Authenticate', 'Basic realm=\"Test\"')
self.send_header('Content-type', 'text/html')
self.end_headers()# >>FOLD>>
def do_POST(self):# <<FOLD<<
"""Serve a POST request."""
r, info = self.deal_post_data()
f = StringIO()
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write('<html>\n<title>Upload Result</title>\n')
f.write('<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\n')
f.write('<body>\n<h2>Upload Result</h2>\n')
f.write('\n<br><a href="../">返回上级</a>&nbsp&nbsp<a href="/">返回首页</a>')
f.write('<hr>\n')
if r:
f.write('<strong>Success:</strong>')
else:
f.write('<strong>Failed:</strong>')
f.write(info)
self.send_footer(f)
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
if self.headers['user-agent'] == 'upload/sync':
self.send_header("Sync-Result", info)
self.end_headers()
if f:
self.copyfile(f, self.wfile)
f.close()# >>FOLD>>
def deal_post_data(self):# <<FOLD<<
boundary = self.headers.plisttext.split("=")[1]
remainbytes = int(self.headers['content-length'])
line = self.rfile.readline()
remainbytes -= len(line)
if not boundary in line:
return (False, "Content NOT begin with boundary")
line = self.rfile.readline()
remainbytes -= len(line)
fn = re.findall(r'Content-Disposition.*name="file"; filename="(.*)"', line)
if not fn:
return (False, "Can't find out file name...")
path = self.translate_path(self.path)
osType = platform.system()
try:
if osType != "Linux":
fn = os.path.join(path, fn[0].decode('gbk').encode('utf-8'))
else:
fn = os.path.join(path, fn[0])
except Exception, e:
return (False, "文件名请不要用中文或者使用IE上传中文名的文件。")
fn_utf8 = self.parser.unescape(fn)
if os.path.exists(fn_utf8):
os.remove(fn_utf8)
line = self.rfile.readline()
remainbytes -= len(line)
line = self.rfile.readline()
remainbytes -= len(line)
try:
out = open(fn_utf8, 'wb')
except IOError:
return (False, "Can't create file to write, do you have permission to write?")
preline = self.rfile.readline()
remainbytes -= len(preline)
while remainbytes > 0:
line = self.rfile.readline()
remainbytes -= len(line)
if boundary in line:
preline = preline[0:-1]
if preline.endswith('\r'):
preline = preline[0:-1]
out.write(preline)
out.close()
if os.path.dirname(fn).endswith('/attachment'):
return handle_attachment(fn, self.headers['user-agent'])
else:
return (True, "File '%s' upload success!" % os.path.basename(fn))
else:
out.write(preline)
preline = line
return (False, "Unexpect Ends of data.")# >>FOLD>>
def send_head(self):# <<FOLD<<
path = self.translate_path(self.path)
f = None
if os.path.isdir(path):
if not self.path.endswith('/'):
# redirect browser - doing basically what apache does
self.send_response(301)
self.send_header("Location", self.path + "/")
self.end_headers()
return None
for index in "index.html", "index.htm":
index = os.path.join(path, index)
if os.path.exists(index):
path = index
break
else:
return self.list_directory(path)
ctype = self.guess_type(path)
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
except IOError:
self.send_error(404, "File not found")
return None
self.send_response(200)
self.send_header("Content-type", ctype)
fs = os.fstat(f.fileno())
self.send_header("Content-Length", str(fs[6]))
self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
self.end_headers()
return f# >>FOLD>>
def list_directory(self, path):# <<FOLD<<
try:
list = filter(lambda x: not x.startswith('.'), os.listdir(path))
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
f = StringIO()
displaypath = requests.utils.unquote(self.path)
#displaypath = requests.utils.cgi.escape(requests.utils.unquote(self.path))
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write('<html>\n<head><title>Directory %s</title>\n' % displaypath)
f.write('<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\n')
f.write('<style type="text/css">a:link,a:visited{text-decoration:none;}a:hover{text-decoration:underline;}</style>')
f.write('\n</head>\n<body>\n<h2>当前位置: %s</h2>\n' % displaypath)
if '/' != self.path:
f.write('\n<a href="../">返回上级</a>&nbsp&nbsp<a href="/">返回首页</a>')
f.write('<hr>\n<form ENCTYPE="multipart/form-data" method="post">')
f.write('<input name="file" type="file"/>')
f.write('<input type="submit" value="上传"/>')
f.write('\n</form>\n<hr>\n<table>\n')
if not list:
f.write('空目录\n')
else:
for name in list:
fullname = os.path.join(path, name)
colorName = displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
#colorName = '<span style="color: #0066CC;">' + name + '/</span>'
colorName = name + '/'
displayname = name
linkname = name + "/"
if os.path.isfile(fullname):
colorName = '<span style="color: #9F5000;">' + name + '</span>'
displayname = name
if os.path.islink(fullname):
colorName = '<span style="color: #00CACA;">' + name + '@</span>'
displayname = name
# Note: a link to a directory displays with @ and links with /
filename = os.getcwd() + '/' + displaypath + displayname
f.write(
'<tr><td width="66%%"><a href="%s">%s</a></td><td width="9%%">%s</td><td width="25%%">%s</td></tr>\n'
% (requests.utils.quote(linkname), colorName,
sizeof_fmt(os.path.getsize(filename)), modification_date(filename)))
f.write('</table>')
self.send_footer(f)
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f# >>FOLD>>
def send_footer(self, f):# <<FOLD<<
f.write('\n<hr><b>同步状态:</b>')
for sync_result in THREAD_RESULT.values():
f.write(sync_result)
f.write('\n<hr>\n<b>注意事项:</b>\n<ul>\n')
f.write('<li>上传同名文件时,会覆盖已存在的旧文件</li>\n')
f.write('</ul>')
f.write('\n<hr>\n<b>站外链接:</b>\n<ul>\n')
f.write('<li><a href="https://www.baidu.com/">百度</a></li>\n')
f.write('</ul><hr>\n</body>\n</html>\n')# >>FOLD>>
def translate_path(self, path):# <<FOLD<<
# abandon query parameters
path = path.split('?', 1)[0]
path = path.split('#', 1)[0]
path = os.path.normpath(requests.utils.unquote(path))
words = path.split('/')
words = filter(None, words)
path = os.getcwd()
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path# >>FOLD>>
def copyfile(self, source, outputfile):
shutil.copyfileobj(source, outputfile)
def guess_type(self, path):# <<FOLD<<
base, ext = os.path.splitext(path)
if ext in self.extensions_map:
return self.extensions_map[ext]
ext = ext.lower()
if ext in self.extensions_map:
return self.extensions_map[ext]
else:
return self.extensions_map['']# >>FOLD>>
if not mimetypes.inited:
mimetypes.init() # try to read system mime.types
extensions_map = mimetypes.types_map.copy()
extensions_map.update({
'': 'application/octet-stream', # Default
'.py': 'text/plain',
'.c': 'text/plain',
'.h': 'text/plain',
})# >>FOLD>>
class ThreadingServer(ThreadingMixIn, BaseHTTPServer.HTTPServer):
pass
if __name__ == '__main__':
# test()
# 单线程
# srvr = BaseHTTPServer.HTTPServer(('', 12306), SimpleHTTPRequestHandler)
# 多线程
srvr = ThreadingServer(('', 12306), SimpleHTTPRequestHandler)
print 'Listening ', srvr.server_address ,'...'
srvr.serve_forever()