需求:
监控linux一个变化的文件(比如/var/log/secure , 另一个文件在不断向其中写入新内容)，实时读取其新的内容，显示出来
实现:
测试1. 直接使用python 读取文件
import os
fd = open(r'/var/log/secure')
for line in fd:
print line.strip()
fd.close()
发现文件读取后，就会退出
测试方法2:
不断循环读取文件，发现每次都会读取所有，不满足需求
import os
while True:
fd = open(r'/var/log/secure')
for line in fd:
print line.strip()
fd.close()
测试方法3：
不断循环读取文件，同时记录上次读到的偏移位置，下次接着读取
import os
import time
pos = 0
while True:
fd = open(r'/var/log/secure')
'''
for line in fd:
print line.strip()
fd.close()
'''
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
if line.strip():
print line.strip()
pos = pos + len(line)
if not line.strip():
break
fd.close()
可以不断读取到文件，最新记录
Apr 30 10:26:47 localhost sshd[28290]: Accepted password for root from 192.168.109.133 port 54286 ssh2 Apr 30 10:26:47 localhost sshd[28290]: pam_unix(sshd:session): session opened for user root by (uid=0) Apr 30 10:26:50 localhost sshd[28290]: Received disconnect from 192.168.109.133: 11: disconnected by user Apr 30 10:26:50 localhost sshd[28290]: pam_unix(sshd:session): session closed for user root
但是需要一直不断读取文件，进入死循环阶段。
测试4: 使用pynotify
pynotify 安装 https://github.com/seb-m/pyinotify/wiki/Install
wget http://peak.telecommunity.com/dist/ez_setup.py sudo python ez_setup.py sudo easy_install pyinotify
在文件被修改时，调用文件读取函数
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import os
import datetime
import pyinotify
import logging
pos = 0
def printlog():
global pos
try:
fd = open(r'/var/log/secure')
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
if line.strip():
print line.strip()
pos = pos + len(line)
if not line.strip():
break
fd.close()
except Exception,e:
print str(e)
class MyEventHandler(pyinotify.ProcessEvent):
#当文件被修改时调用函数
def process_IN_MODIFY(self, event):
try:
printlog()
except Exception,e:
print str(e)
def main():
#输出前面的log
printlog()
# watch manager
wm = pyinotify.WatchManager()
wm.add_watch('/var/log/secure', pyinotify.ALL_EVENTS, rec=True)
eh = MyEventHandler()
# notifier
notifier = pyinotify.Notifier(wm, eh)
notifier.loop()
if __name__ == '__main__':
main()
当文件不断增加，不断增加那么文件尺寸会不断变大，文件涉及到切割。
测试5: 监控文件变化，将自动终止监控，重新开始监控
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import os
import datetime
import pyinotify
import logging
import time
pos = 0
def printlog():
global pos
try:
fd = open(r'/var/log/secure')
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
if line.strip():
print line.strip()
pos = pos + len(line)
if not line.strip():
break
fd.close()
except Exception,e:
print str(e)
class MyEventHandler(pyinotify.ProcessEvent):
#当文件被修改时调用函数
def process_IN_MODIFY(self, event):
try:
printlog()
except Exception,e:
print str(e)
#文件自动被删除
# 文件被删除 或者切割
def process_IN_MOVE_SELF(self,event):
global notifier
try:
notifier.stop()
#wm.rm_watch(0)
except Exception,e:
print str(e)
notifier = None
def main():
global notifier,pos
path = '/var/log/secure'
while True:
if os.path.isfile(path):
pos = 0
#输出前面的log
printlog()
# watch manager
wm = pyinotify.WatchManager()
eh = MyEventHandler()
notifier = pyinotify.Notifier(wm, eh)
wm.add_watch('/var/log/secure', pyinotify.ALL_EVENTS, rec=True)
# notifier
try:
notifier.loop()
except Exception,e:
print str(e)
print 'end one time'
else:
time.sleep(60)
if __name__ == '__main__':
main()
文件切割操作:测试过程: 创建logrotate配置文件
cat /etc/logrotate.d/syslog
/var/log/secure
{
daily
missingok
rotate 7
create
dateext
}
手动执行切割操作
[root@localhost logrotate.d]# logrotate -fv syslog reading config file syslog reading config info for /var/log/secure Handling 1 logs rotating pattern: /var/log/secure forced from command line (7 rotations) empty log files are rotated, old logs are removed considering log /var/log/secure log needs rotating rotating log /var/log/secure, log->rotateCount is 7 dateext suffix '-20170430' glob pattern '-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' glob finding old rotated logs failed fscreate context set to unconfined_u:object_r:var_log_t:s0 renaming /var/log/secure to /var/log/secure-20170430 creating new /var/log/secure mode = 0644 uid = 0 gid = 0 [root@localhost logrotate.d]# ls -lh /var/log/secure* -rw-r--r--. 1 root root 0 Apr 30 16:03 /var/log/secure #生成了一个新的文件 -rw-r--r--. 1 root root 5 Apr 30 16:00 /var/log/secure-20170430 #创建了新的备份文件
但是这个过程还是会导致notify监控事件终止
[2017-04-30 16:03:27,182 pyinotify ERROR] The pathname '/var/log/secure' of this watch <Watch wd=1 path=/var/log/secure mask=4095 proc_fun=None auto_add=False exclude_filter=<function <lambda> at 0x85457d4> dir=False > has probably changed and couldn't be updated, so it cannot be trusted anymore. To fix this error move directories/files only between watched parents directories, in this case e.g. put a watch on '/var/log'. 'NoneType' object has no attribute 'cleanup' end one time
由于我们写的是循环，所以可以继续等待1分钟，再次自动启动对文件的检测。
远程登录ssh，
[root@localhost ~]# cat /var/log/secure Apr 30 16:08:24 localhost sshd[3930]: Accepted password for root from 192.168.109.1 port 59052 ssh2 Apr 30 16:08:24 localhost sshd[3930]: pam_unix(sshd:session): session opened for user root by (uid=0) [root@localhost ~]#
我们的程序还是可以获得输出信息
如果程序被异常终止，下次再启动时，还是从头开始读取文件。就会导致读取的文件时重复的...
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import os
import datetime
import pyinotify
import logging
import time
import json
#pos = 0
conffile = 'read.conf'
path = '/var/log/secure'
def printlog():
#global pos
try:
pos = 0
ctime = os.path.getctime(path)
if os.path.isfile(conffile):
fr = open(conffile)
data = json.loads(fr.read())
fr.close()
oldtime = data['ctime']
if oldtime == ctime:
print 'the same'
pos = data['pos']
else:
pass #新创建的文件重头读取
fd = open(path)
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
if line.strip():
print line.strip()
pos = pos + len(line)
if not line.strip():
break
fd.close()
fr = open(conffile,'w')
fr.write(json.dumps({'pos':pos,'ctime':ctime}))
fr.close()
except Exception,e:
print str(e)
class MyEventHandler(pyinotify.ProcessEvent):
#当文件被修改时调用函数
def process_IN_MODIFY(self, event):
try:
printlog()
except Exception,e:
print str(e)
#文件自动被删除
# 文件被删除 或者切割
def process_IN_MOVE_SELF(self,event):
global notifier
try:
notifier.stop()
#wm.rm_watch(0)
except Exception,e:
print str(e)
notifier = None
def main():
global notifier,pos
while True:
if os.path.isfile(path):
#输出前面的log
printlog()
# watch manager
wm = pyinotify.WatchManager()
eh = MyEventHandler()
notifier = pyinotify.Notifier(wm, eh)
wm.add_watch('/var/log/secure', pyinotify.ALL_EVENTS, rec=True)
# notifier
try:
notifier.loop()
except Exception,e:
print str(e)
print 'end one time'
else:
time.sleep(60)
if __name__ == '__main__':
main()
但是发现创建时间每次都发生变化，没有办法作为唯一标识一个文件是否是新创建的。通过读取第一行的方式 判断是否为同一个文件
测试6: 通过第一行比较,判断是否为新文件
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import os
import datetime
import pyinotify
import logging
import time
import json
import hashlib
#pos = 0
conffile = 'read.conf'
path = '/var/log/secure'
def mymd5(line):
try:
m2 = hashlib.md5()
m2.update(line)
return m2.hexdigest()
except Exception,e:
print str(e)
return ''
def printlog():
#global pos
try:
pos = 0
newhash = ''
fd = open(path)
line = fd.readline()
#将文件指针移到行头
fd.seek(0, os.SEEK_SET)
newhash = mymd5(line)
if os.path.isfile(conffile):
fr = open(conffile)
data = json.loads(fr.read())
fr.close()
myhash = data['myhash']
if newhash == myhash:
print 'the same'
pos = data['pos']
else:
pass #新创建的文件重头读取
#fd = open(path)
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
if line.strip():
print line.strip()
pos = pos + len(line)
if not line.strip():
break
fd.close()
fr = open(conffile,'w')
fr.write(json.dumps({'pos':pos,'myhash':newhash}))
fr.close()
except Exception,e:
print str(e)
class MyEventHandler(pyinotify.ProcessEvent):
#当文件被修改时调用函数
def process_IN_MODIFY(self, event):
try:
printlog()
except Exception,e:
print str(e)
#文件自动被删除
# 文件被删除 或者切割
def process_IN_MOVE_SELF(self,event):
global notifier
try:
notifier.stop()
#wm.rm_watch(0)
except Exception,e:
print str(e)
notifier = None
def main():
global notifier,pos
while True:
if os.path.isfile(path):
#输出前面的log
printlog()
# watch manager
wm = pyinotify.WatchManager()
eh = MyEventHandler()
notifier = pyinotify.Notifier(wm, eh)
wm.add_watch('/var/log/secure', pyinotify.ALL_EVENTS, rec=True)
# notifier
try:
notifier.loop()
except Exception,e:
print str(e)
print 'end one time'
else:
time.sleep(60)
if __name__ == '__main__':
main()
import os
import time
pos = 0
while True:
fd = open(r'/var/log/secure')
'''
for line in fd:
print line.strip()
fd.close()
'''
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
print line.strip()
pos = pos + len(line)
print pos
if not line.strip():
print 'end'
break
fd.close()