需求:
監控linux一個變化的文件(比如/var/log/secure , 另一個文件在不斷向其中寫入新內容)，實時讀取其新的內容，顯示出來
實現:
測試1. 直接使用python 讀取文件
import os
fd = open(r'/var/log/secure')
for line in fd:
print line.strip()
fd.close()
發現文件讀取后，就會退出
測試方法2:
不斷循環讀取文件，發現每次都會讀取所有，不滿足需求
import os
while True:
fd = open(r'/var/log/secure')
for line in fd:
print line.strip()
fd.close()
測試方法3：
不斷循環讀取文件，同時記錄上次讀到的偏移位置，下次接着讀取
import os
import time
pos = 0
while True:
fd = open(r'/var/log/secure')
'''
for line in fd:
print line.strip()
fd.close()
'''
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
if line.strip():
print line.strip()
pos = pos + len(line)
if not line.strip():
break
fd.close()
可以不斷讀取到文件，最新記錄
Apr 30 10:26:47 localhost sshd[28290]: Accepted password for root from 192.168.109.133 port 54286 ssh2 Apr 30 10:26:47 localhost sshd[28290]: pam_unix(sshd:session): session opened for user root by (uid=0) Apr 30 10:26:50 localhost sshd[28290]: Received disconnect from 192.168.109.133: 11: disconnected by user Apr 30 10:26:50 localhost sshd[28290]: pam_unix(sshd:session): session closed for user root
但是需要一直不斷讀取文件，進入死循環階段。
測試4: 使用pynotify
pynotify 安裝 https://github.com/seb-m/pyinotify/wiki/Install
wget http://peak.telecommunity.com/dist/ez_setup.py sudo python ez_setup.py sudo easy_install pyinotify
在文件被修改時，調用文件讀取函數
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import os
import datetime
import pyinotify
import logging
pos = 0
def printlog():
global pos
try:
fd = open(r'/var/log/secure')
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
if line.strip():
print line.strip()
pos = pos + len(line)
if not line.strip():
break
fd.close()
except Exception,e:
print str(e)
class MyEventHandler(pyinotify.ProcessEvent):
#當文件被修改時調用函數
def process_IN_MODIFY(self, event):
try:
printlog()
except Exception,e:
print str(e)
def main():
#輸出前面的log
printlog()
# watch manager
wm = pyinotify.WatchManager()
wm.add_watch('/var/log/secure', pyinotify.ALL_EVENTS, rec=True)
eh = MyEventHandler()
# notifier
notifier = pyinotify.Notifier(wm, eh)
notifier.loop()
if __name__ == '__main__':
main()
當文件不斷增加，不斷增加那么文件尺寸會不斷變大，文件涉及到切割。
測試5: 監控文件變化，將自動終止監控，重新開始監控
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import os
import datetime
import pyinotify
import logging
import time
pos = 0
def printlog():
global pos
try:
fd = open(r'/var/log/secure')
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
if line.strip():
print line.strip()
pos = pos + len(line)
if not line.strip():
break
fd.close()
except Exception,e:
print str(e)
class MyEventHandler(pyinotify.ProcessEvent):
#當文件被修改時調用函數
def process_IN_MODIFY(self, event):
try:
printlog()
except Exception,e:
print str(e)
#文件自動被刪除
# 文件被刪除 或者切割
def process_IN_MOVE_SELF(self,event):
global notifier
try:
notifier.stop()
#wm.rm_watch(0)
except Exception,e:
print str(e)
notifier = None
def main():
global notifier,pos
path = '/var/log/secure'
while True:
if os.path.isfile(path):
pos = 0
#輸出前面的log
printlog()
# watch manager
wm = pyinotify.WatchManager()
eh = MyEventHandler()
notifier = pyinotify.Notifier(wm, eh)
wm.add_watch('/var/log/secure', pyinotify.ALL_EVENTS, rec=True)
# notifier
try:
notifier.loop()
except Exception,e:
print str(e)
print 'end one time'
else:
time.sleep(60)
if __name__ == '__main__':
main()
文件切割操作:測試過程: 創建logrotate配置文件
cat /etc/logrotate.d/syslog
/var/log/secure
{
daily
missingok
rotate 7
create
dateext
}
手動執行切割操作
[root@localhost logrotate.d]# logrotate -fv syslog reading config file syslog reading config info for /var/log/secure Handling 1 logs rotating pattern: /var/log/secure forced from command line (7 rotations) empty log files are rotated, old logs are removed considering log /var/log/secure log needs rotating rotating log /var/log/secure, log->rotateCount is 7 dateext suffix '-20170430' glob pattern '-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' glob finding old rotated logs failed fscreate context set to unconfined_u:object_r:var_log_t:s0 renaming /var/log/secure to /var/log/secure-20170430 creating new /var/log/secure mode = 0644 uid = 0 gid = 0 [root@localhost logrotate.d]# ls -lh /var/log/secure* -rw-r--r--. 1 root root 0 Apr 30 16:03 /var/log/secure #生成了一個新的文件 -rw-r--r--. 1 root root 5 Apr 30 16:00 /var/log/secure-20170430 #創建了新的備份文件
但是這個過程還是會導致notify監控事件終止
[2017-04-30 16:03:27,182 pyinotify ERROR] The pathname '/var/log/secure' of this watch <Watch wd=1 path=/var/log/secure mask=4095 proc_fun=None auto_add=False exclude_filter=<function <lambda> at 0x85457d4> dir=False > has probably changed and couldn't be updated, so it cannot be trusted anymore. To fix this error move directories/files only between watched parents directories, in this case e.g. put a watch on '/var/log'. 'NoneType' object has no attribute 'cleanup' end one time
由於我們寫的是循環，所以可以繼續等待1分鍾，再次自動啟動對文件的檢測。
遠程登錄ssh，
[root@localhost ~]# cat /var/log/secure Apr 30 16:08:24 localhost sshd[3930]: Accepted password for root from 192.168.109.1 port 59052 ssh2 Apr 30 16:08:24 localhost sshd[3930]: pam_unix(sshd:session): session opened for user root by (uid=0) [root@localhost ~]#
我們的程序還是可以獲得輸出信息
如果程序被異常終止，下次再啟動時，還是從頭開始讀取文件。就會導致讀取的文件時重復的...
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import os
import datetime
import pyinotify
import logging
import time
import json
#pos = 0
conffile = 'read.conf'
path = '/var/log/secure'
def printlog():
#global pos
try:
pos = 0
ctime = os.path.getctime(path)
if os.path.isfile(conffile):
fr = open(conffile)
data = json.loads(fr.read())
fr.close()
oldtime = data['ctime']
if oldtime == ctime:
print 'the same'
pos = data['pos']
else:
pass #新創建的文件重頭讀取
fd = open(path)
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
if line.strip():
print line.strip()
pos = pos + len(line)
if not line.strip():
break
fd.close()
fr = open(conffile,'w')
fr.write(json.dumps({'pos':pos,'ctime':ctime}))
fr.close()
except Exception,e:
print str(e)
class MyEventHandler(pyinotify.ProcessEvent):
#當文件被修改時調用函數
def process_IN_MODIFY(self, event):
try:
printlog()
except Exception,e:
print str(e)
#文件自動被刪除
# 文件被刪除 或者切割
def process_IN_MOVE_SELF(self,event):
global notifier
try:
notifier.stop()
#wm.rm_watch(0)
except Exception,e:
print str(e)
notifier = None
def main():
global notifier,pos
while True:
if os.path.isfile(path):
#輸出前面的log
printlog()
# watch manager
wm = pyinotify.WatchManager()
eh = MyEventHandler()
notifier = pyinotify.Notifier(wm, eh)
wm.add_watch('/var/log/secure', pyinotify.ALL_EVENTS, rec=True)
# notifier
try:
notifier.loop()
except Exception,e:
print str(e)
print 'end one time'
else:
time.sleep(60)
if __name__ == '__main__':
main()
但是發現創建時間每次都發生變化，沒有辦法作為唯一標識一個文件是否是新創建的。通過讀取第一行的方式 判斷是否為同一個文件
測試6: 通過第一行比較,判斷是否為新文件
# -*- coding: utf-8 -*-
#!/usr/bin/env python
import os
import datetime
import pyinotify
import logging
import time
import json
import hashlib
#pos = 0
conffile = 'read.conf'
path = '/var/log/secure'
def mymd5(line):
try:
m2 = hashlib.md5()
m2.update(line)
return m2.hexdigest()
except Exception,e:
print str(e)
return ''
def printlog():
#global pos
try:
pos = 0
newhash = ''
fd = open(path)
line = fd.readline()
#將文件指針移到行頭
fd.seek(0, os.SEEK_SET)
newhash = mymd5(line)
if os.path.isfile(conffile):
fr = open(conffile)
data = json.loads(fr.read())
fr.close()
myhash = data['myhash']
if newhash == myhash:
print 'the same'
pos = data['pos']
else:
pass #新創建的文件重頭讀取
#fd = open(path)
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
if line.strip():
print line.strip()
pos = pos + len(line)
if not line.strip():
break
fd.close()
fr = open(conffile,'w')
fr.write(json.dumps({'pos':pos,'myhash':newhash}))
fr.close()
except Exception,e:
print str(e)
class MyEventHandler(pyinotify.ProcessEvent):
#當文件被修改時調用函數
def process_IN_MODIFY(self, event):
try:
printlog()
except Exception,e:
print str(e)
#文件自動被刪除
# 文件被刪除 或者切割
def process_IN_MOVE_SELF(self,event):
global notifier
try:
notifier.stop()
#wm.rm_watch(0)
except Exception,e:
print str(e)
notifier = None
def main():
global notifier,pos
while True:
if os.path.isfile(path):
#輸出前面的log
printlog()
# watch manager
wm = pyinotify.WatchManager()
eh = MyEventHandler()
notifier = pyinotify.Notifier(wm, eh)
wm.add_watch('/var/log/secure', pyinotify.ALL_EVENTS, rec=True)
# notifier
try:
notifier.loop()
except Exception,e:
print str(e)
print 'end one time'
else:
time.sleep(60)
if __name__ == '__main__':
main()
import os
import time
pos = 0
while True:
fd = open(r'/var/log/secure')
'''
for line in fd:
print line.strip()
fd.close()
'''
if pos != 0:
fd.seek(pos,0)
while True:
line = fd.readline()
print line.strip()
pos = pos + len(line)
print pos
if not line.strip():
print 'end'
break
fd.close()