python增量计算

    在rsync同步数据后，因业务需要以及文件大小限制，需要对数据做增量分析，而每次都需要拿出文件的增量部分。
    linux有差异计算命令diff以及补丁工具patch，都不是很符合预期。这两种工具都是需要对两个文件进行对比，而若如此做的话，其一计算增量耗时，其二需要有一个原数据的副本文件。多余的副本文件会导致额外的存储开销，以及数据移动成本。
    又因同步过来的数据是多个application的数据，需要针对不同的app进行增量计算，提交给相应的计算任务。希望每次在rsync后直接做增量计算，这里就考虑到直接使用python脚本编写。
   

#! /usr/bin/env python
# -*- coding: utf-8 -*-

import os
import getopt
import sys
import fnmatch  

from datetime import datetime,timedelta,date

#判断是否为新一天的开始,数据文件是按日保存,凌晨定时任务需要对前日的数据进行增量计算
def is_newday_begin(app,daystr):
	file_day_file='/var/local/diff_file_storage/day_'+app
	if os.path.exists(file_day_file):
		try:
			day_file=open(file_day_file,'r')
			oldday=day_file.read()
			if oldday != daystr:
				return True
		finally:
			day_file.close()
	return False
	
def read_position(app,daystr,file):
	file_day_file='/var/local/diff_file_storage/day_'+app
	if os.path.exists(file_day_file):
		try:
			day_file=open(file_day_file,'r')
			oldday=day_file.read()
			if oldday != daystr:
				day_file.close()
				day_file=open(file_day_file,'w+')
				day_file.write(daystr)
				write_position(daystr,file,0)
		finally:
			day_file.close()
	else:
		try:
			day_file=open(file_day_file,'w+')
			day_file.write(daystr)
		finally:
			day_file.close() 
		
        
	position=None
	open_file=None
	position_file=file+'_'+daystr
	try:
		if os.path.exists(position_file):
			open_file=open(position_file,'r')
			position = long(open_file.read())
		else:
			position=0
	finally:
		if open_file is not None:
			open_file.close()
	print 'read from ',position_file
	return position

def write_position(daystr,file,position):
	open_file=None
	position_file=file+'_'+daystr
	try:
		open_file=open(position_file,'w+')
		open_file.write(str(position))
	finally:
		if open_file is not None:
			open_file.close()
	print 'write to ',position_file

def read_diff(app,day,outpath,times):
	filepath='/data/dc_files/'+app+day.strftime('/%Y/%m')
	if is_newday_begin(app,day.strftime('%Y-%m-%d')) and times <= 0:
		read_diff(app,day+timedelta(days=-1),outpath,times+1)
		print 'read_diff old day : ',day
	#get old position of file
	file_position=0
	file_position_dir='/var/local/diff_file_storage/position/'
	filenames=os.listdir(filepath)
				
	if not filepath.endswith('/'):
		filepath = filepath + '/'
	filepre = app+'_'+day.strftime('%Y-%m-%d')+'_' 
        target_file=open(outpath,'a')
	try:
		for line in filenames:
			if line.startswith(filepre):
				print 'read_diff: flush file ',line
				file_position_path=file_position_dir+app+'_'+line.replace(filepre,'')
				#read position from temp
				oldposition = read_position(app,day.strftime('%Y-%m-%d'),file_position_path)
				position=oldposition
				print 'read_diff: old position is ',position
			
				#read data,change position
				try:
					source_file=open(filepath+line,'r')
					#target_file=open(outpath,'w')
					source_file.seek(oldposition)
					for temp_line in source_file:
						target_file.write(temp_line)
						position = position+len(temp_line)
					
				finally:
					source_file.close()
					if position==oldposition:
						#os.remove(outpath)
						print outpath,' empty data'	
				#write position back to temp
				write_position(day.strftime('%Y-%m-%d'),file_position_path,position)
				print 'read_diff: new position is ',position
	finally:
		target_file.close()
    

    如上所示，是diff计算的demo代码。对每次增量计算的位移量进行保存，下次再进行增量计算时，就可以从位移量处直接读取。
    比之用diff,path少了副本相应开销，速度也可观。增加main后也可以直接在shell中调用。


    实现为module，则如下代码所示：
   

    #! /usr/bin/env python
# Filename:diffmodule.py
# -*- coding: utf-8 -*-

import os
import getopt
import sys
import fnmatch

#read diff from sfile to ofile by position_file cached
def read_diff(position_file,sfile,ofile):
        old_position=read_position(position_file)
        tmp_position=0
        source_file=None
        target_file=None
        try:
                source_file=open(sfile,'r')
                source_file.seek(old_position)
                target_file=open(ofile,'a')
                for temp_line in source_file:
                        target_file.write(temp_line)
                        tmp_position = tmp_position+len(temp_line)
        finally:
                if source_file is not None:
                        source_file.close()
                if target_file is not None:
                        target_file.close()
        if not tmp_position==0:
                write_position(position_file,tmp_position+old_position)
        return tmp_position

# read position from file
def read_position(position_file):
        position=None
        open_file=None
        try:
                if os.path.exists(position_file):
                        open_file=open(position_file,'r')
                        position = long(open_file.read())
                else:
                        position=0
        finally:
                if open_file is not None:
                        open_file.close()
        return position

#write position to file
def write_position(position_file,position):
        open_file=None
        result = False
        try:
                open_file=open(position_file,'w+')
                open_file.write(str(position))
        finally:
                if open_file is not None:
                        open_file.close()
                        result = True
        return result
    

   如上代码可供参考，在python 2.7.3下测试OK。