redis去重字符串的示例：

基于Redis实现Bloomfilter去重，代码如下：

# encoding=utf-8

importredis

fromhashlibimportmd5

classSimpleHash(object):

def__init__(self,cap,seed):

self.cap=cap

self.seed=seed

defhash(self,value):

ret=0

foriinrange(len(value)):

ret+=self.seed*ret+ord(value[i])

return(self.cap-1)&ret

classBloomFilter(object):

def__init__(self,host='localhost',port=6379,db=0,blockNum=1,key='bloomfilter'):

"""

:param host: the host of Redis

:param port: the port of Redis

:param db: witch db in Redis

:param blockNum: one blockNum for about 90,000,000; if you have more strings for filtering, increase it.

:param key: the key's name in Redis

"""

self.server=redis.Redis(host=host,port=port,db=db)

self.bit_size=1<<31# Redis的String类型最大容量为512M，现使用256M

self.seeds=[5,7,11,13,31,37,61]

self.key=key

self.blockNum=blockNum

self.hashfunc=[]

forseedinself.seeds:

self.hashfunc.append(SimpleHash(self.bit_size,seed))

defisContains(self,str_input):

ifnotstr_input:

returnFalse

m5=md5()

m5.update(str_input)

str_input=m5.hexdigest()

ret=True

name=self.key+str(int(str_input[0:2],16)%self.blockNum)

forfinself.hashfunc:

loc=f.hash(str_input)

ret=ret&self.server.getbit(name,loc)

returnret

definsert(self,str_input):

m5=md5()

m5.update(str_input)

str_input=m5.hexdigest()

name=self.key+str(int(str_input[0:2],16)%self.blockNum)

forfinself.hashfunc:

loc=f.hash(str_input)

self.server.setbit(name,loc,1)

if__name__=='__main__':

""" 第一次运行时会显示 not exists!，之后再运行会显示 exists! """

bf=BloomFilter()

ifbf.isContains('http://www.baidu.com'):# 判断字符串是否存在

print'exists!'

else:

print'not exists!'

bf.insert('http://www.baidu.com')