使用python opencv对目录下图片进行去重的方法

版本：

平台：ubuntu 14 / I5 / 4G内存

python版本：python2.7

opencv版本：2.13.4

依赖：

如果系统没有python，则需要进行安装

sudo apt-get install python

sudo apt-get install python-dev

sudo apt-get install python-pip

sudo pip install numpy mathplotlib

sudo apt-get install libcv-dev

sudo apt-get install python-opencv

使用感知哈希算法进行图片去重

原理：对每个文件进行遍历所有进行去重，因此图片越多速度越慢，但是可以节省手动操作

感知哈希原理：

1、需要比较的图片都缩放成8*8大小的灰度图

2、获得每个图片每个像素与平均值的比较，得到指纹

3、根据指纹计算汉明距离

5、如果得出的不同的元素小于5则为相同（相似？）的图片

#!/usr/bin/python
# -*- coding: UTF-8 -*-

import cv2
import numpy as np
import os,sys,types

def cmpandremove2(path):
dirs = os.listdir(path)
dirs.sort()
if len(dirs) <= 0:
 return
dict={}
for i in dirs:
 prepath = path + "/" + i
 preimg = cv2.imread(prepath)
 if type(preimg) is types.NoneType:
  continue
 preresize = cv2.resize(preimg, (8,8))
 pregray = cv2.cvtColor(preresize, cv2.COLOR_BGR2GRAY)
 premean = cv2.mean(pregray)[0]
 prearr = np.array(pregray.data)
 for j in range(0,len(prearr)):
  if prearr[j] >= premean:
   prearr[j] = 1
  else:
   prearr[j] = 0
 print "get", prepath
 dict[i] = prearr
dictkeys = dict.keys()
dictkeys.sort()
index = 0
while True:
 if index >= len(dictkeys):
  break
 curkey = dictkeys[index]
 dellist=[]
 print curkey
 index2 = index
 while True:
  if index2 >= len(dictkeys):
   break
  j = dictkeys[index2]
  if curkey == j:
   index2 = index2 + 1
   continue
  arr1 = dict[curkey]
  arr2 = dict[j]
  diff = 0
  for k in range(0,len(arr2)):
   if arr1[k] != arr2[k]:
    diff = diff + 1
  if diff <= 5:
   dellist.append(j)
  index2 = index2 + 1
 if len(dellist) > 0:
  for j in dellist:
   file = path + "/" + j
   print "remove", file
   os.remove(file)
   dict.pop(j)
  dictkeys = dict.keys()
  dictkeys.sort()
 index = index + 1

def cmpandremove(path):
index = 0
flag = 0
dirs = os.listdir(path)
dirs.sort()
if len(dirs) <= 0:
 return 0
while True:
 if index >= len(dirs):
  break
 prepath = path + dirs[index]
 print prepath
 index2 = 0
 preimg = cv2.imread(prepath)
 if type(preimg) is types.NoneType:
  index = index + 1
  continue
 preresize = cv2.resize(preimg,(8,8))
 pregray = cv2.cvtColor(preresize, cv2.COLOR_BGR2GRAY)
 premean = cv2.mean(pregray)[0]
 prearr = np.array(pregray.data)
 for i in range(0,len(prearr)):
  if prearr[i] >= premean:
   prearr[i] = 1
  else:
   prearr[i] = 0
 removepath = []
 while True:
  if index2 >= len(dirs):
   break
  if index2 != index:
   curpath = path + dirs[index2]
   #print curpath
   curimg = cv2.imread(curpath)
   if type(curimg) is types.NoneType:
    index2 = index2 + 1
    continue
   curresize = cv2.resize(curimg, (8,8))
   curgray = cv2.cvtColor(curresize, cv2.COLOR_BGR2GRAY)
   curmean = cv2.mean(curgray)[0]
   curarr = np.array(curgray.data)
   for i in range(0,len(curarr)):
    if curarr[i] >= curmean:
     curarr[i] = 1
    else:
     curarr[i] = 0
   diff = 0
   for i in range(0,len(curarr)):
    if curarr[i] != prearr[i] :
     diff = diff + 1
   if diff <= 5:
    print 'the same'
    removepath.append(curpath)
    flag = 1
  index2 = index2 + 1
 index = index + 1
 if len(removepath) > 0:
  for file in removepath:
   print "remove", file
   os.remove(file)
  dirs = os.listdir(path)
  dirs.sort()
  if len(dirs) <= 0:
   return 0
  #index = 0
return flag

def main(argv):
if len(argv) <= 1:
 print "command error"
 return -1
if os.path.exists(argv[1]) is False:
 return -1
path = argv[1]
'''
while True:
 if cmpandremove(path) == 0:
  break
'''
cmpandremove(path)
return 0

if __name__ == '__main__':
main(sys.argv)

为了节省操作，遍历所有目录，把想要去重的目录遍历一遍

#!/bin/bash
indir=$1
addcount=0
function intest()
{

for file in $1/*
do
 echo $file
 if test -d $file
 then
  ~/similar.py $file/
  intest $file
 fi
done
}

intest $indir

来源：https://blog.csdn.net/shan_xg/article/details/79448314