SimHash去重

今天，学习了网页去重的一些方法，算法当中SimHash算法，我尤为关注。下面我将详细介绍一下这个算法

首先，介绍一下SimHash:如果两个相似文档的语义指纹只相差几个位或更少，这样的语义指纹叫做SimHash

计算海明距离的两种方法：

1，按位与

2，两个长整形异或后，然后计算结果中1的个数

取得每个特征的64位hash值

public static int hamming(long l1, long l2) {
  long lxor = l1 ^ l2;
  return BitUtil.pop(lxor);
 }

SimHash计算过程：

初始化长度为64位的向量，该向量的每个维度都是0

循环处理：取每个特征的64位hash值，如果这个hash值得第i位是1，则将向量的第i个数加上特征权重，反之，如果为0，则减去相应的权重

完成所有特征的处理，向量中某些数为正，某些数为负，正数对应的位为1，负数为0，得到最终64位的SimHash

在写入文件过程中，可以把SimHash值使用差分编码进行压缩后保存，下面是一个简单实现代码

package com.test;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;

public class DetaCompress {

    public static byte[] longToBytes(long n) {
        byte[] buf = new byte[8];// 新建一个byte数组
        for (int i = buf.length - 1; i >= 0; i--) {
            buf[i] = (byte) (n & 0x00000000000000ff);// 取低8位的值
            n >>>= 8;// 右移8位
        }
        return buf;
    }

    // 把一个long型的数据进行压缩
    public static void writeVLong(long i, BufferedOutputStream dos)
            throws IOException {
        while ((i & ~0x7F) != 0) {
            dos.write((byte) ((i & 0x7f) | 0x80)); // 写入低位字节
            i >>>= 7; // 右移7位
        }

        dos.write((byte) i);
        // System.out.println((byte)i+"    写入低位字节");

    }

    // 把一个压缩后的long型的数据读取出来
    static long readVLong(DataInputStream dis) throws IOException {
        byte b = dis.readByte(); // 读入一个字节
        int i = b & 0x7F; // 取低7位的值
        // 每个高位的字节多乘个2的7次方，也就是128
        for (int shift = 7; (b & 0x80) != 0; shift += 7) {
            if (dis.available() != 0) {
                b = dis.readByte();
                i |= (b & 0x7F) << shift; // 当前字节表示的位乘2的shift次方
            }
        }
        return i;// 返回最终结果i
    }

    // 把long型数组simHashSet写入fileName指定的文件中去
    static int write(long[] simHashSet, String fileName) {
        int j = 0;
        try {
            BufferedOutputStream dos = new BufferedOutputStream(
                    new FileOutputStream(fileName));
            byte[] b = longToBytes(simHashSet[0]);// 数组的第一个数字一个转换成二进制
            dos.write(b);// 把它写到文件中
            for (int i = 1; i < simHashSet.length; i++) {
                long lo = simHashSet[i] - simHashSet[i - 1];// 用一个变量记录数组中后一个数减前一个数的差
                writeVLong(lo, dos);// 把这个差值写入文件
            }
            dos.close();
            j = simHashSet.length;
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return j;
    }

    // 从fileName指定的文件中把long型数组写出来
    static long[] read(int len, String fileName) {
        try {
            DataInputStream dis = new DataInputStream(new BufferedInputStream(
                    new FileInputStream(fileName)));
            long[] simHashSet = new long[len];
            simHashSet[0] = dis.readLong();// 从文件读取第一个long型数字放入数组
            for (int i = 1; i < len; i++) {
                simHashSet[i] = readVLong(dis);// 读取文件剩下的元素
                simHashSet[i] = simHashSet[i] + simHashSet[i - 1];  // 将元素都变成数组后一个数和前一个数字的和
            }
            dis.close();
            
            return simHashSet;
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }
}

排重的总体思想是：

先把要检索的f 位指纹集合缩小，将集合f位划分几块，

精确匹配高d位，集合容量缩小变为f'=|s|/2^d'

然后在小集合中检索f-d'位的海明距离

下面是实现的例子：

package com.lietu.simhash;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.StringTokenizer;
import java.util.Map.Entry;

/**
 * 64位分四块，最多找出有3位差别的simhash
 * 
 * @author lg
 * 
 */
// TODO: 保存排序后的中间状态
public class SimHashSet4 implements Iterable<SimHashData> {
    ArrayList<SimHashData> t1 = new ArrayList<SimHashData>();
    ArrayList<SimHashData> t2 = new ArrayList<SimHashData>();
    ArrayList<SimHashData> t3 = new ArrayList<SimHashData>();
    ArrayList<SimHashData> t4 = new ArrayList<SimHashData>();

    public ArrayList<SimHashData> getT1(){
        return t1;
    }
    static Comparator<SimHashData> comp = new Comparator<SimHashData>() {
        public int compare(SimHashData o1, SimHashData o2) {
            if (o1.q == o2.q)
                return 0;
            return (isLessThanUnsigned(o1.q, o2.q)) ? 1 : -1;
        }
    }; // 比较无符号64位
    static Comparator<Long> compHigh = new Comparator<Long>() {
        public int compare(Long o1, Long o2) {
            o1 |= 0xFFFFFFFFFFFFL;
            o2 |= 0xFFFFFFFFFFFFL;
            // System.out.println(Long.toBinaryString(o1));
            // System.out.println(Long.toBinaryString(o2));
            // System.out.println((o1 == o2));
            if (o1.equals(o2))
                return 0;
            return (isLessThanUnsigned(o1, o2)) ? 1 : -1;
        }
    }; // 比较无符号64位中的高16位

    public void load(String fileName) {
        String line = null;

        try {
            InputStream is = new FileInputStream(new File(fileName));

            BufferedReader br = new BufferedReader(new InputStreamReader(is));

            while ((line = br.readLine()) != null) {
                addSimHash(line.trim());
            }
            br.close();

        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static boolean isLessThanUnsigned(long n1, long n2) {
        return (n1 < n2) ^ ((n1 < 0) != (n2 < 0));
    }

    public void sort() {
        t2.clear();
        t3.clear();
        t4.clear();
        for (SimHashData simHash : t1) 
        {
            long t = Long.rotateLeft(simHash.q, 16);
            t2.add(new SimHashData(t, simHash.no));

            t = Long.rotateLeft(t, 16);
            t3.add(new SimHashData(t, simHash.no));

            t = Long.rotateLeft(t, 16);
            t4.add(new SimHashData(t, simHash.no));
        }

        Collections.sort(t1, comp);
        Collections.sort(t2, comp);
        Collections.sort(t3, comp);
        Collections.sort(t4, comp);
    }

    public boolean contains(SimHashData key) {
        int low = 0;
        int high = t1.size() - 1;

        while (low <= high) {
            int mid = (low + high) >>> 1;
            SimHashData midVal = t1.get(mid);
            int cmp = comp.compare(midVal, key);

            if (cmp < 0)
                low = mid + 1;
            else if (cmp > 0)
                high = mid - 1;
            else
                return true; // key found
        }
        return false; // key not found
    }

    /**
     * probe exact match
     * 
     * @param t
     * @return
     */
    public Span probe(ArrayList<SimHashData> t, long key) {
        // System.out.println("t:"+t.size());
        int low = 0;
        int high = t.size() - 1;

        while (low <= high) {
            int mid = (low + high) >>> 1;
            Long midVal = t.get(mid).q;
            int cmp = compHigh.compare(midVal, key);

            if (cmp < 0)
                low = mid + 1;
            else if (cmp > 0)
                high = mid - 1;
            else {
                // key found
                int matchStart = mid;
                int matchEnd = mid;
                while (matchStart > 0) {
                    midVal = t.get(matchStart - 1).q;
                    if (compHigh.compare(midVal, key) == 0) {
                        --matchStart;
                    } else {
                        break;
                    }
                }

                while (matchEnd < (t.size() - 1)) {
                    midVal = t.get(matchEnd + 1).q;
                    if (compHigh.compare(midVal, key) == 0) {
                        ++matchEnd;
                    } else {
                        break;
                    }
                }
                return new Span(matchStart, matchEnd);
            }
        }
        return null; // key not found
    }

    /**
     * get most 3 bit difference.
     * 
     * @param fingerPrint
     * @param k
     * @return
     */
    public HashSet<SimHashData> getSimSet(long fingerPrint, int k) {

        HashSet<SimHashData> retAll = new HashSet<SimHashData>();
        Span s1 = probe(t1, fingerPrint);
        if (s1 != null) {
            // System.out.println("s1:"+s1);
            ArrayList<SimHashData> ret1 = getSim(t1, s1, fingerPrint, k);
            retAll.addAll(ret1);
        }
        long q2 = Long.rotateLeft(fingerPrint, 16);
        Span s2 = probe(t2, q2);
        if (s2 != null) {
            // System.out.println("s2:"+s2);
            ArrayList<SimHashData> ret2 = getSim(t2, s2, q2, k);
            // rotateRight(ret2, 16);
            retAll.addAll(ret2);
        }

        long q3 = Long.rotateLeft(q2, 16);
        Span s3 = probe(t3, q3);
        if (s3 != null) {
            // System.out.println("s3:"+s3);
            ArrayList<SimHashData> ret3 = getSim(t3, s3, q3, k);
            // rotateRight(ret3, 32);
            retAll.addAll(ret3);
        }

        long q4 = Long.rotateLeft(q3, 16);
        Span s4 = probe(t4, q4);
        if (s4 != null) {
        //    System.out.println("s4:" + s4);
            ArrayList<SimHashData> ret4 = getSim(t4, s4, q4, k);
            // rotateRight(ret4, 48);
            retAll.addAll(ret4);
        }
        // System.out.println("o:"+Long.toBinaryString(fingerPrint));
        return retAll;
    }

    /**
     * 从Span找出部分相等的，取出最多差k位的
     * 
     * @param t
     * @param s
     * @param fingerPrint
     * @param k
     * @return
     */
    public ArrayList<SimHashData> getSim(ArrayList<SimHashData> t, Span s,
            long fingerPrint, int k) {
        ArrayList<SimHashData> result = new ArrayList<SimHashData>();

        for (int i = s.getStart(); i <= s.getEnd(); ++i) {
            SimHashData data = t.get(i);
            long q = data.q;
            if (BitUtil.diffIn(fingerPrint, q, k)) {
                result.add(data);
            }
        }

        return result;
    }

    public void addSimHash(String line) {
        StringTokenizer st = new StringTokenizer(line, ":");
        String key = st.nextToken();
        long t = BitUtil.decodeLong(key);
        long no = Long.parseLong(st.nextToken());
        // Long.parseLong(key,2);
        // System.out.println(t);
        t1.add(new SimHashData(t, no));
    }
    
    public void addSimHash(SimHashData key) {
        t1.add(key);
    }

    public void addInc(String key) {
        long t = BitUtil.decodeLong(key);
        // Long.parseLong(key,2);
        // System.out.println(t);
        SimHashData element = new SimHashData(t);
        int insertionPoint = findInsertionPoint(t1, element);
        t1.add(insertionPoint, element);

        long q2 = Long.rotateLeft(t, 16);
        element = new SimHashData(q2);
        insertionPoint = findInsertionPoint(t2, element);
        t2.add(insertionPoint, element);

        long q3 = Long.rotateLeft(q2, 16);
        element = new SimHashData(q3);
        insertionPoint = findInsertionPoint(t3, element);
        t3.add(insertionPoint, element);

        long q4 = Long.rotateLeft(q3, 16);
        element = new SimHashData(q4);
        insertionPoint = findInsertionPoint(t4, element);
        t4.add(insertionPoint, element);
    }

    /**
     * Find the insertion point for the argument in a sorted list.
     * 
     * @param element
     *            find this object's insertion point in the sorted list
     * @return the index of the insertion point
     */
    int findInsertionPoint(ArrayList<SimHashData> list, SimHashData element) {
        // Find the new element's insertion point.
        int insertionPoint = Collections.binarySearch(list, element, comp);
        if (insertionPoint < 0) {
            insertionPoint = -(insertionPoint + 1);
        }
        return insertionPoint;
    }

    public Iterator<SimHashData> iterator() {
        return t1.iterator();
    }

    public void save(String fileName) {
        BufferedWriter writer;
        try {
            writer = new BufferedWriter(new FileWriter(fileName));
            for (SimHashData simhash : t1) {
                //String str=BitUtil.encodeLong(simhash.q).substring(8);
                String str=BitUtil.encodeLong(simhash.q);
                writer.write(str);
//                writer.write(simhash.q+"");
                writer.write(":");
                writer.write(String.valueOf(simhash.no));
                writer.write("\r\n");
            }
            writer.flush();
            writer.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public void save(String fileName, String[] newStr) {
        BufferedWriter writer;
        try {
            OutputStream out = new FileOutputStream(fileName, true);
            OutputStreamWriter outWriter = new OutputStreamWriter(out);
            writer = new BufferedWriter(outWriter);
            for (int i = 0; i < newStr.length; i++) {
                if (newStr[i] != null) {
                    writer.append(newStr[i]);
                    writer.append("\r\n");
                    if (i % 10000 == 0)
                        System.out.println(i + ":" + newStr[i]);
                } else {
                    break;
                }
            }
            writer.flush();
            writer.close();
            System.out.println("结束!");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    // 将数据读成SimHashData对象型集合
    public ArrayList<SimHashData> readData(String path) {
        ArrayList<SimHashData> list = new ArrayList<SimHashData>();

        try {
            InputStream input = new FileInputStream(new File(path));
            BufferedReader br = new BufferedReader(new InputStreamReader(input));
            String line = "";
            while ((line = br.readLine()) != null) {
                StringTokenizer st = new StringTokenizer(line, ":");
                long key = BitUtil.decodeLong(st.nextToken());
                long no = Long.parseLong(st.nextToken());
                list.add(new SimHashData(key, no));
            }
            br.close();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return list;
    }



}

介绍一篇论文：Google Detecting NearDuplicates For Web Crawling 论文介绍了把SimHash用于爬虫抓取过程的网页去重。

最后，说一下分布式文档排重：利用分布式系统框架如hadoop等，使用MapReduce进行文档排重，提高了效率和节省了时间，这已经成为了常用的大数据量的排重方式

以上，是我对SimHash的一些总结，请大家指教！大家共勉