package com.pachira.d;
import java.math.BigInteger;
/**
* SamHash简介
* 1.基本思想
* LSH: The basic idea is to hash the input items so that similar items are mapped to the same buckets with high probability;
* Hamming distance
*
* 2.具体步骤
* a). 对于给定的一段语句，进行分词，得到有效的特征向量;
* b). 为每一个特征向量设置一个权值;
* c). 对每一个特征向量计算hash值，为01组成的n-bit签名;
* d). 所有特征向量进行加权（1则为正，0则为负），然后累加;
* e). 对于n-bit签名的累加结果，如果>0置1，否则置0;
* f). 得到该语句的simhash值;
* g). 根据不同语句simhash的Hamming Distance就来判断相似程度;
*
* 3.算法优势
* 用于比较大文本，如500字以上效果挺好，距离小于3的基本都是相似，误判率也比较低;
* ps: 对于短句或者标题，内容过少，离散性较强，并不足以判断相似性;
* 对于短句或者标题，可以尝试使用ED等（个人理解）
* jecard、cos@
*
* 4.具体流程
* 中国人民解放军解放中国
* |
* |分词
* |
* --------- hash ------- weight ---------------
* | 中 国 | --> | 10010 | ---> | 3 -3 -3 3 -3 |
* | 人 民 | --> | 11001 | ---> | 2 2 -2 -2 2 |
* | 解放军 | --> | 11100 | ---> | 5 5 5 -5 -5 |
* | 解 放 | --> | 11011 | ---> | 2 2 -2 2 2 |
* | 中 国 | --> | 10001 | ---> | 1 -1 -1 -1 1 |
* --------- ------- ---------------
* sign |
* | 1 1 0 0 0 | <--- | 13 5 -3 -3 -3 |
*
* 5.具体实现：
*/
public class SimHash {
/**
* 计算初始的hash值
* @param source 要计算hash的特征字符串
* @param hashbits 指定特定位数的hash
* @return 特征的hash值
*/
public static BigInteger hash(String source, int hashbits) {
if (source == null || source.length() == 0) {
return new BigInteger("0");
} else {
char[] sourceArray = source.toCharArray();
BigInteger hashcode = BigInteger.valueOf(((long) sourceArray[0]) << 7);
BigInteger m = new BigInteger("1000003");
BigInteger mask = new BigInteger("2").pow(hashbits).subtract(new BigInteger("1"));
for (char item : sourceArray) {
BigInteger temp = BigInteger.valueOf((long) item);
hashcode = hashcode.multiply(m).xor(temp).and(mask);
}
hashcode = hashcode.xor(new BigInteger(String.valueOf(source.length())));
if (hashcode.equals(new BigInteger("-1"))) {
hashcode = new BigInteger("-2");
}
return hashcode;
}
}
/**
* 更新向量的维度值信息
* @param hash 某一个特征的hash值
* @param features 基础向量
* @param weight 某一个特征的权重
*/
public static void updatefeatures(BigInteger hash, int[] features, int weight) {
for (int i = 0; i < features.length; i++) {
BigInteger bitmask = new BigInteger("1").shiftLeft(i);
/*
*对某一特征的hash数列进行判断,如果是1000...1,那么数组的第一位和末尾一位加1,
*中间的62位减一,也就是说,逢1加1,逢0减1.一直到把所有的分词hash数列全部判断完毕.
*/
if (hash.and(bitmask).signum() != 0) {
//将该特征的信息更新到基础向量中，如果向量的第i维是大于0，+wegiht，否者-weight
features[i] += weight;
} else {
features[i] -= weight;
}
}
}
/**
* 最后对数组进行判断,大于0的记为1,小于等于0的记为0,得到一个 64bit 的数字指纹/签名.
*
* @param features
* @return
*/
public static String fingerprint(int[] features) {
BigInteger fingerprint = new BigInteger("0");
StringBuffer simHashBuffer = new StringBuffer();
for (int i = 0; i < features.length; i++) {
if (features[i] >= 0) {
fingerprint = fingerprint.add(new BigInteger("1").shiftLeft(i));
simHashBuffer.append("1");
} else {
simHashBuffer.append("0");
}
}
return simHashBuffer.toString();
}
/**
* hamming distance: 获得simHash串中同位置不同的数字的总数
* @param str1 simHash 1
* @param str2 simHash 2
* @return hamming distance
*/
public static int hammingDistance(String str1, String str2) {
int distance;
if (str1.length() != str2.length()) {
distance = -1;
} else {
distance = 0;
for (int i = 0; i < str1.length(); i++) {
if (str1.charAt(i) != str2.charAt(i)) {
distance++;
}
}
}
return distance;
}
public static void main(String[] args) {
String s = "My thesis work focuses on large scale copy detection of digital objects such as textual documents";
String t = "My thesis job focuses on large scale copy detection of digital objects such as textual documents";
int hashbits = 64;
//定义n-bit向量
int[] features = new int[hashbits];
int[] featuret = new int[hashbits];
//获得特征的hash值
BigInteger bs = hash(s, hashbits);
BigInteger bt = hash(t, hashbits);
//该处只是简单表示一个特征，如果对于一个文档一般会提取多个特征
updatefeatures(bs, features, 1);
updatefeatures(bt, featuret, 1);
//获得文档的指纹信息
String fingers = fingerprint(features);
String fingert = fingerprint(featuret);
//计算hamming distance
int dis = hammingDistance(fingers, fingert);
System.out.println(fingers + "\t" + s);
System.out.println(fingert + "\t" + t);
System.out.println(dis);
}
}
