java大作业 KShinglingAlgorithm

wiki上关于KShingling Algorithm（w-shingling）的说明：

http://en.wikipedia.org/wiki/W-shingling

摘要：

In natural language processing a w-shingling is a set of unique "shingles"—contiguous subsequences of tokens in a document—that can be used to gauge the similarity of two documents. The w denotes the number of tokens in each shingle in the set.

The document, "a rose is a rose is a rose" can be tokenized as follows:

(a,rose,is,a,rose,is,a,rose)

The set of all contiguous sequences of 4 tokens (N-grams, here: 4-grams) is

{ (a,rose,is,a), (rose,is,a,rose), (is,a,rose,is), (a,rose,is,a), (rose,is,a,rose) } = { (a,rose,is,a), (rose,is,a,rose), (is,a,rose,is) }

我理解的此算法，是把每段文本都像上述分解后，统计两段文本的合集b，再统计交集a，用a/b得到相似度。

写得有些复杂：

  1 package bigproject2;
  2 
  3 import javax.swing.JOptionPane;
  4 
  5 public class union {
  6     //求子集
  7     public String[] ziji(String str)
  8     {
  9         char[] ch=str.toCharArray();
 10         int c=0;
 11         for(int i=0;i<ch.length;i++)
 12         {
 13             if(ch[i]==' ')
 14                 c++;
 15         }
 16         //建立单词数组
 17         String[] strt=new String[c+1];
 18         for(int i=0;i<c+1;i++)
 19             strt[i]="";
 20         int h=0;
 21         for(int i=0;i<c+1;i++)
 22         {
 23             for(int j=h;j<ch.length;j++)
 24             {
 25                 if(ch[j]==' ')
 26                 {
 27                     h=j+1;
 28                     break;
 29                 }
 30                 else strt[i]+=ch[j];
 31             }
 32         }
 33         return strt;
 34     }
 35     //按k分，并去掉重复子集。
 36     public String[] cut(String[] str,int k) throws MyException{
 37         if(str.length<k)
 38                 throw new MyException("单词数少于"+k+"，无法进行计算！");
 39         String[] t=new String[str.length-k+1];
 40         for(int i=0;i<str.length-k+1;i++)
 41             t[i]="";
 42         int h=0,m=0;
 43         for(;h<str.length-k+1;h++)
 44         {
 45             for(int i=m;i<m+k;i++)
 46                 t[h]+=str[i];
 47             m++;
 48         }
 49         //去掉重复部分
 50         int merge=0;
 51         for(int i=0;i<t.length-1;i++)
 52         {
 53             if(t[i].equals("")) break;
 54             for(int j=i+1;j<t.length;j++)
 55             {
 56                 if(t[i].equals(t[j]))
 57                 {
 58                     merge++;
 59                     int y=j;
 60                     for(;y<t.length-1;y++)
 61                     {
 62                         t[y]=t[y+1];
 63                     }
 64                     t[y]="";
 65                 }
 66             }
 67         }
 68         String[] fin=new String[t.length-merge];
 69         for(int i=0;i<t.length-merge;i++)
 70             fin[i]=t[i];
 71         return fin;
 72     }
 73     public class MyException extends Exception{
 74         public MyException(String str){
 75             JOptionPane.showMessageDialog(null, str,"警告", JOptionPane.INFORMATION_MESSAGE);
 76         }
 77     }
 78     //求两字符串数组合集个数。
 79     public int heji(String[] a,String[] b){
 80         int count=a.length+b.length;
 81         for(int i=0;i<a.length;i++)
 82         {
 83             for(int j=0;j<b.length;j++)
 84             {
 85                 if(a[i].equals(b[j]))
 86                     count--;
 87             }
 88         }
 89         return count;
 90     }
 91     //求两字符串数组交集个数。
 92     public int jiaoji(String[] a,String[] b){
 93         int count=0;
 94         for(int i=0;i<a.length;i++)
 95         {
 96             for(int j=0;j<b.length;j++)
 97             {
 98                 if(a[i].equals(b[j]))
 99                     count++;
100             }
101         }
102         return count;
103     }
104 
105 }

 

 1 package bigproject2;
 2 
 3 
 4 public class KShinglingAlgorithm extends union{
 5     private String text1,text2;
 6     public String getText1()
 7     {
 8         return text1;
 9     }
10     public String getText2()
11     {
12         return text2;
13     }
14     public void setText1(String text1)
15     {
16         this.text1=text1;
17     }
18     public void setText2(String text2)
19     {
20         this.text2=text2;
21     }
22     
23     public float getSimilarity(int k)
24     {
25        union a=new union();
26        String[] t1=a.ziji(this.text1);
27        String[] t2=a.ziji(this.text2);
28        String[] t1t,t2t;
29        try{
30            t1t=a.cut(t1, k);
31            t2t=a.cut(t2, k);
32            
33        }catch(MyException e){
34                return -1;
35        }
36        int he=a.heji(t1t, t2t);
37        int jiao=a.jiaoji(t1t, t2t);
38        return (float)jiao/he;
39     }
40 
41 }

 

 

面板设计部分：

  1 package bigproject2;
  2 import java.awt.*;
  3 import java.awt.event.*;
  4 import java.io.BufferedReader;
  5 import java.io.File;
  6 import java.io.FileNotFoundException;
  7 import java.io.FileReader;
  8 import java.io.IOException;
  9 import java.io.InputStreamReader;
 10 
 11 import javax.swing.*;
 12 import javax.swing.event.*;
 13 import javax.swing.filechooser.FileNameExtensionFilter;
 14 
 15 public class Outlook extends JFrame{
 16     JFrame frm=new JFrame("相似度计算器");
 17     JPanel areabottom=new JPanel();
 18     JPanel areatop=new JPanel();
 19     JPanel areamiddle=new JPanel();
 20     static JTextArea tl=new JTextArea();
 21     static JTextArea tr=new JTextArea();
 22     JScrollPane left=new JScrollPane(tl,JScrollPane.VERTICAL_SCROLLBAR_ALWAYS,
 23             JScrollPane.HORIZONTAL_SCROLLBAR_AS_NEEDED);
 24     JScrollPane right=new JScrollPane(tr,JScrollPane.VERTICAL_SCROLLBAR_ALWAYS,
 25             JScrollPane.HORIZONTAL_SCROLLBAR_AS_NEEDED);
 26     JSplitPane sp=new JSplitPane(JSplitPane.HORIZONTAL_SPLIT,left,right);
 27     static JButton toBig=new JButton("全部大写");
 28     static JButton delbd=new JButton("去掉标点");
 29     static JButton count=new JButton("计算相似度");
 30     JLabel space=new JLabel("                                               ");
 31     JLabel t1=new JLabel("Text1");
 32     JLabel t2=new JLabel("Text2");
 33 
 34     JMenuBar mb=new JMenuBar();
 35     JMenu open=new JMenu("打开");
 36     JMenuItem opent1=new JMenuItem("打开到Text1");
 37     JMenuItem opent2=new JMenuItem("打开到Text2");
 38     
 39     private String str="";
 40     public Outlook()
 41     {
 42         judge();
 43         
 44         frm.setVisible(true);
 45         frm.setBounds(50, 50, 500, 400);
 46         frm.setLayout(new BorderLayout(5,5));
 47         
 48         frm.add("North",areatop);
 49         frm.add("Center",areamiddle);
 50         frm.add("South",areabottom);
 51         
 52         areatop.add(mb);
 53         mb.add(open);        
 54         open.add(opent1);
 55         open.add(opent2);
 56         open.setPreferredSize(new Dimension(40,18));
 57         mb.setBackground(frm.getBackground());
 58         areatop.setLayout(new FlowLayout(FlowLayout.LEFT));
 59         areamiddle.setLayout(new FlowLayout(FlowLayout.LEFT));
 60         
 61         areamiddle.add(t1);
 62         t1.setPreferredSize(new Dimension(frm.getWidth()/2-20,10));
 63         areamiddle.add(t2);
 64         t2.setPreferredSize(new Dimension(50,10));
 65         areamiddle.add(left);
 66         left.setPreferredSize(new Dimension(frm.getWidth()/2-20,frm.getHeight()/2));    
 67         areamiddle.add(right);
 68         right.setPreferredSize(new Dimension(frm.getWidth()/2-20,frm.getHeight()/2));
 69         tl.setLineWrap(true);
 70         tr.setLineWrap(true);
 71         
 72         areabottom.add(toBig);
 73         areabottom.add(delbd);
 74         areabottom.add(space);
 75         areabottom.add(count);
 76         
 77         opent1.addActionListener(new ActionListener(){
 78             public void actionPerformed(ActionEvent e) {
 79                 try {
 80                     openfile();
 81                     tl.setText(str);
 82                 } catch (IOException e1) {
 83                     e1.printStackTrace();
 84                 }
 85                 judge();
 86             }
 87         });
 88         opent2.addActionListener(new ActionListener(){
 89             public void actionPerformed(ActionEvent e) {
 90                 try {
 91                     openfile();
 92                     tr.setText(str);
 93                 } catch (IOException e1) {
 94                     e1.printStackTrace();
 95                 }
 96                 judge();
 97             }
 98         });
 99         toBig.addActionListener(new ActionListener(){
100             public void actionPerformed(ActionEvent e){
101                 tl.setText(tobig(tl.getText()));
102                 tr.setText(tobig(tr.getText()));
103             }
104         });
105         
106         delbd.addActionListener(new ActionListener(){
107             public void actionPerformed(ActionEvent e){
108                 tl.setText(del(tl.getText()));
109                 tr.setText(del(tr.getText()));
110                 judge();
111             }
112             
113         });
114         count.addActionListener(new ActionListener(){
115             public void actionPerformed(ActionEvent e){
116                 KShinglingAlgorithm a=new KShinglingAlgorithm();
117                 a.setText1(tl.getText());
118                 a.setText2(tr.getText());
119                 float b=a.getSimilarity(4);
120                 if(b!=-1)
121                     JOptionPane.showMessageDialog(null, Float.toString(b),"相似度", JOptionPane.INFORMATION_MESSAGE); 
122             }
123         });
124         tr.addKeyListener(new KeyAdapter(){
125             public void keyTyped(KeyEvent e){
126                 judge();
127             }
128         });
129         tl.addKeyListener(new KeyAdapter(){
130             public void keyTyped(KeyEvent e){
131                 judge();
132             }
133         });
134     }
135     public void judge(){
136         if(tl.getText().length()!=0||tr.getText().length()!=0) {
137             toBig.setEnabled(true);
138             delbd.setEnabled(true);
139             count.setEnabled(true);
140         }
141         else{
142             toBig.setEnabled(false);
143             delbd.setEnabled(false);
144             count.setEnabled(false);
145         }    
146     }
147     public void openfile() throws IOException{
148         str="";
149         JFileChooser choose=new JFileChooser();        
150         int result = choose.showOpenDialog(this);
151         File file = null; //注意初始化
152         //加过滤器
153         if (result == JFileChooser.APPROVE_OPTION) {
154             file = choose.getSelectedFile();
155             }
156         else{
157             return; //使点取消后不会抛出异常
158         }
159         FileReader fr=new FileReader(file);
160         BufferedReader br=new BufferedReader(fr);
161         char c[]=new char[512];
162         String strline="";
163         while(br.ready()){
164             strline=br.readLine();
165             str+=strline;
166         };
167         br.close();
168         fr.close();
169     }
170     public String tobig(String str){
171         String temp="";
172         for(int i=0;i<str.length();i++)
173         {
174             if(str.charAt(i)>='a'&&str.charAt(i)<='z')
175             {
176                 char t=str.charAt(i);
177                 t=(char)(str.charAt(i)-32);
178                 temp+=t;
179             }
180             else temp+=str.charAt(i);
181         }
182         return temp;
183     }
184     
185     public String del(String str){
186         String temp="";
187         for(int i=0;i<str.length();i++)
188         {
189             char t=str.charAt(i);
190             if(t>='!'&&t<='/'||t>=58&&t<=64||t>=91&&t<=96||t>=123&&t<=126);
191             else temp+=t;
192         }
193         return temp;
194     }
195     public static void main(String[] args){
196         new Outlook();
197         
198         
199     }
200 }
Outlook

 

posted @ 2014-12-25 12:57  verlen  阅读(352)  评论(0)  编辑  收藏  举报