Daily Report 2012/11/07 陈伯雄(step 8)

　　今天针对PIPE组对数据表的修改，对建立倒排索引做了系统的修改，由于表DOC、VEDIO、QUESTION（由QAPAIR修改为QUESTION）的属性并不完全相同，处理数据方法进行少量修改：

　　DOC表和VEDIO表具有的相同属性：title；

　　DOC独有属性：author，keywords；

　　QUESTION独有属性：question;

　　3个表最后的到的倒排索引结构式相同的，得到WORDLIST和对应ID；

　　以下功能整合到分词模块和更新倒排索引模块中

//分词
        static private List<string> getWords(int type, SqlDataReader reader)
        {
            List<string> listall = new List<string>();
            if (type == 0)
            {
                string title = reader[_Title].ToString();
                string keyword = reader[_KeyWords].ToString();
                string author = reader[_Author].ToString();
                //string description = reader[_Description].ToString();
                List<string> list1 = ChineseWordSegmentation.word_segmentation(title);
                List<string> list2 = keyword.Split(new char[2] { ' ', ':' }, StringSplitOptions.RemoveEmptyEntries).ToList();
                List<string> list3 = author.Split(new char[2] { ' ', '.' }, StringSplitOptions.RemoveEmptyEntries).ToList();
                //List<string> list4 = ChineseWordSegmentation.word_segmentation(description);
                //listall = list1.Union(list2).Union(list3).Union(list4).ToList();
                listall = list1.Union(list2).Union(list3).ToList();
            }
            else if (type == 1)
            {
                string title = reader[_Title].ToString();
                //string description = reader[_Description].ToString();
                //List<string> list1 = ChineseWordSegmentation.word_segmentation(title);
                //List<string> list2 = ChineseWordSegmentation.word_segmentation(description);
                //listall = list1.Union(list2).ToList();
                listall = ChineseWordSegmentation.word_segmentation(title);
            }
            else
            {
                string question = reader[_Question].ToString();
                listall = ChineseWordSegmentation.word_segmentation(question);
            }
            return listall;
        }

        //更新倒排索引
        static private void updateIndex(List<string> words, SqlConnection connection, string ID)
        {
            SqlCommand cmd = new SqlCommand();
            cmd.Connection = connection;
            foreach (string word in words)
            {
                //倒排表中加入新关键词                            
                cmd.CommandText = "SELECT value FROM index3 WHERE value = word";
                object val = cmd.ExecuteScalar();
                if (val == System.DBNull.Value)                               //if(cmd.ExecuteScalar() is DBNull)
                {
                    cmd.CommandText = "INSERT INTO index3 VALUES(word, ID)";
                    cmd.ExecuteNonQuery();
                }
                //倒排索引中存在的关键词，加上属性ID信息
                else
                {
                    string newValue = val.ToString() + "," + ID;
                    cmd.CommandText = "UPDATE index3 SET value = newValue WHERE key = word";
                    cmd.ExecuteNonQuery();
                }
            }
        }

主函数部分：

 List<Result> resultList = new List<Result>();
            string connectionString = GetConnectionString();                            //SQL Server链接字符串   
            using (SqlConnection connection = new SqlConnection(connectionString))      //SQL链接类的实例化
            {
                connection.Open();                                                      //打开数据库
                //建立倒排表
                string sqlstr = "CREATE table index_doc(key varchar(50) primary key, ID varchar(50))";                
                SqlCommand cmd = new SqlCommand();
                cmd.Connection = connection;
                cmd.CommandText = sqlstr;
                cmd.ExecuteNonQuery();
                sqlstr = "CREATE table index_vedio(key varchar(50) primary key, ID varchar(50))";
                cmd.CommandText = sqlstr;
                cmd.ExecuteNonQuery();
                sqlstr = "CREATE table index_question(key varchar(50) primary key, ID varchar(50))";
                cmd.CommandText = sqlstr;
                cmd.ExecuteNonQuery();

                for (int i = 0; i < 3;i++ )
                {
                    string table = "";
                    if (i == 0) table = _TableDoc;
                    else if (i == 1) table = _TableVideo;
                    else table = _TableQuestion;
                    //读取顺序表
                    sqlstr = "SELECT * FROM" + table;
                    cmd.CommandText = sqlstr;
                    SqlDataReader reader = cmd.ExecuteReader();                             
                    try
                    {
                        while (reader.Read())
                        {
                            string ID = reader[_ID].ToString();
                            //分词处理
                            List<string> words = getWords(i, reader);
                            //将keyword信息添加到倒排表
                            updateIndex(words, connection, ID);
                        }
                    }
                    finally
                    {
                        // Always call Close when done reading.
                        reader.Close();
                    }
                }