用事务处理大数据量表数据的导入问题

         两个不同的数据源,有着相同的数据表,要从一个数据源把数据导入到另一个数据源,该如何处理? 
        通常情况下,处理小数据量的情况,我们用DataSet就能成功实现,但是数据量特别大的话,就要考虑使用DataReader了,因为DataSet把数据存储在内存中,在数据量内存都无法存取,怎么取数据呢?
        使用DataReader把数据一条一条取出来,再结合多线程,能够很方便地实现数据的导入,而且速度也会很快.
        以下程序代码用事务处理机制完成数据的导入.
   public bool ImportToClient(string serverSql,string tableName)
    {
        SqlConnection connServer = new SqlConnection(FLServerConnString);
        connServer.Open();
        SqlCommand cmdServer = new SqlCommand();               
        cmdServer.Connection = connServer;      
        cmdServer.CommandTimeout = 0;
        cmdServer.CommandText = serverSql;
        SqlDataReader drServer = cmdServer.ExecuteReader();
       

        SqlConnection connClient = new SqlConnection(FLClientConnString);
        connClient.Open();
        SqlCommand cmdClient = new SqlCommand();
        SqlTransaction clientTran;
        clientTran = connClient.BeginTransaction();
        cmdClient.Connection = connClient;
        cmdClient.Transaction = clientTran;
        cmdClient.CommandTimeout = 0;
        cmdClient.CommandText = "truncate table "+tableName;
        cmdClient.ExecuteNonQuery();

        try
        {
            string insertClientSql = "";
            object[] objectServerColumns = new object[drServer.FieldCount];
            while(drServer.Read())
            {
                insertClientSql = "insert into "+tableName+" values(";
                int j = drServer.GetValues(objectServerColumns);               
                for (int i = 0; i < objectServerColumns.Length - 1; i++)
                {                   
                    if (objectServerColumns[i].GetType().ToString() == "System.DBNull")
                    {
                        insertClientSql += "null,";
                    }                    
                    else
                    {
                        insertClientSql += "'"+objectServerColumns[i].ToString()+"',";                       
                    }
                }
                if (objectServerColumns[objectServerColumns.Length - 1].GetType().ToString() == "System.DBNull")
                {
                    insertClientSql += "null)";
                }
                else
                {
                    insertClientSql += "'"+objectServerColumns[objectServerColumns.Length-1].ToString()+"')";
                }               
                cmdClient.CommandText = insertClientSql;
                cmdClient.ExecuteNonQuery();              
            }
            connServer.Close();//此處一定要關閉
            clientTran.Commit();
            drServer.Close();
            return true;
        }
        catch (Exception ee)
        {           
            clientTran.Rollback();
            return false;
        }
        finally
        {           
            connClient.Close();
        }
    }
   
}

posted @ 2007-05-30 19:21  穹苍之筝  阅读(782)  评论(0编辑  收藏  举报