用事务处理大数据量表数据的导入问题
两个不同的数据源,有着相同的数据表,要从一个数据源把数据导入到另一个数据源,该如何处理?
通常情况下,处理小数据量的情况,我们用DataSet就能成功实现,但是数据量特别大的话,就要考虑使用DataReader了,因为DataSet把数据存储在内存中,在数据量内存都无法存取,怎么取数据呢?
使用DataReader把数据一条一条取出来,再结合多线程,能够很方便地实现数据的导入,而且速度也会很快.
以下程序代码用事务处理机制完成数据的导入.
public bool ImportToClient(string serverSql,string tableName)
{
SqlConnection connServer = new SqlConnection(FLServerConnString);
connServer.Open();
SqlCommand cmdServer = new SqlCommand();
cmdServer.Connection = connServer;
cmdServer.CommandTimeout = 0;
cmdServer.CommandText = serverSql;
SqlDataReader drServer = cmdServer.ExecuteReader();
SqlConnection connClient = new SqlConnection(FLClientConnString);
connClient.Open();
SqlCommand cmdClient = new SqlCommand();
SqlTransaction clientTran;
clientTran = connClient.BeginTransaction();
cmdClient.Connection = connClient;
cmdClient.Transaction = clientTran;
cmdClient.CommandTimeout = 0;
cmdClient.CommandText = "truncate table "+tableName;
cmdClient.ExecuteNonQuery();
try
{
string insertClientSql = "";
object[] objectServerColumns = new object[drServer.FieldCount];
while(drServer.Read())
{
insertClientSql = "insert into "+tableName+" values(";
int j = drServer.GetValues(objectServerColumns);
for (int i = 0; i < objectServerColumns.Length - 1; i++)
{
if (objectServerColumns[i].GetType().ToString() == "System.DBNull")
{
insertClientSql += "null,";
}
else
{
insertClientSql += "'"+objectServerColumns[i].ToString()+"',";
}
}
if (objectServerColumns[objectServerColumns.Length - 1].GetType().ToString() == "System.DBNull")
{
insertClientSql += "null)";
}
else
{
insertClientSql += "'"+objectServerColumns[objectServerColumns.Length-1].ToString()+"')";
}
cmdClient.CommandText = insertClientSql;
cmdClient.ExecuteNonQuery();
}
connServer.Close();//此處一定要關閉
clientTran.Commit();
drServer.Close();
return true;
}
catch (Exception ee)
{
clientTran.Rollback();
return false;
}
finally
{
connClient.Close();
}
}
}