winnet获取web数据的错误
我要一次访问好几百个url,我用单线程试过几次没有出错,但是用多线程,即使两三个线程同时跑(每个线程的URL都不同)也会出错。(session没有重复),出错的地方很奇怪,一般是在
pHttpFile->Close();这行代码。提示访问冲突,看样子内存溢出了还是怎么。
pHttpFile = pConnection->OpenRequest("GET",szObject, NULL, 0, NULL, NULL,
INTERNET_FLAG_KEEP_CONNECTION); //这行也会出错有时。
很痛苦啊,错误是随机的,和某个URL应该没有关系。目前为止我解决了这个问题,是用的另一种方式读取web数据,即使100线程都没有错误了。我先把错误的代码贴出来。
bool GetHttptFile(char *url,char *buf,int *l)
{
ZeroMemory(buf,BUF_LEN);
DWORD dwServiceType = AFX_INET_SERVICE_HTTP;
CString szServer, szObject;
INTERNET_PORT nPort;
if (false==AfxParseURL(url,dwServiceType,szServer,szObject, nPort) )//url需要加http://
{
return false;
}
char session_[MAX_PATH];
ZeroMemory(session_,MAX_PATH);
session++;
itoa(session,session_,10);
CInternetSession mysession(session_);
CHttpConnection* pConnection=NULL;
CHttpFile* pHttpFile=NULL;
DWORD tlen=0;
try
{
pConnection = mysession.GetHttpConnection(szServer,
INTERNET_FLAG_KEEP_CONNECTION,
INTERNET_INVALID_PORT_NUMBER,
NULL, NULL);
pHttpFile = pConnection->OpenRequest("GET",szObject,
NULL, 0, NULL, NULL,
INTERNET_FLAG_KEEP_CONNECTION);
pHttpFile->SendRequest(NULL);
pHttpFile->QueryInfo(HTTP_QUERY_CONTENT_LENGTH,tlen,0);
if (tlen>BUF_LEN+2)//页面数据过大
{
pConnection->Close();
pHttpFile->Close();
mysession.Close();
delete pConnection;
delete pHttpFile;
return false;
}
if (!tlen)//如果tlen为零,可能是函数没有读到头部,要人工定位数据大小
{
char temp[BUFSIZ*2*100];
ZeroMemory(temp,BUFSIZ*2*100);
int pos=0;
while(pHttpFile->Read(temp,BUFSIZ*2*100))
{
if (pos+strlen(temp)>BUF_LEN+2)//页面数据过大
{
mysession.Close();
pConnection->Close();
pHttpFile->Close();
delete pConnection;
delete pHttpFile;
return false;
}
memcpy(buf+pos,temp,strlen(temp));
pos+=strlen(temp);
ZeroMemory(temp,BUFSIZ*2*100);
}
*l=pos;
if (!pos)
{
mysession.Close();
pConnection->Close();//关闭会出错
pHttpFile->Close();
delete pConnection;
delete pHttpFile;
return false;
}
}
else
{
*l = tlen;
UINT uBytesRead = pHttpFile->Read(buf,tlen);
}
}
catch(CInternetException *p)//
{
p->Delete();//
if (pConnection)
{
pConnection->Close();
delete pConnection;
}
if (pHttpFile)
{
pHttpFile->Close();
delete pHttpFile;
}
mysession.Close();
return false;
}
catch (...)//比如超时异常,上面session的超时设置无效
{
if (pConnection)
{
pConnection->Close();
delete pConnection;
}
if (pHttpFile)
{
pHttpFile->Close();
delete pHttpFile;
}
mysession.Close();
return false;
}
pConnection->Close();
pHttpFile->Close();
mysession.Close();
delete pConnection;
delete pHttpFile;
return true;
}
/////////////////我现在用的是自己写的这个函数则没有出错
bool GetHttptFile(char *url,char *buf,int *l)
{
ZeroMemory(buf,BUF_LEN);
char session_[MAX_PATH];
ZeroMemory(session_,MAX_PATH);
session++;
itoa(session,session_,10);
CInternetSession m_Session(session_);
CHttpFile* m_HttpFile = NULL;
m_Session.SetOption(INTERNET_OPTION_CONNECT_TIMEOUT,1000);
m_Session.SetOption(INTERNET_OPTION_DATA_RECEIVE_TIMEOUT, 1000);
m_Session.SetOption(INTERNET_OPTION_DATA_SEND_TIMEOUT, 1000);
m_Session.SetOption(INTERNET_OPTION_CONNECT_BACKOFF,1000);
m_Session.SetOption(INTERNET_OPTION_CONNECT_RETRIES,2);
try
{
m_HttpFile = (CHttpFile*)m_Session.OpenURL(url);
DWORD dwStatusCode;
m_HttpFile-> QueryInfoStatusCode(dwStatusCode);
char temp[BUFSIZ*2*100];
ZeroMemory(temp,BUFSIZ*2*100);
int pos=0;
while(m_HttpFile-> Read(temp,BUFSIZ*2*100))
{
// SetFilePointer(hFile,0,0,FILE_END);
// WriteFile(hFile, myData,myData.GetLength(), &dwWritten, NULL);
if (pos+strlen(temp)>BUF_LEN)//页面数据过大
{
m_HttpFile-> Close();
m_Session.Close();
delete m_HttpFile;
return false;
}
memcpy(buf+pos,temp,strlen(temp));
pos+=strlen(temp);
ZeroMemory(temp,BUFSIZ*2*100);
}
*l=pos;
m_HttpFile->Close();
m_Session.Close();
delete m_HttpFile;
}
catch(CInternetException *p)//必须捕获,不然会内存泄露
{
p->Delete();//不要用delete p;而是执行封装好的方法。
if (m_HttpFile)
{
m_HttpFile->Close();
delete m_HttpFile;
}
m_Session.Close();
return false;
}
catch (...)//比如超时异常,上面session的超时设置无效
{
if (m_HttpFile)
{
m_HttpFile->Close();
delete m_HttpFile;
}
m_Session.Close();
return false;
}
return true;
}
pHttpFile->Close();这行代码。提示访问冲突,看样子内存溢出了还是怎么。
pHttpFile = pConnection->OpenRequest("GET",szObject, NULL, 0, NULL, NULL,
INTERNET_FLAG_KEEP_CONNECTION); //这行也会出错有时。
很痛苦啊,错误是随机的,和某个URL应该没有关系。目前为止我解决了这个问题,是用的另一种方式读取web数据,即使100线程都没有错误了。我先把错误的代码贴出来。
bool GetHttptFile(char *url,char *buf,int *l)
{
ZeroMemory(buf,BUF_LEN);
DWORD dwServiceType = AFX_INET_SERVICE_HTTP;
CString szServer, szObject;
INTERNET_PORT nPort;
if (false==AfxParseURL(url,dwServiceType,szServer,szObject, nPort) )//url需要加http://
{
return false;
}
char session_[MAX_PATH];
ZeroMemory(session_,MAX_PATH);
session++;
itoa(session,session_,10);
CInternetSession mysession(session_);
CHttpConnection* pConnection=NULL;
CHttpFile* pHttpFile=NULL;
DWORD tlen=0;
try
{
pConnection = mysession.GetHttpConnection(szServer,
INTERNET_FLAG_KEEP_CONNECTION,
INTERNET_INVALID_PORT_NUMBER,
NULL, NULL);
pHttpFile = pConnection->OpenRequest("GET",szObject,
NULL, 0, NULL, NULL,
INTERNET_FLAG_KEEP_CONNECTION);
pHttpFile->SendRequest(NULL);
pHttpFile->QueryInfo(HTTP_QUERY_CONTENT_LENGTH,tlen,0);
if (tlen>BUF_LEN+2)//页面数据过大
{
pConnection->Close();
pHttpFile->Close();
mysession.Close();
delete pConnection;
delete pHttpFile;
return false;
}
if (!tlen)//如果tlen为零,可能是函数没有读到头部,要人工定位数据大小
{
char temp[BUFSIZ*2*100];
ZeroMemory(temp,BUFSIZ*2*100);
int pos=0;
while(pHttpFile->Read(temp,BUFSIZ*2*100))
{
if (pos+strlen(temp)>BUF_LEN+2)//页面数据过大
{
mysession.Close();
pConnection->Close();
pHttpFile->Close();
delete pConnection;
delete pHttpFile;
return false;
}
memcpy(buf+pos,temp,strlen(temp));
pos+=strlen(temp);
ZeroMemory(temp,BUFSIZ*2*100);
}
*l=pos;
if (!pos)
{
mysession.Close();
pConnection->Close();//关闭会出错
pHttpFile->Close();
delete pConnection;
delete pHttpFile;
return false;
}
}
else
{
*l = tlen;
UINT uBytesRead = pHttpFile->Read(buf,tlen);
}
}
catch(CInternetException *p)//
{
p->Delete();//
if (pConnection)
{
pConnection->Close();
delete pConnection;
}
if (pHttpFile)
{
pHttpFile->Close();
delete pHttpFile;
}
mysession.Close();
return false;
}
catch (...)//比如超时异常,上面session的超时设置无效
{
if (pConnection)
{
pConnection->Close();
delete pConnection;
}
if (pHttpFile)
{
pHttpFile->Close();
delete pHttpFile;
}
mysession.Close();
return false;
}
pConnection->Close();
pHttpFile->Close();
mysession.Close();
delete pConnection;
delete pHttpFile;
return true;
}
/////////////////我现在用的是自己写的这个函数则没有出错
bool GetHttptFile(char *url,char *buf,int *l)
{
ZeroMemory(buf,BUF_LEN);
char session_[MAX_PATH];
ZeroMemory(session_,MAX_PATH);
session++;
itoa(session,session_,10);
CInternetSession m_Session(session_);
CHttpFile* m_HttpFile = NULL;
m_Session.SetOption(INTERNET_OPTION_CONNECT_TIMEOUT,1000);
m_Session.SetOption(INTERNET_OPTION_DATA_RECEIVE_TIMEOUT, 1000);
m_Session.SetOption(INTERNET_OPTION_DATA_SEND_TIMEOUT, 1000);
m_Session.SetOption(INTERNET_OPTION_CONNECT_BACKOFF,1000);
m_Session.SetOption(INTERNET_OPTION_CONNECT_RETRIES,2);
try
{
m_HttpFile = (CHttpFile*)m_Session.OpenURL(url);
DWORD dwStatusCode;
m_HttpFile-> QueryInfoStatusCode(dwStatusCode);
char temp[BUFSIZ*2*100];
ZeroMemory(temp,BUFSIZ*2*100);
int pos=0;
while(m_HttpFile-> Read(temp,BUFSIZ*2*100))
{
// SetFilePointer(hFile,0,0,FILE_END);
// WriteFile(hFile, myData,myData.GetLength(), &dwWritten, NULL);
if (pos+strlen(temp)>BUF_LEN)//页面数据过大
{
m_HttpFile-> Close();
m_Session.Close();
delete m_HttpFile;
return false;
}
memcpy(buf+pos,temp,strlen(temp));
pos+=strlen(temp);
ZeroMemory(temp,BUFSIZ*2*100);
}
*l=pos;
m_HttpFile->Close();
m_Session.Close();
delete m_HttpFile;
}
catch(CInternetException *p)//必须捕获,不然会内存泄露
{
p->Delete();//不要用delete p;而是执行封装好的方法。
if (m_HttpFile)
{
m_HttpFile->Close();
delete m_HttpFile;
}
m_Session.Close();
return false;
}
catch (...)//比如超时异常,上面session的超时设置无效
{
if (m_HttpFile)
{
m_HttpFile->Close();
delete m_HttpFile;
}
m_Session.Close();
return false;
}
return true;
}