用C++爬取网页

          做了好几天，终于写出来了，以前没有想到过，用C++也可以爬取网页，经过这么多天的努力终于做好了，解决了乱码问题。

从中学到很多，小到一个函数的参数，达到如何使用一个函数。

           还有C++中一直让人头疼的编码问题，unicode编码问题，研究了很多资料，又对Ｍutibytetowidechar和widechartomultibyte进行了重新的认识。

一个重要的关键是windows默认的是ANSI字符集，同时对HTML的格式进行了分析，以判断编码问题。

           感觉那么多天的辛苦没有白费，付出有了收获。不过在此，真的感谢那些牛人，期间也参考了他们的代码。

 代码:

?

#include <iostream>

#include <winsock2.h>

#include <cstring>

#include <fstream>

#pragma comment(lib,"ws2_32.lib")

 

using namespace std;

 

void getWebPage(char *url)

{

    SOCKET sock;

    WSADATA wsa;

    struct sockaddr_in  addrclient;

    ofstream of;

    WSAStartup(MAKEWORD(2,2),&wsa);

    of.open("temp.txt");

    if(!of)

    {

        cout<<"open fail!"<<endl;

        return;

    }

    static char content[100000]="";

    char myurl[256];

    char host[256];

    char dom[256];

    char header[256];

    char type[512];

    char *p;

    memset(myurl,'\0',256);

    memset(host,'\0',256);

    memset(dom,'\0',256);

    memset(header,'\0',256);

    memset(type,'\0',512);

    char *purl=0;

    struct hostent *phost;

    sock=socket(PF_INET,SOCK_STREAM,IPPROTO_TCP);

 

    strcpy(myurl,url);

    for(purl=myurl;*purl!='/'&&purl!='\0';++purl);

    if(int(purl-myurl)==strlen(myurl))

        strcpy(host,"/");

    else

        strcpy(host,purl);

    *purl='\0';

    strcpy(dom,myurl);

 

    cout<<dom<<endl;          //输出域名

    cout<<host<<endl;     //输出地址

    of<<dom<<endl;

    of<<host<<endl;

    phost=gethostbyname(dom);

         

    addrclient.sin_family=AF_INET;

    addrclient.sin_port=htons(80);

    addrclient.sin_addr.S_un.S_addr=*((unsigned long *)phost->h_addr);

     

    connect(sock,(struct sockaddr*)&addrclient,sizeof(addrclient));

     

    strcat(header, "GET ");

    strcat(header, host);

    strcat(header, " HTTP/1.1\r\n");

    strcat(header, "Host: ");

    strcat(header, dom);

    strcat(header, "\r\nConnection: Close\r\n\r\n");

    send(sock,header,strlen(header),0);

    recv(sock,type,512,0);

    cout<<type<<endl;

    of<<type;

    p=strstr(type,"utf-8");

    if(p)

    {

    memset(content,'\0',100000);

    while(recv(sock,content,100000,0)>0)

    {

        int len=MultiByteToWideChar(CP_UTF8, 0, content, -1, NULL,0);

        unsigned short * wszGBK = new unsigned short[len+1];

        memset(wszGBK, 0, len * 2 + 2);

        MultiByteToWideChar(CP_UTF8, 0, content, -1, (LPWSTR)wszGBK, len);

        len = WideCharToMultiByte(CP_ACP, 0, (LPCWSTR)wszGBK, -1, NULL, 0, NULL, NULL); 

        char *szGBK=new char[len + 1];

        memset(szGBK, 0, len + 1);

        WideCharToMultiByte (CP_ACP, 0, (LPCWSTR)wszGBK, -1, szGBK, len, NULL,NULL);

        cout<<szGBK;

        of<<szGBK;

        strnset(content,'\0',100000);

        delete []wszGBK;

        delete [] szGBK;

    }

    }

    else

    {

        memset(type,'\0',512);

        recv(sock,type,512,0);

        cout<<type;

        of<<type;

        p=strstr(type,"gb2312");

        if(p)

        {

            while(recv(sock,content,100000,0))

            {

                cout<<content;

                of<<content;

                strnset(content,'\0',100000);

            }

        }

        else

        {

           while(recv(sock,content,100000,0)>0)

           {

               int len=MultiByteToWideChar(CP_UTF8, 0, content, -1, NULL,0);

               unsigned short * wszGBK = new unsigned short[len+1];

               memset(wszGBK, 0, len * 2 + 2);

               MultiByteToWideChar(CP_UTF8, 0, content, -1, (LPWSTR)wszGBK, len);

               len = WideCharToMultiByte(CP_ACP, 0, (LPCWSTR)wszGBK, -1, NULL, 0, NULL, NULL); 

               char *szGBK=new char[len + 1];

               memset(szGBK, 0, len + 1);

               WideCharToMultiByte (CP_ACP, 0, (LPCWSTR)wszGBK, -1, szGBK, len, NULL,NULL);

               cout<<szGBK;

               of<<szGBK;

               strnset(content,'\0',100000);

               delete []wszGBK;

               delete [] szGBK;

           }

        }

    }

    closesocket(sock);

    WSACleanup();

    of.close();

    cout<<endl;

}

int main()

{

    char url[256];

    cout<<"http://";

    cin>>url;

    getWebPage(url);

    return 0;

}

 对此，又对socket编程产生了兴趣，socket编程魅力无穷。