ERF和PCAP文件中的ipv4流包统计与汇聚

跟着学校老师搞流量识别时写的，能把PCAP或ERF文件中的流分组后排序生成新的文件，思路很乱，好像还有点bug，分析一个114MB的PCAP文件没问题，但是分析1个2.7G的大数据集，在写入新文件的时候就出问题了，Wireshark打不开。有空再改改。

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

#include <winsock2.h>
#include <map>
#include <list>
#include <vector>
#include <cstdio>
#include <omp.h>
#include <algorithm>
#include <cstdlib>
using namespace std;
#define TRACE_PCAP      1
#define TRACE_ERF       2
 
#define LINK_ETH        0x00000001
#define LINK_11C        0x0000000B
 
#define PROT_IP         0x0800
#define PROT_TCP        0x06
#define PROT_UDP        0x11
#define PROT_ICMP       0x01
 
#define PCAP_HDR_LEN    16
#define ERF_HDR_LEN     18                  //including eth protocol header
#define ETH_HDR_LEN     14
#define C11_HDR_LEN     8
 
#define UM_FLOW         WM_USER + 100
 
#define BUFF_LEN        204800
#define HASH_LEN        2000000
#define MICE_FLOW_PKTS  4
 
struct thread_param
{
    u_short thread_id;
    LARGE_INTEGER fp_start;
    LARGE_INTEGER fp_end;
};
struct eth_header
{
    unsigned char       h_dest[6]; /* destination eth addr */
    unsigned char       h_source[6]; /* source ether addr */
    unsigned short      h_proto; /* packet type ID field */
};
 
struct c11_header
{
    unsigned char       h_unkown[6]; /*  */
    unsigned short      h_proto; /* packet type ID field */
};
 
struct ip_header
{
    unsigned char     iphVerLen;      // Version and length
    unsigned char     ipTOS;          // Type of service
    unsigned short    ipLength;       // Total datagram length
    unsigned short    ipID;           // Identification
    unsigned short    ipFlags;        // Flags
    unsigned char     ipTTL;          // Time to live
    unsigned char     ipProtocol;     // Protocol
    unsigned short    ipChecksum;     // Header checksum
    unsigned long     ipSource;       // Source address
    unsigned long     ipDestination;  // Destination address
};
 
struct tcp_header
{
    unsigned short          sourcePort;         // Source Port
    unsigned short          destinationPort;    // Destination Port
    unsigned long           sequenceNumber;     // Number of Sequence
    unsigned long           acknowledgeNumber;  // Number of aknowledge
    unsigned char               dataoffset;         // Pointer to data
    unsigned char               flags;              // Flags
    unsigned short          windows;            // Size of window
    unsigned short          checksum;           // Total checksum
    unsigned short          urgentPointer;      // Urgent pointer
};
 
struct udp_header
{
    unsigned short          sourcePort;         // Source Port
    unsigned short          destinationPort;    // Destination Port
    unsigned short          len;                // Total length
    unsigned short          checksum;           // Total checksum
};
 
struct pcap_file_header
{
    unsigned long           magic;
    unsigned short          version_major;
    unsigned short          version_minor;
    unsigned long           thiszone;
    unsigned long           sigfigs;
    unsigned long           snaplen;
    unsigned long           linktype;
};
 
struct pcap_pkthdr
{
    unsigned long           sec;
    unsigned long           usec;
    unsigned long           cap_len;
    unsigned long           len;
};
 
struct erf_pkthdr
{
    unsigned long           usec;
    unsigned long           sec;
    unsigned char           type;
    unsigned char           flags;
    unsigned short          rlen;
    unsigned short          lctr;
    unsigned short          wlen;
};
 
struct db_pkt
{
    unsigned long           sec;
    unsigned long           usec;
    char                    src_ip[20];
    char                    dst_ip[20];
    unsigned short          src_port;
    unsigned short          dst_port;
    char                    trs_prot[8];
    unsigned short          pkt_len;
};
 
struct trns_pkt_info
{
    unsigned long           sec;
    unsigned long           usec;
    unsigned long           src_ip;         // Source address
    unsigned long           dst_ip;         // Destination address
    unsigned short          src_port;       //Source port
    unsigned short          dst_port;       //Destination port
    unsigned char             trs_prot;
    unsigned short          pkt_len;
    unsigned long             len;
    char*           buff;
};
 
struct flow_info
{
    unsigned long           id;
    unsigned long           src_ip;         // Source address
    unsigned long           dst_ip;         // Destination address
    unsigned short          src_port;       //Source port
    unsigned short          dst_port;       //Destination port
    unsigned long           n_pkt;          //Number of packets
    unsigned int            f_start;        //start position in the flow file
    unsigned int            f_end;          //end  position in the flow file
};
 
int ipcmp(const unsigned long& ip1,const unsigned long& ip2)
{
        if(   (ip1&0x000000ff)          <  (ip2&0x000000ff)           ) return -1;
        if(   (ip1&0x000000ff)          >  (ip2&0x000000ff)           ) return 1;
        if(   (ip1&0x0000ff00)>>8   <  (ip2&0x0000ff00)>>8    ) return -1;
        if(   (ip1&0x0000ff00)>>8   >  (ip2&0x0000ff00)>>8    ) return 1;
        if(   (ip1&0x00ff0000)>>16 <  (ip2&0x00ff0000)>>16  ) return -1;
        if(   (ip1&0x00ff0000)>>16 >  (ip2&0x00ff0000)>>16  ) return 1;
        if(   (ip1&0xff000000)>>24 <  (ip2&0xff000000)>>24  ) return -1;
        if(   (ip1&0xff000000)>>24 >  (ip2&0xff000000)>>24  ) return 1;
        return 0;
}
 
struct session
{
    unsigned long           ip1;
    unsigned long           ip2;  //ip1 always < ip2
    unsigned short          port1;
    unsigned short          port2;
    bool operator <(const session& x) const
    {
 
        if (ipcmp(ip1,x.ip1)<0) return true;
        if (ipcmp(ip1,x.ip1)>0) return false;
        if (ipcmp(ip2,x.ip2)<0) return true;
        if (ipcmp(ip2,x.ip2)>0) return false;
        /*
        if (ip1<x.ip1) return true;
        if (ip1>x.ip1) return false;
        if (ip2<x.ip2) return true;
        if (ip2>x.ip2) return false;
        */
        if (port1<x.port1) return true;
        if (port1>x.port1) return false;
        if (port2<x.port2) return true;
        if (port2>x.port2) return false;
        return false;
    }
};
 
map<session,list<trns_pkt_info> >g_flowmap;
char        g_tracefile[MAX_PATH] = "1.pcap";
int         g_tracetype = TRACE_PCAP;
u_short     g_linktype = LINK_ETH;
FILE        *fp_trace = NULL;
pcap_file_header g_fhdr;
 
long NextTransPacket(trns_pkt_info &pkt, FILE *fp_trace)
{
    int cap_len;                                //cap packet length does not include format header
    char *buff, *p;
    long res = 0;
    memset(&pkt, 0, sizeof(pkt));
    if (!fp_trace)return 0;
    if (feof(fp_trace))return 0;
    if (g_tracetype == TRACE_PCAP)
    {
        struct pcap_pkthdr pkthdrp;
        if ((fread((void *)&pkthdrp, 1, sizeof(struct pcap_pkthdr), fp_trace)) < sizeof(struct pcap_pkthdr))
            return 0;
        cap_len = pkthdrp.cap_len;              //The cap_len of a pcap packet does not include the pcap header, big-endian
        pkt.sec = pkthdrp.sec;
        pkt.usec = pkthdrp.usec;
        res = cap_len + sizeof(struct pcap_pkthdr);
    }
    else if (g_tracetype == TRACE_ERF)
    {
        struct erf_pkthdr pkthdre;
        if ((fread((void *)&pkthdre, 1, sizeof(struct erf_pkthdr), fp_trace) )< sizeof(struct erf_pkthdr))
            return 0;
        cap_len = ntohs(pkthdre.rlen) - sizeof(struct erf_pkthdr) - 2;          //The rlen of a erf packet includes the erf header,little-endian
        pkt.sec = ntohl(pkthdre.sec);
        pkt.usec = ntohl(pkthdre.usec);
        fseek(fp_trace, 2, SEEK_CUR);                                           //Skip eth protocol header
        res = ntohs(pkthdre.rlen);
    }
 
    buff = (char *)malloc(cap_len);
    u_short link_prot;
    struct ip_header *iph;
    struct eth_header *ethh;
    struct c11_header *c11h;
    struct tcp_header *tcph;
    struct udp_header *udph;
    char *pHeader;
//  in_addr addr;
    unsigned int IPHeaderLength;
 
    if (fread(buff, 1, cap_len, fp_trace) < cap_len)
        return FALSE;
    pkt.buff=(char *)malloc(res);
    fseek(fp_trace,-res,SEEK_CUR);
    fread(pkt.buff,1,res,fp_trace);
    pkt.len=res;
    pHeader = buff;
    if (g_linktype == LINK_11C)
    {
        c11h = (struct c11_header *)pHeader;
        link_prot = ntohs(c11h->h_proto);
    }
    else
    {
        ethh = (struct eth_header *)pHeader;
        link_prot = ntohs(ethh->h_proto);
        if (link_prot == 0x0081)                //VLAN tag
        {
            pHeader += 4;
            ethh = (struct eth_header *)pHeader;
            link_prot = ntohs(ethh->h_proto);
        }
    }
    if (link_prot == PROT_IP)  //IP packet
    {
        pHeader += (g_linktype == LINK_11C) ? sizeof(struct c11_header) : sizeof(struct eth_header);
        iph = (struct ip_header *)pHeader;
        IPHeaderLength = ((iph->iphVerLen) & 0x0F) * 4;
        pkt.src_ip = iph->ipSource;
        pkt.dst_ip = iph->ipDestination;
        pkt.trs_prot = iph->ipProtocol;
        if (iph->ipProtocol == PROT_TCP)  //TCP packet
        {
            pHeader += IPHeaderLength;
            tcph = (struct tcp_header *)pHeader;
            pkt.src_port = ntohs(tcph->sourcePort);
            pkt.dst_port = ntohs(tcph->destinationPort);
            pkt.pkt_len = ntohs(iph->ipLength) - IPHeaderLength - sizeof(struct tcp_header);
        }
        else if (iph->ipProtocol == PROT_UDP)  //UDP packet
        {
            pHeader += IPHeaderLength;
            udph = (struct udp_header *)pHeader;
            pkt.src_port = ntohs(udph->sourcePort);
            pkt.dst_port = ntohs(udph->destinationPort);
            pkt.pkt_len = ntohs(iph->ipLength) - IPHeaderLength - sizeof(struct udp_header);
        }
    }
    free(buff);
    return res;
}
 
 
void ReadFlowFile()
{
    fp_trace=fopen(g_tracefile,"rb");
    if (!fp_trace)return;
    if (g_tracetype == TRACE_PCAP)
    {
        fread((void *)&g_fhdr, 1, sizeof(pcap_file_header), fp_trace);
        g_linktype = g_fhdr.linktype;
    }
    trns_pkt_info pkt;
    long res;
    session tmp;
    while(res=NextTransPacket(pkt,fp_trace))
    {
       if(pkt.src_ip==0||pkt.dst_ip==0) continue;
        pkt.buff=(char *)malloc(res);
        fseek(fp_trace,-res,SEEK_CUR);
        fread(pkt.buff,1,res,fp_trace);
        pkt.len=res;
       if(ipcmp(pkt.src_ip,pkt.dst_ip)<0)
       //if(pkt.src_ip<pkt.dst_ip)
        {
            tmp.ip1=pkt.src_ip;
            tmp.ip2=pkt.dst_ip;
            tmp.port1=pkt.src_port;
            tmp.port2=pkt.dst_port;
        }
        else
        {
            tmp.ip1=pkt.dst_ip;
            tmp.ip2=pkt.src_ip;
            tmp.port1=pkt.dst_port;
            tmp.port2=pkt.src_port;
        }
        if(!g_flowmap.count(tmp))
        {
            list<trns_pkt_info> li;
            li.push_back(pkt);
            g_flowmap[tmp]=li;
        }
        else
        {
            g_flowmap[tmp].push_back(pkt);
            pkt.buff=NULL;
        }
    }
    fclose(fp_trace);
    fp_trace=NULL;
}
void ShowAllFlows()
{
    session tmp;
    in_addr addr1,addr2;
    printf("共有%d组数据\n",g_flowmap.size());
    system("PAUSE");
    map<session,list<trns_pkt_info> >::iterator it;
    int i;
    for(it=g_flowmap.begin(),i=1;it!=g_flowmap.end();it++,i++)
    {
        tmp=it->first;
        printf("\n----------------------------------\n");
        printf("第%d组数据:\n",i);
        memcpy(&addr1,&tmp.ip1,4);
        memcpy(&addr2,&tmp.ip2,4);
        printf("ip1:%s  port1:%d\n",inet_ntoa(addr1),tmp.port1);
        printf("ip2:%s  port2:%d\n",inet_ntoa(addr2),tmp.port2);
        printf("共有%d个包:\n",it->second.size());
        for(list<trns_pkt_info>::iterator j=it->second.begin();j!=it->second.end();j++)
        {
            memcpy(&addr1,&j->src_ip,4);
            memcpy(&addr2,&j->dst_ip,4);
            printf("src:%s port:%d  ",inet_ntoa(addr1),j->src_port);
            printf("dst:%s port:%d  \n",inet_ntoa(addr2),j->dst_port);
        }
        printf("----------------------------------\n");
        system("PAUSE");
    }
}
void ConvergeFlows()
{
    char pkt_fname[MAX_PATH];
    char buff[BUFF_LEN];
    if(g_tracetype == TRACE_PCAP)
        sprintf(pkt_fname, "%s.pkt.pcap", g_tracefile);
    else
        sprintf(pkt_fname, "%s.pkt.erf", g_tracefile);
    FILE *fp_pkt = fopen(pkt_fname, "wb");
    if (g_tracetype == TRACE_PCAP)
    {
         struct pcap_pkthdr pkthdrp;
        fwrite((void *)&g_fhdr, 1, sizeof(struct pcap_file_header), fp_pkt);
    }
    map<session,list<trns_pkt_info> >::iterator it;
    trns_pkt_info pkt;
    for(it=g_flowmap.begin();it!=g_flowmap.end();it++)
    {
        for(list<trns_pkt_info>::iterator j=it->second.begin();j!=it->second.end();j++)
        {
            pkt=*j;
            fwrite(pkt.buff, sizeof(char), pkt.len, fp_pkt);
        }
    }
    fclose(fp_pkt);
}
int main()
{
    ReadFlowFile();
   // ShowAllFlows();
    ConvergeFlows();
}