P3

package p3.hadoop.mapred;

import java.io.IOException;
import java.io.InputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.BytesWritable;
import p3.common.lib.BinaryUtils;
import p3.common.lib.Bytes;

public class PcapLineReader
{
  private static final int DEFAULT_BUFFER_SIZE = 2048;
  private int bufferSize;
  private static final int PCAP_FILE_HEADER_LENGTH = 24;
  private static final int PCAP_PACKET_HEADER_LENGTH = 16;
  private static final int PCAP_PACKET_HEADER_CAPLEN_POS = 8;
  private static final int PCAP_PACKET_HEADER_WIREDLEN_POS = 12;
  private static final int PCAP_PACKET_HEADER_CAPLEN_LEN = 4;
  private static final int PCAP_PACKET_HEADER_TIMESTAMP_LEN = 4;
  private static final int PCAP_PACKET_MIN_LEN = 53;
  private static final int PCAP_PACKET_MAX_LEN = 1519;
  private static final int MAGIC_NUMBER = -725372255;
  private static final int MIN_PKT_SIZE = 42;
  private long min_captime;
  private long max_captime;
  private InputStream in;
  private byte[] buffer;
  byte[] pcap_header;
  private int bufferLength;
  int consumed;

  public PcapLineReader(InputStream in, int bufferSize, long min_captime, long max_captime)
  {
    this.bufferSize = 2048;

    this.bufferLength = 0;
    this.consumed = 0;

    this.in = in;
    this.bufferSize = bufferSize;
    this.buffer = new byte[this.bufferSize];
    this.min_captime = min_captime;
    this.max_captime = max_captime;
  }

  public PcapLineReader(InputStream in, Configuration conf)
    throws IOException
  {
    this(in, 2048, 
      conf.getLong("pcap.file.captime.min", 1309412600L), 
      conf.getLong("pcap.file.captime.max", conf.getLong("pcap.file.captime.max", 1309412600L) + 172800L));
  }

  public void close()
    throws IOException
  {
    this.in.close();
  }

  int skipPartialRecord(int fraction)
    throws IOException
  {
    int pos = 0;
    byte[] captured = new byte[fraction];
    byte[] tmpTimestamp1 = new byte[4];
    byte[] tmpTimestamp2 = new byte[4];
    byte[] tmpCapturedLen1 = new byte[4];
    byte[] tmpWiredLen1 = new byte[4];
    byte[] tmpCapturedLen2 = new byte[4];
    byte[] tmpWiredLen2 = new byte[4];
    int caplen1 = 0;
    int wiredlen1 = 0;
    int caplen2 = 0;
    int wiredlen2 = 0;
    long timestamp2 = 0L;

    int size = 0;
    long endureTime = 100L;

    if ((size = this.in.read(captured)) < 42) return 0;

    do
    {
      if ((size - pos < 32) || (size - pos < 53)) {
        pos = size;
        break;
      }

      System.arraycopy(captured, pos, tmpTimestamp1, 0, 4);
      long timestamp1 = Bytes.toLong(BinaryUtils.flipBO(tmpTimestamp1, 4));

      System.arraycopy(captured, pos + 8, tmpCapturedLen1, 0, 4);
      caplen1 = Bytes.toInt(BinaryUtils.flipBO(tmpCapturedLen1, 4));

      System.arraycopy(captured, pos + 12, tmpWiredLen1, 0, 4);
      wiredlen1 = Bytes.toInt(BinaryUtils.flipBO(tmpWiredLen1, 4));

      if ((caplen1 > 53) && (caplen1 < 1519) && (size - pos - 32 - caplen1 > 0))
      {
        System.arraycopy(captured, pos + 16 + caplen1 + 8, tmpCapturedLen2, 0, 4);
        caplen2 = Bytes.toInt(BinaryUtils.flipBO(tmpCapturedLen2, 4));

        System.arraycopy(captured, pos + 16 + caplen1 + 12, tmpWiredLen2, 0, 4);
        wiredlen2 = Bytes.toInt(BinaryUtils.flipBO(tmpWiredLen2, 4));

        System.arraycopy(captured, pos + 16 + caplen1, tmpTimestamp2, 0, 4);
        timestamp2 = Bytes.toLong(BinaryUtils.flipBO(tmpTimestamp2, 4));

        if ((timestamp1 >= this.min_captime) && (timestamp1 < this.max_captime) && (this.min_captime <= timestamp2) && (timestamp2 < this.max_captime) && 
          (wiredlen1 > 53) && (wiredlen1 < 1519) && (wiredlen2 > 53) && (wiredlen2 < 1519) && 
          (caplen1 > 0) && (caplen1 <= wiredlen1) && (caplen2 > 0) && (caplen2 <= wiredlen2) && 
          (timestamp2 >= timestamp1) && (timestamp2 - timestamp1 < endureTime)) {
          return pos;
        }

      }

      ++pos;
    }
    while (pos < size);

    return pos;
  }

  int readPacket(int packetLen)
    throws IOException
  {
    int bufferPosn = 16;
    byte[] tmp_buffer = new byte[packetLen];

    if ((this.bufferLength = this.in.read(tmp_buffer)) < packetLen) {
      System.arraycopy(tmp_buffer, 0, this.buffer, bufferPosn, this.bufferLength);
      bufferPosn += this.bufferLength;

      byte[] newpacket = new byte[packetLen - this.bufferLength];

      if ((this.bufferLength = this.in.read(newpacket)) < 0) return bufferPosn;
      System.arraycopy(newpacket, 0, this.buffer, bufferPosn, this.bufferLength);
    }
    else
    {
      System.arraycopy(tmp_buffer, 0, this.buffer, bufferPosn, this.bufferLength);
    }
    bufferPosn += this.bufferLength;

    return bufferPosn;
  }

  int readPacketHeader()
  {
    int headerLength = 0;
    int headerPosn = 0;
    this.pcap_header = new byte[16];

    byte[] tmp_header = new byte[16];
    BytesWritable capturedLen = new BytesWritable();
    try
    {
      if ((headerLength = this.in.read(this.pcap_header)) < 16)
      {
        if (headerLength == -1) return 0;
        headerPosn += headerLength;

        byte[] newheader = new byte[16 - headerLength];

        if ((headerLength = this.in.read(newheader)) < 0) {
          this.consumed = headerPosn;
          return -1;
        }
        System.arraycopy(newheader, 0, this.pcap_header, headerPosn, headerLength);
      }
      capturedLen.set(this.pcap_header, 8, 4);
      System.arraycopy(this.pcap_header, 0, this.buffer, 0, 16);
      headerPosn = 0;
    }
    catch (IOException e)
    {
      e.printStackTrace();
    }
    return Bytes.toInt(BinaryUtils.flipBO(capturedLen.getBytes(), 4));
  }

  public int readFileHeader()
  {
    try {
      byte[] magic = new byte[4];
      this.bufferLength = this.in.read(this.buffer, 0, 24);
      System.arraycopy(this.buffer, 0, magic, 0, magic.length);

      if (Bytes.toInt(magic) == -725372255) break label50;
      return 0;
    }
    catch (IOException e) {
      e.printStackTrace();
    }
    label50: return this.bufferLength;
  }

  public int readLine(BytesWritable bytes, int maxLineLength, int maxBytesToConsume)
    throws IOException
  {
    bytes.set(new BytesWritable());
    boolean hitEndOfFile = false;
    long bytesConsumed = 0L;

    int caplen = readPacketHeader();

    if (caplen == 0) {
      bytesConsumed = 0L;
    } else if (caplen == -1) {
      bytesConsumed += this.consumed;
    }
    else if ((caplen > 0) && (caplen < 1519)) {
      if ((this.bufferLength = readPacket(caplen)) < caplen + 16) {
        hitEndOfFile = true;
      }
      bytesConsumed += this.bufferLength;

      if (!(hitEndOfFile)) {
        bytes.set(this.buffer, 0, caplen + 16);
      }
    }

    return (int)Math.min(bytesConsumed, 2147483647L);
  }

  public int readLine(BytesWritable str, int maxLineLength)
    throws IOException
  {
    return readLine(str, maxLineLength, 2147483647);
  }

  public int readLine(BytesWritable str)
    throws IOException
  {
    return readLine(str, 2147483647, 2147483647);
  }
}

 

// Decompiled by Jad v1.5.8e2. Copyright 2001 Pavel Kouznetsov.
// Jad home page: http://kpdus.tripod.com/jad.html
// Decompiler options: packimports(3) fieldsfirst ansi space 
// Source File Name:   PcapVlenRecordReader.java

package p3.hadoop.mapred;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.RecordReader;

// Referenced classes of package p3.hadoop.mapred:
//			PcapLineReader

public class PcapVlenRecordReader
	implements RecordReader
{

	private CompressionCodecFactory compressionCodecs;
	private long start;
	private long pos;
	private long end;
	private PcapLineReader in;
	int maxLineLength;
	private boolean fileheader_skip;

	public PcapVlenRecordReader(Configuration job, FileSplit split)
		throws IOException
	{
		compressionCodecs = null;
		fileheader_skip = true;
		maxLineLength = job.getInt("mapred.linerecordreader.maxlength", 0x7fffffff);
		fileheader_skip = job.getBoolean("pcap.file.header.skip", true);
		start = split.getStart();
		end = start + split.getLength();
		Path file = split.getPath();
		compressionCodecs = new CompressionCodecFactory(job);
		CompressionCodec codec = compressionCodecs.getCodec(file);
		FileSystem fs = file.getFileSystem(job);
		FSDataInputStream fileIn = fs.open(split.getPath());
		boolean skipFileHeader = false;
		boolean skipPartialRecord = false;
		int fraction = 4000;
		if (codec != null)
		{
			in = new PcapLineReader(codec.createInputStream(fileIn), job);
			end = 0x7fffffffffffffffL;
			skipFileHeader = true;
		} else
		{
			if (start == 0L)
			{
				skipFileHeader = true;
			} else
			{
				skipPartialRecord = true;
				fileIn.seek(start);
			}
			in = new PcapLineReader(fileIn, job);
		}
		if (skipFileHeader)
			start += in.readFileHeader();
		if (skipPartialRecord)
		{
			int skip;
			for (skip = in.skipPartialRecord(fraction); skip == fraction; skip = in.skipPartialRecord(fraction))
				start += skip;

			start += skip;
			fileIn.seek(start);
			in = new PcapLineReader(fileIn, job);
		}
		pos = start;
	}

	public LongWritable createKey()
	{
		return new LongWritable();
	}

	public BytesWritable createValue()
	{
		return new BytesWritable();
	}

	public synchronized boolean next(LongWritable key, BytesWritable value)
		throws IOException
	{
		while (pos < end) 
		{
			key.set(pos);
			int newSize = in.readLine(value, maxLineLength, Math.max((int)Math.min(0x7fffffffL, end - pos), maxLineLength));
			if (newSize == 0)
			{
				pos = end;
				return false;
			}
			pos += newSize;
			if (newSize < maxLineLength)
				return true;
		}
		return false;
	}

	public float getProgress()
	{
		if (start == end)
			return 0.0F;
		else
			return Math.min(1.0F, (float)(pos - start) / (float)(end - start));
	}

	public synchronized long getPos()
		throws IOException
	{
		return pos;
	}

	public synchronized void close()
		throws IOException
	{
		if (in != null)
			in.close();
	}

	public volatile boolean next(Object obj, Object obj1)
		throws IOException
	{
		return next((LongWritable)obj, (BytesWritable)obj1);
	}

	public volatile Object createValue()
	{
		return createValue();
	}

	public volatile Object createKey()
	{
		return createKey();
	}
}