java的网络编程
1.流式套接字和数据报式套接字以及原始式套接字的区别
流式套接字：面向连接，可靠的数据连接方式，数据无差错，无重复，完全按照发送顺序，可大量传输数据，传输完毕需要释放已经建立的连接，效率不高，遵循三次握手，发送信息，接受信息，反馈已经接受的信息。（TCP协议）
数据报式套接字：无服务连接，数据以独立包的形式发送，不提供数据无差错，无重复保证，且接受是无序的，不需要建立连接，大小限制在64k之内，无需释放资源速度快。（UDP）协议
原始式套接字：允许对较低层次协议，如ip的直接访问，监听网络流量和分析比较有用
2.可用端口
可用端口为1-65535，但是1-1024不建议使用，还有例如3306,8080等端口也相应的被占用
3.dns域名解析过程
首先查找的是浏览器缓存中是否有已经解析的域名地址，如果有，那么直接使用，这个域名解析过程结束，这个时间由ttl来设置，但是也受浏览器缓存大小的限制，缓存时间过长，ip地址变化找不到ip地址，导致域名不能正常解析，时间太短，那么导致每次访问网站都需要重新解析一次域名，如果在缓存中没有找到，那么会相应的在操作系统的缓存中选择是否有相应的解析ip地址解析结果，如果有那么久使用这个ip地址并返回，这里在以前的时候存在域名劫持，因为可以通过c：\windows\systems\driver\etc\hosts 来指定一个解析的ip地址，win7后这个只能读而不能改，当然linux的配置文件/etc/name.conf中也可以修改达到同样的目的，当然也受到缓存时间的影响，假如前两个无法解决，那么会相应的发送到ldns（本地区的域名服务器，公司的服务器），这window下可以使用ipconfig查询，在linux下可以查相应的配置文件 /etc/resolv.conf ,一般而言这个专门的域名解析性能都会比较好。一般会缓存域名解析的结果，也受缓存时间的控制，大约80%的域名解析都是在这里完成的，ldns主要承担了域名的解析工作，这里假如还是没有才会使用root server域名的解析工作，返回时一个逆过程，在dns解析的过程中主要在两个地方缓存结果，一个是ldns一个是本机，其中缓存控制是由缓存时间和缓存大小控制的，最大的缓存时间是ttl，ldns的缓存时间是我们很难由本地介入，本地的缓存可以使用 ipcongfig/flushdns，linux可以/etc/imit.d/nscd restart来清除缓存，当然在windows和linux下都可以使用nslookup来查看相应的解析结果
4.InetAddress与SocketAddress
InetAddress：表示带互联网协议
SocketAddress：不带任何的协议
public class InetAddressTest {
public static void main(String[] args) throws UnknownHostException {
InetAddress byName = InetAddress.getByName("localhost");
System.out.println(byName.getHostName());
System.out.println(byName.getHostAddress());
/*输出
* localhost
* 127.0.0.1
*/
}
5.TCP（Socket与ServerSocket）
备注：网络编程实质是网络版的io流
@SuppressWarnings("all")
public class ServerSocketAndSocketTest {
@Test
public void client() throws UnknownHostException, IOException {
Socket socket = null;
OutputStream outputStream = null;
try {
socket = new Socket("localhost", 9898);
outputStream = socket.getOutputStream();
outputStream.write("hello 你好！".getBytes());
outputStream.flush();
} finally {
if (outputStream != null) {
outputStream.close();
}
if (socket != null) {
socket.close();
}
}
}
@Test
public void server() throws IOException {
ServerSocket serverSocket = null;
Socket accept = null;
InputStream inputStream = null;
BufferedReader bufferedReader = null;
try {
serverSocket = new ServerSocket();
serverSocket.bind(new InetSocketAddress(9898));
// 阻塞式方法
accept = serverSocket.accept();
inputStream = accept.getInputStream();
bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "utf-8"));
String str = null;
while ((str = bufferedReader.readLine()) != null) {
System.out.println(str);
}
// 这个可以使用getOutputStream()传输消息回去
} finally {
if (inputStream != null) {
inputStream.close();
}
if (bufferedReader != null) {
bufferedReader.close();
}
if (accept != null) {
accept.close();
}
if (serverSocket != null) {
serverSocket.close();
}
}
}
}
6.UDP（DatagramSocket与DatagramPacket）
@SuppressWarnings("all")
public class DatagramSocketAndDatagramPacketTest {
@Test
public void client() throws IOException {
DatagramSocket datagramSocket = null;
try {
datagramSocket = new DatagramSocket();
byte[] byt = "hello 你好".getBytes();
DatagramPacket datagramPacket = new DatagramPacket(byt, byt.length,
new InetSocketAddress("localhost", 9898));
datagramSocket.send(datagramPacket);
} finally {
if (datagramSocket != null) {
datagramSocket.close();
}
}
}
@Test
public void server() throws IOException {
DatagramSocket datagramSocket = null;
try {
datagramSocket = new DatagramSocket(9898);
byte[] byt = new byte[1024];
DatagramPacket datagramPacket = new DatagramPacket(byt, byt.length);
datagramSocket.receive(datagramPacket);
int length = datagramPacket.getLength();
byte[] data = datagramPacket.getData();
System.out.println(new String(data, 0, length, "utf-8"));
} finally {
if (datagramSocket != null) {
datagramSocket.close();
}
}
}
}
7.url
public class URLTest {
public static void main(String[] args) throws MalformedURLException {
URL url = new URL("https://www.baidu.com/baidu?tn=monline_3_dg&ie=utf-8&wd=%E7%99%BE%E5%BA%A6");
System.out.println(url.getProtocol());//https
System.out.println(url.getHost());//www.baidu.com
System.out.println(url.getRef());//存在？为空，否则按实输出
System.out.println(url.getPort());//-1 未定义默认的端口号
System.out.println(url.getPath());// /baidu
System.out.println(url.getQuery());//锚点之后的内容
}
}
//爬取图片
//有时会遇到403，这个在爬虫中
public class URLDown { public static void main(String[] args) throws IOException { String path="http://www.umei.cc/meinvtupian/meinvxiezhen/20580.htm"; java.util.List<String> mapPath = getMapPath(path); for (String string : mapPath) { URL url = new URL(string); URLConnection openConnection = url.openConnection(); InputStream inputStream = openConnection.getInputStream(); FileOutputStream fileOutputStream = new FileOutputStream(url.getFile().substring(url.getFile().lastIndexOf("/")+1)); byte [] byts=new byte[1024]; int len=0; while((len=inputStream.read(byts))!=-1){ fileOutputStream.write(byts,0,len); } if(inputStream!=null){ inputStream.close(); } if(fileOutputStream!=null){ fileOutputStream.close(); } } } public static java.util.List<String> getMapPath(String path) throws IOException{ ArrayList<String> list = new ArrayList<>(); InputStream inputStream=null; BufferedReader bufferedReader=null; try { URL url = new URL(path); URLConnection openConnection = url.openConnection(); inputStream = openConnection.getInputStream(); //获取编码格式 String fileEncode = URLDown.getFileEncode(path); bufferedReader = new BufferedReader(new InputStreamReader(inputStream, fileEncode)); String str = null; String regex="http://i1.umei.cc/uploads/tu/201807.*.jpg"; Pattern compile = Pattern.compile(regex); while ((str = bufferedReader.readLine()) != null) { Matcher matcher = compile.matcher(str); while(matcher.find()){ //将符合要求的放入集合 list.add(matcher.group()); } } return list; } finally { if(bufferedReader!=null){ bufferedReader.close(); } if(inputStream!=null){ inputStream.close(); } } } public static String getFileEncode(String path) { /* * detector是探测器，它把探测任务交给具体的探测实现类的实例完成。 * cpDetector内置了一些常用的探测实现类，这些探测实现类的实例可以通过add方法 加进来，如ParsingDetector、 * JChardetFacade、ASCIIDetector、UnicodeDetector。 * detector按照“谁最先返回非空的探测结果，就以该结果为准”的原则返回探测到的 * 字符集编码。使用需要用到三个第三方JAR包：antlr.jar、chardet.jar和cpdetector.jar * cpDetector是基于统计学原理的，不保证完全正确。 */ CodepageDetectorProxy detector = CodepageDetectorProxy.getInstance(); /* * ParsingDetector可用于检查HTML、XML等文件或字符流的编码,构造方法中的参数用于 * 指示是否显示探测过程的详细信息，为false不显示。 */ detector.add(new ParsingDetector(false)); /* * JChardetFacade封装了由Mozilla组织提供的JChardet，它可以完成大多数文件的编码 * 测定。所以，一般有了这个探测器就可满足大多数项目的要求，如果你还不放心，可以 * 再多加几个探测器，比如下面的ASCIIDetector、UnicodeDetector等。 */ detector.add(JChardetFacade.getInstance());// 用到antlr.jar、chardet.jar // ASCIIDetector用于ASCII编码测定 // detector.add(ASCIIDetector.getInstance()); // UnicodeDetector用于Unicode家族编码的测定 // detector.add(UnicodeDetector.getInstance()); java.nio.charset.Charset charset = null; URL url=null; try { url = new URL(path); } catch (MalformedURLException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { charset = detector.detectCodepage(url); } catch (Exception ex) { ex.printStackTrace(); } if (charset != null) return charset.name(); else return null; }; }
//多线程下载
public class UrlThreadDown { public static void main(String[] args) throws IOException, InterruptedException { long start = System.currentTimeMillis(); String pathName = "https://qd.myapp.com/myapp/qqteam/tim/down/tim_pc.exe"; int threadNum = 3; CountDownLatch countDownLatch = new CountDownLatch(threadNum); ThreadDown threadDown = new ThreadDown(pathName, threadNum,countDownLatch); for (int i = 0; i < threadNum; i++) { new Thread(threadDown).start(); } countDownLatch.await(); long end = System.currentTimeMillis(); System.out.println(end-start); threadDown.fileConnect(); end = System.currentTimeMillis(); System.out.println(end-start); } } @SuppressWarnings("all") class ThreadDown implements Runnable { public List<String> list = new ArrayList<>(); private String pathName; private int threadNum; private int threadI; private int contentLength; CountDownLatch countDownLatch; public ThreadDown(String pathName, int threadNum,CountDownLatch countDownLatch) throws IOException { this.pathName = pathName; this.threadNum = threadNum; this.countDownLatch=countDownLatch; URL url=null; try { url = new URL(pathName); URLConnection openConnection = url.openConnection(); contentLength = openConnection.getContentLength(); } catch (Exception e) { System.out.println(pathName+"不正确"); } } @Override public void run() { try { down(); } catch (IOException e) { e.printStackTrace(); } } public void down() throws IOException { URL url = new URL(pathName); if(url==null) { return; } if(contentLength==-1){ System.out.println("大小未知"); return; } String fileName=url.getFile(); URLConnection openConnection = url.openConnection(); //在创建对象之后，建立连接之前，可指定各种选项（例如，doInput 和 UseCaches）。 //连接后再进行设置就会发生错误。连接后才能进行的操作（例如 getContentLength）， //如有必要，将隐式执行连接。 int size = contentLength / threadNum + 1; BufferedInputStream bufferedInputStream = null; BufferedOutputStream bufferedOutputStream = null; try { String bytes = null; String substring = null; synchronized (this) { if (threadI < threadNum - 1) { bytes = "bytes=" + threadI * size + "-" + ((threadI + 1) * size-1); } else { bytes = "bytes=" + (contentLength - ((threadI - 1) * size-1)) + "-" + (contentLength-1); } substring = fileName.substring(fileName.lastIndexOf("/") + 1) + threadI; threadI++;
list.add(substring); }
//包括开头和结尾 openConnection.setRequestProperty("Range", bytes); bufferedInputStream = new BufferedInputStream(openConnection.getInputStream()); bufferedOutputStream = new BufferedOutputStream(new FileOutputStream(substring)); int len = 0; while ((len = bufferedInputStream.read()) != -1) { bufferedOutputStream.write(len); } countDownLatch.countDown(); } finally { if (bufferedInputStream != null) { bufferedInputStream.close(); } if (bufferedOutputStream != null) { bufferedOutputStream.close(); } } } public void fileConnect() throws IOException { if(list.size()<=1){ return; } BufferedOutputStream bufferedOutputStream = null; try { Collections.sort(list); bufferedOutputStream = new BufferedOutputStream( new FileOutputStream(list.get(0).substring(0, list.get(0).length() - 1))); for (String string : list) { BufferedInputStream bufferedInputStream = null; try { bufferedInputStream = new BufferedInputStream(new FileInputStream(string)); int len = 0; while ((len = bufferedInputStream.read()) != -1) { bufferedOutputStream.write(len); } } finally { if (bufferedInputStream != null) { bufferedInputStream.close(); } } } } finally { if (bufferedOutputStream != null) { bufferedOutputStream.close(); } } } }
备注：使用skip循环下载并不适用网络下载，因为网络存在阻塞，出于各种原因，skip 方法最终跳过的字节数可能更少一些，甚至可能为 0。如果 n 为负，则抛出 IOException，例如如下，可以下载Apache下的本地文件，但是网上并不适用，available（）方法，返回不收阻塞的输入流
class ThreadDown2 implements Runnable {
public List<String> list = new ArrayList<>();
private String pathName;
private int threadNum;
private int threadI;
private int contentLength;
CountDownLatch countDownLatch;
public ThreadDown2(String pathName, int threadNum,CountDownLatch countDownLatch) throws IOException {
this.pathName = pathName;
this.threadNum = threadNum;
this.countDownLatch=countDownLatch;
URL url=null;
try {
url = new URL(pathName);
URLConnection openConnection = url.openConnection();
contentLength = openConnection.getContentLength();
System.out.println(contentLength);
} catch (Exception e) {
System.out.println(pathName+"不正确");
}
}
@Override
public void run() {
try {
down();
} catch (IOException e) {
e.printStackTrace();
}
}
public void down() throws IOException {
URL url = new URL(pathName);
if(url==null) {
return;
}
if(contentLength==-1){
System.out.println("大小未知");
return;
}
String fileName=url.getFile();
URLConnection openConnection = url.openConnection();
//在创建对象之后，建立连接之前，可指定各种选项（例如，doInput 和 UseCaches）。
//连接后再进行设置就会发生错误。连接后才能进行的操作（例如 getContentLength），
//如有必要，将隐式执行连接。
int size = contentLength / threadNum + 1;
InputStream inputStream=null;
FileOutputStream fileOutputStream =null;
try {
int startPosition=0;
int endPosition=0;
String substring = null;
synchronized (this) {
startPosition=threadI * size;
if (threadI < threadNum - 1) {
endPosition=(threadI+1)* size;
} else {
endPosition=contentLength;
}
substring = fileName.substring(fileName.lastIndexOf("/") + 1) + threadI;
threadI++;
list.add(substring);
}
String substring2 = substring.substring(substring.length()-1);
if(endPosition-startPosition==size){
inputStream = openConnection.getInputStream();
fileOutputStream = new FileOutputStream(substring);
inputStream.skip(Integer.valueOf(substring2)*size);
System.out.println(substring+":"+Integer.valueOf(substring2)*size+":"+(Integer.valueOf(substring2)*size+size));
byte[] byts = new byte[8192];
for (int i = 0; i <= size / 8192; i++) {
if (i < size / 8192 ) {
inputStream.read(byts);
fileOutputStream.write(byts, 0, 8192);
} else {
inputStream.read(byts);
fileOutputStream.write(byts, 0, size%8192);
}
}
} else {
inputStream = openConnection.getInputStream();
fileOutputStream = new FileOutputStream(substring);
inputStream.skip(Integer.valueOf(substring2)*size);
byte[] byts = new byte[8192];
System.out.println(substring+":"+Integer.valueOf(substring2)*size+":"+(Integer.valueOf(substring2)*size+contentLength % size));
System.out.println(111111111);
for (int i = 0; i <= contentLength % size / 8192; i++) {
int read = inputStream.read(byts);
if(read!=-1){
fileOutputStream.write(byts, 0, read);
}
}
}
countDownLatch.countDown();
} finally {
if (inputStream != null) {
inputStream.close();
}
if (fileOutputStream != null) {
fileOutputStream.close();
}
}
}
public void fileConnect() throws IOException {
if (list.size() <= 1) {
return;
}
BufferedOutputStream bufferedOutputStream = null;
try {
bufferedOutputStream = new BufferedOutputStream(
new FileOutputStream(list.get(0).substring(0, list.get(0).length() - 1)));
for (String string : list) {
BufferedInputStream bufferedInputStream = null;
try {
bufferedInputStream = new BufferedInputStream(new FileInputStream(string));
int len = 0;
while ((len = bufferedInputStream.read()) != -1) {
bufferedOutputStream.write(len);
}
} finally {
if (bufferedInputStream != null) {
bufferedInputStream.close();
}
}
}
} finally {
if (bufferedOutputStream != null) {
bufferedOutputStream.close();
}
}
}
}
