java网络编程总结
OSI网络七层模型
为了不同计算机厂商生产的电脑能够通信，以便在更大的范围内建立计算机网络，有必要建立一个国际范围内的网络体系结构标准，也就有了OSI网络七层模型
TCP是一个重要的传输层协议，提供面向连接、可靠、有序的字节流传输服务，在传输数据前必须先建立tcp连接，传输报文如下：
tcp三次握手和四次挥手
UDP协议提供无连接、不可靠的数据包尽力传输的服务
BIO阻塞式网络编程
bio网络编程基于socket和io，先看一个简单的client程序和server端程序
package com.example.test.BIO; import java.io.*; import java.net.Socket; import java.util.Scanner; /** * @author hehang on 2019-05-24 * @descriptionasd */ public class BioClient { public static void main(String[] args) throws IOException { Socket socket = new Socket("localhost",8080); OutputStream outputStream = socket.getOutputStream(); BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(outputStream,"utf-8")); Scanner scanner = new Scanner(System.in); System.out.println("请输入："); String msg = scanner.nextLine(); bufferedWriter.write(msg); scanner.close(); socket.close(); } }
package com.example.test.BIO; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.ServerSocket; import java.net.Socket; /** * @author hehang on 2019-05-24 * @descriptionasd */ public class BioServer { public static void main(String[] args) throws Exception { ServerSocket serverSocket = new ServerSocket(8080); System.out.println("server启动"); while(!serverSocket.isClosed()){ Socket socket = serverSocket.accept(); try { InputStream in = socket.getInputStream(); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(in)); String msg = null; while ((msg = bufferedReader.readLine()) != null) { if (msg.length() == 0) { break; } System.out.println(msg); } System.out.println("收到消息，来自：" + socket.toString()); } catch (IOException e) { e.printStackTrace(); } finally { socket.close(); } } serverSocket.close(); } }
由于serverSocket.accept()及bufferedReader.readLine()都是阻塞的方法，因此server端是一个单线程的，不能满足多个client同时请求，改造server端，使其支持多线程
package com.example.test.BIO; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.ServerSocket; import java.net.Socket; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** * @author hehang on 2019-05-24 * @descriptionasd */ public class BioServer2 { private static ExecutorService threadPool = Executors.newCachedThreadPool(); public static void main(String[] args) throws Exception { ServerSocket serverSocket = new ServerSocket(8080); System.out.println("server启动"); while(!serverSocket.isClosed()){ Socket socket = serverSocket.accept(); System.out.println("收到新的请求" + socket.toString()); threadPool.execute(()->{ try { InputStream in = socket.getInputStream(); String msg = null; while ((msg = bufferedReader.readLine()) != null) { if (msg.length() == 0) { break; } System.out.println(msg); } System.out.println("收到消息，来自：" + socket.toString()); } catch (IOException e) { e.printStackTrace(); } finally { try { socket.close(); } catch (IOException e) { e.printStackTrace(); } } }); } serverSocket.close(); } }
此时sever端的并发数就受限于线程池的大小，那么浏览器和我们server端如何交互呢，此时就需要对http协议有基础的认识
http请求数据包解析
http响应数据包解析
http响应码状态
对现有的server端程序改造，使其支持http请求
package com.example.test.BIO; import java.io.*; import java.net.ServerSocket; import java.net.Socket; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; /** * @author hehang on 2019-05-24 * @descriptionasd */ public class BioServer3 { private static ExecutorService threadPool = Executors.newCachedThreadPool(); public static void main(String[] args) throws Exception { ServerSocket serverSocket = new ServerSocket(8080); System.out.println("server启动"); while(!serverSocket.isClosed()){ Socket socket = serverSocket.accept(); System.out.println("收到新的请求" + socket.toString()); threadPool.execute(()->{ try { InputStream in = socket.getInputStream(); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(in,"utf-8")); String msg = null; while ((msg = bufferedReader.readLine()) != null) { if (msg.length() == 0) { break; } System.out.println(msg); } System.out.println("收到消息，来自：" + socket.toString()); OutputStream outputStream = socket.getOutputStream(); outputStream.write("HTTP/1.1 200 OK\r\n".getBytes()); outputStream.write("Content-Length: 11\r\n\r\n".getBytes()); outputStream.write("Hello World".getBytes()); outputStream.flush(); bufferedReader.close(); outputStream.close(); } catch (IOException e) { e.printStackTrace(); } finally { try { socket.close(); } catch (IOException e) { e.printStackTrace(); } } }); } serverSocket.close(); } }
改造client端，使其能够接受server端的响应
package com.example.test.BIO; import java.io.*; import java.net.Socket; import java.util.Scanner; /** * @author hehang on 2019-05-24 * @descriptionasd */ public class BioClient { public static void main(String[] args) throws IOException { Socket socket = new Socket("localhost",8080); OutputStream outputStream = socket.getOutputStream(); BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(outputStream,"utf-8")); Scanner scanner = new Scanner(System.in); System.out.println("请输入："); String msg = scanner.nextLine(); bufferedWriter.write(msg); bufferedWriter.flush(); socket.shutdownOutput(); InputStream inputStream =socket.getInputStream(); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); String returnMsg = null; while((returnMsg = bufferedReader.readLine())!=null){ if(returnMsg.length()==0){ break; } System.out.println(returnMsg); } scanner.close(); socket.close(); } }
阻塞式编程相关概念
NIO网络编程
从java1.4开始，出现了新的JAVA IO非阻塞API，NIO有三个核心概念：Buffer缓冲区、Channel通道和Selector选择器
Buffer缓冲区
demo如下
package com.example.test.NIO; import java.nio.ByteBuffer; /** * @author hehang on 2019-05-24 * @descriptionsdf */ public class BufferDemo { public static void main(String[] args) { ByteBuffer byteBuffer = ByteBuffer.allocate(4); System.out.println(String.format("初始化容量：%s,position位置：%s,limit限制：%s", byteBuffer.capacity(),byteBuffer.position(),byteBuffer.limit())); byteBuffer.put((byte) 1); byteBuffer.put((byte) 2); byteBuffer.put((byte) 3); System.out.println(String.format("初始化容量：%s,position位置：%s,limit限制：%s", byteBuffer.capacity(),byteBuffer.position(),byteBuffer.limit())); System.out.println("开始读取"); //切换为读模式 byteBuffer.flip(); byte a = byteBuffer.get(); System.out.println(a); byte c = byteBuffer.get(); System.out.println(c); System.out.println(String.format("初始化容量：%s,position位置：%s,limit限制：%s", byteBuffer.capacity(),byteBuffer.position(),byteBuffer.limit())); //清除已读取的数据，转为写模式 byteBuffer.compact(); System.out.println(String.format("初始化容量：%s,position位置：%s,limit限制：%s", byteBuffer.capacity(),byteBuffer.position(),byteBuffer.limit())); byteBuffer.put((byte) 3); byteBuffer.put((byte) 4); byteBuffer.put((byte) 5); System.out.println(String.format("最终的情况，capacity容量：%s, position位置：%s, limit限制：%s", byteBuffer.capacity(), byteBuffer.position(), byteBuffer.limit())); } }
package com.example.test.NIO; import java.nio.ByteBuffer; /** * @author hehang on 2019-05-24 * @descriptionasd */ public class DerictBufferDemo { public static void main(String[] args) { ByteBuffer byteBuffer = ByteBuffer.allocateDirect(4); System.out.println(String.format("初始化容量：%s,position位置：%s,limit限制：%s", byteBuffer.capacity(),byteBuffer.position(),byteBuffer.limit())); byteBuffer.put((byte) 1); byteBuffer.put((byte) 2); byteBuffer.put((byte) 3); System.out.println(String.format("初始化容量：%s,position位置：%s,limit限制：%s", byteBuffer.capacity(),byteBuffer.position(),byteBuffer.limit())); System.out.println("开始读取"); //切换为读模式 byteBuffer.flip(); byte a = byteBuffer.get(); System.out.println(a); byte c = byteBuffer.get(); System.out.println(c); System.out.println(String.format("初始化容量：%s,position位置：%s,limit限制：%s", byteBuffer.capacity(),byteBuffer.position(),byteBuffer.limit())); //清除已读取的数据，转为写模式 byteBuffer.compact(); System.out.println(String.format("初始化容量：%s,position位置：%s,limit限制：%s", byteBuffer.capacity(),byteBuffer.position(),byteBuffer.limit())); byteBuffer.put((byte) 3); byteBuffer.put((byte) 4); byteBuffer.put((byte) 5); System.out.println(String.format("最终的情况，capacity容量：%s, position位置：%s, limit限制：%s", byteBuffer.capacity(), byteBuffer.position(), byteBuffer.limit())); } }
Channel通道
利用nio构建简单的client程序和server端程序
package com.example.test.NIO; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.SocketChannel; import java.util.Scanner; public class NioClient { public static void main(String[] args) throws IOException { SocketChannel socketChannel =SocketChannel.open(); socketChannel.configureBlocking(false); socketChannel.connect(new InetSocketAddress("127.0.0.1",8080)); while(!socketChannel.finishConnect()){ //没连接上则一直连接 Thread.yield(); } Scanner scanner = new Scanner(System.in); System.out.println("请输入："); String msg = scanner.nextLine(); ByteBuffer byteBuffer = ByteBuffer.wrap(msg.getBytes()); while(byteBuffer.hasRemaining()){ socketChannel.write(byteBuffer); } System.out.println("收到服务端的响应："); ByteBuffer returnBuffer = ByteBuffer.allocate(1024); while(socketChannel.isOpen()&& socketChannel.read(returnBuffer)!=-1){ if(returnBuffer.position()>0){ break; } } returnBuffer.flip(); System.out.println(returnBuffer.limit()); byte[] content = new byte[returnBuffer.limit()]; returnBuffer.get(content); System.out.println(new String(content)); scanner.close(); socketChannel.close(); } }
package com.example.test.NIO; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; public class NioServer { public static void main(String[] args) throws IOException { //创建网络服务 ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); serverSocketChannel.configureBlocking(false);//设置为非阻塞 serverSocketChannel.socket().bind(new InetSocketAddress(8080));//绑定端口 System.out.println("启动成功"); while(true){ SocketChannel socketChannel = serverSocketChannel.accept();//获取tcp连接通道 if(socketChannel!=null){ System.out.println("收到新连接" + socketChannel.getRemoteAddress()); socketChannel.configureBlocking(false);//设置为非阻塞 ByteBuffer requestByte = ByteBuffer.allocate(1024); while(socketChannel.isOpen()&& socketChannel.read(requestByte)!=-1){ //长连接情况下，需要手动判断数据是否读取完毕，此处做一个简单判断，position>0表示读完 if(requestByte.position()>0){ break; } } if(requestByte.position()==0){ continue;//没数据则不继续后续处理 } requestByte.flip(); byte[] content = new byte[requestByte.limit()]; requestByte.get(content); System.out.println(new String(content)); System.out.println("收到数据，来自" + socketChannel.getRemoteAddress()); String response = "HTTP/1.1 200 OK \r\n" + "Content_Length: 11 \r\n\r\n" + "Hello Word"; ByteBuffer returnByteBuffer = ByteBuffer.wrap(response.getBytes()); while(returnByteBuffer.hasRemaining()){ socketChannel.write(returnByteBuffer); } } } } }
此时发现由于循环读取ByteBuffer中的数据，server只能同时接受一个请求，因此需要对server进行改造
package com.example.test.NIO; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.util.ArrayList; import java.util.Iterator; /** * @author hehang on 2019-05-24 * @descriptionasd */ public class NioServer1 { private static ArrayList<SocketChannel> socketChannels = new ArrayList<>(); public static void main(String[] args) throws IOException { //创建网络服务 ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); serverSocketChannel.configureBlocking(false);//设置为非阻塞 serverSocketChannel.socket().bind(new InetSocketAddress(8080));//绑定端口 System.out.println("启动成功"); while(true){ SocketChannel socketChannel = serverSocketChannel.accept();//获取tcp连接通道 if(socketChannel!=null) { System.out.println("收到新连接" + socketChannel.getRemoteAddress()); socketChannel.configureBlocking(false);//设置为非阻塞 socketChannels.add(socketChannel); }else{ //没有新连接的时候，就去处理现有连接的数据 Iterator<SocketChannel> iterator = socketChannels.iterator(); while(iterator.hasNext()){ SocketChannel socketChannel1 = iterator.next(); ByteBuffer requestByte = ByteBuffer.allocate(1024); if(socketChannel1.read(requestByte)==0){ continue; } while(socketChannel1.isOpen()&& socketChannel1.read(requestByte)!=-1){ //长连接情况下，需要手动判断数据是否读取完毕，此处做一个简单判断，position>0表示读完 if(requestByte.position()>0){ break; } } requestByte.flip(); byte[] content = new byte[requestByte.limit()]; requestByte.get(content); System.out.println(new String(content)); System.out.println("收到数据，来自" + socketChannel1.getRemoteAddress()); String response = "HTTP/1.1 200 OK \r\n" + "Content_Length: 11 \r\n\r\n" + "Hello Word"; ByteBuffer returnByteBuffer = ByteBuffer.wrap(response.getBytes()); while(returnByteBuffer.hasRemaining()){ socketChannel1.write(returnByteBuffer); } iterator.remove(); } } } } }
利用ByteBuffer和Channel已经可以进行NIO编程，但是这样却很不方便，JDK为我们提供一个新的API：Selector
改造server端程序如下
package com.example.test.NIO; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.util.Iterator; import java.util.Set; public class NioServer2 { public static void main(String[] args) throws IOException { //创建网络服务 ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); serverSocketChannel.configureBlocking(false);//设置为非阻塞 //构建选择器，将serverSocketChannel注册上去，并且selector对serverSocketChannel上面的accept事件感兴趣 Selector selector = Selector.open(); SelectionKey selectionKey = serverSocketChannel.register(selector,0,serverSocketChannel); selectionKey.interestOps(SelectionKey.OP_ACCEPT); serverSocketChannel.socket().bind(new InetSocketAddress(8080));//绑定端口 System.out.println("启动成功"); while(true){ selector.select();//该方法会阻塞，直到有事件通知才会返回 //获取事件 Set<SelectionKey> selectionKeys = selector.selectedKeys(); Iterator<SelectionKey> iter = selectionKeys.iterator(); while(iter.hasNext()){ // System.out.println("-------------"); SelectionKey key = iter.next(); iter.remove(); if(key.isAcceptable()){ ServerSocketChannel server = (ServerSocketChannel) key.attachment(); // 将拿到的客户端连接通道,注册到selector上面 SocketChannel clientSocketChannel = server.accept(); // mainReactor 轮询accept clientSocketChannel.configureBlocking(false); clientSocketChannel.register(selector, SelectionKey.OP_READ, clientSocketChannel); System.out.println("收到新连接 : " + clientSocketChannel.getRemoteAddress()); } if(key.isReadable()){ SocketChannel socketChannel = (SocketChannel) key.attachment(); try { ByteBuffer requestBuffer = ByteBuffer.allocate(1024); while (socketChannel.isOpen() && socketChannel.read(requestBuffer) != -1) { // 长连接情况下,需要手动判断数据有没有读取结束 (此处做一个简单的判断: 超过0字节就认为请求结束了) if (requestBuffer.position() > 0) break; } if(requestBuffer.position() == 0) continue; // 如果没数据了, 则不继续后面的处理 requestBuffer.flip(); byte[] content = new byte[requestBuffer.limit()]; requestBuffer.get(content); System.out.println(new String(content)); System.out.println("收到数据,来自：" + socketChannel.getRemoteAddress()); // TODO 业务操作 数据库 接口调用等等 // 响应结果 200 String response = "HTTP/1.1 200 OK\r\n" + "Content-Length: 11\r\n\r\n" + "Hello World"; ByteBuffer buffer = ByteBuffer.wrap(response.getBytes()); while (buffer.hasRemaining()) { socketChannel.write(buffer); } } catch (IOException e) { // e.printStackTrace(); key.cancel(); // 取消事件订阅 } } } selector.selectNow(); } } }
在上面的server端程序中，一个selector监听所有事件,一个线程处理所有请求事件，难以利用现代服务器多核特性，会成为性能瓶颈!，因此实际开发中要有多线程的运用，对此，JDK作者给出了NIO与多线程的结合使用：《Scalable IO in java》
下面是多Reactor服务端实现
package com.example.test.NIO; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.*; import java.util.Iterator; import java.util.Random; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; /** * @author hehang on 2020-02-02 * @description */ public class NioServer3 { /** 处理业务操作的线程 */ private static ExecutorService workPool = Executors.newCachedThreadPool(); /** * 封装了selector.select()等事件轮询的代码 */ abstract class ReactorThread extends Thread { Selector selector; LinkedBlockingQueue<Runnable> taskQueue = new LinkedBlockingQueue<>(); /** * Selector监听到有事件后,调用这个方法 */ public abstract void handler(SelectableChannel channel) throws Exception; private ReactorThread() throws IOException { selector = Selector.open(); } volatile boolean running = false; @Override public void run() { // 轮询Selector事件 while (running) { try { // 执行队列中的任务 Runnable task; while ((task = taskQueue.poll()) != null) { task.run(); } selector.select(1000); // 获取查询结果 Set<SelectionKey> selected = selector.selectedKeys(); // 遍历查询结果 Iterator<SelectionKey> iter = selected.iterator(); while (iter.hasNext()) { // 被封装的查询结果 SelectionKey key = iter.next(); iter.remove(); int readyOps = key.readyOps(); // 关注 Read 和 Accept两个事件 if ((readyOps & (SelectionKey.OP_READ | SelectionKey.OP_ACCEPT)) != 0 || readyOps == 0) { try { SelectableChannel channel = (SelectableChannel) key.attachment(); channel.configureBlocking(false); handler(channel); if (!channel.isOpen()) { key.cancel(); // 如果关闭了,就取消这个KEY的订阅 } } catch (Exception ex) { key.cancel(); // 如果有异常,就取消这个KEY的订阅 } } } selector.selectNow(); } catch (IOException e) { e.printStackTrace(); } } } private SelectionKey register(SelectableChannel channel) throws Exception { // 为什么register要以任务提交的形式，让reactor线程去处理？ // 因为线程在执行channel注册到selector的过程中，会和调用selector.select()方法的线程争用同一把锁 // 而select()方法实在eventLoop中通过while循环调用的，争抢的可能性很高，为了让register能更快的执行，就放到同一个线程来处理 FutureTask<SelectionKey> futureTask = new FutureTask<>(() -> channel.register(selector, 0, channel)); taskQueue.add(futureTask); return futureTask.get(); } private void doStart() { if (!running) { running = true; start(); } } } private ServerSocketChannel serverSocketChannel; // 1、创建多个线程 - accept处理reactor线程 (accept线程) private ReactorThread[] mainReactorThreads = new ReactorThread[1]; // 2、创建多个线程 - io处理reactor线程 (I/O线程) private ReactorThread[] subReactorThreads = new ReactorThread[8]; /** * 初始化线程组 */ private void newGroup() throws IOException { // 创建IO线程,负责处理客户端连接以后socketChannel的IO读写 for (int i = 0; i < subReactorThreads.length; i++) { subReactorThreads[i] = new ReactorThread() { @Override public void handler(SelectableChannel channel) throws IOException { // work线程只负责处理IO处理，不处理accept事件 SocketChannel ch = (SocketChannel) channel; ByteBuffer requestBuffer = ByteBuffer.allocate(1024); while (ch.isOpen() && ch.read(requestBuffer) != -1) { // 长连接情况下,需要手动判断数据有没有读取结束 (此处做一个简单的判断: 超过0字节就认为请求结束了) if (requestBuffer.position() > 0) break; } if (requestBuffer.position() == 0) return; // 如果没数据了, 则不继续后面的处理 requestBuffer.flip(); byte[] content = new byte[requestBuffer.limit()]; requestBuffer.get(content); System.out.println(new String(content)); System.out.println(Thread.currentThread().getName() + "收到数据,来自：" + ch.getRemoteAddress()); // TODO 业务操作 数据库、接口... workPool.submit(() -> { }); // 响应结果 200 String response = "HTTP/1.1 200 OK\r\n" + "Content-Length: 11\r\n\r\n" + "Hello World"; ByteBuffer buffer = ByteBuffer.wrap(response.getBytes()); while (buffer.hasRemaining()) { ch.write(buffer); } } }; } // 创建mainReactor线程, 只负责处理serverSocketChannel for (int i = 0; i < mainReactorThreads.length; i++) { mainReactorThreads[i] = new ReactorThread() { AtomicInteger incr = new AtomicInteger(0); @Override public void handler(SelectableChannel channel) throws Exception { // 只做请求分发，不做具体的数据读取 ServerSocketChannel ch = (ServerSocketChannel) channel; SocketChannel socketChannel = ch.accept(); socketChannel.configureBlocking(false); // 收到连接建立的通知之后，分发给I/O线程继续去读取数据 int index = incr.getAndIncrement() % subReactorThreads.length; ReactorThread workEventLoop = subReactorThreads[index]; workEventLoop.doStart(); SelectionKey selectionKey = workEventLoop.register(socketChannel); selectionKey.interestOps(SelectionKey.OP_READ); System.out.println(Thread.currentThread().getName() + "收到新连接 : " + socketChannel.getRemoteAddress()); } }; } } /** * 初始化channel,并且绑定一个eventLoop线程 * * @throws IOException IO异常 */ private void initAndRegister() throws Exception { // 1、 创建ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); serverSocketChannel.configureBlocking(false); // 2、 将serverSocketChannel注册到selector int index = new Random().nextInt(mainReactorThreads.length); mainReactorThreads[index].doStart(); SelectionKey selectionKey = mainReactorThreads[index].register(serverSocketChannel); selectionKey.interestOps(SelectionKey.OP_ACCEPT); } /** * 绑定端口 * * @throws IOException IO异常 */ private void bind() throws IOException { // 1、 正式绑定端口，对外服务 serverSocketChannel.bind(new InetSocketAddress(8080)); System.out.println("启动完成，端口8080"); } public static void main(String[] args) throws Exception { NioServer3 nioServer3 = new NioServer3(); nioServer3.newGroup(); // 1、 创建main和sub两组线程 nioServer3.initAndRegister(); // 2、 创建serverSocketChannel，注册到mainReactor线程上的selector上 nioServer3.bind(); // 3、 为serverSocketChannel绑定端口 } }
