chisel 分析3 业务流缓存

　　目前业务使用chisel 代理内网流量到云端Internet节点使用，本地流量从tun接口上读取引流封包走chisel的ssh 隧道到云端，

本次主要看：chisel的server端业务流分析

client 和server 端读写设计以及怎样mux demux

client 收到浏览器的请求后，进过一些列的处理进入piperemote逻辑；

// newMux returns a mux that runs over the given connection.
func newMux(p packetConn) *mux {
    m := &mux{
        conn:             p,
        incomingChannels: make(chan NewChannel, chanSize),
        globalResponses:  make(chan interface{}, 1),
        incomingRequests: make(chan *Request, chanSize),
        errCond:          newCond(),
    }
    if debugMux {
        m.chanList.offset = atomic.AddUint32(&globalOff, 1)
    }

    go m.loop()
    return m
}

sshConn := p.sshTun.getSSH(ctx) 获取当前ssh conn连接

dst, reqs, err := sshConn.OpenChannel("chisel", []byte(p.remote.Remote())) 计算当期请求的channel id，便于在tcp流中找到属于自己的channel

s, r := cio.Pipe(src, dst) 将浏览器src 和remote chisel server dst 相互proxy-pipe-流量转发。

来看看当前怎样实现pipe的

func Pipe(src io.ReadWriteCloser, dst io.ReadWriteCloser) (int64, int64) {
    var sent, received int64
    var wg sync.WaitGroup
    var o sync.Once
    close := func() {
        src.Close()
        dst.Close()
    }
    wg.Add(2)
    go func() {
        received, _ = io.Copy(src, dst)
        o.Do(close)
        wg.Done()
    }()
    go func() {
        sent, _ = io.Copy(dst, src)
        o.Do(close)
        wg.Done()
    }()
    wg.Wait()
    return sent, received
}

调用Io.copy 实现src 和dst之间的流量转发，io.copy可以参考iocopy分析

由于是多个channel 共用一个tcp ssh 流，那当前这个tcp流的packet是属于哪个channel 呢？和Tcp拆包和粘包一样处理即可！

也就是channel 会自己封装一个read write函数，channel的write函数实现粘包组合，最后调用channel的conn 发送报文走tcp传输

channel的io.ReadWriteCloser 接口实现

channel的初始化

func (m *mux) newChannel(chanType string, direction channelDirection, extraData []byte) *channel {
    ch := &channel{
        remoteWin:        window{Cond: newCond()},
        myWindow:         channelWindowSize,
        pending:          newBuffer(),// channel read 中间缓冲区
        extPending:       newBuffer(),
        direction:        direction,
        incomingRequests: make(chan *Request, chanSize),
        msg:              make(chan interface{}, chanSize),
        chanType:         chanType,
        extraData:        extraData,
        mux:              m,
        packetPool:       make(map[uint32][]byte),
    }
    fmt.Printf("addr:%p newChannel:%#v\n\n", ch, *ch)
    ch.localId = m.chanList.add(ch)
    fmt.Printf("addr:%p newChannel:%#v  ---> localid:%d \n\n", ch, *ch, ch.localId)
    return ch
}

channel的Write接口

func (ch *channel) Write(data []byte) (int, error) {
    if !ch.decided {
        return 0, errUndecided
    }
    return ch.WriteExtended(data, 0)
}

调用WriteExtended 向stream 写入data，

ch.remoteWin.reserve(space) 保留当前channel 的窗口space=(len(data),ch.maxRemotePayload),也就是在channel 应用层也有一个类似与tcp层的窗口（发送窗口以及接收窗口）

也就是当前reserve reserves win from the available window capacity.If no capacity remains, reserve will block. reserve may / return less than requested.

packet = make([]byte, want) 当前ch 的packet

调用for len(data) > 0 阻塞式的发送数据

// WriteExtended writes data to a specific extended stream. These streams are
// used, for example, for stderr.
func (ch *channel) WriteExtended(data []byte, extendedCode uint32) (n int, err error) {
    if ch.sentEOF {
        return 0, io.EOF
    }
    // 1 byte message type, 4 bytes remoteId, 4 bytes data length
    opCode := byte(msgChannelData)
    headerLength := uint32(9)
    if extendedCode > 0 {
        headerLength += 4
        opCode = msgChannelExtendedData
    }

    ch.writeMu.Lock()
    packet := ch.packetPool[extendedCode]
    // We don't remove the buffer from packetPool, so
    // WriteExtended calls from different goroutines will be
    // flagged as errors by the race detector.
    ch.writeMu.Unlock()

    for len(data) > 0 {
        space := min(ch.maxRemotePayload, len(data))
      if space, err = ch.remoteWin.reserve(space); err != nil {
            return n, err
        }
        if want := headerLength + space; uint32(cap(packet)) < want {
            packet = make([]byte, want)
        } else {
            packet = packet[:want]
        }

        todo := data[:space]

        packet[0] = opCode
        binary.BigEndian.PutUint32(packet[1:], ch.remoteId)
        if extendedCode > 0 {
            binary.BigEndian.PutUint32(packet[5:], uint32(extendedCode))
        }
        binary.BigEndian.PutUint32(packet[headerLength-4:], uint32(len(todo)))
        copy(packet[headerLength:], todo)
        if err = ch.writePacket(packet); err != nil {
            return n, err
        }

        n += len(todo)
        data = data[len(todo):]
    }

    ch.writeMu.Lock()
    ch.packetPool[extendedCode] = packet// 保留当前ch的中间缓存空间，避免重复make 申请内存空间释放
    ch.writeMu.Unlock()

    return n, err
}

最后调用mux的conn handshakeTransport 流接口发送数据

// writePacket sends a packet. If the packet is a channel close, it updates
// sentClose. This method takes the lock c.writeMu.
func (ch *channel) writePacket(packet []byte) error {
    ch.writeMu.Lock()
    if ch.sentClose {
        ch.writeMu.Unlock()
        return io.EOF
    }
    ch.sentClose = (packet[0] == msgChannelClose)
    err := ch.mux.conn.writePacket(packet)
    ch.writeMu.Unlock()
    return err
}

根据初始化逻辑

// NewClientConn establishes an authenticated SSH connection using c
// as the underlying transport.  The Request and NewChannel channels
// must be serviced or the connection will hang.
func NewClientConn(c net.Conn, addr string, config *ClientConfig) (Conn, <-chan NewChannel, <-chan *Request, error) {
    
    conn := &connection{
        sshConn: sshConn{conn: c, user: fullConf.User},
    }

    if err := conn.clientHandshake(addr, &fullConf); err != nil {
    
    conn.mux = newMux(conn.transport)----> conn.mux.conn = conn.transport
    return conn, conn.mux.incomingChannels, conn.mux.incomingRequests, nil
}


也就是
c.transport = newClientTransport(
        newTransport(c.sshConn.conn, config.Rand, true /* is client */),
        c.clientVersion, c.serverVersion, config, dialAddress, c.sshConn.RemoteAddr())


c.transport <-----newClientTransport<----newHandshakeTransport<---newTransport

keyconn = transporp  ---> t.conn = keyconn  --->  c.transport = t

func newTransport(rwc io.ReadWriteCloser, rand io.Reader, isClient bool) *transport {
    t := &transport{
        bufReader: bufio.NewReader(rwc),//c.sshConn.conn
        bufWriter: bufio.NewWriter(rwc),//c.sshConn.conn
　　　　  rand: rand, 
　　　　　reader: connectionState{

packetCipher: &streamPacketCipher{cipher: noneCipher{}},

pendingKeyChange: make(chan packetCipher, 1),

writer: connectionState{

packetCipher: &streamPacketCipher{cipher: noneCipher{}},

pendingKeyChange: make(chan packetCipher, 1),

}
}

func newClientTransport(conn keyingTransport, clientVersion, serverVersion []byte, config *ClientConfig, dialAddr string, addr net.Addr) *handshakeTransport {
	t := newHandshakeTransport(conn, &config.Config, clientVersion, serverVersion)
	t.dialAddress = dialAddr
	t.remoteAddr = addr
	t.hostKeyCallback = config.HostKeyCallback
	t.bannerCallback = config.BannerCallback
	if config.HostKeyAlgorithms != nil {
		t.hostKeyAlgorithms = config.HostKeyAlgorithms
	} else {
		t.hostKeyAlgorithms = supportedHostKeyAlgos
	}
	go t.readLoop()
	go t.kexLoop()
	return t
}

调用 handshakeTransport的接口

func (t *handshakeTransport) writePacket(p []byte) error {
    
    t.mu.Lock()
    defer t.mu.Unlock()
    if t.writeError != nil {
        return t.writeError
    }

    if t.sentInitMsg != nil {
        // Copy the packet so the writer can reuse the buffer.
        cp := make([]byte, len(p))
        copy(cp, p)
        t.pendingPackets = append(t.pendingPackets, cp)
        return nil
    }

    if t.writeBytesLeft > 0 {
        t.writeBytesLeft -= int64(len(p))
    } else {
        t.requestKeyExchange()
    }

    if t.writePacketsLeft > 0 {
        t.writePacketsLeft--
    } else {
        t.requestKeyExchange()
    }

    if err := t.pushPacket(p); err != nil {
        t.writeError = err
    }

    return nil
}

**调用HandshakeTransport writePacket**

func (t *handshakeTransport) pushPacket(p []byte) error {
    if debugHandshake {
        t.printPacket(p, true)
    }
    return t.conn.writePacket(p)
}

调用newTransport transport 的 writePacket

func (t *transport) writePacket(packet []byte) error {
    if debugTransport {
        t.printPacket(packet, true)
    }
    return t.writer.writePacket(t.bufWriter, t.rand, packet)
}

调用 (s *connectionState) writePacket(w *bufio.Writer, rand io.Reader, packet []byte) error

最后调用cipher的 writeCipherPacket--->bufWriter: bufio.NewWriter(rwc),//c.sshConn.conn的write

从面向过程就是这样---->

channel的Read接口

func (ch *channel) Read(data []byte) (int, error) {
    if !ch.decided {
        return 0, errUndecided
    }
    return ch.ReadExtended(data, 0)
}

调用ReadExtended向stream 读取data，

n, err = c.pending.Read(data)

c.adjustWindow(uint32(n)) 读取了n byte后告知对端， ch 接收窗口增大n

func (m *mux) newChannel(chanType string, direction channelDirection, extraData []byte) *channel {
    ch := &channel{
        remoteWin:        window{Cond: newCond()},
        myWindow:         channelWindowSize,
        pending:          newBuffer(),
        extPending:       newBuffer(),
        direction:        direction,
        incomingRequests: make(chan *Request, chanSize),
        msg:              make(chan interface{}, chanSize),
        chanType:         chanType,
        extraData:        extraData,
        mux:              m,
        packetPool:       make(map[uint32][]byte),
    }
    return ch
}

再根据mux 创建channel的时候，会给ch创建reading pending 读缓存， tcp流数据写入到此pending ，此时read就会有数据

那么pending: newBuffer(),的数据是什么时候写入的呢？

// newMux returns a mux that runs over the given connection.
func newMux(p packetConn) *mux {
    m := &mux{
        conn:             p,
        incomingChannels: make(chan NewChannel, chanSize),
        globalResponses:  make(chan interface{}, 1),
        incomingRequests: make(chan *Request, chanSize),
        errCond:          newCond(),
    }
    if debugMux {
        m.chanList.offset = atomic.AddUint32(&globalOff, 1)
    }

    go m.loop()
    return m
}

在创建mux的时候，会创建一个loop协程，此协程不停地从mux的ch中读取数据

// loop runs the connection machine. It will process packets until an
// error is encountered. To synchronize on loop exit, use mux.Wait.
func (m *mux) loop() {
    var err error
    for err == nil {
        err = m.onePacket()
    }
----------------------------
}

/ onePacket reads and processes one packet.
func (m *mux) onePacket() error {
    packet, err := m.conn.readPacket()

    switch packet[0] {
    case msgChannelOpen:
        return m.handleChannelOpen(packet)
    case msgGlobalRequest, msgRequestSuccess, msgRequestFailure:
        return m.handleGlobalPacket(packet)
    }

    
    id := binary.BigEndian.Uint32(packet[1:])

    ch := m.chanList.getChan(id)
    
    fmt.Printf("choose ch id:%d process %+#v\n", id, ch)
    return ch.handlePacket(packet)
}


func (ch *channel) handlePacket(packet []byte) error {
    fmt.Printf(" ch handle packet:%d \n", packet[0])
    switch packet[0] {
    case msgChannelData, msgChannelExtendedData:
        return ch.handleData(packet)
    case msgChannelClose:
        ch.sendMessage(channelCloseMsg{PeersID: ch.remoteId})
        ch.mux.chanList.remove(ch.localId)
        ch.close()
        return nil
    case msgChannelEOF:
    ---------------------
        return nil
    }

    decoded, err := decode(packet)
    switch msg := decoded.(type) {
    case *channelOpenFailureMsg:
        case *channelOpenConfirmMsg:
        case *windowAdjustMsg:
       case *channelRequestMsg:
       default:
        ch.msg <- msg
    ---------------------------
    }
    return nil
}

来看packetdate的处理写入pending buf

func (ch *channel) handleData(packet []byte) error {
    headerLen := 9
    isExtendedData := packet[0] == msgChannelExtendedData

    ch.pending.write(data)
    
    return nil
}

仔细分析：m.conn.readPacket() 调用的可是(t *handshakeTransport) readPacket()的接口；然后此接口是从handshakeTransport的incoming chan读取数据。

那么incoming的数据是谁写入的呢？

func (t *handshakeTransport) readPacket() ([]byte, error) {
	p, ok := <-t.incoming
	if !ok {
		return nil, t.readError
	}
	return p, nil
}

在初始化

handshakeTransport的时候会开启一个协程，

func newClientTransport(conn keyingTransport, clientVersion, serverVersion []byte, config *ClientConfig, dialAddr string, addr net.Addr) *handshakeTransport {
	t := newHandshakeTransport(conn, &config.Config, clientVersion, serverVersion)
	-----------------------
	go t.readLoop()
	go t.kexLoop()
	return t
}

go t.readLoop() 此协程从对应的t.conn.readPacket()读取数据也就是到了transport 的 t.reader.readPacket(t.bufReader)接口读取数据

go t.kexLoop()//ssh 的handshake

func (t *handshakeTransport) readLoop() {
	first := true
	for {
		p, err := t.readOnePacket(first)
		first = false
		if err != nil {
			t.readError = err
			close(t.incoming)
			break
		}
		if p[0] == msgIgnore || p[0] == msgDebug {
			continue
		}
		t.incoming <- p
	}

	// Stop writers too.
	t.recordWriteError(t.readError)

	// Unblock the writer should it wait for this.
	close(t.startKex)

	// Don't close t.requestKex; it's also written to from writePacket.
}

posted @ 2024-01-07 14:26 codestacklinuxer 阅读(19) 评论(0) 编辑收藏举报

会员力量，点亮园子希望

刷新页面返回顶部

坐看云起时

乘风好去，长空万里，直下看山河!!! 研究过httpserver、nginx、内核tcpip协议栈源码，内存管理、摄像头-iic-spi等驱动!! 目前瞎搞

chisel 分析3 业务流缓存

client 收到浏览器的请求后，进过一些列的处理进入piperemote逻辑；

channel的io.ReadWriteCloser 接口实现

channel的初始化

channel的Write接口

**调用HandshakeTransport writePacket**

channel的Read接口

坐看云起时

乘风好去，长空万里，直下看山河!!! 研究过httpserver、nginx、内核tcpip协议栈源码，内存管理 、摄像头-iic-spi等驱动!! 目前瞎搞

chisel 分析3 业务流缓存

client 收到浏览器的请求后，进过一些列的处理进入piperemote逻辑；

channel的io.ReadWriteCloser 接口实现

channel的初始化

channel的Write接口

调用HandshakeTransport writePacket

channel的Read接口

乘风好去，长空万里，直下看山河!!! 研究过httpserver、nginx、内核tcpip协议栈源码，内存管理、摄像头-iic-spi等驱动!! 目前瞎搞

**调用HandshakeTransport writePacket**