使用go语言绕过page cache读写文件

有时候，我们希望我们的数据不通过page cache的缓冲直接落盘。go语言里，用参数DIRECT打开文件可以实现这一点要求。

但这样做有一个硬性的要求，就是在读写的时候，对应的数据在内存中的地址一定要满足512对齐，即首地址的2进制形式中后面至少要有9个0结尾，且数据长度为512字节的整数倍，否则读写会失败。

我们用go语言中的切片slice来验证这件事。

首先我们建立一个go语言的切片并随便赋一些值：

buf := make([]byte, 8192)
for i := 0; i < 20; i++ {
	buf[i] = byte(i)
}

我们首先尝试一下正常的读写文件过程，先不使用DIRECT参数绕开page cache。

func writeWithoutAlignmentWithoutDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdb",
		os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[4:516]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}

这段代码的运行结果如下：

buffer  0xc42000a2a0
buffer[0]  0xc42005a000
write with buffer  0xc42005a004
write succeed

可以看出，这个切片的地址是0xc42000a2a0，这个切片内数据的首地址是0xc42005a000，这是一个与c语言不同的地方，c语言的数据首地址即为其中数据的首地址，而go语言中，切片的地址和切片内数据首地址是不同的。

我们要写入的数据的首地址是从切片的第5个元素开始，其首地址为0xc42005a004，虽然并没有512对齐，但是由于我们没有尝试绕过page cache，所以根据WriteAt函数的返回值err可以看到，我们的写入操作是成功的。

下面我们尝试一下使用DIRECT参数打开文件，绕过page cache进行写数据操作，其他参数不变。

func writeWithoutAlignmentWithDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdc",
		os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[4:516]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}

这段代码运行后，我们可以看到如下结果。

buffer  0xc42000a2e0
buffer[0]  0xc42005a000
write with buffer  0xc42005a004
write error  write /dev/sdc: invalid argument

看到了write error，WriteAt函数这次的返回值给出的并不是nil了，我们的写入操作失败了，其返回值返回了一个不可理解的invalid argument（非法参数）。

but我们的参数毫无问题啊！下面我们尝试一下把要写入的数据改为512对齐。

func writeWithAlignmentWithDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdd",
		os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[512 : 512+512]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}

这段代码运行后，结果如下。

white with alignment and DIRECT:
buffer  0xc42000a340
buffer[0]  0xc42005a000
write with buffer  0xc42005a200
write succeed

我们的写操作成功了！而这段代码与上次未成功的不同之处只有一个，那就是将要写入数据的首地址改成了512对齐。

通过这三段go程序，我们很清晰的验证了绕过page cache写文件的条件。

类似的，下面给出验证绕过page cache读文件也需要512对齐条件的代码。

func readWithoutAlignmentWithoutDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdb", os.O_RDONLY, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[2:514]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}
func readWithoutAlignmentWithDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdc", os.O_RDONLY|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[2:514]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}
func readWithAlignmentWithDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdd", os.O_RDONLY|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[512 : 512+512]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}

这三个函数的运行结果分如下。

read with buffer  0xc42005a002
read succeed [4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]

　　

read with buffer  0xc42005a002
read error  read /dev/sdc: invalid argument

　　

read with buffer  0xc42005a200
read succeed [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]

可以看出，由于最初我们将切片的前20位分别赋值为0-20，其他位赋值默认值为0，第一个写入函数将buf[4:516]写入到/dev/sdb中，第二个写入函数写入失败，第三个写入函数将buf[512 : 512+512]写入到/dev/sdd，所以根据读取结果可以看出，我们的读取函数也是ok的。

最后，给出整段测试程序的完整代码。

package main

import (
	"fmt"
	"os"
	"syscall"
	"unsafe"
)

func main() {
	buf := make([]byte, 8192)
	for i := 0; i < 20; i++ {
		buf[i] = byte(i)
	}
	fmt.Println("----------------------------------------")
	fmt.Println("white without alignment and DIRECT:")
	writeWithoutAlignmentWithoutDIRECT(buf)
	fmt.Println("----------------------------------------")
	fmt.Println("white without alignment but with DIRECT:")
	writeWithoutAlignmentWithDIRECT(buf)
	fmt.Println("----------------------------------------")
	fmt.Println("white with alignment and DIRECT:")
	writeWithAlignmentWithDIRECT(buf)
	fmt.Println("----------------------------------------")
	fmt.Println("read without alignment and DIRECT:")
	readWithoutAlignmentWithoutDIRECT(buf)
	fmt.Println("----------------------------------------")
	fmt.Println("read without alignment but with DIRECT:")
	readWithoutAlignmentWithDIRECT(buf)
	fmt.Println("----------------------------------------")
	fmt.Println("read with alignment and DIRECT:")
	readWithAlignmentWithDIRECT(buf)
}

func writeWithoutAlignmentWithoutDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdb",
		os.O_WRONLY|os.O_CREATE, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[4:516]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}
func writeWithoutAlignmentWithDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdc",
		os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[4:516]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}
func writeWithAlignmentWithDIRECT(buf []byte) {
	// open file
	file, err := os.OpenFile("/dev/sdd",
		os.O_WRONLY|os.O_CREATE|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred with file opening or creation\n")
		return
	}
	defer file.Close()

	// write file
	fmt.Println("buffer ", unsafe.Pointer(&buf))
	fmt.Println("buffer[0] ", unsafe.Pointer(&buf[0]))
	buf2 := buf[512 : 512+512]
	fmt.Println("write with buffer ", unsafe.Pointer(&buf2[0]))

	_, err = file.WriteAt(buf2, 512)
	if err != nil {
		fmt.Println("write error ", err)
	} else {
		fmt.Println("write succeed")
	}
}

func readWithoutAlignmentWithoutDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdb", os.O_RDONLY, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[2:514]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}
func readWithoutAlignmentWithDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdc", os.O_RDONLY|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[2:514]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}
func readWithAlignmentWithDIRECT(buf []byte) {
	// read file
	file, err := os.OpenFile("/dev/sdd", os.O_RDONLY|syscall.O_DIRECT, 0666)
	if err != nil {
		fmt.Printf("An error occurred whit file ipening.\n")
		return
	}
	defer file.Close()

	buf = buf[512 : 512+512]
	fmt.Println("read with buffer ", unsafe.Pointer(&buf[0]))

	_, err = file.ReadAt(buf, 512)
	if err != nil {
		fmt.Println("read error ", err)
	} else {
		fmt.Println("read succeed", buf)
	}
}