-----------------------
https://www.jianshu.com/p/964b887da04c
package main
import (
"fmt"
"os"
"syscall"
)
const maxMapSize = 0x8000000000
const maxMmapStep = 1 << 30 // 1GB
func main() {
file, err := os.OpenFile("my.db", os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
panic(err)
}
defer file.Close()
stat, err := os.Stat("my.db")
if err != nil {
panic(err)
}
size, err := mmapSize(int(stat.Size()))
if err != nil {
panic(err)
}
fmt.Println("size:", size)
//// 沒有下面這句，將會報錯
//err = syscall.Ftruncate(int(file.Fd()), 2) //第一種改法, truncate() 這里的2，表示會將參數fd指定的文件大小改為參數length指定的大小
//if err != nil {
// panic(err)
//}
// 第二種改法，對文件執行一下write , 或者WriteAt
file.Write(make([]byte, 10)) // 這里的10， 表示文件大小限定為10了， 后面超多10的寫不進（打開文件時，字符e沒有）。
b, err := syscall.Mmap(int(file.Fd()), 0, 100, syscall.PROT_WRITE|syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
panic(err)
}
for index, bb := range []byte("Hello abcde") {
b[index] = bb
}
err = syscall.Munmap(b)
if err != nil {
panic(err)
}
}
func mmapSize(size int) (int, error) {
// Double the size from 32KB until 1GB.
for i := uint(15); i <= 30; i++ {
if size <= 1<<i {
return 1 << i, nil
}
}
// Verify the requested size is not above the maximum allowed.
if size > maxMapSize {
return 0, fmt.Errorf("mmap too large")
}
// If larger than 1GB then grow by 1GB at a time.
sz := int64(size)
if remainder := sz % int64(maxMmapStep); remainder > 0 {
sz += int64(maxMmapStep) - remainder
}
// Ensure that the mmap size is a multiple of the page size.
// This should always be true since we're incrementing in MBs.
pageSize := int64(os.Getpagesize())
if (sz % pageSize) != 0 {
sz = ((sz / pageSize) + 1) * pageSize
}
// If we've exceeded the max size then only grow up to the max size.
if sz > maxMapSize {
sz = maxMapSize
}
return int(sz), nil
}
另外一個例子：https://gist.github.com/suyash/a19b7f91000b24fde4bc4a015680c611
write.go
package main
import (
"os"
"syscall"
)
func main() {
filename, offset, length := "testgo.out", 10, 20
f, err := os.Create(filename)
if err != nil {
panic(err)
}
f.Write(make([]byte, length))
fd := int(f.Fd())
b, err := syscall.Mmap(fd, 0, length, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
if err != nil {
panic(err)
}
b[offset-1] = 'x'
err = syscall.Munmap(b)
if err != nil {
panic(err)
}
}
reader.go
package main
import (
"fmt"
"os"
"syscall"
)
func main() {
filename, offset := "main.c", 10
f, err := os.Open(filename)
if err != nil {
panic(err)
}
fd := int(f.Fd())
stat, err := f.Stat()
if err != nil {
panic(err)
}
size := int(stat.Size())
b, err := syscall.Mmap(fd, 0, size, syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
panic(err)
}
fmt.Printf("%c\n", b[offset-1])
err = syscall.Munmap(b)
if err != nil {
panic(err)
}
}
reader.c
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
int main () {
const char* filename = "main.c";
int offset = 10;
int fd = open(filename, O_RDONLY);
if (fd == -1) {
printf("cannot open %s\n", filename);
return -1;
}
struct stat sbuf;
if (stat(filename, &sbuf) == -1) {
printf("cannot stat %s\n", filename);
return -1;
}
int size = sbuf.st_size;
char* data = mmap((caddr_t)0, size, PROT_READ, MAP_SHARED, fd, 0);
if (data == -1) {
printf("cannot mmap %s\n", filename);
return -1;
}
printf("byte at offset %d is %c\n", offset, data[offset - 1]);
int err = munmap(data, size);
if (err == -1) {
printf("cannot munmap %s\n", filename);
return -1;
}
return 0;
}
----------------------
最近工作中在研究Hyperledger Fabric區塊鏈開源項目，其中區塊鏈peer節點底層使用了leveldb作為State Database的存儲介質，出於好奇，決定對一些常用的KV存儲做一些研究。
這一次我主要對2種KV存儲的源碼做了分析，一個是BoltDB，這是LMDB的Go語言版本，另一個就是goleveldb。
在閱讀BoltDB項目源碼的過程中，我發現它將持久化文件以只讀模式通過mmap映射到內存空間中，然后通過索引找到內存中key對應的value所指向的空間，然后將這段內存返回給用戶。之前雖然也聽說過內存映射文件技術，但一直沒有實際使用過，因此這一次我決定對mmap做一些嘗試，以下是這次嘗試的過程。
文件寫入
內存映射文件(Memory-mapped file)將一段虛擬內存逐字節對應於一個文件或類文件的資源，使得應用程序處理映射部分如同訪問主存，用於增加I/O性能。Mmap函數存在於Go's syscall package中，它接收一個文件描述符，需要映射的大小(返回的切片容量)以及需要的內存保護和映射類型。
package main import ( "fmt" "os" "syscall" ) const maxMapSize = 0x8000000000 const maxMmapStep = 1 << 30 // 1GB func main() { file, err := os.OpenFile("my.db", os.O_RDWR|os.O_CREATE, 0644) if err != nil { panic(err) } defer file.Close() stat, err := os.Stat("my.db") if err != nil { panic(err) } size, err := mmapSize(int(stat.Size())) if err != nil { panic(err) } b, err := syscall.Mmap(int(file.Fd()), 0, size, syscall.PROT_WRITE|syscall.PROT_READ, syscall.MAP_SHARED) if err != nil { panic(err) } for index, bb := range []byte("Hello world") { b[index] = bb } err = syscall.Munmap(b) if err != nil { panic(err) } } func mmapSize(size int) (int, error) { // Double the size from 32KB until 1GB. for i := uint(15); i <= 30; i++ { if size <= 1<<i { return 1 << i, nil } } // Verify the requested size is not above the maximum allowed. if size > maxMapSize { return 0, fmt.Errorf("mmap too large") } // If larger than 1GB then grow by 1GB at a time. sz := int64(size) if remainder := sz % int64(maxMmapStep); remainder > 0 { sz += int64(maxMmapStep) - remainder } // Ensure that the mmap size is a multiple of the page size. // This should always be true since we're incrementing in MBs. pageSize := int64(os.Getpagesize()) if (sz % pageSize) != 0 { sz = ((sz / pageSize) + 1) * pageSize } // If we've exceeded the max size then only grow up to the max size. if sz > maxMapSize { sz = maxMapSize