core/rawdb: freezer batch write (#23462)
This change is a rewrite of the freezer code. When writing ancient chain data to the freezer, the previous version first encoded each individual item to a temporary buffer, then wrote the buffer. For small item sizes (for example, in the block hash freezer table), this strategy causes a lot of system calls for writing tiny chunks of data. It also allocated a lot of temporary []byte buffers. In the new version, we instead encode multiple items into a re-useable batch buffer, which is then written to the file all at once. This avoids performing a system call for every inserted item. To make the internal batching work, the ancient database API had to be changed. While integrating this new API in BlockChain.InsertReceiptChain, additional optimizations were also added there. Co-authored-by: Felix Lange <fjl@twurst.com>
This commit is contained in:
Martin Holst Swende
committed by
GitHub
GitHub
parent
9a0df80bbc
commit
794c6133ef
@ -18,49 +18,36 @@ package rawdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/metrics"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().Unix())
|
||||
}
|
||||
|
||||
// Gets a chunk of data, filled with 'b'
|
||||
func getChunk(size int, b int) []byte {
|
||||
data := make([]byte, size)
|
||||
for i := range data {
|
||||
data[i] = byte(b)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// TestFreezerBasics test initializing a freezertable from scratch, writing to the table,
|
||||
// and reading it back.
|
||||
func TestFreezerBasics(t *testing.T) {
|
||||
t.Parallel()
|
||||
// set cutoff at 50 bytes
|
||||
f, err := newCustomTable(os.TempDir(),
|
||||
f, err := newTable(os.TempDir(),
|
||||
fmt.Sprintf("unittest-%d", rand.Uint64()),
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// Write 15 bytes 255 times, results in 85 files
|
||||
for x := 0; x < 255; x++ {
|
||||
data := getChunk(15, x)
|
||||
f.Append(uint64(x), data)
|
||||
}
|
||||
writeChunks(t, f, 255, 15)
|
||||
|
||||
//print(t, f, 0)
|
||||
//print(t, f, 1)
|
||||
@ -98,16 +85,21 @@ func TestFreezerBasicsClosing(t *testing.T) {
|
||||
f *freezerTable
|
||||
err error
|
||||
)
|
||||
f, err = newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write 15 bytes 255 times, results in 85 files
|
||||
|
||||
// Write 15 bytes 255 times, results in 85 files.
|
||||
// In-between writes, the table is closed and re-opened.
|
||||
for x := 0; x < 255; x++ {
|
||||
data := getChunk(15, x)
|
||||
f.Append(uint64(x), data)
|
||||
batch := f.newBatch()
|
||||
require.NoError(t, batch.AppendRaw(uint64(x), data))
|
||||
require.NoError(t, batch.commit())
|
||||
f.Close()
|
||||
f, err = newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -124,7 +116,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
|
||||
t.Fatalf("test %d, got \n%x != \n%x", y, got, exp)
|
||||
}
|
||||
f.Close()
|
||||
f, err = newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -137,22 +129,22 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
|
||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64())
|
||||
|
||||
{ // Fill table
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
// Fill table
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write 15 bytes 255 times
|
||||
for x := 0; x < 255; x++ {
|
||||
data := getChunk(15, x)
|
||||
f.Append(uint64(x), data)
|
||||
}
|
||||
writeChunks(t, f, 255, 15)
|
||||
|
||||
// The last item should be there
|
||||
if _, err = f.Retrieve(0xfe); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// open the index
|
||||
idxFile, err := os.OpenFile(filepath.Join(os.TempDir(), fmt.Sprintf("%s.ridx", fname)), os.O_RDWR, 0644)
|
||||
if err != nil {
|
||||
@ -165,9 +157,10 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
|
||||
}
|
||||
idxFile.Truncate(stat.Size() - 4)
|
||||
idxFile.Close()
|
||||
|
||||
// Now open it again
|
||||
{
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -188,22 +181,22 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
|
||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64())
|
||||
|
||||
{ // Fill a table and close it
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
// Fill a table and close it
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write 15 bytes 255 times
|
||||
for x := 0; x < 0xff; x++ {
|
||||
data := getChunk(15, x)
|
||||
f.Append(uint64(x), data)
|
||||
}
|
||||
writeChunks(t, f, 255, 15)
|
||||
|
||||
// The last item should be there
|
||||
if _, err = f.Retrieve(f.items - 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// open the index
|
||||
idxFile, err := os.OpenFile(filepath.Join(os.TempDir(), fmt.Sprintf("%s.ridx", fname)), os.O_RDWR, 0644)
|
||||
if err != nil {
|
||||
@ -213,9 +206,10 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
|
||||
// 0-indexEntry, 1-indexEntry, corrupt-indexEntry
|
||||
idxFile.Truncate(indexEntrySize + indexEntrySize + indexEntrySize/2)
|
||||
idxFile.Close()
|
||||
|
||||
// Now open it again
|
||||
{
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -228,15 +222,17 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
|
||||
t.Errorf("Expected error for missing index entry")
|
||||
}
|
||||
// We should now be able to store items again, from item = 1
|
||||
batch := f.newBatch()
|
||||
for x := 1; x < 0xff; x++ {
|
||||
data := getChunk(15, ^x)
|
||||
f.Append(uint64(x), data)
|
||||
require.NoError(t, batch.AppendRaw(uint64(x), getChunk(15, ^x)))
|
||||
}
|
||||
require.NoError(t, batch.commit())
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// And if we open it, we should now be able to read all of them (new values)
|
||||
{
|
||||
f, _ := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
for y := 1; y < 255; y++ {
|
||||
exp := getChunk(15, ^y)
|
||||
got, err := f.Retrieve(uint64(y))
|
||||
@ -255,22 +251,21 @@ func TestSnappyDetection(t *testing.T) {
|
||||
t.Parallel()
|
||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("snappytest-%d", rand.Uint64())
|
||||
|
||||
// Open with snappy
|
||||
{
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write 15 bytes 255 times
|
||||
for x := 0; x < 0xff; x++ {
|
||||
data := getChunk(15, x)
|
||||
f.Append(uint64(x), data)
|
||||
}
|
||||
writeChunks(t, f, 255, 15)
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Open without snappy
|
||||
{
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -282,7 +277,7 @@ func TestSnappyDetection(t *testing.T) {
|
||||
|
||||
// Open with snappy
|
||||
{
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -292,8 +287,8 @@ func TestSnappyDetection(t *testing.T) {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func assertFileSize(f string, size int64) error {
|
||||
stat, err := os.Stat(f)
|
||||
if err != nil {
|
||||
@ -303,7 +298,6 @@ func assertFileSize(f string, size int64) error {
|
||||
return fmt.Errorf("error, expected size %d, got %d", size, stat.Size())
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// TestFreezerRepairDanglingIndex checks that if the index has more entries than there are data,
|
||||
@ -313,16 +307,15 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
|
||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("dangling_indextest-%d", rand.Uint64())
|
||||
|
||||
{ // Fill a table and close it
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
// Fill a table and close it
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write 15 bytes 9 times : 150 bytes
|
||||
for x := 0; x < 9; x++ {
|
||||
data := getChunk(15, x)
|
||||
f.Append(uint64(x), data)
|
||||
}
|
||||
writeChunks(t, f, 9, 15)
|
||||
|
||||
// The last item should be there
|
||||
if _, err = f.Retrieve(f.items - 1); err != nil {
|
||||
f.Close()
|
||||
@ -331,6 +324,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
|
||||
f.Close()
|
||||
// File sizes should be 45, 45, 45 : items[3, 3, 3)
|
||||
}
|
||||
|
||||
// Crop third file
|
||||
fileToCrop := filepath.Join(os.TempDir(), fmt.Sprintf("%s.0002.rdat", fname))
|
||||
// Truncate third file: 45 ,45, 20
|
||||
@ -345,17 +339,18 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
|
||||
file.Truncate(20)
|
||||
file.Close()
|
||||
}
|
||||
|
||||
// Open db it again
|
||||
// It should restore the file(s) to
|
||||
// 45, 45, 15
|
||||
// with 3+3+1 items
|
||||
{
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
if f.items != 7 {
|
||||
f.Close()
|
||||
t.Fatalf("expected %d items, got %d", 7, f.items)
|
||||
}
|
||||
if err := assertFileSize(fileToCrop, 15); err != nil {
|
||||
@ -365,30 +360,29 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFreezerTruncate(t *testing.T) {
|
||||
|
||||
t.Parallel()
|
||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("truncation-%d", rand.Uint64())
|
||||
|
||||
{ // Fill table
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
// Fill table
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write 15 bytes 30 times
|
||||
for x := 0; x < 30; x++ {
|
||||
data := getChunk(15, x)
|
||||
f.Append(uint64(x), data)
|
||||
}
|
||||
writeChunks(t, f, 30, 15)
|
||||
|
||||
// The last item should be there
|
||||
if _, err = f.Retrieve(f.items - 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Reopen, truncate
|
||||
{
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -401,9 +395,7 @@ func TestFreezerTruncate(t *testing.T) {
|
||||
if f.headBytes != 15 {
|
||||
t.Fatalf("expected %d bytes, got %d", 15, f.headBytes)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// TestFreezerRepairFirstFile tests a head file with the very first item only half-written.
|
||||
@ -412,20 +404,26 @@ func TestFreezerRepairFirstFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("truncationfirst-%d", rand.Uint64())
|
||||
{ // Fill table
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
|
||||
// Fill table
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write 80 bytes, splitting out into two files
|
||||
f.Append(0, getChunk(40, 0xFF))
|
||||
f.Append(1, getChunk(40, 0xEE))
|
||||
batch := f.newBatch()
|
||||
require.NoError(t, batch.AppendRaw(0, getChunk(40, 0xFF)))
|
||||
require.NoError(t, batch.AppendRaw(1, getChunk(40, 0xEE)))
|
||||
require.NoError(t, batch.commit())
|
||||
|
||||
// The last item should be there
|
||||
if _, err = f.Retrieve(f.items - 1); err != nil {
|
||||
if _, err = f.Retrieve(1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Truncate the file in half
|
||||
fileToCrop := filepath.Join(os.TempDir(), fmt.Sprintf("%s.0001.rdat", fname))
|
||||
{
|
||||
@ -439,9 +437,10 @@ func TestFreezerRepairFirstFile(t *testing.T) {
|
||||
file.Truncate(20)
|
||||
file.Close()
|
||||
}
|
||||
|
||||
// Reopen
|
||||
{
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -449,9 +448,14 @@ func TestFreezerRepairFirstFile(t *testing.T) {
|
||||
f.Close()
|
||||
t.Fatalf("expected %d items, got %d", 0, f.items)
|
||||
}
|
||||
|
||||
// Write 40 bytes
|
||||
f.Append(1, getChunk(40, 0xDD))
|
||||
batch := f.newBatch()
|
||||
require.NoError(t, batch.AppendRaw(1, getChunk(40, 0xDD)))
|
||||
require.NoError(t, batch.commit())
|
||||
|
||||
f.Close()
|
||||
|
||||
// Should have been truncated down to zero and then 40 written
|
||||
if err := assertFileSize(fileToCrop, 40); err != nil {
|
||||
t.Fatal(err)
|
||||
@ -468,25 +472,26 @@ func TestFreezerReadAndTruncate(t *testing.T) {
|
||||
t.Parallel()
|
||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("read_truncate-%d", rand.Uint64())
|
||||
{ // Fill table
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
|
||||
// Fill table
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write 15 bytes 30 times
|
||||
for x := 0; x < 30; x++ {
|
||||
data := getChunk(15, x)
|
||||
f.Append(uint64(x), data)
|
||||
}
|
||||
writeChunks(t, f, 30, 15)
|
||||
|
||||
// The last item should be there
|
||||
if _, err = f.Retrieve(f.items - 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Reopen and read all files
|
||||
{
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -497,40 +502,48 @@ func TestFreezerReadAndTruncate(t *testing.T) {
|
||||
for y := byte(0); y < 30; y++ {
|
||||
f.Retrieve(uint64(y))
|
||||
}
|
||||
|
||||
// Now, truncate back to zero
|
||||
f.truncate(0)
|
||||
|
||||
// Write the data again
|
||||
batch := f.newBatch()
|
||||
for x := 0; x < 30; x++ {
|
||||
data := getChunk(15, ^x)
|
||||
if err := f.Append(uint64(x), data); err != nil {
|
||||
t.Fatalf("error %v", err)
|
||||
}
|
||||
require.NoError(t, batch.AppendRaw(uint64(x), getChunk(15, ^x)))
|
||||
}
|
||||
require.NoError(t, batch.commit())
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func TestOffset(t *testing.T) {
|
||||
func TestFreezerOffset(t *testing.T) {
|
||||
t.Parallel()
|
||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("offset-%d", rand.Uint64())
|
||||
{ // Fill table
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
||||
|
||||
// Fill table
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Write 6 x 20 bytes, splitting out into three files
|
||||
f.Append(0, getChunk(20, 0xFF))
|
||||
f.Append(1, getChunk(20, 0xEE))
|
||||
batch := f.newBatch()
|
||||
require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF)))
|
||||
require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE)))
|
||||
|
||||
f.Append(2, getChunk(20, 0xdd))
|
||||
f.Append(3, getChunk(20, 0xcc))
|
||||
require.NoError(t, batch.AppendRaw(2, getChunk(20, 0xdd)))
|
||||
require.NoError(t, batch.AppendRaw(3, getChunk(20, 0xcc)))
|
||||
|
||||
f.Append(4, getChunk(20, 0xbb))
|
||||
f.Append(5, getChunk(20, 0xaa))
|
||||
f.DumpIndex(0, 100)
|
||||
require.NoError(t, batch.AppendRaw(4, getChunk(20, 0xbb)))
|
||||
require.NoError(t, batch.AppendRaw(5, getChunk(20, 0xaa)))
|
||||
require.NoError(t, batch.commit())
|
||||
|
||||
t.Log(f.dumpIndexString(0, 100))
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// Now crop it.
|
||||
{
|
||||
// delete files 0 and 1
|
||||
@ -558,7 +571,7 @@ func TestOffset(t *testing.T) {
|
||||
filenum: tailId,
|
||||
offset: itemOffset,
|
||||
}
|
||||
buf := zeroIndex.marshallBinary()
|
||||
buf := zeroIndex.append(nil)
|
||||
// Overwrite index zero
|
||||
copy(indexBuf, buf)
|
||||
// Remove the four next indices by overwriting
|
||||
@ -567,44 +580,36 @@ func TestOffset(t *testing.T) {
|
||||
// Need to truncate the moved index items
|
||||
indexFile.Truncate(indexEntrySize * (1 + 2))
|
||||
indexFile.Close()
|
||||
|
||||
}
|
||||
|
||||
// Now open again
|
||||
checkPresent := func(numDeleted uint64) {
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.DumpIndex(0, 100)
|
||||
// It should allow writing item 6
|
||||
f.Append(numDeleted+2, getChunk(20, 0x99))
|
||||
defer f.Close()
|
||||
t.Log(f.dumpIndexString(0, 100))
|
||||
|
||||
// It should be fine to fetch 4,5,6
|
||||
if got, err := f.Retrieve(numDeleted); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if exp := getChunk(20, 0xbb); !bytes.Equal(got, exp) {
|
||||
t.Fatalf("expected %x got %x", exp, got)
|
||||
}
|
||||
if got, err := f.Retrieve(numDeleted + 1); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if exp := getChunk(20, 0xaa); !bytes.Equal(got, exp) {
|
||||
t.Fatalf("expected %x got %x", exp, got)
|
||||
}
|
||||
if got, err := f.Retrieve(numDeleted + 2); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if exp := getChunk(20, 0x99); !bytes.Equal(got, exp) {
|
||||
t.Fatalf("expected %x got %x", exp, got)
|
||||
}
|
||||
// It should allow writing item 6.
|
||||
batch := f.newBatch()
|
||||
require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x99)))
|
||||
require.NoError(t, batch.commit())
|
||||
|
||||
// It should error at 0, 1,2,3
|
||||
for i := numDeleted - 1; i > numDeleted-10; i-- {
|
||||
if _, err := f.Retrieve(i); err == nil {
|
||||
t.Fatal("expected err")
|
||||
}
|
||||
}
|
||||
checkRetrieveError(t, f, map[uint64]error{
|
||||
0: errOutOfBounds,
|
||||
1: errOutOfBounds,
|
||||
2: errOutOfBounds,
|
||||
3: errOutOfBounds,
|
||||
})
|
||||
checkRetrieve(t, f, map[uint64][]byte{
|
||||
4: getChunk(20, 0xbb),
|
||||
5: getChunk(20, 0xaa),
|
||||
6: getChunk(20, 0x99),
|
||||
})
|
||||
}
|
||||
checkPresent(4)
|
||||
// Now, let's pretend we have deleted 1M items
|
||||
|
||||
// Edit the index again, with a much larger initial offset of 1M.
|
||||
{
|
||||
// Read the index file
|
||||
p := filepath.Join(os.TempDir(), fmt.Sprintf("%v.ridx", fname))
|
||||
@ -624,13 +629,71 @@ func TestOffset(t *testing.T) {
|
||||
offset: itemOffset,
|
||||
filenum: tailId,
|
||||
}
|
||||
buf := zeroIndex.marshallBinary()
|
||||
buf := zeroIndex.append(nil)
|
||||
// Overwrite index zero
|
||||
copy(indexBuf, buf)
|
||||
indexFile.WriteAt(indexBuf, 0)
|
||||
indexFile.Close()
|
||||
}
|
||||
checkPresent(1000000)
|
||||
|
||||
// Check that existing items have been moved to index 1M.
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
t.Log(f.dumpIndexString(0, 100))
|
||||
|
||||
checkRetrieveError(t, f, map[uint64]error{
|
||||
0: errOutOfBounds,
|
||||
1: errOutOfBounds,
|
||||
2: errOutOfBounds,
|
||||
3: errOutOfBounds,
|
||||
999999: errOutOfBounds,
|
||||
})
|
||||
checkRetrieve(t, f, map[uint64][]byte{
|
||||
1000000: getChunk(20, 0xbb),
|
||||
1000001: getChunk(20, 0xaa),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func checkRetrieve(t *testing.T, f *freezerTable, items map[uint64][]byte) {
|
||||
t.Helper()
|
||||
|
||||
for item, wantBytes := range items {
|
||||
value, err := f.Retrieve(item)
|
||||
if err != nil {
|
||||
t.Fatalf("can't get expected item %d: %v", item, err)
|
||||
}
|
||||
if !bytes.Equal(value, wantBytes) {
|
||||
t.Fatalf("item %d has wrong value %x (want %x)", item, value, wantBytes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkRetrieveError(t *testing.T, f *freezerTable, items map[uint64]error) {
|
||||
t.Helper()
|
||||
|
||||
for item, wantError := range items {
|
||||
value, err := f.Retrieve(item)
|
||||
if err == nil {
|
||||
t.Fatalf("unexpected value %x for item %d, want error %v", item, value, wantError)
|
||||
}
|
||||
if err != wantError {
|
||||
t.Fatalf("wrong error for item %d: %v", item, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Gets a chunk of data, filled with 'b'
|
||||
func getChunk(size int, b int) []byte {
|
||||
data := make([]byte, size)
|
||||
for i := range data {
|
||||
data[i] = byte(b)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// TODO (?)
|
||||
@ -644,53 +707,18 @@ func TestOffset(t *testing.T) {
|
||||
// should be handled already, and the case described above can only (?) happen if an
|
||||
// external process/user deletes files from the filesystem.
|
||||
|
||||
// TestAppendTruncateParallel is a test to check if the Append/truncate operations are
|
||||
// racy.
|
||||
//
|
||||
// The reason why it's not a regular fuzzer, within tests/fuzzers, is that it is dependent
|
||||
// on timing rather than 'clever' input -- there's no determinism.
|
||||
func TestAppendTruncateParallel(t *testing.T) {
|
||||
dir, err := ioutil.TempDir("", "freezer")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
func writeChunks(t *testing.T, ft *freezerTable, n int, length int) {
|
||||
t.Helper()
|
||||
|
||||
f, err := newCustomTable(dir, "tmp", metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, 8, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fill := func(mark uint64) []byte {
|
||||
data := make([]byte, 8)
|
||||
binary.LittleEndian.PutUint64(data, mark)
|
||||
return data
|
||||
}
|
||||
|
||||
for i := 0; i < 5000; i++ {
|
||||
f.truncate(0)
|
||||
data0 := fill(0)
|
||||
f.Append(0, data0)
|
||||
data1 := fill(1)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
f.truncate(0)
|
||||
wg.Done()
|
||||
}()
|
||||
go func() {
|
||||
f.Append(1, data1)
|
||||
wg.Done()
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
if have, err := f.Retrieve(0); err == nil {
|
||||
if !bytes.Equal(have, data0) {
|
||||
t.Fatalf("have %x want %x", have, data0)
|
||||
}
|
||||
batch := ft.newBatch()
|
||||
for i := 0; i < n; i++ {
|
||||
if err := batch.AppendRaw(uint64(i), getChunk(length, i)); err != nil {
|
||||
t.Fatalf("AppendRaw(%d, ...) returned error: %v", i, err)
|
||||
}
|
||||
}
|
||||
if err := batch.commit(); err != nil {
|
||||
t.Fatalf("Commit returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSequentialRead does some basic tests on the RetrieveItems.
|
||||
@ -698,20 +726,17 @@ func TestSequentialRead(t *testing.T) {
|
||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("batchread-%d", rand.Uint64())
|
||||
{ // Fill table
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write 15 bytes 30 times
|
||||
for x := 0; x < 30; x++ {
|
||||
data := getChunk(15, x)
|
||||
f.Append(uint64(x), data)
|
||||
}
|
||||
writeChunks(t, f, 30, 15)
|
||||
f.DumpIndex(0, 30)
|
||||
f.Close()
|
||||
}
|
||||
{ // Open it, iterate, verify iteration
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -732,7 +757,7 @@ func TestSequentialRead(t *testing.T) {
|
||||
}
|
||||
{ // Open it, iterate, verify byte limit. The byte limit is less than item
|
||||
// size, so each lookup should only return one item
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -761,16 +786,13 @@ func TestSequentialReadByteLimit(t *testing.T) {
|
||||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("batchread-2-%d", rand.Uint64())
|
||||
{ // Fill table
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 100, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write 10 bytes 30 times,
|
||||
// Splitting it at every 100 bytes (10 items)
|
||||
for x := 0; x < 30; x++ {
|
||||
data := getChunk(10, x)
|
||||
f.Append(uint64(x), data)
|
||||
}
|
||||
writeChunks(t, f, 30, 10)
|
||||
f.Close()
|
||||
}
|
||||
for i, tc := range []struct {
|
||||
@ -786,7 +808,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
|
||||
{100, 109, 10},
|
||||
} {
|
||||
{
|
||||
f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 100, true)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user