Skip to content

feat: add parquet receipt store with DuckDB range queries #2789

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jewei1997 wants to merge 10 commits into
base: ledger-cache-layer
Choose a base branch
from
+2,400 −73
Open

feat: add parquet receipt store with DuckDB range queries #2789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
215a286
feat: add parquet receipt store with DuckDB range queries
jewei1997 Feb 2, 2026
432bd51
fix: update fakeReceiptStore.FilterLogs to new signature
jewei1997 Feb 2, 2026
a9dd371
make parquet its own package
jewei1997 Feb 4, 2026
4fe5492
Merge branch 'ledger-cache-layer' into parquet-receiptdb
jewei1997 Feb 4, 2026
e744ecd
fix build issue
jewei1997 Feb 4, 2026
9d1d18f
remove duckdb stub
jewei1997 Feb 4, 2026
2c364bc
remove parquet enabled
jewei1997 Feb 4, 2026
e807617
fix go lint
jewei1997 Feb 4, 2026
4b294d9
feat: optimize WAL write path with binary encoding and batched writes
jewei1997 Feb 9, 2026
69423d9
feat: add store helpers and pebble pre-marshaled bytes support
jewei1997 Feb 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 36 additions & 18 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ require (
github.com/confio/ics23/go v0.9.0
github.com/cosmos/cosmos-sdk v0.45.10
github.com/cosmos/go-bip39 v1.0.0
github.com/duckdb/duckdb-go/v2 v2.5.3
github.com/dvsekhvalnov/jose2go v1.5.0
github.com/ethereum/evmc/v12 v12.1.0
github.com/ethereum/go-ethereum v1.16.1
Expand All @@ -32,6 +33,7 @@ require (
github.com/ledgerwatch/erigon-lib v0.0.0-20230210071639-db0e7ed11263
github.com/linxGnu/grocksdb v1.8.11
github.com/mitchellh/mapstructure v1.5.0
github.com/parquet-go/parquet-go v0.25.1
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.23.0
github.com/rakyll/statik v0.1.7
Expand All @@ -49,7 +51,7 @@ require (
github.com/tendermint/tendermint v0.37.0-dev
github.com/tendermint/tm-db v0.6.8-0.20220519162814-e24b96538a12
github.com/tidwall/btree v1.6.0
github.com/tidwall/gjson v1.10.2
github.com/tidwall/gjson v1.14.2
github.com/tidwall/wal v1.2.1
github.com/zbiljic/go-filelock v0.0.0-20170914061330-1dbf7103ab7d
github.com/zeebo/blake3 v0.2.4
Expand All @@ -59,13 +61,13 @@ require (
go.opentelemetry.io/otel/sdk v1.38.0
go.opentelemetry.io/otel/sdk/metric v1.38.0
go.opentelemetry.io/otel/trace v1.38.0
golang.org/x/crypto v0.40.0
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
golang.org/x/crypto v0.41.0
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0
golang.org/x/sync v0.16.0
golang.org/x/sys v0.35.0
golang.org/x/time v0.13.0
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576
google.golang.org/grpc v1.67.3
google.golang.org/grpc v1.75.0
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
)
Expand All @@ -76,7 +78,7 @@ require (
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect
github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect
github.com/regen-network/cosmos-proto v0.3.1
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
)

require (
Expand All @@ -100,6 +102,8 @@ require (
github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
github.com/alexkohler/prealloc v1.0.0 // indirect
github.com/alingse/asasalint v0.0.11 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/apache/arrow-go/v18 v18.4.1 // indirect
github.com/ashanbrown/forbidigo v1.3.0 // indirect
github.com/ashanbrown/makezero v1.1.1 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
Expand Down Expand Up @@ -137,7 +141,7 @@ require (
github.com/curioswitch/go-reassign v0.2.0 // indirect
github.com/daixiang0/gci v0.8.1 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
github.com/denis-tingaikin/go-header v0.4.3 // indirect
Expand All @@ -146,6 +150,14 @@ require (
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dlclark/regexp2 v1.7.0 // indirect
github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 // indirect
github.com/duckdb/duckdb-go-bindings v0.1.23 // indirect
github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.23 // indirect
github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.23 // indirect
github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.23 // indirect
github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.23 // indirect
github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.23 // indirect
github.com/duckdb/duckdb-go/arrowmapping v0.0.26 // indirect
github.com/duckdb/duckdb-go/mapping v0.0.25 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/esimonov/ifshort v1.0.4 // indirect
github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
Expand Down Expand Up @@ -175,15 +187,16 @@ require (
github.com/go-toolsmith/astp v1.0.0 // indirect
github.com/go-toolsmith/strparse v1.0.0 // indirect
github.com/go-toolsmith/typep v1.0.2 // indirect
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/gateway v1.1.0 // indirect
github.com/golang/glog v1.1.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe // indirect
Expand All @@ -193,7 +206,7 @@ require (
github.com/golangci/misspell v0.3.5 // indirect
github.com/golangci/revgrep v0.0.0-20220804021717-745bb2f7c2e6 // indirect
github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
github.com/google/flatbuffers v23.5.26+incompatible // indirect
github.com/google/flatbuffers v25.2.10+incompatible // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/orderedcode v0.0.1 // indirect
github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
Expand Down Expand Up @@ -233,7 +246,7 @@ require (
github.com/kisielk/gotool v1.0.0 // indirect
github.com/kkHAIKE/contextcheck v1.1.3 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kulti/thelper v0.6.3 // indirect
Expand All @@ -251,7 +264,7 @@ require (
github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mbilski/exhaustivestruct v1.2.0 // indirect
github.com/mgechev/revive v1.2.4 // indirect
github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect
Expand All @@ -270,12 +283,13 @@ require (
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/pion/dtls/v2 v2.2.7 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/stun/v2 v2.0.0 // indirect
github.com/pion/transport/v2 v2.2.1 // indirect
github.com/pion/transport/v3 v3.0.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/polyfloyd/go-errorlint v1.0.5 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.65.0 // indirect
Expand Down Expand Up @@ -329,6 +343,7 @@ require (
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
github.com/yagipy/maintidx v1.0.0 // indirect
github.com/yeya24/promlinter v0.2.0 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
github.com/zondax/hid v0.9.1 // indirect
github.com/zondax/ledger-go v0.14.1 // indirect
gitlab.com/bosi/decorder v0.2.3 // indirect
Expand All @@ -340,13 +355,16 @@ require (
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/zap v1.22.0 // indirect
golang.org/x/exp/typeparams v0.0.0-20220827204233-334a2380cb91 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/term v0.33.0 // indirect
golang.org/x/text v0.27.0 // indirect
golang.org/x/tools v0.34.0 // indirect
golang.org/x/mod v0.27.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/term v0.34.0 // indirect
golang.org/x/text v0.28.0 // indirect
golang.org/x/tools v0.36.0 // indirect
golang.org/x/tools/go/expect v0.1.1-deprecated // indirect
golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
google.golang.org/protobuf v1.36.10 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
honnef.co/go/tools v0.3.3 // indirect
Expand Down
Loading
Loading