feat: add parquet receipt store with DuckDB range queries#2789
feat: add parquet receipt store with DuckDB range queries#2789jewei1997 wants to merge 10 commits intoledger-cache-layerfrom
Conversation
|
The latest Buf updates on your PR. Results from workflow Buf / buf (pull_request).
|
Codecov Report✅ All modified and coverable lines are covered by tests. Additional details and impacted files@@ Coverage Diff @@
## ledger-cache-layer #2789 +/- ##
=======================================================
+ Coverage 46.91% 48.32% +1.40%
=======================================================
Files 1969 671 -1298
Lines 160784 50575 -110209
=======================================================
- Hits 75438 24438 -51000
+ Misses 78801 23998 -54803
+ Partials 6545 2139 -4406
Flags with carried forward coverage won't be shown. Click here to find out more. 🚀 New features to boost your workflow:
|
![github-advanced-security[bot]](https://avatars.githubusercontent.com/in/57789?s=60&v=4){kind=small}
| for blockNum, logs := range chunk.logs { | ||
| if blockNum < fromBlock || blockNum > toBlock { | ||
| continue | ||
| } | ||
| for _, lg := range logs { | ||
| if matchLog(lg, crit) { | ||
| logCopy := *lg | ||
| result = append(result, &logCopy) | ||
| } | ||
| } | ||
| } |
Check warning
Code scanning / CodeQL
Iteration over map Warning
| "database/sql" | ||
| "fmt" | ||
| "path/filepath" | ||
| "runtime" |
Check notice
Code scanning / CodeQL
Sensitive package import Note
| go func() { | ||
| for { | ||
| latestVersion := s.latestVersion.Load() | ||
| pruneBeforeBlock := latestVersion - s.config.KeepRecent | ||
| if pruneBeforeBlock > 0 { | ||
| pruned := s.pruneOldFiles(uint64(pruneBeforeBlock)) | ||
| if pruned > 0 && s.log != nil { | ||
| s.log.Info(fmt.Sprintf("Pruned %d parquet file pairs older than block %d", pruned, pruneBeforeBlock)) | ||
| } | ||
| } | ||
|
|
||
| // Add jitter to avoid thundering herd | ||
| jitter := time.Duration(float64(pruneIntervalSeconds)*0.5) * time.Second | ||
| sleepDuration := time.Duration(pruneIntervalSeconds)*time.Second + jitter | ||
|
|
||
| select { | ||
| case <-s.pruneStop: | ||
| return | ||
| case <-time.After(sleepDuration): | ||
| // Continue to next iteration | ||
| } | ||
| } | ||
| }() |
Check notice
Code scanning / CodeQL
Spawning a Go routine Note
| } | ||
|
|
||
| // Add jitter to avoid thundering herd | ||
| jitter := time.Duration(float64(pruneIntervalSeconds)*0.5) * time.Second |
Check notice
Code scanning / CodeQL
Floating point arithmetic Note
2 participants
Summary
This PR adds a parquet-based receipt storage backend with DuckDB for efficient range queries on logs, enabling fast
eth_getLogsqueries across block ranges.Backend: "parquet"in config)KeepRecentconfig-tags duckdbto enable parquet backendThe parquet backend supports the new
FilterLogsrange query API introduced in #2788, enabling efficient cross-block log queries without falling back to per-receipt fetching.Dependencies
Test plan
-tags duckdb