Revert "silkworm: use silkworm-go bindings (erigontech#8829)"

This reverts commit bc0b727.

.github/workflows/ci.yml

@@ -112,7 +112,7 @@ jobs:
   docker-build-check:
     # don't run this on devel - the PR must have run it to be merged and it misleads that this pushes the docker image
     if: (${{ github.event_name == 'push' || !github.event.pull_request.draft }}) && ${{ github.ref != 'refs/heads/devel' }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/checkout@v3
@@ -128,7 +128,7 @@ jobs:
 
 #  automated-tests:
 #    runs-on:
-#      ubuntu-22.04
+#      ubuntu-20.04
 #    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
 #    steps:
 #      - uses: actions/checkout@v3

.github/workflows/test-integration-caplin.yml

@@ -21,7 +21,8 @@ jobs:
     strategy:
       matrix:
 #        disable macos-11 until https://github.com/ledgerwatch/erigon/issues/8789
-        os: [ ubuntu-22.04 ] # list of os: https://github.com/actions/virtual-environments
+#        os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-20.04 ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
 
     steps:

Dockerfile

@@ -15,7 +15,7 @@ ADD . .
 RUN --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/tmp/go-build \
     --mount=type=cache,target=/go/pkg/mod \
-    make BUILD_TAGS=nosqlite,noboltdb,nosilkworm all
+    make all
 
 
 FROM docker.io/library/golang:1.20-alpine3.17 AS tools-builder

Makefile

@@ -236,16 +236,6 @@ git-submodules:
 	@git submodule sync --quiet --recursive || true
 	@git submodule update --quiet --init --recursive --force || true
 
-## install:                            copies binaries and libraries to DIST
-DIST ?= $(CURDIR)/build/dist
-.PHONY: install
-install:
-	mkdir -p "$(DIST)"
-	cp -f "$$($(CURDIR)/turbo/silkworm/silkworm_lib_path.sh)" "$(DIST)"
-	cp -f "$(GOBIN)/"* "$(DIST)"
-	@echo "Copied files to $(DIST):"
-	@ls -al "$(DIST)"
-
 PACKAGE_NAME          := github.com/ledgerwatch/erigon
 GOLANG_CROSS_VERSION  ?= v1.20.7
 

cmd/utils/flags.go

@@ -848,6 +848,11 @@ var (
 		Usage: "Comma separated list of support session ids to connect to",
 	}
 
+	SilkwormLibraryPathFlag = cli.StringFlag{
+		Name:  "silkworm.libpath",
+		Usage: "Path to the Silkworm library",
+		Value: "",
+	}
 	SilkwormExecutionFlag = cli.BoolFlag{
 		Name:  "silkworm.exec",
 		Usage: "Enable Silkworm block execution",
@@ -860,7 +865,6 @@ var (
 		Name:  "silkworm.sentry",
 		Usage: "Enable embedded Silkworm Sentry service",
 	}
-
 	BeaconAPIFlag = cli.BoolFlag{
 		Name:  "beacon.api",
 		Usage: "Enable beacon API",
@@ -1563,7 +1567,10 @@ func setCaplin(ctx *cli.Context, cfg *ethconfig.Config) {
 }
 
 func setSilkworm(ctx *cli.Context, cfg *ethconfig.Config) {
-	cfg.SilkwormExecution = ctx.Bool(SilkwormExecutionFlag.Name)
+	cfg.SilkwormLibraryPath = ctx.String(SilkwormLibraryPathFlag.Name)
+	if ctx.IsSet(SilkwormExecutionFlag.Name) {
+		cfg.SilkwormExecution = ctx.Bool(SilkwormExecutionFlag.Name)
+	}
 	cfg.SilkwormRpcDaemon = ctx.Bool(SilkwormRpcDaemonFlag.Name)
 	cfg.SilkwormSentry = ctx.Bool(SilkwormSentryFlag.Name)
 }

eth/backend.go

@@ -350,8 +350,8 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger
 
 	backend.gasPrice, _ = uint256.FromBig(config.Miner.GasPrice)
 
-	if config.SilkwormExecution || config.SilkwormRpcDaemon || config.SilkwormSentry {
-		backend.silkworm, err = silkworm.New(config.Dirs.DataDir)
+	if config.SilkwormLibraryPath != "" {
+		backend.silkworm, err = silkworm.New(config.SilkwormLibraryPath, config.Dirs.DataDir)
 		if err != nil {
 			return nil, err
 		}
@@ -392,7 +392,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger
 			MaxPeers:    p2pConfig.MaxPeers,
 		}
 
-		silkwormSentryService := silkworm.NewSentryService(backend.silkworm, settings)
+		silkwormSentryService := backend.silkworm.NewSentryService(settings)
 		backend.silkwormSentryService = &silkwormSentryService
 
 		sentryClient, err := sentry_multi_client.GrpcClient(backend.sentryCtx, apiAddr)
@@ -922,7 +922,7 @@ func (s *Ethereum) Init(stack *node.Node, config *ethconfig.Config) error {
 	s.apiList = jsonrpc.APIList(chainKv, ethRpcClient, txPoolRpcClient, miningRpcClient, ff, stateCache, blockReader, s.agg, &httpRpcCfg, s.engine, s.logger)
 
 	if config.SilkwormRpcDaemon && httpRpcCfg.Enabled {
-		silkwormRPCDaemonService := silkworm.NewRpcDaemonService(s.silkworm, chainKv)
+		silkwormRPCDaemonService := s.silkworm.NewRpcDaemonService(chainKv)
 		s.silkwormRPCDaemonService = &silkwormRPCDaemonService
 	} else {
 		go func() {
@@ -1463,9 +1463,7 @@ func (s *Ethereum) Stop() error {
 		}
 	}
 	if s.silkworm != nil {
-		if err := s.silkworm.Close(); err != nil {
-			s.logger.Error("silkworm.Close error", "err", err)
-		}
+		s.silkworm.Close()
 	}
 
 	return nil

eth/ethconfig/config.go

@@ -105,6 +105,9 @@ var Defaults = Config{
 		KeepBlocks: false,
 		Produce:    true,
 	},
+
+	// applies if SilkwormLibraryPath is set
+	SilkwormExecution: true,
 }
 
 func init() {
@@ -251,9 +254,10 @@ type Config struct {
 	ForcePartialCommit bool
 
 	// Embedded Silkworm support
-	SilkwormExecution bool
-	SilkwormRpcDaemon bool
-	SilkwormSentry    bool
+	SilkwormLibraryPath string
+	SilkwormExecution   bool
+	SilkwormRpcDaemon   bool
+	SilkwormSentry      bool
 
 	DisableTxPoolGossip bool
 }

eth/stagedsync/stage_execute.go

@@ -467,7 +467,7 @@ Loop:
 
 		_, isMemoryMutation := txc.Tx.(*membatchwithdb.MemoryMutation)
 		if cfg.silkworm != nil && !isMemoryMutation {
-			blockNum, err = silkworm.ExecuteBlocks(cfg.silkworm, txc.Tx, cfg.chainConfig.ChainID, blockNum, to, uint64(cfg.batchSize), writeChangeSets, writeReceipts, writeCallTraces)
+			blockNum, err = cfg.silkworm.ExecuteBlocks(txc.Tx, cfg.chainConfig.ChainID, blockNum, to, uint64(cfg.batchSize), writeChangeSets, writeReceipts, writeCallTraces)
 		} else {
 			err = executeBlock(block, txc.Tx, batch, cfg, *cfg.vmConfig, writeChangeSets, writeReceipts, writeCallTraces, stateStream, logger)
 		}

go.mod

@@ -4,7 +4,6 @@ go 1.20
 
 require (
 	github.com/erigontech/mdbx-go v0.27.21
-	github.com/erigontech/silkworm-go v0.10.0
 	github.com/ledgerwatch/log/v3 v3.9.0
 	github.com/ledgerwatch/secp256k1 v1.0.0
 )

go.sum

@@ -297,8 +297,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/erigontech/mdbx-go v0.27.21 h1:Pv47QIiRXR8Nv+nltZteLm4xkRwuvqmOCjzZj9X0s1A=
 github.com/erigontech/mdbx-go v0.27.21/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4=
-github.com/erigontech/silkworm-go v0.10.0 h1:oAoptLtJbQXk63mrKYs6qliQlbDrXTSNiZfzv1OMp+Q=
-github.com/erigontech/silkworm-go v0.10.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU=
 github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0=
 github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=

turbo/cli/default_flags.go

@@ -163,6 +163,7 @@ var DefaultFlags = []cli.Flag{
 
 	&utils.OtsSearchMaxCapFlag,
 
+	&utils.SilkwormLibraryPathFlag,
 	&utils.SilkwormExecutionFlag,
 	&utils.SilkwormRpcDaemonFlag,
 	&utils.SilkwormSentryFlag,

turbo/silkworm/load_unix.go

@@ -0,0 +1,37 @@
+//go:build unix
+
+package silkworm
+
+/*
+#cgo LDFLAGS: -ldl
+#include <dlfcn.h>
+#include <stdlib.h>
+*/
+import "C"
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+func OpenLibrary(dllPath string) (unsafe.Pointer, error) {
+	cPath := C.CString(dllPath)
+	defer C.free(unsafe.Pointer(cPath))
+	dllHandle := C.dlopen(cPath, C.RTLD_LAZY)
+	if dllHandle == nil {
+		err := C.GoString(C.dlerror())
+		return nil, fmt.Errorf("failed to load dynamic library %s: %s", dllPath, err)
+	}
+	return dllHandle, nil
+}
+
+func LoadFunction(dllHandle unsafe.Pointer, funcName string) (unsafe.Pointer, error) {
+	cName := C.CString(funcName)
+	defer C.free(unsafe.Pointer(cName))
+	funcPtr := C.dlsym(dllHandle, cName)
+	if funcPtr == nil {
+		err := C.GoString(C.dlerror())
+		return nil, fmt.Errorf("failed to find the %s function: %s", funcName, err)
+	}
+	return funcPtr, nil
+}

turbo/silkworm/load_windows.go

@@ -0,0 +1,16 @@
+//go:build windows
+
+package silkworm
+
+import (
+	"errors"
+	"unsafe"
+)
+
+func OpenLibrary(dllPath string) (unsafe.Pointer, error) {
+	return nil, errors.New("not implemented")
+}
+
+func LoadFunction(dllHandle unsafe.Pointer, funcName string) (unsafe.Pointer, error) {
+	return nil, errors.New("not implemented")
+}