diff --git a/arguments.go b/arguments.go index dd962c0..f4ab4ec 100644 --- a/arguments.go +++ b/arguments.go @@ -16,6 +16,7 @@ type arguments struct { MaxConnections int `short:"c" long:"max-connections" value-name:"" default:"512" description:"Maximum number of HTTP connections"` MaxConnectionsPerHost int `long:"max-connections-per-host" value-name:"" default:"512" description:"Maximum number of HTTP connections per host"` MaxResponseBodySize int `long:"max-response-body-size" value-name:"" default:"10000000" description:"Maximum response body size to read"` + CustomDnsAddr string `long:"custom-dns-addr" description:"Custom DNS server to be used for requests"` RawExcludedPatterns []string `short:"e" long:"exclude" value-name:"..." description:"Exclude URLs matched with given regular expressions"` RawIncludedPatterns []string `short:"i" long:"include" value-name:"..." description:"Include URLs matched with given regular expressions"` FollowRobotsTxt bool `long:"follow-robots-txt" description:"Follow robots.txt when scraping pages"` diff --git a/command.go b/command.go index bb5f8e6..d523c5a 100644 --- a/command.go +++ b/command.go @@ -56,6 +56,7 @@ func (c *command) runWithError(ss []string) (bool, error) { SkipTLSVerification: args.SkipTLSVerification, Timeout: time.Duration(args.Timeout) * time.Second, Header: args.Header, + CustomDnsAddr: args.CustomDnsAddr, }, ), args.RateLimit, diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go index be89341..599529a 100644 --- a/fasthttp_http_client_factory.go +++ b/fasthttp_http_client_factory.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/tls" "net" @@ -24,6 +25,23 @@ func (*fasthttpHttpClientFactory) Create(o httpClientOptions) httpClient { d = fasthttpproxy.FasthttpHTTPDialerTimeout(o.Proxy, tcpTimeout) } + if o.CustomDnsAddr != "" { + d = func(addr string) (net.Conn, error) { + dialer := fasthttp.TCPDialer{ + Concurrency: 1000, + Resolver: &net.Resolver{ + PreferGo: true, + StrictErrors: false, + Dial: func(ctx context.Context, network string, address string) (net.Conn, error) { + internalDialer := net.Dialer{} + return internalDialer.DialContext(ctx, "udp", o.CustomDnsAddr) + }, + }, + } + return dialer.DialTimeout(addr, tcpTimeout) + } + } + return newFasthttpHttpClient( &fasthttp.Client{ MaxConnsPerHost: o.MaxConnectionsPerHost, diff --git a/fasthttp_http_client_factory_test.go b/fasthttp_http_client_factory_test.go index 41f3696..047d615 100644 --- a/fasthttp_http_client_factory_test.go +++ b/fasthttp_http_client_factory_test.go @@ -9,3 +9,7 @@ func TestFasthttpHttpClientFactoryCreate(t *testing.T) { func TestFasthttpHttpClientFactoryCreateWithProxy(t *testing.T) { newFasthttpHttpClientFactory().Create(httpClientOptions{Proxy: "foo"}) } + +func TestFasthttpHttpClientFactoryCreateWithCustomDnsResolver(t *testing.T) { + newFasthttpHttpClientFactory().Create(httpClientOptions{CustomDnsAddr: "1.1.1.1:53"}) +} diff --git a/http_client_factory.go b/http_client_factory.go index ec3c385..cb2ef7e 100644 --- a/http_client_factory.go +++ b/http_client_factory.go @@ -13,6 +13,7 @@ type httpClientOptions struct { SkipTLSVerification bool Timeout time.Duration Header http.Header + CustomDnsAddr string } type httpClientFactory interface {