From 985b1a3299e5cc1e580202a9399736326132094a Mon Sep 17 00:00:00 2001 From: Yota Toyama Date: Wed, 17 Jul 2024 21:33:58 +0900 Subject: [PATCH] Modify DNS resolver option (#400) --- .snapshots/TestHelp | 1 + arguments.go | 2 +- command.go | 2 +- fasthttp_http_client_factory.go | 31 +++++++++++++--------------- fasthttp_http_client_factory_test.go | 2 +- http_client_factory.go | 2 +- 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.snapshots/TestHelp b/.snapshots/TestHelp index c265d59..53f3d1a 100644 --- a/.snapshots/TestHelp +++ b/.snapshots/TestHelp @@ -22,6 +22,7 @@ Application Options: sitemap.xml (deprecated) --header=
... Custom headers -f, --ignore-fragments Ignore URL fragments + --dns-resolver=
Custom DNS resolver --format=[text|json|junit] Output format (default: text) --json Output results in JSON (deprecated) --experimental-verbose-json Include successful results in JSON diff --git a/arguments.go b/arguments.go index f4ab4ec..2c1b472 100644 --- a/arguments.go +++ b/arguments.go @@ -16,7 +16,6 @@ type arguments struct { MaxConnections int `short:"c" long:"max-connections" value-name:"" default:"512" description:"Maximum number of HTTP connections"` MaxConnectionsPerHost int `long:"max-connections-per-host" value-name:"" default:"512" description:"Maximum number of HTTP connections per host"` MaxResponseBodySize int `long:"max-response-body-size" value-name:"" default:"10000000" description:"Maximum response body size to read"` - CustomDnsAddr string `long:"custom-dns-addr" description:"Custom DNS server to be used for requests"` RawExcludedPatterns []string `short:"e" long:"exclude" value-name:"..." description:"Exclude URLs matched with given regular expressions"` RawIncludedPatterns []string `short:"i" long:"include" value-name:"..." description:"Include URLs matched with given regular expressions"` FollowRobotsTxt bool `long:"follow-robots-txt" description:"Follow robots.txt when scraping pages"` @@ -24,6 +23,7 @@ type arguments struct { RawHeaders []string `long:"header" value-name:"
..." description:"Custom headers"` // TODO Remove a short option. IgnoreFragments bool `short:"f" long:"ignore-fragments" description:"Ignore URL fragments"` + DnsResolver string `long:"dns-resolver" value-name:"
" description:"Custom DNS resolver"` Format string `long:"format" description:"Output format" default:"text" choice:"text" choice:"json" choice:"junit"` // TODO Remove this option. JSONOutput bool `long:"json" description:"Output results in JSON (deprecated)"` diff --git a/command.go b/command.go index d523c5a..bea32cb 100644 --- a/command.go +++ b/command.go @@ -56,7 +56,7 @@ func (c *command) runWithError(ss []string) (bool, error) { SkipTLSVerification: args.SkipTLSVerification, Timeout: time.Duration(args.Timeout) * time.Second, Header: args.Header, - CustomDnsAddr: args.CustomDnsAddr, + DnsResolver: args.DnsResolver, }, ), args.RateLimit, diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go index 599529a..29b0bf8 100644 --- a/fasthttp_http_client_factory.go +++ b/fasthttp_http_client_factory.go @@ -17,28 +17,25 @@ func newFasthttpHttpClientFactory() *fasthttpHttpClientFactory { } func (*fasthttpHttpClientFactory) Create(o httpClientOptions) httpClient { - d := func(addr string) (net.Conn, error) { - return fasthttp.DialTimeout(addr, tcpTimeout) + d := func(address string) (net.Conn, error) { + return fasthttp.DialTimeout(address, tcpTimeout) } if o.Proxy != "" { d = fasthttpproxy.FasthttpHTTPDialerTimeout(o.Proxy, tcpTimeout) - } - - if o.CustomDnsAddr != "" { - d = func(addr string) (net.Conn, error) { - dialer := fasthttp.TCPDialer{ - Concurrency: 1000, - Resolver: &net.Resolver{ - PreferGo: true, - StrictErrors: false, - Dial: func(ctx context.Context, network string, address string) (net.Conn, error) { - internalDialer := net.Dialer{} - return internalDialer.DialContext(ctx, "udp", o.CustomDnsAddr) - }, + } else if o.DnsResolver != "" { + nd := &net.Dialer{} + td := &fasthttp.TCPDialer{ + Concurrency: concurrency, + Resolver: &net.Resolver{ + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + return nd.DialContext(ctx, "udp", o.DnsResolver) }, - } - return dialer.DialTimeout(addr, tcpTimeout) + }, + } + + d = func(address string) (net.Conn, error) { + return td.DialTimeout(address, tcpTimeout) } } diff --git a/fasthttp_http_client_factory_test.go b/fasthttp_http_client_factory_test.go index 047d615..3195090 100644 --- a/fasthttp_http_client_factory_test.go +++ b/fasthttp_http_client_factory_test.go @@ -11,5 +11,5 @@ func TestFasthttpHttpClientFactoryCreateWithProxy(t *testing.T) { } func TestFasthttpHttpClientFactoryCreateWithCustomDnsResolver(t *testing.T) { - newFasthttpHttpClientFactory().Create(httpClientOptions{CustomDnsAddr: "1.1.1.1:53"}) + newFasthttpHttpClientFactory().Create(httpClientOptions{DnsResolver: "1.1.1.1:53"}) } diff --git a/http_client_factory.go b/http_client_factory.go index cb2ef7e..0549dc1 100644 --- a/http_client_factory.go +++ b/http_client_factory.go @@ -13,7 +13,7 @@ type httpClientOptions struct { SkipTLSVerification bool Timeout time.Duration Header http.Header - CustomDnsAddr string + DnsResolver string } type httpClientFactory interface {