Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Modify DNS resolver option #400

Merged
merged 9 commits into from
Jul 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .snapshots/TestHelp
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ Application Options:
sitemap.xml (deprecated)
--header=<header>... Custom headers
-f, --ignore-fragments Ignore URL fragments
--dns-resolver=<address> Custom DNS resolver
--format=[text|json|junit] Output format (default: text)
--json Output results in JSON (deprecated)
--experimental-verbose-json Include successful results in JSON
Expand Down
2 changes: 1 addition & 1 deletion arguments.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ type arguments struct {
MaxConnections int `short:"c" long:"max-connections" value-name:"<count>" default:"512" description:"Maximum number of HTTP connections"`
MaxConnectionsPerHost int `long:"max-connections-per-host" value-name:"<count>" default:"512" description:"Maximum number of HTTP connections per host"`
MaxResponseBodySize int `long:"max-response-body-size" value-name:"<size>" default:"10000000" description:"Maximum response body size to read"`
CustomDnsAddr string `long:"custom-dns-addr" description:"Custom DNS server to be used for requests"`
RawExcludedPatterns []string `short:"e" long:"exclude" value-name:"<pattern>..." description:"Exclude URLs matched with given regular expressions"`
RawIncludedPatterns []string `short:"i" long:"include" value-name:"<pattern>..." description:"Include URLs matched with given regular expressions"`
FollowRobotsTxt bool `long:"follow-robots-txt" description:"Follow robots.txt when scraping pages"`
FollowSitemapXML bool `long:"follow-sitemap-xml" description:"Scrape only pages listed in sitemap.xml (deprecated)"`
RawHeaders []string `long:"header" value-name:"<header>..." description:"Custom headers"`
// TODO Remove a short option.
IgnoreFragments bool `short:"f" long:"ignore-fragments" description:"Ignore URL fragments"`
DnsResolver string `long:"dns-resolver" value-name:"<address>" description:"Custom DNS resolver"`
Format string `long:"format" description:"Output format" default:"text" choice:"text" choice:"json" choice:"junit"`
// TODO Remove this option.
JSONOutput bool `long:"json" description:"Output results in JSON (deprecated)"`
Expand Down
2 changes: 1 addition & 1 deletion command.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func (c *command) runWithError(ss []string) (bool, error) {
SkipTLSVerification: args.SkipTLSVerification,
Timeout: time.Duration(args.Timeout) * time.Second,
Header: args.Header,
CustomDnsAddr: args.CustomDnsAddr,
DnsResolver: args.DnsResolver,
},
),
args.RateLimit,
Expand Down
31 changes: 14 additions & 17 deletions fasthttp_http_client_factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,28 +17,25 @@ func newFasthttpHttpClientFactory() *fasthttpHttpClientFactory {
}

func (*fasthttpHttpClientFactory) Create(o httpClientOptions) httpClient {
d := func(addr string) (net.Conn, error) {
return fasthttp.DialTimeout(addr, tcpTimeout)
d := func(address string) (net.Conn, error) {
return fasthttp.DialTimeout(address, tcpTimeout)
}

if o.Proxy != "" {
d = fasthttpproxy.FasthttpHTTPDialerTimeout(o.Proxy, tcpTimeout)
}

if o.CustomDnsAddr != "" {
d = func(addr string) (net.Conn, error) {
dialer := fasthttp.TCPDialer{
Concurrency: 1000,
Resolver: &net.Resolver{
PreferGo: true,
StrictErrors: false,
Dial: func(ctx context.Context, network string, address string) (net.Conn, error) {
internalDialer := net.Dialer{}
return internalDialer.DialContext(ctx, "udp", o.CustomDnsAddr)
},
} else if o.DnsResolver != "" {
nd := &net.Dialer{}
td := &fasthttp.TCPDialer{
Concurrency: concurrency,
Resolver: &net.Resolver{
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
return nd.DialContext(ctx, "udp", o.DnsResolver)
},
}
return dialer.DialTimeout(addr, tcpTimeout)
},
}

d = func(address string) (net.Conn, error) {
return td.DialTimeout(address, tcpTimeout)
}
}

Expand Down
2 changes: 1 addition & 1 deletion fasthttp_http_client_factory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,5 @@ func TestFasthttpHttpClientFactoryCreateWithProxy(t *testing.T) {
}

func TestFasthttpHttpClientFactoryCreateWithCustomDnsResolver(t *testing.T) {
newFasthttpHttpClientFactory().Create(httpClientOptions{CustomDnsAddr: "1.1.1.1:53"})
newFasthttpHttpClientFactory().Create(httpClientOptions{DnsResolver: "1.1.1.1:53"})
}
2 changes: 1 addition & 1 deletion http_client_factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ type httpClientOptions struct {
SkipTLSVerification bool
Timeout time.Duration
Header http.Header
CustomDnsAddr string
DnsResolver string
}

type httpClientFactory interface {
Expand Down
Loading