Skip to content

Commit

Permalink
add in request option param: InsecureSkipVerify; (#279)
Browse files Browse the repository at this point in the history 
Co-authored-by: Kadi Gasanguseynov <kagasanguseynov@avito.ru>
  • Loading branch information
@lif0
lif0 and Kadi Gasanguseynov committed Nov 15, 2023
1 parent 0a1c895 commit 7c67106
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 36 deletions.
7 changes: 4 additions & 3 deletions cycletls/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@ import (

type browser struct {
// Return a greeting that embeds the name in a message.
JA3 string
UserAgent string
Cookies []Cookie
JA3 string
UserAgent string
Cookies []Cookie
InsecureSkipVerify bool
}

var disabledRedirect = func(req *http.Request, via []*http.Request) error {
Expand Down
42 changes: 25 additions & 17 deletions cycletls/index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,19 @@ import (

// Options sets CycleTLS client options
type Options struct {
URL string `json:"url"`
Method string `json:"method"`
Headers map[string]string `json:"headers"`
Body string `json:"body"`
Ja3 string `json:"ja3"`
UserAgent string `json:"userAgent"`
Proxy string `json:"proxy"`
Cookies []Cookie `json:"cookies"`
Timeout int `json:"timeout"`
DisableRedirect bool `json:"disableRedirect"`
HeaderOrder []string `json:"headerOrder"`
OrderAsProvided bool `json:"orderAsProvided"` //TODO
InsecureSkipVerify bool `json:"insecureSkipVerify"`
URL string `json:"url"`
Method string `json:"method"`
Headers map[string]string `json:"headers"`
Body string `json:"body"`
Ja3 string `json:"ja3"`
UserAgent string `json:"userAgent"`
Proxy string `json:"proxy"`
Cookies []Cookie `json:"cookies"`
Timeout int `json:"timeout"`
DisableRedirect bool `json:"disableRedirect"`
HeaderOrder []string `json:"headerOrder"`
OrderAsProvided bool `json:"orderAsProvided"` //TODO
}

type cycleTLSRequest struct {
Expand All @@ -48,6 +49,7 @@ type Response struct {
Status int
Body string
Headers map[string]string
FinalUrl string
}

// JSONBody converts response body to json
Expand All @@ -70,9 +72,10 @@ type CycleTLS struct {
func processRequest(request cycleTLSRequest) (result fullRequest) {

var browser = browser{
JA3: request.Options.Ja3,
UserAgent: request.Options.UserAgent,
Cookies: request.Options.Cookies,
JA3: request.Options.Ja3,
UserAgent: request.Options.UserAgent,
Cookies: request.Options.Cookies,
InsecureSkipVerify: request.Options.InsecureSkipVerify,
}

client, err := newClient(
Expand Down Expand Up @@ -170,18 +173,23 @@ func processRequest(request cycleTLSRequest) (result fullRequest) {

func dispatcher(res fullRequest) (response Response, err error) {
defer res.client.CloseIdleConnections()
finalUrl := res.options.Options.URL

resp, err := res.client.Do(res.req)
if err != nil {

parsedError := parseError(err)

headers := make(map[string]string)
return Response{res.options.RequestID, parsedError.StatusCode, parsedError.ErrorMsg + "-> \n" + string(err.Error()), headers}, nil //normally return error here
return Response{res.options.RequestID, parsedError.StatusCode, parsedError.ErrorMsg + "-> \n" + string(err.Error()), headers, finalUrl}, nil //normally return error here

}
defer resp.Body.Close()

if resp != nil && resp.Request != nil && resp.Request.URL != nil {
finalUrl = resp.Request.URL.String()
}

encoding := resp.Header["Content-Encoding"]
content := resp.Header["Content-Type"]

Expand All @@ -203,7 +211,7 @@ func dispatcher(res fullRequest) (response Response, err error) {
}
}
}
return Response{res.options.RequestID, resp.StatusCode, Body, headers}, nil
return Response{res.options.RequestID, resp.StatusCode, Body, headers, finalUrl}, nil

}

Expand Down
33 changes: 18 additions & 15 deletions cycletls/roundtripper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"sync"

http "github.com/Danny-Dasilva/fhttp"
http2 "github.com/Danny-Dasilva/fhttp/http2"
"github.com/Danny-Dasilva/fhttp/http2"
utls "github.com/Danny-Dasilva/utls"
"golang.org/x/net/proxy"
)
Expand All @@ -23,9 +23,10 @@ type roundTripper struct {
JA3 string
UserAgent string

Cookies []Cookie
cachedConnections map[string]net.Conn
cachedTransports map[string]http.RoundTripper
InsecureSkipVerify bool
Cookies []Cookie
cachedConnections map[string]net.Conn
cachedTransports map[string]http.RoundTripper

dialer proxy.ContextDialer
}
Expand Down Expand Up @@ -105,7 +106,7 @@ func (rt *roundTripper) dialTLS(ctx context.Context, network, addr string) (net.
return nil, err
}

conn := utls.UClient(rawConn, &utls.Config{ServerName: host, InsecureSkipVerify: true}, // MinVersion: tls.VersionTLS10,
conn := utls.UClient(rawConn, &utls.Config{ServerName: host, InsecureSkipVerify: rt.InsecureSkipVerify}, // MinVersion: tls.VersionTLS10,
// MaxVersion: tls.VersionTLS13,

utls.HelloCustom)
Expand Down Expand Up @@ -177,21 +178,23 @@ func newRoundTripper(browser browser, dialer ...proxy.ContextDialer) http.RoundT
return &roundTripper{
dialer: dialer[0],

JA3: browser.JA3,
UserAgent: browser.UserAgent,
Cookies: browser.Cookies,
cachedTransports: make(map[string]http.RoundTripper),
cachedConnections: make(map[string]net.Conn),
JA3: browser.JA3,
UserAgent: browser.UserAgent,
Cookies: browser.Cookies,
cachedTransports: make(map[string]http.RoundTripper),
cachedConnections: make(map[string]net.Conn),
InsecureSkipVerify: browser.InsecureSkipVerify,
}
}

return &roundTripper{
dialer: proxy.Direct,

JA3: browser.JA3,
UserAgent: browser.UserAgent,
Cookies: browser.Cookies,
cachedTransports: make(map[string]http.RoundTripper),
cachedConnections: make(map[string]net.Conn),
JA3: browser.JA3,
UserAgent: browser.UserAgent,
Cookies: browser.Cookies,
cachedTransports: make(map[string]http.RoundTripper),
cachedConnections: make(map[string]net.Conn),
InsecureSkipVerify: browser.InsecureSkipVerify,
}
}
42 changes: 41 additions & 1 deletion cycletls/tests/integration/disable_redirect_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
package cycletls_test

import (
//"fmt"
"log"
"testing"

Expand Down Expand Up @@ -45,3 +44,44 @@ func TestRedirectDisabled(t *testing.T) {
}

}

func TestRedirectEnabled_NewURL(t *testing.T) {
url := "https://rb.gy/3hwz5h"
client := cycletls.Init()
resp, err := client.Do(url, cycletls.Options{
Body: "",
Ja3: "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0",
UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36",
}, "GET")
if err != nil {
log.Print("Request Failed: " + err.Error())
}
if resp.Status != 200 {
t.Fatal("Expected {} Got {} for Status", 200, resp.Status)
}

if resp.FinalUrl == url {
t.Fatal("Expected https://www.google.com Got {} for Url", url, resp.FinalUrl)
}
}

func TestRedirectDisabled_NewURL(t *testing.T) {
url := "https://rb.gy/3hwz5h"
client := cycletls.Init()
resp, err := client.Do(url, cycletls.Options{
Body: "",
Ja3: "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0",
UserAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36",
DisableRedirect: true,
}, "GET")
if err != nil {
log.Print("Request Failed: " + err.Error())
}
if resp.Status != 301 {
t.Fatal("Expected {} Got {} for Status", 200, resp.Status)
}

if resp.FinalUrl != url {
t.Fatal("Expected {} Got {} for Url", url, resp.FinalUrl)
}
}

0 comments on commit 7c67106

Please sign in to comment.