diff --git a/README.md b/README.md index 06eafc96..3b32f7f7 100644 --- a/README.md +++ b/README.md @@ -443,17 +443,25 @@ install npm packages (this command handles the above) `npm run build` ### To recompile Golang files in the golang folder + +All + +`npm run build:go` + Windows -`npm run build:windows` +`npm run build:go:windows:amd64` Linux -`npm run build:linux` +`npm run build:go:linux:amd64` Mac -`npm run build:mac:` +`npm run build:go:mac:arm64` + + +You can view the available compile options within the `package.json` ## Questions @@ -672,9 +680,9 @@ import ( func main() { client := cycletls.Init() jar, err := cookiejar.New(nil) - if err != nil { - log.Fatal(err) - } + if err != nil { + log.Fatal(err) + } // First request to set cookie firstResponse, err := client.Do("https://httpbin.org/cookies/set?a=1&b=2&c=3", cycletls.Options{ Body: "", @@ -687,17 +695,17 @@ func main() { log.Fatal(err) } firstURL, _ := url.Parse(firstResponse.FinalUrl) - jar.SetCookies( firstURL, firstResponse.Cookies) + jar.SetCookies( firstURL, firstResponse.Cookies) // Second request to verify cookies, including the cookies from the first response secondResponse, err := client.Do("https://httpbin.org/cookies", cycletls.Options{ - Body: "", - Ja3: "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0", - UserAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", - Headers: map[string]string{ - "Cookie": getHeadersFromJar(jar, firstURL), - }, + Body: "", + Ja3: "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0", + UserAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + Headers: map[string]string{ + "Cookie": getHeadersFromJar(jar, firstURL), + }, }, "GET") if err != nil { log.Fatal(err) diff --git a/cycletls/index.go b/cycletls/index.go index 7650ea20..a3c59c31 100644 --- a/cycletls/index.go +++ b/cycletls/index.go @@ -5,7 +5,7 @@ import ( "flag" http "github.com/Danny-Dasilva/fhttp" "github.com/gorilla/websocket" - "io/ioutil" + "io" "log" nhttp "net/http" "net/url" @@ -176,7 +176,6 @@ func processRequest(request cycleTLSRequest) (result fullRequest) { func dispatcher(res fullRequest) (response Response, err error) { defer res.client.CloseIdleConnections() finalUrl := res.options.Options.URL - resp, err := res.client.Do(res.req) if err != nil { @@ -195,8 +194,8 @@ func dispatcher(res fullRequest) (response Response, err error) { encoding := resp.Header["Content-Encoding"] content := resp.Header["Content-Type"] + bodyBytes, err := io.ReadAll(resp.Body) - bodyBytes, err := ioutil.ReadAll(resp.Body) if err != nil { log.Print("Parse Bytes" + err.Error()) return response, err @@ -353,7 +352,7 @@ func WSEndpoint(w nhttp.ResponseWriter, r *nhttp.Request) { if err != nil { //Golang Received a non-standard request to this port, printing request var data map[string]interface{} - bodyBytes, err := ioutil.ReadAll(r.Body) + bodyBytes, err := io.ReadAll(r.Body) if err != nil { log.Print("Invalid Request: Body Read Error" + err.Error()) } diff --git a/cycletls/roundtripper.go b/cycletls/roundtripper.go index bf85c309..65a762b2 100644 --- a/cycletls/roundtripper.go +++ b/cycletls/roundtripper.go @@ -144,7 +144,7 @@ func (rt *roundTripper) dialTLS(ctx context.Context, network, addr string) (net. rt.cachedTransports[addr] = &t2 default: // Assume the remote peer is speaking HTTP 1.x + TLS. - rt.cachedTransports[addr] = &http.Transport{DialTLSContext: rt.dialTLS} + rt.cachedTransports[addr] = &http.Transport{DialTLSContext: rt.dialTLS, DisableKeepAlives: true} } diff --git a/cycletls/tests/integration/images_test.go b/cycletls/tests/integration/images_test.go index a8fe68fc..d5903dae 100644 --- a/cycletls/tests/integration/images_test.go +++ b/cycletls/tests/integration/images_test.go @@ -1,12 +1,11 @@ package cycletls_test import ( - "runtime" "bytes" "encoding/base64" - "io/ioutil" "log" "os" + "runtime" "testing" cycletls "github.com/Danny-Dasilva/CycleTLS/cycletls" @@ -66,13 +65,13 @@ func WriteFile(Body string, Filepath string) { } func CompareFiles(filepath1 string, filepath2 string) bool { - f1, err1 := ioutil.ReadFile(filepath1) + f1, err1 := os.ReadFile(filepath1) if err1 != nil { log.Fatal(err1) } - f2, err2 := ioutil.ReadFile(filepath2) + f2, err2 := os.ReadFile(filepath2) if err2 != nil { log.Fatal(err2) diff --git a/cycletls/utils.go b/cycletls/utils.go index 4344a980..2dc990f2 100644 --- a/cycletls/utils.go +++ b/cycletls/utils.go @@ -7,10 +7,9 @@ import ( "crypto/sha256" "encoding/base64" "encoding/json" - "io/ioutil" "strconv" "strings" - + "io" "github.com/andybalholm/brotli" utls "github.com/refraction-networking/utls" ) @@ -85,7 +84,7 @@ func gUnzipData(data []byte) (resData []byte, err error) { return []byte{}, err } defer gz.Close() - respBody, err := ioutil.ReadAll(gz) + respBody, err := io.ReadAll(gz) return respBody, err } func enflateData(data []byte) (resData []byte, err error) { @@ -94,12 +93,12 @@ func enflateData(data []byte) (resData []byte, err error) { return []byte{}, err } defer zr.Close() - enflated, err := ioutil.ReadAll(zr) + enflated, err := io.ReadAll(zr) return enflated, err } func unBrotliData(data []byte) (resData []byte, err error) { br := brotli.NewReader(bytes.NewReader(data)) - respBody, err := ioutil.ReadAll(br) + respBody, err := io.ReadAll(br) return respBody, err } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 37867024..06bc51a1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,16 @@ # CycleTLS Changelog +## 1.0.24 - (11-27-2023) +### Release Highlights +Small bugfixes and ReadMe updates + +### Enhancements +- Update docs +- Add tests for forceHTTP1 +- update to using io instead of ioutil +### Bug Fixes +- Fix issue with keepalives on closed connections when http1 is forced + ## 1.0.23 - (11-27-2023) ### Release Highlights Switch to UTLS diff --git a/tests/forceHTTP1.test.ts b/tests/forceHTTP1.test.ts new file mode 100644 index 00000000..255e65c7 --- /dev/null +++ b/tests/forceHTTP1.test.ts @@ -0,0 +1,49 @@ +import initCycleTLS, { CycleTLSClient } from "../dist/index.js"; + +describe("CycleTLS HTTP Version Tests", () => { + let cycleTLS: CycleTLSClient; + let ja3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0"; + let userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36"; + + beforeAll(async () => { + cycleTLS = await initCycleTLS({ port: 9134 }); + }); + + afterAll(() => { + cycleTLS.exit(); + }); + + test("Should use HTTP/2 by default", async () => { + const response = await cycleTLS( + "https://tls.peet.ws/api/all", + { + body: "", + ja3: ja3, + userAgent: userAgent, + forceHTTP1: false, + }, + "get" + ); + + expect(response.status).toBe(200); + let fullResp = response.body as { http_version: string }; + expect(fullResp.http_version).toBe("h2"); + }); + + test("Should force HTTP/1.1 when specified", async () => { + const response = await cycleTLS( + "https://tls.peet.ws/api/all", + { + body: "", + ja3: ja3, + userAgent: userAgent, + forceHTTP1: true, + }, + "get" + ); + + expect(response.status).toBe(200); + let fullResp = response.body as { http_version: string }; + expect(fullResp.http_version).toBe("HTTP/1.1"); + }); +}); diff --git a/tests/golang/multipart_wip.go b/tests/golang/multipart_wip.go deleted file mode 100644 index 29934064..00000000 --- a/tests/golang/multipart_wip.go +++ /dev/null @@ -1,154 +0,0 @@ -package main - -import ( - "bytes" - "compress/gzip" - "compress/zlib" - "encoding/base64" - "io" - "io/ioutil" - "log" - "mime/multipart" - "net/http" - "os" - "path" - "path/filepath" - "strings" - - "github.com/andybalholm/brotli" -) - -func gUnzipData(data []byte) (resData []byte, err error) { - gz, err := gzip.NewReader(bytes.NewReader(data)) - if err != nil { - return []byte{}, err - } - defer gz.Close() - respBody, err := ioutil.ReadAll(gz) - return respBody, err -} -func enflateData(data []byte) (resData []byte, err error) { - zr, err := zlib.NewReader(bytes.NewReader(data)) - if err != nil { - return []byte{}, err - } - defer zr.Close() - enflated, err := ioutil.ReadAll(zr) - return enflated, err -} -func unBrotliData(data []byte) (resData []byte, err error) { - br := brotli.NewReader(bytes.NewReader(data)) - respBody, err := ioutil.ReadAll(br) - return respBody, err -} -func DecompressBody(Body []byte, encoding []string, content []string) (parsedBody string) { - if len(encoding) > 0 { - if encoding[0] == "gzip" { - unz, err := gUnzipData(Body) - if err != nil { - return string(Body) - } - return string(unz) - } else if encoding[0] == "deflate" { - unz, err := enflateData(Body) - if err != nil { - return string(Body) - } - return string(unz) - } else if encoding[0] == "br" { - unz, err := unBrotliData(Body) - if err != nil { - return string(Body) - } - return string(unz) - } - } else if len(content) > 0 { - decodingTypes := map[string]bool{ - "image/svg+xml": true, - "image/webp": true, - "image/jpeg": true, - "image/png": true, - } - if decodingTypes[content[0]] { - return base64.StdEncoding.EncodeToString(Body) - } - } - parsedBody = string(Body) - return parsedBody - -} - -// func main() { -// fileDir, _ := os.Getwd() -// fileName := "README.md" -// filePath := path.Join(fileDir, fileName) - -// file, _ := os.Open(filePath) -// defer file.Close() - -// body := &bytes.Buffer{} -// writer := multipart.NewWriter(body) -// part, _ := writer.CreateFormFile("file", filepath.Base(file.Name())) -// io.Copy(part, file) -// writer.Close() - -// r, _ := http.NewRequest("POST", "http://httpbin.org/post", body) -// r.Header.Add("Content-Type", writer.FormDataContentType()) -// client := &http.Client{} -// resp, err := client.Do(r) -// if err != nil { -// log.Println("err") -// } -// bodyBytes, err := ioutil.ReadAll(resp.Body) -// encoding := resp.Header["Content-Encoding"] -// content := resp.Header["Content-Type"] - -// Body := DecompressBody(bodyBytes, encoding, content) -// log.Println(Body) - -// } - -func main() { - fileDir, _ := os.Getwd() - fileName := "README.md" - filePath := path.Join(fileDir, fileName) - file, _ := os.Open(filePath) - _ = filepath.Base(file.Name()) - - defer file.Close() - - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - fw, err := writer.CreateFormField("name") - if err != nil { - } - _, err = io.Copy(fw, strings.NewReader("John")) - if err != nil { - log.Println("err") - } - /////////////// - fw, err = writer.CreateFormField("age") - if err != nil { - } - _, err = io.Copy(fw, strings.NewReader("23")) - if err != nil { - log.Println("err") - } - ////////// - writer.Close() - - r, _ := http.NewRequest("POST", "http://httpbin.org/post", body) - r.Header.Add("Content-Type", writer.FormDataContentType()) - client := &http.Client{} - resp, err := client.Do(r) - if err != nil { - log.Println("err") - } - bodyBytes, err := ioutil.ReadAll(resp.Body) - encoding := resp.Header["Content-Encoding"] - content := resp.Header["Content-Type"] - - Body := DecompressBody(bodyBytes, encoding, content) - log.Println(Body) - -}