diff --git a/datasrcs/radb.go b/datasrcs/radb.go deleted file mode 100644 index 2538c61c0..000000000 --- a/datasrcs/radb.go +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright © by Jeff Foley 2017-2023. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. -// SPDX-License-Identifier: Apache-2.0 - -package datasrcs - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "strconv" - "strings" - "time" - - amassnet "github.com/owasp-amass/amass/v3/net" - "github.com/owasp-amass/amass/v3/net/http" - "github.com/owasp-amass/amass/v3/requests" - "github.com/owasp-amass/amass/v3/systems" - "github.com/owasp-amass/resolve" - "github.com/caffix/service" - "github.com/caffix/stringset" - "github.com/miekg/dns" -) - -const ( - // radbWhoisURL is the URL for the RADb whois server. - radbWhoisURL = "whois.radb.net" -) - -// RADb is the Service that handles access to the RADb data source. -type RADb struct { - service.BaseService - - SourceType string - sys systems.System - addr string -} - -// NewRADb returns he object initialized, but not yet started. -func NewRADb(sys systems.System) *RADb { - r := &RADb{ - SourceType: requests.API, - sys: sys, - } - - go r.requests() - r.BaseService = *service.NewBaseService(r, "RADb") - return r -} - -// Description implements the Service interface. -func (r *RADb) Description() string { - return r.SourceType -} - -// OnStart implements the Service interface. -func (r *RADb) OnStart() error { - msg := resolve.QueryMsg(radbWhoisURL, dns.TypeA) - if resp, err := r.sys.TrustedResolvers().QueryBlocking(context.TODO(), msg); err == nil { - if ans := resolve.ExtractAnswers(resp); len(ans) > 0 { - ip := ans[0].Data - if ip != "" { - r.addr = ip - } - } - } - r.SetRateLimit(1) - return nil -} - -func (r *RADb) registryRADbURL(registry string) string { - var url string - - switch registry { - case "arin": - url = "https://rdap.arin.net/registry/" - case "ripencc": - url = "https://rdap.db.ripe.net/" - case "apnic": - url = "https://rdap.apnic.net/" - case "lacnic": - url = "https://rdap.lacnic.net/rdap/" - case "afrinic": - url = "https://rdap.afrinic.net/rdap/" - } - return url -} - -func (r *RADb) requests() { - for { - select { - case <-r.Done(): - return - case in := <-r.Input(): - switch req := in.(type) { - case *requests.ASNRequest: - r.CheckRateLimit() - r.asnRequest(context.TODO(), req) - } - } - } -} - -func (r *RADb) asnRequest(ctx context.Context, req *requests.ASNRequest) { - if req.Address == "" && req.ASN == 0 { - return - } - - r.CheckRateLimit() - if req.Address != "" { - r.executeASNAddrQuery(ctx, req.Address) - return - } - r.executeASNQuery(ctx, req.ASN, "", "") -} - -func (r *RADb) executeASNAddrQuery(ctx context.Context, addr string) { - url := r.getIPURL("arin", addr) - headers := map[string]string{"Content-Type": "application/json"} - resp, err := http.RequestWebPage(ctx, &http.Request{ - URL: url, - Header: headers, - }) - if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 400 { - r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err) - return - } - - var m struct { - Version string `json:"ipVersion"` - ClassName string `json:"objectClassName"` // should be 'ip network' - CIDRs []struct { - V4Prefix string `json:"v4prefix"` - V6Prefix string `json:"v6prefix"` - Length int `json:"length"` - } `json:"cidr0_cidrs"` - } - if err := json.Unmarshal([]byte(resp.Body), &m); err != nil { - r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err) - return - } else if m.ClassName != "ip network" || len(m.CIDRs) == 0 { - r.sys.Config().Log.Printf("%s: %s: The request returned zero results", r.String(), url) - return - } - - var prefix string - switch m.Version { - case "v4": - prefix = m.CIDRs[0].V4Prefix - case "v6": - prefix = m.CIDRs[0].V6Prefix - } - if prefix == "" { - return - } - - cidr := prefix + "/" + strconv.Itoa(m.CIDRs[0].Length) - if asn := r.ipToASN(ctx, cidr); asn != 0 { - r.executeASNQuery(ctx, asn, addr, cidr) - } -} - -func (r *RADb) getIPURL(registry, addr string) string { - format := r.registryRADbURL(registry) + "ip/%s" - - return fmt.Sprintf(format, addr) -} - -func (r *RADb) executeASNQuery(ctx context.Context, asn int, addr, prefix string) { - if asn == 0 { - return - } - - numRateLimitChecks(r, 2) - url := r.getASNURL("arin", strconv.Itoa(asn)) - headers := map[string]string{"Content-Type": "application/json"} - resp, err := http.RequestWebPage(ctx, &http.Request{ - URL: url, - Header: headers, - }) - if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 400 { - r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err) - return - } - - var m struct { - ClassName string `json:"objectClassName"` // interested in "autnum" - Description string `json:"name"` - Dates []struct { - Action string `json:"eventAction"` // interested in "registration" - Date string `json:"eventDate"` - } - } - if err := json.Unmarshal([]byte(resp.Body), &m); err != nil { - r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err) - return - } else if m.ClassName != "autnum" { - r.sys.Config().Log.Printf("%s: %s: The query returned incorrect results", r.String(), url) - return - } - - var registration string - for _, event := range m.Dates { - if event.Action == "registration" { - registration = event.Date - break - } - } - - var at time.Time - if registration != "" { - d, err := time.Parse(time.RFC3339, registration) - if err == nil { - at = d - } - } - - numRateLimitChecks(r, 2) - blocks := stringset.New() - defer blocks.Close() - - if prefix != "" { - blocks.Insert(prefix) - } - - nb := r.netblocks(ctx, asn) - defer nb.Close() - - blocks.Union(nb) - if blocks.Len() == 0 { - r.sys.Config().Log.Printf("%s: %s: The query returned zero netblocks", r.String(), url) - return - } - - r.sys.Cache().Update(&requests.ASNRequest{ - Address: addr, - ASN: asn, - Prefix: prefix, - AllocationDate: at, - Description: m.Description, - Netblocks: blocks.Slice(), - Tag: r.SourceType, - Source: r.String(), - }) -} - -func (r *RADb) getASNURL(registry, asn string) string { - format := r.registryRADbURL(registry) + "autnum/%s" - - return fmt.Sprintf(format, asn) -} - -func (r *RADb) netblocks(ctx context.Context, asn int) *stringset.Set { - netblocks := stringset.New() - - numRateLimitChecks(r, 2) - url := r.getNetblocksURL(strconv.Itoa(asn)) - headers := map[string]string{"Content-Type": "application/json"} - resp, err := http.RequestWebPage(ctx, &http.Request{ - URL: url, - Header: headers, - }) - if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 400 { - r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err) - return netblocks - } - - var m struct { - Results []struct { - Version string `json:"ipVersion"` - ClassName string `json:"objectClassName"` // should be 'ip network' - CIDRs []struct { - V4Prefix string `json:"v4prefix"` - V6Prefix string `json:"v6prefix"` - Length int `json:"length"` - } `json:"cidr0_cidrs"` - } `json:"arin_originas0_networkSearchResults"` - } - if err := json.Unmarshal([]byte(resp.Body), &m); err != nil { - r.sys.Config().Log.Printf("%s: %s: %v", r.String(), url, err) - return netblocks - } - - for _, block := range m.Results { - if block.ClassName != "ip network" { - continue - } - - for _, cidr := range block.CIDRs { - var prefix string - - switch block.Version { - case "v4": - prefix = cidr.V4Prefix - case "v6": - prefix = cidr.V6Prefix - } - - if prefix != "" { - l := strconv.Itoa(cidr.Length) - - netblocks.Insert(prefix + "/" + l) - } - } - } - - if netblocks.Len() == 0 { - r.sys.Config().Log.Printf("%s: Failed to acquire netblocks for ASN %d", r.String(), asn) - } - return netblocks -} - -func (r *RADb) getNetblocksURL(asn string) string { - format := "https://rdap.arin.net/registry/arin_originas0_networksbyoriginas/%s" - - return fmt.Sprintf(format, asn) -} - -func (r *RADb) ipToASN(ctx context.Context, cidr string) int { - numRateLimitChecks(r, 2) - if r.addr == "" { - msg := resolve.QueryMsg(radbWhoisURL, dns.TypeA) - resp, err := r.sys.TrustedResolvers().QueryBlocking(ctx, msg) - if err != nil { - r.sys.Config().Log.Printf("%s: %s: %v", r.String(), radbWhoisURL, err) - return 0 - } - - ans := resolve.ExtractAnswers(resp) - if len(ans) == 0 { - return 0 - } - - ip := ans[0].Data - if ip == "" { - r.sys.Config().Log.Printf("%s: Failed to resolve %s", r.String(), radbWhoisURL) - return 0 - } - r.addr = ip - } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - conn, err := amassnet.DialContext(ctx, "tcp", r.addr+":43") - if err != nil { - r.sys.Config().Log.Printf("%s: %v", r.String(), err) - return 0 - } - defer conn.Close() - - fmt.Fprintf(conn, "!r%s,o\n", cidr) - - var asn int - scanner := bufio.NewScanner(conn) - for scanner.Scan() { - line := scanner.Text() - - if err := scanner.Err(); err != nil { - continue - } - if !strings.HasPrefix(line, "AS") { - continue - } - - line2 := strings.ReplaceAll(line, "AS", "") - nums := strings.Split(strings.TrimSpace(line2), " ") - n := strings.TrimSpace(nums[len(nums)-1]) - if n == "" { - continue - } - asn, err = strconv.Atoi(n) - if err != nil { - asn = 0 - } - } - return asn -} diff --git a/datasrcs/scripting/dns.go b/datasrcs/scripting/dns.go index f747faf41..89a932bbb 100644 --- a/datasrcs/scripting/dns.go +++ b/datasrcs/scripting/dns.go @@ -13,11 +13,11 @@ import ( "sync" "time" + "github.com/miekg/dns" amassnet "github.com/owasp-amass/amass/v3/net" amassdns "github.com/owasp-amass/amass/v3/net/dns" "github.com/owasp-amass/amass/v3/requests" "github.com/owasp-amass/resolve" - "github.com/miekg/dns" bf "github.com/tylertreat/BoomFilters" lua "github.com/yuin/gopher-lua" "golang.org/x/net/publicsuffix" @@ -26,13 +26,21 @@ import ( const ( defaultSweepSize = 250 activeSweepSize = 500 + maxSweepSize = 1000 ) var ( sweepLock sync.Mutex + sweepMaxCh chan struct{} = make(chan struct{}, maxSweepSize) sweepFilter *bf.StableBloomFilter = bf.NewDefaultStableBloomFilter(1000000, 0.01) ) +func init() { + for i := 0; i < maxSweepSize; i++ { + sweepMaxCh <- struct{}{} + } +} + // Wrapper so that scripts can make DNS queries. func (s *Script) resolve(L *lua.LState) int { ctx, err := extractContext(L.CheckUserData(1)) @@ -85,7 +93,7 @@ func (s *Script) resolve(L *lua.LState) int { func (s *Script) fwdQuery(ctx context.Context, name string, qtype uint16) (*dns.Msg, error) { msg := resolve.QueryMsg(name, qtype) - resp, err := s.dnsQuery(ctx, msg, s.sys.Resolvers(), 50) + resp, err := s.dnsQuery(ctx, msg, s.sys.Resolvers(), 5) if err != nil { return resp, err } @@ -93,7 +101,7 @@ func (s *Script) fwdQuery(ctx context.Context, name string, qtype uint16) (*dns. return nil, errors.New("query failed") } - resp, err = s.dnsQuery(ctx, msg, s.sys.TrustedResolvers(), 50) + resp, err = s.dnsQuery(ctx, msg, s.sys.TrustedResolvers(), 3) if resp == nil && err == nil { err = errors.New("query failed") } @@ -190,7 +198,6 @@ func (s *Script) reverseSweep(L *lua.LState) int { } var count int - ch := make(chan *resolve.ExtractedAnswer, 10) for _, ip := range amassnet.CIDRSubset(cidr, addr, size) { select { case <-ctx.Done(): @@ -202,47 +209,38 @@ func (s *Script) reverseSweep(L *lua.LState) int { sweepLock.Lock() if a := ip.String(); !sweepFilter.TestAndAdd([]byte(a)) { count++ - go s.getPTR(ctx, a, ch) + <-sweepMaxCh + go s.getPTR(ctx, a, sweepMaxCh) } sweepLock.Unlock() } - var records []*resolve.ExtractedAnswer - for i := 0; i < count; i++ { - if rr := <-ch; rr != nil { - records = append(records, rr) - } - } - - for _, rr := range records { - s.newPTR(ctx, rr) - } L.Push(lua.LNil) return 1 } -func (s *Script) getPTR(ctx context.Context, addr string, ch chan *resolve.ExtractedAnswer) { +func (s *Script) getPTR(ctx context.Context, addr string, ch chan struct{}) { + defer func() { ch <- struct{}{} }() + if reserved, _ := amassnet.IsReservedAddress(addr); reserved { - ch <- nil return } msg := resolve.ReverseMsg(addr) - resp, err := s.dnsQuery(ctx, msg, s.sys.Resolvers(), 10) + resp, err := s.dnsQuery(ctx, msg, s.sys.Resolvers(), 5) if err != nil || resp == nil { - ch <- nil return } - resp, err = s.dnsQuery(ctx, msg, s.sys.TrustedResolvers(), 10) + resp, err = s.dnsQuery(ctx, msg, s.sys.TrustedResolvers(), 3) if err != nil || resp == nil { - ch <- nil return } if ans := resolve.ExtractAnswers(resp); len(ans) > 0 { if records := resolve.AnswersByType(ans, dns.TypePTR); len(records) > 0 { - ch <- records[0] + s.newPTR(ctx, records[0]) + return } } } diff --git a/datasrcs/scripting/http.go b/datasrcs/scripting/http.go index 3fa4019c2..147b928a3 100644 --- a/datasrcs/scripting/http.go +++ b/datasrcs/scripting/http.go @@ -8,6 +8,7 @@ import ( "context" "net/url" "strings" + "time" "github.com/owasp-amass/amass/v3/net/dns" "github.com/owasp-amass/amass/v3/net/http" @@ -197,6 +198,9 @@ func (s *Script) req(ctx context.Context, url, data string, hdr http.Header, aut } numRateLimitChecks(s, s.seconds) + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + resp, err := http.RequestWebPage(ctx, &http.Request{ URL: url, Method: method, @@ -223,29 +227,33 @@ func (s *Script) crawl(L *lua.LState) int { } u := L.CheckString(2) - if u != "" { - max := L.CheckInt(3) + if u == "" { + return 0 + } - err = http.Crawl(ctx, u, cfg.Domains(), max, func(req *http.Request, resp *http.Response) { - if u, err := url.Parse(req.URL); err == nil { - s.genNewName(ctx, http.CleanName(u.Hostname())) - } - s.internalSendNames(ctx, resp.Body) + max := L.CheckInt(3) + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() - if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 { - for _, name := range http.NamesFromCert(resp.TLS.PeerCertificates[0]) { - s.newNameWithSrc(ctx, http.CleanName(name), "cert", "Active Cert") - } + err = http.Crawl(ctx, u, cfg.Domains(), max, func(req *http.Request, resp *http.Response) { + if u, err := url.Parse(req.URL); err == nil { + s.genNewName(ctx, http.CleanName(u.Hostname())) + } + s.internalSendNames(ctx, resp.Body) + + if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 { + for _, name := range http.NamesFromCert(resp.TLS.PeerCertificates[0]) { + s.newNameWithSrc(ctx, http.CleanName(name), "cert", "Active Cert") } - for k, v := range resp.Header { - if k == "Content-Security-Policy" || - k == "Content-Security-Policy-Report-Only" || - k == "X-Content-Security-Policy" || k == "X-Webkit-CSP" { - s.internalSendNamesWithSrc(ctx, v, s.Description(), "CSP Header") - } + } + for k, v := range resp.Header { + if k == "Content-Security-Policy" || + k == "Content-Security-Policy-Report-Only" || + k == "X-Content-Security-Policy" || k == "X-Webkit-CSP" { + s.internalSendNamesWithSrc(ctx, v, s.Description(), "CSP Header") } - }) - } + } + }) if err != nil && cfg.Verbose { cfg.Log.Printf("%s: %s: %v", s.String(), u, err) diff --git a/datasrcs/sources.go b/datasrcs/sources.go index 4cc85fc73..fc5ae8c51 100644 --- a/datasrcs/sources.go +++ b/datasrcs/sources.go @@ -7,16 +7,16 @@ package datasrcs import ( "sort" + "github.com/caffix/service" + "github.com/caffix/stringset" "github.com/owasp-amass/amass/v3/config" "github.com/owasp-amass/amass/v3/datasrcs/scripting" "github.com/owasp-amass/amass/v3/systems" - "github.com/caffix/service" - "github.com/caffix/stringset" ) // GetAllSources returns a slice of all data source services initialized. func GetAllSources(sys systems.System) []service.Service { - srvs := []service.Service{NewRADb(sys)} + var srvs []service.Service if scripts, err := sys.Config().AcquireScripts(); err == nil { for _, script := range scripts { @@ -62,9 +62,3 @@ func SelectedDataSources(cfg *config.Config, avail []service.Service) []service. }) return results } - -func numRateLimitChecks(srv service.Service, num int) { - for i := 0; i < num; i++ { - srv.CheckRateLimit() - } -} diff --git a/enum/enum.go b/enum/enum.go index 7cce83e46..0665b9cea 100644 --- a/enum/enum.go +++ b/enum/enum.go @@ -20,30 +20,34 @@ import ( // Enumeration is the object type used to execute a DNS enumeration. type Enumeration struct { - Config *config.Config - Sys systems.System - ctx context.Context - graph *netmap.Graph - srcs []service.Service - done chan struct{} - nameSrc *enumSource - subTask *subdomainTask - dnsTask *dnsTask - valTask *dnsTask - store *dataManager - requests queue.Queue - plock sync.Mutex - pending bool + Config *config.Config + Sys systems.System + ctx context.Context + graph *netmap.Graph + srcs []service.Service + done chan struct{} + nameSrc *enumSource + subTask *subdomainTask + dnsTask *dnsTask + valTask *dnsTask + store *dataManager + requests queue.Queue + plock sync.Mutex + pending bool + reqCountSig chan struct{} + reqCountChan chan map[string]int } // NewEnumeration returns an initialized Enumeration that has not been started yet. func NewEnumeration(cfg *config.Config, sys systems.System, graph *netmap.Graph) *Enumeration { return &Enumeration{ - Config: cfg, - Sys: sys, - graph: graph, - srcs: datasrcs.SelectedDataSources(cfg, sys.DataSources()), - requests: queue.NewQueue(), + Config: cfg, + Sys: sys, + graph: graph, + srcs: datasrcs.SelectedDataSources(cfg, sys.DataSources()), + requests: queue.NewQueue(), + reqCountSig: make(chan struct{}, 1), + reqCountChan: make(chan map[string]int, 1), } } @@ -154,6 +158,14 @@ loop: break loop case <-e.ctx.Done(): break loop + case <-e.reqCountSig: + resp := make(map[string]int) + for k, v := range requestsMap { + if l := len(v); l > 0 { + resp[k] = l + } + } + e.reqCountChan <- resp case <-e.requests.Signal(): element, ok := e.requests.Next() if !ok { diff --git a/enum/input.go b/enum/input.go index e7b32985f..0266bf5ed 100644 --- a/enum/input.go +++ b/enum/input.go @@ -6,6 +6,7 @@ package enum import ( "context" + "fmt" "regexp" "strconv" "sync" @@ -189,10 +190,25 @@ func (r *enumSource) Next(ctx context.Context) bool { r.markDone() return false case <-t.C: - if !r.enum.requestsPending() && r.pipeline.DataItemCount() <= 0 { + count := r.pipeline.DataItemCount() + if !r.enum.requestsPending() && count <= 0 { r.markDone() return false } + if r.enum.Config.Verbose { + r.enum.reqCountSig <- struct{}{} + reqs := <-r.enum.reqCountChan + + var lines string + for k, v := range reqs { + lines += fmt.Sprintf("%s: %d requests\n", k, v) + } + + r.enum.Config.Log.Printf("Input Source timed out, not terminating: %d data items on the pipeline\n", count) + if len(reqs) > 0 { + r.enum.Config.Log.Printf("Number of pending requests:\n%s", lines) + } + } r.fillQueue() t.Reset(waitForDuration) case <-r.queue.Signal():