Don't check preconnect links

lychee-lib/Cargo.toml

@@ -63,6 +63,7 @@ features = ["runtime-tokio"]
 
 [dev-dependencies]
 doc-comment = "0.3.3"
+pretty_assertions = "1.4.0"
 tempfile = "3.7.1"
 wiremock = "0.5.19"
 serde_json = "1.0.104"

lychee-lib/src/extract/html/html5ever.rs

@@ -69,6 +69,14 @@ impl TokenSink for LinkExtractor {
                     }
                 }
 
+                // Check and exclude rel=preconnect. Other than prefetch and preload,
+                // preconnect only does DNS lookups and might not be a link to a resource
+                if let Some(rel) = attrs.iter().find(|attr| &attr.name.local == "rel") {
+                    if rel.value.contains("preconnect") {
+                        return TokenSinkResult::Continue;
+                    }
+                }
+
                 for attr in attrs {
                     let urls = LinkExtractor::extract_urls_from_elem_attr(
                         &attr.name.local,
@@ -136,6 +144,8 @@ impl LinkExtractor {
         // and https://html.spec.whatwg.org/multipage/indices.html#attributes-1
 
         match (elem_name, attr_name) {
+            // TODO: Skip <link rel="preconnect">
+
             // Common element/attribute combinations for links
             (_, "href" | "src" | "cite" | "usemap")
             // Less common (but still valid!) combinations
@@ -353,4 +363,14 @@ mod tests {
         let uris = extract_html(input, false);
         assert_eq!(uris, expected);
     }
+
+    #[test]
+    fn test_skip_preconnect() {
+        let input = r#"
+            <link rel="preconnect" href="https://example.com">
+        "#;
+
+        let uris = extract_html(input, false);
+        assert!(uris.is_empty());
+    }
 }

lychee-lib/src/extract/html/html5gum.rs

@@ -4,13 +4,15 @@ use super::{is_email_link, is_verbatim_elem, srcset};
 use crate::{extract::plaintext::extract_plaintext, types::uri::raw::RawUri};
 
 #[derive(Clone)]
+#[allow(clippy::struct_excessive_bools)]
 struct LinkExtractor {
     // note: what html5gum calls a tag, lychee calls an element
     links: Vec<RawUri>,
     current_string: Vec<u8>,
     current_element_name: Vec<u8>,
     current_element_is_closing: bool,
     current_element_nofollow: bool,
+    current_element_preconnect: bool,
     current_attribute_name: Vec<u8>,
     current_attribute_value: Vec<u8>,
     last_start_element: Vec<u8>,
@@ -33,6 +35,7 @@ impl LinkExtractor {
             current_element_name: Vec::new(),
             current_element_is_closing: false,
             current_element_nofollow: false,
+            current_element_preconnect: false,
             current_attribute_name: Vec::new(),
             current_attribute_value: Vec::new(),
             last_start_element: Vec::new(),
@@ -147,7 +150,15 @@ impl LinkExtractor {
             if attr == "rel" && value.contains("nofollow") {
                 self.current_element_nofollow = true;
             }
-            if self.current_element_nofollow {
+
+            // Ignore links with rel=preconnect
+            // Other than prefetch and preload, preconnect only makes
+            // a DNS lookup, so we don't want to extract those links.
+            if attr == "rel" && value.contains("preconnect") {
+                self.current_element_preconnect = true;
+            }
+
+            if self.current_element_nofollow || self.current_element_preconnect {
                 self.current_attribute_name.clear();
                 self.current_attribute_value.clear();
                 return;
@@ -507,4 +518,14 @@ mod tests {
         let uris = extract_html(input, false);
         assert_eq!(uris, expected);
     }
+
+    #[test]
+    fn test_skip_preconnect() {
+        let input = r#"
+            <link rel="preconnect" href="https://example.com">
+        "#;
+
+        let uris = extract_html(input, false);
+        assert!(uris.is_empty());
+    }
 }

lychee-lib/src/extract/mod.rs

@@ -57,6 +57,7 @@ impl Extractor {
 
 #[cfg(test)]
 mod tests {
+    use pretty_assertions::assert_eq;
     use reqwest::Url;
     use std::{collections::HashSet, path::Path};
 
@@ -72,20 +73,33 @@ mod tests {
         let input_content = InputContent::from_string(input, file_type);
 
         let extractor = Extractor::new(false, false);
-        let uris_html5gum = extractor
+        let uris_html5gum: HashSet<Uri> = extractor
             .extract(&input_content)
             .into_iter()
             .filter_map(|raw_uri| Uri::try_from(raw_uri).ok())
             .collect();
+        let uris_html5gum_sorted: Vec<Uri> = {
+            let mut uris = uris_html5gum.clone().into_iter().collect::<Vec<_>>();
+            uris.sort();
+            uris
+        };
 
         let extractor = Extractor::new(true, false);
-        let uris_html5ever = extractor
+        let uris_html5ever: HashSet<Uri> = extractor
             .extract(&input_content)
             .into_iter()
             .filter_map(|raw_uri| Uri::try_from(raw_uri).ok())
             .collect();
+        let uris_html5ever_sorted: Vec<Uri> = {
+            let mut uris = uris_html5ever.into_iter().collect::<Vec<_>>();
+            uris.sort();
+            uris
+        };
 
-        assert_eq!(uris_html5gum, uris_html5ever);
+        assert_eq!(
+            uris_html5gum_sorted, uris_html5ever_sorted,
+            "Mismatch between html5gum and html5ever"
+        );
         uris_html5gum
     }
 
@@ -245,7 +259,8 @@ mod tests {
         let expected_links = IntoIterator::into_iter([
             website("https://example.com/"),
             website("https://example.com/favicon.ico"),
-            website("https://fonts.externalsite.com"),
+            // Note that we exclude `preconnect` links:
+            // website("https://fonts.externalsite.com"),
             website("https://example.com/docs/"),
             website("https://example.com/forum"),
         ])