CVE-2024-0243

Crawler downloads external links despite prevention settings.

• CVSS 8.1
• CWE-918
• Input Validation and Sanitization
• Remote

With the following crawler configuration: “`python from bs4 import BeautifulSoup as Soup url = "https://example.com" loader = RecursiveUrlLoader( url=url, max_depth=2, extractor=lambda x: Soup(x, "html.parser").text ) docs = loader.load() “` An attacker in control of the contents of `https://example.com` could place a malicious HTML file in there with links like "https://example.completely.different/my_file.html" and the crawler would proceed to download that file as well even though `prevent_outside=True`. https://github.com/langchain-ai/langchain/blob/bf0b3cc0b5ade1fb95a5b1b6fa260e99064c2e22/libs/community/langchain_community/document_loaders/recursive_url_loader.py#L51-L51 Resolved in https://github.com/langchain-ai/langchain/pull/15559

CVE

CVE-2024-0243

CWE

CWE-918

CVSS base score

8.1

Published

2024-02-26

OWASP

A01 Broken Access Control

Orthogonal defect classification

Algorithm

Code defect classification

Incorrect Algorithm

Category

Input Validation and Sanitization

Subcategory

Server-Side Request Forgery (SSRF)

Accessibility scope

Remote

Impact

Unauthorized Access

Affected component

Langchain

Fixed by upgrading

Yes

Solution

Upgrade to version 0.0.195 or later, where the vulnerability has been resolved.

from bs4 import BeautifulSoup as Soup
import requests

class RecursiveUrlLoader:
    def __init__(self, url, max_depth, extractor, prevent_outside=True):
        self.url = url
        self.max_depth = max_depth
        self.extractor = extractor
        self.prevent_outside = prevent_outside
        self.visited_urls = set()

    def load(self):
        return self._load_recursive(self.url, 0)

    def _load_recursive(self, url, depth):
        if depth > self.max_depth or url in self.visited_urls:
            return []
        
        self.visited_urls.add(url)
        response = requests.get(url)
        content = self.extractor(response.text)
        links = self.extract_links(response.text)

        documents = [content]
        for link in links:
            documents.extend(self._load_recursive(link, depth + 1))
        
        return documents

    def extract_links(self, html):
        soup = Soup(html, "html.parser")
        return [a['href'] for a in soup.find_all('a', href=True)]

url = "https://example.com"
loader = RecursiveUrlLoader(
    url=url, max_depth=2, extractor=lambda x: Soup(x, "html.parser").text
)
docs = loader.load()

from bs4 import BeautifulSoup as Soup
import requests

class RecursiveUrlLoader:
    def __init__(self, url, max_depth, extractor, prevent_outside=True):
        self.url = url
        self.max_depth = max_depth
        self.extractor = extractor
        self.prevent_outside = prevent_outside
        self.visited_urls = set()

    def check_url(self, url):
        return url.startswith(self.url)

    def load(self):
        return self._load_recursive(self.url, 0)

    def _load_recursive(self, url, depth):
        if depth > self.max_depth or url in self.visited_urls:
            return []
        
        self.visited_urls.add(url)
        response = requests.get(url)
        content = self.extractor(response.text)
        links = self.extract_links(response.text)

        documents = [content]
        for link in links:
            if self.prevent_outside and not self.check_url(link):
                continue
            documents.extend(self._load_recursive(link, depth + 1))
        
        return documents

    def extract_links(self, html):
        soup = Soup(html, "html.parser")
        return [a['href'] for a in soup.find_all('a', href=True)]

url = "https://example.com"
loader = RecursiveUrlLoader(
    url=url, max_depth=2, extractor=lambda x: Soup(x, "html.parser").text
)
docs = loader.load()

<a href="https://example.completely.different/my_file.html">Malicious Link</a>

@misc{vaitp:cve20240243,
  title        = {{Crawler downloads external links despite prevention settings.}},
  author       = {Bogaerts, Fr\'ed\'eric and Ivaki, Naghmeh and Fonseca, Jos\'e},
  year         = {2024},
  note         = {VAITP Python Vulnerability Dataset, entry CVE-2024-0243},
  howpublished = {\url{https://netpack.pt/vaitp/vulnerability/CVE-2024-0243/}}
}