Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #436 from mendableai/mog/fix-infinite-regex

Browse Source 
fix(WebScraper): infinite regex leading to fly.io instance hangs
This commit is contained in:
Nicolas
2024-07-18 13:32:44 -04:00
committed by GitHub
parent 6d1d46a987 0b8047c7a0
commit d338b05446
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/WebScraper/utils/replacePaths.ts
+4 -4
View File
@@ -6,13 +6,13 @@ export const replacePathsWithAbsolutePaths = (documents: Document[]): Document[]
const baseUrl = new URL(document.metadata.sourceURL).origin; const baseUrl = new URL(document.metadata.sourceURL).origin;
const paths = const paths =
document.content.match( document.content.match(
/(!?\[.*?\])\(((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\)|href="([^"]+)"/g /!?\[.*?\]\(.*?\)|href=".+?"/g
) || []; ) || [];
paths.forEach((path: string) => { paths.forEach((path: string) => {
try { try {
const isImage = path.startsWith("!"); const isImage = path.startsWith("!");
let matchedUrl = path.match(/\(([^)]+)\)/) || path.match(/href="([^"]+)"/); let matchedUrl = path.match(/\((.*?)\)/) || path.match(/href="([^"]+)"/);
let url = matchedUrl[1]; let url = matchedUrl[1];
if (!url.startsWith("data:") && !url.startsWith("http")) { if (!url.startsWith("data:") && !url.startsWith("http")) {
@@ -50,11 +50,11 @@ export const replaceImgPathsWithAbsolutePaths = (documents: Document[]): Documen
const baseUrl = new URL(document.metadata.sourceURL).origin; const baseUrl = new URL(document.metadata.sourceURL).origin;
const images = const images =
document.content.match( document.content.match(
/!\[.*?\]\(((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*)\)/g /!\[.*?\]\(.*?\)/g
) || []; ) || [];
images.forEach((image: string) => { images.forEach((image: string) => {
let imageUrl = image.match(/\(([^)]+)\)/)[1]; let imageUrl = image.match(/\((.*?)\)/)[1];
let altText = image.match(/\[(.*?)\]/)[1]; let altText = image.match(/\[(.*?)\]/)[1];
if (!imageUrl.startsWith("data:image")) { if (!imageUrl.startsWith("data:image")) {