Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scraper): improve charset detection regex to accurately parse meta tags (#1265)

Browse Source
This commit is contained in:
Grass Huang
2025-02-27 00:31:06 +08:00
committed by GitHub
parent bf1a79588e
commit 7bf04d409a
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
+1 -1
View File
@@ -74,7 +74,7 @@ export async function scrapeURLWithFetch(
const buf = Buffer.from(await x.arrayBuffer()); const buf = Buffer.from(await x.arrayBuffer());
let text = buf.toString("utf8"); let text = buf.toString("utf8");
const charset = (text.match(/charset=["']?(.+?)["']?>/) ?? [])[1] const charset = (text.match(/<meta\b[^>]*charset\s*=\s*["']?([^"'\s\/>]+)/i) ?? [])[1]
try { try {
if (charset) { if (charset) {
text = new TextDecoder(charset.trim()).decode(buf); text = new TextDecoder(charset.trim()).decode(buf);