Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(scrapeURL): includeTags/excludeTags

Browse Source
This commit is contained in:
Gergő Móricz
2024-11-07 21:10:27 +01:00
parent 8d467c8ca7
commit 552d55c8fc
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
+2 -2
View File
@@ -56,7 +56,7 @@ export const removeUnwantedElements = (
) => { ) => {
const soup = load(html); const soup = load(html);
if (scrapeOptions.includeTags && scrapeOptions.includeTags.length > 0) { if (scrapeOptions.includeTags && scrapeOptions.includeTags.filter(x => x.trim().length !== 0).length > 0) {
// Create a new root element to hold the tags to keep // Create a new root element to hold the tags to keep
const newRoot = load("<div></div>")("div"); const newRoot = load("<div></div>")("div");
scrapeOptions.includeTags.forEach((tag) => { scrapeOptions.includeTags.forEach((tag) => {
@@ -69,7 +69,7 @@ export const removeUnwantedElements = (
soup("script, style, noscript, meta, head").remove(); soup("script, style, noscript, meta, head").remove();
if (scrapeOptions.excludeTags && scrapeOptions.excludeTags.length > 0) { if (scrapeOptions.excludeTags && scrapeOptions.excludeTags.filter(x => x.trim().length !== 0).length > 0) {
scrapeOptions.excludeTags.forEach((tag) => { scrapeOptions.excludeTags.forEach((tag) => {
let elementsToRemove: Cheerio<AnyNode>; let elementsToRemove: Cheerio<AnyNode>;
if (tag.startsWith("*") && tag.endsWith("*")) { if (tag.startsWith("*") && tag.endsWith("*")) {