Nicolas
123 lines
2.8 KiB
TypeScript
123 lines
2.8 KiB
TypeScript
// TODO: refactor
|
|
|
|
import { AnyNode, Cheerio, load } from "cheerio";
|
|
import { ScrapeOptions } from "../../../controllers/v1/types";
|
|
|
|
const excludeNonMainTags = [
|
|
"header",
|
|
"footer",
|
|
"nav",
|
|
"aside",
|
|
".header",
|
|
".top",
|
|
".navbar",
|
|
"#header",
|
|
".footer",
|
|
".bottom",
|
|
"#footer",
|
|
".sidebar",
|
|
".side",
|
|
".aside",
|
|
"#sidebar",
|
|
".modal",
|
|
".popup",
|
|
"#modal",
|
|
".overlay",
|
|
".ad",
|
|
".ads",
|
|
".advert",
|
|
"#ad",
|
|
".lang-selector",
|
|
".language",
|
|
"#language-selector",
|
|
".social",
|
|
".social-media",
|
|
".social-links",
|
|
"#social",
|
|
".menu",
|
|
".navigation",
|
|
"#nav",
|
|
".breadcrumbs",
|
|
"#breadcrumbs",
|
|
"#search-form",
|
|
".search",
|
|
"#search",
|
|
".share",
|
|
"#share",
|
|
".widget",
|
|
"#widget",
|
|
".cookie",
|
|
"#cookie",
|
|
];
|
|
|
|
const forceIncludeMainTags = ["#main"];
|
|
|
|
export const removeUnwantedElements = (
|
|
html: string,
|
|
scrapeOptions: ScrapeOptions,
|
|
) => {
|
|
const soup = load(html);
|
|
|
|
if (
|
|
scrapeOptions.includeTags &&
|
|
scrapeOptions.includeTags.filter((x) => x.trim().length !== 0).length > 0
|
|
) {
|
|
// Create a new root element to hold the tags to keep
|
|
const newRoot = load("<div></div>")("div");
|
|
scrapeOptions.includeTags.forEach((tag) => {
|
|
soup(tag).each((_, element) => {
|
|
newRoot.append(soup(element).clone());
|
|
});
|
|
});
|
|
return newRoot.html() ?? "";
|
|
}
|
|
|
|
soup("script, style, noscript, meta, head").remove();
|
|
|
|
if (
|
|
scrapeOptions.excludeTags &&
|
|
scrapeOptions.excludeTags.filter((x) => x.trim().length !== 0).length > 0
|
|
) {
|
|
scrapeOptions.excludeTags.forEach((tag) => {
|
|
let elementsToRemove: Cheerio<AnyNode>;
|
|
if (tag.startsWith("*") && tag.endsWith("*")) {
|
|
let classMatch = false;
|
|
|
|
const regexPattern = new RegExp(tag.slice(1, -1), "i");
|
|
elementsToRemove = soup("*").filter((i, element) => {
|
|
if (element.type === "tag") {
|
|
const attributes = element.attribs;
|
|
const tagNameMatches = regexPattern.test(element.name);
|
|
const attributesMatch = Object.keys(attributes).some((attr) =>
|
|
regexPattern.test(`${attr}="${attributes[attr]}"`),
|
|
);
|
|
if (tag.startsWith("*.")) {
|
|
classMatch = Object.keys(attributes).some((attr) =>
|
|
regexPattern.test(`class="${attributes[attr]}"`),
|
|
);
|
|
}
|
|
return tagNameMatches || attributesMatch || classMatch;
|
|
}
|
|
return false;
|
|
});
|
|
} else {
|
|
elementsToRemove = soup(tag);
|
|
}
|
|
elementsToRemove.remove();
|
|
});
|
|
}
|
|
|
|
if (scrapeOptions.onlyMainContent) {
|
|
excludeNonMainTags.forEach((tag) => {
|
|
const elementsToRemove = soup(tag).filter(
|
|
forceIncludeMainTags.map((x) => ":not(:has(" + x + "))").join(""),
|
|
);
|
|
|
|
elementsToRemove.remove();
|
|
});
|
|
}
|
|
|
|
const cleanedHtml = soup.html();
|
|
return cleanedHtml;
|
|
};
|