WebScraper refactor into scrapeURL (#714)
* feat: use strictNullChecking * feat: switch logger to Winston * feat(scrapeURL): first batch * fix(scrapeURL): error swallow * fix(scrapeURL): add timeout to EngineResultsTracker * fix(scrapeURL): report unexpected error to sentry * chore: remove unused modules * feat(transfomers/coerce): warn when a format's response is missing * feat(scrapeURL): feature flag priorities, engine quality sorting, PDF and DOCX support * (add note) * feat(scrapeURL): wip readme * feat(scrapeURL): LLM extract * feat(scrapeURL): better warnings * fix(scrapeURL/engines/fire-engine;playwright): fix screenshot * feat(scrapeURL): add forceEngine internal option * feat(scrapeURL/engines): scrapingbee * feat(scrapeURL/transformars): uploadScreenshot * feat(scrapeURL): more intense tests * bunch of stuff * get rid of WebScraper (mostly) * adapt batch scrape * add staging deploy workflow * fix yaml * fix logger issues * fix v1 test schema * feat(scrapeURL/fire-engine/chrome-cdp): remove wait inserts on actions * scrapeURL: v0 backwards compat * logger fixes * feat(scrapeurl): v0 returnOnlyUrls support * fix(scrapeURL/v0): URL leniency * fix(batch-scrape): ts non-nullable * fix(scrapeURL/fire-engine/chromecdp): fix wait action * fix(logger): remove error debug key * feat(requests.http): use dotenv expression * fix(scrapeURL/extractMetadata): extract custom metadata * fix crawl option conversion * feat(scrapeURL): Add retry logic to robustFetch * fix(scrapeURL): crawl stuff * fix(scrapeURL): LLM extract * fix(scrapeURL/v0): search fix * fix(tests/v0): grant larger response size to v0 crawl status * feat(scrapeURL): basic fetch engine * feat(scrapeURL): playwright engine * feat(scrapeURL): add url-specific parameters * Update readme and examples * added e2e tests for most parameters. Still a few actions, location and iframes to be done. * fixed type * Nick: * Update scrape.ts * Update index.ts * added actions and base64 check * Nick: skipTls feature flag? * 403 * todo * todo * fixes * yeet headers from url specific params * add warning when final engine has feature deficit * expose engine results tracker for ScrapeEvents implementation * ingest scrape events * fixed some tests * comment * Update index.test.ts * fixed rawHtml * Update index.test.ts * update comments * move geolocation to global f-e option, fix removeBase64Images * Nick: * trim url-specific params * Update index.ts --------- Co-authored-by: Eric Ciarla <ericciarla@yahoo.com> Co-authored-by: rafaelmmiller <8574157+rafaelmmiller@users.noreply.github.com> Co-authored-by: Nicolas <nicolascamara29@gmail.com>
This commit is contained in:
Gergő Móricz
GitHub
@@ -2,7 +2,6 @@
|
||||
import { WebCrawler } from '../crawler';
|
||||
import axios from 'axios';
|
||||
import robotsParser from 'robots-parser';
|
||||
import { getAdjustedMaxDepth } from '../utils/maxDepthUtils';
|
||||
|
||||
jest.mock('axios');
|
||||
jest.mock('robots-parser');
|
||||
@@ -35,165 +34,6 @@ describe('WebCrawler', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('should filter out links that exceed maxDepth param of 2 based on enterURL depth of 0 ', async () => {
|
||||
const initialUrl = 'http://example.com'; // Set initial URL for this test
|
||||
const enteredMaxCrawledDepth = 2;
|
||||
maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
|
||||
|
||||
|
||||
crawler = new WebCrawler({
|
||||
jobId: "TEST",
|
||||
initialUrl: initialUrl,
|
||||
includes: [],
|
||||
excludes: [],
|
||||
limit: 100,
|
||||
maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
|
||||
});
|
||||
|
||||
// Mock sitemap fetching function to return controlled links
|
||||
crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
|
||||
initialUrl, // depth 0
|
||||
initialUrl + '/page1', // depth 1
|
||||
initialUrl + '/page1/page2', // depth 2
|
||||
initialUrl + '/page1/page2/page3' // depth 3, should be filtered out
|
||||
]);
|
||||
|
||||
const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
|
||||
expect(results).toEqual([
|
||||
{ url: initialUrl, html: '' },
|
||||
{ url: initialUrl + '/page1', html: '' },
|
||||
{ url: initialUrl + '/page1/page2', html: '' }
|
||||
]);
|
||||
|
||||
|
||||
// Ensure that the link with depth 3 is not included
|
||||
expect(results.some(r => r.url === initialUrl + '/page1/page2/page3')).toBe(false);
|
||||
});
|
||||
|
||||
it('should filter out links that exceed maxDepth param of 0 based on enterURL depth of 0 ', async () => {
|
||||
const initialUrl = 'http://example.com'; // Set initial URL for this test
|
||||
const enteredMaxCrawledDepth = 0;
|
||||
maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
|
||||
|
||||
|
||||
crawler = new WebCrawler({
|
||||
jobId: "TEST",
|
||||
initialUrl: initialUrl,
|
||||
includes: [],
|
||||
excludes: [],
|
||||
limit: 100,
|
||||
maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
|
||||
});
|
||||
|
||||
// Mock sitemap fetching function to return controlled links
|
||||
crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
|
||||
initialUrl, // depth 0
|
||||
initialUrl + '/page1', // depth 1
|
||||
initialUrl + '/page1/page2', // depth 2
|
||||
initialUrl + '/page1/page2/page3' // depth 3, should be filtered out
|
||||
]);
|
||||
|
||||
const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
|
||||
expect(results).toEqual([
|
||||
{ url: initialUrl, html: '' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('should filter out links that exceed maxDepth param of 1 based on enterURL depth of 1 ', async () => {
|
||||
const initialUrl = 'http://example.com/page1'; // Set initial URL for this test
|
||||
const enteredMaxCrawledDepth = 1;
|
||||
maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
|
||||
|
||||
|
||||
crawler = new WebCrawler({
|
||||
jobId: "TEST",
|
||||
initialUrl: initialUrl,
|
||||
includes: [],
|
||||
excludes: [],
|
||||
limit: 100,
|
||||
maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
|
||||
});
|
||||
|
||||
// Mock sitemap fetching function to return controlled links
|
||||
crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
|
||||
initialUrl, // depth 0
|
||||
initialUrl + '/page2', // depth 1
|
||||
initialUrl + '/page2/page3', // depth 2
|
||||
initialUrl + '/page2/page3/page4' // depth 3, should be filtered out
|
||||
]);
|
||||
|
||||
const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
|
||||
expect(results).toEqual([
|
||||
{ url: initialUrl, html: '' },
|
||||
{ url: initialUrl + '/page2', html: '' }
|
||||
]);
|
||||
});
|
||||
|
||||
it('should filter out links that exceed maxDepth param of 1 based on enterURL depth of 2 ', async () => {
|
||||
const initialUrl = 'http://example.com/page1'; // Set initial URL for this test
|
||||
const enteredMaxCrawledDepth = 2;
|
||||
maxCrawledDepth = getAdjustedMaxDepth(initialUrl, enteredMaxCrawledDepth);
|
||||
|
||||
|
||||
crawler = new WebCrawler({
|
||||
jobId: "TEST",
|
||||
initialUrl: initialUrl,
|
||||
includes: [],
|
||||
excludes: [],
|
||||
limit: 100,
|
||||
maxCrawledDepth: maxCrawledDepth, // Set maxDepth for testing
|
||||
});
|
||||
|
||||
// Mock sitemap fetching function to return controlled links
|
||||
crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
|
||||
initialUrl, // depth 0
|
||||
initialUrl + '/page2', // depth 1
|
||||
initialUrl + '/page2/page3', // depth 2
|
||||
initialUrl + '/page2/page3/page4' // depth 3, should be filtered out
|
||||
]);
|
||||
|
||||
const results = await crawler.start(undefined, undefined, undefined, undefined, undefined, maxCrawledDepth);
|
||||
expect(results).toEqual([
|
||||
{ url: initialUrl, html: '' },
|
||||
{ url: initialUrl + '/page2', html: '' },
|
||||
{ url: initialUrl + '/page2/page3', html: '' }
|
||||
]);
|
||||
});
|
||||
|
||||
it('should handle allowBackwardCrawling option correctly', async () => {
|
||||
const initialUrl = 'https://mendable.ai/blog';
|
||||
|
||||
// Setup the crawler with the specific test case options
|
||||
const crawler = new WebCrawler({
|
||||
jobId: "TEST",
|
||||
initialUrl: initialUrl,
|
||||
includes: [],
|
||||
excludes: [],
|
||||
limit: 100,
|
||||
maxCrawledDepth: 3, // Example depth
|
||||
allowBackwardCrawling: true
|
||||
});
|
||||
|
||||
// Mock the sitemap fetching function to simulate backward crawling
|
||||
crawler['tryFetchSitemapLinks'] = jest.fn().mockResolvedValue([
|
||||
initialUrl,
|
||||
'https://mendable.ai', // backward link
|
||||
initialUrl + '/page1',
|
||||
initialUrl + '/page1/page2'
|
||||
]);
|
||||
|
||||
const results = await crawler.start();
|
||||
expect(results).toEqual([
|
||||
{ url: initialUrl, html: '' },
|
||||
{ url: 'https://mendable.ai', html: '' }, // Expect the backward link to be included
|
||||
{ url: initialUrl + '/page1', html: '' },
|
||||
{ url: initialUrl + '/page1/page2', html: '' }
|
||||
]);
|
||||
|
||||
// Check that the backward link is included if allowBackwardCrawling is true
|
||||
expect(results.some(r => r.url === 'https://mendable.ai')).toBe(true);
|
||||
});
|
||||
|
||||
it('should respect the limit parameter by not returning more links than specified', async () => {
|
||||
const initialUrl = 'http://example.com';
|
||||
const limit = 2; // Set a limit for the number of links
|
||||
|
||||
Reference in New Issue
Block a user