Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v1 restructure

Browse Source
This commit is contained in:
Gergő Móricz
2024-08-15 21:51:59 +02:00
parent af08ab0b1a
commit 4165de1773
21 changed files with 211 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/controllers/v1/crawl-status.ts
-3
View File
@@ -1,9 +1,6 @@
import { Request, Response } from "express";
import { authenticateUser } from "./auth";
import { RateLimiterMode } from "../../../src/types";
import { addWebScraperJob } from "../../../src/services/queue-jobs";
import { getWebScraperQueue } from "../../../src/services/queue-service";
import { supabaseGetJobById } from "../../../src/lib/supabase-jobs";
import { Logger } from "../../../src/lib/logger";
import { v4 as uuidv4 } from "uuid";
apps/api/src/controllers/v1/crawl.ts
+16 -15
View File
@@ -1,20 +1,16 @@
import { Request, Response } from "express";
import { WebScraperDataProvider } from "../../../src/scraper/WebScraper";
import { billTeam } from "../../../src/services/billing/credit_billing";
import { checkTeamCredits } from "../../../src/services/billing/credit_billing";
import { authenticateUser } from "./auth";
import { RateLimiterMode } from "../../../src/types";
import { addWebScraperJob } from "../../../src/services/queue-jobs";
import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist";
import { logCrawl } from "../../../src/services/logging/crawl_log";
import { validateIdempotencyKey } from "../../../src/services/idempotency/validate";
import { createIdempotencyKey } from "../../../src/services/idempotency/create";
import { defaultCrawlPageOptions, defaultCrawlerOptions, defaultOrigin } from "../../../src/lib/default-values";
import { v4 as uuidv4 } from "uuid";
import { Logger } from "../../../src/lib/logger";
import { checkAndUpdateURL } from "../../../src/lib/validateUrl";
import { CrawlRequest, CrawlResponse } from "./types";
export async function crawlController(req: Request, res: Response) {
export async function crawlController(req: Request<{}, {}, CrawlRequest>, res: Response<CrawlResponse>) {
// expected req.body
// req.body = {
@@ -39,52 +35,57 @@ export async function crawlController(req: Request, res: Response) {
RateLimiterMode.Crawl
);
if (!success) {
return res.status(status).json({ error });
return res.status(status).json({ success: false, error });
}
if (req.headers["x-idempotency-key"]) {
const isIdempotencyValid = await validateIdempotencyKey(req);
if (!isIdempotencyValid) {
return res.status(409).json({ error: "Idempotency key already used" });
return res.status(409).json({ success: false, error: "Idempotency key already used" });
}
try {
createIdempotencyKey(req);
} catch (error) {
Logger.error(error);
return res.status(500).json({ error: error.message });
return res.status(500).json({ success: false, error: error.message });
}
}
const { success: creditsCheckSuccess, message: creditsCheckMessage } =
await checkTeamCredits(team_id, 1);
if (!creditsCheckSuccess) {
return res.status(402).json({ error: "Insufficient credits" });
return res.status(402).json({ success: false, error: "Insufficient credits" });
}
let url = req.body.url;
if (!url) {
return res.status(400).json({ error: "Url is required" });
return res.status(400).json({ success: false, error: "Url is required" });
}
if (isUrlBlocked(url)) {
return res
.status(403)
.json({
success: false,
error:
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
});
}
try {
url = checkAndUpdateURL(url);
url = checkAndUpdateURL(url).url;
} catch (error) {
return res.status(400).json({ error: 'Invalid Url' });
return res.status(400).json({ success: false, error: 'Invalid Url' });
}
// TODO: add job to queue
const id = uuidv4();
return res.status(200).json({ jobId: id, url: `${req.protocol}://${req.get('host')}/v1/crawl/${id}` });
return res.status(200).json({
success: true,
id,
url: `${req.protocol}://${req.get('host')}/v1/crawl/${id}`,
});
// const mode = req.body.mode ?? "crawl";
@@ -134,6 +135,6 @@ export async function crawlController(req: Request, res: Response) {
// res.json({ jobId: job.id });
} catch (error) {
Logger.error(error);
return res.status(500).json({ error: error.message });
return res.status(500).json({ success: false, error: error.message });
}
}
apps/api/src/controllers/v1/map.ts
+10 -19
View File
@@ -1,29 +1,19 @@
import { Request, Response } from "express";
import { WebScraperDataProvider } from "../../../src/scraper/WebScraper";
import { billTeam } from "../../../src/services/billing/credit_billing";
import { checkTeamCredits } from "../../../src/services/billing/credit_billing";
import { authenticateUser } from "./auth";
import { RateLimiterMode } from "../../../src/types";
import { addWebScraperJob } from "../../../src/services/queue-jobs";
import { isUrlBlocked } from "../../../src/scraper/WebScraper/utils/blocklist";
import { logCrawl } from "../../../src/services/logging/crawl_log";
import { validateIdempotencyKey } from "../../../src/services/idempotency/validate";
import { createIdempotencyKey } from "../../../src/services/idempotency/create";
import { defaultCrawlPageOptions, defaultCrawlerOptions, defaultOrigin } from "../../../src/lib/default-values";
import { v4 as uuidv4 } from "uuid";
import { Logger } from "../../../src/lib/logger";
import { checkAndUpdateURL } from "../../../src/lib/validateUrl";
import { MapRequest, MapResponse } from "./types";
export async function mapController(req: Request, res: Response) {
export async function mapController(req: Request<{}, MapResponse, MapRequest>, res: Response<MapResponse>) {
// expected req.body
// req.body = {
// url: string
// ignoreSitemap: true??
// other crawler options?
// crawlerOptions:
// }
try {
const { success, team_id, error, status } = await authenticateUser(
req,
@@ -31,7 +21,7 @@ export async function mapController(req: Request, res: Response) {
RateLimiterMode.Crawl
);
if (!success) {
return res.status(status).json({ error });
return res.status(status).json({ success: false, error });
}
// if (req.headers["x-idempotency-key"]) {
@@ -55,25 +45,26 @@ export async function mapController(req: Request, res: Response) {
let url = req.body.url;
if (!url) {
return res.status(400).json({ error: "Url is required" });
return res.status(400).json({ success: false, error: "Url is required" });
}
if (isUrlBlocked(url)) {
return res
.status(403)
.json({
success: false,
error:
"Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.",
});
}
try {
url = checkAndUpdateURL(url);
url = checkAndUpdateURL(url).url;
} catch (error) {
return res.status(400).json({ error: 'Invalid Url' });
return res.status(400).json({ success: false, error: 'Invalid Url' });
}
return res.status(200).json({ urls: [ "test1", "test2" ] });
return res.status(200).json({ success: true, links: [ "test1", "test2" ] });
// const mode = req.body.mode ?? "crawl";
@@ -123,6 +114,6 @@ export async function mapController(req: Request, res: Response) {
// res.json({ jobId: job.id });
} catch (error) {
Logger.error(error);
return res.status(500).json({ error: error.message });
return res.status(500).json({ success: false, error: error.message });
}
}
apps/api/src/controllers/v1/scrape.ts
+7 -15
View File
@@ -1,19 +1,12 @@
// import { ExtractorOptions, PageOptions } from './../../lib/entities';
import { Request, Response } from "express";
// import { WebScraperDataProvider } from "../../scraper/WebScraper";
// import { billTeam, checkTeamCredits } from "../../services/billing/credit_billing";
import { authenticateUser } from "./auth";
import { RateLimiterMode } from "../../types";
// import { logJob } from "../../services/logging/log_job";
// import { Document } from "../../lib/entities";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; // Import the isUrlBlocked function
// import { numTokensFromString } from '../../lib/LLM-extraction/helpers';
// import { defaultPageOptions, defaultExtractorOptions, defaultTimeout, defaultOrigin } from '../../../src/lib/default-values';
// import { v4 as uuidv4 } from "uuid";
import { Logger } from '../../lib/logger';
import { checkAndUpdateURL } from '../../lib/validateUrl';
import { ScrapeRequest, ScrapeResponse } from "./types";
export async function scrapeController(req: Request, res: Response) {
export async function scrapeController(req: Request<{}, ScrapeResponse, ScrapeRequest>, res: Response<ScrapeResponse>) {
let url = req.body.url;
if (!url) {
return { success: false, error: "Url is required", returnCode: 400 };
@@ -24,7 +17,7 @@ export async function scrapeController(req: Request, res: Response) {
}
try {
url = checkAndUpdateURL(url);
url = checkAndUpdateURL(url).url;
} catch (error) {
return { success: false, error: "Invalid URL", returnCode: 400 };
}
@@ -53,20 +46,19 @@ export async function scrapeController(req: Request, res: Response) {
RateLimiterMode.Scrape
);
if (!success) {
return res.status(status).json({ error });
return res.status(status).json({ success: false, error });
}
// check credits
const result = {
const result: ScrapeResponse = {
success: true,
warning: "test",
data: {
markdown: "test",
content: "test",
html: "test",
rawHtml: "test",
linksOnPage: ["test1", "test2"],
links: ["test1", "test2"],
screenshot: "test",
metadata: {
title: "test",
@@ -174,7 +166,7 @@ export async function scrapeController(req: Request, res: Response) {
// return res.status(result.returnCode).json(result);
} catch (error) {
Logger.error(error);
return res.status(500).json({ error: error.message });
return res.status(500).json({ success: false, error: error.message });
}
}
apps/api/src/controllers/v1/types.ts
+97
View File
@@ -0,0 +1,97 @@
export type Format = "markdown" | "html" | "rawHtml" | "links" | "screenshot" | "screenshot@fullPage";
export type ScrapeRequest = {
url: string;
formats?: Format[];
headers?: { [K: string]: string };
includeTags?: string[];
excludeTags?: string[];
onlyMainContent?: boolean;
timeout?: number;
waitFor?: number;
}
export type CrawlerOptions = {
includePaths?: string[];
excludePaths?: string[];
maxDepth?: number;
limit?: number;
allowBackwardLinks?: boolean; // >> TODO: CHANGE THIS NAME???
allowExternalLinks?: boolean;
ignoreSitemap?: boolean;
};
export type CrawlRequest = {
url: string;
crawlerOptions?: CrawlerOptions;
scrapeOptions?: Exclude<ScrapeRequest, "url">;
};
export type MapRequest = {
url: string;
crawlerOptions?: CrawlerOptions;
};
export type Document = {
markdown?: string,
html?: string,
rawHtml?: string,
links?: string[],
screenshot?: string,
metadata: {
title?: string;
description?: string;
language?: string;
keywords?: string;
robots?: string;
ogTitle?: string;
ogDescription?: string;
ogUrl?: string;
ogImage?: string;
ogAudio?: string;
ogDeterminer?: string;
ogLocale?: string;
ogLocaleAlternate?: string[];
ogSiteName?: string;
ogVideo?: string;
dcTermsCreated?: string;
dcDateCreated?: string;
dcDate?: string;
dcTermsType?: string;
dcType?: string;
dcTermsAudience?: string;
dcTermsSubject?: string;
dcSubject?: string;
dcDescription?: string;
dcTermsKeywords?: string;
modifiedTime?: string;
publishedTime?: string;
articleTag?: string;
articleSection?: string;
sourceURL?: string;
statusCode?: number;
error?: string;
},
}
export type ErrorResponse = {
success: false;
error: string;
};
export type ScrapeResponse = ErrorResponse | {
success: true;
warning?: string;
data: Document;
};
export type CrawlResponse = ErrorResponse | {
success: true;
id: string;
url: string;
}
export type MapResponse = ErrorResponse | {
success: true;
links: string[];
}