Iyeoluwads/firecrawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rust): update rust sdk to support new features (#1446)

Browse Source 
* chore(rust-sdk): cargo fmt

* feat(rust-sdk): implement search api + example + test

* feat(rust-sdk): implement crawl cancel api + example + test

* feat(rust-sdk): implement crawl check errors api + example + test

* feat(rust-sdk): implement batch crawl + test + example

+ Fix MapOptions

* feat(rust-sdk): implement extract api + test + example

* feat(rust-sdk): implement llmtxt api + test + example

* chore(rust-sdk): correct mock tests

* chore(rust-sdk): prep for cargo distribution
This commit is contained in:
kkharji
2025-04-18 07:59:59 +03:00
committed by GitHub
parent 33aece8e96
commit f2c01340d1
20 changed files with 4350 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/rust-sdk/src/batch_scrape.rs
+494
View File
@@ -0,0 +1,494 @@
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use crate::{
crawl::{CrawlErrorsResponse, CrawlStatus, CrawlStatusTypes},
scrape::ScrapeOptions,
FirecrawlApp, FirecrawlError, API_VERSION,
};
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct BatchScrapeParams {
/// List of URLs to scrape
pub urls: Vec<String>,
/// Scrape options to apply to all URLs
#[serde(flatten)]
pub options: Option<ScrapeOptions>,
/// Whether to ignore invalid URLs
#[serde(rename = "ignoreInvalidURLs")]
pub ignore_invalid_urls: bool,
/// ID of an existing job to append these URLs to
pub append_to_id: Option<String>,
/// Webhook configuration
pub webhook: Option<WebhookOptions>,
/// Idempotency key to send to the crawl endpoint.
#[serde(skip)]
pub idempotency_key: Option<String>,
}
/// Options for webhook notifications
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct WebhookOptions {
/// URL to send webhook notifications to
pub url: String,
/// Custom headers to include in webhook requests
pub headers: Option<HashMap<String, String>>,
/// Authentication token for the webhook
pub auth_token: Option<String>,
}
impl From<&str> for WebhookOptions {
fn from(url: &str) -> Self {
Self {
url: url.to_string(),
headers: None,
auth_token: None,
}
}
}
/// Response from initiating a batch scrape job
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct BatchScrapeResponse {
/// Whether the request was successful
pub success: bool,
/// The ID of the batch scrape job
pub id: String,
/// URL to get the status of the batch scrape job
pub url: String,
/// List of URLs that were invalid and could not be processed
pub invalid_urls: Option<Vec<String>>,
}
impl From<String> for WebhookOptions {
fn from(url: String) -> Self {
Self {
url,
headers: None,
auth_token: None,
}
}
}
impl FirecrawlApp {
/// Initiates an asynchronous batch scrape job
pub async fn async_batch_scrape_urls(
&self,
params: BatchScrapeParams,
) -> Result<BatchScrapeResponse, FirecrawlError> {
let headers = self.prepare_headers(params.idempotency_key.as_ref());
let response = self
.client
.post(format!("{}{}/batch/scrape", self.api_url, API_VERSION))
.headers(headers)
.json(&params)
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Initiating batch scrape job".to_string(), e))?;
self.handle_response(response, "initiate batch scrape job")
.await
}
/// Initiates a batch scrape job and waits for completion
pub async fn batch_scrape_urls(
&self,
params: BatchScrapeParams,
poll_interval: Option<u64>,
) -> Result<CrawlStatus, FirecrawlError> {
let poll_interval_ms = poll_interval.unwrap_or(2000);
let response = self.async_batch_scrape_urls(params).await?;
self.monitor_batch_job_status(&response.id, poll_interval_ms)
.await
}
/// Checks the status of a batch scrape job
pub async fn check_batch_scrape_status(
&self,
id: impl AsRef<str>,
) -> Result<CrawlStatus, FirecrawlError> {
let response = self
.client
.get(format!(
"{}{}/batch/scrape/{}",
self.api_url,
API_VERSION,
id.as_ref()
))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(
format!("Checking status of batch scrape {}", id.as_ref()),
e,
)
})?;
let mut status: CrawlStatus = self
.handle_response(
response,
format!("Checking status of batch scrape {}", id.as_ref()),
)
.await?;
if status.status == CrawlStatusTypes::Completed {
while let Some(next) = status.next.clone() {
let new_status = self.check_batch_scrape_status_next(next).await?;
status.data.extend_from_slice(&new_status.data);
status.next = new_status.next;
}
}
Ok(status)
}
/// Helper function to paginate through batch scrape status results
async fn check_batch_scrape_status_next(
&self,
next: impl AsRef<str>,
) -> Result<CrawlStatus, FirecrawlError> {
let response = self
.client
.get(next.as_ref())
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(
format!("Paginating batch scrape using URL {:?}", next.as_ref()),
e,
)
})?;
self.handle_response(
response,
format!("Paginating batch scrape using URL {:?}", next.as_ref()),
)
.await
}
/// Check for errors in a batch scrape job
pub async fn check_batch_scrape_errors(
&self,
id: impl AsRef<str>,
) -> Result<CrawlErrorsResponse, FirecrawlError> {
let response = self
.client
.get(format!(
"{}{}/batch/scrape/{}/errors",
self.api_url,
API_VERSION,
id.as_ref()
))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(
format!("Checking errors for batch scrape {}", id.as_ref()),
e,
)
})?;
self.handle_response(
response,
format!("Checking errors for batch scrape {}", id.as_ref()),
)
.await
}
/// Helper function to poll for batch job status until completion
async fn monitor_batch_job_status(
&self,
id: &str,
poll_interval: u64,
) -> Result<CrawlStatus, FirecrawlError> {
loop {
let status_data = self.check_batch_scrape_status(id).await?;
match status_data.status {
CrawlStatusTypes::Completed => {
break Ok(status_data);
}
CrawlStatusTypes::Scraping => {
tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
}
CrawlStatusTypes::Failed => {
break Err(FirecrawlError::CrawlJobFailed(
"Batch scrape job failed".into(),
status_data,
));
}
CrawlStatusTypes::Cancelled => {
break Err(FirecrawlError::CrawlJobFailed(
"Batch scrape job was cancelled".into(),
status_data,
));
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[tokio::test]
#[ignore = "Makes real network request"]
async fn test_real_batch_scrape() {
let api_url = std::env::var("FIRECRAWL_API_URL")
.expect("Please set the FIRECRAWL_API_URL environment variable");
let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
// Start a batch scrape job
let params = BatchScrapeParams {
urls: vec![
"https://example.com".to_string(),
"https://example.org".to_string(),
],
ignore_invalid_urls: true,
..Default::default()
};
let response = app.async_batch_scrape_urls(params).await.unwrap();
assert!(response.success);
assert!(!response.id.is_empty());
assert!(!response.url.is_empty());
}
#[tokio::test]
async fn test_async_batch_scrape_with_mock() {
let mut server = mockito::Server::new_async().await;
// Set up the mock
let mock = server
.mock("POST", "/v1/batch/scrape")
// Remove the match_body expectation which might be causing issues
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "batch-123",
"url": "https://api.example.com/v1/batch/batch-123",
"invalidUrls": []
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let params = BatchScrapeParams {
urls: vec![
"https://example.com".to_string(),
"https://example.org".to_string(),
],
ignore_invalid_urls: true,
..Default::default()
};
let response = app.async_batch_scrape_urls(params).await.unwrap();
assert!(response.success);
assert_eq!(response.id, "batch-123");
assert_eq!(response.url, "https://api.example.com/v1/batch/batch-123");
assert!(response.invalid_urls.unwrap_or_default().is_empty());
mock.assert();
}
#[tokio::test]
async fn test_batch_scrape_with_webhook() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v1/batch/scrape")
// Remove the match_body expectation to simplify
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "batch-123",
"url": "https://api.example.com/v1/batch/batch-123"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let params = BatchScrapeParams {
urls: vec!["https://example.com".to_string()],
webhook: Some("https://webhook.example.com/notify".into()),
..Default::default()
};
let response = app.async_batch_scrape_urls(params).await.unwrap();
assert!(response.success);
assert_eq!(response.id, "batch-123");
mock.assert();
}
#[tokio::test]
async fn test_check_batch_scrape_status_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/v1/batch/scrape/batch-123")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"status": "completed",
"total": 2,
"completed": 2,
"creditsUsed": 2,
"expiresAt": "2023-12-31T23:59:59Z",
"data": [
{
"metadata": {
"sourceURL": "https://example.com",
"statusCode": 200
},
"markdown": "Example Domain content"
},
{
"metadata": {
"sourceURL": "https://example.org",
"statusCode": 200
},
"markdown": "Another example content"
}
]
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let status = app.check_batch_scrape_status("batch-123").await.unwrap();
assert_eq!(status.total, 2);
assert_eq!(status.completed, 2);
assert_eq!(status.data.len(), 2);
assert_eq!(status.data[0].metadata.source_url, "https://example.com");
assert_eq!(status.data[1].metadata.source_url, "https://example.org");
mock.assert();
}
#[tokio::test]
async fn test_check_batch_scrape_errors_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/v1/batch/scrape/batch-123/errors")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"errors": [
{
"id": "error1",
"timestamp": "2023-01-01T00:00:00Z",
"url": "https://invalid.example.com",
"error": "Failed to load page"
}
],
"robotsBlocked": [
"https://example.com/admin"
]
})
.to_string(),
)
.create_async()
.await;
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let errors = app.check_batch_scrape_errors("batch-123").await.unwrap();
assert_eq!(errors.errors.len(), 1);
assert_eq!(errors.errors[0].url, "https://invalid.example.com");
assert_eq!(errors.robots_blocked.len(), 1);
assert_eq!(errors.robots_blocked[0], "https://example.com/admin");
mock.assert();
}
#[tokio::test]
async fn test_batch_scrape_with_invalid_urls() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v1/batch/scrape")
// Remove the match_body expectation
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "batch-123",
"url": "https://api.example.com/v1/batch/batch-123",
"invalidUrls": ["invalid-url"]
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let params = BatchScrapeParams {
urls: vec!["https://example.com".to_string(), "invalid-url".to_string()],
ignore_invalid_urls: true,
..Default::default()
};
let response = app.async_batch_scrape_urls(params).await.unwrap();
assert!(response.success);
assert_eq!(response.id, "batch-123");
assert_eq!(response.invalid_urls, Some(vec!["invalid-url".to_string()]));
mock.assert();
}
#[tokio::test]
async fn test_batch_scrape_error_response() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v1/batch/scrape")
.with_status(400)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "No valid URLs provided"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let params = BatchScrapeParams::default();
let result = app.async_batch_scrape_urls(params).await;
assert!(result.is_err());
mock.assert();
}
}
apps/rust-sdk/src/crawl.rs
+307 -26
View File
@@ -2,7 +2,11 @@ use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use crate::{document::Document, scrape::{ScrapeFormats, ScrapeOptions}, FirecrawlApp, FirecrawlError, API_VERSION};
use crate::{
document::Document,
scrape::{ScrapeFormats, ScrapeOptions},
FirecrawlApp, FirecrawlError, API_VERSION,
};
#[derive(Deserialize, Serialize, Clone, Copy, Debug)]
pub enum CrawlScrapeFormats {
@@ -23,13 +27,13 @@ pub enum CrawlScrapeFormats {
Links,
/// Will result in a URL to a screenshot of the page.
///
///
/// Can not be used in conjunction with `CrawlScrapeFormats::ScreenshotFullPage`.
#[serde(rename = "screenshot")]
Screenshot,
/// Will result in a URL to a full-page screenshot of the page.
///
///
/// Can not be used in conjunction with `CrawlScrapeFormats::Screenshot`.
#[serde(rename = "screenshot@fullPage")]
ScreenshotFullPage,
@@ -59,12 +63,12 @@ pub struct CrawlScrapeOptions {
pub only_main_content: Option<bool>,
/// HTML tags to exclusively include.
///
///
/// For example, if you pass `div`, you will only get content from `<div>`s and their children.
pub include_tags: Option<Vec<String>>,
/// HTML tags to exclude.
///
///
/// For example, if you pass `img`, you will never get image URLs in your results.
pub exclude_tags: Option<Vec<String>>,
@@ -81,7 +85,9 @@ pub struct CrawlScrapeOptions {
impl From<CrawlScrapeOptions> for ScrapeOptions {
fn from(value: CrawlScrapeOptions) -> Self {
ScrapeOptions {
formats: value.formats.map(|formats| formats.into_iter().map(|x| x.into()).collect()),
formats: value
.formats
.map(|formats| formats.into_iter().map(|x| x.into()).collect()),
only_main_content: value.only_main_content,
include_tags: value.include_tags,
exclude_tags: value.exclude_tags,
@@ -101,12 +107,12 @@ pub struct CrawlOptions {
pub scrape_options: Option<CrawlScrapeOptions>,
/// URL RegEx patterns to (exclusively) include.
///
///
/// For example, if you specified `"blog"`, only pages that have `blog` somewhere in the URL would be crawled.
pub include_paths: Option<Vec<String>>,
/// URL RegEx patterns to exclude.
///
///
/// For example, if you specified `"blog"`, pages that have `blog` somewhere in the URL would not be crawled.
pub exclude_paths: Option<Vec<String>>,
@@ -200,6 +206,29 @@ pub struct CrawlStatus {
pub data: Vec<Document>,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CrawlError {
pub id: String,
pub timestamp: Option<String>,
pub url: String,
pub error: String,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CrawlErrorsResponse {
pub errors: Vec<CrawlError>,
#[serde(rename = "robotsBlocked")]
pub robots_blocked: Vec<String>,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CancelCrawlResponse {
pub status: String,
}
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct CrawlAsyncResponse {
@@ -223,19 +252,20 @@ impl FirecrawlApp {
url: url.as_ref().to_string(),
options: options.unwrap_or_default(),
};
let headers = self.prepare_headers(body.options.idempotency_key.as_ref());
let response = self
.client
.post(&format!("{}{}/crawl", self.api_url, API_VERSION))
.post(format!("{}{}/crawl", self.api_url, API_VERSION))
.headers(headers.clone())
.json(&body)
.send()
.await
.map_err(|e| FirecrawlError::HttpError(format!("Crawling {:?}", url.as_ref()), e))?;
self.handle_response::<CrawlAsyncResponse>(response, "start crawl job").await
self.handle_response::<CrawlAsyncResponse>(response, "start crawl job")
.await
}
/// Performs a crawl job for a URL using the Firecrawl API, waiting for the end result. This may take a long time depending on the size of the target page and your options (namely `CrawlOptions.limit`).
@@ -245,38 +275,65 @@ impl FirecrawlApp {
options: impl Into<Option<CrawlOptions>>,
) -> Result<CrawlStatus, FirecrawlError> {
let options = options.into();
let poll_interval = options.as_ref().and_then(|x| x.poll_interval).unwrap_or(2000);
let poll_interval = options
.as_ref()
.and_then(|x| x.poll_interval)
.unwrap_or(2000);
let res = self.crawl_url_async(url, options).await?;
self.monitor_job_status(&res.id, poll_interval).await
}
async fn check_crawl_status_next(&self, next: impl AsRef<str>) -> Result<CrawlStatus, FirecrawlError> {
async fn check_crawl_status_next(
&self,
next: impl AsRef<str>,
) -> Result<CrawlStatus, FirecrawlError> {
let response = self
.client
.get(next.as_ref())
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| FirecrawlError::HttpError(format!("Paginating crawl using URL {:?}", next.as_ref()), e))?;
.map_err(|e| {
FirecrawlError::HttpError(
format!("Paginating crawl using URL {:?}", next.as_ref()),
e,
)
})?;
self.handle_response(response, format!("Paginating crawl using URL {:?}", next.as_ref())).await
self.handle_response(
response,
format!("Paginating crawl using URL {:?}", next.as_ref()),
)
.await
}
/// Checks for the status of a crawl, based on the crawl's ID. To be used in conjunction with `FirecrawlApp::crawl_url_async`.
pub async fn check_crawl_status(&self, id: impl AsRef<str>) -> Result<CrawlStatus, FirecrawlError> {
pub async fn check_crawl_status(
&self,
id: impl AsRef<str>,
) -> Result<CrawlStatus, FirecrawlError> {
let response = self
.client
.get(&format!(
.get(format!(
"{}{}/crawl/{}",
self.api_url, API_VERSION, id.as_ref()
self.api_url,
API_VERSION,
id.as_ref()
))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| FirecrawlError::HttpError(format!("Checking status of crawl {}", id.as_ref()), e))?;
.map_err(|e| {
FirecrawlError::HttpError(format!("Checking status of crawl {}", id.as_ref()), e)
})?;
let mut status: CrawlStatus = self.handle_response(response, format!("Checking status of crawl {}", id.as_ref())).await?;
let mut status: CrawlStatus = self
.handle_response(
response,
format!("Checking status of crawl {}", id.as_ref()),
)
.await?;
if status.status == CrawlStatusTypes::Completed {
while let Some(next) = status.next {
@@ -304,16 +361,240 @@ impl FirecrawlApp {
tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
}
CrawlStatusTypes::Failed => {
break Err(FirecrawlError::CrawlJobFailed(format!(
"Crawl job failed."
), status_data));
break Err(FirecrawlError::CrawlJobFailed(
"Crawl job failed".into(),
status_data,
));
}
CrawlStatusTypes::Cancelled => {
break Err(FirecrawlError::CrawlJobFailed(format!(
"Crawl job was cancelled."
), status_data));
break Err(FirecrawlError::CrawlJobFailed(
"Crawl job was cancelled.".into(),
status_data,
));
}
}
}
}
/// Cancel an asynchronous crawl job using the Firecrawl API.
///
/// # Returns
///
/// A response indicating whether the cancellation was successful, or a FirecrawlError if the request fails.
pub async fn cancel_crawl(
&self,
id: impl AsRef<str>,
) -> Result<CancelCrawlResponse, FirecrawlError> {
let response = self
.client
.delete(format!(
"{}{}/crawl/{}",
self.api_url,
API_VERSION,
id.as_ref()
))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Cancelling crawl {}", id.as_ref()), e)
})?;
self.handle_response(response, "crawl_cancel").await
}
/// Returns information about crawl errors.
///
/// # Returns
///
/// A response containing information about crawl errors, or a FirecrawlError if the request fails.
pub async fn check_crawl_errors(
&self,
id: impl AsRef<str>,
) -> Result<CrawlErrorsResponse, FirecrawlError> {
let response = self
.client
.get(format!(
"{}{}/crawl/{}/errors",
self.api_url,
API_VERSION,
id.as_ref()
))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Checking errors for crawl {}", id.as_ref()), e)
})?;
self.handle_response(response, "crawl_check").await
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[tokio::test]
#[ignore = "Makes real network request"]
async fn test_real_cancel_crawl() {
let api_url = std::env::var("FIRECRAWL_API_URL")
.expect("Please set the FIRECRAWL_API_URL environment variable");
let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
// First start a crawl job
let crawl_response = app
.crawl_url_async("https://example.com", None)
.await
.unwrap();
// Then cancel it
let cancel_response = app.cancel_crawl(crawl_response.id).await.unwrap();
assert_eq!(cancel_response.status, "cancelled");
}
#[tokio::test]
async fn test_cancel_crawl_with_mock() {
let mut server = mockito::Server::new_async().await;
// Set up the mock for the cancel request
let mock = server
.mock("DELETE", "/v1/crawl/test-crawl-id")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": null,
"status": "cancelled"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = app.cancel_crawl("test-crawl-id").await.unwrap();
assert_eq!(response.status, "cancelled");
mock.assert();
}
#[tokio::test]
async fn test_cancel_crawl_error_response() {
let mut server = mockito::Server::new_async().await;
// Set up the mock for an error response
let mock = server
.mock("DELETE", "/v1/crawl/invalid-id")
.with_status(404)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Crawl job not found"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let result = app.cancel_crawl("invalid-id").await;
assert!(result.is_err());
mock.assert();
}
#[tokio::test]
#[ignore = "Makes real network request"]
async fn test_real_check_crawl_errors() {
let api_url = std::env::var("FIRECRAWL_API_URL")
.expect("Please set the FIRECRAWL_API_URL environment variable");
let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
// First start a crawl job
let crawl_response = app
.crawl_url_async("https://no-wer-agg.invalid", None)
.await
.unwrap();
// Check for errors
let errors_response = app.check_crawl_errors(crawl_response.id).await.unwrap();
println!("{errors_response:?}");
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
assert!(
!errors_response.errors.is_empty(),
"WARN: Error returned related to Supabase not in my environment. It may fail"
);
}
#[tokio::test]
async fn test_check_crawl_errors_with_mock() {
let mut server = mockito::Server::new_async().await;
// Set up the mock for the check errors request
let mock = server
.mock("GET", "/v1/crawl/test-crawl-id/errors")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"errors": [
{
"id": "error1",
"timestamp": "2023-01-01T00:00:00Z",
"url": "https://example.com/error-page",
"error": "Failed to load page"
}
],
"robotsBlocked": [
"https://example.com/blocked-by-robots"
]
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = app.check_crawl_errors("test-crawl-id").await.unwrap();
assert_eq!(response.errors.len(), 1);
assert_eq!(response.errors[0].id, "error1");
assert_eq!(response.errors[0].url, "https://example.com/error-page");
assert_eq!(response.errors[0].error, "Failed to load page");
assert_eq!(response.robots_blocked.len(), 1);
assert_eq!(
response.robots_blocked[0],
"https://example.com/blocked-by-robots"
);
mock.assert();
}
#[tokio::test]
async fn test_check_crawl_errors_error_response() {
let mut server = mockito::Server::new_async().await;
// Set up the mock for an error response
let mock = server
.mock("GET", "/v1/crawl/invalid-id/errors")
.with_status(404)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Crawl job not found"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let result = app.check_crawl_errors("invalid-id").await;
assert!(result.is_err());
mock.assert();
}
}
apps/rust-sdk/src/document.rs
+2 -3
View File
@@ -57,12 +57,12 @@ pub struct Document {
pub markdown: Option<String>,
/// The HTML of the page, present if `ScrapeFormats::HTML` is present in `ScrapeOptions.formats`.
///
///
/// This contains HTML that has non-content tags removed. If you need the original HTML, use `ScrapeFormats::RawHTML`.
pub html: Option<String>,
/// The raw HTML of the page, present if `ScrapeFormats::RawHTML` is present in `ScrapeOptions.formats`.
///
///
/// This contains the original, untouched HTML on the page. If you only need human-readable content, use `ScrapeFormats::HTML`.
pub raw_html: Option<String>,
@@ -83,4 +83,3 @@ pub struct Document {
/// The warning message will contain any errors encountered during the extraction.
pub warning: Option<String>,
}
apps/rust-sdk/src/error.rs
+2
View File
@@ -42,4 +42,6 @@ pub enum FirecrawlError {
APIError(String, FirecrawlAPIError),
#[error("Crawl job failed: {0}")]
CrawlJobFailed(String, CrawlStatus),
#[error("Missuse: {0}")]
Missuse(String),
}
apps/rust-sdk/src/extract.rs
+596
View File
@@ -0,0 +1,596 @@
use std::collections::HashMap;
use schemars::schema_for;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{FirecrawlApp, FirecrawlError, API_VERSION};
/// Parameters for extract requests
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ExtractParams {
/// URLs to extract information from
pub urls: Option<Vec<String>>,
/// Extraction prompt
pub prompt: Option<String>,
/// Schema for structured output
pub schema: Option<Value>,
/// System prompt for the LLM
pub system_prompt: Option<String>,
/// Allow following external links
pub allow_external_links: Option<bool>,
/// Enable web search for additional information
pub enable_web_search: Option<bool>,
/// Show sources in the response
pub show_sources: Option<bool>,
/// Origin information, defaults to "api-sdk"
pub origin: Option<String>,
/// Timeout in milliseconds, defaults to 60000
pub timeout: Option<u32>,
/// Whether to include URL trace information, defaults to false
pub url_trace: Option<bool>,
/// Whether to ignore sitemap, defaults to false
pub ignore_sitemap: Option<bool>,
/// Whether to include subdomains, defaults to true
pub include_subdomains: Option<bool>,
/// Maximum number of URLs to process
pub limit: Option<u32>,
/// Experimental: Stream steps information
#[serde(rename = "__experimental_streamSteps")]
pub experimental_stream_steps: Option<bool>,
/// Experimental: Include LLM usage information
#[serde(rename = "__experimental_llmUsage")]
pub experimental_llm_usage: Option<bool>,
/// Experimental: Show sources information
#[serde(rename = "__experimental_showSources")]
pub experimental_show_sources: Option<bool>,
/// Experimental: Cache key
#[serde(rename = "__experimental_cacheKey")]
pub experimental_cache_key: Option<String>,
/// Experimental: Cache mode, defaults to "direct"
#[serde(rename = "__experimental_cacheMode")]
pub experimental_cache_mode: Option<String>,
}
/// Response from initiating an extract operation
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ExtractResponse {
/// Whether the request was successful
pub success: bool,
/// The ID of the extract job
pub id: String,
/// URL trace information if requested
pub url_trace: Option<Vec<URLTrace>>,
}
/// Information about URL processing during extraction
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct URLTrace {
/// The URL being processed
pub url: String,
/// Status of processing this URL
pub status: String,
/// Timing information for URL processing
pub timing: URLTraceTiming,
/// Error message if processing failed
pub error: Option<String>,
/// Warning message if there were issues
pub warning: Option<String>,
/// Content statistics
pub content_stats: Option<ContentStats>,
/// Relevance score for this URL (0-1)
pub relevance_score: Option<f64>,
/// Whether this URL was used in the final completion
pub used_in_completion: Option<bool>,
/// Fields extracted from this URL
pub extracted_fields: Option<Vec<String>>,
}
/// Timing information for URL processing
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct URLTraceTiming {
/// When the URL was discovered
pub discovered_at: String,
/// When scraping began for this URL
pub scraped_at: Option<String>,
/// When processing was completed for this URL
pub completed_at: Option<String>,
}
/// Statistics about processed content
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ContentStats {
/// Length of the raw content in characters
pub raw_content_length: u32,
/// Length of the processed content in characters
pub processed_content_length: u32,
/// Number of tokens used for this content
pub tokens_used: u32,
}
/// Response for extract status check
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ExtractStatusResponse {
/// Whether the request was successful
pub success: bool,
/// Status of the extract job: "pending", "processing", "completed", "failed"
pub status: String,
/// Extracted data, present when status is "completed"
pub data: Option<Value>,
/// Error message if the job failed
pub error: Option<String>,
/// URL trace information if requested
pub url_trace: Option<Vec<URLTrace>>,
/// Sources information if requested
pub sources: Option<HashMap<String, Vec<String>>>,
}
impl FirecrawlApp {
/// Extracts information from URLs using the Firecrawl API.
///
/// This is the synchronous version that polls until completion.
///
/// Either `params.prompt` or `params.schema` must be provided.
pub async fn extract(
&self,
params: impl Into<ExtractParams>,
) -> Result<ExtractStatusResponse, FirecrawlError> {
let mut params = params.into();
// Validation: Either prompt or schema must be provided
if params.prompt.is_none() && params.schema.is_none() {
return Err(FirecrawlError::APIError(
"Extract validation".to_string(),
crate::error::FirecrawlAPIError {
success: false,
error: "Either prompt or schema must be provided".to_string(),
details: None,
},
));
}
// Set default origin if not provided
if params.origin.is_none() {
params.origin = Some("api-sdk".to_string());
}
// Initiate the extract job asynchronously
let response = self.async_extract(params).await?;
// Poll for the result
let poll_interval = 2000; // Default to 2 seconds
self.monitor_extract_job_status(&response.id, poll_interval)
.await
}
pub async fn extract_with_schemars<T>(
&self,
params: impl Into<ExtractParams>,
) -> Result<ExtractStatusResponse, FirecrawlError>
where
T: schemars::JsonSchema,
{
let mut params = params.into();
let schema = schema_for!(T);
let schema_json = serde_json::to_value(schema).map_err(|e| {
FirecrawlError::APIError(
"Schema serialization".to_string(),
crate::error::FirecrawlAPIError {
success: false,
error: e.to_string(),
details: None,
},
)
})?;
params.schema = Some(schema_json);
self.extract(params).await
}
/// Initiates an asynchronous extract operation.
///
/// # Arguments
///
/// * `params` - Parameters for the extract request
///
/// # Returns
///
/// A response containing the extract job ID, or a FirecrawlError if the request fails.
///
/// # Notes
///
/// Either `params.urls` or `params.prompt` must be provided.
/// Either `params.prompt` or `params.schema` must be provided.
pub async fn async_extract(
&self,
params: impl Into<ExtractParams>,
) -> Result<ExtractResponse, FirecrawlError> {
let params = params.into();
// Validation: Either URLs or prompt must be provided
if params.urls.is_none() && params.prompt.is_none() {
return Err(FirecrawlError::APIError(
"Extract validation".to_string(),
crate::error::FirecrawlAPIError {
success: false,
error: "Either URLs or prompt must be provided".to_string(),
details: None,
},
));
}
// Validation: Either prompt or schema must be provided
if params.prompt.is_none() && params.schema.is_none() {
return Err(FirecrawlError::APIError(
"Extract validation".to_string(),
crate::error::FirecrawlAPIError {
success: false,
error: "Either prompt or schema must be provided".to_string(),
details: None,
},
));
}
let headers = self.prepare_headers(None);
let response = self
.client
.post(format!("{}{}/extract", self.api_url, API_VERSION))
.headers(headers)
.json(&params)
.send()
.await
.map_err(|e| FirecrawlError::HttpError("Initiating extract job".to_string(), e))?;
self.handle_response(response, "initiate extract job").await
}
/// Checks the status of an extract job.
///
/// # Arguments
///
/// * `id` - The ID of the extract job
///
/// # Returns
///
/// A response containing the status of the extract job, or a FirecrawlError if the request fails.
pub async fn get_extract_status(
&self,
id: impl AsRef<str>,
) -> Result<ExtractStatusResponse, FirecrawlError> {
let response = self
.client
.get(format!(
"{}{}/extract/{}",
self.api_url,
API_VERSION,
id.as_ref()
))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Checking status of extract {}", id.as_ref()), e)
})?;
self.handle_response(
response,
format!("Checking status of extract {}", id.as_ref()),
)
.await
}
/// Helper function to poll for extract job status until completion
async fn monitor_extract_job_status(
&self,
id: &str,
poll_interval: u64,
) -> Result<ExtractStatusResponse, FirecrawlError> {
loop {
let status_data = self.get_extract_status(id).await?;
match status_data.status.as_str() {
"completed" => {
break Ok(status_data);
}
"pending" | "processing" => {
tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
}
"failed" => {
let error_msg = status_data
.error
.clone()
.unwrap_or_else(|| "Extract job failed".to_string());
break Err(FirecrawlError::APIError(
"Extract job failed".to_string(),
crate::error::FirecrawlAPIError {
success: false,
error: error_msg,
details: None,
},
));
}
_ => {
break Err(FirecrawlError::APIError(
"Extract job status".to_string(),
crate::error::FirecrawlAPIError {
success: false,
error: format!("Unexpected status: {}", status_data.status),
details: None,
},
));
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[tokio::test]
#[ignore = "Makes real network request"]
async fn test_real_extract() {
let api_url = std::env::var("FIRECRAWL_API_URL")
.expect("Please set the FIRECRAWL_API_URL environment variable");
let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
// Create extract params
let params = ExtractParams {
urls: Some(vec!["https://example.com".to_string()]),
prompt: Some("Extract the title and main content from this page".to_string()),
schema: None,
origin: Some("test".to_string()),
..Default::default()
};
// Start an extract job
let response = app.async_extract(params).await.unwrap();
assert!(response.success);
assert!(!response.id.is_empty());
}
#[tokio::test]
async fn test_async_extract_with_mock() {
let mut server = mockito::Server::new_async().await;
// Set up the mock for the extract request
let mock = server
.mock("POST", "/v1/extract")
.match_body(mockito::Matcher::PartialJson(json!({
"urls": ["https://example.com"],
"prompt": "Extract the title and main content"
})))
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "extract-123",
"urlTrace": []
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let params = ExtractParams {
urls: Some(vec!["https://example.com".to_string()]),
prompt: Some("Extract the title and main content".to_string()),
schema: None,
..Default::default()
};
let response = app.async_extract(params).await.unwrap();
assert!(response.success);
assert_eq!(response.id, "extract-123");
assert!(response.url_trace.unwrap_or_default().is_empty());
mock.assert();
}
#[tokio::test]
async fn test_extract_with_schema() {
let mut server = mockito::Server::new_async().await;
// Set up the mock for the extract request with schema
let mock = server
.mock("POST", "/v1/extract")
.match_body(mockito::Matcher::PartialJson(json!({
"urls": ["https://example.com"],
"schema": {
"type": "object",
"properties": {
"title": { "type": "string" },
"content": { "type": "string" }
}
}
})))
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "extract-123"
})
.to_string(),
)
.create();
// Set up the mock for the status request
let status_mock = server
.mock("GET", "/v1/extract/extract-123")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"status": "completed",
"data": {
"title": "Example Domain",
"content": "This domain is for use in illustrative examples in documents."
}
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let urls = Some(vec!["https://example.com".to_string()]);
let params = ExtractParams {
urls,
schema: Some(json!({
"type": "object",
"properties": {
"title": { "type": "string" },
"content": { "type": "string" }
}
})),
..Default::default()
};
let response = app.extract(params).await.unwrap();
assert!(response.success);
assert_eq!(response.status, "completed");
let data = response.data.unwrap();
assert_eq!(data["title"], "Example Domain");
assert_eq!(
data["content"],
"This domain is for use in illustrative examples in documents."
);
mock.assert();
status_mock.assert();
}
#[tokio::test]
async fn test_extract_status_with_mock() {
let mut server = mockito::Server::new_async().await;
// Set up the mock for the status check
let mock = server
.mock("GET", "/v1/extract/extract-123")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"status": "processing",
"urlTrace": [
{
"url": "https://example.com",
"status": "scraping",
"timing": {
"discoveredAt": "2023-01-01T00:00:00Z"
}
}
]
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let status = app.get_extract_status("extract-123").await.unwrap();
assert!(status.success);
assert_eq!(status.status, "processing");
assert_eq!(status.url_trace.unwrap()[0].url, "https://example.com");
mock.assert();
}
#[tokio::test]
async fn test_extract_validation_errors() {
let app = FirecrawlApp::new_selfhosted("https://example.com", Some("test_key")).unwrap();
// Test missing both URLs and prompt
let result = app.async_extract(ExtractParams::default()).await;
assert!(result.is_err());
// Test having URLs but missing both prompt and schema
let params = ExtractParams {
urls: Some(vec!["https://example.com".to_string()]),
..Default::default()
};
let result = app.async_extract(params).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_extract_api_error() {
let mut server = mockito::Server::new_async().await;
// Set up the mock for an error response
let mock = server
.mock("POST", "/v1/extract")
.with_status(400)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Invalid schema format"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let params = ExtractParams {
urls: Some(vec!["https://example.com".to_string()]),
schema: Some(json!("invalid")), // Invalid schema format
..Default::default()
};
let result = app.async_extract(params).await;
assert!(result.is_err());
mock.assert();
}
}
apps/rust-sdk/src/lib.rs
+34 -11
View File
@@ -2,14 +2,18 @@ use reqwest::{Client, Response};
use serde::de::DeserializeOwned;
use serde_json::Value;
pub mod batch_scrape;
pub mod crawl;
pub mod document;
mod error;
pub mod extract;
pub mod llmstxt;
pub mod map;
pub mod scrape;
pub mod search;
pub use error::FirecrawlError;
use error::FirecrawlAPIError;
pub use error::FirecrawlError;
#[derive(Clone, Debug)]
pub struct FirecrawlApp {
@@ -26,9 +30,12 @@ impl FirecrawlApp {
FirecrawlApp::new_selfhosted(CLOUD_API_URL, Some(api_key))
}
pub fn new_selfhosted(api_url: impl AsRef<str>, api_key: Option<impl AsRef<str>>) -> Result<Self, FirecrawlError> {
pub fn new_selfhosted(
api_url: impl AsRef<str>,
api_key: Option<impl AsRef<str>>,
) -> Result<Self, FirecrawlError> {
let url = api_url.as_ref().to_string();
if url == CLOUD_API_URL && api_key.is_none() {
return Err(FirecrawlError::APIError(
"Configuration".to_string(),
@@ -36,7 +43,7 @@ impl FirecrawlApp {
success: false,
error: "API key is required for cloud service".to_string(),
details: None,
}
},
));
}
@@ -73,27 +80,43 @@ impl FirecrawlApp {
.text()
.await
.map_err(|e| FirecrawlError::ResponseParseErrorText(e))
.and_then(|response_json| serde_json::from_str::<Value>(&response_json).map_err(|e| FirecrawlError::ResponseParseError(e)))
.and_then(|response_json| {
serde_json::from_str::<Value>(&response_json)
.map_err(|e| FirecrawlError::ResponseParseError(e))
.inspect(|data| {
#[cfg(debug_assertions)]
println!("Response JSON: {:#?}", data);
})
})
.and_then(|response_value| {
if response_value["success"].as_bool().unwrap_or(false) {
Ok(serde_json::from_value::<T>(response_value).map_err(|e| FirecrawlError::ResponseParseError(e))?)
if action.as_ref().starts_with("crawl_") // no success in check/cancel crawl responses
|| response_value["success"].as_bool().unwrap_or(false)
{
Ok(serde_json::from_value::<T>(response_value)
.map_err(|e| FirecrawlError::ResponseParseError(e))?)
} else {
Err(FirecrawlError::APIError(
action.as_ref().to_string(),
serde_json::from_value(response_value).map_err(|e| FirecrawlError::ResponseParseError(e))?
serde_json::from_value(response_value)
.map_err(|e| FirecrawlError::ResponseParseError(e))?,
))
}
});
match &response {
Ok(_) => response,
Err(FirecrawlError::ResponseParseError(_)) | Err(FirecrawlError::ResponseParseErrorText(_)) => {
Err(FirecrawlError::ResponseParseError(_))
| Err(FirecrawlError::ResponseParseErrorText(_)) => {
if is_success {
response
} else {
Err(FirecrawlError::HttpRequestFailed(action.as_ref().to_string(), status.as_u16(), status.as_str().to_string()))
Err(FirecrawlError::HttpRequestFailed(
action.as_ref().to_string(),
status.as_u16(),
status.as_str().to_string(),
))
}
},
}
Err(_) => response,
}
}
apps/rust-sdk/src/llmstxt.rs
+426
View File
@@ -0,0 +1,426 @@
use serde::{Deserialize, Serialize};
use crate::{FirecrawlApp, FirecrawlError, API_VERSION};
/// Parameters for generating LLMs.txt
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct GenerateLLMsTextParams {
/// URL for which to generate LLMs.txt
pub url: String,
/// Maximum number of URLs to process. Default: 10
pub max_urls: u32,
/// Whether to show the full LLMs-full.txt in the response. Default: false
pub show_full_text: bool,
/// Experimental streaming option
#[serde(rename = "__experimental_stream")]
pub experimental_stream: bool,
}
impl Default for GenerateLLMsTextParams {
fn default() -> Self {
Self {
url: String::new(),
max_urls: 1,
show_full_text: false,
experimental_stream: false,
}
}
}
/// Response from initiating a LLMs.txt generation job
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct GenerateLLMsTextResponse {
/// Whether the request was successful
pub success: bool,
/// Job ID for the LLMs.txt generation
pub id: String,
}
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct LLMTextData {
#[serde(rename = "llmstxt")]
pub compact: Option<String>,
#[serde(rename = "llmsfulltxt")]
pub full: Option<String>,
}
/// Response from checking the status of a LLMs.txt generation job
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct GenerateLLMsTextStatusResponse {
/// Whether the request was successful
pub success: bool,
/// Status of the job: "pending", "processing", "completed", "failed"
pub status: String,
/// Generated LLMs.txt data, present when status is "completed"
#[serde(default)]
pub data: LLMTextData,
/// Error message if the job failed
pub error: Option<String>,
/// Expiration timestamp for the data
pub expires_at: String,
}
impl FirecrawlApp {
/// Generates LLMs.txt for a given URL and polls until completion.
///
/// # Arguments
///
/// * `params` - Parameters for the LLMs.txt generation
///
/// # Returns
///
/// A response containing the generation results, or a FirecrawlError if the request fails.
pub async fn generate_llms_text(
&self,
params: impl Into<GenerateLLMsTextParams>,
) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {
// Initiate the LLMs.txt generation job asynchronously
let response = self.async_generate_llms_text(params).await?;
// Poll for the result
let poll_interval = 2000; // Default to 2 seconds
self.monitor_llms_text_job_status(&response.id, poll_interval)
.await
}
/// Initiates an asynchronous LLMs.txt generation operation.
///
/// # Arguments
///
/// * `params` - Parameters for the LLMs.txt generation
///
/// # Returns
///
/// A response containing the generation job ID, or a FirecrawlError if the request fails.
pub async fn async_generate_llms_text(
&self,
params: impl Into<GenerateLLMsTextParams>,
) -> Result<GenerateLLMsTextResponse, FirecrawlError> {
let params = params.into();
// Validation: URL must be provided
if params.url.is_empty() {
return Err(FirecrawlError::APIError(
"Generate LLMs.txt validation".to_string(),
crate::error::FirecrawlAPIError {
success: false,
error: "URL must be provided".to_string(),
details: None,
},
));
}
let headers = self.prepare_headers(None);
let response = self
.client
.post(format!("{}{}/llmstxt", self.api_url, API_VERSION))
.headers(headers)
.json(&params)
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError("Initiating LLMs.txt generation".to_string(), e)
})?;
self.handle_response(response, "initiate LLMs.txt generation")
.await
}
/// Checks the status of a LLMs.txt generation operation.
///
/// # Arguments
///
/// * `id` - The ID of the LLMs.txt generation operation
///
/// # Returns
///
/// A response containing the current status and results of the generation operation,
/// or a FirecrawlError if the request fails.
pub async fn check_generate_llms_text_status(
&self,
id: impl AsRef<str>,
) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {
let response = self
.client
.get(format!(
"{}{}/llmstxt/{}",
self.api_url,
API_VERSION,
id.as_ref()
))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(
format!("Checking status of LLMs.txt generation {}", id.as_ref()),
e,
)
})?;
self.handle_response(
response,
format!("Checking status of LLMs.txt generation {}", id.as_ref()),
)
.await
}
/// Helper function to poll for LLMs.txt generation job status until completion
async fn monitor_llms_text_job_status(
&self,
id: &str,
poll_interval: u64,
) -> Result<GenerateLLMsTextStatusResponse, FirecrawlError> {
loop {
let status_data = self.check_generate_llms_text_status(id).await?;
match status_data.status.as_str() {
"completed" => {
break Ok(status_data);
}
"pending" | "processing" => {
tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
}
"failed" => {
let error_msg = status_data
.error
.clone()
.unwrap_or_else(|| "LLMs.txt generation failed".to_string());
break Err(FirecrawlError::APIError(
"LLMs.txt generation failed".to_string(),
crate::error::FirecrawlAPIError {
success: false,
error: error_msg,
details: None,
},
));
}
_ => {
break Err(FirecrawlError::APIError(
"LLMs.txt generation status".to_string(),
crate::error::FirecrawlAPIError {
success: false,
error: format!("Unexpected status: {}", status_data.status),
details: None,
},
));
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[tokio::test]
#[ignore = "Makes real network request"]
async fn test_real_generate_llms_text() {
let api_url = std::env::var("FIRECRAWL_API_URL")
.expect("Please set the FIRECRAWL_API_URL environment variable");
let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
let params = GenerateLLMsTextParams {
url: "https://example.com".to_string(),
max_urls: 5,
show_full_text: true,
..Default::default()
};
let response = app.async_generate_llms_text(params).await.unwrap();
assert!(response.success);
assert!(!response.id.is_empty());
}
#[tokio::test]
async fn test_async_generate_llms_text_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v1/llmstxt")
.match_body(mockito::Matcher::PartialJson(json!({
"url": "https://example.com",
"maxUrls": 5
})))
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "llmstxt-123"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let params = GenerateLLMsTextParams {
url: "https://example.com".to_string(),
max_urls: 5,
..Default::default()
};
let response = app.async_generate_llms_text(params).await.unwrap();
assert!(response.success);
assert_eq!(response.id, "llmstxt-123");
mock.assert();
}
#[tokio::test]
async fn test_check_generate_llms_text_status_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/v1/llmstxt/llmstxt-123")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"status": "processing",
"expiresAt": "2023-01-01T00:00:00Z"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let status = app
.check_generate_llms_text_status("llmstxt-123")
.await
.unwrap();
assert!(status.success);
assert_eq!(status.status, "processing");
assert_eq!(status.expires_at, "2023-01-01T00:00:00Z");
mock.assert();
}
#[tokio::test]
async fn test_generate_llms_text_with_mock() {
let mut server = mockito::Server::new_async().await;
// Set up the mock for the generate request
let mock = server
.mock("POST", "/v1/llmstxt")
.match_body(mockito::Matcher::PartialJson(json!({
"url": "https://example.com",
"showFullText": true
})))
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"id": "llmstxt-123"
})
.to_string(),
)
.create();
// Set up the mock for the status request
let status_mock = server
.mock("GET", "/v1/llmstxt/llmstxt-123")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"status": "completed",
"data": {
"llmstxt": "Allow: /about\nDisallow: /admin\n",
"llmsfulltxt": "# LLMs.txt\n\nAllow: /about\nDisallow: /admin\n"
},
"expiresAt": "2023-01-01T00:00:00Z"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let params = GenerateLLMsTextParams {
url: "https://example.com".to_string(),
show_full_text: true,
..Default::default()
};
let response = app.generate_llms_text(params).await.unwrap();
assert!(response.success);
assert_eq!(response.status, "completed");
let data = response.data;
assert_eq!(
data.compact,
Some("Allow: /about\nDisallow: /admin\n".into())
);
assert_eq!(
data.full,
Some("# LLMs.txt\n\nAllow: /about\nDisallow: /admin\n".into())
);
mock.assert();
status_mock.assert();
}
#[tokio::test]
async fn test_generate_llms_text_validation_errors() {
let app = FirecrawlApp::new_selfhosted("https://example.com", Some("test_key")).unwrap();
// Test missing URL
let params = GenerateLLMsTextParams {
url: "".to_string(),
..Default::default()
};
let result = app.async_generate_llms_text(params).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_generate_llms_text_api_error() {
let mut server = mockito::Server::new_async().await;
// Set up the mock for an error response
let mock = server
.mock("POST", "/v1/llmstxt")
.with_status(400)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Invalid URL format"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let params = GenerateLLMsTextParams {
url: "not-a-valid-url".to_string(),
..Default::default()
};
let result = app.async_generate_llms_text(params).await;
assert!(result.is_err());
mock.assert();
}
}
apps/rust-sdk/src/map.rs
+4 -2
View File
@@ -16,7 +16,7 @@ pub struct MapOptions {
pub include_subdomains: Option<bool>,
/// Maximum number of links to return (default: `5000`)
pub exclude_tags: Option<u32>,
pub limit: Option<u32>,
}
#[derive(Deserialize, Serialize, Debug, Default)]
@@ -59,7 +59,9 @@ impl FirecrawlApp {
.await
.map_err(|e| FirecrawlError::HttpError(format!("Mapping {:?}", url.as_ref()), e))?;
let response = self.handle_response::<MapResponse>(response, "scrape URL").await?;
let response = self
.handle_response::<MapResponse>(response, "scrape URL")
.await?;
Ok(response.links)
}
apps/rust-sdk/src/scrape.rs
+10 -8
View File
@@ -24,26 +24,26 @@ pub enum ScrapeFormats {
Links,
/// Will result in a URL to a screenshot of the page.
///
///
/// Can not be used in conjunction with `ScrapeFormats::ScreenshotFullPage`.
#[serde(rename = "screenshot")]
Screenshot,
/// Will result in a URL to a full-page screenshot of the page.
///
///
/// Can not be used in conjunction with `ScrapeFormats::Screenshot`.
#[serde(rename = "screenshot@fullPage")]
ScreenshotFullPage,
/// Will result in the results of an LLM extraction.
///
///
/// See `ScrapeOptions.extract` for more options.
#[serde(rename = "extract")]
Extract,
}
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default)]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ExtractOptions {
/// Schema the output should adhere to, provided in JSON Schema format.
@@ -56,7 +56,7 @@ pub struct ExtractOptions {
}
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default)]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeOptions {
/// Formats to extract from the page. (default: `[ Markdown ]`)
@@ -66,12 +66,12 @@ pub struct ScrapeOptions {
pub only_main_content: Option<bool>,
/// HTML tags to exclusively include.
///
///
/// For example, if you pass `div`, you will only get content from `<div>`s and their children.
pub include_tags: Option<Vec<String>>,
/// HTML tags to exclude.
///
///
/// For example, if you pass `img`, you will never get image URLs in your results.
pub exclude_tags: Option<Vec<String>>,
@@ -131,7 +131,9 @@ impl FirecrawlApp {
.await
.map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;
let response = self.handle_response::<ScrapeResponse>(response, "scrape URL").await?;
let response = self
.handle_response::<ScrapeResponse>(response, "scrape URL")
.await?;
Ok(response.data)
}
apps/rust-sdk/src/search.rs
+245
View File
@@ -0,0 +1,245 @@
use crate::{scrape::ScrapeOptions, FirecrawlApp, FirecrawlError, API_VERSION};
use serde::{Deserialize, Serialize};
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct SearchParams {
/// The search query string
pub query: String,
/// Maximum number of results to return. Default: 5, Max: 20
pub limit: Option<u32>,
/// Time-based search filter.
#[serde(skip_serializing_if = "Option::is_none")]
pub tbs: Option<String>,
/// Query string to filter search results. Example: "site:example.com"
#[serde(skip_serializing_if = "Option::is_none")]
pub filter: Option<String>,
/// Language code. Default: "en"
pub lang: Option<String>,
/// Country code. Default: "us"
pub country: Option<String>,
/// Geographic location string for local search results
#[serde(skip_serializing_if = "Option::is_none")]
pub location: Option<String>,
/// Origin identifier. Default: "api"
pub origin: Option<String>,
/// Timeout in milliseconds. Default: 60000
pub timeout: Option<u32>,
/// Additional options for webpage scraping behavior
#[serde(skip_serializing_if = "Option::is_none")]
pub scrape_options: Option<ScrapeOptions>,
}
impl Default for SearchParams {
fn default() -> Self {
Self {
query: String::new(),
limit: Some(5),
tbs: None,
filter: None,
lang: Some("en".to_string()),
country: Some("us".to_string()),
location: None,
origin: Some("api".to_string()),
timeout: Some(60000),
scrape_options: None,
}
}
}
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct SearchResponse {
pub success: bool,
pub data: Vec<SearchDocument>,
pub warning: Option<String>,
}
// TODO: Consider merging fields into document::Document (url, title, description) while preserving optionality
/// A document returned from a search or scrape request
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct SearchDocument {
/// Document URL
pub url: String,
/// Document title
pub title: String,
/// Document description
pub description: String,
}
impl FirecrawlApp {
/// Search for content using the Firecrawl API.
///
/// # Arguments
///
/// * `query` - The search query string
/// * `params` - Optional parameters for the search request
///
/// # Returns
///
/// A SearchResponse containing the search results, or a FirecrawlError if the request fails.
pub async fn search(
&self,
query: impl AsRef<str>,
params: impl Into<Option<SearchParams>>,
) -> Result<SearchResponse, FirecrawlError> {
let mut search_params = params.into().unwrap_or_default();
search_params.query = query.as_ref().to_string();
self.search_with_params(search_params).await
}
/// Alternative method that takes SearchParams directly
///
/// # Arguments
///
/// * `params` - Search parameters including the query
///
/// # Returns
///
/// A SearchResponse containing the search results, or a FirecrawlError if the request fails.
pub async fn search_with_params(
&self,
params: SearchParams,
) -> Result<SearchResponse, FirecrawlError> {
let headers = self.prepare_headers(None);
let response = self
.client
.post(format!("{}{}/search", self.api_url, API_VERSION))
.headers(headers)
.json(&params)
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(format!("Searching with query: {:?}", params.query), e)
})?;
self.handle_response::<SearchResponse>(response, "search")
.await
}
}
#[cfg(test)]
pub mod tests {
use super::*;
use serde_json::json;
#[tokio::test]
#[ignore = "Makes real network request"]
async fn test_real_search() {
let api_url = std::env::var("FIRECRAWL_API_URL")
.expect("Please set the FIRECRAWL_API_URL environment variable");
let app = FirecrawlApp::new_selfhosted(api_url, None::<&str>).unwrap();
let response = app.search("test query", None).await.unwrap();
assert!(response.success);
}
#[tokio::test]
async fn test_search_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v1/search")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": [{
"url": "https://example.com",
"title": "Example Domain",
"description": "...."
}],
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = app.search("test", None).await.unwrap();
assert!(response.success);
assert_eq!(response.data.len(), 1);
assert_eq!(response.data[0].url, "https://example.com");
assert_eq!(response.data[0].title, "Example Domain".to_string());
assert_eq!(response.data[0].description, "....".to_string());
mock.assert();
}
#[tokio::test]
async fn test_search_with_params() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v1/search")
.with_header("content-type", "application/json")
.match_body(mockito::Matcher::Json(json!({
"query": "test",
"limit": 10,
"lang": "fr",
"country": "fr",
"origin": "api",
"timeout": 30000
})))
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": [],
"warning": "No results found"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let params = SearchParams {
query: "test".to_string(),
limit: Some(10),
lang: Some("fr".to_string()),
country: Some("fr".to_string()),
timeout: Some(30000),
..Default::default()
};
let response = app.search_with_params(params).await.unwrap();
assert!(response.success);
assert_eq!(response.data.len(), 0);
assert_eq!(response.warning, Some("No results found".to_string()));
mock.assert();
}
#[tokio::test]
async fn test_search_error_response() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v1/search")
.with_status(400)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Invalid query"
})
.to_string(),
)
.create();
let app = FirecrawlApp::new_selfhosted(server.url(), Some("test_key")).unwrap();
let result = app.search("", None).await;
assert!(result.is_err());
mock.assert();
}
#[tokio::test]
async fn test_search_network_error() {
let app = FirecrawlApp::new_selfhosted("http://invalid-url", Some("test_key")).unwrap();
let result = app.search("test", None).await;
assert!(result.is_err());
}
}