Resolve merge conflicts with main

2024-04-30 10:54:18 -03:00
parent 35480bd2ad d3c36adaa7
commit a095e1b63d
73 changed files with 7136 additions and 1052 deletions
@@ -0,0 +1,328 @@
+import request from "supertest";
+import { app } from "../../index";
+import dotenv from "dotenv";
+
+dotenv.config();
+
+// const TEST_URL = 'http://localhost:3002'
+const TEST_URL = "http://127.0.0.1:3002";
+
+
+  describe("E2E Tests for API Routes", () => {
+    beforeAll(() => {
+      process.env.USE_DB_AUTHENTICATION = "true";
+    });
+
+    afterAll(() => {
+      delete process.env.USE_DB_AUTHENTICATION;
+    });
+    describe("GET /", () => {
+      it("should return Hello, world! message", async () => {
+        const response = await request(TEST_URL).get("/");
+
+        expect(response.statusCode).toBe(200);
+        expect(response.text).toContain("SCRAPERS-JS: Hello, world! Fly.io");
+      });
+    });
+
+    describe("GET /test", () => {
+      it("should return Hello, world! message", async () => {
+        const response = await request(TEST_URL).get("/test");
+        expect(response.statusCode).toBe(200);
+        expect(response.text).toContain("Hello, world!");
+      });
+    });
+
+    describe("POST /v0/scrape", () => {
+      it("should require authorization", async () => {
+        const response = await request(app).post("/v0/scrape");
+        expect(response.statusCode).toBe(401);
+      });
+
+      it("should return an error response with an invalid API key", async () => {
+        const response = await request(TEST_URL)
+          .post("/v0/scrape")
+          .set("Authorization", `Bearer invalid-api-key`)
+          .set("Content-Type", "application/json")
+          .send({ url: "https://firecrawl.dev" });
+        expect(response.statusCode).toBe(401);
+      });
+
+      it("should return an error for a blocklisted URL", async () => {
+        const blocklistedUrl = "https://facebook.com/fake-test";
+        const response = await request(TEST_URL)
+          .post("/v0/scrape")
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+          .set("Content-Type", "application/json")
+          .send({ url: blocklistedUrl });
+        expect(response.statusCode).toBe(403);
+        expect(response.body.error).toContain("Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.");
+      });
+
+      it("should return a successful response with a valid preview token", async () => {
+        const response = await request(TEST_URL)
+          .post("/v0/scrape")
+          .set("Authorization", `Bearer this_is_just_a_preview_token`)
+          .set("Content-Type", "application/json")
+          .send({ url: "https://firecrawl.dev" });
+        expect(response.statusCode).toBe(200);
+      }, 10000); // 10 seconds timeout
+
+      it("should return a successful response with a valid API key", async () => {
+        const response = await request(TEST_URL)
+          .post("/v0/scrape")
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+          .set("Content-Type", "application/json")
+          .send({ url: "https://firecrawl.dev" });
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toHaveProperty("data");
+        expect(response.body.data).toHaveProperty("content");
+        expect(response.body.data).toHaveProperty("markdown");
+        expect(response.body.data).toHaveProperty("metadata");
+        expect(response.body.data.content).toContain("🔥 FireCrawl");
+      }, 30000); // 30 seconds timeout
+
+      it('should return a successful response for a valid scrape with PDF file', async () => {
+        const response = await request(TEST_URL)
+          .post('/v0/scrape')
+          .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
+          .set('Content-Type', 'application/json')
+          .send({ url: 'https://arxiv.org/pdf/astro-ph/9301001.pdf' });
+        await new Promise((r) => setTimeout(r, 6000));
+        
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toHaveProperty('data');
+        expect(response.body.data).toHaveProperty('content');
+        expect(response.body.data).toHaveProperty('metadata');
+        expect(response.body.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
+      }, 30000); // 30 seconds
+  
+      it('should return a successful response for a valid scrape with PDF file without explicit .pdf extension', async () => {
+        const response = await request(TEST_URL)
+          .post('/v0/scrape')
+          .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
+          .set('Content-Type', 'application/json')
+          .send({ url: 'https://arxiv.org/pdf/astro-ph/9301001' });
+        await new Promise((r) => setTimeout(r, 6000));
+        
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toHaveProperty('data');
+        expect(response.body.data).toHaveProperty('content');
+        expect(response.body.data).toHaveProperty('metadata');
+        expect(response.body.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
+      }, 30000); // 30 seconds
+    });
+
+    describe("POST /v0/crawl", () => {
+      it("should require authorization", async () => {
+        const response = await request(TEST_URL).post("/v0/crawl");
+        expect(response.statusCode).toBe(401);
+      });
+
+      it("should return an error response with an invalid API key", async () => {
+        const response = await request(TEST_URL)
+          .post("/v0/crawl")
+          .set("Authorization", `Bearer invalid-api-key`)
+          .set("Content-Type", "application/json")
+          .send({ url: "https://firecrawl.dev" });
+        expect(response.statusCode).toBe(401);
+      });
+
+      it("should return an error for a blocklisted URL", async () => {
+        const blocklistedUrl = "https://twitter.com/fake-test";
+        const response = await request(TEST_URL)
+          .post("/v0/crawl")
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+          .set("Content-Type", "application/json")
+          .send({ url: blocklistedUrl });
+        expect(response.statusCode).toBe(403);
+        expect(response.body.error).toContain("Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.");
+      });
+
+      it("should return a successful response with a valid API key", async () => {
+        const response = await request(TEST_URL)
+          .post("/v0/crawl")
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+          .set("Content-Type", "application/json")
+          .send({ url: "https://firecrawl.dev" });
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toHaveProperty("jobId");
+        expect(response.body.jobId).toMatch(
+          /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
+        );
+      });
+
+
+      // Additional tests for insufficient credits?
+    });
+
+    describe("POST /v0/crawlWebsitePreview", () => {
+      it("should require authorization", async () => {
+        const response = await request(TEST_URL).post(
+          "/v0/crawlWebsitePreview"
+        );
+        expect(response.statusCode).toBe(401);
+      });
+
+      it("should return an error response with an invalid API key", async () => {
+        const response = await request(TEST_URL)
+          .post("/v0/crawlWebsitePreview")
+          .set("Authorization", `Bearer invalid-api-key`)
+          .set("Content-Type", "application/json")
+          .send({ url: "https://firecrawl.dev" });
+        expect(response.statusCode).toBe(401);
+      });
+
+      it("should return an error for a blocklisted URL", async () => {
+        const blocklistedUrl = "https://instagram.com/fake-test";
+        const response = await request(TEST_URL)
+          .post("/v0/crawlWebsitePreview")
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+          .set("Content-Type", "application/json")
+          .send({ url: blocklistedUrl });
+        expect(response.statusCode).toBe(403);
+        expect(response.body.error).toContain("Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it.");
+      });
+
+      it("should return a successful response with a valid API key", async () => {
+        const response = await request(TEST_URL)
+          .post("/v0/crawlWebsitePreview")
+          .set("Authorization", `Bearer this_is_just_a_preview_token`)
+          .set("Content-Type", "application/json")
+          .send({ url: "https://firecrawl.dev" });
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toHaveProperty("jobId");
+        expect(response.body.jobId).toMatch(
+          /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
+        );
+      });
+    });
+
+    describe("POST /v0/search", () => {
+      it("should require authorization", async () => {
+        const response = await request(TEST_URL).post("/v0/search");
+        expect(response.statusCode).toBe(401);
+      });
+
+      it("should return an error response with an invalid API key", async () => {
+        const response = await request(TEST_URL)
+          .post("/v0/search")
+          .set("Authorization", `Bearer invalid-api-key`)
+          .set("Content-Type", "application/json")
+          .send({ query: "test" });
+        expect(response.statusCode).toBe(401);
+      });
+
+
+      
+      it("should return a successful response with a valid API key", async () => {
+        const response = await request(TEST_URL)
+          .post("/v0/search")
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+          .set("Content-Type", "application/json")
+          .send({ query: "test" });
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toHaveProperty("success");
+        expect(response.body.success).toBe(true);
+        expect(response.body).toHaveProperty("data");
+      }, 30000); // 30 seconds timeout
+    });
+
+    describe("GET /v0/crawl/status/:jobId", () => {
+      it("should require authorization", async () => {
+        const response = await request(TEST_URL).get("/v0/crawl/status/123");
+        expect(response.statusCode).toBe(401);
+      });
+
+      it("should return an error response with an invalid API key", async () => {
+        const response = await request(TEST_URL)
+          .get("/v0/crawl/status/123")
+          .set("Authorization", `Bearer invalid-api-key`);
+        expect(response.statusCode).toBe(401);
+      });
+
+      it("should return Job not found for invalid job ID", async () => {
+        const response = await request(TEST_URL)
+          .get("/v0/crawl/status/invalidJobId")
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+        expect(response.statusCode).toBe(404);
+      });
+
+      it("should return a successful response for a valid crawl job", async () => {
+        const crawlResponse = await request(TEST_URL)
+          .post("/v0/crawl")
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+          .set("Content-Type", "application/json")
+          .send({ url: "https://firecrawl.dev" });
+        expect(crawlResponse.statusCode).toBe(200);
+
+        const response = await request(TEST_URL)
+          .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toHaveProperty("status");
+        expect(response.body.status).toBe("active");
+
+        // wait for 30 seconds
+        await new Promise((r) => setTimeout(r, 30000));
+
+        const completedResponse = await request(TEST_URL)
+          .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
+          .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
+        expect(completedResponse.statusCode).toBe(200);
+        expect(completedResponse.body).toHaveProperty("status");
+        expect(completedResponse.body.status).toBe("completed");
+        expect(completedResponse.body).toHaveProperty("data");
+        expect(completedResponse.body.data[0]).toHaveProperty("content");
+        expect(completedResponse.body.data[0]).toHaveProperty("markdown");
+        expect(completedResponse.body.data[0]).toHaveProperty("metadata");
+        expect(completedResponse.body.data[0].content).toContain(
+          "🔥 FireCrawl"
+        );
+      }, 60000); // 60 seconds
+
+      it('should return a successful response for a valid crawl job with PDF files without explicit .pdf extension', async () => {
+        const crawlResponse = await request(TEST_URL)
+          .post('/v0/crawl')
+          .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`)
+          .set('Content-Type', 'application/json')
+          .send({ url: 'https://arxiv.org/abs/astro-ph/9301001', crawlerOptions: { limit: 10, excludes: [ 'list/*', 'login', 'abs/*', 'static/*', 'about/*', 'archive/*' ] }});
+        expect(crawlResponse.statusCode).toBe(200);
+  
+        const response = await request(TEST_URL)
+          .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
+          .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`);
+          expect(response.statusCode).toBe(200);
+          expect(response.body).toHaveProperty('status');
+          expect(response.body.status).toBe('active');
+  
+          // wait for 30 seconds
+          await new Promise((r) => setTimeout(r, 30000));
+    
+          const completedResponse = await request(TEST_URL)
+          .get(`/v0/crawl/status/${crawlResponse.body.jobId}`)
+          .set('Authorization', `Bearer ${process.env.TEST_API_KEY}`);
+  
+          expect(completedResponse.statusCode).toBe(200);
+          expect(completedResponse.body).toHaveProperty('status');
+          expect(completedResponse.body.status).toBe('completed');
+          expect(completedResponse.body).toHaveProperty('data');
+          expect(completedResponse.body.data.length).toBeGreaterThan(1);
+          expect(completedResponse.body.data).toEqual(
+            expect.arrayContaining([
+              expect.objectContaining({
+                content: expect.stringContaining('asymmetries might represent, for instance, preferred source orientations to our line of sight.')
+              })
+            ])
+          );
+      }, 60000); // 60 seconds
+    });
+
+    describe("GET /is-production", () => {
+      it("should return the production status", async () => {
+        const response = await request(TEST_URL).get("/is-production");
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toHaveProperty("isProduction");
+      });
+    });
+  });