Spliting relative paths for images

2024-04-16 16:31:33 -03:00
parent 3e4064bce2
commit a04610302a
2 changed files with 32 additions and 9 deletions
@@ -90,6 +90,7 @@ app.post("/v0/scrape", async (req, res) => {
  try {
    // make sure to authenticate user first, Bearer <token>
    const team_id = await authenticateUser(req, res, "scrape");
+    const crawlerOptions = req.body.crawlerOptions ?? {};

    try {
      const { success: creditsCheckSuccess, message: creditsCheckMessage } =
@@ -113,6 +114,9 @@ app.post("/v0/scrape", async (req, res) => {
      await a.setOptions({
        mode: "single_urls",
        urls: [url],
+        crawlerOptions: {
+          ...crawlerOptions,
+        },
      });

      const docs = await a.getDocuments(false);