init

apps/api/src/controllers/scrape.ts

@@ -58,19 +58,26 @@ export async function scrapeHelper(
   }
 
   // make sure doc.content is not empty
-  const filteredDocs = docs.filter(
+  let filteredDocs = docs.filter(
     (doc: { content?: string }) => doc.content && doc.content.trim().length > 0
   );
   if (filteredDocs.length === 0) {
     return { success: true, error: "No page found", returnCode: 200, data: docs[0] };
   }
 
+  // Remove rawHtml if pageOptions.rawHtml is false
+  if (!pageOptions.rawHtml) {
+    filteredDocs.forEach(doc => {
+      delete doc.rawHtml;
+    });
+  }
+
   let creditsToBeBilled = filteredDocs.length;
   const creditsPerLLMExtract = 50;
 
 
 
-  if (extractorOptions.mode === "llm-extraction") {
+  if (extractorOptions.mode === "llm-extraction" || extractorOptions.mode === "llm-extraction-from-raw-html" || extractorOptions.mode === "llm-extraction-from-markdown") {
     creditsToBeBilled = creditsToBeBilled + (creditsPerLLMExtract * filteredDocs.length);
   }
 

apps/api/src/lib/LLM-extraction/index.ts

@@ -8,7 +8,8 @@ import { Document, ExtractorOptions } from "../entities";
 // Generate completion using OpenAI
 export async function generateCompletions(
   documents: Document[],
-  extractionOptions: ExtractorOptions
+  extractionOptions: ExtractorOptions,
+  mode: "markdown" | "raw-html"
 ): Promise<Document[]> {
   // const schema = zodToJsonSchema(options.schema)
 
@@ -28,6 +29,7 @@ export async function generateCompletions(
             document: document,
             schema: schema,
             prompt: prompt,
+            mode: mode,
           });
           // Validate the JSON output against the schema using AJV
           const validate = ajv.compile(schema);

apps/api/src/lib/LLM-extraction/models.ts

@@ -13,26 +13,37 @@ const defaultPrompt =
   "You are a professional web scraper. Extract the contents of the webpage";
 
 function prepareOpenAIDoc(
-  document: Document
+  document: Document,
+  mode: "markdown" | "raw-html"
 ): [OpenAI.Chat.Completions.ChatCompletionContentPart[], number] {
+
   let markdown = document.markdown;
 
-// Check if the markdown content exists in the document
+  let extractionTarget = document.markdown;
-  if (!markdown) {
+
+  if (mode === "raw-html") {
+    extractionTarget = document.rawHtml;
+  }
+
+  // Check if the markdown content exists in the document
+  if (!extractionTarget) {
     throw new Error(
-      "Markdown content is missing in the document. This is likely due to an error in the scraping process. Please try again or reach out to help@mendable.ai"
+      `${mode} content is missing in the document. This is likely due to an error in the scraping process. Please try again or reach out to help@mendable.ai`
     );
   }
 
+
+
+
   // count number of tokens
-  const numTokens = numTokensFromString(document.markdown, "gpt-4");
+  const numTokens = numTokensFromString(extractionTarget, "gpt-4");
 
   if (numTokens > maxTokens) {
     // trim the document to the maximum number of tokens, tokens != characters
-    markdown = markdown.slice(0, (maxTokens * modifier));
+    extractionTarget = extractionTarget.slice(0, (maxTokens * modifier));
   }
 
-  return [[{ type: "text", text: markdown }], numTokens];
+  return [[{ type: "text", text: extractionTarget }], numTokens];
 }
 
 export async function generateOpenAICompletions({
@@ -42,6 +53,7 @@ export async function generateOpenAICompletions({
   schema, //TODO - add zod dynamic type checking
   prompt = defaultPrompt,
   temperature,
+  mode
 }: {
   client: OpenAI;
   model?: string;
@@ -49,9 +61,10 @@ export async function generateOpenAICompletions({
   schema: any; // This should be replaced with a proper Zod schema type when available
   prompt?: string;
   temperature?: number;
+  mode: "markdown" | "raw-html";
 }): Promise<Document> {
   const openai = client as OpenAI;
-  const [content, numTokens] = prepareOpenAIDoc(document);
+  const [content, numTokens] = prepareOpenAIDoc(document, mode);
 
   const completion = await openai.chat.completions.create({
     model,

apps/api/src/lib/entities.ts

@@ -13,6 +13,7 @@ export interface Progress {
 export type PageOptions = {
   onlyMainContent?: boolean;
   includeHtml?: boolean;
+  rawHtml?: boolean;
   fallback?: boolean;
   fetchPageContent?: boolean;
   waitFor?: number;
@@ -25,7 +26,7 @@ export type PageOptions = {
 };
 
 export type ExtractorOptions = {
-  mode: "markdown" | "llm-extraction";
+  mode: "markdown" | "llm-extraction" | "llm-extraction-from-markdown" | "llm-extraction-from-raw-html";
   extractionPrompt?: string;
   extractionSchema?: Record<string, any>;
 }
@@ -73,6 +74,7 @@ export class Document {
   content: string;
   markdown?: string;
   html?: string;
+  rawHtml?: string;
   llm_extraction?: Record<string, any>;
   createdAt?: Date;
   updatedAt?: Date;

apps/api/src/scraper/WebScraper/index.ts

@@ -66,6 +66,7 @@ export class WebScraperDataProvider {
           const result = await scrapSingleUrl(
             url,
             this.pageOptions,
+            this.extractorOptions,
             existingHTML
           );
           processedUrls++;
@@ -269,10 +270,16 @@ export class WebScraperDataProvider {
     // documents = await this.applyImgAltText(documents);
 
     if (
-      this.extractorOptions.mode === "llm-extraction" &&
+      (this.extractorOptions.mode === "llm-extraction" || this.extractorOptions.mode === "llm-extraction-from-markdown") &&
       this.mode === "single_urls"
     ) {
-      documents = await generateCompletions(documents, this.extractorOptions);
+      documents = await generateCompletions(documents, this.extractorOptions, "markdown");
+    }
+    if (
+      (this.extractorOptions.mode === "llm-extraction-from-raw-html") &&
+      this.mode === "single_urls"
+    ) {
+      documents = await generateCompletions(documents, this.extractorOptions, "raw-html");
     }
     return documents.concat(pdfDocuments).concat(docxDocuments);
   }

apps/api/src/scraper/WebScraper/single_url.ts

@@ -2,7 +2,7 @@ import * as cheerio from "cheerio";
 import { ScrapingBeeClient } from "scrapingbee";
 import { extractMetadata } from "./utils/metadata";
 import dotenv from "dotenv";
-import { Document, PageOptions, FireEngineResponse } from "../../lib/entities";
+import { Document, PageOptions, FireEngineResponse, ExtractorOptions } from "../../lib/entities";
 import { parseMarkdown } from "../../lib/html-to-markdown";
 import { urlSpecificParams } from "./utils/custom/website_params";
 import { fetchAndProcessPdf } from "./utils/pdfProcessor";
@@ -303,10 +303,14 @@ export async function scrapSingleUrl(
   pageOptions: PageOptions = {
     onlyMainContent: true,
     includeHtml: false,
+    rawHtml: false,
     waitFor: 0,
     screenshot: false,
     headers: undefined
   },
+  extractorOptions: ExtractorOptions = {
+    mode: "llm-extraction-from-markdown"
+  },
   existingHtml: string = ""
 ): Promise<Document> {
   urlToScrap = urlToScrap.trim();
@@ -465,6 +469,7 @@ export async function scrapSingleUrl(
         content: text,
         markdown: text,
         html: pageOptions.includeHtml ? html : undefined,
+        rawHtml: pageOptions.rawHtml || extractorOptions.mode === "llm-extraction-from-raw-html" ? rawHtml : undefined,
         metadata: {
           ...metadata,
           screenshot: screenshot,
@@ -478,6 +483,7 @@ export async function scrapSingleUrl(
         content: text,
         markdown: text,
         html: pageOptions.includeHtml ? html : undefined,
+        rawHtml: pageOptions.rawHtml || extractorOptions.mode === "llm-extraction-from-raw-html" ? rawHtml : undefined,
         metadata: {
           ...metadata,
           sourceURL: urlToScrap,