map + search + scrape markdown bug

apps/api/src/search/fireEngine.ts

@@ -4,42 +4,41 @@ import { SearchResult } from "../../src/lib/entities";
 
 dotenv.config();
 
-export async function serper_search(q, options: {
+export async function fireEngineSearch(q: string, options: {
     tbs?: string;
     filter?: string;
     lang?: string;
     country?: string;
     location?: string;
-    num_results: number;
+    numResults: number;
     page?: number;
 }): Promise<SearchResult[]> {
   let data = JSON.stringify({
     q: q,
-    hl: options.lang,
-    gl: options.country,
+    lang: options.lang,
+    country: options.country,
     location: options.location,
     tbs: options.tbs,
-    num: options.num_results,
+    num: options.numResults,
     page: options.page ?? 1,
   });
 
+  if (!process.env.FIRE_ENGINE_BETA_URL) {
+    return [];
+  }
+
   let config = {
     method: "POST",
-    url: "https://google.serper.dev/search",
+    url: `${process.env.FIRE_ENGINE_BETA_URL}/search`,
     headers: {
-      "X-API-KEY": process.env.SERPER_API_KEY,
       "Content-Type": "application/json",
     },
     data: data,
   };
   const response = await axios(config);
-  if (response && response.data && Array.isArray(response.data.organic)) {
-    return response.data.organic.map((a) => ({
-      url: a.link,
-      title: a.title,
-      description: a.snippet,
-    }));
-  }else{
+  if (response && response.data) {
+    return response.data
+  } else {
     return [];
   }
 }

apps/api/src/search/googlesearch.ts

@@ -52,7 +52,7 @@ async function _req(term: string, results: number, lang: string, country: string
 
 
 
-export async function google_search(term: string, advanced = false, num_results = 7, tbs = null, filter = null, lang = "en", country = "us", proxy = null, sleep_interval = 0, timeout = 5000, ) :Promise<SearchResult[]> {
+export async function googleSearch(term: string, advanced = false, num_results = 7, tbs = null, filter = null, lang = "en", country = "us", proxy = null, sleep_interval = 0, timeout = 5000, ) :Promise<SearchResult[]> {
     let proxies = null;
     if (proxy) {
         if (proxy.startsWith("https")) {

apps/api/src/search/index.ts

@@ -1,10 +1,7 @@
 import { Logger } from "../../src/lib/logger";
 import { SearchResult } from "../../src/lib/entities";
-import { google_search } from "./googlesearch";
-import { serper_search } from "./serper";
-
-
-
+import { googleSearch } from "./googlesearch";
+import { fireEngineSearch } from "./fireEngine";
 
 export async function search({
   query,
@@ -32,10 +29,10 @@ export async function search({
   timeout?: number;
 }) : Promise<SearchResult[]> {
   try {
-    if (process.env.SERPER_API_KEY ) {
-      return await serper_search(query, {num_results, tbs, filter, lang, country, location});
+    if (process.env.FIRE_ENGINE_BETA_URL) {
+      return await fireEngineSearch(query, {numResults: num_results, tbs, filter, lang, country, location});
     }
-    return await google_search(
+    return await googleSearch(
       query,
       advanced,
       num_results,
@@ -51,5 +48,4 @@ export async function search({
     Logger.error(`Error in search function: ${error}`);
     return []
   }
-  // if process.env.SERPER_API_KEY is set, use serper
 }