Nick: test suite init

apps/test-suite/utils/log.ts

@@ -0,0 +1,10 @@
+import { supabase_service } from "./supabase";
+import { WebsiteScrapeError } from "./types";
+
+export async function logErrors(dataError: WebsiteScrapeError[], time_taken: number, num_tokens:number, score: number) {
+  try {
+    await supabase_service.from("test_suite_logs").insert([{log:dataError, time_taken, num_tokens, score}]);
+  } catch (error) {
+    console.error(`Error logging to supabase: ${error}`);
+  }
+}

apps/test-suite/utils/misc.ts

@@ -0,0 +1,47 @@
+const getRandomLinksFromContent = async (options: {
+    content: string;
+    excludes: string[];
+    limit: number;
+  }): Promise<string[]> => {
+    const regex = /(?<=\()https:\/\/(.*?)(?=\))/g;
+    const links = options.content.match(regex);
+    const filteredLinks = links
+      ? links.filter(
+          (link) => !options.excludes.some((exclude) => link.includes(exclude))
+        )
+      : [];
+    const uniqueLinks = [...new Set(filteredLinks)]; // Ensure all links are unique
+    const randomLinks = [];
+    while (randomLinks.length < options.limit && uniqueLinks.length > 0) {
+      const randomIndex = Math.floor(Math.random() * uniqueLinks.length);
+      randomLinks.push(uniqueLinks.splice(randomIndex, 1)[0]);
+    }
+    return randomLinks;
+  };
+  
+  function fuzzyContains(options: {
+    largeText: string;
+    queryText: string;
+    threshold?: number;
+  }): boolean {
+    // Normalize texts: lowercasing and removing non-alphanumeric characters
+    const normalize = (text: string) =>
+      text.toLowerCase().replace(/[^a-z0-9]+/g, " ");
+  
+    const normalizedLargeText = normalize(options.largeText);
+    const normalizedQueryText = normalize(options.queryText);
+  
+    // Split the query into words
+    const queryWords = normalizedQueryText.split(/\s+/);
+  
+    // Count how many query words are in the large text
+    const matchCount = queryWords.reduce((count, word) => {
+      return count + (normalizedLargeText.includes(word) ? 1 : 0);
+    }, 0);
+  
+    // Calculate the percentage of words matched
+    const matchPercentage = matchCount / queryWords.length;
+  
+    // Check if the match percentage meets or exceeds the threshold
+    return matchPercentage >= (options.threshold || 0.8);
+  }

apps/test-suite/utils/supabase.ts

@@ -0,0 +1,56 @@
+import { createClient, SupabaseClient } from "@supabase/supabase-js";
+import "dotenv/config";
+// SupabaseService class initializes the Supabase client conditionally based on environment variables.
+class SupabaseService {
+  private client: SupabaseClient | null = null;
+
+  constructor() {
+    const supabaseUrl = process.env.SUPABASE_URL;
+    const supabaseServiceToken = process.env.SUPABASE_SERVICE_TOKEN;
+    // Only initialize the Supabase client if both URL and Service Token are provided.
+    if (process.env.USE_DB_AUTHENTICATION === "false") {
+      // Warn the user that Authentication is disabled by setting the client to null
+      console.warn(
+        "\x1b[33mAuthentication is disabled. Supabase client will not be initialized.\x1b[0m"
+      );
+      this.client = null;
+    } else if (!supabaseUrl || !supabaseServiceToken) {
+      console.error(
+        "\x1b[31mSupabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable\x1b[0m"
+      );
+    } else {
+      this.client = createClient(supabaseUrl, supabaseServiceToken);
+    }
+  }
+
+  // Provides access to the initialized Supabase client, if available.
+  getClient(): SupabaseClient | null {
+    return this.client;
+  }
+}
+
+// Using a Proxy to handle dynamic access to the Supabase client or service methods.
+// This approach ensures that if Supabase is not configured, any attempt to use it will result in a clear error.
+export const supabase_service: SupabaseClient = new Proxy(
+  new SupabaseService(),
+  {
+    get: function (target, prop, receiver) {
+      const client = target.getClient();
+      // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
+      if (client === null) {
+        console.error(
+          "Attempted to access Supabase client when it's not configured."
+        );
+        return () => {
+          throw new Error("Supabase client is not configured.");
+        };
+      }
+      // Direct access to SupabaseService properties takes precedence.
+      if (prop in target) {
+        return Reflect.get(target, prop, receiver);
+      }
+      // Otherwise, delegate access to the Supabase client.
+      return Reflect.get(client, prop, receiver);
+    },
+  }
+) as unknown as SupabaseClient;

apps/test-suite/utils/tokens.ts

@@ -0,0 +1,16 @@
+import { encoding_for_model } from "@dqbd/tiktoken";
+import { TiktokenModel } from "@dqbd/tiktoken";
+
+// This function calculates the number of tokens in a text string using GPT-3.5-turbo model
+export function numTokensFromString(message: string, model: string): number {
+  const encoder = encoding_for_model(model as TiktokenModel);
+
+  // Encode the message into tokens
+  const tokens = encoder.encode(message);
+
+  // Free the encoder resources after use
+  encoder.free();
+
+  // Return the number of tokens
+  return tokens.length;
+}

apps/test-suite/utils/types.ts

@@ -0,0 +1,7 @@
+export interface WebsiteScrapeError {
+  website: string;
+  prompt: string;
+  expected_output: string;
+  actual_output: string;
+  error: string;
+}