Nick: from cache fixes

apps/api/src/scraper/scrapeURL/engines/cache/index.ts

@@ -10,6 +10,9 @@ export async function scrapeCache(meta: Meta): Promise<EngineScrapeResult> {
   const entry = await getEntryFromCache(key);
   if (entry === null) throw new EngineError("Cache missed");
 
+  // Set fromCache flag to indicate this document was retrieved from cache
+  meta.internalOptions.fromCache = true;
+
   return {
     url: entry.url,
     html: entry.html,

apps/api/src/scraper/scrapeURL/index.ts

@@ -154,6 +154,7 @@ export type InternalOptions = {
   useCache?: boolean;
   disableSmartWaitCache?: boolean; // Passed along to fire-engine
   isBackgroundIndex?: boolean;
+  fromCache?: boolean; // Indicates if the document was retrieved from cache
 };
 
 export type EngineResultsTracker = {

apps/api/src/scraper/scrapeURL/transformers/cache.ts

@@ -19,6 +19,12 @@ export function saveToCache(meta: Meta, document: Document): Document {
     );
   }
 
+  // If the document was retrieved from cache, we don't need to save it
+  if (meta.internalOptions.fromCache) {
+    return document;
+  }
+
+
   const key = cacheKey(meta.url, meta.options, meta.internalOptions);
 
   if (key !== null) {