updated logs
This commit is contained in:
@@ -4,6 +4,7 @@ import { RateLimiterMode } from "../../src/types";
|
|||||||
import { addWebScraperJob } from "../../src/services/queue-jobs";
|
import { addWebScraperJob } from "../../src/services/queue-jobs";
|
||||||
import { getWebScraperQueue } from "../../src/services/queue-service";
|
import { getWebScraperQueue } from "../../src/services/queue-service";
|
||||||
import { supabaseGetJobById } from "../../src/lib/supabase-jobs";
|
import { supabaseGetJobById } from "../../src/lib/supabase-jobs";
|
||||||
|
import { Logger } from "../../src/lib/logger";
|
||||||
|
|
||||||
export async function crawlStatusController(req: Request, res: Response) {
|
export async function crawlStatusController(req: Request, res: Response) {
|
||||||
try {
|
try {
|
||||||
@@ -44,7 +45,7 @@ export async function crawlStatusController(req: Request, res: Response) {
|
|||||||
partial_data: jobStatus == 'completed' ? [] : partialDocs,
|
partial_data: jobStatus == 'completed' ? [] : partialDocs,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
Logger.error(error);
|
||||||
return res.status(500).json({ error: error.message });
|
return res.status(500).json({ error: error.message });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import { logCrawl } from "../../src/services/logging/crawl_log";
|
|||||||
import { validateIdempotencyKey } from "../../src/services/idempotency/validate";
|
import { validateIdempotencyKey } from "../../src/services/idempotency/validate";
|
||||||
import { createIdempotencyKey } from "../../src/services/idempotency/create";
|
import { createIdempotencyKey } from "../../src/services/idempotency/create";
|
||||||
import { defaultCrawlPageOptions, defaultCrawlerOptions, defaultOrigin } from "../../src/lib/default-values";
|
import { defaultCrawlPageOptions, defaultCrawlerOptions, defaultOrigin } from "../../src/lib/default-values";
|
||||||
|
import { Logger } from "../../src/lib/logger";
|
||||||
|
|
||||||
export async function crawlController(req: Request, res: Response) {
|
export async function crawlController(req: Request, res: Response) {
|
||||||
try {
|
try {
|
||||||
@@ -30,7 +31,7 @@ export async function crawlController(req: Request, res: Response) {
|
|||||||
try {
|
try {
|
||||||
createIdempotencyKey(req);
|
createIdempotencyKey(req);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
Logger.error(error);
|
||||||
return res.status(500).json({ error: error.message });
|
return res.status(500).json({ error: error.message });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -83,7 +84,7 @@ export async function crawlController(req: Request, res: Response) {
|
|||||||
documents: docs,
|
documents: docs,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
Logger.error(error);
|
||||||
return res.status(500).json({ error: error.message });
|
return res.status(500).json({ error: error.message });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -101,7 +102,7 @@ export async function crawlController(req: Request, res: Response) {
|
|||||||
|
|
||||||
res.json({ jobId: job.id });
|
res.json({ jobId: job.id });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
Logger.error(error);
|
||||||
return res.status(500).json({ error: error.message });
|
return res.status(500).json({ error: error.message });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { authenticateUser } from "./auth";
|
|||||||
import { RateLimiterMode } from "../../src/types";
|
import { RateLimiterMode } from "../../src/types";
|
||||||
import { addWebScraperJob } from "../../src/services/queue-jobs";
|
import { addWebScraperJob } from "../../src/services/queue-jobs";
|
||||||
import { isUrlBlocked } from "../../src/scraper/WebScraper/utils/blocklist";
|
import { isUrlBlocked } from "../../src/scraper/WebScraper/utils/blocklist";
|
||||||
|
import { Logger } from "../../src/lib/logger";
|
||||||
|
|
||||||
export async function crawlPreviewController(req: Request, res: Response) {
|
export async function crawlPreviewController(req: Request, res: Response) {
|
||||||
try {
|
try {
|
||||||
@@ -39,7 +40,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
|
|||||||
|
|
||||||
res.json({ jobId: job.id });
|
res.json({ jobId: job.id });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
Logger.error(error);
|
||||||
return res.status(500).json({ error: error.message });
|
return res.status(500).json({ error: error.message });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import { Document } from "../lib/entities";
|
|||||||
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist"; // Import the isUrlBlocked function
|
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist"; // Import the isUrlBlocked function
|
||||||
import { numTokensFromString } from '../lib/LLM-extraction/helpers';
|
import { numTokensFromString } from '../lib/LLM-extraction/helpers';
|
||||||
import { defaultPageOptions, defaultExtractorOptions, defaultTimeout, defaultOrigin } from '../lib/default-values';
|
import { defaultPageOptions, defaultExtractorOptions, defaultTimeout, defaultOrigin } from '../lib/default-values';
|
||||||
|
import { Logger } from '../lib/logger';
|
||||||
|
|
||||||
export async function scrapeHelper(
|
export async function scrapeHelper(
|
||||||
req: Request,
|
req: Request,
|
||||||
@@ -112,7 +113,7 @@ export async function scrapeController(req: Request, res: Response) {
|
|||||||
return res.status(402).json({ error: "Insufficient credits" });
|
return res.status(402).json({ error: "Insufficient credits" });
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
Logger.error(error);
|
||||||
earlyReturn = true;
|
earlyReturn = true;
|
||||||
return res.status(402).json({ error: "Error checking team credits. Please contact hello@firecrawl.com for help." });
|
return res.status(402).json({ error: "Error checking team credits. Please contact hello@firecrawl.com for help." });
|
||||||
}
|
}
|
||||||
@@ -188,7 +189,7 @@ export async function scrapeController(req: Request, res: Response) {
|
|||||||
|
|
||||||
return res.status(result.returnCode).json(result);
|
return res.status(result.returnCode).json(result);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
Logger.error(error);
|
||||||
return res.status(500).json({ error: error.message });
|
return res.status(500).json({ error: error.message });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { logJob } from "../services/logging/log_job";
|
|||||||
import { PageOptions, SearchOptions } from "../lib/entities";
|
import { PageOptions, SearchOptions } from "../lib/entities";
|
||||||
import { search } from "../search";
|
import { search } from "../search";
|
||||||
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
|
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
|
||||||
|
import { Logger } from "../lib/logger";
|
||||||
|
|
||||||
export async function searchHelper(
|
export async function searchHelper(
|
||||||
req: Request,
|
req: Request,
|
||||||
@@ -155,7 +156,7 @@ export async function searchController(req: Request, res: Response) {
|
|||||||
return res.status(402).json({ error: "Insufficient credits" });
|
return res.status(402).json({ error: "Insufficient credits" });
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
Logger.error(error);
|
||||||
return res.status(500).json({ error: "Internal server error" });
|
return res.status(500).json({ error: "Internal server error" });
|
||||||
}
|
}
|
||||||
const startTime = new Date().getTime();
|
const startTime = new Date().getTime();
|
||||||
@@ -183,7 +184,7 @@ export async function searchController(req: Request, res: Response) {
|
|||||||
});
|
});
|
||||||
return res.status(result.returnCode).json(result);
|
return res.status(result.returnCode).json(result);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
Logger.error(error);
|
||||||
return res.status(500).json({ error: error.message });
|
return res.status(500).json({ error: error.message });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { Request, Response } from "express";
|
import { Request, Response } from "express";
|
||||||
import { getWebScraperQueue } from "../../src/services/queue-service";
|
import { getWebScraperQueue } from "../../src/services/queue-service";
|
||||||
import { supabaseGetJobById } from "../../src/lib/supabase-jobs";
|
import { supabaseGetJobById } from "../../src/lib/supabase-jobs";
|
||||||
|
import { Logger } from "../../src/lib/logger";
|
||||||
|
|
||||||
export async function crawlJobStatusPreviewController(req: Request, res: Response) {
|
export async function crawlJobStatusPreviewController(req: Request, res: Response) {
|
||||||
try {
|
try {
|
||||||
@@ -35,7 +36,7 @@ export async function crawlJobStatusPreviewController(req: Request, res: Respons
|
|||||||
partial_data: jobStatus == 'completed' ? [] : partialDocs,
|
partial_data: jobStatus == 'completed' ? [] : partialDocs,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(error);
|
Logger.error(error);
|
||||||
return res.status(500).json({ error: error.message });
|
return res.status(500).json({ error: error.message });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,9 +83,9 @@ if (cluster.isMaster) {
|
|||||||
function startServer(port = DEFAULT_PORT) {
|
function startServer(port = DEFAULT_PORT) {
|
||||||
const server = app.listen(Number(port), HOST, () => {
|
const server = app.listen(Number(port), HOST, () => {
|
||||||
Logger.info(`Worker ${process.pid} listening on port ${port}`);
|
Logger.info(`Worker ${process.pid} listening on port ${port}`);
|
||||||
// Logger.info(
|
Logger.info(
|
||||||
// `For the UI, open: http://${HOST}:${port}/admin/${process.env.BULL_AUTH_KEY}/queues`
|
`For the Queue UI, open: http://${HOST}:${port}/admin/${process.env.BULL_AUTH_KEY}/queues`
|
||||||
// );
|
);
|
||||||
});
|
});
|
||||||
return server;
|
return server;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ const ajv = new Ajv(); // Initialize AJV for JSON schema validation
|
|||||||
|
|
||||||
import { generateOpenAICompletions } from "./models";
|
import { generateOpenAICompletions } from "./models";
|
||||||
import { Document, ExtractorOptions } from "../entities";
|
import { Document, ExtractorOptions } from "../entities";
|
||||||
|
import { Logger } from "../logger";
|
||||||
|
|
||||||
// Generate completion using OpenAI
|
// Generate completion using OpenAI
|
||||||
export async function generateCompletions(
|
export async function generateCompletions(
|
||||||
@@ -44,7 +45,7 @@ export async function generateCompletions(
|
|||||||
|
|
||||||
return completionResult;
|
return completionResult;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error generating completions: ${error}`);
|
Logger.error(`Error generating completions: ${error}`);
|
||||||
throw new Error(`Error generating completions: ${error.message}`);
|
throw new Error(`Error generating completions: ${error.message}`);
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { AuthResponse } from "../../src/types";
|
import { AuthResponse } from "../../src/types";
|
||||||
|
import { Logger } from "./logger";
|
||||||
|
|
||||||
let warningCount = 0;
|
let warningCount = 0;
|
||||||
|
|
||||||
@@ -8,7 +9,7 @@ export function withAuth<T extends AuthResponse, U extends any[]>(
|
|||||||
return async function (...args: U): Promise<T> {
|
return async function (...args: U): Promise<T> {
|
||||||
if (process.env.USE_DB_AUTHENTICATION === "false") {
|
if (process.env.USE_DB_AUTHENTICATION === "false") {
|
||||||
if (warningCount < 5) {
|
if (warningCount < 5) {
|
||||||
console.warn("WARNING - You're bypassing authentication");
|
Logger.warn("You're bypassing authentication");
|
||||||
warningCount++;
|
warningCount++;
|
||||||
}
|
}
|
||||||
return { success: true } as T;
|
return { success: true } as T;
|
||||||
@@ -16,7 +17,7 @@ export function withAuth<T extends AuthResponse, U extends any[]>(
|
|||||||
try {
|
try {
|
||||||
return await originalFunction(...args);
|
return await originalFunction(...args);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error in withAuth function: ", error);
|
Logger.error(`Error in withAuth function: ${error}`);
|
||||||
return { success: false, error: error.message } as T;
|
return { success: false, error: error.message } as T;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { Logger } from "../../../lib/logger";
|
||||||
|
|
||||||
const socialMediaBlocklist = [
|
const socialMediaBlocklist = [
|
||||||
'facebook.com',
|
'facebook.com',
|
||||||
'x.com',
|
'x.com',
|
||||||
@@ -59,7 +61,7 @@ export function isUrlBlocked(url: string): boolean {
|
|||||||
return isBlocked;
|
return isBlocked;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
// If an error occurs (e.g., invalid URL), return false
|
// If an error occurs (e.g., invalid URL), return false
|
||||||
console.error(`Error parsing the following URL: ${url}`);
|
Logger.error(`Error parsing the following URL: ${url}`);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import Anthropic from '@anthropic-ai/sdk';
|
import Anthropic from '@anthropic-ai/sdk';
|
||||||
import axios from 'axios';
|
import axios from 'axios';
|
||||||
|
import { Logger } from '../../../lib/logger';
|
||||||
|
|
||||||
export async function getImageDescription(
|
export async function getImageDescription(
|
||||||
imageUrl: string,
|
imageUrl: string,
|
||||||
@@ -82,7 +83,7 @@ export async function getImageDescription(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error generating image alt text:", error?.message);
|
Logger.error(`Error generating image alt text: ${error}`);
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
import { CheerioAPI } from "cheerio";
|
import { CheerioAPI } from "cheerio";
|
||||||
|
import { Logger } from "../../../lib/logger";
|
||||||
|
|
||||||
interface Metadata {
|
interface Metadata {
|
||||||
title?: string;
|
title?: string;
|
||||||
description?: string;
|
description?: string;
|
||||||
@@ -105,7 +107,7 @@ export function extractMetadata(soup: CheerioAPI, url: string): Metadata {
|
|||||||
dctermsCreated = soup('meta[name="dcterms.created"]').attr("content") || null;
|
dctermsCreated = soup('meta[name="dcterms.created"]').attr("content") || null;
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error extracting metadata:", error);
|
Logger.error(`Error extracting metadata: ${error}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { Logger } from "../../../lib/logger";
|
||||||
import { Document } from "../../../lib/entities";
|
import { Document } from "../../../lib/entities";
|
||||||
|
|
||||||
export const replacePathsWithAbsolutePaths = (documents: Document[]): Document[] => {
|
export const replacePathsWithAbsolutePaths = (documents: Document[]): Document[] => {
|
||||||
@@ -39,7 +40,7 @@ export const replacePathsWithAbsolutePaths = (documents: Document[]): Document[]
|
|||||||
|
|
||||||
return documents;
|
return documents;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error replacing paths with absolute paths", error);
|
Logger.debug(`Error replacing paths with absolute paths: ${error}`);
|
||||||
return documents;
|
return documents;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -78,7 +79,7 @@ export const replaceImgPathsWithAbsolutePaths = (documents: Document[]): Documen
|
|||||||
|
|
||||||
return documents;
|
return documents;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error replacing img paths with absolute paths", error);
|
Logger.error(`Error replacing img paths with absolute paths: ${error}`);
|
||||||
return documents;
|
return documents;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import axios from "axios";
|
import axios from "axios";
|
||||||
import * as cheerio from "cheerio";
|
import * as cheerio from "cheerio";
|
||||||
|
import { Logger } from "../../../lib/logger";
|
||||||
|
|
||||||
|
|
||||||
export async function attemptScrapWithRequests(
|
export async function attemptScrapWithRequests(
|
||||||
@@ -9,13 +10,13 @@ export async function attemptScrapWithRequests(
|
|||||||
const response = await axios.get(urlToScrap, { timeout: 15000 });
|
const response = await axios.get(urlToScrap, { timeout: 15000 });
|
||||||
|
|
||||||
if (!response.data) {
|
if (!response.data) {
|
||||||
console.log("Failed normal requests as well");
|
Logger.debug("Failed normal requests as well");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
return response.data;
|
return response.data;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error in attemptScrapWithRequests: ${error}`);
|
Logger.debug(`Error in attemptScrapWithRequests: ${error}`);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import axios from 'axios';
|
|||||||
import * as cheerio from 'cheerio';
|
import * as cheerio from 'cheerio';
|
||||||
import * as querystring from 'querystring';
|
import * as querystring from 'querystring';
|
||||||
import { SearchResult } from '../../src/lib/entities';
|
import { SearchResult } from '../../src/lib/entities';
|
||||||
|
import { Logger } from '../../src/lib/logger';
|
||||||
|
|
||||||
const _useragent_list = [
|
const _useragent_list = [
|
||||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0',
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0',
|
||||||
@@ -96,7 +97,7 @@ export async function google_search(term: string, advanced = false, num_results
|
|||||||
await new Promise(resolve => setTimeout(resolve, sleep_interval * 1000));
|
await new Promise(resolve => setTimeout(resolve, sleep_interval * 1000));
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error.message === 'Too many requests') {
|
if (error.message === 'Too many requests') {
|
||||||
console.warn('Too many requests, breaking the loop');
|
Logger.warn('Too many requests, breaking the loop');
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
throw error;
|
throw error;
|
||||||
@@ -107,7 +108,7 @@ export async function google_search(term: string, advanced = false, num_results
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (attempts >= maxAttempts) {
|
if (attempts >= maxAttempts) {
|
||||||
console.warn('Max attempts reached, breaking the loop');
|
Logger.warn('Max attempts reached, breaking the loop');
|
||||||
}
|
}
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { Logger } from "../../src/lib/logger";
|
||||||
import { SearchResult } from "../../src/lib/entities";
|
import { SearchResult } from "../../src/lib/entities";
|
||||||
import { google_search } from "./googlesearch";
|
import { google_search } from "./googlesearch";
|
||||||
import { serper_search } from "./serper";
|
import { serper_search } from "./serper";
|
||||||
@@ -47,7 +48,7 @@ export async function search({
|
|||||||
timeout
|
timeout
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error in search function: ", error);
|
Logger.error(`Error in search function: ${error}`);
|
||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
// if process.env.SERPER_API_KEY is set, use serper
|
// if process.env.SERPER_API_KEY is set, use serper
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { Logger } from "../../../src/lib/logger";
|
||||||
import { getWebScraperQueue } from "../queue-service";
|
import { getWebScraperQueue } from "../queue-service";
|
||||||
import { sendSlackWebhook } from "./slack";
|
import { sendSlackWebhook } from "./slack";
|
||||||
|
|
||||||
@@ -9,13 +10,13 @@ export async function checkAlerts() {
|
|||||||
process.env.ALERT_NUM_ACTIVE_JOBS &&
|
process.env.ALERT_NUM_ACTIVE_JOBS &&
|
||||||
process.env.ALERT_NUM_WAITING_JOBS
|
process.env.ALERT_NUM_WAITING_JOBS
|
||||||
) {
|
) {
|
||||||
console.info("Initializing alerts");
|
Logger.info("Initializing alerts");
|
||||||
const checkActiveJobs = async () => {
|
const checkActiveJobs = async () => {
|
||||||
try {
|
try {
|
||||||
const webScraperQueue = getWebScraperQueue();
|
const webScraperQueue = getWebScraperQueue();
|
||||||
const activeJobs = await webScraperQueue.getActiveCount();
|
const activeJobs = await webScraperQueue.getActiveCount();
|
||||||
if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {
|
if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {
|
||||||
console.warn(
|
Logger.warn(
|
||||||
`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`
|
`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`
|
||||||
);
|
);
|
||||||
sendSlackWebhook(
|
sendSlackWebhook(
|
||||||
@@ -23,12 +24,12 @@ export async function checkAlerts() {
|
|||||||
true
|
true
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
console.info(
|
Logger.info(
|
||||||
`Number of active jobs is under ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}`
|
`Number of active jobs is under ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Failed to check active jobs:", error);
|
Logger.error(`Failed to check active jobs: ${error}`);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -38,7 +39,7 @@ export async function checkAlerts() {
|
|||||||
const paused = await webScraperQueue.getPausedCount();
|
const paused = await webScraperQueue.getPausedCount();
|
||||||
|
|
||||||
if (waitingJobs !== paused && waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {
|
if (waitingJobs !== paused && waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {
|
||||||
console.warn(
|
Logger.warn(
|
||||||
`Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.`
|
`Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.`
|
||||||
);
|
);
|
||||||
sendSlackWebhook(
|
sendSlackWebhook(
|
||||||
@@ -57,6 +58,6 @@ export async function checkAlerts() {
|
|||||||
// setInterval(checkAll, 10000); // Run every
|
// setInterval(checkAll, 10000); // Run every
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Failed to initialize alerts:", error);
|
Logger.error(`Failed to initialize alerts: ${error}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import axios from "axios";
|
import axios from "axios";
|
||||||
|
import { Logger } from "../../../src/lib/logger";
|
||||||
|
|
||||||
export async function sendSlackWebhook(
|
export async function sendSlackWebhook(
|
||||||
message: string,
|
message: string,
|
||||||
@@ -16,8 +17,8 @@ export async function sendSlackWebhook(
|
|||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
console.log("Webhook sent successfully:", response.data);
|
Logger.log("Webhook sent successfully:", response.data);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error sending webhook:", error);
|
Logger.error(`Error sending webhook: ${error}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -263,14 +263,14 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (creditUsageError) {
|
if (creditUsageError) {
|
||||||
console.error("Error calculating credit usage:", creditUsageError);
|
Logger.error(`Error calculating credit usage: ${creditUsageError}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (creditUsages && creditUsages.length > 0) {
|
if (creditUsages && creditUsages.length > 0) {
|
||||||
totalCreditsUsed = creditUsages[0].total_credits_used;
|
totalCreditsUsed = creditUsages[0].total_credits_used;
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error calculating credit usage:", error);
|
Logger.error(`Error calculating credit usage: ${error}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Adjust total credits used by subtracting coupon value
|
// Adjust total credits used by subtracting coupon value
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { Request } from "express";
|
import { Request } from "express";
|
||||||
import { supabase_service } from "../supabase";
|
import { supabase_service } from "../supabase";
|
||||||
|
import { Logger } from "../../../src/lib/logger";
|
||||||
|
|
||||||
export async function createIdempotencyKey(
|
export async function createIdempotencyKey(
|
||||||
req: Request,
|
req: Request,
|
||||||
@@ -14,7 +15,7 @@ export async function createIdempotencyKey(
|
|||||||
.insert({ key: idempotencyKey });
|
.insert({ key: idempotencyKey });
|
||||||
|
|
||||||
if (error) {
|
if (error) {
|
||||||
console.error("Failed to create idempotency key:", error);
|
Logger.error(`Failed to create idempotency key: ${error}`);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { Request } from "express";
|
import { Request } from "express";
|
||||||
import { supabase_service } from "../supabase";
|
import { supabase_service } from "../supabase";
|
||||||
import { validate as isUuid } from 'uuid';
|
import { validate as isUuid } from 'uuid';
|
||||||
|
import { Logger } from "../../../src/lib/logger";
|
||||||
|
|
||||||
export async function validateIdempotencyKey(
|
export async function validateIdempotencyKey(
|
||||||
req: Request,
|
req: Request,
|
||||||
@@ -13,7 +14,7 @@ export async function validateIdempotencyKey(
|
|||||||
// Ensure idempotencyKey is treated as a string
|
// Ensure idempotencyKey is treated as a string
|
||||||
const key = Array.isArray(idempotencyKey) ? idempotencyKey[0] : idempotencyKey;
|
const key = Array.isArray(idempotencyKey) ? idempotencyKey[0] : idempotencyKey;
|
||||||
if (!isUuid(key)) {
|
if (!isUuid(key)) {
|
||||||
console.error("Invalid idempotency key provided in the request headers.");
|
Logger.debug("Invalid idempotency key provided in the request headers.");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -23,7 +24,7 @@ export async function validateIdempotencyKey(
|
|||||||
.eq("key", idempotencyKey);
|
.eq("key", idempotencyKey);
|
||||||
|
|
||||||
if (error) {
|
if (error) {
|
||||||
console.error(error);
|
Logger.error(`Error validating idempotency key: ${error}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!data || data.length === 0) {
|
if (!data || data.length === 0) {
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { supabase_service } from "../supabase";
|
import { supabase_service } from "../supabase";
|
||||||
|
import { Logger } from "../../../src/lib/logger";
|
||||||
import "dotenv/config";
|
import "dotenv/config";
|
||||||
|
|
||||||
export async function logCrawl(job_id: string, team_id: string) {
|
export async function logCrawl(job_id: string, team_id: string) {
|
||||||
@@ -13,7 +14,7 @@ export async function logCrawl(job_id: string, team_id: string) {
|
|||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error logging crawl job:\n", error);
|
Logger.error(`Error logging crawl job to supabase:\n${error}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,10 @@ export async function logScrape(
|
|||||||
scrapeLog: ScrapeLog,
|
scrapeLog: ScrapeLog,
|
||||||
pageOptions?: PageOptions
|
pageOptions?: PageOptions
|
||||||
) {
|
) {
|
||||||
|
if (process.env.USE_DB_AUTHENTICATION === "false") {
|
||||||
|
Logger.debug("Skipping logging scrape to Supabase");
|
||||||
|
return;
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
// Only log jobs in production
|
// Only log jobs in production
|
||||||
// if (process.env.ENV !== "production") {
|
// if (process.env.ENV !== "production") {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { supabase_service } from "../supabase";
|
|||||||
import { withAuth } from "../../lib/withAuth";
|
import { withAuth } from "../../lib/withAuth";
|
||||||
import { Resend } from "resend";
|
import { Resend } from "resend";
|
||||||
import { NotificationType } from "../../types";
|
import { NotificationType } from "../../types";
|
||||||
|
import { Logger } from "../../../src/lib/logger";
|
||||||
|
|
||||||
const emailTemplates: Record<
|
const emailTemplates: Record<
|
||||||
NotificationType,
|
NotificationType,
|
||||||
@@ -52,11 +53,11 @@ async function sendEmailNotification(
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (error) {
|
if (error) {
|
||||||
console.error("Error sending email: ", error);
|
Logger.debug(`Error sending email: ${error}`);
|
||||||
return { success: false };
|
return { success: false };
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error sending email (2): ", error);
|
Logger.debug(`Error sending email (2): ${error}`);
|
||||||
return { success: false };
|
return { success: false };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -79,7 +80,7 @@ export async function sendNotificationInternal(
|
|||||||
.lte("sent_date", endDateString);
|
.lte("sent_date", endDateString);
|
||||||
|
|
||||||
if (error) {
|
if (error) {
|
||||||
console.error("Error fetching notifications: ", error);
|
Logger.debug(`Error fetching notifications: ${error}`);
|
||||||
return { success: false };
|
return { success: false };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,7 +94,7 @@ export async function sendNotificationInternal(
|
|||||||
.eq("team_id", team_id);
|
.eq("team_id", team_id);
|
||||||
|
|
||||||
if (emailsError) {
|
if (emailsError) {
|
||||||
console.error("Error fetching emails: ", emailsError);
|
Logger.debug(`Error fetching emails: ${emailsError}`);
|
||||||
return { success: false };
|
return { success: false };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,7 +113,7 @@ export async function sendNotificationInternal(
|
|||||||
]);
|
]);
|
||||||
|
|
||||||
if (insertError) {
|
if (insertError) {
|
||||||
console.error("Error inserting notification record: ", insertError);
|
Logger.debug(`Error inserting notification record: ${insertError}`);
|
||||||
return { success: false };
|
return { success: false };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { PostHog } from 'posthog-node';
|
import { PostHog } from 'posthog-node';
|
||||||
import "dotenv/config";
|
import "dotenv/config";
|
||||||
|
import { Logger } from '../../src/lib/logger';
|
||||||
|
|
||||||
export default function PostHogClient() {
|
export default function PostHogClient() {
|
||||||
const posthogClient = new PostHog(process.env.POSTHOG_API_KEY, {
|
const posthogClient = new PostHog(process.env.POSTHOG_API_KEY, {
|
||||||
@@ -19,7 +20,7 @@ class MockPostHog {
|
|||||||
export const posthog = process.env.POSTHOG_API_KEY
|
export const posthog = process.env.POSTHOG_API_KEY
|
||||||
? PostHogClient()
|
? PostHogClient()
|
||||||
: (() => {
|
: (() => {
|
||||||
console.warn(
|
Logger.warn(
|
||||||
"POSTHOG_API_KEY is not provided - your events will not be logged. Using MockPostHog as a fallback. See posthog.ts for more."
|
"POSTHOG_API_KEY is not provided - your events will not be logged. Using MockPostHog as a fallback. See posthog.ts for more."
|
||||||
);
|
);
|
||||||
return new MockPostHog();
|
return new MockPostHog();
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { createClient, SupabaseClient } from "@supabase/supabase-js";
|
import { createClient, SupabaseClient } from "@supabase/supabase-js";
|
||||||
|
import { Logger } from "../lib/logger";
|
||||||
|
|
||||||
// SupabaseService class initializes the Supabase client conditionally based on environment variables.
|
// SupabaseService class initializes the Supabase client conditionally based on environment variables.
|
||||||
class SupabaseService {
|
class SupabaseService {
|
||||||
@@ -10,13 +11,13 @@ class SupabaseService {
|
|||||||
// Only initialize the Supabase client if both URL and Service Token are provided.
|
// Only initialize the Supabase client if both URL and Service Token are provided.
|
||||||
if (process.env.USE_DB_AUTHENTICATION === "false") {
|
if (process.env.USE_DB_AUTHENTICATION === "false") {
|
||||||
// Warn the user that Authentication is disabled by setting the client to null
|
// Warn the user that Authentication is disabled by setting the client to null
|
||||||
console.warn(
|
Logger.warn(
|
||||||
"\x1b[33mAuthentication is disabled. Supabase client will not be initialized.\x1b[0m"
|
"Authentication is disabled. Supabase client will not be initialized."
|
||||||
);
|
);
|
||||||
this.client = null;
|
this.client = null;
|
||||||
} else if (!supabaseUrl || !supabaseServiceToken) {
|
} else if (!supabaseUrl || !supabaseServiceToken) {
|
||||||
console.error(
|
Logger.error(
|
||||||
"\x1b[31mSupabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable\x1b[0m"
|
"Supabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable"
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
this.client = createClient(supabaseUrl, supabaseServiceToken);
|
this.client = createClient(supabaseUrl, supabaseServiceToken);
|
||||||
@@ -35,10 +36,15 @@ export const supabase_service: SupabaseClient = new Proxy(
|
|||||||
new SupabaseService(),
|
new SupabaseService(),
|
||||||
{
|
{
|
||||||
get: function (target, prop, receiver) {
|
get: function (target, prop, receiver) {
|
||||||
|
if (process.env.USE_DB_AUTHENTICATION === "false") {
|
||||||
|
Logger.debug(
|
||||||
|
"Attempted to access Supabase client when it's not configured."
|
||||||
|
);
|
||||||
|
}
|
||||||
const client = target.getClient();
|
const client = target.getClient();
|
||||||
// If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
|
// If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
|
||||||
if (client === null) {
|
if (client === null) {
|
||||||
console.error(
|
Logger.error(
|
||||||
"Attempted to access Supabase client when it's not configured."
|
"Attempted to access Supabase client when it's not configured."
|
||||||
);
|
);
|
||||||
return () => {
|
return () => {
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { Logger } from "../../src/lib/logger";
|
||||||
import { supabase_service } from "./supabase";
|
import { supabase_service } from "./supabase";
|
||||||
|
|
||||||
export const callWebhook = async (teamId: string, jobId: string,data: any) => {
|
export const callWebhook = async (teamId: string, jobId: string,data: any) => {
|
||||||
@@ -15,10 +16,7 @@ export const callWebhook = async (teamId: string, jobId: string,data: any) => {
|
|||||||
.eq("team_id", teamId)
|
.eq("team_id", teamId)
|
||||||
.limit(1);
|
.limit(1);
|
||||||
if (error) {
|
if (error) {
|
||||||
console.error(
|
Logger.error(`Error fetching webhook URL for team ID: ${teamId}, error: ${error.message}`);
|
||||||
`Error fetching webhook URL for team ID: ${teamId}`,
|
|
||||||
error.message
|
|
||||||
);
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,9 +51,6 @@ export const callWebhook = async (teamId: string, jobId: string,data: any) => {
|
|||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(
|
Logger.error(`Error sending webhook for team ID: ${teamId}, error: ${error.message}`);
|
||||||
`Error sending webhook for team ID: ${teamId}`,
|
|
||||||
error.message
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
import { createClient, SupabaseClient } from "@supabase/supabase-js";
|
import { createClient, SupabaseClient } from "@supabase/supabase-js";
|
||||||
import "dotenv/config";
|
import "dotenv/config";
|
||||||
|
import { Logger } from "../../api/src/lib/logger";
|
||||||
|
|
||||||
// SupabaseService class initializes the Supabase client conditionally based on environment variables.
|
// SupabaseService class initializes the Supabase client conditionally based on environment variables.
|
||||||
class SupabaseService {
|
class SupabaseService {
|
||||||
private client: SupabaseClient | null = null;
|
private client: SupabaseClient | null = null;
|
||||||
@@ -10,13 +12,13 @@ class SupabaseService {
|
|||||||
// Only initialize the Supabase client if both URL and Service Token are provided.
|
// Only initialize the Supabase client if both URL and Service Token are provided.
|
||||||
if (process.env.USE_DB_AUTHENTICATION === "false") {
|
if (process.env.USE_DB_AUTHENTICATION === "false") {
|
||||||
// Warn the user that Authentication is disabled by setting the client to null
|
// Warn the user that Authentication is disabled by setting the client to null
|
||||||
console.warn(
|
Logger.warn(
|
||||||
"\x1b[33mAuthentication is disabled. Supabase client will not be initialized.\x1b[0m"
|
"Authentication is disabled. Supabase client will not be initialized."
|
||||||
);
|
);
|
||||||
this.client = null;
|
this.client = null;
|
||||||
} else if (!supabaseUrl || !supabaseServiceToken) {
|
} else if (!supabaseUrl || !supabaseServiceToken) {
|
||||||
console.error(
|
Logger.error(
|
||||||
"\x1b[31mSupabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable\x1b[0m"
|
"Supabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable"
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
this.client = createClient(supabaseUrl, supabaseServiceToken);
|
this.client = createClient(supabaseUrl, supabaseServiceToken);
|
||||||
@@ -35,10 +37,15 @@ export const supabase_service: SupabaseClient = new Proxy(
|
|||||||
new SupabaseService(),
|
new SupabaseService(),
|
||||||
{
|
{
|
||||||
get: function (target, prop, receiver) {
|
get: function (target, prop, receiver) {
|
||||||
|
if (process.env.USE_DB_AUTHENTICATION === "false") {
|
||||||
|
Logger.debug(
|
||||||
|
"Attempted to access Supabase client when it's not configured."
|
||||||
|
);
|
||||||
|
}
|
||||||
const client = target.getClient();
|
const client = target.getClient();
|
||||||
// If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
|
// If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
|
||||||
if (client === null) {
|
if (client === null) {
|
||||||
console.error(
|
Logger.error(
|
||||||
"Attempted to access Supabase client when it's not configured."
|
"Attempted to access Supabase client when it's not configured."
|
||||||
);
|
);
|
||||||
return () => {
|
return () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user