first commit

This commit is contained in:
2025-11-06 11:08:59 +01:00
commit 3c5117c2c3
85 changed files with 13275 additions and 0 deletions
+208
View File
@@ -0,0 +1,208 @@
const path = require('path');
const fs = require('fs');
const logger = require('./logger');
class ConfigLoader {
constructor() {
this.config = {};
this.env = process.env.NODE_ENV || 'development';
this.loadConfig();
}
loadConfig() {
try {
// Load base configuration
this.loadBaseConfig();
// Load environment-specific configuration
this.loadEnvConfig();
// Validate configuration
this.validateConfig();
logger.info(`Configuration loaded for environment: ${this.env}`);
} catch (error) {
logger.error('Configuration loading failed:', error);
throw error;
}
}
loadBaseConfig() {
// Load from appConfig.js
const { appConfig } = require('../config/appConfig');
this.config = { ...appConfig };
}
loadEnvConfig() {
const envFile = path.join(__dirname, '../../', `env.${this.env}`);
if (fs.existsSync(envFile)) {
logger.info(`Loading environment configuration from: ${envFile}`);
// Parse environment file
const envContent = fs.readFileSync(envFile, 'utf8');
const envVars = this.parseEnvFile(envContent);
// Override configuration with environment-specific values
Object.assign(process.env, envVars);
// Reload configuration with new environment variables
const { appConfig } = require('../config/appConfig');
this.config = { ...appConfig };
} else {
logger.warn(`Environment configuration file not found: ${envFile}`);
}
}
parseEnvFile(content) {
const envVars = {};
const lines = content.split('\n');
for (const line of lines) {
const trimmedLine = line.trim();
// Skip empty lines and comments
if (!trimmedLine || trimmedLine.startsWith('#')) {
continue;
}
// Parse KEY=VALUE format
const equalIndex = trimmedLine.indexOf('=');
if (equalIndex > 0) {
const key = trimmedLine.substring(0, equalIndex).trim();
const value = trimmedLine.substring(equalIndex + 1).trim();
// Remove quotes if present
const cleanValue = value.replace(/^["']|["']$/g, '');
envVars[key] = cleanValue;
}
}
return envVars;
}
validateConfig() {
const errors = [];
const warnings = [];
// Required fields validation
if (!this.config.apis.groq.apiKey) {
errors.push('GROQ_API_KEY is required');
}
if (!this.config.auth.jwtSecret) {
errors.push('JWT_SECRET is required');
}
if (!this.config.database.password) {
errors.push('DB_PASSWORD is required');
}
// Environment-specific validations
if (this.env === 'production') {
if (this.config.auth.jwtSecret === 'dev_secret_key_change_in_production') {
errors.push('JWT_SECRET must be changed for production');
}
if (!this.config.database.ssl) {
warnings.push('Database SSL is recommended for production');
}
}
// Log warnings
warnings.forEach(warning => {
logger.warn(`Configuration warning: ${warning}`);
});
// Throw errors
if (errors.length > 0) {
const errorMessage = `Configuration errors: ${errors.join(', ')}`;
logger.error(errorMessage);
throw new Error(errorMessage);
}
}
get(key, defaultValue = null) {
const keys = key.split('.');
let value = this.config;
for (const k of keys) {
if (value && typeof value === 'object' && k in value) {
value = value[k];
} else {
return defaultValue;
}
}
return value;
}
getAll() {
return this.config;
}
getServerConfig() {
return this.config.server;
}
getDatabaseConfig() {
return this.config.database;
}
getApiConfig() {
return this.config.apis;
}
getAuthConfig() {
return this.config.auth;
}
getUploadConfig() {
return this.config.upload;
}
getRateLimitConfig() {
return this.config.rateLimit;
}
getLoggingConfig() {
return this.config.logging;
}
getModelConfig() {
return this.config.models;
}
getSecurityConfig() {
return this.config.security;
}
getMonitoringConfig() {
return this.config.monitoring;
}
getDevelopmentConfig() {
return this.config.development;
}
isDevelopment() {
return this.env === 'development';
}
isProduction() {
return this.env === 'production';
}
isTest() {
return this.env === 'test';
}
getEnvironment() {
return this.env;
}
}
// Create singleton instance
const configLoader = new ConfigLoader();
module.exports = configLoader;
+110
View File
@@ -0,0 +1,110 @@
const { testConnection, syncDatabase, dropDatabase } = require('../config/database');
const logger = require('./logger');
const initializeDatabase = async (force = false) => {
try {
console.log('🚀 Initializing database...');
// Test connection
console.log('📡 Testing database connection...');
const connected = await testConnection();
if (!connected) {
throw new Error('Database connection failed');
}
console.log('✅ Database connection successful');
// Sync database (create tables)
console.log('🔧 Synchronizing database schema...');
const synced = await syncDatabase(force);
if (!synced) {
throw new Error('Database synchronization failed');
}
console.log('✅ Database schema synchronized');
console.log('🎉 Database initialization completed successfully!');
return true;
} catch (error) {
console.error('❌ Database initialization failed:', error.message);
logger.error('Database initialization error:', error);
return false;
}
};
const resetDatabase = async () => {
try {
console.log('🔄 Resetting database...');
// Drop all tables
console.log('🗑️ Dropping all tables...');
const dropped = await dropDatabase();
if (!dropped) {
throw new Error('Database drop failed');
}
console.log('✅ All tables dropped');
// Reinitialize
console.log('🔧 Reinitializing database...');
const initialized = await initializeDatabase(true);
if (!initialized) {
throw new Error('Database reinitialization failed');
}
console.log('✅ Database reset completed');
return true;
} catch (error) {
console.error('❌ Database reset failed:', error.message);
logger.error('Database reset error:', error);
return false;
}
};
const checkDatabaseStatus = async () => {
try {
console.log('🔍 Checking database status...');
const connected = await testConnection();
if (!connected) {
console.log('❌ Database connection failed');
return false;
}
console.log('✅ Database is connected and ready');
return true;
} catch (error) {
console.error('❌ Database status check failed:', error.message);
return false;
}
};
// Run initialization if this file is executed directly
if (require.main === module) {
const args = process.argv.slice(2);
const command = args[0] || 'init';
switch (command) {
case 'init':
initializeDatabase().then(success => {
process.exit(success ? 0 : 1);
});
break;
case 'reset':
resetDatabase().then(success => {
process.exit(success ? 0 : 1);
});
break;
case 'status':
checkDatabaseStatus().then(success => {
process.exit(success ? 0 : 1);
});
break;
default:
console.log('Usage: node databaseInit.js [init|reset|status]');
process.exit(1);
}
}
module.exports = {
initializeDatabase,
resetDatabase,
checkDatabaseStatus
};
+212
View File
@@ -0,0 +1,212 @@
const { initializeDatabase, resetDatabase, checkDatabaseStatus } = require('./databaseInit');
const { sequelize } = require('../config/database');
const logger = require('./logger');
// Import seeders
const createAdminUser = require('../seeders/001_create_admin_user');
const createSampleTrainingData = require('../seeders/002_create_sample_training_data');
const runMigrations = async () => {
try {
console.log('🔄 Running database migrations...');
// Import all migration files
const migrations = [
require('../migrations/001_create_users'),
require('../migrations/002_create_conversations'),
require('../migrations/003_create_plans'),
require('../migrations/004_create_messages'),
require('../migrations/005_create_documents'),
require('../migrations/006_create_feedback'),
require('../migrations/007_create_model_versions'),
require('../migrations/008_create_training_data'),
require('../migrations/009_create_tool_executions')
];
for (const migration of migrations) {
console.log(` Running migration: ${migration.up.name || 'unnamed'}`);
await migration.up(sequelize.getQueryInterface(), sequelize.constructor);
}
console.log('✅ All migrations completed successfully');
return true;
} catch (error) {
console.error('❌ Migration failed:', error);
logger.error('Migration error:', error);
return false;
}
};
const runSeeders = async () => {
try {
console.log('🌱 Running database seeders...');
// Run seeders
await createAdminUser.up(sequelize.getQueryInterface(), sequelize.constructor);
await createSampleTrainingData.up(sequelize.getQueryInterface(), sequelize.constructor);
console.log('✅ All seeders completed successfully');
return true;
} catch (error) {
console.error('❌ Seeding failed:', error);
logger.error('Seeding error:', error);
return false;
}
};
const setupDatabase = async (options = {}) => {
try {
console.log('🚀 Setting up database...');
const {
force = false,
seed = true,
migrations = true
} = options;
// Check database status
const status = await checkDatabaseStatus();
if (!status) {
throw new Error('Database connection failed');
}
// Run migrations if requested
if (migrations) {
const migrated = await runMigrations();
if (!migrated) {
throw new Error('Migrations failed');
}
}
// Run seeders if requested
if (seed) {
const seeded = await runSeeders();
if (!seeded) {
throw new Error('Seeding failed');
}
}
console.log('🎉 Database setup completed successfully!');
return true;
} catch (error) {
console.error('❌ Database setup failed:', error.message);
logger.error('Database setup error:', error);
return false;
}
};
const getDatabaseInfo = async () => {
try {
console.log('📊 Database Information:');
// Get table information
const [results] = await sequelize.query(`
SELECT
schemaname,
tablename,
tableowner
FROM pg_tables
WHERE schemaname = 'public'
ORDER BY tablename;
`);
console.log(` Tables: ${results.length}`);
results.forEach(table => {
console.log(` - ${table.tablename}`);
});
// Get database size
const [sizeResult] = await sequelize.query(`
SELECT pg_size_pretty(pg_database_size(current_database())) as size;
`);
console.log(` Database size: ${sizeResult[0].size}`);
return {
tables: results,
size: sizeResult[0].size
};
} catch (error) {
console.error('❌ Error getting database info:', error);
return null;
}
};
const backupDatabase = async (backupPath) => {
try {
console.log('💾 Creating database backup...');
// This would typically use pg_dump in a real implementation
console.log(` Backup path: ${backupPath}`);
console.log(' Note: Implement actual backup logic with pg_dump');
return true;
} catch (error) {
console.error('❌ Backup failed:', error);
return false;
}
};
// Command line interface
if (require.main === module) {
const args = process.argv.slice(2);
const command = args[0] || 'help';
const runCommand = async () => {
switch (command) {
case 'init':
await setupDatabase({ force: false, seed: true, migrations: true });
break;
case 'reset':
await resetDatabase();
break;
case 'migrate':
await runMigrations();
break;
case 'seed':
await runSeeders();
break;
case 'status':
await checkDatabaseStatus();
break;
case 'info':
await getDatabaseInfo();
break;
case 'backup':
const backupPath = args[1] || './backup.sql';
await backupDatabase(backupPath);
break;
case 'help':
default:
console.log(`
Database Manager Commands:
init - Initialize database with migrations and seeders
reset - Reset database (drop and recreate)
migrate - Run migrations only
seed - Run seeders only
status - Check database connection status
info - Show database information
backup - Create database backup
help - Show this help message
Usage: node databaseManager.js [command]
`);
break;
}
};
runCommand().then(() => {
process.exit(0);
}).catch(error => {
console.error('Command failed:', error);
process.exit(1);
});
}
module.exports = {
runMigrations,
runSeeders,
setupDatabase,
getDatabaseInfo,
backupDatabase
};
+40
View File
@@ -0,0 +1,40 @@
const winston = require('winston');
const path = require('path');
// Create logs directory if it doesn't exist
const fs = require('fs');
const logsDir = path.join(__dirname, '../../logs');
if (!fs.existsSync(logsDir)) {
fs.mkdirSync(logsDir, { recursive: true });
}
const logger = winston.createLogger({
level: process.env.LOG_LEVEL || 'info',
format: winston.format.combine(
winston.format.timestamp(),
winston.format.errors({ stack: true }),
winston.format.json()
),
defaultMeta: { service: 'reason-flow' },
transports: [
new winston.transports.File({
filename: path.join(logsDir, 'error.log'),
level: 'error'
}),
new winston.transports.File({
filename: path.join(logsDir, 'combined.log')
})
]
});
// Add console transport for development
if (process.env.NODE_ENV !== 'production') {
logger.add(new winston.transports.Console({
format: winston.format.combine(
winston.format.colorize(),
winston.format.simple()
)
}));
}
module.exports = logger;
+134
View File
@@ -0,0 +1,134 @@
const groqService = require('../services/groqService');
const model1Service = require('../services/model1Service');
const queryModelService = require('../services/queryModelService');
const logger = require('./logger');
const testGroqConnection = async () => {
console.log('🧪 Testing Groq API Connection...');
try {
const connectionTest = await groqService.testConnection();
if (connectionTest.success) {
console.log('✅ Groq API connection successful');
console.log(`Model: ${connectionTest.model}`);
console.log(`Response: ${connectionTest.response.content.substring(0, 100)}...`);
} else {
console.log('❌ Groq API connection failed');
console.log(`Error: ${connectionTest.error}`);
}
return connectionTest;
} catch (error) {
console.log('❌ Groq API test failed:', error.message);
return { success: false, error: error.message };
}
};
const testModel1 = async () => {
console.log('\n🧪 Testing MODEL1 (Engineering Plan Generation)...');
try {
const testQuery = "How do I design a bridge that can handle heavy traffic loads?";
const planData = await model1Service.generatePlan(testQuery, {
context: { test: true }
});
console.log('✅ MODEL1 plan generation successful');
console.log(`Title: ${planData.title}`);
console.log(`Description: ${planData.description.substring(0, 100)}...`);
console.log(`Steps: ${planData.steps.length}`);
console.log(`Tools Required: ${planData.toolsRequired.length}`);
console.log(`Processing Time: ${planData.processingTime}s`);
console.log(`Tokens Used: ${planData.tokensUsed}`);
return planData;
} catch (error) {
console.log('❌ MODEL1 test failed:', error.message);
return { success: false, error: error.message };
}
};
const testQueryModel = async () => {
console.log('\n🧪 Testing QUERYMODEL (Plan Execution)...');
try {
const testPlan = "Execute the following engineering plan: Design a bridge for heavy traffic loads. Steps: 1. Calculate load requirements 2. Design structural elements 3. Check safety factors";
const executionResult = await queryModelService.executePlan(testPlan, {
test: true
});
console.log('✅ QUERYMODEL execution successful');
console.log(`Execution Status: ${executionResult.executionResults.executionStatus}`);
console.log(`Steps Completed: ${executionResult.executionResults.stepsCompleted.length}`);
console.log(`Results: ${executionResult.executionResults.results.length}`);
console.log(`Processing Time: ${executionResult.processingTime}s`);
console.log(`Tokens Used: ${executionResult.tokensUsed}`);
return executionResult;
} catch (error) {
console.log('❌ QUERYMODEL test failed:', error.message);
return { success: false, error: error.message };
}
};
const testToolExecution = async () => {
console.log('\n🧪 Testing Tool Execution...');
try {
const toolResult = await queryModelService.executeTool(
'query_expander',
'query_expander',
{ query: 'bridge design', context: { test: true } },
'test-plan-id'
);
console.log('✅ Tool execution successful');
console.log(`Tool: ${toolResult.tool_name}`);
console.log(`Status: ${toolResult.status}`);
console.log(`Execution Time: ${toolResult.execution_time}s`);
return toolResult;
} catch (error) {
console.log('❌ Tool execution test failed:', error.message);
return { success: false, error: error.message };
}
};
const runAllTests = async () => {
console.log('🚀 Starting Groq Integration Tests...\n');
const results = {
connection: await testGroqConnection(),
model1: await testModel1(),
queryModel: await testQueryModel(),
toolExecution: await testToolExecution()
};
console.log('\n📊 Test Results Summary:');
console.log(`Connection: ${results.connection.success ? '✅' : '❌'}`);
console.log(`MODEL1: ${results.model1.success !== false ? '✅' : '❌'}`);
console.log(`QUERYMODEL: ${results.queryModel.success !== false ? '✅' : '❌'}`);
console.log(`Tool Execution: ${results.toolExecution.success !== false ? '✅' : '❌'}`);
const allPassed = Object.values(results).every(result =>
result.success !== false
);
console.log(`\n${allPassed ? '🎉 All tests passed!' : '⚠️ Some tests failed'}`);
return results;
};
// Run tests if this file is executed directly
if (require.main === module) {
runAllTests().catch(console.error);
}
module.exports = {
testGroqConnection,
testModel1,
testQueryModel,
testToolExecution,
runAllTests
};
+196
View File
@@ -0,0 +1,196 @@
const configLoader = require('./configLoader');
const logger = require('./logger');
const validateConfiguration = () => {
try {
console.log('🔍 Validating Reason Flow Configuration...\n');
const config = configLoader.getAll();
const env = configLoader.getEnvironment();
console.log(`Environment: ${env}`);
console.log(`Server: ${config.server.host}:${config.server.port}`);
console.log(`Database: ${config.database.host}:${config.database.port}/${config.database.name}`);
console.log(`CORS Origin: ${config.server.corsOrigin}\n`);
// Check required configurations
const checks = [
{
name: 'Groq API Key',
value: config.apis.groq.apiKey,
required: true,
valid: config.apis.groq.apiKey && config.apis.groq.apiKey !== 'your_groq_api_key_here'
},
{
name: 'JWT Secret',
value: config.auth.jwtSecret,
required: true,
valid: config.auth.jwtSecret && config.auth.jwtSecret !== 'your_jwt_secret_here_make_it_long_and_secure'
},
{
name: 'Database Password',
value: config.database.password,
required: true,
valid: config.database.password && config.database.password !== 'your_password_here'
},
{
name: 'OpenAI API Key',
value: config.apis.openai.apiKey,
required: false,
valid: config.apis.openai.apiKey && config.apis.openai.apiKey !== 'your_openai_api_key_here'
},
{
name: 'SERP API Key',
value: config.apis.serp.apiKey,
required: false,
valid: config.apis.serp.apiKey && config.apis.serp.apiKey !== 'your_serp_api_key_here'
}
];
console.log('Configuration Checks:');
console.log('====================');
let allValid = true;
let hasWarnings = false;
checks.forEach(check => {
const status = check.valid ? '✅' : (check.required ? '❌' : '⚠️');
const required = check.required ? '(Required)' : '(Optional)';
console.log(`${status} ${check.name} ${required}`);
if (!check.valid && check.required) {
allValid = false;
} else if (!check.valid && !check.required) {
hasWarnings = true;
}
});
console.log('\n');
// Environment-specific checks
if (env === 'production') {
console.log('Production Environment Checks:');
console.log('==============================');
const prodChecks = [
{
name: 'JWT Secret Security',
valid: config.auth.jwtSecret !== 'dev_secret_key_change_in_production',
message: 'JWT secret should be changed for production'
},
{
name: 'Database SSL',
valid: config.database.ssl,
message: 'Database SSL is recommended for production'
},
{
name: 'Debug Mode',
valid: !config.development.debugMode,
message: 'Debug mode should be disabled in production'
},
{
name: 'Verbose Logging',
valid: !config.development.verboseLogging,
message: 'Verbose logging should be disabled in production'
}
];
prodChecks.forEach(check => {
const status = check.valid ? '✅' : '⚠️';
console.log(`${status} ${check.name}: ${check.message}`);
if (!check.valid) {
hasWarnings = true;
}
});
}
console.log('\n');
// Summary
if (allValid) {
console.log('🎉 Configuration validation passed!');
if (hasWarnings) {
console.log('⚠️ Some optional configurations are missing or need attention.');
}
return true;
} else {
console.log('❌ Configuration validation failed!');
console.log('Please fix the required configuration issues before starting the application.');
return false;
}
} catch (error) {
console.error('❌ Configuration validation error:', error.message);
logger.error('Configuration validation error:', error);
return false;
}
};
const showConfiguration = () => {
try {
console.log('📋 Current Configuration:');
console.log('========================\n');
const config = configLoader.getAll();
console.log('Server Configuration:');
console.log(` Port: ${config.server.port}`);
console.log(` Host: ${config.server.host}`);
console.log(` Environment: ${config.server.env}`);
console.log(` CORS Origin: ${config.server.corsOrigin}\n`);
console.log('Database Configuration:');
console.log(` Host: ${config.database.host}`);
console.log(` Port: ${config.database.port}`);
console.log(` Name: ${config.database.name}`);
console.log(` User: ${config.database.user}`);
console.log(` SSL: ${config.database.ssl}\n`);
console.log('API Configuration:');
console.log(` Groq Model: ${config.apis.groq.model}`);
console.log(` Groq API Key: ${config.apis.groq.apiKey ? 'Set' : 'Not Set'}`);
console.log(` OpenAI API Key: ${config.apis.openai.apiKey ? 'Set' : 'Not Set'}`);
console.log(` SERP API Key: ${config.apis.serp.apiKey ? 'Set' : 'Not Set'}\n`);
console.log('Security Configuration:');
console.log(` JWT Secret: ${config.auth.jwtSecret ? 'Set' : 'Not Set'}`);
console.log(` JWT Expires In: ${config.auth.jwtExpiresIn}`);
console.log(` Helmet Enabled: ${config.security.helmetEnabled}\n`);
console.log('Development Configuration:');
console.log(` Debug Mode: ${config.development.debugMode}`);
console.log(` Verbose Logging: ${config.development.verboseLogging}`);
console.log(` Hot Reload: ${config.development.hotReload}\n`);
} catch (error) {
console.error('❌ Error showing configuration:', error.message);
}
};
// Run validation if this file is executed directly
if (require.main === module) {
const args = process.argv.slice(2);
const command = args[0] || 'validate';
switch (command) {
case 'validate':
const isValid = validateConfiguration();
process.exit(isValid ? 0 : 1);
break;
case 'show':
showConfiguration();
break;
default:
console.log('Usage: node validateConfig.js [validate|show]');
process.exit(1);
}
}
module.exports = {
validateConfiguration,
showConfiguration
};