Core Components
This section provides detailed documentation of SupaScan's core components and their interactions.
Component Overview
graph TD
A[Main Application] --> B[Configuration]
A --> C[ClickHouse Cluster]
A --> D[Indexing Engine]
A --> E[API Gateway]
A --> F[SupaApps]
A --> G[Services]
D --> H[Blockchain Parser]
D --> I[Data Validator]
D --> J[Message Queue]
E --> K[REST API]
E --> L[SQL API]
E --> M[Webhook Service]
F --> N[Wallet Profiler]
F --> O[KOL Watch]
F --> P[Analytics Engine]
G --> Q[Cache Service]
G --> R[Logger]
G --> S[Auth Service]
1. Main Application (index.ts)
The main entry point that orchestrates all SupaScan functionality.
Initialization Flow
// 1. Load configuration
const config = loadConfig();
// 2. Initialize ClickHouse cluster
await initializeClickHouse();
// 3. Initialize Redis cache
await initializeRedis();
// 4. Start indexing engine
await startIndexingEngine();
// 5. Initialize API gateway
await initializeAPIGateway();
// 6. Start SupaApps
await startSupaApps();
// 7. Start webhook service
await startWebhookService();
// 8. Launch application
app.listen(config.server.port);
Key Responsibilities
- Application Lifecycle Management: Start, stop, graceful shutdown
- Service Coordination: Initializes and manages all services
- Health Monitoring: Monitors system health and performance
- Error Handling: Global error handling and recovery
- Resource Management: Manages connections and resources
Error Handling
// Global error handler
process.on('uncaughtException', (error) => {
logger.error('Uncaught Exception:', error);
process.exit(1);
});
process.on('unhandledRejection', (reason, promise) => {
logger.error('Unhandled Rejection:', reason);
});
// Graceful shutdown
process.once('SIGINT', () => gracefulShutdown());
process.once('SIGTERM', () => gracefulShutdown());
2. Configuration Module (config/index.ts)
Centralized configuration management with validation for all SupaScan services.
Configuration Structure
interface Config {
server: ServerConfig;
clickhouse: ClickHouseConfig;
solana: SolanaConfig;
redis: RedisConfig;
indexing: IndexingConfig;
api: APIConfig;
webhooks: WebhookConfig;
supapps: SupaAppsConfig;
monitoring: MonitoringConfig;
}
Environment Variable Loading
class ConfigLoader {
static load(): Config {
const config = {
server: {
port: parseInt(process.env.PORT || '3000'),
host: process.env.HOST || '0.0.0.0',
cors: process.env.CORS_ORIGINS?.split(',') || ['*']
},
clickhouse: {
host: process.env.CLICKHOUSE_HOST || 'localhost',
port: parseInt(process.env.CLICKHOUSE_PORT || '9000'),
database: process.env.CLICKHOUSE_DATABASE || 'supascan',
username: process.env.CLICKHOUSE_USERNAME || 'default',
password: process.env.CLICKHOUSE_PASSWORD || '',
cluster: process.env.CLICKHOUSE_CLUSTER || 'default'
},
solana: {
rpcUrl: process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com',
rpcUrls: process.env.SOLANA_RPC_URLS?.split(',') || [],
commitment: process.env.SOLANA_COMMITMENT || 'confirmed',
timeout: parseInt(process.env.SOLANA_TIMEOUT || '30000')
},
indexing: {
batchSize: parseInt(process.env.INDEXING_BATCH_SIZE || '1000'),
intervalMs: parseInt(process.env.INDEXING_INTERVAL_MS || '1000'),
maxRetries: parseInt(process.env.INDEXING_MAX_RETRIES || '3'),
parallelWorkers: parseInt(process.env.INDEXING_WORKERS || '4')
}
};
this.validate(config);
return config;
}
private static validate(config: Config): void {
if (!config.solana.rpcUrl) {
throw new Error('SOLANA_RPC_URL is required');
}
if (!config.clickhouse.host) {
throw new Error('CLICKHOUSE_HOST is required');
}
}
}
Configuration Access
// Singleton pattern
export default ConfigLoader.load();
// Usage
import config from './config';
console.log(config.clickhouse.host);
3. ClickHouse Database Module (clickhouse.ts)
ClickHouse cluster wrapper providing high-performance analytics database operations.
Database Schema
-- Main tables
transactions -- All Solana transactions
token_transfers -- Token transfer events
token_swaps -- DEX swap events
token_creations -- New token creation events
wallets -- Wallet information and stats
dex_pools -- DEX pool information
social_signals -- Social media signals
webhook_subscriptions -- Webhook configurations
Core Functions
Data Insertion
export class ClickHouseClient {
private client: ClickHouseClient;
async insertTransaction(transaction: Transaction): Promise<void> {
await this.client.insert({
table: 'transactions',
values: [{
signature: transaction.signature,
slot: transaction.slot,
block_time: transaction.blockTime,
fee: transaction.fee,
success: transaction.success ? 1 : 0,
accounts: transaction.accounts,
instructions: transaction.instructions,
logs: transaction.logs
}]
});
}
async insertTokenTransfer(transfer: TokenTransfer): Promise<void> {
await this.client.insert({
table: 'token_transfers',
values: [{
signature: transfer.signature,
slot: transfer.slot,
block_time: transfer.blockTime,
token_mint: transfer.tokenMint,
from_address: transfer.fromAddress,
to_address: transfer.toAddress,
amount: transfer.amount,
decimals: transfer.decimals,
ui_amount: transfer.uiAmount
}]
});
}
}
Query Operations
export class QueryService {
async getTransactionsByWallet(
wallet: string,
limit: number = 100
): Promise<Transaction[]> {
const result = await this.client.query({
query: `
SELECT * FROM transactions
WHERE has(accounts, '${wallet}')
ORDER BY slot DESC
LIMIT ${limit}
`,
format: 'JSONEachRow'
});
return result.json();
}
async getTokenVolume(
tokenMint: string,
timeRange: string = '24h'
): Promise<number> {
const result = await this.client.query({
query: `
SELECT sum(amount) as total_volume
FROM token_transfers
WHERE token_mint = '${tokenMint}'
AND block_time > now() - INTERVAL ${timeRange}
`,
format: 'JSONEachRow'
});
return result.json()[0].total_volume;
}
}
Batch Operations
export class BatchProcessor {
async insertBatch(
table: string,
data: any[],
batchSize: number = 1000
): Promise<void> {
const batches = this.chunkArray(data, batchSize);
await Promise.all(
batches.map(batch =>
this.client.insert({ table, values: batch })
)
);
}
private chunkArray<T>(array: T[], size: number): T[][] {
const chunks: T[][] = [];
for (let i = 0; i < array.length; i += size) {
chunks.push(array.slice(i, i + size));
}
return chunks;
}
}
4. Indexing Engine (indexing/)
High-performance blockchain indexing system that processes Solana data in real-time.
Block Fetcher
export class BlockFetcher {
private solanaClient: Connection;
private rpcEndpoints: string[];
constructor(rpcEndpoints: string[]) {
this.rpcEndpoints = rpcEndpoints;
this.solanaClient = new Connection(rpcEndpoints[0], 'confirmed');
}
async fetchBlock(slot: number): Promise<BlockResponse | null> {
try {
const block = await this.solanaClient.getBlock(slot, {
maxSupportedTransactionVersion: 0,
transactionDetails: 'full',
rewards: false
});
return block;
} catch (error) {
logger.error(`Failed to fetch block ${slot}:`, error);
return null;
}
}
async fetchMultipleBlocks(slots: number[]): Promise<BlockResponse[]> {
const blocks = await Promise.allSettled(
slots.map(slot => this.fetchBlock(slot))
);
return blocks
.filter((result): result is PromiseFulfilledResult<BlockResponse | null> =>
result.status === 'fulfilled' && result.value !== null
)
.map(result => result.value!);
}
}
Transaction Parser
export class TransactionParser {
parseTransaction(tx: TransactionResponse): ParsedTransaction {
return {
signature: tx.transaction.signatures[0],
slot: tx.slot,
blockTime: tx.blockTime,
fee: tx.meta?.fee || 0,
success: tx.meta?.err === null,
accounts: tx.transaction.message.accountKeys.map(key => key.toString()),
instructions: this.parseInstructions(tx.transaction.message.instructions),
logs: tx.meta?.logMessages || [],
computeUnitsConsumed: tx.meta?.computeUnitsConsumed || 0
};
}
private parseInstructions(instructions: TransactionInstruction[]): ParsedInstruction[] {
return instructions.map(ix => ({
programId: ix.programId.toString(),
accounts: ix.accounts,
data: Buffer.from(ix.data).toString('base64')
}));
}
}
Token Transfer Extractor
export class TokenTransferExtractor {
extractTransfers(tx: ParsedTransaction): TokenTransfer[] {
const transfers: TokenTransfer[] = [];
for (const instruction of tx.instructions) {
if (this.isTokenTransfer(instruction)) {
const transfer = this.parseTokenTransfer(tx, instruction);
if (transfer) {
transfers.push(transfer);
}
}
}
return transfers;
}
private isTokenTransfer(instruction: ParsedInstruction): boolean {
// Check if instruction is from SPL Token program
return instruction.programId === 'TokenkegQfeZyiNwAJbNbGKPF7Wu3B9zH7';
}
private parseTokenTransfer(
tx: ParsedTransaction,
instruction: ParsedInstruction
): TokenTransfer | null {
try {
// Parse SPL Token transfer instruction
const data = Buffer.from(instruction.data, 'base64');
const instructionType = data[0];
if (instructionType === 3) { // Transfer instruction
const amount = data.readBigUInt64LE(1);
return {
signature: tx.signature,
slot: tx.slot,
blockTime: tx.blockTime,
tokenMint: this.getTokenMint(instruction),
fromAddress: this.getFromAddress(instruction),
toAddress: this.getToAddress(instruction),
amount: amount.toString(),
decimals: this.getDecimals(instruction),
uiAmount: Number(amount) / Math.pow(10, this.getDecimals(instruction))
};
}
} catch (error) {
logger.error('Failed to parse token transfer:', error);
}
return null;
}
}
DEX Swap Detector
export class DEXSwapDetector {
private dexPrograms = new Map([
['675kPX9MHTjS2zt1qfr1NYHuzeLXfQM9H24wFSUt1Mp8', 'raydium'],
['Eo7WjKq67rjJQSZxS6z3YkapzY3eMj6Xy8X5EQVn5UaB', 'meteora'],
['6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P', 'pumpfun']
]);
detectSwaps(tx: ParsedTransaction): DEXSwap[] {
const swaps: DEXSwap[] = [];
for (const instruction of tx.instructions) {
const dexProtocol = this.dexPrograms.get(instruction.programId);
if (dexProtocol) {
const swap = this.parseSwap(tx, instruction, dexProtocol);
if (swap) {
swaps.push(swap);
}
}
}
return swaps;
}
private parseSwap(
tx: ParsedTransaction,
instruction: ParsedInstruction,
protocol: string
): DEXSwap | null {
try {
// Parse swap instruction based on protocol
switch (protocol) {
case 'raydium':
return this.parseRaydiumSwap(tx, instruction);
case 'meteora':
return this.parseMeteoraSwap(tx, instruction);
case 'pumpfun':
return this.parsePumpFunSwap(tx, instruction);
default:
return null;
}
} catch (error) {
logger.error(`Failed to parse ${protocol} swap:`, error);
return null;
}
}
}
5. API Gateway (api/)
RESTful API gateway providing access to all SupaScan data and functionality.
REST API Server
export class APIServer {
private app: Express;
private clickhouse: ClickHouseClient;
constructor(clickhouse: ClickHouseClient) {
this.app = express();
this.clickhouse = clickhouse;
this.setupMiddleware();
this.setupRoutes();
}
private setupMiddleware(): void {
this.app.use(cors());
this.app.use(express.json());
this.app.use(rateLimit({
windowMs: 15 * 60 * 1000, // 15 minutes
max: 1000 // limit each IP to 1000 requests per windowMs
}));
this.app.use(authMiddleware);
}
private setupRoutes(): void {
this.app.use('/v1/transactions', transactionRoutes);
this.app.use('/v1/tokens', tokenRoutes);
this.app.use('/v1/wallets', walletRoutes);
this.app.use('/v1/sql', sqlRoutes);
this.app.use('/v1/webhooks', webhookRoutes);
this.app.use('/v1/supapps', supappsRoutes);
}
}
SQL API Gateway
export class SQLGateway {
async executeQuery(
query: string,
userId: string
): Promise<QueryResult> {
// Validate query
this.validateQuery(query);
// Check permissions
await this.checkPermissions(userId, query);
// Execute query
const result = await this.clickhouse.query({
query,
format: 'JSONEachRow'
});
return {
data: result.json(),
rows: result.rows,
executionTime: result.executionTime
};
}
private validateQuery(query: string): void {
// Check for dangerous operations
const dangerousKeywords = ['DROP', 'DELETE', 'UPDATE', 'INSERT', 'ALTER'];
const upperQuery = query.toUpperCase();
for (const keyword of dangerousKeywords) {
if (upperQuery.includes(keyword)) {
throw new Error(`Operation ${keyword} is not allowed`);
}
}
}
}
Webhook Service
export class WebhookService {
async createWebhook(
userId: string,
webhookData: WebhookConfig
): Promise<WebhookSubscription> {
const webhookId = generateId();
await this.clickhouse.insert({
table: 'webhook_subscriptions',
values: [{
webhook_id: webhookId,
user_id: userId,
webhook_url: webhookData.webhookUrl,
filters: JSON.stringify(webhookData.filters),
status: 'active',
created_at: new Date()
}]
});
return { webhookId, ...webhookData };
}
async processEvent(event: BlockchainEvent): Promise<void> {
// Get active webhooks
const webhooks = await this.getActiveWebhooks();
// Check each webhook against event
for (const webhook of webhooks) {
if (this.matchesFilters(event, webhook.filters)) {
await this.sendWebhook(webhook, event);
}
}
}
}
6. SupaApps (supapps/)
Specialized analysis applications built on top of SupaScan data.
Wallet Profiler
export class WalletProfiler {
async analyzeWallet(walletAddress: string): Promise<WalletAnalysis> {
const [
transactions,
tokenBalances,
tradingHistory,
socialSignals
] = await Promise.all([
this.getTransactionHistory(walletAddress),
this.getTokenBalances(walletAddress),
this.getTradingHistory(walletAddress),
this.getSocialSignals(walletAddress)
]);
return {
address: walletAddress,
totalTransactions: transactions.length,
totalVolume: this.calculateTotalVolume(tradingHistory),
winRate: this.calculateWinRate(tradingHistory),
topTokens: this.getTopTokens(tokenBalances),
riskScore: this.calculateRiskScore(transactions),
socialActivity: socialSignals,
lastActivity: this.getLastActivity(transactions)
};
}
private calculateWinRate(trades: Trade[]): number {
const profitableTrades = trades.filter(trade => trade.pnl > 0);
return profitableTrades.length / trades.length;
}
}
KOL Watch
export class KOLWatch {
async trackInfluencer(influencerId: string): Promise<InfluencerActivity> {
const [
recentPosts,
walletActivity,
tokenMentions
] = await Promise.all([
this.getRecentPosts(influencerId),
this.getWalletActivity(influencerId),
this.getTokenMentions(influencerId)
]);
return {
influencerId,
recentPosts,
walletActivity: this.analyzeWalletActivity(walletActivity),
tokenMentions: this.analyzeTokenMentions(tokenMentions),
influenceScore: this.calculateInfluenceScore(recentPosts, walletActivity)
};
}
async detectInfluenceEvents(influencerId: string): Promise<InfluenceEvent[]> {
const events: InfluenceEvent[] = [];
// Check for new posts
const newPosts = await this.getNewPosts(influencerId);
for (const post of newPosts) {
const tokens = this.extractTokenMentions(post.text);
if (tokens.length > 0) {
events.push({
type: 'token_mention',
influencerId,
postId: post.id,
tokens,
timestamp: post.timestamp
});
}
}
return events;
}
}
PnL Detective
export class PnLDetective {
async calculatePnL(walletAddress: string, timeRange: string): Promise<PnLAnalysis> {
const trades = await this.getTradesInRange(walletAddress, timeRange);
let totalPnL = 0;
let totalFees = 0;
const tokenPnL = new Map<string, number>();
for (const trade of trades) {
const pnl = this.calculateTradePnL(trade);
totalPnL += pnl;
totalFees += trade.fee;
const currentPnL = tokenPnL.get(trade.token) || 0;
tokenPnL.set(trade.token, currentPnL + pnl);
}
return {
walletAddress,
timeRange,
totalPnL,
totalFees,
netPnL: totalPnL - totalFees,
tokenBreakdown: Object.fromEntries(tokenPnL),
tradeCount: trades.length,
averageTradeSize: this.calculateAverageTradeSize(trades)
};
}
}
7. Service Layer
Cache Service
Multi-level caching system for optimal performance:
export class CacheService {
private memoryCache: NodeCache;
private redisCache?: Redis;
constructor(redisConfig?: RedisConfig) {
this.memoryCache = new NodeCache({
stdTTL: 300, // 5 minutes default
checkperiod: 60
});
if (redisConfig) {
this.redisCache = new Redis(redisConfig);
}
}
async get<T>(key: string): Promise<T | null> {
// L1: Memory cache
const memResult = this.memoryCache.get<T>(key);
if (memResult) return memResult;
// L2: Redis cache
if (this.redisCache) {
const redisResult = await this.redisCache.get(key);
if (redisResult) {
const parsed = JSON.parse(redisResult);
this.memoryCache.set(key, parsed);
return parsed;
}
}
return null;
}
async set<T>(key: string, value: T, ttl?: number): Promise<void> {
// Set in memory cache
this.memoryCache.set(key, value, ttl);
// Set in Redis if available
if (this.redisCache) {
await this.redisCache.setex(key, ttl || 300, JSON.stringify(value));
}
}
}
Authentication Service
API key management and validation:
export class AuthService {
async validateApiKey(apiKey: string): Promise<AuthResult> {
try {
// Check API key format
if (!this.isValidApiKeyFormat(apiKey)) {
return { valid: false, error: 'INVALID_API_KEY' };
}
// Check if key exists and is active
const keyData = await this.getApiKeyData(apiKey);
if (!keyData || keyData.status !== 'active') {
return { valid: false, error: 'INVALID_API_KEY' };
}
// Check rate limits
if (await this.isRateLimited(apiKey)) {
return { valid: false, error: 'RATE_LIMIT_EXCEEDED' };
}
return {
valid: true,
userId: keyData.userId,
permissions: keyData.permissions
};
} catch (error) {
logger.error('Auth validation error:', error);
return { valid: false, error: 'AUTH_ERROR' };
}
}
private isValidApiKeyFormat(apiKey: string): boolean {
return /^sk_[a-zA-Z0-9]{32}$/.test(apiKey);
}
}
Analytics Engine
Real-time analytics and metrics calculation:
export class AnalyticsEngine {
async calculateTokenMetrics(tokenMint: string): Promise<TokenMetrics> {
const [
volume24h,
priceChange24h,
holderCount,
transactionCount
] = await Promise.all([
this.getVolume24h(tokenMint),
this.getPriceChange24h(tokenMint),
this.getHolderCount(tokenMint),
this.getTransactionCount24h(tokenMint)
]);
return {
tokenMint,
volume24h,
priceChange24h,
holderCount,
transactionCount,
liquidity: await this.getLiquidity(tokenMint),
marketCap: await this.getMarketCap(tokenMint)
};
}
async detectPatterns(walletAddress: string): Promise<TradingPattern[]> {
const trades = await this.getTradingHistory(walletAddress);
const patterns: TradingPattern[] = [];
// Detect swing trading patterns
if (this.isSwingTrader(trades)) {
patterns.push({ type: 'swing_trader', confidence: 0.85 });
}
// Detect arbitrage patterns
if (this.isArbitrageur(trades)) {
patterns.push({ type: 'arbitrageur', confidence: 0.92 });
}
// Detect MEV patterns
if (this.isMEVBot(trades)) {
patterns.push({ type: 'mev_bot', confidence: 0.78 });
}
return patterns;
}
}
Logger Service
Comprehensive logging system:
export class LoggerService {
private logger: winston.Logger;
constructor() {
this.logger = winston.createLogger({
level: process.env.LOG_LEVEL || 'info',
format: winston.format.combine(
winston.format.timestamp(),
winston.format.errors({ stack: true }),
winston.format.json()
),
transports: [
new winston.transports.Console({
format: winston.format.combine(
winston.format.colorize(),
winston.format.simple()
)
}),
new winston.transports.DailyRotateFile({
filename: 'logs/supascan-%DATE%.log',
datePattern: 'YYYY-MM-DD',
maxFiles: '30d',
maxSize: '100m'
})
]
});
}
info(message: string, meta?: any): void {
this.logger.info(message, meta);
}
error(message: string, error?: Error, meta?: any): void {
this.logger.error(message, { error: error?.stack, ...meta });
}
warn(message: string, meta?: any): void {
this.logger.warn(message, meta);
}
}
Component Interaction Examples
Real-time Token Analysis Flow
sequenceDiagram
participant User
participant API
participant Analytics
participant ClickHouse
participant Cache
participant Webhook
User->>API: GET /v1/tokens/So111.../analysis
API->>Cache: Check cached analysis
alt Cache hit
Cache-->>API: Return cached data
else Cache miss
API->>Analytics: Calculate metrics
Analytics->>ClickHouse: Query token data
ClickHouse-->>Analytics: Raw data
Analytics->>Analytics: Process metrics
Analytics-->>API: Calculated metrics
API->>Cache: Store results
end
API-->>User: Token analysis
Note over Analytics: Background processing
Analytics->>Webhook: Check for alerts
Webhook->>User: Send notification (if configured)
Webhook Event Processing Flow
sequenceDiagram
participant Blockchain
participant Indexer
participant FilterEngine
participant WebhookService
participant UserEndpoint
Blockchain->>Indexer: New token created
Indexer->>Indexer: Parse and validate
Indexer->>ClickHouse: Store token data
Indexer->>FilterEngine: Check webhook filters
FilterEngine->>ClickHouse: Query active webhooks
ClickHouse-->>FilterEngine: Matching webhooks
loop For each matching webhook
FilterEngine->>WebhookService: Process event
WebhookService->>UserEndpoint: Send webhook
UserEndpoint-->>WebhookService: Acknowledgment
end
SupaApp Analysis Flow
sequenceDiagram
participant User
participant SupaApp
participant Analytics
participant ClickHouse
participant Cache
User->>SupaApp: Analyze wallet 9WzDX...
SupaApp->>Cache: Check cached analysis
alt Cache hit
Cache-->>SupaApp: Return cached data
else Cache miss
SupaApp->>Analytics: Request analysis
Analytics->>ClickHouse: Query wallet data
ClickHouse-->>Analytics: Transaction history
Analytics->>Analytics: Calculate patterns
Analytics->>Analytics: Generate insights
Analytics-->>SupaApp: Analysis results
SupaApp->>Cache: Store results
end
SupaApp-->>User: Wallet analysis report
Performance Optimizations
Connection Pooling
export class ConnectionPool {
private pools: Map<string, Pool> = new Map();
getConnection(service: string): Pool {
if (!this.pools.has(service)) {
this.pools.set(service, this.createPool(service));
}
return this.pools.get(service)!;
}
private createPool(service: string): Pool {
const config = this.getPoolConfig(service);
return new Pool({
min: config.minConnections,
max: config.maxConnections,
acquireTimeoutMillis: config.acquireTimeout,
createTimeoutMillis: config.createTimeout,
destroyTimeoutMillis: config.destroyTimeout
});
}
}
Query Optimization
export class QueryOptimizer {
optimizeQuery(query: string): string {
// Add LIMIT if missing
if (!query.toUpperCase().includes('LIMIT')) {
query += ' LIMIT 1000';
}
// Optimize time-based filters
query = this.optimizeTimeFilters(query);
// Add appropriate indexes hints
query = this.addIndexHints(query);
return query;
}
private optimizeTimeFilters(query: string): string {
// Replace generic time filters with optimized ones
return query.replace(
/WHERE\s+block_time\s*>\s*now\(\)\s*-\s*INTERVAL\s+(\d+)\s+HOUR/gi,
'WHERE block_time > toDateTime(now() - INTERVAL $1 HOUR)'
);
}
}