/** * Graceful Degradation Tests * * Tests the behavior of the UnifiedStorageManager when some storage tiers are unavailable. */ import * as path from 'path'; import * as os from 'os'; import * as fs from 'fs'; import { StorageTier, StorageConfig } from '../../src/types'; import { createUnifiedStorageManager } from '../../src/storage'; describe('Graceful Degradation Tests', () => { // Create a temporary directory for local storage const tempDir = path.join(os.tmpdir(), 'data-storage-tests', `test-${Date.now()}`); // Storage configuration let config: StorageConfig; beforeAll(() => { // Create temp directory fs.mkdirSync(tempDir, { recursive: true }); // Create storage configuration with multiple tiers config = { providers: { hot: { type: 'redis' as any, config: { host: 'localhost', port: 6379 } }, warm: { type: 'influxdb' as any, config: { url: 'http://localhost:8086', token: 'token', org: 'org', bucket: 'bucket' } }, cold: { type: 'postgresql' as any, config: { host: 'localhost', port: 5432, database: 'db', user: 'user', password: 'password' } }, batching: { timeBased: { enabled: true, intervalMs: 1000 }, sizeBased: { enabled: true, maxBatchSize: 10 }, priorityOverride: true } } }; }); afterAll(() => { // Clean up temp directory fs.rmSync(tempDir, { recursive: true, force: true }); }); it('should continue operation when hot tier is unavailable', async () => { // Arrange const storageManager = createUnifiedStorageManager(config, { localStorage: { directory: tempDir, maxSizeBytes: 10 * 1024 * 1024, // 10MB encrypt: false, compressionLevel: 0 } }); // Mock the provider factory with hot tier unavailable const mockHotStore = jest.fn().mockRejectedValue(new Error('Hot tier unavailable')); const mockWarmStore = jest.fn().mockResolvedValue('warm-id'); const mockColdStore = jest.fn().mockResolvedValue('cold-id'); (storageManager as any).remoteStorageManager = { initialize: jest.fn().mockResolvedValue(undefined), store: jest.fn().mockImplementation((data, category, options) => { if (options?.tier === StorageTier.HOT) { return mockHotStore(data, category, options); } else if (options?.tier === StorageTier.WARM) { return mockWarmStore(data, category, options); } else if (options?.tier === StorageTier.COLD) { return mockColdStore(data, category, options); } else { return mockHotStore(data, category, options); // Default to hot tier } }), retrieve: jest.fn().mockResolvedValue([]), update: jest.fn().mockResolvedValue(undefined), delete: jest.fn().mockResolvedValue(undefined), storeBatch: jest.fn().mockResolvedValue([]), retrieveBatch: jest.fn().mockResolvedValue([]), cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }), migrate: jest.fn().mockResolvedValue(undefined), getStatus: jest.fn().mockResolvedValue({ connected: true, healthy: true, queryPerformance: { hot: { averageQueryTime: 0, queriesPerSecond: 0 }, warm: { averageQueryTime: 100, queriesPerSecond: 5 }, cold: { averageQueryTime: 200, queriesPerSecond: 2 }, overall: 'degraded', providers: { hot: 'unhealthy', warm: 'healthy', cold: 'healthy' }, issues: { severity: 'warning', message: 'Hot tier is unavailable', component: 'hot-tier', timestamp: new Date() } } }), close: jest.fn().mockResolvedValue(undefined) }; // Act await storageManager.initialize(); // Store data in hot tier (should fail and fall back to warm tier) const hotId = await storageManager.store({ value: 1 }, 'test', { tier: StorageTier.HOT }); // Store data in warm tier (should succeed) const warmId = await storageManager.store({ value: 2 }, 'test', { tier: StorageTier.WARM }); // Store data in cold tier (should succeed) const coldId = await storageManager.store({ value: 3 }, 'test', { tier: StorageTier.COLD }); // Get status const status = await storageManager.getStatus(); // Assert expect(mockHotStore).toHaveBeenCalledTimes(1); expect(mockWarmStore).toHaveBeenCalledTimes(2); // Once for warm tier, once for fallback from hot expect(mockColdStore).toHaveBeenCalledTimes(1); expect(hotId).toBe('warm-id'); // Should fall back to warm tier expect(warmId).toBe('warm-id'); expect(coldId).toBe('cold-id'); expect(status.healthy).toBe(false); // System is degraded expect(status.queryPerformance.overall).toBe('degraded'); // Clean up await storageManager.close(); }); it('should continue operation when warm tier is unavailable', async () => { // Arrange const storageManager = createUnifiedStorageManager(config, { localStorage: { directory: tempDir, maxSizeBytes: 10 * 1024 * 1024, // 10MB encrypt: false, compressionLevel: 0 } }); // Mock the provider factory with warm tier unavailable const mockHotStore = jest.fn().mockResolvedValue('hot-id'); const mockWarmStore = jest.fn().mockRejectedValue(new Error('Warm tier unavailable')); const mockColdStore = jest.fn().mockResolvedValue('cold-id'); (storageManager as any).remoteStorageManager = { initialize: jest.fn().mockResolvedValue(undefined), store: jest.fn().mockImplementation((data, category, options) => { if (options?.tier === StorageTier.HOT) { return mockHotStore(data, category, options); } else if (options?.tier === StorageTier.WARM) { return mockWarmStore(data, category, options); } else if (options?.tier === StorageTier.COLD) { return mockColdStore(data, category, options); } else { return mockHotStore(data, category, options); // Default to hot tier } }), retrieve: jest.fn().mockResolvedValue([]), update: jest.fn().mockResolvedValue(undefined), delete: jest.fn().mockResolvedValue(undefined), storeBatch: jest.fn().mockResolvedValue([]), retrieveBatch: jest.fn().mockResolvedValue([]), cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }), migrate: jest.fn().mockResolvedValue(undefined), getStatus: jest.fn().mockResolvedValue({ connected: true, healthy: true, queryPerformance: { hot: { averageQueryTime: 50, queriesPerSecond: 10 }, warm: { averageQueryTime: 0, queriesPerSecond: 0 }, cold: { averageQueryTime: 200, queriesPerSecond: 2 }, overall: 'degraded', providers: { hot: 'healthy', warm: 'unhealthy', cold: 'healthy' }, issues: { severity: 'warning', message: 'Warm tier is unavailable', component: 'warm-tier', timestamp: new Date() } } }), close: jest.fn().mockResolvedValue(undefined) }; // Act await storageManager.initialize(); // Store data in hot tier (should succeed) const hotId = await storageManager.store({ value: 1 }, 'test', { tier: StorageTier.HOT }); // Store data in warm tier (should fail and fall back to hot tier) const warmId = await storageManager.store({ value: 2 }, 'test', { tier: StorageTier.WARM }); // Store data in cold tier (should succeed) const coldId = await storageManager.store({ value: 3 }, 'test', { tier: StorageTier.COLD }); // Get status const status = await storageManager.getStatus(); // Assert expect(mockHotStore).toHaveBeenCalledTimes(2); // Once for hot tier, once for fallback from warm expect(mockWarmStore).toHaveBeenCalledTimes(1); expect(mockColdStore).toHaveBeenCalledTimes(1); expect(hotId).toBe('hot-id'); expect(warmId).toBe('hot-id'); // Should fall back to hot tier expect(coldId).toBe('cold-id'); expect(status.healthy).toBe(false); // System is degraded expect(status.queryPerformance.overall).toBe('degraded'); // Clean up await storageManager.close(); }); it('should continue operation when all remote tiers are unavailable', async () => { // Arrange const storageManager = createUnifiedStorageManager(config, { localStorage: { directory: tempDir, maxSizeBytes: 10 * 1024 * 1024, // 10MB encrypt: false, compressionLevel: 0 } }); // Mock the provider factory with all tiers unavailable const mockHotStore = jest.fn().mockRejectedValue(new Error('Hot tier unavailable')); const mockWarmStore = jest.fn().mockRejectedValue(new Error('Warm tier unavailable')); const mockColdStore = jest.fn().mockRejectedValue(new Error('Cold tier unavailable')); (storageManager as any).remoteStorageManager = { initialize: jest.fn().mockResolvedValue(undefined), store: jest.fn().mockImplementation((data, category, options) => { if (options?.tier === StorageTier.HOT) { return mockHotStore(data, category, options); } else if (options?.tier === StorageTier.WARM) { return mockWarmStore(data, category, options); } else if (options?.tier === StorageTier.COLD) { return mockColdStore(data, category, options); } else { return mockHotStore(data, category, options); // Default to hot tier } }), retrieve: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')), update: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')), delete: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')), storeBatch: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')), retrieveBatch: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')), cleanup: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')), migrate: jest.fn().mockRejectedValue(new Error('Remote storage unavailable')), getStatus: jest.fn().mockResolvedValue({ connected: false, healthy: false, error: 'All remote tiers are unavailable', queryPerformance: { hot: { averageQueryTime: 0, queriesPerSecond: 0 }, warm: { averageQueryTime: 0, queriesPerSecond: 0 }, cold: { averageQueryTime: 0, queriesPerSecond: 0 }, overall: 'unhealthy', providers: { hot: 'unhealthy', warm: 'unhealthy', cold: 'unhealthy' }, issues: { severity: 'error', message: 'All remote tiers are unavailable', component: 'remote-storage', timestamp: new Date() } } }), close: jest.fn().mockResolvedValue(undefined) }; // Create a spy on the local storage provider const localStoreSpy = jest.fn().mockResolvedValue('local-id'); (storageManager as any).localStorageProvider = { initialize: jest.fn().mockResolvedValue(undefined), store: localStoreSpy, retrieve: jest.fn().mockResolvedValue([{ data: { value: 'test' }, metadata: { id: 'local-id', category: 'test' } }]), update: jest.fn().mockResolvedValue(undefined), delete: jest.fn().mockResolvedValue(undefined), storeBatch: jest.fn().mockResolvedValue(['local-id-1', 'local-id-2']), cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }), getStatus: jest.fn().mockResolvedValue({ connected: true, healthy: true, queryPerformance: { hot: { averageQueryTime: 10, queriesPerSecond: 100 }, warm: { averageQueryTime: 10, queriesPerSecond: 100 }, cold: { averageQueryTime: 10, queriesPerSecond: 100 }, overall: 'healthy', providers: { hot: 'healthy', warm: 'healthy', cold: 'healthy' } } }), close: jest.fn().mockResolvedValue(undefined), getLocalStorageStatus: jest.fn().mockResolvedValue({ available: true, healthy: true, totalSizeBytes: 1024 * 1024 * 1024, usedSizeBytes: 1024 * 1024, freeSizeBytes: 1023 * 1024 * 1024, itemCount: 10 }) }; // Act await storageManager.initialize(); // Store data in hot tier (should fail and fall back to local storage) const hotId = await storageManager.store({ value: 1 }, 'test', { tier: StorageTier.HOT }); // Store data in warm tier (should fail and fall back to local storage) const warmId = await storageManager.store({ value: 2 }, 'test', { tier: StorageTier.WARM }); // Store data in cold tier (should fail and fall back to local storage) const coldId = await storageManager.store({ value: 3 }, 'test', { tier: StorageTier.COLD }); // Retrieve data (should use local storage) const results = await storageManager.retrieve({ category: 'test' }); // Get status const status = await storageManager.getStatus(); // Get local storage status const localStatus = await storageManager.getLocalStorageStatus(); // Assert expect(mockHotStore).toHaveBeenCalledTimes(1); expect(mockWarmStore).toHaveBeenCalledTimes(1); expect(mockColdStore).toHaveBeenCalledTimes(1); expect(localStoreSpy).toHaveBeenCalledTimes(3); expect(hotId).toBe('local-id'); expect(warmId).toBe('local-id'); expect(coldId).toBe('local-id'); expect(results.length).toBe(1); expect(results[0].data.value).toBe('test'); expect(status.connected).toBe(true); // System is connected via local storage expect(status.healthy).toBe(false); // System is degraded expect(status.queryPerformance.overall).toBe('unhealthy'); expect(localStatus.available).toBe(true); expect(localStatus.healthy).toBe(true); // Verify pending sync contains the stored items expect((storageManager as any).pendingSync.size).toBe(3); expect((storageManager as any).pendingSync.has(hotId)).toBe(true); expect((storageManager as any).pendingSync.has(warmId)).toBe(true); expect((storageManager as any).pendingSync.has(coldId)).toBe(true); // Clean up await storageManager.close(); }); it('should recover when tiers become available again', async () => { // Arrange const storageManager = createUnifiedStorageManager(config, { localStorage: { directory: tempDir, maxSizeBytes: 10 * 1024 * 1024, // 10MB encrypt: false, compressionLevel: 0 } }); // Mock the provider factory with tiers initially unavailable, then available const mockHotStore = jest.fn() .mockRejectedValueOnce(new Error('Hot tier unavailable')) .mockResolvedValue('hot-id'); const mockWarmStore = jest.fn() .mockRejectedValueOnce(new Error('Warm tier unavailable')) .mockResolvedValue('warm-id'); const mockColdStore = jest.fn() .mockRejectedValueOnce(new Error('Cold tier unavailable')) .mockResolvedValue('cold-id'); (storageManager as any).remoteStorageManager = { initialize: jest.fn().mockResolvedValue(undefined), store: jest.fn().mockImplementation((data, category, options) => { if (options?.tier === StorageTier.HOT) { return mockHotStore(data, category, options); } else if (options?.tier === StorageTier.WARM) { return mockWarmStore(data, category, options); } else if (options?.tier === StorageTier.COLD) { return mockColdStore(data, category, options); } else { return mockHotStore(data, category, options); // Default to hot tier } }), retrieve: jest.fn() .mockRejectedValueOnce(new Error('Remote storage unavailable')) .mockResolvedValue([{ data: { value: 'remote' }, metadata: { id: 'remote-id', category: 'test' } }]), update: jest.fn().mockResolvedValue(undefined), delete: jest.fn().mockResolvedValue(undefined), storeBatch: jest.fn().mockResolvedValue(['remote-id-1', 'remote-id-2']), retrieveBatch: jest.fn().mockResolvedValue([]), cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }), migrate: jest.fn().mockResolvedValue(undefined), getStatus: jest.fn() .mockResolvedValueOnce({ connected: false, healthy: false, error: 'All remote tiers are unavailable', queryPerformance: { hot: { averageQueryTime: 0, queriesPerSecond: 0 }, warm: { averageQueryTime: 0, queriesPerSecond: 0 }, cold: { averageQueryTime: 0, queriesPerSecond: 0 }, overall: 'unhealthy', providers: { hot: 'unhealthy', warm: 'unhealthy', cold: 'unhealthy' } } }) .mockResolvedValue({ connected: true, healthy: true, queryPerformance: { hot: { averageQueryTime: 50, queriesPerSecond: 10 }, warm: { averageQueryTime: 100, queriesPerSecond: 5 }, cold: { averageQueryTime: 200, queriesPerSecond: 2 }, overall: 'healthy', providers: { hot: 'healthy', warm: 'healthy', cold: 'healthy' } } }), close: jest.fn().mockResolvedValue(undefined) }; // Create a spy on the local storage provider const localStoreSpy = jest.fn().mockResolvedValue('local-id'); (storageManager as any).localStorageProvider = { initialize: jest.fn().mockResolvedValue(undefined), store: localStoreSpy, retrieve: jest.fn().mockResolvedValue([{ data: { value: 'local' }, metadata: { id: 'local-id', category: 'test' } }]), update: jest.fn().mockResolvedValue(undefined), delete: jest.fn().mockResolvedValue(undefined), storeBatch: jest.fn().mockResolvedValue(['local-id-1', 'local-id-2']), cleanup: jest.fn().mockResolvedValue({ itemsRemoved: 0, bytesFreed: 0, duration: 0 }), getStatus: jest.fn().mockResolvedValue({ connected: true, healthy: true, queryPerformance: { hot: { averageQueryTime: 10, queriesPerSecond: 100 }, warm: { averageQueryTime: 10, queriesPerSecond: 100 }, cold: { averageQueryTime: 10, queriesPerSecond: 100 }, overall: 'healthy', providers: { hot: 'healthy', warm: 'healthy', cold: 'healthy' } } }), close: jest.fn().mockResolvedValue(undefined), getLocalStorageStatus: jest.fn().mockResolvedValue({ available: true, healthy: true, totalSizeBytes: 1024 * 1024 * 1024, usedSizeBytes: 1024 * 1024, freeSizeBytes: 1023 * 1024 * 1024, itemCount: 10 }) }; // Act await storageManager.initialize(); // First phase: tiers are unavailable // Store data in hot tier (should fail and fall back to local storage) const hotId1 = await storageManager.store({ value: 1 }, 'test', { tier: StorageTier.HOT }); // Retrieve data (should use local storage) const results1 = await storageManager.retrieve({ category: 'test' }); // Get status (should be degraded) const status1 = await storageManager.getStatus(); // Second phase: tiers are available again // Store data in hot tier (should succeed) const hotId2 = await storageManager.store({ value: 2 }, 'test', { tier: StorageTier.HOT }); // Retrieve data (should use remote storage) const results2 = await storageManager.retrieve({ category: 'test' }); // Get status (should be healthy) const status2 = await storageManager.getStatus(); // Force synchronization (should sync local data to remote) const syncResult = await storageManager.forceSynchronization(); // Assert expect(hotId1).toBe('local-id'); expect(hotId2).toBe('hot-id'); expect(results1[0].data.value).toBe('local'); expect(results2[0].data.value).toBe('remote'); expect(status1.healthy).toBe(false); expect(status1.queryPerformance.overall).toBe('unhealthy'); expect(status2.healthy).toBe(true); expect(status2.queryPerformance.overall).toBe('healthy'); expect(syncResult.success).toBe(true); expect(syncResult.itemCount).toBeGreaterThan(0); // Clean up await storageManager.close(); }); });