Data Sync: Offline-First Architecture

Problem

Modern applications require seamless functionality regardless of network connectivity. Users expect to:

• Continue working without interruption when offline
• Automatically sync changes when reconnected
• Avoid data loss or conflicts
• Experience consistent UI state across sessions

Traditional online-first approaches fail when connectivity drops, frustrating users and causing data loss. Offline-first architecture inverts this paradigm: the local database is the source of truth, and the server is a backup.

Solution

Core Principles

1. Local-First Storage

• Store all data locally using IndexedDB, SQLite, or similar
• Treat local database as primary source of truth
• Server acts as backup and sync point

2. Optimistic Updates

• Update UI immediately on user action
• Queue changes for sync
• Rollback if sync fails

3. Conflict Resolution

• Implement Last-Write-Wins (LWW) or Custom Logic
• Track timestamps and version numbers
• Merge strategies for complex data

4. Sync Queue

• Maintain queue of pending operations
• Retry failed syncs with exponential backoff
• Preserve operation order

5. Network Detection

• Monitor online/offline status
• Trigger sync when connectivity restored
• Provide user feedback

Architecture Diagram

┌─────────────────────────────────────────┐
│         User Interface Layer            │
└──────────────┬──────────────────────────┘
               │
┌──────────────▼──────────────────────────┐
│    Local State Management (Redux)       │
└──────────────┬──────────────────────────┘
               │
┌──────────────▼──────────────────────────┐
│   Sync Engine & Conflict Resolution     │
└──────────────┬──────────────────────────┘
               │
┌──────────────▼──────────────────────────┐
│  Local Database (IndexedDB/SQLite)      │
└──────────────┬──────────────────────────┘
               │
        ┌──────┴──────┐
        │             │
   ┌────▼────┐   ┌───▼────┐
   │ Online? │   │ Offline?│
   └────┬────┘   └────┬────┘
        │             │
   ┌────▼──────────────▼────┐
   │   Sync Queue Manager   │
   └────┬──────────────┬────┘
        │              │
   ┌────▼────┐    ┌───▼────┐
   │ Retry   │    │ Batch  │
   │ Logic   │    │ Ops    │
   └────┬────┘    └───┬────┘
        │             │
        └──────┬──────┘
               │
        ┌──────▼──────────┐
        │  Remote Server  │
        └─────────────────┘

Code

1. Sync Engine Core

// syncEngine.ts
interface SyncOperation {
  id: string;
  type: 'CREATE' | 'UPDATE' | 'DELETE';
  entity: string;
  data: any;
  timestamp: number;
  retries: number;
  status: 'PENDING' | 'SYNCING' | 'SYNCED' | 'FAILED';
}

interface ConflictResolution {
  strategy: 'LWW' | 'CUSTOM' | 'MANUAL';
  resolver?: (local: any, remote: any) => any;
}

class SyncEngine {
  private db: IDBDatabase;
  private syncQueue: SyncOperation[] = [];
  private isOnline: boolean = navigator.onLine;
  private isSyncing: boolean = false;
  private conflictStrategy: ConflictResolution;

  constructor(db: IDBDatabase, strategy: ConflictResolution = { strategy: 'LWW' }) {
    this.db = db;
    this.conflictStrategy = strategy;
    this.initializeNetworkListeners();
    this.loadQueueFromStorage();
  }

  private initializeNetworkListeners(): void {
    window.addEventListener('online', () => this.handleOnline());
    window.addEventListener('offline', () => this.handleOffline());
  }

  private handleOnline(): void {
    this.isOnline = true;
    console.log('🟢 Online - Starting sync');
    this.syncAll();
  }

  private handleOffline(): void {
    this.isOnline = false;
    console.log('🔴 Offline - Queuing operations');
  }

  async addOperation(operation: Omit<SyncOperation, 'id' | 'timestamp' | 'retries' | 'status'>): Promise<void> {
    const syncOp: SyncOperation = {
      ...operation,
      id: `${operation.entity}-${Date.now()}-${Math.random()}`,
      timestamp: Date.now(),
      retries: 0,
      status: 'PENDING',
    };

    this.syncQueue.push(syncOp);
    await this.persistQueue();

    if (this.isOnline && !this.isSyncing) {
      this.syncAll();
    }
  }

  private async syncAll(): Promise<void> {
    if (this.isSyncing || this.syncQueue.length === 0) return;

    this.isSyncing = true;

    try {
      const pendingOps = this.syncQueue.filter(op => op.status === 'PENDING');

      for (const operation of pendingOps) {
        await this.syncOperation(operation);
      }

      // Remove synced operations
      this.syncQueue = this.syncQueue.filter(op => op.status !== 'SYNCED');
      await this.persistQueue();
    } catch (error) {
      console.error('Sync failed:', error);
    } finally {
      this.isSyncing = false;
    }
  }

  private async syncOperation(operation: SyncOperation): Promise<void> {
    operation.status = 'SYNCING';

    try {
      const response = await this.sendToServer(operation);

      if (response.conflict) {
        await this.handleConflict(operation, response.serverData);
      } else {
        operation.status = 'SYNCED';
      }
    } catch (error) {
      operation.retries++;

      if (operation.retries < 5) {
        operation.status = 'PENDING';
        await this.exponentialBackoff(operation.retries);
      } else {
        operation.status = 'FAILED';
        console.error(`Operation ${operation.id} failed after 5 retries`);
      }
    }

    await this.persistQueue();
  }

  private async handleConflict(local: SyncOperation, remote: any): Promise<void> {
    let resolved: any;

    switch (this.conflictStrategy.strategy) {
      case 'LWW':
        resolved = local.timestamp > remote.timestamp ? local.data : remote;
        break;
      case 'CUSTOM':
        if (this.conflictStrategy.resolver) {
          resolved = this.conflictStrategy.resolver(local.data, remote);
        }
        break;
      case 'MANUAL':
        // Emit event for UI to handle
        window.dispatchEvent(new CustomEvent('conflict', { detail: { local, remote } }));
        return;
    }

    // Update local database with resolved data
    await this.updateLocalDatabase(local.entity, resolved);
    local.status = 'SYNCED';
  }

  private async sendToServer(operation: SyncOperation): Promise<any> {
    const response = await fetch(`/api/sync/${operation.entity}`, {
      method: operation.type === 'DELETE' ? 'DELETE' : operation.type === 'CREATE' ? 'POST' : 'PUT',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        ...operation.data,
        _syncId: operation.id,
        _timestamp: operation.timestamp,
      }),
    });

    if (!response.ok) throw new Error(`HTTP ${response.status}`);
    return response.json();
  }

  private async exponentialBackoff(retryCount: number): Promise<void> {
    const delay = Math.min(1000 * Math.pow(2, retryCount), 30000);
    return new Promise(resolve => setTimeout(resolve, delay));
  }

  private async updateLocalDatabase(entity: string, data: any): Promise<void> {
    const tx = this.db.transaction(entity, 'readwrite');
    const store = tx.objectStore(entity);
    await store.put(data);
  }

  private async persistQueue(): Promise<void> {
    const tx = this.db.transaction('_syncQueue', 'readwrite');
    const store = tx.objectStore('_syncQueue');
    await store.clear();
    for (const op of this.syncQueue) {
      await store.add(op);
    }
  }

  private async loadQueueFromStorage(): Promise<void> {
    const tx = this.db.transaction('_syncQueue', 'readonly');
    const store = tx.objectStore('_syncQueue');
    this.syncQueue = await store.getAll();
  }

  getQueueStatus(): { pending: number; syncing: number; failed: number } {
    return {
      pending: this.syncQueue.filter(op => op.status === 'PENDING').length,
      syncing: this.syncQueue.filter(op => op.status === 'SYNCING').length,
      failed: this.syncQueue.filter(op => op.status === 'FAILED').length,
    };
  }
}

export default SyncEngine;

2. Local Database Setup

// database.ts
class LocalDatabase {
  private db: IDBDatabase | null = null;

  async initialize(): Promise<IDBDatabase> {
    return new Promise((resolve, reject) => {
      const request = indexedDB.open('AppDB', 1);

      request.onerror = () => reject(request.error);
      request.onsuccess = () => {
        this.db = request.result;
        resolve(this.db);
      };

      request.onupgradeneeded = (event) => {
        const db = (event.target as IDBOpenDBRequest).result;

        // Create object stores
        if (!db.objectStoreNames.contains('todos')) {
          const todoStore = db.createObjectStore('todos', { keyPath: 'id' });
          todoStore.createIndex('status', 'status', { unique: false });
          todoStore.createIndex('createdAt', 'createdAt', { unique: false });
        }

        if (!db.objectStoreNames.contains('users')) {
          db.createObjectStore('users', { keyPath: 'id' });
        }

        if (!db.objectStoreNames.contains('_syncQueue')) {
          db.createObjectStore('_syncQueue', { keyPath: 'id' });
        }

        if (!db.objectStoreNames.contains('_metadata')) {
          db.createObjectStore('_metadata', { keyPath: 'key' });
        }
      };
    });
  }

  async get(storeName: string, key: any): Promise<any> {
    if (!this.db) throw new Error('Database not initialized');
    const tx = this.db.transaction(storeName, 'readonly');
    const store = tx.objectStore(storeName);
    return new Promise((resolve, reject) => {
      const request = store.get(key);
      request.onsuccess = () => resolve(request.result);
      request.onerror = () => reject(request.error);
    });
  }

  async getAll(storeName: string, query?: IDBValidKey | IDBKeyRange): Promise<any[]> {
    if (!this.db) throw new Error('Database not initialized');
    const tx = this.db.transaction(storeName, 'readonly');
    const store = tx.objectStore(storeName);
    return new Promise((resolve, reject) => {
      const request = query ? store.getAll(query) : store.getAll();
      request.onsuccess = () => resolve(request.result);
      request.onerror = () => reject(request.error);
    });
  }

  async put(storeName: string, value: any): Promise<IDBValidKey> {
    if (!this.db) throw new Error('Database not initialized');
    const tx = this.db.transaction(storeName, 'readwrite');
    const store = tx.objectStore(storeName);
    return new Promise((resolve, reject) => {
      const request = store.put(value);
      request.onsuccess = () => resolve(request.result);
      request.onerror = () => reject(request.error);
    });
  }

  async delete(storeName: string, key: any): Promise<void> {
    if (!this.db) throw new Error('Database not initialized');
    const tx = this.db.transaction(storeName, 'readwrite');
    const store = tx.objectStore(storeName);
    return new Promise((resolve, reject) => {
      const request = store.delete(key);
      request.onsuccess = () => resolve();
      request.onerror = () => reject(request.error);
    });
  }

  async clear(storeName: string): Promise<void> {
    if (!this.db) throw new Error('Database not initialized');
    const tx = this.db.transaction(storeName, 'readwrite');
    const store = tx.objectStore(storeName);
    return new Promise((resolve, reject) => {
      const request = store.clear();
      request.onsuccess = () => resolve();
      request.onerror = () => reject(request.error);
    });
  }

  getDatabase(): IDBDatabase {
    if (!this.db) throw new Error('Database not initialized');
    return this.db;
  }
}

export default LocalDatabase;

3. React Integration

// useSyncedData.ts
import { useState, useEffect, useCallback } from 'react';
import SyncEngine from './syncEngine';
import LocalDatabase from './database';

interface UseSyncedDataOptions {
  storeName: string;
  syncEngine: SyncEngine;
  db: LocalDatabase;
}

function useSyncedData<T>(options: UseSyncedDataOptions) {
  const { storeName, syncEngine, db } = options;
  const [data, setData] = useState<T[]>([]);
  const [loading, setLoading] = useState(true);
  const [syncStatus, setSyncStatus] = useState(syncEngine.getQueueStatus());

  // Load initial data
  useEffect(() => {
    const loadData = async () => {
      try {
        const items = await db.getAll(storeName);
        setData(items);
      } catch (error) {
        console.error('Failed to load data:', error);
      } finally {
        setLoading(false);
      }
    };

    loadData();
  }, [storeName, db]);

  // Monitor sync status
  useEffect(() => {
    const interval = setInterval(() => {
      setSyncStatus(syncEngine.getQueueStatus());
    }, 1000);

    return () => clearInterval(interval);
  }, [syncEngine]);

  const create = useCallback(
    async (item: Omit<T, 'id'>) => {
      const id = `${Date.now()}-${Math.random()}`;
      const newItem = { ...item, id } as T;

      // Optimistic update
      setData(prev => [...prev, newItem]);

      // Queue sync
      await syncEngine.addOperation({
        type: 'CREATE',
        entity: storeName,
        data: newItem,
      });

      // Persist locally
      await db.put(storeName, newItem);
    },
    [storeName, syncEngine, db]
  );

  const update = useCallback(
    async (id: string, updates: Partial<T>) => {
      const item = await db.get(storeName, id);
      const updated = { ...item, ...updates };

      // Optimistic update
      setData(prev => prev.map(i => (i.id === id ? updated : i)));

      // Queue sync
      await syncEngine.addOperation({
        type: 'UPDATE',
        entity: storeName,
        data: updated,
      });

      // Persist locally
      await db.put(storeName, updated);
    },
    [storeName, syncEngine, db]
  );

  const remove = useCallback(
    async (id: string) => {
      // Optimistic update
      setData(prev => prev.filter(i => i.id !== id));

      // Queue sync
      await syncEngine.addOperation({
        type: 'DELETE',
        entity: storeName,
        data: { id },
      });

      // Remove locally
      await db.delete(storeName, id);
    },
    [storeName, syncEngine, db]
  );

  return {
    data,
    loading,
    syncStatus,
    create,
    update,
    remove,
  };
}

export default useSyncedData;

4. Component Example

```typescript // TodoApp.tsx import React, { useEffect } from 'react'; import useSyncedData from './useSyncedData'; import SyncEngine from './syncEngine'; import LocalDatabase from './database';

interface Todo { id: string; title: string; completed: boolean; createdAt: number; }

let syncEngine: SyncEngine; let db: LocalDatabase;

async function initializeApp() { db = new LocalDatabase(); await db.initialize(); syncEngine = new SyncEngine(db.getDatabase(), { strategy: 'LWW', }); }

export default function TodoApp() { const { data: todos, loading, syncStatus, create, update, remove } = useSyncedData({ storeName: 'todos', syncEngine, db, });

useEffect(() => { initializeApp(); }, []);

const handleAddTodo = async (title: string) => { await create({ title, completed: false, createdAt: Date.now(), }); };

const handleToggleTodo = async (id: string, completed: boolean) => { await update(id, { completed: !completed }); };

const handleDeleteTodo = async (id: string) => { await remove(id); };

if (loading) return

return (