feat(knowledge): add Live sync option to KB connectors + fix embedding billing

apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts

@@ -13,6 +13,7 @@ import { z } from 'zod'
 import { decryptApiKey } from '@/lib/api-key/crypto'
 import { AuditAction, AuditResourceType, recordAudit } from '@/lib/audit/log'
 import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
+import { hasLiveSyncAccess } from '@/lib/billing/core/subscription'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service'
 import { cleanupUnusedTagDefinitions } from '@/lib/knowledge/tags/service'
@@ -116,6 +117,20 @@ export async function PATCH(request: NextRequest, { params }: RouteParams) {
       )
     }
 
+    if (
+      parsed.data.syncIntervalMinutes !== undefined &&
+      parsed.data.syncIntervalMinutes > 0 &&
+      parsed.data.syncIntervalMinutes < 60
+    ) {
+      const canUseLiveSync = await hasLiveSyncAccess(auth.userId)
+      if (!canUseLiveSync) {
+        return NextResponse.json(
+          { error: 'Live sync requires a Max or Enterprise plan' },
+          { status: 403 }
+        )
+      }
+    }
+
     if (parsed.data.sourceConfig !== undefined) {
       const existingRows = await db
         .select()

apps/sim/app/api/knowledge/[id]/connectors/route.ts

@@ -7,6 +7,7 @@ import { z } from 'zod'
 import { encryptApiKey } from '@/lib/api-key/crypto'
 import { AuditAction, AuditResourceType, recordAudit } from '@/lib/audit/log'
 import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
+import { hasLiveSyncAccess } from '@/lib/billing/core/subscription'
 import { generateRequestId } from '@/lib/core/utils/request'
 import { dispatchSync } from '@/lib/knowledge/connectors/sync-engine'
 import { allocateTagSlots } from '@/lib/knowledge/constants'
@@ -97,6 +98,16 @@ export async function POST(request: NextRequest, { params }: { params: Promise<{
 
     const { connectorType, credentialId, apiKey, sourceConfig, syncIntervalMinutes } = parsed.data
 
+    if (syncIntervalMinutes > 0 && syncIntervalMinutes < 60) {
+      const canUseLiveSync = await hasLiveSyncAccess(auth.userId)
+      if (!canUseLiveSync) {
+        return NextResponse.json(
+          { error: 'Live sync requires a Max or Enterprise plan' },
+          { status: 403 }
+        )
+      }
+    }
+
     const connectorConfig = CONNECTOR_REGISTRY[connectorType]
     if (!connectorConfig) {
       return NextResponse.json(

apps/sim/app/api/knowledge/search/utils.test.ts

@@ -5,6 +5,7 @@
  * @vitest-environment node
  */
 import { createEnvMock, databaseMock, loggerMock } from '@sim/testing'
+import { mockNextFetchResponse } from '@sim/testing/mocks'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 vi.mock('drizzle-orm')
@@ -14,16 +15,6 @@ vi.mock('@/lib/knowledge/documents/utils', () => ({
   retryWithExponentialBackoff: (fn: any) => fn(),
 }))
 
-vi.stubGlobal(
-  'fetch',
-  vi.fn().mockResolvedValue({
-    ok: true,
-    json: async () => ({
-      data: [{ embedding: [0.1, 0.2, 0.3] }],
-    }),
-  })
-)
-
 vi.mock('@/lib/core/config/env', () => createEnvMock())
 
 import {
@@ -178,17 +169,16 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       const result = await generateSearchEmbedding('test query')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         'https://test.openai.azure.com/openai/deployments/text-embedding-ada-002/embeddings?api-version=2024-12-01-preview',
         expect.objectContaining({
           headers: expect.objectContaining({
@@ -209,17 +199,16 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       const result = await generateSearchEmbedding('test query')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         'https://api.openai.com/v1/embeddings',
         expect.objectContaining({
           headers: expect.objectContaining({
@@ -243,17 +232,16 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       await generateSearchEmbedding('test query')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         expect.stringContaining('api-version='),
         expect.any(Object)
       )
@@ -273,17 +261,16 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       await generateSearchEmbedding('test query', 'text-embedding-3-small')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         'https://test.openai.azure.com/openai/deployments/custom-embedding-model/embeddings?api-version=2024-12-01-preview',
         expect.any(Object)
       )
@@ -311,13 +298,12 @@ describe('Knowledge Search Utils', () => {
         KB_OPENAI_MODEL_NAME: 'text-embedding-ada-002',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
+      mockNextFetchResponse({
         ok: false,
         status: 404,
         statusText: 'Not Found',
-        text: async () => 'Deployment not found',
-      } as any)
+        text: 'Deployment not found',
+      })
 
       await expect(generateSearchEmbedding('test query')).rejects.toThrow('Embedding API failed')
 
@@ -332,13 +318,12 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
+      mockNextFetchResponse({
         ok: false,
         status: 429,
         statusText: 'Too Many Requests',
-        text: async () => 'Rate limit exceeded',
-      } as any)
+        text: 'Rate limit exceeded',
+      })
 
       await expect(generateSearchEmbedding('test query')).rejects.toThrow('Embedding API failed')
 
@@ -356,17 +341,16 @@ describe('Knowledge Search Utils', () => {
         KB_OPENAI_MODEL_NAME: 'text-embedding-ada-002',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       await generateSearchEmbedding('test query')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         expect.any(String),
         expect.objectContaining({
           body: JSON.stringify({
@@ -387,17 +371,16 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
+      mockNextFetchResponse({
+        json: {
           data: [{ embedding: [0.1, 0.2, 0.3] }],
-        }),
-      } as any)
+          usage: { prompt_tokens: 1, total_tokens: 1 },
+        },
+      })
 
       await generateSearchEmbedding('test query', 'text-embedding-3-small')
 
-      expect(fetchSpy).toHaveBeenCalledWith(
+      expect(vi.mocked(fetch)).toHaveBeenCalledWith(
         expect.any(String),
         expect.objectContaining({
           body: JSON.stringify({

apps/sim/app/api/knowledge/utils.test.ts

@@ -77,6 +77,7 @@ vi.stubGlobal(
         { embedding: [0.1, 0.2], index: 0 },
         { embedding: [0.3, 0.4], index: 1 },
       ],
+      usage: { prompt_tokens: 2, total_tokens: 2 },
     }),
   })
 )
@@ -294,7 +295,7 @@ describe('Knowledge Utils', () => {
     it.concurrent('should return same length as input', async () => {
       const result = await generateEmbeddings(['a', 'b'])
 
-      expect(result.length).toBe(2)
+      expect(result.embeddings.length).toBe(2)
     })
 
     it('should use Azure OpenAI when Azure config is provided', async () => {
@@ -313,6 +314,7 @@ describe('Knowledge Utils', () => {
         ok: true,
         json: async () => ({
           data: [{ embedding: [0.1, 0.2], index: 0 }],
+          usage: { prompt_tokens: 1, total_tokens: 1 },
         }),
       } as any)
 
@@ -342,6 +344,7 @@ describe('Knowledge Utils', () => {
         ok: true,
         json: async () => ({
           data: [{ embedding: [0.1, 0.2], index: 0 }],
+          usage: { prompt_tokens: 1, total_tokens: 1 },
         }),
       } as any)
 

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx

@@ -19,26 +19,23 @@ import {
   ModalHeader,
   Tooltip,
 } from '@/components/emcn'
+import { getSubscriptionAccessState } from '@/lib/billing/client'
 import { consumeOAuthReturnContext } from '@/lib/credentials/client-state'
 import { getProviderIdFromServiceId, type OAuthProvider } from '@/lib/oauth'
 import { OAuthModal } from '@/app/workspace/[workspaceId]/components/oauth-modal'
 import { ConnectorSelectorField } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/components/connector-selector-field'
+import { SYNC_INTERVALS } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts'
+import { MaxBadge } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/max-badge'
+import { isBillingEnabled } from '@/app/workspace/[workspaceId]/settings/navigation'
 import { getDependsOnFields } from '@/blocks/utils'
 import { CONNECTOR_REGISTRY } from '@/connectors/registry'
 import type { ConnectorConfig, ConnectorConfigField } from '@/connectors/types'
 import { useCreateConnector } from '@/hooks/queries/kb/connectors'
 import { useOAuthCredentials } from '@/hooks/queries/oauth/oauth-credentials'
+import { useSubscriptionData } from '@/hooks/queries/subscription'
 import type { SelectorKey } from '@/hooks/selectors/types'
 import { useCredentialRefreshTriggers } from '@/hooks/use-credential-refresh-triggers'
 
-const SYNC_INTERVALS = [
-  { label: 'Every hour', value: 60 },
-  { label: 'Every 6 hours', value: 360 },
-  { label: 'Daily', value: 1440 },
-  { label: 'Weekly', value: 10080 },
-  { label: 'Manual only', value: 0 },
-] as const
-
 const CONNECTOR_ENTRIES = Object.entries(CONNECTOR_REGISTRY)
 
 interface AddConnectorModalProps {
@@ -67,6 +64,10 @@ export function AddConnectorModal({ open, onOpenChange, knowledgeBaseId }: AddCo
   const { workspaceId } = useParams<{ workspaceId: string }>()
   const { mutate: createConnector, isPending: isCreating } = useCreateConnector()
 
+  const { data: subscriptionResponse } = useSubscriptionData({ enabled: isBillingEnabled })
+  const subscriptionAccess = getSubscriptionAccessState(subscriptionResponse?.data)
+  const hasMaxAccess = !isBillingEnabled || subscriptionAccess.hasUsableMaxAccess
+
   const connectorConfig = selectedType ? CONNECTOR_REGISTRY[selectedType] : null
   const isApiKeyMode = connectorConfig?.auth.mode === 'apiKey'
   const connectorProviderId = useMemo(
@@ -516,8 +517,13 @@ export function AddConnectorModal({ open, onOpenChange, knowledgeBaseId }: AddCo
                     onValueChange={(val) => setSyncInterval(Number(val))}
                   >
                     {SYNC_INTERVALS.map((interval) => (
-                      <ButtonGroupItem key={interval.value} value={String(interval.value)}>
+                      <ButtonGroupItem
+                        key={interval.value}
+                        value={String(interval.value)}
+                        disabled={interval.requiresMax && !hasMaxAccess}
+                      >
                         {interval.label}
+                        {interval.requiresMax && !hasMaxAccess && <MaxBadge />}
                       </ButtonGroupItem>
                     ))}
                   </ButtonGroup>

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/consts.ts

@@ -0,0 +1,8 @@
+export const SYNC_INTERVALS = [
+  { label: 'Live', value: 5, requiresMax: true },
+  { label: 'Every hour', value: 60, requiresMax: false },
+  { label: 'Every 6 hours', value: 360, requiresMax: false },
+  { label: 'Daily', value: 1440, requiresMax: false },
+  { label: 'Weekly', value: 10080, requiresMax: false },
+  { label: 'Manual only', value: 0, requiresMax: false },
+] as const