Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/appkit-ui/src/react/charts/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import type { Table } from "apache-arrow";
// ============================================================================

/** Supported data formats for analytics queries */
export type DataFormat = "json" | "arrow" | "auto";
export type DataFormat = "json" | "arrow" | "arrow_stream" | "auto";
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in theory arrow is the same as arrow_stream, so I'm not following what's the problem?


/** Chart orientation */
export type Orientation = "vertical" | "horizontal";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ describe("useChartData", () => {
);
});

test("auto-selects JSON by default when no heuristics match", () => {
test("auto-selects ARROW_STREAM by default when no heuristics match", () => {
mockUseAnalyticsQuery.mockReturnValue({
data: [],
loading: false,
Expand All @@ -223,11 +223,11 @@ describe("useChartData", () => {
expect(mockUseAnalyticsQuery).toHaveBeenCalledWith(
"test",
{ limit: 100 },
expect.objectContaining({ format: "JSON" }),
expect.objectContaining({ format: "ARROW_STREAM" }),
);
});

test("defaults to auto format (JSON) when format is not specified", () => {
test("defaults to auto format (ARROW_STREAM) when format is not specified", () => {
mockUseAnalyticsQuery.mockReturnValue({
data: [],
loading: false,
Expand All @@ -243,7 +243,7 @@ describe("useChartData", () => {
expect(mockUseAnalyticsQuery).toHaveBeenCalledWith(
"test",
undefined,
expect.objectContaining({ format: "JSON" }),
expect.objectContaining({ format: "ARROW_STREAM" }),
);
});
});
Expand Down
8 changes: 5 additions & 3 deletions packages/appkit-ui/src/react/hooks/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import type { Table } from "apache-arrow";
// ============================================================================

/** Supported response formats for analytics queries */
export type AnalyticsFormat = "JSON" | "ARROW";
export type AnalyticsFormat = "JSON" | "ARROW" | "ARROW_STREAM";

/**
* Typed Arrow Table - preserves row type information for type inference.
Expand All @@ -32,8 +32,10 @@ export interface TypedArrowTable<
// ============================================================================

/** Options for configuring an analytics SSE query */
export interface UseAnalyticsQueryOptions<F extends AnalyticsFormat = "JSON"> {
/** Response format - "JSON" returns typed arrays, "ARROW" returns TypedArrowTable */
export interface UseAnalyticsQueryOptions<
F extends AnalyticsFormat = "ARROW_STREAM",
> {
/** Response format - "ARROW_STREAM" (default) uses inline Arrow, "JSON" returns typed arrays, "ARROW" uses external links */
format?: F;

/** Maximum size of serialized parameters in bytes */
Expand Down
4 changes: 2 additions & 2 deletions packages/appkit-ui/src/react/hooks/use-analytics-query.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,13 @@ function getArrowStreamUrl(id: string) {
export function useAnalyticsQuery<
T = unknown,
K extends QueryKey = QueryKey,
F extends AnalyticsFormat = "JSON",
F extends AnalyticsFormat = "ARROW_STREAM",
>(
queryKey: K,
parameters?: InferParams<K> | null,
options: UseAnalyticsQueryOptions<F> = {} as UseAnalyticsQueryOptions<F>,
): UseAnalyticsQueryResult<InferResultByFormat<T, K, F>> {
const format = options?.format ?? "JSON";
const format = options?.format ?? "ARROW_STREAM";
const maxParametersSize = options?.maxParametersSize ?? 100 * 1024;
const autoStart = options?.autoStart ?? true;

Expand Down
7 changes: 4 additions & 3 deletions packages/appkit-ui/src/react/hooks/use-chart-data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,11 @@ export interface UseChartDataResult {
function resolveFormat(
format: DataFormat,
parameters?: Record<string, unknown>,
): "JSON" | "ARROW" {
): "JSON" | "ARROW" | "ARROW_STREAM" {
// Explicit format selection
if (format === "json") return "JSON";
if (format === "arrow") return "ARROW";
if (format === "arrow_stream") return "ARROW_STREAM";

// Auto-selection heuristics
if (format === "auto") {
Expand All @@ -72,10 +73,10 @@ function resolveFormat(
return "ARROW";
}

return "JSON";
return "ARROW_STREAM";
}

return "JSON";
return "ARROW_STREAM";
}

// ============================================================================
Expand Down
2 changes: 1 addition & 1 deletion packages/appkit/src/connectors/sql-warehouse/defaults.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ interface ExecuteStatementDefaults {
export const executeStatementDefaults: ExecuteStatementDefaults = {
wait_timeout: "30s",
disposition: "INLINE",
format: "JSON_ARRAY",
format: "ARROW_STREAM",
on_wait_timeout: "CONTINUE",
timeout: 60000,
};
120 changes: 102 additions & 18 deletions packages/appkit/src/plugins/analytics/analytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import { queryDefaults } from "./defaults";
import manifest from "./manifest.json";
import { QueryProcessor } from "./query";
import type {
AnalyticsFormat,
AnalyticsQueryResponse,
IAnalyticsConfig,
IAnalyticsQueryRequest,
Expand Down Expand Up @@ -119,7 +120,8 @@ export class AnalyticsPlugin extends Plugin {
res: express.Response,
): Promise<void> {
const { query_key } = req.params;
const { parameters, format = "JSON" } = req.body as IAnalyticsQueryRequest;
const { parameters, format = "ARROW_STREAM" } =
req.body as IAnalyticsQueryRequest;

// Request-scoped logging with WideEvent tracking
logger.debug(req, "Executing query: %s (format=%s)", query_key, format);
Expand Down Expand Up @@ -155,19 +157,6 @@ export class AnalyticsPlugin extends Plugin {
const userKey = getCurrentUserId();
const executorKey = isAsUser ? userKey : "global";

const queryParameters =
format === "ARROW"
? {
formatParameters: {
disposition: "EXTERNAL_LINKS",
format: "ARROW_STREAM",
},
type: "arrow",
}
: {
type: "result",
};

const hashedQuery = this.queryProcessor.hashQuery(query);

const defaultConfig: PluginExecuteConfig = {
Expand Down Expand Up @@ -197,20 +186,115 @@ export class AnalyticsPlugin extends Plugin {
parameters,
);

const result = await executor.query(
return this._executeWithFormatFallback(
executor,
query,
processedParams,
queryParameters.formatParameters,
format,
signal,
);

return { type: queryParameters.type, ...result };
},
streamExecutionSettings,
executorKey,
);
}

/** Format configurations in fallback order. */
private static readonly FORMAT_CONFIGS = {
ARROW_STREAM: {
formatParameters: { disposition: "INLINE", format: "ARROW_STREAM" },
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

from this URL
https://docs.databricks.com/api/workspace/statementexecution/executestatement#format

Important: The formats ARROW_STREAM and CSV are supported only with EXTERNAL_LINKS disposition. JSON_ARRAY is supported in INLINE and EXTERNAL_LINKS disposition.

so before changing anything this was already supporting arrow, can I know what's the case where this was failing? I would like to see it

type: "result" as const,
},
JSON: {
formatParameters: undefined,
type: "result" as const,
},
ARROW: {
formatParameters: {
disposition: "EXTERNAL_LINKS",
format: "ARROW_STREAM",
},
type: "arrow" as const,
},
};

/**
* Execute a query with automatic format fallback.
*
* For the default ARROW_STREAM format, tries formats in order until one
* succeeds: ARROW_STREAM → JSON → ARROW. This handles warehouses that
* only support a subset of format/disposition combinations.
*
* Explicit format requests (JSON, ARROW) are not retried.
*/
private async _executeWithFormatFallback(
executor: AnalyticsPlugin,
query: string,
processedParams:
| Record<string, SQLTypeMarker | null | undefined>
| undefined,
requestedFormat: AnalyticsFormat,
signal?: AbortSignal,
): Promise<{ type: string; [key: string]: any }> {
// Explicit format — no fallback.
if (requestedFormat === "JSON" || requestedFormat === "ARROW") {
const config = AnalyticsPlugin.FORMAT_CONFIGS[requestedFormat];
const result = await executor.query(
query,
processedParams,
config.formatParameters,
signal,
);
return { type: config.type, ...result };
}

// Default (ARROW_STREAM) — try each format in order.
const fallbackOrder: AnalyticsFormat[] = ["ARROW_STREAM", "JSON", "ARROW"];

for (let i = 0; i < fallbackOrder.length; i++) {
const fmt = fallbackOrder[i];
const config = AnalyticsPlugin.FORMAT_CONFIGS[fmt];
try {
const result = await executor.query(
query,
processedParams,
config.formatParameters,
signal,
);
if (i > 0) {
logger.info(
"Query succeeded with fallback format %s (preferred %s was rejected)",
fmt,
fallbackOrder[0],
);
}
return { type: config.type, ...result };
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
const isFormatError =
msg.includes("ARROW_STREAM") ||
msg.includes("JSON_ARRAY") ||
msg.includes("EXTERNAL_LINKS") ||
msg.includes("INVALID_PARAMETER_VALUE") ||
msg.includes("NOT_IMPLEMENTED");

if (!isFormatError || i === fallbackOrder.length - 1) {
throw err;
}

logger.warn(
"Format %s rejected by warehouse, falling back to %s: %s",
fmt,
fallbackOrder[i + 1],
msg,
);
}
}

// Unreachable — last format in fallbackOrder throws on failure.
throw new Error("All format fallbacks exhausted");
}

/**
* Execute a SQL query using the current execution context.
*
Expand Down
Loading
Loading