From 78b34939d7101a1bcfc3d6b228c397b9d04a80b0 Mon Sep 17 00:00:00 2001 From: mpcgird Date: Thu, 15 Jan 2026 01:24:39 +0200 Subject: [PATCH 1/8] removed EVENT_REPOSITORY_CLICKHOUSE_ROLLOUT_PERCENT added support for tasks_v1 for logs cleaned the No logs message extended LogsPresenter with BasePresenter for spans --- apps/webapp/app/components/logs/LogsTable.tsx | 16 +-- apps/webapp/app/env.server.ts | 1 - .../v3/LogDetailPresenter.server.ts | 26 +++- .../presenters/v3/LogsListPresenter.server.ts | 116 +++++++++++------- .../route.tsx | 26 +++- ...projectParam.env.$envParam.logs.$logId.tsx | 25 ++-- .../app/v3/eventRepository/index.server.ts | 50 +++++--- internal-packages/clickhouse/src/index.ts | 12 +- .../clickhouse/src/taskEvents.ts | 58 ++++++++- 9 files changed, 233 insertions(+), 97 deletions(-) diff --git a/apps/webapp/app/components/logs/LogsTable.tsx b/apps/webapp/app/components/logs/LogsTable.tsx index 169e65515b..c9e442a824 100644 --- a/apps/webapp/app/components/logs/LogsTable.tsx +++ b/apps/webapp/app/components/logs/LogsTable.tsx @@ -27,7 +27,6 @@ import { PopoverMenuItem } from "~/components/primitives/Popover"; type LogsTableProps = { logs: LogEntry[]; - hasFilters: boolean; searchTerm?: string; isLoading?: boolean; isLoadingMore?: boolean; @@ -59,7 +58,6 @@ function getLevelBorderColor(level: LogEntry["level"]): string { export function LogsTable({ logs, - hasFilters, searchTerm, isLoading = false, isLoadingMore = false, @@ -126,11 +124,7 @@ export function LogsTable({ - {logs.length === 0 && !hasFilters ? ( - - {!isLoading && } - - ) : logs.length === 0 ? ( + {logs.length === 0 ? ( window.location.reload()} /> ) : ( logs.map((log) => { @@ -214,14 +208,6 @@ export function LogsTable({ ); } -function NoLogs({ title }: { title: string }) { - return ( -
- {title} -
- ); -} - function BlankState({ isLoading, onRefresh }: { isLoading?: boolean; onRefresh?: () => void }) { if (isLoading) return ; diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 1b4a96aaf9..82ba812a67 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1220,7 +1220,6 @@ const EnvironmentSchema = z .number() .int() .default(60_000 * 5), // 5 minutes - EVENT_REPOSITORY_CLICKHOUSE_ROLLOUT_PERCENT: z.coerce.number().optional(), EVENT_REPOSITORY_DEFAULT_STORE: z .enum(["postgres", "clickhouse", "clickhouse_v2"]) .default("postgres"), diff --git a/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts b/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts index 5921090d70..37a000653f 100644 --- a/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts @@ -2,6 +2,8 @@ import { type ClickHouse } from "@internal/clickhouse"; import { type PrismaClientOrTransaction } from "@trigger.dev/database"; import { convertClickhouseDateTime64ToJsDate } from "~/v3/eventRepository/clickhouseEventRepository.server"; import { kindToLevel } from "~/utils/logUtils"; +import { getConfiguredEventRepository } from "~/v3/eventRepository/index.server"; +import { ServiceValidationError } from "~/v3/services/baseService.server"; export type LogDetailOptions = { environmentId: string; @@ -24,8 +26,28 @@ export class LogDetailPresenter { public async call(options: LogDetailOptions) { const { environmentId, organizationId, projectId, spanId, traceId, startTime } = options; - // Build ClickHouse query - const queryBuilder = this.clickhouse.taskEventsV2.logDetailQueryBuilder(); + // Determine which store to use based on organization configuration + const { store } = await getConfiguredEventRepository(organizationId); + + // Throw error if postgres is detected + if (store === "postgres") { + throw new ServiceValidationError( + "Log details are not available for PostgreSQL event store. Please contact support." + ); + } + + // Throw error if clickhouse v1 is detected (not supported) + if (store === "postgres") { + throw new ServiceValidationError( + "Log details are not available for postgres event store. Please contact support." + ); + } + + // Build ClickHouse query - only v2 is supported for log details + const isClickhouseV2 = store === "clickhouse_v2"; + const queryBuilder = isClickhouseV2 + ? this.clickhouse.taskEventsV2.logDetailQueryBuilder() + : this.clickhouse.taskEvents.logDetailQueryBuilder(); // Required filters - spanId, traceId, and startTime uniquely identify the log // Multiple events can share the same spanId (span, span events, logs), so startTime is needed diff --git a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts index 0b5f2c175a..cb7cf53287 100644 --- a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts @@ -8,6 +8,7 @@ import { TaskRunStatus as TaskRunStatusEnum, TaskTriggerSource, } from "@trigger.dev/database"; +import { getConfiguredEventRepository } from "~/v3/eventRepository/index.server"; // Create a schema that validates TaskRunStatus enum values const TaskRunStatusSchema = z.array(z.nativeEnum(TaskRunStatusEnum)); @@ -23,6 +24,8 @@ import { convertClickhouseDateTime64ToJsDate, } from "~/v3/eventRepository/clickhouseEventRepository.server"; import { kindToLevel, type LogLevel, LogLevelSchema } from "~/utils/logUtils"; +import { BasePresenter } from "~/presenters/v3/basePresenter.server"; + export type { LogLevel }; @@ -131,9 +134,7 @@ function decodeCursor(cursor: string): LogCursor | null { } // Convert display level to ClickHouse kinds and statuses -function levelToKindsAndStatuses( - level: LogLevel -): { kinds?: string[]; statuses?: string[] } { +function levelToKindsAndStatuses(level: LogLevel): { kinds?: string[]; statuses?: string[] } { switch (level) { case "DEBUG": return { kinds: ["DEBUG_EVENT", "LOG_DEBUG"] }; @@ -150,7 +151,6 @@ function levelToKindsAndStatuses( } } - function convertDateToNanoseconds(date: Date): bigint { return BigInt(date.getTime()) * 1_000_000n; } @@ -168,11 +168,13 @@ function formatNanosecondsForClickhouse(ns: bigint): string { return padded.slice(0, 10) + "." + padded.slice(10); } -export class LogsListPresenter { +export class LogsListPresenter extends BasePresenter { constructor( private readonly replica: PrismaClientOrTransaction, private readonly clickhouse: ClickHouse - ) {} + ) { + super(); + } public async call( organizationId: string, @@ -242,10 +244,7 @@ export class LogsListPresenter { (search !== undefined && search !== "") || !time.isDefault; - const possibleTasksAsync = getAllTaskIdentifiers( - this.replica, - environmentId - ); + const possibleTasksAsync = getAllTaskIdentifiers(this.replica, environmentId); const bulkActionsAsync = this.replica.bulkActionGroup.findMany({ select: { @@ -264,31 +263,26 @@ export class LogsListPresenter { take: 20, }); - const [possibleTasks, bulkActions, displayableEnvironment] = - await Promise.all([ - possibleTasksAsync, - bulkActionsAsync, - findDisplayableEnvironment(environmentId, userId), - ]); - - if ( - bulkId && - !bulkActions.some((bulkAction) => bulkAction.friendlyId === bulkId) - ) { - const selectedBulkAction = - await this.replica.bulkActionGroup.findFirst({ - select: { - friendlyId: true, - type: true, - createdAt: true, - name: true, - }, - where: { - friendlyId: bulkId, - projectId, - environmentId, - }, - }); + const [possibleTasks, bulkActions, displayableEnvironment] = await Promise.all([ + possibleTasksAsync, + bulkActionsAsync, + findDisplayableEnvironment(environmentId, userId), + ]); + + if (bulkId && !bulkActions.some((bulkAction) => bulkAction.friendlyId === bulkId)) { + const selectedBulkAction = await this.replica.bulkActionGroup.findFirst({ + select: { + friendlyId: true, + type: true, + createdAt: true, + name: true, + }, + where: { + friendlyId: bulkId, + projectId, + environmentId, + }, + }); if (selectedBulkAction) { bulkActions.push(selectedBulkAction); @@ -371,7 +365,29 @@ export class LogsListPresenter { } } - const queryBuilder = this.clickhouse.taskEventsV2.logsListQueryBuilder(); + // Determine which store to use based on organization configuration + const { store } = await getConfiguredEventRepository(organizationId); + + // Throw error if postgres is detected + if (store === "postgres") { + throw new ServiceValidationError( + "Logs are not available for PostgreSQL event store. Please contact support." + ); + } + + // Throw error if clickhouse v1 is detected (not supported) + if (store === "postgres") { + throw new ServiceValidationError( + "Logs are not available for Postgres event store. Please contact support." + ); + } + + // Get the appropriate query builder based on store type + const isClickhouseV2 = store === "clickhouse_v2"; + + const queryBuilder = isClickhouseV2 + ? this.clickhouse.taskEventsV2.logsListQueryBuilder() + : this.clickhouse.taskEvents.logsListQueryBuilder(); queryBuilder.prewhere("environment_id = {environmentId: String}", { environmentId, @@ -382,12 +398,17 @@ export class LogsListPresenter { }); queryBuilder.where("project_id = {projectId: String}", { projectId }); - // Time filters - inserted_at in PREWHERE for partition pruning, start_time in WHERE + // Time filters - inserted_at in PREWHERE only for v2, start_time in WHERE for both if (effectiveFrom) { const fromNs = convertDateToNanoseconds(effectiveFrom); - queryBuilder.prewhere("inserted_at >= {insertedAtStart: DateTime64(3)}", { - insertedAtStart: convertDateToClickhouseDateTime(effectiveFrom), - }); + + // Only use inserted_at for partition pruning if v2 + if (isClickhouseV2) { + queryBuilder.prewhere("inserted_at >= {insertedAtStart: DateTime64(3)}", { + insertedAtStart: convertDateToClickhouseDateTime(effectiveFrom), + }); + } + queryBuilder.where("start_time >= {fromTime: String}", { fromTime: formatNanosecondsForClickhouse(fromNs), }); @@ -396,9 +417,14 @@ export class LogsListPresenter { if (effectiveTo) { const clampedTo = effectiveTo > new Date() ? new Date() : effectiveTo; const toNs = convertDateToNanoseconds(clampedTo); - queryBuilder.prewhere("inserted_at <= {insertedAtEnd: DateTime64(3)}", { - insertedAtEnd: convertDateToClickhouseDateTime(clampedTo), - }); + + // Only use inserted_at for partition pruning if v2 + if (isClickhouseV2) { + queryBuilder.prewhere("inserted_at <= {insertedAtEnd: DateTime64(3)}", { + insertedAtEnd: convertDateToClickhouseDateTime(clampedTo), + }); + } + queryBuilder.where("start_time <= {toTime: String}", { toTime: formatNanosecondsForClickhouse(toNs), }); @@ -428,7 +454,6 @@ export class LogsListPresenter { ); } - if (levels && levels.length > 0) { const conditions: string[] = []; const params: Record = {}; @@ -477,7 +502,6 @@ export class LogsListPresenter { queryBuilder.where("NOT (kind = 'SPAN' AND status = 'PARTIAL')"); - // Cursor pagination const decodedCursor = cursor ? decodeCursor(cursor) : null; if (decodedCursor) { @@ -529,7 +553,7 @@ export class LogsListPresenter { try { let attributes = log.attributes as ErrorAttributes; - if (attributes?.error?.message && typeof attributes.error.message === 'string') { + if (attributes?.error?.message && typeof attributes.error.message === "string") { displayMessage = attributes.error.message; } } catch { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx index aad6a2be53..3d392be1e7 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -1,5 +1,6 @@ import { type LoaderFunctionArgs , redirect} from "@remix-run/server-runtime"; import { type MetaFunction, useFetcher, useNavigation, useLocation } from "@remix-run/react"; +import { ServiceValidationError } from "~/v3/services/baseService.server"; import { TypedAwait, typeddefer, @@ -86,7 +87,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const showDebug = url.searchParams.get("showDebug") === "true"; const presenter = new LogsListPresenter($replica, clickhouseClient); - const list = presenter.call(project.organizationId, environment.id, { + + const listPromise = presenter.call(project.organizationId, environment.id, { userId, projectId: project.id, ...filters, @@ -94,6 +96,11 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { levels, includeDebugLogs: isAdmin && showDebug, defaultPeriod: "1h", + }).catch((error) => { + if (error instanceof ServiceValidationError) { + return { error: error.message }; + } + throw error; }); const session = await setRootOnlyFilterPreference(filters.rootOnly, request); @@ -101,7 +108,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { return typeddefer( { - data: list, + data: listPromise, rootOnlyDefault: filters.rootOnly, filters, isAdmin, @@ -149,10 +156,20 @@ export default function Page() { } > - {(list) => { + {(result) => { + // Check if result contains an error + if ("error" in result) { + return ( +
+ + {result.error} + +
+ ); + } return ( { const presenter = new LogDetailPresenter($replica, clickhouseClient); - const result = await presenter.call({ - environmentId: environment.id, - organizationId: project.organizationId, - projectId: project.id, - spanId, - traceId, - startTime, - }); + let result; + try { + result = await presenter.call({ + environmentId: environment.id, + organizationId: project.organizationId, + projectId: project.id, + spanId, + traceId, + startTime, + }); + } catch (error) { + if (error instanceof ServiceValidationError) { + throw new Response(error.message, { status: 400 }); + } + throw error; + } if (!result) { throw new Response("Log not found", { status: 404 }); diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index a8f66dab70..79f54dbaeb 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -24,6 +24,42 @@ export function resolveEventRepositoryForStore(store: string | undefined): IEven return eventRepository; } +export async function getConfiguredEventRepository( + organizationId: string +): Promise<{ repository: IEventRepository; store: string }> { + const organization = await prisma.organization.findFirst({ + select: { + id: true, + featureFlags: true, + }, + where: { + id: organizationId, + }, + }); + + if (!organization) { + throw new Error('Organization not found when configuring event repository'); + } + + // resolveTaskEventRepositoryFlag checks: + // 1. organization.featureFlags (highest priority) + // 2. global feature flags (via flags() function) + // 3. env.EVENT_REPOSITORY_DEFAULT_STORE (fallback) + const taskEventStore = await resolveTaskEventRepositoryFlag( + (organization.featureFlags as Record | null) ?? undefined + ); + + if (taskEventStore === "clickhouse_v2") { + return { repository: clickhouseEventRepositoryV2, store: "clickhouse_v2" }; + } + + if (taskEventStore === "clickhouse") { + return { repository: clickhouseEventRepository, store: "clickhouse" }; + } + + return { repository: eventRepository, store: 'postgres' }; +} + export async function getEventRepository( featureFlags: Record | undefined, parentStore: string | undefined @@ -92,20 +128,6 @@ async function resolveTaskEventRepositoryFlag( return "clickhouse"; } - if (env.EVENT_REPOSITORY_CLICKHOUSE_ROLLOUT_PERCENT) { - const rolloutPercent = env.EVENT_REPOSITORY_CLICKHOUSE_ROLLOUT_PERCENT; - - const randomNumber = Math.random(); - - if (randomNumber < rolloutPercent) { - // Use the default store when rolling out (could be clickhouse or clickhouse_v2) - if (env.EVENT_REPOSITORY_DEFAULT_STORE === "clickhouse_v2") { - return "clickhouse_v2"; - } - return "clickhouse"; - } - } - return flag; } diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index f6a014da5c..4f98cfbca1 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -23,8 +23,10 @@ import { getTraceSummaryQueryBuilderV2, insertTaskEvents, insertTaskEventsV2, - getLogsListQueryBuilder, - getLogDetailQueryBuilder, + getLogsListQueryBuilderV2, + getLogDetailQueryBuilderV2, + getLogsListQueryBuilderV1, + getLogDetailQueryBuilderV1, } from "./taskEvents.js"; import { Logger, type LogLevel } from "@trigger.dev/core/logger"; import type { Agent as HttpAgent } from "http"; @@ -210,6 +212,8 @@ export class ClickHouse { traceSummaryQueryBuilder: getTraceSummaryQueryBuilder(this.reader), traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilder(this.reader), spanDetailsQueryBuilder: getSpanDetailsQueryBuilder(this.reader), + logsListQueryBuilder: getLogsListQueryBuilderV1(this.reader, this.logsQuerySettings?.list), + logDetailQueryBuilder: getLogDetailQueryBuilderV1(this.reader, this.logsQuerySettings?.detail), }; } @@ -219,8 +223,8 @@ export class ClickHouse { traceSummaryQueryBuilder: getTraceSummaryQueryBuilderV2(this.reader), traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilderV2(this.reader), spanDetailsQueryBuilder: getSpanDetailsQueryBuilderV2(this.reader), - logsListQueryBuilder: getLogsListQueryBuilder(this.reader, this.logsQuerySettings?.list), - logDetailQueryBuilder: getLogDetailQueryBuilder(this.reader, this.logsQuerySettings?.detail), + logsListQueryBuilder: getLogsListQueryBuilderV2(this.reader, this.logsQuerySettings?.list), + logDetailQueryBuilder: getLogDetailQueryBuilderV2(this.reader, this.logsQuerySettings?.detail), }; } } diff --git a/internal-packages/clickhouse/src/taskEvents.ts b/internal-packages/clickhouse/src/taskEvents.ts index f526cdf0b6..f8e59f36d3 100644 --- a/internal-packages/clickhouse/src/taskEvents.ts +++ b/internal-packages/clickhouse/src/taskEvents.ts @@ -255,7 +255,7 @@ export const LogsListResult = z.object({ export type LogsListResult = z.output; -export function getLogsListQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) { +export function getLogsListQueryBuilderV2(ch: ClickhouseReader, settings?: ClickHouseSettings) { return ch.queryBuilderFast({ name: "getLogsList", table: "trigger_dev.task_events_v2", @@ -301,7 +301,7 @@ export const LogDetailV2Result = z.object({ export type LogDetailV2Result = z.output; -export function getLogDetailQueryBuilder(ch: ClickhouseReader, settings?: ClickHouseSettings) { +export function getLogDetailQueryBuilderV2(ch: ClickhouseReader, settings?: ClickHouseSettings) { return ch.queryBuilderFast({ name: "getLogDetail", table: "trigger_dev.task_events_v2", @@ -325,3 +325,57 @@ export function getLogDetailQueryBuilder(ch: ClickhouseReader, settings?: ClickH settings, }); } + +// ============================================================================ +// Logs List Query Builders for V1 (task_events_v1) +// ============================================================================ + +export function getLogsListQueryBuilderV1(ch: ClickhouseReader, settings?: ClickHouseSettings) { + return ch.queryBuilderFast({ + name: "getLogsListV1", + table: "trigger_dev.task_events_v1", + columns: [ + "environment_id", + "organization_id", + "project_id", + "task_identifier", + "run_id", + "start_time", + "trace_id", + "span_id", + "parent_span_id", + { name: "message", expression: "LEFT(message, 512)" }, + "kind", + "status", + "duration", + "metadata", + "attributes" + ], + settings, + }); +} + +export function getLogDetailQueryBuilderV1(ch: ClickhouseReader, settings?: ClickHouseSettings) { + return ch.queryBuilderFast({ + name: "getLogDetailV1", + table: "trigger_dev.task_events_v1", + columns: [ + "environment_id", + "organization_id", + "project_id", + "task_identifier", + "run_id", + "start_time", + "trace_id", + "span_id", + "parent_span_id", + "message", + "kind", + "status", + "duration", + "metadata", + "attributes", + ], + settings, + }); +} From 46db1b25e5d0fac0a559155b95f95cd7be70b928 Mon Sep 17 00:00:00 2001 From: mpcgird Date: Thu, 15 Jan 2026 01:41:05 +0200 Subject: [PATCH 2/8] added hasLogsPageAccess featureFlag for logs page --- .../app/components/navigation/SideMenu.tsx | 2 +- .../v3/LogDetailPresenter.server.ts | 7 ---- .../presenters/v3/LogsListPresenter.server.ts | 7 ---- .../route.tsx | 38 ++++++++++++++++++- apps/webapp/app/v3/featureFlags.server.ts | 2 + 5 files changed, 40 insertions(+), 16 deletions(-) diff --git a/apps/webapp/app/components/navigation/SideMenu.tsx b/apps/webapp/app/components/navigation/SideMenu.tsx index 86678bc1ca..76c974d596 100644 --- a/apps/webapp/app/components/navigation/SideMenu.tsx +++ b/apps/webapp/app/components/navigation/SideMenu.tsx @@ -269,7 +269,7 @@ export function SideMenu({ to={v3DeploymentsPath(organization, project, environment)} data-action="deployments" /> - {(isAdmin || user.isImpersonating) && ( + {(user.admin || user.isImpersonating || featureFlags.hasLogsPageAccess) && ( { ]; }; +async function hasLogsPageAccess( + userId: string, + isAdmin: boolean, + isImpersonating: boolean, + organizationSlug: string +): Promise { + if (isAdmin || isImpersonating) { + return true; + } + + // Check organization feature flags + const organization = await prisma.organization.findFirst({ + where: { + slug: organizationSlug, + members: { some: { userId } }, + }, + select: { + featureFlags: true, + }, + }); + + if (!organization?.featureFlags) { + return false; + } + + const flags = organization.featureFlags as Record; + const hasLogsPageAccessResult = validateFeatureFlagValue( + FEATURE_FLAG.hasLogsPageAccess, + flags.hasLogsPageAccess + ); + + return hasLogsPageAccessResult.success && hasLogsPageAccessResult.data === true; +} + + export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); const userId = user.id; diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts index 81dff31ffa..605c11defc 100644 --- a/apps/webapp/app/v3/featureFlags.server.ts +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -6,6 +6,7 @@ export const FEATURE_FLAG = { runsListRepository: "runsListRepository", taskEventRepository: "taskEventRepository", hasQueryAccess: "hasQueryAccess", + hasLogsPageAccess: "hasLogsPageAccess", } as const; const FeatureFlagCatalog = { @@ -13,6 +14,7 @@ const FeatureFlagCatalog = { [FEATURE_FLAG.runsListRepository]: z.enum(["clickhouse", "postgres"]), [FEATURE_FLAG.taskEventRepository]: z.enum(["clickhouse", "clickhouse_v2", "postgres"]), [FEATURE_FLAG.hasQueryAccess]: z.coerce.boolean(), + [FEATURE_FLAG.hasLogsPageAccess]: z.coerce.boolean(), }; type FeatureFlagKey = keyof typeof FeatureFlagCatalog; From 5e920b9657a07ba43766fd2ca5f64ebd5261258e Mon Sep 17 00:00:00 2001 From: mpcgird Date: Thu, 15 Jan 2026 04:18:12 +0200 Subject: [PATCH 3/8] query optimization --- .../app/presenters/v3/LogDetailPresenter.server.ts | 13 ++++--------- .../app/presenters/v3/LogsListPresenter.server.ts | 8 ++++---- .../route.tsx | 2 +- internal-packages/clickhouse/src/taskEvents.ts | 12 ++++++------ 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts b/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts index a975fdec1a..cb193ab186 100644 --- a/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts @@ -84,15 +84,10 @@ export class LogDetailPresenter { } try { - // Handle attributes which could be a JSON object or string - if (log.attributes) { - if (typeof log.attributes === "string") { - parsedAttributes = JSON.parse(log.attributes) as Record; - rawAttributesString = log.attributes; - } else if (typeof log.attributes === "object") { - parsedAttributes = log.attributes as Record; - rawAttributesString = JSON.stringify(log.attributes); - } + // Handle attributes_text which is a string + if (log.attributes_text) { + parsedAttributes = JSON.parse(log.attributes_text) as Record; + rawAttributesString = log.attributes_text; } } catch { // Ignore parse errors diff --git a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts index a8e41f1fb0..a9b09a107f 100644 --- a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts @@ -19,12 +19,12 @@ import { findDisplayableEnvironment } from "~/models/runtimeEnvironment.server"; import { getAllTaskIdentifiers } from "~/models/task.server"; import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; import { ServiceValidationError } from "~/v3/services/baseService.server"; +import { kindToLevel, type LogLevel, LogLevelSchema } from "~/utils/logUtils"; +import { BasePresenter } from "~/presenters/v3/basePresenter.server"; import { convertDateToClickhouseDateTime, convertClickhouseDateTime64ToJsDate, } from "~/v3/eventRepository/clickhouseEventRepository.server"; -import { kindToLevel, type LogLevel, LogLevelSchema } from "~/utils/logUtils"; -import { BasePresenter } from "~/presenters/v3/basePresenter.server"; export type { LogLevel }; @@ -542,9 +542,9 @@ export class LogsListPresenter extends BasePresenter { let displayMessage = log.message; // For error logs with status ERROR, try to extract error message from attributes - if (log.status === "ERROR" && log.attributes) { + if (log.status === "ERROR" && log.attributes_text) { try { - let attributes = log.attributes as ErrorAttributes; + let attributes = JSON.parse(log.attributes_text) as ErrorAttributes; if (attributes?.error?.message && typeof attributes.error.message === "string") { displayMessage = attributes.error.message; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx index 1049b05f33..334f2afcd4 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -227,7 +227,7 @@ function LogsList({ showDebug, defaultPeriod, }: { - list: Awaited["data"]>; + list: Exclude["data"]>, { error: string }>; //exclude error, it is handled rootOnlyDefault: boolean; isAdmin: boolean; showDebug: boolean; diff --git a/internal-packages/clickhouse/src/taskEvents.ts b/internal-packages/clickhouse/src/taskEvents.ts index f8e59f36d3..3e40f13c17 100644 --- a/internal-packages/clickhouse/src/taskEvents.ts +++ b/internal-packages/clickhouse/src/taskEvents.ts @@ -250,7 +250,7 @@ export const LogsListResult = z.object({ status: z.string(), duration: z.number().or(z.string()), metadata: z.string(), - attributes: z.any(), + attributes_text: z.string(), }); export type LogsListResult = z.output; @@ -274,7 +274,7 @@ export function getLogsListQueryBuilderV2(ch: ClickhouseReader, settings?: Click "status", "duration", "metadata", - "attributes" + "attributes_text" ], settings, }); @@ -296,7 +296,7 @@ export const LogDetailV2Result = z.object({ status: z.string(), duration: z.number().or(z.string()), metadata: z.string(), - attributes: z.any() + attributes_text: z.string() }); export type LogDetailV2Result = z.output; @@ -320,7 +320,7 @@ export function getLogDetailQueryBuilderV2(ch: ClickhouseReader, settings?: Clic "status", "duration", "metadata", - "attributes", + "attributes_text", ], settings, }); @@ -349,7 +349,7 @@ export function getLogsListQueryBuilderV1(ch: ClickhouseReader, settings?: Click "status", "duration", "metadata", - "attributes" + "attributes_text" ], settings, }); @@ -374,7 +374,7 @@ export function getLogDetailQueryBuilderV1(ch: ClickhouseReader, settings?: Clic "status", "duration", "metadata", - "attributes", + "attributes_text", ], settings, }); From c55ab1e2346f233c0e946bdd5f55945877f5f9ef Mon Sep 17 00:00:00 2001 From: mpcgird Date: Thu, 15 Jan 2026 04:33:23 +0200 Subject: [PATCH 4/8] query optimization --- internal-packages/clickhouse/src/taskEvents.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal-packages/clickhouse/src/taskEvents.ts b/internal-packages/clickhouse/src/taskEvents.ts index 3e40f13c17..fa64a908dd 100644 --- a/internal-packages/clickhouse/src/taskEvents.ts +++ b/internal-packages/clickhouse/src/taskEvents.ts @@ -249,7 +249,6 @@ export const LogsListResult = z.object({ kind: z.string(), status: z.string(), duration: z.number().or(z.string()), - metadata: z.string(), attributes_text: z.string(), }); @@ -273,7 +272,6 @@ export function getLogsListQueryBuilderV2(ch: ClickhouseReader, settings?: Click "kind", "status", "duration", - "metadata", "attributes_text" ], settings, @@ -295,7 +293,6 @@ export const LogDetailV2Result = z.object({ kind: z.string(), status: z.string(), duration: z.number().or(z.string()), - metadata: z.string(), attributes_text: z.string() }); @@ -319,7 +316,6 @@ export function getLogDetailQueryBuilderV2(ch: ClickhouseReader, settings?: Clic "kind", "status", "duration", - "metadata", "attributes_text", ], settings, @@ -348,7 +344,6 @@ export function getLogsListQueryBuilderV1(ch: ClickhouseReader, settings?: Click "kind", "status", "duration", - "metadata", "attributes_text" ], settings, @@ -373,7 +368,6 @@ export function getLogDetailQueryBuilderV1(ch: ClickhouseReader, settings?: Clic "kind", "status", "duration", - "metadata", "attributes_text", ], settings, From d181640e631e77d68be5222fa745a6b046e30616 Mon Sep 17 00:00:00 2001 From: mpcgird Date: Thu, 15 Jan 2026 12:07:36 +0200 Subject: [PATCH 5/8] review changes --- .../app/presenters/v3/LogDetailPresenter.server.ts | 12 +----------- .../app/presenters/v3/LogsListPresenter.server.ts | 2 +- .../route.tsx | 14 +++++++++++--- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts b/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts index cb193ab186..e5e022bdb5 100644 --- a/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts @@ -70,18 +70,10 @@ export class LogDetailPresenter { const log = records[0]; - // Parse metadata and attributes - let parsedMetadata: Record = {}; + let parsedAttributes: Record = {}; let rawAttributesString = ""; - try { - if (log.metadata) { - parsedMetadata = JSON.parse(log.metadata) as Record; - } - } catch { - // Ignore parse errors - } try { // Handle attributes_text which is a string @@ -107,10 +99,8 @@ export class LogDetailPresenter { status: log.status, duration: typeof log.duration === "number" ? log.duration : Number(log.duration), level: kindToLevel(log.kind, log.status), - metadata: parsedMetadata, attributes: parsedAttributes, // Raw strings for display - rawMetadata: log.metadata, rawAttributes: rawAttributesString, }; } diff --git a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts index a9b09a107f..f45ab1fa79 100644 --- a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts @@ -544,7 +544,7 @@ export class LogsListPresenter extends BasePresenter { // For error logs with status ERROR, try to extract error message from attributes if (log.status === "ERROR" && log.attributes_text) { try { - let attributes = JSON.parse(log.attributes_text) as ErrorAttributes; + const attributes = JSON.parse(log.attributes_text) as ErrorAttributes; if (attributes?.error?.message && typeof attributes.error.message === "string") { displayMessage = attributes.error.message; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx index 334f2afcd4..54903c6a8d 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -98,12 +98,20 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = user.id; const isAdmin = user.admin || user.isImpersonating; - if (!isAdmin) { - throw redirect("/"); - } const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params); + const canAccess = await hasLogsPageAccess( + userId, + user.admin, + user.isImpersonating, + organizationSlug + ); + + if(!canAccess) { + throw redirect("/"); + } + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { throw new Response("Project not found", { status: 404 }); From e2e8dd4d8cd0eb0831150d7d6d70119828222809 Mon Sep 17 00:00:00 2001 From: mpcgird Date: Thu, 15 Jan 2026 12:17:23 +0200 Subject: [PATCH 6/8] fix comment --- apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts | 1 - .../route.tsx | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts b/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts index e5e022bdb5..4272892e65 100644 --- a/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/LogDetailPresenter.server.ts @@ -36,7 +36,6 @@ export class LogDetailPresenter { ); } - // Build ClickHouse query - only v2 is supported for log details const isClickhouseV2 = store === "clickhouse_v2"; const queryBuilder = isClickhouseV2 ? this.clickhouse.taskEventsV2.logDetailQueryBuilder() diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx index 54903c6a8d..b4a614d74a 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -108,7 +108,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { organizationSlug ); - if(!canAccess) { + if (!canAccess) { throw redirect("/"); } From 013da38eb03f88bc60aa3ee646a6189e448eea06 Mon Sep 17 00:00:00 2001 From: mpcgird Date: Thu, 15 Jan 2026 12:40:43 +0200 Subject: [PATCH 7/8] review suggested fixes --- apps/webapp/app/presenters/v3/LogsListPresenter.server.ts | 5 +---- .../route.tsx | 1 - apps/webapp/app/v3/eventRepository/index.server.ts | 4 ++-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts index f45ab1fa79..35ba43117f 100644 --- a/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/LogsListPresenter.server.ts @@ -2,11 +2,9 @@ import { z } from "zod"; import { type ClickHouse, type LogsListResult } from "@internal/clickhouse"; import { MachinePresetName } from "@trigger.dev/core/v3"; import { - type PrismaClient, type PrismaClientOrTransaction, type TaskRunStatus, TaskRunStatus as TaskRunStatusEnum, - TaskTriggerSource, } from "@trigger.dev/database"; import { getConfiguredEventRepository } from "~/v3/eventRepository/index.server"; @@ -26,7 +24,6 @@ import { convertClickhouseDateTime64ToJsDate, } from "~/v3/eventRepository/clickhouseEventRepository.server"; - export type { LogLevel }; type ErrorAttributes = { @@ -173,7 +170,7 @@ export class LogsListPresenter extends BasePresenter { private readonly replica: PrismaClientOrTransaction, private readonly clickhouse: ClickHouse ) { - super(); + super(undefined, replica); } public async call( diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx index b4a614d74a..b1cb9aeaf1 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -92,7 +92,6 @@ async function hasLogsPageAccess( return hasLogsPageAccessResult.success && hasLogsPageAccessResult.data === true; } - export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await requireUser(request); const userId = user.id; diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index 79f54dbaeb..62cc9e6448 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -38,7 +38,7 @@ export async function getConfiguredEventRepository( }); if (!organization) { - throw new Error('Organization not found when configuring event repository'); + throw new Error("Organization not found when configuring event repository"); } // resolveTaskEventRepositoryFlag checks: @@ -57,7 +57,7 @@ export async function getConfiguredEventRepository( return { repository: clickhouseEventRepository, store: "clickhouse" }; } - return { repository: eventRepository, store: 'postgres' }; + return { repository: eventRepository, store: "postgres" }; } export async function getEventRepository( From 764e88bb1dc0d9c6304d7b74a3907606b656e84d Mon Sep 17 00:00:00 2001 From: mpcgird Date: Thu, 15 Jan 2026 13:03:16 +0200 Subject: [PATCH 8/8] review suggested fixes --- .../route.tsx | 2 +- .../app/v3/eventRepository/index.server.ts | 22 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx index b1cb9aeaf1..44fbd437f5 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -141,7 +141,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { defaultPeriod: "1h", }).catch((error) => { if (error instanceof ServiceValidationError) { - return { error: error.message }; + return { error: "Failed to load logs. Please refresh and try again." }; } throw error; }); diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index 62cc9e6448..cb211e2b02 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -5,7 +5,7 @@ import { clickhouseEventRepositoryV2, } from "./clickhouseEventRepositoryInstance.server"; import { IEventRepository, TraceEventOptions } from "./eventRepository.types"; -import { $replica, prisma } from "~/db.server"; +import { prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; import { FEATURE_FLAG, flags } from "../featureFlags.server"; import { getTaskEventStore } from "../taskEventStore.server"; @@ -24,9 +24,17 @@ export function resolveEventRepositoryForStore(store: string | undefined): IEven return eventRepository; } + export const EVENT_STORE_TYPES = { + POSTGRES: "postgres", + CLICKHOUSE: "clickhouse", + CLICKHOUSE_V2: "clickhouse_v2", + } as const; + +export type EventStoreType = typeof EVENT_STORE_TYPES[keyof typeof EVENT_STORE_TYPES]; + export async function getConfiguredEventRepository( organizationId: string -): Promise<{ repository: IEventRepository; store: string }> { +): Promise<{ repository: IEventRepository; store: EventStoreType }> { const organization = await prisma.organization.findFirst({ select: { id: true, @@ -49,15 +57,15 @@ export async function getConfiguredEventRepository( (organization.featureFlags as Record | null) ?? undefined ); - if (taskEventStore === "clickhouse_v2") { - return { repository: clickhouseEventRepositoryV2, store: "clickhouse_v2" }; + if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE_V2) { + return { repository: clickhouseEventRepositoryV2, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 }; } - if (taskEventStore === "clickhouse") { - return { repository: clickhouseEventRepository, store: "clickhouse" }; + if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE) { + return { repository: clickhouseEventRepository, store: EVENT_STORE_TYPES.CLICKHOUSE }; } - return { repository: eventRepository, store: "postgres" }; + return { repository: eventRepository, store: EVENT_STORE_TYPES.POSTGRES }; } export async function getEventRepository(