diff --git a/clickhouse/src/queries/click-house-log-query/ClickHouseLogQuery.tsx b/clickhouse/src/queries/click-house-log-query/ClickHouseLogQuery.tsx index b1235da44..8d4d1df06 100644 --- a/clickhouse/src/queries/click-house-log-query/ClickHouseLogQuery.tsx +++ b/clickhouse/src/queries/click-house-log-query/ClickHouseLogQuery.tsx @@ -11,11 +11,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { parseVariables } from '@perses-dev/plugin-system'; +import { parseVariables, LogQueryPlugin } from '@perses-dev/plugin-system'; import { getClickHouseLogData } from './get-click-house-log-data'; import { ClickHouseLogQueryEditor } from './ClickHouseLogQueryEditor'; import { ClickHouseLogQuerySpec } from './click-house-log-query-types'; -import { LogQueryPlugin } from './log-query-plugin-interface'; export const ClickHouseLogQuery: LogQueryPlugin = { getLogData: getClickHouseLogData, @@ -28,4 +27,6 @@ export const ClickHouseLogQuery: LogQueryPlugin = { variables: allVariables, }; }, + // TODO: Implement proper SQL parsing for volume queries + createVolumeQuery: () => null, }; diff --git a/clickhouse/src/queries/click-house-log-query/click-house-log-query-types.test.ts b/clickhouse/src/queries/click-house-log-query/click-house-log-query-types.test.ts index 65b8f8aa7..8c31875d8 100644 --- a/clickhouse/src/queries/click-house-log-query/click-house-log-query-types.test.ts +++ b/clickhouse/src/queries/click-house-log-query/click-house-log-query-types.test.ts @@ -11,9 +11,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +import { LogQueryContext } from '@perses-dev/plugin-system'; import { ClickHouseDatasource, ClickHouseDatasourceSpec } from '../../datasources/click-house-datasource'; import { ClickHouseQueryResponse } from '../../model/click-house-client'; -import { ClickHouseQueryContext } from './log-query-plugin-interface'; import { ClickHouseLogQuery } from './ClickHouseLogQuery'; const datasource: ClickHouseDatasourceSpec = { @@ -35,8 +35,8 @@ const getDatasourceClient: jest.Mock = jest.fn(() => { return clickhouseStubClient; }); -const createStubContext = (): ClickHouseQueryContext => { - const stubLogContext: ClickHouseQueryContext = { +const createStubContext = (): LogQueryContext => { + const stubLogContext: LogQueryContext = { datasourceStore: { getDatasource: jest.fn(), getDatasourceClient: getDatasourceClient, @@ -51,6 +51,7 @@ const createStubContext = (): ClickHouseQueryContext => { start: new Date('01-02-2025'), }, variableState: {}, + refreshKey: '', }; return stubLogContext; }; diff --git a/clickhouse/src/queries/click-house-log-query/get-click-house-log-data.ts b/clickhouse/src/queries/click-house-log-query/get-click-house-log-data.ts index 5ea71ec56..1edda577a 100644 --- a/clickhouse/src/queries/click-house-log-query/get-click-house-log-data.ts +++ b/clickhouse/src/queries/click-house-log-query/get-click-house-log-data.ts @@ -11,12 +11,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { replaceVariables } from '@perses-dev/plugin-system'; +import { replaceVariables, LogQueryPlugin } from '@perses-dev/plugin-system'; import { LogEntry, LogData } from '@perses-dev/core'; import { ClickHouseClient, ClickHouseQueryResponse } from '../../model/click-house-client'; import { DEFAULT_DATASOURCE } from '../constants'; import { ClickHouseLogQuerySpec } from './click-house-log-query-types'; -import { LogQueryPlugin } from './log-query-plugin-interface'; function flattenObject( obj: Record, diff --git a/clickhouse/src/queries/click-house-log-query/log-query-plugin-interface.ts b/clickhouse/src/queries/click-house-log-query/log-query-plugin-interface.ts deleted file mode 100644 index 3ae031699..000000000 --- a/clickhouse/src/queries/click-house-log-query/log-query-plugin-interface.ts +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright The Perses Authors -// Licensed under the Apache License, Version 2.0 (the \"License\"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an \"AS IS\" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import { AbsoluteTimeRange, UnknownSpec, LogData } from '@perses-dev/core'; -import { DatasourceStore, Plugin, VariableStateMap } from '@perses-dev/plugin-system'; - -export interface LogQueryResult { - logs: LogData; - timeRange: AbsoluteTimeRange; - metadata?: { - executedQueryString: string; - }; -} - -export interface ClickHouseQueryContext { - timeRange: AbsoluteTimeRange; - variableState: VariableStateMap; - datasourceStore: DatasourceStore; -} - -type LogQueryPluginDependencies = { - variables?: string[]; -}; - -export interface LogQueryPlugin extends Plugin { - getLogData: (spec: Spec, ctx: ClickHouseQueryContext) => Promise; - dependsOn?: (spec: Spec, ctx: ClickHouseQueryContext) => LogQueryPluginDependencies; -} diff --git a/logexplorer/.cjs.swcrc b/logexplorer/.cjs.swcrc new file mode 100644 index 000000000..2ed65083d --- /dev/null +++ b/logexplorer/.cjs.swcrc @@ -0,0 +1,20 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "jsc": { + "parser": { + "syntax": "typescript", + "tsx": true + }, + "target": "es2022", + "transform": { + "react": { + "runtime": "automatic", + "useBuiltins": true + } + } + }, + "module": { + "type": "commonjs" + }, + "exclude": ["\\.(stories|test)\\."] +} diff --git a/logexplorer/.gitignore b/logexplorer/.gitignore new file mode 100644 index 000000000..fe8baa8ba --- /dev/null +++ b/logexplorer/.gitignore @@ -0,0 +1,21 @@ +.idea/ + +# Local +.DS_Store +*.local +*.log* + +# Dist +node_modules +dist/ + +# IDE +.vscode/* +!.vscode/extensions.json +.idea + +# generated archives +*.tar.gz + +# external CUE dependencies +/*/cue.mod/pkg/ diff --git a/logexplorer/.swcrc b/logexplorer/.swcrc new file mode 100644 index 000000000..feaf67637 --- /dev/null +++ b/logexplorer/.swcrc @@ -0,0 +1,21 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "jsc": { + "parser": { + "syntax": "typescript", + "tsx": true + }, + "target": "es2022", + "transform": { + "react": { + "runtime": "automatic", + "useBuiltins": true + } + } + }, + "module": { + "type": "es6" + }, + "sourceMaps": true, + "exclude": ["\\.(stories|test)\\."] +} diff --git a/logexplorer/README.md b/logexplorer/README.md new file mode 100644 index 000000000..58090454c --- /dev/null +++ b/logexplorer/README.md @@ -0,0 +1,41 @@ +# Plugin Module: log-explorer + +### How to install + +This plugin requires react and react-dom 18 + +Install peer dependencies: + +```bash +npm install react@18 react-dom@18 +``` + +Install the plugin: + +```bash +npm install @my-org/log-explorer +``` + +## Development + +### Setup + +Install dependencies: + +```bash +npm install +``` + +### Get Started + +Start the dev server: + +```bash +npm run dev +``` + +Build the plugin for distribution: + +```bash +npm run build +``` diff --git a/logexplorer/cue.mod/module.cue b/logexplorer/cue.mod/module.cue new file mode 100644 index 000000000..5ae153349 --- /dev/null +++ b/logexplorer/cue.mod/module.cue @@ -0,0 +1,7 @@ +module: "github.com/perses-dev/log-explorer@v0" +language: { + version: "v0.12.0" +} +source: { + kind: "git" +} diff --git a/logexplorer/jest.config.ts b/logexplorer/jest.config.ts new file mode 100644 index 000000000..13276c6cd --- /dev/null +++ b/logexplorer/jest.config.ts @@ -0,0 +1,26 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { Config } from "@jest/types"; +import shared from "../jest.shared"; + +const jestConfig: Config.InitialOptions = { + ...shared, + + setupFilesAfterEnv: [ + ...(shared.setupFilesAfterEnv ?? []), + "/src/setup-tests.ts", + ], +}; + +export default jestConfig; diff --git a/logexplorer/package.json b/logexplorer/package.json new file mode 100644 index 000000000..84c35f7e7 --- /dev/null +++ b/logexplorer/package.json @@ -0,0 +1,71 @@ +{ + "name": "@perses-dev/log-explorer", + "version": "0.1.0", + "homepage": "https://github.com/perses/plugins/blob/main/README.md", + "repository": { + "type": "git", + "url": "git+https://github.com/perses/plugins.git" + }, + "bugs": { + "url": "https://github.com/perses/plugins/issues" + }, + "scripts": { + "dev": "rsbuild dev", + "build": "npm run build-mf && concurrently \"npm:build:*\"", + "build-mf": "rsbuild build", + "build:cjs": "swc ./src -d dist/lib/cjs --strip-leading-paths --config-file .cjs.swcrc", + "build:esm": "swc ./src -d dist/lib --strip-leading-paths --config-file .swcrc", + "build:types": "tsc --project tsconfig.build.json", + "lint": "eslint src --ext .ts,.tsx", + "test": "cross-env LC_ALL=C TZ=UTC jest --passWithNoTests", + "type-check": "tsc --noEmit" + }, + "main": "lib/cjs/index.js", + "module": "lib/index.js", + "types": "lib/index.d.ts", + "devDependencies": { + "@types/qs": "^6.9.18" + }, + "peerDependencies": { + "@emotion/react": "^11.7.1", + "@emotion/styled": "^11.6.0", + "@hookform/resolvers": "^3.2.0", + "@perses-dev/components": "^0.54.0-beta.1", + "@perses-dev/core": "^0.53.0", + "@perses-dev/dashboards": "^0.54.0-beta.1", + "@perses-dev/explore": "^0.54.0-beta.1", + "@perses-dev/plugin-system": "^0.54.0-beta.1", + "@tanstack/react-query": "^4.39.1", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", + "echarts": "5.5.0", + "immer": "^10.1.1", + "lodash": "^4.17.21", + "react": "^17.0.2 || ^18.0.0", + "react-dom": "^17.0.2 || ^18.0.0", + "react-hook-form": "^7.52.2", + "react-router-dom": "^5 || ^6 || ^7" + }, + "files": [ + "lib/**/*", + "__mf/**/*", + "mf-manifest.json", + "mf-stats.json" + ], + "perses": { + "moduleName": "LogExplorer", + "moduleOrg": "perses.dev", + "schemasPath": "schemas", + "plugins": [ + { + "kind": "Explore", + "spec": { + "display": { + "name": "Log Explorer" + }, + "name": "LogExplorer" + } + } + ] + } +} diff --git a/logexplorer/rsbuild.config.ts b/logexplorer/rsbuild.config.ts new file mode 100644 index 000000000..a46647113 --- /dev/null +++ b/logexplorer/rsbuild.config.ts @@ -0,0 +1,46 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { pluginReact } from '@rsbuild/plugin-react'; +import { createConfigForPlugin } from '../rsbuild.shared'; + +export default createConfigForPlugin({ + name: 'LogExplorer', + rsbuildConfig: { + server: { port: 3009 }, + plugins: [pluginReact()], + }, + moduleFederation: { + exposes: { + './LogExplorer': './src/explore/log-explorer', + }, + shared: { + react: { requiredVersion: '18.2.0', singleton: true }, + 'react-dom': { requiredVersion: '18.2.0', singleton: true }, + echarts: { singleton: true }, + 'date-fns': { singleton: true }, + 'date-fns-tz': { singleton: true }, + lodash: { singleton: true }, + '@perses-dev/components': { singleton: true }, + '@perses-dev/plugin-system': { singleton: true }, + '@perses-dev/explore': { singleton: true }, + '@perses-dev/dashboards': { singleton: true }, + '@emotion/react': { requiredVersion: '^11.11.3', singleton: true }, + '@emotion/styled': { singleton: true }, + '@hookform/resolvers': { singleton: true }, + '@tanstack/react-query': { singleton: true }, + 'react-hook-form': { singleton: true }, + 'react-router-dom': { singleton: true }, + }, + }, +}); diff --git a/logexplorer/src/bootstrap.tsx b/logexplorer/src/bootstrap.tsx new file mode 100644 index 000000000..cabf17b20 --- /dev/null +++ b/logexplorer/src/bootstrap.tsx @@ -0,0 +1,18 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import React from 'react'; +import ReactDOM from 'react-dom/client'; + +const root = ReactDOM.createRoot(document.getElementById('root')!); +root.render(); diff --git a/logexplorer/src/env.d.ts b/logexplorer/src/env.d.ts new file mode 100644 index 000000000..e216e17b1 --- /dev/null +++ b/logexplorer/src/env.d.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// diff --git a/logexplorer/src/explore/index.ts b/logexplorer/src/explore/index.ts new file mode 100644 index 000000000..617b5ed84 --- /dev/null +++ b/logexplorer/src/explore/index.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './log-explorer'; diff --git a/logexplorer/src/explore/log-explorer/LogExplorer.tsx b/logexplorer/src/explore/log-explorer/LogExplorer.tsx new file mode 100644 index 000000000..45f1f537f --- /dev/null +++ b/logexplorer/src/explore/log-explorer/LogExplorer.tsx @@ -0,0 +1,201 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { Box, Stack } from '@mui/material'; +import { + DataQueriesProvider, + LogQueryContext, + MultiQueryEditor, + useAllVariableValues, + useDatasourceStore, + useListPluginMetadata, + usePluginRegistry, + useTimeRange, +} from '@perses-dev/plugin-system'; +import { ReactElement, useEffect, useMemo, useState } from 'react'; +import { QueryDefinition } from '@perses-dev/core'; +import { Panel } from '@perses-dev/dashboards'; +import { useExplorerManagerContext } from '@perses-dev/explore'; + +interface LogExplorerQueryParams { + queries?: QueryDefinition[]; +} + +const PANEL_PREVIEW_HEIGHT = 700; +const HISTOGRAM_HEIGHT = 200; +const EMPTY_QUERIES: QueryDefinition[] = []; + +function LogsTablePanel({ queries }: { queries: QueryDefinition[] }): ReactElement { + const height = PANEL_PREVIEW_HEIGHT; + + const definitions = useMemo( + () => + queries.length + ? queries.map((query) => ({ + kind: query.spec.plugin.kind, + spec: query.spec.plugin.spec, + })) + : [], + [queries] + ); + + return ( + + + + + + ); +} + +function VolumeHistogramPanel({ queries }: { queries: QueryDefinition[] }): ReactElement { + const definitions = useMemo( + () => + queries.length + ? queries.map((query) => ({ + kind: query.spec.plugin.kind, + spec: query.spec.plugin.spec, + })) + : [], + [queries] + ); + + return ( + + + + + + ); +} + +export function LogExplorer(): ReactElement { + const { + data: { queries = EMPTY_QUERIES }, + setData, + } = useExplorerManagerContext(); + + const { getPlugin } = usePluginRegistry(); + const { absoluteTimeRange } = useTimeRange(); + const datasourceStore = useDatasourceStore(); + const variableState = useAllVariableValues(); + + const logQueryContext: LogQueryContext = useMemo( + () => ({ + timeRange: absoluteTimeRange, + variableState, + datasourceStore, + refreshKey: '', + }), + [absoluteTimeRange, variableState, datasourceStore] + ); + + // Get all datasource plugins that support LogQuery + const { data: datasourcePlugins } = useListPluginMetadata(['Datasource']); + + const logDatasourcePlugins = useMemo( + () => + datasourcePlugins + ?.filter((plugin) => { + const pluginSpec = plugin.spec as { supportedQueryTypes?: string[] }; + return pluginSpec?.supportedQueryTypes?.includes('LogQuery'); + }) + .map((p) => p.kind) ?? [], + [datasourcePlugins] + ); + + const [volumeQueries, setVolumeQueries] = useState([]); + + useEffect(() => { + const generateVolumeQueries = async (): Promise => { + if (queries.length === 0) { + setVolumeQueries((prev) => (prev.length === 0 ? prev : [])); + return; + } + + const volumeQueryPromises = queries.map(async (query) => { + if (query.kind !== 'LogQuery') { + return null; + } + + try { + const pluginKind = query.spec.plugin.kind; + const plugin = await getPlugin('LogQuery', pluginKind); + + if (plugin?.createVolumeQuery) { + return plugin.createVolumeQuery(query.spec.plugin.spec, logQueryContext); + } + } catch (error) { + console.error(`[LogExplorer] Failed to create volume query for ${query.spec.plugin.kind}:`, error); + } + + return null; + }); + + const results = await Promise.all(volumeQueryPromises); + const validVolumeQueries = results.filter((q: QueryDefinition | null): q is QueryDefinition => q !== null); + setVolumeQueries(validVolumeQueries); + }; + + generateVolumeQueries(); + }, [queries, getPlugin, logQueryContext]); + + return ( + + setData({ queries: state })} + onQueryRun={() => setData({ queries })} + /> + {volumeQueries.length > 0 && } + + + ); +} diff --git a/logexplorer/src/explore/log-explorer/index.ts b/logexplorer/src/explore/log-explorer/index.ts new file mode 100644 index 000000000..45f264f12 --- /dev/null +++ b/logexplorer/src/explore/log-explorer/index.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './LogExplorer'; diff --git a/logexplorer/src/getPluginModule.ts b/logexplorer/src/getPluginModule.ts new file mode 100644 index 000000000..ee2f25a74 --- /dev/null +++ b/logexplorer/src/getPluginModule.ts @@ -0,0 +1,30 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { PluginModuleResource, PluginModuleSpec } from '@perses-dev/plugin-system'; +import packageJson from '../package.json'; + +/** + * Returns the plugin module information from package.json + */ +export function getPluginModule(): PluginModuleResource { + const { name, version, perses } = packageJson; + return { + kind: 'PluginModule', + metadata: { + name, + version, + }, + spec: perses as PluginModuleSpec, + }; +} diff --git a/logexplorer/src/index-federation.ts b/logexplorer/src/index-federation.ts new file mode 100644 index 000000000..36f748007 --- /dev/null +++ b/logexplorer/src/index-federation.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import('./bootstrap'); diff --git a/logexplorer/src/index.ts b/logexplorer/src/index.ts new file mode 100644 index 000000000..5d31f9b0f --- /dev/null +++ b/logexplorer/src/index.ts @@ -0,0 +1,15 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export { getPluginModule } from './getPluginModule'; +export * from './explore'; diff --git a/logexplorer/src/setup-tests.ts b/logexplorer/src/setup-tests.ts new file mode 100644 index 000000000..c4b091083 --- /dev/null +++ b/logexplorer/src/setup-tests.ts @@ -0,0 +1,17 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import '@testing-library/jest-dom'; + +// Always mock e-charts during tests since we don't have a proper canvas in jsdom +jest.mock('echarts/core'); diff --git a/logexplorer/tsconfig.build.json b/logexplorer/tsconfig.build.json new file mode 100644 index 000000000..fc0aafe27 --- /dev/null +++ b/logexplorer/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["**/*.stories.*", "**/*.test.*", "**/*.map"], + "compilerOptions": { + "emitDeclarationOnly": true, + "declaration": true, + "preserveWatchOutput": true + } +} diff --git a/logexplorer/tsconfig.json b/logexplorer/tsconfig.json new file mode 100644 index 000000000..d8471c931 --- /dev/null +++ b/logexplorer/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "outDir": "./dist/lib", + "rootDir": "./src", + "target": "es2022", + "lib": ["dom", "dom.iterable", "esnext"], + "module": "esnext", + "jsx": "react-jsx", + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "node", + "resolveJsonModule": true, + "isolatedModules": true, + "noUncheckedIndexedAccess": true, + "declaration": true, + "declarationMap": true, + "pretty": true + }, + "include": ["src"] +} diff --git a/loki/src/queries/loki-log-query/LokiLogQuery.tsx b/loki/src/queries/loki-log-query/LokiLogQuery.tsx index 5ceeb473f..e5be8f0a4 100644 --- a/loki/src/queries/loki-log-query/LokiLogQuery.tsx +++ b/loki/src/queries/loki-log-query/LokiLogQuery.tsx @@ -11,11 +11,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { parseVariables } from '@perses-dev/plugin-system'; +import { QueryDefinition } from '@perses-dev/core'; +import { LogQueryPlugin, LogQueryContext, calculateVolumeInterval, parseVariables } from '@perses-dev/plugin-system'; import { getLokiLogData } from './get-loki-log-data'; import { LokiLogQueryEditor } from './LokiLogQueryEditor'; import { LokiLogQuerySpec } from './loki-log-query-types'; -import { LogQueryPlugin } from './log-query-plugin-interface'; export const LokiLogQuery: LogQueryPlugin = { getLogData: getLokiLogData, @@ -28,4 +28,26 @@ export const LokiLogQuery: LogQueryPlugin = { variables: allVariables, }; }, + createVolumeQuery: (spec: LokiLogQuerySpec, ctx: LogQueryContext): QueryDefinition | null => { + if (!spec.query || !spec.query.trim()) { + return null; + } + + const interval = calculateVolumeInterval(ctx.timeRange.end.getTime() - ctx.timeRange.start.getTime()); + const volumeQuery = `sum by (level, detected_level) (count_over_time(${spec.query}[${interval}]))`; + + return { + kind: 'TimeSeriesQuery', + spec: { + plugin: { + kind: 'LokiTimeSeriesQuery', + spec: { + query: volumeQuery, + datasource: spec.datasource, + step: interval, + }, + }, + }, + }; + }, }; diff --git a/loki/src/queries/loki-log-query/get-loki-log-data.ts b/loki/src/queries/loki-log-query/get-loki-log-data.ts index 1f5fa2f5f..182d1abee 100644 --- a/loki/src/queries/loki-log-query/get-loki-log-data.ts +++ b/loki/src/queries/loki-log-query/get-loki-log-data.ts @@ -11,13 +11,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { replaceVariables } from '@perses-dev/plugin-system'; +import { replaceVariables, LogQueryPlugin, LogQueryContext } from '@perses-dev/plugin-system'; import { LogEntry, LogData } from '@perses-dev/core'; import { LokiStreamResult } from '../../model/loki-client-types'; import { LokiClient } from '../../model/loki-client'; import { DEFAULT_DATASOURCE } from '../constants'; import { LokiLogQuerySpec } from './loki-log-query-types'; -import { LogQueryPlugin, LogQueryContext } from './log-query-plugin-interface'; function convertStreamsToLogs(streams: LokiStreamResult[]): LogData { const entries: LogEntry[] = []; diff --git a/loki/src/queries/loki-log-query/log-query-plugin-interface.ts b/loki/src/queries/loki-log-query/log-query-plugin-interface.ts deleted file mode 100644 index e67d21167..000000000 --- a/loki/src/queries/loki-log-query/log-query-plugin-interface.ts +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright The Perses Authors -// Licensed under the Apache License, Version 2.0 (the \"License\"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an \"AS IS\" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import { LogData, AbsoluteTimeRange, UnknownSpec } from '@perses-dev/core'; -import { DatasourceStore, Plugin, VariableStateMap } from '@perses-dev/plugin-system'; - -export interface LogQueryResult { - logs: LogData; - timeRange: AbsoluteTimeRange; - metadata?: { - executedQueryString: string; - }; -} - -export interface LogQueryContext { - timeRange: AbsoluteTimeRange; - variableState: VariableStateMap; - datasourceStore: DatasourceStore; -} - -type LogQueryPluginDependencies = { - variables?: string[]; -}; - -export interface LogQueryPlugin extends Plugin { - getLogData: (spec: Spec, ctx: LogQueryContext) => Promise; - dependsOn?: (spec: Spec, ctx: LogQueryContext) => LogQueryPluginDependencies; -} diff --git a/loki/src/queries/loki-log-query/loki-log-query-types.test.ts b/loki/src/queries/loki-log-query/loki-log-query-types.test.ts index 06bd7df4a..9318046d4 100644 --- a/loki/src/queries/loki-log-query/loki-log-query-types.test.ts +++ b/loki/src/queries/loki-log-query/loki-log-query-types.test.ts @@ -11,11 +11,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +import { LogQueryContext } from '@perses-dev/plugin-system'; import { LokiQueryRangeStreamsResponse, LokiQueryRangeResponse } from '../../model/loki-client-types'; import { LokiDatasource } from '../../datasources/loki-datasource'; import { LokiDatasourceSpec } from '../../datasources/loki-datasource/loki-datasource-types'; import { LokiLogQuery } from './LokiLogQuery'; -import { LogQueryContext } from './log-query-plugin-interface'; const datasource: LokiDatasourceSpec = { directUrl: '/test', @@ -63,6 +63,7 @@ const createStubContext = (): LogQueryContext => { start: new Date('01-02-2025'), }, variableState: {}, + refreshKey: '', }; return stubLogContext; }; diff --git a/package-lock.json b/package-lock.json index 6c8c7cd5f..70e8c861f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "heatmapchart", "histogramchart", "jaeger", + "logexplorer", "logstable", "loki", "markdown", @@ -251,6 +252,33 @@ "use-resize-observer": "^9.0.0" } }, + "logexplorer": { + "name": "@perses-dev/log-explorer", + "version": "0.1.0", + "devDependencies": { + "@types/qs": "^6.9.18" + }, + "peerDependencies": { + "@emotion/react": "^11.7.1", + "@emotion/styled": "^11.6.0", + "@hookform/resolvers": "^3.2.0", + "@perses-dev/components": "^0.54.0-beta.1", + "@perses-dev/core": "^0.53.0", + "@perses-dev/dashboards": "^0.54.0-beta.1", + "@perses-dev/explore": "^0.54.0-beta.1", + "@perses-dev/plugin-system": "^0.54.0-beta.1", + "@tanstack/react-query": "^4.39.1", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", + "echarts": "5.5.0", + "immer": "^10.1.1", + "lodash": "^4.17.21", + "react": "^17.0.2 || ^18.0.0", + "react-dom": "^17.0.2 || ^18.0.0", + "react-hook-form": "^7.52.2", + "react-router-dom": "^5 || ^6 || ^7" + } + }, "logstable": { "name": "@perses-dev/logs-table-plugin", "version": "0.2.1", @@ -4023,6 +4051,10 @@ "resolved": "jaeger", "link": true }, + "node_modules/@perses-dev/log-explorer": { + "resolved": "logexplorer", + "link": true + }, "node_modules/@perses-dev/logs-table-plugin": { "resolved": "logstable", "link": true diff --git a/package.json b/package.json index 39771d349..c5908e987 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "heatmapchart", "histogramchart", "jaeger", + "logexplorer", "logstable", "loki", "markdown", diff --git a/victorialogs/src/queries/victorialogs-log-query/VictoriaLogsLogQuery.tsx b/victorialogs/src/queries/victorialogs-log-query/VictoriaLogsLogQuery.tsx index d9e021363..68e07f3a5 100644 --- a/victorialogs/src/queries/victorialogs-log-query/VictoriaLogsLogQuery.tsx +++ b/victorialogs/src/queries/victorialogs-log-query/VictoriaLogsLogQuery.tsx @@ -11,11 +11,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { parseVariables } from '@perses-dev/plugin-system'; +import { QueryDefinition } from '@perses-dev/core'; +import { LogQueryPlugin, LogQueryContext, calculateVolumeInterval, parseVariables } from '@perses-dev/plugin-system'; import { getVictoriaLogsLogData } from './query'; import { VictoriaLogsLogQueryEditor } from './VictoriaLogsLogQueryEditor'; import { VictoriaLogsLogQuerySpec } from './types'; -import { LogQueryPlugin } from './interface'; export const VictoriaLogsLogQuery: LogQueryPlugin = { getLogData: getVictoriaLogsLogData, @@ -28,4 +28,26 @@ export const VictoriaLogsLogQuery: LogQueryPlugin = { variables: allVariables, }; }, + createVolumeQuery: (spec: VictoriaLogsLogQuerySpec, ctx: LogQueryContext): QueryDefinition | null => { + if (!spec.query || !spec.query.trim()) { + return null; + } + + const interval = calculateVolumeInterval(ctx.timeRange.end.getTime() - ctx.timeRange.start.getTime()); + const volumeQuery = `${spec.query} | stats by (_time:${interval}, _stream) count() as volume`; + + return { + kind: 'TimeSeriesQuery', + spec: { + plugin: { + kind: 'VictoriaLogsTimeSeriesQuery', + spec: { + query: volumeQuery, + datasource: spec.datasource, + step: interval, + }, + }, + }, + }; + }, }; diff --git a/victorialogs/src/queries/victorialogs-log-query/interface.ts b/victorialogs/src/queries/victorialogs-log-query/interface.ts deleted file mode 100644 index e67d21167..000000000 --- a/victorialogs/src/queries/victorialogs-log-query/interface.ts +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright The Perses Authors -// Licensed under the Apache License, Version 2.0 (the \"License\"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an \"AS IS\" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import { LogData, AbsoluteTimeRange, UnknownSpec } from '@perses-dev/core'; -import { DatasourceStore, Plugin, VariableStateMap } from '@perses-dev/plugin-system'; - -export interface LogQueryResult { - logs: LogData; - timeRange: AbsoluteTimeRange; - metadata?: { - executedQueryString: string; - }; -} - -export interface LogQueryContext { - timeRange: AbsoluteTimeRange; - variableState: VariableStateMap; - datasourceStore: DatasourceStore; -} - -type LogQueryPluginDependencies = { - variables?: string[]; -}; - -export interface LogQueryPlugin extends Plugin { - getLogData: (spec: Spec, ctx: LogQueryContext) => Promise; - dependsOn?: (spec: Spec, ctx: LogQueryContext) => LogQueryPluginDependencies; -} diff --git a/victorialogs/src/queries/victorialogs-log-query/query.ts b/victorialogs/src/queries/victorialogs-log-query/query.ts index 9551f1935..1e80e34c8 100644 --- a/victorialogs/src/queries/victorialogs-log-query/query.ts +++ b/victorialogs/src/queries/victorialogs-log-query/query.ts @@ -11,13 +11,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { replaceVariables } from '@perses-dev/plugin-system'; +import { replaceVariables, LogQueryPlugin, LogQueryContext } from '@perses-dev/plugin-system'; import { LogEntry, LogData } from '@perses-dev/core'; import { VictoriaLogsStreamQueryRangeResponse } from '../../model/types'; import { VictoriaLogsClient } from '../../model/client'; import { DEFAULT_DATASOURCE } from '../constants'; import { VictoriaLogsLogQuerySpec } from './types'; -import { LogQueryPlugin, LogQueryContext } from './interface'; function convertStreamToLogs(data: VictoriaLogsStreamQueryRangeResponse, defaultTime: string): LogData { const entries: LogEntry[] = []; diff --git a/victorialogs/src/queries/victorialogs-log-query/types.test.ts b/victorialogs/src/queries/victorialogs-log-query/types.test.ts index bc7587d41..8800e5cf1 100644 --- a/victorialogs/src/queries/victorialogs-log-query/types.test.ts +++ b/victorialogs/src/queries/victorialogs-log-query/types.test.ts @@ -11,11 +11,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +import { LogQueryContext } from '@perses-dev/plugin-system'; import { VictoriaLogsStreamQueryRangeResponse } from '../../model/types'; import { VictoriaLogsDatasource } from '../../datasources/victorialogs-datasource'; import { VictoriaLogsDatasourceSpec } from '../../datasources/victorialogs-datasource/types'; import { VictoriaLogsLogQuery } from './VictoriaLogsLogQuery'; -import { LogQueryContext } from './interface'; const datasource: VictoriaLogsDatasourceSpec = { directUrl: '/test', @@ -55,6 +55,7 @@ const createStubContext = (): LogQueryContext => { start: new Date('01-02-2025'), }, variableState: {}, + refreshKey: '', }; return stubLogContext; }; diff --git a/victorialogs/src/queries/victorialogs-time-series-query/query.ts b/victorialogs/src/queries/victorialogs-time-series-query/query.ts index 0f62ada05..abd60b295 100644 --- a/victorialogs/src/queries/victorialogs-time-series-query/query.ts +++ b/victorialogs/src/queries/victorialogs-time-series-query/query.ts @@ -79,9 +79,12 @@ function formatStepForVictoriaLogs(stepSeconds: number): string { function convertMatrixToTimeSeries(matrix: VictoriaLogsMatrixResult[]): TimeSeries[] { return matrix.map((series) => { - const { _stream, ...labels } = series.metric; - if (_stream) { - const match = _stream.match(/{([^}]+)}/); + const labels = { ...series.metric }; + delete labels._stream; + delete labels.__name__; + + if (series.metric._stream) { + const match = series.metric._stream.match(/{([^}]+)}/); if (match && match[1]) { match[1].split(',').forEach((labelPair) => { const [key, val] = labelPair.split('=').map((s) => s.trim().replace(/^"|"$/g, ''));