Skip to content

Commit 9345d90

Browse files
shah-harshitshah-harshit
committed
fix: add data product filter to Data Quality dashboard
Fixes #27124 Made-with: Cursor
1 parent 426081b commit 9345d90

9 files changed

Lines changed: 255 additions & 26 deletions

File tree

‎openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/DataQualityDashboard.spec.ts‎

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313

1414
import test, { expect, Page } from '@playwright/test';
1515
import { getCurrentMillis } from '../../../../src/utils/date-time/DateTimeUtils';
16+
import { DataProduct } from '../../../support/domain/DataProduct';
17+
import { Domain } from '../../../support/domain/Domain';
1618
import { TableClass } from '../../../support/entity/TableClass';
1719
import { Glossary } from '../../../support/glossary/Glossary';
1820
import { GlossaryTerm } from '../../../support/glossary/GlossaryTerm';
@@ -48,6 +50,8 @@ const tag = new TagClass({ classification: classification.data.name });
4850
const tier = new TagClass({ classification: 'Tier' });
4951
const glossary = new Glossary();
5052
const glossaryTerm = new GlossaryTerm(glossary);
53+
const domain = new Domain();
54+
const dataProduct = new DataProduct([domain]);
5155

5256
const testCaseResult = {
5357
result: 'Found min=10001, max=27809 vs. the expected min=90001, max=96162.',
@@ -76,6 +80,13 @@ test.beforeAll('setup pre-test', async ({ browser }) => {
7680
await table1.create(apiContext);
7781
await table2.create(apiContext);
7882
await table3.create(apiContext);
83+
await domain.create(apiContext);
84+
await dataProduct.create(apiContext);
85+
await dataProduct.addAssets(apiContext, [
86+
{ id: table1.entityResponseData.id, type: 'table' },
87+
{ id: table2.entityResponseData.id, type: 'table' },
88+
{ id: table3.entityResponseData.id, type: 'table' },
89+
]);
7990
for (const table of [table1, table2, table3]) {
8091
await table.patch({
8192
apiContext,
@@ -147,22 +158,6 @@ test.beforeAll('setup pre-test', async ({ browser }) => {
147158
await afterAction();
148159
});
149160

150-
test.afterAll('cleanup', async ({ browser }) => {
151-
const { apiContext, afterAction } = await createNewPage(browser);
152-
153-
await table1.delete(apiContext);
154-
await table2.delete(apiContext);
155-
await table3.delete(apiContext);
156-
await user1.delete(apiContext);
157-
await glossaryTerm.delete(apiContext);
158-
await glossary.delete(apiContext);
159-
await tag.delete(apiContext);
160-
await tier.delete(apiContext);
161-
await classification.delete(apiContext);
162-
163-
await afterAction();
164-
});
165-
166161
const waitForDashboardApiResponses = (page: Page, key: string) => {
167162
const testCaseStatusResponse = page.waitForResponse(
168163
`/api/v1/dataQuality/testSuites/dataQualityReport?q=*${key}*&index=testCase&aggregationQuery=bucketName%3Dstatus%3AaggType%3Dterms%3Afield%3DtestCaseResult.testCaseStatus`
@@ -307,6 +302,31 @@ test('DataQualityDashboardTab', async ({ page }) => {
307302

308303
expect(responseData.ok()).toBeTruthy();
309304
}
305+
306+
await page.getByRole('button', { name: 'Data Product' }).click();
307+
await page.getByTestId('search-input').click();
308+
const dataProductSearchApi = page.waitForResponse(
309+
'/api/v1/search/query?*q=*index=dataProduct*'
310+
);
311+
await page
312+
.getByTestId('search-input')
313+
.fill(dataProduct.responseData.displayName ?? dataProduct.data.displayName);
314+
await dataProductSearchApi;
315+
await page
316+
.getByText(
317+
dataProduct.responseData.displayName ?? dataProduct.data.displayName
318+
)
319+
.click();
320+
const dataProductApiResponse = waitForDashboardApiResponses(
321+
page,
322+
encodeURIComponent(dataProduct.data.name)
323+
);
324+
await page.getByTestId('update-btn').click();
325+
for (const apiRes of dataProductApiResponse) {
326+
const responseData = await apiRes;
327+
328+
expect(responseData.ok()).toBeTruthy();
329+
}
310330
});
311331

312332
test('Dimension card click should redirect to test cases with applied filters', async ({

‎openmetadata-ui/src/main/resources/ui/src/components/DataQuality/DataQualityDashboard/DataQualityDashboard.component.tsx‎

Lines changed: 122 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,9 @@ const DataQualityDashboard = ({
7878
}: {
7979
initialFilters?: DqDashboardChartFilters;
8080
hideFilterBar?: boolean;
81-
hiddenFilters?: Array<'owner' | 'tier' | 'tags' | 'glossaryTerms'>;
81+
hiddenFilters?: Array<
82+
'owner' | 'tier' | 'tags' | 'glossaryTerms' | 'dataProducts'
83+
>;
8284
isGovernanceView?: boolean;
8385
className?: string;
8486
}) => {
@@ -136,9 +138,20 @@ const DataQualityDashboard = ({
136138
options: [],
137139
});
138140
const [isTagLoading, setIsTagLoading] = useState(false);
141+
const [dataProductOptions, setDataProductOptions] = useState<{
142+
defaultOptions: SearchDropdownOption[];
143+
options: SearchDropdownOption[];
144+
}>({
145+
defaultOptions: [],
146+
options: [],
147+
});
148+
const [isDataProductLoading, setIsDataProductLoading] = useState(false);
139149
const [selectedTagFilter, setSelectedTagFilter] = useState<
140150
SearchDropdownOption[]
141151
>([]);
152+
const [selectedDataProductFilter, setSelectedDataProductFilter] = useState<
153+
SearchDropdownOption[]
154+
>([]);
142155
const [selectedGlossaryTermFilter, setSelectedGlossaryTermFilter] = useState<
143156
SearchDropdownOption[]
144157
>([]);
@@ -182,6 +195,7 @@ const DataQualityDashboard = ({
182195
ownerFqn: defaultFilters.ownerFqn,
183196
tags: defaultFilters.tags,
184197
tier: defaultFilters.tier,
198+
dataProductFqns: defaultFilters.dataProductFqns,
185199
startTs: defaultFilters.startTs,
186200
endTs: defaultFilters.endTs,
187201
domainFqn: defaultFilters.domainFqn,
@@ -190,6 +204,7 @@ const DataQualityDashboard = ({
190204
defaultFilters.ownerFqn,
191205
defaultFilters.tier,
192206
defaultFilters.tags,
207+
defaultFilters.dataProductFqns,
193208
defaultFilters.startTs,
194209
defaultFilters.endTs,
195210
defaultFilters.domainFqn,
@@ -242,6 +257,16 @@ const DataQualityDashboard = ({
242257
}));
243258
};
244259

260+
const handleDataProductChange = (
261+
dataProducts: SearchDropdownOption[] = []
262+
) => {
263+
setSelectedDataProductFilter(dataProducts);
264+
setChartFilter((prev) => ({
265+
...prev,
266+
dataProductFqns: dataProducts.map((dp) => dp.key),
267+
}));
268+
};
269+
245270
const fetchTagOptions = async (query = WILD_CARD_CHAR) => {
246271
const response = await searchQuery({
247272
searchIndex: SearchIndex.TAG,
@@ -262,6 +287,25 @@ const DataQualityDashboard = ({
262287
return tagFilterOptions;
263288
};
264289

290+
const fetchDataProductOptions = async (query = WILD_CARD_CHAR) => {
291+
const response = await searchQuery({
292+
searchIndex: SearchIndex.DATA_PRODUCT,
293+
query: query === WILD_CARD_CHAR ? query : `*${query}*`,
294+
pageSize: PAGE_SIZE_BASE,
295+
});
296+
const hits = response.hits.hits;
297+
const dataProductFilterOptions = hits.map((hit) => {
298+
const source = hit._source;
299+
300+
return {
301+
key: source.fullyQualifiedName ?? source.name,
302+
label: source.displayName ?? source.fullyQualifiedName ?? source.name,
303+
};
304+
});
305+
306+
return dataProductFilterOptions;
307+
};
308+
265309
const handleTagSearch = async (query: string) => {
266310
if (isEmpty(query)) {
267311
setTagOptions((prev) => ({
@@ -380,6 +424,28 @@ const DataQualityDashboard = ({
380424
}
381425
};
382426

427+
const handleDataProductSearch = async (query: string) => {
428+
if (isEmpty(query)) {
429+
setDataProductOptions((prev) => ({
430+
...prev,
431+
options: prev.defaultOptions,
432+
}));
433+
} else {
434+
setIsDataProductLoading(true);
435+
try {
436+
const response = await fetchDataProductOptions(query);
437+
setDataProductOptions((prev) => ({
438+
...prev,
439+
options: response,
440+
}));
441+
} catch {
442+
// we will not show the toast error message for suggestion API
443+
} finally {
444+
setIsDataProductLoading(false);
445+
}
446+
}
447+
};
448+
383449
const fetchDefaultGlossaryTermOptions = async () => {
384450
if (glossaryTermOptions.defaultOptions.length) {
385451
setGlossaryTermOptions((prev) => ({
@@ -405,6 +471,31 @@ const DataQualityDashboard = ({
405471
}
406472
};
407473

474+
const fetchDefaultDataProductOptions = async () => {
475+
if (dataProductOptions.defaultOptions.length) {
476+
setDataProductOptions((prev) => ({
477+
...prev,
478+
options: [...selectedDataProductFilter, ...prev.defaultOptions],
479+
}));
480+
481+
return;
482+
}
483+
484+
try {
485+
setIsDataProductLoading(true);
486+
const response = await fetchDataProductOptions();
487+
setDataProductOptions((prev) => ({
488+
...prev,
489+
defaultOptions: response,
490+
options: response,
491+
}));
492+
} catch {
493+
// we will not show the toast error message for search API
494+
} finally {
495+
setIsDataProductLoading(false);
496+
}
497+
};
498+
408499
const getTierTag = async () => {
409500
setTier((prev) => ({ ...prev, isLoading: true }));
410501
try {
@@ -454,6 +545,9 @@ const DataQualityDashboard = ({
454545
const showGlossaryTermsFilter = !hiddenFilters.includes(
455546
DQ_FILTER_KEYS.GLOSSARY_TERMS
456547
);
548+
const showDataProductsFilter = !hiddenFilters.includes(
549+
DQ_FILTER_KEYS.DATA_PRODUCTS
550+
);
457551

458552
useEffect(() => {
459553
if (hideFilterBar) {
@@ -468,6 +562,9 @@ const DataQualityDashboard = ({
468562
if (showGlossaryTermsFilter) {
469563
fetchDefaultGlossaryTermOptions();
470564
}
565+
if (showDataProductsFilter) {
566+
fetchDefaultDataProductOptions();
567+
}
471568
}, [hideFilterBar, hiddenFiltersKey]);
472569

473570
const tags = useMemo(
@@ -511,12 +608,25 @@ const DataQualityDashboard = ({
511608
[selectedTierFilter, tier]
512609
);
513610

611+
const dataProducts = useMemo(
612+
() => ({
613+
options: uniqBy(dataProductOptions.options, 'key'),
614+
selectedKeys: selectedDataProductFilter,
615+
onChange: handleDataProductChange,
616+
onGetInitialOptions: fetchDefaultDataProductOptions,
617+
onSearch: handleDataProductSearch,
618+
isSuggestionsLoading: isDataProductLoading,
619+
}),
620+
[isDataProductLoading, dataProductOptions, selectedDataProductFilter]
621+
);
622+
514623
const showFilterBar = !hideFilterBar;
515624
const hasVisibleFilters =
516625
showOwnerFilter ||
517626
showTierFilter ||
518627
showTagsFilter ||
519-
showGlossaryTermsFilter;
628+
showGlossaryTermsFilter ||
629+
showDataProductsFilter;
520630

521631
const cardClassName = classNames('data-quality-dashboard-card-section', {
522632
'tw:ring-0': isGovernanceView,
@@ -596,6 +706,16 @@ const DataQualityDashboard = ({
596706
/>
597707
)}
598708

709+
{showDataProductsFilter && (
710+
<SearchDropdown
711+
hideCounts
712+
label={t('label.data-product')}
713+
searchKey="dataProduct"
714+
triggerButtonSize="middle"
715+
{...dataProducts}
716+
/>
717+
)}
718+
599719
{showGlossaryTermsFilter && (
600720
<SearchDropdown
601721
hideCounts

‎openmetadata-ui/src/main/resources/ui/src/components/DataQuality/DataQualityDashboard/DataQualityDashboard.interface.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ export type DqDashboardChartFilters = {
1515
glossaryTerms?: string[];
1616
tags?: string[];
1717
tier?: string[];
18+
dataProductFqns?: string[];
1819
startTs?: number;
1920
endTs?: number;
2021
entityFQN?: string;

‎openmetadata-ui/src/main/resources/ui/src/components/DataQuality/DataQualityDashboard/DataQualityDashboard.test.tsx‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,9 @@ describe('DataQualityDashboard', () => {
327327
screen.getByTestId('search-dropdown-label.tier')
328328
).toBeInTheDocument();
329329
expect(screen.getByTestId('search-dropdown-label.tag')).toBeInTheDocument();
330+
expect(
331+
screen.getByTestId('search-dropdown-label.data-product')
332+
).toBeInTheDocument();
330333
expect(
331334
screen.getByTestId('search-dropdown-label.glossary-term')
332335
).toBeInTheDocument();
@@ -746,7 +749,7 @@ describe('DataQualityDashboard', () => {
746749
jest.clearAllMocks();
747750
});
748751

749-
it('hides Owner, Tier, Tag and Glossary Term dropdowns when hideFilterBar is true', async () => {
752+
it('hides Owner, Tier, Tag, Data Product and Glossary Term dropdowns when hideFilterBar is true', async () => {
750753
render(<DataQualityDashboard hideFilterBar />, { wrapper: MemoryRouter });
751754

752755
expect(
@@ -758,6 +761,9 @@ describe('DataQualityDashboard', () => {
758761
expect(
759762
screen.queryByTestId('search-dropdown-label.tag')
760763
).not.toBeInTheDocument();
764+
expect(
765+
screen.queryByTestId('search-dropdown-label.data-product')
766+
).not.toBeInTheDocument();
761767
expect(
762768
screen.queryByTestId('search-dropdown-label.glossary-term')
763769
).not.toBeInTheDocument();
@@ -1054,9 +1060,12 @@ describe('DataQualityDashboard', () => {
10541060
});
10551061

10561062
it('does not call tag search API when tags is in hiddenFilters', async () => {
1057-
render(<DataQualityDashboard hiddenFilters={['tags']} />, {
1058-
wrapper: MemoryRouter,
1059-
});
1063+
render(
1064+
<DataQualityDashboard hiddenFilters={['tags', 'dataProducts']} />,
1065+
{
1066+
wrapper: MemoryRouter,
1067+
}
1068+
);
10601069

10611070
await waitFor(() => {
10621071
expect(
@@ -1069,9 +1078,14 @@ describe('DataQualityDashboard', () => {
10691078
});
10701079

10711080
it('does not call glossary term search API when glossaryTerms is in hiddenFilters', async () => {
1072-
render(<DataQualityDashboard hiddenFilters={['glossaryTerms']} />, {
1073-
wrapper: MemoryRouter,
1074-
});
1081+
render(
1082+
<DataQualityDashboard
1083+
hiddenFilters={['glossaryTerms', 'dataProducts']}
1084+
/>,
1085+
{
1086+
wrapper: MemoryRouter,
1087+
}
1088+
);
10751089

10761090
await waitFor(() => {
10771091
expect(
@@ -1101,6 +1115,9 @@ describe('DataQualityDashboard', () => {
11011115
expect(
11021116
screen.queryByTestId('search-dropdown-label.tag')
11031117
).not.toBeInTheDocument();
1118+
expect(
1119+
screen.queryByTestId('search-dropdown-label.data-product')
1120+
).not.toBeInTheDocument();
11041121
});
11051122
});
11061123

@@ -1299,7 +1316,7 @@ describe('DataQualityDashboard', () => {
12991316
(args[0] as Record<string, unknown>).query === '***'
13001317
);
13011318

1302-
expect(wildcardCalls.length).toBeGreaterThanOrEqual(2); // tags + glossaryTerms
1319+
expect(wildcardCalls.length).toBeGreaterThanOrEqual(3); // tags + glossaryTerms + data products
13031320
expect(tripleStarCalls).toHaveLength(0);
13041321
});
13051322

‎openmetadata-ui/src/main/resources/ui/src/constants/DataQuality.constants.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,4 +81,5 @@ export const DQ_FILTER_KEYS = {
8181
TIER: 'tier',
8282
TAGS: 'tags',
8383
GLOSSARY_TERMS: 'glossaryTerms',
84+
DATA_PRODUCTS: 'dataProducts',
8485
} as const;

0 commit comments

Comments
 (0)