diff --git a/cypress/e2e/ingestion/files-ui.cy.ts b/cypress/e2e/ingestion/files-ui.cy.ts index 68e1d74d..443f3cfe 100644 --- a/cypress/e2e/ingestion/files-ui.cy.ts +++ b/cypress/e2e/ingestion/files-ui.cy.ts @@ -2,6 +2,7 @@ /// import { byTestId, testIds } from '../../support/selectors'; +import { continuePastMissingNodeCoverageWarning } from '../../support/journey-helpers'; describe('File Handling and Processing', () => { const nodeFile = 'AngularTesting_nodelist_withseqs_TN93_BS.csv'; @@ -9,6 +10,11 @@ describe('File Handling and Processing', () => { const linkFile = 'AngularTesting_Epi_linklist_BS.csv'; const additionalNodeFile = compatibleNodeFile; const geoJSONLocationFile = 'map-node-locations.geojson'; + const coverageNodeFile = 'missing-node-coverage-nodes.csv'; + const coverageLinkFile = 'missing-node-coverage-links.csv'; + const matchedCoverageNodeFile = 'missing-node-coverage-matched-nodes.csv'; + const matchedCoverageLinkFile = 'missing-node-coverage-matched-links.csv'; + const extraCoverageNodeFile = 'missing-node-coverage-extra-node.csv'; const loadNodeFile = () => cy.loadFiles([{ name: nodeFile, datatype: 'node' }]); beforeEach(() => { @@ -180,6 +186,7 @@ describe('File Handling and Processing', () => { cy.contains('#file-table .file-table-row', linkFile, { timeout: 20000 }).should('be.visible'); cy.get('.runtime-error-banner').should('not.exist'); cy.get('#launch').should('not.be.disabled').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.get('.lm_tab.lm_active', { timeout: 20000 }).should('contain.text', '2D Network'); }); @@ -286,6 +293,7 @@ describe('File Handling and Processing', () => { ]); cy.get('#launch').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.get('.lm_tab.lm_active', { timeout: 20000 }).should('contain.text', '2D Network'); cy.window().its('commonService.session.data.nodes').should('have.length.greaterThan', 0); @@ -303,6 +311,110 @@ describe('File Handling and Processing', () => { }); }); + it('gates launch when imported node and topology identifiers do not match', () => { + cy.loadFiles([ + { name: coverageNodeFile, datatype: 'node', field1: 'id' }, + { name: coverageLinkFile, datatype: 'link', field1: 'source', field2: 'target' }, + ]); + + cy.get('#launch').click({ force: true }); + cy.get(byTestId(testIds.filesMissingNodeCoverageDialog), { timeout: 10000 }) + .should('be.visible') + .parents('.p-dialog') + .as('coverageDialog'); + + cy.get('@coverageDialog') + .find(byTestId(testIds.filesMissingNodeDataSection)) + .should('contain.text', 'Missing node data (1)') + .and('contain.text', 'D'); + cy.get('@coverageDialog') + .find(byTestId(testIds.filesMissingLinkInformationSection)) + .should('contain.text', 'Missing link information (1)') + .and('contain.text', 'C'); + cy.window().then((win: any) => { + expect(win.commonService.session.network.launched).to.equal(false); + expect(win.commonService.session.data.nodes).to.have.length(0); + }); + + cy.get('@coverageDialog') + .find(byTestId(testIds.filesMissingNodeCoverageCancel)) + .click({ force: true }); + cy.get(byTestId(testIds.filesMissingNodeCoverageDialog)).should('not.exist'); + cy.get('.lm_tab.lm_active').should('contain.text', 'Files'); + cy.window().then((win: any) => { + expect(win.commonService.session.network.launched).to.equal(false); + expect(win.commonService.session.files.map((file: any) => file.name)).to.include.members([ + coverageNodeFile, + coverageLinkFile, + ]); + }); + + cy.get('#launch').click({ force: true }); + cy.get(byTestId(testIds.filesMissingNodeCoverageDialog), { timeout: 10000 }) + .should('be.visible') + .parents('.p-dialog') + .find(byTestId(testIds.filesMissingNodeCoverageContinue)) + .click({ force: true }); + + cy.window({ timeout: 30000 }) + .its('commonService.session.network.isFullyLoaded') + .should('be.true'); + cy.get('.lm_tab.lm_active', { timeout: 20000 }).should('contain.text', '2D Network'); + }); + + it('launches without a coverage dialog when explicit topology is absent or fully matched', () => { + cy.loadFiles([{ name: coverageNodeFile, datatype: 'node', field1: 'id' }]); + cy.get('#launch').click({ force: true }); + cy.get(byTestId(testIds.filesMissingNodeCoverageDialog)).should('not.exist'); + cy.window({ timeout: 30000 }) + .its('commonService.session.network.isFullyLoaded') + .should('be.true'); + + cy.visit('/?skipEula=1&skipDemoSession=1'); + cy.get('#fileDropRef', { timeout: 15000 }).should('exist'); + + cy.loadFiles([ + { name: matchedCoverageNodeFile, datatype: 'node', field1: 'id' }, + { name: matchedCoverageLinkFile, datatype: 'link', field1: 'source', field2: 'target' }, + ]); + cy.get('#launch').click({ force: true }); + cy.get(byTestId(testIds.filesMissingNodeCoverageDialog)).should('not.exist'); + cy.window({ timeout: 30000 }) + .its('commonService.session.network.isFullyLoaded') + .should('be.true'); + }); + + it('cancels an update without replacing the currently processed network', () => { + cy.loadFiles([ + { name: matchedCoverageNodeFile, datatype: 'node', field1: 'id' }, + { name: matchedCoverageLinkFile, datatype: 'link', field1: 'source', field2: 'target' }, + ]); + cy.get('#launch').click({ force: true }); + cy.window({ timeout: 30000 }) + .its('commonService.session.network.isFullyLoaded') + .should('be.true'); + + cy.get(byTestId(testIds.appFileMenuButton)).click({ force: true }); + cy.contains('[role="menuitem"]', 'Add Data').click({ force: true }); + cy.get('.lm_tab.lm_active', { timeout: 20000 }).should('contain.text', 'Files'); + cy.loadFiles([{ name: extraCoverageNodeFile, datatype: 'node', field1: 'id' }]); + + cy.get(byTestId(testIds.filesUpdateButton)).click({ force: true }); + cy.get(byTestId(testIds.filesMissingNodeCoverageDialog), { timeout: 10000 }) + .should('be.visible') + .parents('.p-dialog') + .find(byTestId(testIds.filesMissingNodeCoverageCancel)) + .click({ force: true }); + + cy.window().then((win: any) => { + expect(win.commonService.session.network.launched).to.equal(true); + expect(win.commonService.session.data.nodes.map((node: any) => node._id).sort()) + .to.deep.equal(['A', 'B']); + expect(win.commonService.session.files.map((file: any) => file.name)) + .to.include(extraCoverageNodeFile); + }); + }); + it('detects a GeoJSON location file, joins feature centers to nodes, and stores the layer', () => { cy.attach_files('#fileDropRef', [nodeFile, geoJSONLocationFile]); @@ -351,6 +463,7 @@ describe('File Handling and Processing', () => { ]); cy.get('#launch').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.get('.lm_tab.lm_active', { timeout: 20000 }).should('contain.text', '2D Network'); cy.get(byTestId(testIds.appFileMenuButton)).click({ force: true }); @@ -383,6 +496,7 @@ describe('File Handling and Processing', () => { ]); cy.get('#launch').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.window({ timeout: 30000 }) .its('commonService.session.network.isFullyLoaded') .should('be.true'); @@ -410,6 +524,7 @@ describe('File Handling and Processing', () => { cy.loadFiles([{ name: linkFile, datatype: 'link' }]); cy.get('#launch').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.window({ timeout: 30000 }) .its('commonService.session.network.isFullyLoaded') .should('be.true'); @@ -428,6 +543,7 @@ describe('File Handling and Processing', () => { ]); cy.get('#launch').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.window({ timeout: 30000 }) .its('commonService.session.network.isFullyLoaded') .should('be.true'); @@ -449,6 +565,7 @@ describe('File Handling and Processing', () => { .find('.flaticon-delete-1') .click({ force: true }); cy.get('#launch').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.window({ timeout: 30000 }) .its('commonService.session.network.isFullyLoaded') .should('be.true'); @@ -461,6 +578,7 @@ describe('File Handling and Processing', () => { cy.loadFiles([{ name: compatibleNodeFile, datatype: 'node', field1: '_id', field2: 'seq' }]); cy.get('#launch').should('contain.text', 'Update').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.window({ timeout: 30000 }) .its('commonService.session.network.isFullyLoaded') .should('be.true'); @@ -482,6 +600,7 @@ describe('File Handling and Processing', () => { ]); cy.get('#launch').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.window({ timeout: 30000 }) .its('commonService.session.network.isFullyLoaded') .should('be.true'); @@ -504,6 +623,7 @@ describe('File Handling and Processing', () => { cy.get('#launch-reset-settings', { timeout: 20000 }) .should('exist') .click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.window({ timeout: 30000 }) .its('commonService.session.network.isFullyLoaded') .should('be.true'); diff --git a/cypress/e2e/journeys/flows/bubble-direct-launch-uploaded.cy.ts b/cypress/e2e/journeys/flows/bubble-direct-launch-uploaded.cy.ts index 7148ba1b..985ff943 100644 --- a/cypress/e2e/journeys/flows/bubble-direct-launch-uploaded.cy.ts +++ b/cypress/e2e/journeys/flows/bubble-direct-launch-uploaded.cy.ts @@ -6,6 +6,7 @@ import { applyPreLaunchFileSettings, assertBubbleReady, assertSessionAfterLaunchCounts, + continuePastMissingNodeCoverageWarning, ensurePreLaunchProfileSynced, visitAppAndAcceptEula, } from '../../../support/journey-helpers'; @@ -31,6 +32,7 @@ const launchProfileDirectToBubble = (profile: DatasetProfile): void => { applyPreLaunchFileSettings(profile); ensurePreLaunchProfileSynced(profile); cy.get('#launch', { timeout: 15000 }).should('not.be.disabled').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.window({ timeout: 120000 }) .its('commonService.session.network.isFullyLoaded') .should('equal', true); diff --git a/cypress/e2e/journeys/flows/map-direct-launch-uploaded.cy.ts b/cypress/e2e/journeys/flows/map-direct-launch-uploaded.cy.ts index 15716556..ee51a40d 100644 --- a/cypress/e2e/journeys/flows/map-direct-launch-uploaded.cy.ts +++ b/cypress/e2e/journeys/flows/map-direct-launch-uploaded.cy.ts @@ -7,6 +7,7 @@ import { assertMapReady, assertMapRenderedCounts, assertSessionAfterLaunchCounts, + continuePastMissingNodeCoverageWarning, ensurePreLaunchProfileSynced, openMapSettingsDialog, selectMapField, @@ -33,6 +34,7 @@ const launchProfileDirectToMap = (profile: DatasetProfile): void => { useOfflineMapLayers({ surviveLaunchReset: true }); cy.get('#launch', { timeout: 15000 }).should('not.be.disabled').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.get('#loading-information', { timeout: 120000 }).should('not.exist'); cy.window({ timeout: 120000 }) .its('commonService.session.network.isFullyLoaded') diff --git a/cypress/e2e/journeys/flows/twod-mixed-origin-coloring.cy.ts b/cypress/e2e/journeys/flows/twod-mixed-origin-coloring.cy.ts index 211bc8aa..88cd722b 100644 --- a/cypress/e2e/journeys/flows/twod-mixed-origin-coloring.cy.ts +++ b/cypress/e2e/journeys/flows/twod-mixed-origin-coloring.cy.ts @@ -4,6 +4,7 @@ import { getProfile } from '../datasets/profile'; import { applyPreLaunchFileSettings, assertAfterLaunchCounts, + continuePastMissingNodeCoverageWarning, ensurePreLaunchProfileSynced, ensureTwoDNetworkView, launchAndWaitForProcessing, @@ -229,6 +230,7 @@ describe('Journey Flow - 2D mixed-origin link coloring', () => { } else { cy.get('[data-testid="files-update-button"]', { timeout: 20000 }).should('not.be.disabled').click({ force: true }); } + continuePastMissingNodeCoverageWarning(); waitForProcessingDialogToClear(); ensureTwoDNetworkView(); }; diff --git a/cypress/e2e/view-state/dashboard-layout-core.cy.ts b/cypress/e2e/view-state/dashboard-layout-core.cy.ts index b88905b1..cb72b851 100644 --- a/cypress/e2e/view-state/dashboard-layout-core.cy.ts +++ b/cypress/e2e/view-state/dashboard-layout-core.cy.ts @@ -89,7 +89,11 @@ describe('Dashboard View State - Golden Layout mechanics', () => { }); }); + cy.window().then((win) => { + win.commonService.visuals.bubble.goldenLayoutComponentResize(); + }); closeDashboardTab('Bubble'); + cy.wait(250); assertOpenDashboardTabs(['2D Network', 'Map', 'Table']); focusDashboardTab('2D Network'); diff --git a/cypress/e2e/view-state/heatmap-view.cy.ts b/cypress/e2e/view-state/heatmap-view.cy.ts index 1615a52e..a54d83a8 100644 --- a/cypress/e2e/view-state/heatmap-view.cy.ts +++ b/cypress/e2e/view-state/heatmap-view.cy.ts @@ -81,7 +81,7 @@ const assertHeatmapColorbarFormat = (expectedPercentageFormat: boolean): void => describe('Heatmap View', () => { beforeEach(() => { - visitAppAndAcceptEula({ skipDemoSession: false }); + visitAppAndAcceptEula({ skipDemoSession: false, dismissWelcomeOverlay: true }); cy.window({ timeout: 60000 }) .its('commonService.session.network.isFullyLoaded') .should('equal', true); diff --git a/cypress/e2e/view-state/map-view.cy.ts b/cypress/e2e/view-state/map-view.cy.ts index 3499aa3a..12184bb3 100644 --- a/cypress/e2e/view-state/map-view.cy.ts +++ b/cypress/e2e/view-state/map-view.cy.ts @@ -611,7 +611,7 @@ describe('Map View', () => { }, end)) container.dispatchEvent(mm1); - const me1 = new MouseEvent('mouseend', Object.assign({ + const me1 = new MouseEvent('mouseup', Object.assign({ bubbles: true, cancelable: true, composed: true, pointerId: 1, pointerType: 'mouse', isPrimary: true, button: 0 }, end)) @@ -628,13 +628,13 @@ describe('Map View', () => { cy.get('#centerMapButton').click({ force: true }); cy.wait(500); - cy.window().then((win: any) => { + cy.window().should((win: any) => { const lmap = win.commonService.visuals.gisMap.lmap; const c = lmap.getCenter(); newCenter = { lat: c.lat, lng: c.lng }; - const latDiff = Math.abs(newCenter.lat - initialCenter.lat); - const lngDiff = Math.abs(newCenter.lng - initialCenter.lng); - expect(latDiff < 0.05 && lngDiff < 0.05).to.equal(true); + const zoom = lmap.getZoom(); + const pixelDiff = lmap.project(newCenter, zoom).distanceTo(lmap.project(initialCenter, zoom)); + expect(pixelDiff, 'center difference in rendered map pixels').to.be.at.most(1.5); }) }); diff --git a/cypress/e2e/view-state/timeline-epicurve-view.cy.ts b/cypress/e2e/view-state/timeline-epicurve-view.cy.ts index 6bffec49..51ece2dc 100644 --- a/cypress/e2e/view-state/timeline-epicurve-view.cy.ts +++ b/cypress/e2e/view-state/timeline-epicurve-view.cy.ts @@ -1,3 +1,9 @@ +import { + goToEpiCurveView, + setGlobalLinkThreshold, + visitAppAndAcceptEula, +} from '../../support/journey-helpers'; + describe('Epi Curve / Timeline View', () => { const selectors = { container: '#epiCurve', @@ -6,15 +12,9 @@ describe('Epi Curve / Timeline View', () => { }; beforeEach(() => { - cy.visit('/'); - cy.wait(6000); - - cy.contains('button', 'Continue with Sample Dataset', { timeout: 10000 }).click({ force: true }); - cy.get('#overlay').should('not.be.visible', { timeout: 10000 }); - - cy.contains('button', 'View').click(); - cy.contains('button[mat-menu-item]', 'Epi Curve').click(); - + visitAppAndAcceptEula({ skipDemoSession: false, dismissWelcomeOverlay: true }); + goToEpiCurveView(); + cy.get(selectors.container, { timeout: 15000 }).should('be.visible'); }); @@ -129,24 +129,16 @@ describe('Epi Curve / Timeline View', () => { let expectedClusterCount = 2 cy.contains('#global-settings-modal .nav-link', 'Filtering').click(); - for (let i = 0; i < 6; i++) { - cy.get('#link-threshold').type('{uparrow}'); - } - - cy.wait(2000); - cy.window().then((win: any) => { - expect(win.commonService.session.style.widgets['link-threshold']).to.eq(22); - expect(win.commonService.session.data.clusters.length).to.eq(expectedClusterCount) - }); + setGlobalLinkThreshold(22); + cy.window() + .its('commonService.session.data.clusters') + .should('have.length', expectedClusterCount); cy.get('#epiCurveSVG .epiCurve-epi-curve circle') - .its('length') - .then((count) => { - expect(count).to.be.eq(expectedClusterCount); - }); + .should('have.length', expectedClusterCount); cy.get('#epiCurveSVG .epiCurve-epi-curve rect') - .then(($rects) => { + .should(($rects) => { const rectFills = [...$rects] .map((c) => c.getAttribute('fill') || '') .filter(Boolean) @@ -155,25 +147,17 @@ describe('Epi Curve / Timeline View', () => { expect(rectFills.some((fill) => fill === '#b732cc' || fill === 'rgb(183,50,204)' || fill === 'rgb(183, 50, 204)')).to.equal(false); }); - for (let i = 0; i < 8; i++) { - cy.get('#link-threshold').type('{downarrow}'); - } - - cy.wait(2000); - cy.window().then((win: any) => { - expectedClusterCount = 4 - expect(win.commonService.session.style.widgets['link-threshold']).to.eq(14); - expect(win.commonService.session.data.clusters.length).to.eq(expectedClusterCount) - }); + expectedClusterCount = 4 + setGlobalLinkThreshold(14); + cy.window() + .its('commonService.session.data.clusters') + .should('have.length', expectedClusterCount); cy.get('#epiCurveSVG .epiCurve-epi-curve circle') - .its('length') - .then((count) => { - expect(count).to.be.eq(expectedClusterCount); - }); + .should('have.length', expectedClusterCount); cy.get('#epiCurveSVG .epiCurve-epi-curve rect') - .then(($rects) => { + .should(($rects) => { const rectFills = [...$rects] .map((c) => c.getAttribute('fill') || '') .filter(Boolean) @@ -200,28 +184,20 @@ describe('Epi Curve / Timeline View', () => { cy.window() .its('commonService.GlobalSettingsModel.SelectedNodeColorTableTypesVariable') .should('equal', 'Show'); - cy.get('#key-tables-node-table').should('be.visible'); + cy.get('#node-color-table:visible').should('be.visible'); cy.contains('#global-settings-modal .nav-link', 'Filtering').click(); let expectedClusterCount = 4 - for (let i = 0; i < 2; i++) { - cy.get('#link-threshold').type('{downarrow}'); - } - - cy.wait(2000); - cy.window().then((win: any) => { - expect(win.commonService.session.style.widgets['link-threshold']).to.eq(14); - expect(win.commonService.session.data.clusters.length).to.eq(expectedClusterCount) - }); + setGlobalLinkThreshold(14); + cy.window() + .its('commonService.session.data.clusters') + .should('have.length', expectedClusterCount); cy.get('#epiCurveSVG .epiCurve-epi-curve circle') - .its('length') - .then((count) => { - expect(count).to.be.eq(expectedClusterCount); - }); + .should('have.length', expectedClusterCount); cy.get('#epiCurveSVG .epiCurve-epi-curve rect') - .then(($rects) => { + .should(($rects) => { const rectFills = [...$rects] .map((c) => c.getAttribute('fill') || '') .filter(Boolean) @@ -231,9 +207,9 @@ describe('Epi Curve / Timeline View', () => { expect(rectFills.some((fill) => fill === '#b732cc' || fill === 'rgb(183,50,204)' || fill === 'rgb(183, 50, 204)')).to.equal(true); }); - cy.get('#key-tables-node-table td input').first().invoke('val', '#777777').trigger('input').trigger('change'); + cy.get('#node-color-table:visible td input').first().invoke('val', '#777777').trigger('input').trigger('change'); cy.get('#epiCurveSVG .epiCurve-epi-curve rect') - .then(($rects) => { + .should(($rects) => { const rectFills = [...$rects] .map((c) => c.getAttribute('fill') || '') .filter(Boolean) @@ -249,7 +225,7 @@ describe('Epi Curve / Timeline View', () => { cy.get('#node-color').invoke('val', '#ff0000').trigger('input'); cy.get('#epiCurveSVG .epiCurve-epi-curve rect') - .then(($rects) => { + .should(($rects) => { const rectFills = [...$rects] .map((c) => c.getAttribute('fill') || '') .filter(Boolean) @@ -378,16 +354,26 @@ describe('Epi Curve / Timeline View', () => { cy.contains('.p-dialog-title', 'Epi Curve Settings') .parents('.p-dialog') .within(() => { - cy.contains('label', field) - .parents('.form-group.row') - .first() - .find('p-select') - .click(); + cy.contains('label', field) + .parents('.form-group.row') + .first() + .find('p-select') + .as('epiFieldSelect') + .click(); }); - cy.get('.p-select-option') - .contains(value) - .click(); + // PrimeNG keeps off-screen options in the active listbox so Cypress can + // scroll them into view. Scope to the newest listbox without filtering + // the option itself through jQuery's stricter :visible semantics. + cy.get('[role="listbox"]') + .last() + .within(() => { + cy.contains('.p-select-option', value).click(); + }); + + cy.get('@epiFieldSelect') + .find('[role="combobox"]') + .should('not.have.attr', 'aria-expanded', 'true'); const normalize = (s: string) => s.replace(/_/g, '').toLowerCase(); let widgetLocation: string; @@ -409,7 +395,7 @@ describe('Epi Curve / Timeline View', () => { cy.window() .its(widgetLocation) - .then(widgetValue => + .should(widgetValue => expect(normalize(widgetValue)).to.equal(normalize(value)) ) @@ -421,9 +407,8 @@ describe('Epi Curve / Timeline View', () => { cy.get('#epiCurveSVG .epiCurve-epi-curve text') .eq(textBoxNumber) - .invoke('text') - .then((textValue) => { - expect(normalize(textValue)).to.equal(normalize(value)); + .should(($text) => { + expect(normalize($text.text())).to.equal(normalize(value)); }); } } @@ -499,7 +484,7 @@ describe('Epi Curve / Timeline View', () => { .should('contain.text', `Date (${binSizeLabel} Bins)`); cy.get('#epiCurveSVG .epiCurve-epi-curve rect') - .then(($rects) => { + .should(($rects) => { const nextRectCount = $rects.length; const nextFirstRectWidth = Number($rects.first().attr('width') || 0); @@ -553,7 +538,7 @@ describe('Epi Curve / Timeline View', () => { .its('commonService.session.style.widgets.epiCurve-cumulative') .should('equal', cumulative); - cy.get('#epiCurveSVG .epiCurve-epi-curve rect').then(($rects) => { + cy.get('#epiCurveSVG .epiCurve-epi-curve rect').should(($rects) => { const nextHeights = [...$rects].map((rect) => Number(rect.getAttribute('height') || 0)); expect(nextHeights.length, 'rect count after cumulative toggle').to.be.greaterThan(1); expect(nextHeights.length, 'rect count should stay stable after cumulative toggle').to.equal(previousHeights.length); diff --git a/cypress/fixtures/missing-node-coverage-extra-node.csv b/cypress/fixtures/missing-node-coverage-extra-node.csv new file mode 100644 index 00000000..3cc9364d --- /dev/null +++ b/cypress/fixtures/missing-node-coverage-extra-node.csv @@ -0,0 +1,2 @@ +id,metadata +C,gamma diff --git a/cypress/fixtures/missing-node-coverage-links.csv b/cypress/fixtures/missing-node-coverage-links.csv new file mode 100644 index 00000000..af41c76d --- /dev/null +++ b/cypress/fixtures/missing-node-coverage-links.csv @@ -0,0 +1,3 @@ +source,target +A,B +B,D diff --git a/cypress/fixtures/missing-node-coverage-matched-links.csv b/cypress/fixtures/missing-node-coverage-matched-links.csv new file mode 100644 index 00000000..39761f70 --- /dev/null +++ b/cypress/fixtures/missing-node-coverage-matched-links.csv @@ -0,0 +1,2 @@ +source,target +A,B diff --git a/cypress/fixtures/missing-node-coverage-matched-nodes.csv b/cypress/fixtures/missing-node-coverage-matched-nodes.csv new file mode 100644 index 00000000..27111748 --- /dev/null +++ b/cypress/fixtures/missing-node-coverage-matched-nodes.csv @@ -0,0 +1,3 @@ +id,metadata +A,alpha +B,beta diff --git a/cypress/fixtures/missing-node-coverage-nodes.csv b/cypress/fixtures/missing-node-coverage-nodes.csv new file mode 100644 index 00000000..2c913979 --- /dev/null +++ b/cypress/fixtures/missing-node-coverage-nodes.csv @@ -0,0 +1,4 @@ +id,metadata +A,alpha +B,beta +C,gamma diff --git a/cypress/support/crosstab-helpers.ts b/cypress/support/crosstab-helpers.ts index de9ceab4..0fec0878 100644 --- a/cypress/support/crosstab-helpers.ts +++ b/cypress/support/crosstab-helpers.ts @@ -257,7 +257,16 @@ export const selectCrosstabField = ( .should('be.visible') .click({ force: true }); - cy.contains('li[role="option"]', label, { timeout: 15000 }).click({ force: true }); + cy.contains('li[role="option"]:visible', label, { timeout: 15000 }) + .should('be.visible') + .click({ force: true }); + + // PrimeNG keeps the listbox DOM measurable during its close transition, + // so jQuery's :visible is not a reliable completion signal. Wait for the + // owning control to report that it is no longer expanded instead. + cy.get(dialogAlias) + .find(`#${selectId} [role="combobox"]`) + .should('not.have.attr', 'aria-expanded', 'true'); cy.window() .its(`commonService.session.style.widgets.${expectedWidgetKey}`) diff --git a/cypress/support/journey-helpers.ts b/cypress/support/journey-helpers.ts index 3f8454ee..a733da51 100644 --- a/cypress/support/journey-helpers.ts +++ b/cypress/support/journey-helpers.ts @@ -681,9 +681,27 @@ export function applyPreLaunchFileSettings(profile: DatasetProfile): void { }); } +export function continuePastMissingNodeCoverageWarning(): void { + cy.get('body').then(($body) => { + const coverageDialog = $body + .find(byTestId(testIds.filesMissingNodeCoverageDialog)) + .filter(':visible') + .closest('.p-dialog'); + + if (!coverageDialog.length) { + return; + } + + cy.wrap(coverageDialog) + .find(byTestId(testIds.filesMissingNodeCoverageContinue)) + .click({ force: true }); + }); +} + export function launchAndWaitForProcessing(timeout = 60000): void { cy.get('#launch', { timeout: 15000 }).should('not.be.disabled'); cy.get('#launch').click({ force: true }); + continuePastMissingNodeCoverageWarning(); cy.get('#loading-information', { timeout }).should('not.exist'); cy.window({ timeout }) .its('commonService.session.network.isFullyLoaded') @@ -785,9 +803,22 @@ export function assertCrosstabReady(timeout = 30000): void { .should('exist'); } +function focusOpenedView(viewName: string, timeout = 15000): void { + cy.window({ timeout }) + .should((win: unknown) => { + const app = (win as WinWithMT).commonService?.visuals?.microbeTrace; + const openViews = (app?.homepageTabs || []).map((tab: { label: string }) => tab.label); + expect(openViews, 'open dashboard views').to.include(viewName); + }) + .then((win: unknown) => { + const app = (win as WinWithMT).commonService.visuals.microbeTrace; + app.Viewclick(viewName); + }); +} + export function assertEpiCurveReady(timeout = 30000): void { - cy.get('#epiCurve', { timeout }).should('be.visible'); - cy.get('#epiCurveSVG', { timeout }).should('be.visible'); + cy.get('#epiCurve:visible', { timeout }).should('be.visible'); + cy.get('#epiCurve:visible #epiCurveSVG', { timeout }).should('be.visible'); cy.window({ timeout }) .its('commonService.visuals.epiCurve') .should('exist'); @@ -802,8 +833,8 @@ export function assertWaterfallReady(timeout = 30000): void { } export function assertHeatmapReady(timeout = 30000): void { - cy.get('#heatmap', { timeout }).should('be.visible'); - cy.get('#heatmap svg.main-svg', { timeout }).should('exist'); + cy.get('#heatmap:visible', { timeout }).should('be.visible'); + cy.get('#heatmap:visible svg.main-svg', { timeout }).should('exist'); cy.window({ timeout }) .its('commonService.visuals.heatmap') .should('exist'); @@ -915,6 +946,7 @@ export function goToEpiCurveView(): void { cy.get(byTestId(testIds.appViewMenuButton), { timeout: 15000 }).click({ force: true }); cy.contains('button[mat-menu-item]', 'Epi Curve', { timeout: 15000 }).click({ force: true }); + focusOpenedView('Epi Curve'); assertEpiCurveReady(); } @@ -929,6 +961,7 @@ export function goToHeatmapView(): void { cy.get(byTestId(testIds.appViewMenuButton), { timeout: 15000 }).click({ force: true }); cy.contains('button[mat-menu-item]', 'Heatmap', { timeout: 15000 }).click({ force: true }); + focusOpenedView('Heatmap'); assertHeatmapReady(); } diff --git a/cypress/support/selectors.ts b/cypress/support/selectors.ts index d3c53c0c..c31a1c7f 100644 --- a/cypress/support/selectors.ts +++ b/cypress/support/selectors.ts @@ -20,6 +20,12 @@ export const testIds = { filesSmartLaunchButton: 'files-smart-launch-button', filesSmartLaunchInfo: 'files-smart-launch-info', filesSmartLaunchTooltip: 'files-smart-launch-tooltip', + filesUpdateButton: 'files-update-button', + filesMissingNodeCoverageDialog: 'files-missing-node-coverage-dialog', + filesMissingNodeCoverageCancel: 'files-missing-node-coverage-cancel', + filesMissingNodeCoverageContinue: 'files-missing-node-coverage-continue', + filesMissingNodeDataSection: 'files-missing-node-data-section', + filesMissingLinkInformationSection: 'files-missing-link-information-section', filterEpsilon: 'filter-epsilon', filterMinimumClusterSize: 'filter-minimum-cluster-size', filterRevealEverything: 'filter-reveal-everything', diff --git a/examples/issue-1728-node-coverage/01-mismatch-links.csv b/examples/issue-1728-node-coverage/01-mismatch-links.csv new file mode 100644 index 00000000..3f3fcb3f --- /dev/null +++ b/examples/issue-1728-node-coverage/01-mismatch-links.csv @@ -0,0 +1,3 @@ +source,target,relationship +A,B,contact +B,D,contact diff --git a/examples/issue-1728-node-coverage/01-mismatch-nodes.csv b/examples/issue-1728-node-coverage/01-mismatch-nodes.csv new file mode 100644 index 00000000..2c913979 --- /dev/null +++ b/examples/issue-1728-node-coverage/01-mismatch-nodes.csv @@ -0,0 +1,4 @@ +id,metadata +A,alpha +B,beta +C,gamma diff --git a/examples/issue-1728-node-coverage/02-matched-links.csv b/examples/issue-1728-node-coverage/02-matched-links.csv new file mode 100644 index 00000000..a3c0316f --- /dev/null +++ b/examples/issue-1728-node-coverage/02-matched-links.csv @@ -0,0 +1,2 @@ +source,target,relationship +A,B,contact diff --git a/examples/issue-1728-node-coverage/02-matched-nodes.csv b/examples/issue-1728-node-coverage/02-matched-nodes.csv new file mode 100644 index 00000000..27111748 --- /dev/null +++ b/examples/issue-1728-node-coverage/02-matched-nodes.csv @@ -0,0 +1,3 @@ +id,metadata +A,alpha +B,beta diff --git a/examples/issue-1728-node-coverage/03-topology-only-links.csv b/examples/issue-1728-node-coverage/03-topology-only-links.csv new file mode 100644 index 00000000..e1ddc8db --- /dev/null +++ b/examples/issue-1728-node-coverage/03-topology-only-links.csv @@ -0,0 +1,3 @@ +source,target,relationship +A,B,contact +B,C,contact diff --git a/examples/issue-1728-node-coverage/04-node-only-nodes.csv b/examples/issue-1728-node-coverage/04-node-only-nodes.csv new file mode 100644 index 00000000..2c913979 --- /dev/null +++ b/examples/issue-1728-node-coverage/04-node-only-nodes.csv @@ -0,0 +1,4 @@ +id,metadata +A,alpha +B,beta +C,gamma diff --git a/examples/issue-1728-node-coverage/05-standalone-sequences.fasta b/examples/issue-1728-node-coverage/05-standalone-sequences.fasta new file mode 100644 index 00000000..d091c28c --- /dev/null +++ b/examples/issue-1728-node-coverage/05-standalone-sequences.fasta @@ -0,0 +1,6 @@ +>A +ACGTACGT +>B +ACGTACGA +>C +ACGTTCGA diff --git a/examples/issue-1728-node-coverage/06-remapped-links.csv b/examples/issue-1728-node-coverage/06-remapped-links.csv new file mode 100644 index 00000000..ae4c8a9c --- /dev/null +++ b/examples/issue-1728-node-coverage/06-remapped-links.csv @@ -0,0 +1,3 @@ +from_id,to_id,relationship +X,Y,contact +Y,W,contact diff --git a/examples/issue-1728-node-coverage/06-remapped-nodes.csv b/examples/issue-1728-node-coverage/06-remapped-nodes.csv new file mode 100644 index 00000000..03f255c1 --- /dev/null +++ b/examples/issue-1728-node-coverage/06-remapped-nodes.csv @@ -0,0 +1,4 @@ +sample_code,group +X,one +Y,one +Z,two diff --git a/examples/issue-1728-node-coverage/07-undeclared-endpoint.graphml b/examples/issue-1728-node-coverage/07-undeclared-endpoint.graphml new file mode 100644 index 00000000..64e8f028 --- /dev/null +++ b/examples/issue-1728-node-coverage/07-undeclared-endpoint.graphml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/examples/issue-1728-node-coverage/08-matched-tree.newick b/examples/issue-1728-node-coverage/08-matched-tree.newick new file mode 100644 index 00000000..db7ea1ac --- /dev/null +++ b/examples/issue-1728-node-coverage/08-matched-tree.newick @@ -0,0 +1 @@ +(A:0.1,(B:0.2,C:0.3):0.1); diff --git a/examples/issue-1728-node-coverage/09-unmatched-enrichment.geojson b/examples/issue-1728-node-coverage/09-unmatched-enrichment.geojson new file mode 100644 index 00000000..4454fcf7 --- /dev/null +++ b/examples/issue-1728-node-coverage/09-unmatched-enrichment.geojson @@ -0,0 +1,16 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [-84.39, 33.75] + }, + "properties": { + "id": "Z", + "label": "Enrichment-only identifier" + } + } + ] +} diff --git a/examples/issue-1728-node-coverage/10-self-links-links.csv b/examples/issue-1728-node-coverage/10-self-links-links.csv new file mode 100644 index 00000000..7c752b68 --- /dev/null +++ b/examples/issue-1728-node-coverage/10-self-links-links.csv @@ -0,0 +1,5 @@ +source,target,relationship +A,A,self +A,B,contact +B,B,self +C,C,self diff --git a/examples/issue-1728-node-coverage/10-self-links-nodes.csv b/examples/issue-1728-node-coverage/10-self-links-nodes.csv new file mode 100644 index 00000000..2c913979 --- /dev/null +++ b/examples/issue-1728-node-coverage/10-self-links-nodes.csv @@ -0,0 +1,4 @@ +id,metadata +A,alpha +B,beta +C,gamma diff --git a/examples/issue-1728-node-coverage/11-case-sensitive-links.csv b/examples/issue-1728-node-coverage/11-case-sensitive-links.csv new file mode 100644 index 00000000..a3c0316f --- /dev/null +++ b/examples/issue-1728-node-coverage/11-case-sensitive-links.csv @@ -0,0 +1,2 @@ +source,target,relationship +A,B,contact diff --git a/examples/issue-1728-node-coverage/11-case-sensitive-nodes.csv b/examples/issue-1728-node-coverage/11-case-sensitive-nodes.csv new file mode 100644 index 00000000..086ecd76 --- /dev/null +++ b/examples/issue-1728-node-coverage/11-case-sensitive-nodes.csv @@ -0,0 +1,3 @@ +id,metadata +a,lowercase +B,uppercase diff --git a/examples/issue-1728-node-coverage/README.md b/examples/issue-1728-node-coverage/README.md new file mode 100644 index 00000000..78240ed7 --- /dev/null +++ b/examples/issue-1728-node-coverage/README.md @@ -0,0 +1,128 @@ +# Issue 1728: Node Coverage Test Data + +Upload the files for one scenario at a time from the Files view. For CSV files, +confirm the listed file type and field mappings before launching. + +## 1. Mismatch in both directions + +Files: + +- `01-mismatch-nodes.csv` as Node List, ID = `id` +- `01-mismatch-links.csv` as Link List, Source = `source`, Target = `target` + +Expected warning: + +- Missing node data (1): `D` +- Missing link information (1): `C` + +Cancel should leave both files loaded and the current network unchanged. Continue +should launch a network containing the identifiers represented by ingestion. + +## 2. Fully matched node and link files + +Files: + +- `02-matched-nodes.csv` as Node List, ID = `id` +- `02-matched-links.csv` as Link List, Source = `source`, Target = `target` + +Expected: no coverage warning. + +## 3. Topology-only link file + +File: + +- `03-topology-only-links.csv` as Link List, Source = `source`, Target = `target` + +Expected warning: + +- Missing node data (3): `A`, `B`, `C` +- No missing-link-information section + +## 4. Node-only file + +File: + +- `04-node-only-nodes.csv` as Node List, ID = `id` + +Expected: no coverage warning because there is no explicit topology source. + +## 5. Standalone FASTA + +File: + +- `05-standalone-sequences.fasta` as FASTA + +Expected: no coverage warning because sequence-derived links do not exist yet. + +## 6. Custom field mappings + +Files: + +- `06-remapped-nodes.csv` as Node List, ID = `sample_code` +- `06-remapped-links.csv` as Link List, Source = `from_id`, Target = `to_id` + +Expected warning: + +- Missing node data (1): `W` +- Missing link information (1): `Z` + +## 7. GraphML with an undeclared edge endpoint + +File: + +- `07-undeclared-endpoint.graphml` as GraphML + +Expected warning: + +- Missing node data (1): `C` +- Missing link information (1): `A` + +This verifies that declared GraphML nodes and edge endpoints are analyzed +separately. + +## 8. Newick tree + +File: + +- `08-matched-tree.newick` as Newick + +Expected: no coverage warning. Newick leaves count as both node and topology IDs. + +## 9. GeoJSON remains enrichment-only + +Files: + +- `02-matched-nodes.csv` as Node List, ID = `id` +- `02-matched-links.csv` as Link List, Source = `source`, Target = `target` +- `09-unmatched-enrichment.geojson` as GeoJSON + +Expected: no coverage warning. The unmatched identifier `Z` in the GeoJSON file +does not participate in coverage analysis. + +## 10. Self-links are ignored + +Files: + +- `10-self-links-nodes.csv` as Node List, ID = `id` +- `10-self-links-links.csv` as Link List, Source = `source`, Target = `target` + +Expected warning: + +- No missing-node-data section +- Missing link information (1): `C` + +The `A-A`, `B-B`, and `C-C` rows are ignored. Only the valid `A-B` topology +contributes endpoint IDs. + +## 11. IDs remain case-sensitive + +Files: + +- `11-case-sensitive-nodes.csv` as Node List, ID = `id` +- `11-case-sensitive-links.csv` as Link List, Source = `source`, Target = `target` + +Expected warning: + +- Missing node data (1): `A` +- Missing link information (1): `a` + diff --git a/src/app/contactTraceCommonServices/import-node-coverage.service.spec.ts b/src/app/contactTraceCommonServices/import-node-coverage.service.spec.ts new file mode 100644 index 00000000..3263927e --- /dev/null +++ b/src/app/contactTraceCommonServices/import-node-coverage.service.spec.ts @@ -0,0 +1,314 @@ +import * as XLSX from 'xlsx'; +import { GraphMLService } from './graphml.service'; +import { + IMPORT_NODE_COVERAGE_DISPLAY_LIMIT, + ImportNodeCoverageService, +} from './import-node-coverage.service'; + +describe('ImportNodeCoverageService', () => { + let service: ImportNodeCoverageService; + + beforeEach(() => { + service = new ImportNodeCoverageService(new GraphMLService()); + }); + + it('finds normalized, deduplicated mismatches in both directions', () => { + const result = service.analyze([ + { + name: 'nodes.csv', + extension: 'csv', + format: 'node', + field1: 'sample', + contents: 'sample,value\n A ,one\nB,two\nB,duplicate\nC,three\n,blank', + }, + { + name: 'links.csv', + extension: 'csv', + format: 'link', + field1: 'from', + field2: 'to', + contents: 'from,to\nA,B\nB,D\nD,D\n,ignored', + }, + ]); + + expect(result.status).toBe('complete'); + if (result.status !== 'complete') return; + + expect(result.summary.missingNodeData).toEqual({ + total: 1, + ids: ['D'], + hidden: 0, + }); + expect(result.summary.missingLinkInformation).toEqual({ + total: 1, + ids: ['C'], + hidden: 0, + }); + }); + + it('honors remapped JSON fields and preserves case', () => { + const result = service.analyze([ + { + name: 'nodes.json', + extension: 'json', + format: 'node', + field1: 'chosenId', + contents: JSON.stringify([ + { ignored: 'wrong-1', chosenId: 'CaseA' }, + { ignored: 'wrong-2', chosenId: 'casea' }, + ]), + }, + { + name: 'links.json', + extension: 'json', + format: 'link', + field1: 'left', + field2: 'right', + contents: JSON.stringify([{ left: 'CaseA', right: 'Other' }]), + }, + ]); + + expect(result.status).toBe('complete'); + if (result.status !== 'complete') return; + + expect(result.summary.missingNodeData.ids).toEqual(['Other']); + expect(result.summary.missingLinkInformation.ids).toEqual(['casea']); + }); + + it('caps each displayed mismatch group at 100 sorted IDs', () => { + const nodeRows = Array.from( + { length: IMPORT_NODE_COVERAGE_DISPLAY_LIMIT + 5 }, + (_, index) => `N${index + 1}`, + ); + const result = service.analyze([ + { + name: 'nodes.csv', + extension: 'csv', + format: 'node', + field1: 'id', + contents: `id\n${nodeRows.join('\n')}`, + }, + { + name: 'links.csv', + extension: 'csv', + format: 'link', + field1: 'source', + field2: 'target', + contents: 'source,target\nA,B', + }, + ]); + + expect(result.status).toBe('complete'); + if (result.status !== 'complete') return; + + expect(result.summary.missingLinkInformation.total).toBe(105); + expect(result.summary.missingLinkInformation.ids.length).toBe(100); + expect(result.summary.missingLinkInformation.hidden).toBe(5); + expect(result.summary.missingLinkInformation.ids.slice(0, 3)).toEqual(['N1', 'N2', 'N3']); + }); + + it('skips node and FASTA inputs when no explicit topology exists', () => { + const result = service.analyze([ + { + name: 'nodes.csv', + extension: 'csv', + format: 'node', + field1: 'id', + contents: 'id\nA\nB', + }, + { + name: 'sequences.fasta', + extension: 'fasta', + format: 'fasta', + contents: '>A\nAAAA\n>B\nCCCC', + }, + { + name: 'locations.geojson', + extension: 'geojson', + format: 'geojson', + contents: '{"type":"FeatureCollection","features":[]}', + }, + ]); + + expect(result).toEqual({ status: 'skipped' }); + }); + + it('treats matrices, Newick, and Auspice leaves as node and topology coverage', () => { + const result = service.analyze([ + { + name: 'matrix.csv', + extension: 'csv', + format: 'matrix', + contents: ',M1,M2\nM1,0,1\nM2,1,0', + }, + { + name: 'tree.nwk', + extension: 'nwk', + format: 'newick', + contents: '(N1:1,N2:1);', + }, + { + name: 'auspice.json', + extension: 'json', + format: 'auspice', + contents: { + tree: { + name: 'root', + children: [{ name: 'A1' }, { name: 'A2' }], + }, + }, + }, + ]); + + expect(result.status).toBe('complete'); + if (result.status !== 'complete') return; + + expect(result.summary.missingNodeData.total).toBe(0); + expect(result.summary.missingLinkInformation.total).toBe(0); + }); + + it('reads remapped Excel node and link identifiers', () => { + const makeWorkbook = (rows: Record[]) => { + const workbook = XLSX.utils.book_new(); + XLSX.utils.book_append_sheet(workbook, XLSX.utils.json_to_sheet(rows), 'Sheet1'); + return XLSX.write(workbook, { type: 'array', bookType: 'xlsx' }); + }; + const result = service.analyze([ + { + name: 'nodes.xlsx', + extension: 'xlsx', + format: 'node', + field1: 'chosen', + contents: makeWorkbook([{ other: 'wrong', chosen: 'X1' }]), + }, + { + name: 'links.xlsx', + extension: 'xlsx', + format: 'link', + field1: 'left', + field2: 'right', + contents: makeWorkbook([{ left: 'X1', right: 'X2' }]), + }, + ]); + + expect(result.status).toBe('complete'); + if (result.status !== 'complete') return; + + expect(result.summary.missingNodeData.ids).toEqual(['X2']); + expect(result.summary.missingLinkInformation.total).toBe(0); + }); + + it('compares declared nodes and endpoints in supported network documents', () => { + const result = service.analyze([ + { + name: 'network.dot', + extension: 'dot', + format: 'network', + contents: 'graph G { A; B; C; A -- B; }', + }, + ]); + + expect(result.status).toBe('complete'); + if (result.status !== 'complete') return; + + expect(result.summary.missingNodeData.total).toBe(0); + expect(result.summary.missingLinkInformation.ids).toEqual(['C']); + }); + + it('does not count generated network endpoint nodes as declarations', () => { + const result = service.analyze([ + { + name: 'network.graphml', + extension: 'graphml', + format: 'network', + contents: [ + '', + '', + '', + '', + '', + '', + ].join(''), + }, + ]); + + expect(result.status).toBe('complete'); + if (result.status !== 'complete') return; + + expect(result.summary.missingNodeData.ids).toEqual(['endpoint-only']); + expect(result.summary.missingLinkInformation.total).toBe(0); + }); + + it('analyzes each supported network document family through the shared importer', () => { + const documents = [ + { + name: 'network.graphml', + contents: '', + }, + { + name: 'network.gexf', + contents: '', + }, + { + name: 'network.xgmml', + contents: '', + }, + { + name: 'network.cx2', + contents: JSON.stringify([ + { CXVersion: '2.0' }, + { nodes: [{ id: 1 }, { id: 2 }, { id: 3 }] }, + { edges: [{ id: 1, s: 1, t: 2 }] }, + { status: [{ success: true }] }, + ]), + }, + { + name: 'network.dot', + contents: 'graph G { 1; 2; 3; 1 -- 2; }', + }, + { + name: 'network.gv', + contents: 'graph G { 1; 2; 3; 1 -- 2; }', + }, + { + name: 'network.gml', + contents: 'graph [ node [ id 1 ] node [ id 2 ] node [ id 3 ] edge [ source 1 target 2 ] ]', + }, + ]; + + documents.forEach(document => { + const result = service.analyze([{ + ...document, + extension: document.name.split('.').pop(), + format: 'network', + }]); + + expect(result.status).withContext(document.name).toBe('complete'); + if (result.status !== 'complete') return; + expect(result.summary.missingNodeData.total).withContext(document.name).toBe(0); + expect(result.summary.missingLinkInformation.ids).withContext(document.name).toEqual(['3']); + }); + }); + + it('returns unavailable instead of a partial result when a contributing file cannot be parsed', () => { + const result = service.analyze([ + { + name: 'nodes.csv', + extension: 'csv', + format: 'node', + field1: 'id', + contents: 'id\nA', + }, + { + name: 'broken.json', + extension: 'json', + format: 'link', + field1: 'source', + field2: 'target', + contents: '{not json}', + }, + ]); + + expect(result.status).toBe('unavailable'); + }); +}); diff --git a/src/app/contactTraceCommonServices/import-node-coverage.service.ts b/src/app/contactTraceCommonServices/import-node-coverage.service.ts new file mode 100644 index 00000000..a0f6d8ca --- /dev/null +++ b/src/app/contactTraceCommonServices/import-node-coverage.service.ts @@ -0,0 +1,309 @@ +import { Injectable } from '@angular/core'; +import * as Papa from 'papaparse'; +import * as patristic from 'patristic'; +import * as XLSX from 'xlsx'; +import { GraphMLService } from './graphml.service'; + +export const IMPORT_NODE_COVERAGE_DISPLAY_LIMIT = 100; + +export interface ImportNodeCoverageGroup { + total: number; + ids: string[]; + hidden: number; +} + +export interface ImportNodeCoverageSummary { + missingNodeData: ImportNodeCoverageGroup; + missingLinkInformation: ImportNodeCoverageGroup; +} + +export type ImportNodeCoverageAnalysis = + | { status: 'complete'; summary: ImportNodeCoverageSummary } + | { status: 'skipped' } + | { status: 'unavailable'; error: unknown }; + +interface ImportCoverageAccumulator { + nodeIds: Set; + topologyIds: Set; + hasExplicitTopology: boolean; +} + +@Injectable({ + providedIn: 'root' +}) +export class ImportNodeCoverageService { + constructor(private graphMLService: GraphMLService) {} + + analyze(files: ReadonlyArray): ImportNodeCoverageAnalysis { + const coverage: ImportCoverageAccumulator = { + nodeIds: new Set(), + topologyIds: new Set(), + hasExplicitTopology: false, + }; + + try { + for (const file of files || []) { + this.addFileCoverage(file, coverage); + } + } catch (error) { + console.warn('Unable to complete the missing-node import preflight.', error); + return { status: 'unavailable', error }; + } + + if (!coverage.hasExplicitTopology) { + return { status: 'skipped' }; + } + + const missingNodeData = Array.from(coverage.topologyIds) + .filter(id => !coverage.nodeIds.has(id)); + const missingLinkInformation = Array.from(coverage.nodeIds) + .filter(id => !coverage.topologyIds.has(id)); + + return { + status: 'complete', + summary: { + missingNodeData: this.summarize(missingNodeData), + missingLinkInformation: this.summarize(missingLinkInformation), + } + }; + } + + private addFileCoverage(file: any, coverage: ImportCoverageAccumulator): void { + const format = String(file?.format ?? '').toLowerCase(); + + switch (format) { + case 'node': + this.addNodeTableCoverage(file, coverage); + return; + case 'link': + this.addLinkTableCoverage(file, coverage); + return; + case 'matrix': + this.addMatrixCoverage(file, coverage); + return; + case 'fasta': + this.addFastaCoverage(file, coverage); + return; + case 'newick': + this.addNewickCoverage(file, coverage); + return; + case 'auspice': + this.addAuspiceCoverage(file, coverage); + return; + case 'network': + case 'graphml': + this.addNetworkDocumentCoverage(file, coverage); + return; + // GeoJSON enriches nodes that another source creates. Session/style and + // unknown files likewise do not define either side of this comparison. + case 'geojson': + default: + return; + } + } + + private addNodeTableCoverage(file: any, coverage: ImportCoverageAccumulator): void { + const idField = String(file?.field1 ?? ''); + this.readTableRecords(file).forEach(record => { + this.addId(coverage.nodeIds, record?.[idField]); + }); + } + + private addLinkTableCoverage(file: any, coverage: ImportCoverageAccumulator): void { + coverage.hasExplicitTopology = true; + const sourceField = String(file?.field1 ?? ''); + const targetField = String(file?.field2 ?? ''); + + this.readTableRecords(file).forEach(record => { + this.addTopologyPair( + coverage.topologyIds, + record?.[sourceField], + record?.[targetField], + ); + }); + } + + private addMatrixCoverage(file: any, coverage: ImportCoverageAccumulator): void { + coverage.hasExplicitTopology = true; + const rows = this.readMatrixRows(file); + if (!rows.length) { + return; + } + + const ids = new Set(); + rows[0].slice(1).forEach(value => this.addId(ids, value)); + rows.slice(1).forEach(row => this.addId(ids, row?.[0])); + ids.forEach(id => { + coverage.nodeIds.add(id); + coverage.topologyIds.add(id); + }); + } + + private addFastaCoverage(file: any, coverage: ImportCoverageAccumulator): void { + const contents = String(file?.contents ?? ''); + contents.split(/[\r\n]+/g).forEach(line => { + if (line.startsWith('>')) { + this.addId(coverage.nodeIds, line.slice(1)); + } + }); + } + + private addNewickCoverage(file: any, coverage: ImportCoverageAccumulator): void { + coverage.hasExplicitTopology = true; + const tree = patristic.parseNewick(String(file?.contents ?? '')); + tree.getLeaves().forEach(leaf => { + const id = this.normalizeId(leaf?.id); + if (!id) { + return; + } + + coverage.nodeIds.add(id); + coverage.topologyIds.add(id); + }); + } + + private addAuspiceCoverage(file: any, coverage: ImportCoverageAccumulator): void { + coverage.hasExplicitTopology = true; + const contents = typeof file?.contents === 'string' + ? JSON.parse(file.contents) + : file?.contents; + const root = contents?.tree; + if (!root || typeof root !== 'object') { + throw new Error(`Auspice file ${file?.name || ''} does not contain a tree.`); + } + + const visit = (node: any): void => { + const children = Array.isArray(node?.children) ? node.children : []; + if (!children.length) { + const id = this.normalizeId(node?.name); + if (id) { + coverage.nodeIds.add(id); + coverage.topologyIds.add(id); + } + return; + } + + children.forEach(visit); + }; + + visit(root); + } + + private addNetworkDocumentCoverage(file: any, coverage: ImportCoverageAccumulator): void { + coverage.hasExplicitTopology = true; + const document = this.graphMLService.importNetworkDocument(file?.contents, { + sourceName: file?.name, + }); + + document.nodes + .filter(node => node?.mt_generated_endpoint !== true && node?.dot_implicit_node !== true) + .forEach(node => this.addId(coverage.nodeIds, node?._id ?? node?.id)); + document.links.forEach(link => { + this.addTopologyPair(coverage.topologyIds, link?.source, link?.target); + }); + } + + private readTableRecords(file: any): any[] { + const extension = this.getExtension(file); + if (extension === 'xls' || extension === 'xlsx') { + const workbook = XLSX.read(file?.contents, { + type: 'array', + cellDates: true, + dateNF: 'mm/dd/yyyy', + }); + return XLSX.utils.sheet_to_json( + workbook.Sheets[workbook.SheetNames[0]], + { raw: false, dateNF: 'mm/dd/yyyy' }, + ); + } + + if (extension === 'json') { + const parsed = typeof file?.contents === 'string' + ? JSON.parse(file.contents) + : file?.contents; + if (Array.isArray(parsed)) { + return parsed; + } + return parsed && typeof parsed === 'object' ? [parsed] : []; + } + + const result = Papa.parse>(String(file?.contents ?? ''), { + header: true, + dynamicTyping: true, + skipEmptyLines: true, + }); + return result.data; + } + + private readMatrixRows(file: any): any[][] { + const extension = this.getExtension(file); + if (extension === 'xls' || extension === 'xlsx') { + const workbook = XLSX.read(file?.contents, { + type: 'array', + cellDates: true, + dateNF: 'mm/dd/yyyy', + }); + return XLSX.utils.sheet_to_json( + workbook.Sheets[workbook.SheetNames[0]], + { header: 1, raw: true, dateNF: 'mm/dd/yyyy' }, + ) as any[][]; + } + + const result = Papa.parse(String(file?.contents ?? ''), { + skipEmptyLines: 'greedy', + }); + return result.data; + } + + private getExtension(file: any): string { + const explicit = String(file?.extension ?? '').toLowerCase(); + if (explicit) { + return explicit; + } + + const name = String(file?.name ?? ''); + return name.includes('.') ? name.split('.').pop()?.toLowerCase() || '' : ''; + } + + private addTopologyPair(ids: Set, rawSource: any, rawTarget: any): void { + const source = this.normalizeId(rawSource); + const target = this.normalizeId(rawTarget); + if (!source || !target || source === target) { + return; + } + + ids.add(source); + ids.add(target); + } + + private addId(ids: Set, rawValue: any): void { + const id = this.normalizeId(rawValue); + if (id) { + ids.add(id); + } + } + + private normalizeId(value: any): string | null { + if (value === null || value === undefined) { + return null; + } + + const id = String(value).trim(); + return id.length ? id : null; + } + + private summarize(ids: string[]): ImportNodeCoverageGroup { + const sorted = [...ids].sort((left, right) => left.localeCompare( + right, + 'en', + { numeric: true, sensitivity: 'variant' }, + )); + const displayed = sorted.slice(0, IMPORT_NODE_COVERAGE_DISPLAY_LIMIT); + + return { + total: sorted.length, + ids: displayed, + hidden: sorted.length - displayed.length, + }; + } +} diff --git a/src/app/filesComponent/files-plugin.component.html b/src/app/filesComponent/files-plugin.component.html index dc61af2c..ea4b8dcf 100644 --- a/src/app/filesComponent/files-plugin.component.html +++ b/src/app/filesComponent/files-plugin.component.html @@ -314,6 +314,79 @@

Please add data files to load...

+ + @if (missingNodeCoverageSummary) { +
+

+ Some identifiers are not represented in both the imported node data and the imported link or topology data. + You can continue with the available data or cancel and correct the uploaded files. + Please upload a corrected dataset. +

+ + @if (missingNodeCoverageSummary.missingNodeData.total > 0) { +
+

Missing node data ({{ missingNodeCoverageSummary.missingNodeData.total }})

+

These identifiers occur in link or topology data but not in node data.

+
    + @for (nodeId of missingNodeCoverageSummary.missingNodeData.ids; track nodeId) { +
  1. {{ nodeId }}
  2. + } +
+ @if (missingNodeCoverageSummary.missingNodeData.hidden > 0) { +

+ {{ missingNodeCoverageSummary.missingNodeData.hidden }} additional IDs not shown. +

+ } +
+ } + + @if (missingNodeCoverageSummary.missingLinkInformation.total > 0) { +
+

Missing link information ({{ missingNodeCoverageSummary.missingLinkInformation.total }})

+

These identifiers occur in node data but not in link or topology data.

+
    + @for (nodeId of missingNodeCoverageSummary.missingLinkInformation.ids; track nodeId) { +
  1. {{ nodeId }}
  2. + } +
+ @if (missingNodeCoverageSummary.missingLinkInformation.hidden > 0) { +

+ {{ missingNodeCoverageSummary.missingLinkInformation.hidden }} additional IDs not shown. +

+ } +
+ } +
+ } + + +
+ + +
+
+
+