Add MobilityData Validator

__tests__/e2e/server/Dockerfile

@@ -1,5 +1,6 @@
 # syntax=docker/dockerfile:1
-FROM maven:3.8.6-openjdk-8
+FROM maven:3.8.7-openjdk-18
+
 WORKDIR /datatools
 
 ARG E2E_AUTH0_USERNAME
@@ -32,6 +33,7 @@ ARG AWS_SECRET_ACCESS_KEY
 
 # Grab latest dev build of Datatools Server
 RUN git clone https://github.com/ibi-group/datatools-server.git
+RUN microdnf install wget
 WORKDIR /datatools/datatools-server
 
 RUN mvn package -DskipTests

__tests__/test-utils/mock-data/manager.js

@@ -340,6 +340,7 @@ export const mockFeedVersion = {
     feedVersionId: 'mock-feed-version-id',
     loadFailureReason: null,
     loadStatus: 'SUCCESS',
+    mobilityDataResult: {},
     routeCount: 10,
     startDate: '20180801',
     stopCount: 237,

lib/manager/components/validation/GtfsValidationViewer.js

@@ -30,6 +30,7 @@ import type {Props as FeedVersionViewerProps} from '../version/FeedVersionViewer
 import type {ValidationResult} from '../../../types'
 
 import ValidationErrorItem from './ValidationErrorItem'
+import MobilityDataValidationResult from './MobilityDataValidationResult'
 
 const DEFAULT_LIMIT = 10
 
@@ -266,9 +267,21 @@ export default class GtfsValidationViewer extends Component<Props, State> {
           </Alert>
         }
         <Panel>
-          <Panel.Heading><Panel.Title componentClass='h2'>{this.messages('title')}</Panel.Title></Panel.Heading>
+          <Panel.Heading>
+            <Panel.Title componentClass='h2'>{this.messages('title')}</Panel.Title>
+          </Panel.Heading>
           {validationContent}
         </Panel>
+        <Panel><Panel.Heading>Mobility Data Validation Issues</Panel.Heading>
+          <ListGroup>
+            {version.mobilityDataResult && version.mobilityDataResult.notices.map(notice => (
+              <MobilityDataValidationResult notice={notice} />
+            ))}
+            {(!version.mobilityDataResult || version.mobilityDataResult.notices.length === 0) && <ListGroupItem className='validation-item'>
+              The MobilityData validator has not produced any errors or warnings. This may be because the validator is still running.
+            </ListGroupItem>}
+          </ListGroup>
+        </Panel>
       </div>
     )
   }

lib/manager/components/validation/MobilityDataValidationResult.js

@@ -0,0 +1,154 @@
+/* eslint-disable jsx-a11y/no-noninteractive-element-interactions */
+/* eslint-disable no-fallthrough */
+import React, { useState } from 'react'
+import Icon from '@conveyal/woonerf/components/icon'
+import { ListGroupItem, Table } from 'react-bootstrap'
+import Markdown from 'markdown-to-jsx'
+
+import toSentenceCase from '../../../common/util/to-sentence-case'
+import {
+  mobilityDataValidationErrorMapping,
+  validationErrorIconLookup
+} from '../../util/version'
+
+import rules from './rules.json'
+
+// from https://stackoverflow.com/a/4149671
+function unCamelCase (s) {
+  return s
+    .split(/(?=[A-Z])/)
+    .join(' ')
+    .toLowerCase()
+}
+
+const NoticeTable = ({ headerOverides = {'stopSequence1': 'Stop seq-uence 1', 'stopSequence2': 'Stop seq-uence 2'}, notices }) => {
+  if (notices.length === 0) return null
+
+  const headers = Object.keys(notices[0])
+
+  return (
+    <Table bordered className='table-fixed' fill hover striped style={{display: 'relative', borderCollapse: 'collapse'}}>
+      <thead>
+        <tr>
+          {headers.map((header) => (
+            <th style={{position: 'sticky', top: 0, background: '#fefefe'}}>
+              {headerOverides[header] || toSentenceCase(unCamelCase(header))}
+            </th>
+          ))}
+        </tr>
+      </thead>
+      <tbody>
+        {notices.map((notice) => (
+          <tr>
+            {headers.map((header, index) => {
+              const FieldWrapper =
+                (header === 'fieldValue' || header === 'message') ? 'pre' : React.Fragment
+
+              let field = notice[header]
+              if (header.endsWith('Km') || header.endsWith('Kph')) {
+                field = Math.round(field)
+              }
+
+              return (
+                <td key={`${header}-${index}`}>
+                  <FieldWrapper>{field}</FieldWrapper>
+                </td>
+              )
+            })}
+          </tr>
+        ))}
+      </tbody>
+    </Table>
+  )
+}
+
+// eslint-disable-next-line complexity
+const renderNoticeDetail = (notice) => {
+  switch (notice.code) {
+    case 'too_many_rows':
+      notice.csvRowNumber = notice.rowNumber
+    case 'fare_transfer_rule_duration_limit_type_without_duration_limit':
+    case 'fare_transfer_rule_duration_limit_without_type':
+    case 'fare_transfer_rule_missing_transfer_count':
+    case 'fare_transfer_rule_with_forbidden_transfer_count':
+      notice.filename = 'fare_transfer_rules.txt'
+    case 'empty_file':
+    case 'emtpy_row':
+    case 'missing_timepoint_column':
+    case 'missing_required_file':
+    case 'missing_recommended_file':
+    case 'unknown_file':
+      return (
+        <ul>
+          {notice.sampleNotices.map((notice) => (
+            <li>
+              {notice.filename}
+              {notice.csvRowNumber && `: row ${notice.csvRowNumber}`}
+            </li>
+          ))}
+        </ul>
+      )
+    default:
+      return (
+        <NoticeTable notices={notice.sampleNotices} />
+      )
+  }
+}
+
+const MobilityDataValidationResult = (props) => {
+  const { notice } = props
+  const rule = rules.find((rd) => rd.rule === notice.code)
+  if (!rule) return null
+
+  const errorClass = `gtfs-error-${mobilityDataValidationErrorMapping[notice.severity]}`
+  const [expanded, setExpanded] = useState(notice.totalNotices < 2)
+
+  const onRowSelect = () => setExpanded(!expanded)
+
+  return (
+    <ListGroupItem style={{ padding: 0 }}>
+      <div style={{ padding: '10px 15px' }}>
+        <h4
+          onClick={onRowSelect}
+          onKeyDown={onRowSelect}
+          style={{ cursor: 'pointer' }}
+        >
+          <span
+            className={`buffer-icon ${errorClass}`}
+            title={`${toSentenceCase(notice.severity)} priority`}
+          >
+            <Icon
+              type={validationErrorIconLookup[mobilityDataValidationErrorMapping[notice.severity]]}
+            />
+          </span>
+          {toSentenceCase(notice.code.replaceAll('_', ' ').toLowerCase())}
+          <span className={errorClass}>
+            {' '}
+            &mdash; {notice.totalNotices} case
+            {notice.totalNotices > 1 ? 's' : ''} found
+          </span>
+          <span className={`pull-right`}>
+            <Icon type={expanded ? 'caret-up' : 'caret-down'} />
+          </span>
+        </h4>
+        {expanded && (
+          <>
+            <p><Markdown>{rule.description}</Markdown></p>
+            <p>
+              <a
+                href={`https://github.com/MobilityData/gtfs-validator/blob/master/RULES.md#${notice.code}`}
+                target='_blank'
+                rel='noreferrer'
+              >
+                More details
+              </a>
+            </p>
+          </>
+        )}
+      </div>
+      <div>{expanded && renderNoticeDetail(notice)}</div>
+    </ListGroupItem>
+  )
+}
+
+export default MobilityDataValidationResult

lib/manager/components/version/FeedVersionViewer.js

@@ -23,7 +23,7 @@ import * as plusActions from '../../../gtfsplus/actions/gtfsplus'
 import ActiveGtfsPlusVersionSummary from '../../../gtfsplus/containers/ActiveGtfsPlusVersionSummary'
 import NotesViewer from '../NotesViewer'
 import TransformationsViewer from '../TransformationsViewer'
-import { errorPriorityLevels, getTableFatalExceptions } from '../../util/version'
+import { errorPriorityLevels, getTableFatalExceptions, mobilityDataValidationErrorMapping } from '../../util/version'
 import GtfsValidationViewer from '../validation/GtfsValidationViewer'
 import type { Feed, FeedVersion, GtfsPlusValidation, Note, Project } from '../../../types'
 import type { GtfsState, ManagerUserState } from '../../../types/reducers'
@@ -32,6 +32,7 @@ import VersionButtonToolbar from './VersionButtonToolbar'
 import VersionDateLabel from './VersionDateLabel'
 import FeedVersionReport from './FeedVersionReport'
 import DeltaStat from './DeltaStat'
+import numeral from 'numeral'
 
 export type Props = {
   comparedVersion?: FeedVersion,
@@ -150,6 +151,7 @@ type SelectorProps = {
 }
 
 class VersionSectionSelector extends Component<SelectorProps> {
+  // eslint-disable-next-line complexity
   _renderIssuesLabel = () => {
     const { comparedVersion, version } = this.props
     const { feedLoadResult, validationResult, validationSummary } = version
@@ -171,9 +173,29 @@ class VersionSectionSelector extends Component<SelectorProps> {
       })
     }
 
-    const text = hasCriticalError
+    // Do the same for the mobility data results
+    let highestMBPriority = 'UNKNOWN'
+    if (version && version.mobilityDataResult) {
+      version.mobilityDataResult.notices.forEach(notice => {
+        const level = errorPriorityLevels[mobilityDataValidationErrorMapping[notice.severity]]
+        if (level < errorPriorityLevels[highestMBPriority]) {
+          highestMBPriority = mobilityDataValidationErrorMapping[notice.severity]
+        }
+      })
+    }
+
+    const dtValidationCount = hasCriticalError
       ? 'critical error'
-      : validationSummary.errorCount
+      : numeral(validationSummary.errorCount).format('0a')
+
+    const mbValidationCount =
+      version.mobilityDataResult &&
+      version.mobilityDataResult.notices &&
+      numeral(
+        version.mobilityDataResult.notices.reduce((prev, cur) => {
+          return prev + cur.totalNotices
+        }, 0)
+      ).format('0a')
 
     let diffLabel
     if (comparedVersion) {
@@ -190,8 +212,11 @@ class VersionSectionSelector extends Component<SelectorProps> {
     return (
       <div style={{ display: 'inline' }}>
         <BsLabel className={`gtfs-error-${highestPriority}-label-solid`}>
-          {text}
+          DT {dtValidationCount}
         </BsLabel>
+        {mbValidationCount && <BsLabel style={{marginLeft: '1ch'}} className={`gtfs-error-${highestMBPriority}-label-solid`}>
+          MD {mbValidationCount}
+        </BsLabel> }
         {diffLabel}
       </div>
     )

lib/manager/containers/__tests__/__snapshots__/DeploymentsPanel.js.snap

@@ -108,6 +108,7 @@ exports[`lib > manager > DeploymentsPanel should render with the list of deploym
                   "feedVersionId": "mock-feed-version-id",
                   "loadFailureReason": null,
                   "loadStatus": "SUCCESS",
+                  "mobilityDataResult": Object {},
                   "routeCount": 10,
                   "startDate": "20180801",
                   "stopCount": 237,
@@ -344,6 +345,7 @@ exports[`lib > manager > DeploymentsPanel should render with the list of deploym
                     "feedVersionId": "mock-feed-version-id",
                     "loadFailureReason": null,
                     "loadStatus": "SUCCESS",
+                    "mobilityDataResult": Object {},
                     "routeCount": 10,
                     "startDate": "20180801",
                     "stopCount": 237,
@@ -637,6 +639,7 @@ exports[`lib > manager > DeploymentsPanel should render with the list of deploym
                             "feedVersionId": "mock-feed-version-id",
                             "loadFailureReason": null,
                             "loadStatus": "SUCCESS",
+                            "mobilityDataResult": Object {},
                             "routeCount": 10,
                             "startDate": "20180801",
                             "stopCount": 237,
@@ -801,6 +804,7 @@ exports[`lib > manager > DeploymentsPanel should render with the list of deploym
                               "feedVersionId": "mock-feed-version-id",
                               "loadFailureReason": null,
                               "loadStatus": "SUCCESS",
+                              "mobilityDataResult": Object {},
                               "routeCount": 10,
                               "startDate": "20180801",
                               "stopCount": 237,
@@ -1153,6 +1157,7 @@ exports[`lib > manager > DeploymentsPanel should render with the list of deploym
                                         "feedVersionId": "mock-feed-version-id",
                                         "loadFailureReason": null,
                                         "loadStatus": "SUCCESS",
+                                        "mobilityDataResult": Object {},
                                         "routeCount": 10,
                                         "startDate": "20180801",
                                         "stopCount": 237,
@@ -1282,6 +1287,7 @@ exports[`lib > manager > DeploymentsPanel should render with the list of deploym
                                             "feedVersionId": "mock-feed-version-id",
                                             "loadFailureReason": null,
                                             "loadStatus": "SUCCESS",
+                                            "mobilityDataResult": Object {},
                                             "routeCount": 10,
                                             "startDate": "20180801",
                                             "stopCount": 237,
@@ -1690,6 +1696,7 @@ exports[`lib > manager > DeploymentsPanel should render with the list of deploym
                                             "feedVersionId": "mock-feed-version-id",
                                             "loadFailureReason": null,
                                             "loadStatus": "SUCCESS",
+                                            "mobilityDataResult": Object {},
                                             "routeCount": 10,
                                             "startDate": "20180801",
                                             "stopCount": 237,