Merge pull request #133 from hed-standard/value-blacklist

Refactor interim value validation
hed-standard · Feb 24, 2024 · 65ebfdb · 65ebfdb
2 parents 18443fc + 57552e3
commit 65ebfdb
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 6 deletions.
diff --git a/parser/main.js b/parser/main.js
@@ -14,8 +14,7 @@ const delimiters = new Set([','])
  */
 const substituteCharacters = function (hedString) {
   const issues = []
-  const illegalCharacterMap = { '\0': ['ASCII NUL', ' '] }
-  const flaggedCharacters = /[^\w\d./$ :-]/g
+  const illegalCharacterMap = { '\0': ['ASCII NUL', ' '], '\t': ['Tab', ' '] }
   const replaceFunction = function (match, offset) {
     if (match in illegalCharacterMap) {
       const [name, replacement] = illegalCharacterMap[match]
@@ -31,7 +30,7 @@ const substituteCharacters = function (hedString) {
       return match
     }
   }
-  const fixedString = hedString.replace(flaggedCharacters, replaceFunction)
+  const fixedString = hedString.replace(/./g, replaceFunction)
 
   return [fixedString, issues]
 }

diff --git a/parser/parsedHedTag.js b/parser/parsedHedTag.js
@@ -233,6 +233,11 @@ export class ParsedHed3Tag extends ParsedHedTag {
    * @param {string} schemaName The label of this tag's schema in the dataset's schema spec.
    */
   _convertTag(hedString, hedSchemas, schemaName) {
+    const hed3ValidCharacters = /^[^{}[\]()~,\0\t]+$/
+    if (!hed3ValidCharacters.test(this.originalTag)) {
+      throw new Error('The parser failed to properly remove an illegal or special character.')
+    }
+
     if (hedSchemas.isSyntaxOnly) {
       this.canonicalTag = this.originalTag
       this.conversionIssues = []

diff --git a/parser/splitHedString.js b/parser/splitHedString.js
@@ -390,7 +390,6 @@ const checkTagForInvalidCharacters = function (hedString, tagSpec, tag, invalidS
   for (let i = 0; i < tag.length; i++) {
     const character = tag.charAt(i)
     if (invalidSet.has(character)) {
-      tagSpec.invalidCharacter = true
       issues.push(
         generateIssue('invalidCharacter', {
           character: character,

diff --git a/tests/event.spec.js b/tests/event.spec.js
@@ -235,6 +235,7 @@ describe('HED string and event validation', () => {
       it('should substitute and warn for certain illegal characters', () => {
         const testStrings = {
           nul: '/Attribute/Object side/Left,/Participant/Effect/Body part/Arm\0',
+          tab: '/Attribute/Object side/Left,/Participant/Effect/Body part/Arm\t',
         }
         const expectedIssues = {
           nul: [
@@ -244,6 +245,13 @@ describe('HED string and event validation', () => {
               string: testStrings.nul,
             }),
           ],
+          tab: [
+            generateIssue('invalidCharacter', {
+              character: 'Tab',
+              index: 61,
+              string: testStrings.tab,
+            }),
+          ],
         }
         // No-op function as this check is done during the parsing stage.
         // eslint-disable-next-line no-unused-vars

diff --git a/validator/event/hed3.js b/validator/event/hed3.js
@@ -19,12 +19,14 @@ const topLevelTagGroupType = 'topLevelTagGroup'
 export class Hed3Validator extends HedValidator {
   /**
    * The parsed definitions.
+   *
    * @type {Map<string, ParsedHedGroup>}
    */
   definitions
 
   /**
    * Constructor.
+   *
    * @param {ParsedHedString} parsedString The parsed HED string to be validated.
    * @param {Schemas} hedSchemas The collection of HED schemas.
    * @param {Map<string, ParsedHedGroup>} definitions The parsed definitions.
@@ -126,6 +128,7 @@ export class Hed3Validator extends HedValidator {
 
   /**
    * Check that the unit is valid for the tag's unit class.
+   *
    * @param {ParsedHed3Tag} tag A HED tag.
    */
   checkIfTagUnitClassUnitsAreValid(tag) {
@@ -281,6 +284,7 @@ export class Hed3Validator extends HedValidator {
 
   /**
    * Validate a unit and strip it from the value.
+   *
    * @param {ParsedHed3Tag} tag A HED tag.
    * @returns {[boolean, boolean, string]} Whether a unit was found, whether it was valid, and the stripped value.
    */
@@ -337,6 +341,8 @@ export class Hed3Validator extends HedValidator {
    *
    * @param {string} value The stripped value.
    * @param {boolean} isNumeric Whether the tag is numeric.
+   * @returns {boolean} Whether the stripped value is valid.
+   * @todo This function is a placeholder until support for value classes is implemented.
    */
   validateValue(value, isNumeric) {
     if (value === '#') {
@@ -346,8 +352,8 @@ export class Hed3Validator extends HedValidator {
     if (isNumeric) {
       return isNumber(value)
     }
-    const hed3ValidValueCharacters = /^[-a-zA-Z0-9.$%^+_; ]+$/
-    return hed3ValidValueCharacters.test(value)
+    // TODO: Placeholder.
+    return true
   }
 
   /**