add support for endIndex in node metadata (5.x edition)

docs/v4, v5/2.XMLparseOptions.md

@@ -1515,12 +1515,15 @@ The MetaData object is not available for nodes that resolve as strings or arrays
 
 ```js
 const parser = new XMLParser({ignoreAttributes: false, captureMetaData: true});
-const jsonObj = parser.parse(`<root><thing name="zero"/><thing name="one"/></root>`);
+const xml = `<root><thing name="zero"/><thing name="one"/></root>`;
+const jsonObj = parser.parse(xml);
 const META_DATA_SYMBOL = XMLParser.getMetaDataSymbol();
-// get the char offset of the start of the tag for <thing name="zero"/>
+// get the char offsets of the tag <thing name="zero"/>
 const thingZero = jsonObj.root.thing[0];
 const thingZeroMetaData = thingZero[META_DATA_SYMBOL];
 const thingZeroStartIndex = thingZeroMetaData.startIndex; // 6
+const thingZeroEndIndex = thingZeroMetaData.endIndex;     // 26
+xml.slice(thingZeroStartIndex, thingZeroEndIndex);        // '<thing name="zero"/>'
 ```
 
 [> Next: XmlBuilder](./3.XMLBuilder.md)

spec/endIndex_spec.js

@@ -0,0 +1,97 @@
+
+import { XMLParser } from "../src/fxp.js";
+
+const XML_METADATA = XMLParser.getMetaDataSymbol();
+
+/** Collect [rawTagName, metadata] for every node carrying metadata, in document order. */
+function collectMeta(node, out = []) {
+    if (node === null || typeof node !== "object") return out;
+    if (node[XML_METADATA]) {
+        const tag = Object.keys(node).find((k) => k !== ":@" && k !== "#text");
+        out.push([tag, node[XML_METADATA]]);
+    }
+    if (Array.isArray(node)) {
+        node.forEach((n) => collectMeta(n, out));
+    } else {
+        for (const k of Object.keys(node)) collectMeta(node[k], out);
+    }
+    return out;
+}
+
+/** Parse xml with metadata capture on and return a Map of rawTagName -> raw text span. */
+function parseSpans(xml) {
+  const parser = new XMLParser({ preserveOrder: true, ignoreAttributes: false, captureMetaData: true });
+  const result = parser.parse(xml);
+  return new Map(collectMeta(result).map(([tag, m]) => [tag, xml.slice(m.startIndex, m.endIndex)]));
+}
+
+describe("XMLParser captureMetaData endIndex", function () {
+  it("does not add metadata (start or end) when captureMetaData is off", function () {
+    const xml = `<root><child/></root>`;
+    const parser = new XMLParser({ preserveOrder: true, ignoreAttributes: false });
+    const result = parser.parse(xml);
+    expect(collectMeta(result).length).toBe(0);
+  });
+
+  it("records an exclusive endIndex", function () {
+        const xml = `<root><foo/><bar type="quux"/><baz type="foo">FOO</baz></root>`;
+        const parser = new XMLParser({ preserveOrder: true, ignoreAttributes: false, captureMetaData: true });
+        const result = parser.parse(xml);
+
+        const meta = collectMeta(result);
+        const spans = meta.map(([tag, m]) => [tag, xml.slice(m.startIndex, m.endIndex)]);
+
+        expect(spans).toEqual([
+            ["root", `<root><foo/><bar type="quux"/><baz type="foo">FOO</baz></root>`],
+            ["foo", `<foo/>`],
+            ["bar", `<bar type="quux"/>`],
+            ["baz", `<baz type="foo">FOO</baz>`],
+        ]);
+    });
+
+    it("covers self-closing elements", function () {
+        const xml = `<a><c/></a>`;
+        expect(parseSpans(xml).get("c")).toBe(`<c/>`);
+    });
+
+    it("covers paired elements with text content", function () {
+        const xml = `<a><d>text</d></a>`;
+        expect(parseSpans(xml).get("d")).toBe(`<d>text</d>`);
+    });
+
+    it("covers elements with inline attributes", function () {
+        const xml = `<a><e id="1" name="foo"/><f class="bar">text</f></a>`;
+        const byTag = parseSpans(xml);
+        expect(byTag.get("e")).toBe(`<e id="1" name="foo"/>`);
+        expect(byTag.get("f")).toBe(`<f class="bar">text</f>`);
+    });
+
+    it("covers deeply nested elements", function () {
+        const xml = `<a><b><c/></b><d>text</d></a>`;
+        const byTag = parseSpans(xml);
+        expect(byTag.get("a")).toBe(`<a><b><c/></b><d>text</d></a>`);
+        expect(byTag.get("b")).toBe(`<b><c/></b>`);
+    });
+
+    it("covers processing-instruction nodes", function () {
+        const xml = `<?xml version="1.0"?><a><?pi target?><c/></a>`;
+        const byTag = parseSpans(xml);
+        expect(byTag.get("?xml")).toBe(`<?xml version="1.0"?>`);
+        expect(byTag.get("?pi")).toBe(`<?pi target?>`);
+    });
+
+    it("does not corrupt a sibling's endIndex when updateTag drops a node", function () {
+        const xml = `<a><b>x</b><skip/><?skip pi?><skip>y</skip></a>`;
+        const parser = new XMLParser({
+            preserveOrder: true,
+            ignoreAttributes: false,
+            captureMetaData: true,
+            updateTag: (tagName) => (tagName === "skip" || tagName === "?skip" ? false : tagName),
+        });
+        const result = parser.parse(xml);
+
+        const byTag = new Map(collectMeta(result).map(([tag, m]) => [tag, xml.slice(m.startIndex, m.endIndex)]));
+        expect(byTag.get("b")).toBe(`<b>x</b>`);
+        expect(byTag.get("a")).toBe(xml);
+    });
+});

spec/startIndex_spec.js

@@ -9,22 +9,22 @@ describe("XMLParser", function () {
     it("should support captureMetadata && !preserveOrder", function () {
         const expected = {
             root: {
-                [XML_METADATA]: { startIndex: 0 },
+                [XML_METADATA]: { startIndex: 0, endIndex: 79 },
                 foo: '',
                 bar: [
                     {
-                        [XML_METADATA]: { startIndex: 12 },
+                        [XML_METADATA]: { startIndex: 12, endIndex: 30 },
                         "@_type": 'quux'
                     },
                     {
-                        [XML_METADATA]: { startIndex: 30 },
+                        [XML_METADATA]: { startIndex: 30, endIndex: 47 },
                         "@_type": 'bat'
                     },
                 ],
                 baz: {
                     '@_type': 'foo',
                     '#text': 'FOO',
-                    [XML_METADATA]: {startIndex: 47},
+                    [XML_METADATA]: { startIndex: 47, endIndex: 72 },
                 }
             }
         };
@@ -38,24 +38,24 @@ describe("XMLParser", function () {
         const expected = [
             {
                 root: [
-                    { foo: [], [XML_METADATA]: { startIndex: 6 } },
+                    { foo: [], [XML_METADATA]: { startIndex: 6, endIndex: 12 } },
                     {
                         bar: [],
                         ':@': { "@_type": 'quux' },
-                        [XML_METADATA]: { startIndex: 12 },
+                        [XML_METADATA]: { startIndex: 12, endIndex: 30 },
                     },
                     {
                         bar: [],
                         ':@': { "@_type": 'bat' },
-                        [XML_METADATA]: { startIndex: 30 },
+                        [XML_METADATA]: { startIndex: 30, endIndex: 47 },
                     },
                     {
                         baz: [{ '#text': 'FOO' }],
                         ':@': { '@_type': 'foo' },
-                        [XML_METADATA]: {startIndex: 47},
+                        [XML_METADATA]: { startIndex: 47, endIndex: 72 },
                     },
                 ],
-                [XML_METADATA]: { startIndex: 0 },
+                [XML_METADATA]: { startIndex: 0, endIndex: 79 },
             }
         ];
 
@@ -69,28 +69,28 @@ describe("XMLParser", function () {
     it("should support captureMetadata && isArray && stopNodes && unpairedTags && updateTag", function () {
         const expected = {
             ROOT: {
-                [XML_METADATA]: { startIndex: 0 },
+                [XML_METADATA]: { startIndex: 0, endIndex: 138 },
                 foo: [''],
                 bar: [
                     {
-                        [XML_METADATA]: { startIndex: 12 },
+                        [XML_METADATA]: { startIndex: 12, endIndex: 30 },
                         "@_type": 'quux'
                     },
                     {
-                        [XML_METADATA]: { startIndex: 30 },
+                        [XML_METADATA]: { startIndex: 30, endIndex: 47 },
                         "@_type": 'bat'
                     },
                 ],
                 baz: {
                     '#text': 'FOO',
                     '@_type': 'foo',
-                    [XML_METADATA]: { startIndex: 47 },
+                    [XML_METADATA]: { startIndex: 47, endIndex: 72 },
                 },
                 // no metadata on stop nodes.
                 stop: 'This is a <b>stop</b> node.',
                 unpaired: {
                     '@_attr': '1',
-                    [XML_METADATA]: { startIndex: 112 },
+                    [XML_METADATA]: { startIndex: 112, endIndex: 131 },
                 }
             }
         };
@@ -100,7 +100,7 @@ describe("XMLParser", function () {
             isArray(tagName) {
                 return (tagName == 'foo');
             },
-            stopNodes: [ 'root.stop' ], unpairedTags: ['unpaired'], 
+            stopNodes: [ 'root.stop' ], unpairedTags: ['unpaired'],
             updateTag(tagName) {
                 if (tagName === 'root') {
                     tagName = 'ROOT';

src/fxp.d.ts

@@ -749,4 +749,6 @@ export class XMLBuilder {
 export interface XMLMetaData {
   /** The index, if available, of the character where the XML node began in the input stream. */
   startIndex?: number;
+  /** The index, if available, of the character where the XML node ended in the input stream. */
+  endIndex?: number;
 }

src/xmlparser/OrderedObjParser.js

@@ -335,6 +335,10 @@ const parseXml = function (xmlData) {
         this.isCurrentNodeStopNode = false; // Reset flag when closing tag
 
         currentNode = this.tagsNodeStack.pop();//avoid recursion, set the parent tag scope
+
+        if (options.captureMetaData && currentNode) {
+          currentNode.addEndIndex(closeIndex + 1);
+        }
         textData = "";
         i = closeIndex;
       } else if (c1 === 63) { //'?'
@@ -360,6 +364,11 @@ const parseXml = function (xmlData) {
             childNode[":@"] = attsMap
           }
           this.addChild(currentNode, childNode, this.readonlyMatcher, i);
+
+          if (options.captureMetaData) {
+            // closeIndex points at '?' of the closing '?>'
+            currentNode.addEndIndex(tagData.closeIndex + 2);
+          }
         }
 
 
@@ -522,6 +531,10 @@ const parseXml = function (xmlData) {
           this.isCurrentNodeStopNode = false; // Reset flag
 
           this.addChild(currentNode, childNode, this.readonlyMatcher, startIndex);
+
+          if (options.captureMetaData) {
+            currentNode.addEndIndex(i + 1);
+          }
         } else {
           //selfClosing tag
           if (isSelfClosing) {
@@ -532,6 +545,10 @@ const parseXml = function (xmlData) {
               childNode[":@"] = prefixedAttrs;
             }
             this.addChild(currentNode, childNode, this.readonlyMatcher, startIndex);
+
+            if (options.captureMetaData) {
+              currentNode.addEndIndex(closeIndex + 1);
+            }
             this.matcher.pop(); // Pop self-closing tag
             this.isCurrentNodeStopNode = false; // Reset flag
           }
@@ -541,6 +558,10 @@ const parseXml = function (xmlData) {
               childNode[":@"] = prefixedAttrs;
             }
             this.addChild(currentNode, childNode, this.readonlyMatcher, startIndex);
+
+            if (options.captureMetaData) {
+              currentNode.addEndIndex(result.closeIndex + 1);
+            }
             this.matcher.pop(); // Pop unpaired tag
             this.isCurrentNodeStopNode = false; // Reset flag
             i = result.closeIndex;

src/xmlparser/xmlNode.js

@@ -27,12 +27,26 @@ export default class XmlNode {
       this.child.push({ [node.tagname]: node.child });
     }
     // if requested, add the startIndex
+    this.addStartIndex(startIndex);
+  }
+
+  addStartIndex(startIndex) {
     if (startIndex !== undefined) {
       // Note: for now we just overwrite the metadata. If we had more complex metadata,
       // we might need to do an object append here:  metadata = { ...metadata, startIndex }
       this.child[this.child.length - 1][METADATA_SYMBOL] = { startIndex };
     }
   }
+
+  addEndIndex(endIndex) {
+    const lastChild = this.child[this.child.length - 1];
+    // endIndex is write-once: when updateTag drops a node, the last child is a
+    // previously completed sibling whose endIndex must not be overwritten
+    if (lastChild !== undefined && lastChild[METADATA_SYMBOL] !== undefined
+      && lastChild[METADATA_SYMBOL].endIndex === undefined) {
+      lastChild[METADATA_SYMBOL].endIndex = endIndex;
+    }
+  }
   /** symbol used for metadata */
   static getMetaDataSymbol() {
     return METADATA_SYMBOL;