TDML zipping 1272

.github/workflows/documentaion.yml

@@ -40,7 +40,7 @@ jobs:
           script: |
             const prBody = context.payload.pull_request.body || "";
             const checkbox1Text = "- [x] I have determined that no documentation updates are needed for these changes";
-            const checkbox2Text = "- [x] I have added following documentation for these changes";
+            const checkbox2Text = "- [x] I have added the following documentation for these changes";
 
             if (!prBody.includes(checkbox1Text) && !prBody.includes(checkbox2Text)) {
               core.setFailed("❌ Required documentation checkbox not checked. Please check one of the the box before merging.");

doc/Wiki.md

@@ -60,9 +60,8 @@ The purpose of Apache Daffodil™ Extension for Visual Studio Code is to ease th
 - [Data Editor](#data-editor-1)
    * [Navigation](#navigation)
    * [Keyboard Shortcuts](#keyboard-shortcuts)
-- [Known Issues in v1.4.1](#known-issues-in-v141)
+- [Known Issues in v1.6.0](#known-issues-in-v160)
    * [General Issues](#general-issues)
-   * [Debugger Issues Originating from 1.4.0](#debugger-issues-originating-from-140)
 - [Reporting Problems and Requesting New Features](#reporting-problems-and-requesting-new-features)
 - [Getting Help](#getting-help)
 - [Contributing](#contributing)
@@ -468,7 +467,16 @@ The original default test case from the temp directory will be appended to the s
 
 </details>
 
-Once the Daffodil Parse has finished, an infoset will be created, and a test case will be added to the existing TDML file. To create an archive for a TDML file with multiple test cases, the same guidelines for creating an archive from a TDML file created from a 'Generate TDML' operation should be followed. All DFDL schema files, input data files, the TDML file, and, optionally, the infosets should be added to the archive. Additionally, any directory structure should be preserved in the archive to allow for the relative paths in the TDML file to be resolved.
+### Zipping the TDML file & associated files
+
+To generate a compressed archive of the TDML test cases, you simply need to have the TDML file opened (and in the active tab) in either the TDML editor or a text editor. Then you open the command palette and find the "Zip TDML file" command and execute it.
+The system will automatically collect the appropriate files: first the TDML file itself, then the files specified by the test case(s) in the file. Each test case will have its associated schema, data, and infoset files placed into a folder with the name of the test case. The TDML file contents will be automatically edited to show the updated location of these files. 
+
+The zip file created will be in the same location as the TDML file and will have the same name as the TDML file with the added extension ".zip" This file can then be transmitted to other users.
+
+If the execution of the test case contained an error or failed to complete parsing for any reason, the associated infoset will be a zero length file.
+
+**Note:** The TDML function does not currently support other, local, schema files included into a primary or base schema. Further development will be required to scan the primary schema file to identify these types of files so that they are included into the TDML file and any zipped archive created for that TDML file.
 
 When running a zip archive created by another user, extract the archive into your workspace folder. If there is an infoset in the zip archive that you wish to compare with your infoset, make sure that the infoset from the zip archive is not located at the same place as the default infoset for the Daffodil Parse that will be run when executing a test case from the TDML file. This is because the Daffodil Parse run by executing the TDML test case uses the default location for its infoset and will overwrite anything that already exists there.
 
@@ -624,15 +632,12 @@ When using `Single Byte Editing Mode`, `CTRL-ENTER` will insert a byte to the le
 
 When browsing the data in the `Physical` or `Logical` viewports, `Home` will take you to the top of the edited file, `End` will take you to the end of the edited file, `Page-Up` will give you the previous page of the edited file, `Page-Down` will give you the next page of the edited file, `Arrow-Up` will give you the previous line of the edited file, and `Arrow-Down` will give you the next line of the edited file. 
 
-# Known Issues in v1.4.1
+# Known Issues in v1.6.0
 
 ## General Issues
 * Some nightly tests are still failing intermittently due to GitHub runners. 
-* TDML Copy, Execute, and Append Functionality is currently not working on the MacOS Platform 
-
-## Debugger Issues Originating from 1.4.0
-
-* At this time the debugger step into and step out actions have no code behind them, using either button results in an unrecoverable error. We have not found a way to disable the step into and step out buttons. This problem occurs in all Operating Systems. This is [noted as a GitHub Issue](https://github.com/apache/daffodil-vscode/issues/5).
+* TDML Zipping does not support nested/included schema files.
+* TDML cannot handle files that are located on a different drive (Windows). for example, don't run off of C:, then use data or infoset files from D:
 
 # Reporting Problems and Requesting New Features
 

package.json

@@ -68,7 +68,8 @@
     "wait-port": "1.1.0",
     "xdg-app-paths": "8.3.0",
     "xml-formatter": "^3.7.0",
-    "xml-js": "^1.6.11"
+    "xml-js": "^1.6.11",
+    "jszip": "^3.10.1"
   },
   "devDependencies": {
     "@sveltejs/adapter-static": "3.0.8",
@@ -267,6 +268,10 @@
         {
           "command": "extension.dfdl-debug.executeTDML",
           "when": "!inDebugMode && (editorLangId == 'tdml' || activeEditor == 'tdml-editor.editor' || dfdl-debug.tdml-editor-active == 'tdml-editor.editor')"
+        },
+        {
+          "command": "extension.dfdl-debug.zipTDML",
+          "when": "!inDebugMode && (editorLangId == 'tdml' || activeEditor == 'tdml-editor.editor' || dfdl-debug.tdml-editor-active == 'tdml-editor.editor')"
         }
       ],
       "commandPalette": [
@@ -290,6 +295,10 @@
           "command": "extension.dfdl-debug.createTDML",
           "when": "true"
         },
+        {
+          "command": "extension.dfdl-debug.zipTDML",
+          "when": "true"
+        },
         {
           "command": "extension.data.edit",
           "when": "true"
@@ -370,6 +379,12 @@
         "category": "Daffodil Debug",
         "enablement": "!inDebugMode && (editorLangId == 'tdml' || activeEditor == 'tdml-editor.editor' || dfdl-debug.tdml-editor-active == 'tdml-editor.editor')"
       },
+      {
+        "command": "extension.dfdl-debug.zipTDML",
+        "title": "Zip TDML File",
+        "category": "Daffodil Debug",
+        "enablement": "!inDebugMode && (editorLangId == 'tdml' || activeEditor == 'tdml-editor.editor' || dfdl-debug.tdml-editor-active == 'tdml-editor.editor')"
+      },
       {
         "command": "extension.dfdl-debug.createTDML",
         "title": "Create TDML File",

src/adapter/activateDaffodilDebug.ts

@@ -16,6 +16,8 @@ import * as dataEditClient from '../dataEditor'
 import * as tdmlEditor from '../tdmlEditor'
 import * as rootCompletion from '../rootCompletion'
 import { tmpdir } from 'os'
+import JSZip from 'jszip'
+import { rm } from 'node:fs/promises'
 
 import {
   CancellationToken,
@@ -31,6 +33,7 @@ import { handleDebugEvent } from './daffodilEvent'
 import { InlineDebugAdapterFactory } from './extension'
 import {
   appendTestCase,
+  readTDMLFileContents,
   getTmpTDMLFilePath,
   copyTestCase,
   TMP_TDML_FILENAME,
@@ -39,10 +42,33 @@ import xmlFormat from 'xml-formatter'
 import { CommandsProvider } from '../views/commands'
 import * as daffodilDebugErrors from './daffodilDebugErrors'
 import { TDMLProvider } from '../tdmlEditor/TDMLProvider'
+import { getTestCaseDisplayData } from '../tdmlEditor/utilities/tdmlXmlUtils'
 
 export const outputChannel: vscode.OutputChannel =
   vscode.window.createOutputChannel('Daffodil')
 
+async function createDirectory(directoryPath: string): Promise<void> {
+  try {
+    await fs.promises.mkdir(directoryPath, { recursive: true })
+    console.log(`Directory created successfully at ${directoryPath}`)
+  } catch (error) {
+    console.error(`Error creating directory:`, error)
+    throw error
+  }
+}
+
+async function copyFileAsync(src: string, dest: string) {
+  try {
+    // Ensure directory exists first
+    await fs.promises.mkdir(path.dirname(dest), { recursive: true })
+
+    await fs.promises.copyFile(src, dest)
+    console.log(`'${src}' was copied to '${dest}'`)
+  } catch (err) {
+    console.error('Error copying file:', err)
+  }
+}
+
 /** Method to file path for schema and data
  * Details:
  *   Required so that the vscode api commands:
@@ -153,8 +179,11 @@ async function createDebugRunFileConfigs(
   if (!targetResource) {
     if (vscode.window.activeTextEditor) {
       targetResource = vscode.window.activeTextEditor.document.uri
-    } else if (TDMLProvider.getDocumentUri()) {
-      targetResource = TDMLProvider.getDocumentUri()
+    } else {
+      const tdmlUri = TDMLProvider.getDocumentUri()
+      if (tdmlUri) {
+        targetResource = tdmlUri
+      }
     }
   }
 
@@ -323,12 +352,144 @@ export function activateDaffodilDebug(
               console.log(reason)
             })
         }
+      }
+    ),
+    vscode.commands.registerCommand(
+      'extension.dfdl-debug.zipTDML',
+      async (resource: vscode.Uri) => {
+        let targetResource: vscode.Uri | undefined = resource
+        if (vscode.window.activeTextEditor) {
+          targetResource = vscode.window.activeTextEditor.document.uri
+        } else {
+          const tdmlUri = TDMLProvider.getDocumentUri()
+          if (tdmlUri) {
+            targetResource = tdmlUri
+          }
+        }
+
+        const resolvedResource = targetResource
+
+        // create temp zip folder
+        let tmpDir = path.dirname(getTmpTDMLFilePath())
+        const zipDir = path.posix.join(tmpDir, '_zipdir')
+        await createDirectory(zipDir)
+
+        // copy TDML file to zip folder
+        await copyFileAsync(
+          resolvedResource.fsPath,
+          path.posix.join(zipDir, path.basename(resolvedResource.fsPath))
+        )
+
+        // read TDML file to see what files are required...
+        await readTDMLFileContents(
+          path.posix.join(zipDir, path.basename(resolvedResource.fsPath))
+        ).then(async (xmlBuffer) => {
+          await getTestCaseDisplayData(xmlBuffer).then((testSuiteData) => {
+            testSuiteData.testCases.forEach((testCase) => {
+              // create subdir for testcase
+              let testCaseDir = path.posix.join(zipDir, testCase.testCaseName)
+              createDirectory(testCaseDir)
+              // copy schema file
+              let xsdFile = testCase.testCaseModel
+              let xsdFileSrc = path.posix.join(
+                path.dirname(resolvedResource.fsPath),
+                xsdFile
+              )
+              let xsdFileDest = path.posix.join(
+                testCaseDir,
+                path.basename(xsdFile)
+              )
+              copyFileAsync(xsdFileSrc, xsdFileDest)
 
-        // fs.copyFile(
-        //   getTmpTDMLFilePath(),
-        //   targetResource as unknown as string,
-        //   (_) => {}
-        // )
+              // edit path in copied TDML file
+              let updatedBuffer = xmlBuffer.replace(
+                `model="${xsdFile}"`,
+                `model="${path.posix.join(testCase.testCaseName, path.basename(xsdFile))}"`
+              )
+              xmlBuffer = updatedBuffer
+
+              // copy data file
+              testCase.dataDocuments.forEach((dataDocuments) => {
+                let dataFile = path.basename(dataDocuments.trim())
+                let dataFileSrc = path.posix.join(
+                  path.dirname(resolvedResource.fsPath),
+                  dataDocuments.trim()
+                )
+                let dataFileDest = path.posix.join(
+                  testCaseDir,
+                  path.basename(dataFile)
+                )
+                copyFileAsync(dataFileSrc, dataFileDest)
+
+                // edit path in copied TDML file
+                let updatedBuffer = xmlBuffer.replace(
+                  dataDocuments.trim(),
+                  path.posix.join(testCase.testCaseName, dataFile)
+                )
+                xmlBuffer = updatedBuffer
+              })
+
+              // copy infoset file
+              testCase.dfdlInfosets.forEach((dfdlInfosets) => {
+                let infoFile = path.basename(dfdlInfosets.trim())
+                let infoSrc = path.posix.join(
+                  path.dirname(resolvedResource.fsPath),
+                  dfdlInfosets.trim()
+                )
+                let infoDest = path.posix.join(
+                  testCaseDir,
+                  path.basename(dfdlInfosets.trim())
+                )
+                copyFileAsync(infoSrc, infoDest)
+
+                // edit path in copied TDML file
+                let infoUpdatedBuffer = xmlBuffer.replace(
+                  dfdlInfosets.trim(),
+                  path.posix.join(testCase.testCaseName, infoFile)
+                )
+                xmlBuffer = infoUpdatedBuffer
+              })
+            })
+          })
+          // write updated info back to TDML file
+          try {
+            // Synchronously writes data to a file, replacing it if it already exists
+            fs.writeFileSync(
+              path.posix.join(zipDir, path.basename(resolvedResource.fsPath)),
+              xmlBuffer,
+              { encoding: 'utf8' }
+            )
+            console.log('Updated schema file written successfully')
+          } catch (err) {
+            console.error('Error writing updated schema file:', err)
+          }
+        })
+
+        // zip folders
+        const zip = new JSZip()
+
+        // Add the folder content recursively
+        addFolderToZip(zipDir, zip)
+
+        // Generate, save, and then clean up
+        let targetZip = targetResource.fsPath.replace(/tdml$/, 'tdml.zip')
+        try {
+          const content = await zip.generateAsync({ type: 'nodebuffer' })
+          fs.writeFileSync(targetZip, content)
+          console.log(`Zip file written successfully: '${targetZip}'`)
+          vscode.window.showInformationMessage(
+            `Zip file successfully created: '${targetZip}'`
+          )
+          await rm(zipDir, { recursive: true, force: true })
+          console.log(`Temp directory successfully removed: '${zipDir}'`)
+        } catch (err) {
+          console.error(
+            `Error while creating zip file '${targetZip}' or deleting temp directory '${zipDir}': ${err}`
+          )
+          vscode.window.showErrorMessage(
+            `Failed to create zip file: '${targetZip}'`
+          )
+        }
       }
     ),
     vscode.commands.registerCommand(
@@ -677,3 +838,29 @@ export const workspaceFileAccessor: FileAccessor = {
     }
   },
 }
+
+/**
+ * Recursively adds files and folders to a JSZip instance.
+ * @param dirPath The folder to add
+ * @param zip The JSZip instance
+ */
+function addFolderToZip(dirPath: string, zip: JSZip) {
+  const files = fs.readdirSync(dirPath)
+
+  for (const file of files) {
+    const filePath = path.posix.join(dirPath, file)
+    const stats = fs.statSync(filePath)
+
+    if (stats.isDirectory()) {
+      // Create a subfolder in the ZIP and recurse
+      const subFolder = zip.folder(file)
+      if (subFolder) {
+        addFolderToZip(filePath, subFolder)
+      }
+    } else {
+      // Read file data and add to current zip/folder
+      const fileData = fs.readFileSync(filePath)
+      zip.file(file, fileData)
+    }
+  }
+}