Changeset 6471

Timestamp:

Nov 4, 2021, 2:16:10 PM (15 months ago)

Author:

Nicklas Nordborg

Message:

Fixes #1348: Improvments to the Map external data wizard

Implemented support for using tab-, comma- and semicolon-separated files. Auto-detection is made when a file is selected but it is possible to manually override.

Location:

extensions/net.sf.basedb.reggie/branches/4.33-stable

Files:

• resources/personal/map-external-data.js (modified) (5 diffs)
• resources/personal/map-external-data.jsp (modified) (2 diffs)
• src/net/sf/basedb/reggie/servlet/ExportServlet.java (modified) (5 diffs)

extensions/net.sf.basedb.reggie/branches/4.33-stable/resources/personal/map-external-data.js

@@ -56 +56 @@
     var lineNo = 0;
     var mapFrom = null;
+    var separator = null;
     var preview = '';
     while (lineNo < lines.length && lineNo < 5)
     {
       var line = lines[lineNo];
-      if (lineNo < 3) 
+      if (lineNo < 4) 
       {
         preview += Strings.encodeTags(line) + '\n';
       }
+      if (separator == null) separator = map.detectSeparator(line);
       if (mapFrom == null) 
       {
-        if (line.match(/^[QCS]\d+\t.*$/))// We look for lines starting with 'Q', 'C' or 'S' followed by {digits} and a {tab}
+        if (line.match(/^[QCS]\d+([\t,;].*)?$/))// We look for lines starting with 'Q', 'C' or 'S' followed by {digits} and a {tab/comma/semicolon}
         {
           var firstChar = line.charAt(0);
@@ -82 +84 @@
           }
         }
-        else if (lineNo == 0 && line.match(/^[^\t]*pad/i)) // If the first header column contains 'pad' (ignoring case) we assume it contains PAD values
+        else if (lineNo == 0 && line.match(/^[^\t,;]*pad/i)) // If the first header column contains 'pad' (ignoring case) we assume it contains PAD values
         {
           mapFrom = 'PAD';
@@ -89 +91 @@
       lineNo++;
     }
-    if (lines.length > 3) preview += '<div id="more-lines">+' + (lines.length-3) + ' more lines...</div>';
+    if (lines.length > 4) preview += '<div id="more-lines">+' + (lines.length-4) + ' more lines...</div>';
+    frm.separator.value = separator ? separator.id : 'tab';
     frm.mapFrom.value = mapFrom || 'PersonalNo';
     Events.sendChangeEvent(frm.mapFrom);
@@ -95 +98 @@
   }
 
+  /**
+    Detect the "best" separator to use by selecting the one 
+    that appear the most times in the given text line. If no
+    separator is found null is returned.
+  */
+  map.detectSeparator = function(text)
+  {
+    var separators =
+    [
+      { 'id': 'tab', 'test': /\t/g, 'sep': '\t'},
+      { 'id': 'comma', 'test': /,/g, 'sep': ','},
+      { 'id': 'semicolon', 'test': /;/g, 'sep': ';'}
+    ];
+    var maxCount = 0;
+    var bestMatch = 0;
+    for (var i=0; i < separators.length; i++)
+    {
+      var sep = separators[i];
+      var m = text.match(sep.test);
+      if (m && m.length > maxCount)
+      {
+        maxCount = m.length;
+        bestMatch = i;
+      }
+    }
+    return maxCount == 0 ? null : separators[bestMatch];
+  }
   
   map.validateStep1 = function(event)
@@ -113 +143 @@
     var url = '../Export.servlet?ID='+App.getSessionId();
     url += '&cmd=MapDataToReleases';
+    url += '&separator='+frm.separator.value;
     url += '&mapFrom='+Strings.encodeTags(frm.mapFrom.value);
     url += '&exportCaseId='+(frm.exportCaseId.checked ? 1 : 0);

extensions/net.sf.basedb.reggie/branches/4.33-stable/resources/personal/map-external-data.jsp

@@ -81 +81 @@
         <td class="help" rowspan="2"><span id="file.message" class="message"></span>
           <ul>
-          <li>Select a tab-separated text file in UTF-8 encoding. 
+          <li>Select a tab-, comma-, or semicolon-separated text file in UTF-8 encoding. 
           <li>The first line should contain column headers.
           <li>The first column should contain the personal number, or a release id value as specified by the <b>Map from</b> option.
@@ -93 +93 @@
         <td>
           <div id="preview"><i>No file loaded...</i></div>
+        </td>
+        <td></td>
+      </tr>
+      <tr>
+        <td class="prompt">Separator</td>
+        <td>
+          <select name="separator" id="separator">
+            <option value="tab">Tab
+            <option value="comma">Comma
+            <option value="semicolon">Semi-colon
+          </select>
         </td>
         <td></td>

extensions/net.sf.basedb.reggie/branches/4.33-stable/src/net/sf/basedb/reggie/servlet/ExportServlet.java

@@ -538 +538 @@
         ReggieRole.checkPermission(dc, "'" + cmd + "' wizard", ReggieRole.PATIENT_CURATOR, ReggieRole.ADMINISTRATOR);
         
+        String separator = Values.getString(req.getParameter("separator"), "tab");
         String mapFrom = Values.getString(req.getParameter("mapFrom"), "PersonalNo");
         boolean exportCaseId = Values.getBoolean(req.getParameter("exportCaseId"));
@@ -544 +545 @@
         boolean sortOutput = Values.getBoolean(req.getParameter("sortOutput"));
 
+        String separatorChar = "\t";
+        if ("comma".equals(separator))
+        {
+          separatorChar = ",";
+        }
+        else if ("semicolon".equals(separator))
+        {
+          separatorChar = ";";
+        }
+        
         SnapshotManager manager = new SnapshotManager();
         List<OutputColumn> outColumns = new ArrayList<>();
@@ -605 +616 @@
           String header = uploadReader.readLine();
           long parsedCharacters = header.length()+1;
-          String[] dataIn = header.split("\t", 2);
+          String[] dataIn = header.split(separatorChar, 2);
           if (dataIn.length == 2) outColumns.add(new RestOfData(dataIn[1]));
         
@@ -627 +638 @@
             progress.display((int)(100*parsedCharacters / totalBytes), "Working... " + inputLine.lineNo + " lines parsed so far...");
             
-            inputLine.cols = data.split("\t", 2);
+            inputLine.cols = data.split(separatorChar, 2);
             inputLine.fromId = Values.getStringOrNull(inputLine.cols[0]);
             if (inputLine.fromId == null)
@@ -669 +680 @@
           tmpOut = new CipherOutputStream(Application.getStaticCache().write(outputFilePath, 5), cipher);
           tmpWriter = new TableWriter(new OutputStreamWriter(tmpOut, "UTF-8"));
+          tmpWriter.setDataSeparator(separatorChar);
           tmpWriter.tablePrintData(headerOut);
           for (Object[] outLines : dataOut)