Changeset 6471


Ignore:
Timestamp:
Nov 4, 2021, 2:16:10 PM (15 months ago)
Author:
Nicklas Nordborg
Message:

Fixes #1348: Improvments to the Map external data wizard

Implemented support for using tab-, comma- and semicolon-separated files. Auto-detection is made when a file is selected but it is possible to manually override.

Location:
extensions/net.sf.basedb.reggie/branches/4.33-stable
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.reggie/branches/4.33-stable/resources/personal/map-external-data.js

    r5740 r6471  
    5656    var lineNo = 0;
    5757    var mapFrom = null;
     58    var separator = null;
    5859    var preview = '';
    5960    while (lineNo < lines.length && lineNo < 5)
    6061    {
    6162      var line = lines[lineNo];
    62       if (lineNo < 3)
     63      if (lineNo < 4)
    6364      {
    6465        preview += Strings.encodeTags(line) + '\n';
    6566      }
     67      if (separator == null) separator = map.detectSeparator(line);
    6668      if (mapFrom == null)
    6769      {
    68         if (line.match(/^[QCS]\d+\t.*$/))// We look for lines starting with 'Q', 'C' or 'S' followed by {digits} and a {tab}
     70        if (line.match(/^[QCS]\d+([\t,;].*)?$/))// We look for lines starting with 'Q', 'C' or 'S' followed by {digits} and a {tab/comma/semicolon}
    6971        {
    7072          var firstChar = line.charAt(0);
     
    8284          }
    8385        }
    84         else if (lineNo == 0 && line.match(/^[^\t]*pad/i)) // If the first header column contains 'pad' (ignoring case) we assume it contains PAD values
     86        else if (lineNo == 0 && line.match(/^[^\t,;]*pad/i)) // If the first header column contains 'pad' (ignoring case) we assume it contains PAD values
    8587        {
    8688          mapFrom = 'PAD';
     
    8991      lineNo++;
    9092    }
    91     if (lines.length > 3) preview += '<div id="more-lines">+' + (lines.length-3) + ' more lines...</div>';
     93    if (lines.length > 4) preview += '<div id="more-lines">+' + (lines.length-4) + ' more lines...</div>';
     94    frm.separator.value = separator ? separator.id : 'tab';
    9295    frm.mapFrom.value = mapFrom || 'PersonalNo';
    9396    Events.sendChangeEvent(frm.mapFrom);
     
    9598  }
    9699
     100  /**
     101    Detect the "best" separator to use by selecting the one
     102    that appear the most times in the given text line. If no
     103    separator is found null is returned.
     104  */
     105  map.detectSeparator = function(text)
     106  {
     107    var separators =
     108    [
     109      { 'id': 'tab', 'test': /\t/g, 'sep': '\t'},
     110      { 'id': 'comma', 'test': /,/g, 'sep': ','},
     111      { 'id': 'semicolon', 'test': /;/g, 'sep': ';'}
     112    ];
     113    var maxCount = 0;
     114    var bestMatch = 0;
     115    for (var i=0; i < separators.length; i++)
     116    {
     117      var sep = separators[i];
     118      var m = text.match(sep.test);
     119      if (m && m.length > maxCount)
     120      {
     121        maxCount = m.length;
     122        bestMatch = i;
     123      }
     124    }
     125    return maxCount == 0 ? null : separators[bestMatch];
     126  }
    97127 
    98128  map.validateStep1 = function(event)
     
    113143    var url = '../Export.servlet?ID='+App.getSessionId();
    114144    url += '&cmd=MapDataToReleases';
     145    url += '&separator='+frm.separator.value;
    115146    url += '&mapFrom='+Strings.encodeTags(frm.mapFrom.value);
    116147    url += '&exportCaseId='+(frm.exportCaseId.checked ? 1 : 0);
  • extensions/net.sf.basedb.reggie/branches/4.33-stable/resources/personal/map-external-data.jsp

    r5740 r6471  
    8181        <td class="help" rowspan="2"><span id="file.message" class="message"></span>
    8282          <ul>
    83           <li>Select a tab-separated text file in UTF-8 encoding.
     83          <li>Select a tab-, comma-, or semicolon-separated text file in UTF-8 encoding.
    8484          <li>The first line should contain column headers.
    8585          <li>The first column should contain the personal number, or a release id value as specified by the <b>Map from</b> option.
     
    9393        <td>
    9494          <div id="preview"><i>No file loaded...</i></div>
     95        </td>
     96        <td></td>
     97      </tr>
     98      <tr>
     99        <td class="prompt">Separator</td>
     100        <td>
     101          <select name="separator" id="separator">
     102            <option value="tab">Tab
     103            <option value="comma">Comma
     104            <option value="semicolon">Semi-colon
     105          </select>
    95106        </td>
    96107        <td></td>
  • extensions/net.sf.basedb.reggie/branches/4.33-stable/src/net/sf/basedb/reggie/servlet/ExportServlet.java

    r6470 r6471  
    538538        ReggieRole.checkPermission(dc, "'" + cmd + "' wizard", ReggieRole.PATIENT_CURATOR, ReggieRole.ADMINISTRATOR);
    539539       
     540        String separator = Values.getString(req.getParameter("separator"), "tab");
    540541        String mapFrom = Values.getString(req.getParameter("mapFrom"), "PersonalNo");
    541542        boolean exportCaseId = Values.getBoolean(req.getParameter("exportCaseId"));
     
    544545        boolean sortOutput = Values.getBoolean(req.getParameter("sortOutput"));
    545546
     547        String separatorChar = "\t";
     548        if ("comma".equals(separator))
     549        {
     550          separatorChar = ",";
     551        }
     552        else if ("semicolon".equals(separator))
     553        {
     554          separatorChar = ";";
     555        }
     556       
    546557        SnapshotManager manager = new SnapshotManager();
    547558        List<OutputColumn> outColumns = new ArrayList<>();
     
    605616          String header = uploadReader.readLine();
    606617          long parsedCharacters = header.length()+1;
    607           String[] dataIn = header.split("\t", 2);
     618          String[] dataIn = header.split(separatorChar, 2);
    608619          if (dataIn.length == 2) outColumns.add(new RestOfData(dataIn[1]));
    609620       
     
    627638            progress.display((int)(100*parsedCharacters / totalBytes), "Working... " + inputLine.lineNo + " lines parsed so far...");
    628639           
    629             inputLine.cols = data.split("\t", 2);
     640            inputLine.cols = data.split(separatorChar, 2);
    630641            inputLine.fromId = Values.getStringOrNull(inputLine.cols[0]);
    631642            if (inputLine.fromId == null)
     
    669680          tmpOut = new CipherOutputStream(Application.getStaticCache().write(outputFilePath, 5), cipher);
    670681          tmpWriter = new TableWriter(new OutputStreamWriter(tmpOut, "UTF-8"));
     682          tmpWriter.setDataSeparator(separatorChar);
    671683          tmpWriter.tablePrintData(headerOut);
    672684          for (Object[] outLines : dataOut)
Note: See TracChangeset for help on using the changeset viewer.