source: trunk/www/common/plugin/parse_file.jsp @ 7494

<%-- $Id: parse_file.jsp 7494 2018-06-04 06:42:46Z nicklas $
  ------------------------------------------------------------------
  Copyright (C) 2006 Johan Enell, Nicklas Nordborg
  Copyright (C) 2007 Nicklas Nordborg

  This file is part of BASE - BioArray Software Environment.
  Available at http://base.thep.lu.se/

  BASE is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License
  as published by the Free Software Foundation; either version 3
  of the License, or (at your option) any later version.

  BASE is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with BASE. If not, see <http://www.gnu.org/licenses/>.
  ------------------------------------------------------------------

  @author Nicklas
  @version 2.0
--%>
<%@ page pageEncoding="UTF-8" session="false"
  contentType="text/html; charset=UTF-8"  
  import="net.sf.basedb.core.SessionControl"
  import="net.sf.basedb.core.DbControl"
  import="net.sf.basedb.core.Config"
  import="net.sf.basedb.core.Item"
  import="net.sf.basedb.core.File"
  import="net.sf.basedb.core.Path"
  import="net.sf.basedb.core.Location"
  import="net.sf.basedb.core.PluginConfigurationRequest"
  import="net.sf.basedb.core.plugin.Plugin"
  import="net.sf.basedb.util.parser.FlatFileParser"
  import="net.sf.basedb.util.parser.WrappedConfigureByExample"
  import="net.sf.basedb.clients.web.Base"
  import="net.sf.basedb.util.Values"
  import="net.sf.basedb.util.fuzzy.StringMatcher"
  import="net.sf.basedb.util.fuzzy.StringMatcher.FuzzyMatch"
  import="net.sf.basedb.clients.web.util.HTML"
  import="java.util.regex.Pattern"
  import="java.util.List"
  import="java.util.ArrayList"
  import="java.util.Set"
  import="java.io.InputStream"
  import="org.json.simple.JSONArray"
  import="org.json.simple.JSONObject"
%>
<%@ taglib prefix="base" uri="/WEB-INF/base.tld" %>
<%@ taglib prefix="m" uri="/WEB-INF/menu.tld" %>
<%@ taglib prefix="t" uri="/WEB-INF/tab.tld" %>
<%@ taglib prefix="tbl" uri="/WEB-INF/table.tld" %>

<%
request.setCharacterEncoding("UTF-8");
final SessionControl sc = Base.getExistingSessionControl(pageContext, true);
final String ID = sc.getId();
final DbControl dc = sc.newDbControl();
final float scale = Base.getScale(sc);
InputStream fileInputStream = null;
try
{
  String path = request.getParameter("path");
  String charsetName = Values.getString(request.getParameter("charset"), Config.getCharset());
  PluginConfigurationRequest pcRequest = (PluginConfigurationRequest)sc.getSessionSetting("plugin.configure.request");
  Plugin plugin = pcRequest.getPlugin();
  
  File file = null;
  FlatFileParser parser = null;
  FlatFileParser.LineType lastLine = null;
  Pattern splitter = null;
  boolean dataIsFound = false;
  int maxLines = Values.getInt(request.getParameter("maxLines"), FlatFileParser.DEFAULT_MAX_UNKNOWN_LINES);
  
  List<String> messages = new ArrayList<String>();
  JSONArray linePatterns = new JSONArray();
  JSONArray jsonFuzzy = new JSONArray();
  
  if (path != null)
  {
    Path p = new Path(path, Path.Type.FILE);
    file = File.getByPath(dc, p, false);
    parser = new FlatFileParser();
    parser.setMaxUnknownLines(maxLines);
    
    String header = Values.getStringOrNull(request.getParameter("header"));
    if (header != null)
    {
      try
      {
        parser.setHeaderRegexp(Pattern.compile(header));
      }
      catch (Throwable t)
      {
        messages.add("Invalid regular expression for header: " + HTML.encodeTags(header));
      }
    }
    
    String dataSplitter = Values.getStringOrNull(request.getParameter("dataSplitter"));
    if (dataSplitter != null)
    {
      try
      {
        splitter = Pattern.compile(dataSplitter);
        parser.setDataSplitterRegexp(splitter);
      }
      catch (Throwable t)
      {
        messages.add("Invalid regular expression for data splitter: " + HTML.encodeTags(dataSplitter));
      }
    }

    String dataHeader = Values.getStringOrNull(request.getParameter("dataHeader"));
    if (dataHeader != null)
    {
      try
      {
        parser.setDataHeaderRegexp(Pattern.compile(dataHeader));
      }
      catch (Throwable t)
      {
        messages.add("Invalid regular expression for data header: " + HTML.encodeTags(dataHeader));
      }
    }
    
    String dataFooter = Values.getStringOrNull(request.getParameter("dataFooter"));
    if (dataFooter != null)
    {
      try
      {
        parser.setDataFooterRegexp(Pattern.compile(dataFooter));
      }
      catch (Throwable t)
      {
        messages.add("Invalid regular expression for data footer: " + HTML.encodeTags(dataFooter));
      }
    }
    
    String ignore = Values.getStringOrNull(request.getParameter("ignore"));
    if (ignore != null)
    {
      try
      {
        parser.setIgnoreRegexp(Pattern.compile(ignore));
      }
      catch (Throwable t)
      {
        messages.add("Invalid regular expression for ignore: " + HTML.encodeTags(ignore));
      }
    }

    parser.setTrimQuotes(Values.getBoolean(request.getParameter("trimQuotes")));
    parser.setMinDataColumns(Values.getInt(request.getParameter("minDataColumns"), 0));
    parser.setMaxDataColumns(Values.getInt(request.getParameter("maxDataColumns"), -1));
    
    fileInputStream = file.getDownloadStream(0);
    if (plugin instanceof WrappedConfigureByExample)
    {
      WrappedConfigureByExample wex = (WrappedConfigureByExample)plugin;
      fileInputStream = wex.wrapInputStream(fileInputStream);
    }
    parser.setInputStream(fileInputStream, charsetName);
    lastLine = parser.parseHeaders();
    dataIsFound = lastLine == FlatFileParser.LineType.DATA || lastLine == FlatFileParser.LineType.DATA_HEADER;
    if (!dataIsFound)
    {
      messages.add("Start of data couldn't be found. Please try again with different settings.");
    }
  }
  %>
  <base:page type="iframe" title="Test with file">
  <base:head scripts="tabcontrol-2.js,~parse_file.js" styles="table.css,tabcontrol.css">
    <style>
    #fileData td
    {
      white-space: nowrap;
      max-width: 20em;
      min-width: 4em;
      overflow: hidden;
      text-overflow: ellipsis;
    }
    #fileData td:first-child
    {
      font-weight: bold;
    }
    .skipped
    {
      padding: 1px 2px 1px 2px;
      color: #777777;
      font-style: italic;
    }
    </style>
  </base:head>
  <base:body>
  <div class="absolutefull bottomborder">

  <t:tabcontrol 
    id="parsedFile"
    subclass="absolutefull dialogtabcontrol"
    position="top" active="filedata"
    >
  <t:tab
    id="filedata"
    title="File data"
    >
    
    <%
    if (file == null)
    {
      %>
      <div class="absolutefull bg-filled-50">
      <div class="messagecontainer error">
        No file selected. Please select a file to test above.
      </div>
      </div>
      <%
    }
    else
    {
      if (messages.size() > 0)
      {
        %>
        <div class="bg-filled-50" style="padding: 5px;">
        <div class="messagecontainer error" style="margin: 0;">
        <b>Could not parse the file <code><%=HTML.encodeTags(path)%></code></b>
        <ol>
        <li><%=Values.getString(messages, "<li>", true)%>
        </ol>
        </div>
        </div>
        <%
      }
      %>
      <div class="itemlist" style="<%=messages.size()==0 ? "" : "border-top-width: 1px;"%>">
        <div class="data">
        <table style="border: 0;" id="fileData">
          <thead class="bg-filled-100">
          <tr>
            <th>Line</th>
            <th>Columns</th>
            <th>Type</th>
            <th>Use as</th>
            <th colspan="99">File data</th>
          </tr>
          </thead>
          <tbody class="rows">
      
        <%
        int currentLine = 0;
        int headerLines = parser.getLineCount();
        String[] rowclass = new String[] { "bg-oddrow", "bg-evenrow" };
        int rowClassIndex = 0;

        while (currentLine < maxLines && currentLine < headerLines)
        {
          FlatFileParser.Line line = parser.getLine(currentLine);
          linePatterns.add(Pattern.quote(line.line()));
          String[] data = null;
          int numMoreColumns = 0;
          if (line.type() == FlatFileParser.LineType.HEADER)
          {
            data = new String[] { line.name(), line.value() };
          }
          else if (splitter != null)
          {
            data = parser.trimQuotes(splitter.split(line.line(), 99));
            if (data.length == 99)
            {
              numMoreColumns = splitter.split(line.line()).length - 99;
            }
          }
          else
          {
            data = new String[] { line.line() };
          }
          %>
          <tr class="<%=rowclass[rowClassIndex]%> highlight">
            <td class="cell"><%=line.lineNo()%></td>
            <td class="cell"><%=data.length+numMoreColumns%></td>
            <td class="cell"><%=line.type()%></td>
            <td class="cell">
              <select class="auto-init" 
                data-auto-init="line-pattern"
                data-line-no="<%=currentLine%>">
              <option value="">
              <option value="dataHeader">Data header
              <option value="dataFooter">Data footer
              </select>
            </td>
            <%
            int colNo = 0;
            for (String d : data)
            {
              %>
              <td class="cell"><%=HTML.encodeTags(d) %></td>
              <%
              colNo++;
              if (colNo == 99) break;
            }
            if (colNo < 99)
            {
              %>
              <td class="cell" colspan="<%=99-colNo%>"></td>
              <%
            }
            %>
          </tr>
          <%
          currentLine++;
          rowClassIndex = 1 - rowClassIndex;
        }
        if (currentLine < maxLines)
        {
          if (lastLine == FlatFileParser.LineType.DATA) parser.nextData();
          while (currentLine < maxLines && parser.hasMoreData())
          {
            FlatFileParser.Data data = parser.nextData();
            linePatterns.add(Pattern.quote(data.line()));
            if (parser.getNumSkippedLines() > 0)
            {
              %>
              <tr>
                <td class="skipped bg-filled-100" colspan="103">
                Skipped <%=parser.getNumSkippedLines()%> lines 
                (<%=parser.getIgnoredLines()%> ignored; <%=parser.getUnknownLines()%> unknown)
                </td>
              </tr>
              <%
            }
            %>
            <tr class="<%=rowclass[rowClassIndex]%> highlight">
              <td class="cell"><%=data.lineNo()%></td>
              <td class="cell"><%=data.columns()%></td>
              <td class="cell">Data</td>
              <td class="cell">
                <select class="auto-init" 
                  data-auto-init="line-pattern"
                  data-line-no="<%=currentLine%>">
                <option value="">
                <option value="dataHeader">Data header
                <option value="dataFooter">Data footer
                </select>
              </td>
              <%
              for (int i = 0; i < data.columns() && i < 99; ++i)
              {
                %>
                <td class="cell"><%=HTML.encodeTags(data.get(i))%></td>
                <%
              }
              if (data.columns() < 99)
              {
                %>
                <td class="cell" colspan="<%=99-data.columns()%>"></td>
                <%
              }
              %>
            </tr>
            <%
            currentLine++;
            rowClassIndex = 1 - rowClassIndex;
          }
          if (parser.getNumSkippedLines() > 0)
          {
            %>
            <tr>
              <td class="error" colspan="99">
              Skipped <%=parser.getNumSkippedLines()%> lines 
              (<%=parser.getIgnoredLines()%> ignored; <%=parser.getUnknownLines()%> unknown)
              </td>
            </tr>
            <%
          }
        }
        %>
        </tbody>
        </table>
      </div>
      </div>
      <%
    }
    %>
    </t:tab>

    <t:tab
      id="mappings"
      title="Column mappings"
      visible="<%=dataIsFound && request.getParameter("mappingParameterNames") != null%>"
      >
      <%
      StringBuilder sb = new StringBuilder();
      List<String> headers = parser.getColumnHeaders();
      int index = 0;
      if (headers != null)
      {
        for (String header : headers)
        {
          sb.append("<option value=\"" + index + "\">" + HTML.encodeTags(header));
          index++;
        }
      }
      else
      {
        int maxDataColumns = Values.getInt(request.getParameter("maxDataColumns"), -1);
        while (index < maxDataColumns)
        {
          sb.append("<option value=\"" + index + "\">" + index);
          index++;
        }
      }
      String mappings = sb.toString();
      String[] mappingParameters = request.getParameter("mappingParameterNames").split(",");
      if (headers != null)
      {
        List<String> labels = new ArrayList<String>();
        for (String name : mappingParameters)
        {
          labels.add(request.getParameter("mapping." + name + ".label"));
        }
        StringMatcher sm = new StringMatcher();
        List<FuzzyMatch> matches = sm.getBestPairs(labels, headers);
        int i = 0;
        for (FuzzyMatch fm : matches)
        {
          String name = mappingParameters[i];
          if (fm != null)
          {
            JSONObject jsonFuzzyMatch = new JSONObject();
            jsonFuzzyMatch.put("name", mappingParameters[i]);
            jsonFuzzyMatch.put("columnIndex", headers.indexOf(fm.getValue()));
            jsonFuzzyMatch.put("score", fm.getScore());
            jsonFuzzy.add(jsonFuzzyMatch);
          }
          else
          {
            jsonFuzzy.add(null);
          }
          ++i;
        }
      }
      %>
      <form name="mappings">
      <div class="absolutefull bg-filled-100" style="height: 2em;">
        <table>
        <tr>
          <td style="padding-left: 4px;">
            <b>Mapping style:</b>
            <input type="radio" name="expressionStyle" id="expressionStyleSimple" value="1" checked><label for="expressionStyleSimple">Simple</label>
            <input type="radio" name="expressionStyle" id="expressionStyleExpression" value="2"><label for="expressionStyleExpression">Expression</label>
          </td>
          <%
          if (jsonFuzzy.size() > 0)
          {
            %>
            <td style="padding-left: 4px;">
            <base:button id="btnFuzzyMatches" title="Auto generate" 
              tooltip="Generate mappings by finding the best match between Property and File column"/>
            </td>
            <td style="padding-left: 4px; width: 50%;">
              Similarity score: 
              <input type="text" id="similarityScore" name="similarity" value="0.85">
              (0 = bad; 1 = good)
            </td>
            <%
          }
          %>
        </tr>
        </table>
      </div>
      
      <div class="absolutefull input100" style="top: 2em;">

        <tbl:table id="col-mappings">
          <tbl:columndef 
            id="property"
            title="Property"
          />
          <tbl:columndef 
            id="expression"
            title="Mapping expression"
          />
          <tbl:columndef 
            id="columns"
            title="File columns"
          />
          <tbl:data>
            <tbl:headers>
              <tbl:headerrow>
                <tbl:columnheaders />
              </tbl:headerrow>
            </tbl:headers>            
            <tbl:rows>
            <%
            for (String name : mappingParameters)
            {
              %>
              <tbl:row>
                <tbl:cell column="property"><%=HTML.encodeTags(request.getParameter("mapping." + name + ".label"))%></tbl:cell>
                <tbl:cell column="expression">
                
                    <table style="width: 100%;">
                    <tr>
                      <td>
                      <input type="text" class="text auto-init" data-auto-init="column-mapping"
                        name="mapping.<%=name%>.expression"
                        maxlength="80" 
                        value="<%=HTML.encodeTags(request.getParameter("mapping." + name + ".expression"))%>">
                      </td>
                    <td style="width: 22px;">
                      <base:icon image="cancel.png" subclass="auto-init" id="<%="clear."+name %>"
                        data-auto-init="column-mapping-clear" data-mapping="<%=name%>"
                        tooltip="Clear this expression"/>
                    </td>
                    </tr>
                    </table>
                </tbl:cell>
                
                <tbl:cell column="columns">
                  <select name="list.<%=name%>" class="auto-init"
                    data-auto-init="column-mapping-preset" data-mapping="<%=name%>">
                  <option value="">
                  <%=mappings%>
                  </select>
                </tbl:cell>
              
              </tbl:row>
              <%
            }
            %>
            </tbl:rows>
          </tbl:data>
        </tbl:table>
      </div>
    </form>
    </t:tab>
      
    </t:tabcontrol>
    
      <div id="page-data" class="datacontainer"
        data-line-patterns="<%=HTML.encodeTags(linePatterns.toJSONString())%>"
        <%
        if (parser != null && parser.getColumnHeaders() != null)
        {
          %>
          data-column-headers="<%=HTML.encodeTags(JSONArray.toJSONString(parser.getColumnHeaders())) %>"
          <%
        }
        %>
        data-fuzzy-matches="<%=HTML.encodeTags(jsonFuzzy.toJSONString()) %>"
        ></div>
    </div>
  </base:body>
  </base:page>
  <%
}
finally
{
  if (fileInputStream != null) fileInputStream.close();
  if (dc != null) dc.close();
}
%>