Changeset 3023

Timestamp:

Dec 12, 2006, 1:32:16 PM (16 years ago)

Author:

Nicklas Nordborg

Message:

Fixes #311: Allow parsing of numbers with different locale settings

Location:

trunk/src

Files:

• core/net/sf/basedb/core/ExtendedProperty.java (modified) (2 diffs)
• core/net/sf/basedb/core/Type.java (modified) (7 diffs)
• core/net/sf/basedb/util/NumberFormatUtil.java (added)
• core/net/sf/basedb/util/parser/ColFunction.java (modified) (3 diffs)
• core/net/sf/basedb/util/parser/ColumnMapper.java (modified) (5 diffs)
• core/net/sf/basedb/util/parser/ConstantMapper.java (modified) (2 diffs)
• core/net/sf/basedb/util/parser/FlatFileParser.java (modified) (7 diffs)
• core/net/sf/basedb/util/parser/JepMapper.java (modified) (5 diffs)
• plugins/core/net/sf/basedb/plugins/AbstractFlatFileImporter.java (modified) (6 diffs)
• plugins/core/net/sf/basedb/plugins/RawDataFlatFileImporter.java (modified) (8 diffs)
• plugins/core/net/sf/basedb/plugins/ReporterFlatFileImporter.java (modified) (9 diffs)

trunk/src/core/net/sf/basedb/core/ExtendedProperty.java

@@ -24 +24 @@
 package net.sf.basedb.core;
 
+import java.text.NumberFormat;
 import java.util.List;
 
@@ -179 +180 @@
   }
 
+  /**
+    Parse a string and return an object of the correct type for this property.
+    Numeric properties are parsed with the specified number format.
+    @param value The value to parse
+    @param numberFormat The number format, or null to use Float.valueOf or Double.valueOf
+    @return An object
+    @throws InvalidDataException If the string cannot be converted to the correct type
+    @see #getType()
+    @since 2.2
+  */
+  public Object parseString(String value, NumberFormat numberFormat)
+  {
+    return getType().parseString(value, numberFormat);
+  }
+  
   /**
     Check if an object is valid according to the specifications of this property.

trunk/src/core/net/sf/basedb/core/Type.java

@@ -37 +37 @@
 import org.hibernate.type.NullableType;
 
+import java.text.NumberFormat;
+import java.text.ParsePosition;
 import java.util.Date;
 import java.util.Map;
@@ -70 +72 @@
       return new Integer(Float.valueOf(value).intValue());
     }
+    public Number convertNumber(Number num)
+    {
+      return Integer.valueOf(num.intValue());
+    }
   },
 
@@ -90 +96 @@
       return new Long(Float.valueOf(value).intValue());
     }
+    public Number convertNumber(Number num)
+    {
+      return Long.valueOf(num.longValue());
+    }
   },
 
@@ -105 +115 @@
       return value == null ? 0 : 4;
     }
+    public Number convertNumber(Number num)
+    {
+      return Float.valueOf(num.floatValue());
+    }
   },
 
@@ -119 +133 @@
     {
       return value == null ? 0 : 8;
+    }
+    public Number convertNumber(Number num)
+    {
+      return Double.valueOf(num.doubleValue());
     }
   },
@@ -351 +369 @@
   
   /**
+    Convert a number to the appropriate class. The default implementation
+    returns null. Each numeric type overrides this method to create a number of
+    the correct type.
+    @since 2.2
+  */
+  public Number convertNumber(Number num)
+  {
+    return null;
+  }
+  
+  /**
     Parse a string and return a value of the correct type.
   */
@@ -363 +392 @@
     {
       return getHibernateType().fromStringValue(value);
+    }
+  }
+  
+  /**
+    Parse a string and return a value of the correct type using a specific
+    number format for numerical values.
+    @param value The string value to pars
+    @param numberFormat The number format, or null to use {@link #parseString(String)}
+    @since 2.2
+  */
+  public Object parseString(String value, NumberFormat numberFormat)
+  {
+    if (value == null)
+    {
+      return null;
+    }
+    else if (!isNumerical || numberFormat == null)
+    {
+      return parseString(value);
+    }
+    else
+    {
+      ParsePosition pos = new ParsePosition(0);
+      Number num = numberFormat.parse(value, pos);
+      if (pos.getIndex() < value.length())
+      {
+        throw new NumberFormatException("For input string: \"" + value + "\"");
+      }
+      return convertNumber(num);
     }
   }

trunk/src/core/net/sf/basedb/util/parser/ColFunction.java

@@ -24 +24 @@
 package net.sf.basedb.util.parser;
 
+import java.text.NumberFormat;
+import java.text.ParsePosition;
 import java.util.HashMap;
 import java.util.List;
@@ -59 +61 @@
   
   private final Map<String, Integer> columnHeaders;
+  private final NumberFormat numberFormat;
+  private final ParsePosition pos;
   private FlatFileParser.Data data;
   
-  public ColFunction(Map<String, Integer> columnHeaders)
+  public ColFunction(Map<String, Integer> columnHeaders, NumberFormat numberFormat)
   {
     this.columnHeaders = columnHeaders;
+    this.numberFormat = numberFormat;
+    this.pos = new ParsePosition(0);
   }
   
-  public ColFunction(List<String> columnHeaders)
+  public ColFunction(List<String> columnHeaders, NumberFormat numberFormat)
   {
+    this.numberFormat = numberFormat;
+    this.pos = new ParsePosition(0);
     this.columnHeaders = new HashMap<String, Integer>(columnHeaders.size());
     int index = 0;
@@ -138 +146 @@
       throw new ParseException("Invalid parameter type: " + argument + "; expected number");
     }
-    try
+    Double d = null;
+    if (value != null)
     {
-      stack.push(value == null ? null : Double.valueOf(value));
+      if (numberFormat == null)
+      {
+        try
+        {
+          d = Double.valueOf(value);
+        }
+        catch (NumberFormatException ex)
+        {
+          stack.push(value);
+          return;
+        }
+      }
+      else
+      {
+        pos.setIndex(0);
+        d = numberFormat.parse(value, pos).doubleValue();
+        if (pos.getIndex() < value.length())
+        {
+          stack.push(value);
+          return;
+        }
+      }
     }
-    catch (NumberFormatException ex)
-    {
-      stack.push(value);
-    }
+    stack.push(d);
   }
   // -------------------------------------------

trunk/src/core/net/sf/basedb/util/parser/ColumnMapper.java

@@ -24 +24 @@
 package net.sf.basedb.util.parser;
 
+import java.text.NumberFormat;
+import java.text.ParsePosition;
+
 import net.sf.basedb.util.parser.FlatFileParser.Data;
 
@@ -40 +43 @@
   private final int index;
   private final String name;
+  private final NumberFormat parser;
+  private final ParsePosition pos;
 
   /**
@@ -49 +54 @@
   public ColumnMapper(int index, String name)
   {
+    this(index, name, null);
+  }
+  
+  /**
+    Create a new column mapper using a specific number format.
+    @param index The index of the data column to use, starting at 0
+    @param name An optional name of the column, use in the <code>toString</code>
+      method
+    @param parser The parser to use or null to use Float.valueOf()
+   */
+  public ColumnMapper(int index, String name, NumberFormat parser)
+  {
     this.index = index;
     this.name = name;
+    this.parser = parser;
+    this.pos = parser == null ? null : new ParsePosition(0);
   }
   
@@ -68 +87 @@
   public Integer getInt(Data data)
   {
-    String stringValue = getValue(data);
-    return stringValue == null ? null : Float.valueOf(stringValue).intValue();
+    return getFloat(getValue(data)).intValue();
   }
   public Float getFloat(Data data)
   {
-    String stringValue = getValue(data);
-    return stringValue == null ? null : Float.valueOf(stringValue);
+    return getFloat(getValue(data));
   }
   // -------------------------------------------
@@ -87 +104 @@
   }
   // -------------------------------------------
+  
+  private Float getFloat(String value)
+  {
+    if (value == null) return null;
+    Float f = null;
+    if (parser == null)
+    {
+      f = Float.valueOf(value);
+    }
+    else
+    {
+      pos.setIndex(0);
+      f = parser.parse(value, pos).floatValue();
+      if (pos.getIndex() < value.length())
+      {
+        throw new NumberFormatException("For input string: \"" + value + "\"");
+      }
+    }
+    return f;
+  }
+  
 }
 

trunk/src/core/net/sf/basedb/util/parser/ConstantMapper.java

@@ -24 +24 @@
 package net.sf.basedb.util.parser;
 
+import java.text.NumberFormat;
+import java.text.ParsePosition;
+
 import net.sf.basedb.util.parser.FlatFileParser.Data;
 
@@ -43 +46 @@
   private NumberFormatException floatException;
   
+  /**
+    Create a constant mapper.
+  */
   public ConstantMapper(String constant)
+  {
+    this(constant, null);
+  }
+  
+  /**
+    Create a constant mapper using a specific number formatter as it's parser.
+    @param constant The constant expression
+    @param parser The number format to use or null to use Float.valueOf()
+    @since 2.2
+  */
+  public ConstantMapper(String constant, NumberFormat parser)
   {
     this.constant = constant;
     if (constant != null)
     {
-      try
+      if (parser != null)
       {
-        asFloat = Float.valueOf(constant);
-      }
-      catch (NumberFormatException ex)
-      {
-        floatException = ex;
-      }
-      if (asFloat != null)
-      {
-        asInteger = asFloat.intValue();
+        ParsePosition pos = new ParsePosition(0);
+        asFloat = parser.parse(constant, pos).floatValue();
+        if (pos.getIndex() < constant.length())
+        {
+          asFloat = null;
+          floatException = new NumberFormatException("For input string: \"" + constant + "\"");
+          integerException = floatException;
+        }
+        else
+        {
+          asInteger = asFloat.intValue();
+        }
       }
       else
       {
-        integerException = floatException;
+        try
+        {
+          asFloat = Float.valueOf(constant);
+        }
+        catch (NumberFormatException ex)
+        {
+          floatException = ex;
+        }
+        if (asFloat != null)
+        {
+          asInteger = asFloat.intValue();
+        }
+        else
+        {
+          integerException = floatException;
+        }
       }
     }

trunk/src/core/net/sf/basedb/util/parser/FlatFileParser.java

@@ -30 +30 @@
 import java.nio.charset.Charset;
 
+import java.text.NumberFormat;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
@@ -238 +239 @@
   
   /**
+    The default number formatter to use for creating mappers.
+  */
+  private NumberFormat numberFormat = null;
+  
+  /**
     List of lines parsed by the {@link #parseHeaders()} method.
   */
@@ -753 +759 @@
   
   /**
+    Set the default number format to use when creating mappers.
+    @param numberFormat The number format to use, or null to parse
+      numbers with Float.valueOf or Double.valueOf
+    @since 2.2
+    @see #getMapper(String)
+    @see #getMapper(String, NumberFormat)
+   */
+  public void setDefaultNumberFormat(NumberFormat numberFormat)
+  {
+    this.numberFormat = numberFormat;
+  }
+  
+  /**
+    Get the default number format.
+    @return The number format, or null if none has been specified
+    @since 2.2
+  */
+  public NumberFormat getDefaultNumberFormat()
+  {
+    return numberFormat;
+  }
+  
+  /**
+    Get a mapper using the default number format.
+    @see #getMapper(String, NumberFormat)
+  */
+  public Mapper getMapper(String expression)
+  {
+    return getMapper(expression, numberFormat);
+  }
+  
+  /**
     Create a mapper object that maps an expression string to a value.
     An expression string is a regular string which contains placeholders
@@ -772 +810 @@
 
     @param expression The string containing the mapping expression
+    @param numberFormat The number format the mapper should use for
+      parsing numbers, or null to use Float.valueOf or Double.valueOf
     @return A mapper object
-  */
-  public Mapper getMapper(String expression)
+    @since 2.2
+  */
+  public Mapper getMapper(String expression, NumberFormat numberFormat)
   {
     Mapper mapper = null;
     if (expression == null || expression.length() == 0)
     {
-      mapper = new ConstantMapper(emptyIsNull ? null : "");
+      mapper = new ConstantMapper(emptyIsNull ? null : "", numberFormat);
     }
     else if (expression.startsWith("="))
     {
-      mapper = new JepMapper(expression.substring(1), columnHeaders);
+      mapper = new JepMapper(expression.substring(1), columnHeaders, numberFormat);
     }
     else
@@ -795 +836 @@
         if (matchStart > nextStart)
         {
-          mappers.add(new ConstantMapper(expression.substring(nextStart, matchStart)));
+          mappers.add(new ConstantMapper(expression.substring(nextStart, matchStart), numberFormat));
         }
         try
         {
           int column = Integer.parseInt(m.group(1));
-          mappers.add(new ColumnMapper(column, null));
+          mappers.add(new ColumnMapper(column, null, numberFormat));
         }
         catch (NumberFormatException ex)
@@ -810 +851 @@
             if (column >= 0) 
             {
-              mappers.add(new ColumnMapper(column, name));
+              mappers.add(new ColumnMapper(column, name, numberFormat));
             }
             else
@@ -822 +863 @@
       if (nextStart < expression.length())
       {
-        mappers.add(new ConstantMapper(expression.substring(nextStart)));
+        mappers.add(new ConstantMapper(expression.substring(nextStart), numberFormat));
       }
       if (mappers.size() == 1)

trunk/src/core/net/sf/basedb/util/parser/JepMapper.java

@@ -24 +24 @@
 package net.sf.basedb.util.parser;
 
+import java.text.NumberFormat;
 import java.util.List;
 import org.nfunk.jep.JEP;
@@ -56 +57 @@
   private final JEP parser;
   private final ColFunction colFunction;
+  private final NumberFormat numberFormat;
   
   /**
@@ -64 +66 @@
   public JepMapper(String expression, List<String> columnHeaders)
   {
+    this(expression, columnHeaders, null);
+  }
+  
+  public JepMapper(String expression, List<String> columnHeaders, NumberFormat numberFormat)
+  {
     this.expression = expression;
-    this.colFunction = new ColFunction(columnHeaders);
+    this.numberFormat = numberFormat;
+    this.colFunction = new ColFunction(columnHeaders, numberFormat);
     // Replace: \ColumnName\ with: col('ColumnName')
     expression = expression.replaceAll("\\\\([^\\\\]*)\\\\", "col('$1')");
@@ -88 +96 @@
     colFunction.setData(data);
     if (parser.hasError()) Jep.clearErrorList(parser);
-    return new Integer((int)parser.getValue());
+    return Integer.valueOf((int)parser.getValue());
   }
   public Float getFloat(Data data)
@@ -94 +102 @@
     colFunction.setData(data);
     if (parser.hasError()) Jep.clearErrorList(parser);
-    return new Float(parser.getValue());
+    return Float.valueOf((float)parser.getValue());
   }
   // -------------------------------------------

trunk/src/plugins/core/net/sf/basedb/plugins/AbstractFlatFileImporter.java

@@ -51 +51 @@
 import net.sf.basedb.core.plugin.Plugin;
 
+import net.sf.basedb.util.NumberFormatUtil;
 import net.sf.basedb.util.error.ClassMapErrorHandler;
 import net.sf.basedb.util.error.ErrorHandler;
@@ -58 +59 @@
 import net.sf.basedb.util.parser.Mapper;
 
+import java.text.NumberFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -272 +274 @@
     new StringParameterType(255, Config.getCharset(), false, 1, 0, 0, 
     new ArrayList<String>(Charset.availableCharsets().keySet()));
+  
+  /**
+    Type for selecting decimal separator.
+  */
+  protected static final StringParameterType decimalSeparatorType =
+    new StringParameterType(255, "dot", false, 1, 0, 0,
+    Arrays.asList(new String[] { "dot", "comma" })
+  );
   
   /**
@@ -482 +492 @@
     FlatFileParser.Data dataline = null;
     ffp.setInputStream(in, getCharset());
+    ffp.setDefaultNumberFormat(getNumberFormat());
     boolean isSuccess = false; // false until we have finished parsing
     try
@@ -714 +725 @@
   
   /**
+    Get the decimal separator used by numbers in the file. This method first 
+    checks the job parameters for a value, then the configuration parameters. 
+    If not found null is returned.
+    @return The decimal separator or null to use Float.valueOf() or Double.valueOf()
+    @since 2.2
+  */
+  protected String getDecimalSeparator()
+  {
+    String ds = null;
+    if (job != null) ds = (String)job.getValue(DECIMAL_SEPARATOR);
+    if (ds == null && configuration != null) 
+    {
+      ds = (String)configuration.getValue(DECIMAL_SEPARATOR);
+    }
+    return ds;
+  }
+  
+  /**
+    Get a number formatter that is able to parse numbers with the
+    specified decimal separator. Returns null if no decimal separator
+    has been specified which causes numbers to be parsed with Float.valueOf()
+    or Double.valueOf().
+    @return The number format or null to use Float.valueOf() or Double.valueOf()
+    @since 2.2
+  */
+  protected NumberFormat getNumberFormat()
+  {
+    String decimalSeparator = getDecimalSeparator();
+    char ds = 0;
+    if ("dot".equals(decimalSeparator))
+    {
+      ds = '.';
+    }
+    else if ("comma".equals(decimalSeparator))
+    {
+      ds = ',';
+    }
+    return ds == 0 ? null : NumberFormatUtil.getNumberFormat(ds, ';');
+  }
+  
+  /**
     The name of the parameter that asks for the character set.
     @see #getCharsetParameter(String, String, String)
@@ -740 +792 @@
     return new PluginParameter<String>(
       CHARSET, label, description, defaultValue, charsetType
+    );
+  }
+
+  /**
+    The name of the parameter that asks for the decimal separator.
+    @see #getCharsetParameter(String, String, String)
+  */
+  protected static final String DECIMAL_SEPARATOR = "decimalSeparator";
+  
+  /**
+    Parameter definition that asks for the decimal separator used by numeric values 
+    in the file that is imported. This can be both a job parameter and a configuration
+    parameter. The implementation first checks in the job parameter and
+    if not found in the configuration parameter. If still not found the
+    no decimal separator is used and number parsing is done by Float.valueOf
+    or Double.valueOf.
+    @param label The label to use for the parameter or null to use the default label
+      (Decimal separator)
+    @param description The description to use for the parameter or null to use the
+      default description
+    @param defaultValue The default value for the decimal separator
+    @since 2.2
+  */
+  protected PluginParameter<String> getDecimalSeparatorParameter(String label, String description, String defaultValue)
+  {
+    if (label == null) label = "Decimal separator";
+    if (description == null) 
+    {
+      description = "The decimal separator used in numeric values, " +
+        "if not specified dot is assumed.";
+    }
+    return new PluginParameter<String>(
+      DECIMAL_SEPARATOR, label, description, defaultValue, decimalSeparatorType
     );
   }

trunk/src/plugins/core/net/sf/basedb/plugins/RawDataFlatFileImporter.java

@@ -59 +59 @@
 import net.sf.basedb.core.Job;
 
+import java.text.NumberFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -208 +209 @@
   private int numInserted;
   private FlatFileParser ffp;
+  private NumberFormat numberFormat;
   private RawDataBatcher batcher;
   private RawBioAssay rawBioAssay;
@@ -374 +376 @@
         storeValue(configuration, request, maxDataColumnsParameter);
         storeValue(configuration, request, ri.getParameter(CHARSET));
+        storeValue(configuration, request, ri.getParameter(DECIMAL_SEPARATOR));
         
         // Column mappings
@@ -396 +399 @@
         storeValue(job, request, fileParameter);
         storeValue(job, request, ri.getParameter(CHARSET));
+        storeValue(job, request, ri.getParameter(DECIMAL_SEPARATOR));
         
         // Error handling parameters
@@ -437 +441 @@
     numInserted = 0;
     this.ffp = ffp;
+    this.numberFormat = ffp.getDefaultNumberFormat();
     
     // Set up error handling
@@ -534 +539 @@
       RawDataProperty ep = entry.getKey();
       Mapper m = entry.getValue();
-      raw.setExtended(ep.getName(), ep.parseString(m.getValue(data)));
+      raw.setExtended(ep.getName(), ep.parseString(m.getValue(data), numberFormat));
     }
 
@@ -613 +618 @@
       parameters.add(fileParameter);
       parameters.add(getCharsetParameter(null, null, (String)configuration.getValue(CHARSET)));
+      parameters.add(getDecimalSeparatorParameter(null, null, (String)configuration.getValue(DECIMAL_SEPARATOR)));
       
       // Error handling parameters
@@ -712 +718 @@
       parameters.add(maxDataColumnsParameter);
       parameters.add(getCharsetParameter(null, null, null));
+      parameters.add(getDecimalSeparatorParameter(null, null, null));
 
       // Column mappings

trunk/src/plugins/core/net/sf/basedb/plugins/ReporterFlatFileImporter.java

@@ -66 +66 @@
 import net.sf.basedb.util.parser.Mapper;
 
+import java.text.NumberFormat;
 import java.util.Collection;
 import java.util.EnumSet;
@@ -305 +306 @@
         storeValue(configuration, request, maxDataColumnsParameter);
         storeValue(configuration, request, ri.getParameter(CHARSET));
+        storeValue(configuration, request, ri.getParameter(DECIMAL_SEPARATOR));
         
         // Column mappings
@@ -332 +334 @@
         storeValue(job, request, fileParameter);
         storeValue(job, request, ri.getParameter(CHARSET));
+        storeValue(job, request, ri.getParameter(DECIMAL_SEPARATOR));
         storeValue(job, request, updateExistingParameter);
         
@@ -372 +375 @@
   private Map<String, Float> deferred;
   private FlatFileParser ffp;
+  private NumberFormat numberFormat;
   
   /**
@@ -395 +399 @@
     updateExisting = (Boolean)job.getValue("updateExisting");
     this.ffp = ffp;
+    this.numberFormat = ffp.getDefaultNumberFormat();
     numInserted = 0;
     numUpdated = 0;
@@ -508 +513 @@
         Float score = scoreMapper == null ? 
             reporterList.getScore(reporter) : 
-            (Float)Type.FLOAT.parseString(scoreMapper.getValue(data));
+            (Float)Type.FLOAT.parseString(scoreMapper.getValue(data), numberFormat);
         if (currentId == 0)
         {
@@ -542 +547 @@
           ExtendedProperty ep = entry.getKey();
           Mapper m = entry.getValue();
-          reporter.setExtended(ep.getName(), ep.parseString(m.getValue(data)));
+          reporter.setExtended(ep.getName(), ep.parseString(m.getValue(data), numberFormat));
         }
         if (currentId != 0)
@@ -620 +625 @@
       }
       parameters.add(fileParameter);
+      parameters.add(updateExistingParameter);
       parameters.add(getCharsetParameter(null, null, (String)configuration.getValue(CHARSET)));
-      parameters.add(updateExistingParameter);
+      parameters.add(getDecimalSeparatorParameter(null, null, (String)configuration.getValue(DECIMAL_SEPARATOR)));
       
       parameters.add(errorSection);
@@ -677 +683 @@
       parameters.add(maxDataColumnsParameter);
       parameters.add(getCharsetParameter(null, null, null));
+      parameters.add(getDecimalSeparatorParameter(null, null, null));
 
       // Column mappings