Changeset 4096

Timestamp:

Jan 22, 2008, 2:22:03 PM (15 years ago)

Author:

Nicklas Nordborg

Message:

Fixes #902: Support for multiple configuration files for raw data types and extended properties

Location:

trunk

Files:

• build.xml (modified) (1 diff)
• config/dist/extended-properties (added)
• config/dist/extended-properties/readme.txt (added)
• config/dist/raw-data-types (added)
• config/dist/raw-data-types/readme.txt (added)
• doc/src/docbook/appendix/extended_properties.xml (modified) (1 diff)
• src/core/net/sf/basedb/core/Application.java (modified) (4 diffs)
• src/core/net/sf/basedb/core/Config.java (modified) (5 diffs)
• src/core/net/sf/basedb/core/ExtendedProperties.java (modified) (15 diffs)
• src/core/net/sf/basedb/core/RawDataTypes.java (modified) (15 diffs)

trunk/build.xml

@@ -133 +133 @@
   <!-- pattern for configuration files use by copy.config -->
   <patternset id="config.files">
-    <include name="*.*" />
+    <include name="**/*.*" />
     <exclude name="web.xml" />
     <exclude name="readme.txt" />

trunk/doc/src/docbook/appendix/extended_properties.xml

@@ -123 +123 @@
   </itemizedlist>
 
+  <bridgehead>Multiple configuration files</bridgehead>
+  <para>
+    Starting with BASE 2.6 it is possible to use multiple configuration
+    files for extended properties. This is useful only if you want to add
+    new columns. To remove or change existing columns you still have
+    to modify the original extended properties file. It is rather simple
+    to use this feature. Create a new subdirectory with the name 
+    <filename>extended-properties</filename> in the same directory as 
+    the <filename>extended-properties.xml</filename>. In this directory
+    you can put additional extended property files. They should have the
+    same format as the default file.
+  </para>
+  
+  <tip>
+    We recommend that you don't modify the default <filename>extended-properties.xml</filename>
+    file at all (unless you want to remove some of the columns). This will make it
+    easier when upgrading BASE since you don't have to worry about losing
+    your own changes.
+  </tip>
 
   <bridgehead>Format of the extended-properties.xml file</bridgehead>

trunk/src/core/net/sf/basedb/core/Application.java

@@ -29 +29 @@
 import net.sf.basedb.core.data.SchemaVersionData;
 import net.sf.basedb.core.authentication.Authenticator;
+import net.sf.basedb.util.FileUtil;
+import net.sf.basedb.util.RegexpFileFilter;
 import net.sf.basedb.util.timer.Scheduler;
 
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.GregorianCalendar;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.HashMap;
@@ -394 +398 @@
       log.info("db.queries = " + queryFile);
       
-      extendedPropertiesFile = Config.getString("db.extended-properties");
+      extendedPropertiesFile = Config.getString("db.extended-properties", "/extended-properties.xml");
       log.info("db.extended-properties = " + extendedPropertiesFile);
       
-      rawDataTypesFile = Config.getString("db.raw-data-types");
+      rawDataTypesFile = Config.getString("db.raw-data-types", "/raw-data-types.xml");
       log.info("db.raw-data-types = " + rawDataTypesFile);
       
@@ -707 +711 @@
     @see ExtendedProperties
   */
-  static String getExtendedPropertiesFile()
-  {
-    return extendedPropertiesFile;
+  static List<String> getExtendedPropertyFiles()
+  {
+    List<String> files = new ArrayList<String>();
+    // Main extended properties file
+    files.add(extendedPropertiesFile);  
+    // Scan extended-properties directory
+    String configDir = Config.getConfigDirectory().getAbsolutePath();
+    java.io.File propConfigDir = new java.io.File(configDir, "extended-properties");
+    if (propConfigDir.exists())
+    {
+      // Find all XML files
+      List<java.io.File> xmlFiles = FileUtil.findFiles(propConfigDir, 
+        new RegexpFileFilter(Pattern.compile(".*\\.xml"), null));
+      if (xmlFiles != null)
+      {
+        for (java.io.File xmlFile : xmlFiles)
+        {
+          String path = xmlFile.getAbsolutePath().replace(configDir, "");
+          files.add(path);
+        }
+      }
+    }
+    return files; 
   }
 
@@ -716 +740 @@
     @see RawDataTypes
   */
-  static String getRawDataTypesFile()
-  {
-    return rawDataTypesFile;
+  static List<String> getRawDataTypeFiles()
+  {
+    List<String> files = new ArrayList<String>();
+    // Main raw data types file
+    files.add(rawDataTypesFile);  
+    // Scan raw-data-types directory
+    String configDir = Config.getConfigDirectory().getAbsolutePath();
+    java.io.File rawConfigDir = new java.io.File(configDir, "raw-data-types");
+    if (rawConfigDir.exists())
+    {
+      // Find all XML files
+      List<java.io.File> xmlFiles = FileUtil.findFiles(rawConfigDir, new RegexpFileFilter(Pattern.compile(".*\\.xml"), null));
+      if (xmlFiles != null)
+      {
+        for (java.io.File xmlFile : xmlFiles)
+        {
+          String path = xmlFile.getAbsolutePath().replace(configDir, "");
+          files.add(path);
+        }
+      }
+    }
+    return files;
   }
 

trunk/src/core/net/sf/basedb/core/Config.java

@@ -27 +27 @@
 import java.util.Properties;
 import java.io.InputStream;
-import java.io.IOException;
 import java.net.URL;
 
@@ -50 +49 @@
   private static boolean isInitialised = false;
   
+  private static java.io.File configDir = null;
+  
   /**
     Loads the settings from the configuration file.
@@ -62 +63 @@
     {
       URL baseConfig = Config.class.getResource("/base.config");
+      configDir = new java.io.File(baseConfig.toURI()).getParentFile();
       InputStream is = baseConfig == null ? null : baseConfig.openStream();
       if (is == null)
@@ -71 +73 @@
       is.close();
     }
-    catch (IOException ex)
+    catch (Exception ex)
     {
       config = null;
@@ -86 +88 @@
     isInitialised = false;
     config = null;
+    configDir = null;
+  }
+  
+  /**
+    Get the directory where the BASE configuration files are located.
+    @return The directory as a <code>java.io.File</code> object
+    @since 2.6
+  */
+  public static java.io.File getConfigDirectory()
+  {
+    return configDir;
   }
   

trunk/src/core/net/sf/basedb/core/ExtendedProperties.java

@@ -44 +44 @@
 /**
   This class is used for reading XML files with information
-  about extended properties. The file should be located at the path
-  returned by {@link Application#getExtendedPropertiesFile()
-  Application.getExtendedPropertiesFile}.
+  about extended properties. All files returned by {@link Application#getExtendedPropertyFiles()}
+  are parsed.
 
   @see net.sf.basedb.core.data.ExtendableData
@@ -73 +72 @@
     that class. The list holds {@link ExtendedProperty} objects.
   */
-  private static Map<String, List<ExtendedProperty>> properties = null;
+  private static Map<String, List<ExtendedProperty>> allProperties = null;
 
   /**
@@ -91 +90 @@
     if (isInitialised) return;
     classes = new ArrayList<String>();
-    properties = new HashMap<String, List<ExtendedProperty>>();
+    allProperties = new HashMap<String, List<ExtendedProperty>>();
     loadExtendedPropertiesFile();
     isInitialised = true;
@@ -103 +102 @@
     if (classes != null) classes.clear();
     classes = null;
-    if (properties != null) properties.clear();
-    properties = null;
+    if (allProperties != null) allProperties.clear();
+    allProperties = null;
     isInitialised = false;
   }
@@ -111 +110 @@
     Load and parse the file with predefined queries.
     This method will populate the {@link #classes} and
-    {@link #properties} variables.
+    {@link #allProperties} variables.
   */
   private static synchronized void loadExtendedPropertiesFile()
     throws BaseException
   {
-    String propertiesFile = Application.getExtendedPropertiesFile();
-    if (propertiesFile == null) return;
+    List<String> files = Application.getExtendedPropertyFiles();
     try
     {
-      Document dom = XMLUtil.getValidatedXml(ExtendedProperties.class.getResource(propertiesFile), dtdFile);
-      loadClasses(dom);
-      log.info("Loaded extended properties from file: " + propertiesFile);
+      Set<String> usedNames = new HashSet<String>();
+      for (String xmlFile : files)
+      {
+        Document dom = XMLUtil.getValidatedXml(ExtendedProperties.class.getResource(xmlFile), dtdFile);
+        loadClasses(usedNames, dom, xmlFile);
+        log.info("Loaded extended properties from file: " + xmlFile);
+      }
     }
     catch (Exception ex)
@@ -154 +156 @@
       className = className.substring(index+1);
     }
-    return properties.containsKey(className);
+    return allProperties.containsKey(className);
   }
 
@@ -172 +174 @@
       className = className.substring(index+1);
     }
-    List<ExtendedProperty> l = (List<ExtendedProperty>)properties.get(className);
+    List<ExtendedProperty> l = (List<ExtendedProperty>)allProperties.get(className);
     return l;
   }
@@ -181 +183 @@
   */
   @SuppressWarnings("unchecked")
-  private static void loadClasses(Document dom)
+  private static void loadClasses(Set<String> usedNames, Document dom, String xmlFile)
   {
     List<Element> classTags = (List<Element>)dom.getRootElement().getChildren("class");
@@ -187 +189 @@
     {
       String className = el.getAttributeValue("name");
-      classes.add(className);
-      properties.put(className, loadProperties(el));
+      if (!classes.contains(className)) 
+      {
+        classes.add(className);
+        allProperties.put(className, new ArrayList<ExtendedProperty>());
+      }
+      loadProperties(usedNames, el, xmlFile);
     }
   }
@@ -194 +200 @@
   /**
     Load the properties for the specified class node and
-    put the list in the {@link #properties} variable.
+    put the list in the {@link #allProperties} variable.
   */
   @SuppressWarnings("unchecked")
-  private static List<ExtendedProperty> loadProperties(Element classElement)
-  {
-    List<ExtendedProperty> properties = new ArrayList<ExtendedProperty>();
+  private static void loadProperties(Set<String> usedNames, Element classElement, String xmlFile)
+  {
+    String className = classElement.getAttributeValue("name");
+    List<ExtendedProperty> properties = allProperties.get(className);
     List<Element> children = (List<Element>)classElement.getChildren("property");
     DbEngine engine = HibernateUtil.getDbEngine();
-    String className = classElement.getAttributeValue("name");
-    Set<String> usedNames = new HashSet<String>();
     for (Element property : children)
     {
@@ -209 +214 @@
       if (!ExtendedProperty.isValidName(name))
       {
-        throw new InvalidDataException("Invalid property for class " + 
-          className + ": name=" + name);
+        throw new InvalidDataException("Invalid property for class '" + 
+          className + "' in file '" + xmlFile + "': name=" + name);
       }
       if (usedNames.contains("name:" + name))
       {
-        throw new InvalidDataException("Duplicate property for class " + 
-          className + ": name=" + name);
+        throw new InvalidDataException("Duplicate property for class '" + 
+          className + "' in file '" + xmlFile + "': name=" + name);
       }
       usedNames.add("name:" + name);
@@ -223 +228 @@
       if (!engine.isValidColumnName(column))
       {
-        throw new InvalidDataException("Invalid column for property " + 
-          className + "[" + name + "]: column=" + column);
+        throw new InvalidDataException("Invalid column for property '" + 
+          className + "[" + name + "]' in file '" + xmlFile + "': column=" + column);
       }
       if (usedNames.contains("column:" + column))
       {
-        throw new InvalidDataException("Duplicate column for property " + 
-          className + "[" + name + "]: column=" + column);
+        throw new InvalidDataException("Duplicate column for property '" + 
+          className + "[" + name + "]' in file '" + xmlFile + "': column=" + column);
       }
       usedNames.add("column:" + column);
@@ -266 +271 @@
           if (url == null)
           {
-            throw new InvalidDataException("Missing url for property link " + 
-              className + "[" + name + "]: regexp=" + regexp);
+            throw new InvalidDataException("Missing url for property link '" + 
+              className + "[" + name + "]' in file '" + xmlFile + "': regexp=" + regexp);
           }
           try
@@ -275 +280 @@
           catch (PatternSyntaxException ex)
           {
-            throw new InvalidDataException("Invalid regexp for property link " + 
-              className + "[" + name + "]: regexp=" + regexp, ex);
+            throw new InvalidDataException("Invalid regexp for property link '" + 
+              className + "[" + name + "]' in file '" + xmlFile + "': regexp=" + regexp, ex);
           }
         }
@@ -282 +287 @@
       properties.add(new ExtendedProperty(name, title, description, column, type, length, nullable, insertable, updateable, averageMethod, epLinks));
     }
-    return properties;
   }
 }

trunk/src/core/net/sf/basedb/core/RawDataTypes.java

@@ -49 +49 @@
 /**
   This class is used for reading the XML configuration file with 
-  information about raw data types. The file should be located at the path
-  returned by {@link Application#getRawDataTypesFile()}.
+  information about raw data types. All files returned by {@link 
+  Application#getRawDataTypeFiles()} are parsed.
 
   @author Nicklas
@@ -63 +63 @@
   */
   private static final org.apache.log4j.Logger log = 
-    org.apache.log4j.LogManager.getLogger("net.sf.basedb.core");
+    org.apache.log4j.LogManager.getLogger("net.sf.basedb.core.RawDataTypes");
 
   /**
@@ -79 +79 @@
     The DTD which is used to validate the XML file.
   */
-  private static final URL dtdFile = RawDataTypes.class.getResource("/net/sf/basedb/core/dtd/raw-data-types.dtd");
+  private static final URL dtdFile = 
+    RawDataTypes.class.getResource("/net/sf/basedb/core/dtd/raw-data-types.dtd");
   
   private static boolean isInitialised = false;
@@ -164 +165 @@
     throws BaseException
   {
-    String xmlFile = Application.getRawDataTypesFile();
-    if (xmlFile == null) xmlFile = "/raw-data-types.xml";
+    List<String> files = Application.getRawDataTypeFiles();
     try
     {
-      Document dom = XMLUtil.getValidatedXml(RawDataTypes.class.getResource(xmlFile), dtdFile);
-      loadRawDataTypes(dom);
-      log.info("Loaded raw data types from file: " + xmlFile);
+      for (String xmlFile : files)
+      {
+        Document dom = XMLUtil.getValidatedXml(RawDataTypes.class.getResource(xmlFile), dtdFile);
+        loadRawDataTypes(dom, xmlFile);
+        log.info("Loaded raw data types from file: " + xmlFile);
+      }
     }
     catch (Exception ex)
@@ -287 +290 @@
   */
   @SuppressWarnings({"unchecked"})
-  private static void loadRawDataTypes(Document dom)
+  private static void loadRawDataTypes(Document dom, String xmlFile)
   {
     List<Element> rawDataTypeTags = dom.getRootElement().getChildren("raw-data-type");
@@ -303 +306 @@
       if (!ExtendedProperty.isValidName(id))
       {
-        throw new InvalidDataException("Invalid id for raw data type: " + id);
+        throw new InvalidDataException("Invalid id for raw data type '" + 
+          id + "' in file '" + xmlFile + "'");
+      }
+      if (usedNames.contains("id:" + id))
+      {
+        throw new InvalidDataException("Duplicate id for raw data type '" + 
+            id +"' in file '" + xmlFile);
       }
       if (usedNames.contains("name:" + name))
       {
-        throw new InvalidDataException("Duplicate name for raw data type " + 
-          id +": name=" + name);
-      }
+        throw new InvalidDataException("Duplicate name for raw data type '" + 
+            id +"' in file '" + xmlFile + "': name=" + name);
+      }
+      usedNames.add("id:" + id);
       usedNames.add("name:" + name);
       if ("database".equals(storage))
@@ -315 +325 @@
         if (!engine.isValidTableName(table))
         {
-          throw new InvalidDataException("Invalid table for raw data type " + 
-              id +": table=" + table);
+          throw new InvalidDataException("Invalid table for raw data type '" + 
+              id +"' in file '" + xmlFile + "': table=" + table);
         }
         if (usedNames.contains("table:" + table))
         {
-          throw new InvalidDataException("Duplicate table for raw data type " + 
-            id +": table=" + table);
+          throw new InvalidDataException("Duplicate table for raw data type '" + 
+              id +"' in file '" + xmlFile + "': table=" + table);
         }
         usedNames.add("table:" + table);
@@ -329 +339 @@
         throw new ConfigurationException("Attribute storage='" + storage + 
           "' is no longer supported. Please remove declaration for <rawdatatype id='" + id + 
-          "'> from " + Application.getRawDataTypesFile());
+          "'> from file '" + xmlFile + "'");
       }
       if (channels <= 0)
       {
-        throw new InvalidDataException("Number of channels must be > 0 for raw data type "+
-            id + ": channels=" + channels);
-      }
-      List<RawDataProperty> properties = loadProperties(el, channels);
-      List<IntensityFormula> formulas = loadIntensityFormulas(el, channels);
+        throw new InvalidDataException("Number of channels must be > 0 for raw data type '"+
+            id +"' in file '" + xmlFile + "': channels=" + channels);
+      }
+      List<RawDataProperty> properties = loadProperties(el, channels, xmlFile);
+      List<IntensityFormula> formulas = loadIntensityFormulas(el, channels, xmlFile);
       RawDataType rdt = new RawDataType(id, name, description, channels, 
           table, properties, formulas);
@@ -350 +360 @@
   */
   @SuppressWarnings({"unchecked"})
-  private static List<RawDataProperty> loadProperties(Element rawDataTypeElement, int channels)
+  private static List<RawDataProperty> loadProperties(Element rawDataTypeElement, int channels, String xmlFile)
   {
     List<RawDataProperty> properties = new ArrayList<RawDataProperty>();
@@ -362 +372 @@
       if (!ExtendedProperty.isValidName(name))
       {
-        throw new InvalidDataException("Invalid property for raw data type " + 
-          rawDataType + ": name=" + name);
+        throw new InvalidDataException("Invalid property for raw data type '" + 
+          rawDataType + "' in file '" + xmlFile + "': name=" + name);
       }
       if (usedNames.contains("name:" + name))
       {
-        throw new InvalidDataException("Duplicate property for raw data type " + 
-          rawDataType + ": name=" + name);
+        throw new InvalidDataException("Duplicate property for raw data type '" + 
+          rawDataType + "' in file '" + xmlFile + "': name=" + name);
       }
       usedNames.add("name:" + name);
@@ -376 +386 @@
       if (!engine.isValidColumnName(column))
       {
-        throw new InvalidDataException("Invalid column for property " + 
-          rawDataType + "[" + name + "]: column=" + column);
+        throw new InvalidDataException("Invalid column for raw data property '" + 
+          rawDataType + "[" + name + "]' in file '" + xmlFile + "': column=" + column);
       }
       if (usedNames.contains("column:" + column))
       {
-        throw new InvalidDataException("Duplicate column for property " + 
-          rawDataType + "[" + name + "]: column=" + column);
+        throw new InvalidDataException("Duplicate column for raw data property '" + 
+          rawDataType + "[" + name + "]' in file '" + xmlFile + "': column=" + column);
       }
       usedNames.add("column:" + column);
@@ -408 +418 @@
       if (channel < 0 || channel > channels)
       {
-        throw new InvalidDataException("Channel for property " + rawDataType+ "[" + name + "]" 
-            +  " must be >= 0 and < " + channels + ": channel=" + channel);
+        throw new InvalidDataException("Channel for raw data property '" + 
+            rawDataType+ "[" + name + "]' in file '" + xmlFile + "'" +
+            " must be >= 0 and < " + channels + ": channel=" + channel);
       }
       properties.add(new RawDataProperty(name, title, description, column, type, length, nullable, averageMethod, channel));
@@ -421 +432 @@
   */
   @SuppressWarnings({"unchecked"})
-  private static List<IntensityFormula> loadIntensityFormulas(Element rawDataTypeElement, int channels)
+  private static List<IntensityFormula> loadIntensityFormulas(Element rawDataTypeElement, int channels, String xmlFile)
   {
     List<IntensityFormula> formulas = new ArrayList<IntensityFormula>();
@@ -432 +443 @@
       if (!ExtendedProperty.isValidName(name))
       {
-        throw new InvalidDataException("Invalid intensity formula for raw data type " + 
-          rawDataType + ": name=" + name);
+        throw new InvalidDataException("Invalid intensity formula for raw data type '" + 
+          rawDataType + "' in file '" + xmlFile + "': name=" + name);
       }
       if (usedNames.contains("name:" + name))
       {
-        throw new InvalidDataException("Duplicate intensity formula for raw data type " + 
-          rawDataType + ": name=" + name);
+        throw new InvalidDataException("Duplicate intensity formula for raw data type '" + 
+          rawDataType + "' in file '" + xmlFile + "': name=" + name);
       }
       usedNames.add("name:" + name);
@@ -451 +462 @@
         if (channel <= 0 || channel > channels)
         {
-          throw new InvalidDataException("Channel for intensity formula " + 
-              rawDataType+ "[" + name + "]" 
-              +  " must be > 0 and < " + channels + ": channel=" + channel);
+          throw new InvalidDataException("Channel for intensity formula '" + 
+              rawDataType+ "[" + name + "]' in file '" + xmlFile + "'"  +
+              " must be > 0 and < " + channels + ": channel=" + channel);
         }
         if (expressions[channel-1] != null)
         {
-          throw new InvalidDataException("Duplicate expression for intensity formula " + 
-            rawDataType + "[" + name + "]: channel=" + channel);
+          throw new InvalidDataException("Duplicate expression for intensity formula '" + 
+            rawDataType + "[" + name + "]' in file '" + xmlFile + "': channel=" + channel);
         }
         if (exp == null)
         {
-          throw new InvalidDataException("Missing expression for formula " + 
-            rawDataType + "[" + name + "]: channel=" + channel);
+          throw new InvalidDataException("Missing expression for formula '" + 
+            rawDataType + "[" + name + "]' in file '" + xmlFile + "': channel=" + channel);
         }
         expressions[channel-1] = exp;