Changeset 3818

Timestamp:

Oct 12, 2007, 11:14:35 AM (15 years ago)

Author:

Nicklas Nordborg

Message:

References #721: Store data in files instead of in the database

• Synchronized Javadoc with code
• Added missing methods to some classes

Location:

branches/filedb

Files:

• src/core/common-queries.xml (modified) (1 diff)
• src/core/net/sf/basedb/core/ArrayDesign.java (modified) (4 diffs)
• src/core/net/sf/basedb/core/DataFileType.java (modified) (7 diffs)
• src/core/net/sf/basedb/core/FileSet.java (modified) (8 diffs)
• src/core/net/sf/basedb/core/FileStoreEnabled.java (modified) (1 diff)
• src/core/net/sf/basedb/core/FileStoreUtil.java (modified) (4 diffs)
• src/core/net/sf/basedb/core/Install.java (modified) (1 diff)
• src/core/net/sf/basedb/core/Platform.java (modified) (8 diffs)
• src/core/net/sf/basedb/core/PlatformFileType.java (modified) (5 diffs)
• src/core/net/sf/basedb/core/PlatformVariant.java (modified) (2 diffs)
• src/core/net/sf/basedb/core/RawBioAssay.java (modified) (9 diffs)
• src/core/net/sf/basedb/core/RawDataTypes.java (modified) (1 diff)
• src/core/net/sf/basedb/core/data/DataFileTypeData.java (modified) (1 diff)
• www/views/rawbioassays/edit_rawbioassay.jsp (modified) (1 diff)

branches/filedb/src/core/common-queries.xml

@@ -2881 +2881 @@
     </description>
   </query>
+
+  <query id="GET_PLATFORMFILETYPE_FOR_DATAFILETYPE" type="HQL">
+    <sql>
+      SELECT {1}
+      FROM PlatformFileTypeData pft
+      WHERE pft.dataFileType = :dataFileType
+    </sql>
+    <description>
+      A Hibernate query that gets platform file types of a specific 
+      data file type.
+    </description>
+  </query>
   
   <query id="GET_FILESETMEMBER_FOR_DATAFILETYPE" type="HQL">

branches/filedb/src/core/net/sf/basedb/core/ArrayDesign.java

@@ -98 +98 @@
     @return The new <code>ArrayDesign</code> item
     @throws BaseException If there is an error
+    @since 2.5
   */
   public static ArrayDesign getNew(DbControl dc, Platform platform)
@@ -116 +117 @@
     @return The new <code>ArrayDesign</code> item
     @throws BaseException If there is an error
+    @since 2.5
   */
   public static ArrayDesign getNew(DbControl dc, PlatformVariant variant)
@@ -195 +197 @@
     -------------------------------------------
   */
+  /**
+    @since 2.5
+  */
   public FileSet getFileSet()
   {
@@ -207 +212 @@
     return fs;
   }
+  /**
+    @since 2.5
+  */
   public boolean hasFileSet()
   {
     return getData().getFileSet() != null;
   }
+  /**
+    @since 2.5
+  */
   public Platform getPlatform()
   {
     return getDbControl().getItem(Platform.class, getData().getPlatform());
   }
+  /**
+    @since 2.5
+  */
   public PlatformVariant getVariant()
   {
     return getDbControl().getItem(PlatformVariant.class, getData().getVariant());
   }
+  /**
+    There are no parent file sets.
+    @return Always null
+    @since 2.5
+  */
   public Collection<FileSet> getParentFileSets()
   {

branches/filedb/src/core/net/sf/basedb/core/DataFileType.java

@@ -34 +34 @@
 
 /**
-  This class is used to represent a type of file that is used to
-  hold data instead of importing it into the database.
+  This class is used to represent types of files that are used
+  to hold data instead of importing the data into into the database.
+  <p>
+  This class can define a {@link DataFileValidator} and a 
+  {@link DataFileMetadataReader} that can be used to validate the if a
+  file is actually of the correct type and, if it is, extract metadata from it.
 
   @author Nicklas
@@ -53 +57 @@
   public static final Item TYPE = Item.DATAFILETYPE;
 
+  /**
+    The external ID for the file type representing
+    Affymetrix CDF files.
+  */
   public static String AFFYMETRIX_CDF = "affymetrix.cdf";
+
+  /**
+    The external ID for the file type representing
+    Affymetrix CEL files.
+  */
   public static String AFFYMETRIX_CEL = "affymetrix.cel";
+
+  /**
+    The external ID for the file type representing
+    a generic raw data file of any type.
+  */
   public static String GENERIC_RAW_DATA = "generic.rawdata";
+
+  /**
+    The external ID for the file type representing
+    a generic reporter map (array design layout) of any type.
+  */
   public static String GENERIC_REPORTER_MAP = "generic.reportermap";
+
+  /**
+    The external ID for the file type representing
+    a generic print map (array design layout) of any type.
+  */
   public static String GENERIC_PRINT_MAP = "generic.printmap";
 
@@ -218 +246 @@
     <ul>
     <li>A file has been tagged with this type in a file set
+    <li>A platform is using this file type
     </ul>
   */
@@ -234 +263 @@
     query.setEntity("dataFileType", this.getData());
     long count = HibernateUtil.loadData(Long.class, query);
+    if (count == 0)
+    {
+      query = HibernateUtil.getPredefinedQuery(session, 
+        "GET_PLATFORMFILETYPE_FOR_DATAFILETYPE", "count(*)");
+      /*
+        SELECT {1}
+        FROM PlatformFileTypeData pft
+        WHERE pft.dataFileType = :dataFileType
+      */
+      query.setEntity("dataFileType", this.getData());
+      count = HibernateUtil.loadData(Long.class, query);
+    }
     return count > 0 || super.isUsed();
   }
@@ -240 +281 @@
     <ul>
     <li>{@link FileSetMember}:s using this file type
+    <li>{@link PlatformFileType}:s using this file type
     <ul>
   */
@@ -256 +298 @@
     query.setEntity("dataFileType", this.getData());
     addUsingItems(using, Item.FILESETMEMBER, query);
+    query = HibernateUtil.getPredefinedQuery(session, 
+      "GET_PLATFORMFILETYPE_FOR_DATAFILETYPE", "pft.id");
+      /*
+        SELECT {1}
+        FROM PlatformFileTypeData pft
+        WHERE pft.dataFileType = :dataFileType
+      */
+    query.setEntity("dataFileType", this.getData());
+    addUsingItems(using, Item.PLATFORMFILETYPE, query);
     return using;
   }
@@ -466 +517 @@
   }
 
-  
   /**
     Get a query that returns all platforms/variants that are using this

branches/filedb/src/core/net/sf/basedb/core/FileSet.java

@@ -257 +257 @@
   /**
     Get a query that returns all members in a file set with a specific 
-    generic type. Note! The query check the generic type of the 
+    generic {@link FileType}. Note! The query checks the generic type from the 
     {@link DataFileType#getGenericType()}, not the {@link File#getFileType()}.
     
@@ -350 +350 @@
   
   /**
-    Remove the member of the given type.
+    Remove the member of the given type. If a {@link DataFileMetadataReader}
+    is associated with the file type this method will call
+    {@link DataFileMetadataReader#resetMetadata(DbControl)}.
+
     @param type A <code>DataFileType</code> object 
     @throws InvalidUseOfNullException If type is null
     @throws PermissionDeniedException If the logged in user
       doesn't have write permission for this file set
-   */
+  */
   public void removeMember(DataFileType type)
   {
@@ -398 +401 @@
   {
     ItemQuery<DataFileType> query = DataFileType.getQuery();
-    Set<DataFileTypeData> memberTypes = getData().getMembers().keySet();
-    Set<Integer> memberIds = new HashSet<Integer>(memberTypes.size());
-    for (DataFileTypeData member : memberTypes)
-    {
-      memberIds.add(member.getId());
-    }
+    query.joinPermanent(Hql.innerJoin("members", Item.FILESETMEMBER.getAlias()));
     query.restrictPermanent(
-      Restrictions.in(
-        Hql.property("id"), 
-        Expressions.parameter("members")
+      Restrictions.eq(
+        Hql.property(Item.FILESETMEMBER.getAlias(), "fileSet"), 
+        Hql.entity(this)
       )
     );
-    query.setPermanentParameter("members", memberIds, null);
     return query;
   }
@@ -450 +447 @@
   
   /**
-    Get a query that is returning all data file types that
-    have been 
+    Get a query that is returning all file types that
+    have been marked as required by the associated platform but has no member
+    in this file set.
     @return A query or null if no platform is associated with this
       item
@@ -497 +495 @@
     share the same instance. This method will not throw any errors
     that have been caused by the validation process. Errors are returned
-    in the list and is reported to by setting the {@link FileSetMember#isValid()}
-    and {@link FileSetMember#getErrorMessage()} properties for each invalid file.
+    as a list of {@link Throwable}:s and is also reported by setting the 
+    {@link FileSetMember#isValid()} and {@link FileSetMember#getErrorMessage()} 
+    properties for each file.
+    <p>
+    Note! {@link PermissionDeniedException}:s are not considered validation errors.
+    They are ignored and doesn't change the validation status.
+    <p>
+    {@link InvalidRelationException} is a less severe type of exception
+    and can be used by a validator when the actual file it is validating
+    is valid but a dependent file is not. For example, a valid CEL file has
+    been attached to a raw bioassay, but it doesn't match the CDF file attached 
+    to the array design. If a validator throws this exception the member status
+    will be set to invalid, but metadata extraction is still done.
+    
     @param dc The DbControl to use in the validation process
     @param extractMetadata TRUE to also extract metadata, FALSE otherwise
@@ -564 +574 @@
           invalidRelation = t instanceof InvalidRelationException;
           ignore = t instanceof PermissionDeniedException;
-          errors.add(t);
           isValid = false;
           errorMessage = t.getMessage();
-          if (!ignore && !invalidRelation) handler.resetMetadata(dc);
+          if (!ignore)
+          {
+            errors.add(t);
+            if (!invalidRelation) handler.resetMetadata(dc);
+          }
         }
       }
@@ -579 +592 @@
         {
           ignore = t instanceof PermissionDeniedException;
-          errors.add(t);
           isValid = false;
           errorMessage = t.getMessage();
-          if (!ignore) handler.resetMetadata(dc);
+          if (!ignore)
+          {
+            errors.add(t);
+            handler.resetMetadata(dc);
+          }
         }
       }
@@ -599 +615 @@
     }
   }
+  
+  
+  /**
+    Wraps a {@link DataFileValidator} and /or {@link DataFileMetadataReader}.
+    as a single item. This class takes care of calling the validator/metadata reader
+    and to set validation status and error messages on the associated file set members.
+  */
   private static class DataFileHandlerWrapper
   {

branches/filedb/src/core/net/sf/basedb/core/FileStoreEnabled.java

@@ -28 +28 @@
 /**
   This interface is implemented by items that can use files instead
-  of the database for storing data. For example, it replaced raw data
+  of the database for storing data. For example, to store raw data
   for a {@link RawBioAssay} and features for a {@link ArrayDesign}.
 

branches/filedb/src/core/net/sf/basedb/core/FileStoreUtil.java

@@ -37 +37 @@
 
 /**
-  Utility class working with {@link FileStoreEnabled} items.
+  Utility class working with {@link FileStoreEnabled} items. 
 
   @author nicklas
@@ -111 +111 @@
   
   /**
-    Get all files with a specific generic file type in a given item's file
+    Get all files with a specific generic {@link FileType} in a given item's file
     set. Note! This method will only return files that the logged in user
-    has permission to read. Other files in the file set will be ignored.
+    has permission to read. Other files in the file set will be ignored. 
+    Note! This method checks the file type from {@link DataFileType#getGenericType()},
+    not the file type from {@link File#getFileType()}.
     
     @param dc A DbControl to use for database access
@@ -174 +176 @@
   /**
     Set or remove a file to {@link FileStoreEnabled} item based
-    on the file's generic {@link FileType}.
+    on the file's generic {@link FileType}. Note! This method checks the file 
+    type from {@link DataFileType#getGenericType()},
+    not the file type from {@link File#getFileType()}.
   
     @param dc A DbControl to use for database access
@@ -224 +228 @@
     }
     return member;
-    
-    
   }
   

branches/filedb/src/core/net/sf/basedb/core/Install.java

@@ -2239 +2239 @@
         {
           platform.setChannels(channels);
+          platform.setRawDataType("platform." + externalId);
         }
         else

branches/filedb/src/core/net/sf/basedb/core/Platform.java

@@ -33 +33 @@
 
 /**
-  This class is used to represent an experimental
-  platform in BASE.
+  This class is used to represent an experimental platform in BASE.
+  An experimental platform is used to define the types of files
+  that are used as data files. For example, the {@link Platform#AFFYMETRIX}
+  platform defines {@link DataFileType#AFFYMETRIX_CDF CDF} and 
+  {@link DataFileType#AFFYMETRIX_CEL CEL} files.
+  <p>
+  A platform may have variants. A {@link PlatformVariant} has independent
+  settings but may inherit file types from the main platform.
+  <p>
+  Some platforms are file-only platforms, and has data that can't be imported
+  into the database, for example the Affymetrix platform. Platforms that
+  are not file-only can also import data into the database. Such platforms
+  may be locked to a single {@link RawDataType} or they may be {@link Platform#GENERIC}.
 
   @author Nicklas
@@ -53 +64 @@
 
   /**
-    The id for the <code>Generic</code> platform. It allows
+    The external id for the <code>Generic</code> platform. It allows
     importing raw data into the database for all raw data
     type.
+    @see Platform#getByExternalId(DbControl, String)
   */
   public static final String GENERIC = "generic";
   
   /**
-    The id for the <code>Affymetrix</code> platform. It 
+    The external id for the <code>Affymetrix</code> platform. It 
     is a file-only platform.
+    @see Platform#getByExternalId(DbControl, String)
   */
   public static final String AFFYMETRIX = "affymetrix";
@@ -79 +92 @@
     Platform p = dc.newItem(Platform.class);
     p.getData().setFileOnly(true);
+    p.getData().setRawDataType("platform." + externalId);
     p.setExternalId(externalId);
     p.setChannels(channels);
@@ -292 +306 @@
   }
   /**
-    Deny deletion of the GENERIC platform. It is required by the
-    system.
+    Deny deletion of the GENERIC and Affymetrix platform. The GENERIC
+    platform is required by the system and the AFFYMETRIX platform is
+    required for backwards compatibility reasons.
   */
   @Override
   protected void initPermissions(int granted, int denied)
   {
-    if (GENERIC.equals(getExternalId())) 
+    if (GENERIC.equals(getExternalId()) || AFFYMETRIX.equals(getExternalId())) 
     {
       denied |= Permission.deny(Permission.DELETE);
@@ -370 +385 @@
     Get the raw data type this platform is locked to. File-only platforms
     are always locked to a raw data type. Platforms that support import of
-    data to the database may or may not be locked.
+    data to the database may or may not be locked. Note! If the variant is
+    a file-only variant that hasn't been committed to the database a 
+    new instance of {@link RawDataType} is returned for each call to this
+    method.
     @return The raw data type or null if the platform isn't locked
   */
   public RawDataType getRawDataType()
   {
-    RawDataType rdt = null;
-    if (isFileOnly())
-    {
-      rdt = RawDataTypes.getRawDataType("platform." + getExternalId());
-    }
-    else
-    {
-      rdt = RawDataTypes.getRawDataType(getData().getRawDataType());
-    }
-    return rdt;
+    return isInDatabase() ?
+        RawDataTypes.getRawDataType(getData().getRawDataType()) :
+        new RawDataType(getData());
   }
   
@@ -405 +416 @@
   /**
     Create a new {@link PlatformVariant} of this platform. 
-    @param systemId The systemId of the new variant (must ne unique)
+    @param systemId The systemId of the new variant (must be unique)
     @return The new <code>PlatformVariant</code> object
     @see PlatformVariant#getNew(DbControl, Platform, String)
@@ -415 +426 @@
   
   /**
+    Get the file types registered with this platform.
     @see PlatformFileType#getQuery(Platform, PlatformVariant, boolean)
   */
@@ -483 +495 @@
   }
   
-  
+  /**
+    Get information about a file type that has been registered with this
+    platform/variant.
+    
+    @param type The file type to get information for
+    @param variant An optional variant
+    @return Null if the file type isn't registered with the platform/variant 
+      or a {@link PlatformFileType} object
+  */
   public PlatformFileType getFileType(DataFileType type, PlatformVariant variant)
   {

branches/filedb/src/core/net/sf/basedb/core/PlatformFileType.java

@@ -55 +55 @@
     new QueryRuntimeFilterFactory.ChildFilter(TYPE, Item.PLATFORM);
 
+  /**
+    Create a new item linking a file type to a platform/variant.
+    @param dc
+    @param platform The platform
+    @param variant The variant (optional)
+    @param fileType The file type
+    @param required If files of this type are required when using the platform or no
+  */
   static PlatformFileType getNew(DbControl dc, Platform platform, PlatformVariant variant,
     DataFileType fileType, boolean required)
@@ -67 +75 @@
   }
   
+  /**
+    Get a <code>PlatformFileType</code> object when you know the ID.
+  
+    @param dc The <code>DbControl</code> which will be used for
+      permission checking and database access.
+    @param id The ID of the item to load
+    @return The <code>PlatformFileType</code> item
+    @throws ItemNotFoundException If an item with the specified ID is not found
+    @throws PermissionDeniedException If the logged in user doesn't have 
+      {@link Permission#READ} permission to the item
+    @throws BaseException If there is another error
+  */
+  public static PlatformFileType getById(DbControl dc, int id)
+    throws ItemNotFoundException, PermissionDeniedException, BaseException
+  {
+    PlatformFileType p = dc.loadItem(PlatformFileType.class, id);
+    if (p == null) throw new ItemNotFoundException("PlatformFileType[id="+id+"]");
+    return p;
+  }
   
   /**
@@ -77 +104 @@
       <code>variant</code> is null
     <li>If <code>variant</code> is null, the query will return file
-      types for all variants if <code>restrict=false</code>. If 
+      types the platform and all variants if <code>restrict=false</code>. If 
       <code>restrict=true</code> the query will only return items
-      registered as common file types for all variants
+      registered directly with the platform
     <li>If <code>variant</code> isn't null, the query will return file
       types that have been registered for that variant plus file
-      types registered as common for all variants if <code>restrict=false</code>.
+      types registered with the platform if <code>restrict=false</code>.
       If <code>restrict=true</code> the query will only return file types
       registered for that particular variant.
@@ -173 +200 @@
   }
   // -------------------------------------------
-
-  
   
   /**
@@ -209 +234 @@
     hint to client applications so they can create a proper GUI. To check
     if a {@link FileSet} contains all files that have been marked as required
-    use TODO.
+    use {@link FileSet#hasAllRequiredFiles()}.
   */
   public boolean isRequired()

branches/filedb/src/core/net/sf/basedb/core/PlatformVariant.java

@@ -93 +93 @@
     v.setPlatform(platform);
     v.getData().setFileOnly(true);
+    v.getData().setRawDataType("variant." + externalId);
     v.setExternalId(externalId);
     v.setChannels(channels);
@@ -400 +401 @@
     Get the raw data type this platform variant is locked to. File-only variants
     are always locked to a raw data type. Variants that support import of
-    data to the database may or may not be locked.
+    data to the database may or may not be locked. Note! If the variant is
+    a file-only variant that hasn't been committed to the database a 
+    new instance of {@link RawDataType} is returned for each call to this
+    method.
     @return The raw data type or null if the platform isn't locked
   */
   public RawDataType getRawDataType()
   {
-    RawDataType rdt = null;
-    if (isFileOnly())
-    {
-      rdt = RawDataTypes.getRawDataType("variant." + getExternalId());
-    }
-    else
-    {
-      rdt = RawDataTypes.getRawDataType(getData().getRawDataType());
-    }
-    return rdt;
+    return isInDatabase() ?
+        RawDataTypes.getRawDataType(getData().getRawDataType()) :
+        new RawDataType(getData());
   }
   

branches/filedb/src/core/net/sf/basedb/core/RawBioAssay.java

@@ -273 +273 @@
     -------------------------------------------
   */
+  /**
+    @since 2.5
+  */
   public FileSet getFileSet()
   {
@@ -285 +288 @@
     return fs;
   }
+  /**
+    @since 2.5
+  */
   public boolean hasFileSet()
   {
     return getData().getFileSet() != null;
   }
+  /**
+    @since 2.5
+  */
   public Platform getPlatform()
   {
     return getDbControl().getItem(Platform.class, getData().getPlatform());
   }
+  /**
+    @since 2.5
+  */
   public PlatformVariant getVariant()
   {
@@ -299 +311 @@
   /**
     Get the file set from the array design, if it exists.
+    @since 2.5
   */
   public Collection<FileSet> getParentFileSets()
@@ -798 +811 @@
   /**
     If raw data has been added to this raw bioassay or not. Note that this method
-    can return true even if the {@link #getSpots()} returns 0 if the raw data
-    type uses files to store it's data.
+    checks both data in the database and data in the form of files.
   */
   public boolean hasData()
@@ -878 +890 @@
   }
   
+  /**
+    Remove all headers from the raw bioassay.
+    @since 2.5
+  */
   public void removeHeaders()
   {
@@ -884 +900 @@
   }
 
+  /**
+    Remove a specific header from the raw bioassay.
+    @param name The name of the header
+    @since 2.5
+  */
   public void removeHeader(String name)
   {
@@ -940 +961 @@
   
   /**
-    Delete the raw data that has been added to the raw bioassay. You should not
-    call this method in the same transaction as adding raw data with the
-    batcher from {@link #getRawDataBatcher()}.
+    Delete the raw data that has been added to the database for this
+    raw bioassay. You should not call this method in the same transaction as 
+    adding raw data with the batcher from {@link #getRawDataBatcher()}.
 
     @throws PermissionDeniedException If the logged in user doesn't have
@@ -962 +983 @@
       throw new PermissionDeniedException("The raw bioassay is used in one or more experiments: "+this);
     }
-    if (getSpots() > 0)
+    if (getNumDbSpots() > 0)
     {
       RawDataBatcher rdb = RawDataBatcher.getNew(getDbControl(), this);
@@ -1079 +1100 @@
       DataResultIterator<RawData> result = rawQuery.iterate(dc);
       int numValidated = 0;
-      int numTotal = getSpots();
+      int numTotal = getNumDbSpots();
       while (result.hasNext())
       {

branches/filedb/src/core/net/sf/basedb/core/RawDataTypes.java

@@ -224 +224 @@
     }
   }
-  
   
   private static RawDataType getPlatformRawDataType(String id)

branches/filedb/src/core/net/sf/basedb/core/data/DataFileTypeData.java

@@ -208 +208 @@
   }
   
+  private Set<FileSetMemberData> members;
+  /**
+    This is the inverse end.
+    @see FileSetMemberData#getDataFileType()
+    @hibernate.set lazy="true" inverse="true" 
+    @hibernate.collection-key column="`datafiletype_id`"
+    @hibernate.collection-one-to-many class="net.sf.basedb.core.data.FileSetMemberData"
+  */
+  Set<FileSetMemberData> getMembers()
+  {
+    return members;
+  }
+  void setMembers(Set<FileSetMemberData> members)
+  {
+    this.members = members;
+  }
+
 }

branches/filedb/www/views/rawbioassays/edit_rawbioassay.jsp

@@ -262 +262 @@
   %>
 
-  <base:page type="popup" title="<%=title%>">
+  <%@page import="net.sf.basedb.core.Include"%>
+<base:page type="popup" title="<%=title%>">
   <base:head scripts="tabcontrol.js,annotations.js,platforms.js" styles="tabcontrol.css">
     <script language="JavaScript">