Changeset 3818


Ignore:
Timestamp:
Oct 12, 2007, 11:14:35 AM (15 years ago)
Author:
Nicklas Nordborg
Message:

References #721: Store data in files instead of in the database

  • Synchronized Javadoc with code
  • Added missing methods to some classes
Location:
branches/filedb
Files:
14 edited

Legend:

Unmodified
Added
Removed
  • branches/filedb/src/core/common-queries.xml

    r3817 r3818  
    28812881    </description>
    28822882  </query>
     2883
     2884  <query id="GET_PLATFORMFILETYPE_FOR_DATAFILETYPE" type="HQL">
     2885    <sql>
     2886      SELECT {1}
     2887      FROM PlatformFileTypeData pft
     2888      WHERE pft.dataFileType = :dataFileType
     2889    </sql>
     2890    <description>
     2891      A Hibernate query that gets platform file types of a specific
     2892      data file type.
     2893    </description>
     2894  </query>
    28832895 
    28842896  <query id="GET_FILESETMEMBER_FOR_DATAFILETYPE" type="HQL">
  • branches/filedb/src/core/net/sf/basedb/core/ArrayDesign.java

    r3817 r3818  
    9898    @return The new <code>ArrayDesign</code> item
    9999    @throws BaseException If there is an error
     100    @since 2.5
    100101  */
    101102  public static ArrayDesign getNew(DbControl dc, Platform platform)
     
    116117    @return The new <code>ArrayDesign</code> item
    117118    @throws BaseException If there is an error
     119    @since 2.5
    118120  */
    119121  public static ArrayDesign getNew(DbControl dc, PlatformVariant variant)
     
    195197    -------------------------------------------
    196198  */
     199  /**
     200    @since 2.5
     201  */
    197202  public FileSet getFileSet()
    198203  {
     
    207212    return fs;
    208213  }
     214  /**
     215    @since 2.5
     216  */
    209217  public boolean hasFileSet()
    210218  {
    211219    return getData().getFileSet() != null;
    212220  }
     221  /**
     222    @since 2.5
     223  */
    213224  public Platform getPlatform()
    214225  {
    215226    return getDbControl().getItem(Platform.class, getData().getPlatform());
    216227  }
     228  /**
     229    @since 2.5
     230  */
    217231  public PlatformVariant getVariant()
    218232  {
    219233    return getDbControl().getItem(PlatformVariant.class, getData().getVariant());
    220234  }
     235  /**
     236    There are no parent file sets.
     237    @return Always null
     238    @since 2.5
     239  */
    221240  public Collection<FileSet> getParentFileSets()
    222241  {
  • branches/filedb/src/core/net/sf/basedb/core/DataFileType.java

    r3812 r3818  
    3434
    3535/**
    36   This class is used to represent a type of file that is used to
    37   hold data instead of importing it into the database.
     36  This class is used to represent types of files that are used
     37  to hold data instead of importing the data into into the database.
     38  <p>
     39  This class can define a {@link DataFileValidator} and a
     40  {@link DataFileMetadataReader} that can be used to validate the if a
     41  file is actually of the correct type and, if it is, extract metadata from it.
    3842
    3943  @author Nicklas
     
    5357  public static final Item TYPE = Item.DATAFILETYPE;
    5458
     59  /**
     60    The external ID for the file type representing
     61    Affymetrix CDF files.
     62  */
    5563  public static String AFFYMETRIX_CDF = "affymetrix.cdf";
     64
     65  /**
     66    The external ID for the file type representing
     67    Affymetrix CEL files.
     68  */
    5669  public static String AFFYMETRIX_CEL = "affymetrix.cel";
     70
     71  /**
     72    The external ID for the file type representing
     73    a generic raw data file of any type.
     74  */
    5775  public static String GENERIC_RAW_DATA = "generic.rawdata";
     76
     77  /**
     78    The external ID for the file type representing
     79    a generic reporter map (array design layout) of any type.
     80  */
    5881  public static String GENERIC_REPORTER_MAP = "generic.reportermap";
     82
     83  /**
     84    The external ID for the file type representing
     85    a generic print map (array design layout) of any type.
     86  */
    5987  public static String GENERIC_PRINT_MAP = "generic.printmap";
    6088
     
    218246    <ul>
    219247    <li>A file has been tagged with this type in a file set
     248    <li>A platform is using this file type
    220249    </ul>
    221250  */
     
    234263    query.setEntity("dataFileType", this.getData());
    235264    long count = HibernateUtil.loadData(Long.class, query);
     265    if (count == 0)
     266    {
     267      query = HibernateUtil.getPredefinedQuery(session,
     268        "GET_PLATFORMFILETYPE_FOR_DATAFILETYPE", "count(*)");
     269      /*
     270        SELECT {1}
     271        FROM PlatformFileTypeData pft
     272        WHERE pft.dataFileType = :dataFileType
     273      */
     274      query.setEntity("dataFileType", this.getData());
     275      count = HibernateUtil.loadData(Long.class, query);
     276    }
    236277    return count > 0 || super.isUsed();
    237278  }
     
    240281    <ul>
    241282    <li>{@link FileSetMember}:s using this file type
     283    <li>{@link PlatformFileType}:s using this file type
    242284    <ul>
    243285  */
     
    256298    query.setEntity("dataFileType", this.getData());
    257299    addUsingItems(using, Item.FILESETMEMBER, query);
     300    query = HibernateUtil.getPredefinedQuery(session,
     301      "GET_PLATFORMFILETYPE_FOR_DATAFILETYPE", "pft.id");
     302      /*
     303        SELECT {1}
     304        FROM PlatformFileTypeData pft
     305        WHERE pft.dataFileType = :dataFileType
     306      */
     307    query.setEntity("dataFileType", this.getData());
     308    addUsingItems(using, Item.PLATFORMFILETYPE, query);
    258309    return using;
    259310  }
     
    466517  }
    467518
    468  
    469519  /**
    470520    Get a query that returns all platforms/variants that are using this
  • branches/filedb/src/core/net/sf/basedb/core/FileSet.java

    r3817 r3818  
    257257  /**
    258258    Get a query that returns all members in a file set with a specific
    259     generic type. Note! The query check the generic type of the
     259    generic {@link FileType}. Note! The query checks the generic type from the
    260260    {@link DataFileType#getGenericType()}, not the {@link File#getFileType()}.
    261261   
     
    350350 
    351351  /**
    352     Remove the member of the given type.
     352    Remove the member of the given type. If a {@link DataFileMetadataReader}
     353    is associated with the file type this method will call
     354    {@link DataFileMetadataReader#resetMetadata(DbControl)}.
     355
    353356    @param type A <code>DataFileType</code> object
    354357    @throws InvalidUseOfNullException If type is null
    355358    @throws PermissionDeniedException If the logged in user
    356359      doesn't have write permission for this file set
    357    */
     360  */
    358361  public void removeMember(DataFileType type)
    359362  {
     
    398401  {
    399402    ItemQuery<DataFileType> query = DataFileType.getQuery();
    400     Set<DataFileTypeData> memberTypes = getData().getMembers().keySet();
    401     Set<Integer> memberIds = new HashSet<Integer>(memberTypes.size());
    402     for (DataFileTypeData member : memberTypes)
    403     {
    404       memberIds.add(member.getId());
    405     }
     403    query.joinPermanent(Hql.innerJoin("members", Item.FILESETMEMBER.getAlias()));
    406404    query.restrictPermanent(
    407       Restrictions.in(
    408         Hql.property("id"),
    409         Expressions.parameter("members")
     405      Restrictions.eq(
     406        Hql.property(Item.FILESETMEMBER.getAlias(), "fileSet"),
     407        Hql.entity(this)
    410408      )
    411409    );
    412     query.setPermanentParameter("members", memberIds, null);
    413410    return query;
    414411  }
     
    450447 
    451448  /**
    452     Get a query that is returning all data file types that
    453     have been
     449    Get a query that is returning all file types that
     450    have been marked as required by the associated platform but has no member
     451    in this file set.
    454452    @return A query or null if no platform is associated with this
    455453      item
     
    497495    share the same instance. This method will not throw any errors
    498496    that have been caused by the validation process. Errors are returned
    499     in the list and is reported to by setting the {@link FileSetMember#isValid()}
    500     and {@link FileSetMember#getErrorMessage()} properties for each invalid file.
     497    as a list of {@link Throwable}:s and is also reported by setting the
     498    {@link FileSetMember#isValid()} and {@link FileSetMember#getErrorMessage()}
     499    properties for each file.
     500    <p>
     501    Note! {@link PermissionDeniedException}:s are not considered validation errors.
     502    They are ignored and doesn't change the validation status.
     503    <p>
     504    {@link InvalidRelationException} is a less severe type of exception
     505    and can be used by a validator when the actual file it is validating
     506    is valid but a dependent file is not. For example, a valid CEL file has
     507    been attached to a raw bioassay, but it doesn't match the CDF file attached
     508    to the array design. If a validator throws this exception the member status
     509    will be set to invalid, but metadata extraction is still done.
     510   
    501511    @param dc The DbControl to use in the validation process
    502512    @param extractMetadata TRUE to also extract metadata, FALSE otherwise
     
    564574          invalidRelation = t instanceof InvalidRelationException;
    565575          ignore = t instanceof PermissionDeniedException;
    566           errors.add(t);
    567576          isValid = false;
    568577          errorMessage = t.getMessage();
    569           if (!ignore && !invalidRelation) handler.resetMetadata(dc);
     578          if (!ignore)
     579          {
     580            errors.add(t);
     581            if (!invalidRelation) handler.resetMetadata(dc);
     582          }
    570583        }
    571584      }
     
    579592        {
    580593          ignore = t instanceof PermissionDeniedException;
    581           errors.add(t);
    582594          isValid = false;
    583595          errorMessage = t.getMessage();
    584           if (!ignore) handler.resetMetadata(dc);
     596          if (!ignore)
     597          {
     598            errors.add(t);
     599            handler.resetMetadata(dc);
     600          }
    585601        }
    586602      }
     
    599615    }
    600616  }
     617 
     618 
     619  /**
     620    Wraps a {@link DataFileValidator} and /or {@link DataFileMetadataReader}.
     621    as a single item. This class takes care of calling the validator/metadata reader
     622    and to set validation status and error messages on the associated file set members.
     623  */
    601624  private static class DataFileHandlerWrapper
    602625  {
  • branches/filedb/src/core/net/sf/basedb/core/FileStoreEnabled.java

    r3812 r3818  
    2828/**
    2929  This interface is implemented by items that can use files instead
    30   of the database for storing data. For example, it replaced raw data
     30  of the database for storing data. For example, to store raw data
    3131  for a {@link RawBioAssay} and features for a {@link ArrayDesign}.
    3232
  • branches/filedb/src/core/net/sf/basedb/core/FileStoreUtil.java

    r3817 r3818  
    3737
    3838/**
    39   Utility class working with {@link FileStoreEnabled} items.
     39  Utility class working with {@link FileStoreEnabled} items. 
    4040
    4141  @author nicklas
     
    111111 
    112112  /**
    113     Get all files with a specific generic file type in a given item's file
     113    Get all files with a specific generic {@link FileType} in a given item's file
    114114    set. Note! This method will only return files that the logged in user
    115     has permission to read. Other files in the file set will be ignored.
     115    has permission to read. Other files in the file set will be ignored.
     116    Note! This method checks the file type from {@link DataFileType#getGenericType()},
     117    not the file type from {@link File#getFileType()}.
    116118   
    117119    @param dc A DbControl to use for database access
     
    174176  /**
    175177    Set or remove a file to {@link FileStoreEnabled} item based
    176     on the file's generic {@link FileType}.
     178    on the file's generic {@link FileType}. Note! This method checks the file
     179    type from {@link DataFileType#getGenericType()},
     180    not the file type from {@link File#getFileType()}.
    177181 
    178182    @param dc A DbControl to use for database access
     
    224228    }
    225229    return member;
    226    
    227    
    228230  }
    229231 
  • branches/filedb/src/core/net/sf/basedb/core/Install.java

    r3817 r3818  
    22392239        {
    22402240          platform.setChannels(channels);
     2241          platform.setRawDataType("platform." + externalId);
    22412242        }
    22422243        else
  • branches/filedb/src/core/net/sf/basedb/core/Platform.java

    r3807 r3818  
    3333
    3434/**
    35   This class is used to represent an experimental
    36   platform in BASE.
     35  This class is used to represent an experimental platform in BASE.
     36  An experimental platform is used to define the types of files
     37  that are used as data files. For example, the {@link Platform#AFFYMETRIX}
     38  platform defines {@link DataFileType#AFFYMETRIX_CDF CDF} and
     39  {@link DataFileType#AFFYMETRIX_CEL CEL} files.
     40  <p>
     41  A platform may have variants. A {@link PlatformVariant} has independent
     42  settings but may inherit file types from the main platform.
     43  <p>
     44  Some platforms are file-only platforms, and has data that can't be imported
     45  into the database, for example the Affymetrix platform. Platforms that
     46  are not file-only can also import data into the database. Such platforms
     47  may be locked to a single {@link RawDataType} or they may be {@link Platform#GENERIC}.
    3748
    3849  @author Nicklas
     
    5364
    5465  /**
    55     The id for the <code>Generic</code> platform. It allows
     66    The external id for the <code>Generic</code> platform. It allows
    5667    importing raw data into the database for all raw data
    5768    type.
     69    @see Platform#getByExternalId(DbControl, String)
    5870  */
    5971  public static final String GENERIC = "generic";
    6072 
    6173  /**
    62     The id for the <code>Affymetrix</code> platform. It
     74    The external id for the <code>Affymetrix</code> platform. It
    6375    is a file-only platform.
     76    @see Platform#getByExternalId(DbControl, String)
    6477  */
    6578  public static final String AFFYMETRIX = "affymetrix";
     
    7992    Platform p = dc.newItem(Platform.class);
    8093    p.getData().setFileOnly(true);
     94    p.getData().setRawDataType("platform." + externalId);
    8195    p.setExternalId(externalId);
    8296    p.setChannels(channels);
     
    292306  }
    293307  /**
    294     Deny deletion of the GENERIC platform. It is required by the
    295     system.
     308    Deny deletion of the GENERIC and Affymetrix platform. The GENERIC
     309    platform is required by the system and the AFFYMETRIX platform is
     310    required for backwards compatibility reasons.
    296311  */
    297312  @Override
    298313  protected void initPermissions(int granted, int denied)
    299314  {
    300     if (GENERIC.equals(getExternalId()))
     315    if (GENERIC.equals(getExternalId()) || AFFYMETRIX.equals(getExternalId()))
    301316    {
    302317      denied |= Permission.deny(Permission.DELETE);
     
    370385    Get the raw data type this platform is locked to. File-only platforms
    371386    are always locked to a raw data type. Platforms that support import of
    372     data to the database may or may not be locked.
     387    data to the database may or may not be locked. Note! If the variant is
     388    a file-only variant that hasn't been committed to the database a
     389    new instance of {@link RawDataType} is returned for each call to this
     390    method.
    373391    @return The raw data type or null if the platform isn't locked
    374392  */
    375393  public RawDataType getRawDataType()
    376394  {
    377     RawDataType rdt = null;
    378     if (isFileOnly())
    379     {
    380       rdt = RawDataTypes.getRawDataType("platform." + getExternalId());
    381     }
    382     else
    383     {
    384       rdt = RawDataTypes.getRawDataType(getData().getRawDataType());
    385     }
    386     return rdt;
     395    return isInDatabase() ?
     396        RawDataTypes.getRawDataType(getData().getRawDataType()) :
     397        new RawDataType(getData());
    387398  }
    388399 
     
    405416  /**
    406417    Create a new {@link PlatformVariant} of this platform.
    407     @param systemId The systemId of the new variant (must ne unique)
     418    @param systemId The systemId of the new variant (must be unique)
    408419    @return The new <code>PlatformVariant</code> object
    409420    @see PlatformVariant#getNew(DbControl, Platform, String)
     
    415426 
    416427  /**
     428    Get the file types registered with this platform.
    417429    @see PlatformFileType#getQuery(Platform, PlatformVariant, boolean)
    418430  */
     
    483495  }
    484496 
    485  
     497  /**
     498    Get information about a file type that has been registered with this
     499    platform/variant.
     500   
     501    @param type The file type to get information for
     502    @param variant An optional variant
     503    @return Null if the file type isn't registered with the platform/variant
     504      or a {@link PlatformFileType} object
     505  */
    486506  public PlatformFileType getFileType(DataFileType type, PlatformVariant variant)
    487507  {
  • branches/filedb/src/core/net/sf/basedb/core/PlatformFileType.java

    r3800 r3818  
    5555    new QueryRuntimeFilterFactory.ChildFilter(TYPE, Item.PLATFORM);
    5656
     57  /**
     58    Create a new item linking a file type to a platform/variant.
     59    @param dc
     60    @param platform The platform
     61    @param variant The variant (optional)
     62    @param fileType The file type
     63    @param required If files of this type are required when using the platform or no
     64  */
    5765  static PlatformFileType getNew(DbControl dc, Platform platform, PlatformVariant variant,
    5866    DataFileType fileType, boolean required)
     
    6775  }
    6876 
     77  /**
     78    Get a <code>PlatformFileType</code> object when you know the ID.
     79 
     80    @param dc The <code>DbControl</code> which will be used for
     81      permission checking and database access.
     82    @param id The ID of the item to load
     83    @return The <code>PlatformFileType</code> item
     84    @throws ItemNotFoundException If an item with the specified ID is not found
     85    @throws PermissionDeniedException If the logged in user doesn't have
     86      {@link Permission#READ} permission to the item
     87    @throws BaseException If there is another error
     88  */
     89  public static PlatformFileType getById(DbControl dc, int id)
     90    throws ItemNotFoundException, PermissionDeniedException, BaseException
     91  {
     92    PlatformFileType p = dc.loadItem(PlatformFileType.class, id);
     93    if (p == null) throw new ItemNotFoundException("PlatformFileType[id="+id+"]");
     94    return p;
     95  }
    6996 
    7097  /**
     
    77104      <code>variant</code> is null
    78105    <li>If <code>variant</code> is null, the query will return file
    79       types for all variants if <code>restrict=false</code>. If
     106      types the platform and all variants if <code>restrict=false</code>. If
    80107      <code>restrict=true</code> the query will only return items
    81       registered as common file types for all variants
     108      registered directly with the platform
    82109    <li>If <code>variant</code> isn't null, the query will return file
    83110      types that have been registered for that variant plus file
    84       types registered as common for all variants if <code>restrict=false</code>.
     111      types registered with the platform if <code>restrict=false</code>.
    85112      If <code>restrict=true</code> the query will only return file types
    86113      registered for that particular variant.
     
    173200  }
    174201  // -------------------------------------------
    175 
    176  
    177202 
    178203  /**
     
    209234    hint to client applications so they can create a proper GUI. To check
    210235    if a {@link FileSet} contains all files that have been marked as required
    211     use TODO.
     236    use {@link FileSet#hasAllRequiredFiles()}.
    212237  */
    213238  public boolean isRequired()
  • branches/filedb/src/core/net/sf/basedb/core/PlatformVariant.java

    r3807 r3818  
    9393    v.setPlatform(platform);
    9494    v.getData().setFileOnly(true);
     95    v.getData().setRawDataType("variant." + externalId);
    9596    v.setExternalId(externalId);
    9697    v.setChannels(channels);
     
    400401    Get the raw data type this platform variant is locked to. File-only variants
    401402    are always locked to a raw data type. Variants that support import of
    402     data to the database may or may not be locked.
     403    data to the database may or may not be locked. Note! If the variant is
     404    a file-only variant that hasn't been committed to the database a
     405    new instance of {@link RawDataType} is returned for each call to this
     406    method.
    403407    @return The raw data type or null if the platform isn't locked
    404408  */
    405409  public RawDataType getRawDataType()
    406410  {
    407     RawDataType rdt = null;
    408     if (isFileOnly())
    409     {
    410       rdt = RawDataTypes.getRawDataType("variant." + getExternalId());
    411     }
    412     else
    413     {
    414       rdt = RawDataTypes.getRawDataType(getData().getRawDataType());
    415     }
    416     return rdt;
     411    return isInDatabase() ?
     412        RawDataTypes.getRawDataType(getData().getRawDataType()) :
     413        new RawDataType(getData());
    417414  }
    418415 
  • branches/filedb/src/core/net/sf/basedb/core/RawBioAssay.java

    r3817 r3818  
    273273    -------------------------------------------
    274274  */
     275  /**
     276    @since 2.5
     277  */
    275278  public FileSet getFileSet()
    276279  {
     
    285288    return fs;
    286289  }
     290  /**
     291    @since 2.5
     292  */
    287293  public boolean hasFileSet()
    288294  {
    289295    return getData().getFileSet() != null;
    290296  }
     297  /**
     298    @since 2.5
     299  */
    291300  public Platform getPlatform()
    292301  {
    293302    return getDbControl().getItem(Platform.class, getData().getPlatform());
    294303  }
     304  /**
     305    @since 2.5
     306  */
    295307  public PlatformVariant getVariant()
    296308  {
     
    299311  /**
    300312    Get the file set from the array design, if it exists.
     313    @since 2.5
    301314  */
    302315  public Collection<FileSet> getParentFileSets()
     
    798811  /**
    799812    If raw data has been added to this raw bioassay or not. Note that this method
    800     can return true even if the {@link #getSpots()} returns 0 if the raw data
    801     type uses files to store it's data.
     813    checks both data in the database and data in the form of files.
    802814  */
    803815  public boolean hasData()
     
    878890  }
    879891 
     892  /**
     893    Remove all headers from the raw bioassay.
     894    @since 2.5
     895  */
    880896  public void removeHeaders()
    881897  {
     
    884900  }
    885901
     902  /**
     903    Remove a specific header from the raw bioassay.
     904    @param name The name of the header
     905    @since 2.5
     906  */
    886907  public void removeHeader(String name)
    887908  {
     
    940961 
    941962  /**
    942     Delete the raw data that has been added to the raw bioassay. You should not
    943     call this method in the same transaction as adding raw data with the
    944     batcher from {@link #getRawDataBatcher()}.
     963    Delete the raw data that has been added to the database for this
     964    raw bioassay. You should not call this method in the same transaction as
     965    adding raw data with the batcher from {@link #getRawDataBatcher()}.
    945966
    946967    @throws PermissionDeniedException If the logged in user doesn't have
     
    962983      throw new PermissionDeniedException("The raw bioassay is used in one or more experiments: "+this);
    963984    }
    964     if (getSpots() > 0)
     985    if (getNumDbSpots() > 0)
    965986    {
    966987      RawDataBatcher rdb = RawDataBatcher.getNew(getDbControl(), this);
     
    10791100      DataResultIterator<RawData> result = rawQuery.iterate(dc);
    10801101      int numValidated = 0;
    1081       int numTotal = getSpots();
     1102      int numTotal = getNumDbSpots();
    10821103      while (result.hasNext())
    10831104      {
  • branches/filedb/src/core/net/sf/basedb/core/RawDataTypes.java

    r3817 r3818  
    224224    }
    225225  }
    226  
    227226 
    228227  private static RawDataType getPlatformRawDataType(String id)
  • branches/filedb/src/core/net/sf/basedb/core/data/DataFileTypeData.java

    r3801 r3818  
    208208  }
    209209 
     210  private Set<FileSetMemberData> members;
     211  /**
     212    This is the inverse end.
     213    @see FileSetMemberData#getDataFileType()
     214    @hibernate.set lazy="true" inverse="true"
     215    @hibernate.collection-key column="`datafiletype_id`"
     216    @hibernate.collection-one-to-many class="net.sf.basedb.core.data.FileSetMemberData"
     217  */
     218  Set<FileSetMemberData> getMembers()
     219  {
     220    return members;
     221  }
     222  void setMembers(Set<FileSetMemberData> members)
     223  {
     224    this.members = members;
     225  }
     226
    210227}
  • branches/filedb/www/views/rawbioassays/edit_rawbioassay.jsp

    r3817 r3818  
    262262  %>
    263263
    264   <base:page type="popup" title="<%=title%>">
     264  <%@page import="net.sf.basedb.core.Include"%>
     265<base:page type="popup" title="<%=title%>">
    265266  <base:head scripts="tabcontrol.js,annotations.js,platforms.js" styles="tabcontrol.css">
    266267    <script language="JavaScript">
Note: See TracChangeset for help on using the changeset viewer.