Changeset 5719

Timestamp:

Sep 6, 2011, 1:56:29 PM (12 years ago)

Author:

Nicklas Nordborg

Message:

References #1604: Support for multiple files of the same type in a FileSet?

Batch importers now have support for adding multiple files of the same type. This must be done by having one line in the import file for each file to add.

Location:

trunk/src

Files:

• core/net/sf/basedb/core/FileSet.java (modified) (4 diffs)
• core/net/sf/basedb/core/FileStoreUtil.java (modified) (2 diffs)
• core/net/sf/basedb/core/data/FileSetMemberData.java (modified) (3 diffs)
• plugins/core/net/sf/basedb/plugins/ManualTransformCreator.java (modified) (4 diffs)
• plugins/core/net/sf/basedb/plugins/batchimport/AbstractItemImporter.java (modified) (3 diffs)
• plugins/core/net/sf/basedb/plugins/batchimport/ArrayDesignImporter.java (modified) (1 diff)
• plugins/core/net/sf/basedb/plugins/batchimport/DerivedBioAssayImporter.java (modified) (1 diff)
• plugins/core/net/sf/basedb/plugins/batchimport/RawBioAssayImporter.java (modified) (1 diff)
• plugins/core/net/sf/basedb/plugins/batchimport/ScanImporter.java (modified) (1 diff)
• test/data/test.batchimport.rawbioassays.txt (modified) (1 diff)

trunk/src/core/net/sf/basedb/core/FileSet.java

@@ -307 +307 @@
     is called. Calling this method is equivalent to first call
     {@link #removeAllOfType(DataFileType)} and then {@link 
-    #addMember(File, DataFileType)}.
+    #addMember(File, DataFileType)} except that if the given file
+    already is a member that member entry is kept.
     
     @param file The file to set as a member (or null to remove it)
@@ -329 +330 @@
     if (file != null) file.checkPermission(Permission.USE);
 
-    removeAllOfType(type);
-    return file == null ? null : addMember(file, type);
+    FileSetMember member = removeAllOfTypeExcept(type, file);
+    if (member == null && file != null)
+    {
+      member = addMember(file, type);
+    }
+    return member;
   }
   
@@ -408 +413 @@
       of the specified type exists
   */
-  @Deprecated
   public FileSetMember getMember(DataFileType type)
   {
@@ -537 +541 @@
     return removed.size();
   }
+  
+  /**
+    Remove all member files of the given type. 
+  
+    @param type A <code>DataFileType</code> object 
+    @throws InvalidUseOfNullException If type is null
+    @throws PermissionDeniedException If the logged in user
+      doesn't have write permission for this file set
+    @return The number of removed files
+    @since 3.0
+  */
+  private FileSetMember removeAllOfTypeExcept(DataFileType type, File file)
+  {
+    DbControl dc = getDbControl();
+    int typeId = type.getId();
+    int fileId = file.getId();
+    FileSetMember remain = null;
+    Iterator<FileSetMemberData> it = getData().getMembers().iterator();
+    // Keep track of removed members so that we can invoke metadata extensions
+    List<ValidatingFileSetMember> removed = new ArrayList<ValidatingFileSetMember>();
+    while (it.hasNext())
+    {
+      FileSetMemberData member = it.next();
+      if (member.getDataFileType().getId() == typeId)
+      {
+        // Type is matching -- what about the file?
+        if (member.getFile().getId() == fileId)
+        {
+          // Keep
+          remain = dc.getItem(FileSetMember.class, member);
+        }
+        else
+        {
+          // Remove
+          it.remove();
+          if (member.getId() > 0)
+          {
+            FileSetMember m = dc.getItem(FileSetMember.class, member);
+            dc.deleteItem(m);
+            removed.add(new ValidatingFileSetMember(m));
+          }
+        }
+      }
+    }
+    if (removed.size() > 0)
+    {
+      // Create context and invoke the extensions system
+      ExtensionsInvoker<ValidationAction> invoker = getInvoker(dc);
+      
+      // Reset the metadata
+      invoker.render(new ResetMetadataRenderer(removed));
+    }
+    return remain;
+  }
+
   
   /**

trunk/src/core/net/sf/basedb/core/FileStoreUtil.java

@@ -219 +219 @@
     FileSet fileSet = item.getFileSet();
     DataFileType type = DataFileType.getByExternalId(dc, dataFileType);
-    FileSetMember member = null;
-    if (file == null)
-    {
-      // TODO #1604
-//      fileSet.removeMember(type);
-    }
-    else
-    {
-      member = fileSet.setMember(file, type);
-    }
+    FileSetMember member = fileSet.setMember(file, type);
     return member;
   }
@@ -276 +267 @@
     // Now, add/remove the file to the fileset
     FileSet fileSet = item.getFileSet();
-    FileSetMember member = null;
-    if (file == null)
-    {
-      // TODO #1604
-//      fileSet.removeMember(type);
-    }
-    else
-    {
-      member = fileSet.setMember(file, type);
-    }
+    FileSetMember member = fileSet.setMember(file, type);
     return member;
   }

trunk/src/core/net/sf/basedb/core/data/FileSetMemberData.java

@@ -43 +43 @@
     Get the file set this file belongs to.
     @hibernate.many-to-one outer-join="false" update="false"
-    @hibernate.column name="`fileset_id`" not-null="true"
+    @hibernate.column name="`fileset_id`" not-null="true" unique-key="uniquefile"
   */
   public FileSetData getFileSet()
@@ -58 +58 @@
     Get the type of this file. A set may only contain one file for each type.
     @hibernate.many-to-one outer-join="false" update="false"
-    @hibernate.column name="`datafiletype_id`" not-null="true"
+    @hibernate.column name="`datafiletype_id`" not-null="true" unique-key="uniquefile"
   */
   public DataFileTypeData getDataFileType()
@@ -73 +73 @@
     Get the file.
     @hibernate.many-to-one outer-join="false"
-    @hibernate.column name="`file_id`" not-null="true"
+    @hibernate.column name="`file_id`" not-null="true" unique-key="uniquefile"
   */
   public FileData getFile()

trunk/src/plugins/core/net/sf/basedb/plugins/ManualTransformCreator.java

@@ -192 +192 @@
         for (DataFileType ft : fileTypes)
         {
-          File file = (File)job.getValue("file." + ft.getExternalId());
-          if (file != null)
+          List<File> files = (List<File>)job.getValues("file." + ft.getExternalId());
+          if (files != null && files.size() > 0)
           {
-            fs.setMember(file, ft);
+            for (File f : files)
+            {
+              fs.addMember(f, ft);
+            }
           }
         }
@@ -274 +277 @@
         for (DataFileType ft : fileTypes)
         {
-          storeValue(job, request, ri.getParameter("file." + ft.getExternalId()));
+          storeValues(job, request, ri.getParameter("file." + ft.getExternalId()));
         }
       }
@@ -460 +463 @@
               ft.getName(),
               ft.getDescription(),
-              new FileParameterType()
+              new FileParameterType(null, false, 0)
           ));
         }
@@ -524 +527 @@
         "fileTypes",
         "File types",
-        "Select all registered file types that the external tool generates as" +
+        "Select all registered file types that the external tool generates as " +
         "output.",
         new ItemParameterType<DataFileType>(DataFileType.class, null, false, 0, fileTypes)

trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/AbstractItemImporter.java

@@ -28 +28 @@
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -48 +49 @@
 import net.sf.basedb.core.Extract;
 import net.sf.basedb.core.File;
+import net.sf.basedb.core.FileSet;
+import net.sf.basedb.core.FileSetMember;
+import net.sf.basedb.core.FileStoreEnabled;
 import net.sf.basedb.core.Hardware;
 import net.sf.basedb.core.ItemParameterType;
@@ -1915 +1919 @@
   }
   
+  private Set<String> fileCache;
+  /**
+    Set or add a file to a file set. The first time this method is called for
+    a given item and file type combination {@link FileSet#setMember(File, DataFileType)}
+    is used to add the file to the file set (which will replace all other files of
+    the same file type. If this method is called more times then 
+    {@link FileSet#addMember(File, DataFileType)} is used so that already added
+    files are preserved. 
+    
+    If the file is null the first time, all files of the given file types are removed.
+    If the file is null after the first time, this call is ignored.
+    
+    @param item The item to set/add a file to
+    @param fileType The type of the file
+    @param file The file, or null
+    @return The new member entry, or null if the call didn't result in
+      an addition
+    @since 3.0
+  */
+  protected FileSetMember setOrAddFile(FileStoreEnabled item, DataFileType fileType, File file)
+  {
+    if (fileCache == null) fileCache = new HashSet<String>();
+    String cacheId = item.getType().name() + "#" + item.getId() + "#" + fileType.getId();
+    FileSetMember added = null;
+    if (!fileCache.contains(cacheId))
+    {
+      // Has been called at least one time before
+      if (file != null) 
+      {
+        added = item.getFileSet().addMember(file, fileType);
+      }
+    }
+    else
+    {
+      added = item.getFileSet().setMember(file, fileType);
+      fileCache.add(cacheId);
+    }
+    return added;
+  }
+  
   /**
     Cache that holds loaded/created items. The cache key is item type + identifier,

trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/ArrayDesignImporter.java

@@ -313 +313 @@
           file = File.getByPath(dc, new Path(filePath, Path.Type.FILE), false);
         }
-        if (file != null)
-        {
-          design.getFileSet().setMember(file, fileType);
-        }
-        else
-        {
-          // TODO #1604
-          //design.getFileSet().removeMember(fileType);
-        }
+        setOrAddFile(design, fileType, file);
       }
     }

trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/DerivedBioAssayImporter.java

@@ -391 +391 @@
           file = File.getByPath(dc, new Path(filePath, Path.Type.FILE), false);
         }
-        if (file != null)
-        {
-          bioAssay.getFileSet().setMember(file, fileType);
-        }
-        else
-        {
-          // TODO #1604
-          //bioAssay.getFileSet().removeMember(fileType);
-        }
+        setOrAddFile(bioAssay, fileType, file);
       }
     }

trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/RawBioAssayImporter.java

@@ -461 +461 @@
           file = File.getByPath(dc, new Path(filePath, Path.Type.FILE), false);
         }
-        if (file != null)
-        {
-          rba.getFileSet().setMember(file, fileType);
-        }
-        else
-        {
-          // TODO #1604
-          //rba.getFileSet().removeMember(fileType);
-        }
+        setOrAddFile(rba, fileType, file);
       }
     }

trunk/src/plugins/core/net/sf/basedb/plugins/batchimport/ScanImporter.java

@@ -255 +255 @@
       File imageFile = File.getByPath(dc, new Path(filePath, Path.Type.FILE), false);
       DataFileType imageType = DataFileType.getByExternalId(dc, DataFileType.MICROARRAY_IMAGE);
-      if (imageFile != null)
-      {
-        scan.getFileSet().setMember(imageFile, imageType);
-      }
-      else
-      {
-        // TODO #1604
-        //scan.getFileSet().removeMember(imageType);
-      }
+      setOrAddFile(scan, imageType, imageFile);
     }
   }

trunk/src/test/data/test.batchimport.rawbioassays.txt

@@ -2 +2 @@
 Raw data 1  The first data set  1 Generic   genepix Scan 1  Feature Feature Generic.1 Generic raw data  test.gpr
 Raw data 2  The second  1 Generic Variant GenePix Scan 2  Feature Feature Generic.2 Generic raw data  test.gpr
+Raw data 2                    Generic raw data  test.cel
 Raw data 3.1  From same hyb 1 Affymetrix      Scan 3  Feature Feature Affy  CEL file  test.cel
 Raw data 3.2  From same hyb 2 Affymetrix      Scan 3  Feature Feature Affy  CEL file  test.cel