Changeset 3719


Ignore:
Timestamp:
Sep 12, 2007, 3:02:59 PM (14 years ago)
Author:
Nicklas Nordborg
Message:

References #411: Most of the basic functionality is now in place and has been documented
On major issue is that the Affymetrix Fusion SDK will not work with compressed files since
it expects direct access to the java.io.File object. I will try to solve this by modifying the
Affymetric Fusion SDK to work with InputStream:s instead. This would be better than to decompress
CEL and CDF files everytime they are needed. In most cases we only scan the headers anyway, so
it would be good if we can avoid copying the entire 40+ MB file(s) each time.

Location:
trunk
Files:
8 added
51 edited

Legend:

Unmodified
Added
Removed
  • trunk/config/dist/base.config

    r3698 r3719  
    151151helptext.update = true
    152152
     153# Set to 1 to disable auto-compression of uploaded files
     154autocompress.disable = 0
     155
     156
  • trunk/doc/src/docbook/userdoc/file_system.xml

    r3675 r3719  
    279279                  still possible to change other metadata, such as
    280280                  the name, description, file type, MIME type, etc.
     281                </para>
     282              </listitem>
     283            </varlistentry>
     284            <varlistentry>
     285              <term>
     286                <guilabel>Store compressed</guilabel>
     287              </term>
     288              <listitem>
     289                <para>
     290                  You can select if you want BASE to store your file in a
     291                  compressed format or in it's normal format. Compressing
     292                  the file may save a lot of disk space and it also uses
     293                  less quota. There are three options:
     294                  <itemizedlist>
     295                  <listitem>
     296                    <para>
     297                      <guilabel>auto</guilabel>: Let BASE automatically
     298                      decide if the file should be compressed or not. The file
     299                      is compressed if (1): it is uploaded to a directory that
     300                      has the <emphasis>compress files</emphasis> flag set
     301                      or (2): if the matching MIME type has the
     302                      <emphasis>compress files</emphasis> flag set.
     303                    </para>
     304                  </listitem>
     305                  <listitem>
     306                    <para>
     307                      <guilabel>yes</guilabel>: Store the file in a compressed
     308                      format.
     309                    </para>
     310                  </listitem>
     311                  <listitem>
     312                    <para>
     313                      <guilabel>no</guilabel>: Store the file in it's normal format.
     314                    </para>
     315                  </listitem>
     316                  </itemizedlist>
     317                 
    281318                </para>
    282319              </listitem>
     
    765802              <varlistentry>
    766803                <term>
     804                  <guilabel>Compress files</guilabel>
     805                </term>
     806                <listitem>
     807                  <para>
     808                    Enable this option to let BASE store files that
     809                    are uploaded to this directory in a compressed format.
     810                    This option only affect files that are uploaded later,
     811                    it doesn't affect already existing files or files that
     812                    are moved between directories.
     813                  </para>
     814                </listitem>
     815              </varlistentry>
     816              <varlistentry>
     817                <term>
    767818                  <guilabel>Description</guilabel>
    768819                </term>
  • trunk/src/core/common-queries.xml

    r3679 r3719  
    27152715  </query>
    27162716 
     2717  <query id="SET_AUTOCOMPRESS_ON_MIMETYPES" type="HQL">
     2718    <sql>
     2719      UPDATE MimeTypeData mt
     2720      SET mt.autoCompress = false
     2721      WHERE mt.autoCompress IS NULL
     2722    </sql>
     2723    <description>
     2724      A HQL query that sets the autoCompress property to false for all
     2725      MIME types with a null value.
     2726    </description>
     2727  </query>
     2728 
     2729  <query id="SET_AUTOCOMPRESS_ON_DIRECTORIES" type="HQL">
     2730    <sql>
     2731      UPDATE DirectoryData dir
     2732      SET dir.autoCompress = false
     2733      WHERE dir.autoCompress IS NULL
     2734    </sql>
     2735    <description>
     2736      A HQL query that sets the autoCompress property to false for all
     2737      directories with a null value.
     2738    </description>
     2739  </query>
     2740
     2741  <query id="SET_COMPRESSED_ON_FILES" type="HQL">
     2742    <sql>
     2743      UPDATE FileData f
     2744      SET
     2745        f.compressed = false,
     2746        f.compressedSize = f.size
     2747      WHERE f.compressed IS NULL
     2748    </sql>
     2749    <description>
     2750      A HQL query that sets the compressed property to false and the
     2751      compressedSize to size for all files with a null value.
     2752    </description>
     2753  </query>
    27172754
    27182755</predefined-queries>
  • trunk/src/core/net/sf/basedb/core/Application.java

    r3698 r3719  
    168168 
    169169  /**
     170    If auto-compression has been enabled or not.
     171  */
     172  private static boolean noAutoCompression;
     173 
     174  /**
    170175    Get the major version.
    171176  */
     
    406411      nonCorePluginDirectory = pluginDir != null ? new java.io.File(pluginDir) : null;
    407412      log.info("plugins.dir = " + nonCorePluginDirectory);
     413     
     414      noAutoCompression = Config.getBoolean("autocompress.disable");
     415      log.info("autocompress.disable = " + noAutoCompression);
    408416     
    409417      // Create a cache for SessionControl objects
     
    734742  }
    735743 
     744  /**
     745    Check if auto-compression has been enabled or not. Files that
     746    are checked with the {@link File#setMimeTypeAuto(String, FileType)}
     747    and will automaically be compressed if auto-compression is enabled and
     748    the {@link MimeType#getAutoCompress()} returns TRUE.
     749    @return TRUE if auto-compression is enabled
     750    @since 2.5
     751  */
     752  public static boolean autoCompressionEnabled()
     753  {
     754    return !noAutoCompression;
     755  }
    736756 
    737757  /**
  • trunk/src/core/net/sf/basedb/core/Config.java

    r3679 r3719  
    150150    return result;
    151151  }
     152 
     153  /**
     154    Get the configuration setting specified by <code>key</code> as a long value.
     155 
     156    @param key The key for the setting
     157    @param defaultValue The value to return if the <code>key</code> is not found
     158    @return The value specified by <code>key</code> or <code>defaultValue</code>
     159      if it is not found
     160  */
     161  public static long getLong(String key, long defaultValue)
     162  {
     163    assert isInitialised : "Config has not been initialised";
     164    long result = defaultValue;
     165    String setting = config.getProperty(key);
     166    if (setting != null)
     167    {
     168      try
     169      {
     170        result = Long.parseLong(setting);
     171      }
     172      catch (NumberFormatException ex)
     173      {}
     174    }
     175    return result;
     176  }
    152177
    153178  /**
  • trunk/src/core/net/sf/basedb/core/Directory.java

    r3679 r3719  
    9999    d.setName("New directory");
    100100    d.setParent(parent);
     101    d.setAutoCompress(parent.getAutoCompress());
    101102    return d;
    102103  }
     
    342343      copy.setName(child.getName());
    343344      copy.setDescription(child.getDescription());
     345      copy.setAutoCompress(child.getAutoCompress());
    344346      dc.saveItem(copy);
    345347      copySubdirectories(dc, child, copy, owner);
     
    622624 
    623625  /**
     626    If files that are uploaded to this directory should be
     627    compressed or not. This setting only affects files when they
     628    are uploaded. It doesn't affect files that are moved between
     629    directories.
     630    @return TRUE to automatically compress files uploaded to this directory
     631    @since 2.5
     632  */
     633  public boolean getAutoCompress()
     634  {
     635    return getData().getAutoCompress();
     636  }
     637 
     638  /**
     639    Set if files of should automatically
     640    be compressed when they are uploaded to this directory.
     641    @param autoCompress TRUE to enable auto-compression
     642    @throws PermissionDeniedException If the logged in user doesn't have
     643      write permission
     644    @since 2.5
     645  */
     646  public void setAutoCompress(boolean autoCompress)
     647    throws PermissionDeniedException
     648  {
     649    checkPermission(Permission.WRITE);
     650    getData().setAutoCompress(autoCompress);
     651  }
     652 
     653 
     654  /**
    624655    Returns TRUE if this directories parent equals null and therfore is a root directory.
    625656   
  • trunk/src/core/net/sf/basedb/core/File.java

    r3679 r3719  
    4949import java.util.Random;
    5050import java.util.Set;
     51import java.util.zip.GZIPInputStream;
     52import java.util.zip.GZIPOutputStream;
    5153import java.security.MessageDigest;
    5254
     
    9597    file.setLocation(Location.OFFLINE);
    9698    file.setName("New file");
     99    file.uploadCompressed = directory.getAutoCompress();
    97100    return file;
    98101  }
     
    293296 
    294297  /**
     298    If the next upload should use compression or not.
     299  */
     300  private boolean uploadCompressed;
     301 
     302  /**
    295303    Creates a new file item from the given data.
    296304   
     
    336344  public long getBytes()
    337345  {
    338     return getData().getSize();
     346    return getData().getCompressedSize();
    339347  }
    340348  /**
     
    614622
    615623  /**
    616     Get the size of this file.
    617     @return the size in bytes
     624    Get the (uncompressed) size of this file.
     625    @return The size in bytes
     626    @see #getCompressedSize()
    618627  */
    619628  public long getSize()
    620629  {
    621     // Due to bug #575, can be removed once an update script has been created in the trunk
    622     long size = getData().getSize();
    623     java.io.File file = newInternalFile != null ? newInternalFile : getAbsolutePath(getInternalName());
    624     if (file.exists())
    625     {
    626       size = file.length();
    627     }
    628     return size;
     630    return getData().getSize();
     631  }
     632 
     633  /**
     634    Get the compressed size of this file. If the file is not stored in a compressed
     635    format this method return the same value as {@link #getSize()}
     636    @return The compressed size in bytes
     637    @see #isCompressed()
     638    @since 2.5
     639  */
     640  public long getCompressedSize()
     641  {
     642    return getData().getCompressedSize();
     643  }
     644
     645  /**
     646    If this file is stored in a compressed format or not.
     647    @return TRUE if the file is compressed
     648    @since 2.5
     649  */
     650  public boolean isCompressed()
     651  {
     652    return getData().isCompressed();
    629653  }
    630654 
     
    758782    Set the mime type automatically by checking the extension of the
    759783    name. If no match is found, the mime type is set to the default value
    760     if specified, or left unchanged if the default is null.
     784    if specified, or left unchanged if the default is null. This method will
     785    also check the {@link MimeType#getAutoCompress()} flag and select
     786    auto-compression for the next upload if it is set. Auto-compression
     787    is also selected if the {@link Directory#getAutoCompress()} flag is
     788    set.
     789   
    761790    @param defaultMimeType The default value to use if no match is found for the file
    762791      extension
     
    765794    @throws PermissionDeniedException If the logged in user doesn't have
    766795      {@link Permission#WRITE} permission
     796    @see #uploadCompressed()
    767797  */
    768798  public void setMimeTypeAuto(String defaultMimeType, FileType defaultFileType)
     
    791821          setMimeType(mimeType.getName());
    792822          fileType = mimeType.getFileType();
     823          uploadCompressed |= mimeType.getAutoCompress() && Application.autoCompressionEnabled();
    793824        }
    794825        else if (defaultMimeType != null)
     
    811842 
    812843  /**
     844    Return the result of auto-compress check from the
     845    last call to {@linik #setMimeTypeAuto(String, FileType)}.
     846    If this method returns true, the next call to {@link #upload(InputStream, boolean)}
     847    or {@link #getUploadStream(boolean)} will store the uploaded file in a
     848    compressed format.
     849   
     850    @return TRUE if the next upload should be compressed
     851    @since 2.5
     852  */
     853  public boolean uploadCompressed()
     854  {
     855    return uploadCompressed;
     856  }
     857 
     858  /**
     859    Compress the file stored on the disk. If the file is already
     860    compressed or not located in the {@link Location#PRIMARY}
     861    location this method does nothing.
     862    @param progress An optional progress reporter
     863    @throws PermissionDeniedException If the logged in user doesn't have write
     864      permission on the file
     865    @since 2.5
     866  */
     867  public void compress(ProgressReporter progress)
     868  {
     869    checkPermission(Permission.WRITE);
     870    if (!isCompressed() && getLocation() == Location.PRIMARY)
     871    {
     872      compressOrDecompress(progress, true);
     873    }
     874  }
     875 
     876  /**
     877    Decompress the file stored on the disk. If the file isn't
     878    compressed or not located in the {@link Location#PRIMARY}
     879    location this method does nothing.
     880    @param progress An optional progress reporter
     881    @throws PermissionDeniedException If the logged in user doesn't have write
     882      permission on the file
     883    @since 2.5
     884  */
     885  public void decompress(ProgressReporter progress)
     886  {
     887    if (isCompressed() && getLocation() == Location.PRIMARY)
     888    {
     889      compressOrDecompress(progress, false);
     890    }
     891  }
     892 
     893  private void compressOrDecompress(ProgressReporter progress, boolean compress)
     894  {
     895    InputStream compressed = getDownloadStream(0);
     896    OutputStream uncompressed = getUploadStream(true, compress);
     897    AbsoluteProgressReporter aProgress = null;
     898    if (progress != null)
     899    {
     900      aProgress = new SimpleAbsoluteProgressReporter(progress, getSize());
     901      String msg = compress ? "Compressing " : "Decompressing ";
     902      aProgress.displayAbsolute(0, msg + getName());
     903    }
     904    try
     905    {
     906      FileUtil.copy(compressed, uncompressed, aProgress);
     907    }
     908    catch (IOException ex)
     909    {
     910      throw new BaseException(ex);
     911    }
     912    finally
     913    {
     914      try
     915      {
     916        uncompressed.close();
     917        compressed.close();
     918      }
     919      catch (IOException ex)
     920      {}
     921    }
     922  }
     923 
     924  /**
     925    Upload a physical file for this file item. This method calls
     926    {@link #upload(InputStream, boolean, Boolean)}. A null value is used
     927    for the compress parameter to enable auto-compression.
     928  */
     929  public void upload(InputStream in, boolean checkMd5)
     930    throws PermissionDeniedException, BaseException
     931  {
     932    upload(in, checkMd5, null);
     933  }
     934
     935  /**
    813936    Upload a physical file for this file item.
    814937    @param in An <code>InputStream</code> which will be read
     
    817940      be checked against the MD5 sum already stored in the database.
    818941      If no MD5 exists in the database this parameter is ignored
     942    @param compress TRUE to store the file in a compressed format, or null
     943      to use whatever the {@link Directory#getAutoCompress()} is specifying
     944      or what auto-detection from the {@link #setMimeTypeAuto(String, FileType)}
     945      found
    819946    @throws PermissionDeniedException If the logged in user doesn't have
    820947      {@link Permission#WRITE} permission for this item, or if the
    821948      file is write protected
    822949    @throws BaseException If there is another error
    823   */
    824   public void upload(InputStream in, boolean checkMd5)
     950    @since 2.5
     951  */
     952  public void upload(InputStream in, boolean checkMd5, Boolean compress)
    825953    throws PermissionDeniedException, BaseException
    826954  {
    827     OutputStream out = getUploadStream(checkMd5);
     955    OutputStream out = getUploadStream(checkMd5, compress);
    828956    try
    829957    {
     
    847975  }
    848976
     977  /**
     978    Get an <code>OutputStream</code> that can be used
     979    to upload a physical file for this file item. This method calls
     980    {@link #getUploadStream(boolean, Boolean)}. A null value is used
     981    for the compress parameter to enable auto-compression.
     982  */
     983  public OutputStream getUploadStream(boolean checkMd5)
     984  {
     985    return getUploadStream(checkMd5, null);
     986  }
     987 
    849988  /**
    850989    Get an <code>OutputStream</code> that can be used
    851990    to upload a physical file for this file item. The client
    852991    application should call <code>close()</code> method to
    853     finish the upload.
     992    finish the upload. Note! Calling this method multiple times
     993    without closing the returned stream will return the same stream
     994    and the checkMd5 and compress parameters are ignored.
     995   
    854996    @param checkMd5 If the MD5 sum of the uploaded data should
    855997      be checked against the MD5 sum already stored in the database.
    856998      If no MD5 exists in the database this parameter is ignored
     999    @param compress TRUE to store the file in a compressed format, or null
     1000      to use whatever the {@link Directory#getAutoCompress()} is specifying
     1001      or what auto-detection from the {@link #setMimeTypeAuto(String, FileType)}
     1002      found
    8571003    @return An <code>OutputStream</code> object
    8581004    @throws PermissionDeniedException If the logged in user doesn't have
     
    8601006      file is write protected
    8611007    @throws BaseException If there is another error
    862   */
    863   public OutputStream getUploadStream(boolean checkMd5)
     1008    @since 2.5
     1009  */
     1010  public OutputStream getUploadStream(boolean checkMd5, Boolean compress)
    8641011    throws PermissionDeniedException, BaseException
    8651012  {
     
    8741021    }
    8751022    if (uploadStream != null) return uploadStream;
     1023    if (compress == null) compress = uploadCompressed;
    8761024    if (newInternalFile == null)
    8771025    {
    878       newInternalFile = getNewFile();
     1026      newInternalFile = getNewFile(compress);
    8791027    }
    8801028    try
    8811029    {
    882       uploadStream = new UploadStream(newInternalFile, true, checkMd5);
     1030      uploadStream = new UploadStream(newInternalFile, true, checkMd5, compress);
    8831031    }
    8841032    catch (IOException ex)
     
    9491097    {
    9501098      InputStream fis = new BufferedInputStream(new FileInputStream(file));
     1099      if (isCompressed())
     1100      {
     1101        fis = new GZIPInputStream(fis);
     1102      }
    9511103      if (offset > 0)
    9521104      {
     
    9711123    @throws BaseException If there is an error.
    9721124  */
    973   private java.io.File getNewFile()
     1125  private java.io.File getNewFile(boolean compress)
    9741126    throws BaseException
    9751127  {
     
    9951147        name = sb.toString()+(getName() != null ? getName() : "");
    9961148        if (name.length() > 30) name = name.substring(0, 30);
     1149        if (compress) name += ".gz";
    9971150        f = new java.io.File(directory, name);
    9981151      } while (!f.createNewFile());
     
    11321285    */
    11331286    private MessageDigest md5 = null;
     1287   
     1288    /**
     1289      The file to write to.
     1290    */
     1291    private java.io.File file = null;
     1292   
     1293    /**
     1294      If the file should be compressed or not.
     1295    */
     1296    private boolean compress;
    11341297   
    11351298    /**
     
    11481311      @throws IOException If there is an error during the upload
    11491312    */
    1150     UploadStream(java.io.File file, boolean calculateMd5, boolean checkMd5)
     1313    private UploadStream(java.io.File file, boolean calculateMd5, boolean checkMd5, boolean compress)
    11511314      throws IOException
    11521315    {
    1153       super(new FileOutputStream(file));
     1316      super(compress ? new GZIPOutputStream(new FileOutputStream(file)) : new FileOutputStream(file));
     1317      this.file = file;
     1318      this.compress = compress;
    11541319      if (calculateMd5)
    11551320      {
     
    12071372      super.close();
    12081373      getData().setSize(totalBytes);
     1374      getData().setCompressedSize(file.length());
     1375      getData().setCompressed(compress);
    12091376      getData().setLocation(Location.PRIMARY.getValue());
    12101377      getData().setRemoved(false);
     
    12211388      }
    12221389      uploadStream = null;
     1390      uploadCompressed = false;
    12231391      closed = true;
    12241392    }
  • trunk/src/core/net/sf/basedb/core/Install.java

    r3679 r3719  
    104104    method.
    105105  */
    106   public static final int NEW_SCHEMA_VERSION = Integer.valueOf(40).intValue();
     106  public static final int NEW_SCHEMA_VERSION = Integer.valueOf(41).intValue();
    107107 
    108108  public static synchronized void createTables(boolean update, final ProgressReporter progress)
     
    449449      // Directory
    450450      if (progress != null) progress.display((int)(12*progress_factor), "--Creating directories...");
    451       DirectoryData rootDirectory = createDirectory(rootUser, null, Directory.ROOT, "root", "This is the root directory", keyEveryoneRead);
    452       DirectoryData homeDirectory = createDirectory(rootUser, rootDirectory, Directory.HOME, "home", "This is the directory where the user's home directories are located", keyEveryoneRead);
    453       DirectoryData templateDirectory = createDirectory(rootUser, rootDirectory, Directory.TEMPLATE, "template", "The substructure of this directory is used to create the directory structure for new home directories", null);
    454       createDirectory(rootUser, templateDirectory, null, "images", "Place your images here", null);
    455       createDirectory(rootUser, templateDirectory, null, "raw data", "Place your raw data files here", null);
    456       createDirectory(rootUser, templateDirectory, null, "projects", "Create a subdirectory for project-related files in this directory.", null);
     451      DirectoryData rootDirectory = createDirectory(rootUser, null, Directory.ROOT, "root", "This is the root directory", keyEveryoneRead, false);
     452      DirectoryData homeDirectory = createDirectory(rootUser, rootDirectory, Directory.HOME, "home", "This is the directory where the user's home directories are located", keyEveryoneRead, false);
     453      DirectoryData templateDirectory = createDirectory(rootUser, rootDirectory, Directory.TEMPLATE, "template", "The substructure of this directory is used to create the directory structure for new home directories", null, false);
     454      createDirectory(rootUser, templateDirectory, null, "images", "Place your images here", null, false);
     455      createDirectory(rootUser, templateDirectory, null, "raw data", "Place your raw data files here", null, true);
     456      createDirectory(rootUser, templateDirectory, null, "projects", "Create a subdirectory for project-related files in this directory.", null, false);
    457457 
    458458      // FileTypes
     
    470470      // MimeTypes
    471471      if (progress != null) progress.display((int)(14*progress_factor), "--Creating MIME types...");
    472       createMimeType("text/css", "Cascading style sheet", "css", null);
    473       createMimeType("text/plain", "Text files", "txt", null);
    474       createMimeType("text/html", "Hypertext markup language files", "htm", null);
    475       createMimeType("text/html", "Hypertext markup language files", "html", null);
    476       createMimeType("image/jpeg", "JPEG image", "jpg", imageType);
    477       createMimeType("image/jpeg", "JPEG image","jpeg", imageType);
    478       createMimeType("image/jpeg", "JPEG image", "jpe", imageType);
    479       createMimeType("image/gif", "GIF image", "gif", imageType);
    480       createMimeType("image/png", "PNG image", "png", imageType);
    481       createMimeType("image/tiff", "TIFF image", "tiff", imageType);
    482       createMimeType("image/tiff", "TIFF image", "tif", imageType);
    483       createMimeType("image/rgb", "RGB image", "rgb", imageType);
    484       createMimeType("image/x-pict", "Macintosh PICT format", "pict", imageType);
    485       createMimeType("image/x-ms-bmp", "Microsoft Windows bitmap", "bmp", imageType);
    486       createMimeType("application/postscript", "PostScript", "ai", null);
    487       createMimeType("application/postscript", "PostScript","eps", null);
    488       createMimeType("application/postscript", "PostScript", "ps", null);
    489       createMimeType("application/pdf", "Adobe Acrobat PDF", "pdf", null);
    490       createMimeType("application/x-latex", "LaTeX document","latex", null);
    491       createMimeType("application/msword", "MS word document", "doc", null);
    492       createMimeType("application/rtf", "Microsoft Rich Text Format", "rtf", null);
    493       createMimeType("application/octet-stream", "PC executable", "exe", null);
    494       createMimeType("application/x-perl", "Perl program", "pl", null);
    495       createMimeType("application/xml", "Extended markup language", "xml", null);
    496       createMimeType("application/xml-dtd", "Document type definition for XML", "dtd", null);
    497       createMimeType("application/vnd.ms-powerpoint", "PowerPoint (Microsoft)", "ppt", null);
    498       createMimeType("application/vnd.ms-powerpoint", "PowerPoint (Microsoft)", "ppz", null);
    499       createMimeType("application/vnd.ms-powerpoint", "PowerPoint (Microsoft)", "pps", null);
    500       createMimeType("application/vnd.ms-powerpoint", "PowerPoint (Microsoft)", "pot", null);
    501       createMimeType("application/vnd.ms-excel", "Excel spreadsheet (Microsoft)", "xls", null);
    502       createMimeType("application/vnd.ms-excel", "Excel spreadsheet (Microsoft)", "xlc", null);
    503       createMimeType("application/vnd.ms-excel", "Excel spreadsheet (Microsoft)", "xll", null);
    504       createMimeType("application/vnd.ms-excel", "Excel spreadsheet (Microsoft)", "xlm", null);
    505       createMimeType("application/vnd.ms-excel", "Excel spreadsheet (Microsoft)", "xlw", null);
    506       createMimeType("application/x-compress", "Unix compressed file", "z", null);
    507       createMimeType("application/zip", "DOS/PC - Pkzipped archive", "zip", null);
    508       createMimeType("application/x-compressed", "Gzip compressed archive", "tgz", null);
    509       createMimeType("application/x-gtar", "Gzipped archive", "gtar", null);
    510       createMimeType("application/x-gzip", "Gzip compressed file", "gz", null);
     472      createMimeType("text/css", "Cascading style sheet", "css", null, false);
     473      createMimeType("text/plain", "Text files", "txt", null, false);
     474      createMimeType("text/html", "Hypertext markup language files", "htm", null, false);
     475      createMimeType("text/html", "Hypertext markup language files", "html", null, false);
     476      createMimeType("image/jpeg", "JPEG image", "jpg", imageType, false);
     477      createMimeType("image/jpeg", "JPEG image","jpeg", imageType, false);
     478      createMimeType("image/jpeg", "JPEG image", "jpe", imageType, false);
     479      createMimeType("image/gif", "GIF image", "gif", imageType, false);
     480      createMimeType("image/png", "PNG image", "png", imageType, false);
     481      createMimeType("image/tiff", "TIFF image", "tiff", imageType, true);
     482      createMimeType("image/tiff", "TIFF image", "tif", imageType, true);
     483      createMimeType("image/rgb", "RGB image", "rgb", imageType, true);
     484      createMimeType("image/x-pict", "Macintosh PICT format", "pict", imageType, false);
     485      createMimeType("image/x-ms-bmp", "Microsoft Windows bitmap", "bmp", imageType, true);
     486      createMimeType("application/postscript", "PostScript", "ai", null, false);
     487      createMimeType("application/postscript", "PostScript","eps", null, false);
     488      createMimeType("application/postscript", "PostScript", "ps", null, false);
     489      createMimeType("application/pdf", "Adobe Acrobat PDF", "pdf", null, false);
     490      createMimeType("application/x-latex", "LaTeX document","latex", null, false);
     491      createMimeType("application/msword", "MS word document", "doc", null, true);
     492      createMimeType("application/rtf", "Microsoft Rich Text Format", "rtf", null, true);
     493      createMimeType("application/octet-stream", "PC executable", "exe", null, false);
     494      createMimeType("application/x-perl", "Perl program", "pl", null, false);
     495      createMimeType("application/xml", "Extended markup language", "xml", null, false);
     496      createMimeType("application/xml-dtd", "Document type definition for XML", "dtd", null, false);
     497      createMimeType("application/vnd.ms-powerpoint", "PowerPoint (Microsoft)", "ppt", null, true);
     498      createMimeType("application/vnd.ms-powerpoint", "PowerPoint (Microsoft)", "ppz", null, true);
     499      createMimeType("application/vnd.ms-powerpoint", "PowerPoint (Microsoft)", "pps", null, true);
     500      createMimeType("application/vnd.ms-powerpoint", "PowerPoint (Microsoft)", "pot", null, true);
     501      createMimeType("application/vnd.ms-excel", "Excel spreadsheet (Microsoft)", "xls", null, true);
     502      createMimeType("application/vnd.ms-excel", "Excel spreadsheet (Microsoft)", "xlc", null, true);
     503      createMimeType("application/vnd.ms-excel", "Excel spreadsheet (Microsoft)", "xll", null, true);
     504      createMimeType("application/vnd.ms-excel", "Excel spreadsheet (Microsoft)", "xlm", null, true);
     505      createMimeType("application/vnd.ms-excel", "Excel spreadsheet (Microsoft)", "xlw", null, true);
     506      createMimeType("application/x-compress", "Unix compressed file", "z", null, false);
     507      createMimeType("application/zip", "DOS/PC - Pkzipped archive", "zip", null, false);
     508      createMimeType("application/x-compressed", "Gzip compressed archive", "tgz", null, false);
     509      createMimeType("application/x-gtar", "Gzipped archive", "gtar", null, false);
     510      createMimeType("application/x-gzip", "Gzip compressed file", "gz", null, false);
    511511 
    512512      // Microarray/BASE-specific mime types
    513       createMimeType("text/plain", "GenePix Results file", "gpr", rawDataType);
    514       createMimeType("text/plain", "GenePix Array List file", "gal", reporterMapType);
    515       createMimeType("text/plain", "Spot file format", "spot", rawDataType);
    516       createMimeType("text/plain", "Spotfinder file format", "tav", rawDataType);
    517       createMimeType("text/plain", "Affymetrix CEL file", "cel", rawDataType);
    518       createMimeType("text/plain", "Affymetrix CDF file", "cdf", reporterMapType);
    519       createMimeType("text/plain", "FASTA formatted file", "fasta", null);
    520       createMimeType("text/plain", "BASEfile formatted file", "base", null);
    521       createMimeType("text/plain", "Biorobotics TAM format", "tam", printMapType);
    522       createMimeType("text/plain", "Molecularware MWBR format", "mvbr", printMapType);
     513      createMimeType("text/plain", "GenePix Results file", "gpr", rawDataType, true);
     514      createMimeType("text/plain", "GenePix Array List file", "gal", reporterMapType, true);
     515      createMimeType("text/plain", "Spot file format", "spot", rawDataType, true);
     516      createMimeType("text/plain", "Spotfinder file format", "tav", rawDataType, true);
     517      createMimeType("text/plain", "Affymetrix CEL file", "cel", rawDataType, true);
     518      createMimeType("text/plain", "Affymetrix CDF file", "cdf", reporterMapType, true);
     519      createMimeType("text/plain", "FASTA formatted file", "fasta", null, true);
     520      createMimeType("text/plain", "BASEfile formatted file", "base", null, true);
     521      createMimeType("text/plain", "Biorobotics TAM format", "tam", printMapType, true);
     522      createMimeType("text/plain", "Molecularware MWBR format", "mvbr", printMapType, true);
    523523 
    524524      // Plate geometries
     
    724724    Create a {@link Directory}.
    725725  */
    726   private static DirectoryData createDirectory(UserData owner, DirectoryData parent, String systemId, String name, String description, ItemKeyData itemKey)
     726  private static DirectoryData createDirectory(UserData owner, DirectoryData parent, String systemId, String name, String description, ItemKeyData itemKey, boolean autoCompress)
    727727    throws BaseException
    728728  {
     
    747747      if (directory != null)
    748748      {
     749        if (schemaVersion.getSchemaVersion() < 41) directory.setAutoCompress(autoCompress);
    749750        log.info("createDirectory: EXISTS [parent="+parent+"; name="+name+"]");
    750751        HibernateUtil.commit(tx);
     
    759760        directory.setOwner(owner);
    760761        directory.setItemKey(itemKey);
     762        directory.setAutoCompress(autoCompress);
    761763        HibernateUtil.saveData(session, directory);
    762764        HibernateUtil.commit(tx);
     
    777779    Create a {@link MimeType}.
    778780  */
    779   private static MimeTypeData createMimeType(String name, String description, String extension, FileTypeData fileType)
     781  private static MimeTypeData createMimeType(String name, String description, String extension, FileTypeData fileType, boolean autoCompress)
    780782    throws BaseException
    781783  {
     
    795797      {
    796798        if (schemaVersion.getSchemaVersion() < 17) mimeType.setFileType(fileType);
     799        if (schemaVersion.getSchemaVersion() < 41) mimeType.setAutoCompress(autoCompress);
    797800        log.info("createMimeType: EXISTS [extension="+extension+"; name="+name+"]");
    798801        HibernateUtil.commit(tx);
     
    805808        mimeType.setExtension(extension);
    806809        mimeType.setFileType(fileType);
     810        mimeType.setAutoCompress(autoCompress);
    807811        HibernateUtil.saveData(session, mimeType);
    808812        HibernateUtil.commit(tx);
  • trunk/src/core/net/sf/basedb/core/Job.java

    r3679 r3719  
    12231223          JobData data = job.getData();
    12241224          data.setPercentComplete(percent);
    1225           data.setStatusMessage(message);
     1225          if (message != null) data.setStatusMessage(message);
    12261226          dc.commit();
    12271227        }
  • trunk/src/core/net/sf/basedb/core/MimeType.java

    r3679 r3719  
    272272  }
    273273 
     274  /**
     275    If files of this MIME type should automatically be compressed
     276    when they are uploaded to BASE. Note! Auto-compression must be
     277    enabled. See {@link Application#autoCompressionEnabled()}.
     278    @return TRUE to automatically compress files of this type
     279    @since 2.5
     280  */
     281  public boolean getAutoCompress()
     282  {
     283    return getData().getAutoCompress();
     284  }
     285 
     286  /**
     287    Set if files of this MIME type should automatically
     288    be compressed when they are uploaded to BASE. Note! Auto-compression must be
     289    enabled. See {@link Application#autoCompressionEnabled()}.
     290    @param autoCompress TRUE to enable auto-compression
     291    @throws PermissionDeniedException If the logged in user doesn't have
     292      write permission
     293    @since 2.5
     294  */
     295  public void setAutoCompress(boolean autoCompress)
     296    throws PermissionDeniedException
     297  {
     298    checkPermission(Permission.WRITE);
     299    getData().setAutoCompress(autoCompress);
     300  }
     301
    274302}
  • trunk/src/core/net/sf/basedb/core/SimpleProgressReporter.java

    r3675 r3719  
    6161  {
    6262    this.percent = percent;
    63     this.message = message;
     63    if (message != null) this.message = message;
    6464    if (progress != null) progress.display(percent, message);
    6565  }
  • trunk/src/core/net/sf/basedb/core/Update.java

    r3679 r3719  
    472472      Change invalid property filter on 'guiContexts.itemType' to
    473473      '$ctx.itemType'.
     474    </td>
     475  </tr>
     476 
     477  <tr>
     478    <td>41</td>
     479    <td>
     480      <ul>
     481      <li>Added {@link net.sf.basedb.core.data.MimeTypeData#getAutoCompress()}
     482      <li>Added {@link net.sf.basedb.core.data.FileData#isCompressed()}.
     483      <li>Added {@link net.sf.basedb.core.data.DirectoryData#getAutoCompress()}.
     484      </ul>
     485      The update sets all values on existing items to false.
    474486    </td>
    475487  </tr>
     
    677689      }
    678690     
    679       /*
    680691      if (schemaVersion < 41)
    681692      {
    682693        if (progress != null) progress.display((int)(40*progress_factor), "--Updating schema version: " + schemaVersion + " -> 41...");
    683694        schemaVersion = setSchemaVersionInTransaction(session, 41);
     695      }
     696     
     697      /*
     698      if (schemaVersion < 42)
     699      {
     700        if (progress != null) progress.display((int)(41*progress_factor), "--Updating schema version: " + schemaVersion + " -> 42...");
     701        schemaVersion = setSchemaVersionInTransaction(session, 42);
    684702        - or -
    685         schemaVersion = updateToSchemaVersion41(session);
     703        schemaVersion = updateToSchemaVersion42(session);
    686704      }
    687705      ... etc...
     
    18951913        }
    18961914       
     1915        if (schemaVersion < 41)
     1916        {
     1917          // Set autoCompress to false for all MIME types with a null value
     1918          org.hibernate.Query query = HibernateUtil.getPredefinedQuery(session,
     1919            "SET_AUTOCOMPRESS_ON_MIMETYPES");
     1920          /*
     1921            UPDATE MimeTypeData mt
     1922            SET mt.autoCompress = false
     1923            WHERE mt.autoCompress IS NULL
     1924          */
     1925          HibernateUtil.executeUpdate(query);
     1926         
     1927          // Set autoCompress to false for all directories types with a null value
     1928          query = HibernateUtil.getPredefinedQuery(session,
     1929            "SET_AUTOCOMPRESS_ON_DIRECTORIES");
     1930          /*
     1931            UPDATE DirectoryData dir
     1932            SET dir.autoCompress = false
     1933            WHERE dir.autoCompress IS NULL
     1934          */
     1935          HibernateUtil.executeUpdate(query);
     1936         
     1937          // Set compressed to false and compressedSize to size
     1938          // for all files with a null value
     1939          query = HibernateUtil.getPredefinedQuery(session,
     1940            "SET_COMPRESSED_ON_FILES");
     1941          /*
     1942            UPDATE FileData f
     1943            SET f.compressed = false,
     1944            SET f.compressedSize = f.size
     1945            WHERE f.compressed IS NULL
     1946          */
     1947          HibernateUtil.executeUpdate(query);
     1948        }
     1949       
     1950       
    18971951        //  Commit the changes
    18981952        HibernateUtil.commit(tx);
  • trunk/src/core/net/sf/basedb/core/data/DirectoryData.java

    r3679 r3719  
    3636  @see <a href="../../../../../../development/overview/data/file.html">File and directory overview</a>
    3737  @base.modified $Date$
    38   @hibernate.class table="`Directories`" lazy="false"
     38  @hibernate.class table="`Directories`" lazy="true"
    3939 */
    4040public class DirectoryData
     
    8888  }
    8989 
     90  private boolean autoCompress;
     91  /**
     92    If files that are uploaded to this directory be compressed or not.
     93    @since 2.5
     94    @hibernate.property column="`auto_compress`" type="boolean" not-null="true"
     95  */
     96  public boolean getAutoCompress()
     97  {
     98    return autoCompress;
     99  }
     100  public void setAutoCompress(boolean autoCompress)
     101  {
     102    this.autoCompress = autoCompress;
     103  }
     104 
    90105  private Set<FileData> files;
    91106  /**
  • trunk/src/core/net/sf/basedb/core/data/FileData.java

    r3679 r3719  
    206206  }
    207207 
     208  private boolean compressed;
     209  /**
     210    If the file is stored in a compressed format or not.
     211    @since 2.5
     212    @hibernate.property column="`compressed`" type="boolean" not-null="true"
     213  */
     214  public boolean isCompressed()
     215  {
     216    return compressed;
     217  }
     218  public void setCompressed(boolean compressed)
     219  {
     220    this.compressed = compressed;
     221  }
     222 
     223  private long compressedSize;
     224  /**
     225    Get the size in bytes that this <code>FileData</code> object
     226    uses after compression. If the file is not compressed, this value
     227    is the same as {@link #getSize()}
     228    @hibernate.property column="`compressed_size`" type="long" not-null="true"
     229  */
     230  public long getCompressedSize()
     231  {
     232    return compressedSize;
     233  }
     234  public void setCompressedSize(long compressedSize)
     235  {
     236    this.compressedSize = compressedSize;
     237  }
     238 
    208239  private String internalName;
    209240  /**
  • trunk/src/core/net/sf/basedb/core/data/MimeTypeData.java

    r3679 r3719  
    111111  }
    112112
     113  private boolean autoCompress;
     114  /**
     115    If files of this MIME type should use auto-compression or not.
     116    @since 2.5
     117    @hibernate.property column="`auto_compress`" type="boolean" not-null="true"
     118  */
     119  public boolean getAutoCompress()
     120  {
     121    return autoCompress;
     122  }
     123  public void setAutoCompress(boolean autoCompress)
     124  {
     125    this.autoCompress = autoCompress;
     126  }
    113127 
    114128}
  • trunk/src/core/net/sf/basedb/util/ConsoleProgressReporter.java

    r3679 r3719  
    5959  {
    6060    if (needNewLine) System.out.print(useNewLine ? "\n" : "\r");
    61     System.out.print("["+percent + "%]\t" + message);
     61    System.out.print("["+percent + "%]\t" + message == null ? "" : message);
    6262    needNewLine = true;
    6363  }
  • trunk/src/core/net/sf/basedb/util/FileUtil.java

    r3700 r3719  
    2525package net.sf.basedb.util;
    2626
     27import net.sf.basedb.core.AbsoluteProgressReporter;
    2728import net.sf.basedb.core.BaseException;
    2829import net.sf.basedb.core.DbControl;
     
    7677    throws IOException
    7778  {
     79    return copy(in, out, null);
     80  }
     81
     82  /**
     83    Copy from the input stream to the output stream
     84    until end of file is reached.
     85    @param in The <code>InputStream</code> to read from
     86    @param out The <code>OutputStream</code> to write to
     87    @param progress An optional progress reporter
     88    @return The number of bytes copied
     89    @throws IOException This exception is thrown if there is an error
     90    @since 2.5
     91  */
     92  public static long copy(InputStream in, OutputStream out, AbsoluteProgressReporter progress)
     93    throws IOException
     94  {
    7895    int bytes = 0;
    7996    long totalBytes = 0;
    8097    byte[] buffer = new byte[BUFFER_SIZE];
    81 
     98 
    8299    while (bytes != -1) // -1 = end of stream
    83100    {
     
    87104        totalBytes += bytes;
    88105        out.write(buffer, 0, bytes);
     106        if (progress != null) progress.displayAbsolute(totalBytes, null);
    89107      }
    90108    }
    91109    return totalBytes;
    92110  }
     111
    93112 
    94113  /**
  • trunk/src/test/TestAnnotationFlatFileImporter.java

    r3675 r3719  
    7070    int sampleDup2 = TestSample.test_create(0, "Sample dup", false);
    7171   
    72     int fileId = TestFile.test_create("data/test.annotation.import.txt", false);
     72    int fileId = TestFile.test_create("data/test.annotation.import.txt", false, false);
    7373   
    7474    // Create job
  • trunk/src/test/TestAnyToAny.java

    r3675 r3719  
    4949    // Standard tests: create, load, list
    5050    int sampleId = TestSample.test_create(0, null, true);
    51     int fileId1 = TestFile.test_create("Linked file #1", true);
    52     int fileId2 = TestFile.test_create("Linked file #2", true);
     51    int fileId1 = TestFile.test_create("Linked file #1", false, true);
     52    int fileId2 = TestFile.test_create("Linked file #2", false, true);
    5353    int id = test_create(sampleId, fileId1, "first");
    5454    int id2 = test_create(sampleId, fileId2, "second");
  • trunk/src/test/TestBase1PluginExecuter.java

    r3675 r3719  
    6262        new String[] { "raw('ch1FgMedian')", "raw('ch2FgMedian')" }, false);
    6363   
    64     int fileId = TestFile.test_create("data/test.plugin.base", false);
     64    int fileId = TestFile.test_create("data/test.plugin.base", false, false);
    6565    int directoryId = TestDirectory.test_create(true, "Base1PluginExecuter");
    6666   
  • trunk/src/test/TestDirectory.java

    r3679 r3719  
    9191        d.setName(directoryName);
    9292        d.setDescription("Added at "+new Date());
     93        d.setAutoCompress(true);
    9394      }
    9495      dc.saveItem(d);
     
    304305    if (!TestUtil.getSilent())
    305306    {
    306       write("   \tID \tName      \tDescription\tParent\tPath");
    307       write("-- \t-- \t--------- \t-----------\t------\t----");
     307      write("   \tID \tName      \tDescription\tAuto compress\tParent\tPath");
     308      write("-- \t-- \t--------- \t-----------\t-------------\t------\t----");
    308309    }
    309310  }
     
    311312    throws BaseException
    312313  {
    313     if (!TestUtil.getSilent()) System.out.println(i+":\t"+d.getId()+"\t"+d.getName()+"\t"+d.getDescription()+"\t"+d.getParent()+"\t"+d.getPath());
     314    if (!TestUtil.getSilent()) System.out.println(i+":\t"+d.getId()+"\t"+d.getName()+"\t"+
     315      d.getDescription()+"\t"+d.getAutoCompress()+"\t"+d.getParent()+"\t"+d.getPath());
    314316  }
    315317 
  • trunk/src/test/TestDirty.java

    r3675 r3719  
    7878   
    7979    int rawBioAssayId = TestRawBioAssay.test_create("genepix", 0, 0, 0, arrayDesignId, false);
    80     int fileId = TestFile.test_create("data/test.import.dirty.txt", false);
     80    int fileId = TestFile.test_create("data/test.import.dirty.txt", false, false);
    8181   
    8282    int pluginDefinitionId = TestPluginDefinition.test_get("net.sf.basedb.plugins.RawDataFlatFileImporter");
  • trunk/src/test/TestDiskUsage.java

    r3679 r3719  
    4848
    4949    // Standard tests: only list is meaningful
    50     int file_id = TestFile.test_create("test.upload.txt", false);
     50    int file_id = TestFile.test_create("test.upload.txt", false, false);
    5151    test_list();
    5252   
  • trunk/src/test/TestFile.java

    r3679 r3719  
    6363    write_header();
    6464    // Standard tests: create, load, list
    65     int id1 = test_create("data/test.upload.txt", true);
    66     int id2 = test_create("data/test.upload.txt", false);
     65    int id1 = test_create("data/test.upload.txt", true, true);
     66    int id2 = test_create("data/test.upload.txt", false, false);
    6767    test_load(id1);
    6868    test_list();
    6969   
    7070    // Extra test: update
    71     test_update("data/test.upload.png", id1);
     71    test_update("data/test.upload.png", id2);
    7272
    7373    // Extra test: unpack zip file
     
    8585  }
    8686 
    87   static int test_create(String filename, boolean setAll)
     87  static int test_create(String filename, boolean compress, boolean setAll)
    8888  {
    8989    if (!TestUtil.hasPermission(Permission.CREATE, Item.FILE)) return 0;
     
    106106      {
    107107        in = new FileInputStream(fileToUpload);
    108         file.upload(in, true);
     108        file.upload(in, true, compress);
    109109        if (file.getSize() != fileToUpload.length())
    110110        {
     
    232232    if (!TestUtil.getSilent())
    233233    {
    234       write("   \tID \tName      \tDescription\tFiletype\tUsed\tLocation\tAction\tSize    \tMime type\tPath");
    235       write("-- \t-- \t--------- \t-----------\t--------\t----\t--------\t------\t--------\t---------\t----");
     234      write("   \tID \tName      \tDescription\tFiletype\tUsed\tLocation\tAction\tSize    \tCompressed size\tMime type\tPath");
     235      write("-- \t-- \t--------- \t-----------\t--------\t----\t--------\t------\t--------\t---------------\t---------\t----");
    236236    }
    237237  }
     
    241241    if (!TestUtil.getSilent()) System.out.println(i+":\t"+fi.getId()+"\t"+fi.getName()+"\t"+fi.getDescription()+"\t"+
    242242      fi.getFileType()+"\t"+fi.isUsed()+"\t"+fi.getLocation()+"\t"+fi.getAction()+"\t"+
    243       fi.getSize()+"\t"+fi.getMimeType()+"\t"+fi.getPath());
     243      fi.getSize()+"\t" + fi.getCompressedSize() + "\t"+fi.getMimeType()+"\t"+fi.getPath());
    244244  }
    245245  static void write(String message)
  • trunk/src/test/TestIlluminaImporter.java

    r3675 r3719  
    4444    // Test import rawdata without lims connection
    4545    TestReporter.test_import_from_file("data/test.illumina.reporters.txt", "Search_key,Target,ProbeId.*", ",(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))", 1, 1);
    46     int fileId = TestFile.test_create("data/test.illumina.rawdata.txt", false);
     46    int fileId = TestFile.test_create("data/test.illumina.rawdata.txt", false, false);
    4747    int hybId = TestHybridization.test_create(0, false, new int[] {});
    4848    int scanId = TestScan.test_create(hybId, 0, 0, false);
  • trunk/src/test/TestMimeType.java

    r3679 r3719  
    7777        mt.setDescription("Added at: "+new Date());
    7878        mt.setExtension("x-test");
     79        mt.setAutoCompress(true);
    7980      }
    8081      dc.saveItem(mt);
     
    173174    if (!TestUtil.getSilent())
    174175    {
    175       write("   \tID \tName      \tDescription\tExtension");
    176       write("-- \t-- \t--------- \t-----------\t----------");
     176      write("   \tID \tName      \tDescription\tExtension\tAuto-compress");
     177      write("-- \t-- \t--------- \t-----------\t---------\t-------------");
    177178    }
    178179  }
     
    183184  static void write_item(int i, MimeType mt)
    184185  {
    185     if (!TestUtil.getSilent()) System.out.println(i+":\t"+mt.getId()+"\t"+mt.getName()+"\t"+mt.getDescription()+"\t"+mt.getExtension());
     186    if (!TestUtil.getSilent())
     187    {
     188      System.out.println(i+":\t"+mt.getId()+"\t"+mt.getName()+"\t"+mt.getDescription()+"\t"+
     189        mt.getExtension() + "\t" + mt.getAutoCompress());
     190    }
    186191  }
    187192}
  • trunk/src/test/TestPlateFlatFileImporter.java

    r3679 r3719  
    5959    // Create upload file and platetype
    6060    TestReporter.test_import_from_file("data/test.plate96.import.txt", "Plate\\tRow\\tColumn\\tCluster ID\\tClone ID\\tGene Symbol\\tBarcode", "\\t", 4, 5);
    61     int fileId = TestFile.test_create("data/test.plate96.import.txt", false);
     61    int fileId = TestFile.test_create("data/test.plate96.import.txt", false, false);
    6262    int plateGeometry = TestPlateGeometry.test_create(8, 12, false);
    6363    int plateTypeId = TestPlateType.test_create(plateGeometry, false);
  • trunk/src/test/TestPrintMapFlatFileImporter.java

    r3679 r3719  
    7373
    7474    // Print map import
    75     int fileId = TestFile.test_create("data/test.parse.printmap.tam", false);
     75    int fileId = TestFile.test_create("data/test.parse.printmap.tam", false, false);
    7676    int printMapPluginDefinitionId  = TestPluginDefinition.test_get("net.sf.basedb.plugins.PrintMapFlatFileImporter");
    7777    int jobId = test_create_job_printmap(printMapPluginDefinitionId, fileId, arrayDesignId);
  • trunk/src/test/TestProtocol.java

    r3679 r3719  
    5757
    5858    // Extra tests: attach file to protocol
    59     int fileId = TestFile.test_create("test.upload.txt", false);
     59    int fileId = TestFile.test_create("test.upload.txt", false, false);
    6060    test_attachfile(id1, fileId);
    6161   
  • trunk/src/test/TestRawDataFlatFileImporter.java

    r3679 r3719  
    4444    // Test import rawdata without lims connection
    4545    TestReporter.test_import_from_file("data/test.rawdata.import.txt", "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", "\\t", 4, 3);
    46     int fileId = TestFile.test_create("data/test.rawdata.import.txt", false);
     46    int fileId = TestFile.test_create("data/test.rawdata.import.txt", false, false);
    4747    int arrayDesignId = TestArrayDesign.test_create(false, false);
    4848    TestArrayDesign.test_import_from_file(arrayDesignId, "data/test.rawdata.import.txt", "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", 0, 2, 1, 4);
  • trunk/src/test/TestReporterFlatFileImporter.java

    r3679 r3719  
    5959    // Create reporter type and upload file
    6060    int reporterTypeId = TestReporterType.test_create();
    61     int fileId = TestFile.test_create("data/test.reporter.import.txt", false);
     61    int fileId = TestFile.test_create("data/test.reporter.import.txt", false, false);
    6262   
    6363    // Create plugin configuration and job
  • trunk/src/test/TestReporterMapFlatFileImporter.java

    r3679 r3719  
    6262    int reporterTypeId = TestReporterType.test_create();
    6363    int arrayDesignId = TestArrayDesign.test_create(false, false);
    64     int fileId = TestFile.test_create("data/test.reportermap.import.txt", false);
     64    int fileId = TestFile.test_create("data/test.reportermap.import.txt", false, false);
    6565   
    6666    // Create reporter plugin configuration and job
  • trunk/src/test/TestSpotImages.java

    r3679 r3719  
    6565    TestRawBioAssay.test_import_from_file(rawBioAssayId, "data/test.rawdata.import.txt");
    6666   
    67     int imageFileId = TestFile.test_create("data/spots.tif", true);
    68     int spotImagesFileId = TestFile.test_create("test"+System.currentTimeMillis()+".zip", true);
     67    int imageFileId = TestFile.test_create("data/spots.tif", false, true);
     68    int spotImagesFileId = TestFile.test_create("test"+System.currentTimeMillis()+".zip", false, true);
    6969    test_create_spotimages(rawBioAssayId, imageFileId, spotImagesFileId);
    7070
  • trunk/www/admin/mimetypes/edit_mimetype.jsp

    r3679 r3719  
    2828--%>
    2929<%@ page session="false"
     30  import="net.sf.basedb.core.Application"
    3031  import="net.sf.basedb.core.SessionControl"
    3132  import="net.sf.basedb.core.DbControl"
     
    6061  boolean readCurrentFileType = true;
    6162  int currentFileTypeId = 0;
     63  boolean autoCompress = false;
    6264
    6365  if (itemId == 0)
     
    7072      currentFileTypeId = Values.getInt(cc.getRecent(Item.FILETYPE.name(), 0), 0);
    7173    }
     74    autoCompress = Values.getBoolean(cc.getPropertyValue("autoCompress"), autoCompress);
    7275  }
    7376  else
     
    7679    cc.setObject("item", mimeType);
    7780    title = "Edit MIME type -- " + HTML.encodeTags(mimeType.getName() + " (" + mimeType.getExtension() + ")");
     81    autoCompress = mimeType.getAutoCompress();
    7882    try
    7983    {
     
    150154
    151155    <h3 class="docked"><%=title%> <base:help tabcontrol="settings" /></h3>
    152     <t:tabcontrol id="settings" contentstyle="<%="height: "+(int)(scale*200)+"px;"%>"
     156    <t:tabcontrol id="settings" contentstyle="<%="height: "+(int)(scale*240)+"px;"%>"
    153157      position="bottom" remember="<%=mimeType != null%>">
    154158    <t:tab id="info" title="MIME type" validate="validateMimeType()" helpid="mimetype.edit">
     
    198202        </td>
    199203      </tr>
     204      <tr>
     205        <td class="prompt">Auto compress</td>
     206        <td>
     207          <input type="radio" name="autoCompress" value="0" <%=!autoCompress ? "checked" : ""%>>no
     208          <input type="radio" name="autoCompress" value="1" <%=autoCompress ? "checked" : ""%>>yes
     209         
     210          <%
     211          if (Application.autoCompressionEnabled())
     212          {
     213            %>
     214            <base:note type="info" style="background: #ffffcc;">
     215              Auto-compression has been disabled in <code>base.config</code>
     216            </base:note>
     217            <%
     218          }
     219          %>
     220        </td>
     221      </tr>
    200222      <tr valign=top>
    201223        <td class="prompt">Description</td>
  • trunk/www/admin/mimetypes/index.jsp

    r3679 r3719  
    142142    mimeType.setName(Values.getStringOrNull(request.getParameter("name")));
    143143    mimeType.setDescription(Values.getStringOrNull(request.getParameter("description")));
     144    mimeType.setAutoCompress(Values.getBoolean(request.getParameter("autoCompress")));
    144145    int fileTypeId = Values.getInt(request.getParameter("filetype_id"));
    145146    if (fileTypeId >= 0) // < 0 = denied or unchanged
  • trunk/www/admin/mimetypes/list_mimetypes.jsp

    r3679 r3719  
    221221        enumeration="<%=fileTypes%>"
    222222        title="File type"
     223        sortable="true"
     224        filterable="true"
     225        exportable="true"
     226      />
     227      <tbl:columndef
     228        id="autoCompress"
     229        property="autoCompress"
     230        datatype="boolean"
     231        title="Auto compress"
    223232        sortable="true"
    224233        filterable="true"
     
    375384                  onclick="itemOnClick(<%=writePermission ? "event" : null%>, <%=itemId%>)"
    376385                  title="<%=tooltip%>"><%=HTML.encodeTags(item.getExtension())%></div></tbl:cell>
    377                 <tbl:cell column="name"><%=name%></a></tbl:cell>
     386                <tbl:cell column="name"><%=name%></tbl:cell>
    378387                <tbl:cell column="fileType"
    379388                  ><base:propertyvalue
     
    383392                    enablePropertyLink="<%=mode.hasPropertyLink()%>"
    384393                  /></tbl:cell>
     394                <tbl:cell column="autoCompress"><%=item.getAutoCompress() ? "yes" : "no"%></a></tbl:cell>
    385395                <tbl:cell column="description"><%=HTML.encodeTags(item.getDescription())%></tbl:cell>
    386396              </tbl:row>
  • trunk/www/admin/mimetypes/view_mimetype.jsp

    r3679 r3719  
    180180      </tr>
    181181      <tr>
     182        <td class="prompt">Auto compress</td>
     183        <td><%=mimeType.getAutoCompress() ? "yes" : "no"%></td>
     184      </tr>
     185      <tr>
    182186        <td class="prompt">Description</td>
    183187        <td><%=HTML.niceFormat(mimeType.getDescription())%></td>
  • trunk/www/filemanager/directories/edit_directory.jsp

    r3679 r3719  
    6060  Directory directory = null;
    6161  Directory parent = null;
     62  boolean autoCompress = false;
    6263
    6364  if (itemId == 0)
     
    6768      SystemItems.getId(Directory.ROOT))
    6869    );
     70    autoCompress = parent.getAutoCompress();
    6971    cc.removeObject("item");
    7072  }
     
    7375    directory = Directory.getById(dc, itemId);
    7476    parent = directory.getParent();
     77    autoCompress = directory.getAutoCompress();
    7578    cc.setObject("item", directory);
    7679    title = "Edit directory -- " + HTML.encodeTags(directory.getName());
     
    146149          size="40" maxlength="<%=Directory.MAX_NAME_LENGTH%>"></td>
    147150      </tr>
     151      <tr>
     152        <td class="prompt">Compress files</td>
     153        <td>
     154          <input type="radio" name="autoCompress" value="0" <%=!autoCompress ? "checked" : ""%>>no
     155          <input type="radio" name="autoCompress" value="1" <%=autoCompress ? "checked" : ""%>>yes
     156        </td>
     157      </tr>
    148158      <tr valign=top>
    149159        <td class="prompt">Description</td>
  • trunk/www/filemanager/directories/index.jsp

    r3679 r3719  
    144144    }
    145145    directory.setName(Values.getStringOrNull(request.getParameter("name")));
     146    directory.setAutoCompress(Values.getBoolean(request.getParameter("autoCompress")));
    146147    directory.setDescription(Values.getStringOrNull(request.getParameter("description")));
    147148    dc.commit();
  • trunk/www/filemanager/directories/list_directories.jsp

    r3679 r3719  
    5757  for (Directory child : children)
    5858  {
    59     String folderIcon = child.isHomeDirectory() ? "UserHome" : "Folder";
    60     if (child.isRemoved()) folderIcon = "FolderDeleted";
     59    String folderIcon = "Folder";
     60    if (child.isHomeDirectory())
     61    {
     62      folderIcon = "UserHome";
     63    }
     64    else if (child.isRemoved())
     65    {
     66      folderIcon = "FolderDeleted";
     67    }
     68    else if (child.getAutoCompress())
     69    {
     70      folderIcon = "FolderCompressed";
     71    }
    6172    sb.append("var dir").append(child.getId()).append(" = JoustMenu.addChildItem(").append(parentId);
    6273    sb.append(",'").append(folderIcon).append("'");
  • trunk/www/filemanager/files/index.jsp

    r3679 r3719  
    4646  import="net.sf.basedb.util.ShareableUtil"
    4747  import="net.sf.basedb.util.OwnableUtil"
     48  import="net.sf.basedb.core.SimpleProgressReporter"
    4849  import="net.sf.basedb.clients.web.Base"
    4950  import="net.sf.basedb.clients.web.WebException"
     
    465466    forward = root + "filemanager/files/download_file.jsp?ID="+ID;
    466467  }
     468  else if ("CompressItem".equals(cmd))
     469  {
     470    // Compress a single item and then return to the view page
     471    SimpleProgressReporter progress = new SimpleProgressReporter(null);
     472    sc.setSessionSetting("progress.compress", progress);
     473    dc = sc.newDbControl();
     474    ItemContext cc = Base.getAndSetCurrentContext(sc, itemType, pageContext, defaultContext);
     475    File f = File.getById(dc, cc.getId());
     476    f.compress(progress);
     477    dc.commit();
     478    redirect = viewPage;
     479  }
     480  else if ("DecompressItem".equals(cmd))
     481  {
     482    // Compress a single item and then return to the view page
     483    SimpleProgressReporter progress = new SimpleProgressReporter(null);
     484    sc.setSessionSetting("progress.decompress", progress);
     485    dc = sc.newDbControl();
     486    ItemContext cc = Base.getAndSetCurrentContext(sc, itemType, pageContext, defaultContext);
     487    File f = File.getById(dc, cc.getId());
     488    f.decompress(progress);
     489    dc.commit();
     490    redirect = viewPage;
     491  }
    467492  else
    468493  {
  • trunk/www/filemanager/files/list_files.jsp

    r3679 r3719  
    3232  import="net.sf.basedb.core.SystemItems"
    3333  import="net.sf.basedb.core.Item"
     34  import="net.sf.basedb.core.Type"
    3435  import="net.sf.basedb.core.Location"
    3536  import="net.sf.basedb.core.File"
     
    4950  import="net.sf.basedb.core.query.Hql"
    5051  import="net.sf.basedb.core.query.Orders"
    51   import="net.sf.basedb.core.query.Hql"
    5252  import="net.sf.basedb.core.plugin.GuiContext"
    5353  import="net.sf.basedb.core.plugin.Plugin"
     
    117117  filter = cc.getPropertyFilter("description");
    118118  if (filter != null) directoryQuery.restrict(filter.getRestriction());
     119  filter = cc.getPropertyFilter("compressed");
     120  if (filter != null)
     121  {
     122    directoryQuery.restrict(Restrictions.eq(Hql.property("autoCompress"), Expressions.parameter("autoCompress")));
     123    directoryQuery.setParameter("autoCompress", filter.getValueAsObject(), Type.BOOLEAN);
     124  }
     125
    119126 
    120127  if (cc.getItemPermission() != null) directoryQuery.setItemPermission(cc.getItemPermission());
     
    390397      />
    391398      <tbl:columndef
     399        id="compressedSize"
     400        property="compressedSize"
     401        datatype="long"
     402        title="Compressed size"
     403        sortable="true"
     404        filterable="true"
     405        exportable="true"
     406      />
     407      <tbl:columndef
     408        id="compressed"
     409        property="compressed"
     410        datatype="boolean"
     411        title="Is compressed"
     412        sortable="true"
     413        filterable="true"
     414        exportable="true"
     415      />
     416      <tbl:columndef
    392417        id="writeProtected"
    393418        property="writeProtected"
     
    645670              String name = HTML.encodeTags(item.getName());
    646671              String path = HTML.encodeTags(item.getPath().toString());
     672              String icon = item.getAutoCompress() ? "directory_compressed.gif" : "directory.gif";
    647673              %>
    648674              <tbl:row>
     
    650676                  clazz="index"
    651677                  ><base:icon
    652                     image="directory.gif"
     678                    image="<%=icon%>"
    653679                    tooltip="<%=path%>"
    654680                  /></tbl:header>
     
    682708                <tbl:cell column="name"><div class="link" onclick="directoryOnClick(event, <%=itemId%>)" title="Edit this directory"><%=name%></div></tbl:cell>
    683709                <tbl:cell column="size">&nbsp;</tbl:cell>
     710                <tbl:cell column="compressedSize">&nbsp;</tbl:cell>
     711                <tbl:cell column="compressed"><%=item.getAutoCompress() ? "yes" : "no"%></tbl:cell>
    684712                <tbl:cell column="fileType">&nbsp;</tbl:cell>
    685713                <tbl:cell column="mimeType">&nbsp;</tbl:cell>
     
    727755              String urlPath = HTML.urlEncode(path);
    728756              String tagPath = HTML.encodeTags(path);
     757              String icon = item.isCompressed() ? "file_compressed.gif" : "file.gif";
    729758              index++;
    730759              numListed++;
     
    734763                  clazz="index"
    735764                  ><base:icon
    736                     image="file.gif"
     765                    image="<%=icon%>"
    737766                    tooltip="<%=tagPath%>"
    738767                  /></tbl:header>
     
    775804                  title="<%=tooltip%>"><%=name%></div></tbl:cell>
    776805                <tbl:cell column="size"><%=Values.formatBytes(item.getSize())%></tbl:cell>
     806                <tbl:cell column="compressedSize"><%=Values.formatBytes(item.getCompressedSize())%></tbl:cell>
     807                <tbl:cell column="compressed"><%=item.isCompressed() ? "yes" : "no"%></tbl:cell>
    777808                <tbl:cell column="writeProtected"><%=item.isWriteProtected() ? "yes" : "no"%></tbl:cell>
    778809                <tbl:cell column="fileType"
  • trunk/www/filemanager/files/view_file.jsp

    r3679 r3719  
    128128      location.href = "index.jsp?ID=<%=ID%>&cmd=MoveItem&move=0&item_id=<%=itemId%>";
    129129    }
     130    function compress()
     131    {
     132      Main.openPopup('../../common/progress_reporter.jsp?ID=<%=ID%>&progress=compress&title=Compressing file', 'CompressFile', 400, 200);
     133      location.replace('index.jsp?ID=<%=ID%>&cmd=CompressItem&item_id=<%=itemId%>');
     134    }
     135    function decompress()
     136    {
     137      Main.openPopup('../../common/progress_reporter.jsp?ID=<%=ID%>&progress=decompress&title=Decompressing file', 'UncompressFile', 400, 200);
     138      location.replace('index.jsp?ID=<%=ID%>&cmd=DecompressItem&item_id=<%=itemId%>');
     139    }
    130140    </script>
    131141  </base:head>
     
    170180        visible="<%=!isOwner%>"
    171181        tooltip="<%=setOwnerPermission ? "Take ownership of this item" : "You do not have permission to take ownership of this item"%>"
     182      />
     183      <tbl:button
     184        disabled="<%=!writePermission%>"
     185        image="<%=writePermission ? "compress.png" : "compress_disabled.png"%>"
     186        onclick="compress()"
     187        title="Compress&hellip;"
     188        visible="<%=!file.isCompressed()%>"
     189        tooltip="Store this file in a compressed format on the disk"
     190      />
     191      <tbl:button
     192        disabled="<%=!writePermission%>"
     193        image="<%=writePermission ? "decompress.png" : "decompress_disabled.png"%>"
     194        onclick="decompress()"
     195        title="Decompress&hellip;"
     196        visible="<%=file.isCompressed()%>"
     197        tooltip="Decompress this file and store it in it's normal format on the disk"
    172198      />
    173199      <tbl:button
     
    320346        <td><%=Values.formatBytes(file.getSize())%></td>
    321347      </tr>
     348      <%
     349      if (file.isCompressed())
     350      {
     351        %>
     352        <tr>
     353          <td class="prompt">Compressed Size</td>
     354          <td><%=Values.formatBytes(file.getCompressedSize())%></td>
     355        </tr>
     356        <%
     357      }
     358      %>
    322359      <tr>
    323360        <td class="prompt">Last update</td>
     
    327364        <td class="prompt">Owner</td>
    328365        <td><base:propertyvalue item="<%=file%>" property="owner" /></td>
     366      </tr>
     367      <tr>
     368        <td class="prompt">MD5</td>
     369        <td><%=file.getMd5()%></td>
    329370      </tr>
    330371      <tr>
  • trunk/www/filemanager/upload/select.jsp

    r3679 r3719  
    2929--%>
    3030<%@ page session="false"
     31  import="net.sf.basedb.core.Application"
    3132  import="net.sf.basedb.core.SessionControl"
    3233  import="net.sf.basedb.core.DbControl"
     
    7677  boolean readCurrentFileType = true;
    7778  int currentFileTypeId = 0;
     79  Boolean compress = Application.autoCompressionEnabled() ? null : false; // null == auto
    7880
    7981  if (itemId == 0)
     
    8789    directory = Directory.getById(dc, Values.getInt(request.getParameter("directory_id"), SystemItems.getId(Directory.ROOT)));
    8890    cc.removeObject("item");
     91    if (directory.getAutoCompress()) compress = true;
    8992  }
    9093  else
     
    163166        frm.check_md5.value = frm.temp_check_md5 && frm.temp_check_md5.checked ? "1" : "0";
    164167        frm.write_protected.value = frm.temp_write_protected.checked ? "1" : "0";
     168        frm.compressed.value = frm.temp_compressed && frm.temp_compressed[frm.temp_compressed.selectedIndex].value;
    165169        if (frm.temp_zip_format)
    166170        {
     
    236240        frm['temp_zip_overwrite'].disabled = !isZip || !frm['temp_zip_unpack'].checked;
    237241        frm['temp_zip_keep'].disabled = !isZip || !frm['temp_zip_unpack'].checked;
     242        frm['temp_compressed'].disabled = isZip;
     243        if (isZip)
     244        {
     245          // Select '-auto-' if present, otherwise 'no'
     246          Forms.selectListOption(frm['temp_compressed'], '0');
     247          Forms.selectListOption(frm['temp_compressed'], '');
     248        }
    238249      }
    239250    }
     
    345356    <input type="hidden" name="check_md5" value="0">
    346357    <input type="hidden" name="write_protected" value="0">
     358    <input type="hidden" name="compressed" value="">
    347359    <input type="hidden" name="zip_format" value="">
    348360    <input type="hidden" name="zip_unpack" value="0">
     
    374386          <%=(file != null && file.isWriteProtected()) ||
    375387            (file == null && Values.getBoolean(cc.getPropertyValue("writeProtected"))) ? "checked" : ""%>>
     388        </td>
     389      </tr>
     390      <tr>
     391        <td class="prompt">Store compressed</td>
     392        <td>
     393          <select name="temp_compressed">
     394            <%
     395            if (Application.autoCompressionEnabled())
     396            {
     397              %>
     398              <option value="" <%=compress == null ? "selected" : "" %>>- auto -
     399              <%
     400            }
     401            %>
     402            <option value="1" <%=Boolean.TRUE.equals(compress) ? "selected" : ""%>>yes
     403            <option value="0" <%=Boolean.FALSE.equals(compress) ? "selected" : ""%>>no
     404          </select>
    376405        </td>
    377406      </tr>
  • trunk/www/filemanager/upload/upload.jsp

    r3679 r3719  
    9292  boolean checkMd5 = Values.getBoolean(upload.getParameter("check_md5"), false);
    9393  boolean writeProtected = Values.getBoolean(upload.getParameter("write_protected"), false);
     94  Boolean compress = null; // null == auto by MIME type
     95  if (upload.getParameter("compressed") != null)
     96  {
     97    compress = Values.getBoolean(upload.getParameter("compressed"));
     98  }
    9499  int fileTypeId = Values.getInt(upload.getParameter("filetype_id"));
    95100  String description = upload.getParameter("description");
     
    152157      }
    153158      f.setDescription(description);
    154       toFile = f.getUploadStream(checkMd5);
     159      toFile = f.getUploadStream(checkMd5, compress);
    155160      f.setWriteProtected(writeProtected);
    156161    }
  • trunk/www/include/scripts/main.js

    r3679 r3719  
    387387    this.controllers['PROJECT'] = { url:'my_base/projects/index.jsp', width:570, height:400 };
    388388    this.controllers['FILE'] = { url:'filemanager/index.jsp', width:500, height:360 };
    389     this.controllers['DIRECTORY'] = { url:'filemanager/directories/index.jsp', width:450, height:340, xpopup:true };
     389    this.controllers['DIRECTORY'] = { url:'filemanager/directories/index.jsp', width:500, height:360, xpopup:true };
    390390    this.controllers['ARRAYBATCH'] = { url:'lims/arraybatches/index.jsp', width:800, height:500 };
    391391    this.controllers['ARRAYDESIGN'] = { url:'lims/arraydesigns/index.jsp', width:800, height:500 };
     
    407407    this.controllers['HARDWARE'] = { url:'admin/hardware/index.jsp', width:500, height:340 };
    408408    this.controllers['HARDWARETYPE'] = { url:'admin/hardwaretypes/index.jsp', width:500, height:340 };
    409     this.controllers['MIMETYPE'] = { url:'admin/mimetypes/index.jsp', width:500, height:340 };
     409    this.controllers['MIMETYPE'] = { url:'admin/mimetypes/index.jsp', width:540, height:400 };
    410410    this.controllers['NEWS'] = { url:'admin/news/index.jsp', width:540, height:400 };
    411411    this.controllers['PLUGINCONFIGURATION'] = { url:'admin/pluginconfigurations/index.jsp', width:500, height:340 };
  • trunk/www/include/scripts/newjoust.js

    r3679 r3719  
    452452    this.addIcon('FolderExpandedSelected', path + 'folderselected.gif', width, height);
    453453    this.addIcon('FolderDeleted', path + 'folderdeleted.gif', width, height);
     454    this.addIcon('FolderCompressed', path + 'foldercompressed.gif', width, height);
     455    this.addIcon('FolderCompressedExpanded', path + 'foldercompressed.gif', width, height);
     456    this.addIcon('FolderCompressedSelected', path + 'foldercompressedselected.gif', width, height);
     457    this.addIcon('FolderCompressedExpandedSelected', path + 'foldercompressedselected.gif', width, height);
    454458    this.addIcon('Root', path + 'root.gif', width, height);
    455459    this.addIcon('Home', path + 'home.gif', width, height);
  • trunk/www/include/styles/main.css

    r3679 r3719  
    232232
    233233.note {
     234  background: #E0E0E0;
    234235}
    235236
    236237.note .main {
    237   background: #E0E0E0;
    238238  border: 1px solid #999999;
    239239}
Note: See TracChangeset for help on using the changeset viewer.