Changeset 7623

Timestamp:

Mar 7, 2019, 10:55:50 AM (5 years ago)

Author:

Nicklas Nordborg

Message:

References #2156: Check UTF-8 for text files that are uploaded without a selected character set

Implemented a utility class CharsetDetector that can be used for simple testing of encoding in text files. It works best with encodings that can be technically detected. For example, UTF-8 is very unlikely to be mixed up with other encodings while any of the ISO-8859-x encodings can typically be used for all files. The StringDetector is intended to be used for discriminating between ISO-8859-x encodings but it requires prior knowledge of text that is expected to be found in the file that is unique to an encoding.

The file upload functionality has been extended to check for UTF-8 text files. It is enabled automatically when the MIME type is set to something in the 'text/*' subset and no character set has been explicitely specified.

Location:

trunk/src/core/net/sf/basedb

Files:

• core/File.java (modified) (11 diffs)
• util/charset (added)
• util/charset/CharsetDetector.java (added)
• util/charset/SimpleStringDetector.java (added)
• util/charset/StringDetector.java (added)

trunk/src/core/net/sf/basedb/core/File.java

@@ -32 +32 @@
 import net.sf.basedb.core.data.MimeTypeData;
 import net.sf.basedb.core.hibernate.TypeWrapper;
+import net.sf.basedb.util.charset.CharsetDetector;
 import net.sf.basedb.util.EqualsHelper;
 import net.sf.basedb.util.FileUtil;
@@ -51 +52 @@
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -1409 +1412 @@
       newInternalFile = getNewFile(compress);
     }
+    String mimeType = getMimeType();
+    String charset = getCharacterSet();
+    // If the file is a text file and no charset has been detected we try UTF-8 detection
+    boolean utf8Detect = mimeType != null && mimeType.startsWith("text/") && charset == null;
     try
     {
-      uploadStream = new UploadStream(newInternalFile, true, checkMd5, compress);
+      uploadStream = new UploadStream(newInternalFile, true, checkMd5, compress, utf8Detect);
     }
     catch (IOException ex)
@@ -1689 +1696 @@
     private boolean checkMd5;
 
+    private CharsetTesterThread utf8Tester;
+    
     /**
       Create a new <code>UploadStream</code> and connect it to the
@@ -1697 +1706 @@
         previously stored md5 sum, useful for making sure secondary storage is
         working properly
+      @param utf8Detect If set, we try to parse the file as UTF-8. If this works we set the
+        encoding to UTF-8
       @throws IOException If there is an error during the upload
     */
-    private UploadStream(java.io.File file, boolean calculateMd5, boolean checkMd5, boolean compress)
+    private UploadStream(java.io.File file, boolean calculateMd5, boolean checkMd5, boolean compress, boolean utf8Detect)
       throws IOException
     {
@@ -1712 +1723 @@
       }
       closed = false;
+      if (utf8Detect)
+      {
+        // Start a thread for parallell testing if the file can be parsed as UTF-8
+        utf8Tester = new CharsetTesterThread(Charset.forName("UTF-8"));
+        new Thread(utf8Tester).start();
+      }
     }
     
@@ -1721 +1738 @@
       if (md5 != null) md5.update(b, off, len);
       super.write(b, off, len);
-    }
-    
+      if (utf8Tester != null) utf8Tester.write(b, off, len);
+    }
+      
     @Override
     public void write(byte[] b)
@@ -1744 +1762 @@
       if (md5 != null) md5.update((byte)b);
       super.write(b);
+      if (utf8Tester != null) utf8Tester.write(b);
     }
 
@@ -1759 +1778 @@
         return;
       }
+      String charset = null;
+      if (utf8Tester != null)
+      {
+        utf8Tester.close();
+        if (utf8Tester.couldParse()) 
+        {
+          charset = utf8Tester.getCharset().name();
+        }
+      }
       super.close();
       FileData data = getData();
@@ -1767 +1795 @@
       data.setRemovedBy(null);
       data.setUrl(null);
+      if (charset != null && data.getCharacterSet() == null)
+      {
+        data.setCharacterSet(charset);
+      }
       if (md5 != null)
       {
@@ -1783 +1815 @@
     }
   }
+  
+  class CharsetTesterThread
+    implements Runnable
+  {
+    private final PipedOutputStream pop;
+    private final PipedInputStream pip;
+    private final CharsetDetector detector;
+    
+    private Thread thread;
+    // These variables are used by both threads
+    private volatile boolean couldParse;
+    private volatile boolean isRunning;
+    
+    CharsetTesterThread(Charset charset)
+      throws IOException
+    {
+      this.detector = new CharsetDetector(charset);
+      this.pop = new PipedOutputStream();
+      this.pip = new PipedInputStream(pop, 4096);
+      this.isRunning = true;
+    }
+    
+    /**
+      Runs in a separate thread.
+    */
+    @Override
+    public void run() 
+    {
+      thread = Thread.currentThread();
+      couldParse = detector.testIt(pip);
+      isRunning = false;
+    }
+
+    public Charset getCharset()
+    {
+      return detector.getCharset();
+    }
+    
+    /**
+      Returns TRUE if the file seems like valid (UTF-8). Note
+      that this may not be correct until the close() method has
+      been called.
+    */
+    public boolean couldParse()
+    {
+      return couldParse;
+    }
+    
+    /**
+      Runs in the main thread. Copy bytes to charset detector via
+      the piped streams.
+    */
+    public void write(byte[] b, int off, int len)
+    {
+      if (isRunning) 
+      {
+        try
+        {
+          pop.write(b, off, len);
+        }
+        catch (IOException ex)
+        {
+          isRunning = false;
+        }
+      }
+    }
+    
+    /**
+      Runs in the main thread. Copy bytes to charset detector via
+      the piped streams.
+    */
+    public void write(int b)
+    {
+      if (isRunning)
+      {
+        try
+        {
+          pop.write(b);
+        }
+        catch (IOException ex)
+        {
+          isRunning = false;
+        }
+      }
+    }
+    
+    /**
+      The file has been uploaded. Wait for the charset detector
+      threas to finish and the close everything.
+     */
+    public void close()
+    {
+      try
+      {
+        // Flush and close the pipe and...
+        pop.flush();
+        FileUtil.close(pop);
+        if (isRunning && thread != null)
+        {
+          // ... wait for the reading thread to terminate
+          thread.join();
+        }
+      }
+      catch (IOException | InterruptedException ex)
+      {}
+      finally
+      {
+        thread = null;
+        FileUtil.close(pop);
+        FileUtil.close(pip);
+      }
+    }
+  }
+  
 }