Changeset 2512

Timestamp:

Jun 18, 2014, 10:52:55 AM (9 years ago)

Author:

Nicklas Nordborg

Message:

References #533: Add secondary analysis section to Reggie

Added support for using InputStream/OutputStream? when reading/writing files to remote hosts.

Location:

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/ssh

Files:

• CloseResourceOutputStream.java (added)
• CopyFilesToBase.java (modified) (7 diffs)
• SshHost.java (modified) (5 diffs)

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/ssh/CopyFilesToBase.java

@@ -1 +1 @@
 package net.sf.basedb.reggie.ssh;
 
+import java.io.InputStream;
 import java.util.Arrays;
 
@@ -7 +8 @@
 
 import net.schmizz.sshj.SSHClient;
+import net.schmizz.sshj.sftp.SFTPClient;
 import net.sf.basedb.core.AnyToAny;
 import net.sf.basedb.core.Application;
@@ -99 +101 @@
     DbControl dc = null;
     SSHClient ssh = null;
+    SFTPClient sftp = null;
     try
     {
@@ -116 +119 @@
       try
       {
+        sftp = ssh.newSFTPClient();
         // Calculate total file size
         long totalSize = 0;
@@ -123 +127 @@
           totalSize += ((Number)jsonFile.get("size")).longValue();
         }
-        System.out.println("item AnyToAny: " + item);
         long copiedSoFar = 0;
         int numCopiedFiles = 0;
@@ -171 +174 @@
             file.setRemoved(false);
             
-            // Actual SCP transfer
-            from.downloadFile(ssh, rootFolder+"/"+path, new BaseFileDestFile(file));
+            InputStream in = from.readFile(sftp, rootFolder+"/"+path);
+            try
+            {
+              file.upload(in, false);
+            }
+            finally
+            {
+              in.close();
+            }
           }
           
@@ -233 +243 @@
     {
       if (dc != null) dc.close();
+      if (sftp != null) SshUtil.close(sftp);
       if (ssh != null) SshUtil.close(ssh);
     }

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/ssh/SshHost.java

@@ -3 +3 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.security.PublicKey;
 import java.text.SimpleDateFormat;
 import java.util.Date;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
@@ -20 +23 @@
 import net.schmizz.sshj.connection.channel.direct.Session;
 import net.schmizz.sshj.connection.channel.direct.Session.Command;
+import net.schmizz.sshj.sftp.FileAttributes;
+import net.schmizz.sshj.sftp.OpenMode;
+import net.schmizz.sshj.sftp.RemoteFile;
+import net.schmizz.sshj.sftp.SFTPClient;
 import net.schmizz.sshj.transport.verification.PromiscuousVerifier;
 import net.schmizz.sshj.xfer.LocalDestFile;
@@ -26 +33 @@
 import net.sf.basedb.reggie.converter.StringToDateConverter;
 import net.sf.basedb.util.FileCopyRunnable;
+import net.sf.basedb.util.uri.CloseResourceInputStream;
 
 /**
@@ -326 +334 @@
   }
   
+  public OutputStream writeFile(SFTPClient sftp, String toPath, FileAttributes attributes)
+  {
+    if (logger.isDebugEnabled())
+    {
+      logger.debug("Writing to " + getAddress() + ":" + getPort() + ":" + toPath);
+    }
+    try
+    {
+      String toDir = toPath.substring(0, toPath.lastIndexOf("/"));
+      sftp.mkdirs(toDir);
+      RemoteFile rf = sftp.open(toPath, EnumSet.of(OpenMode.CREAT, OpenMode.TRUNC, OpenMode.WRITE), attributes);
+      return new CloseResourceOutputStream(rf.new RemoteFileOutputStream(), rf);
+    }
+    catch (IOException ex)
+    {
+      logger.error("File upload failed", ex);
+      throw new RuntimeException(ex);
+    }
+    finally
+    {}
+  }
+  
   public void uploadFile(SSHClient ssh, LocalSourceFile file, String toPath)
   {
@@ -357 +387 @@
   }
 
+  /**
+    Read from a file on the remote server. This method need an
+    SFTPClient object which can be created from SSHClient.newSFTPClient().
+
+    @param sftp A SFTPClient instance connected to the remote host
+    @param fromPath The path of the remote file
+    @return An InputStream reading from the remote file
+  */
+  public InputStream readFile(SFTPClient sftp, String fromPath)
+  {
+    if (logger.isDebugEnabled())
+    {
+      logger.debug("Reading file '" + fromPath + "' from " + getAddress() + ":" + getPort());
+    }
+    try
+    {
+      RemoteFile rf = sftp.open(fromPath, EnumSet.of(OpenMode.READ));
+      return new CloseResourceInputStream(rf.new RemoteFileInputStream(), rf);
+    }
+    catch (IOException ex)
+    {
+      logger.error("File download failed", ex);
+      throw new RuntimeException(ex);
+    }
+    finally
+    {}
+  }
+  
+  /**
+    Download a remote file to a local destination.
+    @param ssh A SSHClient instance connected to the remote host
+    @param fromPath The path of the remote file
+    @param file The local destination the remote file is downloaded to
+  */
   public void downloadFile(SSHClient ssh, String fromPath, LocalDestFile file)
   {