Changeset 5582


Ignore:
Timestamp:
Mar 15, 2011, 1:55:59 PM (13 years ago)
Author:
Nicklas Nordborg
Message:

References #1582: Extended support for external files

Created framework for external file support. A registry (ConnectionManagerRegistry) is keeping track of registered factories (ConnectionManagerFactory). Typcially, a specific protocol is handled by a single factory.

An implementation for http and https have been created. I also have code for HDFS but I think that should be made as an extension. To do this we first need #1593 and parts of #1592.

The current implementation only supports automatic selection of connection factory (based on the protocol). This will probably not work with, for example, Amazon S3, since this also uses http (+custom authentication procedure). We need a way to manually select a factory to use. This is probably best done at the FileServer level.

Location:
trunk/src
Files:
10 added
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/clients/web/net/sf/basedb/clients/web/servlet/Download.java

    r5384 r5582  
    159159        mimeType = MimeType.getMimeType(dc, f.getName(), defaultMimeType);
    160160      }
    161       dc.close();
    162161      response.setContentType(mimeType);
    163162      if (f.getCharacterSet() != null) response.setCharacterEncoding(f.getCharacterSet());
    … …  
    166165      if (size >= 0 && size < Integer.MAX_VALUE) response.setContentLength((int)size);
    167166      in = f.getDownloadStream(0);
     167      dc.close();
    168168      out = response.getOutputStream();
    169169      FileUtil.copy(in, out);

  • trunk/src/core/net/sf/basedb/core/Application.java

    r5442 r5582  
    3535import net.sf.basedb.util.StaticCache;
    3636import net.sf.basedb.util.timer.Scheduler;
     37import net.sf.basedb.util.uri.ConnectionManagerRegistry;
    3738
    3839import java.util.ArrayList;
    … …  
    192193  */
    193194  private static LogManagerFactory logManagerFactory;
     195 
     196  /**
     197    The URI handler registry that is currently in use.
     198  */
     199  private static ConnectionManagerRegistry connectionManagerRegistry;
    194200 
    195201  /**
    … …  
    520526          logManagerFactory = (LogManagerFactory)factoryClass.newInstance();
    521527        }
     528       
     529        connectionManagerRegistry = new ConnectionManagerRegistry();
     530        ConnectionManagerRegistry.registerDefaultUriHandlers(connectionManagerRegistry);
     531        // TODO - load more handlers as configured in .... (base.config?)
     532        connectionManagerRegistry.lock();
    522533       
    523534        // Adding a task that cleans the session control cache at regular intervals
    … …  
    640651    cleanSessionControlCache(true);
    641652    sessionCache = null;
    642 
     653    connectionManagerRegistry = null;
     654   
    643655    Keyring.unload();
    644656    SystemItems.unload();
    … …  
    703715  {
    704716    return logManagerFactory;
     717  }
     718 
     719  /**
     720    Get the (read-only) connection manager registry that contains factories
     721    for accessing file on external locations.
     722    @since 3.0
     723  */
     724  public static ConnectionManagerRegistry getConnectionManagerRegistry()
     725  {
     726    return connectionManagerRegistry;
    705727  }
    706728 

  • trunk/src/core/net/sf/basedb/core/File.java

    r5384 r5582  
    2727import net.sf.basedb.core.data.FileAttachableData;
    2828import net.sf.basedb.core.data.FileData;
    29 import net.sf.basedb.core.data.FileServerData;
    3029import net.sf.basedb.core.data.FileSetData;
    3130import net.sf.basedb.core.data.FileTypeData;
    … …  
    3332import net.sf.basedb.util.EqualsHelper;
    3433import net.sf.basedb.util.FileUtil;
    35 import net.sf.basedb.util.HttpUtil;
    3634import net.sf.basedb.util.MD5;
    3735import net.sf.basedb.util.StreamCacher;
    38 import net.sf.basedb.util.ssl.SSLUtil;
     36import net.sf.basedb.util.uri.ConnectionParameters;
     37import net.sf.basedb.util.uri.ConnectionManager;
     38import net.sf.basedb.util.uri.ConnectionManagerFactory;
     39import net.sf.basedb.util.uri.ConnectionManagerRegistry;
     40import net.sf.basedb.util.uri.UriMetadata;
    3941
    4042import java.io.BufferedInputStream;
    … …  
    4446import java.io.FileNotFoundException;
    4547import java.io.FileOutputStream;
    46 import java.io.FilterInputStream;
    4748import java.io.IOException;
    4849import java.io.InputStream;
    4950import java.io.OutputStream;
    50 import java.net.MalformedURLException;
    51 import java.net.URL;
     51import java.net.URI;
     52import java.net.URISyntaxException;
    5253import java.nio.charset.Charset;
    5354import java.text.SimpleDateFormat;
    … …  
    5960import java.util.Random;
    6061import java.util.Set;
    61 import java.util.regex.Matcher;
    62 import java.util.regex.Pattern;
    6362import java.util.zip.GZIPInputStream;
    6463import java.util.zip.GZIPOutputStream;
    6564import java.security.MessageDigest;
    66 
    67 
    68 import org.apache.http.HttpEntity;
    69 import org.apache.http.HttpResponse;
    70 import org.apache.http.auth.AuthScope;
    71 import org.apache.http.auth.Credentials;
    72 import org.apache.http.auth.UsernamePasswordCredentials;
    73 import org.apache.http.client.HttpClient;
    74 import org.apache.http.client.methods.HttpGet;
    75 import org.apache.http.client.methods.HttpHead;
    76 import org.apache.http.conn.scheme.Scheme;
    77 import org.apache.http.conn.ssl.SSLSocketFactory;
    78 import org.apache.http.impl.client.DefaultHttpClient;
    79 
    8065
    8166/**
    … …  
    897882 
    898883  /**
     884    Get the URL of an external file as an URI. This property is only set/valid
     885    for files that are stored externally (getLocation() == Location.EXTERNAL).
     886    If the URL is is not a valid URI an exception is thrown.
     887    @return An URI or null
     888    @since 3.0
     889  */
     890  public URI getURI()
     891  {
     892    return getLocation() == Location.EXTERNAL ? URI.create(getData().getUrl()) : null;
     893  }
     894 
     895  /**
    899896    Set the URL to an external file that should represent this file
    900897    object. NOTE! The location, size, md5, internal file, etc. will be reset
    … …  
    916913   
    917914    data.setUrl(StringUtil.setNotNullString(url, "url", MAX_URL_LENGTH));
    918     URL u = null;
     915
     916    // Validate the URL
     917    URI uri = null;
     918    ConnectionManagerRegistry registry = Application.getConnectionManagerRegistry();
     919    ConnectionManagerFactory factory = null;
    919920    try
    920921    {
    921       // Validate the URL
    922       u = new URL(url);
    923       if (!"http".equals(u.getProtocol()) && !"https".equals(u.getProtocol()))
    924       {
    925         throw new InvalidDataException("Only http or https URLs are supported: " + url);
    926       }
    927     }
    928     catch (MalformedURLException ex)
     922      uri = new URI(url);
     923      factory = registry.findFactory(uri);
     924      if (factory == null)
     925      {
     926        throw new InvalidDataException("Can't find any connection manager for URL: " + url);
     927      }
     928    }
     929    catch (URISyntaxException ex)
    929930    {
    930931      throw new InvalidDataException("Invalid URL: " + url, ex);
    … …  
    943944    if (loadMetadata)
    944945    {
    945       HttpClient client = null;
     946      ConnectionParameters server = getServerInfo();
     947      ConnectionManager manager = factory.createConnectionManager(uri, server);
    946948      try
    947949      {
    948         client = getHttpClient();
    949         HttpHead head = new HttpHead(url);
    950         HttpResponse response = client.execute(head);
    951 
    952         int responseCode = response.getStatusLine().getStatusCode();
    953         if (responseCode >= 200 &&  responseCode < 300)
    954         {
    955           setMetadataFromHttpResponse(response);
    956         }
    957         else if (responseCode == 404)
    958         {
    959           throw new InvalidDataException("404 URL not found: " + url);
    960         }
    961         else if (responseCode == 401)
    962         {
    963           throw new InvalidDataException("401 Authorization required: " + url);
    964         }
    965         else
    966         {
    967           throw new IOException(responseCode + " " + response.getStatusLine().getReasonPhrase());
    968         }
     950        setMetadataFromURI(manager.getMetadata());
    969951      }
    970952      catch (IOException ex)
    971953      {
    972954        throw new BaseException("Could not load metadata for URL:" + url, ex);
    973       }
    974       finally
    975       {
    976         if (client != null) HttpUtil.shutdown(client);
    977955      }
    978956    }
    … …  
    1009987  }
    1010988 
    1011   /**
    1012     Get a HttpClient object that has been configured to download this file.
    1013     When the calling code is done with the client, it should call
    1014     HttpUtil.shutdown(client) to properly shut down the client.
    1015     @since 2.16
    1016   */
    1017   HttpClient getHttpClient()
    1018     throws IOException
    1019   {
    1020     DefaultHttpClient httpClient = new DefaultHttpClient();
    1021 
    1022     // Do we need to provide username/password?
    1023     FileServerData server = getData().getFileServer();
    1024     if (server != null && server.getUsername() != null)
    1025     {
    1026       Credentials c = new UsernamePasswordCredentials(server.getUsername(), server.getPassword());
    1027       httpClient.getCredentialsProvider().setCredentials(AuthScope.ANY, c );
    1028     }
    1029 
    1030     SSLSocketFactory sslFactory = null;
    1031     if (server != null)
    1032     {
    1033       sslFactory = SSLUtil.getSSLSocketFactory(server.getServerCertificate(),
    1034         server.getClientCertificate(), server.getClientCertificatePassword());
    1035     }
    1036     else
    1037     {
    1038       sslFactory = SSLUtil.getSSLSocketFactory();
    1039     }
    1040     // Adds support for https urls
    1041     Scheme https = new Scheme("https", sslFactory, 443);
    1042     httpClient.getConnectionManager().getSchemeRegistry().register(https);
    1043       return httpClient;
    1044   }
    1045  
    1046   private void setMetadataFromHttpResponse(HttpResponse response)
     989  private ConnectionParameters getServerInfo()
     990  {
     991    return ConnectionParameters.create(getData().getFileServer());
     992  }
     993 
     994  private void setMetadataFromURI(UriMetadata metadata)
    1047995  {
    1048996    FileData data = getData();
    1049     data.setSize(HttpUtil.getContentLength(response));
    1050     Date lastUpdated = HttpUtil.getLastModified(response);
     997    Long length = metadata.getLength();
     998    data.setSize(length == null ? -1 : length);
     999   
     1000    Date lastUpdated = metadata.getLastModified();
    10511001    if (lastUpdated != null) data.setLastUpdate(lastUpdated);
    1052     String mimeType = HttpUtil.getContentType(response);
    1053     String charset = null;
    1054     if (mimeType != null)
    1055     {
    1056       Pattern p = Pattern.compile("(\\w+/\\w+)\\s*;\\s*charset=(.*)");
    1057       Matcher m = p.matcher(mimeType);
    1058       if (m.matches())
    1059       {
    1060         mimeType = m.group(1);
    1061         charset = m.group(2);
    1062       }
    1063     }
     1002   
     1003    String mimeType = metadata.getMimeType();
    10641004    if (mimeType != null) data.setMimeType(mimeType);
     1005   
     1006    String charset = metadata.getCharacterSet();
    10651007    if (charset != null) data.setCharacterSet(charset);
     1008   
     1009    String md5 = metadata.getMd5();
     1010    if (md5 != null) data.setMd5(md5);
    10661011  }
    10671012 
    … …  
    14041349      if (location == Location.EXTERNAL)
    14051350      {
    1406         final HttpClient client = getHttpClient();
    1407         try
     1351        URI uri = getURI();
     1352        ConnectionParameters server = getServerInfo();
     1353        ConnectionManagerRegistry registry = Application.getConnectionManagerRegistry();
     1354        ConnectionManager manager = registry.createConnectionManager(uri, server);
     1355        download = manager.getInputStream();
     1356        setMetadataFromURI(manager.getMetadata());
     1357        if (download == null)
    14081358        {
    1409           HttpGet get = new HttpGet(getUrl());
    1410           HttpResponse response = client.execute(get);
    1411           setMetadataFromHttpResponse(response);
    1412           HttpEntity entity = response.getEntity();
    1413           if (entity != null)
    1414           {
    1415             download = new FilterInputStream(entity.getContent())
    1416             {
    1417               /**
    1418                 Make sure the client connection manager is closed when the
    1419                 file has been completely downloaded.
    1420               */
    1421               @Override
    1422               public void close()
    1423                 throws IOException
    1424               {
    1425                 super.close();
    1426                 HttpUtil.shutdown(client);
    1427               }
    1428             };
    1429           }
    1430           else
    1431           {
    1432             download = new ByteArrayInputStream(new byte[0]);
    1433           }
    1434         }
    1435         catch (RuntimeException ex)
    1436         {
    1437           if (client != null) HttpUtil.shutdown(client);
     1359          // For 0-length documents
     1360          download = new ByteArrayInputStream(new byte[0]);
    14381361        }
    14391362      }
Note: See TracChangeset for help on using the changeset viewer.