Changeset 5194


Ignore:
Timestamp:
Dec 4, 2009, 1:15:25 PM (13 years ago)
Author:
Nicklas Nordborg
Message:

References #1444: Implement generic BFS writers and readers/parsers

Added writer/parser for data files. Some changes to event handling.

Location:
trunk/src
Files:
4 added
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/core/net/sf/basedb/util/bfs/AnnotationWriter.java

    r5193 r5194  
    145145    usedIds.add(id);
    146146    print(id);
    147     print("\t");
    148     tablePrintData(data);
     147    if (data.length > 0)
     148    {
     149      print("\t");
     150      tablePrintData(data);
     151    }
     152    else
     153    {
     154      print("\n");
     155    }
    149156  }
    150157 

  • trunk/src/core/net/sf/basedb/util/bfs/MetadataModel.java

    r5193 r5194  
    2222package net.sf.basedb.util.bfs;
    2323
     24import java.io.Writer;
    2425import java.util.ArrayList;
    2526import java.util.List;
    … …  
    2829  A model container of the contents of a BFS metadata file. This class
    2930  is primarily intended to be used when parsing a BFS metadata file
    30   (see {@link MetadataParser}), but it can be used to programmatically
    31   create a BFS metadata file and then write it out to a file (see
    32   {@link MetadataWriter}).
     31  (see {@link MetadataParser#parse(java.io.InputStream, EventHandler)}),
     32  but it can be used to programmatically create a BFS metadata file and
     33  then write it out to a file (see {@link #print(Writer)}).
    3334  <p>
    3435 
    … …  
    4344*/
    4445public class MetadataModel
     46  implements EventHandler
    4547{
    4648
    … …  
    4850  private List<Section> sections;
    4951  private Section files;
     52  private Section currentSection;
    5053 
    5154  /**
    … …  
    5760  }
    5861
     62  /*
     63    From the EventHandler interface
     64    -------------------------------
     65  */
     66  @Override
     67  public void handleEvent(EventType eventType, Object eventData)
     68  {
     69    if (eventType == MetadataParser.BOF_EVENT)
     70    {
     71      setSubtype((String)eventData);
     72    }
     73    else if (eventType == MetadataParser.SECTION_EVENT)
     74    {
     75      currentSection = addSection((String)eventData);
     76    }
     77    else if (eventType == MetadataParser.ENTRY_EVENT)
     78    {
     79      currentSection.addEntry((SectionEntry)eventData);
     80    }
     81  }
     82  // ------------------------------------
     83
     84  /**
     85    Utility method for printing the contents of this model object
     86    to a stream.
     87    @param out The output stream to write to
     88  */
     89  public void print(Writer out)
     90  {
     91    MetadataWriter bfs = new MetadataWriter(out);
     92    bfs.setSubtype(getSubtype());
     93    for (int i = 0; i < getSectionCount(); ++i)
     94    {
     95      Section s = getSection(i);
     96      bfs.bfsPrintSection(s.getName(), false, false);
     97      for (int j = 0; j < s.getEntryCount(); ++j)
     98      {
     99        SectionEntry e = s.getEntry(j);
     100        bfs.bfsPrintValue(e.getKey(), e.getValues());
     101      }
     102    }
     103    bfs.flush();
     104  }
     105 
    59106  /**
    60107    Set the subtype of the BFS.

  • trunk/src/core/net/sf/basedb/util/bfs/MetadataParser.java

    r5193 r5194  
    115115  }
    116116
    117   /**
    118     Parse the input stream and return the result as a metadata model object.
    119     The input stream is expected to be in UTF-8 format.
    120    
    121     @param in The stream to parse
    122     @return A metadata model object
    123     @throws IOException If there is an error reading the stream data
    124     @throws NullPointerException If the stream is null
    125   */
    126   public MetadataModel parse(InputStream in)
    127     throws IOException
    128   {
    129     DefaultEventHandler handler = new DefaultEventHandler();
    130     parse(in, handler);
    131     return handler.getModel();
    132   }
    133  
    134117  /**
    135118    Parse the input stream and notify the specified event handler with events.
    … …  
    262245    handler.handleEvent(END_OF_FILE_EVENT, null);
    263246  }
    264 
    265  
    266   /**
    267     Default event handler implementation for the metadata parser. This
    268     implementation generates a {@link MetadataModel} object from the
    269     information in the file.
    270   */
    271   public static class DefaultEventHandler
    272     implements EventHandler
    273   {
    274  
    275     private MetadataModel bfs;
    276     private MetadataModel.Section currentSection;
    277    
    278     public DefaultEventHandler()
    279     {}
    280 
    281     /*
    282       From the EventHandler interface
    283       -------------------------------
    284     */
    285     @Override
    286     public void handleEvent(EventType eventType, Object eventData)
    287     {
    288       if (eventType == BOF_EVENT)
    289       {
    290         handleBof((String)eventData);
    291       }
    292       else if (eventType == SECTION_EVENT)
    293       {
    294         handleSection((String)eventData);
    295       }
    296       else if (eventType == ENTRY_EVENT)
    297       {
    298         handleEntry((MetadataModel.SectionEntry)eventData);
    299       }
    300     }
    301     // ------------------------------------
    302    
    303     /**
    304       Handles the beginning-of-file marker event.
    305       @see MetadataParser#BOF_EVENT
    306     */
    307     public void handleBof(String subtype)
    308     {
    309       bfs = new MetadataModel();
    310       bfs.setSubtype(subtype);
    311     }
    312 
    313     /**
    314       Handles a section event.
    315       @see MetadataParser#SECTION_EVENT
    316     */
    317     public void handleSection(String name)
    318     {
    319       currentSection = bfs.addSection(name);
    320     }
    321 
    322     /**
    323       Handles a section entry event.
    324       @see MetadataParser#ENTRY_EVENT
    325     */
    326     public void handleEntry(MetadataModel.SectionEntry entry)
    327     {
    328       currentSection.addEntry(entry);
    329     }
    330    
    331     /**
    332       Get the generated metadata model object after the parsing has
    333       been finished.
    334     */
    335     public MetadataModel getModel()
    336     {
    337       return bfs;
    338     }
    339 
    340   }
    341  
    342247}

  • trunk/src/core/net/sf/basedb/util/bfs/MetadataWriter.java

    r5193 r5194  
    5555  public static final String BOF_MARKER = "BFSformat";
    5656
    57   private final String subtype;
    5857  private final EncoderDecoder encoder;
     58  private String subtype;
    5959  private Set<String> usedSections;
    6060  private Set<String> usedKeys;
    … …  
    6969   
    7070    @param out The parent writer which this writer will print to
    71     @param subtype The BFS subtype, or null if no subtype is needed
    72   */
    73   public MetadataWriter(Writer out, String subtype)
     71  */
     72  public MetadataWriter(Writer out)
    7473  {
    7574    super(out);
    76     this.subtype = subtype;
    7775    this.usedSections = new HashSet<String>();
    7876    this.usedKeys = new HashSet<String>();
    7977    this.encoder = new TabCrLfEncoderDecoder(true);
    80     internalPrintBofMarker();
    8178  }
    8279 
    … …  
    9188  }
    9289
     90  /**
     91    Set the BFS subtype for this writer. This method must be called
     92    before any printing method is called.
     93    @param subtype The subtype
     94    @throws IllegalStateException If data has already been written
     95  */
     96  public void setSubtype(String subtype)
     97  {
     98    if (getLineCount() > 0)
     99    {
     100      throw new IllegalStateException("Can't change the subtype after data has been printed");
     101    }
     102    this.subtype = subtype;
     103  }
     104 
    93105  /**
    94106    Get the name of the current section.
    … …  
    168180  public void bfsPrintEmptyLine(int num)
    169181  {
     182    if (getLineCount() == 0) internalPrintBofMarker();
    170183    for (int i = 0; i < num; ++i)
    171184    {
    172185      print("\n");
    173186    }
    174     if (num > 0) lineCount += num;
     187    lineCount += num;
    175188  }
    176189 
    … …  
    184197  {
    185198    if (comment == null) return;
     199    if (getLineCount() == 0) internalPrintBofMarker();
    186200    String[] lines = comment.split("\\n|\\r|\\r\\n");
    187201    for (String line : lines)
    … …  
    204218  public void bfsPrintSection(String name, boolean unique, boolean uniqueKeys)
    205219  {
     220    if (getLineCount() == 0) internalPrintBofMarker();
    206221    if (name == null) throw new NullPointerException("name");
    207222    if (unique)

  • trunk/src/test/TestBfs.java

    r5193 r5194  
    2222import java.io.File;
    2323import java.io.FileInputStream;
     24import java.io.FileOutputStream;
    2425import java.io.FileWriter;
     26import java.io.OutputStreamWriter;
    2527import java.io.Writer;
    2628import java.util.Arrays;
    … …  
    2830import net.sf.basedb.core.BaseException;
    2931import net.sf.basedb.util.Values;
     32import net.sf.basedb.util.bfs.AnnotationModel;
    3033import net.sf.basedb.util.bfs.AnnotationParser;
    3134import net.sf.basedb.util.bfs.AnnotationWriter;
    32 import net.sf.basedb.util.bfs.EventHandler;
    33 import net.sf.basedb.util.bfs.EventType;
     35import net.sf.basedb.util.bfs.DataParser;
     36import net.sf.basedb.util.bfs.DataWriter;
     37import net.sf.basedb.util.bfs.MatrixModel;
    3438import net.sf.basedb.util.bfs.MetadataModel;
    3539import net.sf.basedb.util.bfs.MetadataParser;
    … …  
    5458   
    5559    File metadata = test_create_temp_file("metadata");
     60    File metadata_copy = test_create_temp_file("metadata_copy");
    5661    test_write_metadata(metadata);
    57     test_read_metadata(metadata);
     62    test_read_metadata(metadata, metadata_copy);
     63    test_read_metadata(metadata_copy, null);
    5864   
    5965    File annotations = test_create_temp_file("annotations");
    … …  
    6167    test_read_annotations(annotations);
    6268   
     69    File data = test_create_temp_file("data");
     70    test_write_data(data);
     71    test_read_data(data);
     72   
    6373    if (TestUtil.waitBeforeDelete()) TestUtil.waitForEnter();
    6474   
    … …  
    91101    {
    92102      Writer out = new FileWriter(file);
    93       bfs = new MetadataWriter(out, "test");
     103      bfs = new MetadataWriter(out);
     104      bfs.setSubtype("test");
    94105     
    95106      // Test comments and empty lines
    … …  
    155166  }
    156167
    157   static void test_read_metadata(File file)
     168  static void test_read_metadata(File file, File copyTo)
    158169  {
    159170    if (file == null) return;
    … …  
    163174    {
    164175      MetadataParser bfsParser = new MetadataParser();
    165       MetadataModel bfs = bfsParser.parse(new FileInputStream(file));
     176      MetadataModel bfs = new MetadataModel();
     177      bfsParser.parse(new FileInputStream(file), bfs);
    166178     
    167179      if (log)
    … …  
    183195        throw new BaseException("Unexpected BFS subtype: " + bfs.getSubtype());
    184196      }
    185        
     197     
    186198      if (bfs.getSectionCount() != 3)
    187199      {
    … …  
    245257        throw new BaseException("Unexpected file, index 'file-2': " + bfs.getFile("file-2"));       
    246258      }
    247 
     259     
     260      if (copyTo != null)
     261      {
     262        bfs.print(new OutputStreamWriter(new FileOutputStream(copyTo), "UTF-8"));
     263      }
     264     
    248265      write("--Test BFS Metadata parser OK");
    249266    }
    … …  
    327344    {
    328345      AnnotationParser bfsParser = new AnnotationParser();
    329      
    330       String[] headers;
    331      
    332       bfsParser.parse(new FileInputStream(file),
    333         new EventHandler()
     346      AnnotationModel bfs = new AnnotationModel();
     347      bfsParser.parse(new FileInputStream(file), bfs);
     348
     349      String[] headers = bfs.getHeaders();
     350      int rowCount = bfs.getRowCount();
     351      int columnCount = bfs.getColumnCount();
     352     
     353      if (log)
     354      {
     355        write("  Rows: " + rowCount + "; columns: " + columnCount);
     356        write("  Header: " +
     357            Values.getString(Arrays.asList(headers), "\t", false));
     358       
     359        for (int r = 0; r < rowCount; r++)
    334360        {
    335           private int dataLines;
    336        
    337           @Override
    338           public <T> void handleEvent(EventType<T> eventType, T eventData)
     361          int rowId = bfs.getRowId(r);
     362          String[] row = bfs.getRowById(rowId);
     363          write("  Line " + r + ": " +
     364              Values.getString(Arrays.asList(row), "\t", false));
     365          if (row.length != columnCount)
    339366          {
    340             if (eventType == AnnotationParser.HEADER_EVENT)
    341             {
    342               String[] headers = (String[])eventData;
    343               if (headers.length != 4)
    344               {
    345                 throw new BaseException("Expected 4 header columns, not " + headers.length);
    346               }
    347               if (log)
    348               {
    349                 write("  Header: " +
    350                   Values.getString(Arrays.asList(headers), "\t", false));
    351               }
    352             }
    353             else if (eventType == AnnotationParser.DATA_EVENT)
    354             {
    355               String[] data = (String[])eventData;
    356               if (data.length != 4)
    357               {
    358                 throw new BaseException("Expected 4 data columns, not " + data.length);
    359               }
    360               dataLines++;
    361               if (log)
    362               {
    363                 write("  Line " + dataLines + ": " +
    364                   Values.getString(Arrays.asList(data), "\t", false));
    365               }
    366             }
    367             else if (eventType == AnnotationParser.END_OF_FILE_EVENT)
    368             {
    369               if (dataLines != 2)
    370               {
    371                 throw new BaseException("Expected 2 lines with data, not " + dataLines);
    372               }
    373             }         
     367            throw new BaseException("Expected " + columnCount +
     368                " data columns, not " + row.length);
    374369          }
    375370        }
    376       );
    377 
     371      }
     372     
     373      if (columnCount != 4)
     374      {
     375        throw new BaseException("Expected 4 header columns, not " + headers.length);
     376      }
     377     
     378      if (rowCount != 2)
     379      {
     380        throw new BaseException("Expected 2 lines with data, not " + rowCount);
     381      }
     382     
     383      if (bfs.getRowId(0) != 1)
     384      {
     385        throw new BaseException("Unexpected row id at line 0: " + bfs.getRowId(0));
     386      }
     387      if (bfs.getRowIndex(2) != 1)
     388      {
     389        throw new BaseException("Unexpected row indx for row id '2': " + bfs.getRowIndex(2));
     390      }
     391     
    378392      write("--Test BFS Annotation parser OK");
    379393    }
    … …  
    386400  }
    387401 
     402  static void test_write_data(File file)
     403  {
     404    if (file == null) return;
     405    DataWriter bfs = null;
     406    try
     407    {
     408      Writer out = new FileWriter(file);
     409      bfs = new DataWriter(out);
     410     
     411      bfs.setColumnCount(4);
     412     
     413
     414      // Two valid lines with data
     415      bfs.bfsPrintData(1, "item-1", "The first item", 55.3);
     416      bfs.bfsPrintData(2, "item-2", "The second item has a <\t> (tab) and <\n> (linebreak)", -44);
     417     
     418      // Incorrect number of columns
     419      try
     420      {
     421        bfs.bfsPrintData(3, "item-3", "The third line is missing a column");
     422        throw new BaseException("DataWriter did not detect incorrect number of columns");
     423      }
     424      catch (IllegalArgumentException ex)
     425      {} // Expected
     426     
     427      bfs.flush();
     428      bfs.close();
     429      write("--Test BFS Data writer OK");
     430    }
     431    catch (Throwable ex)
     432    {
     433      write("--Test BFS Data writer FAILED: " + file);
     434      ex.printStackTrace();
     435      ok = false;
     436    }
     437    finally
     438    {
     439      if (bfs != null)
     440      {
     441        bfs.flush();
     442        bfs.close();
     443      }
     444    }
     445  }
     446
     447  static void test_read_data(File file)
     448  {
     449    if (file == null) return;
     450    final boolean log = !TestUtil.getSilent();
     451    if (log) write("  Parsing BFS Data: " + file);
     452    try
     453    {
     454      DataParser bfsParser = new DataParser();
     455      MatrixModel bfs = new MatrixModel();
     456      bfsParser.parse(new FileInputStream(file), bfs);
     457
     458      int rowCount = bfs.getRowCount();
     459      int columnCount = bfs.getColumnCount();
     460     
     461      if (log)
     462      {
     463        write("  Rows: " + rowCount + "; columns: " + columnCount);
     464       
     465        for (int r = 0; r < rowCount; r++)
     466        {
     467          String[] row = bfs.getRowByIndex(r);
     468          write("  Line " + r + ": " +
     469              Values.getString(Arrays.asList(row), "\t", false));
     470          if (row.length != columnCount)
     471          {
     472            throw new BaseException("Expected " + columnCount +
     473                " data columns, not " + row.length);
     474          }
     475        }
     476      }
     477     
     478      if (columnCount != 4)
     479      {
     480        throw new BaseException("Expected 4 data columns, not " + columnCount);
     481      }
     482     
     483      if (rowCount != 2)
     484      {
     485        throw new BaseException("Expected 2 lines with data, not " + rowCount);
     486      }
     487           
     488      write("--Test BFS Data parser OK");
     489    }
     490    catch (Throwable ex)
     491    {
     492      write("--Test BFS Data parser FAILED: " + file);
     493      ex.printStackTrace();
     494      ok = false;
     495    }
     496  }
     497
     498 
    388499  static void write(String message)
    389500  {
Note: See TracChangeset for help on using the changeset viewer.