CSVREAD / CSVWRITE: instead of setting the options one by one, all options can…

CSVREAD / CSVWRITE: instead of setting the options one by one, all options can be combined into a space separated key-value pairs.

h2/src/docsrc/help/help.csv

@@ -124,7 +124,7 @@ statements; each statement must end with ';'. This command can be used to
 restore a database from a backup. The password must be in single quotes; it is
 case sensitive and can contain spaces.
 
-Instead of a file name, an URL may be used. 
+Instead of a file name, an URL may be used.
 To read a stream from the classpath, use the prefix 'classpath:'.
 
 The compression algorithm must match the one used when creating the script. When
@@ -1651,10 +1651,21 @@ CONSTRAINT CONST_ID
 
 "Other Grammar","Csv Options","
 charsetString [, fieldSepString [, fieldDelimString [, escString [, nullString]]]]]
+    | optionString
 ","
 Optional parameters for CSVREAD and CSVWRITE.
+Instead of setting the options one by one, all options can be
+combined into a space separated key-value pairs, as follows:
+'charset=UTF-8 escape="" fieldDelimiter="" fieldSeparator=, ' ||
+'lineComment=# lineSeparator=\n null= rowSeparator='.
+The following options are supported:
+charset, escape, fieldDelimiter, fieldSeparator,
+lineComment (# for H2 version 1.2, disabled for H2 version 1.3),
+lineSeparator, null, rowSeparator (not set by default).
+The options text is encoded like a Java string.
 ","
 CALL CSVWRITE('test2.csv', 'SELECT * FROM TEST', 'UTF-8', '|');
+CALL CSVWRITE('test2.csv', 'SELECT * FROM TEST', 'charset=UTF-8 fieldSeparator=|');
 "
 
 "Other Grammar","Data Type","
@@ -3375,7 +3386,7 @@ FILE_READ(fileNameString [,encodingString])
 Returns the contents of a file. If only one parameter is supplied, the data are
 returned as a BLOB. If two parameters are used, the data is returned as a CLOB
 (text). The second parameter is the character set to use, NULL meaning the
-default character set for this system. 
+default character set for this system.
 
 File names and URLs are supported.
 To read a stream from the classpath, use the prefix ""classpath:"".

h2/src/main/org/h2/bnf/Bnf.java

@@ -72,10 +72,12 @@ public class Bnf {
         return head;
     }
 
-    private void parse(Reader csv) throws SQLException, IOException {
+    private void parse(Reader reader) throws SQLException, IOException {
         Rule functions = null;
         statements = New.arrayList();
-        ResultSet rs = Csv.getInstance().read(csv, null);
+        Csv csv = Csv.getInstance();
+        csv.setLineCommentCharacter('#');
+        ResultSet rs = csv.read(reader, null);
         for (int id = 0; rs.next(); id++) {
             String section = rs.getString("SECTION").trim();
             if (section.startsWith("System")) {

h2/src/main/org/h2/expression/Function.java

@@ -1046,15 +1046,21 @@ public class Function extends Expression implements FunctionCall {
         case CSVREAD: {
             String fileName = v0.getString();
             String columnList = v1 == null ? null : v1.getString();
-            String charset = v2 == null ? null : v2.getString();
-            String fieldSeparatorRead = v3 == null ? null : v3.getString();
-            String fieldDelimiter = v4 == null ? null : v4.getString();
-            String escapeCharacter = v5 == null ? null : v5.getString();
-            Value v6 = getNullOrValue(session, argList, 6);
-            String nullString = v6 == null ? null : v6.getString();
             Csv csv = Csv.getInstance();
-            setCsvDelimiterEscape(csv, fieldSeparatorRead, fieldDelimiter, escapeCharacter);
-            csv.setNullString(nullString);
+            String options = v2 == null ? null : v2.getString();
+            String charset = null;
+            if (options != null && options.indexOf('=') >= 0) {
+                charset = csv.setOptions(options);
+            } else {
+                charset = options;
+                String fieldSeparatorRead = v3 == null ? null : v3.getString();
+                String fieldDelimiter = v4 == null ? null : v4.getString();
+                String escapeCharacter = v5 == null ? null : v5.getString();
+                Value v6 = getNullOrValue(session, argList, 6);
+                String nullString = v6 == null ? null : v6.getString();
+                setCsvDelimiterEscape(csv, fieldSeparatorRead, fieldDelimiter, escapeCharacter);
+                csv.setNullString(nullString);
+            }
             char fieldSeparator = csv.getFieldSeparatorRead();
             String[] columns = StringUtils.arraySplit(columnList, fieldSeparator, true);
             try {
@@ -1076,19 +1082,25 @@ public class Function extends Expression implements FunctionCall {
         case CSVWRITE: {
             session.getUser().checkAdmin();
             Connection conn = session.createConnection(false);
-            String charset = v2 == null ? null : v2.getString();
-            String fieldSeparatorWrite = v3 == null ? null : v3.getString();
-            String fieldDelimiter = v4 == null ? null : v4.getString();
-            String escapeCharacter = v5 == null ? null : v5.getString();
-            Value v6 = getNullOrValue(session, argList, 6);
-            String nullString = v6 == null ? null : v6.getString();
-            Value v7 = getNullOrValue(session, argList, 7);
-            String lineSeparator = v7 == null ? null : v7.getString();
             Csv csv = Csv.getInstance();
-            setCsvDelimiterEscape(csv, fieldSeparatorWrite, fieldDelimiter, escapeCharacter);
-            csv.setNullString(nullString);
-            if (lineSeparator != null) {
-                csv.setLineSeparator(lineSeparator);
+            String options = v2 == null ? null : v2.getString();
+            String charset = null;
+            if (options != null && options.indexOf('=') >= 0) {
+                charset = csv.setOptions(options);
+            } else {
+                charset = options;
+                String fieldSeparatorWrite = v3 == null ? null : v3.getString();
+                String fieldDelimiter = v4 == null ? null : v4.getString();
+                String escapeCharacter = v5 == null ? null : v5.getString();
+                Value v6 = getNullOrValue(session, argList, 6);
+                String nullString = v6 == null ? null : v6.getString();
+                Value v7 = getNullOrValue(session, argList, 7);
+                String lineSeparator = v7 == null ? null : v7.getString();
+                setCsvDelimiterEscape(csv, fieldSeparatorWrite, fieldDelimiter, escapeCharacter);
+                csv.setNullString(nullString);
+                if (lineSeparator != null) {
+                    csv.setLineSeparator(lineSeparator);
+                }
             }
             try {
                 int rows = csv.write(conn, v0.getString(), v1.getString(), charset);
@@ -1900,12 +1912,18 @@ public class Function extends Expression implements FunctionCall {
                 throw DbException.get(ErrorCode.PARAMETER_NOT_SET_1, "fileName");
             }
             String columnList = argList.length < 2 ? null : argList[1].getValue(session).getString();
-            String charset = argList.length < 3 ? null : argList[2].getValue(session).getString();
-            String fieldSeparatorRead = argList.length < 4 ? null : argList[3].getValue(session).getString();
-            String fieldDelimiter = argList.length < 5 ? null : argList[4].getValue(session).getString();
-            String escapeCharacter = argList.length < 6 ? null : argList[5].getValue(session).getString();
             Csv csv = Csv.getInstance();
-            setCsvDelimiterEscape(csv, fieldSeparatorRead, fieldDelimiter, escapeCharacter);
+            String options = argList.length < 3 ? null : argList[2].getValue(session).getString();
+            String charset = null;
+            if (options != null && options.indexOf('=') >= 0) {
+                charset = csv.setOptions(options);
+            } else {
+                charset = options;
+                String fieldSeparatorRead = argList.length < 4 ? null : argList[3].getValue(session).getString();
+                String fieldDelimiter = argList.length < 5 ? null : argList[4].getValue(session).getString();
+                String escapeCharacter = argList.length < 6 ? null : argList[5].getValue(session).getString();
+                setCsvDelimiterEscape(csv, fieldSeparatorRead, fieldDelimiter, escapeCharacter);
+            }
             char fieldSeparator = csv.getFieldSeparatorRead();
             String[] columns = StringUtils.arraySplit(columnList, fieldSeparator, true);
             ResultSet rs = null;

h2/src/main/org/h2/res/help.csv

@@ -546,6 +546,7 @@ CONSTRAINT [ IF NOT EXISTS ] newConstraintName
 Defines a constraint name."
 "Other Grammar","Csv Options","
 charsetString [, fieldSepString [, fieldDelimString [, escString [, nullString]]]]]
+    | optionString
 ","
 Optional parameters for CSVREAD and CSVWRITE."
 "Other Grammar","Data Type","

h2/src/main/org/h2/table/MetaTable.java

@@ -934,7 +934,9 @@ public class MetaTable extends Table {
             try {
                 byte[] data = Utils.getResource(resource);
                 Reader reader = new InputStreamReader(new ByteArrayInputStream(data));
-                ResultSet rs = Csv.getInstance().read(reader, null);
+                Csv csv = Csv.getInstance();
+                csv.setLineCommentCharacter('#');
+                ResultSet rs = csv.read(reader, null);
                 for (int i = 0; rs.next(); i++) {
                     add(rows,
                         // ID

h2/src/main/org/h2/tools/Csv.java

@@ -32,6 +32,7 @@ import org.h2.message.DbException;
 import org.h2.util.IOUtils;
 import org.h2.util.JdbcUtils;
 import org.h2.util.New;
+import org.h2.util.StringUtils;
 
 /**
  * A facility to read from and write to CSV (comma separated values) files. When
@@ -42,16 +43,22 @@ import org.h2.util.New;
  */
 public class Csv implements SimpleRowSource {
 
-    private String streamCharset = SysProperties.FILE_ENCODING;
     private String[] columnNames;
+
+    private String characterSet = SysProperties.FILE_ENCODING;
+    private char escapeCharacter = '\"';
+    private char fieldDelimiter = '\"';
     private char fieldSeparatorRead = ',';
-    private char commentLineStart = '#';
     private String fieldSeparatorWrite = ",";
-    private String rowSeparatorWrite;
-    private char fieldDelimiter = '\"';
-    private char escapeCharacter = '\"';
+
+    // TODO change the docs at setLineCommentCharacter
+    // TODO also change help.csv
+    private char lineComment = Constants.VERSION_MINOR == 3 ? 0 : '#';
+
     private String lineSeparator = SysProperties.LINE_SEPARATOR;
     private String nullString = "";
+
+    private String rowSeparatorWrite;
     private String fileName;
     private Reader input;
     private char[] inputBuffer;
@@ -218,25 +225,28 @@ public class Csv implements SimpleRowSource {
 
     private void makeColumnNamesUnique() {
         for (int i = 0; i < columnNames.length; i++) {
-            String x = columnNames[i];
-            if (x == null || x.length() == 0) {
-                x = "C" + (i + 1);
+            StringBuilder buff = new StringBuilder();
+            String n = columnNames[i];
+            if (n == null || n.length() == 0) {
+                buff.append('C').append(i + 1);
+            } else {
+                buff.append(n);
             }
             for (int j = 0; j < i; j++) {
                 String y = columnNames[j];
-                if (x.equals(y)) {
-                    x += "1";
+                if (buff.toString().equals(y)) {
+                    buff.append('1');
                     j = -1;
                 }
             }
-            columnNames[i] = x;
+            columnNames[i] = buff.toString();
         }
     }
 
     private void init(String newFileName, String charset) {
         this.fileName = newFileName;
         if (charset != null) {
-            this.streamCharset = charset;
+            this.characterSet = charset;
         }
     }
 
@@ -245,7 +255,7 @@ public class Csv implements SimpleRowSource {
             try {
                 OutputStream out = IOUtils.openFileOutputStream(fileName, false);
                 out = new BufferedOutputStream(out, Constants.IO_BUFFER_SIZE);
-                output = new BufferedWriter(new OutputStreamWriter(out, streamCharset));
+                output = new BufferedWriter(new OutputStreamWriter(out, characterSet));
             } catch (Exception e) {
                 close();
                 throw DbException.convertToIOException(e);
@@ -306,7 +316,7 @@ public class Csv implements SimpleRowSource {
             try {
                 InputStream in = IOUtils.openFileInputStream(fileName);
                 in = new BufferedInputStream(in, Constants.IO_BUFFER_SIZE);
-                input = new InputStreamReader(in, streamCharset);
+                input = new InputStreamReader(in, characterSet);
             } catch (IOException e) {
                 close();
                 throw e;
@@ -481,7 +491,7 @@ public class Csv implements SimpleRowSource {
             } else if (ch <= ' ') {
                 // ignore spaces
                 continue;
-            } else if (ch == commentLineStart) {
+            } else if (lineComment != 0 && ch == lineComment) {
                 // comment until end of line
                 inputBufferStart = -1;
                 while (true) {
@@ -581,11 +591,6 @@ public class Csv implements SimpleRowSource {
 
     private SQLException convertException(String message, Exception e) {
         return DbException.get(ErrorCode.IO_EXCEPTION_1, e, message).getSQLException();
-//        SQLException s = new SQLException(message, "CSV");
-//        //## Java 1.4 begin ##
-//        s.initCause(e);
-//        //## Java 1.4 end ##
-//        return s;
     }
 
     /**
@@ -661,6 +666,25 @@ public class Csv implements SimpleRowSource {
         this.rowSeparatorWrite = rowSeparatorWrite;
     }
 
+    /**
+     * Set the line comment character. The default is character code 0 (line
+     * comments are disabled) for H2 version 1.3, and '#' for H2 version 1.2.
+     *
+     * @param lineCommentCharacter the line comment character
+     */
+    public void setLineCommentCharacter(char lineCommentCharacter) {
+        this.lineComment = lineCommentCharacter;
+    }
+
+    /**
+     * Get the line comment character.
+     *
+     * @return the line comment character, or 0 if disabled
+     */
+    public char getLineCommentCharacter() {
+        return lineComment;
+    }
+
     /**
      * Set the field delimiter. The default is " (a double quote).
      * The value 0 means no field delimiter is used.
@@ -728,6 +752,15 @@ public class Csv implements SimpleRowSource {
         this.lineSeparator = lineSeparator;
     }
 
+    /**
+     * Get the current line separator.
+     *
+     * @return the line separator
+     */
+    public String getLineSeparator() {
+        return lineSeparator;
+    }
+
     /**
      * Set the value that represents NULL.
      *
@@ -746,4 +779,53 @@ public class Csv implements SimpleRowSource {
         return nullString;
     }
 
+    /**
+     * INTERNAL.
+     * Parse and set the CSV options.
+     *
+     * @param options the the options
+     * @return the character set
+     */
+    public String setOptions(String options) {
+        String charset = null;
+        options = StringUtils.javaDecode(options);
+        String[] keyValuePairs = StringUtils.arraySplit(options, ' ', false);
+        for (String pair : keyValuePairs) {
+            int index = pair.indexOf('=');
+            String key = StringUtils.trim(pair.substring(0, index), true, true, " ");
+            String value = StringUtils.trim(pair.substring(index + 1), true, true, " ");
+            char ch = value.length() == 0 ? 0 : value.charAt(0);
+            if (isParam(key, "escape", "esc", "escapeCharacter")) {
+                setEscapeCharacter(ch);
+            } else if (isParam(key, "fieldDelimiter", "fieldDelim")) {
+                setFieldDelimiter(ch);
+            } else if (isParam(key, "fieldSeparator", "fieldSep")) {
+                setFieldSeparatorRead(ch);
+                setFieldSeparatorWrite(value);
+            } else if (isParam(key, "lineComment", "lineCommentCharacter")) {
+                setLineCommentCharacter(ch);
+            } else if (isParam(key, "lineSeparator", "lineSep")) {
+                setLineSeparator(value);
+            } else if (isParam(key, "null", "nullString")) {
+                setNullString(value);
+            } else if (isParam(key, "rowSeparator", "rowSep")) {
+                setRowSeparatorWrite(value);
+            } else if (isParam(key, "charset", "characterSet")) {
+                charset = value;
+            } else {
+                throw DbException.get(ErrorCode.UNSUPPORTED_SETTING_1, key);
+            }
+        }
+        return charset;
+    }
+
+    private boolean isParam(String key, String... values) {
+        for (String v : values) {
+            if (key.equalsIgnoreCase(v)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
 }

h2/src/test/org/h2/test/db/TestCsv.java

@@ -20,6 +20,10 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Random;
+import org.h2.constant.ErrorCode;
+import org.h2.constant.SysProperties;
+import org.h2.engine.Constants;
+import org.h2.message.DbException;
 import org.h2.store.fs.FileObject;
 import org.h2.store.fs.FileSystem;
 import org.h2.test.TestBase;
@@ -49,6 +53,7 @@ public class TestCsv extends TestBase {
     }
 
     public void test() throws Exception {
+        testOptions();
         testPseudoBom();
         testWriteRead();
         testColumnNames();
@@ -63,6 +68,53 @@ public class TestCsv extends TestBase {
         deleteDb("csv");
     }
 
+    private void testOptions() {
+        Csv csv = Csv.getInstance();
+        assertEquals(",", csv.getFieldSeparatorWrite());
+        assertEquals(SysProperties.LINE_SEPARATOR, csv.getLineSeparator());
+        assertEquals("", csv.getNullString());
+        assertEquals(null, csv.getRowSeparatorWrite());
+        assertEquals('\"', csv.getEscapeCharacter());
+        assertEquals('"', csv.getFieldDelimiter());
+        assertEquals(',', csv.getFieldSeparatorRead());
+        assertEquals(",", csv.getFieldSeparatorWrite());
+        assertEquals(Constants.VERSION_MINOR == 3 ? 0 : '#', csv.getLineCommentCharacter());
+
+        String charset = csv.setOptions("escape=1x fieldDelimiter=2x fieldSeparator=3x " +
+                "lineComment=4x lineSeparator=5x " +
+                "null=6x rowSeparator=7x charset=8x");
+        assertEquals('1', csv.getEscapeCharacter());
+        assertEquals('2', csv.getFieldDelimiter());
+        assertEquals('3', csv.getFieldSeparatorRead());
+        assertEquals("3x", csv.getFieldSeparatorWrite());
+        assertEquals('4', csv.getLineCommentCharacter());
+        assertEquals("5x", csv.getLineSeparator());
+        assertEquals("6x", csv.getNullString());
+        assertEquals("7x", csv.getRowSeparatorWrite());
+        assertEquals("8x", charset);
+
+        charset = csv.setOptions("escape= fieldDelimiter= fieldSeparator= " +
+                "lineComment= lineSeparator=\r\n " +
+                "null=\0 rowSeparator= charset=");
+        assertEquals(0, csv.getEscapeCharacter());
+        assertEquals(0, csv.getFieldDelimiter());
+        assertEquals(0, csv.getFieldSeparatorRead());
+        assertEquals("", csv.getFieldSeparatorWrite());
+        assertEquals(0, csv.getLineCommentCharacter());
+        assertEquals("\r\n", csv.getLineSeparator());
+        assertEquals("\0", csv.getNullString());
+        assertEquals("", csv.getRowSeparatorWrite());
+        assertEquals("", charset);
+
+        try {
+            csv.setOptions("escape=a error=b");
+            fail();
+        } catch (DbException e) {
+            assertEquals(ErrorCode.UNSUPPORTED_SETTING_1, e.getErrorCode());
+            assertEquals('a', csv.getEscapeCharacter());
+        }
+    }
+
     private void testPseudoBom() throws Exception {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         // UTF-8 "BOM" / marker
@@ -86,6 +138,14 @@ public class TestCsv extends TestBase {
         assertEquals("First Name", rs.getMetaData().getColumnName(2));
         assertEquals("2x", rs.getMetaData().getColumnName(3));
         assertEquals("_X2", rs.getMetaData().getColumnName(4));
+
+        rs = Csv.getInstance().read(new StringReader("a,a\n1,2"), null);
+        assertEquals("A", rs.getMetaData().getColumnName(1));
+        assertEquals("A1", rs.getMetaData().getColumnName(2));
+
+        rs = Csv.getInstance().read(new StringReader("1,2"), new String[]{"", null});
+        assertEquals("C1", rs.getMetaData().getColumnName(1));
+        assertEquals("C2", rs.getMetaData().getColumnName(2));
     }
 
     private void testSpaceSeparated() throws SQLException {

h2/src/tools/org/h2/build/doc/GenerateDoc.java

@@ -68,7 +68,7 @@ public class GenerateDoc {
         session.put("stableVersion", Constants.getVersionStable());
         session.put("stableVersionDate", Constants.BUILD_DATE_STABLE);
         // String help = "SELECT * FROM INFORMATION_SCHEMA.HELP WHERE SECTION";
-        String help = "SELECT ROWNUM ID, * FROM CSVREAD('" + inHelp + "') WHERE SECTION ";
+        String help = "SELECT ROWNUM ID, * FROM CSVREAD('" + inHelp + "', NULL, 'lineComment=#') WHERE SECTION ";
         map("commands", help + "LIKE 'Commands%' ORDER BY ID", true);
         map("commandsDML", help + "= 'Commands (DML)' ORDER BY ID", false);
         map("commandsDDL", help + "= 'Commands (DDL)' ORDER BY ID", false);

h2/src/tools/org/h2/build/doc/GenerateHelp.java

@@ -32,7 +32,9 @@ public class GenerateHelp {
     private void run() throws Exception {
         String in = "src/docsrc/help/help.csv";
         String out = "src/main/org/h2/res/help.csv";
-        ResultSet rs = Csv.getInstance().read(in, null, null);
+        Csv csv = Csv.getInstance();
+        csv.setLineCommentCharacter('#');
+        ResultSet rs = csv.read(in, null, null);
         SimpleResultSet rs2 = new SimpleResultSet();
         ResultSetMetaData meta = rs.getMetaData();
         int columnCount = meta.getColumnCount() - 1;
@@ -58,7 +60,7 @@ public class GenerateHelp {
                 "# Version 1.0, and under the Eclipse Public License, Version 1.0\n" +
                 "# (http://h2database.com/html/license.html).\n" +
                 "# Initial Developer: H2 Group)\n");
-        Csv csv = Csv.getInstance();
+        csv = Csv.getInstance();
         csv.setLineSeparator("\n");
         csv.write(writer, rs2);
     }

h2/src/tools/org/h2/jcr/Railroads.java

@@ -47,7 +47,9 @@ public class Railroads {
     private void process() throws Exception {
         RailroadImages.main();
         bnf = Bnf.getInstance(getReader());
-        ResultSet rs = Csv.getInstance().read(getReader(), null);
+        Csv csv = Csv.getInstance();
+        csv.setLineCommentCharacter('#');
+        ResultSet rs = csv.read(getReader(), null);
         map("grammar", rs, true);
         processHtml("jcr-sql2.html");
     }