Skip to content

H
h2database
提交 d64f9ca9 authored 作者: Thomas Mueller's avatar Thomas Mueller
浏览文件
操作

CSVREAD / CSVWRITE: instead of setting the options one by one, all options can… 

CSVREAD / CSVWRITE: instead of setting the options one by one, all options can be combined into a space separated key-value pairs.
上级 a9f0448f
显示空白字符变更
内嵌 并排
正在显示 包含 234 行增加 和 54 行删除
h2/src/docsrc/help/help.csv
浏览文件 @ d64f9ca9
......@@ -1651,10 +1651,21 @@ CONSTRAINT CONST_ID
"Other Grammar","Csv Options","
charsetString [, fieldSepString [, fieldDelimString [, escString [, nullString]]]]]
| optionString
","
Optional parameters for CSVREAD and CSVWRITE.
Instead of setting the options one by one, all options can be
combined into a space separated key-value pairs, as follows:
'charset=UTF-8 escape="" fieldDelimiter="" fieldSeparator=, ' ||
'lineComment=# lineSeparator=\n null= rowSeparator='.
The following options are supported:
charset, escape, fieldDelimiter, fieldSeparator,
lineComment (# for H2 version 1.2, disabled for H2 version 1.3),
lineSeparator, null, rowSeparator (not set by default).
The options text is encoded like a Java string.
","
CALL CSVWRITE('test2.csv', 'SELECT * FROM TEST', 'UTF-8', '|');
CALL CSVWRITE('test2.csv', 'SELECT * FROM TEST', 'charset=UTF-8 fieldSeparator=|');
"
"Other Grammar","Data Type","
......
h2/src/main/org/h2/bnf/Bnf.java
浏览文件 @ d64f9ca9
......@@ -72,10 +72,12 @@ public class Bnf {
return head;
}
private void parse(Reader csv) throws SQLException, IOException {
private void parse(Reader reader) throws SQLException, IOException {
Rule functions = null;
statements = New.arrayList();
ResultSet rs = Csv.getInstance().read(csv, null);
Csv csv = Csv.getInstance();
csv.setLineCommentCharacter('#');
ResultSet rs = csv.read(reader, null);
for (int id = 0; rs.next(); id++) {
String section = rs.getString("SECTION").trim();
if (section.startsWith("System")) {
......
h2/src/main/org/h2/expression/Function.java
浏览文件 @ d64f9ca9
......@@ -1046,15 +1046,21 @@ public class Function extends Expression implements FunctionCall {
case CSVREAD: {
String fileName = v0.getString();
String columnList = v1 == null ? null : v1.getString();
String charset = v2 == null ? null : v2.getString();
Csv csv = Csv.getInstance();
String options = v2 == null ? null : v2.getString();
String charset = null;
if (options != null && options.indexOf('=') >= 0) {
charset = csv.setOptions(options);
} else {
charset = options;
String fieldSeparatorRead = v3 == null ? null : v3.getString();
String fieldDelimiter = v4 == null ? null : v4.getString();
String escapeCharacter = v5 == null ? null : v5.getString();
Value v6 = getNullOrValue(session, argList, 6);
String nullString = v6 == null ? null : v6.getString();
Csv csv = Csv.getInstance();
setCsvDelimiterEscape(csv, fieldSeparatorRead, fieldDelimiter, escapeCharacter);
csv.setNullString(nullString);
}
char fieldSeparator = csv.getFieldSeparatorRead();
String[] columns = StringUtils.arraySplit(columnList, fieldSeparator, true);
try {
......@@ -1076,7 +1082,13 @@ public class Function extends Expression implements FunctionCall {
case CSVWRITE: {
session.getUser().checkAdmin();
Connection conn = session.createConnection(false);
String charset = v2 == null ? null : v2.getString();
Csv csv = Csv.getInstance();
String options = v2 == null ? null : v2.getString();
String charset = null;
if (options != null && options.indexOf('=') >= 0) {
charset = csv.setOptions(options);
} else {
charset = options;
String fieldSeparatorWrite = v3 == null ? null : v3.getString();
String fieldDelimiter = v4 == null ? null : v4.getString();
String escapeCharacter = v5 == null ? null : v5.getString();
......@@ -1084,12 +1096,12 @@ public class Function extends Expression implements FunctionCall {
String nullString = v6 == null ? null : v6.getString();
Value v7 = getNullOrValue(session, argList, 7);
String lineSeparator = v7 == null ? null : v7.getString();
Csv csv = Csv.getInstance();
setCsvDelimiterEscape(csv, fieldSeparatorWrite, fieldDelimiter, escapeCharacter);
csv.setNullString(nullString);
if (lineSeparator != null) {
csv.setLineSeparator(lineSeparator);
}
}
try {
int rows = csv.write(conn, v0.getString(), v1.getString(), charset);
result = ValueInt.get(rows);
......@@ -1900,12 +1912,18 @@ public class Function extends Expression implements FunctionCall {
throw DbException.get(ErrorCode.PARAMETER_NOT_SET_1, "fileName");
}
String columnList = argList.length < 2 ? null : argList[1].getValue(session).getString();
String charset = argList.length < 3 ? null : argList[2].getValue(session).getString();
Csv csv = Csv.getInstance();
String options = argList.length < 3 ? null : argList[2].getValue(session).getString();
String charset = null;
if (options != null && options.indexOf('=') >= 0) {
charset = csv.setOptions(options);
} else {
charset = options;
String fieldSeparatorRead = argList.length < 4 ? null : argList[3].getValue(session).getString();
String fieldDelimiter = argList.length < 5 ? null : argList[4].getValue(session).getString();
String escapeCharacter = argList.length < 6 ? null : argList[5].getValue(session).getString();
Csv csv = Csv.getInstance();
setCsvDelimiterEscape(csv, fieldSeparatorRead, fieldDelimiter, escapeCharacter);
}
char fieldSeparator = csv.getFieldSeparatorRead();
String[] columns = StringUtils.arraySplit(columnList, fieldSeparator, true);
ResultSet rs = null;
......
h2/src/main/org/h2/res/help.csv
浏览文件 @ d64f9ca9
......@@ -546,6 +546,7 @@ CONSTRAINT [ IF NOT EXISTS ] newConstraintName
Defines a constraint name."
"Other Grammar","Csv Options","
charsetString [, fieldSepString [, fieldDelimString [, escString [, nullString]]]]]
| optionString
","
Optional parameters for CSVREAD and CSVWRITE."
"Other Grammar","Data Type","
......
h2/src/main/org/h2/table/MetaTable.java
浏览文件 @ d64f9ca9
......@@ -934,7 +934,9 @@ public class MetaTable extends Table {
try {
byte[] data = Utils.getResource(resource);
Reader reader = new InputStreamReader(new ByteArrayInputStream(data));
ResultSet rs = Csv.getInstance().read(reader, null);
Csv csv = Csv.getInstance();
csv.setLineCommentCharacter('#');
ResultSet rs = csv.read(reader, null);
for (int i = 0; rs.next(); i++) {
add(rows,
// ID
......
h2/src/main/org/h2/tools/Csv.java
浏览文件 @ d64f9ca9
......@@ -32,6 +32,7 @@ import org.h2.message.DbException;
import org.h2.util.IOUtils;
import org.h2.util.JdbcUtils;
import org.h2.util.New;
import org.h2.util.StringUtils;
/**
* A facility to read from and write to CSV (comma separated values) files. When
......@@ -42,16 +43,22 @@ import org.h2.util.New;
*/
public class Csv implements SimpleRowSource {
private String streamCharset = SysProperties.FILE_ENCODING;
private String[] columnNames;
private String characterSet = SysProperties.FILE_ENCODING;
private char escapeCharacter = '\"';
private char fieldDelimiter = '\"';
private char fieldSeparatorRead = ',';
private char commentLineStart = '#';
private String fieldSeparatorWrite = ",";
private String rowSeparatorWrite;
private char fieldDelimiter = '\"';
private char escapeCharacter = '\"';
// TODO change the docs at setLineCommentCharacter
// TODO also change help.csv
private char lineComment = Constants.VERSION_MINOR == 3 ? 0 : '#';
private String lineSeparator = SysProperties.LINE_SEPARATOR;
private String nullString = "";
private String rowSeparatorWrite;
private String fileName;
private Reader input;
private char[] inputBuffer;
......@@ -218,25 +225,28 @@ public class Csv implements SimpleRowSource {
private void makeColumnNamesUnique() {
for (int i = 0; i < columnNames.length; i++) {
String x = columnNames[i];
if (x == null || x.length() == 0) {
x = "C" + (i + 1);
StringBuilder buff = new StringBuilder();
String n = columnNames[i];
if (n == null || n.length() == 0) {
buff.append('C').append(i + 1);
} else {
buff.append(n);
}
for (int j = 0; j < i; j++) {
String y = columnNames[j];
if (x.equals(y)) {
x += "1";
if (buff.toString().equals(y)) {
buff.append('1');
j = -1;
}
}
columnNames[i] = x;
columnNames[i] = buff.toString();
}
}
private void init(String newFileName, String charset) {
this.fileName = newFileName;
if (charset != null) {
this.streamCharset = charset;
this.characterSet = charset;
}
}
......@@ -245,7 +255,7 @@ public class Csv implements SimpleRowSource {
try {
OutputStream out = IOUtils.openFileOutputStream(fileName, false);
out = new BufferedOutputStream(out, Constants.IO_BUFFER_SIZE);
output = new BufferedWriter(new OutputStreamWriter(out, streamCharset));
output = new BufferedWriter(new OutputStreamWriter(out, characterSet));
} catch (Exception e) {
close();
throw DbException.convertToIOException(e);
......@@ -306,7 +316,7 @@ public class Csv implements SimpleRowSource {
try {
InputStream in = IOUtils.openFileInputStream(fileName);
in = new BufferedInputStream(in, Constants.IO_BUFFER_SIZE);
input = new InputStreamReader(in, streamCharset);
input = new InputStreamReader(in, characterSet);
} catch (IOException e) {
close();
throw e;
......@@ -481,7 +491,7 @@ public class Csv implements SimpleRowSource {
} else if (ch <= ' ') {
// ignore spaces
continue;
} else if (ch == commentLineStart) {
} else if (lineComment != 0 && ch == lineComment) {
// comment until end of line
inputBufferStart = -1;
while (true) {
......@@ -581,11 +591,6 @@ public class Csv implements SimpleRowSource {
private SQLException convertException(String message, Exception e) {
return DbException.get(ErrorCode.IO_EXCEPTION_1, e, message).getSQLException();
// SQLException s = new SQLException(message, "CSV");
// //## Java 1.4 begin ##
// s.initCause(e);
// //## Java 1.4 end ##
// return s;
}
/**
......@@ -661,6 +666,25 @@ public class Csv implements SimpleRowSource {
this.rowSeparatorWrite = rowSeparatorWrite;
}
/**
* Set the line comment character. The default is character code 0 (line
* comments are disabled) for H2 version 1.3, and '#' for H2 version 1.2.
*
* @param lineCommentCharacter the line comment character
*/
public void setLineCommentCharacter(char lineCommentCharacter) {
this.lineComment = lineCommentCharacter;
}
/**
* Get the line comment character.
*
* @return the line comment character, or 0 if disabled
*/
public char getLineCommentCharacter() {
return lineComment;
}
/**
* Set the field delimiter. The default is " (a double quote).
* The value 0 means no field delimiter is used.
......@@ -728,6 +752,15 @@ public class Csv implements SimpleRowSource {
this.lineSeparator = lineSeparator;
}
/**
* Get the current line separator.
*
* @return the line separator
*/
public String getLineSeparator() {
return lineSeparator;
}
/**
* Set the value that represents NULL.
*
......@@ -746,4 +779,53 @@ public class Csv implements SimpleRowSource {
return nullString;
}
/**
* INTERNAL.
* Parse and set the CSV options.
*
* @param options the the options
* @return the character set
*/
public String setOptions(String options) {
String charset = null;
options = StringUtils.javaDecode(options);
String[] keyValuePairs = StringUtils.arraySplit(options, ' ', false);
for (String pair : keyValuePairs) {
int index = pair.indexOf('=');
String key = StringUtils.trim(pair.substring(0, index), true, true, " ");
String value = StringUtils.trim(pair.substring(index + 1), true, true, " ");
char ch = value.length() == 0 ? 0 : value.charAt(0);
if (isParam(key, "escape", "esc", "escapeCharacter")) {
setEscapeCharacter(ch);
} else if (isParam(key, "fieldDelimiter", "fieldDelim")) {
setFieldDelimiter(ch);
} else if (isParam(key, "fieldSeparator", "fieldSep")) {
setFieldSeparatorRead(ch);
setFieldSeparatorWrite(value);
} else if (isParam(key, "lineComment", "lineCommentCharacter")) {
setLineCommentCharacter(ch);
} else if (isParam(key, "lineSeparator", "lineSep")) {
setLineSeparator(value);
} else if (isParam(key, "null", "nullString")) {
setNullString(value);
} else if (isParam(key, "rowSeparator", "rowSep")) {
setRowSeparatorWrite(value);
} else if (isParam(key, "charset", "characterSet")) {
charset = value;
} else {
throw DbException.get(ErrorCode.UNSUPPORTED_SETTING_1, key);
}
}
return charset;
}
private boolean isParam(String key, String... values) {
for (String v : values) {
if (key.equalsIgnoreCase(v)) {
return true;
}
}
return false;
}
}
h2/src/test/org/h2/test/db/TestCsv.java
浏览文件 @ d64f9ca9
......@@ -20,6 +20,10 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Random;
import org.h2.constant.ErrorCode;
import org.h2.constant.SysProperties;
import org.h2.engine.Constants;
import org.h2.message.DbException;
import org.h2.store.fs.FileObject;
import org.h2.store.fs.FileSystem;
import org.h2.test.TestBase;
......@@ -49,6 +53,7 @@ public class TestCsv extends TestBase {
}
public void test() throws Exception {
testOptions();
testPseudoBom();
testWriteRead();
testColumnNames();
......@@ -63,6 +68,53 @@ public class TestCsv extends TestBase {
deleteDb("csv");
}
private void testOptions() {
Csv csv = Csv.getInstance();
assertEquals(",", csv.getFieldSeparatorWrite());
assertEquals(SysProperties.LINE_SEPARATOR, csv.getLineSeparator());
assertEquals("", csv.getNullString());
assertEquals(null, csv.getRowSeparatorWrite());
assertEquals('\"', csv.getEscapeCharacter());
assertEquals('"', csv.getFieldDelimiter());
assertEquals(',', csv.getFieldSeparatorRead());
assertEquals(",", csv.getFieldSeparatorWrite());
assertEquals(Constants.VERSION_MINOR == 3 ? 0 : '#', csv.getLineCommentCharacter());
String charset = csv.setOptions("escape=1x fieldDelimiter=2x fieldSeparator=3x " +
"lineComment=4x lineSeparator=5x " +
"null=6x rowSeparator=7x charset=8x");
assertEquals('1', csv.getEscapeCharacter());
assertEquals('2', csv.getFieldDelimiter());
assertEquals('3', csv.getFieldSeparatorRead());
assertEquals("3x", csv.getFieldSeparatorWrite());
assertEquals('4', csv.getLineCommentCharacter());
assertEquals("5x", csv.getLineSeparator());
assertEquals("6x", csv.getNullString());
assertEquals("7x", csv.getRowSeparatorWrite());
assertEquals("8x", charset);
charset = csv.setOptions("escape= fieldDelimiter= fieldSeparator= " +
"lineComment= lineSeparator=\r\n " +
"null=\0 rowSeparator= charset=");
assertEquals(0, csv.getEscapeCharacter());
assertEquals(0, csv.getFieldDelimiter());
assertEquals(0, csv.getFieldSeparatorRead());
assertEquals("", csv.getFieldSeparatorWrite());
assertEquals(0, csv.getLineCommentCharacter());
assertEquals("\r\n", csv.getLineSeparator());
assertEquals("\0", csv.getNullString());
assertEquals("", csv.getRowSeparatorWrite());
assertEquals("", charset);
try {
csv.setOptions("escape=a error=b");
fail();
} catch (DbException e) {
assertEquals(ErrorCode.UNSUPPORTED_SETTING_1, e.getErrorCode());
assertEquals('a', csv.getEscapeCharacter());
}
}
private void testPseudoBom() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
// UTF-8 "BOM" / marker
......@@ -86,6 +138,14 @@ public class TestCsv extends TestBase {
assertEquals("First Name", rs.getMetaData().getColumnName(2));
assertEquals("2x", rs.getMetaData().getColumnName(3));
assertEquals("_X2", rs.getMetaData().getColumnName(4));
rs = Csv.getInstance().read(new StringReader("a,a\n1,2"), null);
assertEquals("A", rs.getMetaData().getColumnName(1));
assertEquals("A1", rs.getMetaData().getColumnName(2));
rs = Csv.getInstance().read(new StringReader("1,2"), new String[]{"", null});
assertEquals("C1", rs.getMetaData().getColumnName(1));
assertEquals("C2", rs.getMetaData().getColumnName(2));
}
private void testSpaceSeparated() throws SQLException {
......
h2/src/tools/org/h2/build/doc/GenerateDoc.java
浏览文件 @ d64f9ca9
......@@ -68,7 +68,7 @@ public class GenerateDoc {
session.put("stableVersion", Constants.getVersionStable());
session.put("stableVersionDate", Constants.BUILD_DATE_STABLE);
// String help = "SELECT * FROM INFORMATION_SCHEMA.HELP WHERE SECTION";
String help = "SELECT ROWNUM ID, * FROM CSVREAD('" + inHelp + "') WHERE SECTION ";
String help = "SELECT ROWNUM ID, * FROM CSVREAD('" + inHelp + "', NULL, 'lineComment=#') WHERE SECTION ";
map("commands", help + "LIKE 'Commands%' ORDER BY ID", true);
map("commandsDML", help + "= 'Commands (DML)' ORDER BY ID", false);
map("commandsDDL", help + "= 'Commands (DDL)' ORDER BY ID", false);
......
h2/src/tools/org/h2/build/doc/GenerateHelp.java
浏览文件 @ d64f9ca9
......@@ -32,7 +32,9 @@ public class GenerateHelp {
private void run() throws Exception {
String in = "src/docsrc/help/help.csv";
String out = "src/main/org/h2/res/help.csv";
ResultSet rs = Csv.getInstance().read(in, null, null);
Csv csv = Csv.getInstance();
csv.setLineCommentCharacter('#');
ResultSet rs = csv.read(in, null, null);
SimpleResultSet rs2 = new SimpleResultSet();
ResultSetMetaData meta = rs.getMetaData();
int columnCount = meta.getColumnCount() - 1;
......@@ -58,7 +60,7 @@ public class GenerateHelp {
"# Version 1.0, and under the Eclipse Public License, Version 1.0\n" +
"# (http://h2database.com/html/license.html).\n" +
"# Initial Developer: H2 Group)\n");
Csv csv = Csv.getInstance();
csv = Csv.getInstance();
csv.setLineSeparator("\n");
csv.write(writer, rs2);
}
......
h2/src/tools/org/h2/jcr/Railroads.java
浏览文件 @ d64f9ca9
......@@ -47,7 +47,9 @@ public class Railroads {
private void process() throws Exception {
RailroadImages.main();
bnf = Bnf.getInstance(getReader());
ResultSet rs = Csv.getInstance().read(getReader(), null);
Csv csv = Csv.getInstance();
csv.setLineCommentCharacter('#');
ResultSet rs = csv.read(getReader(), null);
map("grammar", rs, true);
processHtml("jcr-sql2.html");
}
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论