You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@carbondata.apache.org by "Praveen M P (JIRA)" <ji...@apache.org> on 2018/05/09 12:03:00 UTC
[jira] [Created] (CARBONDATA-2462) Carbon writer fails when the
record is given from avro file(binary)
Praveen M P created CARBONDATA-2462:
---------------------------------------
Summary: Carbon writer fails when the record is given from avro file(binary)
Key: CARBONDATA-2462
URL: https://issues.apache.org/jira/browse/CARBONDATA-2462
Project: CarbonData
Issue Type: Bug
Reporter: Praveen M P
public class AvroCarbonWrte { static String avroschema = readFileAndConvertoStringObject("D:/Carbon/sdk/avro.json");
static Schema parsedAschema = new org.apache.avro.Schema.Parser().parse(avroschema);
static Properties properties = new Properties();
public static void main(String[] args) throws Exception {
String path = "conf.properties";
if (args.length > 0) {
path = args[0];
}
properties.load(new FileReader(path));
// readOneResult();
//writerTest("D:/Carbon/sdk/avrodata/ok.avro");
readTest("D:/Carbon/sdk/avrodata/ok.avro");
//dataGenerator("D:/Carbon/sdk/avrodata/simuldata/simu.avro");
}
public static void readTest(String path) throws IOException, InvalidLoadOptionException {
File file = new File(path);
GenericDatumReader<GenericRecord> genericRecordGenericDatumReader = new GenericDatumReader<GenericRecord>(parsedAschema);
DataFileReader<GenericRecord> genericRecords = new DataFileReader<>(file, genericRecordGenericDatumReader);
//Field[] fields = new Array[Field](8);
Field[] fields = new Field[8];
fields[0] = new Field("timeStamp", DataTypes.LONG);
fields[1] = new Field("vin", DataTypes.STRING);
fields[2] = new Field("event", DataTypes.STRING);
ArrayList<StructField> fld_s = new ArrayList<StructField>();
fld_s.add(new StructField("valueType", DataTypes.STRING));
fld_s.add(new StructField("status", DataTypes.STRING));
fld_s.add(new StructField("code", DataTypes.STRING));
fld_s.add(new StructField("name_en", DataTypes.STRING));
fld_s.add(new StructField("name", DataTypes.STRING));
fld_s.add(new StructField("value", DataTypes.STRING));
ArrayList<StructField> fld_s_n = new ArrayList<StructField>();
fld_s_n.add(new StructField("signals", DataTypes.createStructType(fld_s)));
fields[3] = new Field("signals", "array", fld_s_n);
fields[4] = new Field("metaData", DataTypes.STRING);
fields[5] = new Field("orginalReport", DataTypes.STRING);
fields[6] = new Field("uploadType", DataTypes.STRING);
fields[7] = new Field("uploadTime", DataTypes.LONG);
while (genericRecords.hasNext()) {
GenericRecord next = genericRecords.next();
byte[] bytes = new JsonAvroConverter().convertToJson(next);
System.out.println(next);
// CarbonWriter writer = CarbonWriter.builder().withSchema(new org.apache.carbondata.sdk.file.Schema(fields)).outputPath(path).buildWriterForAvroInput();
CarbonWriter writer = CarbonWriter.builder().withSchema(new org.apache.carbondata.sdk.file.Schema(fields))
.outputPath("D:/Carbon/sdk/avro.carbon").isTransactionalTable(false).buildWriterForAvroInput();
// GenericData.Record record = new JsonAvroConverter().convertToGenericDataRecord(bytes, parsedAschema);
writer.write(next);
writer.close();
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)