You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/02/14 23:26:50 UTC
[1/3] incubator-carbondata git commit: csvReader code improvements
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 25de27fcf -> f3496ee0b
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/processing/src/test/resources/csv/data.csv
----------------------------------------------------------------------
diff --git a/processing/src/test/resources/csv/data.csv b/processing/src/test/resources/csv/data.csv
new file mode 100644
index 0000000..6407b10
--- /dev/null
+++ b/processing/src/test/resources/csv/data.csv
@@ -0,0 +1,1001 @@
+ID,date,country,name,phonetype,serialname,salary
+1,2015/7/23,china,aaa1,phone197,ASD69643,15000
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001
+3,2015/7/25,china,aaa3,phone1904,ASD37014,15002
+4,2015/7/26,china,aaa4,phone2435,ASD66902,15003
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004
+6,2015/7/28,china,aaa6,phone294,ASD59961,15005
+7,2015/7/29,china,aaa7,phone610,ASD14875,15006
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009
+11,2015/7/18,china,aaa11,phone1554,ASD26101,15010
+12,2015/7/19,china,aaa12,phone1781,ASD85711,15011
+13,2015/7/20,china,aaa13,phone943,ASD39200,15012
+14,2015/7/21,china,aaa14,phone1954,ASD80468,15013
+15,2015/7/22,china,aaa15,phone451,ASD1954,15014
+16,2015/7/23,china,aaa16,phone390,ASD38513,15015
+17,2015/7/24,china,aaa17,phone1929,ASD86213,15016
+18,2015/7/25,usa,aaa18,phone910,ASD88812,15017
+19,2015/7/26,china,aaa19,phone2151,ASD9316,15018
+20,2015/7/27,china,aaa20,phone2625,ASD62597,15019
+21,2015/7/28,china,aaa21,phone1371,ASD27896,15020
+22,2015/7/29,china,aaa22,phone945,ASD79760,15021
+23,2015/7/30,china,aaa23,phone2177,ASD45410,15022
+24,2015/7/31,china,aaa24,phone1586,ASD80645,15023
+25,2015/8/1,china,aaa25,phone1310,ASD36408,15024
+26,2015/8/2,china,aaa26,phone1579,ASD14571,15025
+27,2015/8/3,china,aaa27,phone2123,ASD36243,15026
+28,2015/8/4,china,aaa28,phone2334,ASD57825,15027
+29,2015/8/5,china,aaa29,phone1166,ASD26161,15028
+30,2015/8/6,china,aaa30,phone2248,ASD47899,15029
+31,2015/8/7,china,aaa31,phone475,ASD89811,15030
+32,2015/8/8,china,aaa32,phone2499,ASD87974,15031
+33,2015/8/9,china,aaa33,phone2333,ASD62408,15032
+34,2015/8/10,china,aaa34,phone1128,ASD73138,15033
+35,2015/8/11,china,aaa35,phone1063,ASD29573,15034
+36,2015/8/12,china,aaa36,phone1633,ASD82574,15035
+37,2015/8/13,china,aaa37,phone775,ASD47938,15036
+38,2015/8/14,china,aaa38,phone817,ASD40947,15037
+39,2015/8/15,china,aaa39,phone2221,ASD6379,15038
+40,2015/8/16,china,aaa40,phone2289,ASD48374,15039
+41,2015/8/17,china,aaa41,phone599,ASD44560,15040
+42,2015/8/18,china,aaa42,phone384,ASD613,15041
+43,2015/8/19,china,aaa43,phone731,ASD66050,15042
+44,2015/8/20,china,aaa44,phone2128,ASD39759,15043
+45,2015/8/21,china,aaa45,phone1503,ASD31200,15044
+46,2015/8/22,china,aaa46,phone1833,ASD22945,15045
+47,2015/8/23,china,aaa47,phone2346,ASD80162,15046
+48,2015/8/24,china,aaa48,phone2714,ASD27822,15047
+49,2015/8/25,china,aaa49,phone1582,ASD21279,15048
+50,2015/8/26,china,aaa50,phone83,ASD17242,15049
+51,2015/8/27,china,aaa51,phone54,ASD29131,15050
+52,2015/8/28,china,aaa52,phone526,ASD73647,15051
+53,2015/8/29,china,aaa53,phone1308,ASD80493,15052
+54,2015/8/30,china,aaa54,phone2785,ASD30573,15053
+55,2015/8/31,china,aaa55,phone2133,ASD49757,15054
+56,2015/9/1,china,aaa56,phone871,ASD54753,15055
+57,2015/9/2,china,aaa57,phone1570,ASD25758,15056
+58,2015/9/3,china,aaa58,phone434,ASD30291,15057
+59,2015/9/4,china,aaa59,phone2023,ASD60739,15058
+60,2015/9/5,china,aaa60,phone1755,ASD4955,15059
+61,2015/9/6,china,aaa61,phone1120,ASD41678,15060
+62,2015/9/7,china,aaa62,phone526,ASD73647,15061
+63,2015/9/8,china,aaa63,phone111,ASD20917,15062
+64,2015/9/9,china,aaa64,phone2477,ASD78171,15063
+65,2015/9/10,china,aaa65,phone1458,ASD3023,15064
+66,2015/9/11,china,aaa66,phone33,ASD54379,15065
+67,2015/9/12,china,aaa67,phone1710,ASD65296,15066
+68,2015/9/13,china,aaa68,phone118,ASD4568,15067
+69,2015/9/14,china,aaa69,phone2772,ASD42161,15068
+70,2015/9/15,china,aaa70,phone1013,ASD88261,15069
+71,2015/9/16,china,aaa71,phone1606,ASD33903,15070
+72,2015/9/17,china,aaa72,phone2800,ASD60308,15071
+73,2015/9/18,china,aaa73,phone2461,ASD14645,15072
+74,2015/9/19,china,aaa74,phone1038,ASD66620,15073
+75,2015/9/20,china,aaa75,phone2882,ASD23220,15074
+76,2015/9/21,china,aaa76,phone1665,ASD31618,15075
+77,2015/9/22,china,aaa77,phone2991,ASD37964,15076
+78,2015/9/23,china,aaa78,phone620,ASD7257,15077
+79,2015/9/24,china,aaa79,phone1097,ASD12510,15078
+80,2015/9/25,usa,aaa80,phone1668,ASD41149,15079
+81,2015/9/26,china,aaa81,phone2869,ASD95862,15080
+82,2015/9/27,china,aaa82,phone2506,ASD77011,15081
+83,2015/9/28,china,aaa83,phone2897,ASD6674,15082
+84,2015/9/29,china,aaa84,phone954,ASD72595,15083
+85,2015/9/30,china,aaa85,phone1382,ASD86617,15084
+86,2015/10/1,china,aaa86,phone284,ASD31454,15085
+87,2015/10/2,china,aaa87,phone1000,ASD1404,15086
+88,2015/10/3,china,aaa88,phone1813,ASD6955,15087
+89,2015/10/4,china,aaa89,phone2301,ASD14198,15088
+90,2015/10/5,france,aaa90,phone791,ASD13426,15089
+91,2015/10/6,china,aaa91,phone1885,ASD67230,15090
+92,2015/10/7,china,aaa92,phone2055,ASD37734,15091
+93,2015/10/8,china,aaa93,phone710,ASD20056,15092
+94,2015/10/9,china,aaa94,phone1661,ASD62717,15093
+95,2015/10/10,china,aaa95,phone890,ASD68112,15094
+96,2015/10/11,china,aaa96,phone1291,ASD43253,15095
+97,2015/10/12,china,aaa97,phone1754,ASD17116,15096
+98,2015/10/13,china,aaa98,phone1162,ASD9615,15097
+99,2015/10/14,china,aaa99,phone1202,ASD38525,15098
+100,2015/10/15,china,aaa100,phone2568,ASD16162,15099
+101,2015/10/16,china,aaa101,phone722,ASD82042,15100
+102,2015/10/17,china,aaa102,phone1986,ASD55325,15101
+103,2015/10/18,china,aaa103,phone2370,ASD86596,15102
+104,2015/10/19,china,aaa104,phone2477,ASD78171,15103
+105,2015/10/20,china,aaa105,phone2047,ASD38944,15104
+106,2015/10/21,china,aaa106,phone16,ASD96002,15105
+107,2015/10/22,china,aaa107,phone1319,ASD79661,15106
+108,2015/10/23,china,aaa108,phone1375,ASD63986,15107
+109,2015/10/24,china,aaa109,phone638,ASD22875,15108
+110,2015/10/25,china,aaa110,phone1520,ASD75562,15109
+111,2015/10/26,china,aaa111,phone868,ASD43647,15110
+112,2015/10/27,china,aaa112,phone1688,ASD69512,15111
+113,2015/10/28,china,aaa113,phone1359,ASD65796,15112
+114,2015/10/29,china,aaa114,phone1819,ASD14490,15113
+115,2015/10/30,china,aaa115,phone2686,ASD70591,15114
+116,2015/10/31,china,aaa116,phone1999,ASD59245,15115
+117,2015/11/1,china,aaa117,phone1708,ASD47454,15116
+118,2015/11/2,china,aaa118,phone868,ASD43647,15117
+119,2015/11/3,china,aaa119,phone2708,ASD80313,15118
+120,2015/11/4,china,aaa120,phone2930,ASD29572,15119
+121,2015/11/5,china,aaa121,phone482,ASD37161,15120
+122,2015/11/6,china,aaa122,phone147,ASD86280,15121
+123,2015/11/7,china,aaa123,phone22,ASD15140,15122
+124,2015/11/8,china,aaa124,phone2900,ASD81501,15123
+125,2015/11/9,china,aaa125,phone2698,ASD94746,15124
+126,2015/11/10,china,aaa126,phone740,ASD87038,15125
+127,2015/11/11,china,aaa127,phone2330,ASD51162,15126
+128,2015/11/12,china,aaa128,phone1825,ASD39324,15127
+129,2015/11/13,china,aaa129,phone1910,ASD38521,15128
+130,2015/11/14,china,aaa130,phone2391,ASD77728,15129
+131,2015/11/15,china,aaa131,phone138,ASD67641,15130
+132,2015/11/16,uk,aaa132,phone1860,ASD84718,15131
+133,2015/11/17,uk,aaa133,phone1957,ASD8256,15132
+134,2015/11/18,uk,aaa134,phone738,ASD55955,15133
+135,2015/11/19,uk,aaa135,phone412,ASD58554,15134
+136,2015/11/20,uk,aaa136,phone2913,ASD38036,15135
+137,2015/11/21,uk,aaa137,phone2397,ASD17008,15136
+138,2015/11/22,uk,aaa138,phone2231,ASD59268,15137
+139,2015/11/23,uk,aaa139,phone1909,ASD32664,15138
+140,2015/11/24,uk,aaa140,phone2339,ASD54735,15139
+141,2015/11/25,china,aaa141,phone1086,ASD70962,15140
+142,2015/11/26,china,aaa142,phone2462,ASD4442,15141
+143,2015/11/27,china,aaa143,phone1628,ASD24356,15142
+144,2015/11/28,china,aaa144,phone1399,ASD18683,15143
+145,2015/11/29,china,aaa145,phone1512,ASD42324,15144
+146,2015/11/30,china,aaa146,phone845,ASD73765,15145
+147,2015/12/1,china,aaa147,phone2338,ASD49799,15146
+148,2015/12/2,china,aaa148,phone112,ASD95309,15147
+149,2015/12/3,china,aaa149,phone2008,ASD99124,15148
+150,2015/12/4,china,aaa150,phone1504,ASD18487,15149
+151,2015/12/5,china,aaa151,phone2538,ASD86901,15150
+152,2015/12/6,china,aaa152,phone253,ASD56839,15151
+153,2015/12/7,china,aaa153,phone543,ASD59832,15152
+154,2015/12/8,china,aaa154,phone251,ASD59211,15153
+155,2015/12/9,china,aaa155,phone2898,ASD58932,15154
+156,2015/12/10,china,aaa156,phone219,ASD78593,15155
+157,2015/12/11,china,aaa157,phone599,ASD44560,15156
+158,2015/12/12,china,aaa158,phone2775,ASD33598,15157
+159,2015/12/13,china,aaa159,phone873,ASD94122,15158
+160,2015/12/14,china,aaa160,phone1871,ASD21810,15159
+161,2015/12/15,china,aaa161,phone1871,ASD21810,15160
+162,2015/12/16,china,aaa162,phone1626,ASD5437,15161
+163,2015/12/17,china,aaa163,phone825,ASD71225,15162
+164,2015/12/18,china,aaa164,phone2660,ASD27594,15163
+165,2015/12/19,china,aaa165,phone388,ASD76310,15164
+166,2015/12/20,china,aaa166,phone2583,ASD77454,15165
+167,2015/12/21,china,aaa167,phone2870,ASD61497,15166
+168,2015/12/22,china,aaa168,phone2785,ASD30573,15167
+169,2015/12/23,china,aaa169,phone293,ASD78900,15168
+170,2015/12/24,china,aaa170,phone116,ASD86473,15169
+171,2015/12/25,china,aaa171,phone1244,ASD30971,15170
+172,2015/12/26,china,aaa172,phone2175,ASD33484,15171
+173,2015/12/27,china,aaa173,phone1909,ASD32664,15172
+174,2015/12/28,china,aaa174,phone1921,ASD28833,15173
+175,2015/12/29,china,aaa175,phone374,ASD67524,15174
+176,2015/12/30,china,aaa176,phone465,ASD75494,15175
+177,2015/12/31,china,aaa177,phone395,ASD63501,15176
+178,2016/1/1,china,aaa178,phone1695,ASD92758,15177
+179,2016/1/2,china,aaa179,phone1396,ASD53292,15178
+180,2016/1/3,china,aaa180,phone1377,ASD12856,15179
+181,2016/1/4,china,aaa181,phone2385,ASD70717,15180
+182,2016/1/5,china,aaa182,phone2083,ASD3221,15181
+183,2016/1/6,china,aaa183,phone609,ASD54015,15182
+184,2016/1/7,china,aaa184,phone1732,ASD35453,15183
+185,2016/1/8,china,aaa185,phone926,ASD46831,15184
+186,2016/1/9,china,aaa186,phone52,ASD46238,15185
+187,2016/1/10,china,aaa187,phone401,ASD28870,15186
+188,2016/1/11,china,aaa188,phone304,ASD52886,15187
+189,2016/1/12,china,aaa189,phone1586,ASD80645,15188
+190,2016/1/13,china,aaa190,phone358,ASD23673,15189
+191,2016/1/14,china,aaa191,phone1930,ASD58097,15190
+192,2016/1/15,china,aaa192,phone663,ASD44218,15191
+193,2016/1/16,china,aaa193,phone1243,ASD37727,15192
+194,2016/1/17,china,aaa194,phone1588,ASD91226,15193
+195,2016/1/18,china,aaa195,phone2527,ASD80035,15194
+196,2016/1/19,china,aaa196,phone1522,ASD62002,15195
+197,2016/1/20,china,aaa197,phone1387,ASD49312,15196
+198,2016/1/21,china,aaa198,phone2493,ASD55143,15197
+199,2016/1/22,china,aaa199,phone2307,ASD10530,15198
+200,2016/1/23,china,aaa200,phone2940,ASD57500,15199
+201,2016/1/24,canada,aaa201,phone1208,ASD8177,15200
+202,2016/1/25,canada,aaa202,phone1383,ASD13789,15201
+203,2016/1/26,canada,aaa203,phone1016,ASD68460,15202
+204,2016/1/27,canada,aaa204,phone2832,ASD89285,15203
+205,2016/1/28,canada,aaa205,phone2199,ASD21040,15204
+206,2016/1/29,canada,aaa206,phone1105,ASD41184,15205
+207,2016/1/30,canada,aaa207,phone1997,ASD79237,15206
+208,2016/1/31,canada,aaa208,phone663,ASD44218,15207
+209,2016/2/1,canada,aaa209,phone767,ASD85191,15208
+210,2016/2/2,canada,aaa210,phone1232,ASD55928,15209
+211,2016/2/3,canada,aaa211,phone2533,ASD86033,15210
+212,2016/2/4,canada,aaa212,phone861,ASD12777,15211
+213,2016/2/5,canada,aaa213,phone1334,ASD47037,15212
+214,2016/2/6,canada,aaa214,phone1438,ASD77362,15213
+215,2016/2/7,canada,aaa215,phone160,ASD78770,15214
+216,2015/7/8,canada,aaa216,phone2437,ASD99353,15215
+217,2015/7/8,canada,aaa217,phone1430,ASD42465,15216
+218,2015/7/8,china,aaa218,phone2989,ASD31575,15217
+219,2015/7/8,china,aaa219,phone1315,ASD42644,15218
+220,2015/7/8,china,aaa220,phone2866,ASD29661,15219
+221,2015/7/8,china,aaa221,phone1616,ASD16671,15220
+222,2015/7/8,china,aaa222,phone2087,ASD48037,15221
+223,2015/7/8,china,aaa223,phone2573,ASD82694,15222
+224,2015/7/8,china,aaa224,phone913,ASD21179,15223
+225,2015/7/8,china,aaa225,phone900,ASD46140,15224
+226,2015/7/8,china,aaa226,phone1087,ASD43483,15225
+227,2015/7/8,china,aaa227,phone1647,ASD74002,15226
+228,2015/7/8,china,aaa228,phone2341,ASD82304,15227
+229,2015/7/8,china,aaa229,phone1901,ASD34911,15228
+230,2015/7/8,china,aaa230,phone2649,ASD69130,15229
+231,2015/7/8,china,aaa231,phone2514,ASD24211,15230
+232,2015/7/8,china,aaa232,phone133,ASD22945,15231
+233,2015/7/8,china,aaa233,phone1048,ASD5352,15232
+234,2015/7/8,uk,aaa234,phone10,ASD18389,15233
+235,2015/7/8,france,aaa235,phone319,ASD14310,15234
+236,2015/7/8,france,aaa236,phone169,ASD18752,15235
+237,2015/7/8,france,aaa237,phone1792,ASD12930,15236
+238,2015/7/8,france,aaa238,phone2914,ASD66266,15237
+239,2015/7/8,france,aaa239,phone371,ASD78389,15238
+240,2015/7/8,france,aaa240,phone739,ASD43117,15239
+241,2015/7/8,france,aaa241,phone791,ASD13426,15240
+242,2015/7/8,france,aaa242,phone2594,ASD53050,15241
+243,2015/7/8,france,aaa243,phone696,ASD31569,15242
+244,2015/7/8,france,aaa244,phone2029,ASD19595,15243
+245,2015/7/8,france,aaa245,phone795,ASD24648,15244
+246,2015/7/8,france,aaa246,phone865,ASD20613,15245
+247,2015/7/8,france,aaa247,phone471,ASD5440,15246
+248,2015/7/8,france,aaa248,phone2399,ASD39891,15247
+249,2015/7/8,france,aaa249,phone56,ASD79971,15248
+250,2015/7/8,france,aaa250,phone1371,ASD27896,15249
+251,2015/7/8,france,aaa251,phone1746,ASD68481,15250
+252,2015/7/8,france,aaa252,phone147,ASD86280,15251
+253,2015/7/8,france,aaa253,phone2766,ASD60381,15252
+254,2015/7/8,france,aaa254,phone2932,ASD12302,15253
+255,2015/7/8,france,aaa255,phone1866,ASD21033,15254
+256,2015/7/8,france,aaa256,phone1631,ASD67070,15255
+257,2015/7/8,france,aaa257,phone1386,ASD80979,15256
+258,2015/7/8,france,aaa258,phone2384,ASD97760,15257
+259,2015/7/8,france,aaa259,phone1977,ASD29691,15258
+260,2015/7/8,france,aaa260,phone316,ASD96088,15259
+261,2015/7/8,france,aaa261,phone763,ASD7447,15260
+262,2015/7/8,france,aaa262,phone2791,ASD86621,15261
+263,2015/7/8,france,aaa263,phone401,ASD28870,15262
+264,2015/7/8,france,aaa264,phone1534,ASD59748,15263
+265,2015/7/8,france,aaa265,phone2815,ASD51862,15264
+266,2015/7/8,france,aaa266,phone2277,ASD52116,15265
+267,2015/7/8,france,aaa267,phone1458,ASD3023,15266
+268,2015/7/8,china,aaa268,phone1173,ASD44668,15267
+269,2015/7/8,china,aaa269,phone2428,ASD94819,15268
+270,2015/7/8,china,aaa270,phone2565,ASD41930,15269
+271,2015/7/8,china,aaa271,phone2064,ASD14615,15270
+272,2015/7/8,china,aaa272,phone1332,ASD35057,15271
+273,2015/7/8,china,aaa273,phone1117,ASD96168,15272
+274,2015/7/8,china,aaa274,phone801,ASD10049,15273
+275,2015/7/8,china,aaa275,phone316,ASD96088,15274
+276,2015/7/8,china,aaa276,phone2098,ASD4166,15275
+277,2015/7/8,china,aaa277,phone1590,ASD74144,15276
+278,2015/7/8,china,aaa278,phone148,ASD26033,15277
+279,2015/7/8,china,aaa279,phone1139,ASD42486,15278
+280,2015/7/8,china,aaa280,phone225,ASD13992,15279
+281,2015/7/8,china,aaa281,phone2490,ASD25777,15280
+282,2015/7/8,china,aaa282,phone807,ASD97797,15281
+283,2015/7/8,china,aaa283,phone25,ASD67930,15282
+284,2015/7/8,china,aaa284,phone1101,ASD35178,15283
+285,2015/7/8,china,aaa285,phone1211,ASD27032,15284
+286,2015/7/8,china,aaa286,phone1092,ASD30977,15285
+287,2015/7/8,china,aaa287,phone268,ASD20002,15286
+288,2015/7/8,china,aaa288,phone1017,ASD2489,15287
+289,2015/7/8,china,aaa289,phone672,ASD18712,15288
+290,2015/7/8,china,aaa290,phone320,ASD5030,15289
+291,2015/7/8,china,aaa291,phone2022,ASD53089,15290
+292,2015/7/8,china,aaa292,phone817,ASD40947,15291
+293,2015/7/8,china,aaa293,phone1513,ASD63237,15292
+294,2015/7/8,china,aaa294,phone1913,ASD29267,15293
+295,2015/7/8,china,aaa295,phone233,ASD94159,15294
+296,2015/7/8,china,aaa296,phone2613,ASD36379,15295
+297,2015/7/8,china,aaa297,phone492,ASD85148,15296
+298,2015/7/8,china,aaa298,phone2896,ASD87790,15297
+299,2015/7/8,china,aaa299,phone1395,ASD5086,15298
+300,2015/7/8,china,aaa300,phone2405,ASD98593,15299
+301,2015/7/8,china,aaa301,phone1837,ASD76100,15300
+302,2015/7/8,china,aaa302,phone690,ASD96405,15301
+303,2015/7/8,china,aaa303,phone1006,ASD22695,15302
+304,2015/7/8,china,aaa304,phone1980,ASD31405,15303
+305,2015/7/8,china,aaa305,phone170,ASD78727,15304
+306,2015/7/8,china,aaa306,phone1139,ASD42486,15305
+307,2015/7/8,china,aaa307,phone1033,ASD7342,15306
+308,2015/7/8,china,aaa308,phone2020,ASD48268,15307
+309,2015/7/8,china,aaa309,phone1629,ASD59143,15308
+310,2015/7/8,china,aaa310,phone222,ASD42940,15309
+311,2015/7/8,china,aaa311,phone574,ASD18673,15310
+312,2015/7/8,china,aaa312,phone2371,ASD40072,15311
+313,2015/7/8,china,aaa313,phone1817,ASD40255,15312
+314,2015/7/8,china,aaa314,phone1153,ASD74976,15313
+315,2015/7/8,china,aaa315,phone2836,ASD74087,15314
+316,2015/7/8,china,aaa316,phone89,ASD88184,15315
+317,2015/7/8,china,aaa317,phone1988,ASD6141,15316
+318,2015/7/8,china,aaa318,phone2641,ASD75257,15317
+319,2015/7/8,china,aaa319,phone106,ASD31344,15318
+320,2015/7/8,china,aaa320,phone426,ASD54309,15319
+321,2015/7/8,china,aaa321,phone1551,ASD30475,15320
+322,2015/7/8,china,aaa322,phone1241,ASD73185,15321
+323,2015/7/8,china,aaa323,phone2249,ASD87147,15322
+324,2015/7/8,china,aaa324,phone1644,ASD12948,15323
+325,2015/7/8,china,aaa325,phone2574,ASD33664,15324
+326,2015/7/8,china,aaa326,phone180,ASD40830,15325
+327,2015/7/8,china,aaa327,phone2382,ASD36810,15326
+328,2015/7/8,china,aaa328,phone1651,ASD91529,15327
+329,2015/7/8,china,aaa329,phone1272,ASD69717,15328
+330,2015/7/8,china,aaa330,phone1623,ASD48772,15329
+331,2015/7/8,china,aaa331,phone982,ASD12783,15330
+332,2015/7/8,china,aaa332,phone2929,ASD37120,15331
+333,2015/7/8,china,aaa333,phone894,ASD84235,15332
+334,2015/7/8,china,aaa334,phone2522,ASD55429,15333
+335,2015/7/8,china,aaa335,phone1347,ASD67484,15334
+336,2015/7/8,china,aaa336,phone471,ASD5440,15335
+337,2015/7/8,china,aaa337,phone2515,ASD2692,15336
+338,2015/7/8,china,aaa338,phone1337,ASD614,15337
+339,2015/7/8,china,aaa339,phone1619,ASD4540,15338
+340,2015/7/8,china,aaa340,phone1262,ASD35650,15339
+341,2015/7/8,china,aaa341,phone2909,ASD62684,15340
+342,2015/7/8,china,aaa342,phone49,ASD54022,15341
+343,2015/7/8,china,aaa343,phone1989,ASD56363,15342
+344,2015/7/8,china,aaa344,phone911,ASD95311,15343
+345,2015/7/8,china,aaa345,phone2139,ASD68016,15344
+346,2015/7/8,china,aaa346,phone2097,ASD15979,15345
+347,2015/7/8,china,aaa347,phone1134,ASD97664,15346
+348,2015/7/8,china,aaa348,phone2347,ASD53040,15347
+349,2015/7/8,china,aaa349,phone2263,ASD40179,15348
+350,2015/7/8,china,aaa350,phone1539,ASD96530,15349
+351,2015/7/8,china,aaa351,phone1123,ASD45798,15350
+352,2015/7/8,china,aaa352,phone2969,ASD18127,15351
+353,2015/7/8,china,aaa353,phone2968,ASD47580,15352
+354,2015/7/8,china,aaa354,phone996,ASD78710,15353
+355,2015/7/8,china,aaa355,phone2700,ASD15021,15354
+356,2015/7/8,china,aaa356,phone753,ASD22367,15355
+357,2015/7/8,china,aaa357,phone2640,ASD29615,15356
+358,2015/7/8,china,aaa358,phone2768,ASD23094,15357
+359,2015/7/8,china,aaa359,phone418,ASD53383,15358
+360,2015/7/8,china,aaa360,phone1612,ASD55774,15359
+361,2015/7/8,china,aaa361,phone451,ASD1954,15360
+362,2015/7/8,china,aaa362,phone2096,ASD61555,15361
+363,2015/7/8,china,aaa363,phone1979,ASD7220,15362
+364,2015/7/8,china,aaa364,phone325,ASD97321,15363
+365,2015/7/8,china,aaa365,phone1138,ASD40077,15364
+366,2015/7/8,china,aaa366,phone1866,ASD21033,15365
+367,2015/7/8,china,aaa367,phone683,ASD26163,15366
+368,2015/7/8,china,aaa368,phone1412,ASD19745,15367
+369,2015/7/8,china,aaa369,phone2578,ASD40786,15368
+370,2015/7/8,china,aaa370,phone950,ASD5064,15369
+371,2015/7/8,china,aaa371,phone1981,ASD11729,15370
+372,2015/7/8,china,aaa372,phone1367,ASD71011,15371
+373,2015/7/8,china,aaa373,phone2867,ASD24364,15372
+374,2015/7/8,china,aaa374,phone2284,ASD32419,15373
+375,2015/7/8,china,aaa375,phone2463,ASD33813,15374
+376,2015/7/8,china,aaa376,phone368,ASD34269,15375
+377,2015/7/8,china,aaa377,phone428,ASD1961,15376
+378,2015/7/8,china,aaa378,phone2527,ASD80035,15377
+379,2015/7/8,china,aaa379,phone260,ASD2976,15378
+380,2015/7/8,china,aaa380,phone1671,ASD73981,15379
+381,2015/7/8,china,aaa381,phone1379,ASD62041,15380
+382,2015/7/8,china,aaa382,phone124,ASD28539,15381
+383,2015/7/8,china,aaa383,phone95,ASD88540,15382
+384,2015/7/8,china,aaa384,phone2487,ASD6907,15383
+385,2015/7/8,china,aaa385,phone1420,ASD49252,15384
+386,2015/7/8,china,aaa386,phone429,ASD61651,15385
+387,2015/7/8,china,aaa387,phone2953,ASD3445,15386
+388,2015/7/8,china,aaa388,phone1645,ASD20063,15387
+389,2015/7/8,china,aaa389,phone283,ASD16039,15388
+390,2015/7/8,china,aaa390,phone2927,ASD21076,15389
+391,2015/7/8,china,aaa391,phone2855,ASD27086,15390
+392,2015/7/8,china,aaa392,phone1952,ASD50922,15391
+393,2015/7/8,china,aaa393,phone229,ASD66817,15392
+394,2015/7/8,brazil,aaa394,phone1883,ASD33806,15393
+395,2015/7/8,brazil,aaa395,phone2148,ASD60441,15394
+396,2015/7/8,brazil,aaa396,phone190,ASD29041,15395
+397,2015/7/8,brazil,aaa397,phone356,ASD62581,15396
+398,2015/7/8,brazil,aaa398,phone2992,ASD71446,15397
+399,2015/7/8,brazil,aaa399,phone2213,ASD54693,15398
+400,2015/7/8,brazil,aaa400,phone165,ASD10848,15399
+401,2015/7/8,brazil,aaa401,phone1579,ASD14571,15400
+402,2015/7/8,brazil,aaa402,phone251,ASD59211,15401
+403,2015/7/8,brazil,aaa403,phone1522,ASD62002,15402
+404,2015/7/8,brazil,aaa404,phone2873,ASD12631,15403
+405,2015/7/8,brazil,aaa405,phone1376,ASD68677,15404
+406,2015/7/8,brazil,aaa406,phone1587,ASD94026,15405
+407,2015/7/8,brazil,aaa407,phone2792,ASD55146,15406
+408,2015/7/8,brazil,aaa408,phone2774,ASD91112,15407
+409,2015/7/8,brazil,aaa409,phone2439,ASD9880,15408
+410,2015/7/8,brazil,aaa410,phone1399,ASD18683,15409
+411,2015/7/8,brazil,aaa411,phone150,ASD69229,15410
+412,2015/7/8,china,aaa412,phone2895,ASD1765,15411
+413,2015/7/8,china,aaa413,phone2551,ASD77958,15412
+414,2015/7/8,china,aaa414,phone694,ASD92230,15413
+415,2015/7/8,china,aaa415,phone2233,ASD84617,15414
+416,2015/7/8,china,aaa416,phone146,ASD65240,15415
+417,2015/7/8,china,aaa417,phone707,ASD5628,15416
+418,2015/7/8,china,aaa418,phone2658,ASD4726,15417
+419,2015/7/8,china,aaa419,phone1020,ASD4161,15418
+420,2015/7/8,china,aaa420,phone535,ASD97410,15419
+421,2015/7/8,china,aaa421,phone624,ASD51860,15420
+422,2015/7/8,usa,aaa422,phone2102,ASD80410,15421
+423,2015/7/8,china,aaa423,phone1112,ASD28257,15422
+424,2015/7/8,china,aaa424,phone1347,ASD67484,15423
+425,2015/7/8,china,aaa425,phone1306,ASD5266,15424
+426,2015/7/8,china,aaa426,phone2458,ASD89845,15425
+427,2015/7/8,china,aaa427,phone1701,ASD65854,15426
+428,2015/7/8,china,aaa428,phone1439,ASD55081,15427
+429,2015/7/8,china,aaa429,phone2952,ASD29721,15428
+430,2015/7/8,china,aaa430,phone2890,ASD73575,15429
+431,2015/7/8,china,aaa431,phone1817,ASD40255,15430
+432,2015/7/8,china,aaa432,phone1264,ASD10727,15431
+433,2015/7/8,china,aaa433,phone2819,ASD32336,15432
+434,2015/7/8,china,aaa434,phone1046,ASD84173,15433
+435,2015/7/8,china,aaa435,phone281,ASD29929,15434
+436,2015/7/8,china,aaa436,phone731,ASD66050,15435
+437,2015/7/8,china,aaa437,phone244,ASD24103,15436
+438,2015/7/8,china,aaa438,phone993,ASD22962,15437
+439,2015/7/8,china,aaa439,phone2984,ASD28539,15438
+440,2015/7/8,china,aaa440,phone1078,ASD93778,15439
+441,2015/7/8,china,aaa441,phone2844,ASD33646,15440
+442,2015/7/8,china,aaa442,phone2251,ASD59852,15441
+443,2015/7/8,china,aaa443,phone598,ASD1303,15442
+444,2015/7/8,china,aaa444,phone1286,ASD96687,15443
+445,2015/7/8,china,aaa445,phone2346,ASD80162,15444
+446,2015/7/8,china,aaa446,phone729,ASD69936,15445
+447,2015/7/8,china,aaa447,phone1946,ASD69048,15446
+448,2015/7/8,china,aaa448,phone2832,ASD89285,15447
+449,2015/7/8,china,aaa449,phone1334,ASD47037,15448
+450,2015/7/8,china,aaa450,phone2434,ASD67700,15449
+451,2015/7/8,china,aaa451,phone595,ASD51331,15450
+452,2015/7/8,china,aaa452,phone2018,ASD63383,15451
+453,2015/7/8,china,aaa453,phone1226,ASD69240,15452
+454,2015/7/8,china,aaa454,phone710,ASD20056,15453
+455,2015/7/8,china,aaa455,phone674,ASD61044,15454
+456,2015/7/8,china,aaa456,phone1921,ASD28833,15455
+457,2015/7/8,china,aaa457,phone696,ASD31569,15456
+458,2015/7/8,china,aaa458,phone1393,ASD44216,15457
+459,2015/7/8,china,aaa459,phone101,ASD28691,15458
+460,2015/7/8,china,aaa460,phone2383,ASD76851,15459
+461,2015/7/8,china,aaa461,phone1828,ASD67859,15460
+462,2015/7/8,usa,aaa462,phone1596,ASD74089,15461
+463,2015/7/8,china,aaa463,phone211,ASD91508,15462
+464,2015/7/8,china,aaa464,phone2782,ASD74449,15463
+465,2015/7/8,china,aaa465,phone2912,ASD78786,15464
+466,2015/7/8,china,aaa466,phone844,ASD29001,15465
+467,2015/7/8,china,aaa467,phone2009,ASD14090,15466
+468,2015/7/8,china,aaa468,phone1795,ASD42863,15467
+469,2015/7/8,china,aaa469,phone1208,ASD8177,15468
+470,2015/7/8,china,aaa470,phone225,ASD13992,15469
+471,2015/7/8,china,aaa471,phone102,ASD16814,15470
+472,2015/7/8,china,aaa472,phone1481,ASD59601,15471
+473,2015/7/8,china,aaa473,phone1147,ASD24488,15472
+474,2015/7/8,china,aaa474,phone2077,ASD13520,15473
+475,2015/7/8,china,aaa475,phone2883,ASD44011,15474
+476,2015/7/8,china,aaa476,phone734,ASD81584,15475
+477,2015/7/8,china,aaa477,phone2658,ASD4726,15476
+478,2015/7/8,china,aaa478,phone2454,ASD11864,15477
+479,2015/7/8,china,aaa479,phone1528,ASD82259,15478
+480,2015/7/8,china,aaa480,phone337,ASD7849,15479
+481,2015/7/8,china,aaa481,phone412,ASD58554,15480
+482,2015/7/8,china,aaa482,phone124,ASD28539,15481
+483,2015/7/8,china,aaa483,phone521,ASD63356,15482
+484,2015/7/8,china,aaa484,phone243,ASD62881,15483
+485,2015/7/8,china,aaa485,phone486,ASD78645,15484
+486,2015/7/8,china,aaa486,phone267,ASD84287,15485
+487,2015/7/8,china,aaa487,phone1056,ASD59874,15486
+488,2015/7/8,china,aaa488,phone843,ASD72585,15487
+489,2015/7/8,china,aaa489,phone1319,ASD79661,15488
+490,2015/7/8,china,aaa490,phone1049,ASD37264,15489
+491,2015/7/8,china,aaa491,phone369,ASD19326,15490
+492,2015/7/8,china,aaa492,phone1878,ASD44601,15491
+493,2015/7/8,china,aaa493,phone2458,ASD89845,15492
+494,2015/7/8,china,aaa494,phone2014,ASD36354,15493
+495,2015/7/8,china,aaa495,phone1597,ASD96755,15494
+496,2015/7/8,china,aaa496,phone609,ASD54015,15495
+497,2015/7/8,china,aaa497,phone2921,ASD89806,15496
+498,2015/7/8,china,aaa498,phone534,ASD79006,15497
+499,2015/7/8,china,aaa499,phone1172,ASD38872,15498
+500,2015/7/8,china,aaa500,phone529,ASD57718,15499
+501,2015/7/8,china,aaa501,phone1152,ASD67729,15500
+502,2015/7/8,china,aaa502,phone2211,ASD76824,15501
+503,2015/7/8,china,aaa503,phone478,ASD78007,15502
+504,2015/7/8,china,aaa504,phone1851,ASD23414,15503
+505,2015/7/8,china,aaa505,phone1955,ASD79904,15504
+506,2015/7/8,china,aaa506,phone545,ASD56156,15505
+507,2015/7/8,china,aaa507,phone44,ASD20640,15506
+508,2015/7/8,china,aaa508,phone437,ASD15325,15507
+509,2015/7/8,china,aaa509,phone1907,ASD4995,15508
+510,2015/7/8,china,aaa510,phone2926,ASD57045,15509
+511,2015/7/8,china,aaa511,phone1774,ASD12773,15510
+512,2015/7/8,china,aaa512,phone1451,ASD46252,15511
+513,2015/7/8,china,aaa513,phone2050,ASD67274,15512
+514,2015/7/8,china,aaa514,phone1359,ASD65796,15513
+515,2015/7/8,china,aaa515,phone2472,ASD11464,15514
+516,2015/7/8,china,aaa516,phone1430,ASD42465,15515
+517,2015/7/8,china,aaa517,phone1946,ASD69048,15516
+518,2015/7/8,china,aaa518,phone1445,ASD154,15517
+519,2015/7/8,china,aaa519,phone1689,ASD25206,15518
+520,2015/7/8,china,aaa520,phone2262,ASD79744,15519
+521,2015/7/8,china,aaa521,phone2950,ASD52102,15520
+522,2015/7/8,china,aaa522,phone764,ASD28348,15521
+523,2015/7/8,china,aaa523,phone318,ASD68522,15522
+524,2015/7/8,china,aaa524,phone1548,ASD95032,15523
+525,2015/7/8,china,aaa525,phone915,ASD65907,15524
+526,2015/7/8,china,aaa526,phone2672,ASD16946,15525
+527,2015/7/8,china,aaa527,phone538,ASD90796,15526
+528,2015/7/8,china,aaa528,phone2806,ASD83826,15527
+529,2015/7/8,china,aaa529,phone2158,ASD5227,15528
+530,2015/7/8,china,aaa530,phone2415,ASD30793,15529
+531,2015/7/8,china,aaa531,phone1017,ASD2489,15530
+532,2015/7/8,china,aaa532,phone779,ASD34772,15531
+533,2015/7/8,china,aaa533,phone2092,ASD72501,15532
+534,2015/7/8,china,aaa534,phone2684,ASD32921,15533
+535,2015/7/8,china,aaa535,phone842,ASD28065,15534
+536,2015/7/8,china,aaa536,phone1628,ASD24356,15535
+537,2015/7/8,china,aaa537,phone2185,ASD42692,15536
+538,2015/7/8,china,aaa538,phone16,ASD96002,15537
+539,2015/7/8,china,aaa539,phone2065,ASD1428,15538
+540,2015/7/8,china,aaa540,phone2804,ASD81203,15539
+541,2015/7/8,china,aaa541,phone1013,ASD88261,15540
+542,2015/7/8,china,aaa542,phone604,ASD9791,15541
+543,2015/7/8,china,aaa543,phone2243,ASD89678,15542
+544,2015/7/8,china,aaa544,phone12,ASD4745,15543
+545,2015/7/8,china,aaa545,phone1834,ASD9095,15544
+546,2015/7/8,china,aaa546,phone2737,ASD884,15545
+547,2015/7/8,china,aaa547,phone2132,ASD28336,15546
+548,2015/7/8,china,aaa548,phone1621,ASD95449,15547
+549,2015/7/8,china,aaa549,phone407,ASD40614,15548
+550,2015/7/8,china,aaa550,phone630,ASD44723,15549
+551,2015/7/8,china,aaa551,phone1586,ASD80645,15550
+552,2015/7/8,china,aaa552,phone1192,ASD76894,15551
+553,2015/7/8,china,aaa553,phone2394,ASD61316,15552
+554,2015/7/8,china,aaa554,phone406,ASD41369,15553
+555,2015/7/8,china,aaa555,phone287,ASD14590,15554
+556,2015/7/8,china,aaa556,phone1794,ASD81307,15555
+557,2015/7/8,china,aaa557,phone1086,ASD70962,15556
+558,2015/7/8,china,aaa558,phone1406,ASD92347,15557
+559,2015/7/8,china,aaa559,phone2886,ASD42613,15558
+560,2015/7/8,china,aaa560,phone901,ASD53764,15559
+561,2015/7/8,china,aaa561,phone328,ASD22482,15560
+562,2015/7/8,china,aaa562,phone2867,ASD24364,15561
+563,2015/7/8,china,aaa563,phone2723,ASD86337,15562
+564,2015/7/8,china,aaa564,phone1339,ASD47999,15563
+565,2015/7/8,china,aaa565,phone403,ASD93198,15564
+566,2015/7/8,china,aaa566,phone557,ASD9926,15565
+567,2015/7/8,china,aaa567,phone107,ASD8537,15566
+568,2015/7/8,china,aaa568,phone2095,ASD78485,15567
+569,2015/7/8,china,aaa569,phone764,ASD28348,15568
+570,2015/7/8,china,aaa570,phone1483,ASD74306,15569
+571,2015/7/8,china,aaa571,phone1304,ASD82559,15570
+572,2015/7/8,china,aaa572,phone2712,ASD30690,15571
+573,2015/7/8,china,aaa573,phone1878,ASD44601,15572
+574,2015/7/8,china,aaa574,phone1707,ASD67861,15573
+575,2015/7/8,china,aaa575,phone2161,ASD88219,15574
+576,2015/7/8,china,aaa576,phone714,ASD96718,15575
+577,2015/7/8,china,aaa577,phone218,ASD11450,15576
+578,2015/7/8,china,aaa578,phone1751,ASD11957,15577
+579,2015/7/8,china,aaa579,phone2442,ASD37341,15578
+580,2015/7/8,china,aaa580,phone2769,ASD17811,15579
+581,2015/7/8,china,aaa581,phone1666,ASD44070,15580
+582,2015/7/8,china,aaa582,phone1262,ASD35650,15581
+583,2015/7/8,china,aaa583,phone90,ASD81472,15582
+584,2015/7/8,china,aaa584,phone1020,ASD4161,15583
+585,2015/7/8,china,aaa585,phone286,ASD23637,15584
+586,2015/7/8,china,aaa586,phone2419,ASD95733,15585
+587,2015/7/8,china,aaa587,phone373,ASD79963,15586
+588,2015/7/8,china,aaa588,phone1872,ASD38362,15587
+589,2015/7/8,china,aaa589,phone847,ASD14247,15588
+590,2015/7/8,china,aaa590,phone966,ASD38194,15589
+591,2015/7/8,china,aaa591,phone2377,ASD50473,15590
+592,2015/7/8,china,aaa592,phone90,ASD81472,15591
+593,2015/7/8,china,aaa593,phone1608,ASD6315,15592
+594,2015/7/8,china,aaa594,phone1955,ASD79904,15593
+595,2015/7/8,china,aaa595,phone331,ASD42591,15594
+596,2015/7/8,china,aaa596,phone2294,ASD18187,15595
+597,2015/7/8,china,aaa597,phone1084,ASD36336,15596
+598,2015/7/8,china,aaa598,phone795,ASD24648,15597
+599,2015/7/8,china,aaa599,phone1496,ASD55988,15598
+600,2015/7/8,china,aaa600,phone1714,ASD16522,15599
+601,2015/7/8,china,aaa601,phone402,ASD189,15600
+602,2015/7/8,china,aaa602,phone729,ASD69936,15601
+603,2015/7/8,china,aaa603,phone1962,ASD85121,15602
+604,2015/7/8,china,aaa604,phone980,ASD28156,15603
+605,2015/7/8,china,aaa605,phone937,ASD89596,15604
+606,2015/7/8,china,aaa606,phone651,ASD26845,15605
+607,2015/7/8,china,aaa607,phone679,ASD45839,15606
+608,2015/7/8,china,aaa608,phone150,ASD69229,15607
+609,2015/7/8,china,aaa609,phone710,ASD20056,15608
+610,2015/7/8,china,aaa610,phone64,ASD27340,15609
+611,2015/7/8,china,aaa611,phone2216,ASD22126,15610
+612,2015/7/8,china,aaa612,phone342,ASD23743,15611
+613,2015/7/8,china,aaa613,phone1769,ASD5421,15612
+614,2015/7/8,china,aaa614,phone1397,ASD82977,15613
+615,2015/7/8,china,aaa615,phone397,ASD53191,15614
+616,2015/7/8,china,aaa616,phone1670,ASD93177,15615
+617,2015/7/8,china,aaa617,phone1264,ASD10727,15616
+618,2015/7/8,china,aaa618,phone2944,ASD16610,15617
+619,2015/7/8,china,aaa619,phone2594,ASD53050,15618
+620,2015/7/8,china,aaa620,phone2759,ASD88298,15619
+621,2015/7/8,china,aaa621,phone704,ASD25128,15620
+622,2015/7/8,china,aaa622,phone2414,ASD74847,15621
+623,2015/7/8,china,aaa623,phone1983,ASD86611,15622
+624,2015/7/8,china,aaa624,phone2098,ASD4166,15623
+625,2015/7/8,china,aaa625,phone84,ASD82187,15624
+626,2015/7/8,china,aaa626,phone388,ASD76310,15625
+627,2015/7/8,china,aaa627,phone308,ASD78701,15626
+628,2015/7/8,china,aaa628,phone1729,ASD83353,15627
+629,2015/7/8,china,aaa629,phone2269,ASD47888,15628
+630,2015/7/8,china,aaa630,phone1233,ASD41299,15629
+631,2015/7/8,china,aaa631,phone330,ASD86359,15630
+632,2015/7/8,china,aaa632,phone1537,ASD46367,15631
+633,2015/7/8,china,aaa633,phone2722,ASD11045,15632
+634,2015/7/8,china,aaa634,phone2691,ASD44656,15633
+635,2015/7/8,china,aaa635,phone2645,ASD76308,15634
+636,2015/7/8,china,aaa636,phone1333,ASD58033,15635
+637,2015/7/8,china,aaa637,phone1776,ASD19877,15636
+638,2015/7/8,china,aaa638,phone979,ASD92290,15637
+639,2015/7/8,china,aaa639,phone2065,ASD1428,15638
+640,2015/7/8,china,aaa640,phone1912,ASD36354,15639
+641,2015/7/8,china,aaa641,phone2275,ASD22269,15640
+642,2015/7/8,china,aaa642,phone2292,ASD10586,15641
+643,2015/7/8,china,aaa643,phone815,ASD65508,15642
+644,2015/7/8,china,aaa644,phone2840,ASD40033,15643
+645,2015/7/8,china,aaa645,phone1615,ASD28864,15644
+646,2015/7/8,china,aaa646,phone1617,ASD77126,15645
+647,2015/7/8,china,aaa647,phone2158,ASD5227,15646
+648,2015/7/8,china,aaa648,phone2121,ASD15680,15647
+649,2015/7/8,china,aaa649,phone205,ASD10000,15648
+650,2015/7/8,china,aaa650,phone1579,ASD14571,15649
+651,2015/7/8,china,aaa651,phone411,ASD94954,15650
+652,2015/7/8,china,aaa652,phone1662,ASD64961,15651
+653,2015/7/8,china,aaa653,phone247,ASD55961,15652
+654,2015/7/8,china,aaa654,phone2169,ASD31622,15653
+655,2015/7/8,china,aaa655,phone1652,ASD14782,15654
+656,2015/7/8,china,aaa656,phone2398,ASD13395,15655
+657,2015/7/8,china,aaa657,phone1395,ASD5086,15656
+658,2015/7/8,china,aaa658,phone975,ASD96997,15657
+659,2015/7/8,china,aaa659,phone1488,ASD18972,15658
+660,2015/7/8,china,aaa660,phone787,ASD84225,15659
+661,2015/7/8,china,aaa661,phone507,ASD61725,15660
+662,2015/7/8,china,aaa662,phone1450,ASD32252,15661
+663,2015/7/8,china,aaa663,phone2011,ASD41588,15662
+664,2015/7/8,china,aaa664,phone2687,ASD14311,15663
+665,2015/7/8,china,aaa665,phone2326,ASD65102,15664
+666,2015/7/8,china,aaa666,phone2195,ASD46852,15665
+667,2015/7/8,china,aaa667,phone563,ASD16790,15666
+668,2015/7/8,china,aaa668,phone747,ASD582,15667
+669,2015/7/8,china,aaa669,phone974,ASD33679,15668
+670,2015/7/8,china,aaa670,phone2223,ASD95804,15669
+671,2015/7/8,china,aaa671,phone1389,ASD60468,15670
+672,2015/7/8,china,aaa672,phone1062,ASD91980,15671
+673,2015/7/8,china,aaa673,phone1373,ASD84913,15672
+674,2015/7/8,china,aaa674,phone2,ASD53470,15673
+675,2015/7/8,china,aaa675,phone45,ASD16172,15674
+676,2015/7/8,china,aaa676,phone2260,ASD37237,15675
+677,2015/7/8,china,aaa677,phone2914,ASD66266,15676
+678,2015/7/8,china,aaa678,phone1162,ASD9615,15677
+679,2015/7/8,china,aaa679,phone725,ASD9665,15678
+680,2015/7/8,china,aaa680,phone2279,ASD2685,15679
+681,2015/7/8,china,aaa681,phone820,ASD77271,15680
+682,2015/7/8,china,aaa682,phone1072,ASD27741,15681
+683,2015/7/8,china,aaa683,phone167,ASD11931,15682
+684,2015/7/8,china,aaa684,phone1740,ASD86624,15683
+685,2015/7/8,china,aaa685,phone1233,ASD41299,15684
+686,2015/7/8,china,aaa686,phone2965,ASD20537,15685
+687,2015/7/8,china,aaa687,phone806,ASD97088,15686
+688,2015/7/8,china,aaa688,phone2202,ASD32371,15687
+689,2015/7/8,china,aaa689,phone455,ASD74141,15688
+690,2015/7/8,china,aaa690,phone17,ASD28968,15689
+691,2015/7/8,china,aaa691,phone928,ASD24015,15690
+692,2015/7/8,china,aaa692,phone374,ASD67524,15691
+693,2015/7/8,china,aaa693,phone2677,ASD50629,15692
+694,2015/7/8,china,aaa694,phone2003,ASD31544,15693
+695,2015/7/8,china,aaa695,phone806,ASD97088,15694
+696,2015/7/8,china,aaa696,phone1482,ASD11869,15695
+697,2015/7/8,china,aaa697,phone2059,ASD79306,15696
+698,2015/7/8,china,aaa698,phone461,ASD47039,15697
+699,2015/7/8,china,aaa699,phone2364,ASD92646,15698
+700,2015/7/8,china,aaa700,phone977,ASD26350,15699
+701,2015/7/8,china,aaa701,phone1148,ASD72903,15700
+702,2015/7/8,china,aaa702,phone2405,ASD98593,15701
+703,2015/7/8,china,aaa703,phone4,ASD8078,15702
+704,2015/7/8,china,aaa704,phone1376,ASD68677,15703
+705,2015/7/8,china,aaa705,phone1929,ASD86213,15704
+706,2015/7/8,china,aaa706,phone58,ASD12989,15705
+707,2015/7/8,china,aaa707,phone1974,ASD25534,15706
+708,2015/7/8,china,aaa708,phone2186,ASD22358,15707
+709,2015/7/8,china,aaa709,phone80,ASD32082,15708
+710,2015/7/8,china,aaa710,phone271,ASD6466,15709
+711,2015/7/8,china,aaa711,phone1600,ASD8619,15710
+712,2015/7/8,china,aaa712,phone780,ASD88786,15711
+713,2015/7/8,china,aaa713,phone1036,ASD67659,15712
+714,2015/7/8,china,aaa714,phone2511,ASD8513,15713
+715,2015/7/8,china,aaa715,phone2902,ASD73129,15714
+716,2015/7/8,china,aaa716,phone183,ASD4907,15715
+717,2015/7/8,china,aaa717,phone1458,ASD3023,15716
+718,2015/7/8,china,aaa718,phone1472,ASD74556,15717
+719,2015/7/8,china,aaa719,phone216,ASD67830,15718
+720,2015/7/8,china,aaa720,phone2248,ASD47899,15719
+721,2015/7/8,china,aaa721,phone105,ASD99347,15720
+722,2015/7/8,china,aaa722,phone334,ASD50262,15721
+723,2015/7/8,china,aaa723,phone1687,ASD45699,15722
+724,2015/7/8,china,aaa724,phone2794,ASD41977,15723
+725,2015/7/8,china,aaa725,phone2143,ASD18823,15724
+726,2015/7/8,china,aaa726,phone2166,ASD12014,15725
+727,2015/7/8,china,aaa727,phone838,ASD49156,15726
+728,2015/7/8,china,aaa728,phone2345,ASD96756,15727
+729,2015/7/8,china,aaa729,phone649,ASD17968,15728
+730,2015/7/8,china,aaa730,phone2974,ASD3384,15729
+731,2015/7/8,china,aaa731,phone911,ASD95311,15730
+732,2015/7/8,china,aaa732,phone612,ASD15305,15731
+733,2015/7/8,china,aaa733,phone1935,ASD97299,15732
+734,2015/7/8,china,aaa734,phone2044,ASD96688,15733
+735,2015/7/8,china,aaa735,phone2796,ASD55699,15734
+736,2015/7/8,china,aaa736,phone1519,ASD48720,15735
+737,2015/7/8,china,aaa737,phone2376,ASD91452,15736
+738,2015/7/8,china,aaa738,phone1331,ASD16918,15737
+739,2015/7/8,china,aaa739,phone2267,ASD50219,15738
+740,2015/7/8,china,aaa740,phone1104,ASD27032,15739
+741,2015/7/8,china,aaa741,phone617,ASD67442,15740
+742,2015/7/8,china,aaa742,phone1451,ASD46252,15741
+743,2015/7/8,china,aaa743,phone398,ASD20972,15742
+744,2015/7/8,china,aaa744,phone2007,ASD1018,15743
+745,2015/7/8,china,aaa745,phone18,ASD54919,15744
+746,2015/7/8,china,aaa746,phone1589,ASD61107,15745
+747,2015/7/8,china,aaa747,phone512,ASD71511,15746
+748,2015/7/8,china,aaa748,phone933,ASD61444,15747
+749,2015/7/8,china,aaa749,phone1271,ASD341,15748
+750,2015/7/8,china,aaa750,phone1826,ASD90555,15749
+751,2015/7/8,china,aaa751,phone223,ASD86899,15750
+752,2015/7/8,china,aaa752,phone2374,ASD85089,15751
+753,2015/7/8,china,aaa753,phone1683,ASD38383,15752
+754,2015/7/8,china,aaa754,phone2226,ASD50408,15753
+755,2015/7/8,china,aaa755,phone1124,ASD42246,15754
+756,2015/7/8,china,aaa756,phone2882,ASD23220,15755
+757,2015/7/8,china,aaa757,phone99,ASD91117,15756
+758,2015/7/8,china,aaa758,phone2590,ASD7295,15757
+759,2015/7/8,china,aaa759,phone863,ASD52426,15758
+760,2015/7/8,china,aaa760,phone1786,ASD24292,15759
+761,2015/7/8,china,aaa761,phone2826,ASD53395,15760
+762,2015/7/8,china,aaa762,phone2145,ASD94322,15761
+763,2015/7/8,china,aaa763,phone689,ASD93487,15762
+764,2015/7/8,china,aaa764,phone2326,ASD65102,15763
+765,2015/7/8,china,aaa765,phone614,ASD5636,15764
+766,2015/7/8,china,aaa766,phone2799,ASD48495,15765
+767,2015/7/8,china,aaa767,phone2385,ASD70717,15766
+768,2015/7/8,china,aaa768,phone2018,ASD63383,15767
+769,2015/7/8,china,aaa769,phone53,ASD15511,15768
+770,2015/7/8,china,aaa770,phone1201,ASD19134,15769
+771,2015/7/8,china,aaa771,phone851,ASD85009,15770
+772,2015/7/8,china,aaa772,phone2496,ASD15218,15771
+773,2015/7/8,china,aaa773,phone2441,ASD90633,15772
+774,2015/7/8,china,aaa774,phone256,ASD52156,15773
+775,2015/7/8,china,aaa775,phone1587,ASD94026,15774
+776,2015/7/8,china,aaa776,phone609,ASD54015,15775
+777,2015/7/8,china,aaa777,phone1548,ASD95032,15776
+778,2015/7/8,china,aaa778,phone2649,ASD69130,15777
+779,2015/7/8,china,aaa779,phone2978,ASD26464,15778
+780,2015/7/8,china,aaa780,phone1480,ASD56214,15779
+781,2015/7/8,china,aaa781,phone2588,ASD88133,15780
+782,2015/7/8,china,aaa782,phone675,ASD8012,15781
+783,2015/7/8,china,aaa783,phone2179,ASD16589,15782
+784,2015/7/8,china,aaa784,phone1730,ASD67148,15783
+785,2015/7/8,china,aaa785,phone1395,ASD5086,15784
+786,2015/7/8,china,aaa786,phone2788,ASD94870,15785
+787,2015/7/8,china,aaa787,phone820,ASD77271,15786
+788,2015/7/8,china,aaa788,phone2064,ASD14615,15787
+789,2015/7/8,china,aaa789,phone1423,ASD46576,15788
+790,2015/7/8,china,aaa790,phone2615,ASD67163,15789
+791,2015/7/8,china,aaa791,phone1122,ASD3869,15790
+792,2015/7/8,china,aaa792,phone2595,ASD16833,15791
+793,2015/7/8,china,aaa793,phone115,ASD92586,15792
+794,2015/7/8,china,aaa794,phone2173,ASD36027,15793
+795,2015/7/8,france,aaa795,phone1004,ASD41910,15794
+796,2015/7/8,france,aaa796,phone2785,ASD30573,15795
+797,2015/7/8,france,aaa797,phone657,ASD24464,15796
+798,2015/7/8,france,aaa798,phone1192,ASD76894,15797
+799,2015/7/8,france,aaa799,phone379,ASD93682,15798
+800,2015/7/8,france,aaa800,phone2,ASD53470,15799
+801,2015/7/8,france,aaa801,phone2245,ASD65691,15800
+802,2015/7/8,france,aaa802,phone1826,ASD90555,15801
+803,2015/7/8,france,aaa803,phone2103,ASD93488,15802
+804,2015/7/8,france,aaa804,phone730,ASD97134,15803
+805,2015/7/8,france,aaa805,phone1414,ASD22547,15804
+806,2015/7/8,france,aaa806,phone2406,ASD99089,15805
+807,2015/7/8,france,aaa807,phone1874,ASD16827,15806
+808,2015/7/8,france,aaa808,phone1801,ASD80314,15807
+809,2015/7/8,france,aaa809,phone56,ASD79971,15808
+810,2015/7/8,france,aaa810,phone378,ASD63016,15809
+811,2015/7/8,france,aaa811,phone1536,ASD99595,15810
+812,2015/7/8,france,aaa812,phone871,ASD54753,15811
+813,2015/7/8,france,aaa813,phone412,ASD58554,15812
+814,2015/7/8,france,aaa814,phone287,ASD14590,15813
+815,2015/7/8,france,aaa815,phone244,ASD24103,15814
+816,2015/7/8,france,aaa816,phone2629,ASD73394,15815
+817,2015/7/8,france,aaa817,phone1032,ASD564,15816
+818,2015/7/8,france,aaa818,phone1013,ASD88261,15817
+819,2015/7/8,france,aaa819,phone2584,ASD77392,15818
+820,2015/7/8,france,aaa820,phone348,ASD99862,15819
+821,2015/7/8,france,aaa821,phone2606,ASD80756,15820
+822,2015/7/8,france,aaa822,phone2077,ASD13520,15821
+823,2015/7/8,france,aaa823,phone118,ASD4568,15822
+824,2015/7/8,france,aaa824,phone2542,ASD14441,15823
+825,2015/7/8,france,aaa825,phone2371,ASD40072,15824
+826,2015/7/8,france,aaa826,phone424,ASD43956,15825
+827,2015/7/8,france,aaa827,phone2540,ASD37847,15826
+828,2015/7/8,france,aaa828,phone2899,ASD55392,15827
+829,2015/7/8,france,aaa829,phone2489,ASD14625,15828
+830,2015/7/8,france,aaa830,phone1390,ASD17540,15829
+831,2015/7/8,france,aaa831,phone1843,ASD49395,15830
+832,2015/7/8,france,aaa832,phone2596,ASD9629,15831
+833,2015/7/8,france,aaa833,phone2301,ASD14198,15832
+834,2015/7/8,france,aaa834,phone789,ASD96642,15833
+835,2015/7/8,france,aaa835,phone2390,ASD48294,15834
+836,2015/7/8,france,aaa836,phone148,ASD26033,15835
+837,2015/7/8,france,aaa837,phone2513,ASD49231,15836
+838,2015/7/8,france,aaa838,phone1713,ASD37721,15837
+839,2015/7/8,france,aaa839,phone2091,ASD9590,15838
+840,2015/7/8,france,aaa840,phone741,ASD6749,15839
+841,2015/7/8,france,aaa841,phone1564,ASD34639,15840
+842,2015/7/8,france,aaa842,phone281,ASD29929,15841
+843,2015/7/8,france,aaa843,phone2331,ASD3487,15842
+844,2015/7/8,france,aaa844,phone1499,ASD60999,15843
+845,2015/7/8,france,aaa845,phone268,ASD20002,15844
+846,2015/7/8,france,aaa846,phone1733,ASD25344,15845
+847,2015/7/8,france,aaa847,phone389,ASD2510,15846
+848,2015/7/8,france,aaa848,phone1341,ASD89162,15847
+849,2015/7/8,france,aaa849,phone1328,ASD14048,15848
+850,2015/7/8,france,aaa850,phone2675,ASD85267,15849
+851,2015/7/8,france,aaa851,phone2953,ASD3445,15850
+852,2015/7/8,france,aaa852,phone955,ASD52222,15851
+853,2015/7/8,france,aaa853,phone35,ASD58328,15852
+854,2015/7/8,france,aaa854,phone2466,ASD68864,15853
+855,2015/7/8,france,aaa855,phone2320,ASD47630,15854
+856,2015/7/8,france,aaa856,phone1711,ASD45523,15855
+857,2015/7/8,france,aaa857,phone1860,ASD84718,15856
+858,2015/7/8,france,aaa858,phone1572,ASD62120,15857
+859,2015/7/8,france,aaa859,phone413,ASD54204,15858
+860,2015/7/8,france,aaa860,phone730,ASD97134,15859
+861,2015/7/8,france,aaa861,phone2618,ASD95562,15860
+862,2015/7/8,china,aaa862,phone1216,ASD29709,15861
+863,2015/7/8,china,aaa863,phone1001,ASD67736,15862
+864,2015/7/8,china,aaa864,phone2818,ASD45929,15863
+865,2015/7/8,china,aaa865,phone527,ASD88720,15864
+866,2015/7/8,china,aaa866,phone2514,ASD24211,15865
+867,2015/7/8,china,aaa867,phone1814,ASD76811,15866
+868,2015/7/8,china,aaa868,phone1385,ASD31455,15867
+869,2015/7/8,china,aaa869,phone1226,ASD69240,15868
+870,2015/7/8,china,aaa870,phone1908,ASD99635,15869
+871,2015/7/8,china,aaa871,phone865,ASD20613,15870
+872,2015/7/8,china,aaa872,phone257,ASD49390,15871
+873,2015/7/8,china,aaa873,phone717,ASD35874,15872
+874,2015/7/8,china,aaa874,phone2089,ASD22717,15873
+875,2015/7/8,china,aaa875,phone299,ASD59117,15874
+876,2015/7/8,china,aaa876,phone1261,ASD26668,15875
+877,2015/7/8,china,aaa877,phone2011,ASD41588,15876
+878,2015/7/8,china,aaa878,phone1711,ASD45523,15877
+879,2015/7/8,china,aaa879,phone2345,ASD96756,15878
+880,2015/7/8,china,aaa880,phone1799,ASD65044,15879
+881,2015/7/8,china,aaa881,phone1117,ASD96168,15880
+882,2015/7/8,china,aaa882,phone585,ASD13591,15881
+883,2015/7/8,china,aaa883,phone2051,ASD57553,15882
+884,2015/7/8,china,aaa884,phone601,ASD54723,15883
+885,2015/7/8,china,aaa885,phone627,ASD63991,15884
+886,2015/7/8,china,aaa886,phone598,ASD1303,15885
+887,2015/7/8,china,aaa887,phone2518,ASD26102,15886
+888,2015/7/8,china,aaa888,phone629,ASD70956,15887
+889,2015/7/8,china,aaa889,phone2235,ASD66575,15888
+890,2015/7/8,china,aaa890,phone1559,ASD64160,15889
+891,2015/7/8,china,aaa891,phone2292,ASD10586,15890
+892,2015/7/8,china,aaa892,phone2595,ASD16833,15891
+893,2015/7/8,china,aaa893,phone1964,ASD61085,15892
+894,2015/7/8,china,aaa894,phone1074,ASD35633,15893
+895,2015/7/8,china,aaa895,phone2271,ASD52445,15894
+896,2015/7/8,china,aaa896,phone2823,ASD66619,15895
+897,2015/7/8,china,aaa897,phone873,ASD94122,15896
+898,2015/7/8,china,aaa898,phone1198,ASD56813,15897
+899,2015/7/8,china,aaa899,phone323,ASD18440,15898
+900,2015/7/8,china,aaa900,phone2465,ASD37753,15899
+901,2015/7/8,china,aaa901,phone2009,ASD14090,15900
+902,2015/7/8,china,aaa902,phone574,ASD18673,15901
+903,2015/7/8,china,aaa903,phone72,ASD64081,15902
+904,2015/7/8,china,aaa904,phone2654,ASD25597,15903
+905,2015/7/8,china,aaa905,phone2156,ASD42573,15904
+906,2015/7/8,china,aaa906,phone1378,ASD95469,15905
+907,2015/7/8,china,aaa907,phone825,ASD71225,15906
+908,2015/7/8,china,aaa908,phone1774,ASD12773,15907
+909,2015/7/8,china,aaa909,phone125,ASD76214,15908
+910,2015/7/8,china,aaa910,phone1587,ASD94026,15909
+911,2015/7/8,china,aaa911,phone180,ASD40830,15910
+912,2015/7/8,china,aaa912,phone2960,ASD80233,15911
+913,2015/7/8,china,aaa913,phone1406,ASD92347,15912
+914,2015/7/8,china,aaa914,phone33,ASD54379,15913
+915,2015/7/8,china,aaa915,phone1517,ASD34441,15914
+916,2015/7/8,china,aaa916,phone979,ASD92290,15915
+917,2015/7/8,china,aaa917,phone118,ASD4568,15916
+918,2015/7/8,china,aaa918,phone491,ASD56573,15917
+919,2015/7/8,china,aaa919,phone383,ASD91162,15918
+920,2015/7/8,china,aaa920,phone310,ASD71880,15919
+921,2015/7/8,china,aaa921,phone2742,ASD98716,15920
+922,2015/7/8,china,aaa922,phone448,ASD8620,15921
+923,2015/7/8,china,aaa923,phone2179,ASD16589,15922
+924,2015/7/8,china,aaa924,phone986,ASD24553,15923
+925,2015/7/8,china,aaa925,phone2413,ASD21779,15924
+926,2015/7/8,china,aaa926,phone2819,ASD32336,15925
+927,2015/7/8,china,aaa927,phone818,ASD56003,15926
+928,2015/7/8,china,aaa928,phone1797,ASD67181,15927
+929,2015/7/8,china,aaa929,phone1832,ASD65098,15928
+930,2015/7/8,china,aaa930,phone2080,ASD45469,15929
+931,2015/7/8,china,aaa931,phone1746,ASD68481,15930
+932,2015/7/8,china,aaa932,phone111,ASD20917,15931
+933,2015/7/8,china,aaa933,phone1823,ASD84425,15932
+934,2015/7/8,china,aaa934,phone2828,ASD70558,15933
+935,2015/7/8,china,aaa935,phone2371,ASD40072,15934
+936,2015/7/8,china,aaa936,phone446,ASD33003,15935
+937,2015/7/8,china,aaa937,phone977,ASD26350,15936
+938,2015/7/8,china,aaa938,phone402,ASD189,15937
+939,2015/7/8,china,aaa939,phone912,ASD65122,15938
+940,2015/7/8,china,aaa940,phone957,ASD71030,15939
+941,2015/7/8,china,aaa941,phone946,ASD53839,15940
+942,2015/7/8,china,aaa942,phone790,ASD67729,15941
+943,2015/7/8,china,aaa943,phone2377,ASD50473,15942
+944,2015/7/8,china,aaa944,phone2230,ASD91897,15943
+945,2015/7/8,china,aaa945,phone1524,ASD35504,15944
+946,2015/7/8,china,aaa946,phone2906,ASD36901,15945
+947,2015/7/8,china,aaa947,phone2651,ASD53635,15946
+948,2015/7/8,china,aaa948,phone1187,ASD61164,15947
+949,2015/7/8,china,aaa949,phone2699,ASD85196,15948
+950,2015/7/8,china,aaa950,phone596,ASD15295,15949
+951,2015/7/8,china,aaa951,phone1531,ASD24387,15950
+952,2015/7/8,china,aaa952,phone2534,ASD26601,15951
+953,2015/7/8,china,aaa953,phone208,ASD50274,15952
+954,2015/7/8,china,aaa954,phone87,ASD87141,15953
+955,2015/7/8,china,aaa955,phone1098,ASD91413,15954
+956,2015/7/8,china,aaa956,phone2807,ASD17710,15955
+957,2015/7/8,china,aaa957,phone1867,ASD99034,15956
+958,2015/7/8,china,aaa958,phone681,ASD96136,15957
+959,2015/7/8,china,aaa959,phone165,ASD10848,15958
+960,2015/7/8,china,aaa960,phone775,ASD47938,15959
+961,2015/7/8,china,aaa961,phone1695,ASD92758,15960
+962,2015/7/8,china,aaa962,phone2662,ASD85295,15961
+963,2015/7/8,china,aaa963,phone52,ASD46238,15962
+964,2015/7/8,china,aaa964,phone268,ASD20002,15963
+965,2015/7/8,china,aaa965,phone19,ASD69283,15964
+966,2015/7/8,china,aaa966,phone2250,ASD13619,15965
+967,2015/7/8,china,aaa967,phone1659,ASD49066,15966
+968,2015/7/8,china,aaa968,phone294,ASD59961,15967
+969,2015/7/8,china,aaa969,phone1537,ASD46367,15968
+970,2015/7/8,china,aaa970,phone2427,ASD76544,15969
+971,2015/7/8,china,aaa971,phone121,ASD56494,15970
+972,2015/7/8,china,aaa972,phone1683,ASD38383,15971
+973,2015/7/8,china,aaa973,phone1971,ASD50880,15972
+974,2015/7/8,china,aaa974,phone2423,ASD91802,15973
+975,2015/7/8,china,aaa975,phone2425,ASD69636,15974
+976,2015/7/8,china,aaa976,phone1677,ASD88004,15975
+977,2015/7/8,china,aaa977,phone1057,ASD75780,15976
+978,2015/7/8,china,aaa978,phone568,ASD95931,15977
+979,2015/7/8,china,aaa979,phone2629,ASD73394,15978
+980,2015/7/8,china,aaa980,phone2778,ASD48823,15979
+981,2015/7/8,china,aaa981,phone1875,ASD28544,15980
+982,2015/7/8,china,aaa982,phone2257,ASD41709,15981
+983,2015/7/8,china,aaa983,phone2519,ASD60789,15982
+984,2015/7/8,china,aaa984,phone892,ASD47520,15983
+985,2015/7/8,china,aaa985,phone1096,ASD92338,15984
+986,2015/7/8,china,aaa986,phone691,ASD93561,15985
+987,2015/7/8,china,aaa987,phone1437,ASD46394,15986
+988,2015/7/8,china,aaa988,phone2150,ASD32526,15987
+989,2015/7/8,china,aaa989,phone2367,ASD25023,15988
+990,2015/7/8,china,aaa990,phone2413,ASD21779,15989
+991,2015/7/8,china,aaa991,phone2601,ASD4891,15990
+992,2015/7/8,china,aaa992,phone395,ASD63501,15991
+993,2015/7/8,china,aaa993,phone165,ASD10848,15992
+994,2015/7/8,china,aaa994,phone1553,ASD58083,15993
+995,2015/7/8,china,aaa995,phone90,ASD81472,15994
+996,2015/7/8,china,aaa996,phone1421,ASD33736,15995
+997,2015/7/8,china,aaa997,phone1588,ASD91226,15996
+998,2015/7/8,china,aaa998,phone1862,ASD47472,15997
+999,2015/7/8,china,aaa999,phone309,ASD78895,15998
+1000,2015/7/8,china,aaa1000,phone1318,ASD85027,15999
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/processing/src/test/resources/csv/data.csv.bz2
----------------------------------------------------------------------
diff --git a/processing/src/test/resources/csv/data.csv.bz2 b/processing/src/test/resources/csv/data.csv.bz2
new file mode 100644
index 0000000..72ea1b0
Binary files /dev/null and b/processing/src/test/resources/csv/data.csv.bz2 differ
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/processing/src/test/resources/csv/data.csv.gz
----------------------------------------------------------------------
diff --git a/processing/src/test/resources/csv/data.csv.gz b/processing/src/test/resources/csv/data.csv.gz
new file mode 100644
index 0000000..a523280
Binary files /dev/null and b/processing/src/test/resources/csv/data.csv.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/processing/src/test/resources/csv/data.csv.lz4
----------------------------------------------------------------------
diff --git a/processing/src/test/resources/csv/data.csv.lz4 b/processing/src/test/resources/csv/data.csv.lz4
new file mode 100644
index 0000000..5288e02
Binary files /dev/null and b/processing/src/test/resources/csv/data.csv.lz4 differ
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/processing/src/test/resources/csv/data.csv.snappy
----------------------------------------------------------------------
diff --git a/processing/src/test/resources/csv/data.csv.snappy b/processing/src/test/resources/csv/data.csv.snappy
new file mode 100644
index 0000000..ff8c8f0
Binary files /dev/null and b/processing/src/test/resources/csv/data.csv.snappy differ
[2/3] incubator-carbondata git commit: csvReader code improvements
Posted by ja...@apache.org.
csvReader code improvements
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d6ceb1d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d6ceb1d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d6ceb1d3
Branch: refs/heads/master
Commit: d6ceb1d3dd5110a85e861b5b5b2dd1021d1714ef
Parents: 25de27f
Author: Jihong Ma <ji...@apache.org>
Authored: Mon Jan 30 21:41:18 2017 -0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue Feb 14 15:08:20 2017 -0800
----------------------------------------------------------------------
.../carbondata/hadoop/CarbonInputFormat.java | 6 +-
.../carbondata/hadoop/csv/CSVInputFormat.java | 288 -----
.../recorditerator/RecordReaderIterator.java | 98 --
.../hadoop/io/BoundedInputStream.java | 129 ---
.../hadoop/io/StringArrayWritable.java | 70 --
.../hadoop/readsupport/CarbonReadSupport.java | 15 +-
.../AbstractDictionaryDecodedReadSupport.java | 84 --
.../impl/ArrayWritableReadSupport.java | 48 -
.../impl/DictionaryDecodeReadSupport.java | 94 ++
.../impl/DictionaryDecodedReadSupportImpl.java | 34 -
.../readsupport/impl/RawDataReadSupport.java | 17 +-
.../hadoop/csv/CSVInputFormatTest.java | 169 ---
hadoop/src/test/resources/data.csv.bz2 | Bin 10572 -> 0 bytes
hadoop/src/test/resources/data.csv.gz | Bin 14710 -> 0 bytes
hadoop/src/test/resources/data.csv.lz4 | Bin 24495 -> 0 bytes
hadoop/src/test/resources/data.csv.snappy | Bin 24263 -> 0 bytes
.../spark/rdd/NewCarbonDataLoadRDD.scala | 8 +-
.../carbondata/spark/util/CommonUtil.scala | 2 +-
.../spark/util/GlobalDictionaryUtil.scala | 4 +-
.../readsupport/SparkRowReadSupportImpl.java | 4 +-
.../readsupport/SparkRowReadSupportImpl.java | 4 +-
.../processing/csvload/BoundedInputStream.java | 129 +++
.../processing/csvload/CSVInputFormat.java | 285 +++++
.../csvload/CSVRecordReaderIterator.java | 97 ++
.../processing/csvload/StringArrayWritable.java | 70 ++
.../processing/csvload/CSVInputFormatTest.java | 167 +++
processing/src/test/resources/csv/data.csv | 1001 ++++++++++++++++++
processing/src/test/resources/csv/data.csv.bz2 | Bin 0 -> 10572 bytes
processing/src/test/resources/csv/data.csv.gz | Bin 0 -> 14710 bytes
processing/src/test/resources/csv/data.csv.lz4 | Bin 0 -> 24495 bytes
.../src/test/resources/csv/data.csv.snappy | Bin 0 -> 24263 bytes
31 files changed, 1870 insertions(+), 953 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 4f81438..8187089 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -60,7 +60,7 @@ import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
-import org.apache.carbondata.hadoop.readsupport.impl.DictionaryDecodedReadSupportImpl;
+import org.apache.carbondata.hadoop.readsupport.impl.DictionaryDecodeReadSupport;
import org.apache.carbondata.hadoop.util.BlockLevelTraverser;
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
@@ -641,7 +641,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
public CarbonReadSupport<T> getReadSupportClass(Configuration configuration) {
String readSupportClass = configuration.get(CARBON_READ_SUPPORT);
//By default it uses dictionary decoder read class
- CarbonReadSupport readSupport = null;
+ CarbonReadSupport<T> readSupport = null;
if (readSupportClass != null) {
try {
Class<?> myClass = Class.forName(readSupportClass);
@@ -656,7 +656,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
LOG.error("Error while creating " + readSupportClass, ex);
}
} else {
- readSupport = new DictionaryDecodedReadSupportImpl();
+ readSupport = new DictionaryDecodeReadSupport<>();
}
return readSupport;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/csv/CSVInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/csv/CSVInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/csv/CSVInputFormat.java
deleted file mode 100644
index 9e35d13..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/csv/CSVInputFormat.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.hadoop.csv;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-
-import org.apache.carbondata.hadoop.io.BoundedInputStream;
-import org.apache.carbondata.hadoop.io.StringArrayWritable;
-
-import com.univocity.parsers.csv.CsvParser;
-import com.univocity.parsers.csv.CsvParserSettings;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.CodecPool;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
-import org.apache.hadoop.io.compress.CompressionInputStream;
-import org.apache.hadoop.io.compress.Decompressor;
-import org.apache.hadoop.io.compress.SplitCompressionInputStream;
-import org.apache.hadoop.io.compress.SplittableCompressionCodec;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-import org.apache.hadoop.util.LineReader;
-
-/**
- * An {@link org.apache.hadoop.mapreduce.InputFormat} for csv files. Files are broken into lines.
- * Values are the line of csv files.
- */
-public class CSVInputFormat extends FileInputFormat<NullWritable, StringArrayWritable> {
-
- public static final String DELIMITER = "carbon.csvinputformat.delimiter";
- public static final String DELIMITER_DEFAULT = ",";
- public static final String COMMENT = "carbon.csvinputformat.comment";
- public static final String COMMENT_DEFAULT = "#";
- public static final String QUOTE = "carbon.csvinputformat.quote";
- public static final String QUOTE_DEFAULT = "\"";
- public static final String ESCAPE = "carbon.csvinputformat.escape";
- public static final String ESCAPE_DEFAULT = "\\";
- public static final String HEADER_PRESENT = "caron.csvinputformat.header.present";
- public static final boolean HEADER_PRESENT_DEFAULT = false;
- public static final String READ_BUFFER_SIZE = "carbon.csvinputformat.read.buffer.size";
- public static final String READ_BUFFER_SIZE_DEFAULT = "65536";
-
- @Override
- public RecordReader<NullWritable, StringArrayWritable> createRecordReader(InputSplit inputSplit,
- TaskAttemptContext context) throws IOException, InterruptedException {
- return new CSVRecordReader();
- }
-
- @Override
- protected boolean isSplitable(JobContext context, Path file) {
- final CompressionCodec codec = new CompressionCodecFactory(context.getConfiguration())
- .getCodec(file);
- if (null == codec) {
- return true;
- }
- return codec instanceof SplittableCompressionCodec;
- }
-
- /**
- * Sets the comment char to configuration. Default it is #.
- * @param configuration
- * @param commentChar
- */
- public static void setCommentCharacter(Configuration configuration, String commentChar) {
- if (commentChar != null && !commentChar.isEmpty()) {
- configuration.set(COMMENT, commentChar);
- }
- }
-
- /**
- * Sets the delimiter to configuration. Default it is ','
- * @param configuration
- * @param delimiter
- */
- public static void setCSVDelimiter(Configuration configuration, String delimiter) {
- if (delimiter != null && !delimiter.isEmpty()) {
- configuration.set(DELIMITER, delimiter);
- }
- }
-
- /**
- * Sets the escape character to configuration. Default it is \
- * @param configuration
- * @param escapeCharacter
- */
- public static void setEscapeCharacter(Configuration configuration, String escapeCharacter) {
- if (escapeCharacter != null && !escapeCharacter.isEmpty()) {
- configuration.set(ESCAPE, escapeCharacter);
- }
- }
-
- /**
- * Whether header needs to read from csv or not. By default it is false.
- * @param configuration
- * @param headerExtractEnable
- */
- public static void setHeaderExtractionEnabled(Configuration configuration,
- boolean headerExtractEnable) {
- configuration.set(HEADER_PRESENT, String.valueOf(headerExtractEnable));
- }
-
- /**
- * Sets the quote character to configuration. Default it is "
- * @param configuration
- * @param quoteCharacter
- */
- public static void setQuoteCharacter(Configuration configuration, String quoteCharacter) {
- if (quoteCharacter != null && !quoteCharacter.isEmpty()) {
- configuration.set(QUOTE, quoteCharacter);
- }
- }
-
- /**
- * Sets the read buffer size to configuration.
- * @param configuration
- * @param bufferSize
- */
- public static void setReadBufferSize(Configuration configuration, String bufferSize) {
- if (bufferSize != null && !bufferSize.isEmpty()) {
- configuration.set(READ_BUFFER_SIZE, bufferSize);
- }
- }
-
- /**
- * Treats value as line in file. Key is null.
- */
- public static class CSVRecordReader extends RecordReader<NullWritable, StringArrayWritable> {
-
- private long start;
- private long end;
- private BoundedInputStream boundedInputStream;
- private Reader reader;
- private CsvParser csvParser;
- private StringArrayWritable value;
- private String[] columns;
- private Seekable filePosition;
- private boolean isCompressedInput;
- private Decompressor decompressor;
-
- @Override
- public void initialize(InputSplit inputSplit, TaskAttemptContext context)
- throws IOException, InterruptedException {
- FileSplit split = (FileSplit) inputSplit;
- start = split.getStart();
- end = start + split.getLength();
- Path file = split.getPath();
- Configuration job = context.getConfiguration();
- CompressionCodec codec = (new CompressionCodecFactory(job)).getCodec(file);
- FileSystem fs = file.getFileSystem(job);
- int bufferSize = Integer.parseInt(job.get(READ_BUFFER_SIZE, READ_BUFFER_SIZE_DEFAULT));
- FSDataInputStream fileIn = fs.open(file, bufferSize);
- InputStream inputStream;
- if (codec != null) {
- isCompressedInput = true;
- decompressor = CodecPool.getDecompressor(codec);
- if (codec instanceof SplittableCompressionCodec) {
- SplitCompressionInputStream scIn = ((SplittableCompressionCodec) codec)
- .createInputStream(fileIn, decompressor, start, end, SplittableCompressionCodec
- .READ_MODE.BYBLOCK);
- start = scIn.getAdjustedStart();
- end = scIn.getAdjustedEnd();
- if (start != 0) {
- LineReader lineReader = new LineReader(scIn, 1);
- start += lineReader.readLine(new Text(), 0);
- }
- filePosition = scIn;
- inputStream = scIn;
- } else {
- CompressionInputStream cIn = codec.createInputStream(fileIn, decompressor);
- filePosition = cIn;
- inputStream = cIn;
- }
- } else {
- fileIn.seek(start);
- if (start != 0) {
- LineReader lineReader = new LineReader(fileIn, 1);
- start += lineReader.readLine(new Text(), 0);
- }
- boundedInputStream = new BoundedInputStream(fileIn, end - start);
- filePosition = fileIn;
- inputStream = boundedInputStream;
- }
- reader = new InputStreamReader(inputStream);
- csvParser = new CsvParser(extractCsvParserSettings(job));
- csvParser.beginParsing(reader);
- }
-
- private CsvParserSettings extractCsvParserSettings(Configuration job) {
- CsvParserSettings parserSettings = new CsvParserSettings();
- parserSettings.getFormat().setDelimiter(job.get(DELIMITER, DELIMITER_DEFAULT).charAt(0));
- parserSettings.getFormat().setComment(job.get(COMMENT, COMMENT_DEFAULT).charAt(0));
- parserSettings.setLineSeparatorDetectionEnabled(true);
- parserSettings.setNullValue("");
- parserSettings.setIgnoreLeadingWhitespaces(false);
- parserSettings.setIgnoreTrailingWhitespaces(false);
- parserSettings.setSkipEmptyLines(false);
- // TODO get from csv file.
- parserSettings.setMaxColumns(1000);
- parserSettings.getFormat().setQuote(job.get(QUOTE, QUOTE_DEFAULT).charAt(0));
- parserSettings.getFormat().setQuoteEscape(job.get(ESCAPE, ESCAPE_DEFAULT).charAt(0));
- if (start == 0) {
- parserSettings.setHeaderExtractionEnabled(job.getBoolean(HEADER_PRESENT,
- HEADER_PRESENT_DEFAULT));
- }
- return parserSettings;
- }
-
- @Override
- public boolean nextKeyValue() throws IOException, InterruptedException {
- columns = csvParser.parseNext();
- if (columns == null) {
- value = null;
- return false;
- }
- if (value == null) {
- value = new StringArrayWritable();
- }
- value.set(columns);
- return true;
- }
-
- @Override
- public NullWritable getCurrentKey() throws IOException, InterruptedException {
- return NullWritable.get();
- }
-
- @Override
- public StringArrayWritable getCurrentValue() throws IOException, InterruptedException {
- return value;
- }
-
- private long getPos() throws IOException {
- long retVal = start;
- if (null != boundedInputStream) {
- retVal = end - boundedInputStream.getRemaining();
- } else if (isCompressedInput && null != filePosition) {
- retVal = filePosition.getPos();
- }
- return retVal;
- }
-
- @Override
- public float getProgress() throws IOException, InterruptedException {
- return start == end ? 0.0F : Math.min(1.0F, (float) (getPos() -
- start) / (float) (end - start));
- }
-
- @Override
- public void close() throws IOException {
- try {
- if (reader != null) {
- reader.close();
- }
- } finally {
- if (decompressor != null) {
- CodecPool.returnDecompressor(decompressor);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/csv/recorditerator/RecordReaderIterator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/csv/recorditerator/RecordReaderIterator.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/csv/recorditerator/RecordReaderIterator.java
deleted file mode 100644
index 39dd916..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/csv/recorditerator/RecordReaderIterator.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.hadoop.csv.recorditerator;
-
-import java.io.IOException;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.hadoop.io.StringArrayWritable;
-import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
-
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-/**
- * It is wrapper iterator around @{@link RecordReader}.
- */
-public class RecordReaderIterator extends CarbonIterator<Object []> {
-
- private RecordReader<NullWritable, StringArrayWritable> recordReader;
-
- /**
- * It is just a little hack to make recordreader as iterator. Usually we cannot call hasNext
- * multiple times on record reader as it moves another line. To avoid that situation like hasNext
- * only tells whether next row is present or not and next will move the pointer to next row after
- * consuming it.
- */
- private boolean isConsumed;
-
- private InputSplit split;
-
- private TaskAttemptContext context;
-
- public RecordReaderIterator(RecordReader<NullWritable, StringArrayWritable> recordReader,
- InputSplit split, TaskAttemptContext context) {
- this.recordReader = recordReader;
- this.split = split;
- this.context = context;
- }
-
- @Override
- public boolean hasNext() {
- try {
- if (!isConsumed) {
- isConsumed = recordReader.nextKeyValue();
- return isConsumed;
- }
- return true;
- } catch (Exception e) {
- throw new CarbonDataLoadingException(e);
- }
- }
-
- @Override
- public Object[] next() {
- try {
- String[] data = recordReader.getCurrentValue().get();
- isConsumed = false;
- return data;
- } catch (Exception e) {
- throw new CarbonDataLoadingException(e);
- }
- }
-
- @Override
- public void initialize() {
- try {
- recordReader.initialize(split, context);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void close() {
- try {
- recordReader.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/io/BoundedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/io/BoundedInputStream.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/io/BoundedInputStream.java
deleted file mode 100644
index 4451400..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/io/BoundedInputStream.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.hadoop.io;
-
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Custom reader class to read the data from file it will take care of reading
- * till the limit assigned to this class
- */
-public class BoundedInputStream extends InputStream {
-
- /**
- * byte value of the new line character
- */
- private static final byte END_OF_LINE_BYTE_VALUE = '\n';
-
- /**
- * number of extra character to read
- */
- private static final int NUMBER_OF_EXTRA_CHARACTER_TO_READ = 100;
-
- /**
- * number of bytes remaining
- */
- private long remaining;
- /**
- * to check whether end of line is found
- */
- private boolean endOfLineFound = false;
-
- private DataInputStream in;
-
- public BoundedInputStream(DataInputStream in, long limit) {
- this.in = in;
- this.remaining = limit;
- }
-
- /**
- * Below method will be used to read the data from file
- *
- * @throws IOException
- * problem while reading
- */
- @Override
- public int read() throws IOException {
- if (this.remaining == 0) {
- return -1;
- } else {
- int var1 = this.in.read();
- if (var1 >= 0) {
- --this.remaining;
- }
-
- return var1;
- }
- }
-
- /**
- * Below method will be used to read the data from file. If limit reaches in
- * that case it will read until new line character is reached
- *
- * @param buffer
- * buffer in which data will be read
- * @param offset
- * from position to buffer will be filled
- * @param length
- * number of character to be read
- * @throws IOException
- * problem while reading
- */
- @Override
- public int read(byte[] buffer, int offset, int length) throws IOException {
- if (this.remaining == 0) {
- return -1;
- } else {
- if (this.remaining < length) {
- length = (int) this.remaining;
- }
-
- length = this.in.read(buffer, offset, length);
- if (length >= 0) {
- this.remaining -= length;
- if (this.remaining == 0 && !endOfLineFound) {
- endOfLineFound = true;
- this.remaining += NUMBER_OF_EXTRA_CHARACTER_TO_READ;
- } else if (endOfLineFound) {
- int end = offset + length;
- for (int i = offset; i < end; i++) {
- if (buffer[i] == END_OF_LINE_BYTE_VALUE) {
- this.remaining = 0;
- return (i - offset) + 1;
- }
- }
- this.remaining += NUMBER_OF_EXTRA_CHARACTER_TO_READ;
- }
- }
- return length;
- }
- }
-
- @Override
- public void close() throws IOException {
- if (in != null) {
- in.close();
- }
- }
-
- public long getRemaining() {
- return this.remaining;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/io/StringArrayWritable.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/io/StringArrayWritable.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/io/StringArrayWritable.java
deleted file mode 100644
index 6f5ae43..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/io/StringArrayWritable.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.hadoop.io;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.Arrays;
-
-import org.apache.hadoop.io.Writable;
-
-/**
- * A String sequence that is usable as a key or value.
- */
-public class StringArrayWritable implements Writable {
- private String[] values;
-
- public String[] toStrings() {
- return values;
- }
-
- public void set(String[] values) {
- this.values = values;
- }
-
- public String[] get() {
- return values;
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- int length = in.readInt();
- values = new String[length];
- for (int i = 0; i < length; i++) {
- byte[] b = new byte[in.readInt()];
- in.readFully(b);
- values[i] = new String(b, Charset.defaultCharset());
- }
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(values.length); // write values
- for (int i = 0; i < values.length; i++) {
- byte[] b = values[i].getBytes(Charset.defaultCharset());
- out.writeInt(b.length);
- out.write(b);
- }
- }
-
- @Override
- public String toString() {
- return Arrays.toString(values);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
index 0ee23c9..b535aea 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
@@ -22,24 +22,27 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
/**
- * It converts to the desired class while reading the rows from RecordReader
+ * This is the interface to convert data reading from RecordReader to row representation.
*/
public interface CarbonReadSupport<T> {
/**
- * It can use [{@link CarbonColumn}] array to create its own schema to create its row.
+ * Initialization if needed based on the projected column list
*
- * @param carbonColumns
+ * @param carbonColumns column list
+ * @param absoluteTableIdentifier table identifier
*/
void initialize(CarbonColumn[] carbonColumns,
AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException;
+ /**
+ * convert column data back to row representation
+ * @param data column data
+ */
T readRow(Object[] data);
/**
- * This method will be used to clear the dictionary cache and update access count for each
- * column involved which will be used during eviction of columns from LRU cache if memory
- * reaches threshold
+ * cleanup step if necessary
*/
void close();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
deleted file mode 100644
index 7723737..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.hadoop.readsupport.impl;
-
-import java.io.IOException;
-
-import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
-
-/**
- * Its an abstract class provides necessary information to decode dictionary data
- */
-public abstract class AbstractDictionaryDecodedReadSupport<T> implements CarbonReadSupport<T> {
-
- protected Dictionary[] dictionaries;
-
- protected DataType[] dataTypes;
- /**
- * carbon columns
- */
- protected CarbonColumn[] carbonColumns;
-
- /**
- * It would be instantiated in side the task so the dictionary would be loaded inside every mapper
- * instead of driver.
- *
- * @param carbonColumns
- * @param absoluteTableIdentifier
- */
- @Override public void initialize(CarbonColumn[] carbonColumns,
- AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
- this.carbonColumns = carbonColumns;
- dictionaries = new Dictionary[carbonColumns.length];
- dataTypes = new DataType[carbonColumns.length];
- for (int i = 0; i < carbonColumns.length; i++) {
- if (carbonColumns[i].hasEncoding(Encoding.DICTIONARY) && !carbonColumns[i]
- .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- CacheProvider cacheProvider = CacheProvider.getInstance();
- Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider
- .createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath());
- dataTypes[i] = carbonColumns[i].getDataType();
- dictionaries[i] = forwardDictionaryCache.get(new DictionaryColumnUniqueIdentifier(
- absoluteTableIdentifier.getCarbonTableIdentifier(),
- carbonColumns[i].getColumnIdentifier(), dataTypes[i]));
- } else {
- dataTypes[i] = carbonColumns[i].getDataType();
- }
- }
- }
-
- /**
- * This method iwll be used to clear the dictionary cache and update access count for each
- * column involved which will be used during eviction of columns from LRU cache if memory
- * reaches threshold
- */
- @Override public void close() {
- for (int i = 0; i < dictionaries.length; i++) {
- CarbonUtil.clearDictionaryCache(dictionaries[i]);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/ArrayWritableReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/ArrayWritableReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/ArrayWritableReadSupport.java
deleted file mode 100644
index 50272e5..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/ArrayWritableReadSupport.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.hadoop.readsupport.impl;
-
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
-
-import org.apache.hadoop.io.ArrayWritable;
-
-public class ArrayWritableReadSupport implements CarbonReadSupport<ArrayWritable> {
-
- @Override public void initialize(CarbonColumn[] carbonColumns,
- AbsoluteTableIdentifier absoluteTableIdentifier) {
- }
-
- @Override public ArrayWritable readRow(Object[] data) {
-
- String[] writables = new String[data.length];
- for (int i = 0; i < data.length; i++) {
- writables[i] = data[i].toString();
- }
- return new ArrayWritable(writables);
- }
-
- /**
- * This method iwll be used to clear the dictionary cache and update access count for each
- * column involved which will be used during eviction of columns from LRU cache if memory
- * reaches threshold
- */
- @Override public void close() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
new file mode 100644
index 0000000..43953d0
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.hadoop.readsupport.impl;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
+
+/**
+ * This is the class to decode dictionary encoded column data back to its original value.
+ */
+public class DictionaryDecodeReadSupport<T> implements CarbonReadSupport<T> {
+
+ protected Dictionary[] dictionaries;
+
+ protected DataType[] dataTypes;
+ /**
+ * carbon columns
+ */
+ protected CarbonColumn[] carbonColumns;
+
+ /**
+ * This initialization is done inside executor task
+ * for column dictionary involved in decoding.
+ *
+ * @param carbonColumns column list
+ * @param absoluteTableIdentifier table identifier
+ */
+ @Override public void initialize(CarbonColumn[] carbonColumns,
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
+ this.carbonColumns = carbonColumns;
+ dictionaries = new Dictionary[carbonColumns.length];
+ dataTypes = new DataType[carbonColumns.length];
+ for (int i = 0; i < carbonColumns.length; i++) {
+ if (carbonColumns[i].hasEncoding(Encoding.DICTIONARY) && !carbonColumns[i]
+ .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ CacheProvider cacheProvider = CacheProvider.getInstance();
+ Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider
+ .createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath());
+ dataTypes[i] = carbonColumns[i].getDataType();
+ dictionaries[i] = forwardDictionaryCache.get(new DictionaryColumnUniqueIdentifier(
+ absoluteTableIdentifier.getCarbonTableIdentifier(),
+ carbonColumns[i].getColumnIdentifier(), dataTypes[i]));
+ } else {
+ dataTypes[i] = carbonColumns[i].getDataType();
+ }
+ }
+ }
+
+ @Override public T readRow(Object[] data) {
+ assert (data.length == dictionaries.length);
+ for (int i = 0; i < dictionaries.length; i++) {
+ if (dictionaries[i] != null) {
+ data[i] = dictionaries[i].getDictionaryValueForKey((int) data[i]);
+ }
+ }
+ return (T)data;
+ }
+
+ /**
+ * to book keep the dictionary cache or update access count for each
+ * column involved during decode, to facilitate LRU cache policy if memory
+ * threshold is reached
+ */
+ @Override public void close() {
+ for (int i = 0; i < dictionaries.length; i++) {
+ CarbonUtil.clearDictionaryCache(dictionaries[i]);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodedReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodedReadSupportImpl.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodedReadSupportImpl.java
deleted file mode 100644
index ec02ab0..0000000
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodedReadSupportImpl.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.hadoop.readsupport.impl;
-
-/**
- * It decodes the dictionary values to actual values.
- */
-public class DictionaryDecodedReadSupportImpl
- extends AbstractDictionaryDecodedReadSupport<Object[]> {
-
- @Override public Object[] readRow(Object[] data) {
- assert (data.length == dictionaries.length);
- for (int i = 0; i < dictionaries.length; i++) {
- if (dictionaries[i] != null) {
- data[i] = dictionaries[i].getDictionaryValueForKey((int) data[i]);
- }
- }
- return data;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java
index f627b26..7e7d414 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java
@@ -27,26 +27,17 @@ public class RawDataReadSupport implements CarbonReadSupport<InternalRow> {
@Override
public void initialize(CarbonColumn[] carbonColumns,
- AbsoluteTableIdentifier absoluteTableIdentifier) {
- }
+ AbsoluteTableIdentifier absoluteTableIdentifier) { }
/**
- * Just return same data.
+ * return column data as InternalRow
*
- * @param data
- * @return
+ * @param data column data
*/
@Override
public InternalRow readRow(Object[] data) {
return new GenericInternalRow(data);
}
- /**
- * This method iwll be used to clear the dictionary cache and update access count for each
- * column involved which will be used during eviction of columns from LRU cache if memory
- * reaches threshold
- */
- @Override public void close() {
-
- }
+ @Override public void close() { }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/test/java/org/apache/carbondata/hadoop/csv/CSVInputFormatTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/csv/CSVInputFormatTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/csv/CSVInputFormatTest.java
deleted file mode 100644
index 38b8b52..0000000
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/csv/CSVInputFormatTest.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.hadoop.csv;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import org.apache.carbondata.hadoop.io.StringArrayWritable;
-
-import junit.framework.TestCase;
-import org.junit.Assert;
-import org.junit.Test;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.compress.BZip2Codec;
-import org.apache.hadoop.io.compress.CompressionOutputStream;
-import org.apache.hadoop.io.compress.GzipCodec;
-import org.apache.hadoop.io.compress.Lz4Codec;
-import org.apache.hadoop.io.compress.SnappyCodec;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-
-public class CSVInputFormatTest extends TestCase {
-
- /**
- * generate compressed files, no need to call this method.
- * @throws Exception
- */
- public void generateCompressFiles() throws Exception {
- String pwd = new File("src/test/resources").getCanonicalPath();
- String inputFile = pwd + "/data.csv";
- FileInputStream input = new FileInputStream(inputFile);
- Configuration conf = new Configuration();
-
- // .gz
- String outputFile = pwd + "/data.csv.gz";
- FileOutputStream output = new FileOutputStream(outputFile);
- GzipCodec gzip = new GzipCodec();
- gzip.setConf(conf);
- CompressionOutputStream outputStream = gzip.createOutputStream(output);
- int i = -1;
- while ((i = input.read()) != -1) {
- outputStream.write(i);
- }
- outputStream.close();
- input.close();
-
- // .bz2
- input = new FileInputStream(inputFile);
- outputFile = pwd + "/data.csv.bz2";
- output = new FileOutputStream(outputFile);
- BZip2Codec bzip2 = new BZip2Codec();
- bzip2.setConf(conf);
- outputStream = bzip2.createOutputStream(output);
- i = -1;
- while ((i = input.read()) != -1) {
- outputStream.write(i);
- }
- outputStream.close();
- input.close();
-
- // .snappy
- input = new FileInputStream(inputFile);
- outputFile = pwd + "/data.csv.snappy";
- output = new FileOutputStream(outputFile);
- SnappyCodec snappy = new SnappyCodec();
- snappy.setConf(conf);
- outputStream = snappy.createOutputStream(output);
- i = -1;
- while ((i = input.read()) != -1) {
- outputStream.write(i);
- }
- outputStream.close();
- input.close();
-
- //.lz4
- input = new FileInputStream(inputFile);
- outputFile = pwd + "/data.csv.lz4";
- output = new FileOutputStream(outputFile);
- Lz4Codec lz4 = new Lz4Codec();
- lz4.setConf(conf);
- outputStream = lz4.createOutputStream(output);
- i = -1;
- while ((i = input.read()) != -1) {
- outputStream.write(i);
- }
- outputStream.close();
- input.close();
-
- }
-
- /**
- * CSVCheckMapper check the content of csv files.
- */
- public static class CSVCheckMapper extends Mapper<NullWritable, StringArrayWritable, NullWritable,
- NullWritable> {
- @Override
- protected void map(NullWritable key, StringArrayWritable value, Context context)
- throws IOException, InterruptedException {
- String[] columns = value.get();
- int id = Integer.parseInt(columns[0]);
- int salary = Integer.parseInt(columns[6]);
- Assert.assertEquals(id - 1, salary - 15000);
- }
- }
-
- /**
- * test read csv files
- * @throws Exception
- */
- @Test public void testReadCSVFiles() throws Exception{
- Configuration conf = new Configuration();
- prepareConf(conf);
- Job job = Job.getInstance(conf, "CSVInputFormat_normal");
- job.setJarByClass(CSVInputFormatTest.class);
- job.setMapperClass(CSVCheckMapper.class);
- job.setNumReduceTasks(0);
- job.setInputFormatClass(CSVInputFormat.class);
-
- String inputFolder = new File("src/test/resources").getCanonicalPath();
- FileInputFormat.addInputPath(job, new Path(inputFolder + File.separator + "data.csv"));
- FileInputFormat.addInputPath(job, new Path(inputFolder + File.separator + "data.csv.bz2"));
- FileInputFormat.addInputPath(job, new Path(inputFolder + File.separator + "data.csv.gz"));
- // FileInputFormat.addInputPath(job, new Path(inputFolder + File.separator + "data.csv.lz4"));
- // FileInputFormat.addInputPath(job, new Path(inputFolder + File.separator + "data.csv.snappy"));
-
- File output = new File("target/output_CSVInputFormatTest");
- deleteOutput(output);
- FileOutputFormat.setOutputPath(job, new Path(output.getCanonicalPath()));
-
- Assert.assertTrue(job.waitForCompletion(true));
- }
-
- private void prepareConf(Configuration conf) {
- conf.setBoolean(CSVInputFormat.HEADER_PRESENT, true);
- }
-
- private void deleteOutput(File output) {
- if (output.exists()) {
- if (output.isDirectory()) {
- for(File file : output.listFiles()) {
- deleteOutput(file);
- }
- output.delete();
- } else {
- output.delete();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/test/resources/data.csv.bz2
----------------------------------------------------------------------
diff --git a/hadoop/src/test/resources/data.csv.bz2 b/hadoop/src/test/resources/data.csv.bz2
deleted file mode 100644
index 72ea1b0..0000000
Binary files a/hadoop/src/test/resources/data.csv.bz2 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/test/resources/data.csv.gz
----------------------------------------------------------------------
diff --git a/hadoop/src/test/resources/data.csv.gz b/hadoop/src/test/resources/data.csv.gz
deleted file mode 100644
index a523280..0000000
Binary files a/hadoop/src/test/resources/data.csv.gz and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/test/resources/data.csv.lz4
----------------------------------------------------------------------
diff --git a/hadoop/src/test/resources/data.csv.lz4 b/hadoop/src/test/resources/data.csv.lz4
deleted file mode 100644
index 5288e02..0000000
Binary files a/hadoop/src/test/resources/data.csv.lz4 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/hadoop/src/test/resources/data.csv.snappy
----------------------------------------------------------------------
diff --git a/hadoop/src/test/resources/data.csv.snappy b/hadoop/src/test/resources/data.csv.snappy
deleted file mode 100644
index ff8c8f0..0000000
Binary files a/hadoop/src/test/resources/data.csv.snappy and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 9ed0913..cb2bd3e 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -41,8 +41,8 @@ import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory}
-import org.apache.carbondata.hadoop.csv.CSVInputFormat
-import org.apache.carbondata.hadoop.csv.recorditerator.RecordReaderIterator
+import org.apache.carbondata.processing.csvload.CSVInputFormat
+import org.apache.carbondata.processing.csvload.CSVRecordReaderIterator
import org.apache.carbondata.processing.csvreaderstep.BlockDetails
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.newflow.DataLoadExecutor
@@ -207,7 +207,7 @@ class NewCarbonDataLoadRDD[K, V](
val readers =
split.partitionBlocksDetail.map(format.createRecordReader(_, hadoopAttemptContext))
readers.zipWithIndex.map { case (reader, index) =>
- new RecordReaderIterator(reader,
+ new CSVRecordReaderIterator(reader,
split.partitionBlocksDetail(index),
hadoopAttemptContext)
}
@@ -233,7 +233,7 @@ class NewCarbonDataLoadRDD[K, V](
val readers =
split.nodeBlocksDetail.map(format.createRecordReader(_, hadoopAttemptContext))
readers.zipWithIndex.map { case (reader, index) =>
- new RecordReaderIterator(reader, split.nodeBlocksDetail(index), hadoopAttemptContext)
+ new CSVRecordReaderIterator(reader, split.nodeBlocksDetail(index), hadoopAttemptContext)
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index 2d62fe6..7592e4e 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -33,7 +33,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.statusmanager.SegmentStatusManager
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.hadoop.csv.CSVInputFormat
+import org.apache.carbondata.processing.csvload.CSVInputFormat
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 09c049d..ef759cf 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -50,8 +50,8 @@ import org.apache.carbondata.core.service.CarbonCommonFactory
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.core.writer.CarbonDictionaryWriter
-import org.apache.carbondata.hadoop.csv.CSVInputFormat
-import org.apache.carbondata.hadoop.io.StringArrayWritable
+import org.apache.carbondata.processing.csvload.CSVInputFormat
+import org.apache.carbondata.processing.csvload.StringArrayWritable
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.spark.CarbonSparkFactory
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index d7fce90..7333115 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -25,13 +25,13 @@ import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.hadoop.readsupport.impl.AbstractDictionaryDecodedReadSupport;
+import org.apache.carbondata.hadoop.readsupport.impl.DictionaryDecodeReadSupport;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.catalyst.expressions.GenericRow;
import org.apache.spark.unsafe.types.UTF8String;
-public class SparkRowReadSupportImpl extends AbstractDictionaryDecodedReadSupport<Row> {
+public class SparkRowReadSupportImpl extends DictionaryDecodeReadSupport<Row> {
@Override public void initialize(CarbonColumn[] carbonColumns,
AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index 652f56c..b848543 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -21,12 +21,12 @@ import java.io.IOException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.hadoop.readsupport.impl.AbstractDictionaryDecodedReadSupport;
+import org.apache.carbondata.hadoop.readsupport.impl.DictionaryDecodeReadSupport;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
-public class SparkRowReadSupportImpl extends AbstractDictionaryDecodedReadSupport<InternalRow> {
+public class SparkRowReadSupportImpl extends DictionaryDecodeReadSupport<InternalRow> {
@Override public void initialize(CarbonColumn[] carbonColumns,
AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/processing/src/main/java/org/apache/carbondata/processing/csvload/BoundedInputStream.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/BoundedInputStream.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/BoundedInputStream.java
new file mode 100644
index 0000000..9f80c07
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/BoundedInputStream.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.csvload;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Customarized reader class to read data from file
+ * untill the upper threshold reached.
+ */
+public class BoundedInputStream extends InputStream {
+
+ /**
+ * byte value of the new line character
+ */
+ private static final byte END_OF_LINE_BYTE_VALUE = '\n';
+
+ /**
+ * number of extra character to read
+ */
+ private static final int NUMBER_OF_EXTRA_CHARACTER_TO_READ = 100;
+
+ /**
+ * number of bytes remaining
+ */
+ private long remaining;
+ /**
+ * to check whether end of line is found
+ */
+ private boolean endOfLineFound = false;
+
+ private DataInputStream in;
+
+ public BoundedInputStream(DataInputStream in, long limit) {
+ this.in = in;
+ this.remaining = limit;
+ }
+
+ /**
+ * Below method will be used to read the data from file
+ *
+ * @throws IOException
+ * problem while reading
+ */
+ @Override
+ public int read() throws IOException {
+ if (this.remaining == 0) {
+ return -1;
+ } else {
+ int var1 = this.in.read();
+ if (var1 >= 0) {
+ --this.remaining;
+ }
+
+ return var1;
+ }
+ }
+
+ /**
+ * Below method will be used to read the data from file. If limit reaches in
+ * that case it will read until new line character is reached
+ *
+ * @param buffer
+ * buffer in which data will be read
+ * @param offset
+ * from position to buffer will be filled
+ * @param length
+ * number of character to be read
+ * @throws IOException
+ * problem while reading
+ */
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ if (this.remaining == 0) {
+ return -1;
+ } else {
+ if (this.remaining < length) {
+ length = (int) this.remaining;
+ }
+
+ length = this.in.read(buffer, offset, length);
+ if (length >= 0) {
+ this.remaining -= length;
+ if (this.remaining == 0 && !endOfLineFound) {
+ endOfLineFound = true;
+ this.remaining += NUMBER_OF_EXTRA_CHARACTER_TO_READ;
+ } else if (endOfLineFound) {
+ int end = offset + length;
+ for (int i = offset; i < end; i++) {
+ if (buffer[i] == END_OF_LINE_BYTE_VALUE) {
+ this.remaining = 0;
+ return (i - offset) + 1;
+ }
+ }
+ this.remaining += NUMBER_OF_EXTRA_CHARACTER_TO_READ;
+ }
+ }
+ return length;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (in != null) {
+ in.close();
+ }
+ }
+
+ public long getRemaining() {
+ return this.remaining;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
new file mode 100644
index 0000000..f38175d
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.csvload;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+
+import com.univocity.parsers.csv.CsvParser;
+import com.univocity.parsers.csv.CsvParserSettings;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Seekable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.compress.CodecPool;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.io.compress.CompressionInputStream;
+import org.apache.hadoop.io.compress.Decompressor;
+import org.apache.hadoop.io.compress.SplitCompressionInputStream;
+import org.apache.hadoop.io.compress.SplittableCompressionCodec;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.util.LineReader;
+
+/**
+ * An {@link org.apache.hadoop.mapreduce.InputFormat} for csv files. Files are broken into lines.
+ * Values are the line of csv files.
+ */
+public class CSVInputFormat extends FileInputFormat<NullWritable, StringArrayWritable> {
+
+ public static final String DELIMITER = "carbon.csvinputformat.delimiter";
+ public static final String DELIMITER_DEFAULT = ",";
+ public static final String COMMENT = "carbon.csvinputformat.comment";
+ public static final String COMMENT_DEFAULT = "#";
+ public static final String QUOTE = "carbon.csvinputformat.quote";
+ public static final String QUOTE_DEFAULT = "\"";
+ public static final String ESCAPE = "carbon.csvinputformat.escape";
+ public static final String ESCAPE_DEFAULT = "\\";
+ public static final String HEADER_PRESENT = "caron.csvinputformat.header.present";
+ public static final boolean HEADER_PRESENT_DEFAULT = false;
+ public static final String READ_BUFFER_SIZE = "carbon.csvinputformat.read.buffer.size";
+ public static final String READ_BUFFER_SIZE_DEFAULT = "65536";
+
+ @Override
+ public RecordReader<NullWritable, StringArrayWritable> createRecordReader(InputSplit inputSplit,
+ TaskAttemptContext context) throws IOException, InterruptedException {
+ return new CSVRecordReader();
+ }
+
+ @Override
+ protected boolean isSplitable(JobContext context, Path file) {
+ final CompressionCodec codec = new CompressionCodecFactory(context.getConfiguration())
+ .getCodec(file);
+ if (null == codec) {
+ return true;
+ }
+ return codec instanceof SplittableCompressionCodec;
+ }
+
+ /**
+ * Sets the comment char to configuration. Default it is #.
+ * @param configuration
+ * @param commentChar
+ */
+ public static void setCommentCharacter(Configuration configuration, String commentChar) {
+ if (commentChar != null && !commentChar.isEmpty()) {
+ configuration.set(COMMENT, commentChar);
+ }
+ }
+
+ /**
+ * Sets the delimiter to configuration. Default it is ','
+ * @param configuration
+ * @param delimiter
+ */
+ public static void setCSVDelimiter(Configuration configuration, String delimiter) {
+ if (delimiter != null && !delimiter.isEmpty()) {
+ configuration.set(DELIMITER, delimiter);
+ }
+ }
+
+ /**
+ * Sets the escape character to configuration. Default it is \
+ * @param configuration
+ * @param escapeCharacter
+ */
+ public static void setEscapeCharacter(Configuration configuration, String escapeCharacter) {
+ if (escapeCharacter != null && !escapeCharacter.isEmpty()) {
+ configuration.set(ESCAPE, escapeCharacter);
+ }
+ }
+
+ /**
+ * Whether header needs to read from csv or not. By default it is false.
+ * @param configuration
+ * @param headerExtractEnable
+ */
+ public static void setHeaderExtractionEnabled(Configuration configuration,
+ boolean headerExtractEnable) {
+ configuration.set(HEADER_PRESENT, String.valueOf(headerExtractEnable));
+ }
+
+ /**
+ * Sets the quote character to configuration. Default it is "
+ * @param configuration
+ * @param quoteCharacter
+ */
+ public static void setQuoteCharacter(Configuration configuration, String quoteCharacter) {
+ if (quoteCharacter != null && !quoteCharacter.isEmpty()) {
+ configuration.set(QUOTE, quoteCharacter);
+ }
+ }
+
+ /**
+ * Sets the read buffer size to configuration.
+ * @param configuration
+ * @param bufferSize
+ */
+ public static void setReadBufferSize(Configuration configuration, String bufferSize) {
+ if (bufferSize != null && !bufferSize.isEmpty()) {
+ configuration.set(READ_BUFFER_SIZE, bufferSize);
+ }
+ }
+
+ /**
+ * Treats value as line in file. Key is null.
+ */
+ public static class CSVRecordReader extends RecordReader<NullWritable, StringArrayWritable> {
+
+ private long start;
+ private long end;
+ private BoundedInputStream boundedInputStream;
+ private Reader reader;
+ private CsvParser csvParser;
+ private StringArrayWritable value;
+ private String[] columns;
+ private Seekable filePosition;
+ private boolean isCompressedInput;
+ private Decompressor decompressor;
+
+ @Override
+ public void initialize(InputSplit inputSplit, TaskAttemptContext context)
+ throws IOException, InterruptedException {
+ FileSplit split = (FileSplit) inputSplit;
+ start = split.getStart();
+ end = start + split.getLength();
+ Path file = split.getPath();
+ Configuration job = context.getConfiguration();
+ CompressionCodec codec = (new CompressionCodecFactory(job)).getCodec(file);
+ FileSystem fs = file.getFileSystem(job);
+ int bufferSize = Integer.parseInt(job.get(READ_BUFFER_SIZE, READ_BUFFER_SIZE_DEFAULT));
+ FSDataInputStream fileIn = fs.open(file, bufferSize);
+ InputStream inputStream;
+ if (codec != null) {
+ isCompressedInput = true;
+ decompressor = CodecPool.getDecompressor(codec);
+ if (codec instanceof SplittableCompressionCodec) {
+ SplitCompressionInputStream scIn = ((SplittableCompressionCodec) codec)
+ .createInputStream(fileIn, decompressor, start, end, SplittableCompressionCodec
+ .READ_MODE.BYBLOCK);
+ start = scIn.getAdjustedStart();
+ end = scIn.getAdjustedEnd();
+ if (start != 0) {
+ LineReader lineReader = new LineReader(scIn, 1);
+ start += lineReader.readLine(new Text(), 0);
+ }
+ filePosition = scIn;
+ inputStream = scIn;
+ } else {
+ CompressionInputStream cIn = codec.createInputStream(fileIn, decompressor);
+ filePosition = cIn;
+ inputStream = cIn;
+ }
+ } else {
+ fileIn.seek(start);
+ if (start != 0) {
+ LineReader lineReader = new LineReader(fileIn, 1);
+ start += lineReader.readLine(new Text(), 0);
+ }
+ boundedInputStream = new BoundedInputStream(fileIn, end - start);
+ filePosition = fileIn;
+ inputStream = boundedInputStream;
+ }
+ reader = new InputStreamReader(inputStream);
+ csvParser = new CsvParser(extractCsvParserSettings(job));
+ csvParser.beginParsing(reader);
+ }
+
+ private CsvParserSettings extractCsvParserSettings(Configuration job) {
+ CsvParserSettings parserSettings = new CsvParserSettings();
+ parserSettings.getFormat().setDelimiter(job.get(DELIMITER, DELIMITER_DEFAULT).charAt(0));
+ parserSettings.getFormat().setComment(job.get(COMMENT, COMMENT_DEFAULT).charAt(0));
+ parserSettings.setLineSeparatorDetectionEnabled(true);
+ parserSettings.setNullValue("");
+ parserSettings.setIgnoreLeadingWhitespaces(false);
+ parserSettings.setIgnoreTrailingWhitespaces(false);
+ parserSettings.setSkipEmptyLines(false);
+ // TODO get from csv file.
+ parserSettings.setMaxColumns(1000);
+ parserSettings.getFormat().setQuote(job.get(QUOTE, QUOTE_DEFAULT).charAt(0));
+ parserSettings.getFormat().setQuoteEscape(job.get(ESCAPE, ESCAPE_DEFAULT).charAt(0));
+ if (start == 0) {
+ parserSettings.setHeaderExtractionEnabled(job.getBoolean(HEADER_PRESENT,
+ HEADER_PRESENT_DEFAULT));
+ }
+ return parserSettings;
+ }
+
+ @Override
+ public boolean nextKeyValue() throws IOException, InterruptedException {
+ columns = csvParser.parseNext();
+ if (columns == null) {
+ value = null;
+ return false;
+ }
+ if (value == null) {
+ value = new StringArrayWritable();
+ }
+ value.set(columns);
+ return true;
+ }
+
+ @Override
+ public NullWritable getCurrentKey() throws IOException, InterruptedException {
+ return NullWritable.get();
+ }
+
+ @Override
+ public StringArrayWritable getCurrentValue() throws IOException, InterruptedException {
+ return value;
+ }
+
+ private long getPos() throws IOException {
+ long retVal = start;
+ if (null != boundedInputStream) {
+ retVal = end - boundedInputStream.getRemaining();
+ } else if (isCompressedInput && null != filePosition) {
+ retVal = filePosition.getPos();
+ }
+ return retVal;
+ }
+
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
+ return start == end ? 0.0F : Math.min(1.0F, (float) (getPos() -
+ start) / (float) (end - start));
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ if (reader != null) {
+ reader.close();
+ }
+ } finally {
+ if (decompressor != null) {
+ CodecPool.returnDecompressor(decompressor);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
new file mode 100644
index 0000000..10a036a
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVRecordReaderIterator.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.processing.csvload;
+
+import java.io.IOException;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * It is wrapper iterator around @{@link RecordReader}.
+ */
+public class CSVRecordReaderIterator extends CarbonIterator<Object []> {
+
+ private RecordReader<NullWritable, StringArrayWritable> recordReader;
+
+ /**
+ * It is just a little hack to make recordreader as iterator. Usually we cannot call hasNext
+ * multiple times on record reader as it moves another line. To avoid that situation like hasNext
+ * only tells whether next row is present or not and next will move the pointer to next row after
+ * consuming it.
+ */
+ private boolean isConsumed;
+
+ private InputSplit split;
+
+ private TaskAttemptContext context;
+
+ public CSVRecordReaderIterator(RecordReader<NullWritable, StringArrayWritable> recordReader,
+ InputSplit split, TaskAttemptContext context) {
+ this.recordReader = recordReader;
+ this.split = split;
+ this.context = context;
+ }
+
+ @Override
+ public boolean hasNext() {
+ try {
+ if (!isConsumed) {
+ isConsumed = recordReader.nextKeyValue();
+ return isConsumed;
+ }
+ return true;
+ } catch (Exception e) {
+ throw new CarbonDataLoadingException(e);
+ }
+ }
+
+ @Override
+ public Object[] next() {
+ try {
+ String[] data = recordReader.getCurrentValue().get();
+ isConsumed = false;
+ return data;
+ } catch (Exception e) {
+ throw new CarbonDataLoadingException(e);
+ }
+ }
+
+ @Override
+ public void initialize() {
+ try {
+ recordReader.initialize(split, context);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() {
+ try {
+ recordReader.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/processing/src/main/java/org/apache/carbondata/processing/csvload/StringArrayWritable.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/StringArrayWritable.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/StringArrayWritable.java
new file mode 100644
index 0000000..7eb3ec9
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/StringArrayWritable.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.csvload;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * A String sequence that is usable as a key or value.
+ */
+public class StringArrayWritable implements Writable {
+ private String[] values;
+
+ public String[] toStrings() {
+ return values;
+ }
+
+ public void set(String[] values) {
+ this.values = values;
+ }
+
+ public String[] get() {
+ return values;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ int length = in.readInt();
+ values = new String[length];
+ for (int i = 0; i < length; i++) {
+ byte[] b = new byte[in.readInt()];
+ in.readFully(b);
+ values[i] = new String(b, Charset.defaultCharset());
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(values.length); // write values
+ for (int i = 0; i < values.length; i++) {
+ byte[] b = values[i].getBytes(Charset.defaultCharset());
+ out.writeInt(b.length);
+ out.write(b);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return Arrays.toString(values);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d6ceb1d3/processing/src/test/java/org/apache/carbondata/processing/csvload/CSVInputFormatTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/csvload/CSVInputFormatTest.java b/processing/src/test/java/org/apache/carbondata/processing/csvload/CSVInputFormatTest.java
new file mode 100644
index 0000000..66aedb6
--- /dev/null
+++ b/processing/src/test/java/org/apache/carbondata/processing/csvload/CSVInputFormatTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.csvload;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import junit.framework.TestCase;
+import org.junit.Assert;
+import org.junit.Test;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.io.compress.Lz4Codec;
+import org.apache.hadoop.io.compress.SnappyCodec;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+public class CSVInputFormatTest extends TestCase {
+
+ /**
+ * generate compressed files, no need to call this method.
+ * @throws Exception
+ */
+ public void generateCompressFiles() throws Exception {
+ String pwd = new File("src/test/resources/csv").getCanonicalPath();
+ String inputFile = pwd + "/data.csv";
+ FileInputStream input = new FileInputStream(inputFile);
+ Configuration conf = new Configuration();
+
+ // .gz
+ String outputFile = pwd + "/data.csv.gz";
+ FileOutputStream output = new FileOutputStream(outputFile);
+ GzipCodec gzip = new GzipCodec();
+ gzip.setConf(conf);
+ CompressionOutputStream outputStream = gzip.createOutputStream(output);
+ int i = -1;
+ while ((i = input.read()) != -1) {
+ outputStream.write(i);
+ }
+ outputStream.close();
+ input.close();
+
+ // .bz2
+ input = new FileInputStream(inputFile);
+ outputFile = pwd + "/data.csv.bz2";
+ output = new FileOutputStream(outputFile);
+ BZip2Codec bzip2 = new BZip2Codec();
+ bzip2.setConf(conf);
+ outputStream = bzip2.createOutputStream(output);
+ i = -1;
+ while ((i = input.read()) != -1) {
+ outputStream.write(i);
+ }
+ outputStream.close();
+ input.close();
+
+ // .snappy
+ input = new FileInputStream(inputFile);
+ outputFile = pwd + "/data.csv.snappy";
+ output = new FileOutputStream(outputFile);
+ SnappyCodec snappy = new SnappyCodec();
+ snappy.setConf(conf);
+ outputStream = snappy.createOutputStream(output);
+ i = -1;
+ while ((i = input.read()) != -1) {
+ outputStream.write(i);
+ }
+ outputStream.close();
+ input.close();
+
+ //.lz4
+ input = new FileInputStream(inputFile);
+ outputFile = pwd + "/data.csv.lz4";
+ output = new FileOutputStream(outputFile);
+ Lz4Codec lz4 = new Lz4Codec();
+ lz4.setConf(conf);
+ outputStream = lz4.createOutputStream(output);
+ i = -1;
+ while ((i = input.read()) != -1) {
+ outputStream.write(i);
+ }
+ outputStream.close();
+ input.close();
+
+ }
+
+ /**
+ * CSVCheckMapper check the content of csv files.
+ */
+ public static class CSVCheckMapper extends Mapper<NullWritable, StringArrayWritable, NullWritable,
+ NullWritable> {
+ @Override
+ protected void map(NullWritable key, StringArrayWritable value, Context context)
+ throws IOException, InterruptedException {
+ String[] columns = value.get();
+ int id = Integer.parseInt(columns[0]);
+ int salary = Integer.parseInt(columns[6]);
+ Assert.assertEquals(id - 1, salary - 15000);
+ }
+ }
+
+ /**
+ * test read csv files
+ * @throws Exception
+ */
+ @Test public void testReadCSVFiles() throws Exception{
+ Configuration conf = new Configuration();
+ prepareConf(conf);
+ Job job = Job.getInstance(conf, "CSVInputFormat_normal");
+ job.setJarByClass(CSVInputFormatTest.class);
+ job.setMapperClass(CSVCheckMapper.class);
+ job.setNumReduceTasks(0);
+ job.setInputFormatClass(CSVInputFormat.class);
+
+ String inputFolder = new File("src/test/resources/csv").getCanonicalPath();
+ FileInputFormat.addInputPath(job, new Path(inputFolder + File.separator + "data.csv"));
+ FileInputFormat.addInputPath(job, new Path(inputFolder + File.separator + "data.csv.bz2"));
+ FileInputFormat.addInputPath(job, new Path(inputFolder + File.separator + "data.csv.gz"));
+ // FileInputFormat.addInputPath(job, new Path(inputFolder + File.separator + "data.csv.lz4"));
+ // FileInputFormat.addInputPath(job, new Path(inputFolder + File.separator + "data.csv.snappy"));
+
+ File output = new File("target/output_CSVInputFormatTest");
+ deleteOutput(output);
+ FileOutputFormat.setOutputPath(job, new Path(output.getCanonicalPath()));
+
+ Assert.assertTrue(job.waitForCompletion(true));
+ }
+
+ private void prepareConf(Configuration conf) {
+ conf.setBoolean(CSVInputFormat.HEADER_PRESENT, true);
+ }
+
+ private void deleteOutput(File output) {
+ if (output.exists()) {
+ if (output.isDirectory()) {
+ for(File file : output.listFiles()) {
+ deleteOutput(file);
+ }
+ output.delete();
+ } else {
+ output.delete();
+ }
+ }
+ }
+}
[3/3] incubator-carbondata git commit: [CARBONDATA-681] CSVReader
related code improvement This closes #573
Posted by ja...@apache.org.
[CARBONDATA-681] CSVReader related code improvement This closes #573
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f3496ee0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f3496ee0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f3496ee0
Branch: refs/heads/master
Commit: f3496ee0bf5d98ed65210208d3e364e95b6c0962
Parents: 25de27f d6ceb1d
Author: jackylk <ja...@huawei.com>
Authored: Tue Feb 14 15:26:40 2017 -0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue Feb 14 15:26:40 2017 -0800
----------------------------------------------------------------------
.../carbondata/hadoop/CarbonInputFormat.java | 6 +-
.../carbondata/hadoop/csv/CSVInputFormat.java | 288 -----
.../recorditerator/RecordReaderIterator.java | 98 --
.../hadoop/io/BoundedInputStream.java | 129 ---
.../hadoop/io/StringArrayWritable.java | 70 --
.../hadoop/readsupport/CarbonReadSupport.java | 15 +-
.../AbstractDictionaryDecodedReadSupport.java | 84 --
.../impl/ArrayWritableReadSupport.java | 48 -
.../impl/DictionaryDecodeReadSupport.java | 94 ++
.../impl/DictionaryDecodedReadSupportImpl.java | 34 -
.../readsupport/impl/RawDataReadSupport.java | 17 +-
.../hadoop/csv/CSVInputFormatTest.java | 169 ---
hadoop/src/test/resources/data.csv.bz2 | Bin 10572 -> 0 bytes
hadoop/src/test/resources/data.csv.gz | Bin 14710 -> 0 bytes
hadoop/src/test/resources/data.csv.lz4 | Bin 24495 -> 0 bytes
hadoop/src/test/resources/data.csv.snappy | Bin 24263 -> 0 bytes
.../spark/rdd/NewCarbonDataLoadRDD.scala | 8 +-
.../carbondata/spark/util/CommonUtil.scala | 2 +-
.../spark/util/GlobalDictionaryUtil.scala | 4 +-
.../readsupport/SparkRowReadSupportImpl.java | 4 +-
.../readsupport/SparkRowReadSupportImpl.java | 4 +-
.../processing/csvload/BoundedInputStream.java | 129 +++
.../processing/csvload/CSVInputFormat.java | 285 +++++
.../csvload/CSVRecordReaderIterator.java | 97 ++
.../processing/csvload/StringArrayWritable.java | 70 ++
.../processing/csvload/CSVInputFormatTest.java | 167 +++
processing/src/test/resources/csv/data.csv | 1001 ++++++++++++++++++
processing/src/test/resources/csv/data.csv.bz2 | Bin 0 -> 10572 bytes
processing/src/test/resources/csv/data.csv.gz | Bin 0 -> 14710 bytes
processing/src/test/resources/csv/data.csv.lz4 | Bin 0 -> 24495 bytes
.../src/test/resources/csv/data.csv.snappy | Bin 0 -> 24263 bytes
31 files changed, 1870 insertions(+), 953 deletions(-)
----------------------------------------------------------------------