You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ignite.apache.org by wt <wa...@gmail.com> on 2018/09/17 10:33:31 UTC
IgniteDataStreamer - into table and not key\value
I am trying to load ignite using the data streamer into an ignite table. The
streamer works but when querying the table it is empty. Upon further
investigation it looks like the streamer is creating a key\value store in
the same cache that the table resided. When querying the metadata it shows
no changes to the cache. If i query the cache parsing in the surrogate key i
injected into the code using the rest service i can see the records are
there but in what looks like a key\value store. Can the IgniteDataStreamer
load a table?
here is the .net code
try { connection.Open();
command = new SqlCommand(sql, connection);
dataReader = command.ExecuteReader();
using (var ldr = ignite.GetDataStreamer<int,
IndexHistory>("IndexcompositionCache"))
{
ldr.PerNodeBufferSize = 1024;
int i = 1;
while (dataReader.Read())
{
var o = new IndexHistory()
{
VD = (int)dataReader.GetValue(0),
SHARESININDEX = (double)dataReader.GetValue(1),
NAME = (string)dataReader.GetValue(2),
ISIN = (string)dataReader.GetValue(3),
SEDOL = (string)dataReader.GetValue(4),
RIC = (string)dataReader.GetValue(5),
CLOSE = (double)dataReader.GetValue(6),
RATE = (decimal)dataReader.GetValue(7)
};
ldr.AddData(i, o );
i = i + 1;
}
}
dataReader.Close();
command.Dispose();
connection.Close(); }
catch (Exception ex)
{
Console.WriteLine(ex.Message.ToString());
}
--
Sent from: http://apache-ignite-developers.2346864.n4.nabble.com/
Re: IgniteDataStreamer - into table and not key\value
Posted by Ilya Kasnacheev <il...@gmail.com>.
Hello!
You don't need to remove those properties, just remove package names:
<property name="keyType" value="IndexcompositionKey"/>
<property name="valueType" value="Indexcomposition"/>
Please try and read mails carefully.
Regards,
--
Ilya Kasnacheev
пн, 17 сент. 2018 г. в 17:05, wt <wa...@gmail.com>:
> this is making no sense. There is absolutely nothing in the documentation
> that says remove those items from the configuration. All i am looking for
> is
> an example of a table that can be loaded with the data streamer without the
> data ending up outside of the table. I have tried with and without that
> simple mapper in th config and it makes no difference.
>
> simple example i have:
>
> 1) config (has no caches specified - contents placed at the end)
> 2) create table in jdbc
>
> CREATE TABLE IF NOT EXISTS Testy (
> id int,
> name varchar,
> PRIMARY KEY (id)
> ) WITH "template=partitioned,backups=0";
>
> 3) table is visible in ignite
>
> {
> "successStatus": 0,
> "error": null,
> "sessionToken": null,
> "response": [
> {
> "cacheName": "SQL_PUBLIC_TESTY",
> "types": [
>
> "SQL_PUBLIC_TESTY_07052f5a_37ac_4992_8a19_79da93e3df8a"
> ],
> "keyClasses": {
>
> "SQL_PUBLIC_TESTY_07052f5a_37ac_4992_8a19_79da93e3df8a":
> "java.lang.Integer"
> },
> "valClasses": {
>
> "SQL_PUBLIC_TESTY_07052f5a_37ac_4992_8a19_79da93e3df8a":
> "java.lang.Object"
> },
> "fields": {
>
> "SQL_PUBLIC_TESTY_07052f5a_37ac_4992_8a19_79da93e3df8a": {
> "ID": "java.lang.Integer",
> "NAME": "java.lang.String"
> }
> },
> "indexes": {
>
> "SQL_PUBLIC_TESTY_07052f5a_37ac_4992_8a19_79da93e3df8a": []
> }
> }}}
>
>
> 4) simple .net project
>
>
> public class Testy
> {
>
> [QuerySqlField]
> public int id { get; set; }
> [QuerySqlField]
> public string name { get; set; }
>
> }
>
> var cfg = new IgniteConfiguration()
> {
> SpringConfigUrl = @"C:\Cluster-client.xml"
> };
>
> var ignite = Ignition.Start(cfg);
> using (var ldr = ignite.GetDataStreamer<int,
> Testy>("SQL_PUBLIC_TESTY"))
> {
> ldr.AddData(1, new Testy() {id=1, name = "ffff" });
> }
>
>
> 5) db beaver
>
> SELECT count(*) FROM testy (result is 0)
>
> 6) rest query
>
>
> http://127.0.0.1:8080/ignite?cmd=get&cacheName=SQL_PUBLIC_TESTY&keyType=int&key=1
>
> {
> "successStatus": 0,
> "affinityNodeId": "9a006702-e11e-4fdc-bf84-7f0b28fc498f",
> "error": null,
> "sessionToken": null,
> "response": {
> "name": "ffff",
> "id": 1
> }
> }
>
>
>
>
> -----------------
> config file
>
> <beans xmlns="http://www.springframework.org/schema/beans"
> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
> xmlns:util="http://www.springframework.org/schema/util"
> xsi:schemaLocation="http://www.springframework.org/schema/beans
>
> http://www.springframework.org/schema/beans/spring-beans.xsd
> http://www.springframework.org/schema/util
>
> http://www.springframework.org/schema/util/spring-util.xsd">
>
> <bean class="org.apache.ignite.configuration.IgniteConfiguration">
> <property name="clientMode" value="true"/>
> <property name="igniteInstanceName" value="Cluster"/>
>
> <property name="discoverySpi">
> <bean
> class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
> <property name="ipFinder">
> <bean
>
> class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
> <property name="addresses">
> <list>
> <value>127.0.0.1:47500..47510</value>
> </list>
> </property>
> </bean>
> </property>
> </bean>
> </property>
>
> <property name="binaryConfiguration">
> <bean
> class="org.apache.ignite.configuration.BinaryConfiguration">
> <property name="nameMapper">
> <bean
> class="org.apache.ignite.binary.BinaryBasicNameMapper">
> <property name="simpleName" value="true"/>
> </bean>
> </property>
> </bean>
> </property>
>
> <property name="dataStorageConfiguration">
> <bean
> class="org.apache.ignite.configuration.DataStorageConfiguration">
> <property name="defaultDataRegionConfiguration">
> <bean
> class="org.apache.ignite.configuration.DataRegionConfiguration">
> <property name="maxSize" value="3145728000"/>
> <property name="persistenceEnabled" value="true"/>
> </bean>
> </property>
> </bean>
> </property>
>
> </bean>
> </beans>
>
>
>
>
>
> --
> Sent from: http://apache-ignite-developers.2346864.n4.nabble.com/
>
Re: IgniteDataStreamer - into table and not key\value
Posted by wt <wa...@gmail.com>.
this is making no sense. There is absolutely nothing in the documentation
that says remove those items from the configuration. All i am looking for is
an example of a table that can be loaded with the data streamer without the
data ending up outside of the table. I have tried with and without that
simple mapper in th config and it makes no difference.
simple example i have:
1) config (has no caches specified - contents placed at the end)
2) create table in jdbc
CREATE TABLE IF NOT EXISTS Testy (
id int,
name varchar,
PRIMARY KEY (id)
) WITH "template=partitioned,backups=0";
3) table is visible in ignite
{
"successStatus": 0,
"error": null,
"sessionToken": null,
"response": [
{
"cacheName": "SQL_PUBLIC_TESTY",
"types": [
"SQL_PUBLIC_TESTY_07052f5a_37ac_4992_8a19_79da93e3df8a"
],
"keyClasses": {
"SQL_PUBLIC_TESTY_07052f5a_37ac_4992_8a19_79da93e3df8a":
"java.lang.Integer"
},
"valClasses": {
"SQL_PUBLIC_TESTY_07052f5a_37ac_4992_8a19_79da93e3df8a":
"java.lang.Object"
},
"fields": {
"SQL_PUBLIC_TESTY_07052f5a_37ac_4992_8a19_79da93e3df8a": {
"ID": "java.lang.Integer",
"NAME": "java.lang.String"
}
},
"indexes": {
"SQL_PUBLIC_TESTY_07052f5a_37ac_4992_8a19_79da93e3df8a": []
}
}}}
4) simple .net project
public class Testy
{
[QuerySqlField]
public int id { get; set; }
[QuerySqlField]
public string name { get; set; }
}
var cfg = new IgniteConfiguration()
{
SpringConfigUrl = @"C:\Cluster-client.xml"
};
var ignite = Ignition.Start(cfg);
using (var ldr = ignite.GetDataStreamer<int,
Testy>("SQL_PUBLIC_TESTY"))
{
ldr.AddData(1, new Testy() {id=1, name = "ffff" });
}
5) db beaver
SELECT count(*) FROM testy (result is 0)
6) rest query
http://127.0.0.1:8080/ignite?cmd=get&cacheName=SQL_PUBLIC_TESTY&keyType=int&key=1
{
"successStatus": 0,
"affinityNodeId": "9a006702-e11e-4fdc-bf84-7f0b28fc498f",
"error": null,
"sessionToken": null,
"response": {
"name": "ffff",
"id": 1
}
}
-----------------
config file
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util
http://www.springframework.org/schema/util/spring-util.xsd">
<bean class="org.apache.ignite.configuration.IgniteConfiguration">
<property name="clientMode" value="true"/>
<property name="igniteInstanceName" value="Cluster"/>
<property name="discoverySpi">
<bean
class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
<property name="ipFinder">
<bean
class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder">
<property name="addresses">
<list>
<value>127.0.0.1:47500..47510</value>
</list>
</property>
</bean>
</property>
</bean>
</property>
<property name="binaryConfiguration">
<bean
class="org.apache.ignite.configuration.BinaryConfiguration">
<property name="nameMapper">
<bean
class="org.apache.ignite.binary.BinaryBasicNameMapper">
<property name="simpleName" value="true"/>
</bean>
</property>
</bean>
</property>
<property name="dataStorageConfiguration">
<bean
class="org.apache.ignite.configuration.DataStorageConfiguration">
<property name="defaultDataRegionConfiguration">
<bean
class="org.apache.ignite.configuration.DataRegionConfiguration">
<property name="maxSize" value="3145728000"/>
<property name="persistenceEnabled" value="true"/>
</bean>
</property>
</bean>
</property>
</bean>
</beans>
--
Sent from: http://apache-ignite-developers.2346864.n4.nabble.com/
Re: IgniteDataStreamer - into table and not key\value
Posted by Ilya Kasnacheev <il...@gmail.com>.
<property name="keyType"
value="com.gmail.waynetheron.model.IndexcompositionKey"/>
<property name="valueType"
value="com.gmail.waynetheron.model.Indexcomposition"/>
Do you have them package names in your config? ^^^ You have to remove those
when using simple name mapper.
If you're using those types from Java you will also need to turn on the
simple name mapper in Java config.
Regards
--
Ilya Kasnacheev
пн, 17 сент. 2018 г. в 15:07, wt <wa...@gmail.com>:
> that is a typo, changing the name of the class to Indexcomposition doesn't
> make any difference. I also added the following to the config
>
>
> <property name="binaryConfiguration">
> <bean
> class="org.apache.ignite.configuration.BinaryConfiguration">
> <property name="nameMapper">
> <bean
> class="org.apache.ignite.binary.BinaryBasicNameMapper">
> <property name="simpleName" value="true"/>
> </bean>
> </property>
> </bean>
> </property>
>
> the class names in .net are matching the java names in the config file
>
>
> public class Indexcomposition
> {
>
> [QuerySqlField]
> public int vd { get; set; }
>
> [QuerySqlField]
> public double sharesinindex { get; set; }
>
> [QuerySqlField]
> public string name { get; set; }
>
> [QuerySqlField]
> public string isin { get; set; }
>
> [QuerySqlField]
> public string sedol { get; set; }
>
> [QuerySqlField]
> public string ric { get; set; }
>
> [QuerySqlField]
> public double close { get; set; }
>
> [QuerySqlField]
> public decimal rate { get; set; }
>
> }
>
>
> <property name="queryEntities">
> .....
>
> <property name="fields">
> <map>
> <entry key="vd"
> value="java.lang.Integer"/>
> <entry key="sharesinindex"
> value="java.lang.Double"/>
> <entry key="name"
> value="java.lang.String"/>
> <entry key="isin"
> value="java.lang.String"/>
> <entry key="sedol"
> value="java.lang.String"/>
> <entry key="ric"
> value="java.lang.String"/>
> <entry key="close"
> value="java.lang.Double"/>
> <entry key="rate"
> value="java.math.BigDecimal"/>
> ...
> </property>
>
>
>
>
> --
> Sent from: http://apache-ignite-developers.2346864.n4.nabble.com/
>
Re: IgniteDataStreamer - into table and not key\value
Posted by wt <wa...@gmail.com>.
that is a typo, changing the name of the class to Indexcomposition doesn't
make any difference. I also added the following to the config
<property name="binaryConfiguration">
<bean
class="org.apache.ignite.configuration.BinaryConfiguration">
<property name="nameMapper">
<bean
class="org.apache.ignite.binary.BinaryBasicNameMapper">
<property name="simpleName" value="true"/>
</bean>
</property>
</bean>
</property>
the class names in .net are matching the java names in the config file
public class Indexcomposition
{
[QuerySqlField]
public int vd { get; set; }
[QuerySqlField]
public double sharesinindex { get; set; }
[QuerySqlField]
public string name { get; set; }
[QuerySqlField]
public string isin { get; set; }
[QuerySqlField]
public string sedol { get; set; }
[QuerySqlField]
public string ric { get; set; }
[QuerySqlField]
public double close { get; set; }
[QuerySqlField]
public decimal rate { get; set; }
}
<property name="queryEntities">
.....
<property name="fields">
<map>
<entry key="vd"
value="java.lang.Integer"/>
<entry key="sharesinindex"
value="java.lang.Double"/>
<entry key="name"
value="java.lang.String"/>
<entry key="isin"
value="java.lang.String"/>
<entry key="sedol"
value="java.lang.String"/>
<entry key="ric"
value="java.lang.String"/>
<entry key="close"
value="java.lang.Double"/>
<entry key="rate"
value="java.math.BigDecimal"/>
...
</property>
--
Sent from: http://apache-ignite-developers.2346864.n4.nabble.com/
Re: IgniteDataStreamer - into table and not key\value
Posted by Ilya Kasnacheev <il...@gmail.com>.
Hello!
I don't see IndexHistory anywhere in your config file.
Most likely you need to make sure that binary object names for keys and
values match between your SQL, Java and .Net.
To do that, you would likely need to enable simple name binary mapping, and
make sure that all simple names are spelled identically.
Please see https://apacheignite-net.readme.io/docs/platform-interoperability
Regards,
--
Ilya Kasnacheev
пн, 17 сент. 2018 г. в 13:40, wt <wa...@gmail.com>:
> hi Ilya
>
> Table initialised? it is a rmdbs integrated here is the config file
> Cluster-server.xml
> <
> http://apache-ignite-developers.2346864.n4.nabble.com/file/t604/Cluster-server.xml>
>
>
> here is the class
>
> class IndexHistory
> {
>
> [QuerySqlField]
> public int VD { get; set; }
>
> [QuerySqlField]
> public double SHARESININDEX { get; set; }
>
> [QuerySqlField]
> public string NAME { get; set; }
>
> [QuerySqlField]
> public string ISIN { get; set; }
>
> [QuerySqlField]
> public string SEDOL { get; set; }
>
> [QuerySqlField]
> public string RIC { get; set; }
>
> [QuerySqlField]
> public double CLOSE { get; set; }
>
> [QuerySqlField]
> public decimal RATE { get; set; }
>
> }
>
>
>
> --
> Sent from: http://apache-ignite-developers.2346864.n4.nabble.com/
>
Re: IgniteDataStreamer - into table and not key\value
Posted by wt <wa...@gmail.com>.
hi Ilya
Table initialised? it is a rmdbs integrated here is the config file
Cluster-server.xml
<http://apache-ignite-developers.2346864.n4.nabble.com/file/t604/Cluster-server.xml>
here is the class
class IndexHistory
{
[QuerySqlField]
public int VD { get; set; }
[QuerySqlField]
public double SHARESININDEX { get; set; }
[QuerySqlField]
public string NAME { get; set; }
[QuerySqlField]
public string ISIN { get; set; }
[QuerySqlField]
public string SEDOL { get; set; }
[QuerySqlField]
public string RIC { get; set; }
[QuerySqlField]
public double CLOSE { get; set; }
[QuerySqlField]
public decimal RATE { get; set; }
}
--
Sent from: http://apache-ignite-developers.2346864.n4.nabble.com/
Re: IgniteDataStreamer - into table and not key\value
Posted by Ilya Kasnacheev <il...@gmail.com>.
Hello!
Can you please show how your table is initialized? Also, can you show your
IndexHistory class?
Regards,
--
Ilya Kasnacheev
пн, 17 сент. 2018 г. в 13:33, wt <wa...@gmail.com>:
> I am trying to load ignite using the data streamer into an ignite table.
> The
> streamer works but when querying the table it is empty. Upon further
> investigation it looks like the streamer is creating a key\value store in
> the same cache that the table resided. When querying the metadata it shows
> no changes to the cache. If i query the cache parsing in the surrogate key
> i
> injected into the code using the rest service i can see the records are
> there but in what looks like a key\value store. Can the IgniteDataStreamer
> load a table?
>
> here is the .net code
>
> try { connection.Open();
> command = new SqlCommand(sql, connection);
> dataReader = command.ExecuteReader();
> using (var ldr = ignite.GetDataStreamer<int,
> IndexHistory>("IndexcompositionCache"))
> {
> ldr.PerNodeBufferSize = 1024;
> int i = 1;
> while (dataReader.Read())
> {
> var o = new IndexHistory()
> {
> VD = (int)dataReader.GetValue(0),
> SHARESININDEX = (double)dataReader.GetValue(1),
> NAME = (string)dataReader.GetValue(2),
> ISIN = (string)dataReader.GetValue(3),
> SEDOL = (string)dataReader.GetValue(4),
> RIC = (string)dataReader.GetValue(5),
> CLOSE = (double)dataReader.GetValue(6),
> RATE = (decimal)dataReader.GetValue(7)
> };
> ldr.AddData(i, o );
>
> i = i + 1;
> }
>
> }
> dataReader.Close();
> command.Dispose();
> connection.Close(); }
> catch (Exception ex)
> {
> Console.WriteLine(ex.Message.ToString());
> }
>
>
>
> --
> Sent from: http://apache-ignite-developers.2346864.n4.nabble.com/
>