You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2016/02/02 22:19:01 UTC
usergrid git commit: Fix issue with timestamps being set with longs
that do not correspond to time. Remove unused dependencies. Remove
CountMinSketch depedency and leverage RX distinct to filter duplicated
devices from being emitted.
Repository: usergrid
Updated Branches:
refs/heads/master 47a86701e -> f7b5516b5
Fix issue with timestamps being set with longs that do not correspond to time. Remove unused dependencies. Remove CountMinSketch depedency and leverage RX distinct to filter duplicated devices from being emitted.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f7b5516b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f7b5516b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f7b5516b
Branch: refs/heads/master
Commit: f7b5516b56a13594709a8b33faa7bd972e81cd4c
Parents: 47a8670
Author: Michael Russo <mi...@gmail.com>
Authored: Tue Feb 2 13:17:45 2016 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Tue Feb 2 13:17:45 2016 -0800
----------------------------------------------------------------------
stack/core/pom.xml | 6 ---
.../corepersistence/CpEntityManager.java | 50 +++++++++++---------
.../java/org/apache/usergrid/mq/Message.java | 4 +-
.../usergrid/persistence/EntityManager.java | 4 +-
.../org/apache/usergrid/utils/UUIDUtils.java | 11 -----
stack/pom.xml | 6 ---
stack/services/pom.xml | 11 -----
.../services/notifications/gcm/GCMAdapter.java | 24 +++++-----
.../impl/ApplicationQueueManagerImpl.java | 23 ++-------
.../org/apache/usergrid/python/JythonTest.java | 39 ---------------
10 files changed, 48 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f7b5516b/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 49082b6..76ff694 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -429,12 +429,6 @@
<version>${metrics.version}</version>
</dependency>
- <dependency>
- <groupId>com.clearspring.analytics</groupId>
- <artifactId>stream</artifactId>
- <version>2.7.0</version>
- </dependency>
-
</dependencies>
<!--
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f7b5516b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index ed1d2de..a7cb8f4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -363,12 +363,10 @@ public class CpEntityManager implements EntityManager {
@Override
public Entity create( UUID importId, String entityType, Map<String, Object> properties ) throws Exception {
- UUID timestampUuid = importId != null ? importId : UUIDUtils.newTimeUUID();
-
//Adding graphite metrics
Timer.Context timeCassCreation = entCreateTimer.time();
- Entity entity = batchCreate( entityType, null, properties, importId, timestampUuid );
+ Entity entity = batchCreate( entityType, null, properties, importId);
timeCassCreation.stop();
return entity;
@@ -380,7 +378,7 @@ public class CpEntityManager implements EntityManager {
//Adding graphite metrics
Timer.Context timeCassCreation = entCreateTimer.time();
- Entity entity = batchCreate( id.getType(), null, properties, id.getUuid(), UUIDUtils.newTimeUUID() );
+ Entity entity = batchCreate( id.getType(), null, properties, id.getUuid());
timeCassCreation.stop();
@@ -404,12 +402,10 @@ public class CpEntityManager implements EntityManager {
public <A extends Entity> A create( String entityType, Class<A> entityClass,
Map<String, Object> properties, UUID importId ) throws Exception {
- UUID timestampUuid = importId != null ? importId : UUIDUtils.newTimeUUID();
-
Timer.Context timeEntityCassCreation = entCreateBatchTimer.time();
- A entity = batchCreate( entityType, entityClass, properties, importId, timestampUuid );
+ A entity = batchCreate( entityType, entityClass, properties, importId);
//Adding graphite metrics
timeEntityCassCreation.stop();
@@ -1657,7 +1653,7 @@ public class CpEntityManager implements EntityManager {
String propertyName, UUID ownerId, Map<String, Object> additionalProperties ) throws Exception {
UUID timestampUuid = UUIDUtils.newTimeUUID();
- long timestamp = UUIDUtils.getUUIDLong( timestampUuid );
+ long timestamp = UUIDUtils.getTimestampInMicros( timestampUuid );
Map<String, Object> properties = new TreeMap<>( CASE_INSENSITIVE_ORDER );
properties.put( PROPERTY_TYPE, Role.ENTITY_TYPE );
@@ -1672,7 +1668,7 @@ public class CpEntityManager implements EntityManager {
}
UUID id = UUIDGenerator.newTimeUUID();
- batchCreate( Role.ENTITY_TYPE, null, properties, id, timestampUuid );
+ batchCreate( Role.ENTITY_TYPE, null, properties, id);
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
CassandraPersistenceUtils.addInsertToMutator( batch, ENTITY_DICTIONARIES,
@@ -2472,8 +2468,8 @@ public class CpEntityManager implements EntityManager {
@Override
- public <A extends Entity> A batchCreate( String entityType, Class<A> entityClass, Map<String, Object> properties,
- UUID importId, UUID timestampUuid )
+ public <A extends Entity> A batchCreate(String entityType, Class<A> entityClass, Map<String, Object> properties,
+ UUID importId)
throws Exception {
String eType = Schema.normalizeEntityType( entityType );
@@ -2486,7 +2482,24 @@ public class CpEntityManager implements EntityManager {
return null;
}
- long timestamp = UUIDUtils.getUUIDLong( timestampUuid );
+
+ long timestamp = UUIDUtils.getTimestampInMicros( UUIDUtils.newTimeUUID() );
+
+ // if the entity UUID is provided, attempt to get a time from the UUID or from it's created property
+ if ( importId != null ) {
+ long timestampFromImport = -1L;
+ if ( UUIDUtils.isTimeBased( importId ) ) {
+ timestampFromImport = UUIDUtils.getTimestampInMicros( importId );
+ }
+ else if ( properties.get( PROPERTY_CREATED ) != null ) {
+ // the entity property would be stored as milliseconds
+ timestampFromImport = getLong( properties.get( PROPERTY_CREATED ) ) * 1000;
+
+ }
+ if (timestampFromImport >= 0){
+ timestamp = timestampFromImport;
+ }
+ }
UUID itemId = UUIDGenerator.newTimeUUID();
@@ -2497,17 +2510,10 @@ public class CpEntityManager implements EntityManager {
itemId = importId;
}
if ( properties == null ) {
- properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
+ properties = new TreeMap<>( CASE_INSENSITIVE_ORDER );
}
- if ( importId != null ) {
- if ( UUIDUtils.isTimeBased( importId ) ) {
- timestamp = UUIDUtils.getTimestampInMicros( importId );
- }
- else if ( properties.get( PROPERTY_CREATED ) != null ) {
- timestamp = getLong( properties.get( PROPERTY_CREATED ) ) * 1000;
- }
- }
+
if ( entityClass == null ) {
entityClass = ( Class<A> ) Schema.getDefaultSchema().getEntityClass( entityType );
@@ -2715,7 +2721,7 @@ public class CpEntityManager implements EntityManager {
boolean removeFromDictionary, UUID timestampUuid )
throws Exception {
- long timestamp = UUIDUtils.getUUIDLong( timestampUuid );
+ long timestamp = UUIDUtils.getTimestampInMicros( timestampUuid );
// dictionaryName = dictionaryName.toLowerCase();
if ( elementCoValue == null ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f7b5516b/stack/core/src/main/java/org/apache/usergrid/mq/Message.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/Message.java b/stack/core/src/main/java/org/apache/usergrid/mq/Message.java
index 73c8f7d..aea24ec 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/Message.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/Message.java
@@ -30,7 +30,6 @@ import java.util.UUID;
import javax.xml.bind.annotation.XmlRootElement;
-import com.clearspring.analytics.hash.MurmurHash;
import org.apache.usergrid.utils.UUIDUtils;
import com.fasterxml.jackson.annotation.JsonAnyGetter;
@@ -55,7 +54,6 @@ import static org.apache.usergrid.utils.ConversionUtils.getInt;
import static org.apache.usergrid.utils.ConversionUtils.uuid;
import static org.apache.usergrid.utils.MapUtils.hashMap;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMillis;
-import static org.apache.usergrid.utils.UUIDUtils.isTimeBased;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
@@ -486,7 +484,7 @@ public class Message {
public void setUuid( UUID uuid ) {
properties.put(MESSAGE_ID, uuid);
- properties.put(MESSAGE_TIMESTAMP, UUIDUtils.getUUIDLong(uuid));
+ properties.put(MESSAGE_TIMESTAMP, UUIDUtils.getTimestampInMillis(uuid));
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f7b5516b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 2203cdf..7725202 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -626,8 +626,8 @@ public interface EntityManager {
public void revokeGroupPermission( UUID groupId, String permission ) throws Exception;
- <A extends Entity> A batchCreate( String entityType, Class<A> entityClass, Map<String, Object> properties, UUID
- importId, UUID timestampUuid ) throws Exception;
+ <A extends Entity> A batchCreate(String entityType, Class<A> entityClass, Map<String, Object> properties, UUID
+ importId) throws Exception;
/**
* Batch dictionary property.
*
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f7b5516b/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java b/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java
index a43049d..931d1bd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java
@@ -24,7 +24,6 @@ import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
-import com.clearspring.analytics.hash.MurmurHash;
import com.fasterxml.uuid.EthernetAddress;
import com.fasterxml.uuid.UUIDComparator;
@@ -355,16 +354,6 @@ public class UUIDUtils {
return tryGetUUID( s.substring( offset, offset + 36 ) );
}
- public static long getUUIDLong(UUID id){
- long timestamp = 0;
- if(UUIDUtils.isTimeBased(id)) {
- timestamp = UUIDUtils.getTimestampInMicros(id);
- }else{
- timestamp = MurmurHash.hash64(id);
- }
- return timestamp;
- }
-
public static String toBase64( UUID id ) {
if ( id == null ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f7b5516b/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 32f5f22..f052bf9 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -796,12 +796,6 @@
<!--</dependency>-->
<dependency>
- <groupId>org.python</groupId>
- <artifactId>jython</artifactId>
- <version>2.5.0</version>
- </dependency>
-
- <dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
<version>${org.springframework.version}</version>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f7b5516b/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index 1127198..b880e8e 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -317,11 +317,6 @@
</dependency>
<dependency>
- <groupId>org.python</groupId>
- <artifactId>jython</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
</dependency>
@@ -453,12 +448,6 @@
</dependency>
<dependency>
- <groupId>com.clearspring.analytics</groupId>
- <artifactId>stream</artifactId>
- <version>2.7.0</version>
- </dependency>
-
- <dependency>
<groupId>com.github.fernandospr</groupId>
<artifactId>java-wns</artifactId>
<version>1.3</version>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f7b5516b/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
index f267fab..e334a54 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
@@ -16,7 +16,6 @@
*/
package org.apache.usergrid.services.notifications.gcm;
-import com.clearspring.analytics.hash.MurmurHash;
import com.google.android.gcm.server.*;
import org.apache.usergrid.persistence.entities.Notification;
import org.apache.usergrid.persistence.entities.Notifier;
@@ -61,7 +60,7 @@ public class GCMAdapter implements ProviderAdapter {
public void testConnection() throws Exception {
Sender sender = new Sender(notifier.getApiKey());
Message message = new Message.Builder().addData("registration_id", "").build();
- List<String> ids = new ArrayList<String>();
+ List<String> ids = new ArrayList<>();
ids.add("device_token");
try {
MulticastResult result = sender.send(message, ids, 1);
@@ -103,11 +102,14 @@ public class GCMAdapter implements ProviderAdapter {
private Batch getBatch( Map<String, Object> payload) {
synchronized (this) {
- long hash = MurmurHash.hash64(payload);
- Batch batch = batches.get(hash);
- if (batch == null && payload != null) {
- batch = new Batch(notifier, payload);
- batches.put(hash, batch);
+ Batch batch = new Batch(notifier,null);
+ if( payload != null ) {
+ long hash = payload.hashCode(); // assume there won't be collisions in our amount of concurrency
+ batch = batches.get(hash);
+ if (batch == null) {
+ batch = new Batch(notifier, payload);
+ batches.put(hash, batch);
+ }
}
return batch;
}
@@ -188,13 +190,13 @@ public class GCMAdapter implements ProviderAdapter {
private Map payload;
private List<String> ids;
private List<TaskTracker> trackers;
- private Map<String, Date> inactiveDevices = new HashMap<String, Date>();
+ private Map<String, Date> inactiveDevices = new HashMap<>();
Batch(Notifier notifier, Map<String,Object> payload) {
this.notifier = notifier;
this.payload = payload;
- this.ids = new ArrayList<String>();
- this.trackers = new ArrayList<TaskTracker>();
+ this.ids = new ArrayList<>();
+ this.trackers = new ArrayList<>();
}
synchronized Map<String, Date> getAndClearInactiveDevices() {
@@ -262,7 +264,7 @@ public class GCMAdapter implements ProviderAdapter {
this.trackers.clear();
return;
-
+
}else {
throw new ConnectionException(e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f7b5516b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index 42ef433..c956417 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -16,8 +16,6 @@
*/
package org.apache.usergrid.services.notifications.impl;
-import com.clearspring.analytics.hash.MurmurHash;
-import com.clearspring.analytics.stream.frequency.CountMinSketch;
import com.codahale.metrics.Meter;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.persistence.*;
@@ -114,7 +112,6 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
em.update(notification);
return;
}
- final CountMinSketch sketch = new CountMinSketch(0.0001,.99,7364181); //add probablistic counter to find dups
final UUID appId = em.getApplication().getUuid();
final Map<String,Object> payloads = notification.getPayloads();
@@ -133,15 +130,6 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
if (logger.isTraceEnabled()) {
logger.trace("notification {} starting to queue device {} ", notification.getUuid(), deviceRef.getUuid());
}
- long hash = MurmurHash.hash(deviceRef.getUuid());
- if (sketch.estimateCount(hash) > 0) { //look for duplicates
- if (logger.isDebugEnabled()) {
- logger.debug("Maybe Found duplicate device: {}", deviceRef.getUuid());
- }
- continue;
- } else {
- sketch.add(hash, 1);
- }
String notifierId = null;
String notifierKey = null;
@@ -194,13 +182,10 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
//process up to 10 concurrently
Observable o = rx.Observable.create( new IteratorObservable<Entity>( iterator ) )
-
- .flatMap(entity -> Observable.just(entity).map(entityListFunct)
- .doOnError(throwable -> {
- logger.error("Failed while writing",
- throwable);
- })
- , 10);
+ .distinct( entity -> entity.getUuid() )
+ .flatMap(entity ->
+ Observable.just(entity).map(entityListFunct)
+ .doOnError(throwable -> logger.error("Failed while writing", throwable)) , 10);
o.toBlocking().lastOrDefault( null );
if (logger.isTraceEnabled()) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f7b5516b/stack/services/src/test/java/org/apache/usergrid/python/JythonTest.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/python/JythonTest.java b/stack/services/src/test/java/org/apache/usergrid/python/JythonTest.java
deleted file mode 100644
index 874e5d4..0000000
--- a/stack/services/src/test/java/org/apache/usergrid/python/JythonTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.python;
-
-
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class JythonTest {
- private static final Logger logger = LoggerFactory.getLogger( JythonTest.class );
-
-
- @Test
- public void testJythonObjectFactory() throws Exception {
- logger.info( "Jython tests disabled" );
- }
-
-
- @Test
- public void testJythonBasedService() throws Exception {
- logger.info( "Jython tests disabled" );
- }
-}