You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by le...@apache.org on 2017/10/27 22:39:48 UTC
[11/51] [partial] incubator-sdap-nexus git commit: SDAP-1 Import all
code under the SDAP SGA
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/stream-definitions
----------------------------------------------------------------------
diff --git a/nexus-ingest/developer-box/stream-definitions b/nexus-ingest/developer-box/stream-definitions
new file mode 100644
index 0000000..0188a98
--- /dev/null
+++ b/nexus-ingest/developer-box/stream-definitions
@@ -0,0 +1,64 @@
+stream create --name ingest-mur --definition "scan-for-mur-granules: file --dir=/vagrant/data/mur/ --mode=ref --pattern=*.nc --maxMessages=1 --fixedDelay=60 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=5184 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.kelvintocelsius.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=analysed_sst,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=sst --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy |
set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-mur-clim --definition "scan-for-mur-clim-granules: file --dir=/vagrant/data/mur-clim/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=5184 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.kelvintocelsius.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=analysed_sst,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=mean_sst --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groov
y | add-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-from-granulename.groovy --variables='regex=^mur_(\\d{3}),dateformat=DDD' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^mur_(\\d{3})' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=JPL-L4_GHRSST-SSTfnd-MUR-GLOB-v02.0-fv04.1_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-modis --definition "scan-for-modis-granules: file --dir=/vagrant/data/modis/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=4096 | join-with-path: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=sst,LATITUDE=lat,LONGITUDE=lon,READER=GRIDTILE --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --environment=STORED_VAR_NAME=sst --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-time: script --script=file:/
//home/vagrant/xd-nexus-shared/add-time-from-granulename.groovy --variables='regex=^.*(\d{7})\.,dateformat=yyyyDDD' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_L3_sst_4km' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-modis-clim --definition "scan-for-modisclim-granules: file --dir=/vagrant/data/modis-clim/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=4096 | join-with-path: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=sst,LATITUDE=lat,LONGITUDE=lon,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 --environment=STORED_VAR_NAME=mean_sst | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-from-granulename.groov
y --variables='regex=^A(\\d{3})\\.,dateformat=DDD' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^A(\\d{3})\\.' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_L3m_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-avhrr --definition "scan-for-avhrr-granules: file --dir=/vagrant/data/avhrr/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=1296 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.kelvintocelsius.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=analysed_sst,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=analysed_sst --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | s
et-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=AVHRR_OI_L4_GHRSST_NCEI' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-avhrr-clim --definition "scan-for-avhrr-clim-granules: file --dir=/vagrant/data/avhrr-clim/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=1296 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.kelvintocelsius.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=analysed_sst,META=analysed_sst_std,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=analysed_sst --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nex
us-shared/generate-tile-id.groovy | add-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-from-granulename.groovy --variables='regex=^(\\d{3}),dateformat=DDD' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^(\\d{3})' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=AVHRR_OI_L4_GHRSST_NCEI_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name "ingest-ascatb-u" --definition "scan-for-ascat-granules: file --dir=/vagrant/data/ascatb/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=NUMROWS,NUMCELLS --sliceByDimension=NUMROWS | join-with-path: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_swath_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.winddirspeedtouv.transform:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=wind_speed,LATITUDE=lat,LONGITUDE=lon,TIME=time,META=wind_dir,READER=SWATHTILE,TEMP_DIR=/tmp,U_OR_V=u,STORED_VAR_NAME=wind_u --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/
vagrant/xd-nexus-shared/generate-tile-id.groovy --variables='salt=wind_u' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=ASCATB-L2-Coastal' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042 --insertBuffer=10"
+
+stream create --name "ingest-ascatb-v" --definition "scan-for-ascat-granules: file --dir=/vagrant/data/ascatb/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=NUMROWS,NUMCELLS --sliceByDimension=NUMROWS | join-with-path: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_swath_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.winddirspeedtouv.transform:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=wind_speed,LATITUDE=lat,LONGITUDE=lon,TIME=time,META=wind_dir,READER=SWATHTILE,TEMP_DIR=/tmp,U_OR_V=v,STORED_VAR_NAME=wind_u --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/
vagrant/xd-nexus-shared/generate-tile-id.groovy --variables='salt=wind_u' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=ASCATB-L2-Coastal' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042 --insertBuffer=10"
+
+stream create --name ingest-trmm --definition "file --dir=/vagrant/data/trmm/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=500 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=TRMM_3B42_daily_precipitation_V7,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --sc
ript=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=TRMM_3B42_daily' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-modis-aqua-aod-500 --definition "file --dir=/vagrant/data/modis_aqua_aod/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=500 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=MYD08_D3_6_Aerosol_Optical_Depth_Land_Ocean_Mean,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-i
d.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_AQUA_AOD_500' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-modis-aqua-aod-16 --definition "file --dir=/vagrant/data/modis_aqua_aod/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=16 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=MYD08_D3_6_Aerosol_Optical_Depth_Land_Ocean_Mean,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.
groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_AQUA_AOD_16' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-modis-terra-aod-500 --definition "file --dir=/vagrant/data/modis_terra_aod/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=500 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=MOD08_D3_6_Aerosol_Optical_Depth_Land_Ocean_Mean,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile
-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_TERRA_AOD_500' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-modis-terra-aod-16 --definition "file --dir=/vagrant/data/modis_terra_aod/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=16 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=MOD08_D3_6_Aerosol_Optical_Depth_Land_Ocean_Mean,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-i
d.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_TERRA_AOD_16' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name "ingest-smap-sss" --definition "scan-for-smap-granules: file --dir=/vagrant/data/smap/ --mode=ref --pattern=*.h5 --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=0,1 --sliceByDimension=1 | join-with-path: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=smap_sss,LATITUDE=lat,LONGITUDE=lon,TIME=row_time,GLBLATTR_DAY=REV_START_TIME,GLBLATTR_DAY_FORMAT=%Y-%jT%H:%M:%S.%f,READER=SWATHTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus
-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=SMAP_L2B_SSS' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-grace --definition "scan-for-grace-granules: file --dir=/vagrant/data/grace/ --mode=ref --pattern=*.nc --maxMessages=1 --fixedDelay=60 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=1296 --splitResult=true | join-with-static-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=152' | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=lwe_thickness,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | subtract180: tcpshell --command='python -u -m nexusxd.subtract180longitude' --bufferSize=1000000 | normalize-time: tcpshell --command='python -u -m nexusxd.normalizetimebeginningofmonth' --bufferSize=1000000 | tile-summ
ary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --environment=STORED_VAR_NAME=lwe_thickness --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=TELLUS_GRACE_MASCON_CRI_GRID_RL05_V2_OCEAN' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-ecco --definition "ecco: file --dir=/vagrant/data/ecco/ --mode=ref --pattern=*.nc --maxMessages=1 --fixedDelay=60 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=latitude,longitude --tilesDesired=500 --splitResult=true | join-with-static-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=240' | tile-reader: tcpshell --command='python -u -m nexusxd.tilereadingprocessor' --environment=VARIABLE=MXLDEPTH,LATITUDE=latitude,LONGITUDE=longitude,TIME=time,READER=GRIDTILE --bufferSize=1000000 --remoteReplyTimeout=1300000 | filter-empty: tcpshell --command='python -u -m nexusxd.emptytilefilter' --bufferSize=1000000 | normalize-time: tcpshell --command='python -u -m nexusxd.normalizetimebeginningofmonth' --bufferSize=1000000 | tile-summary: tcpshell --command='python -u -m nexusxd.tilesumarizingprocessor' --environment=STORED_VAR_NAME=MXL
DEPTH --bufferSize=1000000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MXLDEPTH_ECCO_version4_release1' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+
+stream create --name "ingest-ccmp-u" --definition "scan-for-ccmp-granules: file --dir=/vagrant/data/ccmp/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=latitude,longitude --tilesDesired=270 --splitResult=true | join-with-time-slice: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=4' | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.computespeeddirfromuv.transform:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=uwnd,LATITUDE=latitude,LONGITUDE=longitude,TIME=time,META=vwnd,READER=GRIDTILE,TEMP_DIR=/tmp,WIND_U=uwnd,WIND_V=vwnd,STORED_VAR_NAME=wind_u --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file://
/home/vagrant/xd-nexus-shared/generate-tile-id.groovy --variables='salt=wind_u' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=CCMP_V2.0_L3.0' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042 --insertBuffer=75"
+
+stream create --name "ingest-ccmp-spd" --definition "scan-for-ccmp-granules: file --dir=/vagrant/data/ccmp/ --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=latitude,longitude --tilesDesired=270 --splitResult=true | join-with-time-slice: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=4' | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=wind_speed,LATITUDE=latitude,LONGITUDE=longitude,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=wind_speed --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy --variab
les='salt=wind_speed' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=CCMP_V2.0_L3.0_WIND_SPEED' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042 --insertBuffer=75"
+
+stream create --name "ingest-ccmp-spd-clim" --definition "scan-for-ccmp-clim-granules: file --dir=/vagrant/data/ccmp/climatology --mode=ref --pattern=*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=latitude,longitude --tilesDesired=270 --splitResult=true | join-with-time-slice: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=4' | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=mean,META=stddev,LATITUDE=latitude,LONGITUDE=longitude,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=mean_wind_speed --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-t
ile-id.groovy --variables='salt=wind_speed' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^CCMP_Wind_Analysis_V02.0_L3.0_RSS_(\\d{3})_1day_clim_pixelMean' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=CCMP_V2.0_L3.0_WIND_SPEED_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042 --insertBuffer=75"
+
+
+
+stream create --name "ingest-measures-ssh-native" --definition "scan-for-measures-granules: file --dir=/vagrant/data/measures_alt/ --mode=ref --pattern=ssh_grids*.nc --maxMessages=1 | to-filename: transform --expression=payload.absolutePath | permute-dimensions: tcpshell --command='python -u -m nexusxd.callncpdq' --environment=DIMENSION_ORDER=Time:Latitude:Longitude | file-ref: transform --expression=\"new java.io.File(new java.lang.String(payload))\" | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=Latitude,Longitude --tilesDesired=1296 | join-with-static-time: transform --expression=\"'Time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';Time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180lon
gitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=SLA,LATITUDE=Latitude,LONGITUDE=Longitude,TIME=Time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=SLA --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MEASURES_SLA_JPL_1603' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name "ingest-measures-ssh-1x1monthly" --definition "scan-for-measures-granules: file --dir=/vagrant/data/measures_alt/ --mode=ref --pattern=avg-regrid1x1-ssh_grids*.nc --maxMessages=1 --fixedDelay=1 | to-filename: transform --expression=payload.absolutePath | permute-dimensions: tcpshell --command='python -u -m nexusxd.callncpdq' --environment=DIMENSION_ORDER=Time:Latitude:Longitude | file-ref: transform --expression=\"new java.io.File(new java.lang.String(payload))\" | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=Latitude,Longitude --tilesDesired=1296 | join-with-static-time: transform --expression=\"'Time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';Time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_e
mpty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=SLA,LATITUDE=Latitude,LONGITUDE=Longitude,TIME=Time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=SLA --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MEASURES_SLA_JPL_1603_1x1Monthly' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name "ingest-measures-ssh-avg" --definition "scan-for-measures-granules: file --dir=/vagrant/data/measures_alt/ --mode=ref --pattern=avg-regrid1x1-ssh_grids*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=Latitude,Longitude --tilesDesired=1296 | join-with-static-time: transform --expression=\"'Time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';Time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=SLA,LATITUDE=Latitude,LONGITUDE=Longitude,TIME=Time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=SLA --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///
home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MEASURES_SLA_JPL_1603' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name "ingest-measures-ssh-clim" --definition "scan-for-measures-clim-granules: file --dir=/vagrant/data/measures_alt/climatology --mode=ref --pattern=ssh_grids*.nc --maxMessages=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=Latitude,Longitude --tilesDesired=1296 | join-with-static-time: transform --expression=\"'Time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';Time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=mean,META=stddev,LATITUDE=Latitude,LONGITUDE=Longitude,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=mean_SLA --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --scri
pt=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-time: script --script=file:///home/vagrant/xd-nexus-shared/add-time-from-granulename.groovy --variables='regex=^ssh_grids_v1609_(\\d{3})_5day_clim_pixelMean,dateformat=DDD' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^ssh_grids_v1609_(\\d{3})_5day_clim_pixelMean' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MEASURES_SLA_JPL_1603_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+
+stream create --name "ingest-ice_shelf-1x1" --definition "scan-for-ice_shelf-granules: file --dir=/vagrant/data/ice_shelf/ --mode=ref --pattern=1x1regrid-ice_shelf_dh_v1.nc --maxMessages=1 --fixedDelay=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=4 | join-with-time-slice: script --script=file:///home/vagrant/xd-nexus-shared/add-time-to-spatial-spec.groovy --variables='timelen=72' | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=height_filt,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=height_filt --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.
groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=ICE_SHELF_DH_V1_1x1Monthly' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-modis-chl_a --definition "scan-for-chla-granules: file --dir=/vagrant/data/modis_aqua_chl --mode=ref --pattern=*.nc --maxMessages=2 --fixedDelay=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=1296 | join: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=chlor_a,LATITUDE=lat,LONGITUDE=lon,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=chlor_a --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-time: script --script=file:///home/vagrant/xd-nexus-shared/ad
d-time-from-granulename.groovy --variables='regex=^A(\\d{7}),dateformat=yyyyDDD' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_L3m_DAY_CHL_chlor_a_4km' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-modis-chl_a-clim --definition "scan-for-chla-clim-granules: file --dir=/vagrant/data/modis_aqua_chl/climatology --mode=ref --pattern=*.nc --maxMessages=2 --fixedDelay=1 | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=1296 | join: transform --expression=\"payload.stream().collect(T(java.util.stream.Collectors).joining(';'))+';file://'+headers['absolutefilepath']\" | python-chain: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=mean,META=stddev,LATITUDE=lat,LONGITUDE=lon,READER=GRIDTILE,TEMP_DIR=/tmp,STORED_VAR_NAME=mean_chlor_a --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-time: script --script=fil
e:///home/vagrant/xd-nexus-shared/add-time-from-granulename.groovy --variables='regex=^A(\\d{3}),dateformat=DDD' | add-day-atr: script --script=file:///home/vagrant/xd-nexus-shared/add-day-of-year-attribute.groovy --variables='regex=^A(\\d{3})' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=MODIS_L3m_DAY_CHL_chlor_a_4km_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+
+
+stream create --name ingest-smapl3 --definition "file --dir=/vagrant/data/smapl3/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=500 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=sss_smap,LATITUDE=lat,LONGITUDE=lon,TIME=time,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-sha
red/set-dataset-name.groovy --variables='datasetname=SMAP_SSS_L3_MONTHLY_500' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+stream create --name ingest-smapl3-clim --definition "file --dir=/vagrant/sync/data/smapl3-clim/ --mode=ref --pattern=*.nc | header-absolutefilepath: header-enricher --headers={\"absolutefilepath\":\"payload\"} | dataset-tiler --dimensions=lat,lon --tilesDesired=500 | join-with-static-time: transform --expression=\"'time:0:1,'+payload.stream().collect(T(java.util.stream.Collectors).joining(';time:0:1,'))+';file://'+headers['absolutefilepath']\" | process-tiles: tcpshell --command='python -u -m nexusxd.processorchain' --environment=CHAIN=nexusxd.tilereadingprocessor.read_grid_data:nexusxd.emptytilefilter.filter_empty_tiles:nexusxd.subtract180longitude.transform:nexusxd.tilesumarizingprocessor.summarize_nexustile,VARIABLE=mean,META=stddev,LATITUDE=lat,LONGITUDE=lon,READER=GRIDTILE,TEMP_DIR=/tmp --bufferSize=1000000 --remoteReplyTimeout=360000 | add-id: script --script=file:///home/vagrant/xd-nexus-shared/generate-tile-id.groovy | add-day-atr: script --script=file:///home/vagrant/xd-n
exus-shared/add-day-of-year-attribute.groovy --variables='regex=^sss_smap_L3m_clim_doy(\\d{3})' | set-dataset-name: script --script=file:///home/vagrant/xd-nexus-shared/set-dataset-name.groovy --variables='datasetname=SMAP_SSS_L3_MONTHLY_500_CLIM' | nexus --cassandraContactPoints=10.0.2.2 --cassandraKeyspace=nexustiles --solrUrl=http://10.0.2.2:8983/solr/ --solrCollection=nexustiles --cassandraPort=9042"
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/user-init.sh
----------------------------------------------------------------------
diff --git a/nexus-ingest/developer-box/user-init.sh b/nexus-ingest/developer-box/user-init.sh
new file mode 100755
index 0000000..d831fe8
--- /dev/null
+++ b/nexus-ingest/developer-box/user-init.sh
@@ -0,0 +1,17 @@
+homedir=$1
+scriptdir=`dirname $0`
+
+pushd $homedir
+
+mkdir nexus
+pushd nexus
+git init
+popd
+
+mkdir xd-nexus-shared
+
+touch /tmp/xdcommand
+
+popd
+
+$scriptdir/user-update.sh $homedir
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/user-update.sh
----------------------------------------------------------------------
diff --git a/nexus-ingest/developer-box/user-update.sh b/nexus-ingest/developer-box/user-update.sh
new file mode 100755
index 0000000..2648a99
--- /dev/null
+++ b/nexus-ingest/developer-box/user-update.sh
@@ -0,0 +1,86 @@
+scriptdir=`dirname $0`
+
+if [ -z "$1" ]; then
+ homedir="/home/vagrant"
+else
+ homedir=$1
+fi
+
+if [ -z "$2" ]; then
+ condaenv="nexus-xd-python-modules"
+else
+ condaenv=$2
+fi
+
+pushd $homedir
+
+pushd nexus
+git pull https://github.com/dataplumber/nexus.git
+popd
+
+source activate $condaenv
+
+# Install spring-xd python module
+pushd nexus/nexus-ingest/spring-xd-python
+python setup.py install --force
+popd
+
+# Install protobuf generated artifacts
+pushd nexus/nexus-ingest/nexus-messages
+./gradlew clean build writeNewPom
+
+pomfile=`find build/poms/*.xml`
+jarfile=`find build/libs/*.jar`
+mvn install:install-file -DpomFile=$pomfile -Dfile=$jarfile
+
+pushd build/python/nexusproto
+python setup.py install --force
+popd
+popd
+
+# Install ingestion modules
+pushd nexus/nexus-ingest/nexus-xd-python-modules
+python setup.py install --force
+popd
+
+# Install shared Groovy scripts
+pushd nexus/nexus-ingest/groovy-scripts
+rm $homedir/xd-nexus-shared/*.groovy
+cp *.groovy $homedir/xd-nexus-shared
+popd
+
+# Delete all streams in Spring XD so we can update the custom modules
+echo stream all destroy --force > /tmp/xdcommand
+$homedir/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell --cmdfile /tmp/xdcommand
+
+# Build and upload dataset-tiler
+pushd nexus/nexus-ingest/dataset-tiler
+./gradlew clean build
+jarfile=`find build/libs/*.jar`
+touch /tmp/moduleupload
+echo module upload --type processor --name dataset-tiler --file $jarfile --force > /tmp/xdcommand
+$homedir/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell --cmdfile /tmp/xdcommand
+popd
+
+# Build and upload tcp-shell
+pushd nexus/nexus-ingest/tcp-shell
+./gradlew clean build
+jarfile=`find build/libs/*.jar`
+touch /tmp/moduleupload
+echo module upload --type processor --name tcpshell --file $jarfile --force > /tmp/xdcommand
+$homedir/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell --cmdfile /tmp/xdcommand
+popd
+
+# Build and upload nexus-sink
+pushd nexus/nexus-ingest/nexus-sink
+./gradlew clean build
+jarfile=`find build/libs/*.jar`
+touch /tmp/moduleupload
+echo module upload --type sink --name nexus --file $jarfile --force > /tmp/xdcommand
+$homedir/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell --cmdfile /tmp/xdcommand
+popd
+
+# Create the streams
+$homedir/spring-xd-1.3.1.RELEASE/shell/bin/xd-shell --cmdfile $scriptdir/stream-definitions
+
+popd
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/developer-box/xd-singlenode-logback.groovy
----------------------------------------------------------------------
diff --git a/nexus-ingest/developer-box/xd-singlenode-logback.groovy b/nexus-ingest/developer-box/xd-singlenode-logback.groovy
new file mode 100644
index 0000000..5aec45b
--- /dev/null
+++ b/nexus-ingest/developer-box/xd-singlenode-logback.groovy
@@ -0,0 +1,91 @@
+/*****************************************************************************
+* Copyright (c) 2016 Jet Propulsion Laboratory,
+* California Institute of Technology. All rights reserved
+*****************************************************************************/
+import org.springframework.xd.dirt.util.logging.CustomLoggerConverter
+import org.springframework.xd.dirt.util.logging.VersionPatternConverter
+import ch.qos.logback.classic.encoder.PatternLayoutEncoder
+import ch.qos.logback.core.rolling.RollingFileAppender
+
+// We highly recommended that you always add a status listener just
+// after the last import statement and before all other statements
+// NOTE - this includes logging configuration in the log and stacktraces in the event of errors
+// statusListener(OnConsoleStatusListener)
+
+// Emulates Log4j formatting
+conversionRule("category", CustomLoggerConverter)
+
+//XD Version
+conversionRule("version", VersionPatternConverter)
+
+def ISO8601 = "yyyy-MM-dd'T'HH:mm:ssZ"
+def datePattern = ISO8601
+
+appender("STDOUT", ConsoleAppender) {
+ encoder(PatternLayoutEncoder) {
+ pattern = "%d{${datePattern}} %version %level{5} %thread %category{2} - %msg%n"
+ }
+}
+
+def logfileNameBase = "${System.getProperty('xd.home')}/logs/singlenode-${System.getProperty('PID')}"
+
+appender("FILE", RollingFileAppender) {
+ file = "${logfileNameBase}.log"
+ append = false
+ rollingPolicy(TimeBasedRollingPolicy) {
+ fileNamePattern = "${logfileNameBase}-%d{yyyy-MM-dd}.%i.log"
+ timeBasedFileNamingAndTriggeringPolicy(SizeAndTimeBasedFNATP) {
+ maxFileSize = "100KB"
+ }
+ }
+
+ encoder(PatternLayoutEncoder) {
+ pattern = "%d{${datePattern}} %version %level{5} %thread %category{2} - %msg%n"
+ }
+}
+
+root(WARN, ["STDOUT", "FILE"])
+
+logger("org.nasa", INFO)
+logger("org.springframework.xd", WARN)
+logger("org.springframework.xd.dirt.server", INFO)
+logger("org.springframework.xd.dirt.util.XdConfigLoggingInitializer", INFO)
+logger("xd.sink", INFO)
+logger("org.springframework.xd.sqoop", INFO)
+// This is for the throughput-sampler sink module
+logger("org.springframework.xd.integration.throughput", INFO)
+
+logger("org.springframework", WARN)
+logger("org.springframework.boot", WARN)
+logger("org.springframework.integration", WARN)
+logger("org.springframework.retry", WARN)
+logger("org.springframework.amqp", WARN)
+
+// Below this line are specific settings for things that are too noisy
+logger("org.springframework.beans.factory.config", ERROR)
+logger("org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer", ERROR)
+
+// This prevents the WARN level InstanceNotFoundException: org.apache.ZooKeeperService:name0=StandaloneServer_port-1
+logger("org.apache.zookeeper.jmx.MBeanRegistry", ERROR)
+
+
+// This prevents the WARN level about a non-static, @Bean method in Spring Batch that is irrelevant
+logger("org.springframework.context.annotation.ConfigurationClassEnhancer", ERROR)
+
+// This prevents the "Error:KeeperErrorCode = NodeExists" INFO messages
+// logged by ZooKeeper when a parent node does not exist while
+// invoking Curator's creatingParentsIfNeeded node builder.
+logger("org.apache.zookeeper.server.PrepRequestProcessor", WARN)
+
+
+// This prevents boot LoggingApplicationListener logger's misleading warning message
+logger("org.springframework.boot.logging.LoggingApplicationListener", ERROR)
+
+
+
+// This prevents Hadoop configuration warnings
+logger("org.apache.hadoop.conf.Configuration", ERROR)
+
+// Suppress json-path warning until SI 4.2 is released
+logger("org.springframework.integration.config.IntegrationRegistrar", ERROR)
+
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/.gitignore
----------------------------------------------------------------------
diff --git a/nexus-ingest/groovy-scripts/.gitignore b/nexus-ingest/groovy-scripts/.gitignore
new file mode 100644
index 0000000..21e72df
--- /dev/null
+++ b/nexus-ingest/groovy-scripts/.gitignore
@@ -0,0 +1,4 @@
+*.iml
+
+.idea
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/README.md
----------------------------------------------------------------------
diff --git a/nexus-ingest/groovy-scripts/README.md b/nexus-ingest/groovy-scripts/README.md
new file mode 100644
index 0000000..92e66ba
--- /dev/null
+++ b/nexus-ingest/groovy-scripts/README.md
@@ -0,0 +1,3 @@
+# groovy-scripts
+
+Various groovy scripts that can be used as part of an [XD Stream](http://docs.spring.io/spring-xd/docs/current/reference/html/#script).
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/add-day-of-year-attribute.groovy
----------------------------------------------------------------------
diff --git a/nexus-ingest/groovy-scripts/add-day-of-year-attribute.groovy b/nexus-ingest/groovy-scripts/add-day-of-year-attribute.groovy
new file mode 100644
index 0000000..daf533a
--- /dev/null
+++ b/nexus-ingest/groovy-scripts/add-day-of-year-attribute.groovy
@@ -0,0 +1,48 @@
+/*****************************************************************************
+* Copyright (c) 2016 Jet Propulsion Laboratory,
+* California Institute of Technology. All rights reserved
+*****************************************************************************/
+/**
+ * Created by greguska on 3/29/16.
+ */
+
+@Grapes([
+ @Grab(group = 'org.nasa.jpl.nexus', module = 'nexus-messages', version = '1.0.0.RELEASE')
+])
+
+import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent
+
+if (regex == null ){
+ throw new RuntimeException("This script requires a regex to use for matching against the granulename.")
+}
+if (!(payload instanceof byte[])){
+ throw new RuntimeException("Can't handle messages that are not byte[]. Got payload of type ${payload.class}")
+}
+
+def pattern = ~"$regex"
+def tileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(payload)
+
+def tileSummary = tileBuilder.summaryBuilder
+
+def granulename = tileSummary.granule
+
+def matches = (granulename =~ pattern)
+
+if (!matches.hasGroup()){
+ throw new RuntimeException("regex did not return any groups.")
+}
+if (1 != matches.size()){
+ throw new RuntimeException("regex did not return *one* group.")
+}
+if (2 != matches[0].size()){
+ throw new RuntimeException("group does not contain match.")
+}
+
+def dayOfYear = matches[0][1]
+
+tileSummary.addGlobalAttributes(NexusContent.Attribute.newBuilder()
+ .setName("day_of_year_i")
+ .addValues(dayOfYear.toString())
+ .build())
+
+return tileBuilder.build().toByteArray()
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/add-time-from-granulename.groovy
----------------------------------------------------------------------
diff --git a/nexus-ingest/groovy-scripts/add-time-from-granulename.groovy b/nexus-ingest/groovy-scripts/add-time-from-granulename.groovy
new file mode 100644
index 0000000..1591018
--- /dev/null
+++ b/nexus-ingest/groovy-scripts/add-time-from-granulename.groovy
@@ -0,0 +1,63 @@
+/*****************************************************************************
+* Copyright (c) 2016 Jet Propulsion Laboratory,
+* California Institute of Technology. All rights reserved
+*****************************************************************************/
+/**
+ * Created by greguska on 3/29/16.
+ */
+
+@Grapes([
+ @Grab(group = 'org.nasa.jpl.nexus', module = 'nexus-messages', version = '1.0.0.RELEASE')
+])
+
+import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent
+
+
+if (regex == null ){
+ throw new RuntimeException("This script requires a regex to use for matching against the granulename.")
+}
+if (dateformat == null ){
+ throw new RuntimeException("This script requires a date format to parse the date extracted using the regex.")
+}
+if (!(payload instanceof byte[])){
+ throw new RuntimeException("Can't handle messages that are not byte[]. Got payload of type ${payload.class}")
+}
+
+def pattern = ~"$regex"
+def tileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(payload)
+
+switch(tileBuilder.tile.tileTypeCase){
+ case NexusContent.TileData.TileTypeCase.GRID_TILE:
+ def gridtilebuilder = tileBuilder.tileBuilder.gridTileBuilder
+ def tilesummary = tileBuilder.summaryBuilder
+
+ def granulename = tilesummary.granule
+
+ def matches = (granulename =~ pattern)
+
+ if (!matches.hasGroup()){
+ throw new RuntimeException("regex did not return any groups.")
+ }
+ if (1 != matches.size()){
+ throw new RuntimeException("regex did not return *one* group.")
+ }
+ if (2 != matches[0].size()){
+ throw new RuntimeException("group does not contain match.")
+ }
+
+ def datestring = matches[0][1]
+
+ def time = Date.parse(dateformat, datestring.toString(), TimeZone.getTimeZone("UTC"))
+
+
+ def secondsSinceEpoch = (time.time / 1000).toLong()
+
+ gridtilebuilder.setTime(secondsSinceEpoch)
+ tilesummary.statsBuilder.setMinTime(secondsSinceEpoch)
+ tilesummary.statsBuilder.setMaxTime(secondsSinceEpoch)
+ break
+ default:
+ throw new RuntimeException("Can only handle GridTile at this time.")
+}
+
+return tileBuilder.build().toByteArray()
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/add-time-to-spatial-spec.groovy
----------------------------------------------------------------------
diff --git a/nexus-ingest/groovy-scripts/add-time-to-spatial-spec.groovy b/nexus-ingest/groovy-scripts/add-time-to-spatial-spec.groovy
new file mode 100644
index 0000000..393164c
--- /dev/null
+++ b/nexus-ingest/groovy-scripts/add-time-to-spatial-spec.groovy
@@ -0,0 +1,34 @@
+/*****************************************************************************
+ * Copyright (c) 2016 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+/**
+ * Created by Nga Quach on 8/31/16.
+ */
+
+if (timelen == null) {
+ throw new RuntimeException("This script requires the length of the time array.")
+}
+def time = 'time'
+if (binding.variables.get("timevar") != null) {
+ time = timevar
+}
+
+def specsIn = payload
+if (payload instanceof String) {
+ specsIn = [payload]
+}
+
+def length = timelen.toInteger()
+def specsOut = []
+for (i = 0; i < length; i++) {
+ specsOut.addAll(specsIn.collect{ spec ->
+ "$time:$i:${i+1},$spec"
+ })
+}
+
+def sectionSpec = specsOut.join(';')
+sectionSpec <<= ';file://'
+sectionSpec <<= headers.absolutefilepath
+
+return sectionSpec.toString()
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/dev-install.sh
----------------------------------------------------------------------
diff --git a/nexus-ingest/groovy-scripts/dev-install.sh b/nexus-ingest/groovy-scripts/dev-install.sh
new file mode 100755
index 0000000..44f1bda
--- /dev/null
+++ b/nexus-ingest/groovy-scripts/dev-install.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+scriptdir=`dirname $0`
+sharedscripts=/home/vagrant/xd-nexus-shared
+
+rm $sharedscripts/*.groovy
+cp $scriptdir/*.groovy $sharedscripts
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/generate-tile-id.groovy
----------------------------------------------------------------------
diff --git a/nexus-ingest/groovy-scripts/generate-tile-id.groovy b/nexus-ingest/groovy-scripts/generate-tile-id.groovy
new file mode 100644
index 0000000..7c4d7de
--- /dev/null
+++ b/nexus-ingest/groovy-scripts/generate-tile-id.groovy
@@ -0,0 +1,37 @@
+/*****************************************************************************
+* Copyright (c) 2016 Jet Propulsion Laboratory,
+* California Institute of Technology. All rights reserved
+*****************************************************************************/
+/**
+ * Created by greguska on 3/29/16.
+ */
+
+@Grapes([
+ @Grab(group = 'org.nasa.jpl.nexus', module = 'nexus-messages', version = '1.0.0.RELEASE')
+])
+
+import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent
+
+if (!(payload instanceof byte[])){
+ throw new RuntimeException("Can't handle messages that are not byte[]. Got payload of type ${payload.class}")
+}
+thesalt = binding.variables.get("salt")
+
+
+def tileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(payload)
+def summaryBuilder = tileBuilder.getSummaryBuilder()
+
+def granule = summaryBuilder.hasGranule()?summaryBuilder.granule:"${headers['absolutefilepath']}".split(File.separator)[-1]
+def originalSpec = summaryBuilder.hasSectionSpec()?summaryBuilder.sectionSpec:"${headers['spec']}"
+
+
+def tileId = UUID.nameUUIDFromBytes("${granule[0..-4]}$originalSpec${thesalt==null?'':thesalt}".toString().bytes).toString()
+
+summaryBuilder.setGranule(granule)
+summaryBuilder.setTileId(tileId)
+summaryBuilder.setSectionSpec(originalSpec)
+
+tileBuilder.getTileBuilder().setTileId(tileId)
+
+return tileBuilder.build().toByteArray()
+
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/nexustile-to-string.groovy
----------------------------------------------------------------------
diff --git a/nexus-ingest/groovy-scripts/nexustile-to-string.groovy b/nexus-ingest/groovy-scripts/nexustile-to-string.groovy
new file mode 100644
index 0000000..39030e7
--- /dev/null
+++ b/nexus-ingest/groovy-scripts/nexustile-to-string.groovy
@@ -0,0 +1,23 @@
+/*****************************************************************************
+ * Copyright (c) 2016 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+/**
+ * Created by greguska on 7/18/16.
+ */
+
+
+@Grapes([
+ @Grab(group = 'org.nasa.jpl.nexus', module = 'nexus-messages', version = '1.0.0.RELEASE')
+])
+
+import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent
+
+
+includeData = Boolean.parseBoolean(binding.variables.get("includeData"))
+
+if(includeData){
+ NexusContent.NexusTile.newBuilder().mergeFrom(payload).toString()
+}else {
+ NexusContent.NexusTile.newBuilder().mergeFrom(payload).summary.toString()
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/set-dataset-name.groovy
----------------------------------------------------------------------
diff --git a/nexus-ingest/groovy-scripts/set-dataset-name.groovy b/nexus-ingest/groovy-scripts/set-dataset-name.groovy
new file mode 100644
index 0000000..ce2f6a2
--- /dev/null
+++ b/nexus-ingest/groovy-scripts/set-dataset-name.groovy
@@ -0,0 +1,28 @@
+/*****************************************************************************
+* Copyright (c) 2016 Jet Propulsion Laboratory,
+* California Institute of Technology. All rights reserved
+*****************************************************************************/
+/**
+ * Created by greguska on 3/29/16.
+ */
+
+@Grapes([
+ @Grab(group = 'org.nasa.jpl.nexus', module = 'nexus-messages', version = '1.0.0.RELEASE')
+])
+
+import org.nasa.jpl.nexus.ingest.wiretypes.NexusContent
+
+if (!(payload instanceof byte[])){
+ throw new RuntimeException("Can't handle messages that are not byte[]. Got payload of type ${payload.class}")
+}
+
+def tileBuilder = NexusContent.NexusTile.newBuilder().mergeFrom(payload)
+
+
+assert datasetname != null : "This script requires a variable called datasetname."
+
+def summaryBuilder = tileBuilder.getSummaryBuilder()
+summaryBuilder.setDatasetName(datasetname)
+
+return tileBuilder.build().toByteArray()
+
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/groovy-scripts/test/TestAddTimeToSpatialSpec.groovy
----------------------------------------------------------------------
diff --git a/nexus-ingest/groovy-scripts/test/TestAddTimeToSpatialSpec.groovy b/nexus-ingest/groovy-scripts/test/TestAddTimeToSpatialSpec.groovy
new file mode 100644
index 0000000..0eb2b07
--- /dev/null
+++ b/nexus-ingest/groovy-scripts/test/TestAddTimeToSpatialSpec.groovy
@@ -0,0 +1,82 @@
+/*****************************************************************************
+ * Copyright (c) 2017 Jet Propulsion Laboratory,
+ * California Institute of Technology. All rights reserved
+ *****************************************************************************/
+/**
+ * Created by greguska on 3/28/17.
+ */
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+import static groovy.test.GroovyAssert.shouldFail
+import static org.junit.Assert.assertEquals
+
+class TestAddTimeToSpatialSpec {
+ GroovyShell shell
+ Binding binding
+ PrintStream orig
+ ByteArrayOutputStream out
+
+ File script
+
+ @Before
+ void setUp() {
+ orig = System.out
+ out = new ByteArrayOutputStream()
+ System.setOut(new PrintStream(out))
+ binding = new Binding()
+ shell = new GroovyShell(binding)
+
+ script = new File('add-time-to-spatial-spec.groovy')
+ }
+
+ @After
+ void tearDown() {
+ System.setOut(orig)
+ }
+
+ @Test
+ void testMissingTimeLen() {
+ def e = shouldFail RuntimeException, {
+ binding.timelen = null
+ shell.evaluate(script)
+ }
+ assert 'This script requires the length of the time array.' == e.message
+ }
+
+ @Test
+ void testStringPayload() {
+ binding.timelen = 4
+ binding.payload = "test:0:1,script:3:4"
+ binding.headers = ['absolutefilepath': 'afilepath']
+
+ def expected = [
+ "time:0:1,test:0:1,script:3:4",
+ "time:1:2,test:0:1,script:3:4",
+ "time:2:3,test:0:1,script:3:4",
+ "time:3:4,test:0:1,script:3:4"
+ ].join(';') + ';file://afilepath'
+
+ def result = shell.evaluate(script)
+ assertEquals expected, result
+ }
+
+ @Test
+ void testListPayload() {
+ binding.timelen = 2
+ binding.payload = ["test:0:1,script:3:4", "test:1:2,script:4:5"]
+ binding.headers = ['absolutefilepath': 'afilepath']
+
+ def expected = [
+ "time:0:1,test:0:1,script:3:4",
+ "time:0:1,test:1:2,script:4:5",
+ "time:1:2,test:0:1,script:3:4",
+ "time:1:2,test:1:2,script:4:5"
+ ].join(';') + ';file://afilepath'
+
+ def result = shell.evaluate(script)
+ assertEquals expected, result
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/.gitignore
----------------------------------------------------------------------
diff --git a/nexus-ingest/nexus-messages/.gitignore b/nexus-ingest/nexus-messages/.gitignore
new file mode 100644
index 0000000..ab9b2f9
--- /dev/null
+++ b/nexus-ingest/nexus-messages/.gitignore
@@ -0,0 +1,35 @@
+.gradle/
+.idea/
+gradlew.bat
+
+distrobution/
+
+gen/
+
+build/*
+!build/reports
+
+build/reports/*
+!build/reports/license
+!build/reports/project
+
+nexusproto.egg-info/
+
+.DS_Store
+*.log
+
+#Idea files
+*.iml
+*.ipr
+*.iws
+
+*.class
+
+# Package Files #
+*.war
+*.ear
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+
+gradle.properties
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/README.md
----------------------------------------------------------------------
diff --git a/nexus-ingest/nexus-messages/README.md b/nexus-ingest/nexus-messages/README.md
new file mode 100644
index 0000000..34bf7c7
--- /dev/null
+++ b/nexus-ingest/nexus-messages/README.md
@@ -0,0 +1,24 @@
+# nexus-messages
+
+This project contains the [protobuf](https://developers.google.com/protocol-buffers/) definition for a NexusTile. By compiling the protobuf specification, both Java and Python objects are generated.
+
+# Developer Installation
+
+1. Run `./gradlew clean build install`
+
+2. cd into `/build/python/nexusproto`
+
+3. Setup a separate conda env or activate an existing one
+
+ ````
+ conda create --name nexus-messages python
+ source activate nexus-messages
+ ````
+
+4. Install Conda dependencies
+
+ ````
+ conda install numpy
+ ````
+
+5. Run `python setup.py install`
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build.gradle
----------------------------------------------------------------------
diff --git a/nexus-ingest/nexus-messages/build.gradle b/nexus-ingest/nexus-messages/build.gradle
new file mode 100644
index 0000000..290a338
--- /dev/null
+++ b/nexus-ingest/nexus-messages/build.gradle
@@ -0,0 +1,195 @@
+import java.nio.file.FileVisitResult
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.SimpleFileVisitor
+import java.nio.file.StandardCopyOption
+import java.nio.file.attribute.BasicFileAttributes
+
+buildscript {
+ repositories {
+ if( project.hasProperty('artifactory_contextUrl') ) {
+ maven {
+ url "${artifactory_contextUrl}"
+ credentials {
+ username = "${artifactory_user}"
+ password = "${artifactory_password}"
+ }
+ }
+ }
+
+ mavenCentral()
+ jcenter()
+ }
+ dependencies {
+ //Check for the latest version here: http://plugins.gradle.org/plugin/com.jfrog.artifactory
+ classpath "org.jfrog.buildinfo:build-info-extractor-gradle:4+"
+ classpath 'com.google.protobuf:protobuf-gradle-plugin:0.7.7'
+ }
+}
+
+if( project.hasProperty('artifactory_contextUrl') ) {
+ allprojects {
+ apply plugin: "com.jfrog.artifactory"
+ }
+
+ artifactory {
+ contextUrl = "${artifactory_contextUrl}"
+ publish {
+ repository {
+ repoKey = "${publish_repoKey}"
+ username = "${artifactory_user}"
+ password = "${artifactory_password}"
+ maven = true
+
+ }
+ defaults {
+ // Reference to Gradle publications defined in the build script.
+ // This is how we tell the Artifactory Plugin which artifacts should be
+ // published to Artifactory.
+ publications('mavenJava')
+ publishArtifacts = true
+ // Publish generated POM files to Artifactory (true by default)
+ publishPom = true
+ }
+ }
+ resolve {
+ repository {
+ repoKey = "${resolve_repoKey}"
+ username = "${artifactory_user}"
+ password = "${artifactory_password}"
+ maven = true
+
+ }
+ }
+ }
+
+ repositories {
+ maven {
+ url "$artifactory_contextUrl/$resolve_repoKey"
+ credentials {
+ username "${artifactory_user}"
+ password "${artifactory_password}"
+ }
+ }
+ }
+
+}else {
+ repositories {
+ mavenCentral()
+ }
+}
+
+apply plugin: 'java'
+apply plugin: 'com.google.protobuf'
+apply plugin: 'maven'
+apply plugin: 'maven-publish'
+apply plugin: 'project-report'
+
+
+group = 'org.nasa.jpl.nexus'
+version = '1.0.0.RELEASE'
+
+ext{
+ genDirectory = "$projectDir/gen"
+ distDirectory = "$projectDir/distrobution"
+ pythonBuildDirPath = "${file(buildDir.path + '/python/nexusproto').path}"
+}
+
+protobuf {
+
+ generatedFilesBaseDir = genDirectory
+
+ // Configure the protoc executable
+ protoc {
+ // Download from repositories
+ artifact = 'com.google.protobuf:protoc:2.6.1'
+ }
+
+ plugins {
+ // Define a plugin with name 'grpc'
+ grpc {
+ path = 'tools/protoc-gen-grpc-java'
+ }
+ }
+
+ generateProtoTasks {
+ all().each { task ->
+ task.builtins {
+ python {
+ outputSubDir = 'python'
+ }
+ }
+ }
+ }
+
+}
+
+task writeNewPom << {
+ pom {}.writeTo(file(buildDir.path + "/poms/${project.name}-${project.version}.xml"))
+}
+
+publishing {
+ publications {
+ mavenJava(MavenPublication) {
+ from components.java
+ }
+ }
+}
+
+assemble.doLast{
+ File pythonbuilddir = file(pythonBuildDirPath)
+
+ File pythonsource = file('src/main/python')
+
+ Files.walkFileTree(pythonsource.toPath(), new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException {
+ Files.createDirectories(pythonbuilddir.toPath().resolve(pythonsource.toPath().relativize(dir)))
+ return FileVisitResult.CONTINUE
+ }
+
+ @Override
+ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
+ Files.copy(file, pythonbuilddir.toPath().resolve(pythonsource.toPath().relativize(file)), StandardCopyOption.REPLACE_EXISTING)
+ return FileVisitResult.CONTINUE
+ }
+ })
+
+ Files.move(file(pythonbuilddir.path + '/nexusproto/setup.py').toPath(), file(pythonbuilddir.path + '/setup.py').toPath(), StandardCopyOption.REPLACE_EXISTING)
+
+ File generatedPython = file("$genDirectory/main/python/NexusContent_pb2.py")
+
+ Files.copy(generatedPython.toPath(), file(pythonbuilddir.path + '/nexusproto/NexusContent_pb2.py').toPath(), StandardCopyOption.REPLACE_EXISTING)
+}
+
+
+if( project.hasProperty('python_executable') ) {
+ task publishPython(type: Exec, group: "publishing", dependsOn: [assemble]) {
+ workingDir "$pythonBuildDirPath"
+ commandLine "$python_executable", 'setup.py', 'bdist_wheel', 'upload', '-r', 'deepdata-artifactory-pypi'
+ }
+}
+
+task tarPython(type: Tar, dependsOn: [assemble]){
+ destinationDir = file("distrobution")
+ archiveName = 'nexusproto.tar.gz'
+ compression = Compression.GZIP
+ from(file(buildDir.path + '/python')){
+ include '**/*'
+ }
+}
+
+clean << {
+ file(genDirectory).deleteDir()
+ file(distDirectory).deleteDir()
+}
+
+dependencies {
+
+ compile 'com.google.protobuf:protobuf-java:2.6.1'
+
+}
+
+task wrapper(type: Wrapper) {
+ gradleVersion = '2.12'
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/license/dependency-license.html
----------------------------------------------------------------------
diff --git a/nexus-ingest/nexus-messages/build/reports/license/dependency-license.html b/nexus-ingest/nexus-messages/build/reports/license/dependency-license.html
new file mode 100644
index 0000000..9c6d93a
--- /dev/null
+++ b/nexus-ingest/nexus-messages/build/reports/license/dependency-license.html
@@ -0,0 +1,62 @@
+<html>
+ <head>
+ <title>HTML License report</title>
+ </head>
+ <style>table {
+ width: 85%;
+ border-collapse: collapse;
+ text-align: center;
+ }
+ .dependencies {
+ text-align: left;
+ }
+ tr {
+ border: 1px solid black;
+ }
+ td {
+ border: 1px solid black;
+ font-weight: bold;
+ color: #2E2E2E
+ }
+ th {
+ border: 1px solid black;
+ }
+ h3 {
+ text-align:center;
+ margin:3px
+ }
+ .license {
+ width:70%
+ }
+
+ .licenseName {
+ width:15%
+ }
+ </style>
+ <body>
+ <table align='center'>
+ <tr>
+ <th>
+ <h3>Dependency</h3>
+ </th>
+ <th>
+ <h3>Jar</h3>
+ </th>
+ <th>
+ <h3>License name</h3>
+ </th>
+ <th>
+ <h3>License text URL</h3>
+ </th>
+ </tr>
+ <tr>
+ <td class='dependencies'>com.google.protobuf:protobuf-java:2.6.1</td>
+ <td class='licenseName'>protobuf-java-2.6.1.jar</td>
+ <td class='licenseName'>New BSD license</td>
+ <td class='license'>
+ <a href='http://www.opensource.org/licenses/bsd-license.php'>Show license agreement</a>
+ </td>
+ </tr>
+ </table>
+ </body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/license/dependency-license.xml
----------------------------------------------------------------------
diff --git a/nexus-ingest/nexus-messages/build/reports/license/dependency-license.xml b/nexus-ingest/nexus-messages/build/reports/license/dependency-license.xml
new file mode 100644
index 0000000..8ed0e97
--- /dev/null
+++ b/nexus-ingest/nexus-messages/build/reports/license/dependency-license.xml
@@ -0,0 +1,6 @@
+<dependencies>
+ <dependency name='com.google.protobuf:protobuf-java:2.6.1'>
+ <file>protobuf-java-2.6.1.jar</file>
+ <license name='New BSD license' url='http://www.opensource.org/licenses/bsd-license.php' />
+ </dependency>
+</dependencies>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/license/license-dependency.html
----------------------------------------------------------------------
diff --git a/nexus-ingest/nexus-messages/build/reports/license/license-dependency.html b/nexus-ingest/nexus-messages/build/reports/license/license-dependency.html
new file mode 100644
index 0000000..d9f5e84
--- /dev/null
+++ b/nexus-ingest/nexus-messages/build/reports/license/license-dependency.html
@@ -0,0 +1,69 @@
+<html>
+ <head>
+ <title>HTML License report</title>
+ </head>
+ <style>table {
+ width: 85%;
+ border-collapse: collapse;
+ text-align: center;
+ }
+
+ .dependencies {
+ text-align: left;
+ width:15%;
+ }
+
+ tr {
+ border: 1px solid black;
+ }
+
+ td {
+ border: 1px solid black;
+ font-weight: bold;
+ color: #2E2E2E
+ }
+
+ th {
+ border: 1px solid black;
+ }
+
+ h3 {
+ text-align:center;
+ margin:3px
+ }
+
+ .license {
+ width:70%
+ }
+
+ .licenseName {
+ width:15%
+ }
+ </style>
+ <body>
+ <table align='center'>
+ <tr>
+ <th>
+ <h3>License</h3>
+ </th>
+ <th>
+ <h3>License text URL</h3>
+ </th>
+ <th>
+ <h3>Dependency</h3>
+ </th>
+ </tr>
+ <tr>
+ <td class='licenseName'>New BSD license</td>
+ <td class='license'>
+ <a href='http://www.opensource.org/licenses/bsd-license.php'>License agreement</a>
+ </td>
+ <td class='dependencies'>
+ <ul>
+ <li>protobuf-java-2.6.1.jar</li>
+ </ul>
+ </td>
+ </tr>
+ </table>
+ </body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/license/license-dependency.xml
----------------------------------------------------------------------
diff --git a/nexus-ingest/nexus-messages/build/reports/license/license-dependency.xml b/nexus-ingest/nexus-messages/build/reports/license/license-dependency.xml
new file mode 100644
index 0000000..66df9d3
--- /dev/null
+++ b/nexus-ingest/nexus-messages/build/reports/license/license-dependency.xml
@@ -0,0 +1,5 @@
+<licenses>
+ <license name='New BSD license' url='http://www.opensource.org/licenses/bsd-license.php'>
+ <dependency>protobuf-java-2.6.1.jar</dependency>
+ </license>
+</licenses>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/project/dependencies.txt
----------------------------------------------------------------------
diff --git a/nexus-ingest/nexus-messages/build/reports/project/dependencies.txt b/nexus-ingest/nexus-messages/build/reports/project/dependencies.txt
new file mode 100644
index 0000000..6698ae1
--- /dev/null
+++ b/nexus-ingest/nexus-messages/build/reports/project/dependencies.txt
@@ -0,0 +1,43 @@
+
+------------------------------------------------------------
+Root project
+------------------------------------------------------------
+
+archives - Configuration for archive artifacts.
+No dependencies
+
+compile - Dependencies for source set 'main'.
+\--- com.google.protobuf:protobuf-java:2.6.1
+
+compileClasspath - Compile classpath for source set 'main'.
+\--- com.google.protobuf:protobuf-java:2.6.1
+
+compileOnly - Compile dependencies for source set 'main'.
+\--- com.google.protobuf:protobuf-java:2.6.1
+
+default - Configuration for default artifacts.
+\--- com.google.protobuf:protobuf-java:2.6.1
+
+protobuf
+No dependencies
+
+protobufToolsLocator_protoc
+\--- com.google.protobuf:protoc:2.6.1
+
+runtime - Runtime dependencies for source set 'main'.
+\--- com.google.protobuf:protobuf-java:2.6.1
+
+testCompile - Dependencies for source set 'test'.
+\--- com.google.protobuf:protobuf-java:2.6.1
+
+testCompileClasspath - Compile classpath for source set 'test'.
+\--- com.google.protobuf:protobuf-java:2.6.1
+
+testCompileOnly - Compile dependencies for source set 'test'.
+\--- com.google.protobuf:protobuf-java:2.6.1
+
+testProtobuf
+No dependencies
+
+testRuntime - Runtime dependencies for source set 'test'.
+\--- com.google.protobuf:protobuf-java:2.6.1
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/base-style.css
----------------------------------------------------------------------
diff --git a/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/base-style.css b/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/base-style.css
new file mode 100644
index 0000000..4afa73e
--- /dev/null
+++ b/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/base-style.css
@@ -0,0 +1,179 @@
+
+body {
+ margin: 0;
+ padding: 0;
+ font-family: sans-serif;
+ font-size: 12pt;
+}
+
+body, a, a:visited {
+ color: #303030;
+}
+
+#content {
+ padding-left: 50px;
+ padding-right: 50px;
+ padding-top: 30px;
+ padding-bottom: 30px;
+}
+
+#content h1 {
+ font-size: 160%;
+ margin-bottom: 10px;
+}
+
+#footer {
+ margin-top: 100px;
+ font-size: 80%;
+ white-space: nowrap;
+}
+
+#footer, #footer a {
+ color: #a0a0a0;
+}
+
+#line-wrapping-toggle {
+ vertical-align: middle;
+}
+
+#label-for-line-wrapping-toggle {
+ vertical-align: middle;
+}
+
+ul {
+ margin-left: 0;
+}
+
+h1, h2, h3 {
+ white-space: nowrap;
+}
+
+h2 {
+ font-size: 120%;
+}
+
+ul.tabLinks {
+ padding-left: 0;
+ padding-top: 10px;
+ padding-bottom: 10px;
+ overflow: auto;
+ min-width: 800px;
+ width: auto !important;
+ width: 800px;
+}
+
+ul.tabLinks li {
+ float: left;
+ height: 100%;
+ list-style: none;
+ padding-left: 10px;
+ padding-right: 10px;
+ padding-top: 5px;
+ padding-bottom: 5px;
+ margin-bottom: 0;
+ -moz-border-radius: 7px;
+ border-radius: 7px;
+ margin-right: 25px;
+ border: solid 1px #d4d4d4;
+ background-color: #f0f0f0;
+}
+
+ul.tabLinks li:hover {
+ background-color: #fafafa;
+}
+
+ul.tabLinks li.selected {
+ background-color: #c5f0f5;
+ border-color: #c5f0f5;
+}
+
+ul.tabLinks a {
+ font-size: 120%;
+ display: block;
+ outline: none;
+ text-decoration: none;
+ margin: 0;
+ padding: 0;
+}
+
+ul.tabLinks li h2 {
+ margin: 0;
+ padding: 0;
+}
+
+div.tab {
+}
+
+div.selected {
+ display: block;
+}
+
+div.deselected {
+ display: none;
+}
+
+div.tab table {
+ min-width: 350px;
+ width: auto !important;
+ width: 350px;
+ border-collapse: collapse;
+}
+
+div.tab th, div.tab table {
+ border-bottom: solid #d0d0d0 1px;
+}
+
+div.tab th {
+ text-align: left;
+ white-space: nowrap;
+ padding-left: 6em;
+}
+
+div.tab th:first-child {
+ padding-left: 0;
+}
+
+div.tab td {
+ white-space: nowrap;
+ padding-left: 6em;
+ padding-top: 5px;
+ padding-bottom: 5px;
+}
+
+div.tab td:first-child {
+ padding-left: 0;
+}
+
+div.tab td.numeric, div.tab th.numeric {
+ text-align: right;
+}
+
+span.code {
+ display: inline-block;
+ margin-top: 0em;
+ margin-bottom: 1em;
+}
+
+span.code pre {
+ font-size: 11pt;
+ padding-top: 10px;
+ padding-bottom: 10px;
+ padding-left: 10px;
+ padding-right: 10px;
+ margin: 0;
+ background-color: #f7f7f7;
+ border: solid 1px #d0d0d0;
+ min-width: 700px;
+ width: auto !important;
+ width: 700px;
+}
+
+span.wrapped pre {
+ word-wrap: break-word;
+ white-space: pre-wrap;
+ word-break: break-all;
+}
+
+label.hidden {
+ display: none;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/style.css
----------------------------------------------------------------------
diff --git a/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/style.css b/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/style.css
new file mode 100644
index 0000000..3186aaa
--- /dev/null
+++ b/nexus-ingest/nexus-messages/build/reports/project/dependencies/css/style.css
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.dependencyTree, #insight {
+ border: solid 2px #d0d0d0;
+ border-radius: 10px;
+ padding: 8px;
+ background-color: #FFFFEE
+}
+.alreadyRendered > a, .unresolvable > a {
+ border-radius: 4px;
+}
+.alreadyRendered > a {
+ border: solid 1px #EEEEEE;
+ background-color: #EEEEEE !IMPORTANT;
+}
+.hasConflict > a {
+ font-weight: bold !IMPORTANT;
+ color: #FAA732 !IMPORTANT;
+}
+.unresolvable > a {
+ border: solid 1px #DD514C;
+ background-color: #DD514C !IMPORTANT;
+ color: white !IMPORTANT;
+}
+.leaf a {
+ font-style: italic !IMPORTANT;
+}
+.configuration h3 {
+ cursor: pointer;
+}
+.configuration h3 ins {
+ background-image: url(../images/d.png); background-repeat:no-repeat; background-color:white;
+ background-position: -18px 0;
+ display: inline-block;
+ width: 18px;
+ height: 18px;
+ position: relative;
+ top: 4px;
+}
+.configuration h3.closed ins {
+ background-position: 0 0;
+}
+#dependencies {
+ margin-right: 30px;
+}
+#insight {
+ position: fixed;
+ right: 50px;
+ top: 50px;
+ display: none;
+ width: 70%;
+ box-shadow: 3px 3px 10px rgba(0, 0, 0, 0.3);
+}
+#dismissInsight {
+ float: right;
+ background:url("../images/d.png") -18px -53px no-repeat !important;
+ width: 20px;
+ height: 20px;
+ cursor: pointer;
+}
+.insightTree {
+ height: 450px;
+ overflow: auto;
+}
+.insightTree a {
+ cursor: default;
+}
+.configurationDescription {
+ font-size: small;
+}