You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ss...@apache.org on 2021/07/15 15:45:18 UTC
[systemds] branch master updated: [SYSTEMDS-3060] Refactor Cleaning
Pipelines - Evaluate with UDF - Pipelines evaluation with
user-defined-function. - Evaluation function is passed as a parameter and
pipelines are evaluated via eval call. - topk_cleaning built-in added -
Test are added for regression and classification
This is an automated email from the ASF dual-hosted git repository.
ssiddiqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new b08b857 [SYSTEMDS-3060] Refactor Cleaning Pipelines - Evaluate with UDF - Pipelines evaluation with user-defined-function. - Evaluation function is passed as a parameter and pipelines are evaluated via eval call. - topk_cleaning built-in added - Test are added for regression and classification
b08b857 is described below
commit b08b857384101e116e36cfe65aaf8a54d018d874
Author: Shafaq Siddiqi <sh...@tugraz.at>
AuthorDate: Thu Jul 15 16:39:17 2021 +0200
[SYSTEMDS-3060] Refactor Cleaning Pipelines - Evaluate with UDF
- Pipelines evaluation with user-defined-function.
- Evaluation function is passed as a parameter and pipelines are evaluated
via eval call.
- topk_cleaning built-in added
- Test are added for regression and classification
Closes #1347.
---
scripts/builtin/bandit.dml | 133 ++----
scripts/builtin/executePipeline.dml | 58 +--
scripts/builtin/imputeByMedian.dml | 2 +-
scripts/builtin/mice.dml | 2 +-
scripts/builtin/topk_cleaning.dml | 145 +++++++
scripts/pipelines/properties/param.csv | 4 +-
scripts/pipelines/properties/primitives.csv | 3 +-
scripts/pipelines/scripts/enumerateLogical.dml | 33 +-
scripts/pipelines/scripts/utils.dml | 399 +++++++++++++-----
.../java/org/apache/sysds/common/Builtins.java | 1 +
... => BuiltinTopkCleaningClassificationTest.java} | 45 +--
...java => BuiltinTopkCleaningRegressionTest.java} | 44 +-
.../pipelines/CleaningTestClassification.java | 116 ------
src/test/resources/datasets/pipelines/dirty.csv | 447 ---------------------
.../functions/builtin/handsOffClustering.dml | 149 +++++++
.../functions/pipelines/compareAccuracy.dml | 135 -------
.../pipelines/intermediates/hyperparams.csv | 2 -
.../functions/pipelines/intermediates/logical.csv | 1 -
.../pipelines/intermediates/pipelines.csv | 2 -
.../scripts/functions/pipelines/mainScript.dml | 371 -----------------
.../functions/pipelines/testClassification.dml | 142 -------
.../scripts/functions/pipelines/testCompare.dml | 124 ------
.../scripts/functions/pipelines/testLogical.dml | 124 ------
.../pipelines/topkcleaningClassificationTest.dml | 155 +++++++
.../pipelines/topkcleaningRegressionTest.dml | 131 ++++++
25 files changed, 992 insertions(+), 1776 deletions(-)
diff --git a/scripts/builtin/bandit.dml b/scripts/builtin/bandit.dml
index 8770cdc..0f99ff6 100644
--- a/scripts/builtin/bandit.dml
+++ b/scripts/builtin/bandit.dml
@@ -19,15 +19,14 @@
#
#-------------------------------------------------------------
-m_bandit = function(Matrix[Double] X_train, Matrix[Double] Y_train, List[Unknown] metaList, List[Unknown] targetList,
- Frame[Unknown] lp, Frame[Unknown] primitives, Frame[Unknown] param, Integer k = 3, Integer R=50,
+m_bandit = function(Matrix[Double] X_train, Matrix[Double] Y_train, List[Unknown] metaList, String evaluationFunc, Matrix[Double] evalFunHp,
+ Frame[Unknown] lp, Frame[Unknown] primitives, Frame[Unknown] param, Integer k = 3, Integer R=50, Double baseLineScore,
Boolean verbose = TRUE)
- return (Frame[Unknown] bestPipeline, Matrix[Double] bestHyperparams, Matrix[Double] bestAccuracy, Frame[String] feaFrameOuter)
+ return (Frame[Unknown] bestPipeline, Matrix[Double] bestHyperparams, Matrix[Double] bestAccuracy, Frame[String] feaFrameOuter)
{
print("Starting optimizer")
NUM_FEATURES = 14
HYPERPARAM_LENGTH = 110
- print("null in data "+sum(is.na(X_train)))
bestPipeline = frame("", rows=1, cols=1)
bestHyperparams = as.matrix(0)
bestAccuracy = as.matrix(0)
@@ -81,7 +80,7 @@ m_bandit = function(Matrix[Double] X_train, Matrix[Double] Y_train, List[Unknown
configurations = configurations[1:n_i, ]
[outPip,outHp, feaFrameOuter] = run_with_hyperparam(configurations, r_i, X_train, Y_train, metaList,
- targetList, param, feaFrameOuter, verbose)
+ evaluationFunc, evalFunHp, param, feaFrameOuter, verbose)
# sort the pipelines by order of accuracy decreasing
a = order(target = outPip, by = 1, decreasing=TRUE, index.return=FALSE)
b = order(target = outHp, by = 1, decreasing=TRUE, index.return=FALSE)
@@ -110,7 +109,7 @@ m_bandit = function(Matrix[Double] X_train, Matrix[Double] Y_train, List[Unknown
hparam[startOut:endOut, 1:ncol(bracket_bestHyperparams)] = bracket_bestHyperparams
}
- [bestPipeline, bestHyperparams] = extractTopK(pipeline, hparam, as.scalar(targetList['dirAcc']), k)
+ [bestPipeline, bestHyperparams] = extractTopK(pipeline, hparam, baseLineScore, k)
bestAccuracy = as.matrix(bestPipeline[,1])
bestPipeline = bestPipeline[,2:ncol(bestPipeline)]
@@ -120,8 +119,6 @@ m_bandit = function(Matrix[Double] X_train, Matrix[Double] Y_train, List[Unknown
print("best pipeline"+ toString(bestPipeline))
print("best hyper-parameters \n"+ toString(bestHyperparams))
print("best accuracy \n"+ toString(bestAccuracy))
- if(as.scalar(targetList['target']) != "compare")
- print("dirty accuracy "+as.scalar(targetList['dirAcc']))
}
}
@@ -199,10 +196,10 @@ get_physical_configurations = function(Frame[String] logical, Scalar[int] numCon
# this method will call the execute pipelines with their hyper-parameters
run_with_hyperparam = function(Frame[Unknown] ph_pip, Integer r_i, Matrix[Double] X, Matrix[Double] Y,
- List[Unknown] metaList, List[Unknown] targetList, Frame[Unknown] param, Frame[Unknown] featureFrameOuter,
+ List[Unknown] metaList, String evaluationFunc, Matrix[Double] evalFunHp, Frame[Unknown] param, Frame[Unknown] featureFrameOuter,
Boolean verbose)
return (Matrix[Double] output_operator, Matrix[Double] output_hyperparam, Frame[Unknown] featureFrameOuter) {
-
+ print("run_with_hyperparam started")
output_hp = matrix(0, nrow(ph_pip)*r_i, 100)
output_accuracy = matrix(0, nrow(ph_pip)*r_i, 1)
output_pipelines = matrix(0, nrow(ph_pip)*r_i, 2)
@@ -214,7 +211,7 @@ run_with_hyperparam = function(Frame[Unknown] ph_pip, Integer r_i, Matrix[Double
id = as.matrix(ph_pip[, 1])
ph_pip = ph_pip[, 2:ncol(ph_pip)]
- feaVec = gatherStats(X, Y, as.matrix(metaList['mask']), as.scalar(targetList['target']))
+ feaVec = gatherStats(X, Y, as.matrix(metaList['mask']))
for(i in 1:nrow(ph_pip))
{
@@ -232,15 +229,15 @@ run_with_hyperparam = function(Frame[Unknown] ph_pip, Integer r_i, Matrix[Double
indexes = cumsum(indexes)
indexes = table(indexes, 1, 1, nrow(hp), 1)
hp_matrix = removeEmpty(target = hp, margin="rows", select = indexes)
- [X, Y, T] = executePipeline(ph_pip[i], X, Y, as.matrix(metaList['mask']), as.matrix(metaList['fd']), hp_matrix, no_of_flag_vars, FALSE)
- if(as.scalar(targetList['target']) == "compare")
- [accuracy, accT] = compareValue(clone_X, X, as.matrix(targetList['cleanData']), as.matrix(metaList['mask']))
- else
- [accuracy, accT] = fclassify(X, Y, as.matrix(metaList['mask']), as.matrix(targetList['mlHp']), as.scalar(targetList['dirAcc']),
- as.scalar(targetList['wAccuracy']), as.scalar(targetList['cv']))
+ [X, Y, T] = executePipeline(ph_pip[i], X, Y, as.matrix(metaList['mask']), as.matrix(metaList['fd']),
+ hp_matrix, no_of_flag_vars, FALSE)
+ argList = list(X=X, Y=Y, Xorig=clone_X, metaList=metaList, evalFunHp=evalFunHp, trainML=0)
+ t1 = time()
+ evalFunOutput = eval(evaluationFunc, argList)
+ accT = floor((time() - t1) / 1e+6)
matrix_width = as.matrix(nrow(hp_matrix) * ncol(hp_matrix))
hp_vec = cbind(matrix_width, matrix(hp_matrix, rows=1, cols=nrow(hp_matrix)*ncol(hp_matrix), byrow=TRUE))
- output_accuracy[index, 1] = accuracy
+ output_accuracy[index, 1] = as.scalar(evalFunOutput[1, 1])
output_hp[index, 1:ncol(hp_vec)] = hp_vec
output_pipelines[index, ] = cbind(as.matrix(index), id[i,1])
X = clone_X
@@ -250,7 +247,7 @@ run_with_hyperparam = function(Frame[Unknown] ph_pip, Integer r_i, Matrix[Double
if(ncol(featureFrameOuter) > 1) {
feaFrame[r, 1:ncol(feaVec)] = as.frame(feaVec)
feaFrame[r, (ncol(feaVec)+1)] = pip_toString
- feaFrame[r, (ncol(feaVec)+2)] = accuracy
+ feaFrame[r, (ncol(feaVec)+2)] = as.scalar(evalFunOutput[1, 1])
feaFrame[r, (ncol(feaVec)+3)] = T
feaFrame[r, (ncol(feaVec)+4)] = accT
}
@@ -322,12 +319,12 @@ getHyperparam = function(Frame[Unknown] pipeline, Frame[Unknown] hpList, Intege
minVal = as.scalar(hpList[index, paramValIndex])
maxVal = as.scalar(hpList[index, paramValIndex + 1])
if(type == "FP") {
- val = rand(rows=no_of_res, cols=1, min=minVal,max=maxVal, pdf="uniform");
+ val = rand(rows=no_of_res, cols=1, min=minVal, max=maxVal, pdf="uniform");
OpParam[, j] = val;
}
else if(type == "INT") {
- val = sample(maxVal, no_of_res, TRUE);
- less_than_min = val < minVal;
+ val = sample(as.integer(maxVal), no_of_res, TRUE);
+ less_than_min = val < as.integer(minVal);
val = (less_than_min * minVal) + val;
OpParam[, j] = val;
}
@@ -450,95 +447,12 @@ return (Frame[Unknown] maxperconf)
maxperconf[1:ncol(tab),] = as.frame(t(colMaxs(tab)))
}
-###########################################################################
-## function to classify the data using cross validation
-############################################################################
-fclassify = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] mask, Matrix[Double] MLhp,
- Double testAccuracy, Boolean isWeighted, Integer cv=3)
- return (Double accuracy, Double T)
-{
-
- if(max(Y) == min(Y)) {
- print("Y contains only one class")
- accuracy = as.double(0)
- }
- else {
- print("STARTING "+cv+" CROSS VALIDATIONS")
- # do the k = 3 cross validations
- [accuracyMatrix, T] = crossV(X, Y, cv, mask, MLhp, isWeighted)
- accuracyMatrix = removeEmpty(target=accuracyMatrix, margin="rows")
- acc = colMeans(accuracyMatrix)
- accuracy = as.scalar(acc[1,1])
- print(cv +" validation accuracy "+accuracy+" in "+T+" ms\n\n")
- }
-}
-
-# # ######################################################################
-# # # # Function for cross validation using hold out method
-# # # # Inputs: The input dataset X, Y and the value of k validation, mask of the
-# # # # dataset for OHE of categorical columns, vector of ML hyper-parameters identified
-# # # # via gridsearch and a boolean value of (un)weighted accuracy.
-# # # # Output: It return a matrix having the accuracy of each fold.
-# # ######################################################################
-
-crossV = function(Matrix[double] X, Matrix[double] y, Integer k, Matrix[Double] mask,
- Matrix[Double] MLhp, Boolean isWeighted)
-return (Matrix[Double] accuracyMatrix, Double T)
-{
- t1 = time()
- accuracyMatrix = matrix(0, k, 1)
- dataList = list()
- testL = list()
- data = order(target = cbind(y, X), by = 1, decreasing=FALSE, index.return=FALSE)
- classes = table(data[, 1], 1)
- ins_per_fold = classes/k
- start_fold = matrix(1, rows=nrow(ins_per_fold), cols=1)
- fold_idxes = cbind(start_fold, ins_per_fold)
-
- start_i = 0; end_i = 0; idx_fold = 1;;
- for(i in 1:k)
- {
- fold_i = matrix(0, 0, ncol(data))
- start=0; end=0;
- for(j in 1:nrow(classes))
- {
- idx = as.scalar(classes[j, 1])
- start = end + 1;
- end = end + idx
- class_j = data[start:end, ]
- start_i = as.scalar(fold_idxes[j, 1]);
- end_i = as.scalar(fold_idxes[j, 2])
- fold_i = rbind(fold_i, class_j[start_i:end_i, ])
- }
- dataList = append(dataList, fold_i)
- fold_idxes[, 1] = fold_idxes[, 2] + 1
- fold_idxes[, 2] += ins_per_fold
- }
-
- parfor(i in seq(1,k))
- {
- [trainList, hold_out] = remove(dataList, i)
- trainset = rbind(trainList)
- testset = as.matrix(hold_out)
- trainX = trainset[, 2:ncol(trainset)]
- trainy = trainset[, 1]
- testX = testset[, 2:ncol(testset)]
- testy = testset[, 1]
- beta = multiLogReg(X=trainX, Y=trainy, icpt=as.scalar(MLhp[1,1]), reg=as.scalar(MLhp[1,2]), tol= 1e-9,
- maxi=as.scalar(MLhp[1,3]), maxii= 50, verbose=FALSE);
- [prob, yhat, a] = multiLogRegPredict(testX, beta, testy, FALSE)
- accuracy = getAccuracy(testy, yhat, isWeighted)
- accuracyMatrix[i] = accuracy
- }
- T = floor((time() - t1) / 1e+6)
-}
-
###############################################################################################
# The function will collect the features like statistics and pipelines and accuracy
# so that they could be used for training a model and predicting pipelines without enumeration
###############################################################################################
-gatherStats = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] mask, String target)
+gatherStats = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] mask)
return (Matrix[Double] features)
{
@@ -562,7 +476,8 @@ return (Matrix[Double] features)
# OHE features
OHE = sum(colMaxs(X) * mask)
features[1, 10] = OHE
- if(target != "compare")
+ distVal = countDistinct(Y)
+ if(nrow(Y) > 1 & distVal <= 10)
{
ctab = table(Y, 1)
features[1, 11] = nrow(ctab) # number of classes
@@ -623,6 +538,6 @@ return (String s)
{
s = ""
for(i in 1:ncol(F))
- s = s + as.scalar(F[,i])+";"
+ s = s + as.scalar(F[1,i])+";"
-}
\ No newline at end of file
+}
diff --git a/scripts/builtin/executePipeline.dml b/scripts/builtin/executePipeline.dml
index 1e1a656..ebc51d9 100644
--- a/scripts/builtin/executePipeline.dml
+++ b/scripts/builtin/executePipeline.dml
@@ -30,7 +30,6 @@ s_executePipeline = function(Frame[String] pipeline, Matrix[Double] X, Matrix[D
print("checks rows in X = "+nrow(X)+" rows in Y = "+nrow(Y)+" cols in X = "+ncol(X)+" col in Y = "+ncol(Y))
print("pipeline in execution "+toString(pipeline))
print("pipeline hps "+toString(hyperParameters))
- while(FALSE){}
}
for(i in 1:ncol(pipeline)) {
op = as.scalar(pipeline[1,i])
@@ -106,11 +105,6 @@ matrixToList = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] mask
# get the number of hyper-parameters and loop till that
no_of_hyperparam = as.scalar(p[1,1])
if(no_of_hyperparam > 0) {
- # if(op == "pca") {
- # # convert the number parameters to a ration related to OHE columns
- # ratio = as.scalar(p[resource_index,1])
- # p[resource_index, 1] = as.integer(ncol(X) - ratio)
- # }
for(i in 1:no_of_hyperparam)
l = append(l, as.scalar(p[1,(i+1)]))
}
@@ -205,14 +199,17 @@ return (Matrix[Double] X)
dummycoding = function(Matrix[Double] X, Matrix[Double] mask)
return (Matrix[Double] dX_train) {
- X = replace(target=X, pattern=NaN, replacement=0)
- idx = vectorToCsv(mask)
-
- # specifications for one-hot encoding of categorical features
- jspecDC = "{ids:true, dummycode:["+idx+"]}";
- # OHE of categorical features
- [dX_train, dM] = transformencode(target=as.frame(X), spec=jspecDC);
+ if(sum(mask) > 0)
+ {
+ X = replace(target=X, pattern=NaN, replacement=1)
+ idx = vectorToCsv(mask)
+ # specifications for one-hot encoding of categorical features
+ jspecDC = "{ids:true, dummycode:["+idx+"]}";
+ # OHE of categorical features
+ [dX_train, dM] = transformencode(target=as.frame(X), spec=jspecDC);
+ }
+ else dX_train = X
}
@@ -306,7 +303,8 @@ return(Matrix[Double] X){
m_pca = function(Matrix[Double] X, Integer K=2, Boolean center=TRUE, Boolean scale=TRUE)
return (Matrix[Double] Xout)
{
- if(K < ncol(X)) {
+
+ if(K < ncol(X) - 1) {
N = nrow(X);
D = ncol(X);
@@ -316,23 +314,25 @@ m_pca = function(Matrix[Double] X, Integer K=2, Boolean center=TRUE, Boolean sca
# co-variance matrix
mu = colSums(X)/N;
C = (t(X) %*% X)/(N-1) - (N/(N-1))*t(mu) %*% mu;
-
# compute eigen vectors and values
[evalues, evectors] = eigen(C);
-
- decreasing_Idx = order(target=evalues,by=1,decreasing=TRUE,index.return=TRUE);
- diagmat = table(seq(1,D),decreasing_Idx);
- # sorts eigenvalues by decreasing order
- evalues = diagmat %*% evalues;
- # sorts eigenvectors column-wise in the order of decreasing eigenvalues
- evectors = evectors %*% diagmat;
-
- eval_dominant = evalues[1:K, 1];
- evec_dominant = evectors[,1:K];
-
- # Construct new data set by treating computed dominant eigenvectors as the basis vectors
- Xout = X %*% evec_dominant;
- Mout = evec_dominant;
+ if(nrow(evalues) > 1 & nrow(evectors) > 1)
+ {
+ decreasing_Idx = order(target=evalues,by=1,decreasing=TRUE,index.return=TRUE);
+ diagmat = table(seq(1,D),decreasing_Idx);
+ # sorts eigenvalues by decreasing order
+ evalues = diagmat %*% evalues;
+ # sorts eigenvectors column-wise in the order of decreasing eigenvalues
+ evectors = evectors %*% diagmat;
+
+ eval_dominant = evalues[1:K, 1];
+ evec_dominant = evectors[,1:K];
+
+ # Construct new data set by treating computed dominant eigenvectors as the basis vectors
+ Xout = X %*% evec_dominant;
+ Mout = evec_dominant;
+ }
+ else Xout = X # these elses could be removed via initiating Xout = X for now they are here for readability
}
else Xout = X
diff --git a/scripts/builtin/imputeByMedian.dml b/scripts/builtin/imputeByMedian.dml
index c6716c6..bf80923 100644
--- a/scripts/builtin/imputeByMedian.dml
+++ b/scripts/builtin/imputeByMedian.dml
@@ -51,7 +51,7 @@ return(Matrix[Double] X)
cols = ncol(nX)
# median imputation
colMedian = matrix(0, 1, cols)
- for(i in 1:cols, check=0)
+ parfor(i in 1:cols, check=0)
colMedian[1, i] = median(X[,i])
X_n = nX + (Mask_n * colMedian)
# mode imputation
diff --git a/scripts/builtin/mice.dml b/scripts/builtin/mice.dml
index a5243c0..f5a373f 100644
--- a/scripts/builtin/mice.dml
+++ b/scripts/builtin/mice.dml
@@ -140,7 +140,7 @@ m_mice= function(Matrix[Double] X, Matrix[Double] cMask, Integer iter = 3,
# learn a regression line
beta = lm(X=train_X, y=train_Y, verbose=FALSE, icpt=1, reg = 1e-7, tol = 1e-7);
# predicting missing values
- pred = lmPredict(X=test_X, B=beta, ytest= matrix(0,1,1), icpt=1)
+ pred = lmPredict(X=test_X, B=beta, ytest= matrix(0,1,1), icpt=1, verbose = FALSE)
# imputing missing column values (assumes Mask_Filled being 0/1-matrix)
R = removeEmpty(target=Mask_Filled[, in_c] * seq(1,n), margin="rows");
# TODO modify removeEmpty to return zero row and n columns
diff --git a/scripts/builtin/topk_cleaning.dml b/scripts/builtin/topk_cleaning.dml
new file mode 100644
index 0000000..2ff7c8d
--- /dev/null
+++ b/scripts/builtin/topk_cleaning.dml
@@ -0,0 +1,145 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+# metaData[3, ncol(X)] : metaData[1] stores mask, metaData[2] stores schema, metaData[3] stores FD mask
+source("scripts/pipelines/scripts/utils.dml") as utils;
+source("scripts/pipelines/scripts/enumerateLogical.dml") as lg;
+
+
+s_topk_cleaning = function(Frame[Unknown] data, Frame[Unknown] metaData = as.frame("NULL"), Frame[Unknown] primitives, Frame[Unknown] parameters,
+ Matrix[Double] cmr = matrix("4 0.7 1", rows=1, cols=3), String evaluationFunc, Matrix[Double] evalFunHp, Integer topK = 5,
+ Integer resource_val = 20, Double sample = 0.1, Boolean isLastLabel = TRUE)
+ return (Frame[Unknown] topKPipelines, Matrix[Double] topKHyperParams, Matrix[Double] topKScores, Frame[Unknown] bestLogical, Frame[Unknown] features, Double dirtyScore)
+{
+ print("starting topk_cleaning")
+ dirtyScore = 100
+ if(as.scalar(metaData[1, 1]) == "NULL")
+ {
+ print("creating meta data")
+ r1 = detectSchema(data)
+ r2 = matrix(0, rows=1, cols=ncol(data))
+ for(i in 1 : ncol(r1))
+ {
+ if(as.scalar(r1[1, i]) == "STRING" | as.scalar(r1[1, i]) == "BOOLEAN")
+ r2[1, i] = 1
+ }
+ schema = r1[, 1:ncol(r1) - 1]
+ mask = r2[, 1:ncol(r2) - 1]
+ fdMask = r2[, 1:ncol(r2) - 1]
+ maskY = as.scalar(r2[,ncol(r2)])
+ }
+ else {
+ schema = metaData[1, 1:ncol(metaData) - 1]
+ mask = as.matrix(metaData[2, 1:ncol(metaData) - 1])
+ fdMask = as.matrix(metaData[3, 1:ncol(metaData) - 1])
+ maskY = as.scalar(metaData[2, ncol(metaData)])
+ }
+ # # keeping the meta list format if we decide to add more stuff in metadata
+ metaList = list(mask=mask, schema=schema, fd=fdMask)
+
+ # separate the label
+ if(isLastLabel ) {
+ X = data[, 1:ncol(data) - 1]
+ label = data[, ncol(data)]
+ # always recode the label
+ if(maskY == 1)
+ [Y, M] = transformencode(target=label, spec= "{ids:true, recode:[1]}");
+ else Y = as.matrix(label)
+ }
+ else
+ {
+ X = data
+ Y = as.matrix(0)
+ }
+
+
+ # # do the string processing
+ X_dirty = X
+ X = utils::stringProcessing(data=X, mask=mask, schema=schema, CorrectTypos=FALSE)
+
+ # # if mask has 1s then there are categorical features
+ if(sum(mask) > 0)
+ {
+ index = vectorToCsv(mask)
+ jspecR = "{ids:true, recode:["+index+"]}"
+ [eX, X_meta] = transformencode(target=X, spec=jspecR);
+ [eX_dirty, X_meta_dirty] = transformencode(target=X_dirty, spec=jspecR);
+
+ }
+ # if no categorical value exist then just cast the frame into matrix
+ else {
+ eX = as.matrix(X)
+ eX_dirty = as.matrix(X_dirty)
+ }
+ # take the sample
+ [eX, Y] = utils::doSample(eX, Y, sample, TRUE)
+ [eX_dirty, Y] = utils::doSample(eX_dirty, Y, sample, FALSE)
+ eX_dirty = utils::dummycoding(eX_dirty, mask)
+ # get the dirty score
+ scoreAndHp = eval(evaluationFunc, list(X=eX_dirty, Y=Y, Xorig=as.matrix(0), metaList=metaList, evalFunHp=evalFunHp, trainML=1))
+ dirtyScore = as.scalar(scoreAndHp[1, 1])
+ evalFunHp = scoreAndHp[1, 2:ncol(scoreAndHp)]
+
+ logicalSeedCI = frame([
+ "4", "MVI", "OTLR", "CI", "SCALE", "0",
+ "1", "MVI", "0", "0", "0", "0",
+ "1", "OTLR", "0", "0", "0", "0",
+ "1", "CI", "0", "0", "0", "0",
+ "2", "MVI", "CI", "0", "0", "0",
+ "2", "MVI", "OTLR", "0", "0", "0",
+ "2", "MVI", "SCALE", "0", "0", "0",
+ "3", "MVI", "SCALE", "OTLR", "0", "0",
+ "4", "OTLR", "MVI", "CI", "SCALE", "0",
+ "5", "MVI", "OTLR", "MVI", "CI", "SCALE"
+ ], rows=10, cols=6)
+
+ logicalSeedNoCI = frame([
+ "3", "MVI", "OTLR", "SCALE", "0",
+ "1", "MVI", "0", "0", "0",
+ "1", "OTLR", "0", "0", "0",
+ "2", "MVI", "OTLR", "0", "0",
+ "2", "MVI", "SCALE", "0", "0",
+ "3", "MVI", "SCALE", "OTLR", "0",
+ "3", "OTLR", "MVI", "SCALE", "0",
+ "4", "MVI", "OTLR", "MVI", "SCALE"
+ ], rows=8, cols=5)
+
+
+ if((nrow(Y) > 0 & countDistinct(Y) < 10))
+ logical = logicalSeedCI
+ else
+ logical = logicalSeedNoCI
+
+ # category = frame(["MVI", "OTLR"], rows=1, cols=2)
+ idx = as.integer(as.scalar(logical[1, 1])) + 1
+
+ category = logical[1, 2:idx]
+
+ [bestLogical, score, T] = lg::enumerateLogical(X=eX, y=Y, cmr=cmr, cat=category, population=logical,
+ max_iter=ceil(resource_val/topK), metaList = metaList, evaluationFunc=evaluationFunc, evalFunHp=evalFunHp,
+ primitives=primitives, param=parameters, num_inst = nrow(primitives), num_exec = topK, isTailed=TRUE, verbose=TRUE)
+
+ topKPipelines = as.frame("NULL"); topKHyperParams = matrix(0,0,0); topKScores = matrix(0,0,0); features = as.frame("NULL")
+
+ [topKPipelines, topKHyperParams, topKScores, features] = bandit(X_train=eX, Y_train=Y, metaList=metaList,
+ evaluationFunc=evaluationFunc, evalFunHp=evalFunHp, lp=bestLogical, primitives=primitives, param=parameters, baseLineScore=dirtyScore,
+ k=topK, R=resource_val, verbose=FALSE);
+}
+
diff --git a/scripts/pipelines/properties/param.csv b/scripts/pipelines/properties/param.csv
index ecd239c..254a0d3 100644
--- a/scripts/pipelines/properties/param.csv
+++ b/scripts/pipelines/properties/param.csv
@@ -1,6 +1,6 @@
name,param_no,maskFlag,FDFlag,yFlag,verboseFlag,dataFlag,dataType,ranges,st1,en1,st2,en1,en2,st3,en3
outlierByIQR,3,0,0,0,1,0,FP,INT,INT,1,5,1,2,1,10
-outlierBySd,3,0,0,0,1,0,FP,INT,INT,1,5,2,2,1,10
+outlierBySd,3,0,0,0,1,0,FP,INT,INT,1,5,1,2,1,10
winsorize,0,0,0,0,1,0,,,,,,,,,
imputeByMean,0,1,0,0,0,2,,,,,,,,,
imputeByMedian,0,1,0,0,0,2,,,,,,,,,
@@ -12,5 +12,5 @@ m_pca,3,0,0,0,0,2,INT,BOOL,BOOL,100,200,0,1,0,0
fillDefault,0,0,0,0,0,2,,,,,,,,,
dummycoding,0,1,0,0,0,2,,,,,,,,,
scale,2,0,0,0,0,0,BOOL,BOOL,0,1,0,1,,,
-forward_fill,1,0,0,0,1,0,BOOL,0,1,,,,,,
+forward_fill,1,0,0,0,1,2,BOOL,0,1,,,,,,
imputeByFd,1,0,1,0,0,2,FP,0.55,1,,,,,,
\ No newline at end of file
diff --git a/scripts/pipelines/properties/primitives.csv b/scripts/pipelines/properties/primitives.csv
index 3bf1bdb..da2bec8 100644
--- a/scripts/pipelines/properties/primitives.csv
+++ b/scripts/pipelines/properties/primitives.csv
@@ -3,4 +3,5 @@ winsorize,imputeByMean,abstain,SMOTE,m_pca,dummycoding,scale
outlierBySd,imputeByMedian,,,,,
outlierByIQR,mice,,,,,
,fillDefault,,,,,
-,imputeByFd,,,,,
\ No newline at end of file
+,imputeByFd,,,,,
+,forward_fill,,,,,
\ No newline at end of file
diff --git a/scripts/pipelines/scripts/enumerateLogical.dml b/scripts/pipelines/scripts/enumerateLogical.dml
index 090d48a..ca14ff7 100644
--- a/scripts/pipelines/scripts/enumerateLogical.dml
+++ b/scripts/pipelines/scripts/enumerateLogical.dml
@@ -48,11 +48,10 @@
# Repeat the process max_iter time. Converge in between if the best_score of previous generation is better then
# best_score of new generation.
-source("scripts/builtin/bandit.dml") as bandit;
-source("scripts/pipelines/scripts/utils.dml") as utils;
-enumerateLogical = function(Matrix[Double] X, Matrix[Double] y, Matrix[Double] cmr, Matrix[Double] cat, Frame[Unknown] population,
- Integer max_iter=10, List[Unknown] metaList, List[Unknown] targetList, Frame[Unknown] primitives, Frame[Unknown] param,
+source("scripts/builtin/bandit.dml") as bandit;
+enumerateLogical = function(Matrix[Double] X, Matrix[Double] y, Matrix[Double] cmr, Frame[Unknown] cat, Frame[Unknown] population,
+ Integer max_iter=10, List[Unknown] metaList, String evaluationFunc, Matrix[Double] evalFunHp, Frame[Unknown] primitives, Frame[Unknown] param,
Integer num_inst, Integer num_exec, Boolean isTailed = TRUE, Boolean verbose)
return (Frame[Unknown] bestLg, Double pre_best, Double T)
{
@@ -63,6 +62,7 @@ return (Frame[Unknown] bestLg, Double pre_best, Double T)
feaFrameOuter = as.frame("NULL")
iter = 1
convergedOuter = FALSE
+
while(iter <= max_iter & !convergedOuter)
{
physicalPipList = list()
@@ -90,7 +90,7 @@ return (Frame[Unknown] bestLg, Double pre_best, Double T)
}
# # execute the physical instances and store the minimum scores, each pipeline is executed num_exec times
[outPip,outHp, feaFrameOuter] = bandit::run_with_hyperparam(physicalConf, num_exec, X, y, metaList,
- targetList, param, as.frame(""), verbose)
+ evaluationFunc, evalFunHp, param, as.frame(""), verbose)
# # sort the configurations groupwise
max_perf = bandit::getMaxPerConf(outPip, nrow(physicalConf))
scores[i] = as.matrix(max_perf[1, 1])
@@ -100,15 +100,9 @@ return (Frame[Unknown] bestLg, Double pre_best, Double T)
selected = order(target = scores, by = 1, decreasing=TRUE, index.return=TRUE)
idxR = as.scalar(selected[1, 1])
best_score = as.scalar(scores[idxR])
- if(verbose)
- {
- print("best score " + best_score)
- print("previous score " + pre_best)
- print("bestLg " + toString(bestLg))
- }
- converged = pre_best > best_score
+ converged = pre_best > best_score
convergedOuter = converged
- if(converged)
+ if(converged & (iter > 1))
{
if(isTailed)
{
@@ -135,7 +129,6 @@ return (Frame[Unknown] bestLg, Double pre_best, Double T)
length_top = as.integer(as.scalar(top[1, 1]))
top = top[, 2:(length_top+1)]
- print("top 1 "+toString(top))
# generate children from crossover
c1 = addition(top, cat, as.scalar(cmr[1,1]))
@@ -156,6 +149,15 @@ return (Frame[Unknown] bestLg, Double pre_best, Double T)
population = children
iter = iter + 1
}
+ if(pre_best == best_score) {
+ print("LogicalENumerator: did not converge after "+max_iter+" iterations")
+ if(isTailed)
+ {
+ lg_tail = frame(["DUMMY", "DIM"], rows=1, cols=2)
+ bestLg = cbind(bestLg, lg_tail)
+ }
+
+ }
T = floor((time() - t1) / 1e+6)
print("time "+T+" ms")
@@ -190,14 +192,12 @@ return (Frame [Unknown] mChild)
random = as.scalar(rand(rows=1, cols=1))
if(random > mutationRate & ncol(child) >= 2)
{
- print("before mutation "+toString(child))
r = sample(ncol(child), 2)
r1 = as.scalar(r[1,1])
r2 = as.scalar(r[2,1])
temp = child[1, r1]
child[1, r1] = child[1, r2]
child[1, r2] = temp
- print("after mutation "+toString(child))
}
mChild = child
}
@@ -223,4 +223,3 @@ return (Frame[Unknown] output)
output = child
print("Starting removal on "+toString(output))
}
-
diff --git a/scripts/pipelines/scripts/utils.dml b/scripts/pipelines/scripts/utils.dml
index df84a1c..38c3b53 100644
--- a/scripts/pipelines/scripts/utils.dml
+++ b/scripts/pipelines/scripts/utils.dml
@@ -55,40 +55,62 @@ return (Frame[Unknown] frameblock)
# Inputs: The input dataset X, Y and sampling ratio between 0 and 1
# Output: sample X and Y
#######################################################################
-doSample = function(Matrix[Double] eX, Matrix[Double] eY, Double ratio)
- return (Matrix[Double] eX, Matrix[Double] eY)
+doSample = function(Matrix[Double] eX, Matrix[Double] eY, Double ratio, Boolean verbose = FALSE)
+ return (Matrix[Double] sampledX, Matrix[Double] sampledY)
{
- MIN_SAMPLE = 10000
+ MIN_SAMPLE = 1000
sampled = floor(nrow(eX) * ratio)
sample = ifelse(sampled > MIN_SAMPLE, TRUE, FALSE)
+ dist = table(eY, 1)
+ dist = nrow(dist)
if(sample)
{
- XY = order(target = cbind(eY, eX), by = 1, decreasing=FALSE, index.return=FALSE)
- # get the class count
- classes = table(eY, 1)
- print("classes")
- print(toString(classes))
- while(FALSE){}
- start_class = 1
- out_s = 1
- out_e = 0
- end_class = 0
- out = matrix(0, sampled, ncol(XY))
- classes_ratio = floor(classes*ratio)
- for(i in 1:nrow(classes))
+ if((nrow(eY) > 1) & (dist < 10)) # for classification
{
- end_class = end_class + as.scalar(classes[i])
- class_t = XY[start_class:end_class, ]
- out_e = out_e + as.scalar(classes_ratio[i])
- out[out_s:out_e, ] = class_t[1:as.scalar(classes_ratio[i]), ]
- out_s = out_e + 1
- start_class = end_class + 1
+ XY = order(target = cbind(eY, eX), by = 1, decreasing=FALSE, index.return=FALSE)
+ # get the class count
+ classes = table(eY, 1)
+ start_class = 1
+ out_s = 1
+ out_e = 0
+ end_class = 0
+ out = matrix(0, sampled, ncol(XY))
+ classes_ratio = floor(classes*ratio)
+ for(i in 1:nrow(classes))
+ {
+ end_class = end_class + as.scalar(classes[i])
+ class_t = XY[start_class:end_class, ]
+ out_e = out_e + as.scalar(classes_ratio[i])
+ out[out_s:out_e, ] = class_t[1:as.scalar(classes_ratio[i]), ]
+ out_s = out_e + 1
+ start_class = end_class + 1
+ }
+ out = removeEmpty(target = out, margin = "rows")
+ sampledY = out[, 1]
+ sampledX = out[, 2:ncol(out)]
}
- out = removeEmpty(target = out, margin = "rows")
- eY = out[, 1]
- eX = out[, 2:ncol(out)]
+ else if(nrow(eY) > 1 & (dist > 10)) # regression
+ {
+ sampledX = eX[1:sampled, ]
+ sampledY = eY[1:sampled, ]
+ }
+ else if(nrow(eY) == 1)
+ {
+ sampledX = eX[1:sampled, ]
+ sampledY = eY
+ }
+ else {
+ sampledX = eX
+ sampledY = eY
+ }
+ }
+ else
+ {
+ sampledX = eX
+ sampledY = eY
}
- print("AFTER SAMPLES "+nrow(eX))
+ if(verbose)
+ print("AFTER SAMPLING: "+nrow(eX))
}
# #######################################################################
@@ -100,56 +122,22 @@ doSample = function(Matrix[Double] eX, Matrix[Double] eY, Double ratio)
dummycoding = function(Matrix[Double] X, Matrix[Double] mask)
return (Matrix[Double] dX_train) {
- idx = vectorToCsv(mask)
-
- # specifications for one-hot encoding of categorical features
- jspecDC = "{ids:true, dummycode:["+idx+"]}";
- # OHE of categorical features
- [dX_train, dM] = transformencode(target=as.frame(X), spec=jspecDC);
-
-}
-
-
-####################################################################
-# Function for classifying the dirty dataset, makes a call to crossV()
-# Inputs: takes the input dataset X, Y and the value of k validation, mask of the
-# dataset for OHE of categorical columns, vector of ML hyper-parameters identified
-# via gridsearch and a boolean value of (un)weighted accuracy.
-# Output: It return a matrix having the accuracy of each fold.
-####################################################################
-classifyDirty = function(Matrix[Double] Xtrain, Matrix[Double] ytrain, Matrix[Double] opt,
- Matrix[Double] mask, Boolean isWeighted = TRUE, Integer cv)
- return (Double accuracy)
-{
if(sum(mask) > 0)
- Xtrain = dummycoding(Xtrain, mask)
- [accuracy, T] = bandit::crossV(Xtrain, ytrain, cv, mask, opt, isWeighted)
- accuracy = mean(accuracy)
- print("cross validated dirty accuracy "+accuracy)
-}
-
-# constraints over hyper parameters
-verifyHp = function(Integer index, Frame[Unknown] pip, Double minVal, Double maxVal, Integer paraNo)
-return (Double minVal, Double maxVal) {
- op = as.scalar(pip[1,index])
- # 1. if next op is pca then current op should not leave NaNs in data
- # 2. if next op is mice then current op should not replace NaNs with zeros
-
- if((op == "outlierBySd" | op == "outlierByIQR") & index < ncol(pip) & paraNo == 2)
{
- nextOp = as.scalar(pip[1, index + 1])
- if(nextOp == "pca" | nextOp == "abstain" | nextOp == "SMOTE")
- {
- maxVal = 1.0
- }
- if(nextOp == "mice")
- {
- minVal = 2.0
- }
+ X = replace(target=X, pattern=NaN, replacement=1)
+ idx = vectorToCsv(mask)
+ # specifications for one-hot encoding of categorical features
+ jspecDC = "{ids:true, dummycode:["+idx+"]}";
+ # OHE of categorical features
+ [dX_train, dM] = transformencode(target=as.frame(X), spec=jspecDC);
}
+ else dX_train = X
}
+
+
+
#####################################
# The function will check if the pipeline have zero hyper-parameters
# then it should not use more resource iterations and should be executed once
@@ -169,15 +157,71 @@ return(Boolean validForResources)
validForResources = count > 0
}
-#####################################
-# Create a pipeline for string processing that needs to be applied before data recoding
-# The pipelines will drop invalid types, transform cases, deduplicate and remove pattern outliers
-######################################
-stringProcessing = function(Frame[Unknown] data, Matrix[Double] mask, Frame[String] schema)
+
+
+
+
+# # ######################################################################
+# # # # Function for cross validation using hold out method
+# # # # Inputs: The input dataset X, Y and the value of k validation, mask of the
+# # # # dataset for OHE of categorical columns, vector of ML hyper-parameters identified
+# # # # via gridsearch and a boolean value of (un)weighted accuracy.
+# # # # Output: It return a matrix having the accuracy of each fold.
+# # ######################################################################
+
+crossVML = function(Matrix[double] X, Matrix[double] y, Integer k, Matrix[Double] MLhp)
+return (Matrix[Double] accuracyMatrix)
+{
+ accuracyMatrix = matrix(0, k, 1)
+ dataList = list()
+ testL = list()
+ data = order(target = cbind(y, X), by = 1, decreasing=FALSE, index.return=FALSE)
+ classes = table(data[, 1], 1)
+ ins_per_fold = classes/k
+ start_fold = matrix(1, rows=nrow(ins_per_fold), cols=1)
+ fold_idxes = cbind(start_fold, ins_per_fold)
+
+ start_i = 0; end_i = 0; idx_fold = 1;;
+ for(i in 1:k)
+ {
+ fold_i = matrix(0, 0, ncol(data))
+ start=0; end=0;
+ for(j in 1:nrow(classes))
+ {
+ idx = as.scalar(classes[j, 1])
+ start = end + 1;
+ end = end + idx
+ class_j = data[start:end, ]
+ start_i = as.scalar(fold_idxes[j, 1]);
+ end_i = as.scalar(fold_idxes[j, 2])
+ fold_i = rbind(fold_i, class_j[start_i:end_i, ])
+ }
+ dataList = append(dataList, fold_i)
+ fold_idxes[, 1] = fold_idxes[, 2] + 1
+ fold_idxes[, 2] += ins_per_fold
+ }
+
+ for(i in seq(1,k))
+ {
+ [trainList, hold_out] = remove(dataList, i)
+ trainset = rbind(trainList)
+ testset = as.matrix(hold_out)
+ trainX = trainset[, 2:ncol(trainset)]
+ trainy = trainset[, 1]
+ testX = testset[, 2:ncol(testset)]
+ testy = testset[, 1]
+ beta = multiLogReg(X=trainX, Y=trainy, icpt=as.scalar(MLhp[1,1]), reg=as.scalar(MLhp[1,2]), tol=as.scalar(MLhp[1,3]),
+ maxi=as.scalar(MLhp[1,4]), maxii=50, verbose=FALSE);
+ [prob, yhat, accuracy] = multiLogRegPredict(testX, beta, testy, FALSE)
+ accuracyMatrix[i] = accuracy
+ }
+
+}
+
+stringProcessing = function(Frame[Unknown] data, Matrix[Double] mask, Frame[String] schema, Boolean CorrectTypos)
return(Frame[Unknown] processedData)
{
- n = nrow(data)
-
+
# step 1 drop invalid types
data = dropInvalidType(data, schema)
@@ -189,46 +233,191 @@ return(Frame[Unknown] processedData)
lowerCase = map(data[, i], "x -> x.toLowerCase()")
data[, i] = lowerCase
}
+
+ }
+
+ if(CorrectTypos)
+ {
+ # recode data to get null mask
+ if(sum(mask) > 0)
+ {
+ # always recode the label
+ index = vectorToCsv(mask)
+ jspecR = "{ids:true, recode:["+index+"]}"
+ [eX, X_meta] = transformencode(target=data, spec=jspecR);
+ }
+ # if no categorical value exist then just cast the frame into matrix
+ else
+ eX = as.matrix(data)
+ nullMask = is.na(eX)
+ print("starting correctTypos ")
+ # fix the typos
+ for(i in 1:ncol(schema))
+ {
+ if(as.scalar(schema[1,i]) == "STRING")
+ data[, i] = correctTypos(data[, i], nullMask[, i], 0.2, 0.9, FALSE, TRUE, FALSE);
+ }
+ # print("after correctTypos "+toString(data, rows=5))
}
+
# TODO add deduplication
processedData = data
-
}
-getOpByTarget = function(Matrix[Double] X, Matrix[Double] y, Matrix[Double] mask, String target)
-return (Matrix[Double] bestOpt)
+
+
+
+topk_gridSearch = function(Matrix[Double] X, Matrix[Double] y, String train, String predict,
+ Integer numB=ncol(X), List[String] params, List[Unknown] paramValues,
+ List[Unknown] trainArgs = list(), List[Unknown] predictArgs = list(),
+ Boolean cv = FALSE, Integer cvk = 5, Boolean verbose = TRUE)
+ return (Matrix[Double] B, Frame[Unknown] opt)
{
- opt = as.frame(0)
- if(target == "multiLogReg")
- {
- params = list("icpt", "reg", "maxii");
- paramRanges = list(10^seq(0,-4), 10^seq(1,-6), 10^seq(1,3));
+ # Step 0) handling default arguments, which require access to passed data
+ if( length(trainArgs) == 0 )
+ trainArgs = list(X=X, y=y, icpt=0, reg=-1, tol=-1, maxi=-1, verbose=FALSE);
+ if( length(predictArgs) == 0 )
+ predictArgs = list(X, y);
+ if( cv & cvk <= 1 ) {
+ print("gridSearch: called with cv=TRUE but cvk="+cvk+", set to default cvk=5.")
+ cvk = 5;
+ }
- if(sum(mask) > 0)
- X = dummycoding(replace(target = X, pattern = NaN, replacement=0), mask)
-
- trainArgs = list(X=X, Y=y, icpt=-1, reg=-1, tol=1e-9, maxi=100, maxii=-1, verbose=FALSE);
- [B1,opt] = gridSearch(X=X, y=y, train="multiLogReg", predict="accuracy", numB=ncol(X)+1,
- params=params, paramValues=paramRanges, trainArgs=trainArgs, verbose=FALSE);
+ # Step 1) preparation of parameters, lengths, and values in convenient form
+ numParams = length(params);
+ paramLens = matrix(0, numParams, 1);
+ for( j in 1:numParams ) {
+ vect = as.matrix(paramValues[j,1]);
+ paramLens[j,1] = nrow(vect);
+ }
+ paramVals = matrix(0, numParams, max(paramLens));
+ for( j in 1:numParams ) {
+ vect = as.matrix(paramValues[j,1]);
+ paramVals[j,1:nrow(vect)] = t(vect);
+ }
+ cumLens = rev(cumprod(rev(paramLens))/rev(paramLens));
+ numConfigs = prod(paramLens);
+
+ # Step 2) materialize hyper-parameter combinations
+ # (simplify debugging and compared to compute negligible)
+ HP = matrix(0, numConfigs, numParams);
+ for( i in 1:nrow(HP) ) {
+ for( j in 1:numParams )
+ HP[i,j] = paramVals[j,as.scalar(((i-1)/cumLens[j,1])%%paramLens[j,1]+1)];
+ }
+
+ if( verbose ) {
+ print("GridSeach: Number of hyper-parameters: \n"+toString(paramLens));
+ print("GridSeach: Hyper-parameter combinations: \n"+toString(HP));
+ }
+
+ # Step 3) training/scoring of parameter combinations
+ Rbeta = matrix(0, nrow(HP), numB);
+ Rloss = matrix(0, nrow(HP), 1);
+ # with cross-validation
+ if( cv & train == "lm") {
+ # a) create folds
+ foldsX = list(); foldsY = list();
+ fs = ceil(nrow(X)/cvk);
+ for( k in 0:(cvk-1) ) {
+ foldsX = append(foldsX, X[(k*fs+1):min((cvk+1)*fs,nrow(X)),]);
+ foldsY = append(foldsY, y[(k*fs+1):min((cvk+1)*fs,nrow(y)),]);
+ }
+ parfor( i in 1:nrow(HP) ) {
+ # a) replace training arguments
+ ltrainArgs = trainArgs;
+ lpredictArgs = predictArgs;
+ for( j in 1:numParams )
+ ltrainArgs[as.scalar(params[j])] = as.scalar(HP[i,j]);
+ # b) cross-validated training/scoring and write-back
+ cvbeta = matrix(0,1,numB);
+ cvloss = matrix(0,1,1);
+ for( k in 1:cvk ) {
+ [tmpX, testX] = remove(foldsX, k);
+ [tmpy, testy] = remove(foldsY, k);
+ ltrainArgs['X'] = rbind(tmpX);
+ ltrainArgs['y'] = rbind(tmpy);
+ lbeta = t(eval(train, ltrainArgs));
+ cvbeta[,1:ncol(lbeta)] = cvbeta[,1:ncol(lbeta)] + lbeta;
+ lpredict = list(as.matrix(testX), as.matrix(testy), t(lbeta), as.scalar(ltrainArgs['icpt']))
+ cvloss += eval(predict, lpredict);
+ }
+ Rbeta[i,] = cvbeta / cvk; # model averaging
+ Rloss[i,] = cvloss / cvk;
+ }
}
- else if(target == "lm")
+ else if(cv & train == "multiLogReg")
{
- params = list("reg", "tol", "maxi");
- paramRanges = list(10^seq(0,-4), 10^seq(1,-6), seq(10,100,10));
+ for( i in 1:nrow(HP) ) {
+ # a) replace training arguments
+ # acc = utils::crossVML(X, y, cvk, HP[i]);
+ k = cvk
+ accuracyMatrix = matrix(0, k, 1)
+ dataList = list()
+ testL = list()
+ data = order(target = cbind(y, X), by = 1, decreasing=FALSE, index.return=FALSE)
+ classes = table(data[, 1], 1)
+ ins_per_fold = classes/k
+ start_fold = matrix(1, rows=nrow(ins_per_fold), cols=1)
+ fold_idxes = cbind(start_fold, ins_per_fold)
- if(sum(mask) > 0)
- X = dummycoding(replace(target = X, pattern = NaN, replacement=0), mask)
+ start_i = 0; end_i = 0; idx_fold = 1;;
+ for(i in 1:k)
+ {
+ fold_i = matrix(0, 0, ncol(data))
+ start=0; end=0;
+ for(j in 1:nrow(classes))
+ {
+ idx = as.scalar(classes[j, 1])
+ start = end + 1;
+ end = end + idx
+ class_j = data[start:end, ]
+ start_i = as.scalar(fold_idxes[j, 1]);
+ end_i = as.scalar(fold_idxes[j, 2])
+ fold_i = rbind(fold_i, class_j[start_i:end_i, ])
+ }
+ dataList = append(dataList, fold_i)
+ fold_idxes[, 1] = fold_idxes[, 2] + 1
+ fold_idxes[, 2] += ins_per_fold
+ }
+
+ for(i in seq(1,k))
+ {
+ [trainList, hold_out] = remove(dataList, i)
+ trainset = rbind(trainList)
+ testset = as.matrix(hold_out)
+ trainX = trainset[, 2:ncol(trainset)]
+ trainy = trainset[, 1]
+ testsetX = testset[, 2:ncol(testset)]
+ testsety = testset[, 1]
+ beta = multiLogReg(X=trainX, Y=trainy, icpt=as.scalar(HP[1,1]), reg=as.scalar(HP[1,2]), tol=as.scalar(HP[1,3]),
+ maxi=as.scalar(HP[1,4]), maxii=50, verbose=FALSE);
+ [prob, yhat, accuracy] = multiLogRegPredict(testsetX, beta, testsety, FALSE)
+ accuracyMatrix[i] = accuracy
+ }
- trainArgs = list(X=X, y=y, icpt=0, reg=-1, tol=-1, maxi=-1, verbose=FALSE);
- [B1, opt] = gridSearch(X=X, y=y, train="lm", predict="l2norm",
- numB=ncol(X), params=params, paramValues=paramRanges, trainArgs=trainArgs, verbose=FALSE);
- }
- else
- print("getOptByTarget: target type not supported. Expected multiLogReg or lm found: "+target)
+ Rloss[i,] = mean(accuracyMatrix)
+ }
- bestOpt = as.matrix(opt)
-}
-
+ }
+ # without cross-validation
+ else {
+ parfor( i in 1:nrow(HP) ) {
+ # a) replace training arguments
+ ltrainArgs = trainArgs;
+ for( j in 1:numParams )
+ ltrainArgs[as.scalar(params[j])] = as.scalar(HP[i,j]);
+ # b) core training/scoring and write-back
+ lbeta = t(eval(train, ltrainArgs))
+ Rbeta[i,1:ncol(lbeta)] = lbeta;
+ Rloss[i,] = eval(predict, append(predictArgs,t(lbeta)));
+ }
+ }
+ # Step 4) select best parameter combination
+ ix = as.scalar(rowIndexMin(t(Rloss)));
+ B = t(Rbeta[ix,]); # optimal model
+ opt = as.frame(HP[ix,]); # optimal hyper-parameters
+}
diff --git a/src/main/java/org/apache/sysds/common/Builtins.java b/src/main/java/org/apache/sysds/common/Builtins.java
index f6cd50f..c12fcd3 100644
--- a/src/main/java/org/apache/sysds/common/Builtins.java
+++ b/src/main/java/org/apache/sysds/common/Builtins.java
@@ -129,6 +129,7 @@ public enum Builtins {
GMM_PREDICT("gmmPredict", true),
GNMF("gnmf", true),
GRID_SEARCH("gridSearch", true),
+ TOPK_CLEANING("topk_cleaning", true),
HOSPITAL_RESIDENCY_MATCH("hospitalResidencyMatch", true),
HYPERBAND("hyperband", true),
IFELSE("ifelse", false),
diff --git a/src/test/java/org/apache/sysds/test/functions/pipelines/CleaningTestLogical.java b/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinTopkCleaningClassificationTest.java
similarity index 62%
rename from src/test/java/org/apache/sysds/test/functions/pipelines/CleaningTestLogical.java
rename to src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinTopkCleaningClassificationTest.java
index 03fc9f9..dae2918 100644
--- a/src/test/java/org/apache/sysds/test/functions/pipelines/CleaningTestLogical.java
+++ b/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinTopkCleaningClassificationTest.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.sysds.test.functions.pipelines;
import org.apache.sysds.common.Types;
@@ -26,10 +25,11 @@ import org.apache.sysds.test.TestUtils;
import org.junit.Assert;
import org.junit.Test;
-public class CleaningTestLogical extends AutomatedTestBase {
- private final static String TEST_NAME = "testLogical";
- private final static String TEST_CLASS_DIR = SCRIPT_DIR + CleaningTestLogical.class.getSimpleName() + "/";
+public class BuiltinTopkCleaningClassificationTest extends AutomatedTestBase {
+ private final static String TEST_NAME1 = "topkcleaningClassificationTest";
+ private final static String TEST_CLASS_DIR = SCRIPT_DIR + BuiltinTopkCleaningClassificationTest.class.getSimpleName() + "/";
+ private final static String TEST_DIR = "functions/pipelines/";
private static final String RESOURCE = SCRIPT_DIR+"functions/pipelines/";
private static final String DATA_DIR = DATASET_DIR+ "pipelines/";
@@ -39,44 +39,29 @@ public class CleaningTestLogical extends AutomatedTestBase {
private static final String PARAM_DIR = "./scripts/pipelines/properties/";
private final static String PARAM = PARAM_DIR + "param.csv";
private final static String PRIMITIVES = PARAM_DIR + "testPrimitives.csv";
- private final static String OUTPUT = RESOURCE+"intermediates/logical.csv";
@Override
public void setUp() {
- addTestConfiguration(TEST_NAME,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME,new String[]{"R"}));
- }
-
- @Test
- public void testLogical1() {
- runTestLogical(10, 4, 2, 2,
- "multiLogReg", Types.ExecMode.SINGLE_NODE);
+ addTestConfiguration(TEST_NAME1,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1,new String[]{"R"}));
}
@Test
- public void testLogical2() {
- runTestLogical(2, 3, 3, 2,
- "multiLogReg", Types.ExecMode.SINGLE_NODE);
+ public void testFindBestPipeline() {
+ runtopkCleaning(0.1, 3,5, TEST_NAME1, Types.ExecMode.SINGLE_NODE);
}
- @Test
- public void testLogicalHybrid() {
- runTestLogical(3, 3, 2, 2,
- "multiLogReg", Types.ExecMode.HYBRID);
- }
- private void runTestLogical(int max_iter, int crossfold,
- int num_inst, int num_exec, String target, Types.ExecMode et) {
+ private void runtopkCleaning(Double sample, int topk, int resources, String testName, Types.ExecMode et) {
- // setOutputBuffering(true);
- String HOME = SCRIPT_DIR+"functions/pipelines/" ;
+ setOutputBuffering(true);
Types.ExecMode modeOld = setExecMode(et);
+ String HOME = SCRIPT_DIR + TEST_DIR;
try {
- loadTestConfiguration(getTestConfiguration(TEST_NAME));
- fullDMLScriptName = HOME + TEST_NAME + ".dml";
+ loadTestConfiguration(getTestConfiguration(testName));
+ fullDMLScriptName = HOME + testName + ".dml";
programArgs = new String[] {"-stats", "-exec", "singlenode", "-nvargs", "dirtyData="+DIRTY,
- "metaData="+META, "primitives="+PRIMITIVES, "parameters="+PARAM, "max_iter="+ max_iter,
- "cv="+ crossfold, "num_inst="+ num_inst, "num_exec="+ num_exec,
- "target="+target, "output="+OUTPUT, "O="+output("O")};
+ "metaData="+META, "primitives="+PRIMITIVES, "parameters="+PARAM, "topk="+ topk, "rv="+ resources,
+ "sample="+sample, "O="+output("O")};
runTest(true, EXCEPTION_NOT_EXPECTED, null, -1);
@@ -87,4 +72,6 @@ public class CleaningTestLogical extends AutomatedTestBase {
resetExecMode(modeOld);
}
}
+
+
}
diff --git a/src/test/java/org/apache/sysds/test/functions/pipelines/CleaningTestCompare.java b/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinTopkCleaningRegressionTest.java
similarity index 61%
rename from src/test/java/org/apache/sysds/test/functions/pipelines/CleaningTestCompare.java
rename to src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinTopkCleaningRegressionTest.java
index fbb253e..ef518b5 100644
--- a/src/test/java/org/apache/sysds/test/functions/pipelines/CleaningTestCompare.java
+++ b/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinTopkCleaningRegressionTest.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.sysds.test.functions.pipelines;
import org.apache.sysds.common.Types;
@@ -24,20 +23,19 @@ import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestConfiguration;
import org.apache.sysds.test.TestUtils;
import org.junit.Assert;
+import org.junit.Ignore;
import org.junit.Test;
-public class CleaningTestCompare extends AutomatedTestBase {
- private final static String TEST_NAME1 = "testCompare";
- private final static String TEST_CLASS_DIR = SCRIPT_DIR + CleaningTestCompare.class.getSimpleName() + "/";
+public class BuiltinTopkCleaningRegressionTest extends AutomatedTestBase{
+ private final static String TEST_NAME1 = "topkcleaningRegressionTest";
+ private final static String TEST_CLASS_DIR = SCRIPT_DIR + BuiltinTopkCleaningRegressionTest.class.getSimpleName() + "/";
- protected static final String RESOURCE = SCRIPT_DIR+"functions/pipelines/";
+ private static final String RESOURCE = SCRIPT_DIR+"functions/pipelines/";
+ // private static final String DATA_DIR = DATASET_DIR+ "pipelines/";
- private final static String DIRTY = DATASET_DIR+ "pipelines/dirty.csv";
- private final static String CLEAN = DATASET_DIR+ "pipelines/clean.csv";
- private final static String META = RESOURCE+ "meta/meta_census.csv";
+ private final static String DIRTY = DATASET_DIR+ "Salaries.csv";
private final static String OUTPUT = RESOURCE+"intermediates/";
-
- protected static final String PARAM_DIR = "./scripts/pipelines/properties/";
+ private static final String PARAM_DIR = "./scripts/pipelines/properties/";
private final static String PARAM = PARAM_DIR + "param.csv";
private final static String PRIMITIVES = PARAM_DIR + "primitives.csv";
@@ -47,29 +45,39 @@ public class CleaningTestCompare extends AutomatedTestBase {
}
@Test
- public void testCP1() {
- runFindPipelineTest(5,10, 2,
- true, "compare", Types.ExecMode.SINGLE_NODE);
+ public void testRegressionPipelinesCP() {
+ runFindPipelineTest(1.0, 5,20, 10,
+ "lm", Types.ExecMode.SINGLE_NODE);
+ }
+
+ @Ignore
+ public void testRegressionPipelinesHybrid() {
+ runFindPipelineTest(1.0, 5,5, 2,
+ "lm", Types.ExecMode.HYBRID);
}
- private void runFindPipelineTest(int topk, int resources, int crossfold,
- boolean weightedAccuracy, String target, Types.ExecMode et) {
- setOutputBuffering(true);
+ private void runFindPipelineTest(Double sample, int topk, int resources, int crossfold,
+ String target, Types.ExecMode et) {
+
+ // setOutputBuffering(true);
String HOME = SCRIPT_DIR+"functions/pipelines/" ;
Types.ExecMode modeOld = setExecMode(et);
try {
loadTestConfiguration(getTestConfiguration(TEST_NAME1));
fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
programArgs = new String[] {"-stats", "-exec", "singlenode", "-nvargs", "dirtyData="+DIRTY,
- "metaData="+META, "primitives="+PRIMITIVES, "parameters="+PARAM, "topk="+ topk, "rv="+ resources,
- "output="+OUTPUT, "target="+target, "cleanData="+CLEAN, "O="+output("O")};
+ "primitives="+PRIMITIVES, "parameters="+PARAM, "sampleSize="+ sample, "topk="+ topk,
+ "rv="+ resources, "sample="+ sample, "output="+OUTPUT, "target="+target, "O="+output("O")};
runTest(true, EXCEPTION_NOT_EXPECTED, null, -1);
+
+ //expected loss smaller than default invocation
Assert.assertTrue(TestUtils.readDMLBoolean(output("O")));
}
finally {
resetExecMode(modeOld);
}
}
+
}
diff --git a/src/test/java/org/apache/sysds/test/functions/pipelines/CleaningTestClassification.java b/src/test/java/org/apache/sysds/test/functions/pipelines/CleaningTestClassification.java
deleted file mode 100644
index 9043fdf..0000000
--- a/src/test/java/org/apache/sysds/test/functions/pipelines/CleaningTestClassification.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.sysds.test.functions.pipelines;
-
-import org.apache.sysds.common.Types;
-import org.apache.sysds.test.AutomatedTestBase;
-import org.apache.sysds.test.TestConfiguration;
-import org.apache.sysds.test.TestUtils;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-
-public class CleaningTestClassification extends AutomatedTestBase {
-// private final static String TEST_NAME1 = "prioritized";
- private final static String TEST_NAME1 = "testClassification";
- private final static String TEST_NAME2 = "compareAccuracy";
- private final static String TEST_CLASS_DIR = SCRIPT_DIR + CleaningTestClassification.class.getSimpleName() + "/";
-
- private static final String RESOURCE = SCRIPT_DIR+"functions/pipelines/";
- private static final String DATA_DIR = DATASET_DIR+ "pipelines/";
-
- private final static String DIRTY = DATA_DIR+ "dirty.csv";
- private final static String CLEAN = DATA_DIR+ "clean.csv";
- private final static String META = RESOURCE+ "meta/meta_census.csv";
- private final static String OUTPUT = RESOURCE+"intermediates/";
-
- private static final String PARAM_DIR = "./scripts/pipelines/properties/";
- private final static String PARAM = PARAM_DIR + "param.csv";
- private final static String PRIMITIVES = PARAM_DIR + "testPrimitives.csv";
-
- @Override
- public void setUp() {
- addTestConfiguration(TEST_NAME1,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1,new String[]{"R"}));
- addTestConfiguration(TEST_NAME2,new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2,new String[]{"R"}));
- }
-
- @Test
- public void testFindBestPipeline() {
- runFindPipelineTest(0.1, 5,5, 2,
- true, "multiLogReg", Types.ExecMode.SINGLE_NODE);
- }
-
- @Ignore
- public void testFindBestPipelineHybrid() {
- runFindPipelineTest(0.1, 5,5, 2,
- true, "multiLogReg", Types.ExecMode.HYBRID);
- }
-
- @Test
- public void testCompareRepairs() {
- runCleanAndCompareTest( Types.ExecMode.SINGLE_NODE);
- }
-
- private void runFindPipelineTest(Double sample, int topk, int resources, int crossfold,
- boolean weightedAccuracy, String target, Types.ExecMode et) {
-
-// setOutputBuffering(true);
- String HOME = SCRIPT_DIR+"functions/pipelines/" ;
- Types.ExecMode modeOld = setExecMode(et);
- try {
- loadTestConfiguration(getTestConfiguration(TEST_NAME1));
- fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
- programArgs = new String[] {"-stats", "-exec", "singlenode", "-nvargs", "dirtyData="+DIRTY,
- "metaData="+META, "primitives="+PRIMITIVES, "parameters="+PARAM,
- "sampleSize="+ sample, "topk="+ topk, "rv="+ resources, "cv="+ crossfold,
- "weighted="+ weightedAccuracy, "output="+OUTPUT, "target="+target, "cleanData="+CLEAN,
- "O="+output("O")};
-
- runTest(true, EXCEPTION_NOT_EXPECTED, null, -1);
-
- //expected loss smaller than default invocation
- Assert.assertTrue(TestUtils.readDMLBoolean(output("O")));
- }
- finally {
- resetExecMode(modeOld);
- }
- }
-
- private void runCleanAndCompareTest( Types.ExecMode et) {
- setOutputBuffering(true);
- String HOME = SCRIPT_DIR+"functions/pipelines/";
- Types.ExecMode modeOld = setExecMode(et);
- try {
- loadTestConfiguration(getTestConfiguration(TEST_NAME2));
- fullDMLScriptName = HOME + TEST_NAME2 + ".dml";
-
- programArgs = new String[] {"-stats", "-exec",
- "singlenode", "-args", DIRTY, CLEAN, META, OUTPUT, output("O")};
-
- runTest(true, EXCEPTION_NOT_EXPECTED, null, -1);
-
- //expected loss smaller than default invocation
- Assert.assertTrue(TestUtils.readDMLBoolean(output("O")));
- }
- finally {
- resetExecMode(modeOld);
- }
- }
-}
\ No newline at end of file
diff --git a/src/test/resources/datasets/pipelines/dirty.csv b/src/test/resources/datasets/pipelines/dirty.csv
index c432dd4..4bef1cf 100644
--- a/src/test/resources/datasets/pipelines/dirty.csv
+++ b/src/test/resources/datasets/pipelines/dirty.csv
@@ -1,217 +1,3 @@
-58.0,Self-employed-not incorporated,4,34,Some college but no degree,0.0,Not in universe,Divorced,Construction,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Head of household,South,Arkansas,Householder,Householder,1053.55,MSA to MSA,NaN,Same county,No,Yes,1.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,-50000
-9.0,Not in universe,0,0,Children,NaN,Not in universe,Never married,Not in universe or children,Not in universe,NaN,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,1758.14,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-10.0,Not in universe,0,0,Children,0.0,NaN,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,1069.16,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-42.0,Private,34,NaN,Bachelors degree(BA AB BS),NaN,NaN,Married-civilian spouse present,Finance insurance and real estate,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,5178.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,NaN,Householder,NaN,Nonmover,Nonmover,NaN,Yes,Not in universe,6.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,NaN,NaN,-50000
-34.0,NaN,4,NaN,Some college but no degree,NaN,NaN,Married-civilian spouse present,NaN,Machine operators assmblrs & inspctrs,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1146.79,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-8.0,Not in universe,0,0,Children,0.0,NaN,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,2466.24,NaN,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-NaN,Private,4,34,NaN,NaN,Not in universe,Married-civilian spouse present,Construction,Precision production craft & repair,White,Do not know,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2441.22,Nonmover,NaN,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,-50000
-46.0,NaN,NaN,31,High school graduate,0.0,Not in universe,Divorced,Business and repair services,Other service,White,Central or South American,Female,Not in universe,Not in universe,NaN,0.0,1590.0,0.0,Single,Not in universe,Not in universe,Householder,Householder,978.16,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,Columbia,Columbia,Columbia,Foreign born- Not a citizen of U S,0,Not in universe,2,52.0,94.0,-50000
-13.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,NaN,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,NaN,Child <18 never marr not in subfamily,Child under 18 never married,1520.08,Nonmover,Nonmover,Nonmover,NaN,NaN,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,0.0,94.0,-50000
-39.0,Not in universe,0,0,10th grade,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,Mexican (Mexicano),Female,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1274.04,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,Mexico,Mexico,Mexico,Foreign born- Not a citizen of U S,0,Not in universe,2,0.0,94.0,-50000
-12.0,Not in universe,0,0,Children,0.0,Not in universe,NaN,Not in universe or children,Not in universe,NaN,Mexican-American,Male,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Nonfiler,South,Utah,NaN,Child under 18 never married,455.02,MSA to MSA,Different region,Different state in South,No,Yes,0.0,Both parents present,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,0,NaN,NaN,-50000
-27.0,Self-employed-not incorporated,4,34,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,NaN,Precision production craft & repair,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1004.69,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,United-States,Native- Born in the United States,1,NaN,2,52.0,94.0,-50000
-46.0,Private,45,12,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,NaN,All other,Male,Not in universe,Not in universe,NaN,0.0,1977.0,NaN,Joint both under 65,Not in universe,NaN,Householder,NaN,999.46,Nonmover,NaN,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,-50000
-55.0,NaN,0,0,Some college but no degree,0.0,NaN,Married-civilian spouse present,NaN,Not in universe,Asian or Pacific Islander,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1483.69,Nonmover,Nonmover,Nonmover,NaN,Not in universe,0.0,NaN,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,0.0,94.0,-50000
-2.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,NaN,1660.53,Nonmover,Nonmover,Nonmover,NaN,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-37.0,Private,4,2,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Construction,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,1669.0,700.0,NaN,Not in universe,NaN,Secondary individual,Nonrelative of householder,1331.35,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,-50000
-46.0,Self-employed-not incorporated,4,34,NaN,0.0,Not in universe,Married-civilian spouse present,Construction,Precision production craft & repair,White,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1629.02,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-25.0,Private,45,23,High school graduate,NaN,Not in universe,Never married,Other professional services,Adm support including clerical,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,NaN,NaN,0.0,Single,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,Child 18 or older,1998.03,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,NaN,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-46.0,Private,NaN,42,High school graduate,0.0,Not in universe,Separated,Wholesale trade,Handlers equip cleaners etc,White,Central or South American,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Householder,Householder,2492.74,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,Peru,NaN,Peru,NaN,0,Not in universe,2,52.0,94.0,-50000
-39.0,Private,34,2,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,1000.0,Joint both under 65,Not in universe,NaN,Householder,Householder,980.1,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-11.0,Not in universe,NaN,0,Children,0.0,Not in universe,Never married,Not in universe or children,NaN,White,Mexican-American,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,498.25,NaN,Nonmover,Nonmover,Yes,Not in universe,0.0,NaN,Mexico,Mexico,NaN,Native- Born in the United States,NaN,Not in universe,NaN,0.0,NaN,-50000
-30.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Never married,Not in universe or children,Not in universe,Black,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Midwest,Minnesota,Householder,Householder,1773.08,MSA to MSA,Same county,Same county,No,No,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-7.0,Not in universe,NaN,0,Children,NaN,Not in universe,Never married,NaN,Not in universe,White,Other Spanish,Male,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Nonfiler,NaN,NaN,NaN,Child under 18 never married,885.84,Nonmover,Nonmover,Nonmover,Yes,NaN,0.0,Both parents present,Dominican-Republic,Puerto-Rico,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-66.0,Self-employed-not incorporated,33,19,7th and 8th grade,0.0,Not in universe,Married-civilian spouse present,Retail trade,Sales,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both 65+,Not in universe,Not in universe,NaN,NaN,4183.26,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,38.0,94.0,-50000
-26.0,Not in universe,0,0,High school graduate,0.0,Not in universe,NaN,Not in universe or children,Not in universe,White,Mexican-American,NaN,Not in universe,NaN,NaN,0.0,0.0,0.0,Joint both under 65,NaN,NaN,Spouse of householder,Spouse of householder,1040.15,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,NaN,Mexico,Mexico,Foreign born- Not a citizen of U S,0,Not in universe,2,0.0,94.0,-50000
-52.0,NaN,0,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1407.39,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0 [...]
-5.0,NaN,NaN,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Nonfiler,West,Alaska,Child <18 never marr not in subfamily,Child under 18 never married,235.92,MSA to MSA,Same county,Same county,No,No,0.0,Mother only present,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-35.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Female,NaN,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,Not in universe,NaN,NaN,Spouse of householder,1669.47,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,NaN,94.0,-50000
-36.0,Private,33,34,High school graduate,0.0,Not in universe,Married-civilian spouse present,Retail trade,Precision production craft & repair,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Midwest,Kansas,Householder,Householder,1812.0,NonMSA to nonMSA,Same county,Same county,No,NaN,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,48.0,NaN,-50000
-42.0,Private,44,NaN,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,NaN,Professional specialty,White,NaN,Female,Not in universe,Not in universe,NaN,0.0,0.0,0.0,NaN,NaN,Not in universe,Spouse of householder,Spouse of householder,2635.89,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,-50000
-1.0,Not in universe,0,0,Children,NaN,Not in universe,Never married,Not in universe or children,Not in universe,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Grandchild <18 never marr child of subfamily RP,Other relative of householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Mother only present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94. [...]
-72.0,Not in universe,0,0,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,NaN,Not in universe,White,All other,Female,Not in universe,Not in universe,NaN,0.0,0.0,NaN,Joint both 65+,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1841.65,Nonmover,NaN,NaN,Yes,Not in universe,0.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,-50000
-41.0,Private,12,35,Associates degree-academic program,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Precision production craft & repair,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,NaN,1095.21,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,NaN,2,Not in universe,2,24.0,NaN,-50000
-90.0,Not in universe,0,NaN,7th and 8th grade,NaN,Not in universe,Widowed,NaN,Not in universe,White,NaN,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Householder,Householder,3666.39,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-39.0,Private,32,16,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Wholesale trade,Sales,White,All other,Male,Not in universe,NaN,Children or Armed Forces,NaN,0.0,0.0,NaN,Not in universe,Not in universe,Householder,NaN,1322.1,Nonmover,NaN,Nonmover,Yes,Not in universe,4.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-0.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,NaN,Child under 18 never married,1475.95,Not in universe,Not in universe,Not in universe,Not in universe under 1 year old,NaN,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0 [...]
-57.0,NaN,NaN,0,NaN,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,NaN,White,Cuban,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,NaN,1133.65,NaN,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,Cuba,Cuba,Cuba,Foreign born- Not a citizen of U S,NaN,Not in universe,2,0.0,94.0,-50000
-17.0,Not in universe,0,0,7th and 8th grade,NaN,High school,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,NaN,NaN,NaN,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,1709.61,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,NaN,-50000
-2.0,Not in universe,0,NaN,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,Black,All other,Female,Not in universe,NaN,Children or Armed Forces,0.0,0.0,NaN,Nonfiler,Not in universe,Not in universe,NaN,NaN,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,NaN,0,0.0,94.0,-50000
-7.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Nonfiler,South,Massachusetts,Child <18 never marr not in subfamily,Child under 18 never married,1905.05,Not identifiable,Same county,Same county,NaN,No,0.0,Both parents present,Germany,United-States,United-States,Native- Born in the United States,0,NaN,0,0.0,94.0,-50000
-40.0,Private,19,NaN,NaN,0.0,Not in universe,Married-civilian spouse present,Manufacturing-nondurable goods,Handlers equip cleaners etc,White,Mexican (Mexicano),Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,Not in universe,NaN,Spouse of householder,Spouse of householder,1265.91,NaN,NaN,Nonmover,Yes,Not in universe,3.0,Not in universe,Mexico,Mexico,Mexico,Foreign born- Not a citizen of U S,0,Not in universe,2,43.0,94.0,-50000
-25.0,NaN,33,29,High school graduate,0.0,Not in universe,Never married,Retail trade,Other service,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Head of household,Not in universe,Not in universe,Child 18+ never marr RP of subfamily,Child 18 or older,765.19,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,4.0,94.0,-50000
-22.0,Private,34,26,NaN,0.0,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Adm support including clerical,NaN,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,NaN,Spouse of householder,1174.34,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-40.0,Private,NaN,NaN,High school graduate,0.0,Not in universe,Married-civilian spouse present,Transportation,Sales,White,NaN,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,NaN,Not in universe,Householder,Householder,2917.4,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-38.0,Not in universe,0,0,Less than 1st grade,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,Child 18 or older,2243.77,Nonmover,Nonmover,Nonmover,Yes,NaN,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-7.0,Not in universe,NaN,NaN,Children,0.0,Not in universe,Never married,Not in universe or children,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,2882.4,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,NaN,0,0.0,94.0,-50000
-49.0,State government,29,26,11th grade,0.0,Not in universe,Married-civilian spouse present,Transportation,Adm support including clerical,White,Mexican-American,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,958.8,Nonmover,NaN,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,-50000
-34.0,Local government,43,10,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Education,Professional specialty,White,All other,Female,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1152.47,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,40.0, [...]
-6.0,Not in universe,0,0,Children,0.0,NaN,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,NaN,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,1117.33,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,NaN,-50000
-31.0,Private,5,39,11th grade,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Transportation and material moving,White,Mexican-American,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,636.85,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in univ [...]
-74.0,Not in universe,NaN,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,5556.0,0.0,0.0,Joint both 65+,Not in universe,Not in universe,Householder,Householder,2730.77,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,NaN,NaN,United-States,NaN,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-0.0,Not in universe,NaN,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,NaN,Mexican-American,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,NaN,Nonfiler,Not in universe,NaN,Child <18 never marr not in subfamily,Child under 18 never married,1428.97,NaN,Not in universe,Not in universe,Not in universe under 1 year old,Not in universe,0.0,Both parents present,Mexico,Mexico,United-States,Native- Born in the United States,0,Not in [...]
-15.0,Not in universe,0,0,7th and 8th grade,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,NaN,NaN,Alaska,Child <18 never marr not in subfamily,Child under 18 never married,215.84,MSA to MSA,Same county,Same county,No,No,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-24.0,NaN,0,0,Bachelors degree(BA AB BS),0.0,College or university,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Northeast,New Mexico,Secondary individual,Nonrelative of householder,1208.48,NaN,Different division same region,NaN,No,No,0.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-43.0,Private,NaN,2,Some college but no degree,0.0,Not in universe,Never married,Finance insurance and real estate,Executive admin and managerial,White,All other,Female,Not in universe,NaN,Children or Armed Forces,0.0,0.0,NaN,Single,Not in universe,Not in universe,Nonfamily householder,NaN,1103.79,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-72.0,Not in universe,0,0,11th grade,0.0,Not in universe,Married-civilian spouse present,NaN,Not in universe,Black,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Householder,Householder,573.79,Nonmover,Nonmover,NaN,Yes,Not in universe,0.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-39.0,Self-employed-not incorporated,4,NaN,NaN,0.0,Not in universe,NaN,NaN,Precision production craft & repair,White,Other Spanish,Male,Not in universe,Other job loser,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,NaN,Householder,1282.49,Nonmover,Nonmover,Nonmover,Yes,NaN,1.0,Not in universe,Dominican-Republic,Dominican-Republic,Dominican-Republic,Foreign born- U S citizen by naturalization,0,NaN,2,52.0,94.0,-50000
-36.0,Private,11,23,NaN,NaN,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Adm support including clerical,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,NaN,Not in universe,Spouse of householder,Spouse of householder,2700.08,Nonmover,Nonmover,NaN,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,-50000
-38.0,Private,39,31,1st 2nd 3rd or 4th grade,0.0,Not in universe,Never married,Personal services except private HH,Other service,White,Central or South American,Female,NaN,NaN,Children or Armed Forces,0.0,0.0,0.0,Head of household,Not in universe,Not in universe,Householder,NaN,NaN,NaN,Nonmover,Nonmover,Yes,NaN,3.0,Not in universe,El-Salvador,El-Salvador,El-Salvador,Foreign born- U S citizen by naturalization,2,Not in universe,2,20.0,94.0,-50000
-49.0,Private,4,2,High school graduate,0.0,Not in universe,Divorced,Construction,Executive admin and managerial,White,All other,NaN,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Single,Midwest,Nevada,Nonfamily householder,Householder,803.77,MSA to MSA,Same county,Same county,No,No,3.0,Not in universe,United-States,Germany,United-States,Native- Born in the United States,1,Not in universe,2,52.0,94.0,-50000
-2.0,Not in universe,0,0,NaN,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,NaN,NaN,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,709.9,Nonmover,Nonmover,NaN,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-33.0,NaN,39,29,Associates degree-academic program,0.0,Not in universe,Never married,Personal services except private HH,Other service,White,All other,Male,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,2117.38,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,Scotland,NaN,Scotland,Foreign born- Not a citizen of U S,NaN,Not in universe,2,52.0,94.0,-50000
-15.0,NaN,0,0,10th grade,0.0,Not in universe,Never married,Not in universe or children,NaN,White,Mexican-American,NaN,Not in universe,New entrant,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,NaN,Child <18 never marr not in subfamily,Child under 18 never married,2288.5,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Both parents present,Mexico,Mexico,United-States,Native- Born in the United States,0,NaN,2,6.0,94.0,-50000
-18.0,NaN,0,0,Some college but no degree,0.0,College or university,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,NaN,Nonfiler,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,Child 18 or older,1680.95,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,NaN,United-States,NaN,Native- Born in the United States,0,Not in universe,2,12.0,94.0,-50000
-36.0,Private,9,36,High school graduate,0.0,Not in universe,NaN,Manufacturing-durable goods,Machine operators assmblrs & inspctrs,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2941.28,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-3.0,Not in universe,NaN,0,Children,NaN,NaN,Never married,Not in universe or children,NaN,White,NaN,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,NaN,Child <18 never marr not in subfamily,NaN,2379.74,Nonmover,Nonmover,Nonmover,Yes,NaN,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-84.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Divorced,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,2784.47,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,0.0,NaN,-50000
-27.0,Not in universe,0,0,9th grade,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Female,Not in universe,NaN,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1620.12,Nonmover,Nonmover,NaN,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-25.0,Self-employed-not incorporated,4,34,High school graduate,0.0,Not in universe,Never married,Construction,Precision production craft & repair,Black,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,NaN,Not in universe,Not in universe,Nonfamily householder,Householder,1427.41,Nonmover,NaN,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-17.0,Not in universe,0,0,NaN,0.0,High school,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,3659.03,Nonmover,Nonmover,NaN,Yes,NaN,0.0,Both parents present,Outlying-U S (Guam USVI etc),Portugal,Portugal,Native- Born abroad of American Parent(s),0,NaN,2,0.0,NaN,-50000
-NaN,Private,34,3,Some college but no degree,0.0,Not in universe,NaN,Finance insurance and real estate,Executive admin and managerial,White,Mexican (Mexicano),Female,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,16.0,NaN,Not in universe,Not in universe,Householder,Householder,472.36,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,Mexico,Mexico,NaN,0,Not in universe,2,52.0,94.0,-50000
-30.0,Self-employed-not incorporated,37,23,High school graduate,0.0,Not in universe,Married-civilian spouse present,Business and repair services,Adm support including clerical,White,Mexican (Mexicano),Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Child 18+ spouse of subfamily RP,Child 18 or older,2111.71,Nonmover,Nonmover,Nonmover,Yes,NaN,1.0,Not in universe,NaN,Mexico,Mexico,Foreign born- U S citizen by naturalization,0,No [...]
-23.0,State government,43,NaN,Associates degree-occup /vocational,450.0,College or university,NaN,NaN,Adm support including clerical,White,All other,Female,No,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Midwest,Tennessee,Householder,Householder,263.45,NonMSA to nonMSA,Different county same state,NaN,No,No,1.0,Not in universe,United-States,United-States,United-States,NaN,2,NaN,2,48.0,94.0,-50000
-37.0,Private,29,38,High school graduate,0.0,Not in universe,Married-civilian spouse present,Transportation,Transportation and material moving,White,All other,Male,Not in universe,NaN,Children or Armed Forces,0.0,NaN,4.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,1712.85,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,-50000
-NaN,State government,4,2,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Construction,Executive admin and managerial,White,NaN,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,70.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1142.73,Nonmover,Nonmover,NaN,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,-50000
-45.0,Self-employed-not incorporated,40,12,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Entertainment,NaN,White,NaN,NaN,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Secondary individual,Nonrelative of householder,3956.64,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-NaN,Private,NaN,30,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,Business and repair services,Other service,White,NaN,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1117.98,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,NaN,2,NaN,NaN,-50000
-28.0,Private,33,3,Some college but no degree,650.0,Not in universe,Married-civilian spouse present,Retail trade,NaN,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1813.28,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,NaN,94.0,-50000
-60.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,NaN,Joint both under 65,Not in universe,Not in universe,NaN,Householder,2420.38,Nonmover,NaN,Nonmover,Yes,NaN,0.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-49.0,NaN,0,0,High school graduate,0.0,Not in universe,Divorced,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,NaN,2275.31,Nonmover,Nonmover,Nonmover,NaN,Not in universe,1.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-24.0,Private,6,23,NaN,1100.0,Not in universe,Never married,Manufacturing-durable goods,Adm support including clerical,White,All other,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,Child 18 or older,2226.61,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,Italy,Italy,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,-50000
-81.0,Not in universe,0,NaN,NaN,0.0,Not in universe,NaN,Not in universe or children,Not in universe,White,All other,NaN,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both 65+,NaN,Not in universe,Spouse of householder,Spouse of householder,1056.14,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,NaN,0,NaN,2,NaN,94.0,-50000
-28.0,Federal government,43,7,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Never married,Education,Professional specialty,White,All other,Male,Yes,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,South,Utah,Secondary individual,Nonrelative of householder,3395.58,NaN,Same county,Same county,No,Yes,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,NaN,2,52.0,94.0,-50000
-NaN,Federal government,50,21,High school graduate,0.0,NaN,Married-civilian spouse present,Public administration,NaN,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,Not in universe,NaN,NaN,Spouse of householder,1589.4,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,NaN,Not in universe,NaN,52.0,94.0,-50000
-34.0,Private,NaN,NaN,NaN,0.0,Not in universe,Divorced,Private household services,Private household services,NaN,Mexican-American,Female,Not in universe,Re-entrant,Children or Armed Forces,NaN,0.0,0.0,Single,West,Colorado,Secondary individual,Nonrelative of householder,1329.22,MSA to MSA,Same county,Same county,No,No,4.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,-50000
-27.0,Private,34,26,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Adm support including clerical,White,NaN,Female,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,7.0,Joint both under 65,Not in universe,NaN,Spouse of householder,NaN,1972.62,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,Mexico,Mexico,Mexico,NaN,0,Not in universe,2,52.0,94.0,-50000
-76.0,Not in universe,0,0,High school graduate,NaN,Not in universe,Widowed,Not in universe or children,Not in universe,White,All other,Female,Not in universe,NaN,Children or Armed Forces,0.0,0.0,NaN,Single,Not in universe,Not in universe,Nonfamily householder,Householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,-50000
-13.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,NaN,Female,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,NaN,NaN,2779.4,Nonmover,Nonmover,Nonmover,NaN,Not in universe,0.0,NaN,United-States,United-States,United-States,NaN,0,Not in universe,NaN,0.0,94.0,-50000
-44.0,Local government,43,12,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Education,Professional specialty,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1643.75,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0 [...]
-33.0,Private,34,23,High school graduate,0.0,Not in universe,Married-civilian spouse present,NaN,Adm support including clerical,NaN,All other,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,NaN,NaN,NaN,289.6,Nonmover,Nonmover,Nonmover,Yes,NaN,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,NaN,-50000
-41.0,Private,32,NaN,Associates degree-occup /vocational,0.0,NaN,Married-civilian spouse present,Wholesale trade,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,1858.26,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-41.0,Self-employed-not incorporated,45,11,NaN,0.0,Not in universe,Never married,Other professional services,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,453.68,NaN,Nonmover,Nonmover,NaN,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,NaN,-50000
-65.0,Not in universe,0,0,Associates degree-occup /vocational,NaN,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,NaN,Female,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Joint both 65+,Not in universe,Not in universe,Spouse of householder,Spouse of householder,703.36,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,Na [...]
-31.0,Private,1,44,High school graduate,0.0,Not in universe,Never married,Agriculture,Farming forestry and fishing,White,NaN,NaN,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Householder,Householder,2419.92,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-NaN,Federal government,50,3,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Public administration,Executive admin and managerial,Other,Central or South American,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2435.08,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,NaN,Italy,Peru,Peru,Foreign born- U S citizen by naturalization,0,Not in universe,2,52.0,NaN,-50000
-NaN,Private,NaN,37,Some college but no degree,0.0,Not in universe,Never married,NaN,Machine operators assmblrs & inspctrs,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1703.78,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,NaN,94.0,-50000
-10.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,2721.59,Nonmover,Nonmover,Nonmover,Yes,NaN,0.0,Both parents present,NaN,United-States,NaN,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-2.0,Not in universe,0,0,Children,0.0,NaN,Never married,Not in universe or children,Not in universe,White,Puerto Rican,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,1383.39,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,Puerto-Rico,Puerto-Rico,United-States,Native- Born in the United States,0,Not in universe,NaN,0.0,94.0,-50000
-31.0,Private,37,26,Bachelors degree(BA AB BS),0.0,NaN,Married-spouse absent,Business and repair services,Adm support including clerical,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Secondary individual,Nonrelative of householder,1444.01,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-1.0,Not in universe,0,0,Children,0.0,NaN,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,NaN,Nonfiler,Not in universe,Not in universe,Child under 18 of RP of unrel subfamily,Nonrelative of householder,581.34,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,NaN,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,0,0.0,NaN,-50000
-15.0,Not in universe,0,0,7th and 8th grade,0.0,Not in universe,NaN,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,1932.66,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,NaN,United-States,United-States,Native- Born in the United States,0,NaN,2,NaN,94.0,-50000
-32.0,Private,33,NaN,Some college but no degree,830.0,NaN,Married-civilian spouse present,NaN,Sales,White,All other,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,45.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,NaN,Nonmover,NaN,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,NaN,94.0,-50000
-27.0,Not in universe,NaN,NaN,High school graduate,0.0,Not in universe,Divorced,Not in universe or children,Not in universe,White,Mexican-American,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Head of household,Not in universe,Not in universe,Householder,NaN,1075.1,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,16.0,94.0,-50000
-6.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,1067.0,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,NaN,United-States,Native- Born in the United States,0,NaN,0,0.0,94.0,-50000
-26.0,NaN,19,NaN,7th and 8th grade,0.0,Not in universe,Married-civilian spouse present,NaN,Other service,White,Mexican (Mexicano),Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Midwest,Kentucky,Householder,Householder,605.73,NonMSA to nonMSA,Same county,Same county,NaN,No,4.0,Not in universe,Mexico,Mexico,Mexico,Foreign born- Not a citizen of U S,0,Not in universe,2,NaN,94.0,-50000
-18.0,NaN,33,42,11th grade,500.0,High school,Never married,Retail trade,NaN,White,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Nonfiler,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,Child 18 or older,1901.27,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,Yugoslavia,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-67.0,Private,34,26,Some college but no degree,0.0,NaN,Divorced,Finance insurance and real estate,Adm support including clerical,White,NaN,Female,NaN,Not in universe,Children or Armed Forces,0.0,0.0,10000.0,Single,Not in universe,Not in universe,Nonfamily householder,NaN,1971.97,Nonmover,Nonmover,Nonmover,Yes,NaN,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,-50000
-NaN,NaN,0,0,NaN,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,728.0,Joint both 65+,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1040.18,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,NaN,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-27.0,Self-employed-not incorporated,4,2,12th grade no diploma,0.0,Not in universe,Married-civilian spouse present,Construction,Executive admin and managerial,White,Puerto Rican,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,NaN,NaN,Householder,Householder,941.93,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,-50000
-45.0,Private,37,12,Bachelors degree(BA AB BS),0.0,NaN,Married-civilian spouse present,Business and repair services,Professional specialty,White,All other,NaN,Not in universe,Not in universe,NaN,NaN,0.0,11.0,Joint both under 65,Midwest,Indiana,Spouse of householder,NaN,1757.67,MSA to MSA,NaN,Same county,No,No,3.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-42.0,Private,37,31,Some college but no degree,0.0,Not in universe,Never married,NaN,Other service,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,9799.0,NaN,Not in universe,Not in universe,NaN,Child 18 or older,5334.5,NaN,NaN,Nonmover,Yes,Not in universe,2.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-26.0,Private,33,19,High school graduate,NaN,Not in universe,Married-civilian spouse present,Retail trade,Sales,NaN,All other,Female,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,NaN,Not in universe,Spouse of householder,Spouse of householder,2032.85,Nonmover,NaN,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,United-States,NaN,0,Not in universe,2,26.0,NaN,-50000
-1.0,Not in universe,NaN,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,NaN,Mexican (Mexicano),Female,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Nonfiler,Not in universe,Not in universe,Other Rel <18 never marr child of subfamily RP,Other relative of householder,NaN,Nonmover,Nonmover,NaN,Yes,Not in universe,0.0,Both parents present,Mexico,Mexico,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-3.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,NaN,NaN,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,1667.99,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,NaN,United-States,United-States,United-States,NaN,0,Not in universe,0,0.0,94.0,-50000
-78.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,NaN,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both 65+,Not in universe,Not in universe,Householder,Householder,1415.56,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,Cambodia,Italy,Italy,Foreign born- U S citizen by naturalization,0,Not in universe,2,0.0,94.0,-50000
-13.0,NaN,0,NaN,Children,0.0,NaN,Never married,Not in universe or children,Not in universe,White,NaN,Female,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,South,Massachusetts,Grandchild <18 never marr not in subfamily,NaN,2058.09,MSA to MSA,Different state same division,Different state in South,No,No,0.0,NaN,United-States,United-States,United-States,NaN,NaN,Not in universe,0,0.0,94.0,-50000
-4.0,NaN,NaN,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,NaN,All other,Male,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,2039.23,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,NaN,United-States,NaN,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-39.0,State government,43,10,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Education,Professional specialty,White,All other,Female,No,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1213.85,Nonmover,Nonmover,Nonmover,NaN,Not in universe,5.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,NaN,-50000
-26.0,Private,33,42,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Retail trade,Handlers equip cleaners etc,White,All other,NaN,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,Child 18 or older,1724.64,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-28.0,State government,43,26,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Education,Adm support including clerical,Asian or Pacific Islander,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,910.37,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,El-Salvador,United-States,Native- Born in the United States,0,Not in universe, [...]
-NaN,Private,33,16,High school graduate,0.0,Not in universe,Married-civilian spouse present,Retail trade,Sales,NaN,All other,Male,No,Not in universe,Children or Armed Forces,0.0,NaN,50.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,616.82,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,-50000
-57.0,Not in universe,NaN,0,10th grade,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,3306.61,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,4.0,94.0,-50000
-NaN,Not in universe,0,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,NaN,Not in universe,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,3464.0,0.0,0.0,Joint both under 65,Abroad,Abroad,Householder,Householder,NaN,Abroad to nonMSA,Abroad,Abroad,No,No,6.0,Not in universe,NaN,United-States,Germany,Native- Born abroad of American Parent(s),0,NaN,2,52.0,NaN,-50000
-6.0,Not in universe,0,0,Children,0.0,Not in universe,NaN,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Nonfiler,Not in universe,Not in universe,NaN,Child under 18 never married,296.63,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Mother only present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,NaN,-50000
-25.0,Private,42,13,NaN,2825.0,Not in universe,Never married,Medical except hospital,Technicians and related support,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,2691.02,NaN,NaN,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,Germany,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,-50000
-90.0,Not in universe,0,0,1st 2nd 3rd or 4th grade,NaN,Not in universe,Widowed,Not in universe or children,Not in universe,Black,All other,Female,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1697.6,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,40.0,94.0,-50000
-59.0,Private,4,34,High school graduate,NaN,Not in universe,Married-civilian spouse present,Construction,Precision production craft & repair,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,NaN,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-14.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,NaN,Child under 18 never married,1168.4,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-23.0,Private,33,26,NaN,0.0,Not in universe,Married-civilian spouse present,Retail trade,Adm support including clerical,Asian or Pacific Islander,All other,Female,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,West,NaN,Spouse of householder,Spouse of householder,2853.63,MSA to MSA,Same county,Same county,No,No,2.0,Not in universe,China,China,Vietnam,Foreign born- Not a citizen of U S,0,Not in universe,2,52.0,94.0,-50000
-24.0,Private,33,29,Bachelors degree(BA AB BS),NaN,Not in universe,NaN,Retail trade,Other service,Asian or Pacific Islander,All other,Female,No,NaN,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,NaN,Not in universe,Spouse of householder,NaN,527.95,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,China,China,China,Foreign born- U S citizen by naturalization,0,Not in universe,2,52.0,94.0,-50000
-52.0,Private,NaN,31,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Personal services except private HH,Other service,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,NaN,NaN,Spouse of householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,NaN,2,NaN,94.0,-50000
-10.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,Black,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Nonfiler,Not in universe,NaN,Child <18 never marr not in subfamily,Child under 18 never married,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Mother only present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,0.0,94.0,-50000
-27.0,Private,29,41,High school graduate,0.0,Not in universe,Married-civilian spouse present,Transportation,Handlers equip cleaners etc,White,NaN,Female,Not in universe,Not in universe,NaN,0.0,NaN,NaN,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,2291.16,Nonmover,NaN,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,-50000
-58.0,Private,36,27,Associates degree-academic program,0.0,Not in universe,Married-civilian spouse present,NaN,Private household services,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Other Rel 18+ ever marr RP of subfamily,Other relative of householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,Poland,Poland,Poland,Foreign born- Not a citizen of U S,0,Not in unive [...]
-NaN,NaN,0,0,High school graduate,0.0,Not in universe,NaN,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,NaN,NaN,Householder,Householder,1176.32,MSA to MSA,Same county,Same county,No,Yes,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,14.0,94.0,-50000
-27.0,Private,16,35,High school graduate,NaN,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Precision production craft & repair,White,Other Spanish,NaN,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,704.01,NaN,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,Dominican-Republic,Dominican-Republic,Dominican-Republic,Foreign born- Not a c [...]
-56.0,Not in universe,0,0,High school graduate,NaN,Not in universe,NaN,Not in universe or children,Not in universe,White,NaN,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Midwest,Minnesota,Other Rel 18+ never marr not in subfamily,Other relative of householder,1164.99,MSA to MSA,Different region,Different state in Midwest,No,No,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,NaN,-50000
-46.0,Private,9,36,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,NaN,Machine operators assmblrs & inspctrs,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,NaN,1294.7,Nonmover,Nonmover,Nonmover,NaN,Not in universe,5.0,Not in universe,NaN,United-States,NaN,Native- Born in the United States,0,Not in universe,2,NaN,94.0,-50000
-15.0,Not in universe,NaN,0,NaN,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Child <18 never marr not in subfamily,NaN,368.23,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,16.0,94.0,-50000
-13.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,218.68,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94. [...]
-22.0,Private,33,41,Associates degree-occup /vocational,0.0,Not in universe,Never married,Retail trade,Handlers equip cleaners etc,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Single,Not in universe,Not in universe,NaN,Householder,2162.94,Nonmover,Nonmover,Nonmover,Yes,NaN,6.0,NaN,United-States,United-States,United-States,NaN,0,Not in universe,2,NaN,94.0,-50000
-63.0,Private,2,44,9th grade,NaN,Not in universe,Married-civilian spouse present,Agriculture,Farming forestry and fishing,Black,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1877.1,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-9.0,Not in universe,0,NaN,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,Black,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Grandchild <18 never marr not in subfamily,NaN,2575.1,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Neither parent present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-NaN,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Nonfiler,Not in universe,NaN,NaN,NaN,1916.92,NaN,Not in universe,Not in universe,Not in universe under 1 year old,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-83.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,NaN,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Spouse of householder,Spouse of householder,2211.06,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-76.0,Private,22,42,5th or 6th grade,0.0,Not in universe,Never married,Manufacturing-nondurable goods,Handlers equip cleaners etc,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Single,NaN,NaN,Nonfamily householder,Householder,2505.84,Nonmover,Nonmover,NaN,Yes,Not in universe,4.0,NaN,United-States,United-States,United-States,NaN,0,No,1,52.0,94.0,-50000
-22.0,Private,33,16,Bachelors degree(BA AB BS),NaN,Not in universe,Never married,Retail trade,Sales,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Midwest,Minnesota,Child 18+ never marr Not in a subfamily,Child 18 or older,1650.78,MSA to MSA,Different county same state,Different county same state,No,No,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,NaN,52.0,NaN,-50000
-40.0,Private,17,26,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,NaN,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,NaN,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94. [...]
-23.0,Private,NaN,23,Some college but no degree,0.0,Not in universe,Never married,Manufacturing-durable goods,Adm support including clerical,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Head of household,Not in universe,Not in universe,Householder,Householder,3840.77,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-43.0,Private,18,33,High school graduate,0.0,NaN,Married-civilian spouse present,Manufacturing-durable goods,Precision production craft & repair,White,Other Spanish,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,Nonmover,Yes,NaN,5.0,Not in universe,NaN,Nicaragua,Nicaragua,Foreign born- Not a citizen of U S,0,Not in universe,2,51.0,94.0,-50000
-34.0,Not in universe,0,0,7th and 8th grade,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,NaN,2594.24,Nonmover,Nonmover,NaN,Yes,NaN,0.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,0.0,94.0,-50000
-26.0,Private,NaN,13,Associates degree-academic program,NaN,NaN,NaN,Hospital services,NaN,White,All other,Female,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,3154.71,Nonmover,Nonmover,Nonmover,Yes,Not in universe,5.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,52.0,94.0,-50000
-NaN,Local government,43,31,High school graduate,0.0,Not in universe,Married-civilian spouse present,Education,Other service,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1816.74,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,-50000
-33.0,Private,24,36,Associates degree-academic program,0.0,Not in universe,Married-spouse absent,Manufacturing-nondurable goods,Machine operators assmblrs & inspctrs,NaN,Mexican-American,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Head of household,Not in universe,Not in universe,Householder,Householder,2230.8,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,-50000
-12.0,Not in universe,NaN,0,Children,0.0,Not in universe,Never married,NaN,Not in universe,White,All other,Male,NaN,Not in universe,NaN,0.0,0.0,0.0,NaN,West,Alaska,Child <18 never marr not in subfamily,Child under 18 never married,223.16,NaN,Same county,NaN,No,No,0.0,Father only present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-8.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,NaN,1823.98,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-34.0,Private,33,24,High school graduate,655.0,Not in universe,Divorced,Retail trade,Adm support including clerical,White,NaN,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Head of household,West,Abroad,Householder,Householder,374.79,Not identifiable,Same county,NaN,No,No,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,-50000
-69.0,Not in universe,0,0,NaN,0.0,Not in universe,Widowed,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Householder,Householder,996.92,Nonmover,Nonmover,Nonmover,Yes,NaN,0.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,0.0,NaN,-50000
-64.0,Not in universe,NaN,0,11th grade,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1701.5,NaN,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,NaN,NaN,Not in universe,2,0.0,94.0,-50000
-23.0,Private,42,13,NaN,1000.0,Not in universe,Never married,Medical except hospital,Technicians and related support,White,All other,NaN,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,West,California,Grandchild 18+ never marr not in subfamily,Other relative of householder,3761.04,MSA to MSA,Different county same state,Different county same state,No,NaN,1.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,22.0,NaN,-50000
-7.0,Not in universe,0,0,Children,NaN,Not in universe,Never married,NaN,Not in universe,White,All other,Female,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,4035.27,Nonmover,Nonmover,NaN,Yes,Not in universe,NaN,Both parents present,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-20.0,State government,43,29,Some college but no degree,486.0,College or university,Married-civilian spouse present,Education,Other service,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,NaN,Not in universe,Householder,Householder,1737.87,Nonmover,Nonmover,NaN,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,30.0,94.0,-50000
-37.0,Private,35,NaN,High school graduate,NaN,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Adm support including clerical,NaN,All other,Female,Not in universe,Not in universe,NaN,0.0,NaN,2.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,NaN,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-9.0,Not in universe,0,0,Children,0.0,Not in universe,NaN,Not in universe or children,Not in universe,Black,Cuban,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Other Rel <18 never marr child of subfamily RP,Other relative of householder,1344.2,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,NaN,0,Not in universe,0,NaN,94.0,-50000
-71.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Nonfiler,Not in universe,Not in universe,Householder,Householder,3687.36,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-44.0,NaN,32,38,Some college but no degree,NaN,Not in universe,NaN,Wholesale trade,Transportation and material moving,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,West,Arizona,Nonfamily householder,Householder,3010.95,MSA to MSA,Same county,Same county,No,Yes,3.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-49.0,Self-employed-not incorporated,42,7,Doctorate degree(PhD EdD),0.0,Not in universe,Married-civilian spouse present,Medical except hospital,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,NaN,Not in universe,Not in universe,Householder,Householder,495.96,Nonmover,Nonmover,NaN,Yes,Not in universe,1.0,Not in universe,Canada,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-75.0,Not in universe,0,0,5th or 6th grade,0.0,NaN,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Nonfiler,Not in universe,NaN,Spouse of householder,Spouse of householder,1066.9,Nonmover,Nonmover,Nonmover,Yes,NaN,0.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-30.0,Private,34,3,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Finance insurance and real estate,Executive admin and managerial,Black,All other,Female,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,100.0,Single,NaN,Not in universe,Nonfamily householder,Householder,1998.28,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-53.0,Not in universe,0,0,High school graduate,0.0,NaN,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,125.0,Nonfiler,Not in universe,Not in universe,Spouse of householder,Spouse of householder,2370.83,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-4.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Nonfiler,Northeast,North Carolina,Child <18 never marr not in subfamily,Child under 18 never married,1641.25,MSA to MSA,Same county,Same county,No,No,0.0,Mother only present,NaN,United-States,United-States,Native- Born in the United States,0,NaN,0,0.0,94.0,-50000
-34.0,Private,33,29,Some college but no degree,525.0,Not in universe,Never married,Retail trade,Other service,White,All other,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Head of household,Not in universe,Not in universe,Householder,Householder,576.26,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-19.0,Private,33,19,High school graduate,0.0,Not in universe,Never married,NaN,Sales,NaN,NaN,Male,Not in universe,Not in universe,NaN,NaN,0.0,0.0,Single,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,Child 18 or older,2798.6,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-NaN,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,2047.69,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,NaN,-50000
-NaN,Not in universe,0,0,Children,0.0,NaN,Never married,Not in universe or children,Not in universe,White,All other,NaN,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,NaN,Child under 18 never married,3564.43,Nonmover,NaN,Nonmover,Yes,Not in universe,0.0,Father only present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,0.0,94.0,-50000
-27.0,Private,33,29,Some college but no degree,0.0,Not in universe,Divorced,Retail trade,Other service,White,Mexican-American,Male,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Single,West,Arizona,NaN,Nonrelative of householder,3828.57,MSA to MSA,NaN,Same county,No,Yes,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-14.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,NaN,NaN,Child <18 never marr not in subfamily,Child under 18 never married,2057.21,Nonmover,NaN,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,NaN,0.0,94.0,-50000
-52.0,Private,8,36,10th grade,NaN,Not in universe,NaN,Manufacturing-durable goods,Machine operators assmblrs & inspctrs,NaN,All other,Male,Not in universe,Re-entrant,Children or Armed Forces,NaN,0.0,0.0,Nonfiler,Not in universe,Not in universe,NaN,Spouse of householder,1406.66,Nonmover,NaN,Nonmover,Yes,Not in universe,0.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,-50000
-24.0,State government,50,23,Some college but no degree,0.0,Not in universe,NaN,NaN,Adm support including clerical,White,Puerto Rican,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,Child 18 or older,985.23,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,NaN,Puerto-Rico,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-54.0,Private,4,34,High school graduate,0.0,Not in universe,Married-civilian spouse present,Construction,Precision production craft & repair,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-41.0,NaN,43,10,NaN,0.0,NaN,Married-civilian spouse present,Education,NaN,White,All other,Female,NaN,Not in universe,Children or Armed Forces,NaN,0.0,50.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,540.55,Nonmover,Nonmover,Nonmover,Yes,Not in universe,5.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,41.0,94.0,-50000
-18.0,NaN,NaN,0,10th grade,NaN,High school,Never married,Not in universe or children,Not in universe,Black,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,NaN,Child 18 or older,1069.95,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-33.0,Private,33,33,NaN,0.0,Not in universe,Never married,Retail trade,Precision production craft & repair,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1008.07,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,-50000
-17.0,Private,39,36,11th grade,0.0,High school,Never married,Personal services except private HH,Machine operators assmblrs & inspctrs,White,All other,Male,Not in universe,Re-entrant,NaN,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,446.1,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,8.0,94.0,-50000
-15.0,Not in universe,0,0,7th and 8th grade,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,3423.07,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe [...]
-37.0,Private,19,26,NaN,0.0,Not in universe,NaN,Manufacturing-nondurable goods,Adm support including clerical,NaN,Mexican-American,Female,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1306.75,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,NaN,United-States,NaN,Native- Born in the United States,0,Not in universe,2,26.0,94.0,-50000
-21.0,Private,34,25,Some college but no degree,625.0,Not in universe,Never married,NaN,Adm support including clerical,NaN,All other,Male,No,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Single,Not in universe,NaN,Child 18+ never marr Not in a subfamily,Child 18 or older,1262.82,Nonmover,NaN,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,NaN,2,52.0,NaN,-50000
-3.0,Not in universe,0,NaN,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,South,Florida,Other Rel <18 never marr child of subfamily RP,Other relative of householder,1785.73,MSA to MSA,Same county,Same county,No,Yes,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-29.0,Private,33,2,Some college but no degree,0.0,Not in universe,Never married,Retail trade,Executive admin and managerial,White,Central or South American,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,NaN,Child 18 or older,1126.36,NaN,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,Peru,Peru,Peru,Foreign born- Not a citizen of U S,NaN,NaN,2,52.0,94.0,-50000
-2.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,Mexican-American,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Grandchild <18 never marr child of subfamily RP,Other relative of householder,1393.61,Nonmover,NaN,Nonmover,Yes,Not in universe,0.0,Mother only present,United-States,United-States,United-States,Native- Born in the United States,0,Not in univ [...]
-37.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1957.61,Nonmover,Nonmover,NaN,NaN,Not in universe,0.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,0.0,NaN,-50000
-NaN,Not in universe,0,0,7th and 8th grade,NaN,Not in universe,NaN,NaN,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Householder,Householder,1536.41,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,NaN,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-63.0,Not in universe,0,0,Some college but no degree,0.0,Not in universe,Divorced,Not in universe or children,Not in universe,White,All other,Female,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,NaN,Not in universe,NaN,Householder,1479.01,Nonmover,Nonmover,Nonmover,Yes,NaN,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-44.0,State government,47,28,High school graduate,0.0,Not in universe,Married-civilian spouse present,NaN,Protective services,NaN,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1772.79,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,NaN,-50000
-0.0,Not in universe,0,0,Children,NaN,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,NaN,NaN,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,579.58,Not in universe,Not in universe,Not in universe,Not in universe under 1 year old,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States [...]
-35.0,Private,9,36,11th grade,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2776.47,Nonmover,Nonmover,NaN,NaN,Not in universe,2.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-3.0,NaN,0,0,Children,NaN,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,West,Vermont,Child <18 never marr not in subfamily,Child under 18 never married,480.28,MSA to MSA,Same county,Same county,No,No,0.0,Both parents present,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-50.0,Private,NaN,16,Associates degree-occup /vocational,0.0,Not in universe,Married-civilian spouse present,NaN,Sales,White,All other,NaN,Not in universe,Not in universe,NaN,NaN,0.0,300.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,829.8,Nonmover,Nonmover,Nonmover,NaN,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
-51.0,Private,42,30,High school graduate,NaN,Not in universe,Married-spouse absent,NaN,Other service,White,All other,Female,No,NaN,Children or Armed Forces,0.0,0.0,0.0,Head of household,Northeast,Maryland,NaN,Householder,573.61,MSA to MSA,Same county,Same county,No,No,6.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,2,Not in universe,2,52.0,94.0,-50000
-46.0,Private,18,37,High school graduate,0.0,Not in universe,Married-spouse absent,Manufacturing-durable goods,Machine operators assmblrs & inspctrs,White,NaN,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,NaN,Secondary individual,Nonrelative of householder,1022.83,Nonmover,Nonmover,NaN,Yes,Not in universe,2.0,NaN,Columbia,Columbia,Columbia,Foreign born- Not a citizen of U S,0,Not in universe,2,52.0,94.0,-50000
-42.0,Local government,43,NaN,NaN,0.0,Not in universe,Married-civilian spouse present,Education,Professional specialty,NaN,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,10.0,NaN,Not in universe,Not in universe,Spouse of householder,Spouse of householder,2142.94,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,52.0,94.0,-50000
-10.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,NaN,0.0,0.0,NaN,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,460.07,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-22.0,Private,33,29,High school graduate,0.0,Not in universe,Never married,Retail trade,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Nonfiler,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,Child 18 or older,2318.96,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,0.0,94.0,-50000
-3.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,NaN,Child <18 never marr not in subfamily,Child under 18 never married,1415.37,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,0,0.0,94.0,-50000
-1.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Child <18 never marr not in subfamily,Child under 18 never married,500.55,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Both parents present,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,0,NaN,94.0,-50000
-28.0,Not in universe,0,0,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,50.0,NaN,Not in universe,NaN,Spouse of householder,Spouse of householder,2551.55,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,0.0,94.0,-50000
-43.0,Local government,43,23,Some college but no degree,0.0,Not in universe,Divorced,Education,Adm support including clerical,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Head of household,Not in universe,Not in universe,Householder,Householder,318.07,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,NaN,52.0,94.0,-50000
-67.0,Private,33,31,Associates degree-academic program,0.0,Not in universe,Married-civilian spouse present,Retail trade,Other service,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,NaN,NaN,NaN,Not in universe,Spouse of householder,Spouse of householder,1674.41,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Yes,1,52.0,94.0,-50000
-63.0,Not in universe,0,0,Associates degree-occup /vocational,0.0,NaN,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,NaN,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,45.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1588.73,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,0. [...]
-38.0,Private,29,26,11th grade,0.0,Not in universe,Married-civilian spouse present,Transportation,Adm support including clerical,White,All other,Female,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,NaN,Householder,Householder,2055.63,Nonmover,NaN,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,United-States,NaN,2,Not in universe,2,52.0,94.0,-50000
-13.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,NaN,White,All other,Male,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Nonfiler,Midwest,NaN,Child <18 never marr not in subfamily,NaN,1729.33,MSA to MSA,Different county same state,Different county same state,No,No,0.0,Both parents present,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,0,0.0,NaN,-50000
-12.0,Not in universe,0,0,Children,0.0,Not in universe,Never married,Not in universe or children,Not in universe,Black,All other,Male,Not in universe,NaN,Children or Armed Forces,0.0,NaN,0.0,Nonfiler,Not in universe,Not in universe,Grandchild <18 never marr not in subfamily,NaN,1205.11,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Neither parent present,United-States,United-States,United-States,NaN,0,Not in universe,0,0.0,94.0,-50000
-50.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Never married,NaN,Not in universe,Black,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,NaN,Not in universe,Householder,Householder,2305.9,Nonmover,NaN,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
-NaN,Private,33,NaN,Some college but no degree,500.0,Not in universe,NaN,Retail trade,Other service,Black,All other,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Head of household,West,Colorado,Householder,Householder,2174.3,MSA to MSA,NaN,Different county same state,No,No,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,-50000
36.0,Private,40,NaN,10th grade,0.0,Not in universe,Never married,Entertainment,Precision production craft & repair,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Secondary individual,Nonrelative of householder,2115.13,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,-50000
20.0,Private,33,41,11th grade,0.0,Not in universe,Never married,Retail trade,Handlers equip cleaners etc,Black,All other,Male,Not in universe,Other job loser,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Child 18+ never marr Not in a subfamily,NaN,3040.78,Nonmover,Nonmover,NaN,Yes,Not in universe,5.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,40.0,94.0,-50000
64.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Nonfiler,Not in universe,Not in universe,Householder,Householder,1897.44,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,-50000
@@ -763,238 +549,5 @@ NaN,Private,29,38,Some college but no degree,0.0,Not in universe,Divorced,Transp
25.0,Self-employed-incorporated,34,19,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Finance insurance and real estate,Sales,White,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,NaN,Nonmover,Nonmover,Nonmover,NaN,Not in universe,1.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,50000+.
40.0,Self-employed-incorporated,45,3,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Other professional services,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2153.92,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States, [...]
64.0,Private,42,12,Doctorate degree(PhD EdD),0.0,Not in universe,Married-civilian spouse present,Medical except hospital,NaN,NaN,All other,Female,NaN,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1731.71,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,40.0,94.0,50000+.
-49.0,Self-employed-not incorporated,4,34,Some college but no degree,0.0,NaN,Married-civilian spouse present,Construction,Precision production craft & repair,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,West,Vermont,Householder,Householder,1015.71,MSA to MSA,Same county,Same county,No,No,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,94.0,50000+.
-39.0,Private,NaN,33,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Precision production craft & repair,White,All other,NaN,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,NaN,Not in universe,Not in universe,Householder,NaN,NaN,NaN,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-36.0,Private,NaN,5,NaN,0.0,Not in universe,Married-civilian spouse present,Wholesale trade,NaN,Asian or Pacific Islander,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1194.78,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,Japan,Japan,Japan,Foreign born- Not a citizen of U S,0,Not in universe,2,52.0,94.0,50000+.
-67.0,Private,33,8,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Retail trade,Professional specialty,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint one under 65 & one 65+,Not in universe,Not in universe,Householder,Householder,NaN,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,Canada,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-83.0,Self-employed-incorporated,2,43,NaN,0.0,Not in universe,Married-civilian spouse present,Agriculture,Farming forestry and fishing,White,All other,Male,Not in universe,NaN,Children or Armed Forces,20051.0,0.0,0.0,Joint both 65+,Not in universe,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,NaN,United-States,NaN,United-States,NaN,0,Not in universe,NaN,52.0,94.0,50000+.
-31.0,Private,32,16,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Wholesale trade,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,7688.0,0.0,350.0,Joint both under 65,NaN,NaN,Householder,Householder,1681.0,MSA to MSA,Same county,NaN,No,Yes,6.0,Not in universe,United-States,NaN,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-33.0,Self-employed-not incorporated,38,33,Some college but no degree,NaN,Not in universe,Married-civilian spouse present,Business and repair services,Precision production craft & repair,White,All other,Male,NaN,Not in universe,Children or Armed Forces,7298.0,0.0,0.0,Joint both under 65,South,Maine,Householder,Householder,2867.52,MSA to MSA,Same county,NaN,No,Yes,1.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,50000+.
-35.0,Self-employed-not incorporated,42,7,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,Medical except hospital,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,1450.0,NaN,West,NaN,Householder,Householder,875.89,Not identifiable,Different county same state,Different county same state,No,Yes,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States, [...]
-66.0,Not in universe,NaN,NaN,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,NaN,All other,Male,NaN,NaN,Children or Armed Forces,0.0,2377.0,9191.0,NaN,Not in universe,Not in universe,Householder,Householder,429.93,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,NaN,0.0,94.0,50000+.
-34.0,Private,12,35,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Precision production craft & repair,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,15024.0,0.0,100.0,Joint both under 65,Not in universe,Not in universe,NaN,NaN,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,30.0,94.0 [...]
-59.0,Not in universe,NaN,0,NaN,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,Cuban,Male,Not in universe,Not in universe,Children or Armed Forces,15024.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,NaN,Householder,1222.77,Nonmover,NaN,NaN,Yes,NaN,6.0,NaN,Cuba,Cuba,Cuba,Foreign born- U S citizen by naturalization,0,Not in universe,2,52.0,94.0,50000+.
-47.0,Private,26,35,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,NaN,Precision production craft & repair,White,All other,NaN,Not in universe,Not in universe,Children or Armed Forces,7298.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1757.67,Nonmover,Nonmover,Nonmover,Yes,Not in universe,5.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,52.0,94.0,50000+.
-51.0,Private,32,18,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Wholesale trade,Sales,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,5000.0,Joint both under 65,Not in universe,Not in universe,NaN,Spouse of householder,1480.52,Nonmover,NaN,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,50000+.
-48.0,Private,30,4,Bachelors degree(BA AB BS),0.0,Not in universe,Married-spouse absent,Communications,Professional specialty,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,1000.0,Single,South,Georgia,Nonfamily householder,Householder,3215.91,NaN,Different state same division,Different state in South,No,Yes,6.0,NaN,United-States,United-States,NaN,Native- Born in the United States,1,Not in universe,2,52.0,NaN,50000+.
-45.0,Not in universe,0,0,Some college but no degree,0.0,Not in universe,Separated,NaN,Not in universe,Black,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Head of household,Midwest,Montana,NaN,Householder,744.6,NaN,Different region,Different state in Midwest,No,NaN,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,0.0,94.0,50000+.
-43.0,NaN,44,2,Masters degree(MA MS MEng MEd MSW MBA),NaN,Not in universe,Married-civilian spouse present,Social services,Executive admin and managerial,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1879.9,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,94.0,50000+.
-46.0,Private,NaN,NaN,Doctorate degree(PhD EdD),9999.0,Not in universe,Never married,Other professional services,Professional specialty,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Single,NaN,Not in universe,Nonfamily householder,Householder,3434.01,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,50000+.
-54.0,Private,29,NaN,Bachelors degree(BA AB BS),NaN,Not in universe,Married-civilian spouse present,Transportation,Technicians and related support,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2652.69,Nonmover,Nonmover,Nonmover,NaN,Not in universe,NaN,NaN,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,50000+.
-28.0,Self-employed-not incorporated,40,12,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Entertainment,Professional specialty,Black,All other,Male,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Single,NaN,Not in universe,Nonfamily householder,NaN,1050.25,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,50000+.
-33.0,Private,19,18,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Manufacturing-nondurable goods,Sales,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2988.6,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,NaN,Nicaragua,Nicaragua,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-28.0,NaN,37,12,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,NaN,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,NaN,Secondary individual,Nonrelative of householder,4323.57,Nonmover,Nonmover,NaN,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,50000+.
-NaN,Private,33,2,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Retail trade,Executive admin and managerial,White,All other,Male,No,Not in universe,Children or Armed Forces,15024.0,0.0,17000.0,Joint both under 65,Midwest,Minnesota,Householder,Householder,1412.23,MSA to MSA,Same county,Same county,No,No,2.0,NaN,United-States,United-States,United-States,Native- Born in the United States,1,Not in universe,2,52.0,NaN,50000+.
-34.0,Private,41,8,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Divorced,Hospital services,Professional specialty,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Single,Not in universe,Not in universe,Child 18+ ever marr Not in a subfamily,Child 18 or older,1389.09,Nonmover,Nonmover,NaN,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0 [...]
-NaN,Private,24,12,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Manufacturing-nondurable goods,NaN,Amer Indian Aleut or Eskimo,All other,Male,No,Not in universe,Children or Armed Forces,0.0,NaN,5.0,Joint both under 65,NaN,Not in universe,Householder,Householder,354.87,Nonmover,NaN,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,NaN,Not in universe,NaN,52.0,94.0,50000+.
-63.0,Private,45,4,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,1300.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,Nonmover,NaN,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in unive [...]
-30.0,Private,37,14,High school graduate,0.0,Not in universe,Married-civilian spouse present,Business and repair services,Technicians and related support,White,All other,Male,Not in universe,Other job loser,Children or Armed Forces,0.0,0.0,150.0,Joint both under 65,NaN,Not in universe,Householder,Householder,3790.25,NaN,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-NaN,Private,42,8,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Medical except hospital,Professional specialty,White,All other,Male,No,Not in universe,Children or Armed Forces,15024.0,0.0,150.0,Joint both under 65,Not in universe,NaN,Householder,Householder,3245.8,Nonmover,Nonmover,Nonmover,NaN,Not in universe,3.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,2,Not in universe,2,52.0,NaN,50000+.
-57.0,Private,43,9,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Education,Professional specialty,Asian or Pacific Islander,All other,Male,NaN,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,South Korea,South Korea,South Korea,Foreign born- U S citizen by naturalization,0,NaN [...]
-42.0,NaN,47,28,Bachelors degree(BA AB BS),2000.0,Not in universe,Married-civilian spouse present,NaN,Protective services,White,All other,Female,NaN,Not in universe,Children or Armed Forces,0.0,0.0,50.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1697.61,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-41.0,Not in universe,0,0,High school graduate,0.0,NaN,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,NaN,Not in universe,NaN,Householder,3164.28,NaN,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,Germany,Native- Born abroad of American Parent(s),0,Not in universe,2,52.0,94.0,50000+.
-26.0,State government,41,8,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Never married,NaN,Professional specialty,White,All other,Female,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1596.19,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,50000+.
-62.0,Self-employed-incorporated,4,2,Bachelors degree(BA AB BS),0.0,Not in universe,Divorced,Construction,Executive admin and managerial,White,Other Spanish,Male,Not in universe,Not in universe,Children or Armed Forces,10520.0,NaN,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,475.5,Nonmover,Nonmover,Nonmover,NaN,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,No,1,52.0,NaN,50000+.
-29.0,Private,25,39,High school graduate,0.0,NaN,Married-civilian spouse present,Manufacturing-nondurable goods,Transportation and material moving,White,All other,NaN,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,14.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,NaN,NaN,NaN,NaN,Yes,Not in universe,NaN,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-36.0,Private,39,2,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Personal services except private HH,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,South,Texas,Householder,Householder,2637.15,MSA to MSA,Same county,Same county,No,Yes,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN, [...]
-NaN,NaN,5,37,NaN,0.0,Not in universe,Never married,Manufacturing-durable goods,NaN,NaN,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,2464.08,NaN,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-47.0,Local government,4,34,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Construction,Precision production craft & repair,White,All other,NaN,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,Northeast,North Carolina,Householder,Householder,NaN,MSA to MSA,Same county,Same county,No,No,6.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-35.0,Private,45,4,Bachelors degree(BA AB BS),NaN,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,7298.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2826.17,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,50000+.
-62.0,Not in universe,0,0,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Widowed,Not in universe or children,Not in universe,Black,NaN,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,2547.0,NaN,Head of household,Not in universe,Not in universe,Householder,Householder,541.38,Nonmover,Nonmover,Nonmover,Yes,NaN,6.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,94.0,50000+.
-43.0,Self-employed-not incorporated,35,17,Bachelors degree(BA AB BS),0.0,Not in universe,NaN,NaN,Sales,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,15000.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,975.87,Nonmover,Nonmover,NaN,Yes,NaN,1.0,Not in universe,United-States,United-States,United-States,NaN,2,Not in universe,2,NaN,94.0,50000+.
-47.0,State government,48,26,Masters degree(MA MS MEng MEd MSW MBA),1070.0,Not in universe,Married-civilian spouse present,Public administration,Adm support including clerical,White,All other,NaN,No,Not in universe,NaN,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,515.39,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in univers [...]
-58.0,Private,44,23,Bachelors degree(BA AB BS),800.0,Not in universe,NaN,Social services,Adm support including clerical,White,NaN,Female,No,Not in universe,Children or Armed Forces,NaN,NaN,NaN,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1485.23,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-58.0,State government,48,NaN,NaN,0.0,Not in universe,Married-civilian spouse present,Public administration,Adm support including clerical,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,15024.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2718.81,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-49.0,Private,22,2,NaN,0.0,Not in universe,Married-civilian spouse present,NaN,Executive admin and managerial,White,All other,NaN,NaN,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,NaN,Householder,2144.73,Nonmover,NaN,Nonmover,NaN,Not in universe,NaN,NaN,United-States,United-States,NaN,Native- Born in the United States,0,NaN,2,52.0,94.0,50000+.
-42.0,Private,30,33,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Communications,Precision production craft & repair,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,South,Missouri,NaN,NaN,2802.34,NonMSA to nonMSA,Different state same division,NaN,No,Yes,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,26.0,94.0,50000+.
-44.0,Self-employed-incorporated,33,16,Some college but no degree,0.0,NaN,Never married,Retail trade,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,2258.0,NaN,Single,Not in universe,Not in universe,Secondary individual,Nonrelative of householder,2993.85,NaN,NaN,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,NaN,1,Not in universe,2,52.0,NaN,50000+.
-48.0,Private,13,33,Some college but no degree,0.0,Not in universe,Divorced,Manufacturing-durable goods,Precision production craft & repair,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,15020.0,0.0,0.0,Head of household,Not in universe,Not in universe,Householder,NaN,1687.68,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-74.0,Not in universe,0,0,Masters degree(MA MS MEng MEd MSW MBA),NaN,Not in universe,Divorced,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,NaN,0.0,2282.0,60000.0,Single,NaN,Not in universe,Nonfamily householder,NaN,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,0.0,94.0,50000+.
-60.0,Private,NaN,NaN,Bachelors degree(BA AB BS),0.0,NaN,Married-civilian spouse present,Finance insurance and real estate,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,South,Oregon,NaN,Householder,2095.61,MSA to MSA,Same county,Same county,No,Yes,4.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,50000+.
-58.0,Not in universe,0,0,Bachelors degree(BA AB BS),0.0,Not in universe,NaN,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,27828.0,0.0,5000.0,Single,Not in universe,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,3.0,94.0,50000+.
-NaN,Private,NaN,4,NaN,0.0,Not in universe,NaN,Manufacturing-durable goods,Professional specialty,NaN,All other,NaN,Not in universe,Not in universe,NaN,0.0,NaN,11.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1994.37,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,NaN,1,Not in universe,2,52.0,94.0,50000+.
-33.0,Local government,47,23,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Public administration,Adm support including clerical,White,Mexican-American,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,Mexico,Mexico,United-States,Native- Born in the United States,0,No [...]
-23.0,NaN,3,39,High school graduate,0.0,Not in universe,Married-civilian spouse present,Mining,Transportation and material moving,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,1902.0,0.0,Joint both under 65,South,Utah,Householder,Householder,2327.7,MSA to MSA,Same county,NaN,No,Yes,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-35.0,Private,3,18,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,NaN,Mining,Sales,White,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,350.0,NaN,Not in universe,Not in universe,Householder,Householder,2134.67,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-56.0,Private,29,39,High school graduate,0.0,Not in universe,Divorced,Transportation,Transportation and material moving,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,NaN,NaN,2129.96,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,52.0,NaN,50000+.
-46.0,Private,37,26,Some college but no degree,0.0,Not in universe,Divorced,Business and repair services,Adm support including clerical,White,All other,Female,NaN,Not in universe,Children or Armed Forces,0.0,0.0,200.0,NaN,Not in universe,Not in universe,Householder,NaN,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,50000+.
-35.0,Private,NaN,33,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,3564.8,Nonmover,Nonmover,NaN,Yes,Not in universe,6.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-49.0,Federal government,43,26,Masters degree(MA MS MEng MEd MSW MBA),0.0,NaN,Married-civilian spouse present,Education,Adm support including clerical,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,17.0,Joint both under 65,West,New York,Householder,Householder,830.17,MSA to MSA,Same county,Same county,No,Yes,NaN,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,50000+.
-43.0,Private,16,22,Some college but no degree,0.0,Not in universe,Never married,Manufacturing-durable goods,Adm support including clerical,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,750.0,Single,Not in universe,Not in universe,Householder,Householder,2066.24,Nonmover,NaN,Nonmover,NaN,Not in universe,6.0,NaN,NaN,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-40.0,Private,37,19,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Business and repair services,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,Not in universe,Not in universe,NaN,NaN,2317.05,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,NaN,94.0,50000+.
-55.0,Private,43,9,Bachelors degree(BA AB BS),0.0,Not in universe,Divorced,Education,Professional specialty,White,All other,Female,No,NaN,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,914.93,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,NaN,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-32.0,Private,45,11,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,Asian or Pacific Islander,All other,Female,Not in universe,Not in universe,Children or Armed Forces,15024.0,0.0,NaN,Joint both under 65,NaN,Not in universe,Householder,Householder,1663.21,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,NaN,NaN,United-States,NaN,0,NaN,2,NaN,NaN,50000+.
-39.0,Self-employed-not incorporated,37,2,High school graduate,0.0,NaN,Married-civilian spouse present,Business and repair services,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,Not in universe,Not in universe,NaN,Householder,1915.24,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN [...]
-NaN,Private,32,18,Bachelors degree(BA AB BS),0.0,NaN,Married-civilian spouse present,Wholesale trade,Sales,White,All other,Male,Not in universe,NaN,Children or Armed Forces,0.0,1902.0,0.0,Joint both under 65,Northeast,NaN,Householder,Householder,1558.29,MSA to MSA,Different state same division,Different state in Northeast,No,NaN,1.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,51.0,94.0,50000+.
-36.0,Not in universe,NaN,0,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,3314.34,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,50000+.
-55.0,State government,NaN,26,Bachelors degree(BA AB BS),0.0,Not in universe,Divorced,Education,Adm support including clerical,NaN,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,10520.0,0.0,0.0,Single,NaN,Colorado,Nonfamily householder,Householder,NaN,MSA to MSA,Different state same division,NaN,No,NaN,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-35.0,Private,37,3,Bachelors degree(BA AB BS),0.0,Not in universe,NaN,Business and repair services,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Joint both under 65,Northeast,North Carolina,Householder,Householder,2116.66,MSA to MSA,Different state same division,Different state in Northeast,No,No,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,94.0,50000+.
-35.0,Private,45,11,NaN,0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,Midwest,Indiana,Householder,Householder,1347.09,MSA to MSA,Same county,Same county,No,No,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,51.0,94.0,50000+.
-NaN,State government,47,12,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Public administration,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,7688.0,0.0,0.0,Joint both under 65,NaN,NaN,NaN,Householder,304.22,NonMSA to MSA,Different county same state,Different county same state,NaN,No,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,1,Not in uni [...]
-30.0,Private,42,12,Bachelors degree(BA AB BS),0.0,Not in universe,Widowed,Medical except hospital,Professional specialty,White,All other,Female,NaN,NaN,Children or Armed Forces,0.0,0.0,NaN,Head of household,Not in universe,Not in universe,NaN,Householder,1622.03,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,NaN,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-40.0,Self-employed-not incorporated,42,7,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Never married,Medical except hospital,Professional specialty,White,All other,Male,Not in universe,Not in universe,NaN,0.0,2444.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,3261.64,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-36.0,Self-employed-incorporated,NaN,33,High school graduate,NaN,Not in universe,Married-civilian spouse present,Construction,Precision production craft & repair,White,All other,Male,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,NaN,Householder,1953.41,Nonmover,Nonmover,NaN,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,51.0,94.0,50000+.
-NaN,Self-employed-incorporated,42,7,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,Medical except hospital,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,2415.0,20.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2087.24,Nonmover,Nonmover,NaN,Yes,Not in universe,2.0,NaN,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,45.0 [...]
-50.0,Not in universe,0,0,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,NaN,Not in universe,NaN,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,NaN,Florida,Spouse of householder,Spouse of householder,1991.44,MSA to MSA,Same county,Same county,No,Yes,0.0,Not in universe,NaN,United-States,United-States,NaN,0,NaN,2,0.0,94.0,50000+.
-45.0,NaN,44,2,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Social services,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,15024.0,NaN,133.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2843.52,Nonmover,Nonmover,NaN,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-41.0,Not in universe,0,0,Doctorate degree(PhD EdD),0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,NaN,Householder,NaN,2013.38,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-47.0,State government,NaN,9,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Education,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1691.77,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-38.0,Private,NaN,2,NaN,0.0,Not in universe,Married-civilian spouse present,Manufacturing-nondurable goods,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,25.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1834.13,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-NaN,Private,37,17,NaN,0.0,Not in universe,Married-civilian spouse present,Business and repair services,Sales,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,100.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2125.3,Nonmover,Nonmover,Nonmover,NaN,Not in universe,4.0,Not in universe,NaN,United-States,United-States,NaN,2,NaN,2,52.0,94.0,50000+.
-46.0,Private,NaN,5,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,NaN,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,20.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2717.15,Nonmover,NaN,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-51.0,State government,50,14,Associates degree-occup /vocational,0.0,Not in universe,Divorced,Public administration,Technicians and related support,NaN,NaN,Male,Not in universe,NaN,Children or Armed Forces,0.0,2444.0,0.0,Single,Not in universe,NaN,Nonfamily householder,Householder,719.08,Nonmover,Nonmover,NaN,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-34.0,Private,31,36,Some college but no degree,0.0,Not in universe,Never married,Utilities and sanitary services,Machine operators assmblrs & inspctrs,White,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,500.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,2790.78,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,England,England,Native- Born abroad of American Parent(s),0,Not in universe,2,52.0,94.0,50000+.
-35.0,NaN,33,16,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Retail trade,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,5178.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Child 18+ ever marr RP of subfamily,Child 18 or older,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,50000+.
-27.0,Private,11,2,Bachelors degree(BA AB BS),NaN,NaN,Married-civilian spouse present,Manufacturing-durable goods,Executive admin and managerial,Asian or Pacific Islander,All other,Female,No,NaN,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Spouse of householder,Spouse of householder,3108.79,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,NaN,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-54.0,Federal government,43,10,Masters degree(MA MS MEng MEd MSW MBA),0.0,NaN,Married-civilian spouse present,Education,Professional specialty,White,All other,Male,Yes,NaN,Children or Armed Forces,7298.0,0.0,NaN,Joint both under 65,Not in universe,Not in universe,NaN,Householder,1458.64,NaN,Nonmover,NaN,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,NaN,2,52.0,94.0,50000+.
-24.0,Private,34,26,High school graduate,NaN,Not in universe,Married-civilian spouse present,Finance insurance and real estate,NaN,White,All other,Female,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,NaN,2128.46,Nonmover,Nonmover,NaN,Yes,Not in universe,2.0,Not in universe,Italy,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,50000+.
-50.0,Private,34,10,Bachelors degree(BA AB BS),0.0,Not in universe,NaN,Finance insurance and real estate,Professional specialty,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Head of household,Not in universe,Not in universe,Other Rel 18+ ever marr RP of subfamily,Other relative of householder,1352.48,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,Ireland,United-States,United-States,Native- Born in the United States,0,Not in [...]
-45.0,Local government,43,NaN,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,NaN,Education,Professional specialty,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,7688.0,0.0,NaN,NaN,Not in universe,Not in universe,Householder,Householder,3488.27,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,39.0,94.0,50000+.
-76.0,Not in universe,0,0,Some college but no degree,0.0,NaN,Married-civilian spouse present,Not in universe or children,NaN,White,All other,Male,Not in universe,Not in universe,NaN,0.0,NaN,0.0,Joint both 65+,Not in universe,NaN,Spouse of householder,Spouse of householder,1630.62,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,NaN,Italy,Italy,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-NaN,Self-employed-not incorporated,NaN,17,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Never married,Finance insurance and real estate,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,200.0,NaN,NaN,Not in universe,NaN,Householder,1763.82,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-43.0,Federal government,49,1,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Public administration,Executive admin and managerial,White,All other,Female,Not in universe,Not in universe,NaN,0.0,0.0,400.0,Joint both under 65,Not in universe,Not in universe,NaN,Spouse of householder,1152.97,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,50000+.
-44.0,Private,45,4,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,NaN,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,7298.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2052.12,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-47.0,Private,4,3,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Construction,Executive admin and managerial,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,1000.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,787.68,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in univers [...]
-51.0,Self-employed-incorporated,32,2,NaN,0.0,Not in universe,Married-civilian spouse present,Wholesale trade,Executive admin and managerial,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,3221.76,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-31.0,Self-employed-incorporated,45,NaN,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Other professional services,Professional specialty,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,1500.0,NaN,Not in universe,Not in universe,Nonfamily householder,Householder,4232.07,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,94 [...]
-43.0,Private,31,4,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Utilities and sanitary services,Professional specialty,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,225.0,Joint both under 65,NaN,Not in universe,Householder,Householder,1444.83,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-34.0,NaN,47,11,NaN,0.0,Not in universe,Married-civilian spouse present,Public administration,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Householder,Householder,1387.78,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-43.0,Private,35,2,NaN,0.0,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Executive admin and managerial,White,NaN,Male,Not in universe,NaN,Children or Armed Forces,0.0,0.0,2500.0,Joint both under 65,NaN,Not in universe,NaN,Householder,2719.7,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,50000+.
51.0,Private,30,NaN,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Communications,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,24000.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1614.85,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-50.0,Private,35,2,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Executive admin and managerial,White,All other,Male,Not in universe,NaN,Children or Armed Forces,NaN,1902.0,0.0,Joint both under 65,NaN,Not in universe,Householder,Householder,3404.17,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,NaN,52.0,94.0,50000+.
-43.0,Local government,43,10,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-spouse absent,Education,Professional specialty,White,All other,Female,NaN,Not in universe,NaN,7430.0,0.0,0.0,Head of household,Not in universe,Not in universe,Householder,Householder,NaN,NaN,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,50000+.
-62.0,Private,29,33,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Transportation,Precision production craft & repair,Black,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1673.74,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,50000+.
-29.0,Private,32,16,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Never married,Wholesale trade,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,13.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,3096.85,Nonmover,NaN,Nonmover,Yes,Not in universe,4.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,50000+.
-56.0,Self-employed-not incorporated,42,7,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,Medical except hospital,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,150.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,3019.13,Nonmover,Nonmover,Nonmover,Yes,NaN,1.0,NaN,England,England,England,Foreign born- Not a citizen of U S,NaN,Not in universe,2,52.0,94.0,50000+.
-30.0,Private,34,5,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1291.74,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,5 [...]
-39.0,Private,33,33,NaN,0.0,Not in universe,Married-civilian spouse present,Retail trade,Precision production craft & repair,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,NaN,Not in universe,NaN,Householder,1453.2,Nonmover,NaN,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,50000+.
-38.0,Private,28,2,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Manufacturing-nondurable goods,Executive admin and managerial,White,All other,NaN,No,NaN,Children or Armed Forces,7688.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-38.0,Private,33,3,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Retail trade,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,1000.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1456.55,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52. [...]
-36.0,Self-employed-not incorporated,45,12,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,White,NaN,NaN,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,150.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1385.85,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in [...]
-55.0,Not in universe,0,0,High school graduate,0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,NaN,Spouse of householder,1210.56,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,Italy,United-States,NaN,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-40.0,Private,33,16,Bachelors degree(BA AB BS),0.0,Not in universe,Divorced,Retail trade,Sales,White,All other,NaN,NaN,NaN,Children or Armed Forces,0.0,1564.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,NaN,1792.6,Nonmover,Nonmover,Nonmover,NaN,Not in universe,1.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-41.0,Private,11,2,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,NaN,0.0,Joint both under 65,Midwest,Oklahoma,Householder,Householder,1748.59,NaN,Same county,Same county,No,No,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,94.0,50000+.
-55.0,Self-employed-not incorporated,34,17,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Sales,White,All other,Male,Not in universe,Not in universe,NaN,15024.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,694.71,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-44.0,Private,42,2,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Medical except hospital,Executive admin and managerial,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,NaN,Spouse of householder,1024.81,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2 [...]
-38.0,Private,30,12,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,NaN,Professional specialty,NaN,All other,Female,Yes,Not in universe,NaN,0.0,0.0,1000.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1811.8,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-NaN,State government,43,9,Doctorate degree(PhD EdD),0.0,Not in universe,Married-civilian spouse present,Education,Professional specialty,White,All other,Female,Yes,Not in universe,Children or Armed Forces,7688.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,3392.62,Nonmover,Nonmover,Nonmover,Yes,Not in universe,5.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,NaN,94.0,50000+.
-62.0,Not in universe,0,0,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,NaN,NaN,NaN,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,1250.0,Joint one under 65 & one 65+,Not in universe,Not in universe,Householder,Householder,1220.55,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,0.0,NaN,50000+.
-51.0,Private,29,39,High school graduate,0.0,Not in universe,NaN,Transportation,Transportation and material moving,White,NaN,Male,Not in universe,NaN,NaN,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,1750.38,Nonmover,Nonmover,Nonmover,Yes,NaN,6.0,NaN,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-32.0,State government,43,7,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,Education,Professional specialty,White,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,170.0,Joint both under 65,South,North Dakota,Householder,Householder,NaN,MSA to MSA,Same county,NaN,No,NaN,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,50000+.
-39.0,Federal government,35,2,High school graduate,NaN,NaN,Married-civilian spouse present,Finance insurance and real estate,Executive admin and managerial,White,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,South,Georgia,NaN,Spouse of householder,2629.88,NonMSA to nonMSA,Different county same state,NaN,No,NaN,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-32.0,Private,45,NaN,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Other professional services,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,1000.0,Single,Not in universe,Not in universe,Nonfamily householder,NaN,3254.84,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-NaN,Local government,47,28,Some college but no degree,NaN,Not in universe,Married-civilian spouse present,Public administration,Protective services,White,All other,Male,Not in universe,Other job loser,NaN,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Householder,Householder,1718.74,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,52.0,94.0,50000+.
-62.0,Not in universe,0,0,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,9181.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1948.13,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,NaN,2,NaN,94.0,50000+.
-NaN,Private,45,NaN,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1142.64,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,50000+.
-78.0,Self-employed-incorporated,9,2,Bachelors degree(BA AB BS),0.0,NaN,Married-civilian spouse present,Manufacturing-durable goods,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,9386.0,0.0,460.0,Joint both 65+,Not in universe,Not in universe,Householder,Householder,3632.17,Nonmover,Nonmover,Nonmover,NaN,Not in universe,NaN,Not in universe,United-States,NaN,NaN,Native- Born in the United States,0,Not in universe,2,50.0,94.0,50000+.
-38.0,Private,34,2,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,250.0,NaN,Not in universe,Not in universe,NaN,Householder,2652.18,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-NaN,Private,23,4,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Manufacturing-nondurable goods,Professional specialty,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,600.0,Joint both under 65,Not in universe,NaN,Householder,Householder,1774.24,Nonmover,NaN,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,England,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-64.0,Private,45,NaN,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,NaN,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1348.23,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,2,NaN,2,52.0,94.0,50000+.
-44.0,Self-employed-not incorporated,42,NaN,NaN,0.0,Not in universe,Married-civilian spouse present,Medical except hospital,Professional specialty,White,All other,Male,NaN,NaN,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1232.4,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-42.0,Private,13,4,High school graduate,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Professional specialty,White,NaN,Male,Not in universe,NaN,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1400.86,Nonmover,Nonmover,Nonmover,Yes,NaN,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,94.0,50000+.
-31.0,Private,33,5,NaN,0.0,Not in universe,Married-civilian spouse present,Retail trade,Professional specialty,White,All other,Female,NaN,Not in universe,Children or Armed Forces,0.0,0.0,900.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,2848.67,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-64.0,Self-employed-not incorporated,40,12,10th grade,0.0,Not in universe,Married-civilian spouse present,NaN,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,3761.78,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,52.0,94.0,50000+.
-55.0,Private,45,15,NaN,0.0,NaN,Married-civilian spouse present,Other professional services,Technicians and related support,White,All other,Female,Not in universe,Not in universe,NaN,0.0,0.0,10000.0,Joint one under 65 & one 65+,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1134.7,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,NaN,England,England,Foreign born- Not a citizen of U S,0,Not in universe,2,52.0,94.0,50000+.
-NaN,State government,43,2,Some college but no degree,0.0,Not in universe,NaN,Education,Executive admin and managerial,Black,All other,Male,Not in universe,Not in universe,Children or Armed Forces,15831.0,0.0,0.0,Single,Not in universe,NaN,Nonfamily householder,Householder,1855.91,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,50000+.
-45.0,State government,48,26,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Public administration,Adm support including clerical,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,10000.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,3465.3,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,NaN,United-States,United-States,United-States,Native- Born in the United States,0, [...]
-56.0,Private,35,16,Bachelors degree(BA AB BS),0.0,NaN,Married-civilian spouse present,Finance insurance and real estate,Sales,White,All other,Male,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Joint both under 65,Not in universe,NaN,Householder,Householder,874.93,Nonmover,Nonmover,Nonmover,Yes,NaN,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,NaN,52.0,94.0,50000+.
-41.0,Private,13,2,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Executive admin and managerial,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,150.0,Joint both under 65,NaN,Not in universe,Householder,Householder,2188.47,NaN,Nonmover,NaN,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,NaN,50000+.
-47.0,Private,NaN,NaN,NaN,0.0,Not in universe,Divorced,Finance insurance and real estate,Sales,White,All other,Male,Not in universe,Not in universe,NaN,27828.0,0.0,0.0,Single,Not in universe,Not in universe,NaN,Householder,2172.64,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-38.0,Federal government,29,NaN,High school graduate,0.0,Not in universe,Never married,Transportation,Adm support including clerical,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,2016.15,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,50000+.
-30.0,Self-employed-incorporated,22,2,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Manufacturing-nondurable goods,Executive admin and managerial,Asian or Pacific Islander,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,1902.0,9.0,Joint both under 65,Not in universe,Not in universe,NaN,Householder,1572.05,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,South Korea,South Korea,South Korea,Foreign born- U S citi [...]
-59.0,Private,9,3,Associates degree-academic program,1250.0,Not in universe,Widowed,Manufacturing-durable goods,Executive admin and managerial,White,All other,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,800.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,3056.61,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,1,Not in universe,2,52.0,94.0,50000+.
-NaN,NaN,NaN,2,11th grade,0.0,Not in universe,Married-civilian spouse present,Personal services except private HH,Executive admin and managerial,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1528.77,Nonmover,NaN,Nonmover,Yes,Not in universe,1.0,Not in universe,El-Salvador,El-Salvador,El-Salvador,Foreign born- Not a citizen of U S,0,Not in universe,2,52.0,94.0,50000+.
-42.0,Self-employed-incorporated,NaN,2,High school graduate,0.0,NaN,Married-civilian spouse present,Business and repair services,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,1977.0,1000.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1820.01,Nonmover,NaN,Nonmover,Yes,Not in universe,3.0,Not in universe,Italy,Italy,NaN,Foreign born- U S citizen by naturalization,0,Not in universe,2,52.0,94.0,50000+.
-NaN,Self-employed-not incorporated,38,33,High school graduate,0.0,Not in universe,Divorced,Business and repair services,Precision production craft & repair,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Single,Not in universe,Not in universe,Nonfamily householder,NaN,266.87,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-49.0,Private,8,3,Associates degree-occup /vocational,0.0,Not in universe,Divorced,Manufacturing-durable goods,Executive admin and managerial,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1517.26,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-43.0,NaN,30,12,Bachelors degree(BA AB BS),790.0,Not in universe,Married-civilian spouse present,Communications,Professional specialty,White,NaN,NaN,No,Not in universe,Children or Armed Forces,0.0,0.0,1000.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1125.93,Nonmover,NaN,Nonmover,Yes,Not in universe,NaN,Not in universe,NaN,United-States,United-States,NaN,0,Not in universe,2,50.0,94.0,50000+.
-62.0,Self-employed-incorporated,40,NaN,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Entertainment,NaN,Black,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,100.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,3130.51,Nonmover,Nonmover,NaN,Yes,Not in universe,2.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Yes,1,52.0,94.0,50000+.
-57.0,Private,4,34,High school graduate,0.0,Not in universe,Married-civilian spouse present,Construction,Precision production craft & repair,Asian or Pacific Islander,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,317.55,NaN,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,NaN,South Korea,South Korea,Foreign born- U S citizen by naturalization,0,Not in universe,2,52.0, [...]
-NaN,Self-employed-incorporated,45,11,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,2600.0,Joint both under 65,NaN,Not in universe,Householder,Householder,3440.05,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not [...]
-73.0,Not in universe,0,0,7th and 8th grade,0.0,Not in universe,Widowed,Not in universe or children,Not in universe,NaN,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,NaN,Nonfamily householder,NaN,3010.72,NaN,Nonmover,Nonmover,Yes,NaN,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,50000+.
-58.0,NaN,43,NaN,Doctorate degree(PhD EdD),0.0,Not in universe,Married-civilian spouse present,Education,NaN,White,All other,Male,Not in universe,Not in universe,NaN,0.0,0.0,10000.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,984.69,NaN,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,2,Not in universe,2,34.0,94.0,50000+.
-39.0,Private,NaN,39,High school graduate,0.0,Not in universe,Married-civilian spouse present,Mining,Transportation and material moving,White,All other,Male,Not in universe,NaN,Children or Armed Forces,7298.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,326.65,Nonmover,Nonmover,NaN,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,50000+.
-42.0,Private,12,4,Doctorate degree(PhD EdD),0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Professional specialty,Asian or Pacific Islander,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,150.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1724.94,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,South Korea,NaN,South Korea,Foreign born- U S citizen by naturalization,0,Not in univer [...]
-75.0,Not in universe,0,0,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,NaN,Not in universe,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both 65+,NaN,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,Nonmover,NaN,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,0.0,94.0,50000+.
-47.0,Local government,43,12,NaN,0.0,NaN,Married-civilian spouse present,Education,Professional specialty,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,200.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,NaN,1632.98,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,40.0,94.0,50000+.
-39.0,Private,NaN,2,Doctorate degree(PhD EdD),0.0,Not in universe,Married-civilian spouse present,NaN,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-64.0,Local government,NaN,33,High school graduate,1570.0,Not in universe,Divorced,Construction,Precision production craft & repair,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,4000.0,Single,Not in universe,NaN,Nonfamily householder,Householder,1882.47,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-44.0,Self-employed-incorporated,38,2,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Business and repair services,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,1260.24,Nonmover,Nonmover,Nonmover,Yes,Not in universe,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not i [...]
-43.0,NaN,47,26,Some college but no degree,0.0,Not in universe,Divorced,Public administration,Adm support including clerical,Black,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1393.32,Nonmover,NaN,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,No,1,52.0,94.0,50000+.
-56.0,Private,4,2,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Construction,Executive admin and managerial,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,3500.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,3579.52,Nonmover,Nonmover,Nonmover,NaN,Not in universe,3.0,Not in universe,England,England,England,Foreign born- U S citizen by naturalization,0,Not in universe,2,52.0,94.0,50000+.
-46.0,Local government,43,10,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Education,Professional specialty,White,All other,Female,Yes,NaN,Children or Armed Forces,0.0,0.0,14690.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1435.68,NaN,Nonmover,Nonmover,NaN,Not in universe,4.0,NaN,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,40.0,94.0,50000+.
-57.0,Private,34,NaN,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,1500.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1081.89,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,NaN,NaN,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,50000+.
-72.0,Not in universe,0,0,Bachelors degree(BA AB BS),0.0,NaN,Married-civilian spouse present,Not in universe or children,NaN,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,30000.0,Joint both 65+,Not in universe,Not in universe,Spouse of householder,NaN,3805.35,Nonmover,Nonmover,Nonmover,NaN,Not in universe,0.0,Not in universe,England,United-States,United-States,Native- Born in the United States,0,NaN,NaN,0.0,94.0,50000+.
-32.0,Private,37,NaN,Some college but no degree,0.0,Not in universe,Divorced,Business and repair services,NaN,White,All other,Male,No,Not in universe,Children or Armed Forces,NaN,0.0,8000.0,Single,West,New Hampshire,Nonfamily householder,NaN,1784.76,MSA to MSA,NaN,Different state in West,No,Yes,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-90.0,Private,35,3,Associates degree-academic program,0.0,Not in universe,Never married,Finance insurance and real estate,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,6097.0,0.0,400.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,2683.23,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,NaN,NaN,NaN,2,52.0,94.0,50000+.
-48.0,Private,15,37,Some college but no degree,0.0,Not in universe,Divorced,Manufacturing-durable goods,Machine operators assmblrs & inspctrs,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1504.18,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,NaN,52.0,94.0,50000+.
-41.0,Private,35,15,Bachelors degree(BA AB BS),0.0,NaN,NaN,Finance insurance and real estate,Technicians and related support,White,All other,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,15000.0,Head of household,Midwest,Oklahoma,Householder,Householder,1732.12,NaN,Same county,Same county,No,No,6.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,NaN,50000+.
-25.0,Private,34,17,NaN,0.0,Not in universe,Never married,Finance insurance and real estate,Sales,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,NaN,Child 18 or older,1476.27,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,NaN,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-37.0,Federal government,47,15,Masters degree(MA MS MEng MEd MSW MBA),0.0,NaN,Married-civilian spouse present,Public administration,Technicians and related support,White,All other,Male,Not in universe,Not in universe,NaN,0.0,0.0,296.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2852.87,Nonmover,NaN,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-52.0,Private,29,2,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Transportation,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,1902.0,150.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1837.06,Nonmover,Nonmover,Nonmover,Yes,NaN,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,NaN,94.0,50000+.
-47.0,Private,7,39,High school graduate,0.0,NaN,Married-civilian spouse present,Manufacturing-durable goods,Transportation and material moving,White,All other,Male,Not in universe,NaN,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Midwest,Iowa,Householder,Householder,4817.03,MSA to MSA,Same county,Same county,No,No,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,50000+.
-NaN,Private,41,8,NaN,0.0,Not in universe,Married-civilian spouse present,Hospital services,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,1612.84,Nonmover,Nonmover,Nonmover,Yes,Not in universe,5.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,NaN,52.0,94.0,50000+.
-48.0,Local government,43,2,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Education,Executive admin and managerial,Black,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,NaN,Spouse of householder,NaN,2559.05,Nonmover,NaN,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,1,Not in universe,2,52.0,94.0 [...]
-43.0,Self-employed-incorporated,32,18,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Wholesale trade,Sales,White,All other,NaN,Not in universe,Not in universe,NaN,15024.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-61.0,Self-employed-not incorporated,45,11,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,908.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2004.82,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States [...]
-67.0,Private,34,2,High school graduate,0.0,Not in universe,Widowed,Finance insurance and real estate,Executive admin and managerial,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Householder,Householder,1539.89,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,Ireland,Ireland,NaN,Native- Born in the United States,0,NaN,NaN,52.0,94.0,50000+.
-31.0,NaN,35,17,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Finance insurance and real estate,Sales,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2011.14,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,Italy,United-States,Native- Born in the United States,0,NaN,2,52.0,NaN,50000+.
-68.0,Not in universe,0,0,NaN,0.0,Not in universe,Married-civilian spouse present,NaN,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,15000.0,Joint both 65+,Not in universe,Not in universe,Spouse of householder,NaN,998.8,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-45.0,State government,43,6,Masters degree(MA MS MEng MEd MSW MBA),NaN,Not in universe,Married-civilian spouse present,Education,Professional specialty,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,NaN,Householder,2278.25,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,India,India,NaN,Foreign born- Not a citizen of U S,NaN,Not in universe,2,52.0,94.0,50000+.
-55.0,Private,2,44,High school graduate,0.0,Not in universe,Married-civilian spouse present,Agriculture,Farming forestry and fishing,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,Not in universe,NaN,Householder,Householder,NaN,Nonmover,Nonmover,NaN,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-47.0,NaN,43,10,Bachelors degree(BA AB BS),NaN,Not in universe,Married-civilian spouse present,Education,Professional specialty,White,All other,Female,Not in universe,NaN,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1138.19,Nonmover,NaN,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-32.0,NaN,45,11,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,Not in universe,NaN,Householder,Householder,1192.59,NaN,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,50000+.
-62.0,Local government,31,2,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Utilities and sanitary services,Executive admin and managerial,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,NaN,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,NaN,Yes,NaN,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-49.0,Self-employed-incorporated,11,2,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,15024.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1622.72,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United Sta [...]
-46.0,Local government,NaN,2,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Education,Executive admin and managerial,White,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,500.0,Joint both under 65,Not in universe,Not in universe,NaN,Spouse of householder,1616.82,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52. [...]
-49.0,Private,NaN,NaN,Bachelors degree(BA AB BS),0.0,Not in universe,Never married,Hospital services,Technicians and related support,White,All other,NaN,Not in universe,Not in universe,Children or Armed Forces,0.0,1564.0,0.0,Single,NaN,Not in universe,Secondary individual,Nonrelative of householder,1007.3,Nonmover,Nonmover,NaN,Yes,NaN,6.0,Not in universe,England,Canada,Canada,Foreign born- U S citizen by naturalization,0,NaN,2,52.0,94.0,50000+.
-40.0,Private,19,NaN,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Manufacturing-nondurable goods,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,NaN,Joint both under 65,West,California,Householder,Householder,835.19,Not identifiable,Different division same region,Different state in West,No,No,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0, [...]
-46.0,Private,30,14,High school graduate,NaN,Not in universe,Married-civilian spouse present,Communications,Technicians and related support,Black,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1555.77,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-38.0,Private,24,36,Bachelors degree(BA AB BS),0.0,NaN,Married-civilian spouse present,Manufacturing-nondurable goods,Machine operators assmblrs & inspctrs,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,NaN,Not in universe,Householder,Householder,1121.25,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,NaN,United-States,NaN,0,Not in universe,2,NaN,94.0,50000+.
-50.0,Private,NaN,2,NaN,0.0,Not in universe,Married-civilian spouse present,Business and repair services,Executive admin and managerial,White,All other,NaN,Not in universe,Not in universe,Children or Armed Forces,15024.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2144.23,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-48.0,Self-employed-incorporated,9,2,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,NaN,Not in universe,Householder,Householder,439.33,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,94. [...]
-34.0,Private,12,26,NaN,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,NaN,Not in universe,Householder,Householder,3616.34,Nonmover,Nonmover,Nonmover,NaN,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,52.0,94.0,50000+.
-46.0,Private,41,8,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Hospital services,Professional specialty,Black,Other Spanish,Female,Not in universe,Not in universe,Children or Armed Forces,NaN,2415.0,100.0,Joint both under 65,Not in universe,NaN,Householder,Householder,988.57,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,Haiti,Haiti,Haiti,Foreign born- U S citizen by naturalization,2,Not in universe,2,52.0,94.0,50000+.
-44.0,Private,32,18,NaN,0.0,Not in universe,Married-civilian spouse present,Wholesale trade,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,987.24,Nonmover,Nonmover,Nonmover,Yes,NaN,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-43.0,Private,11,2,Bachelors degree(BA AB BS),0.0,Not in universe,NaN,Manufacturing-durable goods,NaN,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,NaN,650.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1683.64,Nonmover,Nonmover,Nonmover,NaN,Not in universe,5.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-45.0,Private,32,18,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Wholesale trade,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,NaN,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-45.0,Private,31,26,High school graduate,0.0,Not in universe,Married-civilian spouse present,Utilities and sanitary services,Adm support including clerical,White,All other,Male,Not in universe,Not in universe,NaN,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1000.26,Nonmover,Nonmover,Nonmover,Yes,NaN,1.0,Not in universe,NaN,NaN,United-States,NaN,NaN,Not in universe,2,NaN,NaN,50000+.
-36.0,Self-employed-incorporated,45,12,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,White,All other,Male,NaN,Not in universe,Children or Armed Forces,NaN,0.0,1.0,Joint both under 65,Not in universe,Not in universe,NaN,Spouse of householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,50000+.
-32.0,Self-employed-not incorporated,39,12,NaN,0.0,Not in universe,Married-civilian spouse present,Personal services except private HH,Professional specialty,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,7688.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1123.1,Nonmover,NaN,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0, [...]
-NaN,Not in universe,0,0,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Never married,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,300.0,Single,Not in universe,Not in universe,NaN,Householder,1296.42,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,0,Not in universe,NaN,20.0,94.0,50000+.
-41.0,Private,31,35,Some college but no degree,0.0,Not in universe,Married-civilian spouse present,Utilities and sanitary services,Precision production craft & repair,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,2000.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,225.4,Nonmover,Nonmover,Nonmover,Yes,Not in universe,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the [...]
-73.0,NaN,NaN,43,High school graduate,NaN,Not in universe,NaN,NaN,Farming forestry and fishing,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint one under 65 & one 65+,Not in universe,Not in universe,Householder,Householder,2458.01,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,NaN,94.0,50000+.
-58.0,Private,33,19,High school graduate,0.0,Not in universe,Widowed,Retail trade,NaN,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Secondary individual,Nonrelative of householder,1612.0,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,51.0,94.0,50000+.
-45.0,NaN,13,37,High school graduate,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Machine operators assmblrs & inspctrs,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,NaN,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1558.0,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in univers [...]
-58.0,Private,32,18,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,NaN,NaN,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,NaN,1600.0,Joint both under 65,Not in universe,Not in universe,Householder,NaN,1281.98,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-60.0,State government,50,3,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Public administration,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,300.0,Joint both under 65,NaN,Not in universe,Householder,NaN,1139.03,Nonmover,NaN,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,1,Not in universe,2,52.0,94.0,50000+.
-48.0,Private,37,2,Bachelors degree(BA AB BS),0.0,NaN,Married-civilian spouse present,Business and repair services,Executive admin and managerial,White,All other,Male,NaN,Not in universe,Children or Armed Forces,7688.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,831.05,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,NaN,50000+.
-42.0,Self-employed-incorporated,33,16,NaN,0.0,NaN,Married-civilian spouse present,Retail trade,Sales,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,10.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,2013.94,Nonmover,Nonmover,Nonmover,Yes,NaN,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-58.0,Self-employed-not incorporated,45,4,NaN,0.0,Not in universe,Married-civilian spouse present,Other professional services,Professional specialty,White,All other,NaN,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1406.58,Nonmover,Nonmover,Nonmover,Yes,NaN,NaN,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,30.0,94.0,50000+.
-40.0,NaN,NaN,8,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Hospital services,Professional specialty,White,All other,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,1500.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,NaN,NaN,Nonmover,Nonmover,Yes,NaN,5.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,50.0,94.0,50000+.
-30.0,Private,11,4,Doctorate degree(PhD EdD),0.0,Not in universe,NaN,Manufacturing-durable goods,Professional specialty,Asian or Pacific Islander,All other,Male,No,Not in universe,Children or Armed Forces,0.0,NaN,500.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2756.21,Nonmover,Nonmover,Nonmover,Yes,NaN,6.0,Not in universe,China,China,China,Foreign born- Not a citizen of U S,0,Not in universe,2,52.0,94.0,50000+.
-45.0,Not in universe,0,0,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Not in universe or children,Not in universe,Black,All other,Female,Not in universe,Not in universe,Children or Armed Forces,NaN,0.0,0.0,Joint both under 65,NaN,Not in universe,Spouse of householder,Spouse of householder,2146.09,NaN,Nonmover,Nonmover,NaN,NaN,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,NaN,2,48.0,94.0,50000+.
-23.0,Private,33,29,Some college but no degree,0.0,Not in universe,Separated,NaN,NaN,White,All other,Male,Not in universe,Not in universe,NaN,0.0,2559.0,0.0,Single,West,Abroad,Nonfamily householder,Householder,413.01,Not identifiable,NaN,Same county,No,No,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,44.0,NaN,50000+.
-33.0,Local government,47,28,Associates degree-occup /vocational,0.0,Not in universe,Married-civilian spouse present,Public administration,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,West,West Virginia,Householder,Householder,4216.99,MSA to MSA,Same county,Same county,No,No,4.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,NaN,52.0,94.0,50000+.
-85.0,Not in universe,0,0,Bachelors degree(BA AB BS),0.0,Not in universe,Widowed,Not in universe or children,Not in universe,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,10898.0,NaN,Not in universe,Not in universe,Householder,Householder,879.82,Nonmover,Nonmover,Nonmover,Yes,Not in universe,0.0,Not in universe,United-States,Ireland,United-States,Native- Born in the United States,0,Not in universe,2,0.0,94.0,50000+.
-32.0,Private,33,16,Associates degree-occup /vocational,0.0,NaN,Divorced,Retail trade,Sales,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,South,NaN,Nonfamily householder,Householder,1771.06,MSA to MSA,Same county,NaN,No,Yes,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,50000+.
-51.0,Not in universe,0,0,Doctorate degree(PhD EdD),0.0,Not in universe,NaN,Not in universe or children,NaN,White,All other,Male,Not in universe,Not in universe,NaN,NaN,2824.0,40000.0,Single,Not in universe,Not in universe,NaN,Householder,1837.87,Nonmover,Nonmover,Nonmover,Yes,Not in universe,3.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-32.0,Self-employed-incorporated,32,16,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,NaN,Sales,NaN,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1215.77,Nonmover,NaN,NaN,Yes,Not in universe,1.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,50.0,94.0,50000+.
-47.0,Federal government,29,25,Associates degree-academic program,1700.0,Not in universe,Married-civilian spouse present,NaN,Adm support including clerical,Black,NaN,Male,Yes,NaN,Children or Armed Forces,0.0,0.0,900.0,NaN,Not in universe,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-54.0,Federal government,49,NaN,Some college but no degree,1700.0,Not in universe,Married-civilian spouse present,Public administration,Precision production craft & repair,White,All other,Male,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,NaN,Not in universe,Householder,NaN,860.04,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,NaN,United-States,United-States,Native- Born in the United States,2,Not in universe,2,52.0,94.0,50000+.
-NaN,Private,14,36,High school graduate,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Machine operators assmblrs & inspctrs,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,100.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,NaN,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,United-States,Native- Born in the United States,1,Not in universe,2,52.0,94.0,50000+.
-27.0,NaN,47,28,Some college but no degree,NaN,Not in universe,Never married,Public administration,Protective services,White,All other,Female,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,NaN,Child 18+ never marr Not in a subfamily,Child 18 or older,2315.35,Nonmover,Nonmover,Nonmover,Yes,NaN,6.0,Not in universe,United-States,United-States,NaN,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-66.0,State government,48,1,Prof school degree (MD DDS DVM LLB JD),0.0,NaN,Married-civilian spouse present,Public administration,Executive admin and managerial,Black,All other,Male,Not in universe,Not in universe,Children or Armed Forces,20051.0,0.0,0.0,Joint one under 65 & one 65+,Not in universe,Not in universe,Householder,Householder,1497.69,Nonmover,Nonmover,Nonmover,NaN,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,N [...]
-NaN,Private,33,19,Associates degree-occup /vocational,500.0,NaN,Married-civilian spouse present,Retail trade,Sales,White,All other,Female,No,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,NaN,Spouse of householder,Spouse of householder,2809.29,NaN,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,United-States,NaN,0,Not in universe,2,52.0,94.0,50000+.
-48.0,Private,13,33,High school graduate,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Precision production craft & repair,White,All other,Male,NaN,Not in universe,Children or Armed Forces,0.0,0.0,500.0,NaN,Not in universe,Not in universe,Householder,Householder,1880.65,Nonmover,Nonmover,Nonmover,NaN,Not in universe,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-37.0,Private,45,2,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Other professional services,NaN,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,17500.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,1387.7,Nonmover,NaN,NaN,Yes,NaN,6.0,NaN,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,NaN,94.0,50000+.
-38.0,Private,11,5,NaN,0.0,Not in universe,Never married,NaN,Professional specialty,NaN,All other,NaN,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1122.45,Nonmover,Nonmover,Nonmover,NaN,NaN,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,0,Not in universe,2,52.0,NaN,50000+.
-45.0,Federal government,50,NaN,Bachelors degree(BA AB BS),0.0,Not in universe,Married-civilian spouse present,Public administration,Executive admin and managerial,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,NaN,0.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,797.72,Nonmover,Nonmover,Nonmover,Yes,Not in universe,2.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,1,Not in universe,2 [...]
-49.0,Private,9,2,Masters degree(MA MS MEng MEd MSW MBA),0.0,NaN,NaN,Manufacturing-durable goods,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,1755.0,0.0,Head of household,Northeast,Connecticut,Householder,Householder,1708.18,Not identifiable,Same county,Same county,No,No,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,NaN,Not in universe,2,52.0,94.0,50000+.
-28.0,Private,37,17,Bachelors degree(BA AB BS),NaN,Not in universe,Married-civilian spouse present,Business and repair services,Sales,White,NaN,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,2500.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,NaN,Nonmover,NaN,Nonmover,Yes,NaN,6.0,Not in universe,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,52.0,94.0,50000+.
-49.0,Private,25,4,NaN,0.0,Not in universe,Married-civilian spouse present,Manufacturing-nondurable goods,Professional specialty,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,500.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,2012.55,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,United-States,Native- Born in the United States,2,Not in universe,2,52.0,94.0,50000+.
-57.0,Federal government,35,11,Prof school degree (MD DDS DVM LLB JD),0.0,Not in universe,Divorced,Finance insurance and real estate,NaN,White,All other,Female,No,Not in universe,Children or Armed Forces,0.0,0.0,3000.0,Single,NaN,Not in universe,Nonfamily householder,Householder,694.05,Nonmover,Nonmover,Nonmover,Yes,NaN,NaN,Not in universe,United-States,NaN,NaN,Native- Born in the United States,0,Not in universe,2,52.0,94.0,50000+.
-45.0,Local government,43,10,Masters degree(MA MS MEng MEd MSW MBA),0.0,Not in universe,Married-civilian spouse present,Education,Professional specialty,White,All other,Male,Yes,Not in universe,Children or Armed Forces,0.0,0.0,5200.0,NaN,Not in universe,NaN,Householder,Householder,604.84,NaN,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,NaN,Native- Born in the United States,2,Not in universe,2,52.0,94.0,50000+.
-47.0,Federal government,12,3,Associates degree-academic program,0.0,Not in universe,Married-civilian spouse present,Manufacturing-durable goods,Executive admin and managerial,White,All other,Male,Not in universe,Not in universe,Children or Armed Forces,15024.0,0.0,50.0,Joint both under 65,Not in universe,Not in universe,Householder,Householder,1767.91,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,NaN,United-States,United-States,United-States,NaN,0,Not in universe,NaN,52.0,94.0,50000+.
-48.0,Self-employed-not incorporated,35,2,NaN,0.0,Not in universe,Married-spouse absent,Finance insurance and real estate,Executive admin and managerial,Black,All other,Male,Not in universe,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Single,Not in universe,Not in universe,Nonfamily householder,Householder,1089.62,Nonmover,Nonmover,Nonmover,Yes,Not in universe,1.0,Not in universe,Jamaica,Cuba,Jamaica,Foreign born- U S citizen by naturalization,0,Not in universe,2,50.0,94.0,50000+.
47.0,NaN,43,12,Doctorate degree(PhD EdD),0.0,Not in universe,Married-civilian spouse present,Education,Professional specialty,White,All other,NaN,No,Not in universe,Children or Armed Forces,0.0,0.0,0.0,Joint both under 65,Not in universe,Not in universe,Spouse of householder,Spouse of householder,2625.77,Nonmover,Nonmover,Nonmover,Yes,Not in universe,6.0,Not in universe,United-States,NaN,United-States,Native- Born in the United States,2,Not in universe,2,52.0,94.0,50000+.
\ No newline at end of file
diff --git a/src/test/scripts/functions/builtin/handsOffClustering.dml b/src/test/scripts/functions/builtin/handsOffClustering.dml
new file mode 100644
index 0000000..7b816d9
--- /dev/null
+++ b/src/test/scripts/functions/builtin/handsOffClustering.dml
@@ -0,0 +1,149 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+# Generate the logical pipelines for data cleaning
+
+source("scripts/pipelines/scripts/utils.dml") as utils;
+
+# read the inputs
+F = read($dirtyData, data_type="frame", format="csv", header=TRUE,
+ naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
+
+metaInfo = read($metaData, data_type="frame", format="csv", header=FALSE);
+primitives = read($primitives, data_type = "frame", format="csv", header= TRUE)
+param = read($parameters, data_type = "frame", format="csv", header= TRUE)
+topK = $topk
+resources = $rv
+sample=$sample
+
+if(nrow(metaInfo) < 2)
+ stop("incomplete meta info")
+
+metaInfo = metaInfo[, 2:ncol(metaInfo)-1]
+
+[topKPipelines, topKHyperParams, topKScores, features, dirtyScore] = handsOffCleaning(F, metaInfo, primitives, param,
+ matrix("4 0.7 1", rows=1, cols=3), "evalClassification", as.matrix("0"), 5, 20, sample, TRUE)
+
+print("this is accuracies "+toString(topKScores))
+result = dirtyScore < as.scalar(topKScores[1, 1])
+write(result, $O)
+
+
+
+# UDF for evaluation
+# choice of parameters provided by API, X, Y, clone_X, evalFunHp (hyper-param), trainML (boolean for optimizing hp internally or passed by externally )
+evalClassification = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] Xorig, List[Unknown] metaList,
+ Matrix[Double] evalFunHp, Integer trainML=0)
+
+return(Matrix[Double] output)
+{
+ cv = 2
+ mask = as.matrix(metaList['mask'])
+
+ if(max(Y) == min(Y)) {
+ print("Y contains only one class")
+ accuracy = as.double(0)
+ }
+ else {
+ if(trainML == 1)
+ {
+ # do the gridsearch for hyper-parameters
+ params = list("icpt", "reg", "tol", "maxii")
+ paramRanges = list(seq(0, 2, 1), 10^seq(1,-4), 10^seq(1,-6), 10^seq(1,3));
+
+ if(sum(mask) > 0)
+ X = utils::dummycoding(replace(target = X, pattern = NaN, replacement=0), mask)
+ trainArgs = list(X=X, Y=Y, icpt=-1, reg=-1, tol=-1, maxi=100, maxii=-1, verbose=FALSE);
+ [B1, opt] = gridSearch(X=X, y=Y, train="multiLogReg", predict="W", numB=ncol(X)+1, cv=TRUE, cvk=cv,
+ params=params, paramValues=paramRanges, trainArgs=trainArgs, verbose=TRUE);
+ evalFunHp = as.matrix(opt)
+ }
+
+ # do the k = 3 cross validations
+ # evalFunHpM = as.matrix(evalFunHp)
+ [accuracyMatrix] = crossV(X, Y, cv, evalFunHp, FALSE)
+ accuracyMatrix = removeEmpty(target=accuracyMatrix, margin="rows")
+ score = mean(accuracyMatrix)
+ print(cv +" validation accuracy "+score)
+ }
+ output = cbind(as.matrix(score), evalFunHp)
+
+}
+
+# # ######################################################################
+# # # # Function for cross validation using hold out method
+# # # # Inputs: The input dataset X, Y and the value of k validation, mask of the
+# # # # dataset for OHE of categorical columns, vector of ML hyper-parameters identified
+# # # # via gridsearch and a boolean value of (un)weighted accuracy.
+# # # # Output: It return a matrix having the accuracy of each fold.
+# # ######################################################################
+
+crossV = function(Matrix[double] X, Matrix[double] y, Integer k, Matrix[Double] MLhp, Boolean isWeighted)
+return (Matrix[Double] accuracyMatrix)
+{
+ accuracyMatrix = matrix(0, k, 1)
+ print("ML HP in CV "+toString(MLhp))
+ dataList = list()
+ testL = list()
+ data = order(target = cbind(y, X), by = 1, decreasing=FALSE, index.return=FALSE)
+ classes = table(data[, 1], 1)
+ ins_per_fold = classes/k
+ start_fold = matrix(1, rows=nrow(ins_per_fold), cols=1)
+ fold_idxes = cbind(start_fold, ins_per_fold)
+
+ start_i = 0; end_i = 0; idx_fold = 1;;
+ for(i in 1:k)
+ {
+ fold_i = matrix(0, 0, ncol(data))
+ start=0; end=0;
+ for(j in 1:nrow(classes))
+ {
+ idx = as.scalar(classes[j, 1])
+ start = end + 1;
+ end = end + idx
+ class_j = data[start:end, ]
+ start_i = as.scalar(fold_idxes[j, 1]);
+ end_i = as.scalar(fold_idxes[j, 2])
+ fold_i = rbind(fold_i, class_j[start_i:end_i, ])
+ }
+ dataList = append(dataList, fold_i)
+ fold_idxes[, 1] = fold_idxes[, 2] + 1
+ fold_idxes[, 2] += ins_per_fold
+ }
+
+ for(i in seq(1,k))
+ {
+ [trainList, hold_out] = remove(dataList, i)
+ trainset = rbind(trainList)
+ testset = as.matrix(hold_out)
+ trainX = trainset[, 2:ncol(trainset)]
+ trainy = trainset[, 1]
+ testX = testset[, 2:ncol(testset)]
+ testy = testset[, 1]
+ beta = multiLogReg(X=trainX, Y=trainy, icpt=as.scalar(MLhp[1,1]), reg=as.scalar(MLhp[1,2]), tol=as.scalar(MLhp[1,3]),
+ maxi=as.scalar(MLhp[1,4]), maxii=50, verbose=FALSE);
+ [prob, yhat, a] = multiLogRegPredict(testX, beta, testy, FALSE)
+ accuracy = getAccuracy(testy, yhat, isWeighted)
+ accuracyMatrix[i] = accuracy
+ }
+
+}
+
+
\ No newline at end of file
diff --git a/src/test/scripts/functions/pipelines/compareAccuracy.dml b/src/test/scripts/functions/pipelines/compareAccuracy.dml
deleted file mode 100644
index ba8df06..0000000
--- a/src/test/scripts/functions/pipelines/compareAccuracy.dml
+++ /dev/null
@@ -1,135 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-##################################################################################################################
-# This script will read the dirty and clean data, then it will apply the best pipeline on dirty data
-# and then will classify both cleaned dataset and check if the cleaned dataset is performing same as original dataset
-# in terms of classification accuracy
-
-# Vocab = original data -> dataset without any noise, the original version with ground truths
- # cleaned data -> dirty dataset cleaned by pipeline
-# read the items
-# dirty dataset F
-# clean dataset O
-# metadata (schema and mask)
-# best k pipelines and hyperparameters generated by previous script mainScript.dml
-
-# do the initial preprocessing like dropping invalid values so that pipeline could fix them
-# then recode the data to bring it into matrix format
-# then construct the hyper-parameters list and call the executePipeline() on the dirty dataset
-# for the comparison OHE the original dataset, there is no need to OHE the cleaned dataset because cleaning pipeline
-# has a primitive for this
-# Call the multilogReg on both of the datasets and compare accuracy on k=3 cross validation
-######################################################################################################################
-
-source("scripts/pipelines/scripts/utils.dml") as utils;
-source("scripts/builtin/bandit.dml") as bandit;
-
-
-F = read($1, data_type="frame", format="csv", header=FALSE,
- naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
-# read original clean data
-O = read($2, data_type="frame", format="csv", header=FALSE,
- naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
-
-metaInfo = read($3, data_type="frame", format="csv", header=FALSE);
-input = $4
-pip = read(input+"pipelines.csv", data_type="frame", format="csv", header=FALSE);
-hp = read(input+"hyperparams.csv", data_type="matrix", format="csv", header=FALSE);
-
-getSchema = metaInfo[1, 2:ncol(metaInfo)]
-getMask = as.matrix(metaInfo[2, 2:ncol(metaInfo)])
-getFdMask = as.matrix(metaInfo[3, 2:ncol(metaInfo)]) # columns of interest for FD computation
-
-# # dropInvalid function will remove the values which are not the part
-# # of the column data type
-X = utils::stringProcessing(F, getMask, getSchema)
-
-# encode the categorical data
-if(sum(getMask) > 0)
-{
- # recode the dirty data, always recode the label
- index = utils::vectorToCsv(getMask)
- jspecR = "{ids:true, recode:["+index+"]}"
- [eX, X_meta] = transformencode(target=X, spec=jspecR);
-}
-# if no categorical value exist then just cast the frame into matrix
-else
- eX = as.matrix(X)
-
-# extract the class label
-eY = eX[, ncol(eX)]
-eX = eX[, 1:ncol(eX) - 1]
-
-
-# strip the mask of class label
-getMask = getMask[, 1:ncol(getMask) - 1] # strip the mask of class label
-getSchema = getSchema[, 1:ncol(getSchema) - 1] # strip the mask of class label
-getFdMask = getFdMask[, 1:ncol(getFdMask) - 1] # strip the mask of class label
-
-
-FD = discoverFD(X=replace(target=eX, pattern=NaN, replacement=1), Mask=getFdMask, threshold=0.8)
-FD = (diag(matrix(1, rows=nrow(FD), cols=1)) ==0) * FD
-FD = FD > 0
-# construct the parameter list for best hyper-parameters if the oversampling technique is part of
-# pipeline then take it out because oversampling is not applied on test dataset
-# this condition is unnecessary here in this case because the input dataset is balanced and
-# instead of diving the dataset into train/test I am doing cross validations
-
-no_of_param = as.scalar(hp[1, 1]) + 1
-hp_width= hp[1, 2:no_of_param]
-hp_matrix = matrix(hp_width, rows=ncol(pip), cols=ncol(hp_width)/ncol(pip))
-
-# # clean using best pipeline
-[cX , cY] = executePipeline(pip[1], eX, eY, getMask, FD, hp_matrix, 5, FALSE)
-
-if(sum(getMask) > 0)
-{
- # dummycode the original data
- index = utils::vectorToCsv(getMask)
- jspec = "{ids:true, dummycode:["+index+"], recode:["+ ncol(O)+"] }"
- [oX, X_meta] = transformencode(target=O, spec=jspec);
-}
-# if no categorical value exist then just cast the frame into matrix
-else
- oX = as.matrix(O)
-
-# # extract the class label of original data
-oY = oX[, ncol(oX)]
-oX = oX[, 1:ncol(oX) - 1]
-
-
-
-# do the k cross validations for original clean data
-accuracyMatrix = bandit::crossV(oX, oY, 3, as.matrix(0), matrix("0 0.000001 100", rows=1, cols=3), TRUE)
-accuracyMatrix = removeEmpty(target=accuracyMatrix, margin="rows")
-oAcc = mean(accuracyMatrix)
-
-# do the k cross validations for cleaned data
-accuracyMatrix = bandit::crossV(cX, cY, 3, as.matrix(0), matrix("0 0.000001 100", rows=1, cols=3), TRUE)
-accuracyMatrix = removeEmpty(target=accuracyMatrix, margin="rows")
-cAcc = mean(accuracyMatrix)
-tol = 1
-
-results = (tol+cAcc) >= oAcc
-print("clean accuracy "+cAcc)
-print("original accuracy "+oAcc)
-write(results, $5, format = "text")
-
diff --git a/src/test/scripts/functions/pipelines/intermediates/hyperparams.csv b/src/test/scripts/functions/pipelines/intermediates/hyperparams.csv
deleted file mode 100644
index a4729a4..0000000
--- a/src/test/scripts/functions/pipelines/intermediates/hyperparams.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-12.0,0,1.0,0,0,0,2.0,0,1.0,0,0,0,2.0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-12.0,0,1.0,0,0,0,2.0,0,1.0,0,0,0,2.0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
diff --git a/src/test/scripts/functions/pipelines/intermediates/logical.csv b/src/test/scripts/functions/pipelines/intermediates/logical.csv
deleted file mode 100644
index 20d1e3c..0000000
--- a/src/test/scripts/functions/pipelines/intermediates/logical.csv
+++ /dev/null
@@ -1 +0,0 @@
-MVI,DUMMY,DIM
diff --git a/src/test/scripts/functions/pipelines/intermediates/pipelines.csv b/src/test/scripts/functions/pipelines/intermediates/pipelines.csv
deleted file mode 100644
index f20554a..0000000
--- a/src/test/scripts/functions/pipelines/intermediates/pipelines.csv
+++ /dev/null
@@ -1,2 +0,0 @@
-imputeByMean,dummycoding
-imputeByMedian,dummycoding
diff --git a/src/test/scripts/functions/pipelines/mainScript.dml b/src/test/scripts/functions/pipelines/mainScript.dml
deleted file mode 100644
index 7883e67..0000000
--- a/src/test/scripts/functions/pipelines/mainScript.dml
+++ /dev/null
@@ -1,371 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-# Generate the logical pipelines for data cleaning
-
-source("scripts/pipelines/scripts/utils.dml") as utils;
-source("scripts/pipelines/scripts/logicalFunc.dml") as logical;
-source("scripts/pipelines/scripts/gridsearchMLR.dml") as gs;
-
-# read the inputs
-F = read($dirtyData, data_type="frame", format="csv", header=TRUE,
- naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
-
-metaInfo = read($metaData, data_type="frame", format="csv", header=FALSE);
-primitives = read($primitives, data_type = "frame", format="csv", header= TRUE)
-param = read($parameters, data_type = "frame", format="csv", header= TRUE)
-sample = $sampleSize
-topK = $topk
-resources = $rv
-crossValidations = $cv
-weightedAccuracy = $weighted # accuracy flag
-targetApplicaton = $target # accuracy flag
-cleanData = read($cleanData, data_type="frame", format="csv", header=TRUE,
- naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
-
-
-if(nrow(metaInfo) < 2)
- stop("incomplete meta info")
-
- # Do the initial cleaning
-
-
-getSchema = metaInfo[1, 2:ncol(metaInfo)]
-getMask = as.matrix(metaInfo[2, 2:ncol(metaInfo)])
-getFdMask = as.matrix(metaInfo[3, 2:ncol(metaInfo)]) # columns of interest for FD computation
-
-# 1. dropInvalid function will remove the values which are not the part
-# of the column data type
-
-# X = dropInvalidType(F, getSchema)
- X = F
-
-# 2. encode the categorical data
-if(sum(getMask) > 0)
-{
- # always recode the label
- index = utils::vectorToCsv(getMask)
- jspecR = "{ids:true, recode:["+index+"]}"
- if(targetApplicaton == "compare") {
- [eX, X_meta] = transformencode(target=rbind(cleanData, X), spec=jspecR);
- cleanX = eX[1:nrow(cleanData)]
- eX = eX[nrow(cleanData)+1:nrow(eX)]
- }
- else
- [eX, X_meta] = transformencode(target=X, spec=jspecR);
- # change the schema to reflect the encoded values
- getSchema = map(getSchema, "x->x.replace(\"STRING\", \"INT64\")")
- getSchema = map(getSchema, "x->x.replace(\"BOOLEAN\", \"INT64\")")
-
-
-}
-# if no categorical value exist then just cast the frame into matrix
-else
- eX = as.matrix(X)
-
-
-
-# 3. extract the class label
-if(targetApplicaton == "classification")
-{
- eY = eX[, ncol(eX)]
- eX = eX[, 1:ncol(eX) - 1]
-
- getMask = getMask[, 1:ncol(getMask) - 1] # strip the mask of class label
- getFdMask = getFdMask[, 1:ncol(getFdMask) - 1] # strip the mask of class label
- getSchema = getSchema[, 1:ncol(getSchema) - 1] # strip the mask of class label
-}
-
-# get the logical seed
-if(targetApplicaton == "compare")
- lgSeed = logical::generateLogicalSeed(eX, as.matrix(0), getMask, targetApplicaton)
-else
- lgSeed = logical::generateLogicalSeed(eX, eY, getMask, targetApplicaton)
-allLgs = logical::transformLogical(lgSeed)
-
-
-d_accuracy = 0
-# 4. perform the sampling
-if(targetApplicaton != "compare") {
- [eX, eY] = utils::doSample(eX, eY, sample)
-
- # 5. get train test and validation set with balanced class distribution
- [X_train, y_train, X_test, y_test] = splitBalanced(eX, eY, 0.7, FALSE)
-
- # 6. find the best hyper parameters for classification algorithm
- # for now only find the best values for intercept and maximum outer iteration
- params = list("reg", "maxi");
- paramRanges = list(10^seq(0,-10), seq(10,100, 10));
-
- # if(sum(getMask) > 0)
- # {
- # dX_train = utils::dummycoding(replace(target = rbind(X_train, X_test), pattern = NaN, replacement=0), getMask)
- # dX_test = dX_train[nrow(y_train)+1:nrow(dX_train),]
- # dX_train = dX_train[1:nrow(y_train),]
- # [opt, loss] = gs::gridSearchMLR(dX_train, y_train, dX_test, y_test,
- # "multiLogReg", "lossFunc", params, paramRanges, FALSE);
-# }
- # else
- # [opt, loss] = gs::gridSearchMLR(X_train, y_train, X_test, y_test,
- # "multiLogReg", "lossFunc", params, paramRanges, FALSE);
- opt = matrix("0 100", 1, 2)
-
- # 7. get the cross validated accuracy on dirty dataset (only on training set)
- d_accuracy = classifyDirty(X_train, y_train, opt, getMask, weightedAccuracy, crossValidations)
- # print("dirty accuracy is "+d_accuracy)
- # # [eX, eY] = prioritise(eX, eY, getMask)
-}
-FD = discoverFD(X=replace(target=eX, pattern=NaN, replacement=1), Mask=getFdMask, threshold=0.8)
-FD = (diag(matrix(1, rows=nrow(FD), cols=1)) ==0) * FD
-FD = FD > 0
-
-logical1 = frame(["4", "MVI", "SCALE", "DUMMY", "DIM", "0", "0", "0"], rows=1, cols=8)
-logical2 = frame(["2", "MVI", "DUMMY", "0", "0", "0", "0", "0"], rows=1, cols=8)
-logical3 = frame(["3", "MVI", "SCALE", "DUMMY", "0", "0", "0", "0"], rows=1, cols=8)
-logical4 = frame(["6", "MVI", "OTLR", "CI", "SCALE", "DUMMY", "DIM", "0"], rows=1, cols=8)
-logical5 = frame(["7", "MVI", "OTLR", "MVI", "CI", "SCALE", "DUMMY", "DIM"], rows=1, cols=8)
-logical6 = frame(["6", "OTLR", "MVI", "CI", "SCALE", "DUMMY", "DIM", "0"], rows=1, cols=8)
-
-log = rbind(logical1, logical2)
-log = rbind(log, logical3)
-log = rbind(log, logical4)
-log = rbind(log, logical5)
-log = rbind(log, logical6)
-print("logical permutations "+toString(log))
-
-metaList = list(mask=getMask, schema=getSchema, fd=FD)
-targetClassification = list(target=targetApplicaton, cv=crossValidations, wAccuracy=weightedAccuracy,
- dirtyAcc = d_accuracy, mlHp = opt, cleanData = as.matrix(0))
-
-
-# val = compareValue(replace(target=eX, pattern=NaN, replacement=0), getMask)
-parfor(i in 1:nrow(log))
-{
- lv = as.integer(as.scalar(log[i, 1])) + 1
- [pip, hp, acc, features] = bandit(X_train=eX, Y_train=eY, metaList=metaList, targetList=targetClassification, lp=log[i, 2:lv],
- primitives=primitives, param=param, k=topK, R=resources, verbose=TRUE);
-}
-
-output = $output
-write(features, output+"/features.csv", format="csv")
-
-
-if(as.scalar((is.na(acc[1,1]))) == 1 | as.scalar(acc[1,1]) < d_accuracy)
- stop("warning: no best pipeline found")
-
-# pip = frame(["mice", "scale", "SMOTE", "dummycoding"], rows=1, cols=4)
-# hp = matrix("6.000 1.000 0.704 1.000 0.000 1.000 2.000 6.000 0.000 1.000 0.000 0.000 0.000 0.000 5.000 149.000 1.000 1.000 1.000 2.000 4.000 1.000 0.000 0.000 2.000 0.000 0.000 0.000 0.000 0.000
- # 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000", rows=1, cols=54)
-
-# acc = as.matrix(63)
-
-print("best pipelines")
-print(toString(pip))
-
-print("best hyperparam")
-print(toString(hp))
-
-print("best accuracy")
-print(toString(acc))
-
-# if(targetApplicaton != "compare")
- # clean_accuracy = testBestPipeline(pip=pip[1,], hp=hp[1,], X_train=X_train, y_train=y_train,
- # X_test=X_test, y_test=y_test, cmask=getMask, FD=FD, MLhp=opt, valAcc=as.scalar(acc[1,1]), dirAcc=d_accuracy,
- # isWeighted=weightedAccuracy)
-# else
-clean_accuracy = as.scalar(acc[1,1])
-
-
-result = d_accuracy < clean_accuracy
-print("result satisfied ------------"+result)
-
-accuracies = cbind(as.matrix(d_accuracy), as.matrix(clean_accuracy))
-
-
-tmp_hp = cbind(matrix(NaN, nrow(hp), 1), hp)
-writeResult = cbind(pip, as.frame(tmp_hp))
-writeResult = cbind(writeResult , as.frame(acc))
-
-
-write(pip, output+"/pipelines.csv", format="csv")
-write(hp, output+"/hyperparams.csv", format="csv")
-write(acc, output+"/accuracies.csv", format="csv")
-write(accuracies , output+"/BestAccuracy.csv", format="csv")
-
-
-
-
-####################################################################
-# Function for classifying the dirty dataset, makes a call to crossV()
-# Inputs: takes the input dataset X, Y and the value of k validation, mask of the
-# dataset for OHE of categorical columns, vector of ML hyper-parameters identified
-# via grid-search and a boolean value of (un)weighted accuracy.
-# Output: It return a matrix having the accuracy of each fold.
-####################################################################
-classifyDirty = function(Matrix[Double] Xtrain, Matrix[Double] ytrain, Matrix[Double] opt,
- Matrix[Double] mask, Boolean isWeighted = TRUE, Integer cv)
- return (Double accuracy)
-{
- # # classify without cleaning fill with default values 1
- Xtrain = replace(target = Xtrain, pattern = NaN, replacement=0)
- if(sum(mask) > 0)
- Xtrain = utils::dummycoding(Xtrain, mask)
- # print("rows in data ")
- # print(nrow(dX_train))
- # print("column in data")
- # print(ncol(dX_train))
- accuracy = crossV(Xtrain, ytrain, cv, mask, opt, isWeighted)
- accuracy = mean(accuracy)
- print("cross validated dirty accuracy "+accuracy)
-}
-
-
-
-
-lossFunc = function(Matrix[Double] X, Matrix[Double] y, Matrix[Double] B)
-return (Matrix[Double] loss) {
- [prob, yhat, acc] = multiLogRegPredict(X=X, B=B, Y=y, verbose=FALSE)
- loss = as.matrix(1 - (acc/100))
- # [confusionCount_c, confusionAVG_c] = confusionMatrix(P=yhat, Y=y)
-}
-
-
-# testBestPipeline = function(Frame[Unknown] pip, Matrix[Double] hp, Matrix[Double] X_train, Matrix[Double] y_train,
- # Matrix[Double] X_test, Matrix[Double] y_test, Matrix[Double] cmask, Matrix[Double] FD, Matrix[Double] MLhp,
- # Double valAcc, Double dirAcc, Boolean isWeighted)
- # return (Double result) {
- # print("hp "+toString(hp))
- # lsTrain = list();
- # lsTest = list();
- # i = 1; k = 1
- # trRow=nrow(X_train)
- # # take the oversampling out from the test processing
- # pip1 = as.frame("")
- # # construct the parameter list for best hyper-parameters
- # while(k <= ncol(pip))
- # {
- # end = as.integer(i+as.integer(as.scalar(hp[1,i])))
- # mat = hp[1, i+1:end]
- # i = end + 1
- # lsTrain = append(lsTrain, mat)
- # if(as.scalar(pip[1,k]) != "SMOTE") {
- # pip1 = cbind(pip1, pip[1,k] )
- # lsTest = append(lsTest, mat)
- # }
- # k = k + 1
- # }
-
- # # clean using best pipeline and train model
- # [X_train, y_train] = executePipeline(pip, X_train, y_train, cmask, FD, lsTrain, 1, FALSE)
- # if(ncol(pip1) > 1)
- # [X_test, y_test] = executePipeline(pip1[, 2:ncol(pip1)], X_test, y_test, cmask, FD, lsTest, 1, FALSE)
- # # X_train_clean = X_train[1:trRow, ]
- # # y_train_clean = Y_train[1:trRow, ]
- # # X_test_clean = X_train[trRow+1:nrow(X_train), ]
- # # y_test_clean = Y_train[trRow+1:nrow(X_train), ]
-
- # # classify after cleaning
- # betas = multiLogReg(X=X_train, Y=y_train, icpt=1,
- # reg=as.scalar(MLhp[1,1]), tol= 1e-9, maxi=as.scalar(MLhp[1,2]),
- # maxii= 50, verbose=FALSE);
-
- # [c_prob, c_yhat, c_accuracy] = multiLogRegPredict(X_test, betas, y_test, FALSE)
- # c_accuracy = getAccuracy(y_test, c_yhat, isWeighted)
- # [confusionCount_c, confusionAVG_c] = confusionMatrix(P=c_yhat, Y=y_test)
-
-
- # print("Actual Records \n"+toString(cbind(X_test, y_test)))
- # # print("Clean Records \n"+toString(cbind(X_test, y_test)))
- # print("predictions Records \n"+toString(cbind(X_test, c_yhat)))
- # print("accuracy of dirty data "+dirAcc)
- # print("accuracy of val data "+valAcc)
- # print("accuracy of test accuracy "+c_accuracy)
- # print("clean confusion matrix \n"+toString(confusionCount_c))
-
- # result = c_accuracy
-# }
-
-
-
-# # ######################################################################
-# # # # Function for cross validation using hold out method
-# # # # Inputs: The input dataset X, Y and the value of k validation, mask of the
-# # # # dataset for OHE of categorical columns, vector of ML hyper-parameters identified
-# # # # via gridsearch and a boolean value of (un)weighted accuracy.
-# # # # Output: It return a matrix having the accuracy of each fold.
-# # ######################################################################
-
-crossV = function(Matrix[double] X, Matrix[double] y, Integer k, Matrix[Double] mask,
- Matrix[Double] MLhp, Boolean isWeighted)
-return (Matrix[Double] accuracyMatrix)
-{
-
- accuracyMatrix = matrix(0, k, 1)
-
- dataList = list()
- testL = list()
- data = order(target = cbind(y, X), by = 1, decreasing=FALSE, index.return=FALSE)
- classes = table(data[, 1], 1)
- ins_per_fold = classes/k
- start_fold = matrix(1, rows=nrow(ins_per_fold), cols=1)
- fold_idxes = cbind(start_fold, ins_per_fold)
-
- start_i = 0; end_i = 0; idx_fold = 1;;
- for(i in 1:k)
- {
- fold_i = matrix(0, 0, ncol(data))
- start=0; end=0;
- for(j in 1:nrow(classes))
- {
- idx = as.scalar(classes[j, 1])
- start = end + 1;
- end = end + idx
- class_j = data[start:end, ]
-
- start_i = as.scalar(fold_idxes[j, 1]);
- end_i = as.scalar(fold_idxes[j, 2])
-
- fold_i = rbind(fold_i, class_j[start_i:end_i, ])
- }
-
- dataList = append(dataList, fold_i)
- fold_idxes[, 1] = fold_idxes[, 2] + 1
- fold_idxes[, 2] += ins_per_fold
- while(FALSE){}
- }
-
- for(i in seq(1,k))
- {
- [trainList, hold_out] = remove(dataList, i)
- trainset = rbind(trainList)
- testset = as.matrix(hold_out)
- trainX = trainset[, 2:ncol(trainset)]
- trainy = trainset[, 1]
- testX = testset[, 2:ncol(testset)]
- testy = testset[, 1]
- beta = multiLogReg(X=trainX, Y=trainy, icpt=1, reg=as.scalar(MLhp[1,1]), tol= 1e-9,
- maxi=as.scalar(MLhp[1,2]), maxii= 50, verbose=FALSE);
- [prob, yhat, a] = multiLogRegPredict(testX, beta, testy, FALSE)
- accuracy = getAccuracy(testy, yhat, isWeighted)
- accuracyMatrix[i] = accuracy
- }
-}
-
-
diff --git a/src/test/scripts/functions/pipelines/testClassification.dml b/src/test/scripts/functions/pipelines/testClassification.dml
deleted file mode 100644
index 62147e5..0000000
--- a/src/test/scripts/functions/pipelines/testClassification.dml
+++ /dev/null
@@ -1,142 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-# Generate the logical pipelines for data cleaning
-
-source("scripts/pipelines/scripts/utils.dml") as utils;
-
-# read the inputs
-F = read($dirtyData, data_type="frame", format="csv", header=TRUE,
- naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
-
-metaInfo = read($metaData, data_type="frame", format="csv", header=FALSE);
-primitives = read($primitives, data_type = "frame", format="csv", header= TRUE)
-param = read($parameters, data_type = "frame", format="csv", header= TRUE)
-logical = frame(["MVI", "DUMMY"], rows=1, cols=2)
-sample = $sampleSize
-topK = $topk
-resources = $rv
-crossValidations = $cv
-weightedAccuracy = $weighted # accuracy flag
-targetApplicaton = $target # accuracy flag
-output = $output
-
-if(nrow(metaInfo) < 2)
- stop("incomplete meta info")
-
-getSchema = metaInfo[1, 2:ncol(metaInfo)]
-getMask = as.matrix(metaInfo[2, 2:ncol(metaInfo)])
-getFdMask = as.matrix(metaInfo[3, 2:ncol(metaInfo)]) # columns of interest for FD computation
-
-# 1. dropInvalid function will remove the values which are not the part
-# of the column data type and convert the data to lowercase
-
-X = utils::stringProcessing(F, getMask, getSchema)
-
-# 2. encode the categorical data
-if(sum(getMask) > 0)
-{
- # always recode the label
- index = vectorToCsv(getMask)
- jspecR = "{ids:true, recode:["+index+"]}"
- [eX, X_meta] = transformencode(target=X, spec=jspecR);
- # change the schema to reflect the encoded values
- getSchema = map(getSchema, "x->x.replace(\"STRING\", \"INT64\")")
- getSchema = map(getSchema, "x->x.replace(\"BOOLEAN\", \"INT64\")")
-
-}
-# if no categorical value exist then just cast the frame into matrix
-else
- eX = as.matrix(X)
-
-# 3. extract the class label
-eY = eX[, ncol(eX)]
-eX = eX[, 1:ncol(eX) - 1]
-
-getMask = getMask[, 1:ncol(getMask) - 1] # strip the mask of class label
-getFdMask = getFdMask[, 1:ncol(getFdMask) - 1] # strip the mask of class label
-getSchema = getSchema[, 1:ncol(getSchema) - 1] # strip the mask of class label
-
-
-# 4. perform the sampling
-[eX, eY] = utils::doSample(eX, eY, sample)
-
-# 5. find the best hyper parameters for classification algorithm
-# for now only find the best values for intercept and maximum outer iteration
-opt = matrix("0 0 100", rows=1, cols=3)
-
-
-# 6. get the cross validated accuracy on dirty dataset (only on training set)
-d_accuracy = 0
-d_accuracy = utils::classifyDirty(eX, eY, opt, getMask, weightedAccuracy, crossValidations)
-
-if(sum(getFdMask) > 0)
-{
- FD = discoverFD(X=replace(target=eX, pattern=NaN, replacement=1), Mask=getFdMask, threshold=0.8)
- FD = (diag(matrix(1, rows=nrow(FD), cols=1)) ==0) * FD
- FD = FD > 0
-}
-FD = as.matrix(0)
-
-metaList = list(mask=getMask, schema=getSchema, fd=FD)
-targetClassification = list(target=targetApplicaton, cv=crossValidations, wAccuracy=weightedAccuracy,
- dirAcc = d_accuracy, mlHp = opt, cleanData = as.matrix(0))
-
-# # initialize output variables
-# 7. call the optimizer
-pip = as.frame("NULL"); hp = matrix(0,0,0); acc = matrix(0,0,0); features = as.frame("NULL")
-[pip, hp, acc, features] = bandit(X_train=eX, Y_train=eY, metaList=metaList, targetList=targetClassification, lp=logical,
- primitives=primitives, param=param, k=topK, R=resources, verbose=TRUE);
-
-
-if(as.scalar((is.na(acc[1,1]))) == 1 | as.scalar(acc[1,1]) < d_accuracy)
- stop("warning: no best pipeline found")
-
-print("best pipelines")
-print(toString(pip))
-
-print("best hyperparam")
-print(toString(hp))
-
-print("best accuracy")
-print(toString(acc[1, 1]))
-
-
-clean_accuracy = max(acc[1,1])
-#8. compare results
-result = d_accuracy < clean_accuracy
-print("result satisfied: "+result)
-
-write(pip, output+"/pipelines.csv", format="csv")
-write(hp, output+"/hyperparams.csv", format="csv")
-write(acc, output+"/accuracies.csv", format="csv")
-accuracies = cbind(as.matrix(d_accuracy), acc[1,1])
-write(accuracies , output+"/BestAccuracy.csv", format="csv")
-write(features, output+"/features.csv", format="csv")
-write(result , $O)
-
-accuracy = function(Matrix[Double] X, Matrix[Double] y, Matrix[Double] B)
-return (Matrix[Double] loss) {
- [prob, yhat, acc] = multiLogRegPredict(X=X, B=B, Y=y, verbose=FALSE)
- loss = as.matrix(1 - (acc/100))
-}
-
-
-
diff --git a/src/test/scripts/functions/pipelines/testCompare.dml b/src/test/scripts/functions/pipelines/testCompare.dml
deleted file mode 100644
index 1f0e663..0000000
--- a/src/test/scripts/functions/pipelines/testCompare.dml
+++ /dev/null
@@ -1,124 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-source("scripts/pipelines/scripts/utils.dml") as utils;
-# read the inputs
-F = read($dirtyData, data_type="frame", format="csv", header=TRUE,
- naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
-
-metaInfo = read($metaData, data_type="frame", format="csv", header=FALSE);
-primitives = read($primitives, data_type = "frame", format="csv", header= TRUE)
-param = read($parameters, data_type = "frame", format="csv", header= TRUE)
-topK = $topk
-resources = $rv
-targetApplicaton = $target # accuracy flag
-cleanData = read($cleanData, data_type="frame", format="csv", header=TRUE,
- naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
-
-logical = as.frame("MVI")
-
-# take the sample of 500 rows to avoid java heap issue
-
-F = F[1:200, 1:20]
-cleanData = cleanData[1:200, 1:20]
-metaInfo = metaInfo[, 1:21]
-
-if(nrow(metaInfo) < 2)
- stop("incomplete meta info")
-
-# construct meta vectors
-getSchema = metaInfo[1, 2:ncol(metaInfo)]
-getMask = as.matrix(metaInfo[2, 2:ncol(metaInfo)])
-getFdMask = as.matrix(metaInfo[3, 2:ncol(metaInfo)]) # columns of interest for FD computation
-
-# can not perform string processing as this creates conflicts with original data like the
-# string processing change the case to lower case then the value comparison with original data
-# is not possible
-# X = utils::stringProcessing(F, getMask, getSchema)
-
-X = F
-
-# encode the categorical data
-if(sum(getMask) > 0)
-{
- # always recode the label
- index = vectorToCsv(getMask)
- jspecR = "{ids:true, recode:["+index+"]}"
-
- [eX, X_meta] = transformencode(target=rbind(cleanData, X), spec=jspecR);
- cleanX = eX[1:nrow(cleanData)]
- eX = eX[nrow(cleanData)+1:nrow(eX)]
-
- # change the schema to reflect the encoded values
- getSchema = map(getSchema, "x->x.replace(\"STRING\", \"INT64\")")
- getSchema = map(getSchema, "x->x.replace(\"BOOLEAN\", \"INT64\")")
-
-
-}
-# if no categorical value exist then just cast the frame into matrix
-else {
- eX = as.matrix(X)
- cleanX = as.matrix(cleanData)
-}
-
-
-FD = discoverFD(X=replace(target=eX, pattern=NaN, replacement=1), Mask=getFdMask, threshold=0.8)
-FD = (diag(matrix(1, rows=nrow(FD), cols=1)) ==0) * FD
-FD = FD > 0
-
-expectedAccuracy = 0.5
-
-metaList = list(mask=getMask, schema=getSchema, fd=FD)
-targetCompare = list(target=targetApplicaton, cv=0, wAccuracy=FALSE,
- dirAcc = expectedAccuracy, mlHp = as.matrix(0), cleanData = cleanX)
-
-
-# # initialize output variables
-pip = as.frame("NULL"); hp = matrix(0,0,0); acc = matrix(0,0,0); features = as.frame("NULL")
-[pip, hp, acc, features] = bandit(X_train=eX, Y_train=as.matrix(0), metaList=metaList, targetList=targetCompare,
- lp=logical, primitives=primitives, param=param, k=topK, R=resources, verbose=TRUE);
-
-
-output = $output
-write(features, output+"/features.csv", format="csv")
-
-
-if(as.scalar((is.na(acc[1,1]))) == 1 | as.scalar(acc[1,1]) < expectedAccuracy)
- stop("warning: no best pipeline found")
-
-
-print("best pipelines")
-print(toString(pip))
-
-print("best hyperparam")
-print(toString(hp))
-
-print("best accuracy")
-print(toString(acc))
-
-clean_accuracy = as.scalar(acc[1,1])
-
-
-result = expectedAccuracy < clean_accuracy
-print("result satisfied ------------"+result)
-
-accuracies = cbind(as.matrix(expectedAccuracy), as.matrix(clean_accuracy))
-
-write(result , $O)
\ No newline at end of file
diff --git a/src/test/scripts/functions/pipelines/testLogical.dml b/src/test/scripts/functions/pipelines/testLogical.dml
deleted file mode 100644
index c0a1306..0000000
--- a/src/test/scripts/functions/pipelines/testLogical.dml
+++ /dev/null
@@ -1,124 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-# Generate the logical pipelines for data cleaning
-
-source("scripts/pipelines/scripts/utils.dml") as utils;
-source("scripts/pipelines/scripts/enumerateLogical.dml") as lg;
-
-
-# read the inputs
-X = read($dirtyData, data_type="frame", format="csv", header=TRUE,
- naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
-
-metaInfo = read($metaData, data_type="frame", format="csv", header=FALSE);
-primitives = read($primitives, data_type = "frame", format="csv", header= TRUE)
-param = read($parameters, data_type = "frame", format="csv", header= TRUE)
-weightedAccuracy = FALSE # accuracy flag
-targetApplicaton = $target # accuracy flag
-
-max_iter = $max_iter
-num_inst = $num_inst
-num_exec = $num_exec
-crossValidations = $cv
-output = $output
-
-getSchema = metaInfo[1, 2:ncol(metaInfo)]
-getMask = as.matrix(metaInfo[2, 2:ncol(metaInfo)])
-getFdMask = as.matrix(metaInfo[3, 2:ncol(metaInfo)]) # columns of interest for FD computation
-
-# encode the categorical data
-if(sum(getMask) > 0)
-{
- # always recode the label
- index = vectorToCsv(getMask)
- jspecR = "{ids:true, recode:["+index+"]}"
- [eX, X_meta] = transformencode(target=X, spec=jspecR);
- # change the schema to reflect the encoded values
- getSchema = map(getSchema, "x->x.replace(\"STRING\", \"INT64\")")
- getSchema = map(getSchema, "x->x.replace(\"BOOLEAN\", \"INT64\")")
-
-}
-# if no categorical value exist then just cast the frame into matrix
-else
- eX = as.matrix(X)
-
-# extract the class label
-eY = eX[, ncol(eX)]
-eX = eX[, 1:ncol(eX) - 1]
-
-getMask = getMask[, 1:ncol(getMask) - 1] # strip the mask of class label
-getFdMask = getFdMask[, 1:ncol(getFdMask) - 1] # strip the mask of class label
-getSchema = getSchema[, 1:ncol(getSchema) - 1] # strip the mask of class label
-# hyperparam for classifier
-opt = matrix("0 0 100", rows=1, cols=3)
-print("opt "+toString(opt))
-# get the cross validated accuracy on dirty dataset (only on training set)
-d_accuracy = 0
-d_accuracy = utils::classifyDirty(eX, eY, opt, getMask, weightedAccuracy, crossValidations)
-
-# get FD for IC operations
-FD = discoverFD(X=replace(target=eX, pattern=NaN, replacement=1), Mask=getFdMask, threshold=0.8)
-FD = (diag(matrix(1, rows=nrow(FD), cols=1)) ==0) * FD
-FD = FD > 0
-
-metaList = list(mask=getMask, schema=getSchema, fd=FD)
-targetClassification = list(target=targetApplicaton, cv=crossValidations, wAccuracy=weightedAccuracy,
- dirAcc = d_accuracy, mlHp = opt, cleanData = as.matrix(0))
-
-# # initialize output variables
-pip = as.frame("NULL"); hp = matrix(0,0,0); acc = matrix(0,0,0); features = as.frame("NULL")
-
-
-logical = frame([
- "1", "MVI", "0", "0", "0", "0",
- # "1", "OTLR", "0", "0", "0", "0",
- # "1", "CI", "0", "0", "0", "0",
- # "2", "MVI", "CI", "0", "0", "0",
- "2", "MVI", "OTLR", "0", "0", "0",
- "2", "MVI", "SCALE", "0", "0", "0",
- "3", "MVI", "SCALE", "OTLR", "0", "0"
- # "4", "MVI", "OTLR", "CI", "SCALE", "0",
- # "4", "OTLR", "MVI", "CI", "SCALE", "0",
- # "5", "MVI", "OTLR", "MVI", "CI", "SCALE"
- ], rows=4, cols=6)
-
-
-categories = frame(["MVI", "OTLR", "SCALE"], rows=1, cols=3)
-cmr = matrix("4 0.7 2", rows=1, cols=3)
-[bestLogical, score, T] = lg::enumerateLogical(X=eX, y=eY, cmr=cmr, cat=categories, population=logical,
- max_iter=max_iter, metaList=metaList, targetList=targetClassification, primitives=primitives, param=param,
- num_inst=num_inst, num_exec=num_exec, isTailed=TRUE, verbose=TRUE)
-
-print("score of pipeline: "+toString(score)+" in "+(T/60000)+" mins")
-print("bestLogical "+toString(bestLogical))
-
-result = d_accuracy < score
-print("result satisfied ------------"+result)
-
-write(result , $O)
-write(bestLogical , output, format="csv")
-
-accuracy = function(Matrix[Double] X, Matrix[Double] y, Matrix[Double] B)
-return (Matrix[Double] loss) {
- [prob, yhat, acc] = multiLogRegPredict(X=X, B=B, Y=y, verbose=FALSE)
- loss = as.matrix(1 - (acc/100))
-}
-
diff --git a/src/test/scripts/functions/pipelines/topkcleaningClassificationTest.dml b/src/test/scripts/functions/pipelines/topkcleaningClassificationTest.dml
new file mode 100644
index 0000000..05aaca4
--- /dev/null
+++ b/src/test/scripts/functions/pipelines/topkcleaningClassificationTest.dml
@@ -0,0 +1,155 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+# Generate the logical pipelines for data cleaning
+
+source("scripts/pipelines/scripts/utils.dml") as utils;
+
+# read the inputs
+F = read($dirtyData, data_type="frame", format="csv", header=TRUE,
+ naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
+
+metaInfo = read($metaData, data_type="frame", format="csv", header=FALSE);
+primitives = read($primitives, data_type = "frame", format="csv", header= TRUE)
+param = read($parameters, data_type = "frame", format="csv", header= TRUE)
+topK = $topk
+resources = $rv
+sample=$sample
+
+if(nrow(metaInfo) < 2)
+ stop("incomplete meta info")
+
+metaInfo = metaInfo[, 2:ncol(metaInfo)]
+
+[topKPipelines, topKHyperParams, topKScores, bestLogical, features, dirtyScore] = topk_cleaning(F, metaInfo, primitives, param,
+ matrix("2 0.7 1", rows=1, cols=3), "evalClassification", as.matrix("0"), topK, resources, sample, TRUE)
+
+
+print("dirty accuracy "+toString(dirtyScore))
+print("best logical pipelines "+toString(bestLogical))
+print("topk pipelines "+toString(topKPipelines))
+print("topk hyper params "+toString(topKHyperParams))
+print("topk scores: \n"+toString(topKScores))
+perf = as.double(as.scalar(topKScores[1, 1])) - as.double(dirtyScore)
+print("performce improvemnet "+ perf)
+result = dirtyScore < as.scalar(topKScores[1, 1])
+write(result, $O)
+
+
+
+# UDF for evaluation
+# choice of parameters provided by API, X, Y, clone_X, evalFunHp (hyper-param), trainML (boolean for optimizing hp internally or passed by externally )
+evalClassification = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] Xorig, List[Unknown] metaList,
+ Matrix[Double] evalFunHp, Integer trainML=0)
+
+return(Matrix[Double] output)
+{
+ cv = 2
+ mask = as.matrix(metaList['mask'])
+ print("min and max of y in eval: "+min(Y)+" "+max(Y))
+ if(max(Y) == min(Y)) {
+ print("Y contains only one class")
+ accuracy = as.double(0)
+ }
+ else {
+ if(trainML == 1)
+ {
+ # do the gridsearch for hyper-parameters
+ params = list("icpt", "reg", "tol", "maxii")
+ paramRanges = list(seq(0, 2, 1), 10^seq(1,-4), 10^seq(1,-6), 10^seq(1,3));
+
+ # if(sum(mask) > 0)
+ # X = utils::dummycoding(replace(target = X, pattern = NaN, replacement=0), mask)
+ trainArgs = list(X=X, Y=Y, icpt=-1, reg=-1, tol=-1, maxi=100, maxii=-1, verbose=FALSE);
+ [B1, opt] = utils::topk_gridSearch(X=X, y=Y, train="multiLogReg", predict="W", numB=ncol(X)+1, cv=TRUE, cvk=cv,
+ params=params, paramValues=paramRanges, trainArgs=trainArgs, verbose=FALSE);
+ evalFunHp = as.matrix(opt)
+ }
+
+ # do the k = 3 cross validations
+ # evalFunHpM = as.matrix(evalFunHp)
+ [accuracyMatrix] = crossV(X, Y, cv, evalFunHp, FALSE)
+ accuracyMatrix = removeEmpty(target=accuracyMatrix, margin="rows")
+ score = mean(accuracyMatrix)
+ print(cv +" validation accuracy "+score)
+ }
+ output = cbind(as.matrix(score), evalFunHp)
+
+}
+
+# # ######################################################################
+# # # # Function for cross validation using hold out method
+# # # # Inputs: The input dataset X, Y and the value of k validation, mask of the
+# # # # dataset for OHE of categorical columns, vector of ML hyper-parameters identified
+# # # # via gridsearch and a boolean value of (un)weighted accuracy.
+# # # # Output: It return a matrix having the accuracy of each fold.
+# # ######################################################################
+
+crossV = function(Matrix[double] X, Matrix[double] y, Integer k, Matrix[Double] MLhp, Boolean isWeighted)
+return (Matrix[Double] accuracyMatrix)
+{
+ accuracyMatrix = matrix(0, k, 1)
+ dataList = list()
+ testL = list()
+ data = order(target = cbind(y, X), by = 1, decreasing=FALSE, index.return=FALSE)
+ classes = table(data[, 1], 1)
+ ins_per_fold = classes/k
+ start_fold = matrix(1, rows=nrow(ins_per_fold), cols=1)
+ fold_idxes = cbind(start_fold, ins_per_fold)
+
+ start_i = 0; end_i = 0; idx_fold = 1;;
+ for(i in 1:k)
+ {
+ fold_i = matrix(0, 0, ncol(data))
+ start=0; end=0;
+ for(j in 1:nrow(classes))
+ {
+ idx = as.scalar(classes[j, 1])
+ start = end + 1;
+ end = end + idx
+ class_j = data[start:end, ]
+ start_i = as.scalar(fold_idxes[j, 1]);
+ end_i = as.scalar(fold_idxes[j, 2])
+ fold_i = rbind(fold_i, class_j[start_i:end_i, ])
+ }
+ dataList = append(dataList, fold_i)
+ fold_idxes[, 1] = fold_idxes[, 2] + 1
+ fold_idxes[, 2] += ins_per_fold
+ }
+
+ for(i in seq(1,k))
+ {
+ [trainList, hold_out] = remove(dataList, i)
+ trainset = rbind(trainList)
+ testset = as.matrix(hold_out)
+ trainX = trainset[, 2:ncol(trainset)]
+ trainy = trainset[, 1]
+ testX = testset[, 2:ncol(testset)]
+ testy = testset[, 1]
+ beta = multiLogReg(X=trainX, Y=trainy, icpt=as.scalar(MLhp[1,1]), reg=as.scalar(MLhp[1,2]), tol=as.scalar(MLhp[1,3]),
+ maxi=as.scalar(MLhp[1,4]), maxii=50, verbose=FALSE);
+ [prob, yhat, a] = multiLogRegPredict(testX, beta, testy, FALSE)
+ accuracy = getAccuracy(testy, yhat, isWeighted)
+ accuracyMatrix[i] = accuracy
+ }
+
+}
+
+
\ No newline at end of file
diff --git a/src/test/scripts/functions/pipelines/topkcleaningRegressionTest.dml b/src/test/scripts/functions/pipelines/topkcleaningRegressionTest.dml
new file mode 100644
index 0000000..3038145
--- /dev/null
+++ b/src/test/scripts/functions/pipelines/topkcleaningRegressionTest.dml
@@ -0,0 +1,131 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+source("scripts/pipelines/scripts/utils.dml") as utils;
+
+# read the inputs
+F = read($dirtyData, data_type="frame", format="csv", header=TRUE,
+ naStrings= ["NA", "null"," ","NaN", "nan", "", "?", "99999"]);
+# only for salaries data
+F = F[, 2:ncol(F)]
+# metaInfo = read($metaData, data_type="frame", format="csv", header=FALSE);
+primitives = read($primitives, data_type = "frame", format="csv", header= TRUE)
+param = read($parameters, data_type = "frame", format="csv", header= TRUE)
+topK = $topk
+resources = $rv
+sample=$sample
+
+[topKPipelines, topKHyperParams, topKScores, bestLogical, features, dirtyScore] = topk_cleaning(data=F, primitives=primitives, parameters=param,
+ cmr=matrix("4 0.7 1", rows=1, cols=3), evaluationFunc="evalRegression", evalFunHp=as.matrix("0"), topK=topK, resource_val=resources, sample=sample, isLastLabel=TRUE)
+
+
+print("dirty accuracy "+toString(dirtyScore))
+print("best logical pipelines "+toString(bestLogical))
+print("topk pipelines "+toString(topKPipelines))
+print("topk hyper params "+toString(topKHyperParams))
+print("topk scores: \n"+toString(topKScores))
+perf = as.scalar(topKScores[1, 1]) - dirtyScore
+print("performce improvemnet "+ perf)
+result = dirtyScore < as.scalar(topKScores[1, 1])
+write(result, $O)
+
+
+
+# UDF for evaluation
+# choice of parameters provided by API, X, Y, clone_X, evalFunHp (hyper-param), trainML (boolean for optimizing hp internally or passed by externally )
+evalRegression = function(Matrix[Double] X, Matrix[Double] Y, Matrix[Double] Xorig, List[Unknown] metaList,
+ Matrix[Double] evalFunHp, Integer trainML=0)
+
+return(Matrix[Double] output)
+{
+ cv = 2
+ mask = as.matrix(metaList['mask'])
+
+ if(max(Y) == min(Y)) {
+ print("Y contains only one class")
+ accuracy = as.double(0)
+ }
+ else {
+ if(trainML == 1)
+ {
+ # do the gridsearch for hyper-parameters
+ params = list("icpt","reg", "tol", "maxi");
+ paramRanges = list(seq(0,2),10^seq(0,-4), 10^seq(-6,-12), 10^seq(1,3));
+ [B1, opt] = utils::topk_gridSearch(X=X, y=Y, train="lm", predict="wmape",
+ numB=ncol(X)+1, cv=TRUE, params=params, paramValues=paramRanges);
+ evalFunHp = as.matrix(opt)
+ }
+
+ # do the k = 3 cross validations
+ # evalFunHpM = as.matrix(evalFunHp)
+ [accuracyMatrix] = crossV(X, Y, cv, evalFunHp)
+ accuracyMatrix = removeEmpty(target=accuracyMatrix, margin="rows")
+ score = mean(accuracyMatrix)
+ print(cv +" validation accuracy "+score)
+ }
+ output = cbind(as.matrix(score), evalFunHp)
+
+}
+
+wmape = function(Matrix[Double] X, Matrix[Double] y, Matrix[Double] B, Integer icpt) return (Matrix[Double] loss) {
+ # loss = as.matrix(sum((y - X%*%B)^2));
+ pred = lmPredict(X=X, B=B, ytest=y, icpt=icpt);
+ WMAPE = sum(abs(y - pred))/sum(abs(y)) #this will give the lose into range of [0,1]
+ loss = as.matrix(WMAPE)
+}
+
+
+
+crossV = function(Matrix[Double] X, Matrix[Double] y, Integer k, Matrix[Double] hp) return (Matrix[Double] accuracyMatrix)
+{
+ icpt = as.scalar(hp[1, 1])
+ reg = as.scalar(hp[1, 2])
+ tol = as.scalar(hp[1, 3])
+ maxi = as.scalar(hp[1, 4])
+ M = nrow(X);
+ lim = floor(as.integer(M/k));
+ accuracyMatrix = matrix(0, rows=k, cols=1)
+
+ for (i in 1:k)
+ {
+ testS = ifelse(i==1, 1, ((i-1) * lim)+1)
+ testE = i * lim;
+ testSet = X[testS:testE,];
+ testRes = y[testS:testE,];
+
+ if (i == 1) {
+ trainSet = X[testE+1:M,];
+ trainRes = y[testE+1:M,];
+ }
+ else if(i == k)
+ {
+ trainSet = X[1:testS-1,];
+ trainRes = y[1:testS-1,];
+ }
+ else {
+ trainSet = rbind(X[1:testS-1,], X[testE+1:M,]);
+ trainRes = rbind(y[1:testS-1,], y[testE+1:M,]);
+ }
+ beta = lm(X=trainSet, y=trainRes, icpt=icpt, reg=reg, tol=tol, maxi=maxi);
+ acc = wmape(testSet, testRes, beta, icpt)
+ accuracyMatrix[i] = (1 - acc)
+ }
+}
+
\ No newline at end of file