dpl=FALSE
##################################################
### start up h2o
library(h2o)
h2oServer = h2o.init(max_mem_size="10g", nthreads=-1)

##################################################
###simulate xor data
set.seed(1)

##simulate (x1,x2) in 4 different regions
p11 = cbind(rnorm(n=25,mean=1,sd=0.5),rnorm(n=25,mean=1,sd=0.5))
p12 = cbind(rnorm(n=25,mean=-1,sd=0.5),rnorm(n=25,mean=1,sd=0.5))
p13 = cbind(rnorm(n=25,mean=-1,sd=0.5),rnorm(n=25,mean=-1,sd=0.5))
p14 = cbind(rnorm(n=25,mean=1,sd=0.5),rnorm(n=25,mean=-1,sd=0.5))

##y=g is 1 when x1 !=1 x2 and 0 when x1=x2
g = as.factor(c(rep(0,50),rep(1,50)))
x = rbind(p11,p13,p12,p14)

##big grid in (x1,x2) space
px1 = seq(min(x[,1]), max(x[,1]), length.out = 100)
px2 = seq(min(x[,2]), max(x[,2]), length.out = 100)
gd = expand.grid(x1=px1, x2=px2)

##store simulation in a data frame
dfd = data.frame(x=x,y=g)
names(dfd) =c("x1","x2","y")

######################################################################
### data in h2o form
dftrain = as.h2o(dfd, destination_frame = "xor.train")
dftest = as.h2o(gd, destination_frame = "xor.test")

#look at h2o data objects
print(dftrain) #h2o prints out first 6 rows
print(dftest)

# see that xor.test and xor.train are now on server
print(h2o.ls()) 

#inspect dftrain
cat("str of dftrain:\n")
print(str(dftrain))

cat("is dh2o an S4 class?:\n")
print(isS4(dftrain))

cat("attributes of dftrain:\n")
print(attributes(dftrain))

cat("the h2o id of dftrain is: ",attr(dftrain,"id"),"\n")

######################################################################
### plot data
cv = ifelse(g==1,"cornflowerblue","coral")
plot(x,col=cv,pch=16,cex.lab=1.5,cex.axis=1.5)
legend("topleft",legend=c("y=1","y=0"), col=c("cornflowerblue","coral"),pch=c(16,16))
# dev.copy2pdf(file="plot-xor.pdf",height=10,width=12)

######################################################################
#  1 hidden layer 2 neurons

fp = file.path("./files","xor.mXOR2")
if(file.exists(fp)) {
   mXOR2 = h2o.loadModel(fp)
} else {
   mXOR2 = h2o.deeplearning(x=1:2, y=3,
                         training_frame = dftrain,
                         hidden = c(2),
                         activation = "Tanh",
                         epochs = 100000,
                         export_weights_and_biases = TRUE,
                         model_id = "xor.mXOR2"
                         #seed=99,reproducible=TRUE 
                         )
   h2o.saveModel(mXOR2,path="./files",force=TRUE) #force overwrites an existing)
}

######################################################################
### look at confusion matrix
print(h2o.confusionMatrix(mXOR2,thresholds=.5))

######################################################################
### plot fit 

## phat on test, this will be a matrix with three columns
phat = h2o.predict(mXOR2, dftest)

## contour plot
phatg = matrix(phat[,3],length(px1), length(px2))
phatv = as.vector(phat[,3])
contour(px1, px2, phatg, levels=0.5, labels="", xlab="", ylab="",
        main= "nn h2o fit to xor data")
points(x, col=ifelse(g==1, "cornflowerblue","coral"),pch=16)
points(gd, pch=".", cex=1.5, col=ifelse(phatv>0.5, "cornflowerblue","coral"))
# dev.copy2pdf(file="fit-mXOR2.pdf",height=10,width=12)

######################################################################
### inspect fit 
#see coefficients
# note: this does not work if you used h2o.loadModel to retrieve a saved one
# from x to hidden
h2o.biases(mXOR2, vector_id = 1)
h2o.weights(mXOR2, matrix_id = 1)

#from hidden to y
h2o.biases(mXOR2, vector_id = 2)
h2o.weights(mXOR2, matrix_id = 2)

## compute output of the first layer
tmp.df = as.h2o(data.frame(x1=c(-1, -1, 1, 1), x2=c(-1, 1, -1, 1)), 
                destination_frame = "xor.4points")
trans.features = h2o.deepfeatures(mXOR2, tmp.df, layer = 1)
as.matrix( h2o.cbind(tmp.df, trans.features) )


######################################################################
### single layer, 10 neurons/units, L1 regularization

mXOR10R = h2o.deeplearning(x=1:2, y=3,
                         dftrain,
                         hidden = c(10),
                         activation = "Tanh",
                         epochs = 100000,
                         export_weights_and_biases = TRUE,
                         l1 = 1e-2,
                         model_id = "xor.mXOR10R"
)

print(h2o.confusionMatrix(mXOR10R,thresholds=.5))

phat = h2o.predict(mXOR10R, dftest)

phatg = matrix(phat[,3],length(px1), length(px2))
phatv = as.vector(phat[,3])
contour(px1, px2, phatg, levels=0.5, labels="", xlab="", ylab="",
        main= "nn h2o fit to xor data")
points(x, col=ifelse(g==1, "cornflowerblue","coral"),pch=16)
points(gd, pch=".", cex=1.5, col=ifelse(phatv>0.5, "cornflowerblue","coral"))

# dev.copy2pdf(file="fit-mXOR10R.pdf",height=10,width=12)

#see coefficients
h2o.weights(mXOR10R, matrix_id = 1)
h2o.weights(mXOR10R, matrix_id = 2)


######################################################################
### try deep c(3,4) hidden units, regularized 

mXORDR = h2o.deeplearning(x=1:2, y=3,
                         dftrain,
                         hidden = c(3,4),
                         activation = "Tanh",
                         epochs = 100000,
                         export_weights_and_biases = TRUE,
                         l1 = 1e-2,
                         model_id = "xor.mXORDR"
)

print(h2o.confusionMatrix(mXORDR,thresholds=.5))

phat = h2o.predict(mXORDR, dftest)

phatg = matrix(phat[,3],length(px1), length(px2))
phatv = as.vector(phat[,3])
contour(px1, px2, phatg, levels=0.5, labels="", xlab="", ylab="",
        main= "nn h2o fit to xor data")
points(x, col=ifelse(g==1, "cornflowerblue","coral"),pch=16)
points(gd, pch=".", cex=1.5, col=ifelse(phatv>0.5, "cornflowerblue","coral"))
# dev.copy2pdf(file="fit-modeDR.pdf",height=10,width=12)

## the weights
h2o.weights(mXORDR, matrix_id = 1)
h2o.weights(mXORDR, matrix_id = 2)
h2o.weights(mXORDR, matrix_id = 3)

## deep features
tmp.df = as.h2o(data.frame(x1=c(-1, -1, 1, 1), x2=c(-1, 1, -1, 1)),
                destination_frame = "xor.4points")
trans.features = h2o.deepfeatures(mXORDR, tmp.df, layer = 2)
as.matrix( h2o.cbind(tmp.df, trans.features) )


## in sample performance
h2o.performance(mXORDR)

f1cut = 0.407872
contour(px1, px2, phatg, levels=f1cut, labels="", xlab="", ylab="",
        main= "nn h2o fit to xor data")
points(x, col=ifelse(g==1, "cornflowerblue","coral"),pch=16)
points(gd, pch=".", cex=1.5, col=ifelse(phatv>0.5, "cornflowerblue","coral"))
# dev.copy2pdf(file="fit-mXORDR-F1.pdf",height=10,width=12)