.Rhistory

(len(xc2)/100)^(2/9)
4*(len(xc2)/100)^(2/9)
# 5.2 Sinusoidal function
tt = seq(from=0, to=9*pi, length.out=10000)
W1 = 0.2
Ph1 = pi
# 5.2 Sinusoidal function
tt = seq(from=0, to=20*pi, length.out=10000)
W1 = 0.2
Ph1 = pi
x2 = sin(2*pi*W1*tt + Ph1)
X11();plot(x2, type="l")
aTSA::adf.test(x2)
# cum sum of periodic function  plus a constant ==> it's now non-stationary
offset = 0.1
x21 = x2 + offset
xc2 = cumsum(x21)
X11();plot(xc2, type="l")
aTSA::adf.test(xc2)
d1 = 1.0
while(TRUE) {
xc2f = fracDiff_FFD(xc2, d=d1, thres=1e-5)
res = aTSA::adf.test(xc2f, output=FALSE)$type1
n1 = nrow(res)
cat("d1 : ", d1,  "xc2f len : ", len(na.omit(xc2f)), " p-val : ", res[n1,"p.value"], "\n")
if ( res[n1,"p.value"] < 0.05 )
break
d1 = d1 - 0.01
}
len(xc2)
# 5.2 (b) (iii)
xc2f = fracDiff_FFD(xc2, d=0.75, thres=1e-5)
len(na.omit(xc2f))
res = aTSA::adf.test(xc2f, output=FALSE)
res
# 5.2 (b) (iii)
xc2f = fracDiff_FFD(xc2, d=0.25, thres=1e-5)
len(na.omit(xc2f))
res = aTSA::adf.test(xc2f, output=FALSE)
res
# 5.2 (b) (iii)
xc2f = fracDiff_FFD(xc2, d=0.05, thres=1e-5)
len(na.omit(xc2f))
res = aTSA::adf.test(xc2f, output=FALSE)
res
X11();plot(na.omit(xc2f), type="l")
# 5.2 (b) (iii)
xc2f = fracDiff_FFD(xc2, d=0.05, thres=1e-2)
len(na.omit(xc2f))
res = aTSA::adf.test(xc2f, output=FALSE)
res
# 5.2 (b) (iii)
xc2f = fracDiff_FFD(xc2, d=0.05, thres=1e-1)
len(na.omit(xc2f))
aTSA::adf.test(xc2f, output=)
X11();plot(na.omit(xc2f), type="l")
# 5.2 (b) (iii)
xc2f = fracDiff_FFD(xc2, d=-0.05, thres=1e-1)
len(na.omit(xc2f))
# 5.2 (b) (iii)
xc2f = fracDiff_FFD(xc2, d=-0.05, thres=1e-2)
len(na.omit(xc2f))
# 5.2 (b) (iii)
xc2f = fracDiff_FFD(xc2, d=-0.05, thres=1e-3)
len(na.omit(xc2f))
res = aTSA::adf.test(xc2f, output=TRUE)
len(xc2f)
d1
d1=0.5
xc2f = fracDiff_FFD(xc2, d=d1, thres=1e-5)
res = aTSA::adf.test(xc2f, output=FALSE)$type1
res = aTSA::adf.test(xc2f, output=FALSE)
res
X11();plot(xc2f, type="l")
X11();plot(xc2, type="l")
d1=0.25
xc2f = fracDiff_FFD(xc2, d=d1, thres=1e-5)
res = aTSA::adf.test(xc2f, output=FALSE)
res
n1 = nrow(res$type1)
n1
cat("d1 : ", d1,  "xc2f len : ", len(na.omit(xc2f)), " p-val : ", res$type1[n1,"p.value"], "\n")
if ( res[n1,"p.value"] < 0.05 )
break
if ( res$type1[n1,"p.value"] < 0.05 )
break
X11();plot(xc2f, type="l")
d1=0.99
xc2f = fracDiff_FFD(xc2, d=d1, thres=1e-5)
res = aTSA::adf.test(xc2f, output=FALSE)
n1 = nrow(res$type1)
res
cat("d1 : ", d1,  "xc2f len : ", len(na.omit(xc2f)), " p-val : ", res$type1[n1,"p.value"], "\n")
X11();plot(xc2f, type="l")
res = aTSA::adf.test(xc2f, output=FALSE)
res
aTSA::adf.test(xc2f)
X11();plot(x2, type="l")
aTSA::adf.test(x2)
d1=0.999
xc2f = fracDiff_FFD(xc2, d=d1, thres=1e-5)
X11();plot(xc2f, type="l")
aTSA::adf.test(xc2f, output=TRUE)
# 5.2 Sinusoidal function
tt = seq(from=0, to=8*pi, length.out=10000)
W1 = 0.2
Ph1 = pi
x2 = sin(2*pi*W1*tt + Ph1)
X11();plot(x2, type="l")
# cum sum of periodic function  plus a constant ==> it's now non-stationary
offset = 0.1
x21 = x2 + offset
xc2 = cumsum(x21)
X11();plot(xc2, type="l")
aTSA::adf.test(xc2)
d1
xc2f = fracDiff_FFD(xc2, d=d1, thres=1e-5)
res = aTSA::adf.test(xc2f, output=FALSE)
res
d1
d1=0.9
xc2f = fracDiff_FFD(xc2, d=d1, thres=1e-5)
res = aTSA::adf.test(xc2f, output=FALSE)
res
X11();plot(xc2f, type="l")
d1
d1=0.5
xc2f = fracDiff_FFD(xc2, d=d1, thres=1e-5)
res = aTSA::adf.test(xc2f, output=FALSE)
res
X11();plot(xc2f, type="l")
d1=0.1
xc2f = fracDiff_FFD(xc2, d=d1, thres=1e-5)
res = aTSA::adf.test(xc2f, output=FALSE)
res
X11();plot(xc2f, type="l")
xc2f = fracDiff_FFD(xc2, d=1.0, thres=1e-5)
res = aTSA::adf.test(xc2f, output=FALSE)
res
X11();plot(xc2f, type="l")
X11();plot(xc2, type="l")
X11();plot(x2, type="l")
library(randomForest)
# library
library(igraph)
# create data:
links=data.frame(
source=c("A","A", "A", "A", "A","J", "B", "B", "C", "C", "D","I"),
target=c("B","B", "C", "D", "J","A","E", "F", "G", "H", "I","I")
)
# Turn it into igraph object
network <- graph_from_data_frame(d=links, directed=F)
# Count the number of degree for each node:
deg <- degree(network, mode="all")
# Plot
X11()
plot(network, vertex.size=deg*6, vertex.color=rgb(0.1,0.7,0.8,0.5) )
links
# Turn it into igraph object
network <- graph_from_data_frame(d=links, directed=T)
# Count the number of degree for each node:
deg <- degree(network, mode="all")
X11()
plot(network, vertex.size=deg*6, vertex.color=rgb(0.1,0.7,0.8,0.5) )
# create data:
links <- data.frame(
source=c("A","A", "A", "A", "A","J", "B", "B", "C", "C", "D","I"),
target=c("B","B", "C", "D", "J","A","E", "F", "G", "H", "I","I"),
importance=(sample(1:4, 12, replace=T))
)
nodes <- data.frame(
name=LETTERS[1:10],
carac=c( rep("young",3),rep("adult",2), rep("old",5))
)
# Turn it into igraph object
network <- graph_from_data_frame(d=links, vertices=nodes, directed=F)
# Make a palette of 3 colors
library(RColorBrewer)
coul  <- brewer.pal(3, "Set1")
# Create a vector of color
my_color <- coul[as.numeric(as.factor(V(network)$carac))]
X11()
plot(network, vertex.color=my_color)
# Add a legend
legend("bottomleft", legend=levels(as.factor(V(network)$carac))  , col = coul , bty = "n", pch=20 , pt.cex = 3, cex = 1.5, text.col=coul , horiz = FALSE, inset = c(0.1, 0.1))
X11()
plot(network, vertex.color=my_color, edge.width=E(network)$importance*2 )
legend("bottomleft", legend=levels(as.factor(V(network)$carac))  , col = coul , bty = "n", pch=20 , pt.cex = 3, cex = 1.5, text.col=coul , horiz = FALSE, inset = c(0.1, 0.1))
# Create data
data <- matrix(sample(0:1, 400, replace=TRUE, prob=c(0.8,0.2)), nrow=20)
network <- graph_from_adjacency_matrix(data , mode='undirected', diag=F )
X11()
par(mfrow=c(2,2), mar=c(1,1,1,1))
plot(network, layout=layout.sphere, main="sphere")
plot(network, layout=layout.circle, main="circle")
plot(network, layout=layout.random, main="random")
plot(network, layout=layout.fruchterman.reingold, main="fruchterman.reingold")
install.packages("sigmaNet")
library(igraph)
library(sigmaNet)
data(lesMis)
l <- layout_nicely(lesMis)
sig <- sigmaFromIgraph(graph = lesMis, layout = l)
#render basic visualization by calling the object
sig
#render basic visualization by calling the object
X11()
sig
library(magrittr)
data(lesMis)
l <- layout_nicely(lesMis)
#one size for all nodes
sig <- sigmaFromIgraph(graph = lesMis, layout = l) %>%
addNodeSize(oneSize = 3)
sig
#using a size attribute
sig <- sigmaFromIgraph(graph = lesMis, layout = l) %>%
addNodeSize(sizeMetric = 'degree', minSize = 2, maxSize = 8)
sig
#using a vector
customSize <- log10(degree(lesMis))
sig <- sigmaFromIgraph(graph = lesMis, layout = l) %>%
addNodeSize(sizeVector = customSize)
sig
l <- layout_nicely(lesMis)
sig <- sigmaFromIgraph(graph = lesMis, layout = l) %>%
addNodeLabels(labelAttr = 'label')
sig
l <- layout_nicely(lesMis)
sig <- sigmaFromIgraph(graph = lesMis, layout = l) %>%
addNodeLabels(labelAttr = 'label')
sig
l <- layout_nicely(lesMis)
#one color for all nodes
sig <- sigmaFromIgraph(graph = lesMis, layout = l) %>%
addNodeColors(oneColor = '#D95F02')
sig
#color based on attribute (edge betweenness cluster)
clust <- cluster_edge_betweenness(lesMis)$membership
V(lesMis)$group <- clust
sig <- sigmaFromIgraph(graph = lesMis, layout = l) %>%
addNodeColors(colorAttr = 'group', colorPal = 'Set1')
sig
library(nnet)
?nnet
library(igraph)
library(sigmaNet)
library(magrittr)
data(lesMis)
l <- layout_nicely(lesMis)
sig <- sigmaFromIgraph(graph = lesMis, layout = l)
sig
# library
library(igraph)
# create data:
links=data.frame(
source=c("A","A", "A", "A", "A","J", "B", "B", "C", "C", "D","I"),
target=c("B","B", "C", "D", "J","A","E", "F", "G", "H", "I","I")
)
# Turn it into igraph object
network <- graph_from_data_frame(d=links, directed=T)
# Count the number of degree for each node:
deg <- degree(network, mode="all")
X11()
plot(network, vertex.size=deg*6, vertex.color=rgb(0.1,0.7,0.8,0.5) )
# create data:
links <- data.frame(
source=c("A","A", "A", "A", "A","J", "B", "B", "C", "C", "D","I"),
target=c("B","B", "C", "D", "J","A","E", "F", "G", "H", "I","I"),
importance=(sample(1:4, 12, replace=T))
)
nodes <- data.frame(
name=LETTERS[1:10],
carac=c( rep("young",3),rep("adult",2), rep("old",5))
)
# Turn it into igraph object
network <- graph_from_data_frame(d=links, vertices=nodes, directed=F)
# Make a palette of 3 colors
library(RColorBrewer)
coul  <- brewer.pal(3, "Set1")
# Create a vector of color
my_color <- coul[as.numeric(as.factor(V(network)$carac))]
X11()
plot(network, vertex.color=my_color)
# Create data
data <- matrix(sample(0:1, 400, replace=TRUE, prob=c(0.8,0.2)), nrow=20)
network <- graph_from_adjacency_matrix(data , mode='undirected', diag=F )
X11()
par(mfrow=c(2,2), mar=c(1,1,1,1))
plot(network, layout=layout.sphere, main="sphere")
plot(network, layout=layout.fruchterman.reingold, main="fruchterman.reingold")
# create data:
links=data.frame(
source=c("A","A", "A", "A", "A","J", "B", "B", "C", "C", "D","I"),
target=c("B","B", "C", "D", "J","A","E", "F", "G", "H", "I","I")
)
# Turn it into igraph object
network <- graph_from_data_frame(d=links, directed=T)
# Count the number of degree for each node:
deg <- degree(network, mode="all")
deg
# create data:
links <- data.frame(
source=c("A","A", "A", "A", "A","J", "B", "B", "C", "C", "D","I"),
target=c("B","B", "C", "D", "J","A","E", "F", "G", "H", "I","I"),
importance=(sample(1:4, 12, replace=T))
)
links
?graph_from_data_frame
## A simple example with a couple of actors
## The typical case is that these tables are read in from files....
actors <- data.frame(name=c("Alice", "Bob", "Cecil", "David",
"Esmeralda"),
age=c(48,33,45,34,21),
gender=c("F","M","F","M","F"))
actors
relations <- data.frame(from=c("Bob", "Cecil", "Cecil", "David",
"David", "Esmeralda"),
to=c("Alice", "Bob", "Alice", "Alice", "Bob", "Alice"),
same.dept=c(FALSE,FALSE,TRUE,FALSE,FALSE,TRUE),
friendship=c(4,5,5,2,1,1), advice=c(4,5,5,4,2,3))
relations
g <- graph_from_data_frame(relations, directed=TRUE, vertices=actors)
print(g, e=TRUE, v=TRUE)
g
X11()
plot(g)
tkplot
tkplot(g)
tkplot(network)
E(network)
E(network)$color
E(network)$color <- "grey"
tkplot(network)
E(network)$color <- "red"
tkplot(network)
V(network)
V(network)$color <-"white"
tkplot(network)
V(network)$color
E(network)$color
?E
str(network)
?nls
home = getwd()
homne
home
home = getwd()
home
today = Sys.Date()
today.ch = as.character(today)
if ( file.exists("CoronaV.RData") ) load("CoronaV.RData")
ls()
home = getwd()
home
if ( file.exists("CoronaV.RData") ) load("CoronaV.RData")
ls()
install.packages("pacman")
pacman::p_load_gh(c(
"trinker/textshape",
"trinker/coreNLPsetup",
"trinker/parsent"
))
library(parsent)
pacman::p_load(parsent, magrittr)
txt <- c(
"Really, I like chocolate because it is good. It smells great.",
"Robots are rather evil and most are devoid of decency.",
"He is my friend.",
"Clifford the big red dog ate my lunch.",
"Professor Johns can not teach",
"",
NA
)
if(!exists('parse_ann')) {
parse_ann <- parse_annotator()
}
(x <- parser(txt, parse.annotator = parse_ann))
X11()
par(mar = c(0,0,0,.7) + 0.2)
plot(x[[2]])
X11()
par(
mfrow = c(3, 2),
mar = c(0,0,1,1) + 0.1
)
invisible(lapply(x[1:5], plot))
get_phrase_type(x, "NP") %>% take() %>% get_leaves()
get_phrase_type_regex(x, "VP") %>%
take() %>%
get_phrase_type_regex("(VB|MD)") %>%
take() %>%
get_leaves()
get_phrase_type_regex(x, "VP") %>%
take() %>%
get_phrase_type_regex("NP") %>%
take() %>%
get_leaves()
get_phrase_type(x, "NP")
res = get_phrase_type(x, "NP")
class(res)
len(res)
length(res)
res[[1]]
get_leaves(res[[1]])
get_leaves((res[[1]]))
get_leaves((res[[2]]))
get_leaves((res[[3]]))
x
txt
get_leaves((res[[4]]))
get_leaves((res[[5]]))
get_leaves((res[[6]]))
get_leaves((res[[7]]))
x = matrix(sample(0:9, 20, replace=TRUE), nrow=5)
x
colSums(x)
library(goodies)
library(quantmod)
library(goodies)
library(quantmod)
home = "/mnt/WanChai/Dropbox/GITHUB_REPO/Monumental_Day"
setwd(home)
home = "/mnt/WanChai/Dropbox/GITHUB_REPO/Monumental_Day"
setwd(home)
library(goodies)
library(quantmod)
home = "/mnt/WanChai/Dropbox/GITHUB_REPO/Monumental_Day"
setwd(home)
library(goodies)
library(quantmod)
home = "/mnt/WanChai/Dropbox/GITHUB_REPO/Monumental_Day"
setwd(home)
source(paste(home, "/utils/fun.R", sep=""))
ff = "YF_stock_indices.RData"
stock_mkt_indices = c("GSPC", "NDX", "RUT", "FTSE", "GDAXI", "FCHI", "N225", "HSI", "STI",
"AORD", "JKSE", "KS11", "TWII", "BVSP", "MXX")
data_dir = "/data/"
plot_dir = "/plots/"
setwd(home)
ff = paste( home, data_dir, "YF_stock_indices.RData",sep="")
ff
load(ff)
file.exists(ff)
x = NDX
tail(x)
tkr = strsplit(colnames(x), "\\.")[[1]][[1]]
tkr
# deal with NAs in the yf time series
xx = x[,6]
if ( sum(is.na(xx)) > 0 ) {
rl = rle(as.integer(is.na(xx)))
maxgap = max(rl$lengths[which(rl$values==1)])
cat("maxgap : ", maxgap, "\n")
cat("# NA segments : ", length(which(is.na(xx))), "\n")
x1 = na.spline(xx, maxgap=20, na.rm=TRUE)
cat("aft NA impute, :", length(which(is.na(x1))), "\n\n")
x[,6] = x1
}
x1 = x["2020"]
t1 = which.max(x1["::2020-06-04",6])
t2 = which.min(x1[,6])
t3 = nrow(x1)
# How long did it take to go from record high in Feb to the low in March ?
(dt1 = t2 - t1 + 1) # 23
# How long did it take to recover from March low?
(dt2 = t3 - t2 ) # 53
# Roll a window of 53 trading days over the history of index.
forw = as.data.frame(rollapply(zoo(x[,6]), width=53, look_inside, by=1, align="left"))
forw[,"date"] = as.Date(row.names(forw))
colnames(forw)[1] = "Forw"
backw = as.data.frame(rollapply(zoo(x[,6]), width=23, look_inside, by=1, align="right"))
backw[,"date"] = as.Date(row.names(backw))
colnames(backw)[1] = "Backw"
df = na.omit(merge(forw, backw, by="date", all=TRUE))
df1 = df[which(df[,"Backw"] < -0.2),]
df1[,"r"] = sapply(df1[,"Backw"], recov)
dim(df1)
df1
dte = df1[,"date"]
class(dte)
diff(dte)
len(diff(dte)> 3)
class(diff(dte))
len(as.integer(diff(dte)) > 3)
as.integer(diff(dte)) > 3
sum(as.integer(diff(dte)) > 3)
sum(as.integer(diff(dte)) > 5)
# How many rolling windows of more than 20% decline?
# Many are overlapped, so exclude those that are less than one week apart
sum(as.integer(diff(df1[,"date"])) > 5)
ix = which(df1[,"Forw"] > df1[,"r"])
df1[ix,]
tail(x1)
head(x1)
t1 = which.max(x1["::2020-06-04",6])
t2 = which.min(x1[,6])
t3 = nrow(x1)
t1
x1[t1,]
x1[t2,]
t3
tail(x1)
df1[ix,]
# How many rolling windows of more than 20% decline?
# Many are overlapped, so exclude those that are less than one week apart
sum(as.integer(diff(df1[,"date"])) > 5)
df1
diff(df1[,"date"]) > 5
ix = which(diff(df1[,"date"]) > 5)
df1[ix,]
len(ix)