From 89326426eda8a829c57ad8f5bd90ef507950b489 Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 7 Apr 2017 13:34:50 +0800 Subject: [PATCH] add http header to requests --- README.md | 12 ++++++------ domain.go | 4 ++-- file.go | 2 +- http.go | 6 ++++++ v0.go | 1 + 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f7a11d5..31cf4cb 100644 --- a/README.md +++ b/README.md @@ -39,18 +39,18 @@ copy index `src_index` from `192.168.1.x` to `192.168.1.y:9200` and save with `d support Basic-Auth ``` -./bin/esm -s http://localhost:9200/ -x "src_index" -y "dest_index" -d http://localhost:9201 -n admin:111111 +./bin/esm -s http://localhost:9200 -x "src_index" -y "dest_index" -d http://localhost:9201 -n admin:111111 ``` copy settings and override shard size ``` -./bin/esm -s http://localhost:9200/ -x "src_index" -y "dest_index" -d http://localhost:9201 -m admin:111111 -c 10000 --shards=50 --copy_settings +./bin/esm -s http://localhost:9200 -x "src_index" -y "dest_index" -d http://localhost:9201 -m admin:111111 -c 10000 --shards=50 --copy_settings ``` copy settings and mapping, recreate target index, add query to source fetch, refresh after migration ``` -./bin/esm -s http://localhost:9200/ -x "src_index" -q=query:phone -y "dest_index" -d http://localhost:9201 -c 10000 --shards=5 --copy_settings --copy_mapping --force --refresh +./bin/esm -s http://localhost:9200 -x "src_index" -q=query:phone -y "dest_index" -d http://localhost:9201 -c 10000 --shards=5 --copy_settings --copy_mapping --force --refresh ``` @@ -66,7 +66,7 @@ loading data from dump files, bulk insert to another es instance support proxy ``` - ./bin/esm -d http://123345.ap-northeast-1.aws.found.io:9200/ -y "dest_index" -n admin:111111 -c 5000 -b 1 --refresh -i dump.bin --dest_proxy=http://127.0.0.1:9743 + ./bin/esm -d http://123345.ap-northeast-1.aws.found.io:9200 -y "dest_index" -n admin:111111 -c 5000 -b 1 --refresh -i dump.bin --dest_proxy=http://127.0.0.1:9743 ``` use sliced scroll(only available in elasticsearch v5) to speed scroll, and update shard number @@ -106,8 +106,8 @@ if download version is not fill you environment,you may try to compile it yourse -w, --workers= concurrency number for bulk workers, default is: "1" -b --bulk_size bulk size in MB" default:5 -v --log setting log level,options:trace,debug,info,warn,error - -i --input_file indexing from local dump file - -o --output_file output documents of source index into local file + -i --input_file indexing from local dump file, file format: {"_id":"xxx","_index":"xxx","_source":{"xxx":"xxx"},"_type":"xxx" } + -o --output_file output documents of source index into local file, file format same as input_file. --source_proxy set proxy to source http connections, ie: http://127.0.0.1:8080 --dest_proxy set proxy to destination http connections, ie: http://127.0.0.1:8080 --refresh refresh after migration finished diff --git a/domain.go b/domain.go index e56d724..52831d4 100644 --- a/domain.go +++ b/domain.go @@ -78,9 +78,9 @@ type Migrator struct{ type Config struct { // config options - SourceEs string `short:"s" long:"source" description:"source elasticsearch instance, ie: http://localhost:9200/"` + SourceEs string `short:"s" long:"source" description:"source elasticsearch instance, ie: http://localhost:9200"` Query string `short:"q" long:"query" description:"query against source elasticsearch instance, filter data before migrate, ie: name:medcl"` - TargetEs string `short:"d" long:"dest" description:"destination elasticsearch instance, ie: http://localhost:9201/"` + TargetEs string `short:"d" long:"dest" description:"destination elasticsearch instance, ie: http://localhost:9201"` SourceEsAuthStr string `short:"m" long:"source_auth" description:"basic auth of source elasticsearch instance, ie: user:pass"` TargetEsAuthStr string `short:"n" long:"dest_auth" description:"basic auth of target elasticsearch instance, ie: user:pass"` DocBufferCount int `short:"c" long:"count" description:"number of documents at a time: ie \"size\" in the scroll request" default:"10000"` diff --git a/file.go b/file.go index 7f4d54d..8606803 100644 --- a/file.go +++ b/file.go @@ -112,7 +112,7 @@ func (c *Migrator) NewFileDumpWorker(pb *pb.ProgressBar, wg *sync.WaitGroup) { } jsr,err:=json.Marshal(docI) - log.Debug(string(jsr)) + log.Trace(string(jsr)) if(err!=nil){ log.Error(err) } diff --git a/http.go b/http.go index 13ce402..a5f94be 100644 --- a/http.go +++ b/http.go @@ -32,6 +32,8 @@ func Get(url string,auth *Auth,proxy string) (*http.Response, string, []error) { request.SetBasicAuth(auth.User,auth.Pass) } + request.Header["Content-Type"]= "application/json" + if(len(proxy)>0){ request.Proxy(proxy) } @@ -47,6 +49,8 @@ func Post(url string,auth *Auth, body string,proxy string)(*http.Response, strin request.SetBasicAuth(auth.User,auth.Pass) } + request.Header["Content-Type"]= "application/json" + if(len(proxy)>0){ request.Proxy(proxy) } @@ -108,6 +112,8 @@ func Request(method string,r string,auth *Auth,body *bytes.Buffer,proxy string)( reqest.SetBasicAuth(auth.User,auth.Pass) } + reqest.Header.Set("Content-Type", "application/json") + resp,errs := client.Do(reqest) if errs != nil { diff --git a/v0.go b/v0.go index 05b25f0..776a8ff 100644 --- a/v0.go +++ b/v0.go @@ -43,6 +43,7 @@ func (s *ESAPIV0) ClusterHealth() *ClusterHealth { return &ClusterHealth{Name: s.Host, Status: "unreachable"} } + log.Debug(url) log.Debug(body) health := &ClusterHealth{}