Fixes deploy to Cloud.gov

Makefile

@@ -9,19 +9,21 @@ clean:
 	rm -f cmd/*/service.exe
 
 .PHONY: generate
-generate: config
+generate:
 	cd internal/postgres ; make generate
+	# cd internal/postgres/search_db ; make generate
 
 .PHONY: config
-config: clean
+config:
 	cd config ; make all || exit 1
 
 docker: 
 	docker build -t jemison/dev -f Dockerfile.dev .
 	docker build -t jemison/build -f Dockerfile.build .
 
 .PHONY: build
-build: generate 
+# lint
+build: clean config generate 
 	echo "build migrate"
 	cd cmd/migrate ; make build
 	echo "build admin"
@@ -36,19 +38,11 @@ build: generate
 	cd cmd/pack ; make build
 	echo "build serve"
 	cd cmd/serve ; make build
-	# echo "build validate"
-	# cd cmd/validate ; make build
+	echo "build validate"
+	cd cmd/validate ; make build
 	echo "build walk"
 	cd cmd/walk ; make build
 
-.PHONY: lint
-lint: generate
-	golangci-lint run -v
-
-.PHONY: containerlint
-containerlint:
-	docker run -v ${PWD}:/app -t jemison/build lint
-
 .PHONY: up
 up: build
 	docker compose up
@@ -72,46 +66,28 @@ macup:
 
 .PHONY: run
 run: clean generate
-	cd assets ; unzip -qq -o static.zip > /dev/null 2>&1
 	docker compose up
 
 .PHONY: cloc
 cloc:
 	docker run --rm -v ${PWD}:/tmp aldanial/cloc --exclude-dir=assets .
 
-delete_admin:
-	cf delete -f admin
-
-delete_entree:
-	cf delete -f entree
-
-delete_extract:
-	cf delete -f extract
-
-delete_fetch:
-	cf delete -f fetch
-
-delete_pack:
-	cf delete -f pack
-
-delete_serve:
-	cf delete -f serve
-
-delete_walk:
-	cf delete -f walk
-
 .PHONY: delete_all
-delete_all: delete_admin delete_entree delete_extract delete_fetch delete_pack delete_serve delete_walk
-
+delete_all: 
+	cd terraform ; make cfclean
 
 # I need to delete_all every time, because there is not enough RAM
 # in the sandbox to rolling deploy
 .PHONY: terraform
-terraform: delete_all
+terraform: delete_all build
 	docker run -v ${PWD}:/app -t jemison/build
 	cd terraform ; make apply_all
 
 .PHONY: docker_full_clean
 docker_full_clean:
 	-docker stop $(docker ps -a -q)
 	-docker rm $(docker ps -a -q)
+
+.PHONY: lint
+lint:
+	-golangci-lint run -v

cmd/serve/main.go

@@ -69,6 +69,7 @@ func main() {
 	templateFilesPath := s.GetParamString("template_files_path")
 	staticFilesPath := s.GetParamString("static_files_path")
 
+	externalScheme := s.GetParamString("external_scheme")
 	externalHost := s.GetParamString("external_host")
 	externalPort := s.GetParamInt64("external_port")
 
@@ -97,9 +98,9 @@ func main() {
 	engine.LoadHTMLGlob(templateFilesPath + "/*")
 
 	baseParams := gin.H{
-		"scheme":      "http",
-		"search_host": "localhost",
-		"search_port": "10000",
+		"scheme":      externalScheme,
+		"search_host": externalHost,
+		"search_port": externalPort,
 	}
 
 	engine.GET("/:tld", func(c *gin.Context) {

cmd/serve/templates/index.tmpl

@@ -55,7 +55,7 @@
         function DoTheSearch() {
           // :{{.search_port}}
           var host = window.location.host; 
-          var post_url = "{{.scheme}}://" + host + "/api/search"
+          var post_url = "{{.scheme}}://{{.search_host}}:{{.search_port}}/api/search"
           console.log("post_url", post_url);
           var query = document.getElementById('six-search').value
           console.log(query);

config/services/base.libsonnet

@@ -14,7 +14,7 @@ local parameters = [
   [
     'allowed_hosts',
     {
-      cf: 'all',
+      cf: 'dec15',
       container: 'dec15',
       localhost: 'nih',
     },

config/services/cf.jsonnet

@@ -1,18 +1,21 @@
 local A = import 'admin.libsonnet';
-local E = import 'extract.libsonnet';
+local EN = import 'entree.libsonnet';
+local EX = import 'extract.libsonnet';
 local F = import 'fetch.libsonnet';
 local M = import 'migrate.libsonnet';
 local P = import 'pack.libsonnet';
 local S = import 'serve.libsonnet';
 local V = import 'validate.libsonnet';
 local W = import 'walk.libsonnet';
 
+
 {
   // :: means "not visible in the output"
   EIGHT_SERVICES: {
     'user-provided': [
       A.cf,
-      E.cf,
+      EN.cf,
+      EX.cf,
       F.cf,
       M.cf,
       P.cf,

config/services/entree.libsonnet

@@ -22,7 +22,8 @@ local parameters = [
 {
   creds:: [[service] + x for x in credentials],
   params:: [[service] + x for x in parameters],
-  cf: B.params('credentials', 'cf', service, self.creds) +
+  cf: { name: service } +
+      B.params('credentials', 'cf', service, self.creds) +
       B.params('parameters', 'cf', service, self.params),
   container: { name: service } +
              B.params('credentials', 'container', service, self.creds) +

config/services/extract.libsonnet

@@ -34,7 +34,8 @@ local parameters = [
 {
   creds:: [[service] + x for x in credentials],
   params:: [[service] + x for x in parameters],
-  cf: B.params('credentials', 'cf', service, self.creds) +
+  cf: { name: service } +
+      B.params('credentials', 'cf', service, self.creds) +
       B.params('parameters', 'cf', service, self.params),
   container: { name: service } +
              B.params('credentials', 'container', service, self.creds) +

config/services/fetch.libsonnet

@@ -14,7 +14,7 @@ local parameters = [
   // With 'one_per_domain', 10 is appropriate... but largely immaterial.
   [
     'queue_model',
-    { cf: 'round_robin', container: 'round_robin' },
+    { cf: 'simple', container: 'simple' },
   ],
   [
     'workers',
@@ -45,7 +45,8 @@ local parameters = [
 {
   creds:: [[service] + x for x in credentials],
   params:: [[service] + x for x in parameters],
-  cf: B.params('credentials', 'cf', service, self.creds) +
+  cf: { name: service } +
+      B.params('credentials', 'cf', service, self.creds) +
       B.params('parameters', 'cf', service, self.params),
   container: { name: service } +
              B.params('credentials', 'container', service, self.creds) +

config/services/migrate.libsonnet

@@ -13,7 +13,8 @@ local parameters = [
 {
   creds:: [[service] + x for x in credentials],
   params:: [[service] + x for x in parameters],
-  cf: B.params('credentials', 'cf', service, self.creds) +
+  cf: { name: service } +
+      B.params('credentials', 'cf', service, self.creds) +
       B.params('parameters', 'cf', service, self.params),
   container: { name: service } +
              B.params('credentials', 'container', service, self.creds) +

config/services/pack.libsonnet

@@ -22,7 +22,8 @@ local parameters = [
 {
   creds:: [[service] + x for x in credentials],
   params:: [[service] + x for x in parameters],
-  cf: B.params('credentials', 'cf', service, self.creds) +
+  cf: { name: service } + 
+      B.params('credentials', 'cf', service, self.creds) +
       B.params('parameters', 'cf', service, self.params),
   container: { name: service } +
              B.params('credentials', 'container', service, self.creds) +

config/services/serve.libsonnet

@@ -59,7 +59,8 @@ local parameters = [
 {
   creds:: [[service] + x for x in credentials],
   params:: [[service] + x for x in parameters],
-  cf: B.params('credentials', 'cf', service, self.creds) +
+  cf: { name: service } +
+      B.params('credentials', 'cf', service, self.creds) +
       B.params('parameters', 'cf', service, self.params),
   container: { name: service } +
              B.params('credentials', 'container', service, self.creds) +

config/services/validate.libsonnet

@@ -22,7 +22,8 @@ local parameters = [
 {
   creds:: [[service] + x for x in credentials],
   params:: [[service] + x for x in parameters],
-  cf: B.params('credentials', 'cf', service, self.creds) +
+  cf: { name: service } +
+      B.params('credentials', 'cf', service, self.creds) +
       B.params('parameters', 'cf', service, self.params),
   container: { name: service } +
              B.params('credentials', 'container', service, self.creds) +

config/services/walk.libsonnet

@@ -34,7 +34,8 @@ local parameters = [
 {
   creds:: [[service] + x for x in credentials],
   params:: [[service] + x for x in parameters],
-  cf: B.params('credentials', 'cf', service, self.creds) +
+  cf: { name: service } +
+      B.params('credentials', 'cf', service, self.creds) +
       B.params('parameters', 'cf', service, self.params),
   container: { name: service } +
              B.params('credentials', 'container', service, self.creds) +

terraform/Makefile

@@ -26,7 +26,7 @@ admin: app
 	mkdir -p zips
 	cp ../cmd/admin/service.exe app/admin.exe
 	echo "web: ./admin.exe" > app/Procfile
-	cp ../assets/apt.yml app/apt.yml
+	cp apt.yml app/apt.yml
 	cd app ; zip -r -X ../zips/admin.zip . \
 		-x assets/databases/* \
 		-x assets/static.zip \
@@ -38,11 +38,10 @@ admin: app
 .PHONY: entree
 entree: app
 	mkdir -p zips
-	cp ../cmd/entree/service.exe app/extract.exe
-	cp ../cmd/entree/run.sh app/run.sh
-	echo "web: ./run.sh" > app/Procfile
-	cp ../assets/apt.yml app/apt.yml
+	cp ../cmd/entree/service.exe app/entree.exe
 	cp ../cmd/migrate/service.exe app/migrate.exe
+	echo "web: ./migrate.exe && ./entree.exe" > app/Procfile
+	cp apt.yml app/apt.yml
 	cd app ; zip -r -X ../zips/entree.zip . \
 		-x assets/databases/* \
 		-x assets/static.zip \
@@ -54,7 +53,7 @@ extract: app
 	mkdir -p zips
 	cp ../cmd/extract/service.exe app/extract.exe
 	echo "web: ./extract.exe" > app/Procfile
-	cp ../assets/apt.yml app/apt.yml
+	cp apt.yml app/apt.yml
 	cd app ; zip -r -X ../zips/extract.zip . \
 		-x assets/databases/* \
 		-x assets/static.zip \
@@ -66,7 +65,7 @@ fetch: app
 	mkdir -p zips
 	cp ../cmd/fetch/service.exe app/fetch.exe
 	echo "web: ./fetch.exe" > app/Procfile
-	cp ../assets/apt.yml app/apt.yml
+	cp apt.yml app/apt.yml
 	cd app ; zip -r -X ../zips/fetch.zip . \
 		-x assets/databases/* \
 		-x assets/static.zip \
@@ -77,7 +76,7 @@ fetch: app
 # migrate: app
 # 	cp ../cmd/migrate/service.exe app/migrate.exe
 # 	echo "web: ./migrate.exe" > app/Procfile
-# 	cp ../assets/apt.yml app/apt.yml
+# 	cp apt.yml app/apt.yml
 # 	cd app ; zip -r -X ../zips/migrate.zip . \
 # 		-x assets/databases/* \
 # 		-x assets/static.zip \
@@ -90,7 +89,7 @@ pack: app
 	mkdir -p zips
 	cp ../cmd/pack/service.exe app/pack.exe
 	echo "web: ./pack.exe" > app/Procfile
-	cp ../assets/apt.yml app/apt.yml
+	cp apt.yml app/apt.yml
 	cd app ; zip -r -X ../zips/pack.zip . \
 		-x assets/databases/* \
 		-x assets/static.zip \
@@ -102,22 +101,21 @@ serve: app
 	mkdir -p zips
 	cp ../cmd/serve/service.exe app/serve.exe
 	echo "web: ./serve.exe" > app/Procfile
-	cp ../assets/apt.yml app/apt.yml
-	cp ../assets/static.zip app/static.zip
-	cd app ; unzip -qq static.zip ; rm static.zip 
-	mkdir app/databases ;
-	cp ../assets/static/index.html app/static/index.html
+	cp apt.yml app/apt.yml
+	cp -R ../cmd/serve/static/ app/static/
+	cp -R ../cmd/serve/templates/ app/templates/
 	cd app ; zip -r -X ../zips/serve.zip . \
-		-x assets/databases/*sqlite* \
-		-x assets/static.zip \
 		-x terraform/*
+	rm -rf app/static/
+	rm -rf app/assets/
+	rm -rf app/templates/
 
 .PHONY: walk
 walk: app
 	mkdir -p zips
 	cp ../cmd/walk/service.exe app/walk.exe
 	echo "web: ./walk.exe" > app/Procfile
-	cp ../assets/apt.yml app/apt.yml
+	cp apt.yml app/apt.yml
 	cd app ; zip -r -X ../zips/walk.zip . \
 		-x assets/databases/* \
 		-x assets/static.zip \
@@ -165,7 +163,7 @@ container_build:
 	docker run -v ${PWD}:/app -t jemison/build
 
 .PHONY: plan_all
-plan_all: init admin entree extract fetch pack serve walk
+plan_all: init entree admin extract fetch pack serve walk
 	terraform plan -var-file="terraform.tfvars" -out plans/das.plan
 
 .PHONY: apply
@@ -176,3 +174,10 @@ apply:
 apply_all: clean plan_all apply
 	echo "apply_all"
 
+.PHONY: cfclean
+cfclean:
+	echo "------- CFCLEAN -------"
+	for app in admin entree extract fetch pack serve validate walk ; do cf delete -f -r $${app} ; done
+	# for service in db_queues db_search db_work ; do cf delete-service -f -w $${service} ; done
+	# for service in s3_backup s3_fetch s3_extract s3_meta ; do cf delete-service -f -w $${service} ; done
+