From 57a64ff88c174cdda2822cf6a6570887b6b87c33 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Wed, 15 May 2019 10:25:20 +0000 Subject: [PATCH] Squashed 'tools/' changes from 41c5622..d6cc704 d6cc704 Fix comment 7139116 Revert "Push comments to the left so they don't appear in scripts" e47e58f Push comments to the left so they don't appear in scripts 3945fce Remove nonexistent env var GIT_TAG cd62992 Merge pull request #156 from weaveworks/drop-quay af0eb51 Merge pull request #157 from weaveworks/fix-image-tag-prefix-length 0b9aee4 Fix image-tag object name prefix length to 8 chars. 813c28f Move from CircleCI 1.0 to 2.0 425cf4e Move from quay.io to Dockerhub 87ccf4f Merge pull request #155 from weaveworks/go-1-12 c31bc28 Update lint script to work with Go 1.12 ed8e380 Update to Go 1.12.1 ec369f5 Merge pull request #153 from dholbach/drop-email ef7418d weave-users mailing list is closed: https://groups.google.com/a/weave.works/forum/#!topic/weave-users/0QXWGOPdBfY 6954a57 Merge pull request #144 from weaveworks/golang-1.11.1 9649eed Upgrade build image from golang:1.10.0-strech to 1.11.1-strech 59263a7 Merge pull request #141 from weaveworks/update-context e235c9b Merge pull request #143 from weaveworks/gc-wks-test-vms c865b4c scheduler: please lint/flake8 da61568 scheduler: please lint/yapf ce9d78e scheduler: do not cache discovery doc e4b7873 scheduler: add comment about GCP projects' IAM roles needed to list/delete instances and firewall rules ff7ec8e scheduler: add comment about CircleCI projects' access via the API 2477d98 scheduler: deploy command now sets the current datetime as the version 5fcd880 scheduler: pass CircleCI API token in for private projects 6b8c323 scheduler: more details in case of failure to get running builds from CircleCI 0871aff scheduler: downgrade google-api-python-client from 1.7.4 to 1.6.7 b631e7f scheduler: add GC of WKS test VMs and firewall rules a923a32 scheduler: document setup and deployment 013f508 scheduler: lock dependencies' versions 6965a4a Merge pull request #142 from weaveworks/fix-build 23298c6 Fix golint expects import golang.org/x/lint/golint 482f4cd Context is now part of the Go standard library 2bbc9a0 Merge pull request #140 from weaveworks/sched-http-retry c3726de Add retries to sched util http calls 2cc7b5a Merge pull request #139 from meghalidhoble/master fd9b0a7 Change : Modified the lint tools to skip the shfmt check if not installed. Why the change : For ppc64le the specific version of shfmt is not available, hence skipped completely the installation of shfmt tool. Thus this change made. bc645c7 Merge pull request #138 from dholbach/add-license-file a642e02 license: add Apache 2.0 license text 9bf5956 Merge pull request #109 from hallum/master d971d82 Merge pull request #134 from weaveworks/2018-07-03-gcloud-regepx 32e7aa2 Merge pull request #137 from weaveworks/gcp-fw-allow-kube-apiserver bbb6735 Allow CI to access k8s API server on GCP instances 764d46c Merge pull request #135 from weaveworks/2018-07-04-docker-ansible-playbook ecc2a4e Merge pull request #136 from weaveworks/2018-07-05-gcp-private-ips 209b7fb tools: Add private_ips to the terraform output 369a655 tools: Add an ansible playbook that just installs docker a643e27 tools: Use --filter instead of --regexp with gcloud b8eca88 Merge pull request #128 from weaveworks/actually-say-whats-wrong 379ce2b Merge pull request #133 from weaveworks/fix-decrypt 3b906b5 Fix incompatibility with recent versions of OpenSSL f091ab4 Merge pull request #132 from weaveworks/add-opencontainers-labels-to-dockerfiles 248def1 Inject git revision in Dockerfiles 64f2c28 Add org.opencontainers.image.* labels to Dockerfiles ea96d8e add information about how to get help (#129) f066ccd Make yapf diff failure look like an error 34d81d7 Merge pull request #127 from weaveworks/golang-1.10.0-stretch 89a0b4f Use golang:1.10.0-stretch image. ca69607 Merge pull request #126 from weaveworks/disable-apt-daily-test f5dc5d5 Create "setup-apt" role 7fab441 Rename bazel to bazel-rules (#125) ccc8316 Revert "Gocyclo should return error code if issues detected" (#124) 1fe184f Bazel rules for building gogo protobufs (#123) b917bb8 Merge pull request #122 from weaveworks/fix-scope-gc c029ce0 Add regex to match scope VMs 0d4824b Merge pull request #121 from weaveworks/provisioning-readme-terraform 5a82d64 Move terraform instructions to tf section d285d78 Merge pull request #120 from weaveworks/gocyclo-return-value 76b94a4 Do not spawn subshell when reading cyclo output 93b3c0d Use golang:1.9.2-stretch image d40728f Gocyclo should return error code if issues detected c4ac1c3 Merge pull request #114 from weaveworks/tune-spell-check 8980656 Only check files 12ebc73 Don't spell-check pki files 578904a Special-case spell-check the same way we do code checks e772ed5 Special-case on mime type and extension using just patterns ae82b50 Merge pull request #117 from weaveworks/test-verbose 8943473 Propagate verbose flag to 'go test'. 7c79b43 Merge pull request #113 from weaveworks/update-shfmt-instructions 258ef01 Merge pull request #115 from weaveworks/extra-linting e690202 Use tools in built image to lint itself 126eb56 Add shellcheck to bring linting in line with scope 63ad68f Don't run lint on files under .git 51d908a Update shfmt instructions e91cb0d Merge pull request #112 from weaveworks/add-python-lint-tools 0c87554 Add yapf and flake8 to golang build image 35679ee Merge pull request #110 from weaveworks/parallel-push-errors 3ae41b6 Remove unneeded if block 51ff31a Exit on first error 0faad9f Check for errors when pushing images in parallel d87cd02 Add arg flag override for destination socks host:port in pacfile. 74dc626 Merge pull request #108 from weaveworks/disable-apt-daily b4f1d91 Merge pull request #107 from weaveworks/docker-17-update 7436aa1 Override apt daily job to not run immediately on boot 7980f15 Merge pull request #106 from weaveworks/document-docker-install-role f741e53 Bump to Docker 17.06 from CE repo 61796a1 Update Docker CE Debian repo details 0d86f5e Allow for Docker package to be named docker-ce 065c68d Document selection of Docker installation role. 3809053 Just --porcelain; it defaults to v1 11400ea Merge pull request #105 from weaveworks/remove-weaveplugin-remnants b8b4d64 remove weaveplugin remnants 35099c9 Merge pull request #104 from weaveworks/pull-docker-py cdd48fc Pull docker-py to speed tests/builds up. e1c6c24 Merge pull request #103 from weaveworks/test-build-tags d5d71e0 Add -tags option so callers can pass in build tags 8949b2b Merge pull request #98 from weaveworks/git-status-tag ac30687 Merge pull request #100 from weaveworks/python_linting 4b125b5 Pin yapf & flake8 versions 7efb485 Lint python linting function 444755b Swap diff direction to reflect changes required c5b2434 Install flake8 & yapf 5600eac Lint python in build-tools repo 0b02ca9 Add python linting c011c0d Merge pull request #79 from kinvolk/schu/python-shebang 6577d07 Merge pull request #99 from weaveworks/shfmt-version 00ce0dc Use git status instead of diff to add 'WIP' tag 411fd13 Use shfmt v1.3.0 instead of latest from master. 0d6d4da Run shfmt 1.3 on the code. 5cdba32 Add sudo c322ca8 circle.yml: Install shfmt binary. e59c225 Install shfmt 1.3 binary. 30706e6 Install pyhcl in the build container. 960d222 Merge pull request #97 from kinvolk/alban/update-shfmt-3 1d535c7 shellcheck: fix escaping issue 5542498 Merge pull request #96 from kinvolk/alban/update-shfmt-2 32f7cc5 shfmt: fix coding style 09f72af lint: print the diff in case of error 571c7d7 Merge pull request #95 from kinvolk/alban/update-shfmt bead6ed Update for latest shfmt b08dc4d Update for latest shfmt (#94) 2ed8aaa Add no-race argument to test script (#92) 80dd78e Merge pull request #91 from weaveworks/upgrade-go-1.8.1 08dcd0d Please ./lint as shfmt changed its rules between 1.0.0 and 1.3.0. a8bc9ab Upgrade default Go version to 1.8.1. 31d069d Change Python shebang to `#!/usr/bin/env python` git-subtree-dir: tools git-subtree-split: d6cc704a2892e8d85aa8fa4d201c1a404f02dfa4 --- .circleci/config.yml | 36 ++ COPYING.LGPL-3 | 175 ++++++ LICENSE | 13 + README.md | 17 +- bazel-rules/BUILD.bazel | 26 + bazel-rules/gogo.bzl | 36 ++ build/Makefile | 5 +- build/golang/Dockerfile | 19 +- build/golang/build.sh | 4 +- build/haskell/Dockerfile | 7 + build/haskell/build.sh | 2 +- circle.yml | 50 -- config_management/README.md | 22 + config_management/group_vars/all | 6 +- .../docker-configuration/files/docker.conf | 2 +- .../tasks/debian.yml | 35 ++ .../docker-from-docker-ce-repo/tasks/main.yml | 35 +- .../tasks/redhat.yml | 29 + .../tasks/debian.yml | 2 +- .../setup-apt/files/apt-daily.timer.conf | 2 + .../roles/setup-apt/tasks/main.yml | 10 + .../roles/weave-net-utilities/tasks/main.yml | 9 + config_management/setup_bare_docker.yml | 16 + config_management/setup_weave-net_dev.yml | 1 + config_management/setup_weave-net_test.yml | 1 + dependencies/cross_versions.py | 118 ++-- dependencies/list_os_images.sh | 6 +- dependencies/list_versions.py | 503 ++++++++++-------- image-tag | 7 +- integration/config.sh | 5 +- lint | 95 ++-- provisioning/README.md | 7 +- provisioning/aws/variables.tf | 2 + provisioning/gcp/main.tf | 14 + provisioning/gcp/outputs.tf | 4 + provisioning/setup.sh | 23 +- push-images | 21 +- rebuild-image | 3 +- sched | 21 +- scheduler/README.md | 66 ++- scheduler/main.py | 302 ++++++----- scheduler/requirements.txt | 4 +- socks/Dockerfile | 8 +- socks/Makefile | 3 +- socks/main.go | 19 +- test | 24 +- 46 files changed, 1237 insertions(+), 578 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 COPYING.LGPL-3 create mode 100644 LICENSE create mode 100644 bazel-rules/BUILD.bazel create mode 100644 bazel-rules/gogo.bzl delete mode 100644 circle.yml create mode 100644 config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml create mode 100644 config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml create mode 100644 config_management/roles/setup-apt/files/apt-daily.timer.conf create mode 100644 config_management/roles/setup-apt/tasks/main.yml create mode 100644 config_management/setup_bare_docker.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..785428ae --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,36 @@ +version: 2 +jobs: + build: + # Use 'machine' type so we can run the lint step with directory mounted + machine: + docker_layer_caching: true + working_directory: /home/circleci/src/github.com/weaveworks/build-tools + environment: + GOPATH: /home/circleci/ + steps: + - checkout + - run: cd build; make + - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./lint . + - run: cd cover; make + # Socks makefile needs to overwrite Go std library + - run: sudo chmod a+wr --recursive /usr/local/go/pkg + - run: cd socks; make + - run: cd runner; make + + - deploy: + command: | + if [ "${CIRCLE_BRANCH}" == "master" ]; then + cd build + docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD + for image in $(make images); do + # Push all tags - latest and git-tag + docker push "${image}" + + # Tag the built images with something derived from the base images in + # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a + # base image would lead to a tag of "1.8.0-stretch" + IMG_TAG=$(make "imagetag-${image#weaveworks/build-}") + docker tag "${image}:latest" "${image}:${IMG_TAG}" + docker push "${image}:${IMG_TAG}" + done + fi diff --git a/COPYING.LGPL-3 b/COPYING.LGPL-3 new file mode 100644 index 00000000..f01171d4 --- /dev/null +++ b/COPYING.LGPL-3 @@ -0,0 +1,175 @@ +./integration/assert.sh is a copy of + + https://github.com/lehmannro/assert.sh/blob/master/assert.sh + +Since it was added to this codebase, it has only received cosmetic +modifications. As it is licensed under the LGPL-3, here's the license +text in its entirety: + + + + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..9cd1640b --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright 2018 Weaveworks. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index 9092b8e2..389e4980 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ Included in this repo are tools shared by weave.git and scope.git. They include +- ```bazel-rules```: Bazel build rules used in our projects - ```build```: a set of docker base-images for building weave projects. These should be used instead of giving each project its own build image. @@ -32,7 +33,11 @@ Included in this repo are tools shared by weave.git and scope.git. They include ## Requirements - ```lint``` requires shfmt to lint sh files; get shfmt with - ```go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt``` +``` +curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 +chmod +x shfmt +``` + (we pin that version, and it doesn't build from the source repo any more) ## Using build-tools.git @@ -50,3 +55,13 @@ To update the code in build-tools.git, the process is therefore: - PR into build-tools.git, go through normal review process etc. - Do `git subtree pull --prefix tools https://github.com/weaveworks/build-tools.git master --squash` in your repo, and PR that. + +## Getting Help + +If you have any questions about, feedback for or problems with `build-tools`: + +- Invite yourself to the Weave Users Slack. +- Ask a question on the [#general](https://weave-community.slack.com/messages/general/) slack channel. +- [File an issue](https://github.com/weaveworks/build-tools/issues/new). + +Your feedback is always welcome! diff --git a/bazel-rules/BUILD.bazel b/bazel-rules/BUILD.bazel new file mode 100644 index 00000000..751b3707 --- /dev/null +++ b/bazel-rules/BUILD.bazel @@ -0,0 +1,26 @@ +load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler") + +go_proto_compiler( + name = "gogo_proto", + deps = [ + "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library", + "//vendor/github.com/gogo/protobuf/proto:go_default_library", + "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library", + ], + plugin = "@com_github_gogo_protobuf//protoc-gen-gogoslick", + visibility = ["//visibility:public"], +) + +go_proto_compiler( + name = "gogo_grpc", + deps = [ + "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library", + "//vendor/github.com/gogo/protobuf/proto:go_default_library", + "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + "//vendor/golang.org/x/net/context:go_default_library", + ], + plugin = "@com_github_gogo_protobuf//protoc-gen-gogoslick", + options = ["plugins=grpc"], + visibility = ["//visibility:public"], +) diff --git a/bazel-rules/gogo.bzl b/bazel-rules/gogo.bzl new file mode 100644 index 00000000..82f24461 --- /dev/null +++ b/bazel-rules/gogo.bzl @@ -0,0 +1,36 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_repository") + + +_BUILD_FILE = """ +proto_library( + name = "gogoproto", + srcs = ["gogo.proto"], + deps = [ + "@com_google_protobuf//:descriptor_proto", + ], + visibility = ["//visibility:public"], +) +""" + +def _go_repository_impl(ctx): + ctx.file("BUILD.bazel", content="") + ctx.file("github.com/gogo/protobuf/gogoproto/BUILD.bazel", content=_BUILD_FILE) + ctx.template("github.com/gogo/protobuf/gogoproto/gogo.proto", ctx.attr._proto) + +_gogo_proto_repository = repository_rule( + implementation = _go_repository_impl, + attrs = { + "_proto": attr.label(default="//vendor/github.com/gogo/protobuf/gogoproto:gogo.proto"), + }, +) + +def gogo_dependencies(): + go_repository( + name = "com_github_gogo_protobuf", + importpath = "github.com/gogo/protobuf", + urls = ["https://codeload.github.com/ianthehat/protobuf/zip/2adc21fd136931e0388e278825291678e1d98309"], + strip_prefix = "protobuf-2adc21fd136931e0388e278825291678e1d98309", + type = "zip", + build_file_proto_mode="disable", + ) + _gogo_proto_repository(name = "internal_gogo_proto_repository") diff --git a/build/Makefile b/build/Makefile index cea049be..5c5db8b5 100644 --- a/build/Makefile +++ b/build/Makefile @@ -3,15 +3,16 @@ # Boiler plate for bulding Docker containers. # All this must go at top of file I'm afraid. -IMAGE_PREFIX := quay.io/weaveworks/build- +IMAGE_PREFIX := weaveworks/build- IMAGE_TAG := $(shell ../image-tag) +GIT_REVISION := $(shell git rev-parse HEAD) UPTODATE := .uptodate # Every directory with a Dockerfile in it builds an image called # $(IMAGE_PREFIX). Dependencies (i.e. things that go in the image) # still need to be explicitly declared. %/$(UPTODATE): %/Dockerfile %/* - $(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/ + $(SUDO) docker build --build-arg=revision=$(GIT_REVISION) -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/ $(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG) touch $@ diff --git a/build/golang/Dockerfile b/build/golang/Dockerfile index 6936eec2..e20873a4 100644 --- a/build/golang/Dockerfile +++ b/build/golang/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.8.0-stretch +FROM golang:1.12.1-stretch RUN apt-get update && \ apt-get install -y \ curl \ @@ -11,9 +11,14 @@ RUN apt-get update && \ python-pip \ python-requests \ python-yaml \ + shellcheck \ unzip && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN pip install attrs +RUN pip install attrs pyhcl yapf==0.16.2 flake8==3.3.0 +RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \ + echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2 shfmt" | sha256sum -c && \ + chmod +x shfmt && \ + mv shfmt /usr/bin RUN go clean -i net && \ go install -tags netgo std && \ go install -race -tags netgo std @@ -25,11 +30,10 @@ RUN go get -tags netgo \ github.com/gogo/protobuf/gogoproto \ github.com/gogo/protobuf/protoc-gen-gogoslick \ github.com/golang/dep/... \ - github.com/golang/lint/golint \ + golang.org/x/lint/golint \ github.com/golang/protobuf/protoc-gen-go \ github.com/kisielk/errcheck \ github.com/mjibson/esc \ - github.com/mvdan/sh/cmd/shfmt \ github.com/prometheus/prometheus/cmd/promtool && \ rm -rf /go/pkg /go/src RUN mkdir protoc && \ @@ -44,3 +48,10 @@ RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \ touch /var/run/secrets/kubernetes.io/serviceaccount/token COPY build.sh / ENTRYPOINT ["/build.sh"] + +ARG revision +LABEL maintainer="Weaveworks " \ + org.opencontainers.image.title="golang" \ + org.opencontainers.image.source="https://github.com/weaveworks/build-tools/tree/master/build/golang" \ + org.opencontainers.image.revision="${revision}" \ + org.opencontainers.image.vendor="Weaveworks" diff --git a/build/golang/build.sh b/build/golang/build.sh index e8f9df58..cf70e1c5 100755 --- a/build/golang/build.sh +++ b/build/golang/build.sh @@ -13,8 +13,8 @@ fi # will have awkward ownership. So we switch to a user with the # same user and group IDs as source directory. We have to set a # few things up so that sudo works without complaining later on. -uid=$(stat --format="%u" $SRC_PATH) -gid=$(stat --format="%g" $SRC_PATH) +uid=$(stat --format="%u" "$SRC_PATH") +gid=$(stat --format="%g" "$SRC_PATH") echo "weave:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd echo "weave:*:::::::" >>/etc/shadow echo "weave ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers diff --git a/build/haskell/Dockerfile b/build/haskell/Dockerfile index 8d40c662..79f34a80 100644 --- a/build/haskell/Dockerfile +++ b/build/haskell/Dockerfile @@ -2,3 +2,10 @@ FROM fpco/stack-build:lts-8.9 COPY build.sh / COPY copy-libraries /usr/local/bin/ ENTRYPOINT ["/build.sh"] + +ARG revision +LABEL maintainer="Weaveworks " \ + org.opencontainers.image.title="haskell" \ + org.opencontainers.image.source="https://github.com/weaveworks/build-tools/tree/master/build/haskell" \ + org.opencontainers.image.revision="${revision}" \ + org.opencontainers.image.vendor="Weaveworks" diff --git a/build/haskell/build.sh b/build/haskell/build.sh index bd529053..e80d2abb 100755 --- a/build/haskell/build.sh +++ b/build/haskell/build.sh @@ -9,4 +9,4 @@ if [ -z "${SRC_PATH:-}" ]; then exit 1 fi -make -C $SRC_PATH BUILD_IN_CONTAINER=false $* +make -C "$SRC_PATH" BUILD_IN_CONTAINER=false "$@" diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 57686d69..00000000 --- a/circle.yml +++ /dev/null @@ -1,50 +0,0 @@ -machine: - services: - - docker - environment: - GOPATH: /home/ubuntu - SRCDIR: /home/ubuntu/src/github.com/weaveworks/tools - PATH: $PATH:$HOME/bin - -dependencies: - post: - - sudo chmod a+wr --recursive /usr/local/go/pkg - - go clean -i net - - go install -tags netgo std - - mkdir -p $(dirname $SRCDIR) - - cp -r $(pwd)/ $SRCDIR - - | - cd $SRCDIR; - go get \ - github.com/fzipp/gocyclo \ - github.com/golang/lint/golint \ - github.com/kisielk/errcheck \ - github.com/fatih/hclfmt \ - gopkg.in/mvdan/sh.v1/cmd/shfmt - -test: - override: - - cd $SRCDIR; ./lint . - - cd $SRCDIR/cover; make - - cd $SRCDIR/socks; make - - cd $SRCDIR/runner; make - - cd $SRCDIR/build; make - -deployment: - snapshot: - branch: master - commands: - - docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASS" "$DOCKER_REGISTRY_URL" - - | - cd $SRCDIR/build; - for image in $(make images); do - # Tag the built images with the revision of this repo. - docker push "${image}:${GIT_TAG}" - - # Tag the built images with something derived from the base images in - # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a - # base image would lead to a tag of "1.8.0-stretch" - IMG_TAG=$(make "imagetag-${image#quay.io/weaveworks/build-}") - docker tag "${image}:latest" "${image}:${IMG_TAG}" - docker push "${image}:${IMG_TAG}" - done diff --git a/config_management/README.md b/config_management/README.md index e98b9b4e..bf1f6f65 100644 --- a/config_management/README.md +++ b/config_management/README.md @@ -113,6 +113,28 @@ N.B.: `--ssh-extra-args` is used to provide: * `StrictHostKeyChecking=no`: as VMs come and go, the same IP can be used by a different machine, so checking the host's SSH key may fail. Note that this introduces a risk of a man-in-the-middle attack. * `UserKnownHostsFile=/dev/null`: if you previously connected a VM with the same IP but a different public key, and added it to `~/.ssh/known_hosts`, SSH may still fail to connect, hence we use `/dev/null` instead of `~/.ssh/known_hosts`. + +### Docker installation role + +Various ways to install Docker are provided: + +- `docker-from-docker-ce-repo` +- `docker-from-docker-repo` +- `docker-from-get.docker.com` +- `docker-from-tarball` + +each producing a slightly different outcome, which can be useful for testing various setup scenarios. + +The `docker-install` role selects one of the above ways to install Docker based on the `docker_install_role` variable. +The default value for this variable is configured in `group_vars/all`. +You can however override it with whichever role you would want to run by passing the name of the role as a key-value pair in `extra-vars`, e.g.: + +``` +ansible-playbook .yml \ + --extra-vars "docker_install_role=docker-from-docker-ce-repo" +``` + + ## Resources * [https://www.vagrantup.com/docs/provisioning/ansible.html](https://www.vagrantup.com/docs/provisioning/ansible.html) diff --git a/config_management/group_vars/all b/config_management/group_vars/all index 24ac08cf..d728cce8 100644 --- a/config_management/group_vars/all +++ b/config_management/group_vars/all @@ -1,8 +1,8 @@ --- -go_version: 1.7.4 +go_version: 1.8.1 terraform_version: 0.8.5 -docker_version: 1.11.2 -docker_install_role: 'docker-from-get.docker.com' +docker_version: 17.06 +docker_install_role: 'docker-from-docker-ce-repo' kubernetes_version: 1.6.1 kubernetes_cni_version: 0.5.1 kubernetes_token: '123456.0123456789123456' diff --git a/config_management/roles/docker-configuration/files/docker.conf b/config_management/roles/docker-configuration/files/docker.conf index 6d02b55e..626d8022 100644 --- a/config_management/roles/docker-configuration/files/docker.conf +++ b/config_management/roles/docker-configuration/files/docker.conf @@ -1,3 +1,3 @@ [Service] ExecStart= -ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000" +ExecStart=/usr/bin/dockerd -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000" diff --git a/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml b/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml new file mode 100644 index 00000000..3e2ae127 --- /dev/null +++ b/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml @@ -0,0 +1,35 @@ +--- +# Debian / Ubuntu specific: + +- name: install dependencies for docker repository + package: + name: "{{ item }}" + state: present + with_items: + - apt-transport-https + - ca-certificates + +- name: add apt key for the docker repository + apt_key: + keyserver: hkp://ha.pool.sks-keyservers.net:80 + id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 + state: present + register: apt_key_docker_repo + +- name: add docker's apt repository ({{ ansible_distribution | lower }}-{{ ansible_distribution_release }}) + apt_repository: + repo: deb https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename|lower }} stable + state: present + register: apt_docker_repo + +- name: update apt's cache + apt: + update_cache: yes + when: apt_key_docker_repo.changed or apt_docker_repo.changed + +- name: install docker-engine + package: + name: "{{ item }}" + state: present + with_items: + - docker-ce={{ docker_version }}* diff --git a/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml b/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml index ea9a3fa4..0acb6d8c 100644 --- a/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml +++ b/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml @@ -1,29 +1,10 @@ -# Docker installation from Docker's CentOS Community Edition -# See also: https://docs.docker.com/engine/installation/linux/centos/ +--- +# Set up Docker +# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install -- name: remove all potentially pre existing packages - yum: - name: '{{ item }}' - state: absent - with_items: - - docker - - docker-common - - container-selinux - - docker-selinux - - docker-engine +# Distribution-specific tasks: +- include: debian.yml + when: ansible_os_family == "Debian" -- name: install yum-utils - yum: - name: yum-utils - state: present - -- name: add docker ce repo - command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - -# Note that Docker CE versions do not follow regular Docker versions, but look -# like, for example: "17.03.0.el7" -- name: install docker - yum: - name: 'docker-ce-{{ docker_version }}' - update_cache: yes - state: present +- include: redhat.yml + when: ansible_os_family == "RedHat" diff --git a/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml b/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml new file mode 100644 index 00000000..ea9a3fa4 --- /dev/null +++ b/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml @@ -0,0 +1,29 @@ +# Docker installation from Docker's CentOS Community Edition +# See also: https://docs.docker.com/engine/installation/linux/centos/ + +- name: remove all potentially pre existing packages + yum: + name: '{{ item }}' + state: absent + with_items: + - docker + - docker-common + - container-selinux + - docker-selinux + - docker-engine + +- name: install yum-utils + yum: + name: yum-utils + state: present + +- name: add docker ce repo + command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + +# Note that Docker CE versions do not follow regular Docker versions, but look +# like, for example: "17.03.0.el7" +- name: install docker + yum: + name: 'docker-ce-{{ docker_version }}' + update_cache: yes + state: present diff --git a/config_management/roles/docker-from-get.docker.com/tasks/debian.yml b/config_management/roles/docker-from-get.docker.com/tasks/debian.yml index 97b5b7a3..7444194e 100644 --- a/config_management/roles/docker-from-get.docker.com/tasks/debian.yml +++ b/config_management/roles/docker-from-get.docker.com/tasks/debian.yml @@ -5,4 +5,4 @@ shell: curl -sSL https://get.docker.com/gpg | sudo apt-key add - - name: install docker - shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ | sh' + shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ -e s/docker-ce/docker-ce={{ docker_version }}*/ | sh' diff --git a/config_management/roles/setup-apt/files/apt-daily.timer.conf b/config_management/roles/setup-apt/files/apt-daily.timer.conf new file mode 100644 index 00000000..bd19c61f --- /dev/null +++ b/config_management/roles/setup-apt/files/apt-daily.timer.conf @@ -0,0 +1,2 @@ +[Timer] +Persistent=false diff --git a/config_management/roles/setup-apt/tasks/main.yml b/config_management/roles/setup-apt/tasks/main.yml new file mode 100644 index 00000000..3593cf70 --- /dev/null +++ b/config_management/roles/setup-apt/tasks/main.yml @@ -0,0 +1,10 @@ +--- +# Set up apt + +# Ubuntu runs an apt update process that will run on first boot from image. +# This is of questionable value when the machines are only going to live for a few minutes. +# If you leave them on they will run the process daily. +# Also we have seen the update process create a 'defunct' process which then throws off Weave Net smoke-test checks. +# So, we override the 'persistent' setting so it will still run at the scheduled time but will not try to catch up on first boot. +- name: copy apt daily override + copy: src=apt-daily.timer.conf dest=/etc/systemd/system/apt-daily.timer.d/ diff --git a/config_management/roles/weave-net-utilities/tasks/main.yml b/config_management/roles/weave-net-utilities/tasks/main.yml index 89038597..6883d23a 100644 --- a/config_management/roles/weave-net-utilities/tasks/main.yml +++ b/config_management/roles/weave-net-utilities/tasks/main.yml @@ -45,3 +45,12 @@ - alpine - aanand/docker-dnsutils - weaveworks/hello-world + +- name: docker pull docker-py which is used by tests + docker_image: + name: joffrey/docker-py + tag: '{{ item }}' + state: present + with_items: + - '1.8.1' + - '1.9.0-rc2' diff --git a/config_management/setup_bare_docker.yml b/config_management/setup_bare_docker.yml new file mode 100644 index 00000000..fac8405f --- /dev/null +++ b/config_management/setup_bare_docker.yml @@ -0,0 +1,16 @@ +--- +################################################################################ +# Install Docker from Docker's official repository +################################################################################ + +- name: install docker + hosts: all + gather_facts: false # required in case Python is not available on the host + become: true + become_user: root + + pre_tasks: + - include: library/setup_ansible_dependencies.yml + + roles: + - docker-install diff --git a/config_management/setup_weave-net_dev.yml b/config_management/setup_weave-net_dev.yml index bdfa08e9..1923d011 100644 --- a/config_management/setup_weave-net_dev.yml +++ b/config_management/setup_weave-net_dev.yml @@ -13,6 +13,7 @@ - include: library/setup_ansible_dependencies.yml roles: + - setup-apt - dev-tools - golang-from-tarball - docker-install diff --git a/config_management/setup_weave-net_test.yml b/config_management/setup_weave-net_test.yml index fbd155df..7125d054 100644 --- a/config_management/setup_weave-net_test.yml +++ b/config_management/setup_weave-net_test.yml @@ -13,6 +13,7 @@ - include: library/setup_ansible_dependencies.yml roles: + - setup-apt - docker-install - weave-net-utilities - kubernetes-install diff --git a/dependencies/cross_versions.py b/dependencies/cross_versions.py index bc93cf31..dd920f0e 100755 --- a/dependencies/cross_versions.py +++ b/dependencies/cross_versions.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Generate the cross product of latest versions of Weave Net's dependencies: # - Go @@ -20,74 +20,72 @@ from itertools import product # See also: /usr/include/sysexits.h -_ERROR_RUNTIME=1 -_ERROR_ILLEGAL_ARGS=64 +_ERROR_RUNTIME = 1 +_ERROR_ILLEGAL_ARGS = 64 + def _usage(error_message=None): - if error_message: - stderr.write('ERROR: ' + error_message + linesep) - stdout.write(linesep.join([ - 'Usage:', - ' cross_versions.py [OPTION]...', - 'Examples:', - ' cross_versions.py', - ' cross_versions.py -r', - ' cross_versions.py --rc', - ' cross_versions.py -l', - ' cross_versions.py --latest', - 'Options:', - '-l/--latest Include only the latest version of each major and minor versions sub-tree.', - '-r/--rc Include release candidate versions.', - '-h/--help Prints this!', - '' - ])) + if error_message: + stderr.write('ERROR: ' + error_message + linesep) + stdout.write( + linesep.join([ + 'Usage:', ' cross_versions.py [OPTION]...', 'Examples:', + ' cross_versions.py', ' cross_versions.py -r', + ' cross_versions.py --rc', ' cross_versions.py -l', + ' cross_versions.py --latest', 'Options:', + '-l/--latest Include only the latest version of each major and' + ' minor versions sub-tree.', + '-r/--rc Include release candidate versions.', + '-h/--help Prints this!', '' + ])) + def _validate_input(argv): - try: - config = { - 'rc': False, - 'latest': False - } - opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc']) - for opt, value in opts: - if opt in ('-h', '--help'): - _usage() - exit() - if opt in ('-l', '--latest'): - config['latest'] = True - if opt in ('-r', '--rc'): - config['rc'] = True - if len(args) != 0: - raise ValueError('Unsupported argument(s): %s.' % args) - return config - except GetoptError as e: - _usage(str(e)) - exit(_ERROR_ILLEGAL_ARGS) - except ValueError as e: - _usage(str(e)) - exit(_ERROR_ILLEGAL_ARGS) + try: + config = {'rc': False, 'latest': False} + opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc']) + for opt, value in opts: + if opt in ('-h', '--help'): + _usage() + exit() + if opt in ('-l', '--latest'): + config['latest'] = True + if opt in ('-r', '--rc'): + config['rc'] = True + if len(args) != 0: + raise ValueError('Unsupported argument(s): %s.' % args) + return config + except GetoptError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + except ValueError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + def _versions(dependency, config): - return map(str, - filter_versions( - get_versions_from(DEPS[dependency]['url'], DEPS[dependency]['re']), - DEPS[dependency]['min'], - **config - ) - ) + return map(str, + filter_versions( + get_versions_from(DEPS[dependency]['url'], + DEPS[dependency]['re']), + DEPS[dependency]['min'], **config)) + def cross_versions(config): - docker_versions = _versions('docker', config) - k8s_versions = _versions('kubernetes', config) - return product(docker_versions, k8s_versions) + docker_versions = _versions('docker', config) + k8s_versions = _versions('kubernetes', config) + return product(docker_versions, k8s_versions) + def main(argv): - try: - config = _validate_input(argv) - print(linesep.join('\t'.join(triple) for triple in cross_versions(config))) - except Exception as e: - print(str(e)) - exit(_ERROR_RUNTIME) + try: + config = _validate_input(argv) + print(linesep.join('\t'.join(triple) + for triple in cross_versions(config))) + except Exception as e: + print(str(e)) + exit(_ERROR_RUNTIME) + if __name__ == '__main__': - main(argv[1:]) + main(argv[1:]) diff --git a/dependencies/list_os_images.sh b/dependencies/list_os_images.sh index 7d2495da..139a0814 100755 --- a/dependencies/list_os_images.sh +++ b/dependencies/list_os_images.sh @@ -8,11 +8,11 @@ Dependencies: - gcloud, Google Cloud Platform's CLI - aws, Usage: - $ ./$(basename "$0") PROVIDER OS + \$ ./$(basename "$0") PROVIDER OS PROVIDER={gcp} OS={ubuntu|debian|centos} Example: - $ ./$(basename "$0") gcp ubuntu + \$ ./$(basename "$0") gcp ubuntu ubuntu-os-cloud/ubuntu-1204-lts ubuntu-os-cloud/ubuntu-1404-lts ubuntu-os-cloud/ubuntu-1604-lts @@ -57,7 +57,7 @@ fi case "$1" in 'gcp') - gcloud compute images list --standard-images --regexp=".*?$2.*" \ + gcloud compute images list --standard-images --filter="name~'.*?$2.*'" \ --format="csv[no-heading][separator=/](selfLink.map().scope(projects).segment(0),family)" \ | sort -d ;; diff --git a/dependencies/list_versions.py b/dependencies/list_versions.py index 3b756cd1..e008ecfe 100755 --- a/dependencies/list_versions.py +++ b/dependencies/list_versions.py @@ -1,17 +1,32 @@ -#!/usr/bin/python +#!/usr/bin/env python # List all available versions of Weave Net's dependencies: # - Go # - Docker # - Kubernetes # -# Depending on the parameters passed, it can gather the equivalent of the below bash one-liners: -# git ls-remote --tags https://github.com/golang/go | grep -oP '(?<=refs/tags/go)[\.\d]+$' | sort --version-sort -# git ls-remote --tags https://github.com/golang/go | grep -oP '(?<=refs/tags/go)[\.\d]+rc\d+$' | sort --version-sort | tail -n 1 -# git ls-remote --tags https://github.com/docker/docker | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' | sort --version-sort -# git ls-remote --tags https://github.com/docker/docker | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-rc\d*$' | sort --version-sort | tail -n 1 -# git ls-remote --tags https://github.com/kubernetes/kubernetes | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' | sort --version-sort -# git ls-remote --tags https://github.com/kubernetes/kubernetes | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-beta\.\d+$' | sort --version-sort | tail -n 1 +# Depending on the parameters passed, it can gather the equivalent of the below +# bash one-liners: +# git ls-remote --tags https://github.com/golang/go \ +# | grep -oP '(?<=refs/tags/go)[\.\d]+$' \ +# | sort --version-sort +# git ls-remote --tags https://github.com/golang/go \ +# | grep -oP '(?<=refs/tags/go)[\.\d]+rc\d+$' \ +# | sort --version-sort \ +# | tail -n 1 +# git ls-remote --tags https://github.com/docker/docker \ +# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' \ +# | sort --version-sort +# git ls-remote --tags https://github.com/docker/docker \ +# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-rc\d*$' \ +# | sort --version-sort \ +# | tail -n 1 +# git ls-remote --tags https://github.com/kubernetes/kubernetes \ +# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' \ +# | sort --version-sort +# git ls-remote --tags https://github.com/kubernetes/kubernetes \ +# | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-beta\.\d+$' \ +# | sort --version-sort | tail -n 1 # # Dependencies: # - python @@ -23,7 +38,7 @@ from os import linesep, path from sys import argv, exit, stdout, stderr from getopt import getopt, GetoptError -from subprocess import Popen, PIPE, STDOUT +from subprocess import Popen, PIPE from pkg_resources import parse_version from itertools import groupby from six.moves import filter @@ -31,236 +46,298 @@ import re # See also: /usr/include/sysexits.h -_ERROR_RUNTIME=1 -_ERROR_ILLEGAL_ARGS=64 - -_TAG_REGEX='^[0-9a-f]{40}\s+refs/tags/%s$' -_VERSION='version' -DEPS={ - 'go': { - 'url': 'https://github.com/golang/go', - 're': 'go(?P<%s>[\d\.]+(?:rc\d)*)' % _VERSION, - 'min': None - }, - 'docker': { - 'url': 'https://github.com/docker/docker', - 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-rc\d)*)' % _VERSION, - # Weave Net only works with Docker from 1.10.0 onwards, so we ignore all previous versions: - 'min': '1.10.0' - }, - 'kubernetes': { - 'url': 'https://github.com/kubernetes/kubernetes', - 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-beta\.\d)*)' % _VERSION, - # Weave Kube requires Kubernetes 1.4.2+, so we ignore all previous versions: - 'min': '1.4.2' - } +_ERROR_RUNTIME = 1 +_ERROR_ILLEGAL_ARGS = 64 + +_TAG_REGEX = '^[0-9a-f]{40}\s+refs/tags/%s$' +_VERSION = 'version' +DEPS = { + 'go': { + 'url': 'https://github.com/golang/go', + 're': 'go(?P<%s>[\d\.]+(?:rc\d)*)' % _VERSION, + 'min': None + }, + 'docker': { + 'url': 'https://github.com/docker/docker', + 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-rc\d)*)' % _VERSION, + # Weave Net only works with Docker from 1.10.0 onwards, so we ignore + # all previous versions: + 'min': '1.10.0', + }, + 'kubernetes': { + 'url': 'https://github.com/kubernetes/kubernetes', + 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-beta\.\d)*)' % _VERSION, + # Weave Kube requires Kubernetes 1.4.2+, so we ignore all previous + # versions: + 'min': '1.4.2', + } } + class Version(object): - ''' Helper class to parse and manipulate (sort, filter, group) software versions. ''' - def __init__(self, version): - self.version = version - self.digits = [int(x) if x else 0 for x in re.match('(\d*)\.?(\d*)\.?(\d*).*?', version).groups()] - self.major, self.minor, self.patch = self.digits - self.__parsed = parse_version(version) - self.is_rc = self.__parsed.is_prerelease - def __lt__ (self, other): - return self.__parsed.__lt__(other.__parsed) - def __gt__ (self, other): - return self.__parsed.__gt__(other.__parsed) - def __le__ (self, other): - return self.__parsed.__le__(other.__parsed) - def __ge__ (self, other): - return self.__parsed.__ge__(other.__parsed) - def __eq__ (self, other): - return self.__parsed.__eq__(other.__parsed) - def __ne__ (self, other): - return self.__parsed.__ne__(other.__parsed) - def __str__(self): - return self.version - def __repr__(self): - return self.version + ''' Helper class to parse and manipulate (sort, filter, group) software + versions. ''' + + def __init__(self, version): + self.version = version + self.digits = [ + int(x) if x else 0 + for x in re.match('(\d*)\.?(\d*)\.?(\d*).*?', version).groups() + ] + self.major, self.minor, self.patch = self.digits + self.__parsed = parse_version(version) + self.is_rc = self.__parsed.is_prerelease + + def __lt__(self, other): + return self.__parsed.__lt__(other.__parsed) + + def __gt__(self, other): + return self.__parsed.__gt__(other.__parsed) + + def __le__(self, other): + return self.__parsed.__le__(other.__parsed) + + def __ge__(self, other): + return self.__parsed.__ge__(other.__parsed) + + def __eq__(self, other): + return self.__parsed.__eq__(other.__parsed) + + def __ne__(self, other): + return self.__parsed.__ne__(other.__parsed) + + def __str__(self): + return self.version + + def __repr__(self): + return self.version + def _read_go_version_from_dockerfile(): - # Read Go version from weave/build/Dockerfile - dockerfile_path = path.join(path.dirname(path.dirname(path.dirname(path.realpath(__file__)))), 'build', 'Dockerfile') - with open(dockerfile_path, 'r') as f: - for line in f: - m = re.match('^FROM golang:(\S*)$', line) - if m: - return m.group(1) - raise RuntimeError("Failed to read Go version from weave/build/Dockerfile. You may be running this script from somewhere else than weave/tools.") + # Read Go version from weave/build/Dockerfile + dockerfile_path = path.join( + path.dirname(path.dirname(path.dirname(path.realpath(__file__)))), + 'build', 'Dockerfile') + with open(dockerfile_path, 'r') as f: + for line in f: + m = re.match('^FROM golang:(\S*)$', line) + if m: + return m.group(1) + raise RuntimeError( + "Failed to read Go version from weave/build/Dockerfile." + " You may be running this script from somewhere else than weave/tools." + ) + def _try_set_min_go_version(): - ''' Set the current version of Go used to build Weave Net's containers as the minimum version. ''' - try: - DEPS['go']['min'] = _read_go_version_from_dockerfile() - except IOError as e: - stderr.write('WARNING: No minimum Go version set. Root cause: %s%s' % (e, linesep)) + ''' Set the current version of Go used to build Weave Net's containers as + the minimum version. ''' + try: + DEPS['go']['min'] = _read_go_version_from_dockerfile() + except IOError as e: + stderr.write('WARNING: No minimum Go version set. Root cause: %s%s' % + (e, linesep)) + def _sanitize(out): - return out.decode('ascii').strip().split(linesep) + return out.decode('ascii').strip().split(linesep) + def _parse_tag(tag, version_pattern, debug=False): - ''' Parse Git tag output's line using the provided `version_pattern`, e.g.: - >>> _parse_tag('915b77eb4efd68916427caf8c7f0b53218c5ea4a refs/tags/v1.4.6', 'v(?P\d+\.\d+\.\d+(?:\-beta\.\d)*)') - '1.4.6' - ''' - pattern = _TAG_REGEX % version_pattern - m = re.match(pattern, tag) - if m: - return m.group(_VERSION) - elif debug: - stderr.write('ERROR: Failed to parse version out of tag [%s] using [%s].%s' % (tag, pattern, linesep)) + ''' Parse Git tag output's line using the provided `version_pattern`, e.g.: + >>> _parse_tag( + '915b77eb4efd68916427caf8c7f0b53218c5ea4a refs/tags/v1.4.6', + 'v(?P\d+\.\d+\.\d+(?:\-beta\.\d)*)') + '1.4.6' + ''' + pattern = _TAG_REGEX % version_pattern + m = re.match(pattern, tag) + if m: + return m.group(_VERSION) + elif debug: + stderr.write( + 'ERROR: Failed to parse version out of tag [%s] using [%s].%s' % + (tag, pattern, linesep)) + def get_versions_from(git_repo_url, version_pattern): - ''' Get release and release candidates' versions from the provided Git repository. ''' - git = Popen(shlex.split('git ls-remote --tags %s' % git_repo_url), stdout=PIPE) - out, err = git.communicate() - status_code = git.returncode - if status_code != 0: - raise RuntimeError('Failed to retrieve git tags from %s. Status code: %s. Output: %s. Error: %s' % (git_repo_url, status_code, out, err)) - return list(filter(None, (_parse_tag(line, version_pattern) for line in _sanitize(out)))) + ''' Get release and release candidates' versions from the provided Git + repository. ''' + git = Popen( + shlex.split('git ls-remote --tags %s' % git_repo_url), stdout=PIPE) + out, err = git.communicate() + status_code = git.returncode + if status_code != 0: + raise RuntimeError('Failed to retrieve git tags from %s. ' + 'Status code: %s. Output: %s. Error: %s' % + (git_repo_url, status_code, out, err)) + return list( + filter(None, (_parse_tag(line, version_pattern) + for line in _sanitize(out)))) + def _tree(versions, level=0): - ''' Group versions by major, minor and patch version digits. ''' - if not versions or level >= len(versions[0].digits): - return # Empty versions or no more digits to group by. - versions_tree = [] - for _, versions_group in groupby(versions, lambda v: v.digits[level]): - subtree = _tree(list(versions_group), level+1) - if subtree: - versions_tree.append(subtree) - # Return the current subtree if non-empty, or the list of "leaf" versions: - return versions_tree if versions_tree else versions + ''' Group versions by major, minor and patch version digits. ''' + if not versions or level >= len(versions[0].digits): + return # Empty versions or no more digits to group by. + versions_tree = [] + for _, versions_group in groupby(versions, lambda v: v.digits[level]): + subtree = _tree(list(versions_group), level + 1) + if subtree: + versions_tree.append(subtree) + # Return the current subtree if non-empty, or the list of "leaf" versions: + return versions_tree if versions_tree else versions + def _is_iterable(obj): - ''' - Check if the provided object is an iterable collection, i.e. not a string, e.g. a list, a generator: - >>> _is_iterable('string') - False - >>> _is_iterable([1, 2, 3]) - True - >>> _is_iterable((x for x in [1, 2, 3])) - True - ''' - return hasattr(obj, '__iter__') and not isinstance(obj, str) + ''' + Check if the provided object is an iterable collection, i.e. not a string, + e.g. a list, a generator: + >>> _is_iterable('string') + False + >>> _is_iterable([1, 2, 3]) + True + >>> _is_iterable((x for x in [1, 2, 3])) + True + ''' + return hasattr(obj, '__iter__') and not isinstance(obj, str) + def _leaf_versions(tree, rc): - ''' - Recursively traverse the versions tree in a depth-first fashion, - and collect the last node of each branch, i.e. leaf versions. - ''' - versions = [] - if _is_iterable(tree): - for subtree in tree: - versions.extend(_leaf_versions(subtree, rc)) - if not versions: - if rc: - last_rc = next(filter(lambda v: v.is_rc, reversed(tree)), None) - last_prod = next(filter(lambda v: not v.is_rc, reversed(tree)), None) - if last_rc and last_prod and (last_prod < last_rc): - versions.extend([last_prod, last_rc]) - elif not last_prod: - versions.append(last_rc) - else: - # Either there is no RC, or we ignore the RC as older than the latest production version: - versions.append(last_prod) - else: - versions.append(tree[-1]) - return versions + ''' + Recursively traverse the versions tree in a depth-first fashion, + and collect the last node of each branch, i.e. leaf versions. + ''' + versions = [] + if _is_iterable(tree): + for subtree in tree: + versions.extend(_leaf_versions(subtree, rc)) + if not versions: + if rc: + last_rc = next(filter(lambda v: v.is_rc, reversed(tree)), None) + last_prod = next( + filter(lambda v: not v.is_rc, reversed(tree)), None) + if last_rc and last_prod and (last_prod < last_rc): + versions.extend([last_prod, last_rc]) + elif not last_prod: + versions.append(last_rc) + else: + # Either there is no RC, or we ignore the RC as older than + # the latest production version: + versions.append(last_prod) + else: + versions.append(tree[-1]) + return versions + def filter_versions(versions, min_version=None, rc=False, latest=False): - ''' Filter provided versions - - >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=False, rc=False) - [1.0.0, 1.0.1, 1.1.1, 2.0.0] - - >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=True, rc=False) - [1.0.1, 1.1.1, 2.0.0] - - >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=False, rc=True) - [1.0.0-beta.1, 1.0.0, 1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0] - - >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version='1.1.0', latest=False, rc=True) - [1.1.1, 1.1.2-rc1, 2.0.0] - - >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=True, rc=True) - [1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0] - - >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version='1.1.0', latest=True, rc=True) - [1.1.1, 1.1.2-rc1, 2.0.0] - ''' - versions = sorted([Version(v) for v in versions]) - if min_version: - min_version = Version(min_version) - versions = [v for v in versions if v >= min_version] - if not rc: - versions = [v for v in versions if not v.is_rc] - if latest: - versions_tree = _tree(versions) - return _leaf_versions(versions_tree, rc) - else: - return versions + ''' Filter provided versions + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version=None, latest=False, rc=False) + [1.0.0, 1.0.1, 1.1.1, 2.0.0] + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version=None, latest=True, rc=False) + [1.0.1, 1.1.1, 2.0.0] + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version=None, latest=False, rc=True) + [1.0.0-beta.1, 1.0.0, 1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0] + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version='1.1.0', latest=False, rc=True) + [1.1.1, 1.1.2-rc1, 2.0.0] + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version=None, latest=True, rc=True) + [1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0] + + >>> filter_versions( + ['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], + min_version='1.1.0', latest=True, rc=True) + [1.1.1, 1.1.2-rc1, 2.0.0] + ''' + versions = sorted([Version(v) for v in versions]) + if min_version: + min_version = Version(min_version) + versions = [v for v in versions if v >= min_version] + if not rc: + versions = [v for v in versions if not v.is_rc] + if latest: + versions_tree = _tree(versions) + return _leaf_versions(versions_tree, rc) + else: + return versions + def _usage(error_message=None): - if error_message: - stderr.write('ERROR: ' + error_message + linesep) - stdout.write(linesep.join([ - 'Usage:', - ' list_versions.py [OPTION]... [DEPENDENCY]', - 'Examples:', - ' list_versions.py go', - ' list_versions.py -r docker', - ' list_versions.py --rc docker', - ' list_versions.py -l kubernetes', - ' list_versions.py --latest kubernetes', - 'Options:', - '-l/--latest Include only the latest version of each major and minor versions sub-tree.', - '-r/--rc Include release candidate versions.', - '-h/--help Prints this!', - '' - ])) + if error_message: + stderr.write('ERROR: ' + error_message + linesep) + stdout.write( + linesep.join([ + 'Usage:', ' list_versions.py [OPTION]... [DEPENDENCY]', + 'Examples:', ' list_versions.py go', + ' list_versions.py -r docker', + ' list_versions.py --rc docker', + ' list_versions.py -l kubernetes', + ' list_versions.py --latest kubernetes', 'Options:', + '-l/--latest Include only the latest version of each major and' + ' minor versions sub-tree.', + '-r/--rc Include release candidate versions.', + '-h/--help Prints this!', '' + ])) + def _validate_input(argv): - try: - config = { - 'rc': False, - 'latest': False - } - opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc']) - for opt, value in opts: - if opt in ('-h', '--help'): - _usage() - exit() - if opt in ('-l', '--latest'): - config['latest'] = True - if opt in ('-r', '--rc'): - config['rc'] = True - if len(args) != 1: - raise ValueError('Please provide a dependency to get versions of. Expected 1 argument but got %s: %s.' % (len(args), args)) - dependency=args[0].lower() - if dependency not in DEPS.keys(): - raise ValueError('Please provide a valid dependency. Supported one dependency among {%s} but got: %s.' % (', '.join(DEPS.keys()), dependency)) - return dependency, config - except GetoptError as e: - _usage(str(e)) - exit(_ERROR_ILLEGAL_ARGS) - except ValueError as e: - _usage(str(e)) - exit(_ERROR_ILLEGAL_ARGS) + try: + config = {'rc': False, 'latest': False} + opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc']) + for opt, value in opts: + if opt in ('-h', '--help'): + _usage() + exit() + if opt in ('-l', '--latest'): + config['latest'] = True + if opt in ('-r', '--rc'): + config['rc'] = True + if len(args) != 1: + raise ValueError('Please provide a dependency to get versions of.' + ' Expected 1 argument but got %s: %s.' % + (len(args), args)) + dependency = args[0].lower() + if dependency not in DEPS.keys(): + raise ValueError( + 'Please provide a valid dependency.' + ' Supported one dependency among {%s} but got: %s.' % + (', '.join(DEPS.keys()), dependency)) + return dependency, config + except GetoptError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + except ValueError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + def main(argv): - try: - dependency, config = _validate_input(argv) - if dependency == 'go': - _try_set_min_go_version() - versions = get_versions_from(DEPS[dependency]['url'], DEPS[dependency]['re']) - versions = filter_versions(versions, DEPS[dependency]['min'], **config) - print(linesep.join(map(str, versions))) - except Exception as e: - print(str(e)) - exit(_ERROR_RUNTIME) + try: + dependency, config = _validate_input(argv) + if dependency == 'go': + _try_set_min_go_version() + versions = get_versions_from(DEPS[dependency]['url'], + DEPS[dependency]['re']) + versions = filter_versions(versions, DEPS[dependency]['min'], **config) + print(linesep.join(map(str, versions))) + except Exception as e: + print(str(e)) + exit(_ERROR_RUNTIME) + if __name__ == '__main__': - main(argv[1:]) + main(argv[1:]) diff --git a/image-tag b/image-tag index d1fd2f72..6a11cb24 100755 --- a/image-tag +++ b/image-tag @@ -4,6 +4,9 @@ set -o errexit set -o nounset set -o pipefail -WORKING_SUFFIX=$(if ! git diff --exit-code --quiet HEAD >&2; then echo "-WIP"; else echo ""; fi) +WORKING_SUFFIX=$(if git status --porcelain | grep -qE '^(?:[^?][^ ]|[^ ][^?])\s'; then echo "-WIP"; else echo ""; fi) BRANCH_PREFIX=$(git rev-parse --abbrev-ref HEAD) -echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short HEAD)$WORKING_SUFFIX" + +# Fix the object name prefix length to 8 characters to have it consistent across the system. +# See https://git-scm.com/docs/git-rev-parse#Documentation/git-rev-parse.txt---shortlength +echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short=8 HEAD)$WORKING_SUFFIX" diff --git a/integration/config.sh b/integration/config.sh index 6bf20860..54192192 100644 --- a/integration/config.sh +++ b/integration/config.sh @@ -115,11 +115,8 @@ rm_containers() { start_suite() { for host in $HOSTS; do [ -z "$DEBUG" ] || echo "Cleaning up on $host: removing all containers and resetting weave" - PLUGIN_ID=$(docker_on "$host" ps -aq --filter=name=weaveplugin) - PLUGIN_FILTER="cat" - [ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID" # shellcheck disable=SC2046 - rm_containers "$host" $(docker_on "$host" ps -aq 2>/dev/null | $PLUGIN_FILTER) + rm_containers "$host" $(docker_on "$host" ps -aq 2>/dev/null) run_on "$host" "docker network ls | grep -q ' weave ' && docker network rm weave" || true weave_on "$host" reset 2>/dev/null done diff --git a/lint b/lint index 1589bd9e..40ad4e7f 100755 --- a/lint +++ b/lint @@ -6,7 +6,8 @@ # # For shell files, it runs shfmt. If you don't have that installed, you can get # it with: -# go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt +# curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 +# chmod +x shfmt # # With no arguments, it lints the current files staged # for git commit. Or you can pass it explicit filenames @@ -50,14 +51,6 @@ spell_check() { local filename="$1" local lint_result=0 - # we don't want to spell check tar balls, binaries, Makefile and json files - if file "$filename" | grep executable >/dev/null 2>&1; then - return $lint_result - fi - if [[ $filename == *".tar" || $filename == *".gz" || $filename == *".json" || $(basename "$filename") == "Makefile" ]]; then - return $lint_result - fi - # misspell is completely optional. If you don't like it # don't have it installed. if ! type misspell >/dev/null 2>&1; then @@ -72,6 +65,7 @@ spell_check() { } lint_go() { + # This function is called on a whole directory containing Go files local filename="$1" local lint_result=0 @@ -80,7 +74,7 @@ lint_go() { echo "${filename}: run gofmt -s -w ${filename}" fi - go tool vet "${filename}" || lint_result=$? + go vet "${filename}" || lint_result=$? # golint is completely optional. If you don't like it # don't have it installed. @@ -113,9 +107,12 @@ lint_sh() { local filename="$1" local lint_result=0 - if ! diff <(shfmt -i 4 "${filename}") "${filename}" >/dev/null; then - lint_result=1 - echo "${filename}: run shfmt -i 4 -w ${filename}" + # Skip shfmt validation, if not installed + if type shfmt >/dev/null 2>&1; then + if ! diff -u "${filename}" <(shfmt -i 4 "${filename}"); then + lint_result=1 + echo "${filename}: run shfmt -i 4 -w ${filename}" + fi fi # the shellcheck is completely optional. If you don't like it @@ -131,7 +128,7 @@ lint_tf() { local filename="$1" local lint_result=0 - if ! diff <(hclfmt "${filename}") "${filename}" >/dev/null; then + if ! diff -u <(hclfmt "${filename}") "${filename}"; then lint_result=1 echo "${filename}: run hclfmt -w ${filename}" fi @@ -153,6 +150,21 @@ lint_md() { return $lint_result } +lint_py() { + local filename="$1" + local lint_result=0 + + if yapf --diff "${filename}" | grep -qE '^[+-]'; then + lint_result=1 + echo "${filename} needs reformatting. Run: yapf --in-place ${filename}" + else + # Only run flake8 if yapf passes, since they pick up a lot of similar issues + flake8 "${filename}" || lint_result=1 + fi + + return $lint_result +} + lint() { filename="$1" ext="${filename##*\.}" @@ -170,18 +182,24 @@ lint() { *.pb.go) return ;; esac - if [[ "$(file --mime-type "${filename}" | awk '{print $2}')" == "text/x-shellscript" ]]; then - ext="sh" - fi + mimetype=$(file --mime-type "${filename}" | awk '{print $2}') - case "$ext" in - go) lint_go "${filename}" || lint_result=1 ;; - sh) lint_sh "${filename}" || lint_result=1 ;; - tf) lint_tf "${filename}" || lint_result=1 ;; - md) lint_md "${filename}" || lint_result=1 ;; + case "$mimetype.$ext" in + text/x-shellscript.*) lint_sh "${filename}" || lint_result=1 ;; + *.go) ;; # done at directory level + *.tf) lint_tf "${filename}" || lint_result=1 ;; + *.md) lint_md "${filename}" || lint_result=1 ;; + *.py) lint_py "${filename}" || lint_result=1 ;; esac - spell_check "${filename}" || lint_result=1 + # we don't want to spell check tar balls, binaries, Makefile and json files + case "$mimetype.$ext" in + *.tar | *.gz | *.json) ;; + *.req | *.key | *.pem | *.crt) ;; + application/x-executable.*) ;; + text/x-makefile.*) ;; + *) spell_check "${filename}" || lint_result=1 ;; + esac return $lint_result } @@ -191,7 +209,7 @@ lint_files() { while read -r filename; do lint "${filename}" || lint_result=1 done - exit $lint_result + return $lint_result } matches_any() { @@ -222,18 +240,33 @@ filter_out() { fi } -list_files() { +lint_directory() { + local dirname="$1" + local lint_result=0 + # This test is just checking if there are any Go files in the directory + if compgen -G "$dirname/*.go" >/dev/null; then + lint_go "${dirname}" || lint_result=1 + fi + ls $dirname/* | filter_out "$LINT_IGNORE_FILE" | lint_files + return $lint_result +} + +lint_directories() { + local lint_result=0 + while read -r dirname; do + lint_directory "${dirname}" || lint_result=1 + done + exit $lint_result +} + +list_directories() { if [ $# -gt 0 ]; then - find "$@" | grep -vE '(^|/)vendor/' - else - git ls-files --exclude-standard | grep -vE '(^|/)vendor/' + find "$@" \( -name vendor -o -name .git -o -name .cache -o -name .pkg \) -prune -o -type d fi } if [ $# = 1 ] && [ -f "$1" ]; then lint "$1" -elif [ -n "$PARALLEL" ]; then - list_files "$@" | filter_out "$LINT_IGNORE_FILE" | xargs -n1 -P16 "$0" else - list_files "$@" | filter_out "$LINT_IGNORE_FILE" | lint_files + list_directories "$@" | lint_directories fi diff --git a/provisioning/README.md b/provisioning/README.md index 627bb42e..6ff739ca 100755 --- a/provisioning/README.md +++ b/provisioning/README.md @@ -16,16 +16,15 @@ You can then use these machines as is or run various Ansible playbooks from `../ * On macOS: `brew install vagrant` * On Linux (via Aptitude): `sudo apt install vagrant` - * If you need a specific version: - - curl -fsS https://releases.hashicorp.com/terraform/x.y.z/terraform_x.y.z_linux_amd64.zip | gunzip > terraform && chmod +x terraform && sudo mv terraform /usr/bin - * For other platforms or more details, see [here](https://www.vagrantup.com/docs/installation/) * You will need [Terraform](https://www.terraform.io) installed on your machine and added to your `PATH` in order to be able to provision cloud-hosted machines automatically. * On macOS: `brew install terraform` * On Linux (via Aptitude): `sudo apt install terraform` + * If you need a specific version: + + curl -fsS https://releases.hashicorp.com/terraform/x.y.z/terraform_x.y.z_linux_amd64.zip | gunzip > terraform && chmod +x terraform && sudo mv terraform /usr/bin * For other platforms or more details, see [here](https://www.terraform.io/intro/getting-started/install.html) * Depending on the cloud provider, you may have to create an account, manually onboard, create and register SSH keys, etc. diff --git a/provisioning/aws/variables.tf b/provisioning/aws/variables.tf index ed5b8b4b..5f4b4628 100755 --- a/provisioning/aws/variables.tf +++ b/provisioning/aws/variables.tf @@ -43,9 +43,11 @@ variable "aws_amis" { "eu-west-2" = "ami-23d0da47" # Red Hat Enterprise Linux 7.3 (HVM), SSD Volume Type: + #"us-east-1" = "ami-b63769a1" # CentOS 7 (x86_64) - with Updates HVM + #"us-east-1" = "ami-6d1c2007" } } diff --git a/provisioning/gcp/main.tf b/provisioning/gcp/main.tf index abfddb7d..af5a22eb 100755 --- a/provisioning/gcp/main.tf +++ b/provisioning/gcp/main.tf @@ -77,3 +77,17 @@ resource "google_compute_firewall" "fw-allow-esp" { source_ranges = ["${var.gcp_network_global_cidr}"] } + +# Required for WKS Kubernetes API server access +resource "google_compute_firewall" "fw-allow-kube-apiserver" { + name = "${var.name}-allow-kube-apiserver" + network = "${var.gcp_network}" + target_tags = ["${var.name}"] + + allow { + protocol = "tcp" + ports = ["6443"] + } + + source_ranges = ["${var.client_ip}"] +} diff --git a/provisioning/gcp/outputs.tf b/provisioning/gcp/outputs.tf index 9aa1e33e..210398ba 100755 --- a/provisioning/gcp/outputs.tf +++ b/provisioning/gcp/outputs.tf @@ -6,6 +6,10 @@ output "public_ips" { value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip}"] } +output "private_ips" { + value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.address}"] +} + output "hostnames" { value = "${join("\n", "${formatlist("%v.%v.%v", diff --git a/provisioning/setup.sh b/provisioning/setup.sh index 2eb67170..965ee28f 100755 --- a/provisioning/setup.sh +++ b/provisioning/setup.sh @@ -18,7 +18,8 @@ function decrypt() { echo >&2 "Failed to decode and decrypt $2: no secret key was provided." return 1 fi - echo "$3" | openssl base64 -d | openssl enc -d -aes256 -pass "pass:$1" + # Set md5 because existing keys were encrypted that way and openssl default changed + echo "$3" | openssl base64 -d | openssl enc -md md5 -d -aes256 -pass "pass:$1" } function ssh_private_key() { @@ -304,11 +305,11 @@ function tf_ssh_usage() { ERROR: $1 Usage: - $ tf_ssh [OPTION]... + \$ tf_ssh [OPTION]... Examples: - $ tf_ssh 1 - $ tf_ssh 1 -o LogLevel VERBOSE - $ tf_ssh 1 -i ~/.ssh/custom_private_key_id_rsa + \$ tf_ssh 1 + \$ tf_ssh 1 -o LogLevel VERBOSE + \$ tf_ssh 1 -i ~/.ssh/custom_private_key_id_rsa Available machines: EOF cat -n >&2 <<<"$(terraform output public_etc_hosts)" @@ -330,12 +331,12 @@ function tf_ansi_usage() { ERROR: $1 Usage: - $ tf_ansi [OPTION]... + \$ tf_ansi [OPTION]... Examples: - $ tf_ansi setup_weave-net_dev - $ tf_ansi 1 - $ tf_ansi 1 -vvv --private-key=~/.ssh/custom_private_key_id_rsa - $ tf_ansi setup_weave-kube --extra-vars "docker_version=1.12.6 kubernetes_version=1.5.6" + \$ tf_ansi setup_weave-net_dev + \$ tf_ansi 1 + \$ tf_ansi 1 -vvv --private-key=~/.ssh/custom_private_key_id_rsa + \$ tf_ansi setup_weave-kube --extra-vars "docker_version=1.12.6 kubernetes_version=1.5.6" Available playbooks: EOF cat -n >&2 <<<"$(for file in "$(dirname "${BASH_SOURCE[0]}")"/../../config_management/*.yml; do basename "$file" | sed 's/.yml//'; done)" @@ -348,7 +349,7 @@ function tf_ansi() { shift # Drop the first argument to allow passing other arguments to Ansible using "$@" -- see below. if [[ "$id" =~ ^[0-9]+$ ]]; then local playbooks=(../../config_management/*.yml) - local path="${playbooks[(($id-1))]}" # Select the ith entry in the list of playbooks (0-based). + local path="${playbooks[(($id - 1))]}" # Select the ith entry in the list of playbooks (0-based). else local path="$(dirname "${BASH_SOURCE[0]}")/../../config_management/$id.yml" fi diff --git a/push-images b/push-images index 9f1f16b1..913a8c31 100755 --- a/push-images +++ b/push-images @@ -26,25 +26,28 @@ while [ $# -gt 0 ]; do esac done -push_image() { - local image="$1" - docker push ${image}:${IMAGE_TAG} -} - +pids="" for image in ${IMAGES}; do if [[ "$image" == *"build"* ]]; then continue fi echo "Will push ${image}:${IMAGE_TAG}" - push_image "${image}" & + docker push "${image}:${IMAGE_TAG}" & + pids="$pids $!" - if [ -z "NO_DOCKER_HUB" ]; then + if [ -z "$NO_DOCKER_HUB" ]; then # remove the quey prefix and push to docker hub docker_hub_image=${image#$QUAY_PREFIX} - docker tag ${image}:${IMAGE_TAG} ${docker_hub_image}:${IMAGE_TAG} + docker tag "${image}:${IMAGE_TAG}" "${docker_hub_image}:${IMAGE_TAG}" echo "Will push ${docker_hub_image}:${IMAGE_TAG}" - docker push ${docker_hub_image}:${IMAGE_TAG} + docker push "${docker_hub_image}:${IMAGE_TAG}" & + pids="$pids $!" fi done +# Wait individually for tasks so we fail-exit on any non-zero return code +for p in $pids; do + wait "$p" +done + wait diff --git a/rebuild-image b/rebuild-image index 1f0bb109..cfa4ced8 100755 --- a/rebuild-image +++ b/rebuild-image @@ -9,6 +9,7 @@ IMAGENAME=$1 SAVEDNAME=$(echo "$IMAGENAME" | sed "s/[\/\-]/\./g") IMAGEDIR=$2 shift 2 +GIT_REVISION="$(git rev-parse HEAD)" INPUTFILES=("$@") CACHEDIR=$HOME/docker/ @@ -17,7 +18,7 @@ CACHEDIR=$HOME/docker/ rebuild() { mkdir -p "$CACHEDIR" rm "$CACHEDIR/$SAVEDNAME"* || true - docker build -t "$IMAGENAME" "$IMAGEDIR" + docker build --build-arg=revision="$GIT_REVISION" -t "$IMAGENAME" "$IMAGEDIR" docker save "$IMAGENAME:latest" | gzip - >"$CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1.gz" } diff --git a/sched b/sched index 72eeee65..179c650a 100755 --- a/sched +++ b/sched @@ -1,16 +1,31 @@ -#!/usr/bin/python +#!/usr/bin/env python import sys, string, urllib import requests +from requests.packages.urllib3.util.retry import Retry +from requests.adapters import HTTPAdapter import optparse +session = requests.Session() +adapter = HTTPAdapter( + max_retries=Retry( + connect=5, + status=5, + backoff_factor=0.1, + status_forcelist=[500, 502, 503, 504] + ) +) +session.mount('http://', adapter) +session.mount('https://', adapter) + + def test_time(target, test_name, runtime): - r = requests.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime)) + r = session.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime)) print r.text.encode('utf-8') assert r.status_code == 204 def test_sched(target, test_run, shard_count, shard_id): tests = {'tests': string.split(sys.stdin.read())} - r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), json=tests) + r = session.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), json=tests) assert r.status_code == 200 result = r.json() for test in sorted(result['tests']): diff --git a/scheduler/README.md b/scheduler/README.md index 8489d787..d9c4aa41 100644 --- a/scheduler/README.md +++ b/scheduler/README.md @@ -1,6 +1,64 @@ -To upload newer version: +# scheduler +## Development + +### Dependencies + +Download and install: + +- the [original App Engine SDK for Python](https://cloud.google.com/appengine/docs/standard/python/download) ([Linux](https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.77.zip), [macOS](https://storage.googleapis.com/appengine-sdks/featured/GoogleAppEngineLauncher-1.9.77.dmg)) -- this should add `appcfg.py` to your `PATH` +- `python` 2.7.x +- `pip` + +### Setup + +```console +$ pip install -U virtualenv +$ virtualenv --python=$(which python2.7) $TMPDIR/scheduler +$ source $TMPDIR/scheduler/bin/activate +$ pip install -r requirements.txt -t lib ``` -pip install -r requirements.txt -t lib -appcfg.py update . -``` + +## Deployment + +- Run: + ```console + $ appcfg.py --version $(date '+%Y%m%dt%H%M%S') update . + XX:XX PM Application: positive-cocoa-90213; version: 1 + XX:XX PM Host: appengine.google.com + XX:XX PM Starting update of app: positive-cocoa-90213, version: 1 + XX:XX PM Getting current resource limits. + Your browser has been opened to visit: + + https://accounts.google.com/o/oauth2/auth?scope=... + + If your browser is on a different machine then exit and re-run this + application with the command-line parameter + + --noauth_local_webserver + + Authentication successful. + XX:XX PM Scanning files on local disk. + XX:XX PM Scanned 500 files. + XX:XX PM Scanned 1000 files. + XX:XX PM Cloning 1220 application files. + XX:XX PM Uploading 28 files and blobs. + XX:XX PM Uploaded 28 files and blobs. + XX:XX PM Compilation starting. + XX:XX PM Compilation completed. + XX:XX PM Starting deployment. + XX:XX PM Checking if deployment succeeded. + XX:XX PM Will check again in 1 seconds. + XX:XX PM Checking if deployment succeeded. + XX:XX PM Will check again in 2 seconds. + XX:XX PM Checking if deployment succeeded. + XX:XX PM Will check again in 4 seconds. + XX:XX PM Checking if deployment succeeded. + XX:XX PM Deployment successful. + XX:XX PM Checking if updated app version is serving. + XX:XX PM Completed update of app: positive-cocoa-90213, version: 1 + XX:XX PM Uploading cron entries. + ``` + +- Go to [console.cloud.google.com](https://console.cloud.google.com) > Weave Integration Tests (`positive-cocoa-90213`) > AppEngine > Versions and ensure traffic is being directed to the newly deployed version. +- Click on Tools > Logs, and ensure the application is behaving well. diff --git a/scheduler/main.py b/scheduler/main.py index 4ed88756..733de807 100644 --- a/scheduler/main.py +++ b/scheduler/main.py @@ -19,157 +19,211 @@ # observations faster. alpha = 0.3 + class Test(ndb.Model): - total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA - total_runs = ndb.IntegerProperty(default=0) - - def parallelism(self): - name = self.key.string_id() - m = re.search('(\d+)_test.sh$', name) - if m is None: - return 1 - else: - return int(m.group(1)) + total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA + total_runs = ndb.IntegerProperty(default=0) + + def parallelism(self): + name = self.key.string_id() + m = re.search('(\d+)_test.sh$', name) + if m is None: + return 1 + else: + return int(m.group(1)) + + def cost(self): + p = self.parallelism() + logging.info("Test %s has parallelism %d and avg run time %s", + self.key.string_id(), p, self.total_run_time) + return self.parallelism() * self.total_run_time - def cost(self): - p = self.parallelism() - logging.info("Test %s has parallelism %d and avg run time %s", self.key.string_id(), p, self.total_run_time) - return self.parallelism() * self.total_run_time class Schedule(ndb.Model): - shards = ndb.JsonProperty() + shards = ndb.JsonProperty() + @app.route('/record//', methods=['POST']) @ndb.transactional def record(test_name, runtime): - test = Test.get_by_id(test_name) - if test is None: - test = Test(id=test_name) - test.total_run_time = (test.total_run_time * (1-alpha)) + (float(runtime) * alpha) - test.total_runs += 1 - test.put() - return ('', 204) - -@app.route('/schedule///', methods=['POST']) + test = Test.get_by_id(test_name) + if test is None: + test = Test(id=test_name) + test.total_run_time = (test.total_run_time * + (1 - alpha)) + (float(runtime) * alpha) + test.total_runs += 1 + test.put() + return ('', 204) + + +@app.route( + '/schedule///', methods=['POST']) def schedule(test_run, shard_count, shard): - # read tests from body - test_names = flask.request.get_json(force=True)['tests'] - - # first see if we have a scedule already - schedule_id = "%s-%d" % (test_run, shard_count) - schedule = Schedule.get_by_id(schedule_id) - if schedule is not None: + # read tests from body + test_names = flask.request.get_json(force=True)['tests'] + + # first see if we have a scedule already + schedule_id = "%s-%d" % (test_run, shard_count) + schedule = Schedule.get_by_id(schedule_id) + if schedule is not None: + return flask.json.jsonify(tests=schedule.shards[str(shard)]) + + # if not, do simple greedy algorithm + test_times = ndb.get_multi( + ndb.Key(Test, test_name) for test_name in test_names) + + def avg(test): + if test is not None: + return test.cost() + return 1 + + test_times = [(test_name, avg(test)) + for test_name, test in zip(test_names, test_times)] + test_times_dict = dict(test_times) + test_times.sort(key=operator.itemgetter(1)) + + shards = {i: [] for i in xrange(shard_count)} + while test_times: + test_name, time = test_times.pop() + + # find shortest shard and put it in that + s, _ = min( + ((i, sum(test_times_dict[t] for t in shards[i])) + for i in xrange(shard_count)), + key=operator.itemgetter(1)) + + shards[s].append(test_name) + + # atomically insert or retrieve existing schedule + schedule = Schedule.get_or_insert(schedule_id, shards=shards) return flask.json.jsonify(tests=schedule.shards[str(shard)]) - # if not, do simple greedy algorithm - test_times = ndb.get_multi(ndb.Key(Test, test_name) for test_name in test_names) - def avg(test): - if test is not None: - return test.cost() - return 1 - test_times = [(test_name, avg(test)) for test_name, test in zip(test_names, test_times)] - test_times_dict = dict(test_times) - test_times.sort(key=operator.itemgetter(1)) - - shards = {i: [] for i in xrange(shard_count)} - while test_times: - test_name, time = test_times.pop() - - # find shortest shard and put it in that - s, _ = min(((i, sum(test_times_dict[t] for t in shards[i])) - for i in xrange(shard_count)), key=operator.itemgetter(1)) - - shards[s].append(test_name) - - # atomically insert or retrieve existing schedule - schedule = Schedule.get_or_insert(schedule_id, shards=shards) - return flask.json.jsonify(tests=schedule.shards[str(shard)]) FIREWALL_REGEXES = [ - re.compile(r'^(?P\w+)-allow-(?P\w+)-(?P\d+)-(?P\d+)$'), - re.compile(r'^(?P\w+)-(?P\d+)-(?P\d+)-allow-(?P[\w\-]+)$'), + re.compile( + r'^(?P\w+)-allow-(?P\w+)-(?P\d+)-(?P\d+)$' + ), + re.compile(r'^(?P\w+)-(?P\d+)-(?P\d+)-allow-' + r'(?P[\w\-]+)$'), ] NAME_REGEXES = [ - re.compile(r'^host(?P\d+)-(?P\d+)-(?P\d+)$'), - re.compile(r'^test-(?P\d+)-(?P\d+)-(?P\d+)$'), + re.compile(pat) + for pat in ( + r'^host(?P\d+)-(?P\d+)-(?P\d+)$', + r'^host(?P\d+)-(?P[a-zA-Z0-9-]+)-(?P\d+)' + r'-(?P\d+)$', + r'^test-(?P\d+)-(?P\d+)-(?P\d+)$', ) ] + def _matches_any_regex(name, regexes): - for regex in regexes: - matches = regex.match(name) - if matches: - return matches + for regex in regexes: + matches = regex.match(name) + if matches: + return matches + +# See also: https://circleci.com/account/api +CIRCLE_CI_API_TOKEN = 'cffb83afd920cfa109cbd3e9eecb7511a2d18bb9' + +# N.B.: When adding a project below, please ensure: +# - its CircleCI project is either public, or is followed by the user attached +# to the above API token +# - user positive-cocoa-90213@appspot.gserviceaccount.com has "Compute Admin" +# access to its GCP project (or any other role including +# compute.instances.list/delete and compute.firewalls.list/delete) PROJECTS = [ - ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True), - ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True), - ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False), + ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True, None), + ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True, None), + ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False, + None), + ('weaveworks/wks', 'wks-tests', 'us-central1-a', True, + CIRCLE_CI_API_TOKEN), ] + @app.route('/tasks/gc') def gc(): - # Get list of running VMs, pick build id out of VM name - credentials = GoogleCredentials.get_application_default() - compute = discovery.build('compute', 'v1', credentials=credentials) - - for repo, project, zone, gc_fw in PROJECTS: - gc_project(compute, repo, project, zone, gc_fw) - - return "Done" - -def gc_project(compute, repo, project, zone, gc_fw): - logging.info("GCing %s, %s, %s", repo, project, zone) - # Get list of builds, filter down to running builds: - running = _get_running_builds(repo) - # Stop VMs for builds that aren't running: - _gc_compute_engine_instances(compute, project, zone, running) - # Remove firewall rules for builds that aren't running: - if gc_fw: - _gc_firewall_rules(compute, project, running) - -def _get_running_builds(repo): - result = urlfetch.fetch('https://circleci.com/api/v1/project/%s' % repo, - headers={'Accept': 'application/json'}) - assert result.status_code == 200 - builds = json.loads(result.content) - running = {build['build_num'] for build in builds if not build.get('stop_time')} - logging.info("Runnings builds: %r", running) - return running + # Get list of running VMs, pick build id out of VM name + credentials = GoogleCredentials.get_application_default() + compute = discovery.build( + 'compute', 'v1', credentials=credentials, cache_discovery=False) + + for repo, project, zone, gc_fw, circleci_api_token in PROJECTS: + gc_project(compute, repo, project, zone, gc_fw, circleci_api_token) + + return "Done" + + +def gc_project(compute, repo, project, zone, gc_fw, circleci_api_token): + logging.info("GCing %s, %s, %s", repo, project, zone) + # Get list of builds, filter down to running builds: + running = _get_running_builds(repo, circleci_api_token) + # Stop VMs for builds that aren't running: + _gc_compute_engine_instances(compute, project, zone, running) + # Remove firewall rules for builds that aren't running: + if gc_fw: + _gc_firewall_rules(compute, project, running) + + +def _get_running_builds(repo, circleci_api_token): + if circleci_api_token: + url = 'https://circleci.com/api/v1/project/%s?circle-token=%s' % ( + repo, circleci_api_token) + else: + url = 'https://circleci.com/api/v1/project/%s' % repo + result = urlfetch.fetch(url, headers={'Accept': 'application/json'}) + if result.status_code != 200: + raise RuntimeError( + 'Failed to get builds for "%s". URL: %s, Status: %s. Response: %s' + % (repo, url, result.status_code, result.content)) + builds = json.loads(result.content) + running = { + build['build_num'] + for build in builds if not build.get('stop_time') + } + logging.info("Runnings builds: %r", running) + return running + def _get_hosts_by_build(instances): - host_by_build = collections.defaultdict(list) - for instance in instances['items']: - matches = _matches_any_regex(instance['name'], NAME_REGEXES) - if not matches: - continue - host_by_build[int(matches.group('build'))].append(instance['name']) - logging.info("Running VMs by build: %r", host_by_build) - return host_by_build + host_by_build = collections.defaultdict(list) + for instance in instances['items']: + matches = _matches_any_regex(instance['name'], NAME_REGEXES) + if not matches: + continue + host_by_build[int(matches.group('build'))].append(instance['name']) + logging.info("Running VMs by build: %r", host_by_build) + return host_by_build + def _gc_compute_engine_instances(compute, project, zone, running): - instances = compute.instances().list(project=project, zone=zone).execute() - if 'items' not in instances: - return - host_by_build = _get_hosts_by_build(instances) - stopped = [] - for build, names in host_by_build.iteritems(): - if build in running: - continue - for name in names: - stopped.append(name) - logging.info("Stopping VM %s", name) - compute.instances().delete(project=project, zone=zone, instance=name).execute() - return stopped + instances = compute.instances().list(project=project, zone=zone).execute() + if 'items' not in instances: + return + host_by_build = _get_hosts_by_build(instances) + stopped = [] + for build, names in host_by_build.iteritems(): + if build in running: + continue + for name in names: + stopped.append(name) + logging.info("Stopping VM %s", name) + compute.instances().delete( + project=project, zone=zone, instance=name).execute() + return stopped + def _gc_firewall_rules(compute, project, running): - firewalls = compute.firewalls().list(project=project).execute() - if 'items' not in firewalls: - return - for firewall in firewalls['items']: - matches = _matches_any_regex(firewall['name'], FIREWALL_REGEXES) - if not matches: - continue - if int(matches.group('build')) in running: - continue - logging.info("Deleting firewall rule %s", firewall['name']) - compute.firewalls().delete(project=project, firewall=firewall['name']).execute() + firewalls = compute.firewalls().list(project=project).execute() + if 'items' not in firewalls: + return + for firewall in firewalls['items']: + matches = _matches_any_regex(firewall['name'], FIREWALL_REGEXES) + if not matches: + continue + if int(matches.group('build')) in running: + continue + logging.info("Deleting firewall rule %s", firewall['name']) + compute.firewalls().delete( + project=project, firewall=firewall['name']).execute() diff --git a/scheduler/requirements.txt b/scheduler/requirements.txt index d4d47e6e..872a7c83 100644 --- a/scheduler/requirements.txt +++ b/scheduler/requirements.txt @@ -1,2 +1,2 @@ -flask -google-api-python-client +flask==0.12.4 +google-api-python-client==1.6.7 diff --git a/socks/Dockerfile b/socks/Dockerfile index 867cd6bc..ad0b8938 100644 --- a/socks/Dockerfile +++ b/socks/Dockerfile @@ -1,7 +1,13 @@ FROM gliderlabs/alpine -MAINTAINER Weaveworks Inc WORKDIR / COPY proxy / EXPOSE 8000 EXPOSE 8080 ENTRYPOINT ["/proxy"] + +ARG revision +LABEL maintainer="Weaveworks " \ + org.opencontainers.image.title="socks" \ + org.opencontainers.image.source="https://github.com/weaveworks/build-tools/tree/master/socks" \ + org.opencontainers.image.revision="${revision}" \ + org.opencontainers.image.vendor="Weaveworks" diff --git a/socks/Makefile b/socks/Makefile index 2daeda64..b3358649 100644 --- a/socks/Makefile +++ b/socks/Makefile @@ -2,6 +2,7 @@ IMAGE_TAR=image.tar IMAGE_NAME=weaveworks/socksproxy +GIT_REVISION := $(shell git rev-parse HEAD) PROXY_EXE=proxy NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \ rm $@; \ @@ -15,7 +16,7 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \ all: $(IMAGE_TAR) $(IMAGE_TAR): Dockerfile $(PROXY_EXE) - docker build -t $(IMAGE_NAME) . + docker build --build-arg=revision=$(GIT_REVISION) -t $(IMAGE_NAME) . docker save $(IMAGE_NAME):latest > $@ $(PROXY_EXE): *.go diff --git a/socks/main.go b/socks/main.go index 7cd8c708..2f16d167 100644 --- a/socks/main.go +++ b/socks/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "net" "net/http" @@ -11,23 +12,23 @@ import ( socks5 "github.com/armon/go-socks5" "github.com/weaveworks/common/mflag" "github.com/weaveworks/common/mflagext" - "golang.org/x/net/context" ) type pacFileParameters struct { - HostMatch string - Aliases map[string]string + HostMatch string + SocksDestination string + Aliases map[string]string } const ( pacfile = ` function FindProxyForURL(url, host) { if(shExpMatch(host, "{{.HostMatch}}")) { - return "SOCKS5 localhost:8000"; + return "SOCKS5 {{.SocksDestination}}"; } {{range $key, $value := .Aliases}} if (host == "{{$key}}") { - return "SOCKS5 localhost:8000"; + return "SOCKS5 {{.SocksDestination}}"; } {{end}} return "DIRECT"; @@ -37,11 +38,13 @@ function FindProxyForURL(url, host) { func main() { var ( - as []string - hostMatch string + as []string + hostMatch string + socksDestination string ) mflagext.ListVar(&as, []string{"a", "-alias"}, []string{}, "Specify hostname aliases in the form alias:hostname. Can be repeated.") mflag.StringVar(&hostMatch, []string{"h", "-host-match"}, "*.weave.local", "Specify main host shExpMatch expression in pacfile") + mflag.StringVar(&socksDestination, []string{"d", "-socks-destination"}, "localhost:8000", "Specify destination host:port in pacfile") mflag.Parse() var aliases = map[string]string{} @@ -60,7 +63,7 @@ func main() { t := template.Must(template.New("pacfile").Parse(pacfile)) http.HandleFunc("/proxy.pac", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/x-ns-proxy-autoconfig") - t.Execute(w, pacFileParameters{hostMatch, aliases}) + t.Execute(w, pacFileParameters{hostMatch, socksDestination, aliases}) }) if err := http.ListenAndServe(":8080", nil); err != nil { diff --git a/test b/test index 31805164..c284e494 100755 --- a/test +++ b/test @@ -7,7 +7,9 @@ SLOW= NO_GO_GET=true TAGS= PARALLEL= +RACE="-race -covermode=atomic" TIMEOUT=1m +VERBOSE= usage() { echo "$0 [-slow] [-in-container foo] [-netgo] [-(no-)go-get] [-timeout 1m]" @@ -15,10 +17,18 @@ usage() { while [ $# -gt 0 ]; do case "$1" in + "-v") + VERBOSE="-v" + shift 1 + ;; "-slow") SLOW=true shift 1 ;; + "-no-race") + RACE= + shift 1 + ;; "-no-go-get") NO_GO_GET=true shift 1 @@ -28,9 +38,13 @@ while [ $# -gt 0 ]; do shift 1 ;; "-netgo") - TAGS="-tags netgo" + TAGS="netgo" shift 1 ;; + "-tags") + TAGS="$2" + shift 2 + ;; "-p") PARALLEL=true shift 1 @@ -46,14 +60,14 @@ while [ $# -gt 0 ]; do esac done -GO_TEST_ARGS=($TAGS -cpu 4 -timeout $TIMEOUT) +GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout $TIMEOUT $VERBOSE) if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then SLOW=true fi if [ -n "$SLOW" ]; then - GO_TEST_ARGS=("${GO_TEST_ARGS[@]}" -race -covermode=atomic) + GO_TEST_ARGS=("${GO_TEST_ARGS[@]}" ${RACE}) # shellcheck disable=SC2153 if [ -n "$COVERDIR" ]; then @@ -69,7 +83,7 @@ fail=0 if [ -z "$TESTDIRS" ]; then # NB: Relies on paths being prefixed with './'. - TESTDIRS=($(git ls-files -- '*_test.go' | grep -vE '^(vendor|prog|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|')) + TESTDIRS=($(git ls-files -- '*_test.go' | grep -vE '^(vendor|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|')) else # TESTDIRS on the right side is not really an array variable, it # is just a string with spaces, but it is written like that to @@ -92,7 +106,7 @@ go test -i "${GO_TEST_ARGS[@]}" "${TESTDIRS[@]}" run_test() { local dir=$1 if [ -z "$NO_GO_GET" ]; then - go get -t "$TAGS" "$dir" + go get -t -tags "${TAGS[@]}" "$dir" fi local GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}")