From 837e083365d9d982ce74fc10b3d5adfc39bb9814 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Sat, 10 Jun 2017 00:06:42 +0100 Subject: [PATCH 01/23] Make `notice-up-to-date` part of the verify build stage. So that I don't forget. Also ensures that the Go notices stays up to date (we don't keep our own copy). Additionally doesn't rely on file deletion and git diff to ensure that the notice is updated, which can throw me when I forget. --- .gitlab-ci.yml | 1 - Makefile | 12 +-- NOTICE | 204 ++++++++++++++++++++++++++++++++++++++- _support/notice.template | 68 +++---------- 4 files changed, 222 insertions(+), 63 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b8f75df221..2369a4c74fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,6 @@ test:go1.8: verify: stage: test script: - - make notice-up-to-date - make verify package: diff --git a/Makefile b/Makefile index ac03b0dc45a..b5ddecbc722 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PREFIX=/usr/local PKG=gitlab.com/gitlab-org/gitaly -BUILD_DIR=$(shell pwd) +BUILD_DIR=${CURDIR} BIN_BUILD_DIR=${BUILD_DIR}/_build/bin PKG_BUILD_DIR:=${BUILD_DIR}/_build/src/${PKG} CMDS:=$(shell cd cmd && ls) @@ -29,7 +29,7 @@ install: build mkdir -p $(DESTDIR)${PREFIX}/bin/ cd ${BIN_BUILD_DIR} && install ${CMDS} ${DESTDIR}${PREFIX}/bin/ -verify: lint check-formatting govendor-status +verify: lint check-formatting govendor-status notice-up-to-date check-formatting: install-developer-tools go run _support/gofmt-all.go -n @@ -50,12 +50,12 @@ package: build ./_support/package/package ${CMDS} notice: ${BUILD_DIR}/_build install-developer-tools - rm -f ${PKG_BUILD_DIR}/NOTICE # Avoid NOTICE-in-NOTICE cd ${PKG_BUILD_DIR} && govendor license -template _support/notice.template -o ${BUILD_DIR}/NOTICE -notice-up-to-date: notice - git ls-files --error-unmatch NOTICE # NOTICE is a tracked file - git diff --exit-code # there are no changed files +notice-up-to-date: ${BUILD_DIR}/_build + @cd ${PKG_BUILD_DIR} && govendor license -template _support/notice.template -o ${BUILD_DIR}/_build/nutd.temp + @diff _build/nutd.temp NOTICE + @rm -f _build/nutd.temp clean: clean-build rm -rf internal/testhelper/testdata diff --git a/NOTICE b/NOTICE index e85940267fb..16b0645731b 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,6 @@ The following components are included in Gitaly: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LICENSE - go Copyright (c) 2009 The Go Authors. All rights reserved. @@ -52,7 +53,6 @@ implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LICENSE - gitlab.com/gitlab-org/gitaly The MIT License (MIT) @@ -247,6 +247,208 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +LICENSE - gitlab.com/gitlab-org/gitaly/vendor/github.com/grpc-ecosystem/go-grpc-middleware + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LICENSE - gitlab.com/gitlab-org/gitaly/vendor/github.com/grpc-ecosystem/go-grpc-prometheus Apache License Version 2.0, January 2004 diff --git a/_support/notice.template b/_support/notice.template index 9543d1d69b2..197675a4d82 100644 --- a/_support/notice.template +++ b/_support/notice.template @@ -1,58 +1,16 @@ The following components are included in Gitaly: -LICENSE - go -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +{{ range $index, $t := . -}} + {{- if eq .Path " go" -}} ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -PATENTS - go -Additional IP Rights Grant (Patents) - -"This implementation" means the copyrightable works distributed by -Google as part of the Go project. - -Google hereby grants to You a perpetual, worldwide, non-exclusive, -no-charge, royalty-free, irrevocable (except as stated in this section) -patent license to make, have made, use, offer to sell, sell, import, -transfer and otherwise run, modify and propagate the contents of this -implementation of Go, where such license applies only to those patent -claims, both currently owned or controlled by Google and acquired in -the future, licensable by Google that are necessarily infringed by this -implementation of Go. This grant does not include claims that would be -infringed only as a consequence of further modification of this -implementation. If you or your agent or exclusive licensee institute or -order or agree to the institution of patent litigation against any -entity (including a cross-claim or counterclaim in a lawsuit) alleging -that this implementation of Go or any code incorporated within this -implementation of Go constitutes direct or contributory patent -infringement, or inducement of patent infringement, then any patent -rights granted to you under this License for this implementation of Go -shall terminate as of the date such litigation is filed. - -{{range $index, $t := .}}{{if ne .Path " go" }}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -{{.Filename}} - {{.Path}} -{{.Text}}{{end}}{{end}} +{{ .Filename }} - {{ .Path }} +{{ .Text }} + {{- end -}} +{{- end -}} +{{ range $index, $t := . -}} + {{- if and (ne .Path " go") (or (ne .Path "gitlab.com/gitlab-org/gitaly") (ne .Filename "NOTICE") ) -}} +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +{{ .Filename }} - {{ .Path }} +{{ .Text }} + {{- end -}} +{{- end }} -- GitLab From 3a0f99f97f03d262eb84ea939ab5e8dc7bd6dda3 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Sat, 10 Jun 2017 00:10:38 +0100 Subject: [PATCH 02/23] Debugging the Makefile --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b5ddecbc722..5a0ff43b24a 100644 --- a/Makefile +++ b/Makefile @@ -53,9 +53,9 @@ notice: ${BUILD_DIR}/_build install-developer-tools cd ${PKG_BUILD_DIR} && govendor license -template _support/notice.template -o ${BUILD_DIR}/NOTICE notice-up-to-date: ${BUILD_DIR}/_build - @cd ${PKG_BUILD_DIR} && govendor license -template _support/notice.template -o ${BUILD_DIR}/_build/nutd.temp - @diff _build/nutd.temp NOTICE - @rm -f _build/nutd.temp + cd ${PKG_BUILD_DIR} && govendor license -template _support/notice.template -o ${BUILD_DIR}/_build/nutd.temp + diff _build/nutd.temp NOTICE + rm -f _build/nutd.temp clean: clean-build rm -rf internal/testhelper/testdata -- GitLab From fd8f623ccfd6f967250422ac2f70759f3bb66522 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Mon, 12 Jun 2017 15:12:33 +0100 Subject: [PATCH 03/23] Updated notice with new format --- NOTICE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index f24a4293550..5fd076fe74a 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,6 @@ The following components are included in Gitaly: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LICENSE - go Copyright (c) 2009 The Go Authors. All rights reserved. @@ -52,7 +53,6 @@ implementation of Go constitutes direct or contributory patent infringement, or inducement of patent infringement, then any patent rights granted to you under this License for this implementation of Go shall terminate as of the date such litigation is filed. - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LICENSE - gitlab.com/gitlab-org/gitaly The MIT License (MIT) -- GitLab From e1cd9a8bd7225910960851cbbe71e00e7b7144dc Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Tue, 13 Jun 2017 23:07:00 +0100 Subject: [PATCH 04/23] Support coverage and race testing --- .gitignore | 1 + Makefile | 130 ++++++++++++++++++++++-------- internal/testhelper/testhelper.go | 13 +++ 3 files changed, 109 insertions(+), 35 deletions(-) diff --git a/.gitignore b/.gitignore index 0aba6d4889e..c5d418a9c66 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ /internal/service/smarthttp/testdata /internal/testhelper/testdata /config.toml +_tools diff --git a/Makefile b/Makefile index 74eaf2fe9e4..16d46ed1a61 100644 --- a/Makefile +++ b/Makefile @@ -1,74 +1,134 @@ PREFIX=/usr/local PKG=gitlab.com/gitlab-org/gitaly BUILD_DIR=${CURDIR} -BIN_BUILD_DIR=${BUILD_DIR}/_build/bin -PKG_BUILD_DIR:=${BUILD_DIR}/_build/src/${PKG} +TARGET_DIR=${BUILD_DIR}/_build +BIN_BUILD_DIR=${TARGET_DIR}/bin +PKG_BUILD_DIR=${TARGET_DIR}/src/${PKG} CMDS:=$(shell cd cmd && ls) -TEST_REPO=internal/testhelper/testdata/data/gitlab-test.git VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S) +export TEST_REPO_LOCATION=${TARGET_DIR}/testdata/data +TEST_REPO=${TEST_REPO_LOCATION}/gitlab-test.git +IGNORED_PACKAGES := /vendor/ +COVERAGE_DIR=${TARGET_DIR}/cover -export GOPATH=${BUILD_DIR}/_build +export GOPATH=${TARGET_DIR} export GO15VENDOREXPERIMENT=1 export PATH:=${GOPATH}/bin:$(PATH) .PHONY: all -all: build +all: verify build test -.PHONY: ${BUILD_DIR}/_build -${BUILD_DIR}/_build: - mkdir -p $@/src/${PKG} - tar -cf - --exclude _build --exclude .git . | (cd $@/src/${PKG} && tar -xf -) - touch $@ +${TARGET_DIR}/.ok: Makefile + rm -rf -- "${TARGET_DIR}" + mkdir -p "$(dir ${PKG_BUILD_DIR})" + ln -sf ../../../.. "${PKG_BUILD_DIR}" + mkdir -p "${BIN_BUILD_DIR}" + touch -- "${TARGET_DIR}/.ok" -build: clean-build ${BUILD_DIR}/_build $(shell find . -name '*.go' -not -path './vendor/*' -not -path './_build/*') - rm -f -- "${BIN_BUILD_DIR}/*" - go install -ldflags "-X main.version=${VERSION}" ${PKG}/cmd/... - cp ${BIN_BUILD_DIR}/* ${BUILD_DIR}/ +.PHONY: build +build: ${TARGET_DIR}/.ok + go install -ldflags "-X main.version=${VERSION}" $(foreach cmd,${CMDS},${PKG}/cmd/${cmd}) + cp $(foreach cmd,${CMDS},${BIN_BUILD_DIR}/${cmd}) ${BUILD_DIR}/ +.PHONY: install install: build mkdir -p $(DESTDIR)${PREFIX}/bin/ cd ${BIN_BUILD_DIR} && install ${CMDS} ${DESTDIR}${PREFIX}/bin/ +.PHONY: verify verify: lint check-formatting govendor-status notice-up-to-date -check-formatting: install-developer-tools +.PHONY: check-formatting +check-formatting: go run _support/gofmt-all.go -n -govendor-status: ${BUILD_DIR}/_build install-developer-tools - cd ${PKG_BUILD_DIR} && govendor status +.PHONY: govendor-status +govendor-status: ${TARGET_DIR}/.ok _tools/govendor + cd ${PKG_BUILD_DIR} && _tools/govendor status ${TEST_REPO}: git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git $@ -test: clean-build ${TEST_REPO} ${BUILD_DIR}/_build - go test ${PKG}/... +.PHONY: test +test: ${TARGET_DIR}/.ok ${TEST_REPO} + go test $(allpackages) -lint: install-developer-tools - go run _support/lint.go +.PHONY: test-race +test-race: ${TARGET_DIR}/.ok ${TEST_REPO} + GODEBUG=cgocheck=2 go test -v -race $(allpackages) +.PHONY: lint +lint: ${TARGET_DIR}/.ok _tools/golint + _tools/golint $(allpackages) + +.PHONY: package package: build ./_support/package/package ${CMDS} -notice: ${BUILD_DIR}/_build install-developer-tools - cd ${PKG_BUILD_DIR} && govendor license -template _support/notice.template -o ${BUILD_DIR}/NOTICE +.PHONY: notice +notice: ${TARGET_DIR}/.ok _tools/govendor + cd ${PKG_BUILD_DIR} && _tools/govendor license -template _support/notice.template -o ${BUILD_DIR}/NOTICE -notice-up-to-date: ${BUILD_DIR}/_build - cd ${PKG_BUILD_DIR} && govendor license -template _support/notice.template -o ${BUILD_DIR}/_build/nutd.temp +.PHONY: notice-up-to-date +notice-up-to-date: ${TARGET_DIR}/.ok _tools/govendor + cd ${PKG_BUILD_DIR} && _tools/govendor license -template _support/notice.template -o ${TARGET_DIR}/nutd.temp diff _build/nutd.temp NOTICE rm -f _build/nutd.temp -clean: clean-build - rm -rf internal/testhelper/testdata - rm -f $(foreach cmd,${CMDS},./${cmd}) - -clean-build: - rm -rf ${BUILD_DIR}/_build +.PHONY: clean +clean: + rm -rf -- ${TARGET_DIR} + rm -f -- $(foreach cmd,${CMDS},./${cmd}) .PHONY: format format: - @go run _support/gofmt-all.go -f + go run _support/gofmt-all.go -f + +.PHONY: cover +cover: ${TARGET_DIR}/.ok ${TEST_REPO} _tools/gocovmerge + @echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!" + mkdir -p "${COVERAGE_DIR}" + rm -f ${COVERAGE_DIR}/*.out "${COVERAGE_DIR}/all.merged" "${COVERAGE_DIR}/all.html" + @for MOD in $(allpackages); do \ + echo go test -coverpkg=`echo $(allpackages)|tr " " ","` \ + -coverprofile=${COVERAGE_DIR}/unit-`echo $$MOD|tr "/" "_"`.out $$MOD; \ + go test -coverpkg=`echo $(allpackages)|tr " " ","` \ + -coverprofile=${COVERAGE_DIR}/unit-`echo $$MOD|tr "/" "_"`.out \ + $$MOD 2>&1 | grep -v "no packages being tested depend on"; \ + done + _tools/gocovmerge ${COVERAGE_DIR}/*.out > "${COVERAGE_DIR}/all.merged" + go tool cover -html "${COVERAGE_DIR}/all.merged" -o "${COVERAGE_DIR}/all.html" + @echo "" + @echo "=====> Total test coverage: <=====" + @echo "" + @go tool cover -func "${COVERAGE_DIR}/all.merged" + +list: ${TARGET_DIR}/.ok + @echo $(allpackages) + +# cd into the GOPATH to workaround ./... not following symlinks +_allpackages = $(shell ( cd "${PKG_BUILD_DIR}" && \ + go list ./... 2>&1 1>&3 | \ + grep -v -e "^$$" $(addprefix -e ,$(IGNORED_PACKAGES)) 1>&2 ) 3>&1 | \ + grep -v -e "^$$" $(addprefix -e ,$(IGNORED_PACKAGES))) + +# memoize allpackages, so that it's executed only once and only if used +allpackages = $(if $(__allpackages),,$(eval __allpackages := $$(_allpackages)))$(__allpackages) .PHONY: install-developer-tools -install-developer-tools: - @go run _support/go-get-if-missing.go govendor github.com/kardianos/govendor - @go run _support/go-get-if-missing.go golint github.com/golang/lint/golint +install-developer-tools: _tools/govendor _tools/golint _tools/gocovmerge + +_tools: + mkdir -p _tools + +_tools/govendor: _tools + go get github.com/kardianos/govendor + mv ${BIN_BUILD_DIR}/govendor _tools/ + +_tools/golint: _tools + go get github.com/golang/lint/golint + mv ${BIN_BUILD_DIR}/golint _tools/ + +_tools/gocovmerge: _tools + go get github.com/wadey/gocovmerge + mv ${BIN_BUILD_DIR}/gocovmerge _tools/ diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go index 0a19ee20516..d5ee2fab7d8 100644 --- a/internal/testhelper/testhelper.go +++ b/internal/testhelper/testhelper.go @@ -12,6 +12,8 @@ import ( log "github.com/Sirupsen/logrus" + "path/filepath" + "google.golang.org/grpc" "google.golang.org/grpc/codes" ) @@ -31,6 +33,17 @@ func MustReadFile(t *testing.T, filename string) []byte { // GitlabTestStoragePath returns the storage path to the gitlab-test repo. func GitlabTestStoragePath() string { + // If TEST_REPO_LOCATION has been set (by the Makefile) then use that + testRepoPath := os.Getenv("TEST_REPO_LOCATION") + if testRepoPath != "" { + testRepoPathAbs, err := filepath.Abs(testRepoPath) + if err != nil { + log.Fatal(err) + } + + return testRepoPathAbs + } + _, currentFile, _, ok := runtime.Caller(0) if !ok { log.Fatal("Could not get caller info") -- GitLab From f87b7f0cb89e131103c7d2172bc1cd3cbbe8b53c Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Tue, 13 Jun 2017 23:13:19 +0100 Subject: [PATCH 05/23] Include coverage in the build --- .gitlab-ci.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2369a4c74fd..ba1d16c54d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,30 +1,34 @@ -image: registry.gitlab.com/gitlab-org/gitlab-build-images:golang-1.7-git-2.8.4 +image: registry.gitlab.com/gitlab-org/gitlab-build-images:golang-1.8-git-2.8.4 stages: - test - package - publish +verify: + stage: test + script: + - make verify + .test_template: &test_definition stage: test script: - go version - git version - - make - make test +coverage: + stage: test + script: + - make cover + test:go1.8: <<: *test_definition image: registry.gitlab.com/gitlab-org/gitlab-build-images:golang-1.8-git-2.8.4 -verify: - stage: test - script: - - make verify - package: stage: package - only: + only: - tags script: - apt-get update -qq -- GitLab From 54e6b093bb411db804c2b25d9e4563024b6b207c Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Tue, 13 Jun 2017 23:48:59 +0100 Subject: [PATCH 06/23] Use govendor to list local packages, rather than go list --- Makefile | 50 +++++++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 16d46ed1a61..00d6f181654 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,8 @@ CMDS:=$(shell cd cmd && ls) VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S) export TEST_REPO_LOCATION=${TARGET_DIR}/testdata/data TEST_REPO=${TEST_REPO_LOCATION}/gitlab-test.git -IGNORED_PACKAGES := /vendor/ COVERAGE_DIR=${TARGET_DIR}/cover +TOOLS_DIR=${BUILD_DIR}/_tools export GOPATH=${TARGET_DIR} export GO15VENDOREXPERIMENT=1 @@ -18,7 +18,7 @@ export PATH:=${GOPATH}/bin:$(PATH) .PHONY: all all: verify build test -${TARGET_DIR}/.ok: Makefile +${TARGET_DIR}/.ok: Makefile ${TOOLS_DIR}/govendor rm -rf -- "${TARGET_DIR}" mkdir -p "$(dir ${PKG_BUILD_DIR})" ln -sf ../../../.. "${PKG_BUILD_DIR}" @@ -43,8 +43,8 @@ check-formatting: go run _support/gofmt-all.go -n .PHONY: govendor-status -govendor-status: ${TARGET_DIR}/.ok _tools/govendor - cd ${PKG_BUILD_DIR} && _tools/govendor status +govendor-status: ${TARGET_DIR}/.ok ${TOOLS_DIR}/govendor + cd ${PKG_BUILD_DIR} && ${TOOLS_DIR}/govendor status ${TEST_REPO}: git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git $@ @@ -58,20 +58,20 @@ test-race: ${TARGET_DIR}/.ok ${TEST_REPO} GODEBUG=cgocheck=2 go test -v -race $(allpackages) .PHONY: lint -lint: ${TARGET_DIR}/.ok _tools/golint - _tools/golint $(allpackages) +lint: ${TARGET_DIR}/.ok ${TOOLS_DIR}/golint + ${TOOLS_DIR}/golint $(allpackages) .PHONY: package package: build ./_support/package/package ${CMDS} .PHONY: notice -notice: ${TARGET_DIR}/.ok _tools/govendor - cd ${PKG_BUILD_DIR} && _tools/govendor license -template _support/notice.template -o ${BUILD_DIR}/NOTICE +notice: ${TARGET_DIR}/.ok ${TOOLS_DIR}/govendor + cd ${PKG_BUILD_DIR} && ${TOOLS_DIR}/govendor license -template _support/notice.template -o ${BUILD_DIR}/NOTICE .PHONY: notice-up-to-date -notice-up-to-date: ${TARGET_DIR}/.ok _tools/govendor - cd ${PKG_BUILD_DIR} && _tools/govendor license -template _support/notice.template -o ${TARGET_DIR}/nutd.temp +notice-up-to-date: ${TARGET_DIR}/.ok ${TOOLS_DIR}/govendor + cd ${PKG_BUILD_DIR} && ${TOOLS_DIR}/govendor license -template _support/notice.template -o ${TARGET_DIR}/nutd.temp diff _build/nutd.temp NOTICE rm -f _build/nutd.temp @@ -85,7 +85,7 @@ format: go run _support/gofmt-all.go -f .PHONY: cover -cover: ${TARGET_DIR}/.ok ${TEST_REPO} _tools/gocovmerge +cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge @echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!" mkdir -p "${COVERAGE_DIR}" rm -f ${COVERAGE_DIR}/*.out "${COVERAGE_DIR}/all.merged" "${COVERAGE_DIR}/all.html" @@ -96,7 +96,7 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} _tools/gocovmerge -coverprofile=${COVERAGE_DIR}/unit-`echo $$MOD|tr "/" "_"`.out \ $$MOD 2>&1 | grep -v "no packages being tested depend on"; \ done - _tools/gocovmerge ${COVERAGE_DIR}/*.out > "${COVERAGE_DIR}/all.merged" + ${TOOLS_DIR}/gocovmerge ${COVERAGE_DIR}/*.out > "${COVERAGE_DIR}/all.merged" go tool cover -html "${COVERAGE_DIR}/all.merged" -o "${COVERAGE_DIR}/all.html" @echo "" @echo "=====> Total test coverage: <=====" @@ -106,29 +106,25 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} _tools/gocovmerge list: ${TARGET_DIR}/.ok @echo $(allpackages) -# cd into the GOPATH to workaround ./... not following symlinks -_allpackages = $(shell ( cd "${PKG_BUILD_DIR}" && \ - go list ./... 2>&1 1>&3 | \ - grep -v -e "^$$" $(addprefix -e ,$(IGNORED_PACKAGES)) 1>&2 ) 3>&1 | \ - grep -v -e "^$$" $(addprefix -e ,$(IGNORED_PACKAGES))) +_allpackages = $(shell cd "${PKG_BUILD_DIR}" && govendor list -no-status +local) # memoize allpackages, so that it's executed only once and only if used allpackages = $(if $(__allpackages),,$(eval __allpackages := $$(_allpackages)))$(__allpackages) .PHONY: install-developer-tools -install-developer-tools: _tools/govendor _tools/golint _tools/gocovmerge +install-developer-tools: ${TOOLS_DIR}/govendor ${TOOLS_DIR}/golint ${TOOLS_DIR}/gocovmerge -_tools: - mkdir -p _tools - -_tools/govendor: _tools +${TOOLS_DIR}/govendor: go get github.com/kardianos/govendor - mv ${BIN_BUILD_DIR}/govendor _tools/ + mkdir -p ${TOOLS_DIR} + mv ${BIN_BUILD_DIR}/govendor ${TOOLS_DIR}/ -_tools/golint: _tools +${TOOLS_DIR}/golint: go get github.com/golang/lint/golint - mv ${BIN_BUILD_DIR}/golint _tools/ + mkdir -p ${TOOLS_DIR} + mv ${BIN_BUILD_DIR}/golint ${TOOLS_DIR}/ -_tools/gocovmerge: _tools +${TOOLS_DIR}/gocovmerge: go get github.com/wadey/gocovmerge - mv ${BIN_BUILD_DIR}/gocovmerge _tools/ + mkdir -p ${TOOLS_DIR} + mv ${BIN_BUILD_DIR}/gocovmerge ${TOOLS_DIR}/ -- GitLab From ca26dd588a5bbe7e057b72fdc4e8f360f70c219f Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Tue, 13 Jun 2017 23:51:42 +0100 Subject: [PATCH 07/23] Fixed govendor path --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 00d6f181654..25fd216f898 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,7 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge list: ${TARGET_DIR}/.ok @echo $(allpackages) -_allpackages = $(shell cd "${PKG_BUILD_DIR}" && govendor list -no-status +local) +_allpackages = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) # memoize allpackages, so that it's executed only once and only if used allpackages = $(if $(__allpackages),,$(eval __allpackages := $$(_allpackages)))$(__allpackages) -- GitLab From c44caf2bfd34eef20c178995848360440fca1aea Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Tue, 13 Jun 2017 23:55:33 +0100 Subject: [PATCH 08/23] Debugging gitlab-ci --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba1d16c54d7..85ef4dfd533 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,7 @@ verify: coverage: stage: test script: + - make -d list - make cover test:go1.8: -- GitLab From a5f3b1d9e4f371c4728426cc9122055b3ee71ebd Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 00:01:07 +0100 Subject: [PATCH 09/23] More gitlab-ci testing --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 25fd216f898..8456d7b80a8 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,8 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge @go tool cover -func "${COVERAGE_DIR}/all.merged" list: ${TARGET_DIR}/.ok + echo GOPATH IS $GOPATH + cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local @echo $(allpackages) _allpackages = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) -- GitLab From 7247064fd45895d79c51b962f6248036ffe818d6 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 00:03:39 +0100 Subject: [PATCH 10/23] Debugging gitlab-ci --- Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 8456d7b80a8..9ef5924232a 100644 --- a/Makefile +++ b/Makefile @@ -104,14 +104,11 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge @go tool cover -func "${COVERAGE_DIR}/all.merged" list: ${TARGET_DIR}/.ok - echo GOPATH IS $GOPATH + echo GOPATH IS ${GOPATH} cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local @echo $(allpackages) -_allpackages = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) - -# memoize allpackages, so that it's executed only once and only if used -allpackages = $(if $(__allpackages),,$(eval __allpackages := $$(_allpackages)))$(__allpackages) +allpackages = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) .PHONY: install-developer-tools install-developer-tools: ${TOOLS_DIR}/govendor ${TOOLS_DIR}/golint ${TOOLS_DIR}/gocovmerge -- GitLab From e4aff768da0e139322a88ccb43218b508dd058ae Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 00:08:03 +0100 Subject: [PATCH 11/23] Fixed some expansions --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9ef5924232a..ae4e8170b53 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ BUILD_DIR=${CURDIR} TARGET_DIR=${BUILD_DIR}/_build BIN_BUILD_DIR=${TARGET_DIR}/bin PKG_BUILD_DIR=${TARGET_DIR}/src/${PKG} -CMDS:=$(shell cd cmd && ls) +CMDS=$(shell cd cmd && ls) VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S) export TEST_REPO_LOCATION=${TARGET_DIR}/testdata/data TEST_REPO=${TEST_REPO_LOCATION}/gitlab-test.git @@ -13,7 +13,7 @@ TOOLS_DIR=${BUILD_DIR}/_tools export GOPATH=${TARGET_DIR} export GO15VENDOREXPERIMENT=1 -export PATH:=${GOPATH}/bin:$(PATH) +export PATH=${GOPATH}/bin:${PATH} .PHONY: all all: verify build test -- GitLab From 9dd9f11d110c2b284ed720616ef202c2d773af98 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 00:11:17 +0100 Subject: [PATCH 12/23] Fixed path --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ae4e8170b53..8cd36aa4704 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ TOOLS_DIR=${BUILD_DIR}/_tools export GOPATH=${TARGET_DIR} export GO15VENDOREXPERIMENT=1 -export PATH=${GOPATH}/bin:${PATH} +export PATH:=${GOPATH}/bin:${PATH} .PHONY: all all: verify build test -- GitLab From ebd8a87aec315fc8aa3d3768d88d92004e51318d Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 00:15:27 +0100 Subject: [PATCH 13/23] Debugging make some more --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8cd36aa4704..0470fae4233 100644 --- a/Makefile +++ b/Makefile @@ -106,7 +106,7 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge list: ${TARGET_DIR}/.ok echo GOPATH IS ${GOPATH} cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local - @echo $(allpackages) + @echo "ALL PACKAGES===" $(allpackages) allpackages = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) -- GitLab From ec6b79d7b1f1256b42d1eb34cbe51e8f1a34a4ae Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 00:19:30 +0100 Subject: [PATCH 14/23] Still trying to figure this out --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0470fae4233..ef9030dcdae 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ list: ${TARGET_DIR}/.ok cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local @echo "ALL PACKAGES===" $(allpackages) -allpackages = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) +allpackages = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local 2>&1) .PHONY: install-developer-tools install-developer-tools: ${TOOLS_DIR}/govendor ${TOOLS_DIR}/golint ${TOOLS_DIR}/gocovmerge -- GitLab From f7f0b7a875caa35286501ea7ebf3250aa059a0d7 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 09:24:20 +0100 Subject: [PATCH 15/23] Still diagnosing gitlab-ci issues --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ef9030dcdae..f8c6cccbfa7 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ export TEST_REPO_LOCATION=${TARGET_DIR}/testdata/data TEST_REPO=${TEST_REPO_LOCATION}/gitlab-test.git COVERAGE_DIR=${TARGET_DIR}/cover TOOLS_DIR=${BUILD_DIR}/_tools +allpackages = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) export GOPATH=${TARGET_DIR} export GO15VENDOREXPERIMENT=1 @@ -103,13 +104,12 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge @echo "" @go tool cover -func "${COVERAGE_DIR}/all.merged" +.PHONY: list list: ${TARGET_DIR}/.ok echo GOPATH IS ${GOPATH} cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local @echo "ALL PACKAGES===" $(allpackages) -allpackages = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local 2>&1) - .PHONY: install-developer-tools install-developer-tools: ${TOOLS_DIR}/govendor ${TOOLS_DIR}/golint ${TOOLS_DIR}/gocovmerge -- GitLab From 31a7a9d9179e7d9a437dd737e7562c8f6698869a Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 09:55:25 +0100 Subject: [PATCH 16/23] Use govendor for package listing consistently --- Makefile | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index f8c6cccbfa7..582be638ad9 100644 --- a/Makefile +++ b/Makefile @@ -4,17 +4,20 @@ BUILD_DIR=${CURDIR} TARGET_DIR=${BUILD_DIR}/_build BIN_BUILD_DIR=${TARGET_DIR}/bin PKG_BUILD_DIR=${TARGET_DIR}/src/${PKG} -CMDS=$(shell cd cmd && ls) VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S) export TEST_REPO_LOCATION=${TARGET_DIR}/testdata/data TEST_REPO=${TEST_REPO_LOCATION}/gitlab-test.git COVERAGE_DIR=${TARGET_DIR}/cover TOOLS_DIR=${BUILD_DIR}/_tools -allpackages = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) -export GOPATH=${TARGET_DIR} -export GO15VENDOREXPERIMENT=1 -export PATH:=${GOPATH}/bin:${PATH} +# Returns a list of all non-vendored (local packages) +LOCAL_PACKAGES = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) +COMMAND_PACKAGES = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local +p ./cmd/...) +COMMANDS = $(subst $(PKG)/cmd/,,$(COMMAND_PACKAGES)) + +export GOPATH = ${TARGET_DIR} +export GO15VENDOREXPERIMENT = 1 +export PATH := ${GOPATH}/bin:${PATH} .PHONY: all all: verify build test @@ -28,13 +31,13 @@ ${TARGET_DIR}/.ok: Makefile ${TOOLS_DIR}/govendor .PHONY: build build: ${TARGET_DIR}/.ok - go install -ldflags "-X main.version=${VERSION}" $(foreach cmd,${CMDS},${PKG}/cmd/${cmd}) - cp $(foreach cmd,${CMDS},${BIN_BUILD_DIR}/${cmd}) ${BUILD_DIR}/ + go install -ldflags "-X main.version=${VERSION}" ${COMMAND_PACKAGES} + cp $(foreach cmd,${COMMANDS},${BIN_BUILD_DIR}/${cmd}) ${BUILD_DIR}/ .PHONY: install install: build mkdir -p $(DESTDIR)${PREFIX}/bin/ - cd ${BIN_BUILD_DIR} && install ${CMDS} ${DESTDIR}${PREFIX}/bin/ + cd ${BIN_BUILD_DIR} && install ${COMMANDS} ${DESTDIR}${PREFIX}/bin/ .PHONY: verify verify: lint check-formatting govendor-status notice-up-to-date @@ -52,19 +55,19 @@ ${TEST_REPO}: .PHONY: test test: ${TARGET_DIR}/.ok ${TEST_REPO} - go test $(allpackages) + go test ${LOCAL_PACKAGES} .PHONY: test-race test-race: ${TARGET_DIR}/.ok ${TEST_REPO} - GODEBUG=cgocheck=2 go test -v -race $(allpackages) + GODEBUG=cgocheck=2 go test -v -race ${LOCAL_PACKAGES} .PHONY: lint lint: ${TARGET_DIR}/.ok ${TOOLS_DIR}/golint - ${TOOLS_DIR}/golint $(allpackages) + ${TOOLS_DIR}/golint ${LOCAL_PACKAGES} .PHONY: package package: build - ./_support/package/package ${CMDS} + ./_support/package/package ${COMMANDS} .PHONY: notice notice: ${TARGET_DIR}/.ok ${TOOLS_DIR}/govendor @@ -79,7 +82,7 @@ notice-up-to-date: ${TARGET_DIR}/.ok ${TOOLS_DIR}/govendor .PHONY: clean clean: rm -rf -- ${TARGET_DIR} - rm -f -- $(foreach cmd,${CMDS},./${cmd}) + rm -f -- $(foreach cmd,${COMMANDS},./${cmd}) .PHONY: format format: @@ -90,10 +93,8 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge @echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!" mkdir -p "${COVERAGE_DIR}" rm -f ${COVERAGE_DIR}/*.out "${COVERAGE_DIR}/all.merged" "${COVERAGE_DIR}/all.html" - @for MOD in $(allpackages); do \ - echo go test -coverpkg=`echo $(allpackages)|tr " " ","` \ - -coverprofile=${COVERAGE_DIR}/unit-`echo $$MOD|tr "/" "_"`.out $$MOD; \ - go test -coverpkg=`echo $(allpackages)|tr " " ","` \ + @for MOD in ${LOCAL_PACKAGES}; do \ + go test -coverpkg=`echo ${LOCAL_PACKAGES}|tr " " ","` \ -coverprofile=${COVERAGE_DIR}/unit-`echo $$MOD|tr "/" "_"`.out \ $$MOD 2>&1 | grep -v "no packages being tested depend on"; \ done @@ -106,9 +107,7 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge .PHONY: list list: ${TARGET_DIR}/.ok - echo GOPATH IS ${GOPATH} - cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local - @echo "ALL PACKAGES===" $(allpackages) + @echo ${LOCAL_PACKAGES} .PHONY: install-developer-tools install-developer-tools: ${TOOLS_DIR}/govendor ${TOOLS_DIR}/golint ${TOOLS_DIR}/gocovmerge -- GitLab From 1b67a7c52baeac1bcd6d25e3b34f81494f1c3433 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 10:19:54 +0100 Subject: [PATCH 17/23] Removed debugging --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 85ef4dfd533..ba1d16c54d7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,6 @@ verify: coverage: stage: test script: - - make -d list - make cover test:go1.8: -- GitLab From c6aa8ce1ef0b7962a023cc97ffce28e3180b9fb0 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 10:24:16 +0100 Subject: [PATCH 18/23] Adding list back --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba1d16c54d7..85ef4dfd533 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,7 @@ verify: coverage: stage: test script: + - make -d list - make cover test:go1.8: -- GitLab From 0214dcd334105e0b1ace5ef21e24fd3678861819 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 10:41:35 +0100 Subject: [PATCH 19/23] More tuning --- Makefile | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 582be638ad9..3bcd429d00b 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,25 @@ -PREFIX=/usr/local -PKG=gitlab.com/gitlab-org/gitaly -BUILD_DIR=${CURDIR} -TARGET_DIR=${BUILD_DIR}/_build -BIN_BUILD_DIR=${TARGET_DIR}/bin -PKG_BUILD_DIR=${TARGET_DIR}/src/${PKG} +PREFIX := /usr/local +PKG := gitlab.com/gitlab-org/gitaly +BUILD_DIR := ${CURDIR} +TARGET_DIR := ${BUILD_DIR}/_build +BIN_BUILD_DIR := ${TARGET_DIR}/bin +PKG_BUILD_DIR := ${TARGET_DIR}/src/${PKG} +export TEST_REPO_LOCATION := ${TARGET_DIR}/testdata/data +TEST_REPO := ${TEST_REPO_LOCATION}/gitlab-test.git +COVERAGE_DIR := ${TARGET_DIR}/cover +TOOLS_DIR := ${BUILD_DIR}/_tools + VERSION=$(shell git describe)-$(shell date -u +%Y%m%d.%H%M%S) -export TEST_REPO_LOCATION=${TARGET_DIR}/testdata/data -TEST_REPO=${TEST_REPO_LOCATION}/gitlab-test.git -COVERAGE_DIR=${TARGET_DIR}/cover -TOOLS_DIR=${BUILD_DIR}/_tools + +export GOPATH := ${TARGET_DIR} +export GO15VENDOREXPERIMENT = 1 +export PATH := ${GOPATH}/bin:${PATH} # Returns a list of all non-vendored (local packages) LOCAL_PACKAGES = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) COMMAND_PACKAGES = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local +p ./cmd/...) COMMANDS = $(subst $(PKG)/cmd/,,$(COMMAND_PACKAGES)) -export GOPATH = ${TARGET_DIR} -export GO15VENDOREXPERIMENT = 1 -export PATH := ${GOPATH}/bin:${PATH} - .PHONY: all all: verify build test -- GitLab From e74839ed32246a3de59d0c27b7616b7e51fc058e Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 12:23:14 +0100 Subject: [PATCH 20/23] More buildfile issues --- Makefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3bcd429d00b..ca646c17f14 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ ${TARGET_DIR}/.ok: Makefile ${TOOLS_DIR}/govendor touch -- "${TARGET_DIR}/.ok" .PHONY: build -build: ${TARGET_DIR}/.ok +build: ${TARGET_DIR}/.ok list go install -ldflags "-X main.version=${VERSION}" ${COMMAND_PACKAGES} cp $(foreach cmd,${COMMANDS},${BIN_BUILD_DIR}/${cmd}) ${BUILD_DIR}/ @@ -55,7 +55,7 @@ ${TEST_REPO}: git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git $@ .PHONY: test -test: ${TARGET_DIR}/.ok ${TEST_REPO} +test: ${TARGET_DIR}/.ok ${TEST_REPO} list go test ${LOCAL_PACKAGES} .PHONY: test-race @@ -90,7 +90,7 @@ format: go run _support/gofmt-all.go -f .PHONY: cover -cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge +cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge list @echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!" mkdir -p "${COVERAGE_DIR}" rm -f ${COVERAGE_DIR}/*.out "${COVERAGE_DIR}/all.merged" "${COVERAGE_DIR}/all.html" @@ -108,7 +108,9 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge .PHONY: list list: ${TARGET_DIR}/.ok - @echo ${LOCAL_PACKAGES} + @echo "GOPATH: ${GOPATH}" + @echo "Packages: ${LOCAL_PACKAGES}" + @echo "Commands: ${COMMANDS}" .PHONY: install-developer-tools install-developer-tools: ${TOOLS_DIR}/govendor ${TOOLS_DIR}/golint ${TOOLS_DIR}/gocovmerge -- GitLab From 1cdd8d237916d520a1d1531546a10586b739381d Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 12:26:20 +0100 Subject: [PATCH 21/23] Makefile funtime --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ca646c17f14..95c736518bf 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,8 @@ export GO15VENDOREXPERIMENT = 1 export PATH := ${GOPATH}/bin:${PATH} # Returns a list of all non-vendored (local packages) -LOCAL_PACKAGES = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local) -COMMAND_PACKAGES = $(shell cd "${PKG_BUILD_DIR}" && ${TOOLS_DIR}/govendor list -no-status +local +p ./cmd/...) +LOCAL_PACKAGES = $(shell cd "${PKG_BUILD_DIR}" && GOPATH=${GOPATH} ${TOOLS_DIR}/govendor list -no-status +local) +COMMAND_PACKAGES = $(shell cd "${PKG_BUILD_DIR}" && GOPATH=${GOPATH} ${TOOLS_DIR}/govendor list -no-status +local +p ./cmd/...) COMMANDS = $(subst $(PKG)/cmd/,,$(COMMAND_PACKAGES)) .PHONY: all @@ -77,8 +77,8 @@ notice: ${TARGET_DIR}/.ok ${TOOLS_DIR}/govendor .PHONY: notice-up-to-date notice-up-to-date: ${TARGET_DIR}/.ok ${TOOLS_DIR}/govendor cd ${PKG_BUILD_DIR} && ${TOOLS_DIR}/govendor license -template _support/notice.template -o ${TARGET_DIR}/nutd.temp - diff _build/nutd.temp NOTICE - rm -f _build/nutd.temp + diff ${TARGET_DIR}/nutd.temp NOTICE + rm -f ${TARGET_DIR}/nutd.temp .PHONY: clean clean: -- GitLab From 5b6a799a220744c1f2c6f771600c6d971a372c36 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 12:33:39 +0100 Subject: [PATCH 22/23] Include build phase in the CI process --- .gitlab-ci.yml | 7 ++++++- Makefile | 10 ++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 85ef4dfd533..c8c11137cf9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ image: registry.gitlab.com/gitlab-org/gitlab-build-images:golang-1.8-git-2.8.4 stages: - test + - build - package - publish @@ -20,13 +21,17 @@ verify: coverage: stage: test script: - - make -d list - make cover test:go1.8: <<: *test_definition image: registry.gitlab.com/gitlab-org/gitlab-build-images:golang-1.8-git-2.8.4 +compile: + stage: build + script: + - make build + package: stage: package only: diff --git a/Makefile b/Makefile index 95c736518bf..63bd2ff443b 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ ${TARGET_DIR}/.ok: Makefile ${TOOLS_DIR}/govendor touch -- "${TARGET_DIR}/.ok" .PHONY: build -build: ${TARGET_DIR}/.ok list +build: ${TARGET_DIR}/.ok go install -ldflags "-X main.version=${VERSION}" ${COMMAND_PACKAGES} cp $(foreach cmd,${COMMANDS},${BIN_BUILD_DIR}/${cmd}) ${BUILD_DIR}/ @@ -55,7 +55,7 @@ ${TEST_REPO}: git clone --bare https://gitlab.com/gitlab-org/gitlab-test.git $@ .PHONY: test -test: ${TARGET_DIR}/.ok ${TEST_REPO} list +test: ${TARGET_DIR}/.ok ${TEST_REPO} go test ${LOCAL_PACKAGES} .PHONY: test-race @@ -90,7 +90,7 @@ format: go run _support/gofmt-all.go -f .PHONY: cover -cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge list +cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge @echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!" mkdir -p "${COVERAGE_DIR}" rm -f ${COVERAGE_DIR}/*.out "${COVERAGE_DIR}/all.merged" "${COVERAGE_DIR}/all.html" @@ -108,9 +108,7 @@ cover: ${TARGET_DIR}/.ok ${TEST_REPO} ${TOOLS_DIR}/gocovmerge list .PHONY: list list: ${TARGET_DIR}/.ok - @echo "GOPATH: ${GOPATH}" - @echo "Packages: ${LOCAL_PACKAGES}" - @echo "Commands: ${COMMANDS}" + @echo "${LOCAL_PACKAGES}" .PHONY: install-developer-tools install-developer-tools: ${TOOLS_DIR}/govendor ${TOOLS_DIR}/golint ${TOOLS_DIR}/gocovmerge -- GitLab From b9472ab996c7e264fb03a8d7abf59dc88ac42094 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Wed, 14 Jun 2017 12:43:43 +0100 Subject: [PATCH 23/23] Release now runs verify rather than notice-up-to-date --- _support/release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_support/release b/_support/release index 091004639b7..92e776b7c34 100755 --- a/_support/release +++ b/_support/release @@ -4,7 +4,7 @@ require 'erb' require_relative 'run.rb' def main(version) - run!(%w[make notice-up-to-date]) + run!(%w[make verify]) run!(%w[make test]) puts 'Testing for changed files' -- GitLab