Update DEVELOPER_GUID, Add Coverage + go ling to Makefile, Run CI tests on all branches (#392)

Jakob3xD · web-flow · commit 5b040f0c852f · 2023-10-13T09:40:49.000-04:00
* .github: run test workflows for all branches, test against 2.9,2.10

Signed-off-by: Jakob Hahn &lt;jakob.hahn@hetzner.com&gt;

* Makefile: use docker golangci-lint, run integration test on `.` folder, change coverage generation

Signed-off-by: Jakob Hahn &lt;jakob.hahn@hetzner.com&gt;

* Update DEVELOPER_GUIDE

Signed-off-by: Jakob Hahn &lt;jakob.hahn@hetzner.com&gt;

---------

Signed-off-by: Jakob Hahn &lt;jakob.hahn@hetzner.com&gt;
diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml
@@ -1,11 +1,6 @@
 name: Link Checker
-on:
-  push:
-    branches:
-      - "*"
-  pull_request:
-    branches:
-      - "*"
+
+on: [push, pull_request]
 
 jobs:
   linkchecker:
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -1,12 +1,6 @@
 name: Lint check
 
-on:
-  push:
-    branches:
-      - "*"
-  pull_request:
-    branches:
-      - "*"
+on: [push, pull_request]
 
 env:
   GITHUB_ACTIONS: true
diff --git a/.github/workflows/test-compatibility.yml b/.github/workflows/test-compatibility.yml
@@ -3,7 +3,7 @@ name: Integration for Compatibility
 on: [push, pull_request]
 
 jobs:
-  integration-test-compatibility:
+  integ-test-compat:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -22,6 +22,8 @@ jobs:
           - { opensearch_version: 2.6.0 }
           - { opensearch_version: 2.7.0 }
           - { opensearch_version: 2.8.0 }
+          - { opensearch_version: 2.9.0 }
+          - { opensearch_version: 2.10.0 }
     steps:
       - uses: actions/checkout@v3
         with: { fetch-depth: 1 }
diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
@@ -1,12 +1,6 @@
 name: Integration
 
-on:
-  push:
-    branches:
-      - "*"
-  pull_request:
-    branches:
-      - "*"
+on: [push, pull_request]
 
 env:
   GITHUB_ACTIONS: true
@@ -30,6 +24,9 @@ jobs:
       - name: Launch OpenSearch cluster
         run: |
           make cluster.clean cluster.build cluster.start
+          for attempt in `seq 25`; do sleep 5; \
+          if curl -s localhost:9200; \
+          then echo '=====> ready'; break; fi; if [ $attempt == 25 ]; then exit 1; fi; echo '=====> waiting...'; done
       - run: make test-integ race=true
       - uses: codecov/codecov-action@v1
         with:
@@ -57,4 +54,7 @@ jobs:
       - name: Launch OpenSearch cluster
         run: |
           make cluster.clean cluster.build cluster.start
-      - run: make test-integ-secure
+          for attempt in `seq 25`; do sleep 5; \
+          if curl -s -ku admin:admin https://localhost:9200; \
+          then echo '=====> ready'; break; fi; if [ $attempt == 25 ]; then exit 1; fi; echo '=====> waiting...'; done
+      - run: make test-integ-secure
diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
@@ -1,12 +1,6 @@
 name: Unit
 
-on:
-  push:
-    branches:
-      - "*"
-  pull_request:
-    branches:
-      - "*"
+on: [push, pull_request]
 
 env:
   GITHUB_ACTIONS: true
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -43,7 +43,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Read, close and replace the http Reponse Body ([#300](https://github.com/opensearch-project/opensearch-go/pull/300))
 - Updated and adjusted golangci-lint, solve linting complains for signer ([#352](https://github.com/opensearch-project/opensearch-go/pull/352))
 - Solve linting complains for opensearchtransport ([#353](https://github.com/opensearch-project/opensearch-go/pull/353))
-- Updated Developer guide to include docker build instructions ([#385]https://github.com/opensearch-project/opensearch-go/pull/385)
+- Updated Developer guide to include docker build instructions ([#385](https://github.com/opensearch-project/opensearch-go/pull/385)
+- Test against version 2.9.0,2.10.0, run tests in all branches, change intergration tests to wait for OpenSearch to start ([#392](https://github.com/opensearch-project/opensearch-go/pull/392))
+- Makefile: use docker golangci-lint, run integration test on `.` folder, change coverage generation ([#392](https://github.com/opensearch-project/opensearch-go/pull/392)) 
 
 ### Deprecated
 
@@ -59,4 +61,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Security
 
-[Unreleased]: https://github.com/opensearch-project/opensearch-go/compare/v2.2.0...HEAD
+[Unreleased]: https://github.com/opensearch-project/opensearch-go/compare/v2.2.0...HEAD
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -11,8 +11,11 @@
       - [Execute integration tests from your terminal](#execute-integration-tests-from-your-terminal)
   - [Lint](#lint)
     - [Markdown lint](#markdown-lint)
+    - [Go lint](#go-lint)
+  - [Coverage](#coverage)
   - [Use an Editor](#use-an-editor)
     - [GoLand](#goland)
+    - [Vim](#vim)
 
 # Developer Guide
 
@@ -26,9 +29,9 @@ Fork [opensearch-project/opensearch-go](https://github.com/opensearch-project/op
 
 ### Install Prerequisites
 
-#### Go 1.11
+#### Go 1.15
 
-OpenSearch Go Client builds using [Go](https://golang.org/doc/install) 1.11 at a minimum.
+OpenSearch Go Client builds using [Go](https://golang.org/doc/install) 1.15 at a minimum.
 
 #### Docker
 
@@ -38,7 +41,7 @@ OpenSearch Go Client builds using [Go](https://golang.org/doc/install) 1.11 at a
 
 To build the project on Windows, use [WSL2](https://learn.microsoft.com/en-us/windows/wsl/install), the compatibility layer for running Linux applications.
 
-Install ```make```
+Install `make`
 ```
 sudo apt install make
 ```
@@ -64,11 +67,10 @@ In order to test opensearch-go client, you need a running OpenSearch cluster. Yo
 
 ### Composing an OpenSearch Docker Container
 
-Ensure that Docker is installed on your local machine. You can check by running `docker --version`. Next, navigate to your local OpenSearch repository where the docker-compose.yaml file is stored. Run the following command to build the docker container. 
+Ensure that Docker is installed on your local machine. You can check by running `docker --version`. Next, navigate to your local opensearch-go repository. Run the following command to build and start the OpenSearch docker container. 
 
 ```
-cd opensearch/.ci/opensearch
-docker-compose -f docker-compose.yml up
+make cluster.build cluster.start
 ```
 
 This command will start the OpenSearch container using the `docker-compose.yaml` configuration file. During the build process, the necessary dependencies and files will be downloaded, which may take some time depending on your internet connection and system resources.
@@ -104,6 +106,30 @@ To check the markdown files, run the following command:
 make lint.markdown
 ```
 
+### Go lint
+
+To check all go files, run the following command:
+
+```
+make linters
+```
+
+## Coverage
+
+To get the repository test coverage, run the following command:
+
+For the results to be display in your terminal:
+
+```
+make coverage
+```
+
+For the results to be display in your browser:
+
+```
+make coverage-html
+```
+
 ## Use an Editor
 
 ### GoLand
@@ -114,3 +140,8 @@ You can import the OpenSearch project into GoLand as follows:
 2. In the subsequent dialog navigate to the ~/go/src/opensearch-go and click **Open**
 
 After you have opened your project, you need to specify the location of the Go SDK. You can either specify a local path to the SDK or download it. To set the Go SDK, navigate to **Go | GOROOT** and set accordingly.
+
+### Vim
+
+To improve your vim experience with Go, you might want to check out [fatih/vim-go](https://github.com/fatih/vim-go).
+For example it correctly formats the file and validates it on save.
diff --git a/Makefile b/Makefile
@@ -38,7 +38,7 @@ endif
 		gotestsum --format=short-verbose --junitfile=tmp/integration-report.xml -- $(testintegargs) "./opensearchtransport" "./opensearchapi" "./opensearchutil"; \
 	else \
 		echo "go test -v" $(testintegargs) "."; \
-		go test -v $(testintegargs) "./opensearchtransport" "./opensearchapi" "./opensearchutil"; \
+		go test -v $(testintegargs) "." "./opensearchtransport" "./opensearchapi" "./opensearchutil"; \
 	fi;
 
 test-integ-secure: ##Run secure integration tests
@@ -48,11 +48,24 @@ test-bench:  ## Run benchmarks
 	@printf "\033[2m→ Running benchmarks...\033[0m\n"
 	go test -run=none -bench=. -benchmem ./...
 
-test-coverage:  ## Generate test coverage report
+coverage:  ## Print test coverage report
+	@make gen-coverage
+	@go tool cover -func=$(PWD)/tmp/total.cov
+	@printf "\033[0m--------------------------------------------------------------------------------\n\033[0m"
+
+coverage-html: ## Open test coverage report in browser
+	@make gen-coverage
+	@go tool cover -html $(PWD)/tmp/total.cov
+
+gen-coverage:  ## Generate test coverage report
 	@printf "\033[2m→ Generating test coverage report...\033[0m\n"
-	@go tool cover -html=tmp/unit.cov -o tmp/coverage.html
-	@go tool cover -func=tmp/unit.cov | 'grep' -v 'opensearchapi/api\.' | sed 's/github.com\/opensearch-project\/opensearch-go\///g'
-	@printf "\033[0m--------------------------------------------------------------------------------\nopen tmp/coverage.html\n\n\033[0m"
+	@rm -rf tmp
+	@mkdir tmp
+	@mkdir tmp/unit
+	@mkdir tmp/integration
+	@go test -cover ./... -args -test.gocoverdir="$(PWD)/tmp/unit"
+	@go test -cover -tags='integration' ./... -args -test.gocoverdir="$(PWD)/tmp/integration"
+	@go tool covdata textfmt -i=$(PWD)/tmp/unit,$(PWD)/tmp/integration -o $(PWD)/tmp/total.cov
 
 ##@ Development
 lint:  ## Run lint on the package
@@ -205,7 +218,7 @@ cluster.clean: ## Remove unused Docker volumes and networks
 	docker system prune --volumes --force
 
 linters:
-	./bin/golangci-lint run ./... --timeout=5m
+	docker run -t --rm -v $$(pwd):/app -v ~/.cache/golangci-lint/v1.53.3:/root/.cache -w /app golangci/golangci-lint:v1.53.3 golangci-lint run --timeout=5m
 
 workflow: ## Run all github workflow commands here sequentially
 
